diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..7c4b4ece --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "metplotpy/external/plottable"] + path = metplotpy/external/plottable + url = https://github.com/znstrider/plottable/ diff --git a/metplotpy/external/__init__.py b/metplotpy/external/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/metplotpy/external/plottable b/metplotpy/external/plottable new file mode 160000 index 00000000..553f5bd5 --- /dev/null +++ b/metplotpy/external/plottable @@ -0,0 +1 @@ +Subproject commit 553f5bd5b8b9d92aaa6cfbfbf5e95889c17fbac0 diff --git a/metplotpy/plots/scorecard/gefs_cnt_scorecard.yaml b/metplotpy/plots/scorecard/gefs_cnt_scorecard.yaml new file mode 100644 index 00000000..7638719e --- /dev/null +++ b/metplotpy/plots/scorecard/gefs_cnt_scorecard.yaml @@ -0,0 +1,204 @@ +# +# GEFS vs. AIGEFS SAL1L2 scorecard (command-line, no database) +# +# Run from the METplotpy repo root: +# python -m metplotpy.plots.scorecard.scorecard_plot +# +# Pipeline: reformat .stat files -> subset -> per-case statistics -> p-values +# -> draw scorecard (scorecard_render.py, matplotlib) +# + +# +# OUTPUT and LOG DIRECTORIES +# + +# All output will reside in this specified output directory, including the +# intermediate files and the finished scorecard (/scorecard.png). +output_dir: '/glade/derecho/scratch/harrold/scorecard/minnas_work/output' + +# A single log file will be generated, capturing logging information from +# all intermediate steps +log_dir : '/glade/derecho/scratch/harrold/scorecard/minnas_work/logs/' +log_filename: noaa_gefs.log +log_level: 'DEBUG' + +# +# INPUT FILES +# + +# Where the MET .stat files live. One directory is read for every combination of +# the ${...} variables below: ${base} (top directory), ${model} (each entry of +# 'model') and ${init} (each date from ymd_start to ymd_end, formatted with 'init'). +# NOTE: the words "base", "model" and "init" are substituted as plain text, so +# they must not appear anywhere else in the paths (e.g. a folder called "init_data"). +#met_stat_input: '/Users/minnawin/Python_Scorecard_Dev/Data_Reformatter/point_stat_HRRR_RRFS/sample_data/all_dates' +met_stat_input: '${base}/${model}.${init}' +base: '/glade/derecho/scratch/harrold/scorecard/test_data/GEFS_data' + +# Which files to read from each directory: one glob pattern or a list of them, +# matched against the file name. Optional: if left out, every *.stat file is read. +# Use this to keep grid-to-grid, grid-to-obs and precip files apart. The run stops +# with an error if a directory is missing or nothing matches. +stat_file_pattern: + - '*grid2obs*.stat' + +# What to do when a date directory, or the files you asked for, are missing for +# some model/date. Optional, default 'skip'. +# skip : drop that date for ALL models (so every model is compared on the same +# days), carry on, and print a warning at the end listing what was skipped. +# The scorecard subtitle then shows e.g. "30 of 31 days". +# error : stop at the first missing directory or file. +on_missing_input: 'skip' + +# Directory-name prefixes for ${model}, i.e. the folders gefs.YYYYMMDD and +# aigefs.YYYYMMDD. These are folder names (lower case) and are NOT the same +# thing as the MODEL column values listed under subset_params below. +model: + - gefs + - aigefs + +# Date format of the date directories (Python strftime), the first and last date +# (inclusive, written in that same format) and the step between dates in days. +init: "%Y%m%d" +increment: 1 +ymd_start: '20260301' +ymd_end: '20260331' + +# +# REFORMAT +# +# if data has unlabelled columns (i.e. a MET .stat file from MET point-stat or +# MET grid-stat tool). Set to True to reformat, False otherwise. +# Or if re-running using the same data and reformatting has already been done, +# then set to False for better performance. +reformat_needed: True + +# Set to 'True' if the .stat data was already aggregated with the MET stat_analysis +# tool, or if the line type already has confidence-interval columns (ncu/ncl and/or +# btcu/btcl), as CNT does. Set to 'False' for raw point-stat/grid-stat output; +# the statistic is then calculated for every case (valid time) with METcalcpy +# sum_stat, and the significance test compares those cases. +has_confidence_stats: True + +# MET line type of the input data (SAL1L2, SL1L2, CNT, ...) +linetype: 'CNT' + +# +# SUBSETTING +# +# Filtering data based on model, masking regions, stat names, levels, variable of +# interest (and its values), and fcst_lead +subset_params: + + # Column names in the MET output file. Values are kept only if they match. + + # Forecast lead times in HHMMSS ('60000' is 6 h, '1200000' is 120 h). + # These become the scorecard columns. + 'fcst_lead': + - '0' + - '60000' + - '120000' + - '240000' + - '360000' + - '480000' + - '600000' + - '720000' + - '840000' + - '960000' + - '1080000' + - '1200000' + + # Vertical levels, e.g. P250 = 250 hPa. Each becomes a scorecard row. + 'fcst_lev': + - 'P850' + - 'P500' + - 'P250' + + # Values of the MODEL column. ORDER MATTERS: the first model is "series 1" + # (model A) and the second is "series 2" (model B). A green up-triangle means + # model A is better than model B. + 'model': + - 'AIGEFS' + - 'GEFS' + + # Forecast variables. Each becomes a scorecard row group. + 'fcst_var': + - 'UGRD' + - 'VGRD' + + # Observation type. Use exactly one (e.g. ADPUPA = upper-air, ADPSFC = surface, + # GDAS for grid-to-grid). Mixing types in one scorecard is not supported. + 'obtype': + - 'ADPUPA' + + # Independent variable name and corresponding value(s): the verification region. + indep_variable: 'vx_mask' + + indep_values: + - 'CONUS' + + # A list of one or more statistics of interest (line type specific). + # SAL1L2: MAE works today. ANOM_CORR (and ANOM_CORR_RAW, RMSFA, RMSOA) are derived + # from partial sums and need the METcalcpy/METdataio derive-step fixes first; with + # fail_on_missing_stats: True the run stops with a clear error if one is listed + # before those fixes are in. + stats_list: + - 'MAE' + - 'ANOM_CORR' + +# Stop with an error if a statistic in stats_list is missing from the results +# (it was filtered out or could not be computed). Set to False to log the error +# and still draw whatever is available. Optional, default True. +fail_on_missing_stats: True + +# +# CONFIDENCE INTERVALS via agg_stat.py +# +# NOTE: this setting is not read by scorecard_plot.py at present; it is kept so +# existing configs stay valid. Which route is used is set by has_confidence_stats. +calculate_confidence_intervals: True + +# +# SCORECARD STATISTICS (p-values) +# + +# Append to the scorecard output file with successive runs +append_subsequent: False + +# Method to calculate p-values: 'NCAR' (t-test on the per-case differences, same as +# METviewer) or 'EMC'. Use NCAR. The EMC method decides which model is better from +# the statistic's NAME, and MAE is not among the names it knows (ME, BIAS, RMSE, +# RMSVE), so its better/worse direction for MAE should be treated as unverified. +pval_method: 'EMC' + +# Number of days used by the EMC method only (ignored with NCAR) +sample_size: 120 + + +# +# SCORECARD PLOT +# +# The scorecard is drawn with matplotlib (scorecard_render.py) and written to +# /scorecard.. The default colors follow METviewer: green = +# model A better, pink = model B better, gray = no significant difference, +# hatched = no data; darker shades are 95 / 99 / 99.9 % significance. + +# Title for scorecard (generated automatically as " vs. Scorecard" +# if this is left unspecified) +scorecard_title: 'Scorecard GEFS and AIGEFS' + +# Optional look-and-feel settings. Every key is optional; anything left out uses +# the default shown. Unknown keys give a warning. +#scorecard_style: +# title: 'AIGEFS vs. GEFS: Winds at 250 hPa' # overrides scorecard_title +# subtitle: 'March 2026, CONUS upper-air' # default: line type, region, obs type, dates +# palette: metviewer # metviewer | colorblind (blue/orange) +# show_values: false # print the signed confidence (1 - p) in each cell +# markers: true # triangles showing significance level +# formats: [png] # any of png, pdf, svg +# dpi: 200 +# label_map: # display names for the row labels +# UGRD: 'U wind' +# VGRD: 'V wind' +# P250: '250 hPa' +# model_names: ['AIGEFS', 'GEFS'] # default: the two entries of subset_params model diff --git a/metplotpy/plots/scorecard/gefs_ecnt_scorecard.yaml b/metplotpy/plots/scorecard/gefs_ecnt_scorecard.yaml new file mode 100644 index 00000000..2d3a8d9a --- /dev/null +++ b/metplotpy/plots/scorecard/gefs_ecnt_scorecard.yaml @@ -0,0 +1,118 @@ + +# +# SOURCE CODE DIRECTORY +# + +# Top directory where the source code for METplotpy resides. +# Used for locating the images used in the scorecard table +base_dir: '/Users/minnawin/Python_Scorecard_Dev' + +# +# OUTPUT and LOG DIRECTORIES +# + + +# All output will reside in this specified output directory +output_dir: '/Users/minnawin/Python_Scorecard_Dev/output' + +# A single log file will be generated, capturing logging information from +# all intermediate steps +log_dir : '/Users/minnawin/Python_Scorecard_Dev/logs/' +log_filename: noaa_gefs.log +log_level: 'DEBUG' + +# MET .stat input files +#met_stat_input: '/Users/minnawin/Python_Scorecard_Dev/Data_Reformatter/point_stat_HRRR_RRFS/sample_data/all_dates' +met_stat_input: '${base}/${model}.${init}' +base: '/Users/minnawin/Python_Scorecard_Dev/NOAA_data' +model: + - gefs + - aigefs +init: "%Y%m%d" +increment: 1 +ymd_start: '20260301' +ymd_end: '20260331' + +# +# REFORMAT +# +# if data has unlabelled columns (i.e. a MET .stat file from MET point-stat or +# MET grid-stat tool). Set to True to reformat, False otherwise. +# Or if re-running using the same data and reformatting has already been done, +# then set to False for better performance. +reformat_needed: True + +# Set to 'True' MET .stat file is aggregated using the MET stat-analysis tool, +# or if line type already has CI stats columns (ncu/ncl and/or btcu/btcl). +# Set to 'False' if data doesn't have CI stats. +has_confidence_stats: True +linetype: 'ECNT' + +# +# SUBSETTING +# +# Filtering data based on model, masking regions, stat names, levels, variable of +# interest (and its values), and fcst_lead +subset_params: + + # Column names in the MET output file + 'fcst_lead': + - '0' + - '60000' + - '120000' + - '240000' + - '360000' + - '480000' + + 'fcst_lev': + - 'A6' + + 'model': + - 'AIGEFS' + - 'GEFS' + + 'fcst_var': + - 'APCP_06' + + # independent variable name and corresponding value(s) + indep_variable: 'vx_mask' + + indep_values: + - 'CONUS' + + # a list of one or more statistics of interest (line type specific) + stats_list: + - 'ME' + + +# +# CONFIDENCE INTERVALS via agg_stat.py +# +# Python code will generate the necessary config information to pass to +# the METcalcpy agg_stat.py module. Some line types do not require this step. +# If the MET output does not have btcu/btcl or ncu/ncl columns, then set +# this value to True +calculate_confidence_intervals: True + +# +# SCORECARD STATISTICS (p-values) +# + +# Append to the scorecard output file with successive runs +append_subsequent: False + +# Method to calculate p-values: 'EMC' or 'NCAR' (code currently uses NCAR) +pval_method: 'NCAR' + +# Used in the EMC method of calculating p-values +sample_size: 10 + + +# +# SCORECARD PLOT +# + +# Title for scorecard (this can be automatically generated in the code if +# this is left unspecified) +scorecard_title: 'Scorecard HRRR and RRFS' + diff --git a/metplotpy/plots/scorecard/gefs_sal1l2_scorecard.yaml b/metplotpy/plots/scorecard/gefs_sal1l2_scorecard.yaml new file mode 100644 index 00000000..1ed30c2d --- /dev/null +++ b/metplotpy/plots/scorecard/gefs_sal1l2_scorecard.yaml @@ -0,0 +1,204 @@ +# +# GEFS vs. AIGEFS SAL1L2 scorecard (command-line, no database) +# +# Run from the METplotpy repo root: +# python -m metplotpy.plots.scorecard.scorecard_plot +# +# Pipeline: reformat .stat files -> subset -> per-case statistics -> p-values +# -> draw scorecard (scorecard_render.py, matplotlib) +# + +# +# OUTPUT and LOG DIRECTORIES +# + +# All output will reside in this specified output directory, including the +# intermediate files and the finished scorecard (/scorecard.png). +output_dir: '/glade/derecho/scratch/harrold/scorecard/minnas_work/output' + +# A single log file will be generated, capturing logging information from +# all intermediate steps +log_dir : '/glade/derecho/scratch/harrold/scorecard/minnas_work/logs/' +log_filename: noaa_gefs.log +log_level: 'DEBUG' + +# +# INPUT FILES +# + +# Where the MET .stat files live. One directory is read for every combination of +# the ${...} variables below: ${base} (top directory), ${model} (each entry of +# 'model') and ${init} (each date from ymd_start to ymd_end, formatted with 'init'). +# NOTE: the words "base", "model" and "init" are substituted as plain text, so +# they must not appear anywhere else in the paths (e.g. a folder called "init_data"). +#met_stat_input: '/Users/minnawin/Python_Scorecard_Dev/Data_Reformatter/point_stat_HRRR_RRFS/sample_data/all_dates' +met_stat_input: '${base}/${model}.${init}' +base: '/glade/derecho/scratch/harrold/scorecard/test_data/GEFS_data' + +# Which files to read from each directory: one glob pattern or a list of them, +# matched against the file name. Optional: if left out, every *.stat file is read. +# Use this to keep grid-to-grid, grid-to-obs and precip files apart. The run stops +# with an error if a directory is missing or nothing matches. +stat_file_pattern: + - '*grid2obs*.stat' + +# What to do when a date directory, or the files you asked for, are missing for +# some model/date. Optional, default 'skip'. +# skip : drop that date for ALL models (so every model is compared on the same +# days), carry on, and print a warning at the end listing what was skipped. +# The scorecard subtitle then shows e.g. "30 of 31 days". +# error : stop at the first missing directory or file. +on_missing_input: 'skip' + +# Directory-name prefixes for ${model}, i.e. the folders gefs.YYYYMMDD and +# aigefs.YYYYMMDD. These are folder names (lower case) and are NOT the same +# thing as the MODEL column values listed under subset_params below. +model: + - gefs + - aigefs + +# Date format of the date directories (Python strftime), the first and last date +# (inclusive, written in that same format) and the step between dates in days. +init: "%Y%m%d" +increment: 1 +ymd_start: '20260301' +ymd_end: '20260331' + +# +# REFORMAT +# +# if data has unlabelled columns (i.e. a MET .stat file from MET point-stat or +# MET grid-stat tool). Set to True to reformat, False otherwise. +# Or if re-running using the same data and reformatting has already been done, +# then set to False for better performance. +reformat_needed: True + +# Set to 'True' if the .stat data was already aggregated with the MET stat_analysis +# tool, or if the line type already has confidence-interval columns (ncu/ncl and/or +# btcu/btcl), as CNT does. Set to 'False' for raw point-stat/grid-stat output; +# the statistic is then calculated for every case (valid time) with METcalcpy +# sum_stat, and the significance test compares those cases. +has_confidence_stats: False + +# MET line type of the input data (SAL1L2, SL1L2, CNT, ...) +linetype: 'SAL1L2' + +# +# SUBSETTING +# +# Filtering data based on model, masking regions, stat names, levels, variable of +# interest (and its values), and fcst_lead +subset_params: + + # Column names in the MET output file. Values are kept only if they match. + + # Forecast lead times in HHMMSS ('60000' is 6 h, '1200000' is 120 h). + # These become the scorecard columns. + 'fcst_lead': + - '0' + - '60000' + - '120000' + - '240000' + - '360000' + - '480000' + - '600000' + - '720000' + - '840000' + - '960000' + - '1080000' + - '1200000' + + # Vertical levels, e.g. P250 = 250 hPa. Each becomes a scorecard row. + 'fcst_lev': + - 'P850' + - 'P500' + - 'P250' + + # Values of the MODEL column. ORDER MATTERS: the first model is "series 1" + # (model A) and the second is "series 2" (model B). A green up-triangle means + # model A is better than model B. + 'model': + - 'AIGEFS' + - 'GEFS' + + # Forecast variables. Each becomes a scorecard row group. + 'fcst_var': + - 'UGRD' + - 'VGRD' + + # Observation type. Use exactly one (e.g. ADPUPA = upper-air, ADPSFC = surface, + # GDAS for grid-to-grid). Mixing types in one scorecard is not supported. + 'obtype': + - 'ADPUPA' + + # Independent variable name and corresponding value(s): the verification region. + indep_variable: 'vx_mask' + + indep_values: + - 'CONUS' + + # A list of one or more statistics of interest (line type specific). + # SAL1L2: MAE works today. ANOM_CORR (and ANOM_CORR_RAW, RMSFA, RMSOA) are derived + # from partial sums and need the METcalcpy/METdataio derive-step fixes first; with + # fail_on_missing_stats: True the run stops with a clear error if one is listed + # before those fixes are in. + stats_list: + - 'MAE' +# - 'ANOM_CORR' + +# Stop with an error if a statistic in stats_list is missing from the results +# (it was filtered out or could not be computed). Set to False to log the error +# and still draw whatever is available. Optional, default True. +fail_on_missing_stats: True + +# +# CONFIDENCE INTERVALS via agg_stat.py +# +# NOTE: this setting is not read by scorecard_plot.py at present; it is kept so +# existing configs stay valid. Which route is used is set by has_confidence_stats. +calculate_confidence_intervals: True + +# +# SCORECARD STATISTICS (p-values) +# + +# Append to the scorecard output file with successive runs +append_subsequent: False + +# Method to calculate p-values: 'NCAR' (t-test on the per-case differences, same as +# METviewer) or 'EMC'. Use NCAR. The EMC method decides which model is better from +# the statistic's NAME, and MAE is not among the names it knows (ME, BIAS, RMSE, +# RMSVE), so its better/worse direction for MAE should be treated as unverified. +pval_method: 'NCAR' + +# Number of days used by the EMC method only (ignored with NCAR) +sample_size: 10 + + +# +# SCORECARD PLOT +# +# The scorecard is drawn with matplotlib (scorecard_render.py) and written to +# /scorecard.. The default colors follow METviewer: green = +# model A better, pink = model B better, gray = no significant difference, +# hatched = no data; darker shades are 95 / 99 / 99.9 % significance. + +# Title for scorecard (generated automatically as " vs. Scorecard" +# if this is left unspecified) +scorecard_title: 'Scorecard GEFS and AIGEFS' + +# Optional look-and-feel settings. Every key is optional; anything left out uses +# the default shown. Unknown keys give a warning. +#scorecard_style: +# title: 'AIGEFS vs. GEFS: Winds at 250 hPa' # overrides scorecard_title +# subtitle: 'March 2026, CONUS upper-air' # default: line type, region, obs type, dates +# palette: metviewer # metviewer | colorblind (blue/orange) +# show_values: false # print the signed confidence (1 - p) in each cell +# markers: true # triangles showing significance level +# formats: [png] # any of png, pdf, svg +# dpi: 200 +# label_map: # display names for the row labels +# UGRD: 'U wind' +# VGRD: 'V wind' +# P250: '250 hPa' +# model_names: ['AIGEFS', 'GEFS'] # default: the two entries of subset_params model diff --git a/metplotpy/plots/scorecard/hrrr_rrfs_cnt_scorecard.yaml b/metplotpy/plots/scorecard/hrrr_rrfs_cnt_scorecard.yaml new file mode 100644 index 00000000..32ecc3cd --- /dev/null +++ b/metplotpy/plots/scorecard/hrrr_rrfs_cnt_scorecard.yaml @@ -0,0 +1,122 @@ + +# +# SOURCE CODE DIRECTORY +# + +# Top directory where the source code for METplotpy resides. +# Used for locating the images used in the scorecard table +base_dir: '/Users/minnawin/Test_Scorecard' + +# +# OUTPUT and LOG DIRECTORIES +# + + +# All output will reside in this specified output directory +output_dir: '/Users/minnawin/Test_Scorecard/output' + +# A single log file will be generated, capturing logging information from +# all intermediate steps +log_dir : '/Users/minnawin/Test_Scorecard/logs/' +log_filename: noaa_gefs.log +log_level: 'DEBUG' + +# MET .stat input files +met_stat_input: '${base}' +base: '/Users/minnawin/Python_Scorecard_Dev/Data_Reformatter/point_stat_HRRR_RRFS/avail_dates' +model: + - RRFS + - HRRR +init: "%Y%m%d" +increment: 1 +ymd_start: '20230702' +ymd_end: '20230704' + +# +# REFORMAT +# +# if data has unlabelled columns (i.e. a MET .stat file from MET point-stat or +# MET grid-stat tool). Set to True to reformat, False otherwise. +# Or if re-running using the same data and reformatting has already been done, +# then set to False for better performance. +reformat_needed: True + +# Set to 'True' MET .stat file is aggregated using the MET stat-analysis tool, +# or if line type already has CI stats columns (ncu/ncl and/or btcu/btcl). +# Set to 'False' if data doesn't have CI stats. +has_confidence_stats: True +linetype: 'CNT' + +# +# SUBSETTING +# +# Filtering data based on model, masking regions, stat names, levels, variable of +# interest (and its values), and fcst_lead +subset_params: + + # Column names in the MET output file + 'fcst_lead': + - '0' + - '60000' + - '90000' + - '100000' + - '120000' + - '150000' + - '180000' + - '240000' + - '360000' + - '410000' + - '430000' + + 'fcst_lev': + - 'Z2' + + 'model': + - 'HRRR_verification_mem000' + - 'RRFS_verification_mem000' + + 'fcst_var': + - 'TMP' + + # independent variable name and corresponding value(s) + indep_variable: 'vx_mask' + + indep_values: + - 'CONUS' + + # a list of one or more statistics of interest (line type specific) + stats_list: + - 'ME' + + +# +# CONFIDENCE INTERVALS via agg_stat.py +# +# Python code will generate the necessary config information to pass to +# the METcalcpy agg_stat.py module. Some line types do not require this step. +# If the MET output does not have btcu/btcl or ncu/ncl columns, then set +# this value to True +calculate_confidence_intervals: False + +# +# SCORECARD STATISTICS (p-values) +# + +# Append to the scorecard output file with successive runs +append_subsequent: False + +# Method to calculate p-values: 'EMC' or 'NCAR' (code currently uses NCAR) +pval_method: 'NCAR' + +# Used in the EMC method of calculating p-values +sample_size: 10 + + +# +# SCORECARD PLOT +# + +# Title for scorecard (this can be automatically generated in the code if +# this is left unspecified) +scorecard_title: 'Scorecard HRRR and RRFS' + diff --git a/metplotpy/plots/scorecard/images/blue_square.png b/metplotpy/plots/scorecard/images/blue_square.png new file mode 100644 index 00000000..516bb675 Binary files /dev/null and b/metplotpy/plots/scorecard/images/blue_square.png differ diff --git a/metplotpy/plots/scorecard/images/down_red.png b/metplotpy/plots/scorecard/images/down_red.png new file mode 100644 index 00000000..63a1b21a Binary files /dev/null and b/metplotpy/plots/scorecard/images/down_red.png differ diff --git a/metplotpy/plots/scorecard/images/gray_square.png b/metplotpy/plots/scorecard/images/gray_square.png new file mode 100644 index 00000000..4d524687 Binary files /dev/null and b/metplotpy/plots/scorecard/images/gray_square.png differ diff --git a/metplotpy/plots/scorecard/images/green_square.png b/metplotpy/plots/scorecard/images/green_square.png new file mode 100644 index 00000000..53bedcd2 Binary files /dev/null and b/metplotpy/plots/scorecard/images/green_square.png differ diff --git a/metplotpy/plots/scorecard/images/large_green_tri.png b/metplotpy/plots/scorecard/images/large_green_tri.png new file mode 100644 index 00000000..5c9ba197 Binary files /dev/null and b/metplotpy/plots/scorecard/images/large_green_tri.png differ diff --git a/metplotpy/plots/scorecard/images/large_red_tri.png b/metplotpy/plots/scorecard/images/large_red_tri.png new file mode 100644 index 00000000..63a1b21a Binary files /dev/null and b/metplotpy/plots/scorecard/images/large_red_tri.png differ diff --git a/metplotpy/plots/scorecard/images/pink_square.png b/metplotpy/plots/scorecard/images/pink_square.png new file mode 100644 index 00000000..d0e0867d Binary files /dev/null and b/metplotpy/plots/scorecard/images/pink_square.png differ diff --git a/metplotpy/plots/scorecard/images/small_green_tri.png b/metplotpy/plots/scorecard/images/small_green_tri.png new file mode 100644 index 00000000..92e72868 Binary files /dev/null and b/metplotpy/plots/scorecard/images/small_green_tri.png differ diff --git a/metplotpy/plots/scorecard/images/small_red_down.png b/metplotpy/plots/scorecard/images/small_red_down.png new file mode 100644 index 00000000..a5c36ae2 Binary files /dev/null and b/metplotpy/plots/scorecard/images/small_red_down.png differ diff --git a/metplotpy/plots/scorecard/images/small_red_tri.png b/metplotpy/plots/scorecard/images/small_red_tri.png new file mode 100644 index 00000000..a5c36ae2 Binary files /dev/null and b/metplotpy/plots/scorecard/images/small_red_tri.png differ diff --git a/metplotpy/plots/scorecard/scorecard_plot.py b/metplotpy/plots/scorecard/scorecard_plot.py new file mode 100644 index 00000000..0252cc6d --- /dev/null +++ b/metplotpy/plots/scorecard/scorecard_plot.py @@ -0,0 +1,1012 @@ +""" +Command-line (no database) scorecard. + +One YAML file drives the whole pipeline: + + 1. reformat MET .stat files into per-line-type tables (METdataio) + 2. subset rows using subset_params in the YAML + 3. per-case statistics (SumStat) or aggregation (AggStat) (METcalcpy) + 4. p-values scorecard_stats.txt, one DIFF_SIG value per cell (METcalcpy) + 5. render the scorecard with scorecard_render.py (matplotlib) + +Usage (from the METplotpy repo root): + python -m metplotpy.plots.scorecard.scorecard_plot + +Look and feel (title, palette, labels, formats...) are set in the optional +`scorecard_style` block of the YAML; see scorecard_render.render_from_config. +""" +import os, sys +import fnmatch +import itertools +import requests +import json +import re +import pandas as pd +from datetime import datetime +from metplotpy.plots import util as plot_util +from METreformat import write_stat_ascii +from METdbLoad.ush.read_data_files import ReadDataFiles +from METdbLoad.ush.read_load_xml import XmlLoadFile +from METreformat.write_stat_ascii import WriteStatAscii +from metcalcpy.util.safe_log import safe_log +from metcalcpy import logging_config +from metcalcpy import scorecard +from metcalcpy.agg_stat import AggStat +from metcalcpy.sum_stat import SumStat +try: + from metplotpy.plots.scorecard.scorecard_render import render_from_config +except ImportError: # e.g. run from inside the scorecard directory + from scorecard_render import render_from_config + + +class ScorecardPlot(): + def __init__(self, configs): + # Assign the instance attributes based on config file settings + self.configs = configs + + # + # Logging and output location contain information for + # all steps + # + self.output_dir = configs['output_dir'] + os.makedirs(self.output_dir, exist_ok=True) + self.log_dir = configs['log_dir'] + # self.event_equalize = configs['event_equalize'] + self.event_equalize = False + os.makedirs(self.log_dir, exist_ok=True) + self.log_filename = os.path.join(self.log_dir, configs['log_filename']) + self.log_level = configs['log_level'] + self.logger = logging_config.setup_logging(configs) + + logger = self.logger + safe_log(logger, "debug", "Initializing ScorecardPlot with parameters") + + # + # Input data + # + self.input_dir = configs['met_stat_input'] + self.ymd_start = configs['ymd_start'] + self.ymd_end = configs['ymd_end'] + + # increment datetime in days + self.increment = configs['increment'] + self.init = configs['init'] + + # Optional glob pattern(s) selecting which .stat files to read from each + # date directory, e.g. '*grid2obs*.stat'. Default: every .stat file. + patterns = configs.get('stat_file_pattern', ['*.stat']) + if isinstance(patterns, str): + patterns = [patterns] + self.stat_file_patterns = patterns + + # What to do when a date directory (or the files in it) is missing: + # 'skip' - drop that date for ALL models (keeps the model comparison + # paired), carry on, and report what was skipped + # 'error' - stop at the first missing directory or file + self.on_missing_input = str(configs.get('on_missing_input', 'skip')).lower() + if self.on_missing_input not in ('skip', 'error'): + raise ValueError("on_missing_input must be 'skip' or 'error', " + f"got {configs.get('on_missing_input')!r}") + self.missing_input = [] # (date, model/variable label, reason) + self.dates_total = None + self.dates_used = None + + # Stop with an error if a statistic listed in stats_list never makes it + # into scorecard_stats.txt (otherwise it silently disappears). + self.fail_on_missing_stats = bool(configs.get('fail_on_missing_stats', True)) + + # + # For reformatter + # + self.linetype = str(configs['linetype']) + self.reformat_flag = configs['reformat_needed'] + self.reformat_params = {} + self.reformat_params['input_stats_aggregated'] = configs['has_confidence_stats'] + self.reformat_params['log_directory'] = self.log_dir + self.reformat_params['output_dir'] = self.output_dir + reformat_output_fname_parts = ["reformatted_", self.linetype, ".txt"] + reformat_output_fname = ''.join(reformat_output_fname_parts) + self.reformat_params['output_filename'] = os.path.join(self.output_dir, reformat_output_fname) + self.reformat_params['input_data_dir'] = configs['met_stat_input'] + self.reformat_params['log_directory'] = self.log_dir + self.reformat_params['log_filename'] = self.log_filename + self.reformat_params['log_level'] = self.log_level + self.reformat_params['line_type'] = self.linetype + + # + # For subsetting + # + self.subset_params: dict = configs['subset_params'] + + subsetted_fname = "filtered.txt" + self.subsetted_filename = os.path.join(self.output_dir, subsetted_fname) + if 'fcst_var' not in self.subset_params.keys(): + msg = "Missing fcst var in config file. This is needed to subset the input data." + safe_log(logger, "error", msg) + + + # + # For p-values via METcalcpy scorecard module + # + self.append_sc_runs: bool = configs['append_subsequent'] + + self.derived_series_1: list[list] = self.create_derived_series() + agg_fname = self.linetype.upper() + "_aggregated.data" + self.aggstat_filename = os.path.join(self.output_dir, agg_fname) + + sc_stat_fname = "scorecard_stats.txt" + self.scorecard_stats_output_filename = os.path.join(self.output_dir, sc_stat_fname) + + self.scorecard_stats_indy_var = self.subset_params['indep_variable'] + self.scorecard_stats_indy_vals = self.subset_params['indep_values'] + + # remove the fcs_init_beg key from the subset parameters if it + # is present. The scorecard doesn't use it and it results in the inability + # to correctly create cartesian products. + if 'fcst_init_beg' in self.subset_params.keys(): + self.subset_params.pop('fcst_init_beg') + + self.scorecard_stats_series_val = self.subset_params + self.scorecard_stats_statslist = self.subset_params['stats_list'] + + # num of days, used in bootstrapping + self.sample_size = int(configs['sample_size']) + self.pval_method: str = configs['pval_method'] + + + + def find_input_files(self, final_list, all_values, var_index, all_dates) -> list: + """ + Find the .stat files matching stat_file_pattern in every input directory. + + A date is only used if EVERY directory for that date (i.e. every model) has + matching files. Otherwise the date is dropped for all models, so the models + are compared on the same sample, and what was skipped is recorded in + self.missing_input (printed at the end by report_missing_input()). + With on_missing_input: error the first missing directory/file stops the run. + + Args: + final_list: input directories, one per combination of the ${...} variables + all_values: the variable values behind each directory (same order) + var_index: position -> variable name for the values in all_values + all_dates: every date requested (strings in the 'init' format) + + Returns: + list of file paths + """ + date_pos = [i for i, n in var_index.items() if n == 'init'] + # variables that differ between directories (besides the date), e.g. model + label_pos = [i for i, n in var_index.items() + if n != 'init' and len({v[i] for v in all_values}) > 1] + + files_by_dir, problem_by_dir, info_by_dir = {}, {}, {} + for curr_dir, curr_value in zip(final_list, all_values): + date = curr_value[date_pos[0]] if date_pos else None + label = ", ".join(f"{var_index[i]}={curr_value[i]}" for i in label_pos) + info_by_dir[curr_dir] = (date, label) + if not os.path.isdir(curr_dir): + problem_by_dir[curr_dir] = "directory not found" + continue + stat_files = sorted(f for f in os.listdir(curr_dir) if f.endswith(".stat")) + matched = [f for f in stat_files + if any(fnmatch.fnmatch(f, p) for p in self.stat_file_patterns)] + if matched: + files_by_dir[curr_dir] = [os.path.join(curr_dir, f) for f in matched] + else: + problem_by_dir[curr_dir] = (f"no files matching {self.stat_file_patterns} " + f"(found: {stat_files if stat_files else 'no .stat files'})") + + if problem_by_dir and (self.on_missing_input == 'error' or not date_pos): + bad_dir = next(iter(problem_by_dir)) + raise FileNotFoundError(f"{bad_dir}: {problem_by_dir[bad_dir]}") + + # A date is complete only if every one of its directories has files + bad_dates = {} + for curr_dir, reason in problem_by_dir.items(): + date, label = info_by_dir[curr_dir] + bad_dates.setdefault(date, []).append((label, reason)) + used_dates = [d for d in all_dates if d not in bad_dates] + for date in all_dates: + for label, reason in bad_dates.get(date, []): + self.missing_input.append((date, label, reason)) + safe_log(self.logger, 'warning', + f"Skipping {date} for all models: {label}: {reason}") + self.dates_total, self.dates_used = len(all_dates), len(used_dates) + if not used_dates: + raise FileNotFoundError( + f"No date has matching files for every model ({len(all_dates)} dates " + f"checked, patterns {self.stat_file_patterns}); first problem: " + f"{self.missing_input[0]}") + + all_files = [] + for curr_dir in final_list: + if info_by_dir[curr_dir][0] in used_dates: + all_files.extend(files_by_dir[curr_dir]) + safe_log(self.logger, 'info', + f"Reading {len(all_files)} .stat file(s) matching " + f"{self.stat_file_patterns} for {len(used_dates)} of {len(all_dates)} date(s)") + return all_files + + def report_missing_input(self) -> None: + """Print (and log) which dates were skipped because input was missing.""" + if not self.missing_input: + return + lines = [f"WARNING: {len(self.missing_input)} input problem(s); " + f"used {self.dates_used} of {self.dates_total} date(s).", + "Skipped for ALL models so the comparison stays paired:"] + for date, label, reason in self.missing_input: + lines.append(f" {date} {label}: {reason}") + msg = "\n".join(lines) + print("\n" + msg) + safe_log(self.logger, 'warning', msg) + + def read_input(self) -> pd.DataFrame: + + """ + Retrieve all the input data from directories specified in the YAML config + file from the input_dir, start date, end date, and increment. + + Args: + + Returns: + + a pandas dataframe containing the data + + """ + # Get all the dates of interest + increment = int(self.increment) + date_start = datetime.strptime(self.ymd_start, self.init) + date_end = datetime.strptime(self.ymd_end, self.init) + + all_dates = [] + while date_start <= date_end: + all_dates.append(date_start.strftime(self.init)) + date_start = date_start + pd.Timedelta(days=increment) + + # Substitute variables that aren't "init" + + # keep track of the index of variable names, to be used later when creating + # relevant file directories + matches = re.finditer(r'\$\{([^}^{]+)\}', self.input_dir) + yaml_var_vals = {} + var_index = {} + + for idx, curr_match in enumerate(matches): + curr_matchobj = curr_match[0] + var = re.match(r'.*{(.*)\}', curr_matchobj) + # user's "variable" name in the input_dir setting + match = var.group(1) + + # evaluate the non-init variables from 'init' + if match != 'init': + if isinstance(self.configs[match], list): + # variable's value is a list of settings + var_index[idx] = match + yaml_var_vals[match] = self.configs[match] + else: + # variable's value is a single value + var_index[idx] = match + yaml_var_vals[match] = [self.configs[match]] + else: + # init variable, defines date directories + var_index[idx] = match + yaml_var_vals[match] = all_dates + + all_var_vals = yaml_var_vals.values() + all_values = [] + + # Get cartesian product of the variable values (pass in the unpacked all_var_vals + # to get the desired cartesian product) + for _ in itertools.product(*all_var_vals): + all_values.append(_) + + # Create the full input directories with all variables substituted with actual values + final_list = [] + + for curr_value in all_values: + string = self.input_dir + for v_idx in var_index: + pattern = var_index[v_idx] + repl = curr_value[v_idx] + if v_idx > 0: + string = result + result = re.sub(pattern, repl, string) + + # Remove the $, {, and } from each directory path + result = re.sub('\\$', '', result) + result = re.sub('{', '', result) + result = re.sub('}', '', result) + + final_list.append(result) + + all_files = self.find_input_files(final_list, all_values, var_index, all_dates) + + # Read in the files into a dataframe using METdataio's METdbLoad modules + + # Replacing the need for an XML specification file, pass in the XMLLoadFile and + # ReadDataFile parameters + rdf_obj: ReadDataFiles = ReadDataFiles(self.logger) + xml_loadfile_obj: XmlLoadFile = XmlLoadFile(None) + flags = xml_loadfile_obj.flags + line_types = xml_loadfile_obj.line_types + flags["load_stat"] = True + rdf_obj.read_data(flags, all_files, line_types) + return rdf_obj.stat_data + + + def create_derived_series(self) -> list[list]: + """ + Create all the derived series settings based on the fcst level, model names, + fcst hour, fcst variable, statistics + + Args: + configs (dict): + + Returns: + A list of lists representing the derived series required by METcalcpy's + scorecard.py module + + """ + fcst_leads = self.subset_params['fcst_lead'] + models = self.subset_params['model'] + stats = self.subset_params['stats_list'] + fcst_levs = self.subset_params['fcst_lev'] + fcst_var = self.subset_params['fcst_var'] + + # Any additional fixed filter dimensions in subset_params (e.g. obtype) + # have to be embedded into the derived-series comparison string too, or + # METcalcpy's _get_derived_points() will intersect them away to an empty + # list and silently produce zero derived points. Inserted between + # fcst_var and stats so model stays at index 1 (used below) and stat + # stays last (used by METcalcpy as the stat name). + # If we know all the additional dims, should we just add to known_keyS? + known_keys = {'fcst_lead', 'model', 'stats_list', 'fcst_lev', 'fcst_var', + 'indep_variable', 'indep_values'} + extra_dims = [v for k, v in self.subset_params.items() + if k not in known_keys and isinstance(v, list)] + + # Create the Cartesian product of the above + result = list(itertools.product(fcst_levs, models, fcst_leads, fcst_var, *extra_dims, stats)) + + # Make all elements strings, to enable joining the fcst lead, model name, etc + # based on model name into the format (level model fcst_hr variable stat: + # Z2 ModelA 60000 TMP RMSE + modelA = [] + modelB = [] + + for _ in result: + if _[1] == models[0]: + modelA_strs = [(str(i)) for i in _] + modelA.append(modelA_strs) + + else: + modelB_strs = [(str(i)) for i in _] + modelB.append(modelB_strs) + + # If models don't have the same number of data points, exit with a message. + if len(modelA) != len(modelB): + msg = (f"Different number of {models[0]} and {models[1]} data. Please check your" + f"data. ") + sys.exit(msg) + + # Join the components into one string + modelA_strs = [" ".join(i) for i in modelA] + modelB_strs = [" ".join(i) for i in modelB] + + # Group the modelA and modelB strings with the same fcst level, fcst hr, + # fcst var, and stat values + modelA_B = [list(i) for i in zip(modelA_strs, modelB_strs)] + + # Add the 'DIFF_SIG' directive to each item + [i.append("DIFF_SIG") for i in modelA_B] + + return modelA_B + + + def __repr__(self): + + class_name = type(self).__name__ + return (f"{class_name}(config={self.configs!r},)") + + + def __str__(self): + return f'(config={self.configs!r})' + + + def check_for_invalid_columns(self, df: pd.DataFrame) -> None: + """ + Verify that columns requested in YAML config file + are valid columns. Exit if a requested column does not + exist in the data. Non-existent columns are logged. + + Args: + df (pd.DataFrame): The dataframe containing MET data + + Returns: + None: exits if there are any columns requested that don't exist in the + data. + + """ + + safe_log(self.logger, 'debug', 'Check for requested columns in data.') + + # Retrieve all the column names in the data frame: + all_cols = df.columns.to_list() + + exclude_key = ['Idx', 'indep_values', 'indep_variable', 'stats_list', ] + filter_keys = [cols for cols in self.subset_params.keys() if cols not in exclude_key] + + invalid_cols = [] + # add the independent variable to the keys to check + indep_var = self.subset_params['indep_variable'] + filter_keys.append(indep_var) + + # Check the columns + for cur_fkey in filter_keys: + # don't include the stats and indep values in the check + if cur_fkey not in all_cols: + invalid_cols.append(cur_fkey) + msg = f" Cannot filter on non-existent column. {cur_fkey!r} is not a valid column name. " + safe_log(self.logger, 'error', msg) + + if len(invalid_cols) > 0: + safe_log(self.logger, 'error', 'Exiting. Check your data for the columns requested.') + sys.exit("Error:" + msg) + + + def subset_data(self, subset_df:pd.DataFrame) -> pd.DataFrame: + """ + Invoke this prior to invoking METcalcpy agg_stat. + Subset data based on independent variable and its + corresponding values, and any other columns in the data. + This step eliminates the need for a database. + + Args: + + subset_df (pd.DataFrame): The dataframe containing the MET stat data with + all columns labelled + + Returns: + result (pd.DataFrame): a dataframe after filtering/subsetting data based on criteria in the YAML config file + Also saves a file that contains only the relevant information as specified in + the YAML config file and also + + """ + safe_log(self.logger, 'debug', 'Filter data based on subset_params in the config file.') + working_df = subset_df.copy() + + # Exit if there are any requested columns that don't exist in the data + self.check_for_invalid_columns(working_df) + + # Replace the 'indep_variable' key with its value, + # replace the 'indep_values' key with its values, and + # replace stats_list keyname with stat_name + filter_keys = self.subset_params + indep_variable = filter_keys['indep_variable'] + indep_var_vals = filter_keys['indep_values'] + filter_keys[indep_variable] = indep_var_vals + if self.configs['has_confidence_stats'] == False: + # update the stats names in stats list with the linetype prefixed to the + # stat name + linetyped_stats = [] + for curr in filter_keys['stats_list']: + lt_stat_name = "".join(self.linetype + "_" + curr) + linetyped_stats.append(lt_stat_name) + filter_keys['stat_name'] = linetyped_stats + else: + filter_keys['stat_name'] = filter_keys['stats_list'] + del filter_keys['indep_variable'] + del filter_keys['indep_values'] + del filter_keys['stats_list'] + + # Create queries for column names specified in the subset_params settings. + # 'OR' all the values corresponding to each key, and 'AND' all of the + # key "segments" to create a final query. + + # Keep the query tokens ordered by column names + all_queries_by_cols = {} + + # Generate all the query tokens for each + # column name. A query token is a combination of + # the column with each value with form + # df[column_name] == val + for idx, column_name in enumerate(filter_keys): + queries_by_column = [] + + # Get the values for the current filter key + values = self.subset_params[column_name] + idx_last_value = len(values) - 1 + + for idx_val, cur_value in enumerate(values): + cur_value = cur_value.strip() + if column_name == 'fcst_lead': + query_token = f' ({column_name} == {cur_value} )' + else: + query_token = f' ({column_name} == "{cur_value}" )' + + # Add the 'OR' logic operator + # between each value corresponding to this column + if len(values) > 1 and idx_val != idx_last_value: + query_token = query_token + ' | ' + queries_by_column.append(query_token) + all_queries_for_this_column = ''.join(queries_by_column) + all_queries_by_cols[column_name] = all_queries_for_this_column + + # Add the appropriate parens and the 'AND' + # logical operator between the query tokens based on column + all_columns = all_queries_by_cols.keys() + last_column = len(all_columns) - 1 + for idx, cur_col in enumerate(all_columns): + values_for_col = all_queries_by_cols[cur_col] + + if idx != last_column: + values_for_col_updated = "(" + values_for_col + " ) & " + else: + values_for_col_updated = "(" + values_for_col + " ) " + + # Add this to the "all queries" dictionary to keep the value query + # tokens grouped by the column name (key) + all_queries_by_cols[cur_col] = values_for_col_updated + + # Create the finished query string + full_query_str_list = [] + for v in all_queries_by_cols.values(): + # only collect the values into a list to be joined later to create the full query + full_query_str_list.append(v) + + full_query = "".join(full_query_str_list) + + result: pd.DataFrame = working_df.query(full_query) + + # Pair the models case by case (event equalization), otherwise the + # significance test subtracts arrays of different lengths. + result = self.equalize_cases(result, list(filter_keys.keys())) + + result.to_csv(self.subsetted_filename, header=True, index_label=False, index=False, sep='\t') + + return result + + + def equalize_cases(self, df: pd.DataFrame, filter_columns: list) -> pd.DataFrame: + """ + Event equalization: keep only the cases that every model has. + + A case is one valid time for one scorecard cell (lead, level, variable, + region, stat, ...). If one model is missing a valid time (e.g. GEFS has + 116 cases and AIGEFS 120), the other model's extra rows are dropped so the + per-case differences used for the p-values line up. This is what + METviewer does before computing a scorecard. + + Args: + df: the subsetted data + filter_columns: the subset_params column names used for filtering + + Returns: + the data with unpaired cases removed + """ + date_col = next((c for c in ('fcst_valid_beg', 'fcst_valid', 'fcst_init_beg', 'fcst_init') + if c in df.columns), None) + if date_col is None or 'model' not in df.columns or df.empty: + safe_log(self.logger, 'debug', 'No date or model column; skipping event equalization.') + return df + + key_cols = [date_col] + [c for c in filter_columns + if c in df.columns and c not in ('model', date_col)] + n_models = df['model'].nunique() + models_per_case = df.groupby(key_cols, dropna=False)['model'].transform('nunique') + keep = models_per_case == n_models + n_dropped = int((~keep).sum()) + if n_dropped: + dropped = df.loc[~keep] + per_model = dropped['model'].value_counts().to_dict() + dates = sorted(dropped[date_col].astype(str).unique()) + shown = ", ".join(dates[:10]) + (" ..." if len(dates) > 10 else "") + msg = (f"Event equalization: dropped {n_dropped} row(s) with no matching case in " + f"the other model ({per_model}); {len(dates)} {date_col} value(s): {shown}") + print(msg) + safe_log(self.logger, 'warning', msg) + return df.loc[keep] + + def reformat_met_stat(self) -> pd.DataFrame: + """ + Invoke the METdataio METreformatter's write_stat_ascii module to + label all the headers in the MET .stat file based on linetype (specified in + the YAML config file). + + Args: + + Returns: + + reformatted_df (pd.DataFrame): The pandas dataframe containing the + reformatted data. + + + """ + + if self.reformat_flag: + r_df = self.read_input() + if r_df.size == 0: + safe_log(self.logger, 'ERROR', "ERROR: Input dataframe is empty. Exiting") + sys.exit() + if os.path.exists(self.reformat_params['output_filename']): + safe_log(self.logger, self.log_level, "Output file already exists, removing this file.") + os.remove(self.reformat_params['output_filename']) + + stat_lines_obj: WriteStatAscii = WriteStatAscii(self.reformat_params, self.logger) + reformatted_df = stat_lines_obj.write_stat_ascii(r_df, self.reformat_params) + return reformatted_df + else: + safe_log(self.logger, self.log_level, "Reformatting not requested") + + + + def set_dtypes(self, reformatted_df:pd.DataFrame) -> pd.DataFrame: + """ + Explicitly set the dtypes for all the columns in the reformatted dataframe + to avoid dtype errors and avoid relying on the soon-to-be deprecated + low_memory=False when reading data. + + Use the MET met_column_types.json file to retrieve the appropriate + dtypes. + + Args: + reformatted_df (pd.DataFrame): The pandas dataframe containing data with reformatted MET + data (i.e. all stat columns are split between + the stat_name and stat_val columns, and + all other unlabelled columns are + appropriately labelled). + + Returns: + dtype_df (pd.DataFrame): A dataframe that has had all the column + dtypes explicitly set. + + """ + + # Retrieve the met_column_types.json file from the MET repository's + # develop branch + url = "https://raw.githubusercontent.com/dtcenter/MET/refs/heads/develop/data/table_files/met_column_types.json" + response = requests.get(url) + dtypes_file = os.path.join(os.getcwd(), "dtypes.txt") + with open(dtypes_file, "w") as f: + f.write(response.text) + + with open(dtypes_file) as dtf: + met_types:dict = json.load(dtf) + + linetype = self.linetype.upper() + dtype_df = reformatted_df.copy(deep=True) + all_df_dtypes = dtype_df.dtypes + + # The dtype index corresponds to the name of the column + idx = all_df_dtypes.index + + # Columns that are added and not part of the original MET input + ignore_cols = ['fcst_init_beg', 'stat_name', 'stat_value'] + + # Assign the dtypes from the MET json file to the corresponding column in + # the dataframe (of the reformatted data). + accepted = ["str", "int64", "float64"] + for i in idx: + # column names in json file are upper case, need to convert the dataframe's + # columns to upper case + met_col = str(i).upper() + + if i not in ignore_cols: + if dtype_df[i].dtype not in accepted: + print(f"WARNING {i}: {dtype_df[i].dtype}, converting to {dtype_df[i].astype("str")}") + + dtype_df.astype({i:met_types[linetype][met_col]}).dtypes + + return dtype_df + + def calculate_agg_stats(self, subset_df:pd.DataFrame): + """ + Calculate the CI's using METcalcpy agg_stat.py + + Args: + subset_df (pd.DataFrame): The dataframe containing subsetted data. + + Returns: + aggregated_df (pd.DataFrame): Dataframe with the CI's calculated + + """ + # fname = "reformatted_" + self.configs['linetype'] + ".txt" + fname = "filtered.txt" + input_file = os.path.join(self.configs['output_dir'], fname) + agg_stat_configs = {} + + # Generate the configuration settings needed by METcalcpy's agg_stat + # module from the scorecard yaml config file. + # subset_df.to_csv(input_file) + agg_stat_configs['agg_stat_input'] = input_file + outname = self.configs['linetype'] + "_aggregated.data" + agg_stat_configs['agg_stat_output'] = os.path.join(self.configs['output_dir'], outname ) + agg_stat_configs['alpha'] = 0.05 + agg_stat_configs['append_to_file'] = "null" + agg_stat_configs['circular_block_bootstrap'] = True + agg_stat_configs['derived_series_1'] = [] + agg_stat_configs['derived_series_2'] = [] + agg_stat_configs['event_equal'] = self.event_equalize + + # fcst_var_val_1 is the fcst_var and the stat (stat name is pre-fixed with + # the linetype e.g. RMSE becomes ECNT_RMSE for linetype ECNT and + # stat in the stat_list of the scorecard YAML). + fcst_var = self.configs['subset_params']['fcst_var'][0] + stats_list = self.configs['subset_params']['stat_name'] + agg_stat_configs['fcst_var_val_1']= {fcst_var:stats_list} + agg_stat_configs['fcst_var_val_2'] = {} + agg_stat_configs['indy_vals'] = self.configs['subset_params']['fcst_lead'] + agg_stat_configs['indy_var'] = 'fcst_lead' + agg_stat_configs['line_type'] = str(self.configs['linetype']).lower() + agg_stat_configs['list_stat_1'] = stats_list + agg_stat_configs['list_stat_2'] = [] + agg_stat_configs['method'] = 'perc' + agg_stat_configs['num_iterations'] = 1 + agg_stat_configs['num_threads'] = -1 + agg_stat_configs['random_seed'] = None + # agg_stat_configs['series_val_1'] = {'model':self.configs['subset_params']['model']} + agg_stat_configs['series_val_1'] = self.configs['subset_params'] + agg_stat_configs['series_val_2'] = [] + + msg = "Calculating CI for "+ self.configs['linetype'] +" with agg_stat" + safe_log(self.logger, self.log_level, msg) + AGG_STAT = AggStat(agg_stat_configs) + AGG_STAT.calculate_stats_and_ci() + self.agg_stat_outfile = agg_stat_configs['agg_stat_output'] + safe_log(self.logger, self.log_level, "Finished calculating CI with agg_stat") + + return pd.read_csv(self.agg_stat_outfile, sep='\t', index_col=False) + + def calculate_sum_stats(self, subset_df: pd.DataFrame): + """ + Calculate per-case/per-day statistics using METcalcpy's sum_stat.py, + the same building block METviewer's own scorecard pipeline uses + (see edu.ucar.metviewer.scorecard.rscript.SumPythonManager in + METviewer, which runs sum_stat.py before scorecard.py -- agg_stat.py + is a different METviewer feature, not part of the scorecard chain). + + Unlike calculate_agg_stats()/AggStat, this does NOT collapse multiple + cases into one bootstrapped point estimate. It computes the requested + statistic (e.g. MAE) independently for each row of the input data, + preserving one row per case/valid-date -- which is what the scorecard's + own DIFF_SIG significance calculation (_calculate_diff_sig_ncar/_emc) + actually needs multiple values per (model, lead) group to compute. + + Args: + subset_df (pd.DataFrame): The dataframe containing subsetted data, + as produced by process_X_for_agg() -- i.e. one row per case, + already filtered down to the requested stat (e.g. one row per + case tagged 'SAL1L2_MAE'), with stat_value left as NaN, to be + filled in here. + + Returns: + summary_df (pd.DataFrame): Dataframe with stat_value populated, + one row per case, ready to feed into get_scorecard_stats(). + + """ + fname = "filtered.txt" + input_file = os.path.join(self.configs['output_dir'], fname) + + # sum_stat.py's calculate_statistic() dispatches to calculate_ + # using the *bare* stat name (e.g. 'mae', not 'sal1l2_mae') -- its own + # line_type == 'sal1l2' special case renames columns to match SL1L2's + # naming specifically so it can reuse the bare calculate_mae/etc. + # functions. Our process_X_for_agg() output uses the LINETYPE_STATNAME + # convention (matching AggStat's needs), so it has to be stripped back + # to bare form here before sum_stat.py sees it. + linetype_upper = str(self.configs['linetype']).upper() + prefix = linetype_upper + '_' + sum_stat_input_df = subset_df.copy() + sum_stat_input_df['stat_name'] = sum_stat_input_df['stat_name'].apply( + lambda s: s[len(prefix):] if isinstance(s, str) and s.startswith(prefix) else s) + + sum_stat_input_file = os.path.join(self.configs['output_dir'], "filtered_for_sum_stat.txt") + sum_stat_input_df.to_csv(sum_stat_input_file, index=None, header=True, sep='\t', na_rep="NA") + + sum_stat_configs = {} + + # Generate the configuration settings needed by METcalcpy's sum_stat + # module from the scorecard yaml config file. Deliberately NOT including + # the bootstrap-specific keys AggStat needs (alpha, circular_block_bootstrap, + # method, num_iterations, num_threads, random_seed) -- sum_stat.py doesn't + # use them; it computes one value per row, no resampling. + sum_stat_configs['sum_stat_input'] = sum_stat_input_file + outname = self.configs['linetype'] + "_summary.data" + sum_stat_configs['sum_stat_output'] = os.path.join(self.configs['output_dir'], outname) + sum_stat_configs['event_equal'] = self.event_equalize + sum_stat_configs['line_type'] = str(self.configs['linetype']).lower() + + fcst_var = self.configs['subset_params']['fcst_var'][0] + stats_list = self.configs['subset_params']['stat_name'] + sum_stat_configs['fcst_var_val_1'] = {fcst_var: stats_list} + sum_stat_configs['fcst_var_val_2'] = {} + sum_stat_configs['indy_vals'] = self.configs['subset_params']['fcst_lead'] + sum_stat_configs['indy_var'] = 'fcst_lead' + sum_stat_configs['list_stat_1'] = stats_list + sum_stat_configs['list_stat_2'] = [] + sum_stat_configs['series_val_1'] = self.configs['subset_params'] + sum_stat_configs['series_val_2'] = {} + sum_stat_configs['derived_series_1'] = [] + sum_stat_configs['derived_series_2'] = [] + sum_stat_configs['log_filename'] = self.configs.get('log_filename', '-') + sum_stat_configs['log_level'] = self.configs.get('log_level', 'ERROR') + + msg = "Calculating per-case statistics for " + self.configs['linetype'] + " with sum_stat" + safe_log(self.logger, self.log_level, msg) + SUM_STAT = SumStat(sum_stat_configs) + SUM_STAT.calculate_stats() + self.sum_stat_outfile = sum_stat_configs['sum_stat_output'] + safe_log(self.logger, self.log_level, "Finished calculating per-case statistics with sum_stat") + + return pd.read_csv(self.sum_stat_outfile, sep='\t', index_col=False) + + + + def get_scorecard_stats(self, input_df:pd.DataFrame) -> pd.DataFrame: + """ + Invoke the METcalcpy scorecard module to calculate the p-values. + + + + Args: + input_df (pd.DataFrame): The dataframe containing the reformated, + filtered, and aggregated (if applicable) data. + Returns: + a dataframe that will be used to generate the + scorecard plot. + + """ + + # Create the params expected by METcalcpy scorecard.py + params = {} + params['append_subsequent'] = self.append_sc_runs + params['derived_series_1'] = self.derived_series_1 + params['ndays'] = self.sample_size + params['pval_method'] = self.pval_method + params['log_dir'] = self.log_dir + params['log_filename'] = self.log_filename + params['log_level'] = self.log_level + params['scorecard_output'] = self.scorecard_stats_output_filename + params['list_stat_1'] = self.scorecard_stats_statslist + + # ToDo put logic for whether agg stat was needed and use that output file + # as input (CNT linetype does not require agg_stat.py, there are other linetypes + # that also do not require agg_stat.py) + if self.linetype == 'CNT': + # use the reformatted output for scorecard stats input + # remove the fcst_init_beg + params['sum_stat_input'] = self.subsetted_filename + else: + print("METcalcpy agg_stat necessary") + aggstat_filename = os.path.join(os.getcwd(), self.aggstat_filename) + params['sum_stat_input'] = aggstat_filename + + params['sum_stat_output'] = self.scorecard_stats_output_filename + params['series_val_1'] = self.scorecard_stats_series_val + params['indy_var'] = self.scorecard_stats_indy_var + params['indy_vals'] = self.scorecard_stats_indy_vals + params['stats_list'] = self.scorecard_stats_statslist + params['stat_flag'] = self.pval_method + + calcpy_sc = scorecard.Scorecard(params) + # calcpy_sc.input_data= input_df + calcpy_sc.calculate_scorecard_data() + + + def check_requested_stats(self) -> None: + """ + Make sure every statistic in stats_list actually reached scorecard_stats.txt. + + A statistic can be lost without any error (a name the reformatter never + creates, a filter that matches nothing, ...), which leaves a scorecard that + looks fine but is missing rows. Raise an error, or only log one when + fail_on_missing_stats is False. + """ + stats_file = self.scorecard_stats_output_filename + if not os.path.isfile(stats_file): + raise FileNotFoundError( + f"{stats_file} was not created; METcalcpy scorecard.py produced no output") + stats_df = pd.read_csv(stats_file, sep="\t", dtype=str, keep_default_na=False) + prefix = f"{self.linetype.upper()}_" + found = {n.upper()[len(prefix):] if n.upper().startswith(prefix) else n.upper() + for n in stats_df['stat_name']} + missing = [s for s in self.scorecard_stats_statslist if str(s).upper() not in found] + if missing: + msg = (f"Requested statistic(s) {missing} are not in {stats_file} " + f"(found: {sorted(found)}). They were filtered out or could not be " + f"computed upstream (reformat, subset or per-case step).") + safe_log(self.logger, 'error', msg) + if self.fail_on_missing_stats: + raise RuntimeError(msg) + + def render_scorecard(self) -> list: + """ + Draw the scorecard with scorecard_render.py (matplotlib only). Reads the + scorecard_stats.txt written by METcalcpy and derives the significance + categories itself, for both the NCAR and EMC p-value methods. + + Look and feel come from the YAML: scorecard_title plus an optional + scorecard_style block (title, subtitle, palette, show_values, markers, + formats, dpi, label_map, model_names). See scorecard_render.render_from_config. + + Returns: + list of the files written (/scorecard.) + """ + models = self.subset_params.get('model', []) + names = [models[0], models[1]] if len(models) >= 2 else None + extra = None + if self.dates_total and self.dates_used < self.dates_total: + extra = f"{self.dates_used} of {self.dates_total} days" # some dates skipped + written = render_from_config( + self.scorecard_stats_output_filename, + os.path.join(self.output_dir, "scorecard"), + style=self.configs.get('scorecard_style'), + model_names=names, + date_range=(self.ymd_start, self.ymd_end), + title=self.configs.get('scorecard_title'), + subtitle_extra=extra) + safe_log(self.logger, 'info', f"Scorecard written: {', '.join(written)}") + return written + + +# Should consider adding a step to remove all files associated with the run to make +# sure the current run isn't using old or stale files to generate the scorecard. + + +def main(config_filename=None): + """ + Read in the YAML config file and perform steps needed + to generate a scorecard plot. + + """ + # print(f"reading in the data: {datafile}") + # df = pd.read_table(datafile) + # + # print(f"df: {df['stat_value']}") + + confs = plot_util.get_params(config_filename) + sc = ScorecardPlot(confs) + + # + # Reformat the MET .stat file(s) + # + reformatted_df = sc.reformat_met_stat() + + # + # Explicitly set the dtype for each column using the MET json file + # + # dtype_df = sc.set_dtypes(reformatted_df) + # + # Filter the data based on settings in the YAML config file + # + subset_df = sc.subset_data(reformatted_df) + # + # Calculate the aggregation statistics via METcalcpy agg_stat.py + # module if needed + if confs['has_confidence_stats']: + print(f"Skip running agg_stat.py {sc.linetype} already has CI's calculated ") + aggregated_df = subset_df.copy() + sc.aggstat_filename = sc.subsetted_filename + else: + print(f" Invoke METcalcpy sum_stat.py to calculate per-case statistics for {sc.linetype} ") + aggregated_df = sc.calculate_sum_stats(subset_df) + sc.aggstat_filename = sc.sum_stat_outfile + # + # Get the p-values and scorecard categories + # + + # input is dependent on whether agg_stat.py was used to calculate the CI's + # CNT line type already has CI's + _: pd.DataFrame = sc.get_scorecard_stats(aggregated_df) + + # Every requested statistic must be present, then draw the scorecard. + sc.check_requested_stats() + sc.render_scorecard() + sc.report_missing_input() + print("Finished") + +if __name__ == "__main__": + # print("Scorecard plotting") + # datafile = "/Users/minnawin/Python_Scorecard_Dev/METcalcpy/metcalcpy/output/hrrr_rrfs_output.data" + # main(datafile) + main() diff --git a/metplotpy/plots/scorecard/scorecard_render.py b/metplotpy/plots/scorecard/scorecard_render.py new file mode 100644 index 00000000..5363be69 --- /dev/null +++ b/metplotpy/plots/scorecard/scorecard_render.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +""" +scorecard_render.py -- database-free, HPC-friendly scorecard renderer. + +Reads the scorecard_stats.txt written by the METplotpy scorecard script +(tab-separated DIFF_SIG output) and draws a METviewer-style scorecard with +matplotlib only (Agg backend: no browser, no Plotly/Kaleido, no display). + +Value convention (from METcalcpy scorecard.py / perfect_score_adjustment): + DIFF_SIG stat_value is a signed confidence level (1 - p), already adjusted + for the statistic's perfect score: + positive -> series 1 (first model) is BETTER than series 2 + negative -> series 1 is WORSE than series 2 + Thresholds: |v| >= 0.95 / 0.99 / 0.999 -> 95 / 99 / 99.9 % significance + (the EMC method returns exactly +/-0.95, 0.99, 1, which map the same way). + +Usage: + python scorecard_render.py scorecard_stats.txt -o scorecard.png + python scorecard_render.py scorecard_stats.txt -o scorecard.pdf --show-values + python scorecard_render.py --demo -o demo.png # synthetic data + +Importable: + from scorecard_render import load_scorecard, render_scorecard + fig = render_scorecard(load_scorecard("scorecard_stats.txt"), "out.png") +""" +import argparse +import re +import sys +import warnings + +import matplotlib +matplotlib.use("Agg") # headless: safe on HPC login/batch nodes +import matplotlib.pyplot as plt +from matplotlib.patches import Rectangle +import numpy as np +import pandas as pd + +# -------------------------------------------------------------------------- +# Palettes: index 0 = 95%, 1 = 99%, 2 = 99.9% +# -------------------------------------------------------------------------- +PALETTES = { + # familiar to METviewer users: green = better, pink = worse + "metviewer": { + "better": ["#c7e9c0", "#74c476", "#238b45"], + "worse": ["#fcc5c0", "#f768a1", "#ae017e"], + }, + # colour-vision-deficiency friendly: blue = better, orange = worse + "colorblind": { + "better": ["#c6dbef", "#6baed6", "#2171b5"], + "worse": ["#fdd0a2", "#fd8d3c", "#d94801"], + }, +} +NOSIG_COLOR = "#d9d9d9" +LEVEL_PCT = {1: "95%", 2: "99%", 3: "99.9%"} +MISSING_VALUES = {"NA", "NAN", "", "-9999", "-9999.0"} + +# Label columns: (key, header). The first three always appear; the rest only +# appear when they actually vary in the data (otherwise they go in the subtitle). +ALWAYS_COLS = [("stat", "Statistic"), ("var", "Variable"), ("level", "Level")] +OPTIONAL_COLS = [("thresh", "Threshold"), ("region", "Region"), ("obtype", "Obs type")] + + +# -------------------------------------------------------------------------- +# Data preparation +# -------------------------------------------------------------------------- +def _lead_to_hours(x): + """HHMMSS integer/string (e.g. 60000 -> 6 h, 480000 -> 48 h) -> hours.""" + v = int(float(x)) + h, m = v // 10000, (v % 10000) // 100 + return h + m / 60.0 + + +def _fmt_lead(hours): + h = int(hours) + m = int(round((hours - h) * 60)) + return f"{h}" if m == 0 else f"{h}:{m:02d}" + + +def _sig_code(v): + """Signed significance code: +/-1,2,3 (95/99/99.9%), 0 = not significant, + None = no data.""" + if v is None or (isinstance(v, float) and np.isnan(v)): + return None + a, s = abs(v), (1 if v >= 0 else -1) + eps = 1e-9 + if a >= 0.999 - eps: + return 3 * s + if a >= 0.99 - eps: + return 2 * s + if a >= 0.95 - eps: + return 1 * s + return 0 + + +def _parse_models(model_str): + """'DIFF_SIG(P250 AIGEFS 0 UGRD ADPUPA MAE-P250 GEFS 0 UGRD ADPUPA MAE)' + -> ('AIGEFS', 'GEFS'). Returns (None, None) if the layout is unexpected.""" + m = re.match(r"^DIFF_SIG\((.*)\)$", str(model_str).strip()) + if not m: + return None, None + tok = m.group(1).split() + # "lev model lead var obtype stat-lev model lead var obtype stat" -> 11 tokens, + # because the series are joined as "...stat-lev" (one token). + if len(tok) == 11: + return tok[1], tok[6] + return None, None + + +def load_scorecard(path): + """Read scorecard_stats.txt and return a tidy DataFrame with columns: + stat, var, level, thresh, region, obtype, lead_hr, value, code + plus df.attrs['model1'/'model2'/'linetype'/'valid_range'].""" + df = pd.read_csv(path, sep="\t", dtype=str, keep_default_na=False) + return tidy_scorecard(df) + + +def tidy_scorecard(raw): + need = ["fcst_lead", "fcst_var", "fcst_lev", "stat_name", "stat_value"] + missing = [c for c in need if c not in raw.columns] + if missing: + raise ValueError(f"scorecard input is missing required columns: {missing} " + f"(found: {list(raw.columns)})") + if len(raw) == 0: + raise ValueError("scorecard input has zero rows -- nothing to draw") + + def col(name, default="NA"): + return raw[name].astype(str) if name in raw.columns else pd.Series(default, index=raw.index) + + linetype = col("line_type").iloc[0] + stat = col("stat_name").str.replace(rf"^{re.escape(linetype)}_", "", regex=True) + + def to_val(s): + return np.nan if s.strip().upper() in MISSING_VALUES else float(s) + + out = pd.DataFrame({ + "stat": stat, + "var": col("fcst_var"), + "level": col("fcst_lev"), + "thresh": col("fcst_thresh"), + "region": col("vx_mask"), + "obtype": col("obtype"), + "lead_hr": raw["fcst_lead"].map(_lead_to_hours), + "value": raw["stat_value"].map(to_val), + }) + out["code"] = out["value"].map(_sig_code) + + m1, m2 = _parse_models(raw["model"].iloc[0]) if "model" in raw.columns else (None, None) + out.attrs["model1"], out.attrs["model2"] = m1, m2 + out.attrs["linetype"] = linetype + if "fcst_valid_end" in raw.columns: + d = pd.to_datetime(raw["fcst_valid_end"], errors="coerce").dropna() + out.attrs["valid_range"] = (d.min(), d.max()) if len(d) else None + return out + + +def summarize(df): + """Print what is actually in the file, so a missing statistic is obvious.""" + print("scorecard rows by statistic (cells with a value / total):", file=sys.stderr) + for stat, g in df.groupby("stat", sort=False): + n_ok = int(g["code"].map(lambda c: c is not None and not + (isinstance(c, float) and np.isnan(c))).sum()) + print(f" {stat:<12} {n_ok}/{len(g)} cells, " + f"{g['var'].nunique()} var(s), {g['level'].nunique()} level(s)", file=sys.stderr) + if n_ok == 0: + warnings.warn(f"{stat}: every value is missing (NA/-9999). The statistic was " + "listed but not computed -- look upstream (SumStat/sal1l2 columns).") + + +def make_demo_df(): + """Synthetic data to preview the layout (no pipeline needed).""" + rng = np.random.default_rng(7) + rows = [] + for stat in ["MAE", "ANOM_CORR"]: + for var, levels in [("UGRD", ["P850", "P500", "P250"]), + ("VGRD", ["P850", "P500", "P250"]), + ("TMP", ["P850", "P500"])]: + for lev in levels: + for hr in range(0, 121, 12): + if rng.random() < 0.06: + continue # a few missing cells + sig = rng.choice([0, 0.95, 0.99, 0.999]) + v = sig * rng.choice([-1, 1]) if sig else rng.uniform(-0.9, 0.9) + rows.append(dict(stat=stat, var=var, level=lev, thresh="NA", + region="CONUS", obtype="ADPUPA", + lead_hr=float(hr), value=float(v))) + df = pd.DataFrame(rows) + df["code"] = df["value"].map(_sig_code) + df.attrs.update(model1="AIGEFS", model2="GEFS", linetype="SAL1L2", + valid_range=(pd.Timestamp("2026-03-01"), pd.Timestamp("2026-03-31"))) + return df + + +# -------------------------------------------------------------------------- +# Rendering +# -------------------------------------------------------------------------- +def render_scorecard(df, outfile=None, title=None, subtitle=None, + model1=None, model2=None, palette="metviewer", + show_values=False, markers=True, dpi=200, + label_map=None, date_range=None, subtitle_extra=None): + """Draw the scorecard. Returns the matplotlib Figure; saves if outfile. + + label_map : optional {raw label: display label} for the row labels, e.g. + {"UGRD": "U wind", "P250": "250 hPa"}. + subtitle_extra : optional text appended to the auto-generated subtitle, e.g. + "30 of 31 days" when some dates were skipped. + date_range : optional (start, end) shown in the auto-generated subtitle + (anything pandas can parse, e.g. "20260301"); overrides the + single valid date found in the stats file. + """ + if palette not in PALETTES: + raise ValueError(f"unknown palette {palette!r}; choose from {sorted(PALETTES)}") + label_map = {str(k): str(v) for k, v in (label_map or {}).items()} + + def disp(x): + return label_map.get(str(x), str(x)) + + pal = PALETTES[palette] + m1 = model1 or df.attrs.get("model1") or "Series 1" + m2 = model2 or df.attrs.get("model2") or "Series 2" + + # ---- which label columns to show -------------------------------------- + cols = list(ALWAYS_COLS) + fixed = [] # facts that don't vary -> subtitle + for key, hdr in OPTIONAL_COLS: + n = df[key].nunique() + if n > 1: + cols.append((key, hdr)) + elif n == 1 and df[key].iloc[0] not in ("NA", ""): + fixed.append(df[key].iloc[0]) + keys = [k for k, _ in cols] + + # ---- one row per label tuple, one column per lead --------------------- + dup = df.duplicated(subset=keys + ["lead_hr"], keep="first") + if dup.any(): + warnings.warn(f"{int(dup.sum())} duplicate (row-label, lead) entries found; " + "keeping the first of each. Check subset_params in the YAML " + "(e.g. two obtypes or masks mixed together).") + df = df[~dup] + row_keys = list(dict.fromkeys(map(tuple, df[keys].itertuples(index=False, name=None)))) + leads = sorted(df["lead_hr"].unique()) + r_idx = {k: i for i, k in enumerate(row_keys)} + c_idx = {h: j for j, h in enumerate(leads)} + code = np.full((len(row_keys), len(leads)), np.nan) + val = np.full_like(code, np.nan) + for r in df.itertuples(index=False): + i, j = r_idx[tuple(getattr(r, k) for k in keys)], c_idx[r.lead_hr] + if r.code is not None and not (isinstance(r.code, float) and np.isnan(r.code)): + code[i, j] = r.code + val[i, j] = r.value + nrow, ncol = code.shape + + # ---- geometry, all in inches ------------------------------------------ + FS = 9.0 + cw, rh = 0.55 if not show_values else 0.62, 0.34 + char_w = 0.082 + col_w = [] + for d, (k, hdr) in enumerate(cols): + longest = max([len(disp(x[d])) for x in row_keys] + [len(hdr)]) + col_w.append(longest * char_w + 0.25) + label_w = sum(col_w) + grid_w = ncol * cw + leg_col_w = max(len(f"{m1} better than {m2} (99.9%)"), + len(f"{m2} better than {m1} (99.9%)")) * char_w + 0.5 + content_w = max(label_w + grid_w, 2 * leg_col_w) + margin = 0.35 + W = content_w + 2 * margin + x0 = (W - (label_w + grid_w)) / 2 # center the table + gx = x0 + label_w # left edge of the colour grid + + title_h, banner_h, lead_h = 0.95, 0.30, 0.32 + y_grid = margin + title_h + banner_h + lead_h + y_grid_end = y_grid + nrow * rh + leg_rows = 4 + H = y_grid_end + 0.45 + leg_rows * 0.30 + margin + + fig = plt.figure(figsize=(W, H), facecolor="white") + ax = fig.add_axes([0, 0, 1, 1]) + ax.set_xlim(0, W) + ax.set_ylim(H, 0) + ax.axis("off") + + # ---- title block ------------------------------------------------------- + title = title or f"{m1} vs. {m2} Scorecard" + if subtitle is None: + bits = [f"{df.attrs.get('linetype', '')}".strip()] if df.attrs.get("linetype") not in (None, "NA") else [] + bits += fixed + vr = date_range or df.attrs.get("valid_range") + if vr: + a, b = (pd.to_datetime(str(x)) for x in vr) + bits.append(a.strftime("%Y-%m-%d") if a == b else f"{a:%Y-%m-%d} to {b:%Y-%m-%d}") + if subtitle_extra: + bits.append(subtitle_extra) + subtitle = " \u00b7 ".join(b for b in bits if b) + ax.text(W / 2, margin + 0.18, title, ha="center", va="center", + fontsize=15, fontweight="bold", color="#1a1a1a") + if subtitle: + ax.text(W / 2, margin + 0.50, subtitle, ha="center", va="center", + fontsize=9.5, color="#555555") + + # ---- header ------------------------------------------------------------ + yb = margin + title_h + ax.text(gx + grid_w / 2, yb + banner_h / 2, "Forecast lead time (hours)", + ha="center", va="center", fontsize=FS + 0.5, fontweight="bold", color="#333333") + ax.plot([gx + 0.04, gx + grid_w - 0.04], [yb + banner_h, yb + banner_h], + color="#333333", lw=0.8) + for j, h in enumerate(leads): + ax.text(gx + (j + 0.5) * cw, yb + banner_h + lead_h / 2, _fmt_lead(h), + ha="center", va="center", fontsize=FS, color="#222222") + xs = [x0] + for w in col_w: + xs.append(xs[-1] + w) + for d, (k, hdr) in enumerate(cols): + ax.text(xs[d] + 0.06, yb + banner_h + lead_h / 2, hdr, ha="left", va="center", + fontsize=FS, fontweight="bold", color="#333333") + ax.plot([x0, gx + grid_w], [y_grid, y_grid], color="#1a1a1a", lw=1.4) + + # ---- cells --------------------------------------------------------------- + def colour_for(c, dark_text=False): + if c == 0: + return NOSIG_COLOR + lvl = int(abs(c)) - 1 + return pal["better" if c > 0 else "worse"][lvl] + + for i in range(nrow): + for j in range(ncol): + x, y = gx + j * cw, y_grid + i * rh + c = code[i, j] + if np.isnan(c): + ax.add_patch(Rectangle((x, y), cw, rh, facecolor="white", + edgecolor="#bdbdbd", hatch="////", lw=0.0)) + ax.add_patch(Rectangle((x, y), cw, rh, facecolor="none", + edgecolor="white", lw=1.5)) + continue + ax.add_patch(Rectangle((x, y), cw, rh, facecolor=colour_for(c), + edgecolor="white", lw=1.5)) + txt_col = "white" if abs(c) == 3 else "#1a1a1a" + cx, cy = x + cw / 2, y + rh / 2 + if markers and c != 0: + size = {1: 14, 2: 26, 3: 42}[int(abs(c))] + mx, my = (x + 0.12, y + 0.12) if show_values else (cx, cy) + ax.scatter([mx], [my], marker="^" if c > 0 else "v", s=size, + color=txt_col, linewidths=0, zorder=3) + if show_values and not np.isnan(val[i, j]): + ax.text(cx, cy + (0.03 if (markers and c != 0) else 0), f"{val[i, j]:+.2f}", + ha="center", va="center", fontsize=7.5, color=txt_col, zorder=3) + + # ---- hierarchical row labels + separators ------------------------------- + ncols_l = len(cols) + for d in range(ncols_l): + prefixes = [rk[:d + 1] for rk in row_keys] + s = 0 + while s < nrow: + e = s + while e + 1 < nrow and prefixes[e + 1] == prefixes[s]: + e += 1 + yc = y_grid + (s + e + 1) / 2 * rh + ax.text(xs[d] + 0.06, yc, disp(row_keys[s][d]), ha="left", va="center", + fontsize=FS, color="#1a1a1a", + fontweight="bold" if d == 0 else "normal") + if s > 0: + ys = y_grid + s * rh + lw, colr = (1.2, "#333333") if d == 0 else (0.8, "#888888") if d == 1 else (0.4, "#cccccc") + ax.plot([xs[d], gx + grid_w], [ys, ys], color=colr, lw=lw, zorder=2) + s = e + 1 + ax.plot([x0, gx + grid_w], [y_grid_end, y_grid_end], color="#1a1a1a", lw=1.4) + + # ---- legend ---------------------------------------------------------------- + ly = y_grid_end + 0.45 + lx = [(W - 2 * leg_col_w) / 2, (W - 2 * leg_col_w) / 2 + leg_col_w] + sw = 0.30 + + def key(x, y, c, label): + ax.add_patch(Rectangle((x, y - 0.10), sw, 0.20, + facecolor=colour_for(c) if c is not None else "white", + edgecolor="#999999" if c is None else "white", lw=0.8, + hatch="////" if c is None else None)) + if markers and c not in (None, 0): + ax.scatter([x + sw / 2], [y], marker="^" if c > 0 else "v", + s={1: 14, 2: 26, 3: 42}[abs(c)], linewidths=0, zorder=3, + color="white" if abs(c) == 3 else "#1a1a1a") + ax.text(x + sw + 0.12, y, label, ha="left", va="center", fontsize=8.5, color="#222222") + + for k, lvl in enumerate([3, 2, 1]): + key(lx[0], ly + k * 0.30, lvl, f"{m1} better than {m2} ({LEVEL_PCT[lvl]})") + key(lx[1], ly + k * 0.30, -lvl, f"{m1} worse than {m2} ({LEVEL_PCT[lvl]})") + key(lx[0], ly + 3 * 0.30, 0, "No statistically significant difference") + key(lx[1], ly + 3 * 0.30, None, "No data") + + if outfile: + fig.savefig(outfile, dpi=dpi, facecolor="white") + return fig + + +# -------------------------------------------------------------------------- +STYLE_KEYS = {"title", "subtitle", "palette", "show_values", "markers", "formats", + "dpi", "label_map", "model_names"} + + +def render_from_config(stats_file, out_base, style=None, model_names=None, + date_range=None, title=None, subtitle_extra=None): + """Config-driven entry point used by scorecard_plot.py (and handy on its own). + + stats_file : path to scorecard_stats.txt + out_base : output path WITHOUT extension, e.g. /scorecard + style : dict, typically the `scorecard_style:` block of the YAML. + Recognised keys (all optional): + title, subtitle, palette (metviewer|colorblind), + show_values (bool), markers (bool), formats (list of + png/pdf/svg), dpi, label_map ({raw: display}), + model_names ([series 1, series 2]) + model_names, date_range, title : defaults used when the style block does + not give them (the YAML's own model list, ymd_start/ymd_end, + and scorecard_title). + Returns the list of files written. + """ + style = dict(style or {}) + unknown = sorted(set(style) - STYLE_KEYS) + if unknown: + warnings.warn(f"unknown scorecard_style key(s) ignored: {unknown}; " + f"valid keys: {sorted(STYLE_KEYS)}") + names = style.get("model_names") or model_names or [None, None] + formats = style.get("formats", ["png"]) + if isinstance(formats, str): + formats = [formats] + bad = [f for f in formats if f.lower() not in ("png", "pdf", "svg")] + if bad: + raise ValueError(f"unsupported scorecard format(s) {bad}; use png, pdf or svg") + + df = load_scorecard(stats_file) + summarize(df) + fig = render_scorecard( + df, None, + title=style.get("title") or title, + subtitle=style.get("subtitle"), + model1=names[0], model2=names[1], + palette=style.get("palette", "metviewer"), + show_values=bool(style.get("show_values", False)), + markers=bool(style.get("markers", True)), + dpi=int(style.get("dpi", 200)), + label_map=style.get("label_map"), + date_range=date_range, subtitle_extra=subtitle_extra) + written = [] + for fmt in formats: + path = f"{out_base}.{fmt.lower()}" + fig.savefig(path, dpi=int(style.get("dpi", 200)), facecolor="white") + written.append(path) + plt.close(fig) + return written + + +def main(argv=None): + p = argparse.ArgumentParser(description="Render a METplus DIFF_SIG scorecard with matplotlib.") + p.add_argument("stats_file", nargs="?", help="scorecard_stats.txt (tab-separated)") + p.add_argument("-o", "--output", default="scorecard.png", help="png, pdf or svg") + p.add_argument("--title") + p.add_argument("--subtitle") + p.add_argument("--model1", help="name of series 1 (default: parsed from file)") + p.add_argument("--model2", help="name of series 2 (default: parsed from file)") + p.add_argument("--palette", choices=sorted(PALETTES), default="metviewer") + p.add_argument("--show-values", action="store_true", + help="print the signed DIFF_SIG confidence value in each cell") + p.add_argument("--no-markers", action="store_true", help="colour only, no triangles") + p.add_argument("--dpi", type=int, default=200) + p.add_argument("--demo", action="store_true", help="use synthetic data") + a = p.parse_args(argv) + + if a.demo: + df = make_demo_df() + elif a.stats_file: + df = load_scorecard(a.stats_file) + else: + p.error("give a stats file or --demo") + summarize(df) + render_scorecard(df, a.output, title=a.title, subtitle=a.subtitle, + model1=a.model1, model2=a.model2, palette=a.palette, + show_values=a.show_values, markers=not a.no_markers, dpi=a.dpi) + print(f"wrote {a.output}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/taylor_diagram/test_taylor_diagram.py b/test/taylor_diagram/test_taylor_diagram.py index 8d8f8dbd..630fad62 100644 --- a/test/taylor_diagram/test_taylor_diagram.py +++ b/test/taylor_diagram/test_taylor_diagram.py @@ -6,7 +6,8 @@ # Converts MatplotlibDeprecation warnings (which are DeprecationWarning) into errors as # any DeprecationWarnings should be fixed as soon as possible -# pytestmark = pytest.mark.filterwarnings("error::DeprecationWarning") +#pytestmark = pytest.mark.filterwarnings("error::DeprecationWarning") + @pytest.mark.parametrize("input_yaml,expected_files", [ ("test_pos_corr.yaml", ["test_pos_corr_plot.png"]),