From cbee385ea4934da04c8897399e8d5ff305500e77 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 12 Sep 2024 17:10:10 -0400 Subject: [PATCH] Replace whitespace in Ref, Dev labels with underscores gcpy/benchmark/modules/benchmark_drydep.py - Import "get_common_varnames" from benchmark_utils.py (suggested by Pylint) - Call util.replace_whitespace on refstr and devstr gcpy/benchmark/modules/benchmark_funcs.py - Import replace_whitespace from gcpy.util module - Call replace_whitespace on refstr and devstr in several routines - Update comments gcpy/benchmark/modules/benchmark_gcclassic_stats.py gcpy/benchmark/modules/benchmark_mass_cons_table.py gcpy/benchmark/modules/benchmark_models_vs_sondes.py gcpy/benchmark/modules/benchmark_scrape_gcclassic_timers.py gcpy/benchmark/modules/benchmark_scrape_gchp_timers.py gcpy/benchmark/modules/oh_metrics.py - Import replace_whitespace from gcpy.util module - Call replace_whitespace on ref_label and dev_label gcpy/benchmark/modules/benchmark_models_vs_obs.py - Import replace_whitespace from gcpy.util module - Call replace_whitespace on ref_label and dev_label - Initialize lon, lat, alt to zero (suggested by Pylint) - Align { } brackets gcpy/benchmark/modules/budget_ox.py - Move import gc higher in the list (suggested by Pylint) - Now use "from gcpy import constants" (suggested by Pylint) - Import replace_whitespace from gcpy.util module - Call replace_whitespace on devstr - Trim trailing whitespace (suggested by Pylint) gcpy/benchmark/modules/budget_tt.py - Move import gc higher in the list (suggested by Pylint) - Now use "from gcpy import constants" (suggested by Pylint) - Now use "from gcpy import util" (suggested by Pylint) - Remove "from glob import glob" (suggested by Pylint) - Call util.replace_whitespace on devstr - Trim trailing whitespace (suggested by Pylint) gcpy/benchmark/modules/budget_ox.py - Move import gc higher in the list (suggested by Pylint) - Now use "from gcpy import constants" (suggested by Pylint) - Import replace_whitespace from gcpy.util module - Call replace_whitespace on devstr - Trim trailing whitespace (suggested by Pylint) gcpy/benchmark/modules/ste_flux.py - Call util.replace_whitespace on devstr CHANGELOG.md - Updated accordingly Signed-off-by: Bob Yantosca --- CHANGELOG.md | 1 + gcpy/benchmark/modules/benchmark_drydep.py | 9 ++- gcpy/benchmark/modules/benchmark_funcs.py | 68 ++++++++++++++++++- .../modules/benchmark_gcclassic_stats.py | 8 ++- .../modules/benchmark_mass_cons_table.py | 8 ++- .../modules/benchmark_models_vs_obs.py | 25 ++++--- .../modules/benchmark_models_vs_sondes.py | 6 +- .../benchmark_scrape_gcclassic_timers.py | 4 ++ .../modules/benchmark_scrape_gchp_timers.py | 4 ++ gcpy/benchmark/modules/budget_ox.py | 15 ++-- gcpy/benchmark/modules/budget_tt.py | 9 ++- gcpy/benchmark/modules/oh_metrics.py | 7 +- gcpy/benchmark/modules/ste_flux.py | 2 +- 13 files changed, 134 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f697d1..4bbee08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - In environment files `read_the_docs_environment.yml` and `read_the_docs_requirements.txt` - Update `jinja` to 3.1.4 (fixes a security issue) - Update `gcpy/setup.py` with the new Python package version numbers +- Updated code in `gcpy/benchmark/modules/` to replace whitespace in Ref and Dev labels with underscores ### Fixed - Fixed formatting error in `.github/workflows/stale.yml` that caused the Mark Stale Issues action not to run diff --git a/gcpy/benchmark/modules/benchmark_drydep.py b/gcpy/benchmark/modules/benchmark_drydep.py index aa2809f..948067d 100644 --- a/gcpy/benchmark/modules/benchmark_drydep.py +++ b/gcpy/benchmark/modules/benchmark_drydep.py @@ -1,14 +1,13 @@ """ Specific utilities for creating plots from GEOS-Chem benchmark simulations. """ -import os import gc import numpy as np from gcpy import util from gcpy.plot.compare_single_level import compare_single_level from gcpy.benchmark.modules.benchmark_utils import \ - get_lumped_species_definitions, make_output_dir, \ - pdf_filename, print_sigdiffs, read_ref_and_dev + get_common_varnames, make_output_dir, pdf_filename, \ + print_sigdiffs, read_ref_and_dev # Suppress numpy divide by zero warnings to prevent output spam np.seterr(divide="ignore", invalid="ignore") @@ -95,6 +94,10 @@ def make_benchmark_drydep_plots( msg = "The spcdb_dir argument has not been specified!" raise ValueError(msg) + # Replace whitespace in the ref and dev labels + refstr = util.replace_whitespace(refstr) + devstr = util.replace_whitespace(devstr) + # Create directory for plots (if it doesn't exist) dst = make_output_dir( dst, diff --git a/gcpy/benchmark/modules/benchmark_funcs.py b/gcpy/benchmark/modules/benchmark_funcs.py index 38118dd..34d1ed1 100644 --- a/gcpy/benchmark/modules/benchmark_funcs.py +++ b/gcpy/benchmark/modules/benchmark_funcs.py @@ -16,6 +16,7 @@ from gcpy import util from gcpy.regrid import create_regridders from gcpy.grid import get_troposphere_mask +from gcpy.util import replace_whitespace from gcpy.units import convert_units from gcpy.constants import COL_WIDTH, MW_AIR_g, skip_these_vars, TABLE_WIDTH from gcpy.plot.compare_single_level import compare_single_level @@ -155,6 +156,10 @@ def create_total_emissions_table( quiet=True ) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # ================================================================== # Get the list of emission variables for which we will print totals # ================================================================== @@ -447,6 +452,10 @@ def create_global_mass_table( quiet=True ) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # ================================================================== # Open file for output # ================================================================== @@ -692,6 +701,10 @@ def create_mass_accumulation_table( quiet=True ) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # ================================================================== # Open file for output # ================================================================== @@ -1032,6 +1045,10 @@ def make_benchmark_conc_plots( else: extra_title_txt = None + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # Pick the proper function to read the data reader = util.dataset_reader(time_mean, verbose=verbose) @@ -1710,6 +1727,10 @@ def make_benchmark_emis_plots( else: extra_title_txt = None + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # Get the function that will read the dataset reader = util.dataset_reader(time_mean, verbose=verbose) @@ -2098,6 +2119,10 @@ def make_benchmark_emis_tables( if not os.path.isdir(emisdir): os.mkdir(emisdir) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # ================================================================== # Read data from netCDF into Dataset objects # ================================================================== @@ -2343,6 +2368,10 @@ def make_benchmark_jvalue_plots( # Create the directory for output util.make_directory(dst, overwrite) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # Get the function that will read file(s) into a Dataset reader = util.dataset_reader(time_mean, verbose=verbose) @@ -2727,6 +2756,10 @@ def make_benchmark_aod_plots( else: extra_title_txt = None + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # Get the function that will read file(s) into a dataset reader = util.dataset_reader(time_mean, verbose=verbose) @@ -3009,11 +3042,19 @@ def make_benchmark_mass_tables( which do not contain the Area variable. Default value: '' """ + # ================================================================== + # Initialization + # ================================================================== + # Make sure the species database folder is passed if spcdb_dir is None: msg = "The 'spcdb_dir' argument has not been specified!" raise ValueError(msg) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # ================================================================== # Define destination directory # ================================================================== @@ -3275,11 +3316,19 @@ def make_benchmark_mass_accumulation_tables( Directory of species_datbase.yml file Default value: None """ + # ================================================================== + # Initialization + # ================================================================== + # Make sure the species database folder is passed if spcdb_dir is None: msg = "The 'spcdb_dir' argument has not been specified!" raise ValueError(msg) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # ================================================================== # Define destination directory # ================================================================== @@ -3543,10 +3592,16 @@ def make_benchmark_oh_metrics( """ # ================================================================== - # Define destination directory + # Initialization # ================================================================== + + # Define destination directory util.make_directory(dst, overwrite) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # ================================================================== # Read data from netCDF into Dataset objects # ================================================================== @@ -3833,6 +3888,10 @@ def make_benchmark_wetdep_plots( if not os.path.isdir(targetdst): os.mkdir(targetdst) + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # Get the function that will read file(s) into a dataset reader = util.dataset_reader(time_mean, verbose=verbose) @@ -4388,6 +4447,9 @@ def make_benchmark_operations_budget( Directory containing the species_database.yml file. Default value: None """ + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) # ------------------------------------------ # Column sections @@ -5083,6 +5145,10 @@ def create_benchmark_summary_table( # Open file for output # ================================================================== + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # Create the directory for output util.make_directory(dst, overwrite) diff --git a/gcpy/benchmark/modules/benchmark_gcclassic_stats.py b/gcpy/benchmark/modules/benchmark_gcclassic_stats.py index 819c0a8..bd2089f 100644 --- a/gcpy/benchmark/modules/benchmark_gcclassic_stats.py +++ b/gcpy/benchmark/modules/benchmark_gcclassic_stats.py @@ -9,7 +9,7 @@ """ import sys import requests -from gcpy.util import verify_variable_type +from gcpy.util import replace_whitespace, verify_variable_type # ---------------------------------------------------------------------- # Global variables @@ -113,7 +113,7 @@ def scrape_stats(text): stats["CH3CCl3"] = line.split(":")[1].strip() if line_count == 18 and "Dev" in line: stats["Mean OH"] = line.split(":")[1].strip() - + # Skip commands if "++ sed" in line: line_count += 1 @@ -168,6 +168,10 @@ def main(ref_label, dev_label): verify_variable_type(ref_label, str) verify_variable_type(dev_label, str) + # Replace whitespace in the ref and dev labels + ref_label = replace_whitespace(ref_label) + dev_label = replace_whitespace(dev_label) + # Scrape the log file text into a variable bmk_id = f"gcc-4x5-1Mon-{dev_label}" text = get_text_from_web(LOG_TEMPLATE.replace("ID", bmk_id)) diff --git a/gcpy/benchmark/modules/benchmark_mass_cons_table.py b/gcpy/benchmark/modules/benchmark_mass_cons_table.py index 6ebfeb0..8d4adfa 100644 --- a/gcpy/benchmark/modules/benchmark_mass_cons_table.py +++ b/gcpy/benchmark/modules/benchmark_mass_cons_table.py @@ -9,8 +9,8 @@ from gcpy.constants import skip_these_vars from gcpy.units import convert_units from gcpy.util import \ - replace_whitespace, dataset_reader, get_area_from_dataset, \ - make_directory, read_config_file, verify_variable_type + dataset_reader, get_area_from_dataset, make_directory, \ + read_config_file, replace_whitespace, verify_variable_type from gcpy.benchmark.modules.benchmark_utils import \ get_datetimes_from_filenames @@ -317,6 +317,10 @@ def make_benchmark_mass_conservation_table( # Create the destination folder make_directory(dst, overwrite) + # Replace whitespace in the ref and dev labels + ref_label = replace_whitespace(ref_label) + dev_label = replace_whitespace(dev_label) + # Get a list of properties for the given species metadata = get_passive_tracer_metadata(spcdb_dir) diff --git a/gcpy/benchmark/modules/benchmark_models_vs_obs.py b/gcpy/benchmark/modules/benchmark_models_vs_obs.py index d727a45..0955873 100644 --- a/gcpy/benchmark/modules/benchmark_models_vs_obs.py +++ b/gcpy/benchmark/modules/benchmark_models_vs_obs.py @@ -21,7 +21,8 @@ import numpy as np import xarray as xr from gcpy.constants import skip_these_vars -from gcpy.util import verify_variable_type, dataset_reader, make_directory +from gcpy.util import \ + dataset_reader, make_directory, replace_whitespace, verify_variable_type from gcpy.cstools import extract_grid from gcpy.grid import get_nearest_model_data from gcpy.benchmark.modules.benchmark_utils import \ @@ -51,6 +52,11 @@ def read_nas( if verbose: print(f"read_nas: Reading {input_file}") + # Initialize + lon = 0.0 + lat = 0.0 + alt = 0.0 + with open(input_file, encoding='UTF-8') as the_file: header = np.array( [next(the_file) for x in range(155) ] @@ -113,11 +119,11 @@ def read_nas( index=obs_dataframe.index ) obs_site_coords = { site: - { - 'lon': lon, - 'lat': lat, - 'alt': alt - } + { + 'lon': lon, + 'lat': lat, + 'alt': alt + } } return obs_dataframe, obs_site_coords @@ -194,13 +200,12 @@ def read_model_data( Returns dataarray : xr.DataArray : GEOS-Chem data read from disk """ - # Read the Ref and Dev model data reader = dataset_reader( multi_files=True, verbose=verbose, ) - + # Read data and rename SpeciesConc_ to SpeciesConcVV_, if necessary # (needed for backwards compatibility with older versions.) dataset = reader(filepaths,drop_variables=skip_these_vars).load() @@ -755,6 +760,10 @@ def make_benchmark_models_vs_obs_plots( verify_variable_type(dev_filepaths, (str, list)) verify_variable_type(dev_label, str) + # Replace whitespace in the ref and dev labels + ref_label = replace_whitespace(ref_label) + dev_label = replace_whitespace(dev_label) + # Create the destination folder make_directory( dst, diff --git a/gcpy/benchmark/modules/benchmark_models_vs_sondes.py b/gcpy/benchmark/modules/benchmark_models_vs_sondes.py index 63f85cb..b068c00 100644 --- a/gcpy/benchmark/modules/benchmark_models_vs_sondes.py +++ b/gcpy/benchmark/modules/benchmark_models_vs_sondes.py @@ -11,7 +11,7 @@ import matplotlib.pyplot as plt from gcpy.cstools import extract_grid from gcpy.grid import get_nearest_model_data, get_vert_grid -from gcpy.util import make_directory, verify_variable_type +from gcpy.util import make_directory, replace_whitespace, verify_variable_type from gcpy.benchmark.modules.benchmark_utils import \ read_ref_and_dev, rename_speciesconc_to_speciesconcvv @@ -522,6 +522,10 @@ def make_benchmark_models_vs_sondes_plots( verify_variable_type(dev_filepaths, (str, list)) verify_variable_type(dev_label, str) + # Replace whitespace in the ref and dev labels + ref_label = replace_whitespace(ref_label) + dev_label = replace_whitespace(dev_label) + # Create the destination folder make_directory( dst, diff --git a/gcpy/benchmark/modules/benchmark_scrape_gcclassic_timers.py b/gcpy/benchmark/modules/benchmark_scrape_gcclassic_timers.py index f828e22..3471af6 100644 --- a/gcpy/benchmark/modules/benchmark_scrape_gcclassic_timers.py +++ b/gcpy/benchmark/modules/benchmark_scrape_gcclassic_timers.py @@ -261,6 +261,10 @@ def make_benchmark_gcclassic_timing_table( # Create the destination folder make_directory(dst, overwrite) + # Replace whitespace in the ref and dev labels + ref_label = replace_whitespace(ref_label) + dev_label = replace_whitespace(dev_label) + # Strip timing info from JSON/text file(s) and sum the them. ref_timers = sum_timers(read_gcclassic(ref_files)) dev_timers = sum_timers(read_gcclassic(dev_files)) diff --git a/gcpy/benchmark/modules/benchmark_scrape_gchp_timers.py b/gcpy/benchmark/modules/benchmark_scrape_gchp_timers.py index 4c42b89..c4a6495 100644 --- a/gcpy/benchmark/modules/benchmark_scrape_gchp_timers.py +++ b/gcpy/benchmark/modules/benchmark_scrape_gchp_timers.py @@ -331,6 +331,10 @@ def make_benchmark_gchp_timing_table( # Create the destination folder make_directory(dst, overwrite) + # Replace whitespace in the ref and dev labels + ref_label = replace_whitespace(ref_label) + dev_label = replace_whitespace(dev_label) + # Strip timing info from JSON/text file(s) and sum the them. ref_timers = sum_timers(read_timing_data(ref_files)) dev_timers = sum_timers(read_timing_data(dev_files)) diff --git a/gcpy/benchmark/modules/budget_ox.py b/gcpy/benchmark/modules/budget_ox.py index 43c3d4b..5d4ba07 100644 --- a/gcpy/benchmark/modules/budget_ox.py +++ b/gcpy/benchmark/modules/budget_ox.py @@ -10,15 +10,16 @@ import os import warnings from calendar import monthrange +import gc import numpy as np import xarray as xr -import gcpy.constants as constants +from gcpy import constants from gcpy.grid import get_troposphere_mask from gcpy.util import get_filepath, read_config_file, \ - rename_and_flip_gchp_rst_vars, reshape_MAPL_CS + rename_and_flip_gchp_rst_vars, reshape_MAPL_CS, \ + replace_whitespace from gcpy.benchmark.modules.benchmark_utils import \ add_lumped_species_to_dataset, get_lumped_species_definitions -import gc # Suppress harmless run-time warnings (mostly about underflow in division) warnings.filterwarnings("ignore", category=RuntimeWarning) @@ -75,7 +76,7 @@ def __init__( # -------------------------------------------------------------- # Arguments from outside # -------------------------------------------------------------- - self.devstr = devstr + self.devstr = replace_whitespace(devstr) self.devdir = devdir self.devrstdir = devrstdir self.dst = dst @@ -362,7 +363,7 @@ def init_and_final_mass( g100 = 100.0 / constants.G airmass_ini = (deltap_ini * globvars.area_m2.values) * g100 airmass_end = (deltap_end * globvars.area_m2.values) * g100 - + # Conversion factors mw_ratio = globvars.mw["O3"] / globvars.mw["Air"] kg_to_tg = 1.0e-9 @@ -466,11 +467,11 @@ def annual_average_drydep( mw_avo = (globvars.mw["Ox"] / constants.AVOGADRO) kg_to_tg = 1.0e-9 area_cm2 = globvars.area_cm2.values - + # Get drydep flux of Ox [molec/cm2/s] dry = globvars.ds_dry["DryDep_Ox"].values - # Convert to Tg Ox + # Convert to Tg Ox dry_tot = 0.0 for t in range(globvars.N_MONTHS): dry_tot += np.nansum(dry[t, :, :] * area_cm2) * globvars.frac_of_a[t] diff --git a/gcpy/benchmark/modules/budget_tt.py b/gcpy/benchmark/modules/budget_tt.py index 2475812..cabaa5e 100644 --- a/gcpy/benchmark/modules/budget_tt.py +++ b/gcpy/benchmark/modules/budget_tt.py @@ -10,17 +10,16 @@ # ====================================================================== import os -from glob import glob import warnings from calendar import monthrange +import gc import numpy as np import xarray as xr -import gcpy.constants as constants +from gcpy import constants from gcpy.grid import get_troposphere_mask -import gcpy.util as util +from gcpy import util from gcpy.benchmark.modules.benchmark_utils import \ rename_speciesconc_to_speciesconcvv -import gc # Suppress harmless run-time warnings (mostly about underflow in division) warnings.filterwarnings("ignore", category=RuntimeWarning) @@ -67,7 +66,7 @@ def __init__(self, devstr, devdir, devrstdir, year, dst, is_gchp, # ------------------------------ # Arguments from outside # ------------------------------ - self.devstr = devstr + self.devstr = util.replace_whitespace(devstr) self.devdir = devdir self.devrstdir = devrstdir self.dst = dst diff --git a/gcpy/benchmark/modules/oh_metrics.py b/gcpy/benchmark/modules/oh_metrics.py index 7f9b35a..6f42206 100644 --- a/gcpy/benchmark/modules/oh_metrics.py +++ b/gcpy/benchmark/modules/oh_metrics.py @@ -15,7 +15,7 @@ import numpy as np import xarray as xr import gcpy.constants as const -from gcpy.util import make_directory, read_config_file +from gcpy.util import make_directory, read_config_file, replace_whitespace # ===================================================================== # %%% METHODS %%% @@ -219,7 +219,6 @@ def init_common_vars(ref, refstr, dev, devstr, spcdb_dir): Returns: common_vars: dict """ - # Get species database spcdb = read_config_file( os.path.join( @@ -470,6 +469,10 @@ def make_benchmark_oh_metrics( if spcdb_dir is None: raise ValueError("The 'spcdb_dir' argument has not been specified!") + # Replace whitespace in the ref and dev labels + refstr = replace_whitespace(refstr) + devstr = replace_whitespace(devstr) + # Tell matplotlib not to look for an X-window os.environ["QT_QPA_PLATFORM"] = "offscreen" diff --git a/gcpy/benchmark/modules/ste_flux.py b/gcpy/benchmark/modules/ste_flux.py index 826c469..df40522 100644 --- a/gcpy/benchmark/modules/ste_flux.py +++ b/gcpy/benchmark/modules/ste_flux.py @@ -59,7 +59,7 @@ def __init__(self, devstr, files, dst, year, # ------------------------------ # Arguments from outside # ------------------------------ - self.devstr = devstr + self.devstr = util.replace_whitespace(devstr) self.files = files self.dst = dst self.overwrite = overwrite