From e553a326b8e53938c423e8d24192962b5eb6bb67 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 24 Jul 2024 02:09:41 -0500 Subject: [PATCH 1/5] Add support to parse bash array as python array Adds the capability to parse a bash array in a configuration as a python array. Note: since arrays cannot reliably be exported in shell, such variables should only be used locally in a config or by python that is parsing it. --- src/wxflow/configuration.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 4a152d3..6d944ed 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -156,21 +156,23 @@ def cast_strdict_as_dtypedict(ctx: Dict[str, str]) -> Dict[str, Any]: def cast_as_dtype(string: str) -> Union[str, int, float, bool, Any]: """ Cast a value into known datatype + Parameters ---------- string: str + Returns ------- - value : str or int or float or datetime + value : str, int, float, bool or datetime; or List of these default: str """ TRUTHS = ['y', 'yes', 't', 'true', '.t.', '.true.'] BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS BOOLS = [x.upper() for x in BOOLS] + BOOLS + ['Yes', 'No', 'True', 'False'] - def _cast_or_not(type: Any, string: str): + def _cast_or_not(to_type: Any, string: str): try: - return type(string) + return to_type(string) except ValueError: return string @@ -185,6 +187,9 @@ def _true_or_not(string: str): except Exception as exc: if string in BOOLS: # Likely a boolean, convert to True/False return _true_or_not(string) + elif string.startswith('(') and string.endswith(')'): + # Convert bash array to python array + return [ cast_as_dtype(elem) for elem in string[1:-1].split() ] elif '.' in string: # Likely a number and that too a float return _cast_or_not(float, string) else: # Still could be a number, may be an integer From bb1eab416ba96ac259df389f19b260dd577eb11b Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 25 Jul 2024 05:22:11 -0500 Subject: [PATCH 2/5] Modify list detection to use comma-separated values Previous method of reading bash arrays directly did not work because arrays cannot be exported and the configuration loader works by reading exported variables. Instead, we now assume any variable with a comma is a comma-separated list. --- src/wxflow/configuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 6d944ed..baee3fe 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -187,9 +187,9 @@ def _true_or_not(string: str): except Exception as exc: if string in BOOLS: # Likely a boolean, convert to True/False return _true_or_not(string) - elif string.startswith('(') and string.endswith(')'): - # Convert bash array to python array - return [ cast_as_dtype(elem) for elem in string[1:-1].split() ] + elif ',' in string: + # Convert comma-separated list to python list + return [ cast_as_dtype(elem) for elem in string.split(',') ] elif '.' in string: # Likely a number and that too a float return _cast_or_not(float, string) else: # Still could be a number, may be an integer From ef5ae37f994420f6231818b7d375ddb91481f10f Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 25 Jul 2024 06:09:40 -0500 Subject: [PATCH 3/5] Fix list parsing error, add pytest Checking for commas had to be moved to first in cast_as_dtype() to avoid commas being considered part of the datatime string. Also added pytest for list parsing. --- src/wxflow/configuration.py | 7 ++++--- tests/test_configuration.py | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index baee3fe..da1b530 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -170,6 +170,10 @@ def cast_as_dtype(string: str) -> Union[str, int, float, bool, Any]: BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS BOOLS = [x.upper() for x in BOOLS] + BOOLS + ['Yes', 'No', 'True', 'False'] + if ',' in string: + # Convert comma-separated list to python list + return [ cast_as_dtype(elem.strip()) for elem in string.split(',') ] + def _cast_or_not(to_type: Any, string: str): try: return to_type(string) @@ -187,9 +191,6 @@ def _true_or_not(string: str): except Exception as exc: if string in BOOLS: # Likely a boolean, convert to True/False return _true_or_not(string) - elif ',' in string: - # Convert comma-separated list to python list - return [ cast_as_dtype(elem) for elem in string.split(',') ] elif '.' in string: # Likely a number and that too a float return _cast_or_not(float, string) else: # Still could be a number, may be an integer diff --git a/tests/test_configuration.py b/tests/test_configuration.py index da1f926..04a94a1 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -30,6 +30,11 @@ export SOME_BOOL4=NO export SOME_BOOL5=.false. export SOME_BOOL6=.F. +export SOME_LIST1="3, 15, -999" +export SOME_LIST2="0.2,3.5,-9999." +export SOME_LIST3="20221225, 202212251845" +export SOME_LIST4="YES, .false., .T." +export SOME_LIST5="0.2, 15, 20221225, NO" """ file1 = """#!/bin/bash @@ -60,7 +65,12 @@ 'SOME_BOOL3': True, 'SOME_BOOL4': False, 'SOME_BOOL5': False, - 'SOME_BOOL6': False + 'SOME_BOOL6': False, + 'SOME_LIST1': [3, 15, -999], + 'SOME_LIST2': [0.2,3.5,-9999.], + 'SOME_LIST3': [datetime(2022, 12, 25, 0, 0, 0), datetime(2022, 12, 25, 18, 45, 0)], + 'SOME_LIST4': [True, False, True], + 'SOME_LIST5': [0.2, 15, datetime(2022, 12, 25, 0, 0, 0), False], } file0_dict_set_envvar = file0_dict.copy() @@ -107,6 +117,14 @@ ('20221215T1830Z', datetime(2022, 12, 15, 18, 30, 0)), ] +list_dtypes = [ + ('3, 15, -999', [3, 15, -999]), + ('0.2,3.5,-9999.', [0.2,3.5,-9999.]), + ('20221215,20221215T1830Z', [datetime(2022, 12, 15, 0, 0, 0), datetime(2022, 12, 15, 18, 30, 0)]), + ('YES, .false., .T.', [True, False, True]), + ('0.2, 15, 20221225, NO', [0.2, 15, datetime(2022, 12, 25, 0, 0, 0), False]), +] + def evaluate(dtypes): for pair in dtypes: @@ -133,6 +151,9 @@ def test_cast_as_dtype_bool(): def test_cast_as_dtype_datetimes(): evaluate(datetime_dtypes) +def test_cast_as_dtype_list(): + evaluate(list_dtypes) + @pytest.fixture def create_configs(tmp_path): From 92b525cdffa912a78d0746bdf6b680b2cee9f79c Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 25 Jul 2024 06:16:47 -0500 Subject: [PATCH 4/5] Fix pynorm issues --- src/wxflow/configuration.py | 2 +- tests/test_configuration.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index da1b530..f06a3f3 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -172,7 +172,7 @@ def cast_as_dtype(string: str) -> Union[str, int, float, bool, Any]: if ',' in string: # Convert comma-separated list to python list - return [ cast_as_dtype(elem.strip()) for elem in string.split(',') ] + return [cast_as_dtype(elem.strip()) for elem in string.split(',')] def _cast_or_not(to_type: Any, string: str): try: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 04a94a1..a849c74 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -67,7 +67,7 @@ 'SOME_BOOL5': False, 'SOME_BOOL6': False, 'SOME_LIST1': [3, 15, -999], - 'SOME_LIST2': [0.2,3.5,-9999.], + 'SOME_LIST2': [0.2, 3.5, -9999.], 'SOME_LIST3': [datetime(2022, 12, 25, 0, 0, 0), datetime(2022, 12, 25, 18, 45, 0)], 'SOME_LIST4': [True, False, True], 'SOME_LIST5': [0.2, 15, datetime(2022, 12, 25, 0, 0, 0), False], @@ -119,7 +119,7 @@ list_dtypes = [ ('3, 15, -999', [3, 15, -999]), - ('0.2,3.5,-9999.', [0.2,3.5,-9999.]), + ('0.2,3.5,-9999.', [0.2, 3.5, -9999.]), ('20221215,20221215T1830Z', [datetime(2022, 12, 15, 0, 0, 0), datetime(2022, 12, 15, 18, 30, 0)]), ('YES, .false., .T.', [True, False, True]), ('0.2, 15, 20221225, NO', [0.2, 15, datetime(2022, 12, 25, 0, 0, 0), False]), @@ -151,6 +151,7 @@ def test_cast_as_dtype_bool(): def test_cast_as_dtype_datetimes(): evaluate(datetime_dtypes) + def test_cast_as_dtype_list(): evaluate(list_dtypes) From c68d7dcb0f579b28bee02c81a01ea42c21c1a7d7 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 25 Jul 2024 07:03:44 -0500 Subject: [PATCH 5/5] Update pytest to test string in list --- tests/test_configuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index a849c74..1070226 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -34,7 +34,7 @@ export SOME_LIST2="0.2,3.5,-9999." export SOME_LIST3="20221225, 202212251845" export SOME_LIST4="YES, .false., .T." -export SOME_LIST5="0.2, 15, 20221225, NO" +export SOME_LIST5="0.2, test_str, 15, 20221225, NO" """ file1 = """#!/bin/bash @@ -70,7 +70,7 @@ 'SOME_LIST2': [0.2, 3.5, -9999.], 'SOME_LIST3': [datetime(2022, 12, 25, 0, 0, 0), datetime(2022, 12, 25, 18, 45, 0)], 'SOME_LIST4': [True, False, True], - 'SOME_LIST5': [0.2, 15, datetime(2022, 12, 25, 0, 0, 0), False], + 'SOME_LIST5': [0.2, 'test_str', 15, datetime(2022, 12, 25, 0, 0, 0), False], } file0_dict_set_envvar = file0_dict.copy() @@ -122,7 +122,7 @@ ('0.2,3.5,-9999.', [0.2, 3.5, -9999.]), ('20221215,20221215T1830Z', [datetime(2022, 12, 15, 0, 0, 0), datetime(2022, 12, 15, 18, 30, 0)]), ('YES, .false., .T.', [True, False, True]), - ('0.2, 15, 20221225, NO', [0.2, 15, datetime(2022, 12, 25, 0, 0, 0), False]), + ('0.2, test_str, 15, 20221225, NO', [0.2, 'test_str', 15, datetime(2022, 12, 25, 0, 0, 0), False]), ]