From 7023a32652f9808e5407c124706dd7097017392a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 13:51:49 -0500 Subject: [PATCH 1/3] add tests --- ush/python/pygw/src/pygw/file_utils.py | 76 ++++++++++++++++++ ush/python/pygw/src/pygw/utils.py | 81 -------------------- ush/python/pygw/src/tests/test_file_utils.py | 26 +++++++ 3 files changed, 102 insertions(+), 81 deletions(-) create mode 100644 ush/python/pygw/src/pygw/file_utils.py delete mode 100644 ush/python/pygw/src/pygw/utils.py create mode 100644 ush/python/pygw/src/tests/test_file_utils.py diff --git a/ush/python/pygw/src/pygw/file_utils.py b/ush/python/pygw/src/pygw/file_utils.py new file mode 100644 index 00000000000..2a7e8a6d08a --- /dev/null +++ b/ush/python/pygw/src/pygw/file_utils.py @@ -0,0 +1,76 @@ +from .fsutils import cp, mkdir + +__all__ = ['FileHandler'] + + +class FileHandler: + """Class to manipulate files in bulk for a given configuration + + Parameters + ---------- + config : dict + A dictionary containing the "action" and the "act" in the form of a list + + NOTE + ---- + "action" can be one of mkdir", "copy", etc. + Corresponding "act" would be ['dir1', 'dir2'], [['src1', 'dest1'], ['src2', 'dest2']] + + Attributes + ---------- + config : dict + Dictionary of files to manipulate + """ + + def __init__(self, config): + + self.config = config + self.sync(config) + + @staticmethod + def sync(config): + """ + Method to execute bulk actions on files described in the configuration + """ + sync_factory = { + 'copy': _copy_files, + 'mkdir': _make_dirs, + } + # loop through the configuration keys + for action, files in config.items(): + sync_factory[action](files) + + @staticmethod + def _copy_files(filelist): + """Function to copy all files specified in the list + + `filelist` should be in the form: + - [src, dest] + + Parameters + ---------- + filelist : list + List of lists of [src, dest] + """ + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + dest = sublist[1] + cp(src, dest) + print(f'Copied {src} to {dest}') # TODO use logger + + + @staticmethod + def _make_dirs(dirlist): + """Function to make all directories specified in the list + + Parameters + ---------- + dirlist : list + List of directories to create + """ + for dd in dirlist: + mkdir(dd) + print(f'Created {dd}') # TODO use logger diff --git a/ush/python/pygw/src/pygw/utils.py b/ush/python/pygw/src/pygw/utils.py deleted file mode 100644 index 66f85529d61..00000000000 --- a/ush/python/pygw/src/pygw/utils.py +++ /dev/null @@ -1,81 +0,0 @@ -from .yaml_file import YAMLFile -from .fsutils import cp, mkdir - -__all__ = ['FileHandler'] - - -class FileHandler: - """Class to manipulate files in bulk for a given configuration - - Parameters - ---------- - path : str, optional - Path to input YAML file containing files to manipulate - config : AttrDict, optional - Configuration AttrDict containing files to manipulate - - NOTE - ---- - While both `path` and `config` are optional, exactly one must be specified - or else an exception will be raised. - - Attributes - ---------- - config : AttrDict - Configuration of files to manipulate - """ - - def __init__(self, path=None, config=None): - - if path and config: - raise Exception('Both path and config are defined. Only one can be.') - elif path: - # read in the YAML file for the configuration - self.config = YAMLFile(path=path) - elif config: - self.config = config - else: - raise Exception("Neither 'path' nor 'config' are defined. One needs to be.") - - def sync(self): - """ - Method to execute bulk actions on files described in the configuration - """ - sync_factory = { - 'copy': copy_files, - 'mkdir': make_dirs, - } - # loop through the configuration keys - for action, files in self.config.items(): - sync_factory[action](files) - -def copy_files(filelist): - """Function to copy all files specified in the list - - `filelist` should be in the form: - - [src, dest] - - Parameters - ---------- - filelist : list - List of lists of [src, dest] - """ - for sublist in filelist: - if len(sublist) != 2: - raise Exception(f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - dest = sublist[1] - cp(src, dest) - print(f'Copied {src} to {dest}') # TODO use logger - -def make_dirs(dirlist): - """Function to make all directories specified in the list - - Parameters - ---------- - dirlist : list - List of directories to create - """ - for dd in dirlist: - mkdir(dd) - print(f'Created {dd}') # TODO use logger \ No newline at end of file diff --git a/ush/python/pygw/src/tests/test_file_utils.py b/ush/python/pygw/src/tests/test_file_utils.py new file mode 100644 index 00000000000..faabd28060f --- /dev/null +++ b/ush/python/pygw/src/tests/test_file_utils.py @@ -0,0 +1,26 @@ +from pygw.attrdict import AttrDict +from pygw.file_utils import FileHandler + + +def test_make_dirs(tmp_path): + """ + Test for creating directories: + Parameters + ---------- + tmp_path - pytest fixture + """ + + dir_path = tmp_path / 'my_test_dir' + f1 = f'{dir_path}/my_file1' + f2 = f'{dir_path}/my_file2' + f3 = f'{dir_path}/my_file3' + + # Create config object for FileHandler + config = Attrdict() + config.mkdir = [f1, f2, f3] + + FileHandler(config).sync() + + for dd in config.mkdir: + assert os.path.exists(dd) + From de5259c670e95190b036303628ff0491ceb75cbb Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 14:38:28 -0500 Subject: [PATCH 2/3] add tests for mkdir and copy. copy is failing --- ush/python/pygw/src/pygw/file_utils.py | 11 ++-- ush/python/pygw/src/tests/test_file_utils.py | 55 +++++++++++++++++--- ush/python/pygw/src/tests/test_template.py | 37 +++++++------ 3 files changed, 69 insertions(+), 34 deletions(-) diff --git a/ush/python/pygw/src/pygw/file_utils.py b/ush/python/pygw/src/pygw/file_utils.py index 2a7e8a6d08a..062a707d055 100644 --- a/ush/python/pygw/src/pygw/file_utils.py +++ b/ush/python/pygw/src/pygw/file_utils.py @@ -25,19 +25,17 @@ class FileHandler: def __init__(self, config): self.config = config - self.sync(config) - @staticmethod - def sync(config): + def sync(self): """ Method to execute bulk actions on files described in the configuration """ sync_factory = { - 'copy': _copy_files, - 'mkdir': _make_dirs, + 'copy': self._copy_files, + 'mkdir': self._make_dirs, } # loop through the configuration keys - for action, files in config.items(): + for action, files in self.config.items(): sync_factory[action](files) @staticmethod @@ -61,7 +59,6 @@ def _copy_files(filelist): cp(src, dest) print(f'Copied {src} to {dest}') # TODO use logger - @staticmethod def _make_dirs(dirlist): """Function to make all directories specified in the list diff --git a/ush/python/pygw/src/tests/test_file_utils.py b/ush/python/pygw/src/tests/test_file_utils.py index faabd28060f..d4647e11a1f 100644 --- a/ush/python/pygw/src/tests/test_file_utils.py +++ b/ush/python/pygw/src/tests/test_file_utils.py @@ -1,8 +1,8 @@ -from pygw.attrdict import AttrDict +import os from pygw.file_utils import FileHandler -def test_make_dirs(tmp_path): +def test_mkdir(tmp_path): """ Test for creating directories: Parameters @@ -11,16 +11,55 @@ def test_make_dirs(tmp_path): """ dir_path = tmp_path / 'my_test_dir' - f1 = f'{dir_path}/my_file1' - f2 = f'{dir_path}/my_file2' - f3 = f'{dir_path}/my_file3' + d1 = f'{dir_path}1' + d2 = f'{dir_path}2' + d3 = f'{dir_path}3' # Create config object for FileHandler - config = Attrdict() - config.mkdir = [f1, f2, f3] + config = {'mkdir': [d1, d2, d3]} + # Create d1, d2, d3 FileHandler(config).sync() - for dd in config.mkdir: + # Check if d1, d2, d3 were indeed created + for dd in config['mkdir']: assert os.path.exists(dd) + +def test_copy(tmp_path): + """ + Test for copying files: + Parameters + ---------- + tmp_path - pytest fixture + """ + + input_dir_path = tmp_path / 'my_input_dir' + + # Create the input directory + config = {'mkdir': input_dir_path} + FileHandler(config).sync() + + # Put empty files in input_dir_path + src_files = [input_dir_path / 'a.txt', input_dir_path / 'b.txt'] + for ff in src_files: + ff.touch() + + # Define output_dir_path and expected file names + output_dir_path = tmp_path / 'my_output_dir' + dest_files = [output_dir_path / 'a.txt', output_dir_path / 'bb.txt'] + + copy_list = [] + for src, dest in zip(src_files, dest_files): + print(src, dest) + copy_list.append([src, dest]) + + # Create config object for FileHandler + config = {'copy': copy_list} + + # Copy input files to output files + FileHandler(config).sync() + + # Check if files were indeed copied + for ff in dest_files: + assert os.path.isfile(ff) diff --git a/ush/python/pygw/src/tests/test_template.py b/ush/python/pygw/src/tests/test_template.py index 5d7bb378b9f..f6d594b2d9d 100644 --- a/ush/python/pygw/src/tests/test_template.py +++ b/ush/python/pygw/src/tests/test_template.py @@ -114,9 +114,9 @@ def test_substitute_with_dependencies(): 'a': 1, 'b': 2 }, - 'dd': { '2': 'a', '1': 'b' }, - 'ee': { '3': 'a', '1': 'b' }, - 'ff': { '4': 'a', '1': 'b $(greeting)' }, + 'dd': {'2': 'a', '1': 'b'}, + 'ee': {'3': 'a', '1': 'b'}, + 'ff': {'4': 'a', '1': 'b $(greeting)'}, 'host': { 'name': 'xenon', 'config': '$(root)/hosts', @@ -128,21 +128,20 @@ def test_substitute_with_dependencies(): } } output = {'complex': {'a': 1, 'b': 2}, - 'config': '/home/user/config/config.yaml', - 'config_file': 'config.yaml', - 'dd': {'1': 'b', '2': 'a'}, - 'dictionary': {'a': 1, 'b': 2}, - 'ee': {'1': 'b', '3': 'a'}, - 'ff': {'1': 'b hello world', '4': 'a'}, - 'greeting': 'hello world', - 'host': {'config': '/home/user/hosts', - 'config_file': '/home/user/config/config.yaml/xenon.config.yaml', - 'name': 'xenon', - 'proxy2': {'config': '/home/user/xenon.hello world.yaml', - 'list': [['/home/user/xenon', 'toto.xenon.hello world'], - 'config.yaml']}}, - 'root': '/home/user', - 'world': 'world'} - + 'config': '/home/user/config/config.yaml', + 'config_file': 'config.yaml', + 'dd': {'1': 'b', '2': 'a'}, + 'dictionary': {'a': 1, 'b': 2}, + 'ee': {'1': 'b', '3': 'a'}, + 'ff': {'1': 'b hello world', '4': 'a'}, + 'greeting': 'hello world', + 'host': {'config': '/home/user/hosts', + 'config_file': '/home/user/config/config.yaml/xenon.config.yaml', + 'name': 'xenon', + 'proxy2': {'config': '/home/user/xenon.hello world.yaml', + 'list': [['/home/user/xenon', 'toto.xenon.hello world'], + 'config.yaml']}}, + 'root': '/home/user', + 'world': 'world'} assert Template.substitute_with_dependencies(input, input, TemplateConstants.DOLLAR_PARENTHESES) == output From c1338b1d8fca8008c77457f02e73e681658170a1 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 14:54:06 -0500 Subject: [PATCH 3/3] test passes when output directory exists --- ush/python/pygw/src/tests/test_file_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/python/pygw/src/tests/test_file_utils.py b/ush/python/pygw/src/tests/test_file_utils.py index d4647e11a1f..684c76b650b 100644 --- a/ush/python/pygw/src/tests/test_file_utils.py +++ b/ush/python/pygw/src/tests/test_file_utils.py @@ -37,7 +37,7 @@ def test_copy(tmp_path): input_dir_path = tmp_path / 'my_input_dir' # Create the input directory - config = {'mkdir': input_dir_path} + config = {'mkdir': [input_dir_path]} FileHandler(config).sync() # Put empty files in input_dir_path @@ -45,13 +45,14 @@ def test_copy(tmp_path): for ff in src_files: ff.touch() - # Define output_dir_path and expected file names + # Create output_dir_path and expected file names output_dir_path = tmp_path / 'my_output_dir' + config = {'mkdir': [output_dir_path]} + FileHandler(config).sync() dest_files = [output_dir_path / 'a.txt', output_dir_path / 'bb.txt'] copy_list = [] for src, dest in zip(src_files, dest_files): - print(src, dest) copy_list.append([src, dest]) # Create config object for FileHandler