diff --git a/ush/python/pygw/src/pygw/file_utils.py b/ush/python/pygw/src/pygw/file_utils.py new file mode 100644 index 00000000000..062a707d055 --- /dev/null +++ b/ush/python/pygw/src/pygw/file_utils.py @@ -0,0 +1,73 @@ +from .fsutils import cp, mkdir + +__all__ = ['FileHandler'] + + +class FileHandler: + """Class to manipulate files in bulk for a given configuration + + Parameters + ---------- + config : dict + A dictionary containing the "action" and the "act" in the form of a list + + NOTE + ---- + "action" can be one of mkdir", "copy", etc. + Corresponding "act" would be ['dir1', 'dir2'], [['src1', 'dest1'], ['src2', 'dest2']] + + Attributes + ---------- + config : dict + Dictionary of files to manipulate + """ + + def __init__(self, config): + + self.config = config + + def sync(self): + """ + Method to execute bulk actions on files described in the configuration + """ + sync_factory = { + 'copy': self._copy_files, + 'mkdir': self._make_dirs, + } + # loop through the configuration keys + for action, files in self.config.items(): + sync_factory[action](files) + + @staticmethod + def _copy_files(filelist): + """Function to copy all files specified in the list + + `filelist` should be in the form: + - [src, dest] + + Parameters + ---------- + filelist : list + List of lists of [src, dest] + """ + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + dest = sublist[1] + cp(src, dest) + print(f'Copied {src} to {dest}') # TODO use logger + + @staticmethod + def _make_dirs(dirlist): + """Function to make all directories specified in the list + + Parameters + ---------- + dirlist : list + List of directories to create + """ + for dd in dirlist: + mkdir(dd) + print(f'Created {dd}') # TODO use logger diff --git a/ush/python/pygw/src/pygw/utils.py b/ush/python/pygw/src/pygw/utils.py deleted file mode 100644 index 66f85529d61..00000000000 --- a/ush/python/pygw/src/pygw/utils.py +++ /dev/null @@ -1,81 +0,0 @@ -from .yaml_file import YAMLFile -from .fsutils import cp, mkdir - -__all__ = ['FileHandler'] - - -class FileHandler: - """Class to manipulate files in bulk for a given configuration - - Parameters - ---------- - path : str, optional - Path to input YAML file containing files to manipulate - config : AttrDict, optional - Configuration AttrDict containing files to manipulate - - NOTE - ---- - While both `path` and `config` are optional, exactly one must be specified - or else an exception will be raised. - - Attributes - ---------- - config : AttrDict - Configuration of files to manipulate - """ - - def __init__(self, path=None, config=None): - - if path and config: - raise Exception('Both path and config are defined. Only one can be.') - elif path: - # read in the YAML file for the configuration - self.config = YAMLFile(path=path) - elif config: - self.config = config - else: - raise Exception("Neither 'path' nor 'config' are defined. One needs to be.") - - def sync(self): - """ - Method to execute bulk actions on files described in the configuration - """ - sync_factory = { - 'copy': copy_files, - 'mkdir': make_dirs, - } - # loop through the configuration keys - for action, files in self.config.items(): - sync_factory[action](files) - -def copy_files(filelist): - """Function to copy all files specified in the list - - `filelist` should be in the form: - - [src, dest] - - Parameters - ---------- - filelist : list - List of lists of [src, dest] - """ - for sublist in filelist: - if len(sublist) != 2: - raise Exception(f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - dest = sublist[1] - cp(src, dest) - print(f'Copied {src} to {dest}') # TODO use logger - -def make_dirs(dirlist): - """Function to make all directories specified in the list - - Parameters - ---------- - dirlist : list - List of directories to create - """ - for dd in dirlist: - mkdir(dd) - print(f'Created {dd}') # TODO use logger \ No newline at end of file diff --git a/ush/python/pygw/src/tests/test_file_utils.py b/ush/python/pygw/src/tests/test_file_utils.py new file mode 100644 index 00000000000..684c76b650b --- /dev/null +++ b/ush/python/pygw/src/tests/test_file_utils.py @@ -0,0 +1,66 @@ +import os +from pygw.file_utils import FileHandler + + +def test_mkdir(tmp_path): + """ + Test for creating directories: + Parameters + ---------- + tmp_path - pytest fixture + """ + + dir_path = tmp_path / 'my_test_dir' + d1 = f'{dir_path}1' + d2 = f'{dir_path}2' + d3 = f'{dir_path}3' + + # Create config object for FileHandler + config = {'mkdir': [d1, d2, d3]} + + # Create d1, d2, d3 + FileHandler(config).sync() + + # Check if d1, d2, d3 were indeed created + for dd in config['mkdir']: + assert os.path.exists(dd) + + +def test_copy(tmp_path): + """ + Test for copying files: + Parameters + ---------- + tmp_path - pytest fixture + """ + + input_dir_path = tmp_path / 'my_input_dir' + + # Create the input directory + config = {'mkdir': [input_dir_path]} + FileHandler(config).sync() + + # Put empty files in input_dir_path + src_files = [input_dir_path / 'a.txt', input_dir_path / 'b.txt'] + for ff in src_files: + ff.touch() + + # Create output_dir_path and expected file names + output_dir_path = tmp_path / 'my_output_dir' + config = {'mkdir': [output_dir_path]} + FileHandler(config).sync() + dest_files = [output_dir_path / 'a.txt', output_dir_path / 'bb.txt'] + + copy_list = [] + for src, dest in zip(src_files, dest_files): + copy_list.append([src, dest]) + + # Create config object for FileHandler + config = {'copy': copy_list} + + # Copy input files to output files + FileHandler(config).sync() + + # Check if files were indeed copied + for ff in dest_files: + assert os.path.isfile(ff) diff --git a/ush/python/pygw/src/tests/test_template.py b/ush/python/pygw/src/tests/test_template.py index 5d7bb378b9f..f6d594b2d9d 100644 --- a/ush/python/pygw/src/tests/test_template.py +++ b/ush/python/pygw/src/tests/test_template.py @@ -114,9 +114,9 @@ def test_substitute_with_dependencies(): 'a': 1, 'b': 2 }, - 'dd': { '2': 'a', '1': 'b' }, - 'ee': { '3': 'a', '1': 'b' }, - 'ff': { '4': 'a', '1': 'b $(greeting)' }, + 'dd': {'2': 'a', '1': 'b'}, + 'ee': {'3': 'a', '1': 'b'}, + 'ff': {'4': 'a', '1': 'b $(greeting)'}, 'host': { 'name': 'xenon', 'config': '$(root)/hosts', @@ -128,21 +128,20 @@ def test_substitute_with_dependencies(): } } output = {'complex': {'a': 1, 'b': 2}, - 'config': '/home/user/config/config.yaml', - 'config_file': 'config.yaml', - 'dd': {'1': 'b', '2': 'a'}, - 'dictionary': {'a': 1, 'b': 2}, - 'ee': {'1': 'b', '3': 'a'}, - 'ff': {'1': 'b hello world', '4': 'a'}, - 'greeting': 'hello world', - 'host': {'config': '/home/user/hosts', - 'config_file': '/home/user/config/config.yaml/xenon.config.yaml', - 'name': 'xenon', - 'proxy2': {'config': '/home/user/xenon.hello world.yaml', - 'list': [['/home/user/xenon', 'toto.xenon.hello world'], - 'config.yaml']}}, - 'root': '/home/user', - 'world': 'world'} - + 'config': '/home/user/config/config.yaml', + 'config_file': 'config.yaml', + 'dd': {'1': 'b', '2': 'a'}, + 'dictionary': {'a': 1, 'b': 2}, + 'ee': {'1': 'b', '3': 'a'}, + 'ff': {'1': 'b hello world', '4': 'a'}, + 'greeting': 'hello world', + 'host': {'config': '/home/user/hosts', + 'config_file': '/home/user/config/config.yaml/xenon.config.yaml', + 'name': 'xenon', + 'proxy2': {'config': '/home/user/xenon.hello world.yaml', + 'list': [['/home/user/xenon', 'toto.xenon.hello world'], + 'config.yaml']}}, + 'root': '/home/user', + 'world': 'world'} assert Template.substitute_with_dependencies(input, input, TemplateConstants.DOLLAR_PARENTHESES) == output