From ee34fdfa31d543a827e7179174ff40892b2a3622 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 9 Jul 2017 15:28:49 +0300 Subject: [PATCH] RFC: move cmdline parsing to cli.py and settings RFC This will make the structure of dvc more linear making it easier to read. Plus, these changes are needed in preparation for libdvc, as they move all cmdline parsing to one common place. This patch also refactors our settings system also make it more linear and remove abstraction levels that are not needed and only cause confusion. There is a lot more to be done in preparation for dvc lib and RFC overall, but this a first step in that long journey. Signed-off-by: Ruslan Kuprieiev --- dvc/__init__.py | 1 + dvc/cli.py | 271 +++++++++++++++++++++++++++++++++++++ dvc/command/base.py | 66 +-------- dvc/command/data_sync.py | 10 -- dvc/command/gc.py | 12 -- dvc/command/import_file.py | 15 -- dvc/command/init.py | 13 -- dvc/command/lock.py | 11 -- dvc/command/remove.py | 12 -- dvc/command/repro.py | 19 +-- dvc/command/run.py | 68 ++++++---- dvc/command/target.py | 13 +- dvc/command/test.py | 12 +- dvc/command/traverse.py | 9 -- dvc/config.py | 4 +- dvc/data_cloud.py | 4 +- dvc/main.py | 89 +----------- dvc/progress.py.back | 90 ++++++++++++ dvc/runtime.py | 44 ------ dvc/settings.py | 42 ++++-- setup.py | 2 +- tests/basic_env.py | 1 - tests/test_cmd_gc.py | 8 +- tests/test_cmd_remove.py | 46 ++----- tests/test_cmd_run.py | 11 +- tests/test_cmd_target.py | 36 ++--- tests/test_import.py | 12 +- tests/test_repro.py | 43 +++--- 28 files changed, 522 insertions(+), 442 deletions(-) create mode 100644 dvc/cli.py create mode 100644 dvc/progress.py.back delete mode 100644 dvc/runtime.py diff --git a/dvc/__init__.py b/dvc/__init__.py index e69de29bb2..004727f024 100644 --- a/dvc/__init__.py +++ b/dvc/__init__.py @@ -0,0 +1 @@ +VERSION = '0.8.7' diff --git a/dvc/cli.py b/dvc/cli.py new file mode 100644 index 0000000000..0338674f86 --- /dev/null +++ b/dvc/cli.py @@ -0,0 +1,271 @@ +from __future__ import print_function + +import os +import sys +import argparse +import configparser +from multiprocessing import cpu_count + +from dvc.command.init import CmdInit +from dvc.command.remove import CmdRemove +from dvc.command.run import CmdRun +from dvc.command.repro import CmdRepro +from dvc.command.data_sync import CmdDataSync +from dvc.command.lock import CmdLock +from dvc.command.gc import CmdGC +from dvc.command.import_file import CmdImportFile +from dvc.command.target import CmdTarget +from dvc.command.test import CmdTest +from dvc.config import Config +from dvc import VERSION + +def parse_args(argv=None): + # Common args + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument('-q', + '--quiet', + action='store_true', + default=False, + help='Be quiet.') + parent_parser.add_argument('-v', + '--verbose', + action='store_true', + default=False, + help='Be verbose.') + parent_parser.add_argument('-G', + '--no-git-actions', + action='store_true', + default=False, + help='Skip all git actions including reproducibility check and commits.') + + desc = 'Data Version Control' + parser = argparse.ArgumentParser(description=desc, parents=[parent_parser], + formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('-V', + '--version', + action='version', + version='%(prog)s ' + VERSION) + + # Sub commands + subparsers = parser.add_subparsers( + dest='cmd', + help='Use dvc CMD --help for command-specific help') + + # Init + init_parser = subparsers.add_parser( + 'init', + parents=[parent_parser], + help='Initialize dvc over a directory (should already be a git dir).') + init_parser.add_argument( + '--data-dir', + default='data', + help='Data directory.') + init_parser.add_argument( + '--cache-dir', + default='.cache', + help='Cache directory.') + init_parser.add_argument( + '--state-dir', + default='.state', + help='State directory.') + init_parser.add_argument( + '--target-file', + default=Config.TARGET_FILE_DEFAULT, + help='Target file.') + init_parser.set_defaults(func=CmdInit) + + # Run + run_parser = subparsers.add_parser( + 'run', + parents=[parent_parser], + help='Run command') + run_parser.add_argument( + '--stdout', + help='Output std output to a file.') + run_parser.add_argument( + '--stderr', + help='Output std error to a file.') + run_parser.add_argument('-i', + '--input', + action='append', + help='Declare input data items for reproducible cmd.') + run_parser.add_argument('-o', + '--output', + action='append', + help='Declare output data items for reproducible cmd.') + run_parser.add_argument('-c', + '--code', + action='append', + help='Code dependencies which produce the output.') + run_parser.add_argument( + '--shell', + action='store_true', + default=False, + help='Shell command') + run_parser.add_argument('-l', + '--lock', + action='store_true', + default=False, + help='Lock data item - disable reproduction.') + run_parser.add_argument( + 'command', + nargs=argparse.REMAINDER, + help='Command to execute') + run_parser.set_defaults(func=CmdRun) + + # Sync + sync_parser = subparsers.add_parser( + 'sync', + parents=[parent_parser], + help='Synchronize data file with cloud (cloud settings already setup.') + sync_parser.add_argument( + 'targets', + metavar='', + nargs='*', + help='File or directory to sync.') + sync_parser.add_argument('-j', + '--jobs', + type=int, + default=cpu_count(), + help='Number of jobs to run simultaneously.') + sync_parser.set_defaults(func=CmdDataSync) + + # Repro + repro_parser = subparsers.add_parser( + 'repro', + parents=[parent_parser], + help='Reproduce data') + repro_parser.add_argument( + 'target', + metavar='', + nargs='*', + help='Data items to reproduce.') + repro_parser.add_argument('-f', + '--force', + action='store_true', + default=False, + help='Reproduce even if dependencies were not changed.') + repro_parser.add_argument('-s', + '--single-item', + action='store_true', + default=False, + help='Reproduce only single data item without recursive dependencies check.') + repro_parser.set_defaults(func=CmdRun) + + # Remove + remove_parser = subparsers.add_parser( + 'remove', + parents=[parent_parser], + help='Remove data item from data directory.') + remove_parser.add_argument('target', + metavar='', + nargs='*', + help='Target to remove - file or directory.') + remove_parser.add_argument('-l', + '--keep-in-cloud', + action='store_true', + default=False, + help='Do not remove data from cloud.') + remove_parser.add_argument('-r', + '--recursive', + action='store_true', + help='Remove directory recursively.') + remove_parser.add_argument('-c', + '--keep-in-cache', + action='store_true', + default=False, + help='Do not remove data from cache.') + remove_parser.set_defaults(func=CmdRemove) + + # Import + import_parser = subparsers.add_parser( + 'import', + parents=[parent_parser], + help='Import file to data directory.') + import_parser.add_argument( + 'input', + nargs='+', + help='Input file/files.') + import_parser.add_argument( + 'output', + help='Output file/directory.') + import_parser.add_argument('-l', + '--lock', + action='store_true', + default=False, + help='Lock data item - disable reproduction.') + import_parser.add_argument('-j', + '--jobs', + type=int, + default=cpu_count(), + help='Number of jobs to run simultaneously.') + import_parser.set_defaults(func=CmdImportFile) + + # Lock + lock_parser = subparsers.add_parser( + 'lock', + parents=[parent_parser], + help='Lock') + lock_parser.add_argument('-l', + '--lock', + action='store_true', + default=False, + help='Lock data item - disable reproduction.') + lock_parser.add_argument('-u', + '--unlock', + action='store_true', + default=False, + help='Unlock data item - enable reproduction.') + lock_parser.add_argument( + 'files', + metavar='', + nargs='*', + help='Data items to lock or unlock.') + lock_parser.set_defaults(func=CmdLock) + + # Garbage collector + gc_parser = subparsers.add_parser( + 'gc', + parents=[parent_parser], + help='Collect garbage') + gc_parser.add_argument('target', + metavar='', + nargs='*', + help='Target to remove - file or directory.') + gc_parser.add_argument('-l', + '--keep-in-cloud', + action='store_true', + default=False, + help='Do not remove data from cloud.') + gc_parser.add_argument('-r', + '--recursive', + action='store_true', + help='Remove directory recursively.') + gc_parser.add_argument('-c', + '--keep-in-cache', + action='store_false', + default=False, + help='Do not remove data from cache.') + gc_parser.set_defaults(func=CmdGC) + + # Target + target_parser = subparsers.add_parser( + 'target', + parents=[parent_parser], + help='Set default target') + target_parser.add_argument('target_file', + metavar='', + nargs='?', + help='Target data item.') + target_parser.add_argument('-u', + '--unset', + action='store_true', + default=False, + help='Reset target.') + target_parser.set_defaults(func=CmdTarget) + + if isinstance(argv, str): + argv = argv.split() + + return parser.parse_args(argv) diff --git a/dvc/command/base.py b/dvc/command/base.py index abff5dd7a2..208939b6f7 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -40,29 +40,24 @@ class CmdBase(object): def __init__(self, settings): self._settings = settings - parser = argparse.ArgumentParser() - self.define_common_args(parser) - self.define_args(parser) - - self._parsed_args, self._command_args = parser.parse_known_args(args=self.args) - - self.process_common_args() + if settings._parsed_args.quiet and not settings._parsed_args.verbose: + Logger.be_quiet() + elif not settings._parsed_args.quiet and settings._parsed_args.verbose: + Logger.be_verbose() @property def settings(self): return self._settings + #NOTE: this name is really confusing. It should really be called "command" or smth, + # because it is only used for "command" argument from CmdRun. @property def args(self): return self._settings.args @property def parsed_args(self): - return self._parsed_args - - @property - def command_args(self): - return self._command_args + return self._settings._parsed_args @property def config(self): @@ -76,34 +71,6 @@ def dvc_home(self): def git(self): return self._settings.git - def define_args(self, parser): - pass - - def set_no_git_actions(self, parser): - parser.add_argument('--no-git-actions', '-G', action='store_true', default=False, - help='Skip all git actions including reproducibility check and commits.') - - def set_lock_action(self, parser): - parser.add_argument('--lock', '-l', action='store_true', default=False, - help='Lock data item - disable reproduction. ' + - 'It can be enabled by `dvc lock` command or by forcing reproduction.') - - def define_common_args(self, parser): - parser.add_argument('--quiet', '-q', action='store_true', default=False, help='Be quiet.') - parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Be verbose.') - parser.add_argument('--jobs', '-j', type=int, default=cpu_count(), - help='Number of jobs to run simultaneously.') - - @staticmethod - def set_reset_flag(parser, short_name, long_name, message): - parser.add_argument(short_name, long_name, action='store_true', default=False, help=message) - - def process_common_args(self): - if self._parsed_args.quiet and not self._parsed_args.verbose: - Logger.be_quiet() - elif not self._parsed_args.quiet and self._parsed_args.verbose: - Logger.be_verbose() - @property def no_git_actions(self): return self.parsed_args.no_git_actions @@ -132,24 +99,5 @@ def commit_if_needed(self, message, error=False): def not_committed_changes_warning(): Logger.warn('changes were not committed to git') - def add_string_arg(self, parser, name, message, default = None, - conf_section=None, conf_name=None): - if conf_section and conf_name: - section = self.config[conf_section] - if not section: - raise ConfigError("") - default_value = section.get(conf_section, default) - else: - default_value = default - - parser.add_argument(name, - metavar='', - default=default_value, - help=message) - def run(self): pass - - @staticmethod - def warning_dvc_is_busy(): - Logger.warn('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.') diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 98285b5222..fbe9ff2116 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -3,7 +3,6 @@ from dvc.command.base import CmdBase, DvcLock from dvc.exceptions import DvcException -from dvc.runtime import Runtime from dvc.system import System from dvc.data_cloud import DataCloud from dvc.utils import map_progress @@ -18,12 +17,6 @@ class CmdDataSync(CmdBase): def __init__(self, settings): super(CmdDataSync, self).__init__(settings) - def define_args(self, parser): - parser.add_argument('targets', - metavar='', - help='File or directory to sync.', - nargs='*') - def run(self): with DvcLock(self.is_locker, self.git): cloud = DataCloud(self.settings) @@ -44,6 +37,3 @@ def run(self): map_progress(cloud.sync, targets, self.parsed_args.jobs) pass - -if __name__ == '__main__': - Runtime.run(CmdDataSync) diff --git a/dvc/command/gc.py b/dvc/command/gc.py index b1195ede29..6ff3a5051c 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -3,20 +3,12 @@ from dvc.command.traverse import Traverse from dvc.logger import Logger from dvc.path.data_item import DataItemError -from dvc.runtime import Runtime class CmdGC(Traverse): def __init__(self, settings): super(CmdGC, self).__init__(settings, "garbage collect", do_not_start_from_root=False) - def define_args(self, parser): - super(CmdGC, self).define_args(parser) - parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.') - parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False, - help='Do not remove data from cache.') - pass - def process_file(self, target): Logger.debug(u'[Cmd-GC] GC file {}.'.format(target)) @@ -60,7 +52,3 @@ def no_git_actions(self): @staticmethod def not_committed_changes_warning(): pass - - -if __name__ == '__main__': - Runtime.run(CmdGC) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 65292d81e6..201f93cc71 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -6,7 +6,6 @@ from dvc.data_cloud import sizeof_fmt from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.system import System from dvc.progress import progress @@ -21,16 +20,6 @@ class CmdImportFile(CmdBase): def __init__(self, settings): super(CmdImportFile, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - self.set_lock_action(parser) - - parser.add_argument('input', - help='Input file/files.', - nargs='+') - parser.add_argument('output', - help='Output file/directory.') - def run(self): targets = [] with DvcLock(self.is_locker, self.git): @@ -221,7 +210,3 @@ def create_state_files(self, targets, lock): @staticmethod def is_url(url): return CmdImportFile.URL_REGEX.match(url) is not None - - -if __name__ == '__main__': - Runtime.run(CmdImportFile) diff --git a/dvc/command/init.py b/dvc/command/init.py index fbe38f7bf8..1e0238912e 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -5,7 +5,6 @@ from dvc.logger import Logger from dvc.config import Config from dvc.exceptions import DvcException -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.system import System @@ -62,15 +61,6 @@ class CmdInit(CmdBase): def __init__(self, settings): super(CmdInit, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - - self.add_string_arg(parser, '--data-dir', 'Data directory.', 'data') - self.add_string_arg(parser, '--cache-dir', 'Cache directory.', '.cache') - self.add_string_arg(parser, '--state-dir', 'State directory.', '.state') - self.add_string_arg(parser, '--target-file', 'Target file.', Config.TARGET_FILE_DEFAULT) - pass - def get_not_existing_path(self, dir): path = Path(os.path.join(self.git.git_dir, dir)) if path.exists(): @@ -162,6 +152,3 @@ def modify_gitignore(self, cache_dir_name): fd.write('\n{}'.format(os.path.basename(self.git.lock_file))) Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name)) - -if __name__ == '__main__': - Runtime.run(CmdInit, False) diff --git a/dvc/command/lock.py b/dvc/command/lock.py index f946d49ab9..b2dc9e0bec 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -1,6 +1,5 @@ from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger -from dvc.runtime import Runtime from dvc.state_file import StateFile @@ -8,12 +7,6 @@ class CmdLock(CmdBase): def __init__(self, settings): super(CmdLock, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - CmdLock.set_reset_flag(parser, '-u', '--unlock', 'Unlock data item - enable reproduction.') - parser.add_argument('files', metavar='', help='Data items to lock or unlock.', nargs='*') - pass - def run(self): with DvcLock(self.is_locker, self.git): return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock) @@ -47,7 +40,3 @@ def lock_files(self, files, target): self.commit_if_needed('DVC lock: {}'.format(' '.join(self.args))) return 0 - - -if __name__ == '__main__': - Runtime.run(CmdLock, False) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index db13860bde..21e2f4e94b 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -3,20 +3,12 @@ from dvc.command.traverse import Traverse from dvc.logger import Logger from dvc.path.data_item import DataItemError -from dvc.runtime import Runtime class CmdRemove(Traverse): def __init__(self, settings): super(CmdRemove, self).__init__(settings, "remove") - def define_args(self, parser): - super(CmdRemove, self).define_args(parser) - parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.') - parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False, - help='Do not remove data from cache.') - pass - def process_file(self, target): Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target)) @@ -83,7 +75,3 @@ def remove_file(self, target): def remove_all(self): return self._traverse_all() - - -if __name__ == '__main__': - Runtime.run(CmdRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index b51454c6d5..aa4d9ed4ef 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -7,7 +7,6 @@ from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.path.data_item import NotInDataDirError -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.system import System @@ -28,16 +27,6 @@ def __init__(self, settings): def code_dependencies(self): return self._code - def define_args(self, parser): - self.set_no_git_actions(parser) - - parser.add_argument('target', metavar='', help='Data items to reproduce.', nargs='*') - parser.add_argument('-f', '--force', action='store_true', default=False, - help='Reproduce even if dependencies were not changed.') - parser.add_argument('-s', '--single-item', action='store_true', default=False, - help='Reproduce only single data item without recursive dependencies check.') - pass - @property def lock(self): return True @@ -209,6 +198,10 @@ def reproduce_data_item(self, changed_files): self.state.stdout, self.state.stderr, self.state.shell, + [], + [], + [], + True, check_if_ready=False) != 0: raise ReproError('Run command reproduction failed') return True @@ -317,7 +310,3 @@ def dependencies(self): dependency_data_items.append(data_item) return dependency_data_items - - -if __name__ == '__main__': - Runtime.run(CmdRepro) diff --git a/dvc/command/run.py b/dvc/command/run.py index 4e2d2d5a4a..54eb8175ad 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,4 +1,5 @@ import os +import sys from dvc.command.base import CmdBase, DvcLock from dvc.exceptions import DvcException @@ -6,7 +7,6 @@ from dvc.logger import Logger from dvc.path.data_item import NotInDataDirError, NotInGitDirError from dvc.repository_change import RepositoryChange -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.utils import cached_property @@ -20,21 +20,6 @@ class CmdRun(CmdBase): def __init__(self, settings): super(CmdRun, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - self.set_lock_action(parser) - - parser.add_argument('--stdout', help='Output std output to a file.') - parser.add_argument('--stderr', help='Output std error to a file.') - parser.add_argument('--input', '-i', action='append', - help='Declare input data items for reproducible cmd.') - parser.add_argument('--output', '-o', action='append', - help='Declare output data items for reproducible cmd.') - parser.add_argument('--code', '-c', action='append', - help='Code dependencies which produce the output.') - parser.add_argument('--shell', help='Shell command', action='store_true', default=False) - pass - @property def lock(self): return self.parsed_args.lock @@ -53,8 +38,8 @@ def declaration_output_data_items(self): def run(self): with DvcLock(self.is_locker, self.git): - data_items_from_args, not_data_items_from_args = self.argv_files_by_type(self.command_args) - return self.run_and_commit_if_needed(self.command_args, + data_items_from_args, not_data_items_from_args = self.argv_files_by_type(self.parsed_args.command) + return self.run_and_commit_if_needed(self.parsed_args.command, data_items_from_args, not_data_items_from_args, self.parsed_args.stdout, @@ -63,7 +48,12 @@ def run(self): pass def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args, - stdout, stderr, shell, check_if_ready=True): + stdout, stderr, shell, + output_data_items=None, + input_data_items=None, + code_dependencies=None, + lock=None, + check_if_ready=True): if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): return 1 @@ -72,12 +62,34 @@ def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_ not_data_items_from_args, stdout, stderr, - shell) + shell, + output_data_items, + input_data_items, + code_dependencies, + lock) - return self.commit_if_needed('DVC run: {}'.format(' '.join(self.args))) + return self.commit_if_needed('DVC run: {}'.format(' '.join(command_args))) def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args, - stdout=None, stderr=None, shell=False): + stdout=None, stderr=None, shell=False, + output_data_items=None, + input_data_items=None, + code_dependencies=None, + lock=None): + + # Repro sets these from state file + if output_data_items == None: + output_data_items = self.declaration_output_data_items + + if input_data_items == None: + input_data_items = self.declaration_input_data_items + + if code_dependencies == None: + code_dependencies = self.code_dependencies + + if lock == None: + lock = self.lock + Logger.debug(u'Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'.format( ' '.join(cmd_args), ', '.join([x.data.dvc for x in data_items_from_args]), @@ -91,13 +103,13 @@ def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args, self.remove_new_files(repo_change) raise RunError('Errors occurred.') - output_set = set(self.declaration_output_data_items + repo_change.changed_data_items) + output_set = set(output_data_items + repo_change.changed_data_items) output_files_dvc = [x.data.dvc for x in output_set] - input_set = set(data_items_from_args + self.declaration_input_data_items) - output_set + input_set = set(data_items_from_args + input_data_items) - output_set input_files_dvc = [x.data.dvc for x in input_set] - code_dependencies_dvc = self.git.abs_paths_to_dvc(self.code_dependencies + not_data_items_from_args) + code_dependencies_dvc = self.git.abs_paths_to_dvc(code_dependencies + not_data_items_from_args) result = [] for data_item in repo_change.changed_data_items: @@ -114,7 +126,7 @@ def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args, output_files_dvc, code_dependencies_dvc, argv=cmd_args, - lock=self.lock, + lock=lock, stdout=self._stdout_to_dvc(stdout), stderr=self._stdout_to_dvc(stderr), shell=shell) @@ -176,7 +188,3 @@ def _data_items_from_params(self, files, param_text): param_text, ', '.join(external)) ) return data_items - - -if __name__ == '__main__': - Runtime.run(CmdRun) diff --git a/dvc/command/target.py b/dvc/command/target.py index fdefa93da2..4b3fa80085 100644 --- a/dvc/command/target.py +++ b/dvc/command/target.py @@ -2,22 +2,15 @@ from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger -from dvc.runtime import Runtime class CmdTarget(CmdBase): def __init__(self, settings): super(CmdTarget, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - CmdTarget.set_reset_flag(parser, '-u', '--unset', 'Reset target.') - parser.add_argument('target', metavar='', nargs='?', help='Target data item.') - pass - def run(self): with DvcLock(self.is_locker, self.git): - target = self.parsed_args.target + target = self.parsed_args.target_file unset = self.parsed_args.unset if target and unset: @@ -61,7 +54,3 @@ def unset_target(self, target_conf_file_path): open(target_conf_file_path, 'a').close() return self.commit_if_needed('DVC target unset') - - -if __name__ == '__main__': - Runtime.run(CmdTarget, False) diff --git a/dvc/command/test.py b/dvc/command/test.py index 62a46c3fae..58a477c1fd 100644 --- a/dvc/command/test.py +++ b/dvc/command/test.py @@ -5,14 +5,10 @@ from google.cloud import storage +import dvc from dvc.command.base import CmdBase from dvc.logger import Logger from dvc.config import Config -from dvc.runtime import Runtime - -from dvc.settings import SettingsError - - class CmdTest(CmdBase): def __init__(self, settings): @@ -33,7 +29,7 @@ def run(self): if self.settings._config.gc_project_name == '': Logger.error('Please specify the google cloud project name in dvc.conf') - raise SettingsError('must specify GC ProjectName') + raise dvc.settings.SettingsError('must specify GC ProjectName') try: client = storage.Client(project=self.settings._config.gc_project_name) @@ -52,7 +48,3 @@ def run(self): sys.exit(-1) Logger.info('bucket %s visible: google cloud seems to be configured correctly' % sb) sys.exit(0) - - -if __name__ == '__main__': - Runtime.run(CmdTest, False) diff --git a/dvc/command/traverse.py b/dvc/command/traverse.py index 7826978a4b..8dc07069ae 100644 --- a/dvc/command/traverse.py +++ b/dvc/command/traverse.py @@ -18,15 +18,6 @@ def __init__(self, settings, cmd_name, do_not_start_from_root=True): self.cloud = DataCloud(self.settings) self._do_not_start_from_root = do_not_start_from_root - def define_args(self, parser): - self.set_no_git_actions(parser) - - parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*') - # parser.add_argument('-r', '--recursive', action='store_true', help='CmdGarbage collect directory recursively.') - parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False, - help='Do not remove data from cloud.') - pass - def run(self): with DvcLock(self.is_locker, self.git): if not self._traverse_all(): diff --git a/dvc/config.py b/dvc/config.py index 3866f9f57a..ccfcfb7c0e 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -46,7 +46,7 @@ def target_file(self): class Config(ConfigI): CONFIG = 'dvc.conf' - def __init__(self, conf_file, conf_pseudo_file=None): + def __init__(self, conf_file=CONFIG, conf_pseudo_file=None): """ Params: conf_file (String): configuration file @@ -59,7 +59,7 @@ def __init__(self, conf_file, conf_pseudo_file=None): self._config.readfp(conf_pseudo_file) else: if not os.path.isfile(conf_file): - raise ConfigError('Config file "{}" does not exist'.format(conf_file)) + raise ConfigError('Config file "{}" does not exist {}'.format(conf_file, os.getcwd())) self._config.read(conf_file) level = self._config['Global']['LogLevel'] diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 1e245f3f7c..71e9f8be86 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -8,10 +8,10 @@ from boto.s3.connection import S3Connection from google.cloud import storage as gc +import dvc from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.config import ConfigError -from dvc.settings import Settings from dvc.progress import progress from dvc.utils import copyfile @@ -382,7 +382,7 @@ class DataCloud(object): } def __init__(self, settings): - assert isinstance(settings, Settings) + assert isinstance(settings, dvc.settings.Settings) #To handle ConfigI case if not hasattr(settings.config, '_config'): diff --git a/dvc/main.py b/dvc/main.py index 09c6926a00..a78fcc2876 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,92 +1,11 @@ -from __future__ import print_function - -from dvc.command.import_file import CmdImportFile - """ main entry point / argument parsing for dvc """ import sys - -from dvc.runtime import Runtime -from dvc.command.init import CmdInit -from dvc.command.remove import CmdRemove -from dvc.command.run import CmdRun -from dvc.command.repro import CmdRepro -from dvc.command.data_sync import CmdDataSync -from dvc.command.lock import CmdLock -from dvc.command.gc import CmdGC -from dvc.command.target import CmdTarget -from dvc.command.test import CmdTest - -VERSION = '0.8.7' - -def print_usage(): - usage = ('', - 'usage: dvc [--version] [--help] command []', - '', - 'These are common DVC commands:', - '', - 'start a working area', - ' init Initialize dvc over a directory (should already be a git dir).', - ' run Run command.', - ' import Import file to data directory.', - ' remove Remove data item from data directory.', - '', - 'synchronize data between remote and local', - ' data sync Synchronize data file with cloud (cloud settings already setup).', -) - print('\n'.join(usage)) +from dvc.settings import Settings def main(): - cmds = ['--help', '--version', 'init', 'run', 'sync', 'repro', 'data', 'remove', 'import', 'lock', 'cloud', \ - 'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \ - 'test', 'test-aws', 'test-gcloud', 'test-cloud', 'gc', 'target'] - - if len(sys.argv) < 2 or sys.argv[1] not in cmds: - if len(sys.argv) >= 2: - print('Unimplemented or unrecognized command: ' + ' '.join(sys.argv[1:])) - print_usage() - sys.exit(-1) - - cmd = sys.argv[1] - - if cmd == '--help': - print_usage() - elif cmd == '--version': - print('dvc version {}'.format(VERSION)) - elif cmd == 'init': - Runtime.run(CmdInit, parse_config=False) - elif cmd == 'run': - Runtime.run(CmdRun) - elif cmd == 'repro': - Runtime.run(CmdRepro) - elif cmd == 'sync': - Runtime.run(CmdDataSync) - elif cmd == 'import': - Runtime.run(CmdImportFile) - elif cmd == 'remove': - Runtime.run(CmdRemove) - elif cmd == 'lock': - Runtime.run(CmdLock) - elif cmd == 'gc': - Runtime.run(CmdGC) - elif cmd == 'target': - Runtime.run(CmdTarget) - elif cmd == 'cloud-run': - print('cloud-run unimplemented') - elif cmd == 'cloud-instance-create': - print('cloud-instance-create unimplemented') - elif cmd == 'clould-instance-remove': - print('cloud-instance-remove unimplemented') - elif cmd == 'cloud-instance-describe': - print('cloud-instance-describe unimplemented') - - elif cmd == 'test-aws': - print('TODO: test aws credentials') - elif cmd == 'test-gcloud': - Runtime.run(CmdTest) - else: - print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1])) - print_usage() - sys.exit(-1) + settings = Settings(sys.argv[1:]) + instance = settings._parsed_args.func(settings) + sys.exit(instance.run()) diff --git a/dvc/progress.py.back b/dvc/progress.py.back new file mode 100644 index 0000000000..eac913dde7 --- /dev/null +++ b/dvc/progress.py.back @@ -0,0 +1,90 @@ +from __future__ import print_function +import sys +import threading +from Queue import Queue + +# (2/100): very_long_nam... [######### ] 60% + +class ProgressInfo(object): + name = "" + current = 0 + total = None + finished = False + +class Progress(object): + _n_finished = 0 + _n_total = 0 + _lock = None + _queue = None + + def __init__(self, total): + self._n_total = total + self._lock = threading.Lock() + self._queue = Queue(maxsize=0) + + def clearln(self): + if sys.stdout.isatty(): + print('\r\x1b[K', end='') + + def writeln(self, line): + if sys.stdout.isatty(): + self.clearln() + print(line, end='') + sys.stdout.flush() + + def update_target(self, name, current, total): + info = ProgressInfo() + info.name = name + info.current = current + info.total = total + + self._queue.put(info) + + def finish_target(self, name): + info = ProgressInfo() + info.name = name + info.current = 100 + info.total = 100 + info.finished = True + + self._queue.put(info) + + def finish(self): + if sys.stdout.isatty(): + print() + + def _bar(self, info): + total_len = 80 + bar_len = 40 + + if info.total == None: + progress = 0 + percent = "?% " + else: + progress = (100 * info.current)/info.total + percent = str(progress) + "% " + + num = "({}/{}): ".format(self._n_finished + 1, self._n_total) + + n_sh = (progress * bar_len)/100 + n_sp = bar_len - n_sh + bar = "[" + '#'*n_sh + ' '*n_sp + "] " + + name_len = len(num + bar + percent) + 1 + name = info.name[:name_len] if len(info.name) > name_len else info.name + + return num + bar + percent + name + + def wait(self): + while self._n_finished != self._n_total: + while not self._queue.empty(): + info = self._queue.get() + bar = self._bar(info) + with self._lock: + if info.finished: + self.clearln() + print(bar) + self._n_finished += 1 + else: + self.writeln(bar) + self.finish() diff --git a/dvc/runtime.py b/dvc/runtime.py deleted file mode 100644 index 06cb18151b..0000000000 --- a/dvc/runtime.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import sys - -from dvc.config import Config, ConfigI -from dvc.exceptions import DvcException -from dvc.git_wrapper import GitWrapper -from dvc.logger import Logger -from dvc.settings import Settings -from dvc.system import System - - -class Runtime(object): - CONFIG = 'dvc.conf' - - @staticmethod - def conf_file_path(git_dir): - return System.realpath(os.path.join(git_dir, Runtime.CONFIG)) - - @staticmethod - def run(cmd_class, parse_config=True, args_start_loc=2): - """ - - Arguments: - args_start_loc (int): where the arguments this command should use start - """ - - try: - runtime_git = GitWrapper() - - if parse_config: - runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir)) - else: - runtime_config = ConfigI() - - args = sys.argv[args_start_loc:] - - # To make argparse print "usage: dvc cmd" instead of "usage: dvc" - sys.argv[0] = sys.argv[0] + " " + sys.argv[1] - - instance = cmd_class(Settings(args, runtime_git, runtime_config)) - sys.exit(instance.run()) - except DvcException as e: - Logger.error(e) - sys.exit(1) diff --git a/dvc/settings.py b/dvc/settings.py index 4cc9162dae..a20a4784a5 100644 --- a/dvc/settings.py +++ b/dvc/settings.py @@ -1,6 +1,10 @@ +import os + +from dvc.git_wrapper import GitWrapperI, GitWrapper +from dvc.config import ConfigI, Config from dvc.exceptions import DvcException from dvc.path.factory import PathFactory - +from dvc.cli import parse_args class SettingsError(DvcException): def __init__(self, msg): @@ -8,18 +12,27 @@ def __init__(self, msg): class Settings(object): - def __init__(self, args, git, config): - self._args = args + def __init__(self, argv=None, git=None, config=None): + self._args = None + args = None + + if argv != None and len(argv) != 0: + args = parse_args(argv) + self._args = argv[2:] + + if git == None: + git = GitWrapper() + + if config == None: + if args != None and args.cmd != 'init': + config = Config() + else: + config = ConfigI() + self._git = git self._config = config self._path_factory = PathFactory(git, config) - - # self._dvc_home = os.environ.get('DVC_HOME') - # if not self._dvc_home: - # raise SettingsError('DVC_HOME environment variable is not defined') - # if not os.path.exists(self._dvc_home): - # raise SettingsError("DVC_HOME directory doesn't exists") - pass + self._parsed_args = args @property def args(self): @@ -28,6 +41,13 @@ def args(self): def set_args(self, args): self._args = args + @property + def parsed_args(self): + return self._parsed_args + + def parse_args(self, args): + self._parsed_args = parse_args(args) + @property def git(self): return self._git @@ -42,4 +62,4 @@ def dvc_home(self): @property def path_factory(self): - return self._path_factory \ No newline at end of file + return self._path_factory diff --git a/setup.py b/setup.py index c4628722d0..e48c411515 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import platform from setuptools import setup, find_packages from distutils.errors import DistutilsPlatformError -from dvc.main import VERSION +from dvc import VERSION install_requires = [ 'altgraph==0.13', diff --git a/tests/basic_env.py b/tests/basic_env.py index 8fb4952d7b..448f1b0b97 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -10,7 +10,6 @@ from dvc.system import System from dvc.utils import rmtree - class BasicEnvironment(TestCase): def init_environment(self, test_dir=System.get_long_path(tempfile.mkdtemp()), diff --git a/tests/test_cmd_gc.py b/tests/test_cmd_gc.py index be8e9c13d5..f3d6c82582 100644 --- a/tests/test_cmd_gc.py +++ b/tests/test_cmd_gc.py @@ -10,7 +10,6 @@ from dvc.system import System from tests.basic_env import BasicEnvironment - class TestCmdDataRemove(BasicEnvironment): def setUp(self): self._test_dir = os.path.join(os.path.sep, 'tmp', 'dvc_unit_test') @@ -24,7 +23,7 @@ def setUp(self): self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit) self._config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self._git, self._config) - self.settings = Settings([], self._git, self._config) + self.settings = Settings('gc', self._git, self._config) self.dir1 = 'dir1' @@ -53,11 +52,10 @@ def test_all(self): self.assertEqual(5, self.cache_file_nums('file2*')) self.assertEqual(5, self.cache_file_nums(os.path.join(self.dir1, 'file3*'))) + self.settings.parse_args('gc --no-git-actions data') cmd = CmdGC(self.settings) os.chdir(self._test_git_dir) - cmd.parsed_args.no_git_actions = True - cmd.parsed_args.target = ['data'] cmd.gc_all() self.assertEqual(1, self.cache_file_nums('file1*')) @@ -72,10 +70,10 @@ def test_empty(self): self.assertEqual(5, self.cache_file_nums('file1*')) self.assertEqual(5, self.cache_file_nums('file2*')) + self.settings.parse_args('gc --no-git-actions') cmd = CmdGC(self.settings) os.chdir(self._test_git_dir) - cmd.parsed_args.no_git_actions = True cmd.gc_all() self.assertEqual(5, self.cache_file_nums('file1*')) diff --git a/tests/test_cmd_remove.py b/tests/test_cmd_remove.py index 9ccabac53d..168f3a8c60 100644 --- a/tests/test_cmd_remove.py +++ b/tests/test_cmd_remove.py @@ -3,14 +3,13 @@ from dvc.command.remove import CmdRemove from tests.basic_env import DirHierarchyEnvironment - class TestCmdRemove(DirHierarchyEnvironment): def setUp(self): DirHierarchyEnvironment.init_environment(self) def test_file_by_file_removal(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True dir1_dvc_name = os.path.join('data', self.dir1) self.assertTrue(os.path.exists(dir1_dvc_name)) @@ -19,9 +18,8 @@ def test_file_by_file_removal(self): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_remove_deep_dir(self): + self.settings.parse_args('remove --keep-in-cloud --recursive') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.recursive = True dir1_dvc_name = os.path.join('data', self.dir1) self.assertTrue(os.path.exists(dir1_dvc_name)) @@ -30,9 +28,8 @@ def test_remove_deep_dir(self): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_not_recursive_removal(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.recursive = False dir1_dvc_name = os.path.join('data', self.dir1) self.assertTrue(os.path.exists(dir1_dvc_name)) @@ -41,9 +38,8 @@ def test_not_recursive_removal(self): pass def test_data_dir_removal(self): + self.settings.parse_args('remove --keep-in-cloud --recursive') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.recursive = True data_dir = 'data' self.assertTrue(os.path.exists(data_dir)) @@ -57,8 +53,8 @@ def setUp(self): DirHierarchyEnvironment.init_environment(self) def test_remove_data_instance(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file1)) self.assertTrue(os.path.isfile(self.cache1)) @@ -70,9 +66,8 @@ def test_remove_data_instance(self): self.assertFalse(os.path.exists(self.state1)) def test_keep_cache(self): + self.settings.parse_args('remove --keep-in-cloud --keep-in-cache') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.keep_in_cache = True self.assertTrue(os.path.isfile(self.file1)) self.assertTrue(os.path.isfile(self.cache1)) @@ -84,8 +79,8 @@ def test_keep_cache(self): self.assertFalse(os.path.exists(self.state1)) def test_remove_data_instance_without_cache(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file6)) self.assertIsNone(self.cache6) @@ -101,16 +96,11 @@ def setUp(self): DirHierarchyEnvironment.init_environment(self) def test_end_to_end_with_no_error(self): - cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - dir11_full = os.path.join('data', self.dir11) dir2_full = os.path.join('data', self.dir2) - cmd.parsed_args.target = [dir11_full, self.file5] - cmd.parsed_args.recursive = True - - cmd.parsed_args.no_git_actions = True + self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5)) + cmd = CmdRemove(self.settings) self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) @@ -125,16 +115,11 @@ def test_end_to_end_with_no_error(self): self.assertTrue(os.path.exists(self.file1)) def test_run(self): - cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - dir11_full = os.path.join('data', self.dir11) dir2_full = os.path.join('data', self.dir2) - cmd.parsed_args.target = [dir11_full, self.file5] - cmd.parsed_args.recursive = True - - cmd.parsed_args.no_git_actions = True + self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5)) + cmd = CmdRemove(self.settings) self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) @@ -154,16 +139,11 @@ def setUp(self): DirHierarchyEnvironment.init_environment(self) def test(self): - cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - dir11_full = os.path.join(self._config.data_dir, self.dir11) dir2_full = os.path.join(self._config.data_dir, self.dir2) - cmd.parsed_args.target = [dir11_full, self.file5] - cmd.parsed_args.recursive = True - - cmd.parsed_args.no_git_actions = True + self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5)) + cmd = CmdRemove(self.settings) self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) diff --git a/tests/test_cmd_run.py b/tests/test_cmd_run.py index 51bc0f9f64..32ea39365f 100644 --- a/tests/test_cmd_run.py +++ b/tests/test_cmd_run.py @@ -31,7 +31,7 @@ def setUp(self): self.config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self.git, self.config) - self.settings = Settings([], self.git, self.config) + self.settings = Settings(['run'], self.git, self.config) pass def init_git_repo(self): @@ -65,13 +65,12 @@ class RunTwoFilesBase(RunBasicTest): def setUp(self): super(RunTwoFilesBase, self).setUp() - self.settings._args = [] - cmd_run = CmdRun(self.settings) self.input_param_file = os.path.join('data', 'extra_input') - cmd_run.parsed_args.input = [self.input_param_file] - self.extra_output_file = os.path.join('data', 'extra_output') - cmd_run.parsed_args.output = [self.extra_output_file] + + self.settings.parse_args('run --input {} --output {}'.format(self.input_param_file, self.extra_output_file)) + self.settings._args = [] + cmd_run = CmdRun(self.settings) self.file_name1 = os.path.join('data', 'file1') self.file_name2 = os.path.join('data', 'file2') diff --git a/tests/test_cmd_target.py b/tests/test_cmd_target.py index 050afaac59..9dc8e5e1e7 100644 --- a/tests/test_cmd_target.py +++ b/tests/test_cmd_target.py @@ -22,7 +22,7 @@ def setUp(self): self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit) self._config = ConfigI('data', 'cache', 'state', '.target') self.path_factory = PathFactory(self._git, self._config) - self.settings = Settings([], self._git, self._config) + self.settings = Settings(['target'], self._git, self._config) os.chdir(self._test_git_dir) @@ -45,53 +45,49 @@ def test_initial_default_target(self): self.assertFalse(os.path.exists('.target')) def test_single_file(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file1_data) def test_multiple_files(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False cmd.run() # Another target - cmd.parsed_args.target = self.file2_data + self.settings.parse_args('target {}'.format(self.file2_data)) + cmd = CmdTarget(self.settings) self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file2_data) # Unset target - cmd.parsed_args.target = '' - cmd.parsed_args.unset = True + self.settings.parse_args('target --unset') + cmd = CmdTarget(self.settings) self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), '') def test_initial_unset(self): + self.settings.parse_args('target --unset') cmd = CmdTarget(self.settings) - cmd.parsed_args.target = '' - cmd.parsed_args.unset = True self.assertEqual(cmd.run(), 1) self.assertFalse(os.path.exists('.target')) def test_unset_existing_target(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file1_data) - cmd.parsed_args.target = '' - cmd.parsed_args.unset = True + self.settings.parse_args('target --unset') + cmd = CmdTarget(self.settings) self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), '') def test_the_same(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file1_data) @@ -99,13 +95,11 @@ def test_the_same(self): self.assertEqual(open('.target').read(), self.file1_data) def test_args_conflict(self): + self.settings.parse_args('target {} --unset'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = True self.assertEqual(cmd.run(), 1) def test_no_args(self): + self.settings.parse_args('target --unset') cmd = CmdTarget(self.settings) - cmd.parsed_args.target = '' - cmd.parsed_args.unset = False - self.assertEqual(cmd.run(), 1) \ No newline at end of file + self.assertEqual(cmd.run(), 1) diff --git a/tests/test_import.py b/tests/test_import.py index 4b1cee6d8b..96b24fcc6b 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -18,7 +18,7 @@ def test_import_file_to_dir(self): dir = os.path.join('data', self.dir1) settings = copy(self.settings) - settings.set_args([temp.name, dir]) + settings.parse_args('import {} {}'.format(temp.name, dir)) cmd = CmdImportFile(settings) cmd.run() @@ -44,7 +44,7 @@ def test_import_file_to_file(self): target_file_name = 'targ.txt' target_file = os.path.join('data', self.dir1, target_file_name) settings = copy(self.settings) - settings.set_args([temp.name, target_file]) + settings.parse_args('import {} {}'.format(temp.name, target_file)) cmd = CmdImportFile(settings) cmd.run() @@ -60,7 +60,7 @@ def test_import_to_existing_dir(self): dir = os.path.join('data', self.dir11) settings = copy(self.settings) - settings.set_args([temp.name, dir]) + settings.parse_args('import {} {}'.format(temp.name, dir)) cmd = CmdImportFile(settings) cmd.run() @@ -78,7 +78,7 @@ def test_import_to_not_existing_dir(self): dir = os.path.join('data', self.dir1, 'not_existing_dir/') settings = copy(self.settings) - settings.set_args([temp.name, dir]) + settings.parse_args('import {} {}'.format(temp.name, dir)) cmd = CmdImportFile(settings) with self.assertRaises(ImportFileError): @@ -103,7 +103,7 @@ def test_multiple_import(self): dir = os.path.join('data', self.dir11) settings = copy(self.settings) - settings.set_args([temp1.name, temp2.name, dir]) + settings.parse_args('import {} {} {}'.format(temp1.name, temp2.name, dir)) cmd = CmdImportFile(settings) cmd.run() @@ -115,4 +115,4 @@ def test_multiple_import(self): self.assertTrue(os.path.exists(dvc_name2)) self.assertTrue(open(dvc_name2).read(), content) - pass \ No newline at end of file + pass diff --git a/tests/test_repro.py b/tests/test_repro.py index 20d993e2b1..8a31752963 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -14,8 +14,10 @@ def setUp(self): self.file_name1 = os.path.join('data', 'file1') self.file1_code_file = 'file1.py' self.create_file_and_commit(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")') - self.settings._args = ['python', self.file1_code_file, '--not-repro', - '--stdout', self.file_name1, '--code', self.file1_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name1, + '--code', self.file1_code_file, + 'python', self.file1_code_file, '--not-repro']) cmd_file1 = CmdRun(self.settings) self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file]) cmd_file1.run() @@ -25,16 +27,19 @@ def setUp(self): self.create_file_and_commit(self.file11_code_file, 'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())') - self.settings._args = ['python', self.file11_code_file, self.file_name1, - '--stdout', self.file_name11, '--code', self.file11_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name11, + '--code', self.file11_code_file, + 'python', self.file11_code_file, self.file_name1]) CmdRun(self.settings).run() self.file_name2 = os.path.join('data', 'file2') self.file2_code_file = 'file2.py' self.create_file_and_commit(self.file2_code_file, 'print("Bobby")') - self.settings._args = ['python', self.file2_code_file, - '--not-repro', '--stdout', self.file_name2] + self.settings.parse_args(['run', + '--stdout', self.file_name2, + 'python', self.file2_code_file, '--not-repro']) CmdRun(self.settings).run() self.file_res_code_file = 'code_res.py' @@ -44,11 +49,13 @@ def setUp(self): 'text2 = open(sys.argv[2]).read()' + os.linesep + 'print(text1 + text2)') self.file_name_res = os.path.join('data', 'file_res') - self.settings._args = ['python', self.file_res_code_file, - self.file_name11, - self.file_name2, - '--stdout', self.file_name_res, - '--code', self.file_res_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name_res, + '--code', self.file_res_code_file, + 'python', self.file_res_code_file, + self.file_name11, + self.file_name2]) + cmd_res = CmdRun(self.settings) self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file]) cmd_res.run() @@ -76,7 +83,7 @@ class ReproCodeDependencyTest(ReproBasicEnv): def test(self): self.modify_file_and_commit(self.file_res_code_file) - self.settings._args = [self.file_name_res] + self.settings.parse_args('repro {}'.format(self.file_name_res)) CmdRepro(self.settings).run() self.assertEqual(open(self.file_name_res).read().strip(), 'Hello\nBobby') @@ -86,19 +93,21 @@ class ReproChangedDependency(ReproBasicEnv): def test(self): self.recreate_file1() - self.settings._args = [self.file_name11] + self.settings.parse_args('repro {}'.format(self.file_name11)) CmdRepro(self.settings).run() self.assertEqual(open(self.file_name11).read(), 'Goodbye\n') def recreate_file1(self): - self.settings._args = [self.file_name1, '--keep-in-cloud'] + self.settings.parse_args('remove {} --keep-in-cloud'.format(self.file_name1)) CmdRemove(self.settings).run() file1_code_file = 'file1_2.py' self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")') - self.settings._args = ['python', file1_code_file, '--not-repro', - '--stdout', self.file_name1, '--code', file1_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name1, + '--code', file1_code_file, + 'python', file1_code_file, '--not-repro']) CmdRun(self.settings).run() @@ -107,7 +116,7 @@ class ReproChangedDeepDependency(ReproChangedDependency): def test(self): self.recreate_file1() - self.settings._args = [self.file_name_res] + self.settings.parse_args('repro {}'.format(self.file_name_res)) CmdRepro(self.settings).run() self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')