diff --git a/dvc/__init__.py b/dvc/__init__.py index e69de29bb2..004727f024 100644 --- a/dvc/__init__.py +++ b/dvc/__init__.py @@ -0,0 +1 @@ +VERSION = '0.8.7' diff --git a/dvc/cli.py b/dvc/cli.py new file mode 100644 index 0000000000..0338674f86 --- /dev/null +++ b/dvc/cli.py @@ -0,0 +1,271 @@ +from __future__ import print_function + +import os +import sys +import argparse +import configparser +from multiprocessing import cpu_count + +from dvc.command.init import CmdInit +from dvc.command.remove import CmdRemove +from dvc.command.run import CmdRun +from dvc.command.repro import CmdRepro +from dvc.command.data_sync import CmdDataSync +from dvc.command.lock import CmdLock +from dvc.command.gc import CmdGC +from dvc.command.import_file import CmdImportFile +from dvc.command.target import CmdTarget +from dvc.command.test import CmdTest +from dvc.config import Config +from dvc import VERSION + +def parse_args(argv=None): + # Common args + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument('-q', + '--quiet', + action='store_true', + default=False, + help='Be quiet.') + parent_parser.add_argument('-v', + '--verbose', + action='store_true', + default=False, + help='Be verbose.') + parent_parser.add_argument('-G', + '--no-git-actions', + action='store_true', + default=False, + help='Skip all git actions including reproducibility check and commits.') + + desc = 'Data Version Control' + parser = argparse.ArgumentParser(description=desc, parents=[parent_parser], + formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('-V', + '--version', + action='version', + version='%(prog)s ' + VERSION) + + # Sub commands + subparsers = parser.add_subparsers( + dest='cmd', + help='Use dvc CMD --help for command-specific help') + + # Init + init_parser = subparsers.add_parser( + 'init', + parents=[parent_parser], + help='Initialize dvc over a directory (should already be a git dir).') + init_parser.add_argument( + '--data-dir', + default='data', + help='Data directory.') + init_parser.add_argument( + '--cache-dir', + default='.cache', + help='Cache directory.') + init_parser.add_argument( + '--state-dir', + default='.state', + help='State directory.') + init_parser.add_argument( + '--target-file', + default=Config.TARGET_FILE_DEFAULT, + help='Target file.') + init_parser.set_defaults(func=CmdInit) + + # Run + run_parser = subparsers.add_parser( + 'run', + parents=[parent_parser], + help='Run command') + run_parser.add_argument( + '--stdout', + help='Output std output to a file.') + run_parser.add_argument( + '--stderr', + help='Output std error to a file.') + run_parser.add_argument('-i', + '--input', + action='append', + help='Declare input data items for reproducible cmd.') + run_parser.add_argument('-o', + '--output', + action='append', + help='Declare output data items for reproducible cmd.') + run_parser.add_argument('-c', + '--code', + action='append', + help='Code dependencies which produce the output.') + run_parser.add_argument( + '--shell', + action='store_true', + default=False, + help='Shell command') + run_parser.add_argument('-l', + '--lock', + action='store_true', + default=False, + help='Lock data item - disable reproduction.') + run_parser.add_argument( + 'command', + nargs=argparse.REMAINDER, + help='Command to execute') + run_parser.set_defaults(func=CmdRun) + + # Sync + sync_parser = subparsers.add_parser( + 'sync', + parents=[parent_parser], + help='Synchronize data file with cloud (cloud settings already setup.') + sync_parser.add_argument( + 'targets', + metavar='', + nargs='*', + help='File or directory to sync.') + sync_parser.add_argument('-j', + '--jobs', + type=int, + default=cpu_count(), + help='Number of jobs to run simultaneously.') + sync_parser.set_defaults(func=CmdDataSync) + + # Repro + repro_parser = subparsers.add_parser( + 'repro', + parents=[parent_parser], + help='Reproduce data') + repro_parser.add_argument( + 'target', + metavar='', + nargs='*', + help='Data items to reproduce.') + repro_parser.add_argument('-f', + '--force', + action='store_true', + default=False, + help='Reproduce even if dependencies were not changed.') + repro_parser.add_argument('-s', + '--single-item', + action='store_true', + default=False, + help='Reproduce only single data item without recursive dependencies check.') + repro_parser.set_defaults(func=CmdRun) + + # Remove + remove_parser = subparsers.add_parser( + 'remove', + parents=[parent_parser], + help='Remove data item from data directory.') + remove_parser.add_argument('target', + metavar='', + nargs='*', + help='Target to remove - file or directory.') + remove_parser.add_argument('-l', + '--keep-in-cloud', + action='store_true', + default=False, + help='Do not remove data from cloud.') + remove_parser.add_argument('-r', + '--recursive', + action='store_true', + help='Remove directory recursively.') + remove_parser.add_argument('-c', + '--keep-in-cache', + action='store_true', + default=False, + help='Do not remove data from cache.') + remove_parser.set_defaults(func=CmdRemove) + + # Import + import_parser = subparsers.add_parser( + 'import', + parents=[parent_parser], + help='Import file to data directory.') + import_parser.add_argument( + 'input', + nargs='+', + help='Input file/files.') + import_parser.add_argument( + 'output', + help='Output file/directory.') + import_parser.add_argument('-l', + '--lock', + action='store_true', + default=False, + help='Lock data item - disable reproduction.') + import_parser.add_argument('-j', + '--jobs', + type=int, + default=cpu_count(), + help='Number of jobs to run simultaneously.') + import_parser.set_defaults(func=CmdImportFile) + + # Lock + lock_parser = subparsers.add_parser( + 'lock', + parents=[parent_parser], + help='Lock') + lock_parser.add_argument('-l', + '--lock', + action='store_true', + default=False, + help='Lock data item - disable reproduction.') + lock_parser.add_argument('-u', + '--unlock', + action='store_true', + default=False, + help='Unlock data item - enable reproduction.') + lock_parser.add_argument( + 'files', + metavar='', + nargs='*', + help='Data items to lock or unlock.') + lock_parser.set_defaults(func=CmdLock) + + # Garbage collector + gc_parser = subparsers.add_parser( + 'gc', + parents=[parent_parser], + help='Collect garbage') + gc_parser.add_argument('target', + metavar='', + nargs='*', + help='Target to remove - file or directory.') + gc_parser.add_argument('-l', + '--keep-in-cloud', + action='store_true', + default=False, + help='Do not remove data from cloud.') + gc_parser.add_argument('-r', + '--recursive', + action='store_true', + help='Remove directory recursively.') + gc_parser.add_argument('-c', + '--keep-in-cache', + action='store_false', + default=False, + help='Do not remove data from cache.') + gc_parser.set_defaults(func=CmdGC) + + # Target + target_parser = subparsers.add_parser( + 'target', + parents=[parent_parser], + help='Set default target') + target_parser.add_argument('target_file', + metavar='', + nargs='?', + help='Target data item.') + target_parser.add_argument('-u', + '--unset', + action='store_true', + default=False, + help='Reset target.') + target_parser.set_defaults(func=CmdTarget) + + if isinstance(argv, str): + argv = argv.split() + + return parser.parse_args(argv) diff --git a/dvc/command/base.py b/dvc/command/base.py index abff5dd7a2..208939b6f7 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -40,29 +40,24 @@ class CmdBase(object): def __init__(self, settings): self._settings = settings - parser = argparse.ArgumentParser() - self.define_common_args(parser) - self.define_args(parser) - - self._parsed_args, self._command_args = parser.parse_known_args(args=self.args) - - self.process_common_args() + if settings._parsed_args.quiet and not settings._parsed_args.verbose: + Logger.be_quiet() + elif not settings._parsed_args.quiet and settings._parsed_args.verbose: + Logger.be_verbose() @property def settings(self): return self._settings + #NOTE: this name is really confusing. It should really be called "command" or smth, + # because it is only used for "command" argument from CmdRun. @property def args(self): return self._settings.args @property def parsed_args(self): - return self._parsed_args - - @property - def command_args(self): - return self._command_args + return self._settings._parsed_args @property def config(self): @@ -76,34 +71,6 @@ def dvc_home(self): def git(self): return self._settings.git - def define_args(self, parser): - pass - - def set_no_git_actions(self, parser): - parser.add_argument('--no-git-actions', '-G', action='store_true', default=False, - help='Skip all git actions including reproducibility check and commits.') - - def set_lock_action(self, parser): - parser.add_argument('--lock', '-l', action='store_true', default=False, - help='Lock data item - disable reproduction. ' + - 'It can be enabled by `dvc lock` command or by forcing reproduction.') - - def define_common_args(self, parser): - parser.add_argument('--quiet', '-q', action='store_true', default=False, help='Be quiet.') - parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Be verbose.') - parser.add_argument('--jobs', '-j', type=int, default=cpu_count(), - help='Number of jobs to run simultaneously.') - - @staticmethod - def set_reset_flag(parser, short_name, long_name, message): - parser.add_argument(short_name, long_name, action='store_true', default=False, help=message) - - def process_common_args(self): - if self._parsed_args.quiet and not self._parsed_args.verbose: - Logger.be_quiet() - elif not self._parsed_args.quiet and self._parsed_args.verbose: - Logger.be_verbose() - @property def no_git_actions(self): return self.parsed_args.no_git_actions @@ -132,24 +99,5 @@ def commit_if_needed(self, message, error=False): def not_committed_changes_warning(): Logger.warn('changes were not committed to git') - def add_string_arg(self, parser, name, message, default = None, - conf_section=None, conf_name=None): - if conf_section and conf_name: - section = self.config[conf_section] - if not section: - raise ConfigError("") - default_value = section.get(conf_section, default) - else: - default_value = default - - parser.add_argument(name, - metavar='', - default=default_value, - help=message) - def run(self): pass - - @staticmethod - def warning_dvc_is_busy(): - Logger.warn('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.') diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 98285b5222..fbe9ff2116 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -3,7 +3,6 @@ from dvc.command.base import CmdBase, DvcLock from dvc.exceptions import DvcException -from dvc.runtime import Runtime from dvc.system import System from dvc.data_cloud import DataCloud from dvc.utils import map_progress @@ -18,12 +17,6 @@ class CmdDataSync(CmdBase): def __init__(self, settings): super(CmdDataSync, self).__init__(settings) - def define_args(self, parser): - parser.add_argument('targets', - metavar='', - help='File or directory to sync.', - nargs='*') - def run(self): with DvcLock(self.is_locker, self.git): cloud = DataCloud(self.settings) @@ -44,6 +37,3 @@ def run(self): map_progress(cloud.sync, targets, self.parsed_args.jobs) pass - -if __name__ == '__main__': - Runtime.run(CmdDataSync) diff --git a/dvc/command/gc.py b/dvc/command/gc.py index b1195ede29..6ff3a5051c 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -3,20 +3,12 @@ from dvc.command.traverse import Traverse from dvc.logger import Logger from dvc.path.data_item import DataItemError -from dvc.runtime import Runtime class CmdGC(Traverse): def __init__(self, settings): super(CmdGC, self).__init__(settings, "garbage collect", do_not_start_from_root=False) - def define_args(self, parser): - super(CmdGC, self).define_args(parser) - parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.') - parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False, - help='Do not remove data from cache.') - pass - def process_file(self, target): Logger.debug(u'[Cmd-GC] GC file {}.'.format(target)) @@ -60,7 +52,3 @@ def no_git_actions(self): @staticmethod def not_committed_changes_warning(): pass - - -if __name__ == '__main__': - Runtime.run(CmdGC) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 65292d81e6..201f93cc71 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -6,7 +6,6 @@ from dvc.data_cloud import sizeof_fmt from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.system import System from dvc.progress import progress @@ -21,16 +20,6 @@ class CmdImportFile(CmdBase): def __init__(self, settings): super(CmdImportFile, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - self.set_lock_action(parser) - - parser.add_argument('input', - help='Input file/files.', - nargs='+') - parser.add_argument('output', - help='Output file/directory.') - def run(self): targets = [] with DvcLock(self.is_locker, self.git): @@ -221,7 +210,3 @@ def create_state_files(self, targets, lock): @staticmethod def is_url(url): return CmdImportFile.URL_REGEX.match(url) is not None - - -if __name__ == '__main__': - Runtime.run(CmdImportFile) diff --git a/dvc/command/init.py b/dvc/command/init.py index fbe38f7bf8..1e0238912e 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -5,7 +5,6 @@ from dvc.logger import Logger from dvc.config import Config from dvc.exceptions import DvcException -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.system import System @@ -62,15 +61,6 @@ class CmdInit(CmdBase): def __init__(self, settings): super(CmdInit, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - - self.add_string_arg(parser, '--data-dir', 'Data directory.', 'data') - self.add_string_arg(parser, '--cache-dir', 'Cache directory.', '.cache') - self.add_string_arg(parser, '--state-dir', 'State directory.', '.state') - self.add_string_arg(parser, '--target-file', 'Target file.', Config.TARGET_FILE_DEFAULT) - pass - def get_not_existing_path(self, dir): path = Path(os.path.join(self.git.git_dir, dir)) if path.exists(): @@ -162,6 +152,3 @@ def modify_gitignore(self, cache_dir_name): fd.write('\n{}'.format(os.path.basename(self.git.lock_file))) Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name)) - -if __name__ == '__main__': - Runtime.run(CmdInit, False) diff --git a/dvc/command/lock.py b/dvc/command/lock.py index f946d49ab9..b2dc9e0bec 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -1,6 +1,5 @@ from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger -from dvc.runtime import Runtime from dvc.state_file import StateFile @@ -8,12 +7,6 @@ class CmdLock(CmdBase): def __init__(self, settings): super(CmdLock, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - CmdLock.set_reset_flag(parser, '-u', '--unlock', 'Unlock data item - enable reproduction.') - parser.add_argument('files', metavar='', help='Data items to lock or unlock.', nargs='*') - pass - def run(self): with DvcLock(self.is_locker, self.git): return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock) @@ -47,7 +40,3 @@ def lock_files(self, files, target): self.commit_if_needed('DVC lock: {}'.format(' '.join(self.args))) return 0 - - -if __name__ == '__main__': - Runtime.run(CmdLock, False) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index db13860bde..21e2f4e94b 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -3,20 +3,12 @@ from dvc.command.traverse import Traverse from dvc.logger import Logger from dvc.path.data_item import DataItemError -from dvc.runtime import Runtime class CmdRemove(Traverse): def __init__(self, settings): super(CmdRemove, self).__init__(settings, "remove") - def define_args(self, parser): - super(CmdRemove, self).define_args(parser) - parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.') - parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False, - help='Do not remove data from cache.') - pass - def process_file(self, target): Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target)) @@ -83,7 +75,3 @@ def remove_file(self, target): def remove_all(self): return self._traverse_all() - - -if __name__ == '__main__': - Runtime.run(CmdRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index b51454c6d5..aa4d9ed4ef 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -7,7 +7,6 @@ from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.path.data_item import NotInDataDirError -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.system import System @@ -28,16 +27,6 @@ def __init__(self, settings): def code_dependencies(self): return self._code - def define_args(self, parser): - self.set_no_git_actions(parser) - - parser.add_argument('target', metavar='', help='Data items to reproduce.', nargs='*') - parser.add_argument('-f', '--force', action='store_true', default=False, - help='Reproduce even if dependencies were not changed.') - parser.add_argument('-s', '--single-item', action='store_true', default=False, - help='Reproduce only single data item without recursive dependencies check.') - pass - @property def lock(self): return True @@ -209,6 +198,10 @@ def reproduce_data_item(self, changed_files): self.state.stdout, self.state.stderr, self.state.shell, + [], + [], + [], + True, check_if_ready=False) != 0: raise ReproError('Run command reproduction failed') return True @@ -317,7 +310,3 @@ def dependencies(self): dependency_data_items.append(data_item) return dependency_data_items - - -if __name__ == '__main__': - Runtime.run(CmdRepro) diff --git a/dvc/command/run.py b/dvc/command/run.py index 4e2d2d5a4a..54eb8175ad 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,4 +1,5 @@ import os +import sys from dvc.command.base import CmdBase, DvcLock from dvc.exceptions import DvcException @@ -6,7 +7,6 @@ from dvc.logger import Logger from dvc.path.data_item import NotInDataDirError, NotInGitDirError from dvc.repository_change import RepositoryChange -from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.utils import cached_property @@ -20,21 +20,6 @@ class CmdRun(CmdBase): def __init__(self, settings): super(CmdRun, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - self.set_lock_action(parser) - - parser.add_argument('--stdout', help='Output std output to a file.') - parser.add_argument('--stderr', help='Output std error to a file.') - parser.add_argument('--input', '-i', action='append', - help='Declare input data items for reproducible cmd.') - parser.add_argument('--output', '-o', action='append', - help='Declare output data items for reproducible cmd.') - parser.add_argument('--code', '-c', action='append', - help='Code dependencies which produce the output.') - parser.add_argument('--shell', help='Shell command', action='store_true', default=False) - pass - @property def lock(self): return self.parsed_args.lock @@ -53,8 +38,8 @@ def declaration_output_data_items(self): def run(self): with DvcLock(self.is_locker, self.git): - data_items_from_args, not_data_items_from_args = self.argv_files_by_type(self.command_args) - return self.run_and_commit_if_needed(self.command_args, + data_items_from_args, not_data_items_from_args = self.argv_files_by_type(self.parsed_args.command) + return self.run_and_commit_if_needed(self.parsed_args.command, data_items_from_args, not_data_items_from_args, self.parsed_args.stdout, @@ -63,7 +48,12 @@ def run(self): pass def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args, - stdout, stderr, shell, check_if_ready=True): + stdout, stderr, shell, + output_data_items=None, + input_data_items=None, + code_dependencies=None, + lock=None, + check_if_ready=True): if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): return 1 @@ -72,12 +62,34 @@ def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_ not_data_items_from_args, stdout, stderr, - shell) + shell, + output_data_items, + input_data_items, + code_dependencies, + lock) - return self.commit_if_needed('DVC run: {}'.format(' '.join(self.args))) + return self.commit_if_needed('DVC run: {}'.format(' '.join(command_args))) def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args, - stdout=None, stderr=None, shell=False): + stdout=None, stderr=None, shell=False, + output_data_items=None, + input_data_items=None, + code_dependencies=None, + lock=None): + + # Repro sets these from state file + if output_data_items == None: + output_data_items = self.declaration_output_data_items + + if input_data_items == None: + input_data_items = self.declaration_input_data_items + + if code_dependencies == None: + code_dependencies = self.code_dependencies + + if lock == None: + lock = self.lock + Logger.debug(u'Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'.format( ' '.join(cmd_args), ', '.join([x.data.dvc for x in data_items_from_args]), @@ -91,13 +103,13 @@ def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args, self.remove_new_files(repo_change) raise RunError('Errors occurred.') - output_set = set(self.declaration_output_data_items + repo_change.changed_data_items) + output_set = set(output_data_items + repo_change.changed_data_items) output_files_dvc = [x.data.dvc for x in output_set] - input_set = set(data_items_from_args + self.declaration_input_data_items) - output_set + input_set = set(data_items_from_args + input_data_items) - output_set input_files_dvc = [x.data.dvc for x in input_set] - code_dependencies_dvc = self.git.abs_paths_to_dvc(self.code_dependencies + not_data_items_from_args) + code_dependencies_dvc = self.git.abs_paths_to_dvc(code_dependencies + not_data_items_from_args) result = [] for data_item in repo_change.changed_data_items: @@ -114,7 +126,7 @@ def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args, output_files_dvc, code_dependencies_dvc, argv=cmd_args, - lock=self.lock, + lock=lock, stdout=self._stdout_to_dvc(stdout), stderr=self._stdout_to_dvc(stderr), shell=shell) @@ -176,7 +188,3 @@ def _data_items_from_params(self, files, param_text): param_text, ', '.join(external)) ) return data_items - - -if __name__ == '__main__': - Runtime.run(CmdRun) diff --git a/dvc/command/target.py b/dvc/command/target.py index fdefa93da2..4b3fa80085 100644 --- a/dvc/command/target.py +++ b/dvc/command/target.py @@ -2,22 +2,15 @@ from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger -from dvc.runtime import Runtime class CmdTarget(CmdBase): def __init__(self, settings): super(CmdTarget, self).__init__(settings) - def define_args(self, parser): - self.set_no_git_actions(parser) - CmdTarget.set_reset_flag(parser, '-u', '--unset', 'Reset target.') - parser.add_argument('target', metavar='', nargs='?', help='Target data item.') - pass - def run(self): with DvcLock(self.is_locker, self.git): - target = self.parsed_args.target + target = self.parsed_args.target_file unset = self.parsed_args.unset if target and unset: @@ -61,7 +54,3 @@ def unset_target(self, target_conf_file_path): open(target_conf_file_path, 'a').close() return self.commit_if_needed('DVC target unset') - - -if __name__ == '__main__': - Runtime.run(CmdTarget, False) diff --git a/dvc/command/test.py b/dvc/command/test.py index 62a46c3fae..58a477c1fd 100644 --- a/dvc/command/test.py +++ b/dvc/command/test.py @@ -5,14 +5,10 @@ from google.cloud import storage +import dvc from dvc.command.base import CmdBase from dvc.logger import Logger from dvc.config import Config -from dvc.runtime import Runtime - -from dvc.settings import SettingsError - - class CmdTest(CmdBase): def __init__(self, settings): @@ -33,7 +29,7 @@ def run(self): if self.settings._config.gc_project_name == '': Logger.error('Please specify the google cloud project name in dvc.conf') - raise SettingsError('must specify GC ProjectName') + raise dvc.settings.SettingsError('must specify GC ProjectName') try: client = storage.Client(project=self.settings._config.gc_project_name) @@ -52,7 +48,3 @@ def run(self): sys.exit(-1) Logger.info('bucket %s visible: google cloud seems to be configured correctly' % sb) sys.exit(0) - - -if __name__ == '__main__': - Runtime.run(CmdTest, False) diff --git a/dvc/command/traverse.py b/dvc/command/traverse.py index 7826978a4b..8dc07069ae 100644 --- a/dvc/command/traverse.py +++ b/dvc/command/traverse.py @@ -18,15 +18,6 @@ def __init__(self, settings, cmd_name, do_not_start_from_root=True): self.cloud = DataCloud(self.settings) self._do_not_start_from_root = do_not_start_from_root - def define_args(self, parser): - self.set_no_git_actions(parser) - - parser.add_argument('target', metavar='', help='Target to remove - file or directory.', nargs='*') - # parser.add_argument('-r', '--recursive', action='store_true', help='CmdGarbage collect directory recursively.') - parser.add_argument('-l', '--keep-in-cloud', action='store_true', default=False, - help='Do not remove data from cloud.') - pass - def run(self): with DvcLock(self.is_locker, self.git): if not self._traverse_all(): diff --git a/dvc/config.py b/dvc/config.py index 3866f9f57a..ccfcfb7c0e 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -46,7 +46,7 @@ def target_file(self): class Config(ConfigI): CONFIG = 'dvc.conf' - def __init__(self, conf_file, conf_pseudo_file=None): + def __init__(self, conf_file=CONFIG, conf_pseudo_file=None): """ Params: conf_file (String): configuration file @@ -59,7 +59,7 @@ def __init__(self, conf_file, conf_pseudo_file=None): self._config.readfp(conf_pseudo_file) else: if not os.path.isfile(conf_file): - raise ConfigError('Config file "{}" does not exist'.format(conf_file)) + raise ConfigError('Config file "{}" does not exist {}'.format(conf_file, os.getcwd())) self._config.read(conf_file) level = self._config['Global']['LogLevel'] diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 1e245f3f7c..71e9f8be86 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -8,10 +8,10 @@ from boto.s3.connection import S3Connection from google.cloud import storage as gc +import dvc from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.config import ConfigError -from dvc.settings import Settings from dvc.progress import progress from dvc.utils import copyfile @@ -382,7 +382,7 @@ class DataCloud(object): } def __init__(self, settings): - assert isinstance(settings, Settings) + assert isinstance(settings, dvc.settings.Settings) #To handle ConfigI case if not hasattr(settings.config, '_config'): diff --git a/dvc/main.py b/dvc/main.py index 09c6926a00..a78fcc2876 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,92 +1,11 @@ -from __future__ import print_function - -from dvc.command.import_file import CmdImportFile - """ main entry point / argument parsing for dvc """ import sys - -from dvc.runtime import Runtime -from dvc.command.init import CmdInit -from dvc.command.remove import CmdRemove -from dvc.command.run import CmdRun -from dvc.command.repro import CmdRepro -from dvc.command.data_sync import CmdDataSync -from dvc.command.lock import CmdLock -from dvc.command.gc import CmdGC -from dvc.command.target import CmdTarget -from dvc.command.test import CmdTest - -VERSION = '0.8.7' - -def print_usage(): - usage = ('', - 'usage: dvc [--version] [--help] command []', - '', - 'These are common DVC commands:', - '', - 'start a working area', - ' init Initialize dvc over a directory (should already be a git dir).', - ' run Run command.', - ' import Import file to data directory.', - ' remove Remove data item from data directory.', - '', - 'synchronize data between remote and local', - ' data sync Synchronize data file with cloud (cloud settings already setup).', -) - print('\n'.join(usage)) +from dvc.settings import Settings def main(): - cmds = ['--help', '--version', 'init', 'run', 'sync', 'repro', 'data', 'remove', 'import', 'lock', 'cloud', \ - 'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \ - 'test', 'test-aws', 'test-gcloud', 'test-cloud', 'gc', 'target'] - - if len(sys.argv) < 2 or sys.argv[1] not in cmds: - if len(sys.argv) >= 2: - print('Unimplemented or unrecognized command: ' + ' '.join(sys.argv[1:])) - print_usage() - sys.exit(-1) - - cmd = sys.argv[1] - - if cmd == '--help': - print_usage() - elif cmd == '--version': - print('dvc version {}'.format(VERSION)) - elif cmd == 'init': - Runtime.run(CmdInit, parse_config=False) - elif cmd == 'run': - Runtime.run(CmdRun) - elif cmd == 'repro': - Runtime.run(CmdRepro) - elif cmd == 'sync': - Runtime.run(CmdDataSync) - elif cmd == 'import': - Runtime.run(CmdImportFile) - elif cmd == 'remove': - Runtime.run(CmdRemove) - elif cmd == 'lock': - Runtime.run(CmdLock) - elif cmd == 'gc': - Runtime.run(CmdGC) - elif cmd == 'target': - Runtime.run(CmdTarget) - elif cmd == 'cloud-run': - print('cloud-run unimplemented') - elif cmd == 'cloud-instance-create': - print('cloud-instance-create unimplemented') - elif cmd == 'clould-instance-remove': - print('cloud-instance-remove unimplemented') - elif cmd == 'cloud-instance-describe': - print('cloud-instance-describe unimplemented') - - elif cmd == 'test-aws': - print('TODO: test aws credentials') - elif cmd == 'test-gcloud': - Runtime.run(CmdTest) - else: - print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1])) - print_usage() - sys.exit(-1) + settings = Settings(sys.argv[1:]) + instance = settings._parsed_args.func(settings) + sys.exit(instance.run()) diff --git a/dvc/progress.py.back b/dvc/progress.py.back new file mode 100644 index 0000000000..eac913dde7 --- /dev/null +++ b/dvc/progress.py.back @@ -0,0 +1,90 @@ +from __future__ import print_function +import sys +import threading +from Queue import Queue + +# (2/100): very_long_nam... [######### ] 60% + +class ProgressInfo(object): + name = "" + current = 0 + total = None + finished = False + +class Progress(object): + _n_finished = 0 + _n_total = 0 + _lock = None + _queue = None + + def __init__(self, total): + self._n_total = total + self._lock = threading.Lock() + self._queue = Queue(maxsize=0) + + def clearln(self): + if sys.stdout.isatty(): + print('\r\x1b[K', end='') + + def writeln(self, line): + if sys.stdout.isatty(): + self.clearln() + print(line, end='') + sys.stdout.flush() + + def update_target(self, name, current, total): + info = ProgressInfo() + info.name = name + info.current = current + info.total = total + + self._queue.put(info) + + def finish_target(self, name): + info = ProgressInfo() + info.name = name + info.current = 100 + info.total = 100 + info.finished = True + + self._queue.put(info) + + def finish(self): + if sys.stdout.isatty(): + print() + + def _bar(self, info): + total_len = 80 + bar_len = 40 + + if info.total == None: + progress = 0 + percent = "?% " + else: + progress = (100 * info.current)/info.total + percent = str(progress) + "% " + + num = "({}/{}): ".format(self._n_finished + 1, self._n_total) + + n_sh = (progress * bar_len)/100 + n_sp = bar_len - n_sh + bar = "[" + '#'*n_sh + ' '*n_sp + "] " + + name_len = len(num + bar + percent) + 1 + name = info.name[:name_len] if len(info.name) > name_len else info.name + + return num + bar + percent + name + + def wait(self): + while self._n_finished != self._n_total: + while not self._queue.empty(): + info = self._queue.get() + bar = self._bar(info) + with self._lock: + if info.finished: + self.clearln() + print(bar) + self._n_finished += 1 + else: + self.writeln(bar) + self.finish() diff --git a/dvc/runtime.py b/dvc/runtime.py deleted file mode 100644 index 06cb18151b..0000000000 --- a/dvc/runtime.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import sys - -from dvc.config import Config, ConfigI -from dvc.exceptions import DvcException -from dvc.git_wrapper import GitWrapper -from dvc.logger import Logger -from dvc.settings import Settings -from dvc.system import System - - -class Runtime(object): - CONFIG = 'dvc.conf' - - @staticmethod - def conf_file_path(git_dir): - return System.realpath(os.path.join(git_dir, Runtime.CONFIG)) - - @staticmethod - def run(cmd_class, parse_config=True, args_start_loc=2): - """ - - Arguments: - args_start_loc (int): where the arguments this command should use start - """ - - try: - runtime_git = GitWrapper() - - if parse_config: - runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir)) - else: - runtime_config = ConfigI() - - args = sys.argv[args_start_loc:] - - # To make argparse print "usage: dvc cmd" instead of "usage: dvc" - sys.argv[0] = sys.argv[0] + " " + sys.argv[1] - - instance = cmd_class(Settings(args, runtime_git, runtime_config)) - sys.exit(instance.run()) - except DvcException as e: - Logger.error(e) - sys.exit(1) diff --git a/dvc/settings.py b/dvc/settings.py index 4cc9162dae..a20a4784a5 100644 --- a/dvc/settings.py +++ b/dvc/settings.py @@ -1,6 +1,10 @@ +import os + +from dvc.git_wrapper import GitWrapperI, GitWrapper +from dvc.config import ConfigI, Config from dvc.exceptions import DvcException from dvc.path.factory import PathFactory - +from dvc.cli import parse_args class SettingsError(DvcException): def __init__(self, msg): @@ -8,18 +12,27 @@ def __init__(self, msg): class Settings(object): - def __init__(self, args, git, config): - self._args = args + def __init__(self, argv=None, git=None, config=None): + self._args = None + args = None + + if argv != None and len(argv) != 0: + args = parse_args(argv) + self._args = argv[2:] + + if git == None: + git = GitWrapper() + + if config == None: + if args != None and args.cmd != 'init': + config = Config() + else: + config = ConfigI() + self._git = git self._config = config self._path_factory = PathFactory(git, config) - - # self._dvc_home = os.environ.get('DVC_HOME') - # if not self._dvc_home: - # raise SettingsError('DVC_HOME environment variable is not defined') - # if not os.path.exists(self._dvc_home): - # raise SettingsError("DVC_HOME directory doesn't exists") - pass + self._parsed_args = args @property def args(self): @@ -28,6 +41,13 @@ def args(self): def set_args(self, args): self._args = args + @property + def parsed_args(self): + return self._parsed_args + + def parse_args(self, args): + self._parsed_args = parse_args(args) + @property def git(self): return self._git @@ -42,4 +62,4 @@ def dvc_home(self): @property def path_factory(self): - return self._path_factory \ No newline at end of file + return self._path_factory diff --git a/setup.py b/setup.py index c4628722d0..e48c411515 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import platform from setuptools import setup, find_packages from distutils.errors import DistutilsPlatformError -from dvc.main import VERSION +from dvc import VERSION install_requires = [ 'altgraph==0.13', diff --git a/tests/basic_env.py b/tests/basic_env.py index 8fb4952d7b..448f1b0b97 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -10,7 +10,6 @@ from dvc.system import System from dvc.utils import rmtree - class BasicEnvironment(TestCase): def init_environment(self, test_dir=System.get_long_path(tempfile.mkdtemp()), diff --git a/tests/test_cmd_gc.py b/tests/test_cmd_gc.py index be8e9c13d5..f3d6c82582 100644 --- a/tests/test_cmd_gc.py +++ b/tests/test_cmd_gc.py @@ -10,7 +10,6 @@ from dvc.system import System from tests.basic_env import BasicEnvironment - class TestCmdDataRemove(BasicEnvironment): def setUp(self): self._test_dir = os.path.join(os.path.sep, 'tmp', 'dvc_unit_test') @@ -24,7 +23,7 @@ def setUp(self): self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit) self._config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self._git, self._config) - self.settings = Settings([], self._git, self._config) + self.settings = Settings('gc', self._git, self._config) self.dir1 = 'dir1' @@ -53,11 +52,10 @@ def test_all(self): self.assertEqual(5, self.cache_file_nums('file2*')) self.assertEqual(5, self.cache_file_nums(os.path.join(self.dir1, 'file3*'))) + self.settings.parse_args('gc --no-git-actions data') cmd = CmdGC(self.settings) os.chdir(self._test_git_dir) - cmd.parsed_args.no_git_actions = True - cmd.parsed_args.target = ['data'] cmd.gc_all() self.assertEqual(1, self.cache_file_nums('file1*')) @@ -72,10 +70,10 @@ def test_empty(self): self.assertEqual(5, self.cache_file_nums('file1*')) self.assertEqual(5, self.cache_file_nums('file2*')) + self.settings.parse_args('gc --no-git-actions') cmd = CmdGC(self.settings) os.chdir(self._test_git_dir) - cmd.parsed_args.no_git_actions = True cmd.gc_all() self.assertEqual(5, self.cache_file_nums('file1*')) diff --git a/tests/test_cmd_remove.py b/tests/test_cmd_remove.py index 9ccabac53d..168f3a8c60 100644 --- a/tests/test_cmd_remove.py +++ b/tests/test_cmd_remove.py @@ -3,14 +3,13 @@ from dvc.command.remove import CmdRemove from tests.basic_env import DirHierarchyEnvironment - class TestCmdRemove(DirHierarchyEnvironment): def setUp(self): DirHierarchyEnvironment.init_environment(self) def test_file_by_file_removal(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True dir1_dvc_name = os.path.join('data', self.dir1) self.assertTrue(os.path.exists(dir1_dvc_name)) @@ -19,9 +18,8 @@ def test_file_by_file_removal(self): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_remove_deep_dir(self): + self.settings.parse_args('remove --keep-in-cloud --recursive') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.recursive = True dir1_dvc_name = os.path.join('data', self.dir1) self.assertTrue(os.path.exists(dir1_dvc_name)) @@ -30,9 +28,8 @@ def test_remove_deep_dir(self): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_not_recursive_removal(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.recursive = False dir1_dvc_name = os.path.join('data', self.dir1) self.assertTrue(os.path.exists(dir1_dvc_name)) @@ -41,9 +38,8 @@ def test_not_recursive_removal(self): pass def test_data_dir_removal(self): + self.settings.parse_args('remove --keep-in-cloud --recursive') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.recursive = True data_dir = 'data' self.assertTrue(os.path.exists(data_dir)) @@ -57,8 +53,8 @@ def setUp(self): DirHierarchyEnvironment.init_environment(self) def test_remove_data_instance(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file1)) self.assertTrue(os.path.isfile(self.cache1)) @@ -70,9 +66,8 @@ def test_remove_data_instance(self): self.assertFalse(os.path.exists(self.state1)) def test_keep_cache(self): + self.settings.parse_args('remove --keep-in-cloud --keep-in-cache') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - cmd.parsed_args.keep_in_cache = True self.assertTrue(os.path.isfile(self.file1)) self.assertTrue(os.path.isfile(self.cache1)) @@ -84,8 +79,8 @@ def test_keep_cache(self): self.assertFalse(os.path.exists(self.state1)) def test_remove_data_instance_without_cache(self): + self.settings.parse_args('remove --keep-in-cloud') cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file6)) self.assertIsNone(self.cache6) @@ -101,16 +96,11 @@ def setUp(self): DirHierarchyEnvironment.init_environment(self) def test_end_to_end_with_no_error(self): - cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - dir11_full = os.path.join('data', self.dir11) dir2_full = os.path.join('data', self.dir2) - cmd.parsed_args.target = [dir11_full, self.file5] - cmd.parsed_args.recursive = True - - cmd.parsed_args.no_git_actions = True + self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5)) + cmd = CmdRemove(self.settings) self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) @@ -125,16 +115,11 @@ def test_end_to_end_with_no_error(self): self.assertTrue(os.path.exists(self.file1)) def test_run(self): - cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - dir11_full = os.path.join('data', self.dir11) dir2_full = os.path.join('data', self.dir2) - cmd.parsed_args.target = [dir11_full, self.file5] - cmd.parsed_args.recursive = True - - cmd.parsed_args.no_git_actions = True + self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5)) + cmd = CmdRemove(self.settings) self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) @@ -154,16 +139,11 @@ def setUp(self): DirHierarchyEnvironment.init_environment(self) def test(self): - cmd = CmdRemove(self.settings) - cmd.parsed_args.keep_in_cloud = True - dir11_full = os.path.join(self._config.data_dir, self.dir11) dir2_full = os.path.join(self._config.data_dir, self.dir2) - cmd.parsed_args.target = [dir11_full, self.file5] - cmd.parsed_args.recursive = True - - cmd.parsed_args.no_git_actions = True + self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5)) + cmd = CmdRemove(self.settings) self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) diff --git a/tests/test_cmd_run.py b/tests/test_cmd_run.py index 51bc0f9f64..32ea39365f 100644 --- a/tests/test_cmd_run.py +++ b/tests/test_cmd_run.py @@ -31,7 +31,7 @@ def setUp(self): self.config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self.git, self.config) - self.settings = Settings([], self.git, self.config) + self.settings = Settings(['run'], self.git, self.config) pass def init_git_repo(self): @@ -65,13 +65,12 @@ class RunTwoFilesBase(RunBasicTest): def setUp(self): super(RunTwoFilesBase, self).setUp() - self.settings._args = [] - cmd_run = CmdRun(self.settings) self.input_param_file = os.path.join('data', 'extra_input') - cmd_run.parsed_args.input = [self.input_param_file] - self.extra_output_file = os.path.join('data', 'extra_output') - cmd_run.parsed_args.output = [self.extra_output_file] + + self.settings.parse_args('run --input {} --output {}'.format(self.input_param_file, self.extra_output_file)) + self.settings._args = [] + cmd_run = CmdRun(self.settings) self.file_name1 = os.path.join('data', 'file1') self.file_name2 = os.path.join('data', 'file2') diff --git a/tests/test_cmd_target.py b/tests/test_cmd_target.py index 050afaac59..9dc8e5e1e7 100644 --- a/tests/test_cmd_target.py +++ b/tests/test_cmd_target.py @@ -22,7 +22,7 @@ def setUp(self): self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit) self._config = ConfigI('data', 'cache', 'state', '.target') self.path_factory = PathFactory(self._git, self._config) - self.settings = Settings([], self._git, self._config) + self.settings = Settings(['target'], self._git, self._config) os.chdir(self._test_git_dir) @@ -45,53 +45,49 @@ def test_initial_default_target(self): self.assertFalse(os.path.exists('.target')) def test_single_file(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file1_data) def test_multiple_files(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False cmd.run() # Another target - cmd.parsed_args.target = self.file2_data + self.settings.parse_args('target {}'.format(self.file2_data)) + cmd = CmdTarget(self.settings) self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file2_data) # Unset target - cmd.parsed_args.target = '' - cmd.parsed_args.unset = True + self.settings.parse_args('target --unset') + cmd = CmdTarget(self.settings) self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), '') def test_initial_unset(self): + self.settings.parse_args('target --unset') cmd = CmdTarget(self.settings) - cmd.parsed_args.target = '' - cmd.parsed_args.unset = True self.assertEqual(cmd.run(), 1) self.assertFalse(os.path.exists('.target')) def test_unset_existing_target(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file1_data) - cmd.parsed_args.target = '' - cmd.parsed_args.unset = True + self.settings.parse_args('target --unset') + cmd = CmdTarget(self.settings) self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), '') def test_the_same(self): + self.settings.parse_args('target {}'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = False self.assertEqual(cmd.run(), 0) self.assertEqual(open('.target').read(), self.file1_data) @@ -99,13 +95,11 @@ def test_the_same(self): self.assertEqual(open('.target').read(), self.file1_data) def test_args_conflict(self): + self.settings.parse_args('target {} --unset'.format(self.file1_data)) cmd = CmdTarget(self.settings) - cmd.parsed_args.target = self.file1_data - cmd.parsed_args.unset = True self.assertEqual(cmd.run(), 1) def test_no_args(self): + self.settings.parse_args('target --unset') cmd = CmdTarget(self.settings) - cmd.parsed_args.target = '' - cmd.parsed_args.unset = False - self.assertEqual(cmd.run(), 1) \ No newline at end of file + self.assertEqual(cmd.run(), 1) diff --git a/tests/test_import.py b/tests/test_import.py index 4b1cee6d8b..96b24fcc6b 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -18,7 +18,7 @@ def test_import_file_to_dir(self): dir = os.path.join('data', self.dir1) settings = copy(self.settings) - settings.set_args([temp.name, dir]) + settings.parse_args('import {} {}'.format(temp.name, dir)) cmd = CmdImportFile(settings) cmd.run() @@ -44,7 +44,7 @@ def test_import_file_to_file(self): target_file_name = 'targ.txt' target_file = os.path.join('data', self.dir1, target_file_name) settings = copy(self.settings) - settings.set_args([temp.name, target_file]) + settings.parse_args('import {} {}'.format(temp.name, target_file)) cmd = CmdImportFile(settings) cmd.run() @@ -60,7 +60,7 @@ def test_import_to_existing_dir(self): dir = os.path.join('data', self.dir11) settings = copy(self.settings) - settings.set_args([temp.name, dir]) + settings.parse_args('import {} {}'.format(temp.name, dir)) cmd = CmdImportFile(settings) cmd.run() @@ -78,7 +78,7 @@ def test_import_to_not_existing_dir(self): dir = os.path.join('data', self.dir1, 'not_existing_dir/') settings = copy(self.settings) - settings.set_args([temp.name, dir]) + settings.parse_args('import {} {}'.format(temp.name, dir)) cmd = CmdImportFile(settings) with self.assertRaises(ImportFileError): @@ -103,7 +103,7 @@ def test_multiple_import(self): dir = os.path.join('data', self.dir11) settings = copy(self.settings) - settings.set_args([temp1.name, temp2.name, dir]) + settings.parse_args('import {} {} {}'.format(temp1.name, temp2.name, dir)) cmd = CmdImportFile(settings) cmd.run() @@ -115,4 +115,4 @@ def test_multiple_import(self): self.assertTrue(os.path.exists(dvc_name2)) self.assertTrue(open(dvc_name2).read(), content) - pass \ No newline at end of file + pass diff --git a/tests/test_repro.py b/tests/test_repro.py index 20d993e2b1..8a31752963 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -14,8 +14,10 @@ def setUp(self): self.file_name1 = os.path.join('data', 'file1') self.file1_code_file = 'file1.py' self.create_file_and_commit(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")') - self.settings._args = ['python', self.file1_code_file, '--not-repro', - '--stdout', self.file_name1, '--code', self.file1_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name1, + '--code', self.file1_code_file, + 'python', self.file1_code_file, '--not-repro']) cmd_file1 = CmdRun(self.settings) self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file]) cmd_file1.run() @@ -25,16 +27,19 @@ def setUp(self): self.create_file_and_commit(self.file11_code_file, 'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())') - self.settings._args = ['python', self.file11_code_file, self.file_name1, - '--stdout', self.file_name11, '--code', self.file11_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name11, + '--code', self.file11_code_file, + 'python', self.file11_code_file, self.file_name1]) CmdRun(self.settings).run() self.file_name2 = os.path.join('data', 'file2') self.file2_code_file = 'file2.py' self.create_file_and_commit(self.file2_code_file, 'print("Bobby")') - self.settings._args = ['python', self.file2_code_file, - '--not-repro', '--stdout', self.file_name2] + self.settings.parse_args(['run', + '--stdout', self.file_name2, + 'python', self.file2_code_file, '--not-repro']) CmdRun(self.settings).run() self.file_res_code_file = 'code_res.py' @@ -44,11 +49,13 @@ def setUp(self): 'text2 = open(sys.argv[2]).read()' + os.linesep + 'print(text1 + text2)') self.file_name_res = os.path.join('data', 'file_res') - self.settings._args = ['python', self.file_res_code_file, - self.file_name11, - self.file_name2, - '--stdout', self.file_name_res, - '--code', self.file_res_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name_res, + '--code', self.file_res_code_file, + 'python', self.file_res_code_file, + self.file_name11, + self.file_name2]) + cmd_res = CmdRun(self.settings) self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file]) cmd_res.run() @@ -76,7 +83,7 @@ class ReproCodeDependencyTest(ReproBasicEnv): def test(self): self.modify_file_and_commit(self.file_res_code_file) - self.settings._args = [self.file_name_res] + self.settings.parse_args('repro {}'.format(self.file_name_res)) CmdRepro(self.settings).run() self.assertEqual(open(self.file_name_res).read().strip(), 'Hello\nBobby') @@ -86,19 +93,21 @@ class ReproChangedDependency(ReproBasicEnv): def test(self): self.recreate_file1() - self.settings._args = [self.file_name11] + self.settings.parse_args('repro {}'.format(self.file_name11)) CmdRepro(self.settings).run() self.assertEqual(open(self.file_name11).read(), 'Goodbye\n') def recreate_file1(self): - self.settings._args = [self.file_name1, '--keep-in-cloud'] + self.settings.parse_args('remove {} --keep-in-cloud'.format(self.file_name1)) CmdRemove(self.settings).run() file1_code_file = 'file1_2.py' self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")') - self.settings._args = ['python', file1_code_file, '--not-repro', - '--stdout', self.file_name1, '--code', file1_code_file] + self.settings.parse_args(['run', + '--stdout', self.file_name1, + '--code', file1_code_file, + 'python', file1_code_file, '--not-repro']) CmdRun(self.settings).run() @@ -107,7 +116,7 @@ class ReproChangedDeepDependency(ReproChangedDependency): def test(self): self.recreate_file1() - self.settings._args = [self.file_name_res] + self.settings.parse_args('repro {}'.format(self.file_name_res)) CmdRepro(self.settings).run() self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')