diff --git a/.travis.yml b/.travis.yml index a9c3bb8..a753390 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,10 @@ python: - "nightly" # command to install dependencies install: + - "pip install -r requirements.txt" - "pip install -r dev-requirements.txt" # command to run tests script: - - "python -m pytest tests/ -v" + - "python -m pytest tests/ *.py -v --doctest-modules" notifications: email: false diff --git a/dev-requirements.in b/dev-requirements.in new file mode 100644 index 0000000..0c9c705 --- /dev/null +++ b/dev-requirements.in @@ -0,0 +1,3 @@ +hypothesis +pytest +pytest-mock diff --git a/dev-requirements.txt b/dev-requirements.txt index a78ce5a..0fee336 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,15 @@ -enum34==1.1.6 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file dev-requirements.txt dev-requirements.in +# +enum34==1.1.6 # via hypothesis +funcsigs==1.0.2 # via mock hypothesis==3.6.0 -py==1.4.31 -pytest==3.0.3 +mock==2.0.0 # via pytest-mock +pbr==1.10.0 # via mock +py==1.4.31 # via pytest +pytest-mock==1.4.0 +pytest==3.0.4 +six==1.10.0 # via mock diff --git a/elm_deps_sync.py b/elm_deps_sync.py index 1c498d0..2922fad 100755 --- a/elm_deps_sync.py +++ b/elm_deps_sync.py @@ -2,66 +2,48 @@ from __future__ import print_function import sys -from collections import OrderedDict -import json import argparse +import elm_package + def sync_versions(top_level_file, spec_file, quiet=False, dry=False, note_test_deps=True): - """ first file should be the top level elm-package.json - second file should be the spec file + """ first file should be the top level elm-package.json. + second file should be the spec level elm-package.json. """ with open(top_level_file) as f: - top_level = json.load(f) + top_level = elm_package.load(f) with open(spec_file) as f: - spec = json.load(f, object_pairs_hook=OrderedDict) - - messages = [] - - for (package_name, package_version) in top_level['dependencies'].items(): - if package_name not in spec['dependencies']: - spec['dependencies'][package_name] = package_version - - messages.append('Package {package_name} inserted to {spec_file} for the first time at version "{package_version}"'.format( - package_name=package_name, spec_file=spec_file, package_version=package_version) - ) - elif spec['dependencies'][package_name] != package_version: - messages.append('Changing {package_name} from version {package_version} to {other_package_version}'.format( - package_version=package_version, package_name=package_name, - other_package_version=spec['dependencies'][package_name]) - ) - - spec['dependencies'][package_name] = package_version + spec = elm_package.load(f) - test_deps = {} - - for (package_name, package_version) in spec['dependencies'].items(): - if package_name not in top_level['dependencies']: - test_deps[package_name] = package_version + (messages, new_deps) = elm_package.sync_deps(top_level['dependencies'], spec['dependencies']) + spec['dependencies'] = new_deps if note_test_deps: - spec['test-dependencies'] = sorted_deps(test_deps) - - if len(messages) > 0 or note_test_deps: - print('{number} packages changed.'.format(number=len(messages))) + test_deps = {} - if not dry: - spec['dependencies'] = sorted_deps(spec['dependencies']) - with open(spec_file, 'w') as f: - json.dump(spec, f, sort_keys=False, indent=4, separators=(',', ': ')) - else: - print("No changes written.") + for (package_name, package_version) in spec['dependencies'].items(): + if package_name not in top_level['dependencies']: + test_deps[package_name] = package_version + spec['test-dependencies'] = elm_package.sorted_deps(test_deps) - if not quiet: - print('\n'.join(messages)) - else: + if len(messages) == 0 and not note_test_deps: print('No changes needed.') + return + + print('{number} packages changed.'.format(number=len(messages))) + + if not quiet: + print('\n'.join(messages)) + if dry: + print("No changes written.") + return -def sorted_deps(deps): - return OrderedDict(sorted(deps.items())) + with open(spec_file, 'w') as f: + elm_package.dump(spec, f) def main(): diff --git a/elm_package.py b/elm_package.py new file mode 100644 index 0000000..23daed6 --- /dev/null +++ b/elm_package.py @@ -0,0 +1,49 @@ +#! /usr/bin/env python +""" +Load and save elm-package.json safely. +""" + +# from typing import Dict, Tuple, IO +import copy +from collections import OrderedDict +import json + + +def load(fileobj): + # type: (IO[str]) -> Dict + return json.load(fileobj, object_pairs_hook=OrderedDict) + + +def dump(package, fileobj): + # type: (Dict, IO[str]) -> None + to_save = copy.deepcopy(package) + to_save['dependencies'] = sorted_deps(to_save['dependencies']) + json.dump(to_save, fileobj, sort_keys=False, indent=4, separators=(',', ': ')) + + +def sorted_deps(deps): + # type: (Dict) -> Dict + return OrderedDict(sorted(deps.items())) + + +def sync_deps(from_deps, to_deps): + # type: (Dict, Dict) -> Tuple[List[str], Dict] + messages = [] + result = copy.deepcopy(to_deps) + + for (package_name, package_version) in from_deps.items(): + if package_name not in to_deps: + result[package_name] = package_version + + messages.append('Inserting new package {package_name} at version {package_version}'.format( + package_name=package_name, package_version=package_version) + ) + elif to_deps[package_name] != package_version: + result[package_name] = package_version + + messages.append('Changing {package_name} from version {package_version} to {other_package_version}'.format( + package_version=package_version, package_name=package_name, + other_package_version=to_deps[package_name]) + ) + + return messages, result diff --git a/exact_dependencies.py b/exact_dependencies.py new file mode 100644 index 0000000..585188d --- /dev/null +++ b/exact_dependencies.py @@ -0,0 +1,25 @@ +#! /usr/bin/env python +""" +Load and save exact-dependencies.json safely. + +The format of exact-dependencies is simply a dictionary of +package name and its exact version: + + { + "elm-lang/core": "4.0.5" + } +""" + +# from typing import Dict, IO +import json + +import elm_package + + +load = elm_package.load + + +def dump(package, fileobj): + # type: (Dict, IO[str]) -> None + to_save = elm_package.sorted_deps(package) + json.dump(to_save, fileobj, sort_keys=False, indent=4, separators=(',', ': ')) diff --git a/native_deps_sync.py b/native_deps_sync.py new file mode 100755 index 0000000..e3d4c6d --- /dev/null +++ b/native_deps_sync.py @@ -0,0 +1,57 @@ +#! /usr/bin/env python +from __future__ import print_function + +import sys +import argparse + +import elm_package +import exact_dependencies + + +def sync_versions(top_level_file, spec_file, quiet=False, dry=False, note_test_deps=True): + """ first file should be the top level elm-native-package.json. + second file should be the spec level elm-native-package.json. + """ + + with open(top_level_file) as f: + top_level = exact_dependencies.load(f) + + with open(spec_file) as f: + spec = exact_dependencies.load(f) + + (messages, new_deps) = elm_package.sync_deps(top_level, spec) + spec = new_deps + + if len(messages) == 0: + print('No changes needed.') + return + + print('{number} packages changed.'.format(number=len(messages))) + + if not quiet: + print('\n'.join(messages)) + + if dry: + print("No changes written.") + return + + with open(spec_file, 'w') as f: + exact_dependencies.dump(spec, f) + + + +def main(): + + parser = argparse.ArgumentParser(description='Sync deps between a parent and a sub') + + parser.add_argument('--quiet', '-q', action='store_true', help='don\'t print anything', default=False) + parser.add_argument('--dry', '-d', action='store_true', help='only print possible changes', default=False) + parser.add_argument('top_level_file') + parser.add_argument('spec_file') + args = parser.parse_args() + + sync_versions(args.top_level_file, args.spec_file, quiet=args.quiet, dry=args.dry) + + +if __name__ == '__main__': + main() diff --git a/native_package_install.py b/native_package_install.py index 390ee65..d3cb6cb 100755 --- a/native_package_install.py +++ b/native_package_install.py @@ -5,11 +5,18 @@ import argparse import collections import fnmatch -import json import os import sys import tarfile -import urllib2 +try: + # For Python 3.0 and later + from urllib.request import urlopen +except ImportError: + # Fall back to Python 2's urllib2 + from urllib2 import urlopen + +import elm_package +import exact_dependencies def read_native_elm_package(package_file): @@ -18,44 +25,45 @@ def read_native_elm_package(package_file): """ with open(package_file) as f: - return json.load(f) + return exact_dependencies.load(f) -def format_url(package): +def format_tarball_url(package): """ Creates the url to fetch the tar from github. - >>> format_url({'namespace': 'elm-lang', 'name': 'navigation', 'version': '2.0.0'}) + >>> format_tarball_url({'owner': 'elm-lang', 'project': 'navigation', 'version': '2.0.0'}) 'https://github.com/elm-lang/navigation/archive/2.0.0.tar.gz' """ - return "https://github.com/{namespace}/{name}/archive/{version}.tar.gz".format(**package) + return "https://github.com/{owner}/{project}/archive/{version}.tar.gz".format(**package) -def parse_json(raw): +def packages_from_exact_deps(exact_dependencies): """ - Parses the json and returns a list of {version, namespace, name}. - >>> parse_json({'elm-lang/navigation': '2.0.0'}) - [{'version': '2.0.0', 'namespace': 'elm-lang', 'name': 'navigation'}] + Parses the json and returns a list of {version, owner, project}. + >>> packages_from_exact_deps({'elm-lang/navigation': '2.0.0'}) \ + == [{'version': '2.0.0', 'owner': 'elm-lang', 'project': 'navigation'}] + True """ result = [] - for name, version in raw.items(): - namespace, package_name = name.split('/') + for package, version in exact_dependencies.items(): + owner, project = package.split('/') result.append({ - 'namespace': namespace, - 'name': package_name, + 'owner': owner, + 'project': project, 'version': version }) return result -def format_vendor_dir(base, namespace): +def ensure_vendor_owner_dir(base, owner): """ Creates the path in the vendor folder. - >>> format_vendor_dir('foo', 'bar') + >>> ensure_vendor_owner_dir('foo', 'bar') 'foo/bar' """ - path = os.path.join(base, namespace) + path = os.path.join(base, owner) try: os.makedirs(path) @@ -65,15 +73,16 @@ def format_vendor_dir(base, namespace): return path -def package_dir(vendor_dir, package): +def vendor_package_dir(vendor_dir, package): """ - Creates the path to the elm package. - >>> package_dir('vendor/assets/elm', {'version': '2.0.0', 'namespace': 'elm-lang', 'name': 'navigation'}) + Returns the path to the elm package. Also creates the parent directory if misisng. + >>> vendor_package_dir('vendor/assets/elm', {'version': '2.0.0', 'owner': 'elm-lang', 'project': 'navigation'}) 'vendor/assets/elm/elm-lang/navigation-2.0.0' """ - return "{vendor_dir}/{package_name}-{version}".format( - vendor_dir=format_vendor_dir(vendor_dir, package['namespace']), - package_name=package['name'], + vendor_owner_dir = ensure_vendor_owner_dir(vendor_dir, package['owner']) + return "{vendor_owner_dir}/{project}-{version}".format( + vendor_owner_dir=vendor_owner_dir, + project=package['project'], version=package['version'] ) @@ -83,60 +92,61 @@ def fetch_packages(vendor_dir, packages): Fetches all packages from github. """ for package in packages: - tar_filename = format_tar_file(vendor_dir, package) - vendor = format_vendor_dir(vendor_dir, package['namespace']) - url = format_url(package) + tar_filename = format_tar_path(vendor_dir, package) + vendor_owner_dir = ensure_vendor_owner_dir(vendor_dir, package['owner']) + url = format_tarball_url(package) - print("Downloading {namespace}/{name} {version}".format(**package)) - tar_file = urllib2.urlopen(url) + print("Downloading {owner}/{project} {version}".format(**package)) + tar_file = urlopen(url) with open(tar_filename, 'w') as tar: tar.write(tar_file.read()) with tarfile.open(tar_filename) as tar: - tar.extractall(vendor, members=tar.getmembers()) + tar.extractall(vendor_owner_dir, members=tar.getmembers()) return packages -def format_tar_file(vendor_dir, package): +def format_tar_path(vendor_dir, package): """ - The name of the tar. - >>> format_tar_file('vendor/assets/elm', {'namespace': 'elm-lang', 'name': 'navigation', 'version': '2.0.0'}) + The path of the tar. + >>> format_tar_path('vendor/assets/elm', {'owner': 'elm-lang', 'project': 'navigation', 'version': '2.0.0'}) 'vendor/assets/elm/elm-lang/navigation-2.0.0-tar.gz' """ - vendor = format_vendor_dir(vendor_dir, package['namespace']) - return package_dir(vendor_dir, package) + "-tar.gz" + ensure_vendor_owner_dir(vendor_dir, package['owner']) + return vendor_package_dir(vendor_dir, package) + "-tar.gz" + -def format_native_name(namespace, name): +def format_native_name(owner, project): """ - Formates the package to the namespace used in elm native. + Formates the package to the owner used in elm native. >>> format_native_name('elm-lang', 'navigation') '_elm_lang$navigation' """ - underscored_namespace = namespace.replace("-", "_") - underscored_name = name.replace("-", "_") - return "_{owner}${repo}".format(owner=underscored_namespace, repo=underscored_name) + underscored_owner = owner.replace("-", "_") + underscored_project = project.replace("-", "_") + return "_{owner}${repo}".format(owner=underscored_owner, repo=underscored_project) -def namespace_from_repo(repository): +def package_name_from_repo(repository): """ - Namespace and name from repository. - >>> namespace_from_repo('https://github.com/NoRedInk/noredink.git') - ['NoRedInk', 'noredink'] + Owner and project from repository. + >>> package_name_from_repo('https://github.com/NoRedInk/noredink.git') + ('NoRedInk', 'noredink') """ repo_without_domain = repository.split('https://github.com/')[1].split('.git')[0] - (namespace, name) = repo_without_domain.split('/') - return [namespace, name] + (owner, project) = repo_without_domain.split('/') + return (owner, project) def get_source_dirs(vendor_dir, package): """ get the source-directories out of an elm-package file """ - elm_package_filename = os.path.join(package_dir(vendor_dir, package), 'elm-package.json') + elm_package_filename = os.path.join(vendor_package_dir(vendor_dir, package), 'elm-package.json') with open(elm_package_filename) as f: - data = json.load(f) + data = elm_package.load(f) return data['source-directories'] @@ -165,62 +175,69 @@ def munge_names(vendor_dir, repository, packages): """ Replaces the namespaced function names in all native code by the namespace from the given elm-package.json. """ - namespace, name = namespace_from_repo(repository) + owner, project = package_name_from_repo(repository) for package in packages: - native_files = find_all_native_files(package_dir(vendor_dir, package)) + native_files = find_all_native_files(vendor_package_dir(vendor_dir, package)) for native_file in native_files: replace_in_file( native_file, - format_native_name(package['namespace'], package['name']), - format_native_name(namespace, name) + format_native_name(package['owner'], package['project']), + format_native_name(owner, project) ) -def update_elm_package(vendor_dir, configs, packages): +def update_source_directories(vendor_dir, elm_package_paths, native_packages): """ - Gets the repo name and updates the source-directories in the given elm-package.json. + Updates the source-directories in the given elm-package.json files. + Returns the repository of the last elm-package.json. """ repository = "" - for config in configs: - with open(config) as f: - data = json.load(f, object_pairs_hook=collections.OrderedDict) + for elm_package_path in elm_package_paths: + with open(elm_package_path) as f: + data = elm_package.load(f) repository = data['repository'] source_directories = data['source-directories'] - path = '../' * config.count('/') + elm_package_dir = os.path.dirname(elm_package_path) needs_save = False - for package in packages: - current_package_dirs = get_source_dirs(vendor_dir, package) + for native_package in native_packages: + source_dirs = get_source_dirs(vendor_dir, native_package) - for dir_name in current_package_dirs: - relative_path = os.path.join(path, package_dir(vendor_dir, package), dir_name) + for source_dir in source_dirs: + absolute_source_dir = os.path.join( + vendor_package_dir(vendor_dir, native_package), source_dir) + relative_path = os.path.relpath(absolute_source_dir, elm_package_dir) if relative_path not in data['source-directories']: data['source-directories'].append(relative_path) needs_save = True if needs_save: - with open(config, 'w') as f: - f.write(json.dumps(data, indent=4)) + with open(elm_package_path, 'w') as f: + elm_package.dump(data, f) return repository -def filter_packages(vendor_dir, packages): - return [x for x in packages if not os.path.isdir(format_tar_file(vendor_dir, x))] +def exclude_downloaded_packages(vendor_dir, packages): + return [x for x in packages if not os.path.isfile(format_tar_path(vendor_dir, x))] + +def main(native_elm_package_path, elm_package_paths, vendor_dir): + absolute_vendor_dir = os.path.abspath(vendor_dir) + absolute_elm_package_paths = list(map(os.path.abspath, elm_package_paths)) -def main(native_elm_package, configs, vendor): - raw_json = read_native_elm_package(native_elm_package) - parsed = parse_json(raw_json) - packages = filter_packages(vendor, parsed) - fetch_packages(vendor, packages) - repository = update_elm_package(vendor, configs, packages) - munge_names(vendor, repository, packages) + raw_json = read_native_elm_package(native_elm_package_path) + all_packages = packages_from_exact_deps(raw_json) + required_packages = exclude_downloaded_packages(absolute_vendor_dir, all_packages) + fetch_packages(absolute_vendor_dir, required_packages) + repository = update_source_directories( + absolute_vendor_dir, absolute_elm_package_paths, required_packages) + munge_names(absolute_vendor_dir, repository, required_packages) def test(): diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..f229360 --- /dev/null +++ b/requirements.in @@ -0,0 +1 @@ +requests diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..28a509b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements.txt requirements.in +# +requests==2.12.0 diff --git a/tests/test_native_deps_sync.py b/tests/test_native_deps_sync.py new file mode 100644 index 0000000..9c97235 --- /dev/null +++ b/tests/test_native_deps_sync.py @@ -0,0 +1,46 @@ +from collections import OrderedDict +import json + +from hypothesis import given +import hypothesis.strategies as st + +import native_deps_sync + + +top_level_deps = [ + ('NoRedInk/top-1', '1.0.0 <= v <= 1.0.0'), + ('NoRedInk/top-2', '1.0.0 <= v <= 1.0.0'), + ('NoRedInk/top-3', '1.0.0 <= v <= 1.0.0'), +] + +spec_deps = [ + ('NoRedInk/top-1', '1.0.0 <= v <= 1.0.0'), + ('NoRedInk/top-2', '1.0.0 <= v <= 1.0.0'), + ('NoRedInk/spec-1', '1.0.0 <= v <= 1.0.0'), + ('NoRedInk/spec-2', '1.0.0 <= v <= 1.0.0'), +] + + +@given(top_level_deps=st.permutations(top_level_deps), + spec_deps=st.permutations(spec_deps)) +def test_spec_order_is_preserved( + tmpdir, + top_level_deps, + spec_deps): + top_level_file = tmpdir.join('elm-native-package.json') + spec_file = tmpdir.join('spec-elm-native-package.json') + + top_level = OrderedDict(top_level_deps) + top_level_file.write(json.dumps(top_level)) + + spec = OrderedDict(spec_deps) + spec_file.write(json.dumps(spec)) + + native_deps_sync.sync_versions( + str(top_level_file), + str(spec_file), + quiet=False, + dry=False) + + new_spec = json.loads(spec_file.read(), object_pairs_hook=OrderedDict) + assert list(new_spec.keys()) == ['NoRedInk/spec-1', 'NoRedInk/spec-2', 'NoRedInk/top-1', 'NoRedInk/top-2', 'NoRedInk/top-3'] diff --git a/tests/test_native_package_install.py b/tests/test_native_package_install.py new file mode 100644 index 0000000..6ffd2b2 --- /dev/null +++ b/tests/test_native_package_install.py @@ -0,0 +1,118 @@ +import json +import tarfile +import difflib + +import native_package_install + + +def test_main_does_not_download_twice(tmpdir, mocker): + native_elm_package = {'elm-lang/core': '1.0.0'} + native_elm_package_path = tmpdir.join('elm-native-package.json') + native_elm_package_path.write(json.dumps(native_elm_package)) + + fake_native_tarball_path = tmpdir.join('core.tgz') + with tmpdir.as_cwd(): + with tarfile.open(str(fake_native_tarball_path), 'w') as f: + fake_elm_package = tmpdir.mkdir('core-1.0.0').join('elm-package.json') + fake_elm_package.write(json.dumps({ + 'source-directories': ['src', 'src2'], + })) + f.add(str(fake_elm_package.relto(tmpdir))) + + elm_package_one_path = tmpdir.join('elm-package-one.json') + elm_package_one_path.write(json.dumps({ + 'repository': 'https://github.com/NoRedInk/elm-ops-tooling.git', + 'source-directories': ['.'], + 'dependencies': {}, + })) + + elm_package_two_path = tmpdir.join('elm-package-two.json') + elm_package_two_path.write(json.dumps({ + 'repository': 'https://github.com/NoRedInk/elm-ops-tooling-two.git', + 'source-directories': ['src'], + 'dependencies': {}, + })) + + vendor = tmpdir.mkdir('vendor') + + urlopen = mocker.patch.object(native_package_install, 'urlopen') + with open(str(fake_native_tarball_path)) as f: + urlopen.return_value = f + + native_package_install.main( + str(native_elm_package_path), + list(map(str, (elm_package_one_path, elm_package_two_path))), + str(vendor)) + + with open(str(fake_native_tarball_path)) as f: + urlopen.return_value = f + + native_package_install.main( + str(native_elm_package_path), + list(map(str, (elm_package_one_path, elm_package_two_path))), + str(vendor)) + + assert urlopen.call_count == 1 + + +def test_update_source_directories_makes_minimum_changes(tmpdir): + vendor_dir = tmpdir.mkdir('vendor') + + elm_package_one_path = tmpdir.join('elm-package-one.json') + elm_package_one_path.write(json.dumps({ + 'repository': 'https://github.com/NoRedInk/elm-ops-tooling.git', + 'source-directories': ['.'], + 'dependencies': {}, + }, indent=4, separators=(',', ': '))) + + elm_package_two_path = tmpdir.join('elm-package-two.json') + elm_package_two_path.write(json.dumps({ + 'repository': 'https://github.com/NoRedInk/elm-ops-tooling-two.git', + 'source-directories': ['src'], + 'dependencies': {}, + }, indent=4, separators=(',', ': '))) + + native_packages = [{'owner': 'elm-lang', 'project': 'core', 'version': '1.0.0'}] + vendor_package_dir = vendor_dir.mkdir('elm-lang').mkdir('core-1.0.0') + fake_elm_package = vendor_package_dir.join('elm-package.json') + fake_elm_package.write(json.dumps({ + 'source-directories': ['src', 'src2'], + })) + src_dir = vendor_package_dir.join('src').relto(tmpdir) + src2_dir = vendor_package_dir.join('src2').relto(tmpdir) + + prev_lines = elm_package_one_path.read().splitlines() + + # sanity check: we're operating on a clean JSON + for line in prev_lines: + assert not line.endswith(' ') + + repo = native_package_install.update_source_directories( + str(vendor_dir), + list(map(str, (elm_package_one_path, elm_package_two_path))), + native_packages, + ) + + assert repo == 'https://github.com/NoRedInk/elm-ops-tooling-two.git' + + new_elm_package_one = json.loads(elm_package_one_path.read()) + assert set(new_elm_package_one['source-directories']) == set(('.', str(src_dir), str(src2_dir))) + + new_elm_package_two = json.loads(elm_package_two_path.read()) + assert set(new_elm_package_two['source-directories']) == set(('src', str(src_dir), str(src2_dir))) + + for diff in difflib.ndiff(prev_lines, elm_package_one_path.read().splitlines()): + if diff.startswith(' '): + continue + + # assert there's only expected diffs + if diff.startswith('+ '): + # adds new src dir or trailing comma + assert ('"."' in diff) or (str(src_dir) in diff) or (str(src2_dir) in diff) + elif diff.startswith('- '): + # trailing comma for src dir + assert '"."' in diff + elif diff.startswith('? '): + pass + else: + assert False, 'unexpected diff operator in: ' + diff