diff --git a/python/Makefile b/python/Makefile index baefb4b..2823291 100644 --- a/python/Makefile +++ b/python/Makefile @@ -2,7 +2,8 @@ test: (. venv/bin/activate; \ - tox; \ + python3 -m pytest --cov=. --cov-report=term-missing:skip-covered ;\ + python3 -m flake8 --config setup.cfg ;\ ) venv: @@ -11,7 +12,6 @@ venv: venv/bin/pip install --upgrade setuptools . venv/bin/activate venv/bin/pip install -e . - venv/bin/pip install --upgrade tox publish: python setup.py sdist upload -r pypi diff --git a/python/examples/simple/example.py b/python/examples/simple/example.py index a614433..132e7eb 100755 --- a/python/examples/simple/example.py +++ b/python/examples/simple/example.py @@ -1,34 +1,29 @@ +#!/usr/bin/python from metaparticle_pkg import Containerize -import time - -package_repo = 'repo' -package_name = 'something' -sleep_time = 10 +import time +import logging -@Containerize( - package={'name': package_name, 'repository': package_repo}, - runtime={'ports': [80, 8080]} -) -def container_with_port(): - print('hello container_with_port') - - for i in range(sleep_time): - print('Sleeping ... {} sec'.format(i)) - time.sleep(1) +# all metaparticle output is accessible through the stdlib logger (debug level) +logging.basicConfig(level=logging.INFO) +logging.getLogger('metaparticle_pkg.runner').setLevel(logging.DEBUG) +logging.getLogger('metaparticle_pkg.builder').setLevel(logging.DEBUG) @Containerize( - package={'name': package_name, 'repository': package_repo, 'publish': True} + package={ + 'name': 'simple', + 'repository': 'docker.io/brendanburns', + 'publish': False + } ) -def hihi(): +def main(): print('hello world!') - for i in range(sleep_time): + for i in range(5): print('Sleeping ... {} sec'.format(i)) time.sleep(1) if __name__ == '__main__': - hihi() - container_with_port() + main() diff --git a/python/examples/simple/requirements.txt b/python/examples/simple/requirements.txt index 81fefc0..19bf20f 100644 --- a/python/examples/simple/requirements.txt +++ b/python/examples/simple/requirements.txt @@ -1,2 +1,2 @@ six==1.10.0 -metaparticle_pkg==0.4.1 +metaparticle_pkg diff --git a/python/examples/web/web.py b/python/examples/web/web.py index 13bac51..8a3cee9 100755 --- a/python/examples/web/web.py +++ b/python/examples/web/web.py @@ -1,11 +1,17 @@ #!/usr/bin/python +from metaparticle_pkg import Containerize -from six.moves import SimpleHTTPServer, socketserver +import logging import socket -from metaparticle_pkg.containerize import Containerize +from six.moves import SimpleHTTPServer, socketserver -OK = 200 +# all metaparticle output is accessible through the stdlib logger (debug level) +logging.basicConfig(level=logging.INFO) +logging.getLogger('metaparticle_pkg.runner').setLevel(logging.DEBUG) +logging.getLogger('metaparticle_pkg.builder').setLevel(logging.DEBUG) + +OK = 200 port = 8080 @@ -24,8 +30,17 @@ def do_HEAD(self): @Containerize( - package={'name': 'web', 'repository': 'docker.io/brendanburns'}, - runtime={'ports': [8080], 'executor': 'metaparticle', 'replicas': 3, 'public': True} + package={ + # to run this example you'll need to change these values + 'name': 'web', + 'repository': 'docker.io/brendanburns', + }, + runtime={ + 'ports': [8080], + 'executor': 'metaparticle', + 'replicas': 3, + 'public': True + } ) def main(): Handler = MyHandler diff --git a/python/metaparticle_pkg/builder/docker_builder.py b/python/metaparticle_pkg/builder/docker_builder.py index 2d29d5f..ec7451b 100644 --- a/python/metaparticle_pkg/builder/docker_builder.py +++ b/python/metaparticle_pkg/builder/docker_builder.py @@ -3,7 +3,8 @@ from docker import APIClient -logger = logging.getLogger(__name__) +# use a generic logger name: metaparticle_pkg.builder +logger = logging.getLogger('.'.join(__name__.split('.')[:-1])) class DockerBuilder: diff --git a/python/metaparticle_pkg/containerize.py b/python/metaparticle_pkg/containerize.py index 00c556a..654b9d5 100644 --- a/python/metaparticle_pkg/containerize.py +++ b/python/metaparticle_pkg/containerize.py @@ -41,7 +41,7 @@ def write_dockerfile(package, exec_file): COPY ./ /app/ RUN pip install --no-cache -r /app/requirements.txt -CMD python /app/{exec_file} +CMD python -u /app/{exec_file} """.format(version=package.py_version, exec_file=exec_file)) diff --git a/python/metaparticle_pkg/option.py b/python/metaparticle_pkg/option.py index 8a66ffb..bd12e94 100644 --- a/python/metaparticle_pkg/option.py +++ b/python/metaparticle_pkg/option.py @@ -5,7 +5,7 @@ def load(cls, options): if not isinstance(options, dict): - sys.stderr.write("Must provide an options dict.") + sys.stderr.write("Must provide an options dictionary. Given option: %s" % repr(options)) sys.exit(1) for option in cls.required_options: if option not in options: @@ -22,6 +22,10 @@ class RuntimeOptions(namedtuple('Runtime', 'executor replicas ports public shard required_options = [] def __new__(cls, executor='docker', replicas=0, ports=[], public=False, shardSpec=None, jobSpec=None): + if shardSpec: + shardSpec = load(ShardSpec, shardSpec) + if jobSpec: + jobSpec = load(JobSpec, jobSpec) return super(RuntimeOptions, cls).__new__(cls, executor, replicas, ports, public, shardSpec, jobSpec) @@ -39,9 +43,9 @@ def __new__(cls, iterations=0): return super(JobSpec, cls).__new__(cls, iterations) -class PackageOptions(namedtuple('Package', 'repository name builder publish verbose quiet py_version')): +class PackageOptions(namedtuple('Package', 'repository name builder publish py_version')): required_options = ['repository'] - def __new__(cls, repository, name, builder='docker', publish=False, verbose=True, quiet=False, py_version=3, dockerfile=None): + def __new__(cls, repository, name, builder='docker', publish=False, py_version=3, dockerfile=None): name = name if name else os.path.basename(os.getcwd()) - return super(PackageOptions, cls).__new__(cls, repository, name, builder, publish, verbose, quiet, py_version) + return super(PackageOptions, cls).__new__(cls, repository, name, builder, publish, py_version) diff --git a/python/metaparticle_pkg/runner/docker_runner.py b/python/metaparticle_pkg/runner/docker_runner.py index 42a6064..51e9c19 100644 --- a/python/metaparticle_pkg/runner/docker_runner.py +++ b/python/metaparticle_pkg/runner/docker_runner.py @@ -2,7 +2,8 @@ from docker import APIClient -logger = logging.getLogger(__name__) +# use a generic logger name: metaparticle_pkg.runner +logger = logging.getLogger('.'.join(__name__.split('.')[:-1])) class DockerRunner: @@ -28,22 +29,20 @@ def run(self, img, name, options): # Launch docker container container = self.docker_client.create_container( img, - name=name, - ports=ports, host_config=host_config, + name=name, + ports=ports ) - self.docker_client.start(container=container.get('Id')) - - self.container = container logger.info('Starting container {}'.format(container)) + self.docker_client.start(container=container.get('Id')) + self.container = container + def logs(self, *args, **kwargs): if self.docker_client is None: self.docker_client = APIClient(version='auto') - # seems like we are hitting bug - # https://github.com/docker/docker-py/issues/300 log_stream = self.docker_client.logs( self.container.get('Id'), stream=True, @@ -51,7 +50,7 @@ def logs(self, *args, **kwargs): ) for line in log_stream: - logger.info(line) + logger.info(line.decode("utf-8").strip('\n')) def cancel(self, name): if self.docker_client is None: diff --git a/python/metaparticle_pkg/version.json b/python/metaparticle_pkg/version.json index 831fc0e..3d61c51 100644 --- a/python/metaparticle_pkg/version.json +++ b/python/metaparticle_pkg/version.json @@ -1,5 +1,5 @@ { "version": "0.6.4", "license": "MIT", -"status": "Development", +"status": "Development" } diff --git a/python/setup.cfg b/python/setup.cfg index bd2bcf6..5e3a93d 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -3,3 +3,6 @@ description-file = README.md [flake8] max-line-length = 160 +exclude = + ./.git, + ./venv \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index f444fa0..5483aac 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,7 +1,8 @@ import setuptools import json -config = json.loads('./metaparticle_pkg/version.json') +with open('./metaparticle_pkg/version.json', 'r') as json_file: + config = json.load(json_file) setuptools.setup( name='metaparticle_pkg', diff --git a/python/tox.ini b/python/tox.ini deleted file mode 100644 index 8cf2edd..0000000 --- a/python/tox.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -envlist = py2 - py3 - -[testenv] -commands = coverage erase - coverage run --source metaparticle -m py.test - coverage report -deps= -r{toxinidir}/requirements.txt - pytest - coverage - mock - pytest-mock