Python fix + cleanup

python/Makefile

@@ -2,7 +2,8 @@
 
 test:
 	(. venv/bin/activate; \
-	tox; \
+	python3 -m pytest --cov=. --cov-report=term-missing:skip-covered ;\
+	python3 -m flake8 --config setup.cfg ;\
 	)
 
 venv: 
@@ -11,7 +12,6 @@ venv:
 	venv/bin/pip install --upgrade setuptools
 	. venv/bin/activate
 	venv/bin/pip install -e .
-	venv/bin/pip install --upgrade tox
 
 publish: 
 	python setup.py sdist upload -r pypi

python/examples/simple/example.py

@@ -1,34 +1,29 @@
+#!/usr/bin/python
 from metaparticle_pkg import Containerize
-import time
-
-package_repo = 'repo'
-package_name = 'something'
-sleep_time = 10
 
+import time
+import logging
 
-@Containerize(
-    package={'name': package_name, 'repository': package_repo},
-    runtime={'ports': [80, 8080]}
-)
-def container_with_port():
-    print('hello container_with_port')
-
-    for i in range(sleep_time):
-        print('Sleeping ... {} sec'.format(i))
-        time.sleep(1)
+# all metaparticle output is accessible through the stdlib logger (debug level)
+logging.basicConfig(level=logging.INFO)
+logging.getLogger('metaparticle_pkg.runner').setLevel(logging.DEBUG)
+logging.getLogger('metaparticle_pkg.builder').setLevel(logging.DEBUG)
 
 
 @Containerize(
-    package={'name': package_name, 'repository': package_repo, 'publish': True}
+    package={
+        'name': 'simple',
+        'repository': 'docker.io/brendanburns',
+        'publish': False
+    }
 )
-def hihi():
+def main():
     print('hello world!')
 
-    for i in range(sleep_time):
+    for i in range(5):
         print('Sleeping ... {} sec'.format(i))
         time.sleep(1)
 
 
 if __name__ == '__main__':
-    hihi()
-    container_with_port()
+    main()

python/examples/simple/requirements.txt

@@ -1,2 +1,2 @@
 six==1.10.0
-metaparticle_pkg==0.4.1
+metaparticle_pkg

python/examples/web/web.py

@@ -1,11 +1,17 @@
 #!/usr/bin/python
+from metaparticle_pkg import Containerize
 
-from six.moves import SimpleHTTPServer, socketserver
+import logging
 import socket
-from metaparticle_pkg.containerize import Containerize
+from six.moves import SimpleHTTPServer, socketserver
 
-OK = 200
+# all metaparticle output is accessible through the stdlib logger (debug level)
+logging.basicConfig(level=logging.INFO)
+logging.getLogger('metaparticle_pkg.runner').setLevel(logging.DEBUG)
+logging.getLogger('metaparticle_pkg.builder').setLevel(logging.DEBUG)
 
+
+OK = 200
 port = 8080
 
 
@@ -24,8 +30,17 @@ def do_HEAD(self):
 
 
 @Containerize(
-    package={'name': 'web', 'repository': 'docker.io/brendanburns'},
-    runtime={'ports': [8080], 'executor': 'metaparticle', 'replicas': 3, 'public': True}
+    package={
+        # to run this example you'll need to change these values
+        'name': 'web',
+        'repository': 'docker.io/brendanburns',
+    },
+    runtime={
+        'ports': [8080],
+        'executor': 'metaparticle',
+        'replicas': 3,
+        'public': True
+    }
 )
 def main():
     Handler = MyHandler

python/metaparticle_pkg/builder/docker_builder.py

@@ -3,7 +3,8 @@
 
 from docker import APIClient
 
-logger = logging.getLogger(__name__)
+# use a generic logger name: metaparticle_pkg.builder
+logger = logging.getLogger('.'.join(__name__.split('.')[:-1]))
 
 
 class DockerBuilder:

python/metaparticle_pkg/containerize.py

@@ -41,7 +41,7 @@ def write_dockerfile(package, exec_file):
 COPY ./ /app/
 RUN pip install --no-cache -r /app/requirements.txt
 
-CMD python /app/{exec_file}
+CMD python -u /app/{exec_file}
 """.format(version=package.py_version, exec_file=exec_file))
 
 

python/metaparticle_pkg/option.py

@@ -5,7 +5,7 @@
 
 def load(cls, options):
     if not isinstance(options, dict):
-        sys.stderr.write("Must provide an options dict.")
+        sys.stderr.write("Must provide an options dictionary. Given option: %s" % repr(options))
         sys.exit(1)
     for option in cls.required_options:
         if option not in options:
@@ -22,6 +22,10 @@ class RuntimeOptions(namedtuple('Runtime', 'executor replicas ports public shard
     required_options = []
 
     def __new__(cls, executor='docker', replicas=0, ports=[], public=False, shardSpec=None, jobSpec=None):
+        if shardSpec:
+            shardSpec = load(ShardSpec, shardSpec)
+        if jobSpec:
+            jobSpec = load(JobSpec, jobSpec)
         return super(RuntimeOptions, cls).__new__(cls, executor, replicas, ports, public, shardSpec, jobSpec)
 
 
@@ -39,9 +43,9 @@ def __new__(cls, iterations=0):
         return super(JobSpec, cls).__new__(cls, iterations)
 
 
-class PackageOptions(namedtuple('Package', 'repository name builder publish verbose quiet py_version')):
+class PackageOptions(namedtuple('Package', 'repository name builder publish py_version')):
     required_options = ['repository']
 
-    def __new__(cls, repository, name, builder='docker', publish=False, verbose=True, quiet=False, py_version=3, dockerfile=None):
+    def __new__(cls, repository, name, builder='docker', publish=False, py_version=3, dockerfile=None):
         name = name if name else os.path.basename(os.getcwd())
-        return super(PackageOptions, cls).__new__(cls, repository, name, builder, publish, verbose, quiet, py_version)
+        return super(PackageOptions, cls).__new__(cls, repository, name, builder, publish, py_version)

python/metaparticle_pkg/runner/docker_runner.py

@@ -2,7 +2,8 @@
 
 from docker import APIClient
 
-logger = logging.getLogger(__name__)
+# use a generic logger name: metaparticle_pkg.runner
+logger = logging.getLogger('.'.join(__name__.split('.')[:-1]))
 
 
 class DockerRunner:
@@ -28,30 +29,28 @@ def run(self, img, name, options):
         # Launch docker container
         container = self.docker_client.create_container(
             img,
-            name=name,
-            ports=ports,
             host_config=host_config,
+            name=name,
+            ports=ports
         )
-        self.docker_client.start(container=container.get('Id'))
-
-        self.container = container
 
         logger.info('Starting container {}'.format(container))
 
+        self.docker_client.start(container=container.get('Id'))
+        self.container = container
+
     def logs(self, *args, **kwargs):
         if self.docker_client is None:
             self.docker_client = APIClient(version='auto')
 
-        # seems like we are hitting bug
-        # https://github.com/docker/docker-py/issues/300
         log_stream = self.docker_client.logs(
             self.container.get('Id'),
             stream=True,
             follow=True
         )
 
         for line in log_stream:
-            logger.info(line)
+            logger.info(line.decode("utf-8").strip('\n'))
 
     def cancel(self, name):
         if self.docker_client is None:

python/metaparticle_pkg/version.json

@@ -1,5 +1,5 @@
 {
 "version": "0.6.4",
 "license": "MIT",
-"status": "Development",
+"status": "Development"
 }

python/setup.cfg

@@ -3,3 +3,6 @@ description-file = README.md
 
 [flake8]
 max-line-length = 160
+exclude = 
+    ./.git,
+    ./venv

python/setup.py

@@ -1,7 +1,8 @@
 import setuptools
 import json
 
-config = json.loads('./metaparticle_pkg/version.json')
+with open('./metaparticle_pkg/version.json', 'r') as json_file:
+    config = json.load(json_file)
 
 setuptools.setup(
     name='metaparticle_pkg',