diff --git a/python/.gitignore b/python/.gitignore index 4ab7448..c9e0ea5 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,3 +1,5 @@ +.pytest_cache/ + # This repo is generating etherial dockerfiles Dockerfile diff --git a/python/Makefile b/python/Makefile index 2823291..ea3bae7 100644 --- a/python/Makefile +++ b/python/Makefile @@ -11,6 +11,7 @@ venv: venv/bin/pip install --upgrade pip venv/bin/pip install --upgrade setuptools . venv/bin/activate + venv/bin/pip install -r requirements.txt venv/bin/pip install -e . publish: diff --git a/python/examples/include/data.json b/python/examples/include/data.json new file mode 100644 index 0000000..7d2eb52 --- /dev/null +++ b/python/examples/include/data.json @@ -0,0 +1,4 @@ +{ + "important": "payload", + "goes": "here" +} \ No newline at end of file diff --git a/python/examples/include/example.py b/python/examples/include/example.py new file mode 100755 index 0000000..361160d --- /dev/null +++ b/python/examples/include/example.py @@ -0,0 +1,37 @@ +#!/usr/bin/python +from metaparticle_pkg import Containerize, PackageFile + +import os +import time +import logging + +# all metaparticle output is accessible through the stdlib logger (debug level) +logging.basicConfig(level=logging.INFO) +logging.getLogger('metaparticle_pkg.runner').setLevel(logging.DEBUG) +logging.getLogger('metaparticle_pkg.builder').setLevel(logging.DEBUG) + + +DATA_FILE = '/opt/some/random/spot/data1.json' +SCRIPT = '/opt/another/random/place/get_the_data.sh' + + +@Containerize( + package={ + 'name': 'file-example', + 'repository': 'docker.io/brendanburns', + 'publish': False, + 'additionalFiles': [ + PackageFile(src='./data.json', dest=DATA_FILE, mode='0400'), + PackageFile(src='./get_data.sh', dest=SCRIPT), + ] + } +) +def main(): + os.system(SCRIPT) + for i in range(5): + print('Sleeping ... {} sec'.format(i)) + time.sleep(1) + + +if __name__ == '__main__': + main() diff --git a/python/examples/include/get_data.sh b/python/examples/include/get_data.sh new file mode 100644 index 0000000..e7d991d --- /dev/null +++ b/python/examples/include/get_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "Hello, world!" +ls -al /opt/some/random/spot/data1.json +cat /opt/some/random/spot/data1.json \ No newline at end of file diff --git a/python/examples/include/requirements.txt b/python/examples/include/requirements.txt new file mode 100644 index 0000000..eb3b229 --- /dev/null +++ b/python/examples/include/requirements.txt @@ -0,0 +1,2 @@ +six==1.10.0 +metaparticle_pkg \ No newline at end of file diff --git a/python/metaparticle_pkg/__init__.py b/python/metaparticle_pkg/__init__.py index d6e6f52..26dc652 100644 --- a/python/metaparticle_pkg/__init__.py +++ b/python/metaparticle_pkg/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import -from metaparticle_pkg.containerize import Containerize +from metaparticle_pkg.containerize import Containerize, PackageFile -__all__ = [Containerize] +__all__ = ['Containerize', 'PackageFile'] diff --git a/python/metaparticle_pkg/containerize.py b/python/metaparticle_pkg/containerize.py index 972869f..2bc4e20 100644 --- a/python/metaparticle_pkg/containerize.py +++ b/python/metaparticle_pkg/containerize.py @@ -36,14 +36,17 @@ def write_dockerfile(package, exec_file): shutil.copy(package.dockerfile, 'Dockerfile') return + copy_files = "\n".join([addFile.render() for addFile in package.additionalFiles]) + with open('Dockerfile', 'w+t') as f: f.write("""FROM python:{version}-alpine - COPY ./ /app/ +{copy_files} RUN pip install --no-cache -r /app/requirements.txt - CMD python -u /app/{exec_file} -""".format(version=package.py_version, exec_file=exec_file)) +""".format(version=package.py_version, + exec_file=exec_file, + copy_files=copy_files)) class Containerize(object): @@ -84,3 +87,17 @@ def signal_handler(signal, frame): return self.runner.logs(self.package.name) return wrapped + + +class PackageFile(object): + + def __init__(self, src, dest, mode=None): + self.src = src + self.dest = dest + self.mode = mode + + def render(self): + ret = "COPY {src} {dest}".format(src=self.src, dest=self.dest) + if self.mode: + ret += "\nRUN chmod -R {mode} {dest}".format(mode=self.mode, dest=self.dest) + return ret diff --git a/python/metaparticle_pkg/option.py b/python/metaparticle_pkg/option.py index b0c459c..b4f4d7f 100644 --- a/python/metaparticle_pkg/option.py +++ b/python/metaparticle_pkg/option.py @@ -44,9 +44,9 @@ def __new__(cls, iterations=0): return super(JobSpec, cls).__new__(cls, iterations) -class PackageOptions(namedtuple('Package', 'repository name builder publish py_version')): +class PackageOptions(namedtuple('Package', 'repository name builder publish py_version additionalFiles')): required_options = ['repository'] - def __new__(cls, repository, name, builder='docker', publish=False, py_version=3, dockerfile=None): + def __new__(cls, repository, name, builder='docker', publish=False, py_version=3, dockerfile=None, additionalFiles=tuple()): name = name if name else os.path.basename(os.getcwd()) - return super(PackageOptions, cls).__new__(cls, repository, name, builder, publish, py_version) + return super(PackageOptions, cls).__new__(cls, repository, name, builder, publish, py_version, additionalFiles) diff --git a/python/metaparticle_pkg/test/test_containerize.py b/python/metaparticle_pkg/test/test_containerize.py index 3f72b00..8515c81 100644 --- a/python/metaparticle_pkg/test/test_containerize.py +++ b/python/metaparticle_pkg/test/test_containerize.py @@ -142,12 +142,47 @@ def test_write_dockerfile_with_dockerfile_absent(self): # Input parameters package = MagicMock() package.dockerfile = None + package.py_version = "3.7" exec_file = '/some/fake_exec_file_path' with patch("metaparticle_pkg.containerize.open", mocked_open_function) as mocked_open: containerize.write_dockerfile(package, exec_file) - self.assertEqual(mocked_open().write.call_count, 1) + mocked_open().write.assert_called_once_with("""\ +FROM python:3.7-alpine +COPY ./ /app/ + +RUN pip install --no-cache -r /app/requirements.txt +CMD python -u /app//some/fake_exec_file_path +""") + + def test_write_dockerfile_with_additional_files(self): + '''Test write_dockerfile method in case of Dockerfile is absent''' + + mocked_open_function = mock_open() + + # Input parameters + package = MagicMock() + package.dockerfile = None + package.py_version = "3.7" + package.additionalFiles = [ + containerize.PackageFile(src='/from/somewhere', dest='/to/somewhere'), + containerize.PackageFile(src='/from/somewhere/else', dest='/to/somewhere/else', mode=754) + ] + exec_file = '/some/fake_exec_file_path' + + with patch("metaparticle_pkg.containerize.open", + mocked_open_function) as mocked_open: + containerize.write_dockerfile(package, exec_file) + mocked_open().write.assert_called_once_with("""\ +FROM python:3.7-alpine +COPY ./ /app/ +COPY /from/somewhere /to/somewhere +COPY /from/somewhere/else /to/somewhere/else +RUN chmod -R 754 /to/somewhere/else +RUN pip install --no-cache -r /app/requirements.txt +CMD python -u /app//some/fake_exec_file_path +""") @patch("metaparticle_pkg.containerize.os") @patch("metaparticle_pkg.containerize.builder") @@ -199,6 +234,22 @@ def test_func(): pass self.assertEqual(mocked_runner.select().logs.call_count, 1) self.assertEqual(mocked_builder.select().build.call_count, 1) + def test_packagefile_render(self): + '''Test packagefile.render method go case''' + + package_file = containerize.PackageFile(src='thesrc', dest='thedst') + actual_value = package_file.render() + + self.assertEqual(actual_value, "COPY thesrc thedst") + + def test_packagefile_render_mode(self): + '''Test packagefile.render method with a mode''' + + package_file = containerize.PackageFile(src='thesrc', dest='thedst', mode=444) + actual_value = package_file.render() + + self.assertEqual(actual_value, "COPY thesrc thedst\nRUN chmod -R 444 thedst") + if __name__ == '__main__': unittest.main()