refs #32 : add - python3-theano kernel definition.

python3-theano/Dockerfile

@@ -0,0 +1,90 @@
+FROM ubuntu:16.04
+MAINTAINER Jeongkyu Shin "[email protected]"
+
+# Add an isolated user
+# /home/work: actual working directory for user codes
+# /home/sorna: place for Python and REPL script
+RUN adduser --disabled-password --gecos "" work
+RUN chmod 700 /home/work
+RUN mkdir /home/sorna
+RUN chmod 755 /home/sorna
+RUN chown -R work:work /home/sorna
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV HOME /home/work
+WORKDIR /home/work
+
+# Install Python
+USER root
+ENV SORNA_PYTHON_VERSION 3.6.0
+RUN sed -i 's/archive\.ubuntu\.com/kr.archive.ubuntu.com/' /etc/apt/sources.list
+RUN apt-get update && apt-get build-dep -y python3
+RUN apt-get install -y git-core wget libreadline-dev libsqlite3-dev libssl-dev libbz2-dev libzmq3-dev tk-dev
+ADD install-python.sh /home/sorna/install-python.sh
+ADD pyenv-run.sh /home/sorna/pyenv-run.sh
+RUN chmod +x /home/sorna/*.sh
+USER work
+ENV PYENV_ROOT /home/sorna/.pyenv
+ENV PATH /home/sorna/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+RUN git clone https://github.com/yyuu/pyenv /home/sorna/.pyenv
+RUN /home/sorna/install-python.sh
+
+# Install common Python packages
+USER root
+RUN apt-get install -y pkg-config
+RUN apt-get install -y libjpeg-dev libpng-dev
+RUN apt-get install -y libfreetype6-dev libblas-dev liblapack-dev libatlas-dev gfortran
+USER work
+# matplotlib may install older version of dateutil; we first install it.
+RUN /home/sorna/pyenv-run.sh pip install pyzmq namedlist six python-dateutil nose
+RUN /home/sorna/pyenv-run.sh pip install numpy scipy matplotlib bokeh
+RUN /home/sorna/pyenv-run.sh pip install scikit-learn scikit-image
+RUN /home/sorna/pyenv-run.sh pip install cvxpy seaborn
+RUN /home/sorna/pyenv-run.sh pip install pandas networkx
+RUN /home/sorna/pyenv-run.sh pip install pillow sklearn
+
+# Additional packages
+USER root
+ENV MPLCONFIGDIR /home/sorna/.matplotlib
+RUN mkdir /home/sorna/.matplotlib
+RUN chown -R work:work /home/sorna/.matplotlib
+USER work
+RUN echo 'import matplotlib.pyplot' > /tmp/matplotlib-fontcache.py
+RUN /home/sorna/pyenv-run.sh python /tmp/matplotlib-fontcache.py
+RUN rm /tmp/matplotlib-fontcache.py
+
+# Install Sorna Media support
+USER work
+ADD matplotlibrc /home/sorna/.matplotlib/
+ADD sorna_media-*.whl /tmp
+RUN /home/sorna/pyenv-run.sh pip install /tmp/sorna_media-*.whl
+
+# Install Theano
+USER work
+RUN /home/sorna/pyenv-run.sh pip install Theano
+
+# Secure installation scripts
+USER root
+RUN apt-get install -y libseccomp2
+ADD run.py /home/sorna/run.py
+ADD run.sh /home/sorna/run.sh
+# NOTE: you must copy $GOPATH/bin to <dockerfile_dir>/
+ADD jail /home/sorna/jail
+ADD intra-jail /home/sorna/intra-jail
+RUN chown root:root /home/sorna/*.sh /home/sorna/jail /home/sorna/intra-jail
+RUN chmod 755 /home/sorna/run.sh /home/sorna/jail /home/sorna/intra-jail
+ADD patch-libs.so /home/sorna/patch-libs.so
+ENV LD_PRELOAD /home/sorna/patch-libs.so
+
+VOLUME ["/home/work"]
+EXPOSE 2001
+
+LABEL io.sorna.timeout="30"
+LABEL io.sorna.maxmem="512m"
+LABEL io.sorna.maxcores="4"
+LABEL io.sorna.mode="query"
+LABEL io.sorna.envs.corecount="OPENBLAS_NUM_THREADS"
+
+USER work
+ENV PYTHONUNBUFFERED 1
+CMD /home/sorna/run.sh

python3-theano/install-python.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+eval "$(pyenv init -)"
+pyenv install $SORNA_PYTHON_VERSION
+pyenv shell $SORNA_PYTHON_VERSION
+pyenv rehash
+pip3 install --upgrade -q pip
+pip3 install wheel

python3-theano/matplotlibrc

@@ -0,0 +1,5 @@
+backend : module://sorna.matplotlib.backend_sorna
+figure.facecolor : FFFFFF
+figure.edgecolor : FFFFFF
+savefig.facecolor : FFFFFF
+savefig.edgecolor : FFFFFF

python3-theano/pyenv-run.sh

@@ -0,0 +1,4 @@
+#! /bin/bash
+eval "$(pyenv init -)"
+pyenv shell $SORNA_PYTHON_VERSION
+exec "$@"

python3-theano/run.py

@@ -0,0 +1,164 @@
+#! /usr/bin/env python3
+
+import builtins as builtin_mod
+import code
+import io
+from namedlist import namedtuple, namedlist
+import os
+from os import path
+import sys
+import types
+import zmq
+try:
+    import simplejson
+    has_simplejson = True
+except ImportError:
+    has_simplejson = False
+
+import sorna.drawing
+
+ExceptionInfo = namedtuple('ExceptionInfo', [
+    'exc',
+    ('args', tuple()),
+    ('raised_before_exec', False),
+    ('traceback', None),
+])
+
+Result = namedlist('Result', [
+    ('stdout', ''),
+    ('stderr', ''),
+    ('media', None),
+])
+
+
+@staticmethod
+def _create_excinfo(e, raised_before_exec, tb):
+    assert isinstance(e, Exception)
+    return ExceptionInfo(type(e).__name__, e.args, raised_before_exec, tb)
+ExceptionInfo.create = _create_excinfo
+
+
+class SockWriter(object):
+    def __init__(self, sock, cell_id):
+        self.cell_id_encoded = '{0}'.format(cell_id).encode('ascii')
+        self.sock = sock
+        self.buffer = io.StringIO()
+
+    def write(self, s):
+        if '\n' in s:  # flush on occurrence of a newline.
+            s1, s2 = s.split('\n', maxsplit=1)
+            s0 = self.buffer.getvalue()
+            self.sock.send_multipart([self.cell_id_encoded, (s0 + s1 + '\n').encode('utf8')])
+            self.buffer.seek(0)
+            self.buffer.truncate(0)
+            self.buffer.write(s2)
+        else:
+            self.buffer.write(s)
+        if self.buffer.tell() > 1024:  # flush if the buffer is too large.
+            s0 = self.buffer.getvalue()
+            self.sock.send_multipart([self.cell_id_encoded, s0.encode('utf8')])
+            self.buffer.seek(0)
+            self.buffer.truncate(0)
+        # TODO: timeout to flush?
+
+
+class CodeRunner(object):
+    '''
+    A thin wrapper for REPL.
+
+    It creates a dummy module that user codes run and keeps the references to user-created objects
+    (e.g., variables and functions).
+    '''
+
+    def __init__(self):
+        self.stdout_buffer = io.StringIO()
+        self.stderr_buffer = io.StringIO()
+
+        # Initialize user module and namespaces.
+        user_module = types.ModuleType('__main__',
+                                       doc='Automatically created module for the interactive shell.')
+        user_module.__dict__.setdefault('__builtin__', builtin_mod)
+        user_module.__dict__.setdefault('__builtins__', builtin_mod)
+        self.user_module = user_module
+        self.user_ns = user_module.__dict__
+
+    def execute(self, cell_id, src):
+        self.stdout_writer = self.stdout_buffer
+        self.stderr_writer = self.stderr_buffer
+        sys.stdout, orig_stdout = self.stdout_writer, sys.stdout
+        sys.stderr, orig_stderr = self.stderr_writer, sys.stderr
+
+        exceptions = []
+        result = Result()
+        before_exec = True
+
+        def my_excepthook(type_, value, tb):
+            exceptions.append(ExceptionInfo.create(value, before_exec, tb))
+        sys.excepthook = my_excepthook
+
+        try:
+            code_obj = code.compile_command(src, symbol='exec')
+        except IndentationError as e:
+            exceptions.append(ExceptionInfo.create(e, before_exec, None))
+        except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError) as e:
+            exceptions.append(ExceptionInfo.create(e, before_exec, None))
+        else:
+            self.user_module.__builtins__._sorna_media = []
+            before_exec = False
+            try:
+                exec(code_obj, self.user_ns)
+            except Exception as e:
+                exceptions.append(ExceptionInfo.create(e, before_exec, None))
+
+        sys.excepthook = sys.__excepthook__
+
+        result.stdout = self.stdout_writer.getvalue()
+        result.stderr = self.stderr_writer.getvalue()
+        # TODO: sanitize media?
+        result.media = self.user_module.__builtins__._sorna_media
+        self.stdout_writer.seek(0, io.SEEK_SET)
+        self.stdout_writer.truncate(0)
+        self.stderr_writer.seek(0, io.SEEK_SET)
+        self.stderr_writer.truncate(0)
+
+        sys.stdout = orig_stdout
+        sys.stderr = orig_stderr
+        return exceptions, result
+
+
+if __name__ == '__main__':
+    # Use the "confined" working directory
+    os.chdir('/home/work')
+    # Replace stdin with a "null" file
+    # (trying to read stdin will raise EOFError immediately afterwards.)
+    sys.stdin = open(os.devnull, 'rb')
+
+    # Initialize context object.
+    runner = CodeRunner()
+
+    # Initialize minimal ZMQ server socket.
+    ctx = zmq.Context(io_threads=1)
+    sock = ctx.socket(zmq.REP)
+    sock.bind('tcp://*:2001')
+    print('serving at port 2001...')
+
+    try:
+        while True:
+            data = sock.recv_multipart()
+            exceptions, result = runner.execute(data[0].decode('ascii'),
+                                                data[1].decode('utf8'))
+            response = {
+                'stdout': result.stdout,
+                'stderr': result.stderr,
+                'media': result.media,
+                'exceptions': exceptions,
+            }
+            json_opts = {}
+            if has_simplejson:
+                json_opts['namedtuple_as_object'] = False
+            sock.send_json(response, **json_opts)
+    except (KeyboardInterrupt, SystemExit):
+        pass
+    finally:
+        sock.close()
+        print('exit.')

python3-theano/run.sh

@@ -0,0 +1,4 @@
+#! /bin/bash
+eval "$(pyenv init -)"
+pyenv shell $SORNA_PYTHON_VERSION
+exec /home/sorna/jail python-tensorflow `pyenv which python` /home/sorna/run.py