refs #34, #1, #44, #45, #46: Update base runner and all descendant im…

…ages

 * Base runner now offers the auto-completion handler stub.

 * "python3" kernel now uses the base runner and thus support the batch
   mode with the default build command: "python setup.py develop"
   (skipped with a warning when setup.py is not uploaded together)

 * C/C++ kernels now have the make utility. Ooops.

base-python3-minimal/base_run.py

@@ -12,6 +12,7 @@
 import aiozmq
 import uvloop
 import zmq
+import simplejson as json
 
 log = logging.getLogger()
 
@@ -55,6 +56,7 @@ def __init__(self):
         self.child_env = {}
         self.insock = None
         self.outsock = None
+        self.loop = None
 
     @abstractmethod
     async def build(self, build_cmd):
@@ -68,6 +70,10 @@ async def execute(self, exec_cmd):
     async def query(self, code_text):
         """Run user code by creating a temporary file and compiling it."""
 
+    @abstractmethod
+    async def complete(self, completion_data):
+        """Return the list of strings to be shown in the auto-complete list."""
+
     async def run_subproc(self, cmd):
         """A thin wrapper for an external command."""
         loop = asyncio.get_event_loop()
@@ -132,6 +138,11 @@ async def main_loop(self):
                 elif op_type == 'input':  # query-mode
                     await self.query(text)
                     self.outsock.write([b'finished', b''])
+                elif op_type == 'complete':  # auto-completion
+                    completion_data = json.loads(text)
+                    completions = await self.complete(completion_data)
+                    self.outsock.write([b'completion', json.dumps(completions)])
+                    self.outsock.write([b'finished', b''])
                 await self.outsock.drain()
             except asyncio.CancelledError:
                 break
@@ -151,6 +162,7 @@ def run(self):
         # Initialize event loop.
         asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
         loop = asyncio.get_event_loop()
+        self.loop = loop
         stopped = asyncio.Event()
 
         def interrupt(loop, stopped):

c/Dockerfile

@@ -2,7 +2,7 @@ FROM lablup/kernel-base-python3-minimal:latest
 MAINTAINER DevOps "[email protected]"
 
 # Install minimal C compile environments
-RUN apk add --no-cache gcc musl-dev
+RUN apk add --no-cache gcc musl-dev make
 
 COPY run.py /home/sorna/run.py
 COPY policy.yml /home/sorna/policy.yml

c/run.py

@@ -87,6 +87,9 @@ async def query(self, code_text):
                    f'./main')
             await self.run_subproc(cmd)
 
+    async def complete(self, data):
+        return []
+
 
 if __name__ == '__main__':
     CProgramRunner().run()

cpp/Dockerfile

@@ -2,7 +2,7 @@ FROM lablup/kernel-base-python3-minimal:latest
 MAINTAINER DevOps "[email protected]"
 
 # Install minimal C++ compile environments
-RUN apk add --no-cache g++ libstdc++
+RUN apk add --no-cache g++ libstdc++ make
 
 COPY run.py /home/sorna/run.py
 COPY policy.yml /home/sorna/policy.yml

cpp/run.py

@@ -85,6 +85,9 @@ async def query(self, code_text):
                    f'&& ./main')
             await self.run_subproc(cmd)
 
+    async def complete(self, data):
+        return []
+
 
 if __name__ == '__main__':
     CPPProgramRunner().run()

java8/run.py

@@ -102,6 +102,9 @@ async def query(self, code_text):
             if filename:
                 os.remove(filename)
 
+    async def complete(self, data):
+        return []
+
 
 def main():
     JavaProgramRunner().run()

python3/Dockerfile

@@ -49,6 +49,7 @@ RUN echo 'import matplotlib.pyplot' > /tmp/matplotlib-fontcache.py \
 
 COPY policy.yml /home/sorna/
 COPY run.py /home/sorna/
+COPY inproc_run.py /home/sorna/
 
 LABEL io.sorna.envs.corecount="OPENBLAS_NUM_THREADS,NPROC"
 LABEL io.sorna.features "batch query uid-match user-input"

python3/inproc_run.py

@@ -0,0 +1,172 @@
+import builtins as builtin_mod
+import code
+import enum
+from functools import partial
+import logging
+import os
+import sys
+import time
+import traceback
+import types
+
+import simplejson as json
+from IPython.core.completer import Completer
+
+import getpass
+
+from sorna.types import (
+    InputRequest, ControlRecord, ConsoleRecord, MediaRecord, HTMLRecord, CompletionRecord,
+)
+
+
+log = logging.getLogger()
+
+class StreamToEmitter:
+
+    def __init__(self, emitter, stream_type):
+        self.emit = emitter
+        self.stream_type = stream_type
+
+    def write(self, s):
+        self.emit(ConsoleRecord(self.stream_type, s))
+
+    def flush(self):
+        pass
+
+
+class PythonInprocRunner:
+    '''
+    A thin wrapper for REPL.
+
+    It creates a dummy module that user codes run and keeps the references to user-created objects
+    (e.g., variables and functions).
+    '''
+
+    def __init__(self, runner):
+        self.runner = runner
+        self.insock = runner.insock
+        self.outsock = runner.outsock
+
+        self.stdout_emitter = StreamToEmitter(self.emit, 'stdout')
+        self.stderr_emitter = StreamToEmitter(self.emit, 'stderr')
+
+        # Initialize user module and namespaces.
+        user_module = types.ModuleType('__main__',
+                                       doc='Automatically created module for the interactive shell.')
+        user_module.__dict__.setdefault('__builtin__', builtin_mod)
+        user_module.__dict__.setdefault('__builtins__', builtin_mod)
+        self.user_module = user_module
+        self.user_ns = user_module.__dict__
+
+        self.completer = Completer(namespace=self.user_ns, global_namespace={})
+        self.completer.limit_to__all__ = True
+
+    def handle_input(self, prompt=None, password=False):
+        if prompt is None:
+            prompt = 'Password: ' if password else ''
+        # Use synchronous version of ZeroMQ sockets
+        raw_insock = self.insock.transport._zmq_sock
+        raw_outsock = self.outsock.transport._zmq_sock
+        raw_outsock.send_multipart([
+            b'stdout',
+            prompt.encode('utf8'),
+        ])
+        raw_outsock.send_multipart([
+            b'waiting-input',
+            json.dumps({'is_password': password}).encode('utf8'),
+        ])
+        data = raw_insock.recv_multipart()
+        return data[1].decode('utf8')
+
+    def get_completions(self, data):
+        state = 0
+        matches = []
+        while True:
+            ret = self.completer.complete(data['line'], state)
+            if ret is None:
+                break
+            matches.append(ret)
+            state += 1
+        return matches
+
+    def emit(self, record):
+        if isinstance(record, ConsoleRecord):
+            assert record.target in ('stdout', 'stderr')
+            self.outsock.write([
+                record.target.encode('ascii'),
+                record.data.encode('utf8'),
+            ])
+        elif isinstance(record, MediaRecord):
+            self.outsock.write([
+                b'media',
+                json.dumps({
+                    'type': record.type,
+                    'data': record.data,
+                }).encode('utf8'),
+            ])
+        elif isinstance(record, HTMLRecord):
+            self.outsock.write([
+                b'html',
+                record.html.encode('utf8'),
+            ])
+        elif isinstance(record, InputRequest):
+            self.outsock.write([
+                b'waiting-input',
+                json.dumps({
+                    'is_password': record.is_password,
+                }).encode('utf8'),
+            ])
+        elif isinstance(record, CompletionRecord):
+            self.outsock.write([
+                b'completion',
+                json.dumps(record.matches).encode('utf8'),
+            ])
+        elif isinstance(record, ControlRecord):
+            self.outsock.write([
+                record.event.encode('ascii'),
+                b'',
+            ])
+        else:
+            raise TypeError('Unsupported record type.')
+
+    @staticmethod
+    def strip_traceback(tb):
+        while tb is not None:
+            frame_summary = traceback.extract_tb(tb, limit=1)[0]
+            if frame_summary[0] == '<input>':
+                break
+            tb = tb.tb_next
+        return tb
+
+    def query(self, code_text):
+        # Set Sorna Media handler
+        self.user_module.__builtins__._sorna_emit = self.emit
+
+        # Override interactive input functions
+        self.user_module.__builtins__.input = self.handle_input
+        getpass.getpass = partial(self.handle_input, password=True)
+
+        try:
+            code_obj = code.compile_command(code_text, symbol='exec')
+        except (OverflowError, IndentationError, SyntaxError,
+                ValueError, TypeError, MemoryError) as e:
+            exc_type, exc_val, tb = sys.exc_info()
+            user_tb = type(self).strip_traceback(tb)
+            err_str = ''.join(traceback.format_exception(exc_type, exc_val, user_tb))
+            hdr_str = 'Traceback (most recent call last):\n' if not err_str.startswith('Traceback ') else ''
+            self.emit(ConsoleRecord('stderr', hdr_str + err_str))
+            self.emit(ControlRecord('finished'))
+        else:
+            sys.stdout, orig_stdout = self.stdout_emitter, sys.stdout
+            sys.stderr, orig_stderr = self.stderr_emitter, sys.stderr
+            try:
+                exec(code_obj, self.user_ns)
+            except Exception as e:
+                # strip the first frame
+                exc_type, exc_val, tb = sys.exc_info()
+                user_tb = type(self).strip_traceback(tb)
+                traceback.print_exception(exc_type, exc_val, user_tb)
+            finally:
+                self.emit(ControlRecord('finished'))
+                sys.stdout = orig_stdout
+                sys.stderr = orig_stderr