Code completion, highlighting and std stream redirections

docs/source/usage.md

@@ -86,6 +86,27 @@ In [2]: %%julia
 |__/                   |
 ```
 
+#### IPython configuration
+
+PyJulia-IPython integration can be configured via IPython's
+configuration system.  For the non-default behaviors, add the
+following lines in, e.g.,
+``~/.ipython/profile_default/ipython_config.py`` (see
+[Introduction to IPython configuration](https://ipython.readthedocs.io/en/stable/config/intro.html)).
+
+To disable code completion in ``%julia`` and ``%%julia`` magics, use
+
+```python
+c.JuliaMagics.completion = False  # default: True
+```
+
+To disable code highlighting in ``%%julia`` magic for terminal
+(non-Jupyter) IPython, use
+
+```python
+c.JuliaMagics.highlight = False  # default: True
+```
+
 ### Virtual environments
 
 PyJulia can be used in Python virtual environments created by

julia/core.py

@@ -144,6 +144,11 @@ def __dir__(self):
     # Override __dir__ method so that completing member names work
     # well in Python REPLs like IPython.
 
+    __path__ = ()
+    # Declare that `JuliaModule` is a Python module since any Julia
+    # module can have sub-modules.
+    # See: https://docs.python.org/3/reference/import.html#package-path-rules
+
     def __getattr__(self, name):
         try:
             return self.__try_getattr(name)
@@ -200,9 +205,9 @@ class JuliaImporter(object):
     # find_module was deprecated in v3.4
     def find_module(self, fullname, path=None):
         if fullname.startswith("julia."):
-            pypath = os.path.join(os.path.dirname(__file__),
-                                  "{}.py".format(fullname[len("julia."):]))
-            if os.path.isfile(pypath):
+            filename = fullname.split(".", 2)[1]
+            filepath = os.path.join(os.path.dirname(__file__), filename)
+            if os.path.isfile(filepath + ".py") or os.path.isdir(filepath):
                 return
             return JuliaModuleLoader()
 
@@ -224,7 +229,7 @@ def load_module(self, fullname):
             return self.julia.eval(juliapath)
 
         try:
-            self.julia.eval("import {}".format(juliapath))
+            self.julia.eval("import {}".format(juliapath.split(".", 1)[0]))
         except JuliaError:
             pass
         else:
@@ -969,6 +974,13 @@ def __init__(self, init_julia=True, jl_init_path=None, runtime=None,
             self.eval("@eval Main import Base.MainInclude: eval, include")
             # https://github.com/JuliaLang/julia/issues/28825
 
+        if not isdefined(self, "Main", "_PyJuliaHelper"):
+            self.eval("include")(
+                os.path.join(
+                    os.path.dirname(os.path.realpath(__file__)), "pyjulia_helper.jl"
+                )
+            )
+
     def _call(self, src):
         """
         Low-level call to execute a snippet of Julia source.

julia/ipy/conftest.py

@@ -0,0 +1,8 @@
+import sys
+
+if sys.version_info[0] < 3:
+    collect_ignore = [
+        "monkeypatch_completer.py",
+        "monkeypatch_interactiveshell.py",
+    ]
+    # Theses files are ignored as import fails at collection phase.

julia/ipy/monkeypatch_completer.py

@@ -0,0 +1,89 @@
+"""
+Monkey-patch `IPCompleter` to make code completion work in ``%%julia``.
+
+This is done by monkey-patching because it looks like there is no
+immediate plan for an API to do this:
+https://github.com/ipython/ipython/pull/10722
+"""
+
+from __future__ import print_function, absolute_import
+
+import re
+
+from IPython.core.completer import Completion, IPCompleter
+
+
+class JuliaCompleter(object):
+    def __init__(self, julia=None):
+        from julia import Julia
+
+        self.julia = Julia() if julia is None else julia
+        self.magic_re = re.compile(r".*(\s|^)%%?julia\s*")
+        # With this regexp, "=%julia Cha<tab>" won't work.  But maybe
+        # it's better to be conservative here.
+
+    @property
+    def jlcomplete(self):
+        from julia.Main._PyJuliaHelper import completions
+
+        return completions
+
+    def julia_completions(self, full_text, offset):
+        self.last_text = full_text
+        match = self.magic_re.match(full_text)
+        if not match:
+            return []
+        prefix_len = match.end()
+        jl_pos = offset - prefix_len
+        jl_code = full_text[prefix_len:]
+        texts, (jl_start, jl_end), should_complete = self.jlcomplete(jl_code, jl_pos)
+        start = jl_start - 1 + prefix_len
+        end = jl_end + prefix_len
+        completions = [Completion(start, end, txt) for txt in texts]
+        self.last_completions = completions
+        # if not should_complete:
+        #     return []
+        return completions
+
+
+class IPCompleterPatcher(object):
+    def __init__(self):
+        from julia.Base import VERSION
+
+        if (VERSION.major, VERSION.minor) < (0, 7):
+            return
+
+        self.patch_ipcompleter(IPCompleter, JuliaCompleter())
+
+    def patch_ipcompleter(self, IPCompleter, jlcompleter):
+        orig__completions = IPCompleter._completions
+
+        def _completions(self, full_text, offset, **kwargs):
+            completions = jlcompleter.julia_completions(full_text, offset)
+            if completions:
+                return completions
+            else:
+                return orig__completions(self, full_text, offset, **kwargs)
+
+        IPCompleter._completions = _completions
+
+        self.orig__completions = orig__completions
+        self.patched__completions = _completions
+        self.IPCompleter = IPCompleter
+
+
+# Make it work with reload:
+try:
+    PATCHER
+except NameError:
+    PATCHER = None
+
+
+def patch_ipcompleter():
+    global PATCHER
+    if PATCHER is not None:
+        return
+    PATCHER = IPCompleterPatcher()
+
+
+# TODO: write `unpatch_ipcompleter`

julia/ipy/monkeypatch_interactiveshell.py

@@ -0,0 +1,43 @@
+"""
+Monkey-patch `TerminalInteractiveShell` to highlight code in ``%%julia``.
+"""
+
+from __future__ import print_function, absolute_import
+
+from IPython.terminal.interactiveshell import TerminalInteractiveShell
+from prompt_toolkit.lexers import PygmentsLexer
+from pygments.lexers import JuliaLexer
+
+
+class TerminalInteractiveShellPatcher(object):
+    def __init__(self):
+        self.patch_extra_prompt_options(TerminalInteractiveShell)
+
+    def patch_extra_prompt_options(self, TerminalInteractiveShell):
+        orig__extra_prompt_options = TerminalInteractiveShell._extra_prompt_options
+        self.orig__extra_prompt_options = orig__extra_prompt_options
+
+        def _extra_prompt_options(self):
+            options = orig__extra_prompt_options(self)
+            options["lexer"].magic_lexers["julia"] = PygmentsLexer(JuliaLexer)
+            return options
+
+        TerminalInteractiveShell._extra_prompt_options = _extra_prompt_options
+
+
+# Make it work with reload:
+try:
+    PATCHER
+except NameError:
+    PATCHER = None
+
+
+def patch_interactiveshell(ip):
+    global PATCHER
+    if PATCHER is not None:
+        return
+    if isinstance(ip, TerminalInteractiveShell):
+        PATCHER = TerminalInteractiveShellPatcher()
+
+
+# TODO: write `unpatch_interactiveshell`

julia/magic.py

@@ -18,21 +18,53 @@
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
+
 import sys
+import warnings
 
 from IPython.core.magic import Magics, magics_class, line_cell_magic
-from julia import Julia, JuliaError
+from IPython.utils import py3compat as compat
+from traitlets import Bool, Enum
+
+from .core import Julia, JuliaError
+from .tools import redirect_output_streams
 
 #-----------------------------------------------------------------------------
 # Main classes
 #-----------------------------------------------------------------------------
 
-import IPython.utils.py3compat as compat
 
 @magics_class
 class JuliaMagics(Magics):
     """A set of magics useful for interactive work with Julia.
     """
+
+    highlight = Bool(
+        True,
+        config=True,
+        help="""
+        Enable highlighting in `%%julia` magic by monkey-patching
+        IPython internal (`TerminalInteractiveShell`).
+        """,
+    )
+    completion = Bool(
+        True,
+        config=True,
+        help="""
+        Enable code completion in `%julia` and `%%julia` magics by
+        monkey-patching IPython internal (`IPCompleter`).
+        """,
+    )
+    redirect_output_streams = Enum(
+        ["auto", True, False],
+        "auto",
+        config=True,
+        help="""
+        Connect Julia's stdout and stderr to Python's standard stream.
+        "auto" (default) means to do so only in Jupyter.
+        """,
+    )
+
     def __init__(self, shell):
         """
         Parameters
@@ -73,10 +105,45 @@ def julia(self, line, cell=None):
 )
 
 
+def should_redirect_output_streams():
+    try:
+        OutStream = sys.modules["ipykernel"].iostream.OutStream
+    except (KeyError, AttributeError):
+        return False
+    return isinstance(sys.stdout, OutStream)
+
+
 #-----------------------------------------------------------------------------
 # IPython registration entry point.
 #-----------------------------------------------------------------------------
 
+
 def load_ipython_extension(ip):
     """Load the extension in IPython."""
-    ip.register_magics(JuliaMagics)
+
+    # This is equivalent to `ip.register_magics(JuliaMagics)` (but it
+    # let us access the instance of `JuliaMagics`):
+    magics = JuliaMagics(shell=ip)
+    ip.register_magics(magics)
+
+    template = "Incompatible upstream libraries. Got ImportError: {}"
+    if magics.highlight:
+        try:
+            from .ipy.monkeypatch_interactiveshell import patch_interactiveshell
+        except ImportError as err:
+            warnings.warn(template.format(err))
+        else:
+            patch_interactiveshell(ip)
+
+    if magics.completion:
+        try:
+            from .ipy.monkeypatch_completer import patch_ipcompleter
+        except ImportError as err:
+            warnings.warn(template.format(err))
+        else:
+            patch_ipcompleter()
+
+    if magics.redirect_output_streams is True or (
+        magics.redirect_output_streams == "auto" and should_redirect_output_streams()
+    ):
+        redirect_output_streams()