diff --git a/docs/source/usage.md b/docs/source/usage.md index 410b1ce2..b52347c1 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -86,6 +86,27 @@ In [2]: %%julia |__/ | ``` +#### IPython configuration + +PyJulia-IPython integration can be configured via IPython's +configuration system. For the non-default behaviors, add the +following lines in, e.g., +``~/.ipython/profile_default/ipython_config.py`` (see +[Introduction to IPython configuration](https://ipython.readthedocs.io/en/stable/config/intro.html)). + +To disable code completion in ``%julia`` and ``%%julia`` magics, use + +```python +c.JuliaMagics.completion = False # default: True +``` + +To disable code highlighting in ``%%julia`` magic for terminal +(non-Jupyter) IPython, use + +```python +c.JuliaMagics.highlight = False # default: True +``` + ### Virtual environments PyJulia can be used in Python virtual environments created by diff --git a/julia/core.py b/julia/core.py index 9beb874d..34813f2a 100644 --- a/julia/core.py +++ b/julia/core.py @@ -144,6 +144,11 @@ def __dir__(self): # Override __dir__ method so that completing member names work # well in Python REPLs like IPython. + __path__ = () + # Declare that `JuliaModule` is a Python module since any Julia + # module can have sub-modules. + # See: https://docs.python.org/3/reference/import.html#package-path-rules + def __getattr__(self, name): try: return self.__try_getattr(name) @@ -200,9 +205,9 @@ class JuliaImporter(object): # find_module was deprecated in v3.4 def find_module(self, fullname, path=None): if fullname.startswith("julia."): - pypath = os.path.join(os.path.dirname(__file__), - "{}.py".format(fullname[len("julia."):])) - if os.path.isfile(pypath): + filename = fullname.split(".", 2)[1] + filepath = os.path.join(os.path.dirname(__file__), filename) + if os.path.isfile(filepath + ".py") or os.path.isdir(filepath): return return JuliaModuleLoader() @@ -224,7 +229,7 @@ def load_module(self, fullname): return self.julia.eval(juliapath) try: - self.julia.eval("import {}".format(juliapath)) + self.julia.eval("import {}".format(juliapath.split(".", 1)[0])) except JuliaError: pass else: @@ -969,6 +974,13 @@ def __init__(self, init_julia=True, jl_init_path=None, runtime=None, self.eval("@eval Main import Base.MainInclude: eval, include") # https://github.com/JuliaLang/julia/issues/28825 + if not isdefined(self, "Main", "_PyJuliaHelper"): + self.eval("include")( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), "pyjulia_helper.jl" + ) + ) + def _call(self, src): """ Low-level call to execute a snippet of Julia source. diff --git a/julia/ipy/__init__.py b/julia/ipy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/julia/ipy/conftest.py b/julia/ipy/conftest.py new file mode 100644 index 00000000..fa39351e --- /dev/null +++ b/julia/ipy/conftest.py @@ -0,0 +1,8 @@ +import sys + +if sys.version_info[0] < 3: + collect_ignore = [ + "monkeypatch_completer.py", + "monkeypatch_interactiveshell.py", + ] + # Theses files are ignored as import fails at collection phase. diff --git a/julia/ipy/monkeypatch_completer.py b/julia/ipy/monkeypatch_completer.py new file mode 100644 index 00000000..3feb80ea --- /dev/null +++ b/julia/ipy/monkeypatch_completer.py @@ -0,0 +1,89 @@ +""" +Monkey-patch `IPCompleter` to make code completion work in ``%%julia``. + +This is done by monkey-patching because it looks like there is no +immediate plan for an API to do this: +https://github.com/ipython/ipython/pull/10722 +""" + +from __future__ import print_function, absolute_import + +import re + +from IPython.core.completer import Completion, IPCompleter + + +class JuliaCompleter(object): + def __init__(self, julia=None): + from julia import Julia + + self.julia = Julia() if julia is None else julia + self.magic_re = re.compile(r".*(\s|^)%%?julia\s*") + # With this regexp, "=%julia Cha" won't work. But maybe + # it's better to be conservative here. + + @property + def jlcomplete(self): + from julia.Main._PyJuliaHelper import completions + + return completions + + def julia_completions(self, full_text, offset): + self.last_text = full_text + match = self.magic_re.match(full_text) + if not match: + return [] + prefix_len = match.end() + jl_pos = offset - prefix_len + jl_code = full_text[prefix_len:] + texts, (jl_start, jl_end), should_complete = self.jlcomplete(jl_code, jl_pos) + start = jl_start - 1 + prefix_len + end = jl_end + prefix_len + completions = [Completion(start, end, txt) for txt in texts] + self.last_completions = completions + # if not should_complete: + # return [] + return completions + + +class IPCompleterPatcher(object): + def __init__(self): + from julia.Base import VERSION + + if (VERSION.major, VERSION.minor) < (0, 7): + return + + self.patch_ipcompleter(IPCompleter, JuliaCompleter()) + + def patch_ipcompleter(self, IPCompleter, jlcompleter): + orig__completions = IPCompleter._completions + + def _completions(self, full_text, offset, **kwargs): + completions = jlcompleter.julia_completions(full_text, offset) + if completions: + return completions + else: + return orig__completions(self, full_text, offset, **kwargs) + + IPCompleter._completions = _completions + + self.orig__completions = orig__completions + self.patched__completions = _completions + self.IPCompleter = IPCompleter + + +# Make it work with reload: +try: + PATCHER +except NameError: + PATCHER = None + + +def patch_ipcompleter(): + global PATCHER + if PATCHER is not None: + return + PATCHER = IPCompleterPatcher() + + +# TODO: write `unpatch_ipcompleter` diff --git a/julia/ipy/monkeypatch_interactiveshell.py b/julia/ipy/monkeypatch_interactiveshell.py new file mode 100644 index 00000000..dc8b9705 --- /dev/null +++ b/julia/ipy/monkeypatch_interactiveshell.py @@ -0,0 +1,43 @@ +""" +Monkey-patch `TerminalInteractiveShell` to highlight code in ``%%julia``. +""" + +from __future__ import print_function, absolute_import + +from IPython.terminal.interactiveshell import TerminalInteractiveShell +from prompt_toolkit.lexers import PygmentsLexer +from pygments.lexers import JuliaLexer + + +class TerminalInteractiveShellPatcher(object): + def __init__(self): + self.patch_extra_prompt_options(TerminalInteractiveShell) + + def patch_extra_prompt_options(self, TerminalInteractiveShell): + orig__extra_prompt_options = TerminalInteractiveShell._extra_prompt_options + self.orig__extra_prompt_options = orig__extra_prompt_options + + def _extra_prompt_options(self): + options = orig__extra_prompt_options(self) + options["lexer"].magic_lexers["julia"] = PygmentsLexer(JuliaLexer) + return options + + TerminalInteractiveShell._extra_prompt_options = _extra_prompt_options + + +# Make it work with reload: +try: + PATCHER +except NameError: + PATCHER = None + + +def patch_interactiveshell(ip): + global PATCHER + if PATCHER is not None: + return + if isinstance(ip, TerminalInteractiveShell): + PATCHER = TerminalInteractiveShellPatcher() + + +# TODO: write `unpatch_interactiveshell` diff --git a/julia/magic.py b/julia/magic.py index 96b3d7ae..35d3088c 100644 --- a/julia/magic.py +++ b/julia/magic.py @@ -18,21 +18,53 @@ #----------------------------------------------------------------------------- from __future__ import print_function, absolute_import + import sys +import warnings from IPython.core.magic import Magics, magics_class, line_cell_magic -from julia import Julia, JuliaError +from IPython.utils import py3compat as compat +from traitlets import Bool, Enum + +from .core import Julia, JuliaError +from .tools import redirect_output_streams #----------------------------------------------------------------------------- # Main classes #----------------------------------------------------------------------------- -import IPython.utils.py3compat as compat @magics_class class JuliaMagics(Magics): """A set of magics useful for interactive work with Julia. """ + + highlight = Bool( + True, + config=True, + help=""" + Enable highlighting in `%%julia` magic by monkey-patching + IPython internal (`TerminalInteractiveShell`). + """, + ) + completion = Bool( + True, + config=True, + help=""" + Enable code completion in `%julia` and `%%julia` magics by + monkey-patching IPython internal (`IPCompleter`). + """, + ) + redirect_output_streams = Enum( + ["auto", True, False], + "auto", + config=True, + help=""" + Connect Julia's stdout and stderr to Python's standard stream. + "auto" (default) means to do so only in Jupyter. + """, + ) + def __init__(self, shell): """ Parameters @@ -73,10 +105,45 @@ def julia(self, line, cell=None): ) +def should_redirect_output_streams(): + try: + OutStream = sys.modules["ipykernel"].iostream.OutStream + except (KeyError, AttributeError): + return False + return isinstance(sys.stdout, OutStream) + + #----------------------------------------------------------------------------- # IPython registration entry point. #----------------------------------------------------------------------------- + def load_ipython_extension(ip): """Load the extension in IPython.""" - ip.register_magics(JuliaMagics) + + # This is equivalent to `ip.register_magics(JuliaMagics)` (but it + # let us access the instance of `JuliaMagics`): + magics = JuliaMagics(shell=ip) + ip.register_magics(magics) + + template = "Incompatible upstream libraries. Got ImportError: {}" + if magics.highlight: + try: + from .ipy.monkeypatch_interactiveshell import patch_interactiveshell + except ImportError as err: + warnings.warn(template.format(err)) + else: + patch_interactiveshell(ip) + + if magics.completion: + try: + from .ipy.monkeypatch_completer import patch_ipcompleter + except ImportError as err: + warnings.warn(template.format(err)) + else: + patch_ipcompleter() + + if magics.redirect_output_streams is True or ( + magics.redirect_output_streams == "auto" and should_redirect_output_streams() + ): + redirect_output_streams() diff --git a/julia/pyjulia_helper.jl b/julia/pyjulia_helper.jl new file mode 100644 index 00000000..b411dde9 --- /dev/null +++ b/julia/pyjulia_helper.jl @@ -0,0 +1,94 @@ +module _PyJuliaHelper + +if VERSION >= v"0.7-" + import REPL + + function completions(str, pos) + ret, ran, should_complete = REPL.completions(str, pos) + return ( + map(REPL.completion_text, ret), + (first(ran), last(ran)), + should_complete, + ) + end +end + +module IOPiper + +const orig_stdin = Ref{IO}() +const orig_stdout = Ref{IO}() +const orig_stderr = Ref{IO}() + +function __init__() +@static if VERSION < v"0.7-" + orig_stdin[] = STDIN + orig_stdout[] = STDOUT + orig_stderr[] = STDERR +else + orig_stdin[] = stdin + orig_stdout[] = stdout + orig_stderr[] = stderr +end +end + +""" + num_utf8_trailing(d::Vector{UInt8}) + +If `d` ends with an incomplete UTF8-encoded character, return the number of trailing incomplete bytes. +Otherwise, return `0`. + +Taken from IJulia.jl. +""" +function num_utf8_trailing(d::Vector{UInt8}) + i = length(d) + # find last non-continuation byte in d: + while i >= 1 && ((d[i] & 0xc0) == 0x80) + i -= 1 + end + i < 1 && return 0 + c = d[i] + # compute number of expected UTF-8 bytes starting at i: + n = c <= 0x7f ? 1 : c < 0xe0 ? 2 : c < 0xf0 ? 3 : 4 + nend = length(d) + 1 - i # num bytes from i to end + return nend == n ? 0 : nend +end + +function pipe_stream(sender::IO, receiver, buf::IO = IOBuffer()) + try + while !eof(sender) + nb = bytesavailable(sender) + write(buf, read(sender, nb)) + + # Taken from IJulia.send_stream: + d = take!(buf) + n = num_utf8_trailing(d) + dextra = d[end-(n-1):end] + resize!(d, length(d) - n) + s = String(copy(d)) + + write(buf, dextra) + receiver(s) # check isvalid(String, s)? + end + catch e + if !isa(e, InterruptException) + rethrow() + end + pipe_stream(sender, receiver, buf) + end +end + +const read_stdout = Ref{Base.PipeEndpoint}() +const read_stderr = Ref{Base.PipeEndpoint}() + +function pipe_std_outputs(out_receiver, err_receiver) + global readout_task + global readerr_task + read_stdout[], = redirect_stdout() + readout_task = @async pipe_stream(read_stdout[], out_receiver) + read_stderr[], = redirect_stderr() + readerr_task = @async pipe_stream(read_stderr[], err_receiver) +end + +end # module + +end # module diff --git a/julia/tools.py b/julia/tools.py new file mode 100644 index 00000000..0ed7d836 --- /dev/null +++ b/julia/tools.py @@ -0,0 +1,24 @@ +from __future__ import print_function, absolute_import + +import sys + + +def make_receiver(io): + def receiver(s): + io.write(s) + io.flush() + + return receiver + + +def redirect_output_streams(): + """ + Redirect Julia's stdout and stderr to Python's counter parts. + """ + + from .Main._PyJuliaHelper.IOPiper import pipe_std_outputs + + pipe_std_outputs(make_receiver(sys.stdout), make_receiver(sys.stderr)) + + # TODO: Invoking `redirect_output_streams()` in terminal IPython + # terminates the whole Python process. Find out why. diff --git a/setup.py b/setup.py index 9abfd390..ca67aca5 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ def pyload(path): ], url='http://julialang.org', packages=['julia'], - package_data={'julia': ['fake-julia/*']}, + package_data={'julia': ['fake-julia/*', "*.jl"]}, entry_points={ "console_scripts": [ "python-jl = julia.python_jl:main",