Produce Python AST for require statements and skip self requires

Closes hylang#1211.

Author: brandonwillard

hy/cmdline.py

@@ -13,6 +13,7 @@
 import importlib
 import py_compile
 import runpy
+import types
 
 import astor.code_gen
 
@@ -72,6 +73,20 @@ def __init__(self, spy=False, output_fn=None, locals=None,
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
 
+        # Create a proper module for this REPL so that we can obtain it easily
+        # (e.g. using `importlib.import_module`), have it be distinct from
+        # `__main__`, and use it with `hy_compile`.
+        module_name = self.locals['__name__']
+        self.module = types.ModuleType(module_name)
+        self.module.__dict__.update(self.locals)
+
+        self.locals = self.module.__dict__
+
+        sys.modules[module_name] = self.module
+
+        # Load cmdline-specific macros.
+        requiref('hy.cmdline', '__console__', assignments='ALL')
+
     def runsource(self, source, filename='<input>', symbol='single'):
         global SIMPLE_TRACEBACKS
 
@@ -102,8 +117,7 @@ def ast_callback(main_ast, expr_ast):
                     new_ast = ast.Module(main_ast.body +
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
-            value = hy_eval(do, self.locals, "__console__",
-                            ast_callback)
+            value = hy_eval(do, self.locals, self.module, ast_callback)
         except HyTypeError as e:
             if e.source is None:
                 e.source = source
@@ -181,7 +195,7 @@ def ideas_macro(ETname):
 
 """)])
 
-requiref("hy.cmdline", "__console__", assignments="ALL")
+
 requiref("hy.cmdline", "__main__", assignments="ALL")
 
 SIMPLE_TRACEBACKS = True
@@ -199,7 +213,7 @@ def pretty_error(func, *args, **kw):
 
 def run_command(source):
     tree = hy_parse(source)
-    pretty_error(hy_eval, tree, module_name="__main__")
+    pretty_error(hy_eval, tree, None, importlib.import_module('__main__'))
     return 0
 
 
@@ -208,12 +222,12 @@ def run_repl(hr=None, **kwargs):
     sys.ps1 = "=> "
     sys.ps2 = "... "
 
-    namespace = {'__name__': '__console__', '__doc__': ''}
+    if not hr:
+        hr = HyREPL(**kwargs)
 
-    with completion(Completer(namespace)):
+    namespace = hr.locals
 
-        if not hr:
-            hr = HyREPL(locals=namespace, **kwargs)
+    with completion(Completer(namespace)):
 
         hr.interact("{appname} {version} using "
                     "{py}({build}) {pyversion} on {os}".format(
@@ -409,7 +423,6 @@ def hyc_main():
 # entry point for cmd line script "hy2py"
 def hy2py_main():
     import platform
-    module_name = "<STDIN>"
 
     options = dict(prog="hy2py", usage="%(prog)s [options] [FILE]",
                    formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -448,7 +461,7 @@ def hy2py_main():
         print()
         print()
 
-    _ast = pretty_error(hy_compile, hst, module_name)
+    _ast = pretty_error(hy_compile, hst, '__main__')
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))

hy/compiler.py

@@ -13,14 +13,16 @@
 
 from hy.lex import mangle, unmangle
 
-import hy.macros
-from hy._compat import (
-    str_type, bytes_type, long_type, PY3, PY35, raise_empty)
-from hy.macros import requiref, macroexpand, tag_macroexpand
+from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
+                        PY35, raise_empty)
+from hy.macros import requiref, load_macros, macroexpand, tag_macroexpand
 import hy.importer
 
 import traceback
 import importlib
+import inspect
+import pkgutil
+import types
 import ast
 import sys
 import copy
@@ -283,28 +285,45 @@ def is_unpack(kind, x):
 
 
 class HyASTCompiler(object):
+    """A Hy-to-Python AST compiler"""
 
-    def __init__(self, module_name):
+    def __init__(self, module):
+        """
+        Parameters
+        ----------
+        module: str or types.ModuleType
+            Module in which the Hy tree is evaluated.
+        """
         self.anon_var_count = 0
         self.imports = defaultdict(set)
-        self.module_name = module_name
         self.temp_if = None
+
+        if not inspect.ismodule(module):
+            module = importlib.import_module(module)
+
+        self.module = module
+        self.module_name = module.__name__
+
         self.can_use_stdlib = (
-            not module_name.startswith("hy.core")
-            or module_name == "hy.core.macros")
+            not self.module_name.startswith("hy.core")
+            or self.module_name == "hy.core.macros")
+
+        # Load stdlib macros into the module namespace.
+        load_macros(self.module)
+
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
         if self.can_use_stdlib and not _stdlib:
             # Populate _stdlib.
             import hy.core
-            for module in hy.core.STDLIB:
-                mod = importlib.import_module(module)
-                for e in map(ast_str, mod.EXPORTS):
+            for stdlib_module in hy.core.STDLIB:
+                mod = importlib.import_module(stdlib_module)
+                for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
                     if getattr(mod, e) is not getattr(builtins, e, ''):
                         # Don't bother putting a name in _stdlib if it
                         # points to a builtin with the same name. This
                         # prevents pointless imports.
-                        _stdlib[e] = module
+                        _stdlib[e] = stdlib_module
 
     def get_anon_var(self):
         self.anon_var_count += 1
@@ -1098,11 +1117,6 @@ def compile_unary_operator(self, expr, root, arg):
         brackets(SYM, sym(":as"), _symn) |
         brackets(SYM, brackets(many(_symn + maybe(sym(":as") + _symn)))))])
     def compile_import_or_require(self, expr, root, entries):
-        """
-        TODO for `require`: keep track of what we've imported in this run and
-        then "unimport" it after we've completed `thing' so that we don't
-        pollute other envs.
-        """
         ret = Result()
 
         for entry in entries:
@@ -1128,8 +1142,9 @@ def compile_import_or_require(self, expr, root, entries):
                 else:
                     assignments = [(k, v or k) for k, v in kids]
 
+            ast_module = ast_str(module, piecewise=True)
+
             if root == "import":
-                ast_module = ast_str(module, piecewise=True)
                 module = ast_module.lstrip(".")
                 level = len(ast_module) - len(module)
                 if assignments == "ALL" and prefix == "":
@@ -1150,10 +1165,23 @@ def compile_import_or_require(self, expr, root, entries):
                         for k, v in assignments]
                 ret += node(
                     expr, module=module or None, names=names, level=level)
-            else: # root == "require"
-                importlib.import_module(module)
-                requiref(module, self.module_name,
-                         assignments=assignments, prefix=prefix)
+
+            elif requiref(ast_module, self.module, assignments=assignments,
+                          prefix=prefix):
+                # Actually calling `require` is necessary for macro expansions
+                # occurring during compilation.
+                self.imports['hy.macros'].update(['requiref'])
+                # The `require` we're creating in AST is the same as above, but used at
+                # run-time (e.g. when modules are loaded via bytecode).
+                ret += self.compile(HyExpression([
+                    HySymbol('requiref'),
+                    HyString(ast_module),
+                    HySymbol('None'),
+                    HyKeyword('assignments'),
+                    (HyString("ALL") if assignments == "ALL" else
+                        [[HyString(k), HyString(v)] for k, v in assignments]),
+                    HyKeyword('prefix'),
+                    HyString(prefix)]).replace(expr))
 
         return ret
 
@@ -1484,7 +1512,8 @@ def compile_class_expression(self, expr, root, name, rest):
                 [x for pair in attrs[0] for x in pair]).replace(attrs)))
 
         for e in body:
-            e = self.compile(self._rewire_init(macroexpand(e, self)))
+            e = self.compile(self._rewire_init(
+                macroexpand(e, self.module, self)))
             bodyr += e + e.expr_as_stmt()
 
         return bases + asty.ClassDef(
@@ -1520,28 +1549,24 @@ def compile_dispatch_tag_macro(self, expr, root, tag, arg):
         return self.compile(tag_macroexpand(
             HyString(mangle(tag)).replace(tag),
             arg,
-            self))
-
-    _namespaces = {}
+            self.module))
 
     @special(["eval-and-compile", "eval-when-compile"], [many(FORM)])
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
-        if self.module_name not in self._namespaces:
-            # Initialize a compile-time namespace for this module.
-            self._namespaces[self.module_name] = {
-                'hy': hy, '__name__': self.module_name}
+
         hy.importer.hy_eval(new_expr + body,
-                            self._namespaces[self.module_name],
-                            self.module_name)
+                            self.module.__dict__,
+                            self.module)
+
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
                 else Result())
 
     @builds_model(HyExpression)
     def compile_expression(self, expr):
         # Perform macro expansions
-        expr = macroexpand(expr, self)
+        expr = macroexpand(expr, self.module, self)
         if not isinstance(expr, HyExpression):
             # Go through compile again if the type changed.
             return self.compile(expr)
@@ -1699,20 +1724,41 @@ def compile_dict(self, m):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
-def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
+def hy_compile(tree, module, root=ast.Module, get_expr=False):
     """
-    Compile a HyObject tree into a Python AST Module.
+    Compile a Hy tree into a Python AST tree.
+
+    Parameters
+    ----------
+    module: str or types.ModuleType
+        Module, or name of the module, in which the Hy tree is evaluated.
 
-    If `get_expr` is True, return a tuple (module, last_expression), where
-    `last_expression` is the.
+    root: ast object, optional (ast.Module)
+        Root object for the Python AST tree.
+
+    get_expr: bool, optional (False)
+        If true, return a tuple with `(root_obj, last_expression)`.
+
+    Returns
+    -------
+    out : A Python AST tree
     """
 
+    if isinstance(module, string_types):
+        if module.startswith('<') and module.endswith('>'):
+            module = types.ModuleType(module)
+        else:
+            module = importlib.import_module(ast_str(module, piecewise=True))
+    if not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+
     tree = wrap_value(tree)
     if not isinstance(tree, HyObject):
         raise HyCompileError("`tree` must be a HyObject or capable of "
                              "being promoted to one")
 
-    compiler = HyASTCompiler(module_name)
+    compiler = HyASTCompiler(module)
     result = compiler.compile(tree)
     expr = result.force_expr
 

hy/completer.py

@@ -39,13 +39,15 @@ def __init__(self, namespace={}):
         self.namespace = namespace
         self.path = [hy.compiler._special_form_compilers,
                      builtins.__dict__,
-                     hy.macros._hy_macros[None],
                      namespace]
-        self.tag_path = [hy.macros._hy_tag[None]]
-        if '__name__' in namespace:
-            module_name = namespace['__name__']
-            self.path.append(hy.macros._hy_macros[module_name])
-            self.tag_path.append(hy.macros._hy_tag[module_name])
+
+        self.tag_path = []
+
+        namespace.setdefault('__macros__', {})
+        namespace.setdefault('__tags__', {})
+
+        self.path.append(namespace['__macros__'])
+        self.tag_path.append(namespace['__tags__'])
 
     def attr_matches(self, text):
         # Borrowed from IPython's completer

hy/contrib/walk.hy

@@ -5,6 +5,7 @@
 
 (import [hy [HyExpression HyDict]]
         [functools [partial]]
+        [importlib [import-module]]
         [collections [OrderedDict]]
         [hy.macros [macroexpand :as mexpand]]
         [hy.compiler [HyASTCompiler]])
@@ -42,9 +43,11 @@
 
 (defn macroexpand-all [form &optional module-name]
   "Recursively performs all possible macroexpansions in form."
-  (setv module-name (or module-name (calling-module-name))
+  (setv module (or (and module-name
+                        (import-module module-name))
+                   (calling-module))
         quote-level [0]
-        ast-compiler (HyASTCompiler module-name))  ; TODO: make nonlocal after dropping Python2
+        ast-compiler (HyASTCompiler module))  ; TODO: make nonlocal after dropping Python2
   (defn traverse [form]
     (walk expand identity form))
   (defn expand [form]
@@ -68,7 +71,7 @@
               [(= (first form) (HySymbol "require"))
                (ast-compiler.compile form)
                (return)]
-              [True (traverse (mexpand form ast-compiler))])
+              [True (traverse (mexpand form module ast-compiler))])
         (if (coll? form)
             (traverse form)
             form)))

hy/core/language.hy

@@ -21,7 +21,7 @@
 (import [hy.models [HySymbol HyKeyword]])
 (import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
 (import [hy.compiler [HyASTCompiler]])
-(import [hy.importer [hy-eval :as eval]])
+(import [hy.importer [calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
@@ -295,12 +295,14 @@ Return series of accumulated sums (or other binary function results)."
 (defn macroexpand [form]
   "Return the full macro expansion of `form`."
   (import hy.macros)
-  (hy.macros.macroexpand form (HyASTCompiler (calling-module-name))))
+  (setv module (calling-module))
+  (hy.macros.macroexpand form module (HyASTCompiler module)))
 
 (defn macroexpand-1 [form]
   "Return the single step macro expansion of `form`."
   (import hy.macros)
-  (hy.macros.macroexpand-1 form (HyASTCompiler (calling-module-name))))
+  (setv module (calling-module))
+  (hy.macros.macroexpand-1 form module (HyASTCompiler module)))
 
 (defn merge-with [f &rest maps]
   "Return the map of `maps` joined onto the first via the function `f`.
@@ -467,8 +469,8 @@ Even objects with the __name__ magic will work."
     (or a b)))
 
 (setv EXPORTS
-  '[*map accumulate butlast calling-module-name chain coll? combinations
-    comp complement compress constantly count cycle dec distinct
+  '[*map accumulate butlast calling-module calling-module-name chain coll?
+    combinations comp complement compress constantly count cycle dec distinct
     disassemble drop drop-last drop-while empty? eval even? every? exec first
     filter flatten float? fraction gensym group-by identity inc input instance?
     integer integer? integer-char? interleave interpose islice iterable?