Merge pull request hylang#1682 from brandonwillard/macro-changes

Macro processing updates and fixes

Author: Kodiologist

NEWS.rst

@@ -12,9 +12,15 @@ New Features
 * Keyword objects (not just literal keywords) can be called, as
   shorthand for `(get obj :key)`, and they accept a default value
   as a second argument.
+* Minimal macro expansion namespacing has been implemented.  As a result,
+  external macros no longer have to `require` their own macro dependencies.
+* Macros and tags now reside in module-level `__macros__` and `__tags__`
+  attributes.
 
 Bug Fixes
 ------------------------------
+* `require` now compiles to Python AST.
+* Fixed circular `require`s.
 * Fixed module reloading.
 * Fixed circular imports.
 * Fixed `__main__` file execution.

hy/cmdline.py

@@ -13,6 +13,7 @@
 import importlib
 import py_compile
 import runpy
+import types
 
 import astor.code_gen
 
@@ -47,10 +48,26 @@ def __call__(self, code=None):
 builtins.exit = HyQuitter('exit')
 
 
-class HyREPL(code.InteractiveConsole):
+class HyREPL(code.InteractiveConsole, object):
     def __init__(self, spy=False, output_fn=None, locals=None,
                  filename="<input>"):
 
+        super(HyREPL, self).__init__(locals=locals,
+                                     filename=filename)
+
+        # Create a proper module for this REPL so that we can obtain it easily
+        # (e.g. using `importlib.import_module`).
+        # Also, make sure it's properly introduced to `sys.modules` and
+        # consistently use its namespace as `locals` from here on.
+        module_name = self.locals.get('__name__', '__console__')
+        self.module = sys.modules.setdefault(module_name,
+                                             types.ModuleType(module_name))
+        self.module.__dict__.update(self.locals)
+        self.locals = self.module.__dict__
+
+        # Load cmdline-specific macros.
+        require('hy.cmdline', module_name, assignments='ALL')
+
         self.spy = spy
 
         if output_fn is None:
@@ -65,9 +82,6 @@ def __init__(self, spy=False, output_fn=None, locals=None,
             else:
                 self.output_fn = __builtins__[mangle(output_fn)]
 
-        code.InteractiveConsole.__init__(self, locals=locals,
-                                         filename=filename)
-
         # Pre-mangle symbols for repl recent results: *1, *2, *3
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
@@ -102,8 +116,7 @@ def ast_callback(main_ast, expr_ast):
                     new_ast = ast.Module(main_ast.body +
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
-            value = hy_eval(do, self.locals, "__console__",
-                            ast_callback)
+            value = hy_eval(do, self.locals, self.module, ast_callback)
         except HyTypeError as e:
             if e.source is None:
                 e.source = source
@@ -181,8 +194,6 @@ def ideas_macro(ETname):
 
 """)])
 
-require("hy.cmdline", "__console__", assignments="ALL")
-require("hy.cmdline", "__main__", assignments="ALL")
 
 SIMPLE_TRACEBACKS = True
 
@@ -199,7 +210,8 @@ def pretty_error(func, *args, **kw):
 
 def run_command(source):
     tree = hy_parse(source)
-    pretty_error(hy_eval, tree, module_name="__main__")
+    require("hy.cmdline", "__main__", assignments="ALL")
+    pretty_error(hy_eval, tree, None, importlib.import_module('__main__'))
     return 0
 
 
@@ -208,12 +220,12 @@ def run_repl(hr=None, **kwargs):
     sys.ps1 = "=> "
     sys.ps2 = "... "
 
-    namespace = {'__name__': '__console__', '__doc__': ''}
+    if not hr:
+        hr = HyREPL(**kwargs)
 
-    with completion(Completer(namespace)):
+    namespace = hr.locals
 
-        if not hr:
-            hr = HyREPL(locals=namespace, **kwargs)
+    with completion(Completer(namespace)):
 
         hr.interact("{appname} {version} using "
                     "{py}({build}) {pyversion} on {os}".format(
@@ -409,7 +421,6 @@ def hyc_main():
 # entry point for cmd line script "hy2py"
 def hy2py_main():
     import platform
-    module_name = "<STDIN>"
 
     options = dict(prog="hy2py", usage="%(prog)s [options] [FILE]",
                    formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -448,7 +459,7 @@ def hy2py_main():
         print()
         print()
 
-    _ast = pretty_error(hy_compile, hst, module_name)
+    _ast = pretty_error(hy_compile, hst, '__main__')
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))

hy/compiler.py

@@ -13,14 +13,16 @@
 
 from hy.lex import mangle, unmangle
 
-import hy.macros
-from hy._compat import (
-    str_type, bytes_type, long_type, PY3, PY35, raise_empty)
-from hy.macros import require, macroexpand, tag_macroexpand
+from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
+                        PY35, raise_empty)
+from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 import hy.importer
 
 import traceback
 import importlib
+import inspect
+import pkgutil
+import types
 import ast
 import sys
 import copy
@@ -279,32 +281,48 @@ def is_unpack(kind, x):
             and x[0] == "unpack-" + kind)
 
 
-_stdlib = {}
-
-
 class HyASTCompiler(object):
+    """A Hy-to-Python AST compiler"""
 
-    def __init__(self, module_name):
+    def __init__(self, module):
+        """
+        Parameters
+        ----------
+        module: str or types.ModuleType
+            Module in which the Hy tree is evaluated.
+        """
         self.anon_var_count = 0
         self.imports = defaultdict(set)
-        self.module_name = module_name
         self.temp_if = None
+
+        if not inspect.ismodule(module):
+            module = importlib.import_module(module)
+
+        self.module = module
+        self.module_name = module.__name__
+
         self.can_use_stdlib = (
-            not module_name.startswith("hy.core")
-            or module_name == "hy.core.macros")
+            not self.module_name.startswith("hy.core")
+            or self.module_name == "hy.core.macros")
+
+        # Load stdlib macros into the module namespace.
+        load_macros(self.module)
+
+        self._stdlib = {}
+
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
-        if self.can_use_stdlib and not _stdlib:
+        if self.can_use_stdlib:
             # Populate _stdlib.
             import hy.core
-            for module in hy.core.STDLIB:
-                mod = importlib.import_module(module)
-                for e in map(ast_str, mod.EXPORTS):
+            for stdlib_module in hy.core.STDLIB:
+                mod = importlib.import_module(stdlib_module)
+                for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
                     if getattr(mod, e) is not getattr(builtins, e, ''):
                         # Don't bother putting a name in _stdlib if it
                         # points to a builtin with the same name. This
                         # prevents pointless imports.
-                        _stdlib[e] = module
+                        self._stdlib[e] = stdlib_module
 
     def get_anon_var(self):
         self.anon_var_count += 1
@@ -1098,11 +1116,6 @@ def compile_unary_operator(self, expr, root, arg):
         brackets(SYM, sym(":as"), _symn) |
         brackets(SYM, brackets(many(_symn + maybe(sym(":as") + _symn)))))])
     def compile_import_or_require(self, expr, root, entries):
-        """
-        TODO for `require`: keep track of what we've imported in this run and
-        then "unimport" it after we've completed `thing' so that we don't
-        pollute other envs.
-        """
         ret = Result()
 
         for entry in entries:
@@ -1128,8 +1141,9 @@ def compile_import_or_require(self, expr, root, entries):
                 else:
                     assignments = [(k, v or k) for k, v in kids]
 
+            ast_module = ast_str(module, piecewise=True)
+
             if root == "import":
-                ast_module = ast_str(module, piecewise=True)
                 module = ast_module.lstrip(".")
                 level = len(ast_module) - len(module)
                 if assignments == "ALL" and prefix == "":
@@ -1150,10 +1164,23 @@ def compile_import_or_require(self, expr, root, entries):
                         for k, v in assignments]
                 ret += node(
                     expr, module=module or None, names=names, level=level)
-            else: # root == "require"
-                importlib.import_module(module)
-                require(module, self.module_name,
-                        assignments=assignments, prefix=prefix)
+
+            elif require(ast_module, self.module, assignments=assignments,
+                         prefix=prefix):
+                # Actually calling `require` is necessary for macro expansions
+                # occurring during compilation.
+                self.imports['hy.macros'].update([None])
+                # The `require` we're creating in AST is the same as above, but used at
+                # run-time (e.g. when modules are loaded via bytecode).
+                ret += self.compile(HyExpression([
+                    HySymbol('hy.macros.require'),
+                    HyString(ast_module),
+                    HySymbol('None'),
+                    HyKeyword('assignments'),
+                    (HyString("ALL") if assignments == "ALL" else
+                        [[HyString(k), HyString(v)] for k, v in assignments]),
+                    HyKeyword('prefix'),
+                    HyString(prefix)]).replace(expr))
 
         return ret
 
@@ -1484,7 +1511,8 @@ def compile_class_expression(self, expr, root, name, rest):
                 [x for pair in attrs[0] for x in pair]).replace(attrs)))
 
         for e in body:
-            e = self.compile(self._rewire_init(macroexpand(e, self)))
+            e = self.compile(self._rewire_init(
+                macroexpand(e, self.module, self)))
             bodyr += e + e.expr_as_stmt()
 
         return bases + asty.ClassDef(
@@ -1520,28 +1548,24 @@ def compile_dispatch_tag_macro(self, expr, root, tag, arg):
         return self.compile(tag_macroexpand(
             HyString(mangle(tag)).replace(tag),
             arg,
-            self))
-
-    _namespaces = {}
+            self.module))
 
     @special(["eval-and-compile", "eval-when-compile"], [many(FORM)])
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
-        if self.module_name not in self._namespaces:
-            # Initialize a compile-time namespace for this module.
-            self._namespaces[self.module_name] = {
-                'hy': hy, '__name__': self.module_name}
+
         hy.importer.hy_eval(new_expr + body,
-                            self._namespaces[self.module_name],
-                            self.module_name)
+                            self.module.__dict__,
+                            self.module)
+
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
                 else Result())
 
     @builds_model(HyExpression)
     def compile_expression(self, expr):
         # Perform macro expansions
-        expr = macroexpand(expr, self)
+        expr = macroexpand(expr, self.module, self)
         if not isinstance(expr, HyExpression):
             # Go through compile again if the type changed.
             return self.compile(expr)
@@ -1665,8 +1689,8 @@ def compile_symbol(self, symbol):
                 attr=ast_str(local),
                 ctx=ast.Load())
 
-        if self.can_use_stdlib and ast_str(symbol) in _stdlib:
-            self.imports[_stdlib[ast_str(symbol)]].add(ast_str(symbol))
+        if self.can_use_stdlib and ast_str(symbol) in self._stdlib:
+            self.imports[self._stdlib[ast_str(symbol)]].add(ast_str(symbol))
 
         return asty.Name(symbol, id=ast_str(symbol), ctx=ast.Load())
 
@@ -1699,20 +1723,41 @@ def compile_dict(self, m):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
-def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
+def hy_compile(tree, module, root=ast.Module, get_expr=False):
     """
-    Compile a HyObject tree into a Python AST Module.
+    Compile a Hy tree into a Python AST tree.
+
+    Parameters
+    ----------
+    module: str or types.ModuleType
+        Module, or name of the module, in which the Hy tree is evaluated.
 
-    If `get_expr` is True, return a tuple (module, last_expression), where
-    `last_expression` is the.
+    root: ast object, optional (ast.Module)
+        Root object for the Python AST tree.
+
+    get_expr: bool, optional (False)
+        If true, return a tuple with `(root_obj, last_expression)`.
+
+    Returns
+    -------
+    out : A Python AST tree
     """
 
+    if isinstance(module, string_types):
+        if module.startswith('<') and module.endswith('>'):
+            module = types.ModuleType(module)
+        else:
+            module = importlib.import_module(ast_str(module, piecewise=True))
+    if not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+
     tree = wrap_value(tree)
     if not isinstance(tree, HyObject):
         raise HyCompileError("`tree` must be a HyObject or capable of "
                              "being promoted to one")
 
-    compiler = HyASTCompiler(module_name)
+    compiler = HyASTCompiler(module)
     result = compiler.compile(tree)
     expr = result.force_expr
 

hy/completer.py

@@ -39,13 +39,15 @@ def __init__(self, namespace={}):
         self.namespace = namespace
         self.path = [hy.compiler._special_form_compilers,
                      builtins.__dict__,
-                     hy.macros._hy_macros[None],
                      namespace]
-        self.tag_path = [hy.macros._hy_tag[None]]
-        if '__name__' in namespace:
-            module_name = namespace['__name__']
-            self.path.append(hy.macros._hy_macros[module_name])
-            self.tag_path.append(hy.macros._hy_tag[module_name])
+
+        self.tag_path = []
+
+        namespace.setdefault('__macros__', {})
+        namespace.setdefault('__tags__', {})
+
+        self.path.append(namespace['__macros__'])
+        self.tag_path.append(namespace['__tags__'])
 
     def attr_matches(self, text):
         # Borrowed from IPython's completer