macroexpand: stop pre-running the hygiene pass

This prepares us to delete the separate hygiene pass, and make it part
of lowering resolve-scopes. This affects a few macros which expect to
run macroexpand then mangle the result more, and the result of those
macros is not already wrapped in `esc()`. The damage seems relatively
minor however, and generally a good improvement.

Author: vtjnash

base/docs/Docs.jl

@@ -292,6 +292,16 @@ function unblock(@nospecialize ex)
     return unblock(exs[1])
 end
 
+# peek through ex to figure out what kind of expression it may eventually act like
+# but ignoring scopes and line numbers
+function unescape(@nospecialize ex)
+    ex = unblock(ex)
+    while isexpr(ex, :escape) || isexpr(ex, :var"hygienic-scope")
+       ex = unblock(ex.args[1])
+    end
+    return ex
+end
+
 uncurly(@nospecialize ex) = isexpr(ex, :curly) ? ex.args[1] : ex
 
 namify(@nospecialize x) = astname(x, isexpr(x, :macro))::Union{Symbol,Expr,GlobalRef}
@@ -351,18 +361,19 @@ function metadata(__source__, __module__, expr, ismodule)
         fields = P[]
         last_docstr = nothing
         for each in (expr.args[3]::Expr).args
-            if isa(each, Symbol) || isexpr(each, :(::))
+            eachex = unescape(each)
+            if isa(eachex, Symbol) || isexpr(eachex, :(::))
                 # a field declaration
                 if last_docstr !== nothing
-                    push!(fields, P(namify(each::Union{Symbol,Expr}), last_docstr))
+                    push!(fields, P(namify(eachex::Union{Symbol,Expr}), last_docstr))
                     last_docstr = nothing
                 end
-            elseif isexpr(each, :function) || isexpr(each, :(=))
+            elseif isexpr(eachex, :function) || isexpr(eachex, :(=))
                 break
-            elseif isa(each, String) || isexpr(each, :string) || isexpr(each, :call) ||
-                (isexpr(each, :macrocall) && each.args[1] === Symbol("@doc_str"))
+            elseif isa(eachex, String) || isexpr(eachex, :string) || isexpr(eachex, :call) ||
+                (isexpr(eachex, :macrocall) && eachex.args[1] === Symbol("@doc_str"))
                 # forms that might be doc strings
-                last_docstr = each::Union{String,Expr}
+                last_docstr = each
             end
         end
         dict = :($(Dict{Symbol,Any})($([(:($(P)($(quot(f)), $d)))::Expr for (f, d) in fields]...)))

base/osutils.jl

@@ -16,7 +16,7 @@ macro static(ex)
         @label loop
         hd = ex.head
         if hd ∈ (:if, :elseif, :&&, :||)
-            cond = Core.eval(__module__, ex.args[1])
+            cond = Core.eval(__module__, ex.args[1])::Bool
             if xor(cond, hd === :||)
                 return esc(ex.args[2])
             elseif length(ex.args) == 3

base/util.jl

@@ -604,7 +604,7 @@ macro kwdef(expr)
         kwdefs = nothing
     end
     return quote
-        Base.@__doc__ $(esc(expr))
+        $(esc(:($Base.@__doc__ $expr)))
         $kwdefs
     end
 end

deps/checksums/Pkg-33b5e806ffbb4d61c0216b17fbca159b4a65bdd4.tar.gz/md5

@@ -0,0 +1 @@
+f3d5df21fefba26f5568f719fc237bac

deps/checksums/Pkg-33b5e806ffbb4d61c0216b17fbca159b4a65bdd4.tar.gz/sha512

@@ -0,0 +1 @@
+7ff5f580c1f66e5fff875601193918ec9dc40a9d5bfbb9de1900075d1a52cfbf3c702cd7293a0ddcca1f2f23f70e19d5824257256bc7b208cdc952022961d3a2

deps/checksums/Pkg-daf02a458ae6daa402a5dd6683c40d6910325c4e.tar.gz/md5

@@ -265,12 +265,6 @@ DocMeta.setdocmeta!(
     maybe_revise(:(using Base.BinaryPlatforms));
     recursive=true, warn=false,
 )
-DocMeta.setdocmeta!(
-    Pkg.LazilyInitializedFields,
-    :DocTestSetup,
-    maybe_revise(:(using Pkg.LazilyInitializedFields));
-    recursive=true, warn=false,
-)
 
 let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS)
     global const buildroot = i === nothing ? (@__DIR__) : first(match(r, ARGS[i]).captures)

deps/checksums/Pkg-daf02a458ae6daa402a5dd6683c40d6910325c4e.tar.gz/sha512

@@ -83,7 +83,7 @@ be passed instead!). For example:
 
 ```jldoctest; setup = :(using InteractiveUtils)
 julia> macroexpand(@__MODULE__, :(@edit println("")) )
-:(InteractiveUtils.edit(println, (Base.typesof)("")))
+:($(Expr(Symbol("hygienic-scope"), :(edit($(Expr(:escape, :println)), (Base.typesof)($(Expr(:escape, ""))))), InteractiveUtils)))
 ```
 
 The functions `Base.Meta.show_sexpr` and [`dump`](@ref) are used to display S-expr style views

doc/make.jl

@@ -535,20 +535,21 @@ this is an extremely useful tool for debugging macros):
 
 ```julia-repl sayhello2
 julia> ex = macroexpand(Main, :(@sayhello("human")) )
-:(Main.println("Hello, ", "human"))
+:($(Expr(Symbol("hygienic-scope"), :(println("Hello, ", "human")), Main)))
 
 julia> typeof(ex)
 Expr
 ```
 
-We can see that the `"human"` literal has been interpolated into the expression.
+We can see that the macro expression has been replaced by a `hygienic-scope` expression in
+its place and the `"human"` literal has been interpolated into the expression.
 
 There also exists a macro [`@macroexpand`](@ref) that is perhaps a bit more convenient than the `macroexpand` function:
 
 
 ```jldoctest sayhello2
 julia> @macroexpand @sayhello "human"
-:(println("Hello, ", "human"))
+:($(Expr(Symbol("hygienic-scope"), :(println("Hello, ", "human")), Main)))
 ```
 
 ### Hold up: why macros?
@@ -579,7 +580,7 @@ julia> typeof(ex)
 Expr
 
 julia> ex
-:(println("I execute at runtime. The argument is: ", $(Expr(:copyast, :($(QuoteNode(:((1, 2, 3)))))))))
+:($(Expr(Symbol("hygienic-scope"), :(println("I execute at runtime. The argument is: ", $(Expr(:copyast, :($(QuoteNode(:((1, 2, 3))))))))), Main)))
 
 julia> eval(ex)
 I execute at runtime. The argument is: (1, 2, 3)
@@ -719,18 +720,18 @@ of a macro expansion with the aptly named [`@macroexpand`](@ref) macro:
 
 ```julia-repl assert2
 julia> @macroexpand @assert a == b
-:(if Main.a == Main.b
-        Main.nothing
-    else
-        Main.throw(Main.AssertionError("a == b"))
-    end)
+:($(Expr(Symbol("hygienic-scope"), :(if $(Expr(:escape, :(a == b)))
+      nothing
+  else
+      throw(AssertionError("a == b"))
+  end), Base)))
 
 julia> @macroexpand @assert a==b "a should equal b!"
-:(if Main.a == Main.b
-        Main.nothing
-    else
-        Main.throw(Main.AssertionError("a should equal b!"))
-    end)
+:($(Expr(Symbol("hygienic-scope"), :(if $(Expr(:escape, :(a == b)))
+      nothing
+  else
+      throw(AssertionError("a should equal b!"))
+  end), Base)))
 ```
 
 There is yet another case that the actual `@assert` macro handles: what if, in addition to printing
@@ -768,15 +769,22 @@ of expressions inside the macro body.
 ### Hygiene
 
 An issue that arises in more complex macros is that of [hygiene](https://en.wikipedia.org/wiki/Hygienic_macro).
-In short, macros must ensure that the variables they introduce in their returned expressions do
-not accidentally clash with existing variables in the surrounding code they expand into. Conversely,
-the expressions that are passed into a macro as arguments are often *expected* to evaluate in
-the context of the surrounding code, interacting with and modifying the existing variables. Another
-concern arises from the fact that a macro may be called in a different module from where it was
-defined. In this case we need to ensure that all global variables are resolved to the correct
-module. Julia already has a major advantage over languages with textual macro expansion (like
-C) in that it only needs to consider the returned expression. All the other variables (such as
-`msg` in `@assert` above) follow the [normal scoping block behavior](@ref scope-of-variables).
+We saw this a bit earlier with the hygienic-scope expression in the result of macroexpand
+that demarcated where our expression came from as a result of interpolation of the expanded
+macro into the rest of the expression.
+
+This already gives Julia a major advantage over languages with textual macro expansion (like
+C) in that it only needs to consider the returned expression, and it cannot change the
+parsing of the expression surrounding it. Thus, all the other variables (such as `msg` in
+`@assert` above) follow the [normal scoping block behavior](@ref scope-of-variables).
+
+Hygiene markers exists because macros must ensure that the variables they introduce in their
+returned expressions do not accidentally clash with existing variables in the surrounding
+code they expand into. Conversely, the expressions that are passed into a macro as arguments
+are often *expected* to evaluate in the context of the surrounding code, interacting with
+and modifying the existing variables. Another concern arises from the fact that a macro may
+be called in a different module from where it was defined. In this case we need to ensure
+that all global variables are resolved to the correct module.
 
 To demonstrate these issues, let us consider writing a `@time` macro that takes an expression
 as its argument, records the time, evaluates the expression, records the time again, prints the