provides ways to circumvent unexpected type bindings

This PR fixes issues involved with the syntaxes to specify expression types.

Currently, `@capture` and `@match` can't distinguish between their syntax
to specify a `:head` of `Expr` and a variable name with underscores.
For example, in the example below, `@capture` recognizes `global_string`
as the syntax to specify `Expr`'s head (i.e. `:string`), not as a simple
variable name:
```julia
julia> ex = :(global_string = 10);
julia> @capture(ex, global_string = n_) # tries to match `Expr(:string, ...) = n_` and bound the matched lhs into a variable `global` and the matched rhs into a variable `n`.
false
```
Since an expression can really have an arbitrary head, `@capture` macro
can't really distinguish them (while we can do some assertion when the
syntax to specify atomic expression type, though).

This PR implements new macros `@capture_notb` and `@match_notb`, which
ignore all the expression type matching syntaxes and provide the ways to
circumvent the issue described above:
```julia
julia> ex = :(global_string = 10)
julia> @capture_notb(ex, global_string = n_) # tries to match `global_string = n_` pattern and bound the matched rhs into a variable `n`.
true
```

These changes aren't breaking but they're somewhat a big change, I'd
like to minor version bump.

Project.toml

@@ -1,6 +1,6 @@
 name = "MacroTools"
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.6"
+version = "0.6.0"
 
 [deps]
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"

docs/Manifest.toml

@@ -1,42 +1,66 @@
 # This file is machine-generated - editing it directly is not advised
 
+[[ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+
+[[Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[Dates]]
 deps = ["Printf"]
 uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
 
-[[Distributed]]
-deps = ["Random", "Serialization", "Sockets"]
-uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
-
 [[DocStringExtensions]]
 deps = ["LibGit2", "Markdown", "Pkg", "Test"]
-git-tree-sha1 = "88bb0edb352b16608036faadcc071adda068582a"
+git-tree-sha1 = "9d4f64f79012636741cf01133158a54b24924c32"
 uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
-version = "0.8.1"
+version = "0.8.4"
 
 [[Documenter]]
-deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
-git-tree-sha1 = "646ebc3db49889ffeb4c36f89e5d82c6a26295ff"
+deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
+git-tree-sha1 = "3ebb967819b284dc1e3c0422229b58a40a255649"
 uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
-version = "0.24.7"
+version = "0.26.3"
+
+[[Downloads]]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+
+[[IOCapture]]
+deps = ["Logging"]
+git-tree-sha1 = "377252859f740c217b936cebcd918a44f9b53b59"
+uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
+version = "0.1.1"
 
 [[InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[JSON]]
 deps = ["Dates", "Mmap", "Parsers", "Unicode"]
-git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
+git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "0.21.0"
+version = "0.21.1"
+
+[[LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+
+[[LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
 
 [[LibGit2]]
-deps = ["Printf"]
+deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
+[[LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+
 [[Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
@@ -47,31 +71,41 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 deps = ["Markdown", "Random"]
 path = ".."
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.5"
+version = "0.5.6"
 
 [[Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 
+[[MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+
+[[NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+
 [[Parsers]]
-deps = ["Dates", "Test"]
-git-tree-sha1 = "0c16b3179190d3046c073440d94172cfc3bb0553"
+deps = ["Dates"]
+git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "0.3.12"
+version = "1.1.0"
 
 [[Pkg]]
-deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Printf]]
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
 [[REPL]]
-deps = ["InteractiveUtils", "Markdown", "Sockets"]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
 [[Random]]
@@ -87,8 +121,16 @@ uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 [[Sockets]]
 uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 
+[[TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[[Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+
 [[Test]]
-deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [[UUIDs]]
@@ -97,3 +139,15 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [[Unicode]]
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+
+[[nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+
+[[p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"

docs/src/pattern-matching.md

@@ -68,7 +68,7 @@ Highlander there can only be one (per expression).
 
 ### Matching on expression type
 
-`@capture` can match expressions by their type, which is either the `head` of `Expr`
+`@capture` can match expressions by their type, which is either the `:head` of `Expr`
 objects or the `typeof` atomic stuff like `Symbol`s and `Int`s. For example:
 
 ```julia
@@ -91,6 +91,26 @@ Another common use case is to catch symbol literals, e.g.
 
 which will match e.g. `struct Foo ...` but not `struct Foo{V} ...`
 
+!!! tip "Matching without expression type"
+    [Matching on expression type](@ref) can be useful, but the problem is that `@capture` can't distinguish between
+    its syntax to specify a `:head` of `Expr` and a variable name with underscores:
+
+    For example, in the example below, `@capture` recognizes `global_string` as the syntax to specify `Expr`'s head (i.e. `:string`),
+    not as a simple variable name:
+    ```julia
+    julia> ex = :(global_string = 10);
+    julia> @capture(ex, global_string = n_) # tries to match `Expr(:string, ...) = n_` and bound the matched lhs into a variable `global` and the matched rhs into a variable `n`.
+    false
+    ```
+
+    To circumvent this issue, MacroTools exports `@capture_notb`, which skips all the expression type matching syntaxes:
+    ```julia
+    julia> ex = :(global_string = 10)
+    julia> @capture_notb(ex, global_string = n_) # tries to match `global_string = n_` pattern and bound the matched rhs into a variable `n`.
+    true
+    ```
+
+
 ### Unions
 
 `@capture` can also try to match the expression against one pattern or another,

src/MacroTools.jl

@@ -1,7 +1,7 @@
 module MacroTools
 
 using Markdown, Random
-export @match, @capture
+export @match, @capture, @match_notb, @capture_notb
 
 include("match/match.jl")
 include("match/types.jl")

src/match/macro.jl

@@ -1,17 +1,17 @@
-function allbindings(pat, bs)
+function allbindings(context, pat, bs)
   if isa(pat, QuoteNode)
-    return allbindings(pat.value, bs)
+    return allbindings(context, pat.value, bs)
   end
   return isbinding(pat) || (isslurp(pat) && pat ≠ :__) ? push!(bs, bname(pat)) :
   isa(pat, TypeBind) ? push!(bs, pat.name) :
-  isa(pat, OrBind) ? (allbindings(pat.pat1, bs); allbindings(pat.pat2, bs)) :
-  istb(pat) ? push!(bs, tbname(pat)) :
+  isa(pat, OrBind) ? (allbindings(context, pat.pat1, bs); allbindings(context, pat.pat2, bs)) :
+  istb(context, pat) ? push!(bs, tbname(pat)) :
   isexpr(pat, :$) ? bs :
-  isa(pat, Expr) ? map(pat -> allbindings(pat, bs), [pat.head, pat.args...]) :
+  isa(pat, Expr) ? map(pat -> allbindings(context, pat, bs), [pat.head, pat.args...]) :
   bs
 end
 
-allbindings(pat) = (bs = Any[]; allbindings(pat, bs); bs)
+allbindings(context, pat) = (bs = Any[]; allbindings(context, pat, bs); bs)
 
 function bindinglet(bs, body)
   ex = :(let $(esc(:env)) = env, $((:($(esc(b)) = get(env, $(Expr(:quote, b)), nothing)) for b in bs)...)
@@ -20,9 +20,9 @@ function bindinglet(bs, body)
   return ex
 end
 
-function makeclause(pat, yes, els = nothing)
-  bs = allbindings(pat)
-  pat = subtb(subor(pat))
+function makeclause(context, pat, yes, els = nothing)
+  bs = allbindings(context, pat)
+  pat = subtb(context, subor(pat))
   quote
     env = trymatch($(Expr(:quote, pat)), ex)
     if env != nothing
@@ -46,28 +46,30 @@ function clauses(ex)
   return clauses
 end
 
-macro match(ex, lines)
+macro match(ex, lines) _match(__module__, ex, lines) end
+macro match_notb(ex, lines) _match(nothing, ex, lines) end
+function _match(context, ex, lines)
   @assert isexpr(lines, :block)
   result = quote
     ex = $(esc(ex))
   end
 
-  @static if VERSION < v"0.7.0-"
-    body = foldr((clause, body) -> makeclause(clause..., body),
-                 nothing, clauses(lines))
-  else
-    body = foldr((clause, body) -> makeclause(clause..., body),
+  body = @static VERSION < v"0.7.0-" ?
+         foldr((clause, body) -> makeclause(context, clause..., body),
+                 nothing, clauses(lines)) :
+         foldr((clause, body) -> makeclause(context, clause..., body),
                  clauses(lines); init=nothing)
-  end
 
   push!(result.args, body)
   return result
 end
 
-macro capture(ex, pat)
-  bs = allbindings(pat)
-  pat = subtb(subor(pat))
-  quote
+macro capture(ex, pat) _capture(__module__, ex, pat) end
+macro capture_notb(ex, pat) _capture(nothing, ex, pat) end
+function _capture(context, ex, pat)
+  bs = allbindings(context, pat)
+  pat = subtb(context, subor(pat))
+  return quote
     $([:($(esc(b)) = nothing) for b in bs]...)
     env = trymatch($(esc(Expr(:quote, pat))), $(esc(ex)))
     if env == nothing

src/match/types.jl

@@ -3,18 +3,37 @@ struct TypeBind
   ts::Set{Any}
 end
 
-istb(s) = false
-istb(s::Symbol) = !(endswith(string(s), "_") ||
-                    endswith(string(s), "_str")) &&
-                    occursin("_", string(s))
+istb(::Nothing, _) = false
+istb(::Module, _) = false
+function istb(context::Module, s::Symbol)
+  (endswith(string(s), "_") || endswith(string(s), "_str")) && return false
+  occursin("_", string(s)) || return false
+  ts = map(Symbol, split(string(s), "_"))
+  popfirst!(ts)
+  return all(s->istype(context, s), ts)
+end
+
+function istype(context::Module, s::Symbol)
+  if string(s)[1] in 'A':'Z'
+    if isdefined(context, s) && isa(getfield(context, s), Type)
+      return true
+    else
+      throw(ArgumentError("""
+      the syntax to specify expression type syntax is used, but the given type isn't defined:
+      if you want to ignore the syntaxes to specify expression type, use `@capture_notb` or `@match_notb` instead
+      """))
+    end
+  end
+  return true
+end
 
 tbname(s::Symbol) = Symbol(split(string(s), "_")[1])
 tbname(s::TypeBind) = s.name
 
 totype(s::Symbol) = string(s)[1] in 'A':'Z' ? s : Expr(:quote, s)
 
-function tbnew(s::Symbol)
-  istb(s) || return s
+function tbnew(context::Module, s::Symbol)
+  istb(context, s) || return s
   ts = map(Symbol, split(string(s), "_"))
   name = popfirst!(ts)
   ts = map(totype, ts)
@@ -24,6 +43,7 @@ end
 match_inner(b::TypeBind, ex, env) =
   isexpr(ex, b.ts...) ? (env[tbname(b)] = ex; env) : @nomatch(b, ex)
 
-subtb(s) = s
-subtb(s::Symbol) = tbnew(s)
-subtb(s::Expr) = isexpr(s, :line) ? s : Expr(subtb(s.head), map(subtb, s.args)...)
+subtb(::Nothing, s) = s
+subtb(context::Module, s) = s
+subtb(context::Module, s::Symbol) = tbnew(context, s)
+subtb(context::Module, s::Expr) = isexpr(s, :line) ? s : Expr(subtb(context, s.head), map(s->subtb(context, s), s.args)...)

test/match.jl

@@ -96,3 +96,16 @@ let
   @capture(ex, $f(args__))
   @test args == [:a, :b]
 end
+
+# hint to use `@capture_notb`
+let
+  ex = :(const GLOBAL_STRING = 10)
+  @test_throws ArgumentError macroexpand(@__MODULE__, :(@capture($ex, const GLOBAL_STRING = x_)))
+end
+
+# if we don't want to make `global_string` type bind syntax, we need to use `@capture_notb`
+let
+  ex = :(const global_string = 10)
+  @capture_notb(ex, const global_string = x_)
+  @test x === 10
+end