sem generic proc param types like generic types + static instantiatio…

…n fixes (#24005)

fixes #4228, fixes #4990, fixes #7006, fixes #7008, fixes #8406, fixes
#8551, fixes #11112, fixes #20027, fixes #22647, refs #23854 and #23855
(remaining issue fixed), refs #8545 (works properly now with
`cast[static[bool]]` changed to `cast[bool]`), refs #22342 and #22607
(disabled tests added), succeeds #23194

Parameter and return type nodes in generic procs now undergo the same
`inGenericContext` treatment that nodes in generic type bodies do. This
allows many of the fixes in #22029 and followups to also apply to
generic proc signatures. Like #23983 however this needs some more
compiler fixes, but this time mostly in `sigmatch` and type
instantiations.

1. `tryReadingGenericParam` no longer treats `tyCompositeTypeClass` like
a concrete type anymore, so expressions like `Foo.T` where `Foo` is a
generic type don't look for a parameter of `Foo` in non-generic code
anymore. It also doesn't generate `tyFromExpr` in non-generic code for
any generic LHS. This is to handle a very specific case in `asyncmacro`
which used `FutureVar.astToStr` where `FutureVar` is generic.
2. The `tryResolvingStaticExpr` call when matching `tyFromExpr` in
sigmatch now doesn't consider call nodes in general unresolved, only
nodes with `tyFromExpr` type, which is emitted on unresolved expressions
by increasing `c.inGenericContext`. `c.inGenericContext == 0` is also
now required to attempt instantiating `tyFromExpr`. So matching against
`tyFromExpr` in proc signatures works in general now, but I'm
speculating it depends on constant folding in `semExpr` for statics to
match against it properly.
3. `paramTypesMatch` now doesn't try to change nodes with `tyFromExpr`
type into `tyStatic` type when fitting to a static type, because it
doesn't need to, they'll be handled the same way (this was a workaround
in place of the static type instantiation changes, only one of the
fields in the #22647 test doesn't work with it).
4. `tyStatic` matching now uses `inferStaticParam` instead of just range
type matching, so `Foo[N div 2]` can infer `N` in the same way `array[N
div 2, int]` can. `inferStaticParam` also disabled itself if the
inferred static param type already had a node, but `makeStaticExpr`
generates static types with unresolved nodes, so we only disable it if
it also doesn't have a binding. This might not work very well but the
static type instantiation changes should really lower the amount of
cases where it's encountered.
5. Static types now undergo type instantiation. Previously the branch
for `tyStatic` in `semtypinst` was a no-op, now it acts similarly to
instantiating any other type with the following differences:
- Other types only need instantiation if `containsGenericType` is true,
static types also get instantiated if their value node isn't a literal
node. Ideally any value node that is "already evaluated" should be
ignored, but I'm not sure of a better way to check this, maybe if
`evalConstExpr` emitted a flag. This is purely for optimization though.
- After instantiation, `semConstExpr` is called on the value node if
`not cl.allowMetaTypes` and the type isn't literally a `static` type.
Then the type of the node is set to the base type of the static type to
deal with `semConstExpr` stripping abstract types.
We need to do this because calls like `foo(N)` where `N` is `static int`
and `foo`'s first parameter is just `int` do not generate `tyFromExpr`,
they are fully typed and so `makeStaticExpr` is called on them, giving a
static type with an unresolved node.

compiler/semexprs.nim

@@ -1482,20 +1482,28 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
 
 proc tryReadingGenericParam(c: PContext, n: PNode, i: PIdent, t: PType): PNode =
   case t.kind
-  of tyTypeParamsHolders:
+  of tyGenericInst:
     result = readTypeParameter(c, t, i, n.info)
     if result == c.graph.emptyNode:
-      result = semGenericStmt(c, n)
-      result.typ = makeTypeFromExpr(c, result.copyTree)
+      if c.inGenericContext > 0:
+        result = semGenericStmt(c, n)
+        result.typ = makeTypeFromExpr(c, result.copyTree)
+      else:
+        result = nil
   of tyUserTypeClasses:
     if t.isResolvedUserTypeClass:
       result = readTypeParameter(c, t, i, n.info)
+    elif c.inGenericContext > 0:
+      result = semGenericStmt(c, n)
+      result.typ = makeTypeFromExpr(c, copyTree(result))
     else:
+      result = nil
+  elif t.containsGenericType:
+    if c.inGenericContext > 0:
       result = semGenericStmt(c, n)
       result.typ = makeTypeFromExpr(c, copyTree(result))
-  of tyFromExpr, tyGenericParam, tyAnything:
-    result = semGenericStmt(c, n)
-    result.typ = makeTypeFromExpr(c, copyTree(result))
+    else:
+      result = nil
   else:
     result = nil
 

compiler/semtypes.nim

@@ -1312,6 +1312,9 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
   result = newProcType(c, n.info, prev)
   var check = initIntSet()
   var counter = 0
+  template isCurrentlyGeneric: bool =
+    # genericParams might update as implicit generic params are added
+    genericParams != nil and genericParams.len > 0
 
   for i in 1..<n.len:
     var a = n[i]
@@ -1332,7 +1335,10 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
       hasDefault = a[^1].kind != nkEmpty
 
     if hasType:
+      let isGeneric = isCurrentlyGeneric()
+      inc c.inGenericContext, ord(isGeneric)
       typ = semParamType(c, a[^2], constraint)
+      dec c.inGenericContext, ord(isGeneric)
       # TODO: Disallow typed/untyped in procs in the compiler/stdlib
       if kind in {skProc, skFunc} and (typ.kind == tyTyped or typ.kind == tyUntyped):
         if not isMagic(getCurrOwner(c)):
@@ -1353,7 +1359,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
         message(c.config, a.info, warnImplicitDefaultValue, msg)
       block determineType:
         var defTyp = typ
-        if genericParams != nil and genericParams.len > 0:
+        if isCurrentlyGeneric():
           defTyp = nil
           def = semGenericStmt(c, def)
           if hasUnresolvedArgs(c, def):
@@ -1432,11 +1438,12 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
       onDef(a[j].info, arg)
       a[j] = newSymNode(arg)
 
-  var r: PType =
-    if n[0].kind != nkEmpty:
-      semTypeNode(c, n[0], nil)
-    else:
-      nil
+  var r: PType = nil
+  if n[0].kind != nkEmpty:
+    let isGeneric = isCurrentlyGeneric()
+    inc c.inGenericContext, ord(isGeneric)
+    r = semTypeNode(c, n[0], nil)
+    dec c.inGenericContext, ord(isGeneric)
 
   if r != nil and kind in {skMacro, skTemplate} and r.kind == tyTyped:
     # XXX: To implement the proposed change in the warning, just
@@ -1489,7 +1496,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
           result.flags.excl tfHasMeta
       result.n.typ = r
 
-  if genericParams != nil and genericParams.len > 0:
+  if isCurrentlyGeneric():
     for n in genericParams:
       if {sfUsed, sfAnon} * n.sym.flags == {}:
         result.flags.incl tfUnresolved

compiler/semtypinst.nim

@@ -678,8 +678,31 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
     elif t.elementType.kind != tyNone:
       result = makeTypeDesc(cl.c, replaceTypeVarsT(cl, t.elementType))
 
-  of tyUserTypeClass, tyStatic:
+  of tyUserTypeClass:
     result = t
+
+  of tyStatic:
+    if cl.c.matchedConcept != nil:
+      # allow concepts to not instantiate statics for now
+      # they can't always infer them
+      return
+    if not containsGenericType(t) and (t.n == nil or t.n.kind in nkLiterals):
+      # no need to instantiate
+      return
+    bailout()
+    result = instCopyType(cl, t)
+    cl.localCache[t.itemId] = result
+    for i in FirstGenericParamAt..<result.kidsLen:
+      var r = result[i]
+      if r != nil:
+        r = replaceTypeVarsT(cl, r)
+        result[i] = r
+        propagateToOwner(result, r)
+    result.n = replaceTypeVarsN(cl, result.n)
+    if not cl.allowMetaTypes and result.n != nil and
+        result.base.kind != tyNone:
+      result.n = cl.c.semConstExpr(cl.c, result.n)
+      result.n.typ = result.base
 
   of tyGenericInst, tyUserTypeClassInst:
     bailout()

compiler/sigmatch.nim

@@ -885,6 +885,7 @@ proc maybeSkipDistinct(m: TCandidate; t: PType, callee: PSym): PType =
 
 proc tryResolvingStaticExpr(c: var TCandidate, n: PNode,
                             allowUnresolved = false,
+                            allowCalls = false,
                             expectedType: PType = nil): PNode =
   # Consider this example:
   #   type Value[N: static[int]] = object
@@ -894,7 +895,7 @@ proc tryResolvingStaticExpr(c: var TCandidate, n: PNode,
   # This proc is used to evaluate such static expressions.
   let instantiated = replaceTypesInBody(c.c, c.bindings, n, nil,
                                         allowMetaTypes = allowUnresolved)
-  if instantiated.kind in nkCallKinds:
+  if not allowCalls and instantiated.kind in nkCallKinds:
     return nil
   result = c.c.semExpr(c.c, instantiated)
 
@@ -966,7 +967,8 @@ proc inferStaticParam*(c: var TCandidate, lhs: PNode, rhs: BiggestInt): bool =
 
     else: discard
 
-  elif lhs.kind == nkSym and lhs.typ.kind == tyStatic and lhs.typ.n == nil:
+  elif lhs.kind == nkSym and lhs.typ.kind == tyStatic and
+      (lhs.typ.n == nil or idTableGet(c.bindings, lhs.typ) == nil):
     var inferred = newTypeS(tyStatic, c.c, lhs.typ.elementType)
     inferred.n = newIntNode(nkIntLit, rhs)
     put(c, lhs.typ, inferred)
@@ -1877,7 +1879,11 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
         elif f.base.kind notin {tyNone, tyGenericParam}:
           result = typeRel(c, f.base, a, flags)
           if result != isNone and f.n != nil:
-            if not exprStructuralEquivalent(f.n, aOrig.n):
+            var r = tryResolvingStaticExpr(c, f.n)
+            if r == nil: r = f.n
+            if not exprStructuralEquivalent(r, aOrig.n) and
+                not (aOrig.n.kind == nkIntLit and
+                  inferStaticParam(c, r, aOrig.n.intVal)):
               result = isNone
         elif f.base.kind == tyGenericParam:
           # Handling things like `type A[T; Y: static T] = object`
@@ -1963,23 +1969,29 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
   of tyFromExpr:
     # fix the expression, so it contains the already instantiated types
     if f.n == nil or f.n.kind == nkEmpty: return isGeneric
-    let reevaluated = tryResolvingStaticExpr(c, f.n)
-    if reevaluated == nil:
+    if c.c.inGenericContext > 0:
+      # need to delay until instantiation
+      # also prevent infinite recursion below
+      return isNone
+    inc c.c.inGenericContext # to generate tyFromExpr again if unresolved
+    let reevaluated = tryResolvingStaticExpr(c, f.n, allowCalls = true).typ
+    dec c.c.inGenericContext
+    case reevaluated.kind
+    of tyFromExpr:
+      # not resolved
       result = isNone
-      return
-    case reevaluated.typ.kind
     of tyTypeDesc:
-      result = typeRel(c, a, reevaluated.typ.base, flags)
+      result = typeRel(c, a, reevaluated.base, flags)
     of tyStatic:
-      result = typeRel(c, a, reevaluated.typ.base, flags)
-      if result != isNone and reevaluated.typ.n != nil:
-        if not exprStructuralEquivalent(aOrig.n, reevaluated.typ.n):
+      result = typeRel(c, a, reevaluated.base, flags)
+      if result != isNone and reevaluated.n != nil:
+        if not exprStructuralEquivalent(aOrig.n, reevaluated.n):
           result = isNone
     else:
       # bug #14136: other types are just like 'tyStatic' here:
-      result = typeRel(c, a, reevaluated.typ, flags)
-      if result != isNone and reevaluated.typ.n != nil:
-        if not exprStructuralEquivalent(aOrig.n, reevaluated.typ.n):
+      result = typeRel(c, a, reevaluated, flags)
+      if result != isNone and reevaluated.n != nil:
+        if not exprStructuralEquivalent(aOrig.n, reevaluated.n):
           result = isNone
   of tyNone:
     if a.kind == tyNone: result = isEqual
@@ -2188,7 +2200,11 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
         a = typ
       else:
         if m.callee.kind == tyGenericBody:
-          if f.kind == tyStatic and typeRel(m, f.base, a) != isNone:
+          # we can't use `makeStaticExpr` if `arg` has a generic type
+          # because it generates `tyStatic`, which semtypinst doesn't touch
+          # not sure if checking for `tyFromExpr` is enough
+          if f.kind == tyStatic and typeRel(m, f.base, a) != isNone and
+              a.kind != tyFromExpr:
             result = makeStaticExpr(m.c, arg)
             result.typ.flags.incl tfUnresolved
             result.typ.n = arg

tests/generics/t23854.nim

@@ -63,7 +63,8 @@ template getBits[bits: static int](x: ptr UncheckedArray[BigInt[bits]]): int = b
 
 proc main() =
   let ctx = ECFFT_Descriptor[EC_ShortW_Aff[Fp[BLS12_381]]].new()
-  when false: echo getBits(ctx.rootsOfUnity2) # doesn't work yet?
+  doAssert getBits(ctx.rootsOfUnity1) == 255
+  doAssert getBits(ctx.rootsOfUnity2) == 255
   doAssert ctx.rootsOfUnity1[0].limbs.len == wordsRequired(255)
   doAssert ctx.rootsOfUnity2[0].limbs.len == wordsRequired(255)
 

tests/generics/t23855.nim

@@ -55,7 +55,7 @@ template getBits[bits: static int](x: ptr UncheckedArray[BigInt[bits]]): int = b
 
 proc main() =
   let ctx = ECFFT_Descriptor[EC_ShortW_Aff[Fp[BLS12_381]]].new()
-  when false: echo getBits(ctx.rootsOfUnity) # doesn't work yet?
+  doAssert getBits(ctx.rootsOfUnity) == 255
   doAssert ctx.rootsOfUnity[0].limbs.len == wordsRequired(255)
 
 main()

tests/generics/tuninstantiatedgenericcalls.nim

@@ -239,3 +239,65 @@ block: # version of #23432 with `typed`, don't delay instantiation
   proc f(x: X) = discard
   var v: Future[void].Raising([ValueError])
   f(v)
+
+block: # issue #22647
+  proc c0(n: static int): int = 8
+  proc c1(n: static int): int = n div 2
+  proc c2(n: static int): int = n * 2
+  proc c3(n: static int, n2: int): int = n * n2
+  proc `**`(n: static int, n2: int): int = n * n2
+  proc c4(n: int, n2: int): int = n * n2
+
+  type
+    a[N: static int] = object
+      f0 : array[N, int]
+
+    b[N: static int] = object
+      f0 : a[c0(N)]  # does not work
+      f1 : a[c1(N)]  # does not work
+      f2 : a[c2(N)]  # does not work
+      f3 : a[N * 2]  # does not work
+      f4 : a[N]      # works
+      f5: a[c3(N, 2)]
+      f6: a[N ** 2]
+      f7: a[2 * N]
+      f8: a[c4(N, 2)]
+
+  proc p[N: static int](x : a[N]) = discard x.f0[0]
+  template check(x, s: untyped) =
+    p(x)
+    doAssert x is a[s]
+    doAssert x.N == s
+    doAssert typeof(x).N == s
+    doAssert x.f0 == default(array[s, int])
+    doAssert x.f0.len == s
+    proc p2[N: static int](y : a[N]) {.gensym.} =
+      doAssert y is a[s]
+      doAssert y.N == s
+      doAssert typeof(y).N == s
+      doAssert y.f0 == default(array[s, int])
+      doAssert y.f0.len == s
+    p2(x)
+    proc p3(z: typeof(x)) {.gensym.} = discard
+    p3(default(a[s]))
+  proc p[N: static int](x : b[N]) =
+    x.f0.check(8)
+    x.f1.check(2)
+    x.f2.check(8)
+    x.f3.check(8)
+    x.f4.check(4)
+    x.f5.check(8)
+    x.f6.check(8)
+    x.f7.check(8)
+    x.f8.check(8)
+
+  var x: b[4]
+  x.p()
+
+when false: # issue #22342, type section version of #22607
+  type GenAlias[isInt: static bool] = (
+    when isInt:
+      int
+    else:
+      float
+  )