Add scala2-library-cc-tasty project (#18967)

We can override the sources of the standard library with cc annotation
in this project and generate a TASTy version of the standard library.

Usage:
```
sbt> set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty
sbt> scala3-bootstrapped/testCompilation
sbt> scala3-bootstrapped/test
sbt> scala3-bootstrapped/run ...
```

```
sbt> scala2-library-cc/run diff
sbt> scala2-library-cc/run diff scala/Option.scala
sbt> scala2-library-cc/run diff scala/collection/immutable
```

### Current limitations

#### Empty self types in library
We need to override `Function1` and `PartialFunction` to remove their
empty self type. These cause some problem with capture checking. We need
to fix these cases. I will open an issue once we have a way to reproduce
it with this PR.

#### Cannot use the TASTy jar yet
There is a bug when we try to class load the captured checked library
from TASTy. These are due to some cyclic reference while unpickling the
TASTy.

<del>The current theory is that this due to the CC annotations in the
top-level classes.
If this is the case, we can fix this issue by adding these language
features as TASTy attributes (see #19033).</del>
This was wrong or only part of the problems. The current issue can be
reproduced with #19115.

Author: nicolasstucki

.github/workflows/ci.yaml

@@ -145,7 +145,10 @@ jobs:
           ./project/scripts/sbt ";sjsSandbox/run ;sjsSandbox/test ;sjsJUnitTests/test ;set sjsJUnitTests/scalaJSLinkerConfig ~= switchToESModules ;sjsJUnitTests/test ;sjsCompilerTests/test"
 
       - name: Test with Scala 2 library TASTy (fast)
-        run: ./project/scripts/sbt ";set ThisBuild/Build.useScala2LibraryTasty := true ;scala3-bootstrapped/testCompilation i5; scala3-bootstrapped/testCompilation tests/run/typelevel-peano.scala; scala3-bootstrapped/testOnly dotty.tools.backend.jvm.DottyBytecodeTests" # only test a subset of test to avoid doubling the CI execution time
+        run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryTasty ;scala3-bootstrapped/testCompilation i5; scala3-bootstrapped/testCompilation tests/run/typelevel-peano.scala; scala3-bootstrapped/testOnly dotty.tools.backend.jvm.DottyBytecodeTests" # only test a subset of test to avoid doubling the CI execution time
+
+      - name: Test with Scala 2 library with CC TASTy (fast)
+        run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty ;scala2-library-tasty/compile" # TODO test all the test configurations in non-CC library (currently disabled due to bug while loading the library)
 
   test_scala2_library_tasty:
     runs-on: [self-hosted, Linux]
@@ -186,7 +189,11 @@ jobs:
         run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
 
       - name: Test with Scala 2 library TASTy
-        run: ./project/scripts/sbt ";set ThisBuild/Build.useScala2LibraryTasty := true ;scala3-bootstrapped/test"
+        run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryTasty ;scala3-bootstrapped/test"
+
+      # TODO test all the test configurations in non-CC library (currently disabled due to bug while loading the library)
+      # - name: Test with Scala 2 library with CC TASTy
+      #   run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty ;scala3-bootstrapped/test"
 
 
   test_windows_fast:

build.sbt

@@ -17,6 +17,8 @@ val `scala3-bench-micro` = Build.`scala3-bench-micro`
 val `scala2-library-bootstrapped` = Build.`scala2-library-bootstrapped`
 val `scala2-library-tasty` = Build.`scala2-library-tasty`
 val `scala2-library-tasty-tests` = Build.`scala2-library-tasty-tests`
+val `scala2-library-cc` = Build.`scala2-library-cc`
+val `scala2-library-cc-tasty` = Build.`scala2-library-cc-tasty`
 val `tasty-core` = Build.`tasty-core`
 val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`
 val `tasty-core-scala2` = Build.`tasty-core-scala2`

compiler/test/dotty/Properties.scala

@@ -87,6 +87,9 @@ object Properties {
 
   /** If we are using the scala-library TASTy jar */
   def usingScalaLibraryTasty: Boolean = scalaLibraryTasty.isDefined
+  /** If we are using the scala-library TASTy jar */
+
+  def usingScalaLibraryCCTasty: Boolean = scalaLibraryTasty.exists(_.contains("scala2-library-cc-tasty"))
 
   /** scala-asm jar */
   def scalaAsm: String = sys.props("dotty.tests.classes.scalaAsm")

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -45,9 +45,9 @@ class CompilationTests {
       compileFilesInDir("tests/pos", defaultOptions.and("-Ysafe-init", "-Ylegacy-lazy-vals", "-Ycheck-constraint-deps"), FileFilter.include(TestSources.posLazyValsAllowlist)),
       compileDir("tests/pos-special/java-param-names", defaultOptions.withJavacOnlyOptions("-parameters")),
     ) ::: (
-      // FIXME: This fails due to a bug involving self types and capture checking
-      if Properties.usingScalaLibraryTasty then Nil
-      else List(compileDir("tests/pos-special/stdlib", allowDeepSubtypes))
+      // TODO create a folder for capture checking tests with the stdlib, or use tests/pos-custom-args/captures under this mode?
+      if Properties.usingScalaLibraryCCTasty then List(compileDir("tests/pos-special/stdlib", allowDeepSubtypes))
+      else Nil
     )
 
     if scala.util.Properties.isJavaAtLeast("16") then

project/Build.scala

@@ -173,11 +173,23 @@ object Build {
   // Run tests with filter through vulpix test suite
   val testCompilation = inputKey[Unit]("runs integration test with the supplied filter")
 
+  sealed trait Scala2Library
+  // Use Scala 2 compiled library JAR
+  object Scala2LibraryJar extends Scala2Library
   // Use the TASTy jar from `scala2-library-tasty` in the classpath
   // This only works with `scala3-bootstrapped/scalac` and tests in `scala3-bootstrapped`
   //
-  // Enable in SBT with: `set ThisBuild/Build.useScala2LibraryTasty := true`
-  val useScala2LibraryTasty = settingKey[Boolean]("Use the TASTy jar from `scala2-library-tasty` in the classpath")
+  object Scala2LibraryTasty extends Scala2Library
+  // Use the TASTy jar from `scala2-library-cc-tasty` in the classpath
+  // This only works with `scala3-bootstrapped/scalac` and tests in `scala3-bootstrapped`
+  //
+  object Scala2LibraryCCTasty extends Scala2Library
+
+  // Set in SBT with:
+  //   - `set ThisBuild/Build.scala2Library := Build.Scala2LibraryJar` (default)
+  //   - `set ThisBuild/Build.scala2Library := Build.Scala2LibraryTasty`
+  //   - `set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty`
+  val scala2Library = settingKey[Scala2Library]("Choose which version of the Scala 2 library should be used")
 
   // Used to compile files similar to ./bin/scalac script
   val scalac = inputKey[Unit]("run the compiler using the correct classpath, or the user supplied classpath")
@@ -225,7 +237,7 @@ object Build {
 
     outputStrategy := Some(StdoutOutput),
 
-    useScala2LibraryTasty := false,
+    scala2Library := Scala2LibraryJar,
 
     // enable verbose exception messages for JUnit
     (Test / testOptions) += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "-s"),
@@ -645,12 +657,18 @@ object Build {
         val externalDeps = externalCompilerClasspathTask.value
         val jars = packageAll.value
 
-        val scala2LibraryTasty = jars.get("scala2-library-tasty") match {
-          case Some(scala2LibraryTastyJar) if useScala2LibraryTasty.value =>
-            Seq("-Ddotty.tests.tasties.scalaLibrary=" + scala2LibraryTastyJar)
-          case _ =>
-            if (useScala2LibraryTasty.value) log.warn("useScala2LibraryTasty is ignored on non-bootstrapped compiler")
-            Seq.empty
+        def libraryPathProperty(jarName: String): Seq[String] =
+          jars.get(jarName) match {
+            case Some(jar) =>
+              Seq(s"-Ddotty.tests.tasties.scalaLibrary=$jar")
+            case None =>
+              log.warn("Scala 2 library TASTy is ignored on non-bootstrapped compiler")
+              Seq.empty
+          }
+        val scala2LibraryTasty = scala2Library.value match {
+          case Scala2LibraryJar => Seq.empty
+          case Scala2LibraryTasty => libraryPathProperty("scala2-library-tasty")
+          case Scala2LibraryCCTasty => libraryPathProperty("scala2-library-cc-tasty")
         }
 
         scala2LibraryTasty ++ Seq(
@@ -764,14 +782,24 @@ object Build {
           else if (debugFromTasty) "dotty.tools.dotc.fromtasty.Debug"
           else "dotty.tools.dotc.Main"
 
-        var extraClasspath =
-          scalaLibTastyOpt match {
-            case Some(scalaLibTasty) if useScala2LibraryTasty.value =>
-              Seq(scalaLibTasty, scalaLib, dottyLib)
-            case _ =>
-              if (useScala2LibraryTasty.value) log.warn("useScala2LibraryTasty is ignored on non-bootstrapped compiler")
-              Seq(scalaLib, dottyLib)
-          }
+        val scala2LibraryTasty = scala2Library.value match {
+          case Scala2LibraryJar => Seq.empty
+          case Scala2LibraryTasty =>
+            jars.get("scala2-library-tasty") match {
+              case Some(jar) => Seq(jar)
+              case None =>
+                log.warn("Scala2LibraryTasty is ignored on non-bootstrapped compiler")
+                Seq.empty
+            }
+          case Scala2LibraryCCTasty =>
+            jars.get("scala2-library-cc-tasty") match {
+              case Some(jar) => Seq(jar)
+              case None =>
+                log.warn("Scala2LibraryCCTasty is ignored on non-bootstrapped compiler")
+                Seq.empty
+            }
+        }
+        var extraClasspath = scala2LibraryTasty ++ Seq(scalaLib, dottyLib)
 
         if (decompile && !args.contains("-classpath"))
           extraClasspath ++= Seq(".")
@@ -882,6 +910,7 @@ object Build {
         "scala3-tasty-inspector"  -> (LocalProject("scala3-tasty-inspector") / Compile / packageBin).value.getAbsolutePath,
         "tasty-core"     -> (LocalProject("tasty-core-bootstrapped") / Compile / packageBin).value.getAbsolutePath,
         "scala2-library-tasty" -> (LocalProject("scala2-library-tasty") / Compile / packageBin).value.getAbsolutePath,
+        "scala2-library-cc-tasty" -> (LocalProject("scala2-library-cc-tasty") / Compile / packageBin).value.getAbsolutePath,
       )
     },
 
@@ -1010,8 +1039,24 @@ object Build {
     withCommonSettings(Bootstrapped).
     dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
     settings(commonBootstrappedSettings).
+    settings(scala2LibraryBootstrappedSettings).
+    settings(moduleName := "scala2-library")
+
+  /** Scala 2 library compiled by dotty using the latest published sources of the library.
+   *
+   *  This version of the library is not (yet) TASTy/binary compatible with the Scala 2 compiled library.
+   */
+  lazy val `scala2-library-cc` = project.in(file("scala2-library-cc")).
+    withCommonSettings(Bootstrapped).
+    dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
+    settings(commonBootstrappedSettings).
+    settings(scala2LibraryBootstrappedSettings).
     settings(
-      moduleName := "scala2-library",
+      moduleName := "scala2-library-cc",
+      scalacOptions += "-Ycheck:all",
+    )
+
+  lazy val scala2LibraryBootstrappedSettings = Seq(
       javaOptions := (`scala3-compiler-bootstrapped` / javaOptions).value,
       Compile / scalacOptions ++= {
         Seq("-sourcepath", ((Compile/sourceManaged).value / "scala-library-src").toString)
@@ -1096,13 +1141,13 @@ object Build {
              |   - final val MinorVersion = $minorVersion
              |   - final val ExperimentalVersion = 0
              | * Clean everything to generate a compiler with those new TASTy versions
-             | * Run scala2-library-bootstrapped/tastyMiMaReportIssues
+             | * Run ${name.value}/tastyMiMaReportIssues
              |""".stripMargin)
 
       }).value,
       Compile / exportJars := true,
       artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
-        "scala2-library-" + dottyVersion + "." + artifact.extension
+        moduleName.value + "-" + dottyVersion + "." + artifact.extension
       },
       run := {
         val log = streams.value.log
@@ -1174,7 +1219,7 @@ object Build {
                 |""".stripMargin)
         }
       }
-    )
+  )
 
   /** Packages the TASTy files of `scala2-library-bootstrapped` in a jar */
   lazy val `scala2-library-tasty` = project.in(file("scala2-library-tasty")).
@@ -1187,6 +1232,17 @@ object Build {
       },
     )
 
+  /** Packages the TASTy files of `scala2-library-cc` in a jar */
+  lazy val `scala2-library-cc-tasty` = project.in(file("scala2-library-cc-tasty")).
+    withCommonSettings(Bootstrapped).
+    settings(
+      exportJars := true,
+      Compile / packageBin / mappings := {
+        (`scala2-library-cc` / Compile / packageBin / mappings).value
+          .filter(_._2.endsWith(".tasty"))
+      },
+    )
+
   /** Test the tasty generated by `scala2-library-bootstrapped`
    *
    *  The sources in src are compiled using TASTy from scala2-library-tasty but then run

scala2-library-cc/src/scala/Function1.scala

@@ -0,0 +1,90 @@
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+// GENERATED CODE: DO NOT EDIT. See scala.Function0 for timestamp.
+
+package scala
+
+
+object Function1 {
+
+  implicit final class UnliftOps[A, B] private[Function1](private val f: A => Option[B]) extends AnyVal {
+    /** Converts an optional function to a partial function.
+      *
+      * @example Unlike [[Function.unlift]], this [[UnliftOps.unlift]] method can be used in extractors.
+      *          {{{
+      *          val of: Int => Option[String] = { i =>
+      *            if (i == 2) {
+      *              Some("matched by an optional function")
+      *            } else {
+      *              None
+      *            }
+      *          }
+      *
+      *          util.Random.nextInt(4) match {
+      *            case of.unlift(m) => // Convert an optional function to a pattern
+      *              println(m)
+      *            case _ =>
+      *              println("Not matched")
+      *          }
+      *          }}}
+      */
+    def unlift: PartialFunction[A, B] = Function.unlift(f)
+  }
+
+}
+
+/** A function of 1 parameter.
+ *
+ *  In the following example, the definition of `succ` is
+ *  shorthand, conceptually, for the anonymous class definition
+ *  `anonfun1`, although the implementation details of how the
+ *  function value is constructed may differ:
+ *
+ *  {{{
+ *  object Main extends App {
+ *    val succ = (x: Int) => x + 1
+ *    val anonfun1 = new Function1[Int, Int] {
+ *      def apply(x: Int): Int = x + 1
+ *    }
+ *    assert(succ(0) == anonfun1(0))
+ * }
+ *  }}}
+ *
+ *  Note that the difference between `Function1` and [[scala.PartialFunction]]
+ *  is that the latter can specify inputs which it will not handle.
+ */
+@annotation.implicitNotFound(msg = "No implicit view available from ${T1} => ${R}.")
+trait Function1[@specialized(Specializable.Arg) -T1, @specialized(Specializable.Return) +R] extends AnyRef { // FIXME: self =>
+  /** Apply the body of this function to the argument.
+   *  @return   the result of function application.
+   */
+  def apply(v1: T1): R
+
+  /** Composes two instances of Function1 in a new Function1, with this function applied last.
+   *
+   *  @tparam   A   the type to which function `g` can be applied
+   *  @param    g   a function A => T1
+   *  @return       a new function `f` such that `f(x) == apply(g(x))`
+   */
+  @annotation.unspecialized def compose[A](g: A => T1): A => R = { x => apply(g(x)) }
+
+  /** Composes two instances of Function1 in a new Function1, with this function applied first.
+   *
+   *  @tparam   A   the result type of function `g`
+   *  @param    g   a function R => A
+   *  @return       a new function `f` such that `f(x) == g(apply(x))`
+   */
+  @annotation.unspecialized def andThen[A](g: R => A): T1 => A = { x => g(apply(x)) }
+
+  override def toString(): String = "<function1>"
+}