Add support for Pipelined builds (#18880)

This includes support for a single pass pipelined build, compatible with
sbt's `ThisBuild/usePipelining`,
- adds `-Ypickle-java` and `-Ypickle-write` flags, expected by Zinc when
pipelining is enabled in sbt.
- when `-Ypickle-write <directory|jar>` is set, then write tasty from
pickler to that output, (building upon
#19074 support for Java signatures
in TASTy files).
- call `apiPhaseCompleted` and `dependencyPhaseCompleted` callbacks,
which will activate early downstream compilation
- calls `generatedNonLocalClass` callbacks early, which enables Zinc to
run the incremental algorithm before starting downstream compilation
(including checking for macro definitions).

generally this can be reviewed commit-by-commit, as they each do an
isolated feature.

As well as many tests in the `sbt-test/pipelining` directory, this has
also been tested locally on `akka/akka-http`, `apache/incubator-pekko`,
`lichess-org/lila`, `scalacenter/scaladex`, `typelevel/fs2`,
`typelevel/http4s`, `typelevel/cats`, `slick/slick`.

This PR sets the ground work for an optional 2-pass compile (reusing the
`OUTLINEattr`), which should use a faster frontend (skipping rhs when
possible) before producing tasty signatures

fixes #19743

compiler/src/dotty/tools/backend/jvm/CodeGen.scala

@@ -125,17 +125,16 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
 
   // Creates a callback that will be evaluated in PostProcessor after creating a file
   private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: util.SourceFile)(using Context): AbstractFile => Unit = {
-    val (fullClassName, isLocal) = atPhase(sbtExtractDependenciesPhase) {
-      (ExtractDependencies.classNameAsString(claszSymbol), claszSymbol.isLocal)
+    val isLocal = atPhase(sbtExtractDependenciesPhase) {
+      claszSymbol.isLocal
     }
     clsFile => {
       val className = cls.name.replace('/', '.')
       if (ctx.compilerCallback != null)
         ctx.compilerCallback.onClassGenerated(sourceFile, convertAbstractFile(clsFile), className)
 
-      ctx.withIncCallback: cb =>
-        if (isLocal) cb.generatedLocalClass(sourceFile, clsFile.jpath)
-        else cb.generatedNonLocalClass(sourceFile, clsFile.jpath, className, fullClassName)
+      if isLocal then
+        ctx.withIncCallback(_.generatedLocalClass(sourceFile, clsFile.jpath))
     }
   }
 

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -28,13 +28,16 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
   var tpdTree: tpd.Tree = tpd.EmptyTree
 
   /** Is this the compilation unit of a Java file */
-  def isJava: Boolean = source.file.name.endsWith(".java")
+  def isJava: Boolean = source.file.ext.isJava
 
   /** Is this the compilation unit of a Java file, or TASTy derived from a Java file */
-  def typedAsJava = isJava || {
-    val infoNN = info
-    infoNN != null && infoNN.tastyInfo.exists(_.attributes.isJava)
-  }
+  def typedAsJava =
+    val ext = source.file.ext
+    ext.isJavaOrTasty && (ext.isJava || tastyInfo.exists(_.attributes.isJava))
+
+  def tastyInfo: Option[TastyInfo] =
+    val local = info
+    if local == null then None else local.tastyInfo
 
 
   /** The source version for this unit, as determined by a language import */
@@ -94,12 +97,15 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
     // when this unit is unsuspended.
     depRecorder.clear()
     if !suspended then
-      if (ctx.settings.XprintSuspension.value)
-        report.echo(i"suspended: $this")
-      suspended = true
-      ctx.run.nn.suspendedUnits += this
-      if ctx.phase == Phases.inliningPhase then
-        suspendedAtInliningPhase = true
+      if ctx.settings.YnoSuspendedUnits.value then
+        report.error(i"Compilation unit suspended $this (-Yno-suspended-units is set)")
+      else
+        if (ctx.settings.XprintSuspension.value)
+          report.echo(i"suspended: $this")
+        suspended = true
+        ctx.run.nn.suspendedUnits += this
+        if ctx.phase == Phases.inliningPhase then
+          suspendedAtInliningPhase = true
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -41,13 +41,13 @@ class Compiler {
     List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
     List(new sjs.PrepJSInterop) ::  // Additional checks and transformations for Scala.js (Scala.js only)
-    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new SetRootTree) ::        // Set the `rootTreeOrProvider` on class symbols
     Nil
 
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
     List(new Pickler) ::            // Generate TASTY info
+    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new CheckUnused.PostInlining) ::  // Check for unused elements

compiler/src/dotty/tools/dotc/Driver.scala

@@ -6,7 +6,7 @@ import core.Comments.{ContextDoc, ContextDocstrings}
 import core.Contexts.*
 import core.{MacroClassLoader, TypeError}
 import dotty.tools.dotc.ast.Positioned
-import dotty.tools.io.AbstractFile
+import dotty.tools.io.{AbstractFile, FileExtension}
 import reporting.*
 import core.Decorators.*
 import config.Feature
@@ -97,9 +97,9 @@ class Driver {
           if !file.exists then
             report.error(em"File does not exist: ${file.path}")
             None
-          else file.extension match
-            case "jar" => Some(file.path)
-            case "tasty" =>
+          else file.ext match
+            case FileExtension.Jar => Some(file.path)
+            case FileExtension.Tasty =>
               TastyFileUtil.getClassPath(file) match
                 case Some(classpath) => Some(classpath)
                 case _ =>

compiler/src/dotty/tools/dotc/classpath/AggregateClassPath.scala

@@ -33,25 +33,6 @@ case class AggregateClassPath(aggregates: Seq[ClassPath]) extends ClassPath {
     packageIndex.getOrElseUpdate(pkg.dottedString, aggregates.filter(_.hasPackage(pkg)))
   }
 
-  override def findClass(className: String): Option[ClassRepresentation] = {
-    val (pkg, _) = PackageNameUtils.separatePkgAndClassNames(className)
-
-    def findEntry(isSource: Boolean): Option[ClassRepresentation] =
-      aggregatesForPackage(PackageName(pkg)).iterator.map(_.findClass(className)).collectFirst {
-        case Some(s: SourceFileEntry) if isSource => s
-        case Some(s: BinaryFileEntry) if !isSource => s
-      }
-
-    val classEntry = findEntry(isSource = false)
-    val sourceEntry = findEntry(isSource = true)
-
-    (classEntry, sourceEntry) match {
-      case (Some(c: BinaryFileEntry), Some(s: SourceFileEntry)) => Some(BinaryAndSourceFilesEntry(c, s))
-      case (c @ Some(_), _) => c
-      case (_, s) => s
-    }
-  }
-
   override def asURLs: Seq[URL] = aggregates.flatMap(_.asURLs)
 
   override def asClassPathStrings: Seq[String] = aggregates.map(_.asClassPathString).distinct

compiler/src/dotty/tools/dotc/classpath/ClassPath.scala

@@ -6,6 +6,7 @@ package dotty.tools.dotc.classpath
 import dotty.tools.dotc.classpath.FileUtils.isTasty
 import dotty.tools.io.AbstractFile
 import dotty.tools.io.ClassRepresentation
+import dotty.tools.io.FileExtension
 
 case class ClassPathEntries(packages: scala.collection.Seq[PackageEntry], classesAndSources: scala.collection.Seq[ClassRepresentation]) {
   def toTuple: (scala.collection.Seq[PackageEntry], scala.collection.Seq[ClassRepresentation]) = (packages, classesAndSources)
@@ -52,7 +53,7 @@ sealed trait BinaryFileEntry extends ClassRepresentation {
 object BinaryFileEntry {
   def apply(file: AbstractFile): BinaryFileEntry =
     if file.isTasty then
-      if file.resolveSiblingWithExtension("class") != null then TastyWithClassFileEntry(file)
+      if file.resolveSiblingWithExtension(FileExtension.Class) != null then TastyWithClassFileEntry(file)
       else StandaloneTastyFileEntry(file)
     else
       ClassFileEntry(file)

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -274,23 +274,18 @@ final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends Clas
 }
 
 case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[BinaryFileEntry] with NoSourcePaths {
-  override def findClass(className: String): Option[ClassRepresentation] =
-    findClassFile(className).map(BinaryFileEntry(_))
 
   def findClassFile(className: String): Option[AbstractFile] = {
     val relativePath = FileUtils.dirPath(className)
-    val tastyFile = new JFile(dir, relativePath + ".tasty")
-    if tastyFile.exists then Some(tastyFile.toPath.toPlainFile)
-    else
-      val classFile = new JFile(dir, relativePath + ".class")
-      if classFile.exists then Some(classFile.toPath.toPlainFile)
-      else None
+    val classFile = new JFile(dir, relativePath + ".class")
+    if classFile.exists then Some(classFile.toPath.toPlainFile)
+    else None
   }
 
   protected def createFileEntry(file: AbstractFile): BinaryFileEntry = BinaryFileEntry(file)
 
   protected def isMatchingFile(f: JFile): Boolean =
-    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
+    f.isTasty || (f.isClass && !f.hasSiblingTasty)
 
   private[dotty] def classes(inPackage: PackageName): Seq[BinaryFileEntry] = files(inPackage)
 }
@@ -301,16 +296,5 @@ case class DirectorySourcePath(dir: JFile) extends JFileDirectoryLookup[SourceFi
   protected def createFileEntry(file: AbstractFile): SourceFileEntry = SourceFileEntry(file)
   protected def isMatchingFile(f: JFile): Boolean = endsScalaOrJava(f.getName)
 
-  override def findClass(className: String): Option[ClassRepresentation] = findSourceFile(className).map(SourceFileEntry(_))
-
-  private def findSourceFile(className: String): Option[AbstractFile] = {
-    val relativePath = FileUtils.dirPath(className)
-    val sourceFile = LazyList("scala", "java")
-      .map(ext => new JFile(dir, relativePath + "." + ext))
-      .collectFirst { case file if file.exists() => file }
-
-    sourceFile.map(_.toPath.toPlainFile)
-  }
-
   private[dotty] def sources(inPackage: PackageName): Seq[SourceFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -17,49 +17,52 @@ object FileUtils {
   extension (file: AbstractFile) {
     def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.name)
 
-    def isClass: Boolean = !file.isDirectory && hasClassExtension && !file.name.endsWith("$class.class")
-      // FIXME: drop last condition when we stop being compatible with Scala 2.11
+    def isClass: Boolean = !file.isDirectory && hasClassExtension
 
-    def hasClassExtension: Boolean = file.hasExtension("class")
+    def hasClassExtension: Boolean = file.ext.isClass
 
-    def hasTastyExtension: Boolean = file.hasExtension("tasty")
+    def hasTastyExtension: Boolean = file.ext.isTasty
 
     def isTasty: Boolean = !file.isDirectory && hasTastyExtension
 
     def isScalaBinary: Boolean = file.isClass || file.isTasty
 
-    def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java"))
+    def isScalaOrJavaSource: Boolean = !file.isDirectory && file.ext.isScalaOrJava
 
     // TODO do we need to check also other files using ZipMagicNumber like in scala.tools.nsc.io.Jar.isJarOrZip?
-    def isJarOrZip: Boolean = file.hasExtension("jar") || file.hasExtension("zip")
+    def isJarOrZip: Boolean = file.ext.isJarOrZip
 
     /**
      * Safe method returning a sequence containing one URL representing this file, when underlying file exists,
      * and returning given default value in other case
      */
     def toURLs(default: => Seq[URL] = Seq.empty): Seq[URL] = if (file.file == null) default else Seq(file.toURL)
 
-    /** Returns the tasty file associated with this class file */
-    def classToTasty: Option[AbstractFile] =
-      assert(file.isClass, s"non-class: $file")
-      val tastyName = classNameToTasty(file.name)
-      Option(file.resolveSibling(tastyName))
+    /**
+     * Returns if there is an existing sibling `.tasty` file.
+     */
+    def hasSiblingTasty: Boolean =
+      assert(file.hasClassExtension, s"non-class: $file")
+      file.resolveSibling(classNameToTasty(file.name)) != null
   }
 
   extension (file: JFile) {
     def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.getName)
 
-    def isClass: Boolean = file.isFile && file.getName.endsWith(SUFFIX_CLASS) && !file.getName.endsWith("$class.class")
-    // FIXME: drop last condition when we stop being compatible with Scala 2.11
+    def isClass: Boolean = file.isFile && hasClassExtension
+
+    def hasClassExtension: Boolean = file.getName.endsWith(SUFFIX_CLASS)
 
     def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
 
-    /** Returns the tasty file associated with this class file */
-    def classToTasty: Option[JFile] =
-      assert(file.isClass, s"non-class: $file")
-      val tastyName = classNameToTasty(file.getName.stripSuffix(".class"))
-      val tastyPath = file.toPath.resolveSibling(tastyName)
-      if java.nio.file.Files.exists(tastyPath) then Some(tastyPath.toFile) else None
+    /**
+     * Returns if there is an existing sibling `.tasty` file.
+     */
+    def hasSiblingTasty: Boolean =
+      assert(file.hasClassExtension, s"non-class: $file")
+      val path = file.toPath
+      val tastyPath = path.resolveSibling(classNameToTasty(file.getName))
+      java.nio.file.Files.exists(tastyPath)
 
   }
 

compiler/src/dotty/tools/dotc/classpath/VirtualDirectoryClassPath.scala

@@ -38,22 +38,18 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
   def asURLs: Seq[URL] = Seq(new URI(dir.name).toURL)
   def asClassPathStrings: Seq[String] = Seq(dir.path)
 
-  override def findClass(className: String): Option[ClassRepresentation] =
-    findClassFile(className).map(BinaryFileEntry(_))
-
   def findClassFile(className: String): Option[AbstractFile] = {
     val pathSeq = FileUtils.dirPath(className).split(java.io.File.separator)
     val parentDir = lookupPath(dir)(pathSeq.init.toSeq, directory = true)
-    if parentDir == null then return None
+    if parentDir == null then None
     else
-      Option(lookupPath(parentDir)(pathSeq.last + ".tasty" :: Nil, directory = false))
-        .orElse(Option(lookupPath(parentDir)(pathSeq.last + ".class" :: Nil, directory = false)))
+      Option(lookupPath(parentDir)(pathSeq.last + ".class" :: Nil, directory = false))
   }
 
   private[dotty] def classes(inPackage: PackageName): Seq[BinaryFileEntry] = files(inPackage)
 
   protected def createFileEntry(file: AbstractFile): BinaryFileEntry = BinaryFileEntry(file)
 
   protected def isMatchingFile(f: AbstractFile): Boolean =
-    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
+    f.isTasty || (f.isClass && !f.hasSiblingTasty)
 }

compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala

@@ -45,21 +45,15 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
     with NoSourcePaths {
 
     override def findClassFile(className: String): Option[AbstractFile] =
-      findClass(className).map(_.file)
-
-    // This method is performance sensitive as it is used by SBT's ExtractDependencies phase.
-    override def findClass(className: String): Option[BinaryFileEntry] = {
       val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className)
-      val binaries = files(PackageName(pkg), simpleClassName + ".tasty", simpleClassName + ".class")
-      binaries.find(_.file.isTasty).orElse(binaries.find(_.file.isClass))
-    }
+      file(PackageName(pkg), simpleClassName + ".class").map(_.file)
 
     override private[dotty] def classes(inPackage: PackageName): Seq[BinaryFileEntry] = files(inPackage)
 
     override protected def createFileEntry(file: FileZipArchive#Entry): BinaryFileEntry = BinaryFileEntry(file)
 
     override protected def isRequiredFileType(file: AbstractFile): Boolean =
-      file.isTasty || (file.isClass && file.classToTasty.isEmpty)
+      file.isTasty || (file.isClass && !file.hasSiblingTasty)
   }
 
   /**

compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala

@@ -43,15 +43,6 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends Efficie
     }
     yield createFileEntry(entry)
 
-  protected def files(inPackage: PackageName, names: String*): Seq[FileEntryType] =
-    for {
-      dirEntry <- findDirEntry(inPackage).toSeq
-      name <- names
-      entry <- Option(dirEntry.lookupName(name, directory = false))
-      if isRequiredFileType(entry)
-    }
-    yield createFileEntry(entry)
-
   protected def file(inPackage: PackageName, name: String): Option[FileEntryType] =
     for {
       dirEntry <- findDirEntry(inPackage)

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -31,7 +31,7 @@ object ScalaSettings extends ScalaSettings
 
 // Kept as seperate type to avoid breaking backward compatibility
 abstract class ScalaSettings extends SettingGroup, AllScalaSettings:
-  val settingsByCategory: Map[SettingCategory, List[Setting[_]]] = 
+  val settingsByCategory: Map[SettingCategory, List[Setting[_]]] =
     allSettings.groupBy(_.category)
       .view.mapValues(_.toList).toMap
       .withDefaultValue(Nil)
@@ -43,7 +43,7 @@ abstract class ScalaSettings extends SettingGroup, AllScalaSettings:
   val verboseSettings: List[Setting[_]] = settingsByCategory(VerboseSetting).sortBy(_.name)
   val settingsByAliases: Map[String, Setting[_]] = allSettings.flatMap(s => s.aliases.map(_ -> s)).toMap
 
-  
+
 trait AllScalaSettings extends CommonScalaSettings, PluginSettings, VerboseSettings, WarningSettings, XSettings, YSettings:
   self: SettingGroup =>
 
@@ -380,6 +380,7 @@ private sealed trait YSettings:
   val YprintPos: Setting[Boolean] = BooleanSetting(ForkSetting, "Yprint-pos", "Show tree positions.")
   val YprintPosSyms: Setting[Boolean] = BooleanSetting(ForkSetting, "Yprint-pos-syms", "Show symbol definitions positions.")
   val YnoDeepSubtypes: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-deep-subtypes", "Throw an exception on deep subtyping call stacks.")
+  val YnoSuspendedUnits: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-suspended-units", "Do not suspend units, e.g. when calling a macro defined in the same run. This will error instead of suspending.")
   val YnoPatmatOpt: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-patmat-opt", "Disable all pattern matching optimizations.")
   val YplainPrinter: Setting[Boolean] = BooleanSetting(ForkSetting, "Yplain-printer", "Pretty-print using a plain printer.")
   val YprintSyms: Setting[Boolean] = BooleanSetting(ForkSetting, "Yprint-syms", "When printing trees print info in symbols instead of corresponding info in trees.")
@@ -439,7 +440,7 @@ private sealed trait YSettings:
   val YdebugMacros: Setting[Boolean] = BooleanSetting(ForkSetting, "Ydebug-macros", "Show debug info when quote pattern match fails")
 
   // Pipeline compilation options
-  val YjavaTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
-  val YjavaTastyOutput: Setting[AbstractFile] = OutputSetting(ForkSetting, "Yjava-tasty-output", "directory|jar", "(Internal use only!) destination for generated .tasty files containing Java type signatures.", NoAbstractFile)
+  val YjavaTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Yjava-tasty", "Pickler phase should compute TASTy for .java defined symbols for use by build tools", aliases = List("-Ypickle-java"), preferPrevious = true)
+  val YearlyTastyOutput: Setting[AbstractFile] = OutputSetting(ForkSetting, "Yearly-tasty-output", "directory|jar", "Destination to write generated .tasty files to for use in pipelined compilation.", NoAbstractFile, aliases = List("-Ypickle-write"), preferPrevious = true)
   val YallowOutlineFromTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Yallow-outline-from-tasty", "Allow outline TASTy to be loaded with the -from-tasty option.")
 end YSettings