From 2a503d95e2a10faf122a80cb14b2e0fdf81a6b8c Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Nov 2024 14:37:05 +0100 Subject: [PATCH 1/4] Drop incremental compilation state for Mill modules when version changes Mill used various tricks to map build scripts written by users to generated enhanced versions, including package objects and other stuff. Since some of these techniques may result in Zinc undercompilation, we want avoid downstream issues, especally in the `methodCodeHashSignatures` task, which is sensitive to inconsistent bytecode. We trait potentially longer compilation against incorrect or too coarse code change detection, which may invalidate a lot more downstream tasks than needed due to invalid data. Fix https://github.com/com-lihaoyi/mill/issues/3874 --- .../src/mill/runner/MillBuildRootModule.scala | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 2a193463f59..53947b2ce7b 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -7,7 +7,7 @@ import mill.define.{Discover, Task} import mill.scalalib.{BoundDep, Dep, DepSyntax, Lib, ScalaModule} import mill.util.CoursierSupport import mill.util.Util.millProjectModule -import mill.scalalib.api.Versions +import mill.scalalib.api.{CompilationResult, Versions} import mill.main.client.OutFiles._ import mill.main.client.CodeGenConstants.buildFileExtensions import mill.main.{BuildInfo, RootModule} @@ -261,6 +261,54 @@ abstract class MillBuildRootModule()(implicit /** Used in BSP IntelliJ, which can only work with directories */ def dummySources: Sources = Task.Sources(T.dest) + + def millVersion = T.input { BuildInfo.millVersion } + + override def compile: T[CompilationResult] = T { + val superTask = super.compile + + val sv = scalaVersion() + val mv = millVersion() + + val prevMillVersionFile = T.dest / s"mill-version.json" + val prevMillVersion = Option(prevMillVersionFile) + .filter(os.exists) + .map(os.read(_).trim) + .getOrElse("?") + + if (prevMillVersion != mv) { + // Mill version changed, drop all previous incremental state + // see https://github.com/com-lihaoyi/mill/issues/3874 + T.log.debug( + s"Detected Mill version change ${prevMillVersion} -> ${mv}. Dropping previous incremental compilation state" + ) + os.remove.all(T.dest) + os.makeDir(T.dest) + } + + // copied from `ScalaModule` + val compileResult = zincWorker() + .worker() + .compileMixed( + upstreamCompileOutput = upstreamCompileOutput(), + sources = Agg.from(allSourceFiles().map(_.path)), + compileClasspath = compileClasspath().map(_.path), + javacOptions = javacOptions() ++ mandatoryJavacOptions(), + scalaVersion = sv, + scalaOrganization = scalaOrganization(), + scalacOptions = allScalacOptions(), + compilerClasspath = scalaCompilerClasspath(), + scalacPluginClasspath = scalacPluginClasspath(), + reporter = T.reporter.apply(hashCode), + reportCachedProblems = zincReportCachedProblems(), + incrementalCompilation = zincIncrementalCompilation(), + auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions() + ) + + os.write(prevMillVersionFile, mv) + compileResult + } + } object MillBuildRootModule { From d6d4c529cb77875ff2cb7dea3f3adaa364ac45ce Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Nov 2024 15:51:29 +0100 Subject: [PATCH 2/4] compile task needs to be persistent --- runner/src/mill/runner/MillBuildRootModule.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 53947b2ce7b..ffbb46102e0 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -264,13 +264,13 @@ abstract class MillBuildRootModule()(implicit def millVersion = T.input { BuildInfo.millVersion } - override def compile: T[CompilationResult] = T { + override def compile: T[CompilationResult] = Task(persistent = true) { val superTask = super.compile val sv = scalaVersion() val mv = millVersion() - val prevMillVersionFile = T.dest / s"mill-version.json" + val prevMillVersionFile = T.dest / s"mill-version" val prevMillVersion = Option(prevMillVersionFile) .filter(os.exists) .map(os.read(_).trim) @@ -284,6 +284,7 @@ abstract class MillBuildRootModule()(implicit ) os.remove.all(T.dest) os.makeDir(T.dest) + os.write(prevMillVersionFile, mv) } // copied from `ScalaModule` @@ -305,7 +306,6 @@ abstract class MillBuildRootModule()(implicit auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions() ) - os.write(prevMillVersionFile, mv) compileResult } From c6a822ed748328f81afb4c16b8594848b73169ec Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Nov 2024 16:06:18 +0100 Subject: [PATCH 3/4] Don't need the reference to super --- runner/src/mill/runner/MillBuildRootModule.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index ffbb46102e0..580604cec8f 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -265,8 +265,6 @@ abstract class MillBuildRootModule()(implicit def millVersion = T.input { BuildInfo.millVersion } override def compile: T[CompilationResult] = Task(persistent = true) { - val superTask = super.compile - val sv = scalaVersion() val mv = millVersion() From 4e43f1b4d1c8f1e8747aced617173e3c20fca6d1 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Nov 2024 16:34:22 +0100 Subject: [PATCH 4/4] Cleanup --- runner/src/mill/runner/MillBuildRootModule.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 580604cec8f..855520b6087 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -265,7 +265,6 @@ abstract class MillBuildRootModule()(implicit def millVersion = T.input { BuildInfo.millVersion } override def compile: T[CompilationResult] = Task(persistent = true) { - val sv = scalaVersion() val mv = millVersion() val prevMillVersionFile = T.dest / s"mill-version" @@ -286,14 +285,14 @@ abstract class MillBuildRootModule()(implicit } // copied from `ScalaModule` - val compileResult = zincWorker() + zincWorker() .worker() .compileMixed( upstreamCompileOutput = upstreamCompileOutput(), sources = Agg.from(allSourceFiles().map(_.path)), compileClasspath = compileClasspath().map(_.path), javacOptions = javacOptions() ++ mandatoryJavacOptions(), - scalaVersion = sv, + scalaVersion = scalaVersion(), scalaOrganization = scalaOrganization(), scalacOptions = allScalacOptions(), compilerClasspath = scalaCompilerClasspath(), @@ -303,8 +302,6 @@ abstract class MillBuildRootModule()(implicit incrementalCompilation = zincIncrementalCompilation(), auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions() ) - - compileResult } }