diff --git a/hail/.mill-version b/hail/.mill-version index b80f98e66f1a..aa22d3ce39b2 100644 --- a/hail/.mill-version +++ b/hail/.mill-version @@ -1 +1 @@ -0.11.7 +0.12.3 diff --git a/hail/build.sc b/hail/build.mill similarity index 80% rename from hail/build.sc rename to hail/build.mill index 96a3baf26c27..9da859300a40 100644 --- a/hail/build.sc +++ b/hail/build.mill @@ -1,3 +1,5 @@ +package build + import $ivy.`com.goyeau::mill-scalafix::0.3.2` import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0` import com.goyeau.mill.scalafix.ScalafixModule @@ -15,56 +17,6 @@ object Settings { val hailPatchVersion = "133" } -/** Update the millw script. */ -def millw(): Command[PathRef] = T.command { - val target = - mill.util.Util.download("https://raw.githubusercontent.com/lefou/millw/main/millw") - val millw = T.workspace / "millw" - os.copy.over(target.path, millw) - os.perms.set(millw, os.perms(millw) + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE) - target -} - -def scalaVersion: T[String] = T.input { - val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.15") - if (!v.startsWith("2.12")) - Result.Failure("Hail currently supports only Scala 2.12") - else - v -} - -def javaVersion: T[String] = T.input { - System.getProperty("java.version") -} - -def sparkVersion: T[String] = T.input { - Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0")) -} - -def debugMode: T[Boolean] = T.input { - val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE") - T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode") - isDebug -} - -def debugOrRelease: Task[String] = T.task { - if (debugMode()) "debug" else "release" -} - -def buildInfo: T[PathRef] = T { - val revision = VcsVersion.vcsState().currentRevision - os.write( - T.dest / "build-info.properties", - s"""[Build Metadata] - |revision=$revision - |sparkVersion=${sparkVersion()} - |hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion} - |hailBuildConfiguration=${debugOrRelease()} - |""".stripMargin, - ) - PathRef(T.dest) -} - object Deps { object HTTPComponents { val core = ivy"org.apache.httpcomponents:httpcore:4.4.14" @@ -94,8 +46,8 @@ object Deps { } object Spark { - def core: Task[Dep] = T.task(ivy"org.apache.spark::spark-core:${sparkVersion()}") - def mllib: Task[Dep] = T.task(ivy"org.apache.spark::spark-mllib:${sparkVersion()}") + def core: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-core:${build.env.sparkVersion()}") + def mllib: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-mllib:${build.env.sparkVersion()}") } val samtools = ivy"com.github.samtools:htsjdk:3.0.5" @@ -121,13 +73,13 @@ object Deps { } trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule { outer => - override def scalaVersion: T[String] = build.scalaVersion() + override def scalaVersion: T[String] = build.env.scalaVersion() override def javacOptions: T[Seq[String]] = Seq( "-Xlint:all", "-Werror", - if (debugMode()) "-g" else "-O", - ) ++ (if (!javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq()) + if (build.env.debugMode()) "-g" else "-O", + ) ++ (if (!build.env.javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq()) override def scalacOptions: T[Seq[String]] = T { Seq( @@ -140,7 +92,7 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule "-Xlint", "-Ywarn-unused:_,-explicits,-implicits", ) ++ ( - if (debugMode()) Seq() + if (build.env.debugMode()) Seq() else Seq( "-Xfatal-warnings", "-opt:l:method", @@ -153,7 +105,7 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule override def bspCompileClasspath: T[Agg[UnresolvedPath]] = super.bspCompileClasspath() ++ resources().map(p => UnresolvedPath.ResolvedPath(p.path)) - trait HailTests extends SbtModuleTests with TestNg with ScalafmtModule with ScalafixModule { + trait HailTests extends SbtTests with TestNg with ScalafmtModule with ScalafixModule { override def forkArgs: T[Seq[String]] = Seq("-Xss4m", "-Xmx4096M") override def ivyDeps: T[Agg[Dep]] = @@ -171,7 +123,50 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule } } -object main extends RootModule with HailScalaModule { outer => +object `package` extends RootModule with HailScalaModule { outer => + + object env extends Module { + def scalaVersion: T[String] = Task.Input { + val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.15") + if (!v.startsWith("2.12")) + Result.Failure("Hail currently supports only Scala 2.12") + else + v + } + + def javaVersion: T[String] = Task.Input { + System.getProperty("java.version") + } + + def sparkVersion: T[String] = Task.Input { + Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0")) + } + + def debugMode: T[Boolean] = Task.Input { + val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE") + T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode") + isDebug + } + + def debugOrRelease: Task[String] = Task.Anon { + if (debugMode()) "debug" else "release" + } + } + + def buildInfo: T[PathRef] = T { + val revision = VcsVersion.vcsState().currentRevision + os.write( + T.dest / "build-info.properties", + s"""[Build Metadata] + |revision=$revision + |sparkVersion=${env.sparkVersion()} + |hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion} + |hailBuildConfiguration=${env.debugOrRelease()} + |""".stripMargin, + ) + PathRef(T.dest) + } + override def moduleDeps: Seq[JavaModule] = Seq(memory) override def resources: T[Seq[PathRef]] = Seq( @@ -256,15 +251,15 @@ object main extends RootModule with HailScalaModule { outer => override def javacOptions: T[Seq[String]] = outer.javacOptions() ++ ( - if (javaVersion().startsWith("1.8")) Seq( + if (env.javaVersion().startsWith("1.8")) Seq( "-XDenableSunApiLintControl", "-Xlint:-sunapi", ) else Seq() ) - override def sources: T[Seq[PathRef]] = T.sources { - Seq(PathRef(this.millSourcePath / os.up / "src" / debugOrRelease() / "java")) + override def sources: T[Seq[PathRef]] = Task.Sources { + Seq(PathRef(this.millSourcePath / os.up / "src" / env.debugOrRelease() / "java")) } } diff --git a/hail/mill b/hail/mill new file mode 100755 index 000000000000..f4d81f4419fe --- /dev/null +++ b/hail/mill @@ -0,0 +1,67 @@ +#!/usr/bin/env sh + +# This is a wrapper script, that automatically download mill from GitHub release pages +# You can give the required mill version with MILL_VERSION env variable +# If no version is given, it falls back to the value of DEFAULT_MILL_VERSION + +set -e + +if [ -z "${DEFAULT_MILL_VERSION}" ] ; then + DEFAULT_MILL_VERSION=0.12.3 +fi + +if [ -z "$MILL_VERSION" ] ; then + if [ -f ".mill-version" ] ; then + MILL_VERSION="$(tr '\r' '\n' < .mill-version | head -n 1 2> /dev/null)" + elif [ -f ".config/mill-version" ] ; then + MILL_VERSION="$(tr '\r' '\n' < .config/mill-version | head -n 1 2> /dev/null)" + elif [ -f "mill" ] && [ "$0" != "mill" ] ; then + MILL_VERSION=$(grep -F "DEFAULT_MILL_VERSION=" "mill" | head -n 1 | cut -d= -f2) + else + MILL_VERSION=$DEFAULT_MILL_VERSION + fi +fi + +if [ "x${XDG_CACHE_HOME}" != "x" ] ; then + MILL_DOWNLOAD_PATH="${XDG_CACHE_HOME}/mill/download" +else + MILL_DOWNLOAD_PATH="${HOME}/.cache/mill/download" +fi +MILL_EXEC_PATH="${MILL_DOWNLOAD_PATH}/${MILL_VERSION}" + +version_remainder="$MILL_VERSION" +MILL_MAJOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}" +MILL_MINOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}" + +if [ ! -s "$MILL_EXEC_PATH" ] ; then + mkdir -p "$MILL_DOWNLOAD_PATH" + if [ "$MILL_MAJOR_VERSION" -gt 0 ] || [ "$MILL_MINOR_VERSION" -ge 5 ] ; then + ASSEMBLY="-assembly" + fi + DOWNLOAD_FILE=$MILL_EXEC_PATH-tmp-download + MILL_VERSION_TAG=$(echo $MILL_VERSION | sed -E 's/([^-]+)(-M[0-9]+)?(-.*)?/\1\2/') + MILL_DOWNLOAD_URL="https://repo1.maven.org/maven2/com/lihaoyi/mill-dist/$MILL_VERSION/mill-dist-$MILL_VERSION.jar" + curl --fail -L -o "$DOWNLOAD_FILE" "$MILL_DOWNLOAD_URL" + chmod +x "$DOWNLOAD_FILE" + mv "$DOWNLOAD_FILE" "$MILL_EXEC_PATH" + unset DOWNLOAD_FILE + unset MILL_DOWNLOAD_URL +fi + +if [ -z "$MILL_MAIN_CLI" ] ; then + MILL_MAIN_CLI="${0}" +fi + +MILL_FIRST_ARG="" + + # first arg is a long flag for "--interactive" or starts with "-i" +if [ "$1" = "--bsp" ] || [ "${1#"-i"}" != "$1" ] || [ "$1" = "--interactive" ] || [ "$1" = "--no-server" ] || [ "$1" = "--repl" ] || [ "$1" = "--help" ] ; then + # Need to preserve the first position of those listed options + MILL_FIRST_ARG=$1 + shift +fi + +unset MILL_DOWNLOAD_PATH +unset MILL_VERSION + +exec $MILL_EXEC_PATH $MILL_FIRST_ARG -D "mill.main.cli=${MILL_MAIN_CLI}" "$@"