From 6d5ae93ccf075c9dcaf69eb258977f4c3cb2eeda Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 13:02:04 +0800 Subject: [PATCH 01/17] . --- build.sc | 3 +- main/api/src/mill/api/SystemStreams.scala | 29 +++++++++++++- main/util/src/mill/util/Jvm.scala | 47 ++++------------------- 3 files changed, 37 insertions(+), 42 deletions(-) diff --git a/build.sc b/build.sc index dab0238e412..93bd7a85f48 100644 --- a/build.sc +++ b/build.sc @@ -153,7 +153,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val lambdaTest = ivy"de.tototec:de.tobiasroeser.lambdatest:0.8.0" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.10.2" + val osLib = ivy"com.lihaoyi::os-lib:0.10.2-5-81dd55-DIRTY14e72139" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.0" val millModuledefsVersion = "0.10.9" @@ -618,6 +618,7 @@ object main extends MillStableScalaModule with BuildInfo { ) object api extends MillStableScalaModule with BuildInfo { + def moduleDeps = Seq(client) def buildInfoPackageName = "mill.api" def buildInfoMembers = Seq( BuildInfo.Value("millVersion", millVersion(), "Mill version."), diff --git a/main/api/src/mill/api/SystemStreams.scala b/main/api/src/mill/api/SystemStreams.scala index aecd81a7566..d42cb66ec54 100644 --- a/main/api/src/mill/api/SystemStreams.scala +++ b/main/api/src/mill/api/SystemStreams.scala @@ -59,7 +59,34 @@ object SystemStreams { Console.withIn(systemStreams.in) { Console.withOut(systemStreams.out) { Console.withErr(systemStreams.err) { - t + os.Inherit.in.withValue( + new os.ProcessInput{ + def redirectFrom = ProcessBuilder.Redirect.PIPE + def processInput(processIn: => os.SubProcess.InputStream) = Some( + new mill.main.client.InputPumper(in, processIn, true, () => true) + ) + } + ){ + os.Inherit.out.withValue( + new os.ProcessOutput{ + def redirectTo = ProcessBuilder.Redirect.PIPE + def processOutput(processOut: => os.SubProcess.OutputStream) = Some( + new mill.main.client.InputPumper(processOut, out, false, () => true) + ) + } + ){ + os.Inherit.err.withValue( + new os.ProcessOutput{ + def redirectTo = ProcessBuilder.Redirect.PIPE + def processOutput(processErr: => os.SubProcess.OutputStream) = Some( + new mill.main.client.InputPumper(processErr, err, false, () => true) + ) + } + ){ + t + } + } + } } } } diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index 6267852c5f7..bbfd8f8818c 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -223,46 +223,13 @@ object Jvm extends CoursierSupport { workingDir: os.Path, backgroundOutputs: Option[Tuple2[ProcessOutput, ProcessOutput]] = None ): SubProcess = { - // If System.in is fake, then we pump output manually rather than relying - // on `os.Inherit`. That is because `os.Inherit` does not follow changes - // to System.in/System.out/System.err, so the subprocess's streams get sent - // to the parent process's origin outputs even if we want to direct them - // elsewhere - - if (!SystemStreams.isOriginal()) { - val process = os.proc(commandArgs).spawn( - cwd = workingDir, - env = envArgs, - stdin = if (backgroundOutputs.isEmpty) os.Pipe else "", - stdout = backgroundOutputs.map(_._1).getOrElse(os.Pipe), - stderr = backgroundOutputs.map(_._2).getOrElse(os.Pipe) - ) - - val sources = Seq( - (process.stdout, System.out, "spawnSubprocess.stdout", false, () => true), - (process.stderr, System.err, "spawnSubprocess.stderr", false, () => true), - (System.in, process.stdin, "spawnSubprocess.stdin", true, () => process.isAlive()) - ) - - for ((std, dest, name, checkAvailable, runningCheck) <- sources) { - val t = new Thread( - new InputPumper(std, dest, checkAvailable, () => runningCheck()), - name - ) - t.setDaemon(true) - t.start() - } - - process - } else { - os.proc(commandArgs).spawn( - cwd = workingDir, - env = envArgs, - stdin = if (backgroundOutputs.isEmpty) os.Inherit else "", - stdout = backgroundOutputs.map(_._1).getOrElse(os.Inherit), - stderr = backgroundOutputs.map(_._2).getOrElse(os.Inherit) - ) - } + os.proc(commandArgs).spawn( + cwd = workingDir, + env = envArgs, + stdin = if (backgroundOutputs.isEmpty) os.Inherit else "", + stdout = backgroundOutputs.map(_._1).getOrElse(os.Inherit), + stderr = backgroundOutputs.map(_._2).getOrElse(os.Inherit) + ) } def runLocal( From 4e556af0f5aac6804fbc42fa3fb59ab03f3e5df9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 13:05:33 +0800 Subject: [PATCH 02/17] . --- build.sc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sc b/build.sc index 93bd7a85f48..887efa42de8 100644 --- a/build.sc +++ b/build.sc @@ -153,7 +153,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val lambdaTest = ivy"de.tototec:de.tobiasroeser.lambdatest:0.8.0" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.10.2-5-81dd55-DIRTY14e72139" + val osLib = ivy"com.lihaoyi::os-lib:0.10.2-6-f5af2e" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.0" val millModuledefsVersion = "0.10.9" From 6a1462eb25d57baa5d935007adb54608e9a11187 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 13:37:37 +0800 Subject: [PATCH 03/17] . From 3a46d6a31f73b11d0aa187bf0d531bd23812b04b Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 15:42:19 +0800 Subject: [PATCH 04/17] . --- build.sc | 2 +- main/api/src/mill/api/SystemStreams.scala | 43 ++++++++----------- .../src/mill/main/client/InputPumper.java | 21 +++++---- .../src/mill/main/client/MillClientMain.java | 2 +- main/util/src/mill/util/Jvm.scala | 22 +++++----- runner/src/mill/runner/MillServerMain.scala | 2 +- .../scalajslib/worker/ScalaJSWorkerImpl.scala | 2 +- 7 files changed, 45 insertions(+), 49 deletions(-) diff --git a/build.sc b/build.sc index 887efa42de8..38377750d19 100644 --- a/build.sc +++ b/build.sc @@ -153,7 +153,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val lambdaTest = ivy"de.tototec:de.tobiasroeser.lambdatest:0.8.0" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.10.2-6-f5af2e" + val osLib = ivy"com.lihaoyi::os-lib:0.10.2-7-7492bb-DIRTY7fe7ee8e" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.0" val millModuledefsVersion = "0.10.9" diff --git a/main/api/src/mill/api/SystemStreams.scala b/main/api/src/mill/api/SystemStreams.scala index d42cb66ec54..66b2f7d823d 100644 --- a/main/api/src/mill/api/SystemStreams.scala +++ b/main/api/src/mill/api/SystemStreams.scala @@ -1,7 +1,7 @@ package mill.api -import java.io.{InputStream, PrintStream} - +import java.io.{InputStream, OutputStream, PrintStream} +import mill.main.client.InputPumper /** * Represents a set of streams that look similar to those provided by the * operating system. These may internally be proxied/redirected/processed, but @@ -48,6 +48,18 @@ object SystemStreams { def originalErr: PrintStream = original.err + private class PumpedProcessInput extends os.ProcessInput{ + def redirectFrom = ProcessBuilder.Redirect.PIPE + def processInput(processIn: => os.SubProcess.InputStream) = Some( + new InputPumper(() => System.in, () => processIn, true, () => true) + ) + } + + private class PumpedProcessOutput(dest: OutputStream) extends os.ProcessOutput{ + def redirectTo = ProcessBuilder.Redirect.PIPE + def processOutput(processOut: => os.SubProcess.OutputStream) = + Some(new InputPumper(() => processOut, () => dest, false, () => true)) + } def withStreams[T](systemStreams: SystemStreams)(t: => T): T = { val in = System.in val out = System.out @@ -59,30 +71,9 @@ object SystemStreams { Console.withIn(systemStreams.in) { Console.withOut(systemStreams.out) { Console.withErr(systemStreams.err) { - os.Inherit.in.withValue( - new os.ProcessInput{ - def redirectFrom = ProcessBuilder.Redirect.PIPE - def processInput(processIn: => os.SubProcess.InputStream) = Some( - new mill.main.client.InputPumper(in, processIn, true, () => true) - ) - } - ){ - os.Inherit.out.withValue( - new os.ProcessOutput{ - def redirectTo = ProcessBuilder.Redirect.PIPE - def processOutput(processOut: => os.SubProcess.OutputStream) = Some( - new mill.main.client.InputPumper(processOut, out, false, () => true) - ) - } - ){ - os.Inherit.err.withValue( - new os.ProcessOutput{ - def redirectTo = ProcessBuilder.Redirect.PIPE - def processOutput(processErr: => os.SubProcess.OutputStream) = Some( - new mill.main.client.InputPumper(processErr, err, false, () => true) - ) - } - ){ + os.Inherit.in.withValue(new PumpedProcessInput) { + os.Inherit.out.withValue(new PumpedProcessOutput(System.out)) { + os.Inherit.err.withValue(new PumpedProcessOutput(System.err)) { t } } diff --git a/main/client/src/mill/main/client/InputPumper.java b/main/client/src/mill/main/client/InputPumper.java index cfd5a242884..cbd5ecf460d 100644 --- a/main/client/src/mill/main/client/InputPumper.java +++ b/main/client/src/mill/main/client/InputPumper.java @@ -2,29 +2,34 @@ import java.io.InputStream; import java.io.OutputStream; +import java.util.function.Supplier; public class InputPumper implements Runnable{ - private InputStream src; - private OutputStream dest; + private Supplier src0; + private Supplier dest0; + private Boolean checkAvailable; private java.util.function.BooleanSupplier runningCheck; - public InputPumper(InputStream src, - OutputStream dest, + public InputPumper(Supplier src, + Supplier dest, Boolean checkAvailable){ this(src, dest, checkAvailable, () -> true); } - public InputPumper(InputStream src, - OutputStream dest, + public InputPumper(Supplier src, + Supplier dest, Boolean checkAvailable, java.util.function.BooleanSupplier runningCheck){ - this.src = src; - this.dest = dest; + this.src0 = src; + this.dest0 = dest; this.checkAvailable = checkAvailable; this.runningCheck = runningCheck; } boolean running = true; public void run() { + InputStream src = src0.get(); + OutputStream dest = dest0.get(); + byte[] buffer = new byte[1024]; try{ while(running){ diff --git a/main/client/src/mill/main/client/MillClientMain.java b/main/client/src/mill/main/client/MillClientMain.java index e6b7d3a3969..5687c8b251c 100644 --- a/main/client/src/mill/main/client/MillClientMain.java +++ b/main/client/src/mill/main/client/MillClientMain.java @@ -221,7 +221,7 @@ public static int run( InputStream outErr = ioSocket.getInputStream(); OutputStream in = ioSocket.getOutputStream(); ProxyStreamPumper outPump = new ProxyStreamPumper(outErr, stdout, stderr); - InputPumper inPump = new InputPumper(stdin, in, true); + InputPumper inPump = new InputPumper(() -> stdin, () -> in, true); Thread outThread = new Thread(outPump, "outPump"); outThread.setDaemon(true); Thread inThread = new Thread(inPump, "inPump"); diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index bbfd8f8818c..30228f5cc82 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -207,17 +207,17 @@ object Jvm extends CoursierSupport { spawnSubprocessWithBackgroundOutputs(commandArgs, envArgs, workingDir, backgroundOutputs) } - /** - * Spawns a generic subprocess, streaming the stdout and stderr to the - * console. If the System.out/System.err have been substituted, makes sure - * that the subprocess's stdout and stderr streams go to the subtituted - * streams. - * - * If the process should be spawned in the background, destination streams for out and err - * respectively must be defined in the backgroundOutputs tuple. Nonbackground process should set - * backgroundOutputs to None - */ - def spawnSubprocessWithBackgroundOutputs( + /** + * Spawns a generic subprocess, streaming the stdout and stderr to the + * console. If the System.out/System.err have been substituted, makes sure + * that the subprocess's stdout and stderr streams go to the subtituted + * streams. + * + * If the process should be spawned in the background, destination streams for out and err + * respectively must be defined in the backgroundOutputs tuple. Nonbackground process should set + * backgroundOutputs to None + */ + def spawnSubprocessWithBackgroundOutputs( commandArgs: Seq[String], envArgs: Map[String, String], workingDir: os.Path, diff --git a/runner/src/mill/runner/MillServerMain.scala b/runner/src/mill/runner/MillServerMain.scala index 53ab81c4e5b..0a8fd606293 100644 --- a/runner/src/mill/runner/MillServerMain.scala +++ b/runner/src/mill/runner/MillServerMain.scala @@ -135,7 +135,7 @@ class Server[T]( val pipedInput = new PipedInputStream() val pipedOutput = new PipedOutputStream() pipedOutput.connect(pipedInput) - val pumper = new InputPumper(in, pipedOutput, false) + val pumper = new InputPumper(() => in, () => pipedOutput, false) val pumperThread = new Thread(pumper, "proxyInputStreamThroughPumper") pumperThread.setDaemon(true) pumperThread.start() diff --git a/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala b/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala index 79ab9b78284..e6b2f0f05b1 100644 --- a/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala +++ b/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala @@ -305,7 +305,7 @@ class ScalaJSWorkerImpl extends ScalaJSWorkerApi { for ((std, dest, name, checkAvailable, runningCheck) <- sources) { val t = new Thread( - new mill.main.client.InputPumper(std, dest, checkAvailable, () => runningCheck()), + new mill.main.client.InputPumper(() => std, () => dest, checkAvailable, () => runningCheck()), name ) t.setDaemon(true) From b5d52fff1650104dcd13ad9cd36689719c5ad10b Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 15:48:10 +0800 Subject: [PATCH 05/17] . From 0946e0880de07089d13f153d7be8370093ccbc41 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 15:54:45 +0800 Subject: [PATCH 06/17] . --- main/api/src/mill/api/SystemStreams.scala | 5 +++-- main/util/src/mill/util/Jvm.scala | 22 +++++++++---------- .../scalajslib/worker/ScalaJSWorkerImpl.scala | 7 +++++- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/main/api/src/mill/api/SystemStreams.scala b/main/api/src/mill/api/SystemStreams.scala index 66b2f7d823d..a1324ab4d31 100644 --- a/main/api/src/mill/api/SystemStreams.scala +++ b/main/api/src/mill/api/SystemStreams.scala @@ -2,6 +2,7 @@ package mill.api import java.io.{InputStream, OutputStream, PrintStream} import mill.main.client.InputPumper + /** * Represents a set of streams that look similar to those provided by the * operating system. These may internally be proxied/redirected/processed, but @@ -48,14 +49,14 @@ object SystemStreams { def originalErr: PrintStream = original.err - private class PumpedProcessInput extends os.ProcessInput{ + private class PumpedProcessInput extends os.ProcessInput { def redirectFrom = ProcessBuilder.Redirect.PIPE def processInput(processIn: => os.SubProcess.InputStream) = Some( new InputPumper(() => System.in, () => processIn, true, () => true) ) } - private class PumpedProcessOutput(dest: OutputStream) extends os.ProcessOutput{ + private class PumpedProcessOutput(dest: OutputStream) extends os.ProcessOutput { def redirectTo = ProcessBuilder.Redirect.PIPE def processOutput(processOut: => os.SubProcess.OutputStream) = Some(new InputPumper(() => processOut, () => dest, false, () => true)) diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index 30228f5cc82..bbfd8f8818c 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -207,17 +207,17 @@ object Jvm extends CoursierSupport { spawnSubprocessWithBackgroundOutputs(commandArgs, envArgs, workingDir, backgroundOutputs) } - /** - * Spawns a generic subprocess, streaming the stdout and stderr to the - * console. If the System.out/System.err have been substituted, makes sure - * that the subprocess's stdout and stderr streams go to the subtituted - * streams. - * - * If the process should be spawned in the background, destination streams for out and err - * respectively must be defined in the backgroundOutputs tuple. Nonbackground process should set - * backgroundOutputs to None - */ - def spawnSubprocessWithBackgroundOutputs( + /** + * Spawns a generic subprocess, streaming the stdout and stderr to the + * console. If the System.out/System.err have been substituted, makes sure + * that the subprocess's stdout and stderr streams go to the subtituted + * streams. + * + * If the process should be spawned in the background, destination streams for out and err + * respectively must be defined in the backgroundOutputs tuple. Nonbackground process should set + * backgroundOutputs to None + */ + def spawnSubprocessWithBackgroundOutputs( commandArgs: Seq[String], envArgs: Map[String, String], workingDir: os.Path, diff --git a/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala b/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala index e6b2f0f05b1..d6f2164ff72 100644 --- a/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala +++ b/scalajslib/worker/1/src/mill/scalajslib/worker/ScalaJSWorkerImpl.scala @@ -305,7 +305,12 @@ class ScalaJSWorkerImpl extends ScalaJSWorkerApi { for ((std, dest, name, checkAvailable, runningCheck) <- sources) { val t = new Thread( - new mill.main.client.InputPumper(() => std, () => dest, checkAvailable, () => runningCheck()), + new mill.main.client.InputPumper( + () => std, + () => dest, + checkAvailable, + () => runningCheck() + ), name ) t.setDaemon(true) From 4afb90768e1816b2f04d0bfdbc394acf657c124a Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 16:00:47 +0800 Subject: [PATCH 07/17] . From e83b43c82f1adbf966647c84a606dd9be6819486 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 16:14:34 +0800 Subject: [PATCH 08/17] . From d71fc0b7e326dd64d73dbf7967edc06a6019fe0d Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 16:28:19 +0800 Subject: [PATCH 09/17] . From c1c40efbfede71369659c0cca870b146d8af88b5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 16:37:23 +0800 Subject: [PATCH 10/17] . From 44d6d30d98de0c21076ad769b1209a3ea1bfdf10 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Jul 2024 17:20:57 +0800 Subject: [PATCH 11/17] . --- build.sc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sc b/build.sc index 38377750d19..ad9f5498708 100644 --- a/build.sc +++ b/build.sc @@ -153,7 +153,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val lambdaTest = ivy"de.tototec:de.tobiasroeser.lambdatest:0.8.0" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.10.2-7-7492bb-DIRTY7fe7ee8e" + val osLib = ivy"com.lihaoyi::os-lib:0.10.2-11-6f1c69" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.0" val millModuledefsVersion = "0.10.9" From 63ecea461b211ee65b03ecd95e6550d4b51e68a3 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 20 Jul 2024 10:56:59 +0800 Subject: [PATCH 12/17] Update build.sc --- build.sc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sc b/build.sc index ad9f5498708..3500dc04bf0 100644 --- a/build.sc +++ b/build.sc @@ -153,7 +153,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val lambdaTest = ivy"de.tototec:de.tobiasroeser.lambdatest:0.8.0" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.10.2-11-6f1c69" + val osLib = ivy"com.lihaoyi::os-lib:0.10.2-15-22c94a" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.0" val millModuledefsVersion = "0.10.9" From 16556b048cc36cbc4177503bef4cc480df1bcc64 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Jul 2024 17:31:11 +0800 Subject: [PATCH 13/17] . --- .../feature/subprocess-stdout/repo/build.sc | 18 +++++ .../feature/subprocess-stdout/repo/mill | 67 +++++++++++++++++ .../test/src/SubprocessStdoutTests.scala | 74 +++++++++++++++++++ .../integration/IntegrationTestSuite.scala | 31 ++++++-- 4 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 integration/feature/subprocess-stdout/repo/build.sc create mode 100755 integration/feature/subprocess-stdout/repo/mill create mode 100644 integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala diff --git a/integration/feature/subprocess-stdout/repo/build.sc b/integration/feature/subprocess-stdout/repo/build.sc new file mode 100644 index 00000000000..0e20b33b049 --- /dev/null +++ b/integration/feature/subprocess-stdout/repo/build.sc @@ -0,0 +1,18 @@ +import mill._ + + +def inheritInterleaved = T { + for (i <- Range.inclusive(1, 9)) { + println("print stdout" + i) + os.proc("echo", "proc stdout" + i).call(stdout = os.Inherit) + System.err.println("print stderr" + i) + os.proc("bash", "-c", s"echo proc stderr${i} >&2").call(stderr = os.Inherit) + } +} + +def inheritRaw = T{ + println("print stdoutRaw") + os.proc("echo", "proc stdoutRaw").call(stdout = os.InheritRaw) + System.err.println("print stderrRaw") + os.proc("bash", "-c", "echo proc stderrRaw >&2").call(stderr = os.InheritRaw) +} \ No newline at end of file diff --git a/integration/feature/subprocess-stdout/repo/mill b/integration/feature/subprocess-stdout/repo/mill new file mode 100755 index 00000000000..d3055fe23ef --- /dev/null +++ b/integration/feature/subprocess-stdout/repo/mill @@ -0,0 +1,67 @@ +#!/usr/bin/env sh + +# This is a wrapper script, that automatically download mill from GitHub release pages +# You can give the required mill version with MILL_VERSION env variable +# If no version is given, it falls back to the value of DEFAULT_MILL_VERSION + +set -e + +if [ -z "${DEFAULT_MILL_VERSION}" ] ; then + DEFAULT_MILL_VERSION=0.11.6 +fi + +if [ -z "$MILL_VERSION" ] ; then + if [ -f ".mill-version" ] ; then + MILL_VERSION="$(head -n 1 .mill-version 2> /dev/null)" + elif [ -f ".config/mill-version" ] ; then + MILL_VERSION="$(head -n 1 .config/mill-version 2> /dev/null)" + elif [ -f "mill" ] && [ "$0" != "mill" ] ; then + MILL_VERSION=$(grep -F "DEFAULT_MILL_VERSION=" "mill" | head -n 1 | cut -d= -f2) + else + MILL_VERSION=$DEFAULT_MILL_VERSION + fi +fi + +if [ "x${XDG_CACHE_HOME}" != "x" ] ; then + MILL_DOWNLOAD_PATH="${XDG_CACHE_HOME}/mill/download" +else + MILL_DOWNLOAD_PATH="${HOME}/.cache/mill/download" +fi +MILL_EXEC_PATH="${MILL_DOWNLOAD_PATH}/${MILL_VERSION}" + +version_remainder="$MILL_VERSION" +MILL_MAJOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}" +MILL_MINOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}" + +if [ ! -s "$MILL_EXEC_PATH" ] ; then + mkdir -p "$MILL_DOWNLOAD_PATH" + if [ "$MILL_MAJOR_VERSION" -gt 0 ] || [ "$MILL_MINOR_VERSION" -ge 5 ] ; then + ASSEMBLY="-assembly" + fi + DOWNLOAD_FILE=$MILL_EXEC_PATH-tmp-download + MILL_VERSION_TAG=$(echo $MILL_VERSION | sed -E 's/([^-]+)(-M[0-9]+)?(-.*)?/\1\2/') + MILL_DOWNLOAD_URL="https://repo1.maven.org/maven2/com/lihaoyi/mill-dist/$MILL_VERSION/mill-dist-$MILL_VERSION.jar" + curl --fail -L -o "$DOWNLOAD_FILE" "$MILL_DOWNLOAD_URL" + chmod +x "$DOWNLOAD_FILE" + mv "$DOWNLOAD_FILE" "$MILL_EXEC_PATH" + unset DOWNLOAD_FILE + unset MILL_DOWNLOAD_URL +fi + +if [ -z "$MILL_MAIN_CLI" ] ; then + MILL_MAIN_CLI="${0}" +fi + +MILL_FIRST_ARG="" + + # first arg is a long flag for "--interactive" or starts with "-i" +if [ "$1" = "--bsp" ] || [ "${1#"-i"}" != "$1" ] || [ "$1" = "--interactive" ] || [ "$1" = "--no-server" ] || [ "$1" = "--repl" ] || [ "$1" = "--help" ] ; then + # Need to preserve the first position of those listed options + MILL_FIRST_ARG=$1 + shift +fi + +unset MILL_DOWNLOAD_PATH +unset MILL_VERSION + +exec $MILL_EXEC_PATH $MILL_FIRST_ARG -D "mill.main.cli=${MILL_MAIN_CLI}" "$@" diff --git a/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala b/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala new file mode 100644 index 00000000000..65f323d9d9b --- /dev/null +++ b/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala @@ -0,0 +1,74 @@ +package mill.integration + +import utest._ + +object SubprocessStdoutTests extends IntegrationTestSuite { + val tests: Tests = Tests { + initWorkspace() + + test { + val res1 = evalStdCombined("inheritInterleaved").out + // Make sure that when a lot of printed/inherited stdout/stderr is printed + // in quick succession, the output ordering is preserved and it doesn't get + // jumbled up + assert( + res1.contains( + s"""print stdout1 + |proc stdout1 + |print stderr1 + |proc stderr1 + |print stdout2 + |proc stdout2 + |print stderr2 + |proc stderr2 + |print stdout3 + |proc stdout3 + |print stderr3 + |proc stderr3 + |print stdout4 + |proc stdout4 + |print stderr4 + |proc stderr4 + |print stdout5 + |proc stdout5 + |print stderr5 + |proc stderr5 + |print stdout6 + |proc stdout6 + |print stderr6 + |proc stderr6 + |print stdout7 + |proc stdout7 + |print stderr7 + |proc stderr7 + |print stdout8 + |proc stdout8 + |print stderr8 + |proc stderr8 + |print stdout9 + |proc stdout9 + |print stderr9 + |proc stderr9""".stripMargin + ) + ) + + // Make sure subprocess output that isn't captures by all of Mill's stdout/stderr/os.Inherit + // redirects still gets pikced up from the stdout/stderr log files and displayed. They may + // be out of order from the original Mill stdout/stderr, but they should still at least turn + // up in the console somewhere and not disappear + // + // Note that it should be out of order, because both `print`s will be captured and logged first, + // whereas the two `proc` outputs will get sent to their respective log files and only noticed + // a few milliseconds later as the files are polled for updates + val res2 = evalStdCombined("inheritRaw").out + assert( + res2.contains( + """print stdoutRaw + |print stderrRaw + |proc stdoutRaw + |proc stderrRaw""".stripMargin + ) + ) + } + } +} diff --git a/integration/src/mill/integration/IntegrationTestSuite.scala b/integration/src/mill/integration/IntegrationTestSuite.scala index 9c9cddd1997..6c0cc0c4a1f 100644 --- a/integration/src/mill/integration/IntegrationTestSuite.scala +++ b/integration/src/mill/integration/IntegrationTestSuite.scala @@ -5,7 +5,7 @@ import mill.resolve.SelectMode import mill.runner.RunnerState import os.{Path, Shellable} import utest._ - +import collection.mutable import scala.util.control.NonFatal object IntegrationTestSuite { @@ -38,20 +38,37 @@ abstract class IntegrationTestSuite extends TestSuite { } def evalTimeoutStdout(timeout: Long, s: Shellable*): IntegrationTestSuite.EvalResult = { + val output = mutable.Buffer.empty[String] + val error = mutable.Buffer.empty[String] + + evalTimeoutStdout0(timeout, output, error) + + } - val output = Seq.newBuilder[String] - val error = Seq.newBuilder[String] - val processOutput = os.ProcessOutput.Readlines(output += _) - val processError = os.ProcessOutput.Readlines(error += _) + def evalTimeoutStdout0(timeout: Long, + output: mutable.Buffer[String], + error: mutable.Buffer[String], + s: Shellable*) = { + + val processOutput = os.ProcessOutput.Readlines(s => synchronized(output.append(s))) + val processError = os.ProcessOutput.Readlines(s => synchronized(error.append(s))) val result = evalFork(processOutput, processError, s, timeout) + IntegrationTestSuite.EvalResult( result, - output.result().mkString("\n"), - error.result().mkString("\n") + synchronized(output.mkString("\n")), + synchronized(error.mkString("\n")) ) } + // Combines stdout and stderr into a single stream; useful for testing + // against the combined output and also asserting on ordering + def evalStdCombined(s: Shellable*): IntegrationTestSuite.EvalResult = { + val combined = mutable.Buffer.empty[String] + evalTimeoutStdout0(-1, combined, combined, s:_*) + } + val millReleaseFileOpt: Option[Path] = Option(System.getenv("MILL_TEST_LAUNCHER")).map(os.Path(_, os.pwd)) From 5dc6e6ad7bec2698bec9f89b58006eee69b59b7d Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Jul 2024 17:40:50 +0800 Subject: [PATCH 14/17] . --- .../src/mill/integration/IntegrationTestSuite.scala | 12 +++++++----- main/api/src/mill/api/SystemStreams.scala | 4 ++-- main/util/src/mill/util/Jvm.scala | 1 - 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/integration/src/mill/integration/IntegrationTestSuite.scala b/integration/src/mill/integration/IntegrationTestSuite.scala index 6c0cc0c4a1f..ba9dce80220 100644 --- a/integration/src/mill/integration/IntegrationTestSuite.scala +++ b/integration/src/mill/integration/IntegrationTestSuite.scala @@ -45,10 +45,12 @@ abstract class IntegrationTestSuite extends TestSuite { } - def evalTimeoutStdout0(timeout: Long, - output: mutable.Buffer[String], - error: mutable.Buffer[String], - s: Shellable*) = { + def evalTimeoutStdout0( + timeout: Long, + output: mutable.Buffer[String], + error: mutable.Buffer[String], + s: Shellable* + ): IntegrationTestSuite.EvalResult = { val processOutput = os.ProcessOutput.Readlines(s => synchronized(output.append(s))) val processError = os.ProcessOutput.Readlines(s => synchronized(error.append(s))) @@ -66,7 +68,7 @@ abstract class IntegrationTestSuite extends TestSuite { // against the combined output and also asserting on ordering def evalStdCombined(s: Shellable*): IntegrationTestSuite.EvalResult = { val combined = mutable.Buffer.empty[String] - evalTimeoutStdout0(-1, combined, combined, s:_*) + evalTimeoutStdout0(-1, combined, combined, s: _*) } val millReleaseFileOpt: Option[Path] = diff --git a/main/api/src/mill/api/SystemStreams.scala b/main/api/src/mill/api/SystemStreams.scala index a1324ab4d31..2389ebee320 100644 --- a/main/api/src/mill/api/SystemStreams.scala +++ b/main/api/src/mill/api/SystemStreams.scala @@ -51,14 +51,14 @@ object SystemStreams { private class PumpedProcessInput extends os.ProcessInput { def redirectFrom = ProcessBuilder.Redirect.PIPE - def processInput(processIn: => os.SubProcess.InputStream) = Some( + def processInput(processIn: => os.SubProcess.InputStream): Some[InputPumper] = Some( new InputPumper(() => System.in, () => processIn, true, () => true) ) } private class PumpedProcessOutput(dest: OutputStream) extends os.ProcessOutput { def redirectTo = ProcessBuilder.Redirect.PIPE - def processOutput(processOut: => os.SubProcess.OutputStream) = + def processOutput(processOut: => os.SubProcess.OutputStream): Some[InputPumper] = Some(new InputPumper(() => processOut, () => dest, false, () => true)) } def withStreams[T](systemStreams: SystemStreams)(t: => T): T = { diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index bbfd8f8818c..78417d35ed7 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -2,7 +2,6 @@ package mill.util import mill.api.Loose.Agg import mill.api._ -import mill.main.client.InputPumper import os.{ProcessOutput, SubProcess} import java.io._ From 223ed8fa427ed0f8896a79ae58ad3bbe941853e6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Jul 2024 17:55:00 +0800 Subject: [PATCH 15/17] . --- integration/src/mill/integration/IntegrationTestSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/src/mill/integration/IntegrationTestSuite.scala b/integration/src/mill/integration/IntegrationTestSuite.scala index ba9dce80220..b2224b610a5 100644 --- a/integration/src/mill/integration/IntegrationTestSuite.scala +++ b/integration/src/mill/integration/IntegrationTestSuite.scala @@ -41,7 +41,7 @@ abstract class IntegrationTestSuite extends TestSuite { val output = mutable.Buffer.empty[String] val error = mutable.Buffer.empty[String] - evalTimeoutStdout0(timeout, output, error) + evalTimeoutStdout0(timeout, output, error, s) } @@ -49,7 +49,7 @@ abstract class IntegrationTestSuite extends TestSuite { timeout: Long, output: mutable.Buffer[String], error: mutable.Buffer[String], - s: Shellable* + s: Seq[Shellable] ): IntegrationTestSuite.EvalResult = { val processOutput = os.ProcessOutput.Readlines(s => synchronized(output.append(s))) @@ -68,7 +68,7 @@ abstract class IntegrationTestSuite extends TestSuite { // against the combined output and also asserting on ordering def evalStdCombined(s: Shellable*): IntegrationTestSuite.EvalResult = { val combined = mutable.Buffer.empty[String] - evalTimeoutStdout0(-1, combined, combined, s: _*) + evalTimeoutStdout0(-1, combined, combined, s) } val millReleaseFileOpt: Option[Path] = From 9e58570429d2d25e79f18a1eaf35379465a210c1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Jul 2024 21:01:50 +0800 Subject: [PATCH 16/17] . --- .../test/src/SubprocessStdoutTests.scala | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala b/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala index 65f323d9d9b..d0d353f5643 100644 --- a/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala +++ b/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala @@ -57,18 +57,31 @@ object SubprocessStdoutTests extends IntegrationTestSuite { // be out of order from the original Mill stdout/stderr, but they should still at least turn // up in the console somewhere and not disappear // - // Note that it should be out of order, because both `print`s will be captured and logged first, - // whereas the two `proc` outputs will get sent to their respective log files and only noticed - // a few milliseconds later as the files are polled for updates val res2 = evalStdCombined("inheritRaw").out - assert( - res2.contains( - """print stdoutRaw - |print stderrRaw - |proc stdoutRaw - |proc stderrRaw""".stripMargin + if (integrationTestMode == "fork") { + // For `fork` tests, which represent `-i`/`--interactive`/`--no-server`, the output should + // be properly ordered since it all comes directly from the stdout/stderr of the same process + assert( + res2.contains( + """print stdoutRaw + |proc stdoutRaw + |print stderrRaw + |proc stderrRaw""".stripMargin + ) ) - ) + }else{ + // Note that it should be out of order, because both `print`s will be captured and logged first, + // whereas the two `proc` outputs will get sent to their respective log files and only noticed + // a few milliseconds later as the files are polled for updates + assert( + res2.contains( + """print stdoutRaw + |print stderrRaw + |proc stdoutRaw + |proc stderrRaw""".stripMargin + ) + ) + } } } } From 7504241a31c30e04803f48a712126362eb7cdd8c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Jul 2024 22:13:34 +0800 Subject: [PATCH 17/17] . --- build.sc | 2 +- .../subprocess-stdout/test/src/SubprocessStdoutTests.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sc b/build.sc index 3500dc04bf0..a093c76ae4a 100644 --- a/build.sc +++ b/build.sc @@ -153,7 +153,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val lambdaTest = ivy"de.tototec:de.tobiasroeser.lambdatest:0.8.0" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.10.2-15-22c94a" + val osLib = ivy"com.lihaoyi::os-lib:0.10.3" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.0" val millModuledefsVersion = "0.10.9" diff --git a/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala b/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala index d0d353f5643..958376340d9 100644 --- a/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala +++ b/integration/feature/subprocess-stdout/test/src/SubprocessStdoutTests.scala @@ -69,7 +69,7 @@ object SubprocessStdoutTests extends IntegrationTestSuite { |proc stderrRaw""".stripMargin ) ) - }else{ + } else { // Note that it should be out of order, because both `print`s will be captured and logged first, // whereas the two `proc` outputs will get sent to their respective log files and only noticed // a few milliseconds later as the files are polled for updates