Skip to content

Commit 7e7a54f

Browse files
lihaoyilihaoyi
authored
Allow .mill-jvm-opts to interpolate environment variables, add .mill-opts (#3841)
This PR extends `.mill-jvm-opts` to support interpolating JVM variables. This provides a bit of flexibility in allowing `$PWD` and `$USER` to be used when passing `-Dproperty=` to Mill. This could in theory be fleshed out even further to run arbitrary shell snippets, but for now I this should provide enough flexibility to meet the immediate needs. Also added a `.mill-opts`, which is similar to `.mill-jvm-opts` but used for Mill arguments rather than JVM arguments. This provides a way to persistent required Mill configuration in a format that is similar to what we already have today (CLI args). This would be a common place to put things like `--jobs=0.5C` or `--allow-positional` or `--no-build-lock`. Updated the integration tests and documentation related discussions * #1226 * #3840
1 parent 1d0d5fd commit 7e7a54f

File tree

15 files changed

+145
-91
lines changed

15 files changed

+145
-91
lines changed

‎build.mill

+4-71
Original file line numberDiff line numberDiff line change
@@ -642,81 +642,27 @@ def launcherScript(
642642
Jvm.universalScript(
643643
shellCommands = {
644644
val jvmArgsStr = shellJvmArgs.mkString(" ")
645-
def java(mainClass: String, passMillJvmOpts: Boolean) = {
646-
val millJvmOpts = if (passMillJvmOpts) "$mill_jvm_opts" else ""
647-
s"""exec "$$JAVACMD" $jvmArgsStr $$JAVA_OPTS $millJvmOpts -cp "${shellClassPath.mkString(
648-
":"
649-
)}" $mainClass "$$@""""
650-
}
645+
val classpathStr = shellClassPath.mkString(":")
651646

652647
s"""if [ -z "$$JAVA_HOME" ] ; then
653648
| JAVACMD="java"
654649
|else
655650
| JAVACMD="$$JAVA_HOME/bin/java"
656651
|fi
657652
|
658-
|mill_jvm_opts=""
659-
|init_mill_jvm_opts () {
660-
| if [ -z $$MILL_JVM_OPTS_PATH ] ; then
661-
| mill_jvm_opts_file=".mill-jvm-opts"
662-
| else
663-
| mill_jvm_opts_file=$$MILL_JVM_OPTS_PATH
664-
| fi
665-
|
666-
| if [ -f "$$mill_jvm_opts_file" ] ; then
667-
| # We need to append a newline at the end to fix
668-
| # https://github.com/com-lihaoyi/mill/issues/2140
669-
| newline="
670-
|"
671-
| mill_jvm_opts="$$(
672-
| echo "$$newline" | cat "$$mill_jvm_opts_file" - | (
673-
| while IFS= read line
674-
| do
675-
| mill_jvm_opts="$${mill_jvm_opts} $$(echo $$line | grep -v "^[[:space:]]*[#]")"
676-
| done
677-
| # we are in a sub-shell, so need to return it explicitly
678-
| echo "$${mill_jvm_opts}"
679-
| )
680-
| )"
681-
| mill_jvm_opts="$${mill_jvm_opts} -Dmill.jvm_opts_applied=true"
682-
| fi
683-
|}
684-
|
685653
|# Client-server mode doesn't seem to work on WSL, just disable it for now
686654
|# https://stackoverflow.com/a/43618657/871202
687655
|if grep -qEi "(Microsoft|WSL)" /proc/version > /dev/null 2> /dev/null ; then
688-
| init_mill_jvm_opts
689656
| if [ -z $$COURSIER_CACHE ] ; then
690657
| COURSIER_CACHE=.coursier
691658
| fi
692-
| ${java(millMainClass, true)}
693-
|else
694-
| if [ "$${1%"-i"*}" != "$$1" ] ; then # first arg starts with "-i"
695-
| init_mill_jvm_opts
696-
| ${java(millMainClass, true)}
697-
| else
698-
| case "$$1" in
699-
| -i | --interactive | --repl | --no-server | --bsp )
700-
| init_mill_jvm_opts
701-
| ${java(millMainClass, true)}
702-
| ;;
703-
| *)
704-
| ${java(millMainClass, false)}
705-
| ;;
706-
| esac
707-
| fi
708659
|fi
660+
|exec "$$JAVACMD" $jvmArgsStr $$JAVA_OPTS -cp "$classpathStr" $millMainClass "$$@"
709661
|""".stripMargin
710662
},
711663
cmdCommands = {
712664
val jvmArgsStr = cmdJvmArgs.mkString(" ")
713-
def java(mainClass: String, passMillJvmOpts: Boolean) = {
714-
val millJvmOpts = if (passMillJvmOpts) "!mill_jvm_opts!" else ""
715-
s""""%JAVACMD%" $jvmArgsStr %JAVA_OPTS% $millJvmOpts -cp "${cmdClassPath.mkString(
716-
";"
717-
)}" $mainClass %*"""
718-
}
719-
665+
val classpathStr = cmdClassPath.mkString(";")
720666
s"""setlocal EnableDelayedExpansion
721667
|set "JAVACMD=java.exe"
722668
|if not "%JAVA_HOME%"=="" set "JAVACMD=%JAVA_HOME%\\bin\\java.exe"
@@ -726,21 +672,8 @@ def launcherScript(
726672
|if "%1" == "--no-server" set _I_=true
727673
|if "%1" == "--bsp" set _I_=true
728674
|
729-
|set "mill_jvm_opts="
730-
|set "mill_jvm_opts_file=.mill-jvm-opts"
731-
|if not "%MILL_JVM_OPTS_PATH%"=="" set "mill_jvm_opts_file=%MILL_JVM_OPTS_PATH%"
675+
|"%JAVACMD%" $jvmArgsStr %JAVA_OPTS% -cp "$classpathStr" $millMainClass %*
732676
|
733-
|if defined _I_ (
734-
| if exist %mill_jvm_opts_file% (
735-
| for /f "delims=" %%G in (%mill_jvm_opts_file%) do (
736-
| set line=%%G
737-
| if "!line:~0,2!"=="-X" set "mill_jvm_opts=!mill_jvm_opts! !line!"
738-
| )
739-
| )
740-
| ${java(millMainClass, true)}
741-
|) else (
742-
| ${java(millMainClass, false)}
743-
|)
744677
|endlocal
745678
|""".stripMargin
746679
}

‎docs/modules/ROOT/partials/Installation_IDE_Support.adoc

+32-6
Original file line numberDiff line numberDiff line change
@@ -196,18 +196,44 @@ JVM options, one per line.
196196
For example, if your build requires a lot of memory and bigger stack size, your
197197
`.mill-jvm-opts` could look like this:
198198

199+
_.mill-jvm-opts_
199200
----
200201
-Xss10m
201202
-Xmx10G
202203
----
203204

204-
The file name for passing JVM options to the Mill launcher is configurable. If
205-
for some reason you don't want to use `.mill-jvm-opts` file name, add
206-
`MILL_JVM_OPTS_PATH` environment variable with any other file name.
205+
Note that `.mill-jvm-opts` requires each CLI token to be on a separate line, so
206+
`-Xss10m -Xmx10G` on a single line is not allowed (as it would pass `"-Xss10m -Xmx10G"`
207+
as a single token and fail argument parsing)
207208

208-
NOTE: `.mill-jvm-opts` is for passing JVM options to the JVM running Mill itself.
209-
If you want to pass JVM options to the project that Mill is building and running,
210-
see the section on
209+
`.mill-jvm-opts` also supports environment variable interpolation, e.g.
210+
211+
_.mill-jvm-opts_
212+
----
213+
# PWD on mac/linux
214+
-Dmy.jvm.property=$PWD
215+
----
216+
217+
The file name `.mill-jvm-opts` can be overridden via the `MILL_JVM_OPTS_PATH` environment
218+
variable.
219+
220+
== Custom Mill Options
221+
222+
Mill also supports the `.mill-opts` file for passing a default set of command line
223+
options to Mill itself. For example, if your project's tasks are CPU heavy, you
224+
may want everyone using your project to run only 0.5 concurrent tasks per CPU. This
225+
can be done by setting `.mill-opts` to:
226+
227+
_.mill-opts_
228+
----
229+
--jobs=0.5C
230+
----
231+
232+
The file name `.mill-opts` can be overridden via the `MILL_OPTS_PATH` environment variable.
233+
234+
NOTE: `.mill-jvm-opts` is for passing JVM options to the JVM running Mill,
235+
and `.mill-opts` is for passing options to Mill itself. If you want to pass JVM options
236+
to the project that Mill is building and running, see the section on
211237
xref:javalib/module-config.adoc#_compilation_execution_flags[Compilation and Execution Flags].
212238

213239
== Other installation methods

‎integration/feature/mill-jvm-opts/resources/.mill-jvm-opts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11

22
# comment after an empty line
3-
-DPROPERTY_PROPERLY_SET_VIA_JVM_OPTS=value-from-file
3+
-Dproperty.properly.set.via.jvm.opts=value-from-file
4+
-Dproperty.with.interpolated.working.dir=value-with-$PWD
45
-Xss120m

‎integration/feature/mill-jvm-opts/resources/.mill-jvm-opts-alternate

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
2+
# comment after an empty line
3+
-Dproperty.properly.set.via.jvm.opts=alternate-value-from-file
4+
-Dproperty.with.interpolated.working.dir=alternate-value-with-$PWD
5+
-Xss220m

‎integration/feature/mill-jvm-opts/resources/.mill-opts

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
2+
# comment after an empty line
3+
--jobs=17

‎integration/feature/mill-jvm-opts/resources/.mill-opts-alternate

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
2+
# comment after an empty line
3+
--jobs=29

‎integration/feature/mill-jvm-opts/resources/build.mill

+10-2
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,23 @@ import java.lang.management.ManagementFactory
44
import scala.jdk.CollectionConverters._
55

66
def checkJvmOpts() = Task.Command {
7-
val prop = System.getProperty("PROPERTY_PROPERLY_SET_VIA_JVM_OPTS")
7+
val prop = System.getProperty("property.properly.set.via.jvm.opts")
88
if (prop != "value-from-file") sys.error("jvm-opts not correctly applied, value was: " + prop)
99
val runtime = ManagementFactory.getRuntimeMXBean()
1010
val args = runtime.getInputArguments().asScala.toSet
11-
if (!args.contains("-DPROPERTY_PROPERLY_SET_VIA_JVM_OPTS=value-from-file")) {
11+
if (!args.contains("-Dproperty.properly.set.via.jvm.opts=value-from-file")) {
1212
sys.error("jvm-opts not correctly applied, args were: " + args.mkString)
1313
}
1414
if (!args.contains("-Xss120m")) {
1515
sys.error("jvm-opts not correctly applied, args were: " + args.mkString)
1616
}
1717
()
1818
}
19+
20+
def getEnvJvmOpts() = Task.Command {
21+
System.getProperty("property.with.interpolated.working.dir")
22+
}
23+
24+
def getNonJvmOpts() = Task.Command {
25+
Task.ctx().asInstanceOf[mill.api.Ctx.Jobs].jobs
26+
}

‎integration/feature/mill-jvm-opts/src/MillJvmOptsTests.scala

+34-3
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,45 @@
11
package mill.integration
22

3+
import mill.main.client.Util
34
import mill.testkit.UtestIntegrationTestSuite
4-
55
import utest._
66

77
object MillJvmOptsTests extends UtestIntegrationTestSuite {
88
val tests: Tests = Tests {
9-
test("JVM options from file .mill-jvm-opts are properly read") - integrationTest { tester =>
9+
test("simple") - integrationTest { tester =>
10+
import tester._
11+
val res = eval("checkJvmOpts")
12+
assert(res.isSuccess)
13+
}
14+
test("interpolatedEnvVars") - integrationTest { tester =>
15+
if (!Util.isWindows) { // PWD does not exist on windows
16+
import tester._
17+
val res = eval(("show", "getEnvJvmOpts"))
18+
val out = res.out
19+
val expected = "\"value-with-" + tester.workspacePath + "\""
20+
assert(out == expected)
21+
}
22+
}
23+
test("alternate") - integrationTest { tester =>
24+
if (!Util.isWindows) {
25+
import tester._
26+
val res = eval(
27+
("show", "getEnvJvmOpts"),
28+
env = Map("MILL_JVM_OPTS_PATH" -> ".mill-jvm-opts-alternate")
29+
)
30+
assert(res.out == "\"alternate-value-with-" + tester.workspacePath + "\"")
31+
}
32+
}
33+
test("nonJvmOpts") - integrationTest { tester =>
34+
import tester._
35+
val res = eval(("show", "getNonJvmOpts"))
36+
assert(res.out == "17")
37+
}
38+
test("nonJvmOptsAlternate") - integrationTest { tester =>
1039
import tester._
11-
assert(eval("checkJvmOpts").isSuccess)
40+
val res =
41+
eval(("show", "getNonJvmOpts"), env = Map("MILL_OPTS_PATH" -> ".mill-opts-alternate"))
42+
assert(res.out == "29")
1243
}
1344
}
1445
}

‎main/client/src/mill/main/client/EnvVars.java

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class EnvVars {
2121

2222

2323
public static final String MILL_JVM_OPTS_PATH = "MILL_JVM_OPTS_PATH";
24+
public static final String MILL_OPTS_PATH = "MILL_OPTS_PATH";
2425

2526

2627
/**

‎main/client/src/mill/main/client/Util.java

+25-4
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import java.util.List;
1717
import java.util.LinkedList;
1818
import java.util.Scanner;
19+
import java.util.regex.Matcher;
20+
import java.util.regex.Pattern;
1921

2022
public class Util {
2123
// use methods instead of constants to avoid inlining by compiler
@@ -139,14 +141,12 @@ static String sha1Hash(String path) throws NoSuchAlgorithmException {
139141
*/
140142
public static List<String> readOptsFileLines(final File file) {
141143
final List<String> vmOptions = new LinkedList<>();
142-
try (
143-
final Scanner sc = new Scanner(file)
144-
) {
144+
try (final Scanner sc = new Scanner(file)) {
145145
while (sc.hasNextLine()) {
146146
String arg = sc.nextLine();
147147
String trimmed = arg.trim();
148148
if (!trimmed.isEmpty() && !trimmed.startsWith("#")) {
149-
vmOptions.add(arg);
149+
vmOptions.add(interpolateEnvVars(arg, System.getenv()));
150150
}
151151
}
152152
} catch (FileNotFoundException e) {
@@ -155,4 +155,25 @@ public static List<String> readOptsFileLines(final File file) {
155155
return vmOptions;
156156
}
157157

158+
public static String interpolateEnvVars(String input, Map<String, String> env){
159+
Matcher matcher = envInterpolatorPattern.matcher(input);
160+
// StringBuilder to store the result after replacing
161+
StringBuffer result = new StringBuffer();
162+
163+
while (matcher.find()) {
164+
String match = matcher.group(1);
165+
if (match.equals("$")) {
166+
matcher.appendReplacement(result, "\\$");
167+
} else {
168+
String envVarValue = env.containsKey(match) ? env.get(match) : "";
169+
matcher.appendReplacement(result, envVarValue);
170+
}
171+
}
172+
173+
matcher.appendTail(result); // Append the remaining part of the string
174+
return result.toString();
175+
}
176+
177+
static Pattern envInterpolatorPattern = Pattern.compile("\\$(\\$|[A-Z_][A-Z0-9_]*)");
178+
158179
}

‎main/server/src/mill/main/server/Server.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ abstract class Server[T](
7474
os.remove.all(socketPath)
7575

7676
val relFile = socketPath.relativeTo(os.pwd).toNIO.toFile
77-
serverLog("listening on socket " + relFile)
77+
serverLog("listening on socket " + relFile + " " + os.pwd)
7878
// Use relative path because otherwise the full path might be too long for the socket API
7979
val addr = AFUNIXSocketAddress.of(relFile)
8080
AFUNIXServerSocket.bindOn(addr)

‎runner/client/src/mill/runner/client/MillClientMain.java

+12-1
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,16 @@
33
import java.nio.file.Path;
44
import java.nio.file.Paths;
55
import java.util.Arrays;
6+
import java.util.Collections;
7+
68
import mill.main.client.ServerLauncher;
79
import mill.main.client.Util;
810
import mill.main.client.lock.Locks;
911
import mill.main.client.OutFiles;
1012
import mill.main.client.ServerCouldNotBeStarted;
1113

14+
import static mill.runner.client.MillProcessLauncher.millOptsFile;
15+
1216
/**
1317
* This is a Java implementation to speed up repetitive starts.
1418
* A Scala implementation would result in the JVM loading much more classes almost doubling the start-up times.
@@ -36,7 +40,14 @@ public static void main(String[] args) throws Exception {
3640
MillNoServerLauncher.runMain(args);
3741
} else try {
3842
// start in client-server mode
39-
ServerLauncher launcher = new ServerLauncher(System.in, System.out, System.err, System.getenv(), args, null, -1){
43+
java.util.List<String> optsArgs = Util.readOptsFileLines(millOptsFile());
44+
Collections.addAll(optsArgs, args);
45+
46+
ServerLauncher launcher = new ServerLauncher(
47+
System.in, System.out, System.err,
48+
System.getenv(), optsArgs.toArray(new String[0]),
49+
null, -1
50+
){
4051
public void initServer(Path serverDir, boolean setJnaNoSys, Locks locks) throws Exception{
4152
MillProcessLauncher.launchMillServer(serverDir, setJnaNoSys);
4253
}

‎runner/client/src/mill/runner/client/MillProcessLauncher.java

+9
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ static int launchMillNoServer(String[] args) throws Exception {
2929
l.addAll(millLaunchJvmCommand(setJnaNoSys));
3030
l.add("mill.runner.MillMain");
3131
l.add(processDir.toAbsolutePath().toString());
32+
l.addAll(Util.readOptsFileLines(millOptsFile()));
3233
l.addAll(Arrays.asList(args));
3334

3435
final ProcessBuilder builder = new ProcessBuilder()
@@ -91,6 +92,14 @@ static File millJvmOptsFile() {
9192
return new File(millJvmOptsPath).getAbsoluteFile();
9293
}
9394

95+
static File millOptsFile() {
96+
String millJvmOptsPath = System.getenv(EnvVars.MILL_OPTS_PATH);
97+
if (millJvmOptsPath == null || millJvmOptsPath.trim().equals("")) {
98+
millJvmOptsPath = ".mill-opts";
99+
}
100+
return new File(millJvmOptsPath).getAbsoluteFile();
101+
}
102+
94103
static boolean millJvmOptsAlreadyApplied() {
95104
final String propAppliedProp = System.getProperty("mill.jvm_opts_applied");
96105
return propAppliedProp != null && propAppliedProp.equals("true");

‎testkit/src/mill/testkit/ExampleTester.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package mill.testkit
22
import mill.util.Util
3+
import mill.main.client.EnvVars.MILL_TEST_SUITE
34
import utest._
45

56
/**
@@ -114,7 +115,7 @@ class ExampleTester(
114115
stderr = os.Pipe,
115116
cwd = workspacePath,
116117
mergeErrIntoOut = true,
117-
env = Map("MILL_TEST_SUITE" -> this.getClass().toString()),
118+
env = Map(MILL_TEST_SUITE -> this.getClass().toString()),
118119
check = false
119120
)
120121

0 commit comments

Comments
 (0)