diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 3090a3b10a97c..cdd59d4108524 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -245,11 +245,11 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S error("Must specify a primary resource (JAR or Python or R file)") } if (driverMemory != null - && Try(JavaUtils.byteStringAsBytes(driverMemory)).getOrElse(-1L) <= 0) { + && Try(JavaUtils.byteStringAsMb(driverMemory)).getOrElse(-1L) <= 0) { error("Driver memory must be a positive number") } if (executorMemory != null - && Try(JavaUtils.byteStringAsBytes(executorMemory)).getOrElse(-1L) <= 0) { + && Try(JavaUtils.byteStringAsMb(executorMemory)).getOrElse(-1L) <= 0) { error("Executor memory must be a positive number") } if (executorCores != null && Try(executorCores.toInt).getOrElse(-1) <= 0) { diff --git a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala index 7282a83f0739f..ebd32459e474a 100644 --- a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala +++ b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala @@ -222,7 +222,7 @@ object UnifiedMemoryManager { } // SPARK-12759 Check executor memory to fail fast if memory is insufficient if (conf.contains(config.EXECUTOR_MEMORY)) { - val executorMemory = conf.getSizeAsBytes(config.EXECUTOR_MEMORY.key) + val executorMemory = conf.getSizeAsMb(config.EXECUTOR_MEMORY.key) if (executorMemory < minSystemMemory) { throw new IllegalArgumentException(s"Executor memory $executorMemory must be at least " + s"$minSystemMemory. Please increase executor memory using the " + diff --git a/docs/configuration.md b/docs/configuration.md index 706c2552b1d17..a7b7f417dc16c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -172,7 +172,7 @@ of the most common options to set are: Amount of memory to use for the driver process, i.e. where SparkContext is initialized, in the same format as JVM memory strings with a size unit suffix ("k", "m", "g" or "t") - (e.g. 512m, 2g). + (e.g. 512m, 2g) using "m" as the default unit.
Note: In client mode, this config must not be set through the SparkConf directly in your application, because the driver JVM has already started at that point. @@ -249,7 +249,8 @@ of the most common options to set are: 1g Amount of memory to use per executor process, in the same format as JVM memory strings with - a size unit suffix ("k", "m", "g" or "t") (e.g. 512m, 2g). + a size unit suffix ("k", "m", "g" or "t") (e.g. 512m, 2g) using + "m" as the default unit. 0.7.0 diff --git a/docs/monitoring.md b/docs/monitoring.md index 32959b77c4773..45edf922af521 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -70,7 +70,11 @@ The history server can be configured as follows: Environment VariableMeaning SPARK_DAEMON_MEMORY - Memory to allocate to the history server (default: 1g). + + Memory to allocate to the history server (default: 1g). This can be configured in the same + format as JVM memory strings with a size unit suffix ("k", "m", "g" or "t") using "m" as + the default unit. + SPARK_DAEMON_JAVA_OPTS diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md index f3c479ba26547..9fc6377d4f06e 100644 --- a/docs/spark-standalone.md +++ b/docs/spark-standalone.md @@ -144,7 +144,10 @@ You can optionally configure the cluster further by setting environment variable SPARK_WORKER_MEMORY - Total amount of memory to allow Spark applications to use on the machine, e.g. 1000m, 2g (default: total memory minus 1 GiB); note that each application's individual memory is configured using its spark.executor.memory property. + + Total amount of memory to allow Spark applications to use on the machine, e.g. 1000m, 2g (default: total memory minus 1 GiB); note that each application's individual memory is configured using its spark.executor.memory property. + This can be configured in the same format as JVM memory strings with a size unit suffix ("k", "m", "g" or "t") using "m" as the default unit. + SPARK_WORKER_PORT @@ -164,7 +167,10 @@ You can optionally configure the cluster further by setting environment variable SPARK_DAEMON_MEMORY - Memory to allocate to the Spark master and worker daemons themselves (default: 1g). + + Memory to allocate to the Spark master and worker daemons themselves (default: 1g). This can be configured in the same + format as JVM memory strings with a size unit suffix ("k", "m", "g" or "t") using "m" as the default unit. + SPARK_DAEMON_JAVA_OPTS diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java index 172fb8c560876..fdc93c2d6e601 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java +++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java @@ -328,4 +328,17 @@ static String findJarsDir(String sparkHome, String scalaVersion, boolean failIfN return libdir.getAbsolutePath(); } + /** + * Add "m" as the default suffix unit when no explicit unit is given. + */ + static String addDefaultMSuffixIfNeeded(String memoryString) { + if (memoryString.chars().allMatch(Character::isDigit)) { + System.err.println("Memory setting without explicit unit (" + + memoryString + ") is taken to be in MB by default! For details check SPARK-32293."); + return memoryString + "m"; + } else { + return memoryString; + } + } + } diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java index fd056bb90e0c4..4ce9071c5eae8 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java @@ -108,7 +108,7 @@ public List buildCommand(Map env) } String mem = firstNonEmpty(memKey != null ? System.getenv(memKey) : null, DEFAULT_MEM); - cmd.add("-Xmx" + mem); + cmd.add("-Xmx" + addDefaultMSuffixIfNeeded(mem)); cmd.add(className); cmd.addAll(classArgs); return cmd; diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 383c3f60a595b..b62130afb8edd 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -285,7 +285,7 @@ private List buildSparkSubmitCommand(Map env) isThriftServer(mainClass) ? System.getenv("SPARK_DAEMON_MEMORY") : null; String memory = firstNonEmpty(tsMemory, config.get(SparkLauncher.DRIVER_MEMORY), System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM); - cmd.add("-Xmx" + memory); + cmd.add("-Xmx" + addDefaultMSuffixIfNeeded(memory)); addOptionString(cmd, driverDefaultJavaOptions); addOptionString(cmd, driverExtraJavaOptions); mergeEnvPathList(env, getLibPathEnvName(), diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 752e8d4c23f8b..da221b2e0c667 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -115,6 +115,22 @@ public void testCliParser() throws Exception { Collections.indexOfSubList(cmd, Arrays.asList(parser.CONF, "spark.randomOption=foo")) > 0); } + @Test + public void testParserWithDefaultUnit() throws Exception { + List sparkSubmitArgs = Arrays.asList( + parser.MASTER, + "local", + parser.DRIVER_MEMORY, + "4200", + parser.DRIVER_CLASS_PATH, + "/driverCp", + SparkLauncher.NO_RESOURCE); + Map env = new HashMap<>(); + List cmd = buildCommand(sparkSubmitArgs, env); + + assertTrue("Driver -Xmx should be configured in MB by default.", cmd.contains("-Xmx4200m")); + } + @Test public void testShellCliParser() throws Exception { List sparkSubmitArgs = Arrays.asList( diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh index 813a70c6e7ec3..fdcfe6767074b 100755 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh @@ -78,11 +78,16 @@ case "$1" in ;; executor) shift 1 + MEMORY_WITH_UNIT=$SPARK_EXECUTOR_MEMORY + if [[ $MEMORY_WITH_UNIT =~ ^[0-9]+$ ]] + then + MEMORY_WITH_UNIT="${MEMORY_WITH_UNIT}m" + fi CMD=( ${JAVA_HOME}/bin/java "${SPARK_EXECUTOR_JAVA_OPTS[@]}" - -Xms$SPARK_EXECUTOR_MEMORY - -Xmx$SPARK_EXECUTOR_MEMORY + -Xms$MEMORY_WITH_UNIT + -Xmx$MEMORY_WITH_UNIT -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH" org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL