From ef6c505520d80b898dc5c7a4f2f3bdd8a4c17fe3 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 22 May 2019 11:00:40 +0800 Subject: [PATCH 01/12] improve docs about overhead. --- .../org/apache/spark/internal/config/package.scala | 4 ++-- docs/configuration.md | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 0aed1af023f83..b2bcdbc3573f0 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -60,7 +60,7 @@ package object config { .createWithDefaultString("1g") private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead") - .doc("The amount of off-heap memory to be allocated per driver in cluster mode, " + + .doc("Amount of memory to be allocated outside the driver process in cluster mode, " + "in MiB unless otherwise specified.") .bytesConf(ByteUnit.MiB) .createOptional @@ -185,7 +185,7 @@ package object config { .createWithDefaultString("1g") private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead") - .doc("The amount of off-heap memory to be allocated per executor in cluster mode, " + + .doc("Amount of memory to be allocated outside per executor process in cluster mode, " + "in MiB unless otherwise specified.") .bytesConf(ByteUnit.MiB) .createOptional diff --git a/docs/configuration.md b/docs/configuration.md index d0b2699a5dc77..0ab0fc2296442 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -181,7 +181,7 @@ of the most common options to set are: spark.driver.memoryOverhead driverMemory * 0.10, with minimum of 384 - The amount of off-heap memory to be allocated per driver in cluster mode, in MiB unless + Amount of memory to be allocated outside the driver process in cluster mode, in MiB unless otherwise specified. This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the container size (typically 6-10%). This option is currently supported on YARN, Mesos and Kubernetes. @@ -215,10 +215,10 @@ of the most common options to set are: spark.executor.memoryOverhead executorMemory * 0.10, with minimum of 384 - The amount of off-heap memory to be allocated per executor, in MiB unless otherwise specified. - This is memory that accounts for things like VM overheads, interned strings, other native - overheads, etc. This tends to grow with the executor size (typically 6-10%). - This option is currently supported on YARN and Kubernetes. + Amount of memory to be allocated outside per executor process in cluster mode, in MiB + unless otherwise specified.This is memory that accounts for things like VM overheads, + interned strings, other native overheads, etc. This tends to grow with the executor + size (typically 6-10%).This option is currently supported on YARN and Kubernetes. From d018b35be6e1ba00e6d4c121554daddc3e3bf4ae Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 22 May 2019 16:33:41 +0800 Subject: [PATCH 02/12] expansion of the docs to clarify memoryOverhead. --- .../spark/internal/config/package.scala | 21 +++++++++---- docs/configuration.md | 30 ++++++++++++++----- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index b2bcdbc3573f0..a8734483eeaf1 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -60,8 +60,12 @@ package object config { .createWithDefaultString("1g") private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead") - .doc("Amount of memory to be allocated outside the driver process in cluster mode, " + - "in MiB unless otherwise specified.") + .doc("Amount of non-heap memory to be allocated per driver process in cluster mode" + + " (e.g YARN and Kubernetes), in MiB unless otherwise specified." + + "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + + " and memory used by other non-driver processes running in the same container." + + "The maximum memory size of container to running driver is determined by the sum of" + + " spark.driver.memoryOverhead and spark.driver.memory.") .bytesConf(ByteUnit.MiB) .createOptional @@ -185,8 +189,12 @@ package object config { .createWithDefaultString("1g") private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead") - .doc("Amount of memory to be allocated outside per executor process in cluster mode, " + - "in MiB unless otherwise specified.") + .doc("Amount of non-heap memory to be allocated per executor process in cluster mode " + + "(e.g YARN and Kubernetes), in MiB unless otherwise specified." + + "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + + " and memory used by other non-executor processes running in the same container." + "The maximum memory size of container to running executor is determined by the sum of " + + "spark.executor.memoryOverhead and spark.executor.memory.") .bytesConf(ByteUnit.MiB) .createOptional @@ -201,7 +209,10 @@ package object config { private[spark] val MEMORY_OFFHEAP_ENABLED = ConfigBuilder("spark.memory.offHeap.enabled") .doc("If true, Spark will attempt to use off-heap memory for certain operations. " + - "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive.") + "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive." + + "Note: If off-heap memory use is enabled or off-heap memory size is increased, " + + "recommend raising the non-heap memory size (e.g increase spark.driver.memoryOverhead " + + "or spark.executor.memoryOverhead).") .withAlternative("spark.unsafe.offHeap") .booleanConf .createWithDefault(false) diff --git a/docs/configuration.md b/docs/configuration.md index 0ab0fc2296442..b6a41cf2710c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -181,10 +181,15 @@ of the most common options to set are: spark.driver.memoryOverhead driverMemory * 0.10, with minimum of 384 - Amount of memory to be allocated outside the driver process in cluster mode, in MiB unless - otherwise specified. This is memory that accounts for things like VM overheads, interned strings, - other native overheads, etc. This tends to grow with the container size (typically 6-10%). - This option is currently supported on YARN, Mesos and Kubernetes. + Amount of non-heap memory to be allocated per driver process in cluster mode + (e.g YARN, Mesos and Kubernetes.), in MiB unless otherwise specified. This is memory that + accounts for things like VM overheads, interned strings, other native overheads, etc. + This tends to grow with the container size (typically 6-10%). + Note: These non-heap memory including off-heap memory + (when spark.memory.offHeap.enabled=true) and memory used by other non-driver + processes running in the same container. The maximum memory size of container to running + driver is determined by the sum of spark.driver.memoryOverhead + and spark.driver.memory. @@ -215,10 +220,16 @@ of the most common options to set are: spark.executor.memoryOverhead executorMemory * 0.10, with minimum of 384 - Amount of memory to be allocated outside per executor process in cluster mode, in MiB - unless otherwise specified.This is memory that accounts for things like VM overheads, - interned strings, other native overheads, etc. This tends to grow with the executor - size (typically 6-10%).This option is currently supported on YARN and Kubernetes. + Amount of non-heap memory to be allocated per executor process in cluster mode + (e.g YARN and Kubernetes), in MiB unless otherwise specified. This is memory that accounts for + things like VM overheads, interned strings, other native overheads, etc. This tends to grow with + the executor size (typically 6-10%).This option is currently supported on YARN and Kubernetes. +
+ Note: These non-heap memory including off-heap memory + (when spark.memory.offHeap.enabled=true) and memory used by other non-executor + processes running in the same container. The maximum memory size of container to running executor + is determined by the sum of spark.executor.memoryOverhead and + spark.executor.memory. @@ -1233,6 +1244,9 @@ Apart from these, the following properties are also available, and may be useful If true, Spark will attempt to use off-heap memory for certain operations. If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive. + Note: If off-heap memory use is enabled or off-heap memory size is increased, + recommend raising the non-heap memory size(e.g increase spark.driver.memoryOverhead + or spark.executor.memoryOverhead). From 554caa2b2213c0ba99f9310f4b9d05e6ad01b6b3 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 22 May 2019 16:47:58 +0800 Subject: [PATCH 03/12] Fix Scala style. --- .../main/scala/org/apache/spark/internal/config/package.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index a8734483eeaf1..3064d03bf9f52 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -193,8 +193,8 @@ package object config { "(e.g YARN and Kubernetes), in MiB unless otherwise specified." + "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + " and memory used by other non-executor processes running in the same container." - "The maximum memory size of container to running executor is determined by the sum of " + - "spark.executor.memoryOverhead and spark.executor.memory.") + "The maximum memory size of container to running executor is determined by the sum of" + + " spark.executor.memoryOverhead and spark.executor.memory.") .bytesConf(ByteUnit.MiB) .createOptional From 2fdc94fa083fab9980cf292478a2a2ee4a784ed6 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 22 May 2019 17:04:04 +0800 Subject: [PATCH 04/12] Fix Scala style. --- .../main/scala/org/apache/spark/internal/config/package.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 3064d03bf9f52..0ef08fb2ac159 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -65,7 +65,7 @@ package object config { "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + " and memory used by other non-driver processes running in the same container." + "The maximum memory size of container to running driver is determined by the sum of" + - " spark.driver.memoryOverhead and spark.driver.memory.") + " `spark.driver.memoryOverhead` and `spark.driver.memory`.") .bytesConf(ByteUnit.MiB) .createOptional @@ -194,7 +194,7 @@ package object config { "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + " and memory used by other non-executor processes running in the same container." "The maximum memory size of container to running executor is determined by the sum of" + - " spark.executor.memoryOverhead and spark.executor.memory.") + " `spark.executor.memoryOverhead` and `spark.executor.memory`.") .bytesConf(ByteUnit.MiB) .createOptional From 2c89b426a0845e552400a757ec3853f52b8485e0 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 22 May 2019 17:27:02 +0800 Subject: [PATCH 05/12] Fix Scala style. --- .../org/apache/spark/internal/config/package.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 0ef08fb2ac159..723f075b9b229 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -64,8 +64,8 @@ package object config { " (e.g YARN and Kubernetes), in MiB unless otherwise specified." + "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + " and memory used by other non-driver processes running in the same container." + - "The maximum memory size of container to running driver is determined by the sum of" + - " `spark.driver.memoryOverhead` and `spark.driver.memory`.") + "The maximum memory size of container to running driver is determined by the sum of " + + "spark.driver.memoryOverhead and spark.driver.memory.") .bytesConf(ByteUnit.MiB) .createOptional @@ -192,9 +192,9 @@ package object config { .doc("Amount of non-heap memory to be allocated per executor process in cluster mode " + "(e.g YARN and Kubernetes), in MiB unless otherwise specified." + "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + - " and memory used by other non-executor processes running in the same container." - "The maximum memory size of container to running executor is determined by the sum of" + - " `spark.executor.memoryOverhead` and `spark.executor.memory`.") + " and memory used by other non-executor processes running in the same container." + + "The maximum memory size of container to running executor is determined by the sum of " + + "spark.executor.memoryOverhead and spark.executor.memory.") .bytesConf(ByteUnit.MiB) .createOptional From 3f79e89e00f920af959a6b979e736af5a43a93c7 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 22 May 2019 17:38:11 +0800 Subject: [PATCH 06/12] Fix Scala style. --- .../main/scala/org/apache/spark/internal/config/package.scala | 4 ++-- docs/configuration.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 723f075b9b229..159e33f7f52f7 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -62,7 +62,7 @@ package object config { private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead") .doc("Amount of non-heap memory to be allocated per driver process in cluster mode" + " (e.g YARN and Kubernetes), in MiB unless otherwise specified." + - "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + + "Note: Non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + " and memory used by other non-driver processes running in the same container." + "The maximum memory size of container to running driver is determined by the sum of " + "spark.driver.memoryOverhead and spark.driver.memory.") @@ -191,7 +191,7 @@ package object config { private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead") .doc("Amount of non-heap memory to be allocated per executor process in cluster mode " + "(e.g YARN and Kubernetes), in MiB unless otherwise specified." + - "Note: These non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + + "Note: Non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + " and memory used by other non-executor processes running in the same container." + "The maximum memory size of container to running executor is determined by the sum of " + "spark.executor.memoryOverhead and spark.executor.memory.") diff --git a/docs/configuration.md b/docs/configuration.md index b6a41cf2710c7..e8b6124be1da2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -185,7 +185,7 @@ of the most common options to set are: (e.g YARN, Mesos and Kubernetes.), in MiB unless otherwise specified. This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the container size (typically 6-10%). - Note: These non-heap memory including off-heap memory + Note: Non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other non-driver processes running in the same container. The maximum memory size of container to running driver is determined by the sum of spark.driver.memoryOverhead @@ -225,7 +225,7 @@ of the most common options to set are: things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the executor size (typically 6-10%).This option is currently supported on YARN and Kubernetes.
- Note: These non-heap memory including off-heap memory + Note: Non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other non-executor processes running in the same container. The maximum memory size of container to running executor is determined by the sum of spark.executor.memoryOverhead and From c4ab8a9c792fae39c012ee9987211e57372ff073 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Mon, 27 May 2019 11:31:47 +0800 Subject: [PATCH 07/12] revert comment in core and add pyspark extended comment in docs. --- .../spark/internal/config/package.scala | 21 +++++-------------- docs/configuration.md | 6 ++++-- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 159e33f7f52f7..d7e200397db62 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -60,12 +60,8 @@ package object config { .createWithDefaultString("1g") private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead") - .doc("Amount of non-heap memory to be allocated per driver process in cluster mode" + - " (e.g YARN and Kubernetes), in MiB unless otherwise specified." + - "Note: Non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + - " and memory used by other non-driver processes running in the same container." + - "The maximum memory size of container to running driver is determined by the sum of " + - "spark.driver.memoryOverhead and spark.driver.memory.") + .doc("The amount of non-heap memory to be allocated per driver in cluster mode, " + + "in MiB unless otherwise specified.") .bytesConf(ByteUnit.MiB) .createOptional @@ -189,12 +185,8 @@ package object config { .createWithDefaultString("1g") private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead") - .doc("Amount of non-heap memory to be allocated per executor process in cluster mode " + - "(e.g YARN and Kubernetes), in MiB unless otherwise specified." + - "Note: Non-heap memory including off-heap memory (when spark.memory.offHeap.enabled=true)" + - " and memory used by other non-executor processes running in the same container." + - "The maximum memory size of container to running executor is determined by the sum of " + - "spark.executor.memoryOverhead and spark.executor.memory.") + .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " + + "in MiB unless otherwise specified.") .bytesConf(ByteUnit.MiB) .createOptional @@ -209,10 +201,7 @@ package object config { private[spark] val MEMORY_OFFHEAP_ENABLED = ConfigBuilder("spark.memory.offHeap.enabled") .doc("If true, Spark will attempt to use off-heap memory for certain operations. " + - "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive." + - "Note: If off-heap memory use is enabled or off-heap memory size is increased, " + - "recommend raising the non-heap memory size (e.g increase spark.driver.memoryOverhead " + - "or spark.executor.memoryOverhead).") + "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive.") .withAlternative("spark.unsafe.offHeap") .booleanConf .createWithDefault(false) diff --git a/docs/configuration.md b/docs/configuration.md index e8b6124be1da2..8ea242cd7edc9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -186,7 +186,8 @@ of the most common options to set are: accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the container size (typically 6-10%). Note: Non-heap memory including off-heap memory - (when spark.memory.offHeap.enabled=true) and memory used by other non-driver + (when spark.memory.offHeap.enabled=true) and memory used by other driver processes + (e.g python process that goes with a PySpark driver) and memory used by other non-driver processes running in the same container. The maximum memory size of container to running driver is determined by the sum of spark.driver.memoryOverhead and spark.driver.memory. @@ -226,7 +227,8 @@ of the most common options to set are: the executor size (typically 6-10%).This option is currently supported on YARN and Kubernetes.
Note: Non-heap memory including off-heap memory - (when spark.memory.offHeap.enabled=true) and memory used by other non-executor + (when spark.memory.offHeap.enabled=true) and memory used by other executor processes + (e.g python process that goes with a PySpark executor) and memory used by other non-executor processes running in the same container. The maximum memory size of container to running executor is determined by the sum of spark.executor.memoryOverhead and spark.executor.memory. From f96f936056682950dc354ec8c630e5017375c1a1 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Mon, 27 May 2019 11:36:47 +0800 Subject: [PATCH 08/12] delete a blank. --- .../main/scala/org/apache/spark/internal/config/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index d7e200397db62..8bf615b233835 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -185,7 +185,7 @@ package object config { .createWithDefaultString("1g") private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead") - .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " + + .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " + "in MiB unless otherwise specified.") .bytesConf(ByteUnit.MiB) .createOptional From 1f31fc6aac694889f1b1450be4f30773deb51ad5 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Tue, 28 May 2019 10:22:11 +0800 Subject: [PATCH 09/12] correct some typo. --- docs/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8ea242cd7edc9..f726c2294cbed 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -185,7 +185,7 @@ of the most common options to set are: (e.g YARN, Mesos and Kubernetes.), in MiB unless otherwise specified. This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the container size (typically 6-10%). - Note: Non-heap memory including off-heap memory + Note: Non-heap memory includes off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other driver processes (e.g python process that goes with a PySpark driver) and memory used by other non-driver processes running in the same container. The maximum memory size of container to running @@ -226,7 +226,7 @@ of the most common options to set are: things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the executor size (typically 6-10%).This option is currently supported on YARN and Kubernetes.
- Note: Non-heap memory including off-heap memory + Note: Non-heap memory includes off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other executor processes (e.g python process that goes with a PySpark executor) and memory used by other non-executor processes running in the same container. The maximum memory size of container to running executor @@ -1247,7 +1247,7 @@ Apart from these, the following properties are also available, and may be useful If true, Spark will attempt to use off-heap memory for certain operations. If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive. Note: If off-heap memory use is enabled or off-heap memory size is increased, - recommend raising the non-heap memory size(e.g increase spark.driver.memoryOverhead + recommend raise the non-heap memory size(e.g increase spark.driver.memoryOverhead or spark.executor.memoryOverhead). From f23c1b70b9dcf8e4dce43e5dc217ea8822ddfae3 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 29 May 2019 10:24:09 +0800 Subject: [PATCH 10/12] Adjust docs. --- docs/configuration.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f726c2294cbed..70e0c1ee41ba1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -182,12 +182,12 @@ of the most common options to set are: driverMemory * 0.10, with minimum of 384 Amount of non-heap memory to be allocated per driver process in cluster mode - (e.g YARN, Mesos and Kubernetes.), in MiB unless otherwise specified. This is memory that + (e.g. YARN, Mesos and Kubernetes.), in MiB unless otherwise specified. This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the container size (typically 6-10%). Note: Non-heap memory includes off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other driver processes - (e.g python process that goes with a PySpark driver) and memory used by other non-driver + (e.g. python process that goes with a PySpark driver) and memory used by other non-driver processes running in the same container. The maximum memory size of container to running driver is determined by the sum of spark.driver.memoryOverhead and spark.driver.memory. @@ -221,14 +221,14 @@ of the most common options to set are: spark.executor.memoryOverhead executorMemory * 0.10, with minimum of 384 - Amount of non-heap memory to be allocated per executor process in cluster mode - (e.g YARN and Kubernetes), in MiB unless otherwise specified. This is memory that accounts for + Amount of non-heap memory to be allocated per executor process in cluster mode + (e.g. YARN and Kubernetes), in MiB unless otherwise specified. This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the executor size (typically 6-10%).This option is currently supported on YARN and Kubernetes.
Note: Non-heap memory includes off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other executor processes - (e.g python process that goes with a PySpark executor) and memory used by other non-executor + (e.g. python process that goes with a PySpark executor) and memory used by other non-executor processes running in the same container. The maximum memory size of container to running executor is determined by the sum of spark.executor.memoryOverhead and spark.executor.memory. @@ -1246,9 +1246,8 @@ Apart from these, the following properties are also available, and may be useful If true, Spark will attempt to use off-heap memory for certain operations. If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive. - Note: If off-heap memory use is enabled or off-heap memory size is increased, - recommend raise the non-heap memory size(e.g increase spark.driver.memoryOverhead - or spark.executor.memoryOverhead). + Note: If off-heap memory is enabled, raise the non-heap memory size (e.g. increase + spark.driver.memoryOverhead or spark.executor.memoryOverhead). From 5dbbf4a27d7892a06e6aec09a486ff77ace7e3dd Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Thu, 30 May 2019 10:02:55 +0800 Subject: [PATCH 11/12] revert This option ... --- docs/configuration.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 70e0c1ee41ba1..27bec9bd79cb9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -181,10 +181,10 @@ of the most common options to set are: spark.driver.memoryOverhead driverMemory * 0.10, with minimum of 384 - Amount of non-heap memory to be allocated per driver process in cluster mode - (e.g. YARN, Mesos and Kubernetes.), in MiB unless otherwise specified. This is memory that - accounts for things like VM overheads, interned strings, other native overheads, etc. - This tends to grow with the container size (typically 6-10%). + Amount of non-heap memory to be allocated per driver process in cluster mode, in MiB unless + otherwise specified. This is memory that accounts for things like VM overheads, interned strings, + other native overheads, etc. This tends to grow with the container size (typically 6-10%). + This option is currently supported on YARN, Mesos and Kubernetes. Note: Non-heap memory includes off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other driver processes (e.g. python process that goes with a PySpark driver) and memory used by other non-driver @@ -221,10 +221,10 @@ of the most common options to set are: spark.executor.memoryOverhead executorMemory * 0.10, with minimum of 384 - Amount of non-heap memory to be allocated per executor process in cluster mode - (e.g. YARN and Kubernetes), in MiB unless otherwise specified. This is memory that accounts for - things like VM overheads, interned strings, other native overheads, etc. This tends to grow with - the executor size (typically 6-10%).This option is currently supported on YARN and Kubernetes. + Amount of non-heap memory to be allocated per executor process in cluster mode, in MiB unless + otherwise specified. This is memory that accounts for things like VM overheads, interned strings, + other native overheads, etc. This tends to grow with the executor size (typically 6-10%). + This option is currently supported on YARN and Kubernetes.
Note: Non-heap memory includes off-heap memory (when spark.memory.offHeap.enabled=true) and memory used by other executor processes @@ -1246,8 +1246,9 @@ Apart from these, the following properties are also available, and may be useful If true, Spark will attempt to use off-heap memory for certain operations. If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive. - Note: If off-heap memory is enabled, raise the non-heap memory size (e.g. increase - spark.driver.memoryOverhead or spark.executor.memoryOverhead). + Note: If off-heap memory is enabled, you may need to raise the non-heap memory size + (e.g. increase spark.driver.memoryOverhead or + spark.executor.memoryOverhead). From 7980652db5f97e398c4375fcf6ff2e7e5bcfd9d0 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Thu, 30 May 2019 15:10:26 +0800 Subject: [PATCH 12/12] Add may need to. --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 27bec9bd79cb9..c59626c34858b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1246,7 +1246,7 @@ Apart from these, the following properties are also available, and may be useful If true, Spark will attempt to use off-heap memory for certain operations. If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive. - Note: If off-heap memory is enabled, you may need to raise the non-heap memory size + Note: If off-heap memory is enabled, may need to raise the non-heap memory size (e.g. increase spark.driver.memoryOverhead or spark.executor.memoryOverhead).