Skip to content
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/src/main/scala/org/apache/spark/SparkContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,7 @@ class SparkContext(config: SparkConf) extends Logging {
private[spark] def cleaner: Option[ContextCleaner] = _cleaner

private[spark] var checkpointDir: Option[String] = None
config.getOption(CHECKPOINT_DIR.key).foreach(setCheckpointDir)

// Thread Local variable that can be used by users to pass information down the stack
protected[spark] val localProperties = new InheritableThreadLocal[Properties] {
Expand Down
10 changes: 10 additions & 0 deletions core/src/main/scala/org/apache/spark/internal/config/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1317,6 +1317,16 @@ package object config {
s" be less than or equal to ${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
.createWithDefault(64 * 1024 * 1024)

private[spark] val CHECKPOINT_DIR =
ConfigBuilder("spark.checkpoint.dir")
.doc(
"Equivalent with SparkContext.setCheckpointDir. If set, the path becomes" +
"the default directory for checkpointing. It can be overwritten by" +
"SparkContext.setCheckpointDir.")
.version("4.0.0")
.stringConf
.createOptional

private[spark] val CHECKPOINT_COMPRESS =
ConfigBuilder("spark.checkpoint.compress")
.doc("Whether to compress RDD checkpoints. Generally a good idea. Compression will use " +
Expand Down
10 changes: 10 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1795,6 +1795,16 @@ Apart from these, the following properties are also available, and may be useful
</td>
<td>0.6.0</td>
</tr>
<tr>
<td><code>spark.checkpoint.dir</code></td>
<td>(none)</td>
<td>
Equivalent with SparkContext.setCheckpointDir. If set, the path becomes
the default directory for checkpointing. It can be overwritten by
SparkContext.setCheckpointDir.
</td>
<td>4.0.0</td>
</tr>
<tr>
<td><code>spark.checkpoint.compress</code></td>
<td>false</td>
Expand Down