Skip to content

Commit 76805aa

Browse files
committed
unifies the config properties name prefix; add the configs into docs/configuration.md
1 parent e700d9c commit 76805aa

File tree

4 files changed

+22
-7
lines changed

4 files changed

+22
-7
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ class SparkContext(
129129
// Generate the random name for a temp folder in Tachyon
130130
// Add a timestamp as the suffix here to make it more safe
131131
val tachyonFolderName = "spark-" + randomUUID.toString()
132-
conf.set("spark.tachyonstore.foldername", tachyonFolderName)
132+
conf.set("spark.tachyonStore.folderName", tachyonFolderName)
133133

134134
val isLocal = (master == "local" || master.startsWith("local["))
135135

core/src/main/scala/org/apache/spark/storage/BlockManager.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,10 @@ private[spark] class BlockManager(
5959
private[storage] val diskStore = new DiskStore(this, diskBlockManager)
6060
var tachyonInitialized = false
6161
private[storage] lazy val tachyonStore: TachyonStore = {
62-
val storeDir = conf.get("spark.tachyonstore.dir", System.getProperty("java.io.tmpdir"))
63-
val appFolderName = conf.get("spark.tachyonstore.foldername")
62+
val storeDir = conf.get("spark.tachyonStore.baseDir", System.getProperty("java.io.tmpdir"))
63+
val appFolderName = conf.get("spark.tachyonStore.folderName")
6464
val tachyonStorePath = s"${storeDir}/${appFolderName}/${this.executorId}"
65-
val tachyonMaster = conf.get("spark.tachyonmaster.address", "tachyon://localhost:19998")
65+
val tachyonMaster = conf.get("spark.tachyonStore.URL", "tachyon://localhost:19998")
6666
val tachyonBlockManager = new TachyonBlockManager(
6767
shuffleBlockManager, tachyonStorePath, tachyonMaster)
6868
tachyonInitialized = true

core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ private[spark] class TachyonBlockManager(
5252
private val subDirsPerTachyonDir =
5353
shuffleManager.conf.get("spark.tachyonStore.subDirectories", "64").toInt
5454

55-
// Create one Tachyon directory for each path mentioned in spark.tachyon.dir; then, inside this
56-
// directory, create multiple subdirectories that we will hash files into, in order to avoid
57-
// having really large inodes at the top level in Tachyon.
55+
// Create one Tachyon directory for each path mentioned in spark.tachyonStore.folderName.dir;
56+
// then, inside this directory, create multiple subdirectories that we will hash files into,
57+
// in order to avoid having really large inodes at the top level in Tachyon.
5858
private val tachyonDirs: Array[TachyonFile] = createTachyonDirs()
5959
private val subDirs = Array.fill(tachyonDirs.length)(new Array[TachyonFile](subDirsPerTachyonDir))
6060

docs/configuration.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,21 @@ Apart from these, the following properties are also available, and may be useful
122122
<code>spark.storage.memoryFraction</code>.
123123
</td>
124124
</tr>
125+
<tr>
126+
<td>spark.tachyonStore.baseDir</td>
127+
<td>/tmp/</td>
128+
<td>
129+
Directories of the Tachyon File System that store RDDs. The Tachyon file system's URL is set by <code>spark.tachyonStore.URL</code>.
130+
It can also be a comma-separated list of multiple directories on Tachyon file system.
131+
</td>
132+
</tr>
133+
<tr>
134+
<td>spark.tachyonStore.URL</td>
135+
<td>tachyon://localhost:19998</td>
136+
<td>
137+
The URL of the underlying Tachyon file system in the TachyonStore.
138+
</td>
139+
</tr>
125140
<tr>
126141
<td>spark.mesos.coarse</td>
127142
<td>false</td>

0 commit comments

Comments
 (0)