Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Fix jar path and add missing ones for spark jobs #14020

Merged
merged 4 commits into from
Feb 15, 2019
Merged

Fix jar path and add missing ones for spark jobs #14020

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e985917
Fix jar path and add missing ones for spark jobs
ashutosh-dwivedi-e3502 Jan 29, 2019
9adcedc
fixes scalastyle violations
ashutosh-dwivedi-e3502 Feb 5, 2019
6f4b56f
exclude all of javadoc, sources, bundle, and src while searching for …
ashutosh-dwivedi-e3502 Feb 7, 2019
e70371d
simplfied the exclude experession
ashutosh-dwivedi-e3502 Feb 7, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 22 additions & 20 deletions scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,37 +80,37 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd
System.getProperty("user.dir")
}

private def getJarFilePath(root: String): String = {
for (platform <- List("linux-x86_64-cpu", "linux-x86_64-gpu", "osx-x86_64-cpu")) {
ashutosh-dwivedi-e3502 marked this conversation as resolved.
Show resolved Hide resolved
val jarFiles = new File(s"$root/$platform/target/").listFiles(new FileFilter {
override def accept(pathname: File) = {
pathname.getAbsolutePath.endsWith(".jar") &&
!pathname.getAbsolutePath.contains("javadoc") &&
!pathname.getAbsolutePath.contains("sources")
}
})
if (jarFiles != null && jarFiles.nonEmpty) {
return jarFiles.head.getAbsolutePath
private def findJars(root: String): Array[File] = {
val excludedSuffixes = List("bundle", "src", "javadoc", "sources")
new File(root).listFiles(new FileFilter {
override def accept(pathname: File) = {
pathname.getAbsolutePath.endsWith(".jar") &&
excludedSuffixes.forall(!pathname.getAbsolutePath.contains(_))
}
})
}

private def getJarFilePath(root: String): String = {
val jarFiles = findJars(s"$root/target/")
if (jarFiles != null && jarFiles.nonEmpty) {
jarFiles.head.getAbsolutePath
} else {
null
}
null
}

private def getSparkJar: String = {
val jarFiles = new File(s"$composeWorkingDirPath/target/").listFiles(new FileFilter {
override def accept(pathname: File) = {
pathname.getAbsolutePath.endsWith(".jar") &&
!pathname.getAbsolutePath.contains("javadoc") &&
!pathname.getAbsolutePath.contains("sources")
}
})
val jarFiles = findJars(s"$composeWorkingDirPath/target/")
if (jarFiles != null && jarFiles.nonEmpty) {
jarFiles.head.getAbsolutePath
} else {
null
}
}

private def getNativeJars(root: String): String =
new File(root).listFiles().map(_.toPath).mkString(",")

protected def buildLeNet(): MXNet = {
val workingDir = composeWorkingDirPath
val assemblyRoot = s"$workingDir/../assembly"
Expand All @@ -130,6 +130,8 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd
protected def buildMlp(): MXNet = {
val workingDir = composeWorkingDirPath
val assemblyRoot = s"$workingDir/../assembly"
val nativeRoot = s"$workingDir/../native/target/lib"

new MXNet()
.setBatchSize(128)
.setLabelName("softmax_label")
Expand All @@ -139,7 +141,7 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd
.setNumEpoch(10)
.setNumServer(1)
.setNumWorker(numWorkers)
.setExecutorJars(s"${getJarFilePath(assemblyRoot)},$getSparkJar")
.setExecutorJars(s"${getJarFilePath(assemblyRoot)},$getSparkJar,${getNativeJars(nativeRoot)}")
.setJava("java")
.setTimeout(0)
}
Expand Down