Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ class CreateBatchRequest {
var pyFiles: List[String] = List()
var files: List[String] = List()
var driverMemory: Option[String] = None
var driverCores: Option[Int] = None
var driverCores: Option[Double] = None
var executorMemory: Option[String] = None
var executorCores: Option[Int] = None
var executorCores: Option[Double] = None
var numExecutors: Option[Int] = None
var archives: List[String] = List()
var queue: Option[String] = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ class CreateInteractiveRequest {
var pyFiles: List[String] = List()
var files: List[String] = List()
var driverMemory: Option[String] = None
var driverCores: Option[Int] = None
var driverCores: Option[Double] = None
var executorMemory: Option[String] = None
var executorCores: Option[Int] = None
var executorCores: Option[Double] = None
var numExecutors: Option[Int] = None
var archives: List[String] = List()
var queue: Option[String] = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class SparkProcessBuilder(livyConf: LivyConf) extends Logging {
this
}

def driverCores(driverCores: Int): SparkProcessBuilder = {
def driverCores(driverCores: Double): SparkProcessBuilder = {
this.driverCores(driverCores.toString)
}

Expand All @@ -104,7 +104,7 @@ class SparkProcessBuilder(livyConf: LivyConf) extends Logging {
conf("spark.driver.cores", driverCores)
}

def executorCores(executorCores: Int): SparkProcessBuilder = {
def executorCores(executorCores: Double): SparkProcessBuilder = {
this.executorCores(executorCores.toString)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,31 @@ class CreateBatchRequestSpec extends FunSpec with LivyBaseUnitTestSuite {
assert(req.conf === Map())
}

it("should support integer cores") {
val json = """{ "driverCores" : 1, "executorCores": 2 }"""
val req = mapper.readValue(json, classOf[CreateBatchRequest])
assert(req.driverCores.get === 1)
assert(req.executorCores.get === 2)
}

it("should support float cores") {
val json = """{ "driverCores" : 0.1, "executorCores": 0.2 }"""
val req = mapper.readValue(json, classOf[CreateBatchRequest])
assert(req.driverCores.get === 0.1)
assert(req.executorCores.get === 0.2)
}

it("should not support string cores") {
val json = """{ "driverCores" : "asdf", "executorCores": "0.2" }"""
val req = mapper.readValue(json, classOf[CreateBatchRequest])
intercept[ClassCastException] {
req.driverCores.get
}
intercept[ClassCastException] {
req.executorCores.get
}
}

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,31 @@ class CreateInteractiveRequestSpec extends FunSpec with LivyBaseUnitTestSuite {
assert(req.conf === Map())
}

it("should support integer cores") {
val json = """{ "driverCores" : 1, "executorCores": 2 }"""
val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
assert(req.driverCores.get === 1)
assert(req.executorCores.get === 2)
}

it("should support float cores") {
val json = """{ "driverCores" : 0.1, "executorCores": 0.2 }"""
val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
assert(req.driverCores.get === 0.1)
assert(req.executorCores.get === 0.2)
}

it("should not support string cores") {
val json = """{ "driverCores" : "asdf", "executorCores": "0.2" }"""
val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
intercept[ClassCastException] {
req.driverCores.get
}
intercept[ClassCastException] {
req.executorCores.get
}
}

}

}