Skip to content

Commit

Permalink
[MXNET-1287] Miscellaneous Scala warning fixes (apache#14658)
Browse files Browse the repository at this point in the history
* Remove printing during tests

* Fix macro annotation warnings

* Remove failing links from spark

* Return printed as debug logs
  • Loading branch information
zachgk authored and larroy committed Apr 15, 2019
1 parent a386644 commit 69bc3a6
Show file tree
Hide file tree
Showing 11 changed files with 27 additions and 55 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -57,22 +57,22 @@ class ImageSuite extends FunSuite with BeforeAndAfterAll {

test("Test load image") {
val nd = Image.imRead(imLocation)
logger.info(s"OpenCV load image with shape: ${nd.shape}")
logger.debug(s"OpenCV load image with shape: ${nd.shape}")
require(nd.shape == Shape(576, 1024, 3), "image shape not Match!")
}

test("Test load image from Socket") {
val url = new URL("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg")
val inputStream = url.openStream
val nd = Image.imDecode(inputStream)
logger.info(s"OpenCV load image with shape: ${nd.shape}")
logger.debug(s"OpenCV load image with shape: ${nd.shape}")
require(nd.shape == Shape(576, 1024, 3), "image shape not Match!")
}

test("Test resize image") {
val nd = Image.imRead(imLocation)
val resizeIm = Image.imResize(nd, 224, 224)
logger.info(s"OpenCV resize image with shape: ${resizeIm.shape}")
logger.debug(s"OpenCV resize image with shape: ${resizeIm.shape}")
require(resizeIm.shape == Shape(224, 224, 3), "image shape not Match!")
}

Expand All @@ -94,7 +94,7 @@ class ImageSuite extends FunSuite with BeforeAndAfterAll {
val tempDirPath = System.getProperty("java.io.tmpdir")
val img = Image.toImage(resizeIm)
ImageIO.write(img, "png", new File(tempDirPath + "/inputImages/out.png"))
logger.info(s"converted image stored in ${tempDirPath + "/inputImages/out.png"}")
logger.debug(s"converted image stored in ${tempDirPath + "/inputImages/out.png"}")
}

test("Test draw Bounding box") {
Expand All @@ -107,7 +107,7 @@ class ImageSuite extends FunSuite with BeforeAndAfterAll {
Image.drawBoundingBox(buf, box, Some(names), fontSizeMult = Some(1.4f))
val tempDirPath = System.getProperty("java.io.tmpdir")
ImageIO.write(buf, "png", new File(tempDirPath + "/inputImages/out2.png"))
logger.info(s"converted image stored in ${tempDirPath + "/inputImages/out2.png"}")
logger.debug(s"converted image stored in ${tempDirPath + "/inputImages/out2.png"}")
for (coord <- box) {
val topLeft = buf.getRGB(coord("xmin"), coord("ymin"))
val downLeft = buf.getRGB(coord("xmin"), coord("ymax"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@ class KVStoreSuite extends FunSuite with BeforeAndAfterAll {
val kv = KVStore.create()
val updater = new MXKVStoreUpdater {
override def update(key: Int, input: NDArray, stored: NDArray): Unit = {
// scalastyle:off println
println(s"update on key $key")
// scalastyle:on println
stored += input * 2
}
override def dispose(): Unit = {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,6 @@ class ModelParallelSuite extends FunSuite with BeforeAndAfterAll {
val arrGrad2 = arrGrad.map(_.copyTo(ctx1))
val exec2 = net.bind(ctx1, args = arr2, argsGrad = arrGrad2)

// Show the execution plan that involves copynode
// scalastyle:off println
print(exec1.debugStr)
// scalastyle:on println

exec1.forward()
exec2.forward()
assert(reldiff(exec1.outputs(0).copyTo(ctx1),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -614,9 +614,6 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll
val embed = Symbol.Embedding(name = "embed")()(
Map("data" -> data, "input_dim" -> inDim, "output_dim" -> outDim))
// TODO
// scalastyle:off println
println(s"Embeded symbol: ${embed.toJson}")
// scalastyle:on println
}

// check ops handle duplicate input correctly.
Expand Down Expand Up @@ -983,9 +980,6 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll
test("batch norm") {
val data = Symbol.Variable("data")
val test = Symbol.BatchNorm(name = "bn")()(Map("data" -> data, "fix_gamma" -> "False"))
// scalastyle:off println
println(s"BatchNorm: ${test.toJson}")
// scalastyle:on println
// TODO: check numeric gradient
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,8 @@ class SymbolSuite extends FunSuite with BeforeAndAfterAll {
var net2 = Symbol.FullyConnected(name = "fc3")()(Map("num_hidden" -> 10))
net2 = Symbol.Activation()()(Map("data" -> net2, "act_type" -> "relu"))
net2 = Symbol.FullyConnected(name = "fc4")()(Map("data" -> net2, "num_hidden" -> 20))
// scalastyle:off println
println(s"net2 debug info:\n${net2.debugStr}")
// scalastyle:on println

val composed = net2(name = "composed", Map("fc3_data" -> net1))
// scalastyle:off println
println(s"composed debug info:\n${composed.debugStr}")
// scalastyle:on println
val multiOut = Symbol.Group(composed, net1)
assert(multiOut.listOutputs().length === 2)
}
Expand Down Expand Up @@ -77,20 +71,12 @@ class SymbolSuite extends FunSuite with BeforeAndAfterAll {
val lam = Symbol.Variable("lam")
val rnd = Symbol.random.poisson(lam = Some(lam), shape = Some(Shape(2, 2)))
val rnd2 = Symbol.random.poisson(lam = Some(1f), shape = Some(Shape(2, 2)))
// scalastyle:off println
println(s"Symbol.random.poisson debug info: ${rnd.debugStr}")
println(s"Symbol.random.poisson debug info: ${rnd2.debugStr}")
// scalastyle:on println
}

test("Symbol random module is generated properly - special case of 'normal'") {
val loc = Symbol.Variable("loc")
val scale = Symbol.Variable("scale")
val rnd = Symbol.random.normal(mu = Some(loc), sigma = Some(scale), shape = Some(Shape(2, 2)))
val rnd2 = Symbol.random.normal(mu = Some(1f), sigma = Some(2f), shape = Some(Shape(2, 2)))
// scalastyle:off println
println(s"Symbol.random.sample_normal debug info: ${rnd.debugStr}")
println(s"Symbol.random.random_normal debug info: ${rnd2.debugStr}")
// scalastyle:on println
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ private[mxnet] abstract class GeneratorBase {
*/
protected def structGeneration(c: blackbox.Context)
(funcDef: List[c.universe.DefDef], annottees: c.Expr[Any]*)
: c.Expr[Any] = {
: c.Expr[Nothing] = {
import c.universe._
val inputs = annottees.map(_.tree).toList
// pattern match on the inputs
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,16 @@ import scala.language.experimental.macros
import scala.reflect.macros.blackbox

private[mxnet] class AddNDArrayFunctions(isContrib: Boolean) extends StaticAnnotation {
private[mxnet] def macroTransform(annottees: Any*) = macro NDArrayMacro.addDefs
private[mxnet] def macroTransform(annottees: Any*): Any = macro NDArrayMacro.addDefs
}

private[mxnet] class AddNDArrayAPIs(isContrib: Boolean) extends StaticAnnotation {
private[mxnet] def macroTransform(annottees: Any*) = macro TypedNDArrayAPIMacro.typeSafeAPIDefs
private[mxnet] def macroTransform(annottees: Any*): Any =
macro TypedNDArrayAPIMacro.typeSafeAPIDefs
}

private[mxnet] class AddNDArrayRandomAPIs(isContrib: Boolean) extends StaticAnnotation {
private[mxnet] def macroTransform(annottees: Any*) =
private[mxnet] def macroTransform(annottees: Any*): Any =
macro TypedNDArrayRandomAPIMacro.typeSafeAPIDefs
}

Expand All @@ -39,7 +40,7 @@ private[mxnet] class AddNDArrayRandomAPIs(isContrib: Boolean) extends StaticAnno
*/
private[mxnet] object NDArrayMacro extends GeneratorBase {

def addDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] = {
def addDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Nothing] = {
import c.universe._
val isContrib: Boolean = c.prefix.tree match {
case q"new AddNDArrayFunctions($b)" => c.eval[Boolean](c.Expr(b))
Expand All @@ -49,7 +50,7 @@ private[mxnet] object NDArrayMacro extends GeneratorBase {
}

private def impl(c: blackbox.Context)
(isContrib: Boolean, annottees: c.Expr[Any]*): c.Expr[Any] = {
(isContrib: Boolean, annottees: c.Expr[Any]*): c.Expr[Nothing] = {
import c.universe._

val functions = functionsToGenerate(isSymbol = false, isContrib)
Expand Down Expand Up @@ -82,7 +83,7 @@ private[mxnet] object NDArrayMacro extends GeneratorBase {
*/
private[mxnet] object TypedNDArrayAPIMacro extends GeneratorBase {

def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] = {
def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Nothing] = {
import c.universe._
val isContrib: Boolean = c.prefix.tree match {
case q"new AddNDArrayAPIs($b)" => c.eval[Boolean](c.Expr(b))
Expand Down Expand Up @@ -148,7 +149,7 @@ private[mxnet] object TypedNDArrayAPIMacro extends GeneratorBase {
private[mxnet] object TypedNDArrayRandomAPIMacro extends GeneratorBase
with RandomHelpers {

def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] = {
def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Nothing] = {
// Note: no contrib managed in this module

val functionDefs = typeSafeRandomFunctionsToGenerate(isSymbol = false)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@ import scala.language.experimental.macros
import scala.reflect.macros.blackbox

private[mxnet] class AddSymbolFunctions(isContrib: Boolean) extends StaticAnnotation {
private[mxnet] def macroTransform(annottees: Any*) = macro SymbolMacro.addDefs
private[mxnet] def macroTransform(annottees: Any*): Any = macro SymbolMacro.addDefs
}

private[mxnet] class AddSymbolAPIs(isContrib: Boolean) extends StaticAnnotation {
private[mxnet] def macroTransform(annottees: Any*) = macro TypedSymbolAPIMacro.typeSafeAPIDefs
private[mxnet] def macroTransform(annottees: Any*): Any =
macro TypedSymbolAPIMacro.typeSafeAPIDefs
}

private[mxnet] class AddSymbolRandomAPIs(isContrib: Boolean) extends StaticAnnotation {
private[mxnet] def macroTransform(annottees: Any*) =
private[mxnet] def macroTransform(annottees: Any*): Any =
macro TypedSymbolRandomAPIMacro.typeSafeAPIDefs
}

Expand All @@ -40,7 +41,7 @@ private[mxnet] class AddSymbolRandomAPIs(isContrib: Boolean) extends StaticAnnot
*/
private[mxnet] object SymbolMacro extends GeneratorBase {

def addDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] = {
def addDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Nothing] = {
import c.universe._
val isContrib: Boolean = c.prefix.tree match {
case q"new AddSymbolFunctions($b)" => c.eval[Boolean](c.Expr(b))
Expand All @@ -50,7 +51,7 @@ private[mxnet] object SymbolMacro extends GeneratorBase {
}

private def impl(c: blackbox.Context)
(isContrib: Boolean, annottees: c.Expr[Any]*): c.Expr[Any] = {
(isContrib: Boolean, annottees: c.Expr[Any]*): c.Expr[Nothing] = {
import c.universe._

val functions = functionsToGenerate(isSymbol = false, isContrib)
Expand All @@ -76,7 +77,7 @@ private[mxnet] object SymbolMacro extends GeneratorBase {
*/
private[mxnet] object TypedSymbolAPIMacro extends GeneratorBase {

def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] = {
def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Nothing] = {
import c.universe._
val isContrib: Boolean = c.prefix.tree match {
case q"new AddSymbolAPIs($b)" => c.eval[Boolean](c.Expr(b))
Expand Down Expand Up @@ -140,7 +141,7 @@ private[mxnet] object TypedSymbolAPIMacro extends GeneratorBase {
private[mxnet] object TypedSymbolRandomAPIMacro extends GeneratorBase
with RandomHelpers {

def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] = {
def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Nothing] = {
val functionDefs = typeSafeRandomFunctionsToGenerate(isSymbol = true)
.map(f => buildTypedFunction(c)(f))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,16 @@ import scala.language.experimental.macros
import scala.reflect.macros.blackbox

private[mxnet] class AddJNDArrayAPIs(isContrib: Boolean) extends StaticAnnotation {
private[mxnet] def macroTransform(annottees: Any*) = macro JavaNDArrayMacro.typeSafeAPIDefs
private[mxnet] def macroTransform(annottees: Any*): Any = macro JavaNDArrayMacro.typeSafeAPIDefs
}

private[mxnet] object JavaNDArrayMacro extends GeneratorBase {

// scalastyle:off havetype
def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*) = {
def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*): c.Expr[Nothing] = {
typeSafeAPIImpl(c)(annottees: _*)
}
// scalastyle:off havetype

private def typeSafeAPIImpl(c: blackbox.Context)(annottees: c.Expr[Any]*) : c.Expr[Any] = {
private def typeSafeAPIImpl(c: blackbox.Context)(annottees: c.Expr[Any]*) : c.Expr[Nothing] = {
import c.universe._

val isContrib: Boolean = c.prefix.tree match {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.mxnet.spark
import org.apache.mxnet.NDArray

/**
* A wrapper for serialize & deserialize <pre>[[org.apache.mxnet.NDArray]]</pre> in spark job
* A wrapper for serialize & deserialize ``org.apache.mxnet.NDArray`` in spark job
* @author Yizhi Liu
*/
class MXNDArray(@transient private var ndArray: NDArray) extends Serializable {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import org.apache.spark.SparkContext
import org.apache.spark.mllib.linalg.Vector

/**
* Wrapper for <pre>[[org.apache.mxnet.Model]]</pre> which used in Spark application
* Wrapper for ``org.apache.mxnet.Model`` which used in Spark application
* @author Yizhi Liu
*/
class MXNetModel private[mxnet](
Expand Down

0 comments on commit 69bc3a6

Please sign in to comment.