diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala index b65f237c8621..b6894f148b11 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala @@ -21,6 +21,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context +import org.apache.mxnet.ResourceScope; import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -37,21 +38,23 @@ class CustomOpExampleSuite extends FunSuite with BeforeAndAfterAll { System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { logger.info("CPU test only, skipped...") } else { - logger.info("Downloading mnist model") - val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" - val tempDirPath = System.getProperty("java.io.tmpdir") - val modelDirPath = tempDirPath + File.separator + "mnist/" - val tmpFile = new File(tempDirPath + "/mnist/mnist.zip") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), - tmpFile) + ResourceScope.using() { + logger.info("Downloading mnist model") + val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" + val tempDirPath = System.getProperty("java.io.tmpdir") + val modelDirPath = tempDirPath + File.separator + "mnist/" + val tmpFile = new File(tempDirPath + "/mnist/mnist.zip") + if (!tmpFile.exists()) { + FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), + tmpFile) + } + // TODO: Need to confirm with Windows + Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + + tempDirPath + "/mnist/") ! + val context = Context.cpu() + val output = ExampleCustomOp.test(modelDirPath, context) + assert(output >= 0.95f) } - // TODO: Need to confirm with Windows - Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " - + tempDirPath + "/mnist/") ! - val context = Context.cpu() - val output = ExampleCustomOp.test(modelDirPath, context) - assert(output >= 0.95f) } } @@ -62,18 +65,20 @@ class CustomOpExampleSuite extends FunSuite with BeforeAndAfterAll { if (RTC_fixed) { if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { - logger.info("Downloading mnist model") - val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" - val tempDirPath = System.getProperty("java.io.tmpdir") - val modelDirPath = tempDirPath + File.separator + "mnist/" - Util.downloadUrl(baseUrl + "/mnist/mnist.zip", - tempDirPath + "/mnist/mnist.zip") - // TODO: Need to confirm with Windows - Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " - + tempDirPath + "/mnist/") ! - val context = Context.gpu() - val output = ExampleCustomOpWithRtc.test(modelDirPath, context) - assert(output >= 0.95f) + ResourceScope.using() { + logger.info("Downloading mnist model") + val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" + val tempDirPath = System.getProperty("java.io.tmpdir") + val modelDirPath = tempDirPath + File.separator + "mnist/" + Util.downloadUrl(baseUrl + "/mnist/mnist.zip", + tempDirPath + "/mnist/mnist.zip") + // TODO: Need to confirm with Windows + Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + + tempDirPath + "/mnist/") ! + val context = Context.gpu() + val output = ExampleCustomOpWithRtc.test(modelDirPath, context) + assert(output >= 0.95f) + } } else { logger.info("GPU test only, skipped...") } diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/IMClassificationExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/IMClassificationExampleSuite.scala index e6f4f6fcc908..1b5e362fec2b 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/IMClassificationExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/IMClassificationExampleSuite.scala @@ -19,7 +19,7 @@ package org.apache.mxnetexamples.imclassification import java.io.File -import org.apache.mxnet.{Context, DType} +import org.apache.mxnet.{Context, DType, ResourceScope} import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -35,35 +35,41 @@ class IMClassificationExampleSuite extends FunSuite with BeforeAndAfterAll { test("Example CI: Test MNIST Training") { - logger.info("Downloading mnist model") - val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" - val tempDirPath = System.getProperty("java.io.tmpdir") - val modelDirPath = tempDirPath + File.separator + "mnist/" - logger.info("tempDirPath: %s".format(tempDirPath)) - Util.downloadUrl(baseUrl + "/mnist/mnist.zip", - tempDirPath + "/mnist/mnist.zip") - // TODO: Need to confirm with Windows - Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " - + tempDirPath + "/mnist/") ! + ResourceScope.using() { + logger.info("Downloading mnist model") + val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" + val tempDirPath = System.getProperty("java.io.tmpdir") + val modelDirPath = tempDirPath + File.separator + "mnist/" + logger.info("tempDirPath: %s".format(tempDirPath)) + Util.downloadUrl(baseUrl + "/mnist/mnist.zip", + tempDirPath + "/mnist/mnist.zip") + // TODO: Need to confirm with Windows + Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + + tempDirPath + "/mnist/") ! - var context = Context.cpu() + var context = Context.cpu() - val valAccuracy = TrainModel.test("mlp", modelDirPath) - Process("rm -rf " + modelDirPath) ! + val valAccuracy = TrainModel.test("mlp", modelDirPath) + Process("rm -rf " + modelDirPath) ! - assert(valAccuracy >= 0.95f) + assert(valAccuracy >= 0.95f) + } } for(model <- List("mlp", "lenet", "resnet")) { test(s"Example CI: Test Image Classification Model ${model}") { - val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true) + ResourceScope.using() { + val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true) + } } } for(model <- List("mlp", "lenet", "resnet")) { test(s"Example CI: Test Image Classification Model ${model} with Float64 input") { - val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true, - dtype = DType.Float64) + ResourceScope.using() { + val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true, + dtype = DType.Float64) + } } } diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala index 9c16aca420ef..d7233a277e59 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala @@ -21,6 +21,7 @@ import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory import java.io.File import org.apache.mxnet.Context +import org.apache.mxnet.ResourceScope import org.apache.mxnetexamples.Util import scala.language.postfixOps @@ -49,27 +50,29 @@ class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { Util.downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg") - val modelDirPath = tempDirPath + File.separator + "resnet18/" - val inputImagePath = tempDirPath + File.separator + - "inputImages/resnet18/Pug-Cookie.jpg" - val inputImageDir = tempDirPath + File.separator + "inputImages/resnet18/" + ResourceScope.using() { + val modelDirPath = tempDirPath + File.separator + "resnet18/" + val inputImagePath = tempDirPath + File.separator + + "inputImages/resnet18/Pug-Cookie.jpg" + val inputImageDir = tempDirPath + File.separator + "inputImages/resnet18/" - var context = Context.cpu() - if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { - context = Context.gpu() - } + var context = Context.cpu() + if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && + System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + context = Context.gpu() + } - val output = ImageClassifierExample.runInferenceOnSingleImage(modelDirPath + "resnet-18", - inputImagePath, context) + val output = ImageClassifierExample.runInferenceOnSingleImage(modelDirPath + "resnet-18", + inputImagePath, context) - val outputList = ImageClassifierExample.runInferenceOnBatchOfImage(modelDirPath + "resnet-18", - inputImageDir, context) + val outputList = ImageClassifierExample.runInferenceOnBatchOfImage(modelDirPath + "resnet-18", + inputImageDir, context) - Process("rm -rf " + modelDirPath + " " + inputImageDir) ! + Process("rm -rf " + modelDirPath + " " + inputImageDir) ! - assert(output(0).toList.head._1 === "n02110958 pug, pug-dog") - assert(outputList(0).toList.head._1 === "n02110958 pug, pug-dog") + assert(output(0).toList.head._1 === "n02110958 pug, pug-dog") + assert(outputList(0).toList.head._1 === "n02110958 pug, pug-dog") + } } } diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/predictor/PredictorExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/predictor/PredictorExampleSuite.scala index 97ca33e71e35..f04f06a37cdd 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/predictor/PredictorExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/predictor/PredictorExampleSuite.scala @@ -65,23 +65,25 @@ class PredictorExampleSuite extends FunSuite with BeforeAndAfterAll { } test("test Predictor With Fixed Shape and random shape") { - val inputDesc = IndexedSeq(new DataDesc("data", Shape(1, 3, 224, 224), - DType.Float32, Layout.NCHW)) - val predictor = PredictorExample.loadModel(modelDirPrefix, inputDesc, context, 0) - // fix size - var img = PredictorExample.preProcess(inputImagePath, 224, 224) - var result = PredictorExample.doInference(predictor, img)(0) - var top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray) - assert(top1 === "n02110958 pug, pug-dog") - // random size 512 - img = PredictorExample.preProcess(inputImagePath, 512, 512) - result = PredictorExample.doInference(predictor, img)(0) - top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray) - assert(top1 === "n02110958 pug, pug-dog") - // original size - img = PredictorExample.preProcess(inputImagePath, 1024, 576) - result = PredictorExample.doInference(predictor, img)(0) - top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray) - assert(top1 === "n02110958 pug, pug-dog") + ResourceScope.using() { + val inputDesc = IndexedSeq(new DataDesc("data", Shape(1, 3, 224, 224), + DType.Float32, Layout.NCHW)) + val predictor = PredictorExample.loadModel(modelDirPrefix, inputDesc, context, 0) + // fix size + var img = PredictorExample.preProcess(inputImagePath, 224, 224) + var result = PredictorExample.doInference(predictor, img)(0) + var top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray) + assert(top1 === "n02110958 pug, pug-dog") + // random size 512 + img = PredictorExample.preProcess(inputImagePath, 512, 512) + result = PredictorExample.doInference(predictor, img)(0) + top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray) + assert(top1 === "n02110958 pug, pug-dog") + // original size + img = PredictorExample.preProcess(inputImagePath, 1024, 576) + result = PredictorExample.doInference(predictor, img)(0) + top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray) + assert(top1 === "n02110958 pug, pug-dog") + } } } diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala index 65902c7ad391..45361f79005d 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala @@ -34,16 +34,18 @@ class MultiTaskSuite extends FunSuite { System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { logger.info("Multitask Test...") - val batchSize = 100 - val numEpoch = 3 - val ctx = Context.gpu() + ResourceScope.using() { + val batchSize = 100 + val numEpoch = 3 + val ctx = Context.gpu() - val modelPath = ExampleMultiTask.getTrainingData - val (executor, evalMetric) = ExampleMultiTask.train(batchSize, numEpoch, ctx, modelPath) - evalMetric.get.foreach { case (name, value) => - assert(value >= 0.95f) + val modelPath = ExampleMultiTask.getTrainingData + val (executor, evalMetric) = ExampleMultiTask.train(batchSize, numEpoch, ctx, modelPath) + evalMetric.get.foreach { case (name, value) => + assert(value >= 0.95f) + } + executor.dispose() } - executor.dispose() } else { logger.info("GPU test only, skipped...") } diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/neuralstyle/NeuralStyleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/neuralstyle/NeuralStyleSuite.scala index c93a7d06a452..40033ad5f6d1 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/neuralstyle/NeuralStyleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/neuralstyle/NeuralStyleSuite.scala @@ -17,7 +17,7 @@ package org.apache.mxnetexamples.neuralstyle -import org.apache.mxnet.{Context, NDArrayCollector} +import org.apache.mxnet.{Context, NDArrayCollector, ResourceScope} import org.apache.mxnetexamples.Util import org.apache.mxnetexamples.neuralstyle.end2end.{BoostInference, BoostTrain} import org.scalatest.{BeforeAndAfterAll, FunSuite} @@ -61,8 +61,10 @@ class NeuralStyleSuite extends FunSuite with BeforeAndAfterAll { System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { ctx = Context.gpu() } - BoostInference.runInference(tempDirPath + "/NS/model", tempDirPath + "/NS", 2, - tempDirPath + "/NS/IMG_4343.jpg", ctx) + ResourceScope.using() { + BoostInference.runInference(tempDirPath + "/NS/model", tempDirPath + "/NS", 2, + tempDirPath + "/NS/IMG_4343.jpg", ctx) + } } test("Example CI: Test Boost Training") { @@ -70,8 +72,10 @@ class NeuralStyleSuite extends FunSuite with BeforeAndAfterAll { if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { val ctx = Context.gpu() - BoostTrain.runTraining(tempDirPath + "/NS/images", tempDirPath + "/NS/vgg19.params", ctx, - tempDirPath + "/NS/starry_night.jpg", tempDirPath + "/NS") + ResourceScope.using() { + BoostTrain.runTraining(tempDirPath + "/NS/images", tempDirPath + "/NS/vgg19.params", ctx, + tempDirPath + "/NS/starry_night.jpg", tempDirPath + "/NS") + } } else { logger.info("GPU test only, skip CPU...") } @@ -82,10 +86,12 @@ class NeuralStyleSuite extends FunSuite with BeforeAndAfterAll { if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { val ctx = Context.gpu() - NeuralStyle.runTraining("vgg19", tempDirPath + "/NS/IMG_4343.jpg", - tempDirPath + "/NS/starry_night.jpg", - ctx, tempDirPath + "/NS/vgg19.params", tempDirPath + "/NS", - 1f, 20f, 0.01f, 1, 10f, 60, 600, 50, 0.0005f) + ResourceScope.using() { + NeuralStyle.runTraining("vgg19", tempDirPath + "/NS/IMG_4343.jpg", + tempDirPath + "/NS/starry_night.jpg", + ctx, tempDirPath + "/NS/vgg19.params", tempDirPath + "/NS", + 1f, 20f, 0.01f, 1, 10f, 60, 600, 50, 0.0005f) + } } else { logger.info("GPU test only, skip CPU") } diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala index ff3fbe9e05d2..b4ce66b99acf 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala @@ -18,7 +18,7 @@ package org.apache.mxnetexamples.rnn -import org.apache.mxnet.{Context, NDArrayCollector} +import org.apache.mxnet.{Context, NDArrayCollector, ResourceScope} import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite, Ignore} import org.slf4j.LoggerFactory @@ -44,37 +44,43 @@ class ExampleRNNSuite extends FunSuite with BeforeAndAfterAll { } test("Example CI: Test LSTM Bucketing") { - val tempDirPath = System.getProperty("java.io.tmpdir") - var ctx = Context.cpu() - if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { - ctx = Context.gpu() - } - if (!System.getenv().containsKey("CI")) { - LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt", - tempDirPath + "/RNN/sherlockholmes.valid.txt", Array(ctx), 1) - } else { - logger.info("Skipping test on CI...") + ResourceScope.using() { + val tempDirPath = System.getProperty("java.io.tmpdir") + var ctx = Context.cpu() + if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && + System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + ctx = Context.gpu() + } + if (!System.getenv().containsKey("CI")) { + LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt", + tempDirPath + "/RNN/sherlockholmes.valid.txt", Array(ctx), 1) + } else { + logger.info("Skipping test on CI...") + } } } test("Example CI: Test TrainCharRNN") { - val tempDirPath = System.getProperty("java.io.tmpdir") - if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - System.getenv("SCALA_TEST_ON_GPU").toInt == 1 && - !System.getenv().containsKey("CI")) { - val ctx = Context.gpu() - TrainCharRnn.runTrainCharRnn(tempDirPath + "/RNN/obama.txt", - tempDirPath, ctx, 1) - } else { - logger.info("CPU not supported for this test, skipped...") + ResourceScope.using() { + val tempDirPath = System.getProperty("java.io.tmpdir") + if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && + System.getenv("SCALA_TEST_ON_GPU").toInt == 1 && + !System.getenv().containsKey("CI")) { + val ctx = Context.gpu() + TrainCharRnn.runTrainCharRnn(tempDirPath + "/RNN/obama.txt", + tempDirPath, ctx, 1) + } else { + logger.info("CPU not supported for this test, skipped...") + } } } test("Example CI: Test Inference on CharRNN") { - val tempDirPath = System.getProperty("java.io.tmpdir") - val ctx = Context.gpu() - TestCharRnn.runInferenceCharRNN(tempDirPath + "/RNN/obama.txt", - tempDirPath + "/RNN/obama", "The joke") + ResourceScope.using() { + val tempDirPath = System.getProperty("java.io.tmpdir") + val ctx = Context.gpu() + TestCharRnn.runInferenceCharRNN(tempDirPath + "/RNN/obama.txt", + tempDirPath + "/RNN/obama", "The joke") + } } } diff --git a/scala-package/mxnet-demo/scala-demo/src/main/scala/sample/ImageClassificationExample.scala b/scala-package/mxnet-demo/scala-demo/src/main/scala/sample/ImageClassificationExample.scala index b5af654e6916..bb6114dfede4 100644 --- a/scala-package/mxnet-demo/scala-demo/src/main/scala/sample/ImageClassificationExample.scala +++ b/scala-package/mxnet-demo/scala-demo/src/main/scala/sample/ImageClassificationExample.scala @@ -94,4 +94,4 @@ object ImageClassificationExample { } } -} \ No newline at end of file +}