Fix test memory with ResourceScope (apache#14666)

haohuanw · Jun 23, 2019 · 431dd9f · 431dd9f
1 parent 16ed2d5
commit 431dd9f
Show file tree

Hide file tree

Showing 8 changed files with 151 additions and 121 deletions.
diff --git a/...kage/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala b/...kage/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala
@@ -21,6 +21,7 @@ import java.net.URL
 
 import org.apache.commons.io.FileUtils
 import org.apache.mxnet.Context
+import org.apache.mxnet.ResourceScope;
 import org.apache.mxnetexamples.Util
 import org.scalatest.{BeforeAndAfterAll, FunSuite}
 import org.slf4j.LoggerFactory
@@ -37,21 +38,23 @@ class CustomOpExampleSuite extends FunSuite with BeforeAndAfterAll {
       System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
       logger.info("CPU test only, skipped...")
     } else {
-      logger.info("Downloading mnist model")
-      val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci"
-      val tempDirPath = System.getProperty("java.io.tmpdir")
-      val modelDirPath = tempDirPath + File.separator + "mnist/"
-      val tmpFile = new File(tempDirPath + "/mnist/mnist.zip")
-      if (!tmpFile.exists()) {
-        FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"),
-          tmpFile)
+      ResourceScope.using() {
+        logger.info("Downloading mnist model")
+        val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci"
+        val tempDirPath = System.getProperty("java.io.tmpdir")
+        val modelDirPath = tempDirPath + File.separator + "mnist/"
+        val tmpFile = new File(tempDirPath + "/mnist/mnist.zip")
+        if (!tmpFile.exists()) {
+          FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"),
+                                  tmpFile)
+        }
+        // TODO: Need to confirm with Windows
+        Process("unzip " + tempDirPath + "/mnist/mnist.zip -d "
+                  + tempDirPath + "/mnist/") !
+        val context = Context.cpu()
+        val output = ExampleCustomOp.test(modelDirPath, context)
+        assert(output >= 0.95f)
       }
-      // TODO: Need to confirm with Windows
-      Process("unzip " + tempDirPath + "/mnist/mnist.zip -d "
-        + tempDirPath + "/mnist/") !
-      val context = Context.cpu()
-      val output = ExampleCustomOp.test(modelDirPath, context)
-      assert(output >= 0.95f)
     }
   }
 
@@ -62,18 +65,20 @@ class CustomOpExampleSuite extends FunSuite with BeforeAndAfterAll {
     if (RTC_fixed) {
       if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
         System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
-        logger.info("Downloading mnist model")
-        val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci"
-        val tempDirPath = System.getProperty("java.io.tmpdir")
-        val modelDirPath = tempDirPath + File.separator + "mnist/"
-        Util.downloadUrl(baseUrl + "/mnist/mnist.zip",
-          tempDirPath + "/mnist/mnist.zip")
-        // TODO: Need to confirm with Windows
-        Process("unzip " + tempDirPath + "/mnist/mnist.zip -d "
-          + tempDirPath + "/mnist/") !
-        val context = Context.gpu()
-        val output = ExampleCustomOpWithRtc.test(modelDirPath, context)
-        assert(output >= 0.95f)
+        ResourceScope.using() {
+          logger.info("Downloading mnist model")
+          val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci"
+          val tempDirPath = System.getProperty("java.io.tmpdir")
+          val modelDirPath = tempDirPath + File.separator + "mnist/"
+          Util.downloadUrl(baseUrl + "/mnist/mnist.zip",
+                           tempDirPath + "/mnist/mnist.zip")
+          // TODO: Need to confirm with Windows
+          Process("unzip " + tempDirPath + "/mnist/mnist.zip -d "
+                    + tempDirPath + "/mnist/") !
+          val context = Context.gpu()
+          val output = ExampleCustomOpWithRtc.test(modelDirPath, context)
+          assert(output >= 0.95f)
+        }
       } else {
         logger.info("GPU test only, skipped...")
       }

diff --git a/...c/test/scala/org/apache/mxnetexamples/imclassification/IMClassificationExampleSuite.scala b/...c/test/scala/org/apache/mxnetexamples/imclassification/IMClassificationExampleSuite.scala
@@ -19,7 +19,7 @@ package org.apache.mxnetexamples.imclassification
 
 import java.io.File
 
-import org.apache.mxnet.{Context, DType}
+import org.apache.mxnet.{Context, DType, ResourceScope}
 import org.apache.mxnetexamples.Util
 import org.scalatest.{BeforeAndAfterAll, FunSuite}
 import org.slf4j.LoggerFactory
@@ -35,35 +35,41 @@ class IMClassificationExampleSuite extends FunSuite with BeforeAndAfterAll {
 
   test("Example CI: Test MNIST Training") {
 
-    logger.info("Downloading mnist model")
-    val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci"
-    val tempDirPath = System.getProperty("java.io.tmpdir")
-    val modelDirPath = tempDirPath + File.separator + "mnist/"
-    logger.info("tempDirPath: %s".format(tempDirPath))
-    Util.downloadUrl(baseUrl + "/mnist/mnist.zip",
-      tempDirPath + "/mnist/mnist.zip")
-    // TODO: Need to confirm with Windows
-    Process("unzip " + tempDirPath + "/mnist/mnist.zip -d "
-      + tempDirPath + "/mnist/") !
+    ResourceScope.using() {
+      logger.info("Downloading mnist model")
+      val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci"
+      val tempDirPath = System.getProperty("java.io.tmpdir")
+      val modelDirPath = tempDirPath + File.separator + "mnist/"
+      logger.info("tempDirPath: %s".format(tempDirPath))
+      Util.downloadUrl(baseUrl + "/mnist/mnist.zip",
+        tempDirPath + "/mnist/mnist.zip")
+      // TODO: Need to confirm with Windows
+      Process("unzip " + tempDirPath + "/mnist/mnist.zip -d "
+        + tempDirPath + "/mnist/") !
 
-    var context = Context.cpu()
+      var context = Context.cpu()
 
-    val valAccuracy = TrainModel.test("mlp", modelDirPath)
-    Process("rm -rf " + modelDirPath) !
+      val valAccuracy = TrainModel.test("mlp", modelDirPath)
+      Process("rm -rf " + modelDirPath) !
 
-    assert(valAccuracy >= 0.95f)
+      assert(valAccuracy >= 0.95f)
+    }
   }
 
   for(model <- List("mlp", "lenet", "resnet")) {
     test(s"Example CI: Test Image Classification Model ${model}") {
-      val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true)
+      ResourceScope.using() {
+        val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true)
+      }
     }
   }
 
   for(model <- List("mlp", "lenet", "resnet")) {
     test(s"Example CI: Test Image Classification Model ${model} with Float64 input") {
-      val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true,
-        dtype = DType.Float64)
+      ResourceScope.using() {
+        val valAccuracy = TrainModel.test(model, "", 10, 1, benchmark = true,
+          dtype = DType.Float64)
+      }
     }
   }
 

diff --git a/...st/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/...st/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala
@@ -21,6 +21,7 @@ import org.scalatest.{BeforeAndAfterAll, FunSuite}
 import org.slf4j.LoggerFactory
 import java.io.File
 import org.apache.mxnet.Context
+import org.apache.mxnet.ResourceScope
 import org.apache.mxnetexamples.Util
 
 import scala.language.postfixOps
@@ -49,27 +50,29 @@ class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
     Util.downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg",
       tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg")
 
-    val modelDirPath = tempDirPath + File.separator + "resnet18/"
-    val inputImagePath = tempDirPath + File.separator +
-      "inputImages/resnet18/Pug-Cookie.jpg"
-    val inputImageDir = tempDirPath + File.separator + "inputImages/resnet18/"
+    ResourceScope.using() {
+      val modelDirPath = tempDirPath + File.separator + "resnet18/"
+      val inputImagePath = tempDirPath + File.separator +
+        "inputImages/resnet18/Pug-Cookie.jpg"
+      val inputImageDir = tempDirPath + File.separator + "inputImages/resnet18/"
 
-    var context = Context.cpu()
-    if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
-      System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
-      context = Context.gpu()
-    }
+      var context = Context.cpu()
+      if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
+            System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
+        context = Context.gpu()
+      }
 
-    val output = ImageClassifierExample.runInferenceOnSingleImage(modelDirPath + "resnet-18",
-     inputImagePath, context)
+      val output = ImageClassifierExample.runInferenceOnSingleImage(modelDirPath + "resnet-18",
+                                                                    inputImagePath, context)
 
-    val outputList = ImageClassifierExample.runInferenceOnBatchOfImage(modelDirPath + "resnet-18",
-        inputImageDir, context)
+      val outputList = ImageClassifierExample.runInferenceOnBatchOfImage(modelDirPath + "resnet-18",
+                                                                         inputImageDir, context)
 
-    Process("rm -rf " + modelDirPath + " " + inputImageDir) !
+      Process("rm -rf " + modelDirPath + " " + inputImageDir) !
 
-    assert(output(0).toList.head._1 === "n02110958 pug, pug-dog")
-    assert(outputList(0).toList.head._1 === "n02110958 pug, pug-dog")
+      assert(output(0).toList.head._1 === "n02110958 pug, pug-dog")
+      assert(outputList(0).toList.head._1 === "n02110958 pug, pug-dog")
+    }
 
   }
 }
diff --git a/...mples/src/test/scala/org/apache/mxnetexamples/infer/predictor/PredictorExampleSuite.scala b/...mples/src/test/scala/org/apache/mxnetexamples/infer/predictor/PredictorExampleSuite.scala
@@ -65,23 +65,25 @@ class PredictorExampleSuite extends FunSuite with BeforeAndAfterAll {
   }
 
   test("test Predictor With Fixed Shape and random shape") {
-    val inputDesc = IndexedSeq(new DataDesc("data", Shape(1, 3, 224, 224),
-      DType.Float32, Layout.NCHW))
-    val predictor = PredictorExample.loadModel(modelDirPrefix, inputDesc, context, 0)
-    // fix size
-    var img = PredictorExample.preProcess(inputImagePath, 224, 224)
-    var result = PredictorExample.doInference(predictor, img)(0)
-    var top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray)
-    assert(top1 === "n02110958 pug, pug-dog")
-    // random size 512
-    img = PredictorExample.preProcess(inputImagePath, 512, 512)
-    result = PredictorExample.doInference(predictor, img)(0)
-    top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray)
-    assert(top1 === "n02110958 pug, pug-dog")
-    // original size
-    img = PredictorExample.preProcess(inputImagePath, 1024, 576)
-    result = PredictorExample.doInference(predictor, img)(0)
-    top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray)
-    assert(top1 === "n02110958 pug, pug-dog")
+    ResourceScope.using() {
+      val inputDesc = IndexedSeq(new DataDesc("data", Shape(1, 3, 224, 224),
+                                              DType.Float32, Layout.NCHW))
+      val predictor = PredictorExample.loadModel(modelDirPrefix, inputDesc, context, 0)
+      // fix size
+      var img = PredictorExample.preProcess(inputImagePath, 224, 224)
+      var result = PredictorExample.doInference(predictor, img)(0)
+      var top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray)
+      assert(top1 === "n02110958 pug, pug-dog")
+      // random size 512
+      img = PredictorExample.preProcess(inputImagePath, 512, 512)
+      result = PredictorExample.doInference(predictor, img)(0)
+      top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray)
+      assert(top1 === "n02110958 pug, pug-dog")
+      // original size
+      img = PredictorExample.preProcess(inputImagePath, 1024, 576)
+      result = PredictorExample.doInference(predictor, img)(0)
+      top1 = PredictorExample.postProcess(modelDirPrefix, result.toArray)
+      assert(top1 === "n02110958 pug, pug-dog")
+    }
   }
 }
diff --git a/...a-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala b/...a-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala
@@ -34,16 +34,18 @@ class MultiTaskSuite extends FunSuite {
       System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
       logger.info("Multitask Test...")
 
-      val batchSize = 100
-      val numEpoch = 3
-      val ctx = Context.gpu()
+      ResourceScope.using() {
+        val batchSize = 100
+        val numEpoch = 3
+        val ctx = Context.gpu()
 
-      val modelPath = ExampleMultiTask.getTrainingData
-      val (executor, evalMetric) = ExampleMultiTask.train(batchSize, numEpoch, ctx, modelPath)
-      evalMetric.get.foreach { case (name, value) =>
-        assert(value >= 0.95f)
+        val modelPath = ExampleMultiTask.getTrainingData
+        val (executor, evalMetric) = ExampleMultiTask.train(batchSize, numEpoch, ctx, modelPath)
+        evalMetric.get.foreach { case (name, value) =>
+          assert(value >= 0.95f)
+        }
+        executor.dispose()
       }
-      executor.dispose()
     } else {
       logger.info("GPU test only, skipped...")
     }

diff --git a/...ckage/examples/src/test/scala/org/apache/mxnetexamples/neuralstyle/NeuralStyleSuite.scala b/...ckage/examples/src/test/scala/org/apache/mxnetexamples/neuralstyle/NeuralStyleSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.mxnetexamples.neuralstyle
 
-import org.apache.mxnet.Context
+import org.apache.mxnet.{Context, ResourceScope}
 import org.apache.mxnetexamples.Util
 import org.apache.mxnetexamples.neuralstyle.end2end.{BoostInference, BoostTrain}
 import org.scalatest.{BeforeAndAfterAll, FunSuite}
@@ -61,17 +61,21 @@ class NeuralStyleSuite extends FunSuite with BeforeAndAfterAll {
       System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
       ctx = Context.gpu()
     }
-    BoostInference.runInference(tempDirPath + "/NS/model", tempDirPath + "/NS", 2,
-      tempDirPath + "/NS/IMG_4343.jpg", ctx)
+    ResourceScope.using() {
+      BoostInference.runInference(tempDirPath + "/NS/model", tempDirPath + "/NS", 2,
+                                  tempDirPath + "/NS/IMG_4343.jpg", ctx)
+    }
   }
 
   test("Example CI: Test Boost Training") {
     val tempDirPath = System.getProperty("java.io.tmpdir")
     if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
       System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
       val ctx = Context.gpu()
-      BoostTrain.runTraining(tempDirPath + "/NS/images", tempDirPath + "/NS/vgg19.params", ctx,
-        tempDirPath + "/NS/starry_night.jpg", tempDirPath + "/NS")
+      ResourceScope.using() {
+        BoostTrain.runTraining(tempDirPath + "/NS/images", tempDirPath + "/NS/vgg19.params", ctx,
+                               tempDirPath + "/NS/starry_night.jpg", tempDirPath + "/NS")
+      }
     } else {
       logger.info("GPU test only, skip CPU...")
     }
@@ -82,10 +86,12 @@ class NeuralStyleSuite extends FunSuite with BeforeAndAfterAll {
     if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
       System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
       val ctx = Context.gpu()
-      NeuralStyle.runTraining("vgg19", tempDirPath + "/NS/IMG_4343.jpg",
-        tempDirPath + "/NS/starry_night.jpg",
-        ctx, tempDirPath + "/NS/vgg19.params", tempDirPath + "/NS",
-        1f, 20f, 0.01f, 1, 10f, 60, 600, 50, 0.0005f)
+      ResourceScope.using() {
+        NeuralStyle.runTraining("vgg19", tempDirPath + "/NS/IMG_4343.jpg",
+                                tempDirPath + "/NS/starry_night.jpg",
+                                ctx, tempDirPath + "/NS/vgg19.params", tempDirPath + "/NS",
+                                1f, 20f, 0.01f, 1, 10f, 60, 600, 50, 0.0005f)
+      }
     } else {
       logger.info("GPU test only, skip CPU")
     }