apache#13624 clojure nightly tests (apache#13624)

ci/docker/runtime_functions.sh

@@ -868,6 +868,14 @@ unittest_ubuntu_cpu_clojure() {
     ./contrib/clojure-package/ci-test.sh
 }
 
+unittest_ubuntu_cpu_clojure_integration() {
+    set -ex
+    make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+    make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+    ./contrib/clojure-package/integration-tests.sh
+}
+
+
 unittest_ubuntu_cpugpu_perl() {
     set -ex
     ./perl-package/test.sh

ci/jenkins/Jenkins_steps.groovy

@@ -887,6 +887,19 @@ def test_unix_clojure_cpu() {
     }]
 }
 
+def test_unix_clojure_integration_cpu() {
+    return ['Clojure: CPU Integration': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-clojure-integration-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.unpack_and_init('cpu', mx_lib, true)
+            utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure_integration', false)
+          }
+        }
+      }
+    }]
+}
+
 def test_unix_r_cpu() {
     return ['R: CPU': {
       node(NODE_LINUX_CPU) {

ci/jenkins/Jenkinsfile_unix_cpu

@@ -52,6 +52,7 @@ core_logic: {
     custom_steps.test_unix_scala_cpu(),
     custom_steps.test_unix_scala_mkldnn_cpu(),
     custom_steps.test_unix_clojure_cpu(),
+    custom_steps.test_unix_clojure_integration_cpu(),
     custom_steps.test_unix_perl_cpu(),
     custom_steps.test_unix_r_cpu(),
     custom_steps.test_unix_julia07_cpu(),

contrib/clojure-package/examples/infer/imageclassifier/src/infer/imageclassifier_example.clj

@@ -74,6 +74,8 @@
         image-file-batches (->> input-dir
                                 io/file
                                 file-seq
+                                sort
+                                reverse
                                 (filter #(.isFile %))
                                 (filter #(re-matches #".*\.jpg$" (.getPath %)))
                                 (mapv #(.getPath %))

contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj

@@ -78,32 +78,44 @@
 
 (defn detect-single-image
   "Detect objects in a single image and print top-5 predictions"
-  [detector input-image output-dir]
+  ([detector input-dir] (detect-single-image detector input-dir "results"))
+  ([detector input-image output-dir]
+    (.mkdir (io/file output-dir))
   (let [image (infer/load-image-from-file input-image)
-        topk 5]
+        topk 5
+        res (infer/detect-objects detector image topk)
+        ]
     (process-results
      [input-image]
-     (infer/detect-objects detector image topk)
-     output-dir)))
+     res
+     output-dir)
+    (first res)
+    )))
 
 (defn detect-images-in-dir
   "Detect objects in all jpg images in the directory"
-  [detector input-dir output-dir]
+  ([detector input-dir] (detect-images-in-dir detector input-dir "results"))
+  ([detector input-dir output-dir]
+  (.mkdir (io/file output-dir))
   (let [batch-size 20
         image-file-batches (->> input-dir
                                 io/file
                                 file-seq
+                                sort
                                 (filter #(.isFile %))
                                 (filter #(re-matches #".*\.jpg$" (.getPath %)))
                                 (mapv #(.getPath %))
                                 (partition-all batch-size))]
-    (doall
+    (apply concat
      (for [image-files image-file-batches]
-       (let [image-batch (infer/load-image-paths image-files) topk 5]
+       (let [image-batch (infer/load-image-paths image-files) 
+             topk 5 
+             res (infer/detect-objects-batch detector image-batch topk) ]
          (process-results
           image-files
-          (infer/detect-objects-batch detector image-batch topk)
-          output-dir))))))
+          res
+          output-dir) 
+         res))))))
 
 (defn run-detector
   "Runs an image detector based on options provided"
@@ -119,7 +131,7 @@
         detector (infer/create-object-detector
                   factory
                   {:contexts [(context/default-context)]})]
-    (println "Output results to:" output-dir ":" (.mkdir (io/file output-dir)))
+    (println "Output results to:" output-dir ":")
     (println "Object detection on a single image")
     (detect-single-image detector input-image output-dir)
     (println "Object detection on images in a directory")

contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj

@@ -45,6 +45,7 @@
   (let [detector (create-detector)
         predictions (detect-single-image detector image-file)
         {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)]
+    (clojure.pprint/pprint predictions)
     (is (some? predictions))
     (is (= 5 (count predictions)))
     (is (string? class))
@@ -55,11 +56,13 @@
 (deftest test-batch-detection
   (let [detector (create-detector)
         batch-predictions (detect-images-in-dir detector image-dir)
+        _ (clojure.pprint/pprint batch-predictions)
         predictions (first batch-predictions)
         {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)]
     (is (some? batch-predictions))
     (is (= 5 (count predictions)))
     (is (string? class))
     (is (< 0.8 prob))
+    (println [x-min x-max y-min y-max])
     (every? #(< 0 % 1) [x-min x-max y-min y-max])
-    (is (= #{"dog" "person" "bicycle" "car"} (set (mapv :class predictions))))))
+    (is (= #{"dog" "person"} (set (mapv :class predictions))))))

contrib/clojure-package/integration-tests.sh

@@ -18,11 +18,15 @@
 
 set -evx
 
-MXNET_HOME=$(cd "$(dirname $0)/../.."; pwd)
+MXNET_HOME=${PWD}
+cd ${MXNET_HOME}/contrib/clojure-package
+# first build the package and install it
+lein install
+
+# then run through the examples 
 EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples
-#cd ${MXNET_HOME}/contrib/clojure-package
-#lein test
-#lein cloverage --codecov
-for test_dir in `find ${EXAMPLES_HOME} -name test` ; do
-  cd ${test_dir} && lein test
-done
+# use AWK pattern for blacklisting
+TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|dontselect2/'`
+for i in $TEST_CASES ; do
+ cd ${i} && lein test
+done