[Clojure] enhance draw bounding box (#14567)

* add test for drawing bounding box

* Uses the core image drawing bounding box functionality for the object detection example.

Adjust the specs and names to make it easier to run with object detection and clojure draw bounding box example

* feedback from @Chouffe

* refactor to be 3 top predictions instead of 5 to make the images less crowded

contrib/clojure-package/examples/infer/objectdetector/project.clj

@@ -22,7 +22,6 @@
   :aliases {"run-detector" ["run" "--" "-m" "models/resnet50_ssd/resnet50_ssd_model" "-i" "images/dog.jpg" "-d" "images/"]}
   :dependencies [[org.clojure/clojure "1.9.0"]
                  [org.clojure/tools.cli "0.4.1"]
-                 [origami "4.0.0-3"]
                  [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.5.0-SNAPSHOT"]]
   :main ^:skip-aot infer.objectdetector-example
   :profiles {:uberjar {:aot :all}})

contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj

@@ -17,13 +17,15 @@
 (ns infer.objectdetector-example
   (:require [org.apache.clojure-mxnet.context :as context]
             [org.apache.clojure-mxnet.dtype :as dtype]
+            [org.apache.clojure-mxnet.image :as image]
             [org.apache.clojure-mxnet.infer :as infer]
             [org.apache.clojure-mxnet.layout :as layout]
             [clojure.java.io :as io]
-            [infer.draw :as draw]
-            [clojure.string :refer [join]]
+            [clojure.string :as string]
             [clojure.tools.cli :refer [parse-opts]])
-  (:gen-class))
+  (:gen-class)
+  (:import (javax.imageio ImageIO)
+           (java.io File)))
 
 (defn check-valid-dir
   "Check that the input directory exists"
@@ -54,35 +56,44 @@
     :validate [check-valid-dir "Input directory not found"]]
    ["-h" "--help"]])
 
-(defn result->map [{:keys [class prob x-min y-min x-max y-max]}]
-  (hash-map
-   :label class
-   :confidence (int (* 100 prob))
-   :top-left [x-min y-min]
-   :bottom-right [x-max y-max]))
 
-(defn print-results [results]
-  (doseq [_r results]
-    (println (format "Class: %s Confidence=%s Coords=(%s, %s)"
-                     (_r :label)
-                     (_r :confidence)
-                     (_r :top-left)
-                     (_r :bottom-right)))))
+(defn process-result! [output-dir image-path predictions]
+  (println "looking at image" image-path)
+  (println "predictions: " predictions)
+  (let [buf (ImageIO/read (new File image-path))
+        width (.getWidth buf)
+        height (.getHeight buf)
+        names (mapv :class predictions)
+        coords (mapv (fn [prediction]
+                       (-> prediction
+                           (update :x-min #(* width %))
+                           (update :x-max #(* width %))
+                           (update :y-min #(* height %))
+                           (update :y-max #(* height %))))
+                     predictions)
+        new-img  (-> (ImageIO/read (new File image-path))
+                     (image/draw-bounding-box! coords
+                                               {:stroke 2
+                                                :names (mapv #(str (:class %) "-" (:prob %))
+                                                             predictions)
+                                                :transparency 0.5
+
+                                                :font-size-mult 1.0}))]
+    (->> (string/split image-path #"\/")
+         last
+         (io/file output-dir)
+         (ImageIO/write new-img "jpg"))))
 
 (defn process-results [images results output-dir]
-  (dotimes [i (count images)]
-    (let [image (nth images i) _results (map result->map (nth results i))]
-      (println "processing: " image)
-      (print-results _results)
-      (draw/draw-bounds image _results output-dir))))
+  (doall (map (partial process-result! output-dir) images results)))
 
 (defn detect-single-image
   "Detect objects in a single image and print top-5 predictions"
   ([detector input-dir] (detect-single-image detector input-dir "results"))
   ([detector input-image output-dir]
     (.mkdir (io/file output-dir))
   (let [image (infer/load-image-from-file input-image)
-        topk 5
+        topk 3
         res (infer/detect-objects detector image topk)
         ]
     (process-results
@@ -109,7 +120,7 @@
     (apply concat
      (for [image-files image-file-batches]
        (let [image-batch (infer/load-image-paths image-files) 
-             topk 5 
+             topk 3
              res (infer/detect-objects-batch detector image-batch topk) ]
          (process-results
           image-files
@@ -143,5 +154,5 @@
         (parse-opts args cli-options)]
     (cond
       (:help options) (println summary)
-      (some? errors) (println (join "\n" errors))
+      (some? errors) (println (string/join "\n" errors))
       :else (run-detector options))))

contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj

@@ -47,11 +47,11 @@
         {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)]
     (clojure.pprint/pprint predictions)
     (is (some? predictions))
-    (is (= 5 (count predictions)))
+    (is (= 3 (count predictions)))
     (is (string? class))
     (is (< 0.8 prob))
     (is (every? #(< 0 % 1) [x-min x-max y-min y-max]))
-    (is (= #{"dog" "person" "bicycle" "car"} (set (mapv :class predictions))))))
+    (is (= #{"dog" "bicycle" "car"} (set (mapv :class predictions))))))
 
 (deftest test-batch-detection
   (let [detector (create-detector)
@@ -60,7 +60,7 @@
         predictions (first batch-predictions)
         {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)]
     (is (some? batch-predictions))
-    (is (= 5 (count predictions)))
+    (is (= 3 (count predictions)))
     (is (string? class))
     (is (< 0.8 prob))
     (println [x-min x-max y-min y-max])

contrib/clojure-package/integration-tests.sh

@@ -26,7 +26,7 @@ lein install
 # then run through the examples 
 EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples
 # use AWK pattern for blacklisting
-TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification|gan|neural-style|infer|pre-trained-models/'`
+TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification|gan|neural-style|pre-trained-models/'`
 for i in $TEST_CASES ; do
  cd ${i} && lein test
 done

contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj

@@ -202,11 +202,11 @@
   (Image/toImage input))
 
 (s/def ::buffered-image #(instance? BufferedImage %))
-(s/def ::xmin integer?)
-(s/def ::xmax integer?)
-(s/def ::ymin integer?)
-(s/def ::ymax integer?)
-(s/def ::coordinate (s/keys :req-un [::xmin ::xmax ::ymin ::ymax]))
+(s/def ::x-min number?)
+(s/def ::x-max number?)
+(s/def ::y-min number?)
+(s/def ::y-max number?)
+(s/def ::coordinate (s/keys :req-un [::x-min ::x-max ::y-min ::y-max]))
 (s/def ::coordinates (s/coll-of ::coordinate))
 (s/def ::names (s/nilable (s/coll-of string?)))
 (s/def ::stroke (s/and integer? pos?))
@@ -217,11 +217,11 @@
 
 (defn- convert-coordinate
   "Convert bounding box coordinate to Scala correct types."
-  [{:keys [xmin xmax ymin ymax]}]
-  {:xmin (int xmin)
-   :xmax (int xmax)
-   :ymin (int ymin)
-   :ymax (int ymax)})
+  [{:keys [x-min x-max y-min y-max]}]
+  {:xmin (int x-min)
+   :xmax (int x-max)
+   :ymin (int y-min)
+   :ymax (int y-max)})
 
 (defn draw-bounding-box!
   "Draw bounding boxes on `buffered-image` and Mutate the input image.
@@ -233,9 +233,9 @@
   `transparency`: float in (0.0, 1.0) - Transparency of the bounding box
   returns: Modified `buffered-image`
   Ex:
-    (draw-bounding-box! img [{:xmin 0 :xmax 100 :ymin 0 :ymax 100}])
-    (draw-bounding-box! [{:xmin 190 :xmax 850 :ymin 50 :ymax 450}
-                         {:xmin 200 :xmax 350 :ymin 440 :ymax 530}]
+    (draw-bounding-box! img [{:x-min 0 :x-max 100 :y-min 0 :y-max 100}])
+    (draw-bounding-box! [{:x-min 190 :x-max 850 :y-min 50 :y-max 450}
+                         {:x-min 200 :x-max 350 :y-min 440 :y-max 530}]
                         {:stroke 2
                          :names [\"pug\" \"cookie\"]
                          :transparency 0.8

contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj

@@ -20,7 +20,8 @@
             [org.apache.clojure-mxnet.ndarray :as ndarray]
             [clojure.java.io :as io]
             [clojure.test :refer :all])
-  (:import (javax.imageio ImageIO)))
+  (:import (javax.imageio ImageIO)
+           (java.io File)))
 
 (def tmp-dir (System/getProperty "java.io.tmpdir"))
 (def image-path (.getAbsolutePath (io/file tmp-dir "Pug-Cookie.jpg")))
@@ -76,4 +77,15 @@
   (let [img-arr (image/read-image image-path)
         resized-arr (image/resize-image img-arr 224 224)
         new-img (image/to-image resized-arr)]
-    (is (= true (ImageIO/write new-img "png" (io/file tmp-dir "out.png"))))))
+    (is (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))
+
+(deftest test-draw-bounding-box!
+  (let [orig-img (ImageIO/read (new File image-path))
+        new-img  (-> orig-img
+                     (image/draw-bounding-box! [{:x-min 190 :x-max 850 :y-min 50 :y-max 450}
+                                                {:x-min 200 :x-max 350 :y-min 440 :y-max 530}]
+                                               {:stroke 2
+                                                :names ["pug" "cookie"]
+                                                :transparency 0.8
+                                                :font-size-mult 2.0}))]
+    (is (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))