From cf675ded96a2c7f201c037fabc6d5947414497ed Mon Sep 17 00:00:00 2001 From: Carin Meier Date: Thu, 11 Apr 2019 19:01:55 -0400 Subject: [PATCH] [Clojure] enhance draw bounding box (#14567) * add test for drawing bounding box * Uses the core image drawing bounding box functionality for the object detection example. Adjust the specs and names to make it easier to run with object detection and clojure draw bounding box example * feedback from @chouffe * refactor to be 3 top predictions instead of 5 to make the images less crowded --- .../examples/infer/objectdetector/project.clj | 1 - .../infer/objectdetector/src/infer/draw.clj | 44 -------------- .../src/infer/objectdetector_example.clj | 59 +++++++++++-------- .../infer/objectdetector_example_test.clj | 6 +- contrib/clojure-package/integration-tests.sh | 2 +- .../src/org/apache/clojure_mxnet/image.clj | 26 ++++---- .../org/apache/clojure_mxnet/image_test.clj | 16 ++++- 7 files changed, 66 insertions(+), 88 deletions(-) delete mode 100644 contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj diff --git a/contrib/clojure-package/examples/infer/objectdetector/project.clj b/contrib/clojure-package/examples/infer/objectdetector/project.clj index cdd9a8991dc8..da01797f5a21 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/project.clj +++ b/contrib/clojure-package/examples/infer/objectdetector/project.clj @@ -22,7 +22,6 @@ :aliases {"run-detector" ["run" "--" "-m" "models/resnet50_ssd/resnet50_ssd_model" "-i" "images/dog.jpg" "-d" "images/"]} :dependencies [[org.clojure/clojure "1.9.0"] [org.clojure/tools.cli "0.4.1"] - [origami "4.0.0-3"] [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.5.0-SNAPSHOT"]] :main ^:skip-aot infer.objectdetector-example :profiles {:uberjar {:aot :all}}) diff --git a/contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj b/contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj deleted file mode 100644 index d29b34b5c22a..000000000000 --- a/contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj +++ /dev/null @@ -1,44 +0,0 @@ -;; Licensed to the Apache Software Foundation (ASF) under one or more -;; contributor license agreements. See the NOTICE file distributed with -;; this work for additional information regarding copyright ownership. -;; The ASF licenses this file to You under the Apache License, Version 2.0 -;; (the "License"); you may not use this file except in compliance with -;; the License. You may obtain a copy of the License at -;; -;; http://www.apache.org/licenses/LICENSE-2.0 -;; -;; Unless required by applicable law or agreed to in writing, software -;; distributed under the License is distributed on an "AS IS" BASIS, -;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -;; See the License for the specific language governing permissions and -;; limitations under the License. -;; - -(ns infer.draw - (:require - [opencv4.colors.rgb :as rgb] - [opencv4.core :refer [FONT_HERSHEY_PLAIN imread imwrite new-point put-text! rectangle]])) - -(defn black-boxes! [img results] - (doseq [{confidence :confidence label :label top-left :top-left bottom-right :bottom-right} results] - (let [w (.width img) - h (.height img) - top-left-p (new-point (int (* w (first top-left))) (int (* h (second top-left)))) - bottom-right-p (new-point (int (* w (first bottom-right))) (int (* h (second bottom-right))))] - (if (< 15 confidence) - (do - (rectangle img top-left-p bottom-right-p rgb/white 1) - (put-text! img - (str label "[" confidence "% ]") - top-left-p - FONT_HERSHEY_PLAIN - 1.0 - rgb/white 1))))) - img) - -(defn draw-bounds [image results output-dir] - (let [out-file (str output-dir "/" (.getName (clojure.java.io/as-file image)))] - (-> image - (imread) - (black-boxes! results) - (imwrite out-file)))) \ No newline at end of file diff --git a/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj b/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj index 9331798b038c..65d822ff36aa 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj +++ b/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj @@ -17,13 +17,15 @@ (ns infer.objectdetector-example (:require [org.apache.clojure-mxnet.context :as context] [org.apache.clojure-mxnet.dtype :as dtype] + [org.apache.clojure-mxnet.image :as image] [org.apache.clojure-mxnet.infer :as infer] [org.apache.clojure-mxnet.layout :as layout] [clojure.java.io :as io] - [infer.draw :as draw] - [clojure.string :refer [join]] + [clojure.string :as string] [clojure.tools.cli :refer [parse-opts]]) - (:gen-class)) + (:gen-class) + (:import (javax.imageio ImageIO) + (java.io File))) (defn check-valid-dir "Check that the input directory exists" @@ -54,27 +56,36 @@ :validate [check-valid-dir "Input directory not found"]] ["-h" "--help"]]) -(defn result->map [{:keys [class prob x-min y-min x-max y-max]}] - (hash-map - :label class - :confidence (int (* 100 prob)) - :top-left [x-min y-min] - :bottom-right [x-max y-max])) -(defn print-results [results] - (doseq [_r results] - (println (format "Class: %s Confidence=%s Coords=(%s, %s)" - (_r :label) - (_r :confidence) - (_r :top-left) - (_r :bottom-right))))) +(defn process-result! [output-dir image-path predictions] + (println "looking at image" image-path) + (println "predictions: " predictions) + (let [buf (ImageIO/read (new File image-path)) + width (.getWidth buf) + height (.getHeight buf) + names (mapv :class predictions) + coords (mapv (fn [prediction] + (-> prediction + (update :x-min #(* width %)) + (update :x-max #(* width %)) + (update :y-min #(* height %)) + (update :y-max #(* height %)))) + predictions) + new-img (-> (ImageIO/read (new File image-path)) + (image/draw-bounding-box! coords + {:stroke 2 + :names (mapv #(str (:class %) "-" (:prob %)) + predictions) + :transparency 0.5 + + :font-size-mult 1.0}))] + (->> (string/split image-path #"\/") + last + (io/file output-dir) + (ImageIO/write new-img "jpg")))) (defn process-results [images results output-dir] - (dotimes [i (count images)] - (let [image (nth images i) _results (map result->map (nth results i))] - (println "processing: " image) - (print-results _results) - (draw/draw-bounds image _results output-dir)))) + (doall (map (partial process-result! output-dir) images results))) (defn detect-single-image "Detect objects in a single image and print top-5 predictions" @@ -82,7 +93,7 @@ ([detector input-image output-dir] (.mkdir (io/file output-dir)) (let [image (infer/load-image-from-file input-image) - topk 5 + topk 3 res (infer/detect-objects detector image topk) ] (process-results @@ -109,7 +120,7 @@ (apply concat (for [image-files image-file-batches] (let [image-batch (infer/load-image-paths image-files) - topk 5 + topk 3 res (infer/detect-objects-batch detector image-batch topk) ] (process-results image-files @@ -143,5 +154,5 @@ (parse-opts args cli-options)] (cond (:help options) (println summary) - (some? errors) (println (join "\n" errors)) + (some? errors) (println (string/join "\n" errors)) :else (run-detector options)))) diff --git a/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj b/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj index 696d96b3ae3a..3d20c614918f 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj +++ b/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj @@ -47,11 +47,11 @@ {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)] (clojure.pprint/pprint predictions) (is (some? predictions)) - (is (= 5 (count predictions))) + (is (= 3 (count predictions))) (is (string? class)) (is (< 0.8 prob)) (is (every? #(< 0 % 1) [x-min x-max y-min y-max])) - (is (= #{"dog" "person" "bicycle" "car"} (set (mapv :class predictions)))))) + (is (= #{"dog" "bicycle" "car"} (set (mapv :class predictions)))))) (deftest test-batch-detection (let [detector (create-detector) @@ -60,7 +60,7 @@ predictions (first batch-predictions) {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)] (is (some? batch-predictions)) - (is (= 5 (count predictions))) + (is (= 3 (count predictions))) (is (string? class)) (is (< 0.8 prob)) (println [x-min x-max y-min y-max]) diff --git a/contrib/clojure-package/integration-tests.sh b/contrib/clojure-package/integration-tests.sh index 5ae26e8dfcd4..3df9ba9787b5 100755 --- a/contrib/clojure-package/integration-tests.sh +++ b/contrib/clojure-package/integration-tests.sh @@ -26,7 +26,7 @@ lein install # then run through the examples EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples # use AWK pattern for blacklisting -TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification|gan|neural-style|infer|pre-trained-models/'` +TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification|gan|neural-style|pre-trained-models/'` for i in $TEST_CASES ; do cd ${i} && lein test done diff --git a/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj index e2e98c4e8f01..f81a35803171 100644 --- a/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj +++ b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj @@ -202,11 +202,11 @@ (Image/toImage input)) (s/def ::buffered-image #(instance? BufferedImage %)) -(s/def ::xmin integer?) -(s/def ::xmax integer?) -(s/def ::ymin integer?) -(s/def ::ymax integer?) -(s/def ::coordinate (s/keys :req-un [::xmin ::xmax ::ymin ::ymax])) +(s/def ::x-min number?) +(s/def ::x-max number?) +(s/def ::y-min number?) +(s/def ::y-max number?) +(s/def ::coordinate (s/keys :req-un [::x-min ::x-max ::y-min ::y-max])) (s/def ::coordinates (s/coll-of ::coordinate)) (s/def ::names (s/nilable (s/coll-of string?))) (s/def ::stroke (s/and integer? pos?)) @@ -217,11 +217,11 @@ (defn- convert-coordinate "Convert bounding box coordinate to Scala correct types." - [{:keys [xmin xmax ymin ymax]}] - {:xmin (int xmin) - :xmax (int xmax) - :ymin (int ymin) - :ymax (int ymax)}) + [{:keys [x-min x-max y-min y-max]}] + {:xmin (int x-min) + :xmax (int x-max) + :ymin (int y-min) + :ymax (int y-max)}) (defn draw-bounding-box! "Draw bounding boxes on `buffered-image` and Mutate the input image. @@ -233,9 +233,9 @@ `transparency`: float in (0.0, 1.0) - Transparency of the bounding box returns: Modified `buffered-image` Ex: - (draw-bounding-box! img [{:xmin 0 :xmax 100 :ymin 0 :ymax 100}]) - (draw-bounding-box! [{:xmin 190 :xmax 850 :ymin 50 :ymax 450} - {:xmin 200 :xmax 350 :ymin 440 :ymax 530}] + (draw-bounding-box! img [{:x-min 0 :x-max 100 :y-min 0 :y-max 100}]) + (draw-bounding-box! [{:x-min 190 :x-max 850 :y-min 50 :y-max 450} + {:x-min 200 :x-max 350 :y-min 440 :y-max 530}] {:stroke 2 :names [\"pug\" \"cookie\"] :transparency 0.8 diff --git a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj index 38ab11c86012..23b88d07e896 100644 --- a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj +++ b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj @@ -20,7 +20,8 @@ [org.apache.clojure-mxnet.ndarray :as ndarray] [clojure.java.io :as io] [clojure.test :refer :all]) - (:import (javax.imageio ImageIO))) + (:import (javax.imageio ImageIO) + (java.io File))) (def tmp-dir (System/getProperty "java.io.tmpdir")) (def image-path (.getAbsolutePath (io/file tmp-dir "Pug-Cookie.jpg"))) @@ -76,4 +77,15 @@ (let [img-arr (image/read-image image-path) resized-arr (image/resize-image img-arr 224 224) new-img (image/to-image resized-arr)] - (is (= true (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))) + (is (ImageIO/write new-img "png" (io/file tmp-dir "out.png"))))) + +(deftest test-draw-bounding-box! + (let [orig-img (ImageIO/read (new File image-path)) + new-img (-> orig-img + (image/draw-bounding-box! [{:x-min 190 :x-max 850 :y-min 50 :y-max 450} + {:x-min 200 :x-max 350 :y-min 440 :y-max 530}] + {:stroke 2 + :names ["pug" "cookie"] + :transparency 0.8 + :font-size-mult 2.0}))] + (is (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))