diff --git a/cpp-package/example/inference/unit_test_imagenet_inference.sh b/cpp-package/example/inference/unit_test_imagenet_inference.sh
index cf3e7f061c8b..fb06fa1a06b8 100755
--- a/cpp-package/example/inference/unit_test_imagenet_inference.sh
+++ b/cpp-package/example/inference/unit_test_imagenet_inference.sh
@@ -32,9 +32,6 @@ if [ "$(uname)" == "Darwin" ]; then
 
     echo ">>> INFO: FP32 dummy data"
     DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:../../../lib ./imagenet_inference --symbol_file "./model/Inception-BN-symbol.json" --batch_size 1 --num_inference_batches 500 --benchmark
-
-    echo ">>> INFO: INT8 dummy data"
-    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../lib ./imagenet_inference --symbol_file "./model/resnet50_v1_int8-symbol.json" --batch_size 1 --num_inference_batches 500 --benchmark
 else
 	echo ">>> INFO: FP32 real data"
     LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../lib ./imagenet_inference --symbol_file "./model/Inception-BN-symbol.json" --params_file "./model/Inception-BN-0126.params" --dataset "./data/val_256_q90.rec" --rgb_mean "123.68 116.779 103.939" --batch_size 1 --num_skipped_batches 50 --num_inference_batches 500
@@ -42,6 +39,11 @@ else
     echo ">>> INFO: FP32 dummy data"
     LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../lib ./imagenet_inference --symbol_file "./model/Inception-BN-symbol.json" --batch_size 1 --num_inference_batches 500 --benchmark
 
-    echo ">>> INFO: INT8 dummy data"
-    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../lib ./imagenet_inference --symbol_file "./model/resnet50_v1_int8-symbol.json" --batch_size 1 --num_inference_batches 500 --benchmark
+    lib_name=$(ls -a ../../../lib | grep -oE 'mkldnn' | tail -1)
+    if [[ -n ${lib_name} ]] && [[ 'mkldnn' =~ ${lib_name} ]]; then
+        echo ">>> INFO: INT8 dummy data"
+        LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../lib ./imagenet_inference --symbol_file "./model/resnet50_v1_int8-symbol.json" --batch_size 1 --num_inference_batches 500 --benchmark
+    else
+        echo "Skipped INT8 test because mkldnn was not found which is required for running inference with quantized models."
+    fi
 fi