development for python2

CNNclassifier.py

@@ -0,0 +1,82 @@
+import numpy as np
+
+np.random.seed(0)
+
+from keras.models import Sequential
+from keras.layers import Conv2D, MaxPooling2D, BatchNormalization
+from keras.layers import Activation, Dropout, Flatten, Dense
+from keras.preprocessing.image import ImageDataGenerator
+from keras.models import model_from_json
+
+import h5py
+
+
+def training():
+    train_data_dir = './data/train'
+    validation_data_dir = './data/test'
+
+    model = Sequential()
+    model.add(Conv2D(32, (3, 3), input_shape=(512, 512, 3)))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
+    model.add(Conv2D(32, (3, 3)))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
+    model.add(Conv2D(64, (3, 3)))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
+
+    # the model so far outputs 3D feature maps (height, width, features)
+    model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors
+    model.add(Dense(64))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    # model.add(Dropout(0.5))
+    model.add(Dense(1))
+    model.add(Activation('sigmoid'))
+
+    model.compile(loss='binary_crossentropy',
+                  optimizer='adam',
+                  metrics=['accuracy'])
+
+    batch_size = 16
+    # this is the augmentation configuration we will use for training
+    train_datagen = ImageDataGenerator(
+            rescale=1./255)
+    # this is the augmentation configuration we will use for testing:
+    # only rescaling
+    test_datagen = ImageDataGenerator(rescale=1./255)
+    # this is a generator that will read pictures found in
+    # subfolers of 'data/train', and indefinitely generate
+    # batches of augmented image data
+    train_generator = train_datagen.flow_from_directory(
+            train_data_dir,  # this is the target directory
+            target_size=(512, 512),  # all images will be resized to 512x512
+            batch_size=batch_size,
+            class_mode='binary',   # since we use binary_crossentropy loss, we need binary labels
+            shuffle=False)
+    # this is a similar generator, for validation data
+    validation_generator = test_datagen.flow_from_directory(
+            validation_data_dir,
+            target_size=(512, 512),
+            batch_size=batch_size,
+            class_mode='binary',
+            shuffle=False)
+
+    model.fit_generator(
+            train_generator,
+            steps_per_epoch=320 // batch_size,
+            epochs=10,
+            validation_data=validation_generator,
+            validation_steps=80 // batch_size)
+
+    # save model to json
+    model_json = model.to_json()
+    with open("./models/model.json", "w") as json_file:
+        json_file.write(model_json)
+
+    model.save_weights('./models/first_try.h5')  # always save your weights after training or during training
+    print("Saved model %s" % './models/first_try.h5')

YE358311_Fender_apron/README.md

@@ -0,0 +1 @@
+Dataset folder where subfolders of data to be pasted

data/README.md

@@ -0,0 +1 @@
+data folder for training and testing subfolders

image_preprocess.py

@@ -0,0 +1,121 @@
+# 1. Import the necessary packages
+import numpy as np
+import cv2
+from PIL import Image
+import webcolors
+import time, os, glob
+import sys
+import shutil
+import random
+
+def preprocess_img(filepath):
+    img = cv2.imread(filepath, -1)
+    rgb_planes = cv2.split(img)
+    result_planes = []
+    result_norm_planes = []
+    for plane in rgb_planes:
+        dilated_img = cv2.dilate(plane, np.ones((31,31), np.uint8))
+        bg_img = cv2.medianBlur(dilated_img, 15)
+        diff_img = 255 - cv2.absdiff(plane, bg_img)
+        norm_img = cv2.normalize(diff_img, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1)
+        result_planes.append(diff_img)
+        result_norm_planes.append(norm_img)
+
+    result = cv2.merge(result_planes)
+    result_norm = cv2.merge(result_norm_planes)
+    width = 512
+    height = 512
+    dim = (width, height)
+    resized = cv2.resize(result_norm, dim, interpolation = cv2.INTER_AREA)
+    gray = cv2.cvtColor(resized, cv2.COLOR_BGR2GRAY)
+
+    return gray
+
+def crack_detection_preprocess_image():
+    # define folders for input data and output data
+    folders = ['./YE358311_Fender_apron', './data']
+    # DATA_FOLDER = 'YE358311_Fender_apron/defect1'
+    DATA_FOLDER_DEFECT_HEALTHY = [folders[0] + '/YE358311_defects/YE358311_Crack_and_Wrinkle_defect', folders[0] + '/YE358311_Healthy']
+    DEST_FOLDER = [folders[1] + '/train/defect', folders[1] + '/train/normal']
+    # DATA_FOLDER = '/Users/akshyasingh/Documents/coding_practice_python/Defect-Detection-Classifier/data/normal1'
+    # DEST_FOLDER = '/Users/akshyasingh/Documents/coding_practice_python/Defect-Detection-Classifier/data/normal1_metal'
+
+    if folders[0] is None or folders[1] is None:
+      print "Bad parameters. Please specify input file path and output file path"
+      exit()
+
+    current_dir =  os.path.abspath(os.path.dirname(__file__))
+    path = os.path.join(current_dir, DATA_FOLDER_DEFECT_HEALTHY[0])
+    files = os.listdir(path)
+    files_txt = [i for i in files if i.endswith('.jpg')]
+    no_defect_files = len(files_txt)
+    # no_defect_files = 11
+
+    current = 0
+    for DATA_FOLDER in DATA_FOLDER_DEFECT_HEALTHY:
+        print "\n\n\nEntering data folder ", DATA_FOLDER[2:]
+        count = 1
+        current_dir =  os.path.abspath(os.path.dirname(__file__))
+        path = os.path.join(current_dir, DATA_FOLDER)
+        files = os.listdir(path)
+        files_txt = [i for i in files if i.endswith('.jpg')]
+        for filename in files_txt:
+            if count > no_defect_files:
+                print "Successfully pre-processsed %d files" % (2*(count-1))
+                break
+
+            current_dir =  os.path.abspath(os.path.dirname(__file__))
+            filepath = os.path.join(current_dir, DATA_FOLDER[2:], filename)
+            print "\n\nReading file : ", filename
+
+            # call image preprocess
+            gray = preprocess_img(filepath)
+
+            if not os.path.exists(os.path.join(current_dir, DEST_FOLDER[current][2:])):
+                os.makedirs(os.path.join(current_dir, DEST_FOLDER[current][2:]))
+            savepath = os.path.join(current_dir, DEST_FOLDER[current][2:], str(count) + '.jpg')
+            print "File saving in ", savepath
+            cv2.imwrite(savepath, gray)
+            count += 1
+        current  += 1
+
+    DEST_FOLDER = [folders[1] + '/test/defect', folders[1] + '/test/normal']
+    if not os.path.exists(os.path.join(current_dir, DEST_FOLDER[0][2:])):
+        os.makedirs(os.path.join(current_dir, DEST_FOLDER[0][2:]))
+    if not os.path.exists(os.path.join(current_dir, DEST_FOLDER[1][2:])):
+        os.makedirs(os.path.join(current_dir, DEST_FOLDER[1][2:]))
+    if not os.path.exists(os.path.join(current_dir, 'uploads')):
+        os.makedirs(os.path.join(current_dir, 'uploads'))
+
+    return count-1
+
+
+def move_files_test_folder(no_files=3):
+    folders = ['train/normal', 'train/defect']
+    folders_dest = ['test/normal', 'test/defect']
+    current_dir =  os.path.abspath(os.path.dirname(__file__))
+
+    name_files_to_be_moved = []
+    for i in range(2):
+        print '\n\nFor i = ', i
+        path = os.path.join(current_dir, 'data', folders[i])
+        if i == 0:
+            to_be_moved = random.sample(glob.glob(path + '/*.jpg'), no_files)
+            # print(to_be_moved)
+        else :
+            for file in to_be_moved:
+                file = file.replace('normal', 'defect')
+                name_files_to_be_moved.append(file)
+            to_be_moved = []
+            to_be_moved = name_files_to_be_moved
+            # print(to_be_moved)
+
+        for f in enumerate(to_be_moved, 1):
+            new_f = f[1].replace('train', 'test')
+            dest = os.path.join(current_dir, 'data', folders_dest[i], new_f)
+            # if not os.path.exists(dest):
+                # os.makedirs(dest)
+            shutil.copy(f[1], dest)
+            # print('\nsource file moved to : ', dest)
+            os.remove(f[1])
+            # print('source file deleted from  : ', f[1])

main_predict.py

@@ -0,0 +1,137 @@
+from flask import Flask, render_template, request
+from werkzeug import secure_filename
+from PIL import Image
+from keras.models import Sequential
+from keras.layers import Conv2D, MaxPooling2D, BatchNormalization
+from keras.layers import Activation, Dropout, Flatten, Dense
+from keras.preprocessing.image import ImageDataGenerator
+from keras.models import load_model, model_from_json
+
+import tensorflow as tf
+import numpy as np
+import cv2
+import webcolors
+import time, os
+import sys, h5py
+import shutil
+import random
+
+np.random.seed(0)
+
+from image_preprocess import crack_detection_preprocess_image
+from image_preprocess import move_files_test_folder
+from image_preprocess import preprocess_img
+from CNNclassifier import training
+
+os.environ['KMP_DUPLICATE_LIB_OK']='True'
+
+app = Flask(__name__)
+
+# load json and create model
+json_file = open('./models/model_92.json', 'r')
+loaded_model_json = json_file.read()
+json_file.close()
+
+# load model and weights
+loaded_model = model_from_json(loaded_model_json)
+# loaded_model = load_model("./models/first_try.h5")
+loaded_model.load_weights("./models/first_try_92.h5")
+# print(loaded_model.summary())
+print "Loaded model from disk"
+
+graph = tf.get_default_graph()
+
+# root
+@app.route("/")
+def index():
+    """
+    this is a root dir of my server
+    :return: str
+    """
+    return '''This is testing API :
+    To start predicting, go to http://127.0.0.1:5000/crack_detection_test'''
+
+@app.route('/crack_detection_test', methods = ['GET','POST'])
+def crack_detection_test():
+    class_pred = ''
+    filenames = ''
+    filename = ''
+    # test for input file
+    if request.method =='POST':
+        file = request.files['file[]']
+        # print(file)
+        if file:
+            current_dir =  os.path.abspath(os.path.dirname(__file__))
+            filename = secure_filename(file.filename)
+            if not os.path.exists(os.path.join(current_dir, 'uploads')):
+                os.makedirs(os.path.join(current_dir, 'uploads'))
+            filepath = os.path.join(current_dir, 'uploads', filename)
+            file.save(filepath)
+
+        # print(filepath)
+        img = cv2.imread(filepath)
+        gray = preprocess_img(filepath)
+
+        # new_im = Image.fromarray(gray)
+        # new_im.show()
+
+        # save the test image in test_dir
+        if not os.path.exists(os.path.join(current_dir, 'uploads', 'test', 'images')):
+            os.makedirs(os.path.join(current_dir, 'uploads', 'test', 'images'))
+        test_dir = os.path.join(current_dir, 'uploads', 'test')
+        # save the test file to test directory
+        savepath = os.path.join(test_dir, 'images', filename)
+        # print(filename)
+        # print(savepath)
+        cv2.imwrite(savepath, gray)
+
+        try :
+            batch_size = 16
+            test_datagen = ImageDataGenerator(rescale=1./255)
+            test_generator = test_datagen.flow_from_directory(
+                                test_dir,
+                                target_size=(512, 512),
+                                batch_size=batch_size,
+                                class_mode='binary',
+                                shuffle=False)
+
+            #resetting generator
+            test_generator.reset()
+
+            with graph.as_default():
+                # make predictions
+                pred=loaded_model.predict_generator(test_generator, steps=len(test_generator), verbose=1)
+
+                # Get classes by np.round
+                cl = np.round(pred)
+                # Get filenames (set shuffle=false in generator is important)
+                filenames=test_generator.filenames
+
+                if cl[:,0][0] == np.float32(1.0):
+                    class_pred = 'Healthy'
+                else :
+                    class_pred = 'Defective'
+                # 0 is defective, 1 is healthy
+                print "\nfile : ", filenames ,"\nprediction : ", pred[:,0], "\nclass : ", cl[:,0][0], '\npredicted class : ', class_pred
+
+        except Exception as e:
+            print e
+            print "Please try again.... something has gone wrong"
+
+        finally :
+            # remove all uploaded files
+            os.remove(savepath)
+            os.remove(filepath)
+
+        # # evaluate loaded model on test data
+        # loaded_model.compile(loss='binary_crossentropy',
+        #           optimizer='adam',
+        #           metrics=['accuracy'])
+
+        # predict = loaded_model.predict(gray)
+        # print(predict)
+
+    return render_template('file_upload.html', string_variable= class_pred, image_name= filename)
+
+if __name__ == '__main__':
+   app.run(debug=True)