2. classify_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classify/annotate data (ie draw bounding box)!\n",
    "\n",
    "__After definition of some common functions (for ex sampling database), We can use two ways to classify image, One is semi automatic and another is full automatic! Select your required option while running code :)__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Define variables__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_PATH = \"dataset-raw\"\n",
    "DATASET_SAMPLING = \"dataset-sampling\"\n",
    "HEIGHT_CV2_IMAGE_RESIZE = 720"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Some Important Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Choice from user_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_choice(n):\n",
    "    # Get choice from user\n",
    "    try:\n",
    "        val = int(input(\"Choose an option [0-%d]: \"%(n-1, )))\n",
    "        if val < 0 or val > n:\n",
    "            raise ValueError\n",
    "        return val\n",
    "    except ValueError:\n",
    "        print(\"Invalid option, Choose again \")\n",
    "        return get_choice(n)\n",
    "    \n",
    "def choose(options):\n",
    "    for i in range(len(options)):\n",
    "        print(\"[%d] %s\"%(i, options[i]))\n",
    "    index = get_choice(len(options))\n",
    "    return index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Get random image from our dataset_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, random\n",
    "def datasetImage():\n",
    "    root = DATASET_PATH\n",
    "    folders = random.sample(os.listdir(root), 1)\n",
    "    for folder in folders:\n",
    "        b = os.path.join(root, folder)\n",
    "        f = random.choice(os.listdir(b))\n",
    "        ff = os.path.join(b, f)\n",
    "        print(ff)\n",
    "        return ff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Skip execution of cell_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SkipExecution(Exception):\n",
    "    def _render_traceback_(self):\n",
    "        print(\"this cell has been skipped\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Image transformations and Common operations (opencv and numpy)_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "# get grayscale image\n",
    "def get_grayscale(image):\n",
    "    return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "# noise removal\n",
    "def remove_noise(image):\n",
    "    return cv2.medianBlur(image,5)\n",
    " \n",
    "#thresholding\n",
    "def thresholding(image):\n",
    "    return cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]\n",
    "\n",
    "#dilation\n",
    "def dilate(image):\n",
    "    kernel = np.ones((5,5),np.uint8)\n",
    "    return cv2.dilate(image, kernel, iterations = 1)\n",
    "    \n",
    "#erosion\n",
    "def erode(image):\n",
    "    kernel = np.ones((5,5),np.uint8)\n",
    "    return cv2.erode(image, kernel, iterations = 1)\n",
    "\n",
    "#opening - erosion followed by dilation\n",
    "def opening(image):\n",
    "    kernel = np.ones((5,5),np.uint8)\n",
    "    return cv2.morphologyEx(image, cv2.MORPH_OPEN, kernel)\n",
    "\n",
    "#canny edge detection\n",
    "def canny(image):\n",
    "    return cv2.Canny(image, 100, 200)\n",
    "\n",
    "#skew correction\n",
    "def deskew(image):\n",
    "    coords = np.column_stack(np.where(image > 0))\n",
    "    angle = cv2.minAreaRect(coords)[-1]\n",
    "    if angle < -45:\n",
    "        angle = -(90 + angle)\n",
    "    else:\n",
    "        angle = -angle\n",
    "    (h, w) = image.shape[:2]\n",
    "    center = (w // 2, h // 2)\n",
    "    M = cv2.getRotationMatrix2D(center, angle, 1.0)\n",
    "    rotated = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)\n",
    "    return rotated\n",
    "\n",
    "#template matching\n",
    "def match_template(image, template):\n",
    "    return cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED)\n",
    "\n",
    "def cv_imread(file_path):\n",
    "    # Load image https://stackoverflow.com/a/54491104/4555317\n",
    "    cv_img = cv2.imdecode(np.fromfile(file_path, dtype=np.uint8), 1)\n",
    "    return cv_img\n",
    "\n",
    "def ResizeWithAspectRatio(image, width=None, height=None, inter=cv2.INTER_AREA):\n",
    "    # stackoverflow copied resize cv2 image\n",
    "    dim = None\n",
    "    (h, w) = image.shape[:2]\n",
    "\n",
    "    if width is None and height is None:\n",
    "        return image\n",
    "    if width is None:\n",
    "        r = height / float(h)\n",
    "        dim = (int(w * r), height)\n",
    "    else:\n",
    "        r = width / float(w)\n",
    "        dim = (width, int(h * r))\n",
    "\n",
    "    return cv2.resize(image, dim, interpolation=inter)\n",
    "\n",
    "def quickShow(wt, img):\n",
    "    resize = ResizeWithAspectRatio(img, height=HEIGHT_CV2_IMAGE_RESIZE)\n",
    "    cv2.imshow(wt, resize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Input method to classify images_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0] Semi Automatic (Click on white areas of speech bubble)\n",
      "[1] Automatic (Using tensorflow text detection and inpainting)\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Choose an option [0-1]:  20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Invalid option, Choose again \n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Choose an option [0-1]:  0\n"
     ]
    }
   ],
   "source": [
    "classify_method = choose([\"Semi Automatic (Click on white areas of speech bubble)\", \"Automatic (Using tensorflow text detection and inpainting)\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manual (Semi Automatic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Semi automatic image detection, here you just click on each bubbles in manga image and it automatically tries to detect valid rectangle box around it, note that this might not work all the time and the specific instructions are below!_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Store output data here_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(classify_method != 0): raise SkipExecution\n",
    "SEGMENTS_RECTANGLE = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Some more functions! (to calculate max rectangle and apply firedraw algorithm)_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(classify_method != 0): raise SkipExecution\n",
    "def get_biggest_cont(img, out):\n",
    "    edges = cv2.Canny(img,100,200)\n",
    "    contours, hierarchy = cv2.findContours(edges,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)\n",
    "    contour = max(contours, key = len)\n",
    "    contourImg = cv2.drawContours(out, [contour], -1, 255, 3) \n",
    "    return contourImg\n",
    "\n",
    "def findMaxRect(mask, drawimg):\n",
    "    contours, hierarchy = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)\n",
    "    max_x, max_y, min_x, min_y = -1, -1, 10000, 10000\n",
    "    for c in contours:\n",
    "        x,y,w,h = cv2.boundingRect(c)\n",
    "        if(x == 0 or y == 0): continue\n",
    "        max_x = max(x+w, max_x)\n",
    "        max_y = max(y+h, max_y)\n",
    "        min_x = min(x, min_x)\n",
    "        min_y = min(y, min_y)\n",
    "\n",
    "    mask = cv2.rectangle(drawimg, (min_x, min_y), (max_x, max_y), 127, 5)\n",
    "    # return (min_x, min_y, max_x - min_x, max_y - min_y)\n",
    "    return (min_x, min_y, max_x, max_y)\n",
    "\n",
    "def firedraw(img, pixel):\n",
    "    x, y = pixel\n",
    "    (h, w) = img.shape[:2]\n",
    "    mask = np.zeros((h+2,w+2),np.uint8)\n",
    "    floodflags = 4\n",
    "    floodflags |= cv2.FLOODFILL_MASK_ONLY\n",
    "    floodflags |= (255 << 8)\n",
    "    num,im,mask,rect = cv2.floodFill(img, mask, (x, y), 127, (100,)*3, (100,)*3, floodflags)\n",
    "    return mask\n",
    "\n",
    "def reconstructFromRect():\n",
    "    global pimg, SEGMENTS_RECTANGLE\n",
    "    img = pimg.copy()\n",
    "    for r in SEGMENTS_RECTANGLE:\n",
    "        img = cv2.rectangle(img, (r[0], r[1]), (r[2], r[3]), 127, 5)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Define click events_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(classify_method != 0): raise SkipExecution\n",
    "def click_event(event, x, y, flags, params):\n",
    "    global img\n",
    "    if event == cv2.EVENT_LBUTTONDOWN:\n",
    "        (h, w) = img.shape[:2]\n",
    "        scale = h / HEIGHT_CV2_IMAGE_RESIZE\n",
    "        x = int(x * scale)\n",
    "        y = int(y * scale)\n",
    "        \n",
    "        dilated_img = erode(img)\n",
    "        mask = firedraw(dilated_img, (x, y))\n",
    "        boundary = np.zeros((h+2,w+2),np.uint8)\n",
    "        contour = get_biggest_cont(mask,boundary)\n",
    "\n",
    "        rect = findMaxRect(mask, img)\n",
    "        quickShow('image', img)\n",
    "        SEGMENTS_RECTANGLE.append(rect)\n",
    "    elif event == cv2.EVENT_RBUTTONDOWN:\n",
    "        SEGMENTS_RECTANGLE.pop()\n",
    "        img = reconstructFromRect()\n",
    "        quickShow('image', img)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Save as Pascal VOC XML format_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(classify_method != 0): raise SkipExecution\n",
    "from pascal_voc_writer import Writer\n",
    "import os, shutil\n",
    "import hashlib\n",
    "\n",
    "def save_annotations_data(img_path):\n",
    "    global SEGMENTS_RECTANGLE, img\n",
    "    rootAnnotation = os.path.join(DATASET_SAMPLING, \"annotations\")\n",
    "    rootImgdir = os.path.join(DATASET_SAMPLING, \"images\")\n",
    "    os.makedirs(rootAnnotation, exist_ok=True)\n",
    "    os.makedirs(rootImgdir, exist_ok=True)\n",
    "\n",
    "    hsh = hashlib.md5(img_path.encode('utf-8')).hexdigest()\n",
    "    fn, fe = os.path.splitext(img_path)\n",
    "    \n",
    "    dest_fpath = os.path.join(rootImgdir, hsh+fe)\n",
    "    shutil.copyfile(img_path, dest_fpath)\n",
    "    \n",
    "    (h, w) = img.shape[:2]\n",
    "    writer = Writer(dest_fpath, w, h)\n",
    "    for box in SEGMENTS_RECTANGLE:\n",
    "        writer.addObject('bubble', box[0], box[1], box[2], box[3])\n",
    "    writer.save(os.path.join(rootAnnotation, hsh+\".xml\"))\n",
    "    \n",
    "    SEGMENTS_RECTANGLE = []\n",
    "    print(\"saved:\", hsh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Motor code_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(classify_method != 0): raise SkipExecution\n",
    "def execute_semi_automatic():\n",
    "    global img, pimg\n",
    "    IMAGE_PATH = datasetImage()\n",
    "    img = cv_imread(IMAGE_PATH)\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "    thres, img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n",
    "    pimg = img.copy()\n",
    "    quickShow('image', img)\n",
    "    \n",
    "    cv2.setMouseCallback('image', click_event)\n",
    "    cv2.waitKey(0)\n",
    "    cv2.destroyAllWindows()\n",
    "    save_annotations_data(IMAGE_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_classify as much image you want :o_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Enter number of samples you want to take:  20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset-raw\\Chaos Alchemist\\chapter-197_2.jpg\n",
      "saved: 2d7da550f56a8ec8344e5ab9141d8bd0\n",
      "dataset-raw\\The Great Mage Returns After 4000 Years\\chapter-111_34.jpg\n",
      "saved: 9421aad51318d1f9081daa82dc0923b3\n",
      "dataset-raw\\Almighty Master\\chapter-219_40.jpg\n",
      "saved: 851d5d8a6dbb0fa6c66aac2e9a517db2\n",
      "dataset-raw\\Shen Yi Di Nu\\chapter-463.5_5.jpg\n",
      "saved: dc91be2e7707583b701517c51ea7ac74\n",
      "dataset-raw\\Kingdom Of The Queen\\chapter-123_16.jpg\n",
      "saved: 607b1b76ded162efcc07d91817c77ac1\n",
      "dataset-raw\\The Child Of Light\\chapter-49.2_5.jpg\n",
      "saved: 023e31b2d758c4ce3a20cac02498c2ce\n",
      "dataset-raw\\I Am A Great God\\chapter-232_4.jpg\n",
      "saved: 3e231c1caa722cf879daf7f1c164834f\n",
      "dataset-raw\\The Death Mage Who Doesn't Want A Fourth Time\\chapter-28_29.jpg\n",
      "saved: 70a4b87a94e205bea6c04e332f7bd64c\n",
      "dataset-raw\\I Was Summoned By The Demon Lord, But I Can't Understand Her Language\\chapter-27_14.jpg\n",
      "saved: 305491e122f02a6fd257fe3a490ee438\n",
      "dataset-raw\\Wake Up, Warrior\\chapter-47_13.jpg\n",
      "saved: f2184529ce8478f8578bd0921af89f52\n",
      "dataset-raw\\Demon LordΓÇÖS 5500 Shadows\\chapter-44_12.jpg\n",
      "saved: 3bbaea93e0952b8eed6e102f0a6f01e9\n",
      "dataset-raw\\My Wife Is Back\\chapter-67_15.jpg\n",
      "saved: 2a17b52cf5f73e8effee223515c1a6f7\n",
      "dataset-raw\\Gura Zeni\\chapter-2_12.jpg\n",
      "saved: 941d33142eafaeeecb39a6aa28b28b64\n",
      "dataset-raw\\Strongest Leveling\\chapter-193_6.jpg\n",
      "saved: e50194c2ddf6e74a8adf2d55fb1e27c7\n",
      "dataset-raw\\Save & Load No Dekiru Yadoya-San\\chapter-20_8.jpg\n",
      "saved: f39cc334cd5200ce3b87ea69852d8161\n",
      "dataset-raw\\Demon King, Your Big Brother Wants You Home For Dinner\\chapter-30_2.jpg\n",
      "saved: 6d38c21a30bd8cb3aa7b8c89ef800ab4\n",
      "dataset-raw\\Immersed In Love\\chapter-214_13.jpg\n",
      "saved: 59ebcb444b5303b1eee996b34c9ad259\n",
      "dataset-raw\\The Golden Haired Elementalist\\chapter-79_10.jpg\n",
      "saved: 8d309266c9e575ceda6539703519a4f3\n",
      "dataset-raw\\Isekai Shikkaku\\chapter-16_8.jpg\n",
      "saved: 2fca96e06589a9edbd469a1d3c9e77fe\n",
      "dataset-raw\\Remake Our Life!\\chapter-31_9.jpg\n",
      "saved: 9e0e4515252ed34e6630ecad3c345c68\n"
     ]
    }
   ],
   "source": [
    "if(classify_method != 0): raise SkipExecution\n",
    "NUM_SAMPLES = int(input(\"Enter number of samples you want to take: \"))\n",
    "for i in range(NUM_SAMPLES):\n",
    "    execute_semi_automatic()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Automatic (Full Automatic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Using tensorflow to detect text, inpaint it and use coordinates to effectively guess all the chatboxes, please note this might not be that accurate so you have been warned :) (plus its also slow)_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this cell has been skipped\n"
     ]
    }
   ],
   "source": [
    "if(classify_method != 1): raise SkipExecution\n",
    "print(\"Under development\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What after this?\n",
    "After classifying a good number of samples, you need to convert them to tf_record files (for training etc), thus using the pascal xml and image files you can generate tf_record! for further steps please check next notebook :)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}