notebooks/process_keypoints.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import ndarray\n",
    "\n",
    "from aipose.frame import FrameYoloV7\n",
    "from aipose.models.yolov7.domain import YoloV7PoseKeypoints\n",
    "from aipose.models.yolov7.types import YoloV7PoseJoints\n",
    "from aipose.stream import process_webcam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/user/.cache/torch/hub/WongKinYiu_yolov7_main\n",
      "\n",
      "                 from  n    params  module                                  arguments                     \n",
      "  0                -1  1         0  models.common.ReOrg                     []                            \n",
      "  1                -1  1      7040  models.common.Conv                      [12, 64, 3, 1]                \n",
      "  2                -1  1     73984  models.common.Conv                      [64, 128, 3, 2]               \n",
      "  3                -1  1      8320  models.common.Conv                      [128, 64, 1, 1]               \n",
      "  4                -2  1      8320  models.common.Conv                      [128, 64, 1, 1]               \n",
      "  5                -1  1     36992  models.common.Conv                      [64, 64, 3, 1]                \n",
      "  6                -1  1     36992  models.common.Conv                      [64, 64, 3, 1]                \n",
      "  7                -1  1     36992  models.common.Conv                      [64, 64, 3, 1]                \n",
      "  8                -1  1     36992  models.common.Conv                      [64, 64, 3, 1]                \n",
      "  9  [-1, -3, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 10                -1  1     33024  models.common.Conv                      [256, 128, 1, 1]              \n",
      " 11                -1  1    295424  models.common.Conv                      [128, 256, 3, 2]              \n",
      " 12                -1  1     33024  models.common.Conv                      [256, 128, 1, 1]              \n",
      " 13                -2  1     33024  models.common.Conv                      [256, 128, 1, 1]              \n",
      " 14                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 15                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 16                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 17                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 18  [-1, -3, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 19                -1  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 20                -1  1   1180672  models.common.Conv                      [256, 512, 3, 2]              \n",
      " 21                -1  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 22                -2  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 23                -1  1    590336  models.common.Conv                      [256, 256, 3, 1]              \n",
      " 24                -1  1    590336  models.common.Conv                      [256, 256, 3, 1]              \n",
      " 25                -1  1    590336  models.common.Conv                      [256, 256, 3, 1]              \n",
      " 26                -1  1    590336  models.common.Conv                      [256, 256, 3, 1]              \n",
      " 27  [-1, -3, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 28                -1  1    525312  models.common.Conv                      [1024, 512, 1, 1]             \n",
      " 29                -1  1   3540480  models.common.Conv                      [512, 768, 3, 2]              \n",
      " 30                -1  1    295680  models.common.Conv                      [768, 384, 1, 1]              \n",
      " 31                -2  1    295680  models.common.Conv                      [768, 384, 1, 1]              \n",
      " 32                -1  1   1327872  models.common.Conv                      [384, 384, 3, 1]              \n",
      " 33                -1  1   1327872  models.common.Conv                      [384, 384, 3, 1]              \n",
      " 34                -1  1   1327872  models.common.Conv                      [384, 384, 3, 1]              \n",
      " 35                -1  1   1327872  models.common.Conv                      [384, 384, 3, 1]              \n",
      " 36  [-1, -3, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 37                -1  1   1181184  models.common.Conv                      [1536, 768, 1, 1]             \n",
      " 38                -1  1   7079936  models.common.Conv                      [768, 1024, 3, 2]             \n",
      " 39                -1  1    525312  models.common.Conv                      [1024, 512, 1, 1]             \n",
      " 40                -2  1    525312  models.common.Conv                      [1024, 512, 1, 1]             \n",
      " 41                -1  1   2360320  models.common.Conv                      [512, 512, 3, 1]              \n",
      " 42                -1  1   2360320  models.common.Conv                      [512, 512, 3, 1]              \n",
      " 43                -1  1   2360320  models.common.Conv                      [512, 512, 3, 1]              \n",
      " 44                -1  1   2360320  models.common.Conv                      [512, 512, 3, 1]              \n",
      " 45  [-1, -3, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 46                -1  1   2099200  models.common.Conv                      [2048, 1024, 1, 1]            \n",
      " 47                -1  1   7609344  models.common.SPPCSPC                   [1024, 512, 1]                \n",
      " 48                -1  1    197376  models.common.Conv                      [512, 384, 1, 1]              \n",
      " 49                -1  1         0  torch.nn.modules.upsampling.Upsample    [None, 2, 'nearest']          \n",
      " 50                37  1    295680  models.common.Conv                      [768, 384, 1, 1]              \n",
      " 51          [-1, -2]  1         0  models.common.Concat                    [1]                           \n",
      " 52                -1  1    295680  models.common.Conv                      [768, 384, 1, 1]              \n",
      " 53                -2  1    295680  models.common.Conv                      [768, 384, 1, 1]              \n",
      " 54                -1  1    663936  models.common.Conv                      [384, 192, 3, 1]              \n",
      " 55                -1  1    332160  models.common.Conv                      [192, 192, 3, 1]              \n",
      " 56                -1  1    332160  models.common.Conv                      [192, 192, 3, 1]              \n",
      " 57                -1  1    332160  models.common.Conv                      [192, 192, 3, 1]              \n",
      " 58[-1, -2, -3, -4, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 59                -1  1    590592  models.common.Conv                      [1536, 384, 1, 1]             \n",
      " 60                -1  1     98816  models.common.Conv                      [384, 256, 1, 1]              \n",
      " 61                -1  1         0  torch.nn.modules.upsampling.Upsample    [None, 2, 'nearest']          \n",
      " 62                28  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 63          [-1, -2]  1         0  models.common.Concat                    [1]                           \n",
      " 64                -1  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 65                -2  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 66                -1  1    295168  models.common.Conv                      [256, 128, 3, 1]              \n",
      " 67                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 68                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 69                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 70[-1, -2, -3, -4, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 71                -1  1    262656  models.common.Conv                      [1024, 256, 1, 1]             \n",
      " 72                -1  1     33024  models.common.Conv                      [256, 128, 1, 1]              \n",
      " 73                -1  1         0  torch.nn.modules.upsampling.Upsample    [None, 2, 'nearest']          \n",
      " 74                19  1     33024  models.common.Conv                      [256, 128, 1, 1]              \n",
      " 75          [-1, -2]  1         0  models.common.Concat                    [1]                           \n",
      " 76                -1  1     33024  models.common.Conv                      [256, 128, 1, 1]              \n",
      " 77                -2  1     33024  models.common.Conv                      [256, 128, 1, 1]              \n",
      " 78                -1  1     73856  models.common.Conv                      [128, 64, 3, 1]               \n",
      " 79                -1  1     36992  models.common.Conv                      [64, 64, 3, 1]                \n",
      " 80                -1  1     36992  models.common.Conv                      [64, 64, 3, 1]                \n",
      " 81                -1  1     36992  models.common.Conv                      [64, 64, 3, 1]                \n",
      " 82[-1, -2, -3, -4, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 83                -1  1     65792  models.common.Conv                      [512, 128, 1, 1]              \n",
      " 84                -1  1    295424  models.common.Conv                      [128, 256, 3, 2]              \n",
      " 85          [-1, 71]  1         0  models.common.Concat                    [1]                           \n",
      " 86                -1  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 87                -2  1    131584  models.common.Conv                      [512, 256, 1, 1]              \n",
      " 88                -1  1    295168  models.common.Conv                      [256, 128, 3, 1]              \n",
      " 89                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 90                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 91                -1  1    147712  models.common.Conv                      [128, 128, 3, 1]              \n",
      " 92[-1, -2, -3, -4, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      " 93                -1  1    262656  models.common.Conv                      [1024, 256, 1, 1]             \n",
      " 94                -1  1    885504  models.common.Conv                      [256, 384, 3, 2]              \n",
      " 95          [-1, 59]  1         0  models.common.Concat                    [1]                           \n",
      " 96                -1  1    295680  models.common.Conv                      [768, 384, 1, 1]              \n",
      " 97                -2  1    295680  models.common.Conv                      [768, 384, 1, 1]              \n",
      " 98                -1  1    663936  models.common.Conv                      [384, 192, 3, 1]              \n",
      " 99                -1  1    332160  models.common.Conv                      [192, 192, 3, 1]              \n",
      "100                -1  1    332160  models.common.Conv                      [192, 192, 3, 1]              \n",
      "101                -1  1    332160  models.common.Conv                      [192, 192, 3, 1]              \n",
      "102[-1, -2, -3, -4, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      "103                -1  1    590592  models.common.Conv                      [1536, 384, 1, 1]             \n",
      "104                -1  1   1770496  models.common.Conv                      [384, 512, 3, 2]              \n",
      "105          [-1, 47]  1         0  models.common.Concat                    [1]                           \n",
      "106                -1  1    525312  models.common.Conv                      [1024, 512, 1, 1]             \n",
      "107                -2  1    525312  models.common.Conv                      [1024, 512, 1, 1]             \n",
      "108                -1  1   1180160  models.common.Conv                      [512, 256, 3, 1]              \n",
      "109                -1  1    590336  models.common.Conv                      [256, 256, 3, 1]              \n",
      "110                -1  1    590336  models.common.Conv                      [256, 256, 3, 1]              \n",
      "111                -1  1    590336  models.common.Conv                      [256, 256, 3, 1]              \n",
      "112[-1, -2, -3, -4, -5, -6]  1         0  models.common.Concat                    [1]                           \n",
      "113                -1  1   1049600  models.common.Conv                      [2048, 512, 1, 1]             \n",
      "114                83  1    295424  models.common.Conv                      [128, 256, 3, 1]              \n",
      "115                93  1   1180672  models.common.Conv                      [256, 512, 3, 1]              \n",
      "116               103  1   2655744  models.common.Conv                      [384, 768, 3, 1]              \n",
      "117               113  1   4720640  models.common.Conv                      [512, 1024, 3, 1]             \n"
     ]
    }
   ],
   "source": [
    "class CustomYoloV7PoseKeypointsFrameProcessor(FrameYoloV7):\n",
    "    def on_predict(\n",
    "        self, frame: ndarray, prediction: list[YoloV7PoseKeypoints]\n",
    "    ) -> None | ndarray:\n",
    "        for keypoint in prediction:\n",
    "            right_wrist = keypoint.get_keypoint(YoloV7PoseJoints.RIGHT_WRIST)\n",
    "            left_wrist = keypoint.get_keypoint(YoloV7PoseJoints.LEFT_WRIST)\n",
    "            nose = keypoint.get_keypoint(YoloV7PoseJoints.NOSE)\n",
    "\n",
    "            are_hands_raised = (\n",
    "                right_wrist.y < nose.y\n",
    "                and left_wrist.y < nose.y\n",
    "                and right_wrist.conf > 0.50\n",
    "                and left_wrist.conf > 0.50\n",
    "            )\n",
    "\n",
    "            if are_hands_raised:\n",
    "                self.stop = True\n",
    "\n",
    "\n",
    "custom_keypoints_frame_processor = CustomYoloV7PoseKeypointsFrameProcessor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/aipose/.venv/lib/python3.10/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3190.)\n",
      "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
     ]
    }
   ],
   "source": [
    "process_webcam(custom_keypoints_frame_processor)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.7 ('.venv': pipenv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "9e906b8bce670891410964d9b40cd49b717f5bb944c17329acc0e9f63bf95cb2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}