models/model_eval/visual_detector_eval.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<td>\n",
        "   <a target=\"_blank\" href=\"https://www.clarifai.com/\" ><img src=\"https://upload.wikimedia.org/wikipedia/commons/b/bc/Clarifai_Logo_FC_Web.png\" width=256/></a>\n",
        "</td>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<td>\n",
        "<a href=\"https://colab.research.google.com/github/Clarifai/examples/blob/main/models/model_eval/visual_detector_eval.ipynb\" target=\"_blank\"><img\n",
        "src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Colab\"></a>\n",
        "</td>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Model Evaluation For Visual Detector"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Getting Started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "CR3FPcliNLbm",
        "outputId": "ec5f29bf-0d53-44c1-8c3c-da269e789e74"
      },
      "outputs": [],
      "source": [
        "!pip install clarifai"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kVHIG3J5NR8b"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "#Replace your PAT\n",
        "os.environ['CLARIFAI_PAT'] = \"PAT\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### For Colab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kC4XIq7zNVZ7",
        "outputId": "2c67ee92-2bf6-43b5-bc39-34c8b4992b14"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/Clarifai/examples.git\n",
        "%cd /content/examples/models/model_train"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### For Local"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zV9vz729NX2T",
        "outputId": "758f0cc9-40d2-4c08-f6eb-598c6a2588bd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/examples/models/model_train\n"
          ]
        }
      ],
      "source": [
        "git clone https://github.com/Clarifai/examples.git\n",
        "cd examples"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Setting Up App"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "Cj_poXcDNfWW",
        "outputId": "1b4e370f-6081-4a20-f8dd-549e55c54a96"
      },
      "outputs": [],
      "source": [
        "from clarifai.client.user import User\n",
        "#replace your \"user_id\"\n",
        "client = User(user_id=\"user_id\")\n",
        "app = client.app(app_id=\"app_id\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Train Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O9GCLM0dN3K5"
      },
      "outputs": [],
      "source": [
        "from clarifai.datasets.upload.utils import load_module_dataloader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "-8auvm3JNswc",
        "outputId": "6461f65e-a3b8-40c5-c9a8-bef3ecc3c3ab"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/examples/datasets/upload/image_detection/voc'"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "module_path = os.path.join(os.getcwd().split('/models/model_train')[0],'datasets/upload/image_detection/voc')\n",
        "module_path"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vpsSkgXhNwZI"
      },
      "outputs": [],
      "source": [
        "voc_dataloader = load_module_dataloader(module_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "P3E6Z-TSNy4R",
        "outputId": "25a7f175-f9a4-4460-ded7-3a0293d1c983"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:57:27 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"d47901b9069887c185a8a3ba510a2174\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:57:27\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=708869;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=796749;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"d47901b9069887c185a8a3ba510a2174\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"d47901b9069887c185a8a3ba510a2174\"\n",
            "\n",
            "Uploading Dataset: 100%|██████████| 1/1 [00:09<00:00,  9.13s/it]\n"
          ]
        }
      ],
      "source": [
        "dataset = app.create_dataset(dataset_id=\"train_dataset\")\n",
        "dataset.upload_dataset(dataloader=voc_dataloader)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Setup Trained Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "57P6o_pON9S8",
        "outputId": "ebd4ee3e-b11c-4a1c-87a5-a87bd8d769d5"
      },
      "outputs": [],
      "source": [
        "MODEL_ID = \"model_detector\"\n",
        "# Use trained model from app\n",
        "model = app.model(model_id=MODEL_ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Training Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "iP0neoJLRx4e",
        "outputId": "4bd11125-03d7-46e8-8364-759070500198"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-02 10:23:04 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-02 10:23:04\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=172081;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=818183;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"two\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"visual_detector_eval2\"\n",
              "]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='train_dataset',eval_id='one')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BsI58HYLR4lw"
      },
      "outputs": [],
      "source": [
        "result=model.get_eval_by_id(eval_id=\"one\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P2OFM3YhR75G",
        "outputId": "f4f2a81f-a856-41c3-89f7-168ae2ca3387"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "mean_avg_precision_iou_50: 1.0\n",
              "mean_avg_precision_iou_range: 0.9453125"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "SDifzjruw910",
        "outputId": "fefefde0-8462-43da-985f-3d7c23abce1c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 11:11:07 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"ac06c1c1c5a7a94dfee083e77fd17cfc\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 11:11:07\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=600312;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=144083;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"ac06c1c1c5a7a94dfee083e77fd17cfc\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"ac06c1c1c5a7a94dfee083e77fd17cfc\"\n",
            "\n",
            "Uploading Dataset: 100%|██████████| 1/1 [00:03<00:00,  3.84s/it]\n"
          ]
        }
      ],
      "source": [
        "# use the correct path for test data\n",
        "module_path=\"/content/examples/datasets/upload/data/voc_test\"\n",
        "voc_dataloader = load_module_dataloader(module_path)\n",
        "test_dataset = app.create_dataset(dataset_id=\"test_dataset_1\")\n",
        "test_dataset.upload_dataset(dataloader=voc_dataloader)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Testing Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "_dRxoE0Hztbh",
        "outputId": "d50bd5da-bb83-4b13-aae0-45b931bf31bd"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 11:12:14 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 11:12:14\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=6758;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=919217;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"eight\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"visual_detector_eval2\"\n",
              "]"
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='test_dataset_1',eval_id='two')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tTgZ7MFnztWA"
      },
      "outputs": [],
      "source": [
        "result=model.get_eval_by_id(\"two\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bkpufK6-z_Qp",
        "outputId": "19558b2d-8d70-4873-cd47-2eb80a184fa1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "mean_avg_precision_iou_50: 1.0\n",
              "mean_avg_precision_iou_range: 0.9555555582046509"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Compare Results From Train & Test Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "id": "dPoktPl7zlTY",
        "outputId": "0904054e-ba4c-428b-d7b9-076b7a22db94"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 11:15:29 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Initializing models<span style=\"color: #808000; text-decoration-color: #808000\">...</span>                      <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">91</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 11:15:29\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Initializing models\u001b[33m...\u001b[0m                      \u001b]8;id=863231;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=923536;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\u001b\\\u001b[2m91\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Initializing models...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  *                                          <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">100</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_detector_eval2/model_detector/76b0d                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">               </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  *                                          \u001b]8;id=455596;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=716982;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\u001b\\\u001b[2m100\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_detector_eval2/model_detector/76b0d                             \u001b[2m               \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:* visual_detector_eval2/model_detector/76b0d\n"
          ]
        }
      ],
      "source": [
        "from clarifai.utils.evaluation import EvalResultCompare\n",
        "\n",
        "eval_result = EvalResultCompare(models=[model],datasets= [dataset,test_dataset])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "FsNIvQP70Iy1",
        "outputId": "523cee67-5aa7-4f89-a656-cd4b1836c4d1"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 11:15:42 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Running `detailed_summary`                 <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">123</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 11:15:42\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Running `detailed_summary`                 \u001b]8;id=871503;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=684420;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\u001b\\\u001b[2m123\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Running `detailed_summary`\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 11:15:42 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_detector_eval2/model_detector/76b0d: retrieving                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'metrics_by_class'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: train_dataset2           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 11:15:42\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=314557;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=144174;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_detector_eval2/model_detector/76b0d: retrieving                  \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'metrics_by_class'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: train_dataset2           \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_detector_eval2/model_detector/76b0d: retrieving                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'metrics_by_class'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: train_dataset2           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=788574;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=538749;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_detector_eval2/model_detector/76b0d: retrieving                  \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'metrics_by_class'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: train_dataset2           \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model visual_detector_eval2/model_detector/76b0d: retrieving {'metrics_by_class': True} metrics of dataset: train_dataset2\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 11:15:43 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_detector_eval2/model_detector/76b0d: retrieving                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'metrics_by_class'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: test_dataset_1           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 11:15:43\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=111095;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=664997;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_detector_eval2/model_detector/76b0d: retrieving                  \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'metrics_by_class'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: test_dataset_1           \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 11:15:43 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_detector_eval2/model_detector/76b0d: retrieving                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'metrics_by_class'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: test_dataset_1           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 11:15:43\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=610396;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=130383;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_detector_eval2/model_detector/76b0d: retrieving                  \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'metrics_by_class'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: test_dataset_1           \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model visual_detector_eval2/model_detector/76b0d: retrieving {'metrics_by_class': True} metrics of dataset: test_dataset_1\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "(     Concept  Average Precision  Total Labeled  True Positives  \\\n",
              " 0     id-cow                1.0              2               2   \n",
              " 0   id-horse                1.0              1               1   \n",
              " 0  id-bottle                1.0              2               2   \n",
              " 0    id-sofa                1.0              1               1   \n",
              " 0    id-bird                1.0              1               1   \n",
              " 0     id-cat                1.0              2               2   \n",
              " 0     id-dog                1.0              1               1   \n",
              " 0  id-person                1.0              8               8   \n",
              " 0     id-dog                1.0              1               1   \n",
              " 0  id-person                1.0              3               3   \n",
              " \n",
              "    False Positives  False Negatives  Recall  Precision        F1  \\\n",
              " 0                0                0     1.0      0.841  0.913634   \n",
              " 0                0                0     1.0      0.783  0.878295   \n",
              " 0                0                0     1.0      0.819  0.900495   \n",
              " 0                0                0     1.0      0.769  0.869418   \n",
              " 0                0                0     1.0      0.790  0.882682   \n",
              " 0                0                0     1.0      0.836  0.910675   \n",
              " 0                0                0     1.0      0.763  0.865570   \n",
              " 0                0                0     1.0      0.940  0.969072   \n",
              " 0                0                0     1.0      0.763  0.865570   \n",
              " 0                0                0     1.0      0.884  0.938429   \n",
              " \n",
              "           Dataset  \n",
              " 0  train_dataset2  \n",
              " 0  train_dataset2  \n",
              " 0  train_dataset2  \n",
              " 0  train_dataset2  \n",
              " 0  train_dataset2  \n",
              " 0  train_dataset2  \n",
              " 0  train_dataset2  \n",
              " 0  train_dataset2  \n",
              " 0  test_dataset_1  \n",
              " 0  test_dataset_1  ,\n",
              "             Total Concept  Average Precision  Total Labeled  True Positives  \\\n",
              " 0  Dataset:train_dataset2                1.0             18              18   \n",
              " 0  Dataset:test_dataset_1                1.0              4               4   \n",
              " \n",
              "    False Positives  False Negatives  Recall  Precision   F1  \n",
              " 0                0                0     1.0        1.0  1.0  \n",
              " 0                0                0     1.0        1.0  1.0  )"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "eval_result.detailed_summary()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}