models/model_eval/embedding_classifier_eval.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<td>\n",
        "   <a target=\"_blank\" href=\"https://www.clarifai.com/\" ><img src=\"https://upload.wikimedia.org/wikipedia/commons/b/bc/Clarifai_Logo_FC_Web.png\" width=256/></a>\n",
        "</td>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Model Evaluation For Embedding Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Getting Started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "7hi1mzpc3MG9",
        "outputId": "2e8d41b8-a712-4fe2-dd2c-ff6f8578a3e3"
      },
      "outputs": [],
      "source": [
        "!pip install clarifai"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "sO5-mVM03o0t"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"CLARIFAI_PAT\"] = \"PAT\" # replace with your own PAT key here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W0bRmofA3xoD",
        "outputId": "9c884f05-66a7-4a84-8251-82679a4d44ba"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/Clarifai/examples.git\n",
        "%cd /content/examples/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### For Local"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "git clone https://github.com/Clarifai/examples.git\n",
        "cd examples"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Setting Up App"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "2PSQCcxY3ztI",
        "outputId": "b3f8ada0-8677-46b0-9098-3799f9ea5d58"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-01 13:35:57 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.user:                                                      <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/user.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">user.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/user.py#127\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">127</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         App created                                                                <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">           </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">           </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                          <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">           </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"32233180a5cf39c46502c029b1cdaa2e\"</span>                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">           </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                    <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">           </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-01 13:35:57\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.user:                                                      \u001b]8;id=409120;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/user.py\u001b\\\u001b[2muser.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=863373;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/user.py#127\u001b\\\u001b[2m127\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         App created                                                                \u001b[2m           \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                              \u001b[2m           \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                          \u001b[2m           \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"32233180a5cf39c46502c029b1cdaa2e\"\u001b[0m                                 \u001b[2m           \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                    \u001b[2m           \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.user:\n",
            "App created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"32233180a5cf39c46502c029b1cdaa2e\"\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from clarifai.client.user import User\n",
        "#replace your \"user_id\"\n",
        "client = User(user_id=\"user_id\")\n",
        "app = client.app(app_id=\"app_id\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Train Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 414
        },
        "id": "wmRYMf2e36zN",
        "outputId": "fe4f996e-d38e-4a1d-d5b4-573a2b8e8cfc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:41:21 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"7af05f836e4230bc9d91627ab0139ed6\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:41:21\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=270599;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=352509;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"7af05f836e4230bc9d91627ab0139ed6\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"7af05f836e4230bc9d91627ab0139ed6\"\n",
            "\n",
            "Uploading inputs:   0%|          | 0/2 [00:00<?, ?it/s]"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:41:29 </span><span style=\"color: #800000; text-decoration-color: #800000\">WARNING </span> clarifai.client.input:  Failed to upload <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> inputs..                       <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">input.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">947</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:41:29\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m clarifai.client.input:  Failed to upload \u001b[1;36m0\u001b[0m inputs..                       \u001b]8;id=975545;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\u001b\\\u001b[2minput.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=468212;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\u001b\\\u001b[2m947\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m                                                                                   \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:clarifai.client.input:Failed to upload 0 inputs..\n",
            " \n",
            "Uploading inputs:  50%|█████     | 1/2 [00:07<00:07,  7.54s/it]"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:41:30 </span><span style=\"color: #800000; text-decoration-color: #800000\">WARNING </span> clarifai.client.input:  Failed to upload <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> inputs..                       <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">input.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">947</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:41:30\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m clarifai.client.input:  Failed to upload \u001b[1;36m0\u001b[0m inputs..                       \u001b]8;id=944276;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\u001b\\\u001b[2minput.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=918426;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\u001b\\\u001b[2m947\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m                                                                                   \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:clarifai.client.input:Failed to upload 0 inputs..\n",
            " \n",
            "Uploading inputs: 100%|██████████| 2/2 [00:08<00:00,  4.32s/it]\n"
          ]
        }
      ],
      "source": [
        "#importing load_module_dataloader for calling the dataloader object in dataset.py in the local data folder\n",
        "from clarifai.datasets.upload.utils import load_module_dataloader\n",
        "\n",
        "\n",
        "# Construct the path to the dataset folder\n",
        "CSV_PATH = os.path.join(os.getcwd().split('/models/model_train')[0],'datasets/upload/data/imdb.csv')\n",
        "\n",
        "\n",
        "# Create a Clarifai dataset with the specified dataset_id\n",
        "dataset = app.create_dataset(dataset_id=\"text_dataset2\")\n",
        "# Upload the dataset using the provided dataloader and get the upload status\n",
        "dataset.upload_from_csv(csv_path=CSV_PATH,input_type='text',csv_type='raw', labels=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Setup Trained Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "FeChaZ6m4BSl",
        "outputId": "4cb16a6d-5f5d-460e-e39a-e82a77b3b7a1"
      },
      "outputs": [],
      "source": [
        "MODEL_ID = \"model_text_embedder\"\n",
        "\n",
        "# Use trained model from app\n",
        "model = app.model(model_id=MODEL_ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Training Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "8KmZCOJ14uUV",
        "outputId": "c4824b53-fe0a-4ee0-b21e-b93de96690a1"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:42:38 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:42:38\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=469470;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=841083;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"two\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"embedding_classifier_eval\"\n",
              "]"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='text_dataset2',eval_id='one',eval_info={\"use_kfold\":False})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "U99QJBDiNfab"
      },
      "outputs": [],
      "source": [
        "result=model.get_eval_by_id(\"one\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LRNO9amHN_7A",
        "outputId": "b7ef26bf-62f2-4110-9423-986e65505b3a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "macro_avg_roc_auc: 1.0\n",
            "macro_avg_f1_score: 1.0\n",
            "macro_avg_precision: 1.0\n",
            "macro_avg_recall: 1.0\n",
            "\n"
          ]
        }
      ],
      "source": [
        "print(result.summary)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        },
        "id": "QeF5cLHNOCR0",
        "outputId": "9651ce5c-10b5-48a8-8910-09c38b77a42c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:46:45 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"5d23735b41fc7dca501f1512a258171a\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:46:45\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=756784;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=256345;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"5d23735b41fc7dca501f1512a258171a\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"5d23735b41fc7dca501f1512a258171a\"\n",
            "\n",
            "Uploading inputs:   0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:46:54 </span><span style=\"color: #800000; text-decoration-color: #800000\">WARNING </span> clarifai.client.input:  Failed to upload <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> inputs..                       <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">input.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">947</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:46:54\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m clarifai.client.input:  Failed to upload \u001b[1;36m0\u001b[0m inputs..                       \u001b]8;id=985484;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\u001b\\\u001b[2minput.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=902408;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\u001b\\\u001b[2m947\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m                                                                                   \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:clarifai.client.input:Failed to upload 0 inputs..\n",
            " \n",
            "Uploading inputs: 100%|██████████| 1/1 [00:08<00:00,  8.45s/it]\n"
          ]
        }
      ],
      "source": [
        "#importing load_module_dataloader for calling the dataloader object in dataset.py in the local data folder\n",
        "from clarifai.datasets.upload.utils import load_module_dataloader\n",
        "\n",
        "\n",
        "# Construct the path to the dataset folder\n",
        "CSV_PATH = \"/content/examples/datasets/upload/data/test_imdb.csv\"\n",
        "\n",
        "\n",
        "# Create a Clarifai dataset with the specified dataset_id\n",
        "test_dataset = app.create_dataset(dataset_id=\"test_text_dataset\")\n",
        "# Upload the dataset using the provided dataloader and get the upload status\n",
        "test_dataset.upload_from_csv(csv_path=CSV_PATH,input_type='text',csv_type='raw', labels=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Testing Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "ld8UhF4LOWvB",
        "outputId": "0ac60e07-0857-43aa-8a46-2d1187c4185c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:47:12 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:47:12\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=896458;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=403994;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"three\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"embedding_classifier_eval\"\n",
              "]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='test_text_dataset',eval_id='two',eval_info={\"use_kfold\":False})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "pBjJxTKYOk44"
      },
      "outputs": [],
      "source": [
        "result = model.get_eval_by_id(\"two\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VsazqtQTOn14",
        "outputId": "5ea53ee9-5f88-4695-ebf1-b9310f3a21dc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "macro_avg_roc_auc: 1.0\n",
            "macro_avg_f1_score: 1.0\n",
            "macro_avg_precision: 1.0\n",
            "macro_avg_recall: 1.0\n",
            "\n"
          ]
        }
      ],
      "source": [
        "print(result.summary)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Compare Results From Train & Test Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "id": "rfz1y7c4OqAy",
        "outputId": "961627c6-4d77-424d-cd8d-1104f82df1eb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:48:24 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Initializing models<span style=\"color: #808000; text-decoration-color: #808000\">...</span>                      <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">91</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:48:24\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Initializing models\u001b[33m...\u001b[0m                      \u001b]8;id=257950;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=292732;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\u001b\\\u001b[2m91\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Initializing models...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  *                                          <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">100</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         embedding_classifier_eval/model_text_embedder/23f9a                    <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">               </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  *                                          \u001b]8;id=480824;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=262791;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\u001b\\\u001b[2m100\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         embedding_classifier_eval/model_text_embedder/23f9a                    \u001b[2m               \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:* embedding_classifier_eval/model_text_embedder/23f9a\n"
          ]
        }
      ],
      "source": [
        "from clarifai.utils.evaluation import EvalResultCompare\n",
        "\n",
        "eval_result = EvalResultCompare(models=[model],datasets= [dataset,test_dataset])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        },
        "id": "qhPeuTNpOugM",
        "outputId": "48ad8da2-f4bd-49f3-aaa0-377d24bb8711"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:48:43 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Running `detailed_summary`                 <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">123</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:48:43\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Running `detailed_summary`                 \u001b]8;id=366250;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=261964;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\u001b\\\u001b[2m123\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Running `detailed_summary`\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-05 07:48:43 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: text_dataset2              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-05 07:48:43\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=134437;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=224386;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: text_dataset2              \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: text_dataset2              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=749502;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=505880;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: text_dataset2              \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model embedding_classifier_eval/model_text_embedder/23f9a: retrieving {'binary_metrics': True} metrics of dataset: text_dataset2\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: test_text_dataset          <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=292491;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=410593;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: test_text_dataset          \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: test_text_dataset          <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=253795;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=420134;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         embedding_classifier_eval/model_text_embedder/23f9a: retrieving         \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: test_text_dataset          \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model embedding_classifier_eval/model_text_embedder/23f9a: retrieving {'binary_metrics': True} metrics of dataset: test_text_dataset\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(  Concept  Accuracy (ROC AUC)  Total Labeled  Total Predicted  True Positives  \\\n",
            "0  id-pos                 1.0             80               80              80   \n",
            "0  id-neg                 1.0            120              120             120   \n",
            "0  id-pos                 1.0             31               31              31   \n",
            "0  id-neg                 1.0             40               40              40   \n",
            "\n",
            "   False Negatives  False Positives  Recall  Precision   F1            Dataset  \n",
            "0                0                0     1.0        1.0  1.0      text_dataset2  \n",
            "0                0                0     1.0        1.0  1.0      text_dataset2  \n",
            "0                0                0     1.0        1.0  1.0  test_text_dataset  \n",
            "0                0                0     1.0        1.0  1.0  test_text_dataset  ,                Total Concept  Accuracy (ROC AUC)  Total Labeled  \\\n",
            "0      Dataset:text_dataset2                 1.0            200   \n",
            "0  Dataset:test_text_dataset                 1.0             71   \n",
            "\n",
            "   Total Predicted  True Positives  False Negatives  False Positives  Recall  \\\n",
            "0              200             200                0                0     1.0   \n",
            "0               71              71                0                0     1.0   \n",
            "\n",
            "   Precision   F1  \n",
            "0        1.0  1.0  \n",
            "0        1.0  1.0  )\n"
          ]
        }
      ],
      "source": [
        "print(eval_result.detailed_summary())"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}