DSPyAdvancedPromptEngineering_AIMakerspace.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/edgarbc/LLM_optimizer/blob/main/DSPyAdvancedPromptEngineering_AIMakerspace.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# DSPy - Advanced Prompt Engineering\n",
        "\n",
        "This example is from the AI Makerspace folks at https://www.youtube.com/watch?v=6YtdtjQD1r0.\n",
        "\n",
        "In the following notebook, we'll explore an introduction to DSPy and what it can do in just a few lines of code!\n",
        "\n",
        "To begin, we'll grab the only (top level) dependency we'll need - DSPy!"
      ],
      "metadata": {
        "id": "lNF2FTJcYP7u"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MeoJU4iE3AWt",
        "outputId": "697273c7-9329-493b-c163-9766ffd3e2c6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m220.4/220.4 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.2/302.2 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m325.5/325.5 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m520.4/520.4 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m380.1/380.1 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.8/65.8 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.4/233.4 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.6/78.6 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install -qU dspy-ai"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "DSPy can leverage OpenAI's models under the hood, and still provide an advantage - in order to do so, however, we'll need to provide an OpenAI API Key!"
      ],
      "metadata": {
        "id": "PrDi65cdY0pa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import getpass\n",
        "\n",
        "os.environ['OPENAI_API_KEY'] = getpass.getpass('OpenAI API Key: ')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8lPWT4vL8zFd",
        "outputId": "671e562c-e4b2-4206-b632-c8ada515de98"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "OpenAI API Key: ··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model\n",
        "\n",
        "Now we can setup our OpenAI language model - which we'll use through the remaining cells in the notebook."
      ],
      "metadata": {
        "id": "FW3W8ogDZJS_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy import OpenAI\n",
        "\n",
        "llm = OpenAI(model='gpt-3.5-turbo')"
      ],
      "metadata": {
        "id": "VJAy8_hw8rUu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Similar to other libraries, we can call the LLM directly with a string to get a response!"
      ],
      "metadata": {
        "id": "ONjD_6hKZPsD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llm(\"What is the square root of pi?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BlwRrI1UZOFc",
        "outputId": "9288af5f-864e-412f-b131-ca03e0becf0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['The square root of pi is approximately 1.77245385091.']"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll also set our `setting.configure` with our OpenAI model in the `lm` (Language Model) field for a default LM to use in case we don't specify which LM we'd like to use when calling our DSPy `Predictors`."
      ],
      "metadata": {
        "id": "oAO_nkG_ZVVV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import dspy\n",
        "\n",
        "dspy.settings.configure(lm=llm)"
      ],
      "metadata": {
        "id": "feM8E46m9Gna"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data\n",
        "\n",
        "We're going to be using a dataset that provides a number of example sentences, along with a rating that indicates their \"dopeness\" level."
      ],
      "metadata": {
        "id": "G5H-q4WdZhc3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"llm-wizard/dope_or_nope_v2\")"
      ],
      "metadata": {
        "id": "n__G5nrU-Epz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have a total of 99 rows of data, and will be splitting that into a `trainset` and a `valset` - for training and evaluation."
      ],
      "metadata": {
        "id": "OMgZxUByaz68"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YMBCVzwhDKJr",
        "outputId": "f16d687b-d3c4-4985-ce0f-96b5fd6ed742"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['Sentence', 'Rating', 'Fire Emojis'],\n",
              "        num_rows: 99\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Due to the nature of the dataset, we'll need to shuffle our dataset to ensure our labels are not clumped up, and our `valset` is remotely representative to our `trainset`."
      ],
      "metadata": {
        "id": "ZwyoqDkvcBxr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = dataset.shuffle(seed=42)"
      ],
      "metadata": {
        "id": "RecukH0rVkNY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll move our `Dataset` into the expected format in DSPy which is the [`Example`](https://dspy-docs.vercel.app/docs/deep-dive/data-handling/examples)!\n",
        "\n",
        "\n",
        "Our examples will have two keys:\n",
        "\n",
        "- `sentence`, our input sentence to be rated\n",
        "- `rating`, our rating label\n",
        "\n",
        "We'll specify our input as `sentence` to properly leverage the DSPy framework."
      ],
      "metadata": {
        "id": "nrr-xSbtcPBK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy import Example\n",
        "\n",
        "trainset = []\n",
        "\n",
        "for row in dataset[\"train\"].select(range(0,len(dataset[\"train\"])-10)):\n",
        "  trainset.append(Example(sentence=row[\"Sentence\"], rating=row[\"Rating\"]).with_inputs(\"sentence\"))\n",
        "\n",
        "len(trainset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "07ARkkb_CnBT",
        "outputId": "bf0272bf-4455-4bb6-ec49-b96df1beb9f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "89"
            ]
          },
          "metadata": {},
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll repeat the same process for our `valset` as well."
      ],
      "metadata": {
        "id": "wZC2yAnqc31Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "valset = []\n",
        "\n",
        "for row in dataset[\"train\"].select(range(len(trainset),len(dataset[\"train\"]))):\n",
        "  valset.append(Example(sentence=row[\"Sentence\"], rating=row[\"Rating\"]).with_inputs(\"sentence\"))\n",
        "\n",
        "len(valset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VTDPZJUZD_hM",
        "outputId": "6550158f-138e-43f5-c120-485cdb96d31c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {},
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's take a peek at an example from our `trainset` and `valset`!"
      ],
      "metadata": {
        "id": "GKKim9uRc8Q0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_example = trainset[0]\n",
        "print(f\"Sentence: {train_example.sentence}\")\n",
        "print(f\"Label: {train_example.rating}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PVgP0ctkEJC9",
        "outputId": "0633c489-fe9b-43f1-f53c-a6fa5ef03254"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentence: The results were satisfactory.\n",
            "Label: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "valset_example = valset[0]\n",
        "print(f\"Sentence: {valset_example.sentence}\")\n",
        "print(f\"Label: {valset_example.rating}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zHFDjOT9EUip",
        "outputId": "9713be33-3001-49f1-8713-9d3d179993aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentence: This is top tier.\n",
            "Label: 4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Signature\n",
        "\n",
        "The first foundational unit in DSPy is the `Signature`.\n",
        "\n",
        "In a sense, a `Signature` can be thought of as both a prompt, as well as metadata about that prompt.\n",
        "\n",
        "Going beyong just a simple `SystemMessage`, as seen in other frameworks, the `Signature` helps DSPy validate datatypes, create examples, and more.\n",
        "\n",
        "> NOTE: DSPy's [documentation](https://dspy-docs.vercel.app/docs/deep-dive/signature/understanding-signatures#what-is-a-signature) goes into more detail about what exactly a `Signature` is."
      ],
      "metadata": {
        "id": "lbjjPIJsdAYs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy import Signature, InputField, OutputField\n",
        "\n",
        "class DopeOrNopeSignature(Signature):\n",
        "  \"\"\"Rate a sentence from 0 to 4 on a dopeness scale\"\"\"\n",
        "  sentence: str = InputField()\n",
        "  rating: int = OutputField()"
      ],
      "metadata": {
        "id": "WcshNXosEaRb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Predictor\n",
        "\n",
        "Now that we have our `Signature`, we can build a `Predictor` that leverages it.\n",
        "\n",
        "A `Predictor`, in the simplest terms, is what calls the LLM using our signature. Importantly, the `Predictor` knows how to leverage our signature to call the LLM. From DSPy's documentation, one of the most interesting parts of a `Predictor` is that it can *learn* to become better at the desired task!\n",
        "\n",
        "Let's take a look at our `TypedPredictor` below to see more."
      ],
      "metadata": {
        "id": "R5BnXK3VdnSL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy.functional import TypedPredictor\n",
        "\n",
        "generate_label = TypedPredictor(DopeOrNopeSignature)"
      ],
      "metadata": {
        "id": "xxsLox0KEtwk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generate_label"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uKjZPqJCeeGs",
        "outputId": "cd592ec6-ca32-429b-e9fa-5d45fc495686"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TypedPredictor(DopeOrNopeSignature(sentence -> rating\n",
              "    instructions='Rate a sentence from 0 to 4 on a dopeness scale'\n",
              "    sentence = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Sentence:', 'desc': '${sentence}'})\n",
              "    rating = Field(annotation=int required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Rating:', 'desc': '${rating}'})\n",
              "))"
            ]
          },
          "metadata": {},
          "execution_count": 155
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "label_prediction = generate_label(sentence=valset_example.sentence)\n",
        "print(f\"Sentence: {valset_example.sentence}\")\n",
        "print(f\"Prediction: {label_prediction}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UUjXc2BIE_as",
        "outputId": "41a5e311-0648-4fa6-db07-fe8a8ee5d57b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentence: This is top tier.\n",
            "Prediction: Prediction(\n",
            "    rating=3\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can, at any time, check our LLMs outputs through the `inspect_history`."
      ],
      "metadata": {
        "id": "Fw6qrDP7gbf1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llm.inspect_history(n=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 382
        },
        "id": "ncjnowmCHGd0",
        "outputId": "36c5efb9-d746-4d98-dcd9-87fbee9d35bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "Rate a sentence from 0 to 4 on a dopeness scale\n",
            "\n",
            "---\n",
            "\n",
            "Follow the following format.\n",
            "\n",
            "Sentence: ${sentence}\n",
            "Rating: ${rating} (Respond with a single int value)\n",
            "\n",
            "---\n",
            "\n",
            "Sentence: This is top tier.\n",
            "Rating:\u001b[32m 3\u001b[0m\n",
            "\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\n\\n\\nRate a sentence from 0 to 4 on a dopeness scale\\n\\n---\\n\\nFollow the following format.\\n\\nSentence: ${sentence}\\nRating: ${rating} (Respond with a single int value)\\n\\n---\\n\\nSentence: This is top tier.\\nRating:\\x1b[32m 3\\x1b[0m\\n\\n\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 157
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Notice how, without our input - the `TypedPredictor` has included format instructions to the LLM to help ensure our returned data resembles what we desire."
      ],
      "metadata": {
        "id": "XTiQ2g3afO8k"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's look at another example of a `Predictor` - this time with Chain of Thought.\n",
        "\n",
        "In order to use this - we don't have to do anything with our `Signature`! We can leave it exactly as is - and allow the `Predictor` to adapt to it.\n",
        "\n",
        "> NOTE: We won't be using this predictor going forward - this is just to showcase the ease of using another `Predictor` with a `Signature`."
      ],
      "metadata": {
        "id": "wuuQ1GxNffdm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy.functional import TypedChainOfThought\n",
        "\n",
        "generate_label_with_chain_of_thought = TypedChainOfThought(DopeOrNopeSignature)\n",
        "\n",
        "label_prediction = generate_label_with_chain_of_thought(sentence=valset_example.sentence)"
      ],
      "metadata": {
        "id": "CcRQk4uQHImC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Sentence: {valset_example.sentence}\")\n",
        "print(f\"Reasoning: {label_prediction.reasoning}\")\n",
        "print(f\"Ground Truth Label: {valset_example.rating}\")\n",
        "print(f\"Prediction: {label_prediction.rating}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vZf2PNb8HYaT",
        "outputId": "b64d7b60-5619-4dab-e3df-f9a88d1d9ed2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentence: This is top tier.\n",
            "Reasoning: produce the rating. We first consider the impact of the phrase \"top tier,\" which implies the highest level of quality or excellence. This phrase is commonly used in a positive context and conveys a strong sense of approval or admiration. Additionally, the brevity and simplicity of the sentence add to its effectiveness and emphasis. Overall, the sentence is straightforward and powerful in its expression of high regard.\n",
            "Ground Truth Label: 4\n",
            "Prediction: 3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can, again, check our LLM's history to see what the actual prompt/response is.\n"
      ],
      "metadata": {
        "id": "miii9xQxgAzm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llm.inspect_history(n=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 524
        },
        "id": "dMxOlK65VzCZ",
        "outputId": "a662ffc1-c12c-4524-fe61-c72cd447e9e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "Rate a sentence from 0 to 4 on a dopeness scale\n",
            "\n",
            "---\n",
            "\n",
            "Follow the following format.\n",
            "\n",
            "Sentence: ${sentence}\n",
            "Reasoning: Let's think step by step in order to ${produce the rating}. We ...\n",
            "Rating: ${rating} (Respond with a single int value)\n",
            "\n",
            "---\n",
            "\n",
            "Sentence: This is top tier.\n",
            "Reasoning: Let's think step by step in order to\u001b[32m produce the rating. We first consider the impact of the phrase \"top tier,\" which implies the highest level of quality or excellence. This phrase is commonly used in a positive context and conveys a strong sense of approval or admiration. Additionally, the brevity and simplicity of the sentence add to its effectiveness and emphasis. Overall, the sentence is straightforward and powerful in its expression of high regard.\n",
            "Rating: 3\u001b[0m\n",
            "\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\n\\n\\nRate a sentence from 0 to 4 on a dopeness scale\\n\\n---\\n\\nFollow the following format.\\n\\nSentence: ${sentence}\\nReasoning: Let\\'s think step by step in order to ${produce the rating}. We ...\\nRating: ${rating} (Respond with a single int value)\\n\\n---\\n\\nSentence: This is top tier.\\nReasoning: Let\\'s think step by step in order to\\x1b[32m produce the rating. We first consider the impact of the phrase \"top tier,\" which implies the highest level of quality or excellence. This phrase is commonly used in a positive context and conveys a strong sense of approval or admiration. Additionally, the brevity and simplicity of the sentence add to its effectiveness and emphasis. Overall, the sentence is straightforward and powerful in its expression of high regard.\\nRating: 3\\x1b[0m\\n\\n\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Modules\n",
        "\n",
        "Now that we have our `TypedPredictor`, we can create a `Module`!\n",
        "\n",
        "A `Module` is useful because it allows us to interact with the `Predictor` and `Signature` in a way that DSPy can leverage for optimization.\n",
        "\n",
        "The helps the DSPy framework determine paths through your program - and helps during the `compilation` or optimisation steps (formerly `teleprompting`).\n",
        "\n",
        "> NOTE: You might notice this looks strikingly familiar to PyTorch, and this is by design!"
      ],
      "metadata": {
        "id": "a7U3yeCsg3B2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy import Module, Prediction\n",
        "\n",
        "class DopeOrNopeStudent(Module):\n",
        "  def __init__(self):\n",
        "    super().__init__()\n",
        "\n",
        "    self.generate_rating = TypedPredictor(DopeOrNopeSignature)\n",
        "\n",
        "  def forward(self, sentence):\n",
        "    prediction = self.generate_rating(sentence=sentence)\n",
        "    return Prediction(rating=prediction.rating)"
      ],
      "metadata": {
        "id": "H6GoyWyUJkl6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluate\n",
        "\n",
        "As with any good framework, DSPy has the ability to `Evaluate` - we can leverage this to determine how our current DSPy \"program\" (our `Module` in this case) operates.\n",
        "\n",
        "> NOTE: DSPy's \"program\" could be loosely related to a \"chain\" from the popular LLM Framework LangChain."
      ],
      "metadata": {
        "id": "PS2pf8tjh7lH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy.evaluate.evaluate import Evaluate\n",
        "\n",
        "evaluate_fewshot = Evaluate(devset=valset, num_threads=1, display_progress=True, display_table=10)\n",
        "\n",
        "def exact_match_metric(answer, pred, trace=None):\n",
        "  return answer.rating == pred.rating\n",
        "\n",
        "evaluate_fewshot(DopeOrNopeStudent(), metric=exact_match_metric)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "id": "C9o5461qMKyt",
        "outputId": "5fa218c7-ad70-4e5c-c036-c300e78a4d38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Metric: 3 / 10  (30.0): 100%|██████████| 10/10 [00:01<00:00,  6.33it/s]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<pandas.io.formats.style.Styler at 0x7b11f3349420>"
            ],
            "text/html": [
              "<style type=\"text/css\">\n",
              "#T_f2849 th {\n",
              "  text-align: left;\n",
              "}\n",
              "#T_f2849 td {\n",
              "  text-align: left;\n",
              "}\n",
              "#T_f2849_row0_col0, #T_f2849_row0_col1, #T_f2849_row0_col2, #T_f2849_row0_col3, #T_f2849_row1_col0, #T_f2849_row1_col1, #T_f2849_row1_col2, #T_f2849_row1_col3, #T_f2849_row2_col0, #T_f2849_row2_col1, #T_f2849_row2_col2, #T_f2849_row2_col3, #T_f2849_row3_col0, #T_f2849_row3_col1, #T_f2849_row3_col2, #T_f2849_row3_col3, #T_f2849_row4_col0, #T_f2849_row4_col1, #T_f2849_row4_col2, #T_f2849_row4_col3, #T_f2849_row5_col0, #T_f2849_row5_col1, #T_f2849_row5_col2, #T_f2849_row5_col3, #T_f2849_row6_col0, #T_f2849_row6_col1, #T_f2849_row6_col2, #T_f2849_row6_col3, #T_f2849_row7_col0, #T_f2849_row7_col1, #T_f2849_row7_col2, #T_f2849_row7_col3, #T_f2849_row8_col0, #T_f2849_row8_col1, #T_f2849_row8_col2, #T_f2849_row8_col3, #T_f2849_row9_col0, #T_f2849_row9_col1, #T_f2849_row9_col2, #T_f2849_row9_col3 {\n",
              "  text-align: left;\n",
              "  white-space: pre-wrap;\n",
              "  word-wrap: break-word;\n",
              "  max-width: 400px;\n",
              "}\n",
              "</style>\n",
              "<table id=\"T_f2849\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th class=\"blank level0\" >&nbsp;</th>\n",
              "      <th id=\"T_f2849_level0_col0\" class=\"col_heading level0 col0\" >sentence</th>\n",
              "      <th id=\"T_f2849_level0_col1\" class=\"col_heading level0 col1\" >example_rating</th>\n",
              "      <th id=\"T_f2849_level0_col2\" class=\"col_heading level0 col2\" >pred_rating</th>\n",
              "      <th id=\"T_f2849_level0_col3\" class=\"col_heading level0 col3\" >exact_match_metric</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
              "      <td id=\"T_f2849_row0_col0\" class=\"data row0 col0\" >This is top tier.</td>\n",
              "      <td id=\"T_f2849_row0_col1\" class=\"data row0 col1\" >4</td>\n",
              "      <td id=\"T_f2849_row0_col2\" class=\"data row0 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row0_col3\" class=\"data row0 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
              "      <td id=\"T_f2849_row1_col0\" class=\"data row1 col0\" >Big mood.</td>\n",
              "      <td id=\"T_f2849_row1_col1\" class=\"data row1 col1\" >3</td>\n",
              "      <td id=\"T_f2849_row1_col2\" class=\"data row1 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row1_col3\" class=\"data row1 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
              "      <td id=\"T_f2849_row2_col0\" class=\"data row2 col0\" >The presentation was outstanding.</td>\n",
              "      <td id=\"T_f2849_row2_col1\" class=\"data row2 col1\" >1</td>\n",
              "      <td id=\"T_f2849_row2_col2\" class=\"data row2 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row2_col3\" class=\"data row2 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
              "      <td id=\"T_f2849_row3_col0\" class=\"data row3 col0\" >I'm living my best life.</td>\n",
              "      <td id=\"T_f2849_row3_col1\" class=\"data row3 col1\" >4</td>\n",
              "      <td id=\"T_f2849_row3_col2\" class=\"data row3 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row3_col3\" class=\"data row3 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
              "      <td id=\"T_f2849_row4_col0\" class=\"data row4 col0\" >Sksksksk, that's hilarious.</td>\n",
              "      <td id=\"T_f2849_row4_col1\" class=\"data row4 col1\" >3</td>\n",
              "      <td id=\"T_f2849_row4_col2\" class=\"data row4 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row4_col3\" class=\"data row4 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
              "      <td id=\"T_f2849_row5_col0\" class=\"data row5 col0\" >The report is comprehensive.</td>\n",
              "      <td id=\"T_f2849_row5_col1\" class=\"data row5 col1\" >1</td>\n",
              "      <td id=\"T_f2849_row5_col2\" class=\"data row5 col2\" >2</td>\n",
              "      <td id=\"T_f2849_row5_col3\" class=\"data row5 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
              "      <td id=\"T_f2849_row6_col0\" class=\"data row6 col0\" >This is next level.</td>\n",
              "      <td id=\"T_f2849_row6_col1\" class=\"data row6 col1\" >4</td>\n",
              "      <td id=\"T_f2849_row6_col2\" class=\"data row6 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row6_col3\" class=\"data row6 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
              "      <td id=\"T_f2849_row7_col0\" class=\"data row7 col0\" >The meeting was productive.</td>\n",
              "      <td id=\"T_f2849_row7_col1\" class=\"data row7 col1\" >1</td>\n",
              "      <td id=\"T_f2849_row7_col2\" class=\"data row7 col2\" >2</td>\n",
              "      <td id=\"T_f2849_row7_col3\" class=\"data row7 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
              "      <td id=\"T_f2849_row8_col0\" class=\"data row8 col0\" >The analysis was insightful.</td>\n",
              "      <td id=\"T_f2849_row8_col1\" class=\"data row8 col1\" >1</td>\n",
              "      <td id=\"T_f2849_row8_col2\" class=\"data row8 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row8_col3\" class=\"data row8 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f2849_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
              "      <td id=\"T_f2849_row9_col0\" class=\"data row9 col0\" >I stan a legend.</td>\n",
              "      <td id=\"T_f2849_row9_col1\" class=\"data row9 col1\" >3</td>\n",
              "      <td id=\"T_f2849_row9_col2\" class=\"data row9 col2\" >3</td>\n",
              "      <td id=\"T_f2849_row9_col3\" class=\"data row9 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "30.0"
            ]
          },
          "metadata": {},
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Program Optimization (the Artist Formerly Known as Teleprompting)\n",
        "\n",
        "Optimization is the crux of the DSPy framework - it is what allows it to operate at a level beyond traditional prompt engineering.\n",
        "\n",
        "At a high level, optimisation is a way for the DSPy framework to take the program, a training set, and a metric - and make changes/tweaks to our program to improve our metrics on our dataset.\n",
        "\n",
        "Let's get started with the `LabeledFewShot` optimizer.\n",
        "\n",
        "The `LabeledFewShot` optimizer very simply provides a sample of the `trainset` as few-shot examples!"
      ],
      "metadata": {
        "id": "eIK6OdQFiUww"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy.teleprompt import LabeledFewShot\n",
        "\n",
        "labeled_fewshot_optimizer = LabeledFewShot(k=4)"
      ],
      "metadata": {
        "id": "wBq1Xs-CHphS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Once we define our optimizer, we can compile our program!"
      ],
      "metadata": {
        "id": "KdtrS7hXjHzv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "compiled_dspy = labeled_fewshot_optimizer.compile(student=DopeOrNopeStudent(), trainset=trainset)"
      ],
      "metadata": {
        "id": "eVv4aK2QJD3v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's evaluate!"
      ],
      "metadata": {
        "id": "8k_KhbL_jMXK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_fewshot(compiled_dspy, metric=exact_match_metric)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "id": "oxl_zZ51JQRc",
        "outputId": "e40327c5-0be3-4749-a8b6-6673c514115a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Metric: 4 / 10  (40.0): 100%|██████████| 10/10 [00:04<00:00,  2.21it/s]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<pandas.io.formats.style.Styler at 0x7b11f3348eb0>"
            ],
            "text/html": [
              "<style type=\"text/css\">\n",
              "#T_18235 th {\n",
              "  text-align: left;\n",
              "}\n",
              "#T_18235 td {\n",
              "  text-align: left;\n",
              "}\n",
              "#T_18235_row0_col0, #T_18235_row0_col1, #T_18235_row0_col2, #T_18235_row0_col3, #T_18235_row1_col0, #T_18235_row1_col1, #T_18235_row1_col2, #T_18235_row1_col3, #T_18235_row2_col0, #T_18235_row2_col1, #T_18235_row2_col2, #T_18235_row2_col3, #T_18235_row3_col0, #T_18235_row3_col1, #T_18235_row3_col2, #T_18235_row3_col3, #T_18235_row4_col0, #T_18235_row4_col1, #T_18235_row4_col2, #T_18235_row4_col3, #T_18235_row5_col0, #T_18235_row5_col1, #T_18235_row5_col2, #T_18235_row5_col3, #T_18235_row6_col0, #T_18235_row6_col1, #T_18235_row6_col2, #T_18235_row6_col3, #T_18235_row7_col0, #T_18235_row7_col1, #T_18235_row7_col2, #T_18235_row7_col3, #T_18235_row8_col0, #T_18235_row8_col1, #T_18235_row8_col2, #T_18235_row8_col3, #T_18235_row9_col0, #T_18235_row9_col1, #T_18235_row9_col2, #T_18235_row9_col3 {\n",
              "  text-align: left;\n",
              "  white-space: pre-wrap;\n",
              "  word-wrap: break-word;\n",
              "  max-width: 400px;\n",
              "}\n",
              "</style>\n",
              "<table id=\"T_18235\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th class=\"blank level0\" >&nbsp;</th>\n",
              "      <th id=\"T_18235_level0_col0\" class=\"col_heading level0 col0\" >sentence</th>\n",
              "      <th id=\"T_18235_level0_col1\" class=\"col_heading level0 col1\" >example_rating</th>\n",
              "      <th id=\"T_18235_level0_col2\" class=\"col_heading level0 col2\" >pred_rating</th>\n",
              "      <th id=\"T_18235_level0_col3\" class=\"col_heading level0 col3\" >exact_match_metric</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
              "      <td id=\"T_18235_row0_col0\" class=\"data row0 col0\" >This is top tier.</td>\n",
              "      <td id=\"T_18235_row0_col1\" class=\"data row0 col1\" >4</td>\n",
              "      <td id=\"T_18235_row0_col2\" class=\"data row0 col2\" >4</td>\n",
              "      <td id=\"T_18235_row0_col3\" class=\"data row0 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
              "      <td id=\"T_18235_row1_col0\" class=\"data row1 col0\" >Big mood.</td>\n",
              "      <td id=\"T_18235_row1_col1\" class=\"data row1 col1\" >3</td>\n",
              "      <td id=\"T_18235_row1_col2\" class=\"data row1 col2\" >3</td>\n",
              "      <td id=\"T_18235_row1_col3\" class=\"data row1 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
              "      <td id=\"T_18235_row2_col0\" class=\"data row2 col0\" >The presentation was outstanding.</td>\n",
              "      <td id=\"T_18235_row2_col1\" class=\"data row2 col1\" >1</td>\n",
              "      <td id=\"T_18235_row2_col2\" class=\"data row2 col2\" >3</td>\n",
              "      <td id=\"T_18235_row2_col3\" class=\"data row2 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
              "      <td id=\"T_18235_row3_col0\" class=\"data row3 col0\" >I'm living my best life.</td>\n",
              "      <td id=\"T_18235_row3_col1\" class=\"data row3 col1\" >4</td>\n",
              "      <td id=\"T_18235_row3_col2\" class=\"data row3 col2\" >3</td>\n",
              "      <td id=\"T_18235_row3_col3\" class=\"data row3 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
              "      <td id=\"T_18235_row4_col0\" class=\"data row4 col0\" >Sksksksk, that's hilarious.</td>\n",
              "      <td id=\"T_18235_row4_col1\" class=\"data row4 col1\" >3</td>\n",
              "      <td id=\"T_18235_row4_col2\" class=\"data row4 col2\" >3</td>\n",
              "      <td id=\"T_18235_row4_col3\" class=\"data row4 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
              "      <td id=\"T_18235_row5_col0\" class=\"data row5 col0\" >The report is comprehensive.</td>\n",
              "      <td id=\"T_18235_row5_col1\" class=\"data row5 col1\" >1</td>\n",
              "      <td id=\"T_18235_row5_col2\" class=\"data row5 col2\" >3</td>\n",
              "      <td id=\"T_18235_row5_col3\" class=\"data row5 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
              "      <td id=\"T_18235_row6_col0\" class=\"data row6 col0\" >This is next level.</td>\n",
              "      <td id=\"T_18235_row6_col1\" class=\"data row6 col1\" >4</td>\n",
              "      <td id=\"T_18235_row6_col2\" class=\"data row6 col2\" >3</td>\n",
              "      <td id=\"T_18235_row6_col3\" class=\"data row6 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
              "      <td id=\"T_18235_row7_col0\" class=\"data row7 col0\" >The meeting was productive.</td>\n",
              "      <td id=\"T_18235_row7_col1\" class=\"data row7 col1\" >1</td>\n",
              "      <td id=\"T_18235_row7_col2\" class=\"data row7 col2\" >3</td>\n",
              "      <td id=\"T_18235_row7_col3\" class=\"data row7 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
              "      <td id=\"T_18235_row8_col0\" class=\"data row8 col0\" >The analysis was insightful.</td>\n",
              "      <td id=\"T_18235_row8_col1\" class=\"data row8 col1\" >1</td>\n",
              "      <td id=\"T_18235_row8_col2\" class=\"data row8 col2\" >3</td>\n",
              "      <td id=\"T_18235_row8_col3\" class=\"data row8 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_18235_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
              "      <td id=\"T_18235_row9_col0\" class=\"data row9 col0\" >I stan a legend.</td>\n",
              "      <td id=\"T_18235_row9_col1\" class=\"data row9 col1\" >3</td>\n",
              "      <td id=\"T_18235_row9_col2\" class=\"data row9 col2\" >3</td>\n",
              "      <td id=\"T_18235_row9_col3\" class=\"data row9 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "40.0"
            ]
          },
          "metadata": {},
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see - with no effort at all - we can improve our performance on our `valset`!"
      ],
      "metadata": {
        "id": "htpDiJLcjOOI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's try another optimizer - this time: [`BootstrapFewShot`](https://dspy-docs.vercel.app/docs/deep-dive/teleprompter/bootstrap-fewshot).\n",
        "\n",
        "The key thing to note is that this optimizer works with even very few examples - by way of generating new examples by the LLMs!"
      ],
      "metadata": {
        "id": "fyf5baq0jU1Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dspy.teleprompt import BootstrapFewShot\n",
        "\n",
        "optimizer = BootstrapFewShot(metric=exact_match_metric, max_bootstrapped_demos=4, max_labeled_demos=12)\n",
        "\n",
        "compiled_dspy_BOOTSTRAP = optimizer.compile(student=DopeOrNopeStudent(), trainset=trainset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vPx1wKpAUKBx",
        "outputId": "5b79baed-8b31-43c4-96b5-c3620d4ec86e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  9%|▉         | 8/89 [00:03<00:32,  2.47it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's finally evaluate!"
      ],
      "metadata": {
        "id": "YsF13taMo6dm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_fewshot(compiled_dspy_BOOTSTRAP, metric=exact_match_metric)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "id": "lFwORTZbUcwG",
        "outputId": "8ff37054-bbc5-4a34-bcc3-41e3d7961e13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Metric: 7 / 10  (70.0): 100%|██████████| 10/10 [00:03<00:00,  2.62it/s]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<pandas.io.formats.style.Styler at 0x7b11f9fbb220>"
            ],
            "text/html": [
              "<style type=\"text/css\">\n",
              "#T_187e0 th {\n",
              "  text-align: left;\n",
              "}\n",
              "#T_187e0 td {\n",
              "  text-align: left;\n",
              "}\n",
              "#T_187e0_row0_col0, #T_187e0_row0_col1, #T_187e0_row0_col2, #T_187e0_row0_col3, #T_187e0_row1_col0, #T_187e0_row1_col1, #T_187e0_row1_col2, #T_187e0_row1_col3, #T_187e0_row2_col0, #T_187e0_row2_col1, #T_187e0_row2_col2, #T_187e0_row2_col3, #T_187e0_row3_col0, #T_187e0_row3_col1, #T_187e0_row3_col2, #T_187e0_row3_col3, #T_187e0_row4_col0, #T_187e0_row4_col1, #T_187e0_row4_col2, #T_187e0_row4_col3, #T_187e0_row5_col0, #T_187e0_row5_col1, #T_187e0_row5_col2, #T_187e0_row5_col3, #T_187e0_row6_col0, #T_187e0_row6_col1, #T_187e0_row6_col2, #T_187e0_row6_col3, #T_187e0_row7_col0, #T_187e0_row7_col1, #T_187e0_row7_col2, #T_187e0_row7_col3, #T_187e0_row8_col0, #T_187e0_row8_col1, #T_187e0_row8_col2, #T_187e0_row8_col3, #T_187e0_row9_col0, #T_187e0_row9_col1, #T_187e0_row9_col2, #T_187e0_row9_col3 {\n",
              "  text-align: left;\n",
              "  white-space: pre-wrap;\n",
              "  word-wrap: break-word;\n",
              "  max-width: 400px;\n",
              "}\n",
              "</style>\n",
              "<table id=\"T_187e0\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th class=\"blank level0\" >&nbsp;</th>\n",
              "      <th id=\"T_187e0_level0_col0\" class=\"col_heading level0 col0\" >sentence</th>\n",
              "      <th id=\"T_187e0_level0_col1\" class=\"col_heading level0 col1\" >example_rating</th>\n",
              "      <th id=\"T_187e0_level0_col2\" class=\"col_heading level0 col2\" >pred_rating</th>\n",
              "      <th id=\"T_187e0_level0_col3\" class=\"col_heading level0 col3\" >exact_match_metric</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
              "      <td id=\"T_187e0_row0_col0\" class=\"data row0 col0\" >This is top tier.</td>\n",
              "      <td id=\"T_187e0_row0_col1\" class=\"data row0 col1\" >4</td>\n",
              "      <td id=\"T_187e0_row0_col2\" class=\"data row0 col2\" >4</td>\n",
              "      <td id=\"T_187e0_row0_col3\" class=\"data row0 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
              "      <td id=\"T_187e0_row1_col0\" class=\"data row1 col0\" >Big mood.</td>\n",
              "      <td id=\"T_187e0_row1_col1\" class=\"data row1 col1\" >3</td>\n",
              "      <td id=\"T_187e0_row1_col2\" class=\"data row1 col2\" >3</td>\n",
              "      <td id=\"T_187e0_row1_col3\" class=\"data row1 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
              "      <td id=\"T_187e0_row2_col0\" class=\"data row2 col0\" >The presentation was outstanding.</td>\n",
              "      <td id=\"T_187e0_row2_col1\" class=\"data row2 col1\" >1</td>\n",
              "      <td id=\"T_187e0_row2_col2\" class=\"data row2 col2\" >4</td>\n",
              "      <td id=\"T_187e0_row2_col3\" class=\"data row2 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
              "      <td id=\"T_187e0_row3_col0\" class=\"data row3 col0\" >I'm living my best life.</td>\n",
              "      <td id=\"T_187e0_row3_col1\" class=\"data row3 col1\" >4</td>\n",
              "      <td id=\"T_187e0_row3_col2\" class=\"data row3 col2\" >4</td>\n",
              "      <td id=\"T_187e0_row3_col3\" class=\"data row3 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
              "      <td id=\"T_187e0_row4_col0\" class=\"data row4 col0\" >Sksksksk, that's hilarious.</td>\n",
              "      <td id=\"T_187e0_row4_col1\" class=\"data row4 col1\" >3</td>\n",
              "      <td id=\"T_187e0_row4_col2\" class=\"data row4 col2\" >3</td>\n",
              "      <td id=\"T_187e0_row4_col3\" class=\"data row4 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
              "      <td id=\"T_187e0_row5_col0\" class=\"data row5 col0\" >The report is comprehensive.</td>\n",
              "      <td id=\"T_187e0_row5_col1\" class=\"data row5 col1\" >1</td>\n",
              "      <td id=\"T_187e0_row5_col2\" class=\"data row5 col2\" >1</td>\n",
              "      <td id=\"T_187e0_row5_col3\" class=\"data row5 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
              "      <td id=\"T_187e0_row6_col0\" class=\"data row6 col0\" >This is next level.</td>\n",
              "      <td id=\"T_187e0_row6_col1\" class=\"data row6 col1\" >4</td>\n",
              "      <td id=\"T_187e0_row6_col2\" class=\"data row6 col2\" >4</td>\n",
              "      <td id=\"T_187e0_row6_col3\" class=\"data row6 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
              "      <td id=\"T_187e0_row7_col0\" class=\"data row7 col0\" >The meeting was productive.</td>\n",
              "      <td id=\"T_187e0_row7_col1\" class=\"data row7 col1\" >1</td>\n",
              "      <td id=\"T_187e0_row7_col2\" class=\"data row7 col2\" >2</td>\n",
              "      <td id=\"T_187e0_row7_col3\" class=\"data row7 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
              "      <td id=\"T_187e0_row8_col0\" class=\"data row8 col0\" >The analysis was insightful.</td>\n",
              "      <td id=\"T_187e0_row8_col1\" class=\"data row8 col1\" >1</td>\n",
              "      <td id=\"T_187e0_row8_col2\" class=\"data row8 col2\" >3</td>\n",
              "      <td id=\"T_187e0_row8_col3\" class=\"data row8 col3\" >False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_187e0_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
              "      <td id=\"T_187e0_row9_col0\" class=\"data row9 col0\" >I stan a legend.</td>\n",
              "      <td id=\"T_187e0_row9_col1\" class=\"data row9 col1\" >3</td>\n",
              "      <td id=\"T_187e0_row9_col2\" class=\"data row9 col2\" >3</td>\n",
              "      <td id=\"T_187e0_row9_col3\" class=\"data row9 col3\" >✔️ [True]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "70.0"
            ]
          },
          "metadata": {},
          "execution_count": 166
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can see that this optimization helps our program achieve 30 points higher on our evaluation!"
      ],
      "metadata": {
        "id": "2k9ov1j-pMV8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llm.inspect_history(n=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 764
        },
        "id": "K-7qjXPKWMWF",
        "outputId": "e2717a49-3999-4b62-ec8e-397c1f864b12"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "Rate a sentence from 0 to 4 on a dopeness scale\n",
            "\n",
            "---\n",
            "\n",
            "Follow the following format.\n",
            "\n",
            "Sentence: ${sentence}\n",
            "Rating: ${rating} (Respond with a single int value)\n",
            "\n",
            "---\n",
            "\n",
            "Sentence: The approval was granted.\n",
            "Rating: 1\n",
            "\n",
            "---\n",
            "\n",
            "Sentence: I admire your dedication.\n",
            "Rating: 1\n",
            "\n",
            "---\n",
            "\n",
            "Sentence: Too good to be true.\n",
            "Rating: 4\n",
            "\n",
            "---\n",
            "\n",
            "Sentence: The software was updated.\n",
            "Rating: 1\n",
            "\n",
            "---\n",
            "\n",
            "Sentence: I stan a legend.\n",
            "Rating:\u001b[32m 3\u001b[0m\n",
            "\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\n\\n\\nRate a sentence from 0 to 4 on a dopeness scale\\n\\n---\\n\\nFollow the following format.\\n\\nSentence: ${sentence}\\nRating: ${rating} (Respond with a single int value)\\n\\n---\\n\\nSentence: The approval was granted.\\nRating: 1\\n\\n---\\n\\nSentence: I admire your dedication.\\nRating: 1\\n\\n---\\n\\nSentence: Too good to be true.\\nRating: 4\\n\\n---\\n\\nSentence: The software was updated.\\nRating: 1\\n\\n---\\n\\nSentence: I stan a legend.\\nRating:\\x1b[32m 3\\x1b[0m\\n\\n\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for name, parameter in compiled_dspy_BOOTSTRAP.named_parameters():\n",
        "  print(f\"Parameter {name}: Num Examples: {len(parameter.demos)}, {parameter.demos[0]}\")\n",
        "  print()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XEP0Sf6CUvuo",
        "outputId": "fe87f449-c39b-4122-9ac7-cee5fe3b792f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Parameter generate_rating.predictor: Num Examples: 16, Example({'augmented': True, 'sentence': 'This tea is piping hot.', 'rating': '4'}) (input_keys=None)\n",
            "\n"
          ]
        }
      ]
    }
  ]
}