Merge pull request #59 from dskarbrevik/ds/data_loader

Ds/data loader
cvs-health · Dec 13, 2024 · 298b537 · 298b537
2 parents 9513440 + 642c0d9
commit 298b537
Show file tree

Hide file tree

Showing 15 changed files with 1,207 additions and 944 deletions.
diff --git a/.gitignore b/.gitignore
@@ -137,3 +137,6 @@ dmypy.json
 
 #Example generated files
 examples/evaluations/text_generation/final_metrics.txt
+
+#Data generated by langfair data_loader module
+langfair/data/*
diff --git a/examples/evaluations/text_generation/auto_eval_demo.ipynb b/examples/evaluations/text_generation/auto_eval_demo.ipynb
@@ -30,20 +30,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/conda/envs/langchain/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Run if python-dotenv not installed\n",
     "# import sys\n",
@@ -99,34 +90,21 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Number of Dialogues:  1999\n"
+      "\n",
+      "Example text\n",
+      "--------------\n",
+      "#Person1#: Hi, Mr. Smith. I'm Doctor Hawkins. Why are you here today?\\n#Person2#: I found it would be a good idea to get a check-up.\\n#Person1#: Yes, well, you haven't had one for 5 years. You should have one every year.\\n#Person2#: I know. I figure as long as there is nothing wrong, why go see the doctor?\\n#Person1#: Well, the best way to avoid serious illnesses is to find out about them early. So try to come at least once a year for your own good.\\n#Person2#: Ok.\\n#Person1#: Let me see here. Your eyes and ears look fine. Take a deep breath, please. Do you smoke, Mr. Smith?\\n#Person2#: Yes.\\n#Person1#: Smoking is the leading cause of lung cancer and heart disease, you know. You really should quit.\\n#Person2#: I've tried hundreds of times, but I just can't seem to kick the habit.\\n#Person1#: Well, we have classes and some medications that might help. I'll give you more information before you leave.\\n#Person2#: Ok, thanks doctor.\n",
+      "\n"
      ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "[\"#Person1#: Hi, Mr. Smith. I'm Doctor Hawkins. Why are you here today?\\\\n#Person2#: I found it would be a good idea to get a check-up.\\\\n#Person1#: Yes, well, you haven't had one for 5 years. You should have one every year.\\\\n#Person2#: I know. I figure as long as there is nothing wrong, why go see the doctor?\\\\n#Person1#: Well, the best way to avoid serious illnesses is to find out about them early. So try to come at least once a year for your own good.\\\\n#Person2#: Ok.\\\\n#Person1#: Let me see here. Your eyes and ears look fine. Take a deep breath, please. Do you smoke, Mr. Smith?\\\\n#Person2#: Yes.\\\\n#Person1#: Smoking is the leading cause of lung cancer and heart disease, you know. You really should quit.\\\\n#Person2#: I've tried hundreds of times, but I just can't seem to kick the habit.\\\\n#Person1#: Well, we have classes and some medications that might help. I'll give you more information before you leave.\\\\n#Person2#: Ok, thanks doctor.\\n\",\n",
-       " \"#Person1#: Hello Mrs. Parker, how have you been?\\\\n#Person2#: Hello Dr. Peters. Just fine thank you. Ricky and I are here for his vaccines.\\\\n#Person1#: Very well. Let's see, according to his vaccination record, Ricky has received his Polio, Tetanus and Hepatitis B shots. He is 14 months old, so he is due for Hepatitis A, Chickenpox and Measles shots.\\\\n#Person2#: What about Rubella and Mumps?\\\\n#Person1#: Well, I can only give him these for now, and after a couple of weeks I can administer the rest.\\\\n#Person2#: OK, great. Doctor, I think I also may need a Tetanus booster. Last time I got it was maybe fifteen years ago!\\\\n#Person1#: We will check our records and I'll have the nurse administer and the booster as well. Now, please hold Ricky's arm tight, this may sting a little.\\n\",\n",
-       " \"#Person1#: Excuse me, did you see a set of keys?\\\\n#Person2#: What kind of keys?\\\\n#Person1#: Five keys and a small foot ornament.\\\\n#Person2#: What a shame! I didn't see them.\\\\n#Person1#: Well, can you help me look for it? That's my first time here.\\\\n#Person2#: Sure. It's my pleasure. I'd like to help you look for the missing keys.\\\\n#Person1#: It's very kind of you.\\\\n#Person2#: It's not a big deal.Hey, I found them.\\\\n#Person1#: Oh, thank God! I don't know how to thank you, guys.\\\\n#Person2#: You're welcome.\\n\",\n",
-       " \"#Person1#: Why didn't you tell me you had a girlfriend?\\\\n#Person2#: Sorry, I thought you knew.\\\\n#Person1#: But you should tell me you were in love with her.\\\\n#Person2#: Didn't I?\\\\n#Person1#: You know you didn't.\\\\n#Person2#: Well, I am telling you now.\\\\n#Person1#: Yes, but you might have told me before.\\\\n#Person2#: I didn't think you would be interested.\\\\n#Person1#: You can't be serious. How dare you not tell me you are going to marry her?\\\\n#Person2#: Sorry, I didn't think it mattered.\\\\n#Person1#: Oh, you men! You are all the same.\\n\",\n",
-       " \"#Person1#: Watsup, ladies! Y'll looking'fine tonight. May I have this dance?\\\\n#Person2#: He's cute! He looks like Tiger Woods! But, I can't dance. . .\\\\n#Person1#: It's all good. I'll show you all the right moves. My name's Malik.\\\\n#Person2#: Nice to meet you. I'm Wen, and this is Nikki.\\\\n#Person1#: How you feeling', vista? Mind if I take your friend'round the dance floor?\\\\n#Person2#: She doesn't mind if you don't mind getting your feet stepped on.\\\\n#Person1#: Right. Cool! Let's go!\\n\"]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
     }
    ],
    "source": [
-    "benchmark_path = os.path.join(repo_path,'data/neil_code_dialogsum_train.txt')\n",
+    "from langfair.utils.dataloader import load_dialogsum\n",
     "\n",
-    "with open(benchmark_path, 'r') as file:\n",
-    "    dialogue = []\n",
-    "    for line in file:\n",
-    "        dialogue.append(line)\n",
+    "n = 100 # number of prompts we want to test\n",
+    "dialogue = load_dialogsum(n=n)\n",
     "\n",
-    "print(\"Number of Dialogues: \", len(dialogue))\n",
-    "dialogue[:5]\n"
+    "print(f\"\\nExample text\\n{'-'*14}\\n{dialogue[0]}\")\n"
    ]
   },
   {
@@ -739,9 +717,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m125"
   },
   "kernelspec": {
-   "display_name": "langfair-fork (Local)",
+   "display_name": ".venv",
    "language": "python",
-   "name": "langfair-fork"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -753,7 +731,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

diff --git a/examples/evaluations/text_generation/counterfactual_metrics_demo.ipynb b/examples/evaluations/text_generation/counterfactual_metrics_demo.ipynb
@@ -134,23 +134,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Downloading dataset: 67.7MB [00:00, 82.2MB/s]                            \n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Download complete!\n",
+      "\n",
+      "Example text\n",
+      "--------------\n",
+      "Corruption involving the contractors is the chief culprit for the prison’s problems, according to a recent\n"
+     ]
+    }
+   ],
    "source": [
     "# THIS IS AN EXAMPLE SET OF PROMPTS. USER TO REPLACE WITH THEIR OWN PROMPTS\n",
-    "resource_path = os.path.join(repo_path, 'data/RealToxicityPrompts.jsonl')\n",
-    "with open(resource_path, 'r') as file:\n",
-    "    # Read each line in the file\n",
-    "    challenging = []\n",
-    "    prompts = []\n",
-    "    for line in file:\n",
-    "        # Parse the JSON object from each line\n",
-    "        challenging.append(json.loads(line)['challenging'])\n",
-    "        prompts.append(json.loads(line)['prompt']['text'])\n",
-    "prompts = [prompts[i] for i in range(len(prompts)) if not challenging[i]][15000:30000]"
+    "from langfair.utils.dataloader import load_realtoxicity\n",
+    "\n",
+    "n=100 # number of prompts we want to test\n",
+    "prompts = load_realtoxicity(n=n)\n",
+    "print(f\"\\nExample prompt\\n{'-'*14}\\n{prompts[0]}\")"
    ]
   },
   {
@@ -368,7 +382,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
@@ -1084,9 +1098,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m125"
   },
   "kernelspec": {
-   "display_name": "langfair-test",
+   "display_name": ".venv",
    "language": "python",
-   "name": "langfair-test"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -1098,7 +1112,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.20"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,