build_human_eval_hit_csvs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cb882096",
   "metadata": {},
   "source": [
    "# SBIC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b4dd1be8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os, re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a7a2818d",
   "metadata": {},
   "outputs": [],
   "source": [
    "TASK = \"css_data/sbic\"\n",
    "baseline = f\"{TASK}/T5-finetune-sbic_predict.json\"\n",
    "with open(baseline, \"r\") as infile:\n",
    "    base_json = json.load(infile)\n",
    "sbic = pd.read_csv('css_data/sbic/sbic.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8a5f5e5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims.\"]\n",
      "no match [\"trivializes harm to victims.\"]\n",
      "\tnon-unique [\"trivializes harm to victims.\"]\n",
      "no match [\"trivializes harm to victims.\"]\n",
      "\tnon-unique [\"are marginalized for a joke\"]\n",
      "no match [\"are marginalized for a joke\"]\n",
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims\"]\n",
      "no match [\"trivializes harm to victims\"]\n",
      "\tnon-unique [\"trivializes harm to victims.\"]\n",
      "no match [\"trivializes harm to victims.\"]\n"
     ]
    }
   ],
   "source": [
    "sbic_test = []\n",
    "for idx, lbl in enumerate(base_json['labels']):\n",
    "    pred = base_json['predictions'][idx]\n",
    "    pred = pred.split(\", \")[0].replace(\"[\", \"\").replace('\"', \"\")\n",
    "    consider = sbic[sbic[\"targetStereotype\"]==lbl].copy()\n",
    "    consider['Generated'] = pred\n",
    "    consider['Model'] = \"baseline\"\n",
    "    consider['Task'] = \"sbic\"\n",
    "    if len(consider)>1:\n",
    "        print(\"\\tnon-unique\", lbl)\n",
    "    if len(consider)==1:\n",
    "        sbic_test.append(consider)\n",
    "    else:\n",
    "        print(\"no match\", lbl)\n",
    "    #sbic_test.append(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e454b79b",
   "metadata": {},
   "outputs": [],
   "source": [
    "hit_df = pd.concat(sbic_test)[['post', 'targetStereotype', 'targetMinority', 'Generated', 'Model', 'Task']].sample(frac=1, random_state=7).copy()\n",
    "hit_df.to_csv('hit/input/sbic/sbic_baseline.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "182ac187",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-ada-001\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "text-babbage-001\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "text-davinci-003\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "flan-t5-large\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "flan-t5-small\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "flan-t5-xl\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "flan-t5-xxl\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "text-curie-001\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "text-davinci-002\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "flan-ul2\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "flan-t5-base\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "chatgpt\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "text-davinci-001\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n",
      "\tno match [\"are marginalized for a joke\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims\"]\n",
      "\tno match [\"trivializes harm to victims.\"]\n"
     ]
    }
   ],
   "source": [
    "for fn in glob(\"css_data/sbic/answer*\"):\n",
    "    model = \"-\".join(fn.split(\"-\")[1:])\n",
    "    print(model)\n",
    "    df = pd.read_csv(fn, sep='\\t', names=['idx', 'targetStereotype', 'Generated'])\n",
    "    \n",
    "    sbic_test = []\n",
    "    for _, row in df.iterrows():\n",
    "        pred = row[\"Generated\"].replace(\"&\", \"\")\n",
    "        lbl = row[\"targetStereotype\"]\n",
    "        consider = sbic[sbic[\"targetStereotype\"]==lbl].copy()\n",
    "        consider['Generated'] = pred\n",
    "        consider['Model'] = model\n",
    "        consider['Task'] = \"sbic\"\n",
    "        if len(consider)==1:\n",
    "            sbic_test.append(consider)\n",
    "        else:\n",
    "            print(\"\\tno match\", lbl)\n",
    "    hit_df = pd.concat(sbic_test)[['post', 'targetStereotype', 'targetMinority', 'Generated', 'Model', 'Task']].sample(frac=1, random_state=7).copy()\n",
    "    hit_df.to_csv(f'hit/input/sbic/sbic_{model}.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72e2ac8e",
   "metadata": {},
   "source": [
    "# MRF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fa2712a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os, re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "72e1881c",
   "metadata": {},
   "outputs": [],
   "source": [
    "TASK = \"css_data/mrf\"\n",
    "baseline = f\"{TASK}/T5-finetune-mrf-explain.json\"\n",
    "with open(baseline, \"r\") as infile:\n",
    "    base_json = json.load(infile)\n",
    "mrf = pd.read_csv('css_data/mrf/mrf.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "18679248",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['covid-19 is not real']\n",
      "no match ['covid-19 is not real']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['unknown intent']\n",
      "no match ['unknown intent']\n",
      "\tnon-unique ['covid-19 is not real']\n",
      "no match ['covid-19 is not real']\n"
     ]
    }
   ],
   "source": [
    "mrf_test = []\n",
    "for idx, lbl in enumerate(base_json['labels']):\n",
    "    pred = base_json['predictions'][idx]\n",
    "    pred = pred.split(\", \")[0].replace(\"[\", \"\").replace('\"', \"\")\n",
    "    consider = mrf[mrf[\"writer_intent\"]==lbl].copy()\n",
    "    consider['Generated'] = pred\n",
    "    consider['Model'] = \"baseline\"\n",
    "    consider['Task'] = \"mrf\"\n",
    "    consider['misinfo'] = consider['gold_label']\n",
    "    if len(consider)>1:\n",
    "        print(\"\\tnon-unique\", lbl)\n",
    "    if len(consider)==1:\n",
    "        mrf_test.append(consider)\n",
    "    else:\n",
    "        print(\"no match\", lbl)\n",
    "    #sbic_test.append(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ecddc392",
   "metadata": {},
   "outputs": [],
   "source": [
    "hit_df = pd.concat(mrf_test)[['headline', 'writer_intent', 'misinfo', 'Generated', 'Model', 'Task']].sample(frac=1, random_state=7).copy()\n",
    "hit_df.to_csv('hit/input/mrf/mrf_baseline.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2be9d24a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chatgpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n",
      "Skipping line 21: expected 3 fields, saw 4\n",
      "Skipping line 22: expected 3 fields, saw 4\n",
      "Skipping line 54: expected 3 fields, saw 4\n",
      "Skipping line 63: expected 3 fields, saw 4\n",
      "Skipping line 64: expected 3 fields, saw 4\n",
      "Skipping line 66: expected 3 fields, saw 5\n",
      "Skipping line 107: expected 3 fields, saw 4\n",
      "Skipping line 110: expected 3 fields, saw 5\n",
      "Skipping line 155: expected 3 fields, saw 4\n",
      "Skipping line 168: expected 3 fields, saw 4\n",
      "Skipping line 172: expected 3 fields, saw 4\n",
      "Skipping line 188: expected 3 fields, saw 4\n",
      "Skipping line 193: expected 3 fields, saw 4\n",
      "Skipping line 202: expected 3 fields, saw 4\n",
      "Skipping line 218: expected 3 fields, saw 4\n",
      "Skipping line 219: expected 3 fields, saw 5\n",
      "Skipping line 223: expected 3 fields, saw 4\n",
      "Skipping line 226: expected 3 fields, saw 4\n",
      "Skipping line 228: expected 3 fields, saw 4\n",
      "Skipping line 232: expected 3 fields, saw 4\n",
      "Skipping line 235: expected 3 fields, saw 4\n",
      "Skipping line 237: expected 3 fields, saw 4\n",
      "Skipping line 241: expected 3 fields, saw 4\n",
      "Skipping line 251: expected 3 fields, saw 4\n",
      "Skipping line 255: expected 3 fields, saw 4\n",
      "Skipping line 273: expected 3 fields, saw 4\n",
      "Skipping line 282: expected 3 fields, saw 4\n",
      "Skipping line 294: expected 3 fields, saw 4\n",
      "Skipping line 320: expected 3 fields, saw 4\n",
      "Skipping line 322: expected 3 fields, saw 4\n",
      "Skipping line 332: expected 3 fields, saw 6\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-ada-001\n",
      "\tno match The coronavirus vaccine will cause other more serious diseases to emerge, said Anthony Fauci. \t ['unknown intent']\n",
      "\tno match Otters Show How Predators Can Blunt Climate Damage \t ['unknown intent']\n",
      "\tno match New Evidence That the Ancient Climate Was Warmer than Today's  'Roman Warming was the warmest in the last 2,000 years' \t ['unknown intent']\n",
      "\tno match Punishing Companies For CO2 Emissions Won't Affect Temps, Climate \t ['unknown intent']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tno match Climate report warns of rising risk as U.N. pushes nations to take action \t ['unknown intent']\n",
      "\tno match The end of total quarantine: the Cabinet of Ministers of Ukraine has just made a decision. \t ['unknown intent']\n",
      "\tno match 'The most significant climate legislation ever': How stimulus bill tackles warming planet \t ['unknown intent']\n",
      "\tno match No 10 and Treasury clash over spending on environmental agenda \t ['unknown intent']\n",
      "\tno match The Drilldown: Opposition disappointed with government's climate legislation \t ['unknown intent']\n",
      "\tno match Councils reporting thousands of 'climate-related' incidents, including flooding \t ['unknown intent']\n",
      "\tno match UK to make climate risk reports mandatory for large companies \t ['unknown intent']\n",
      "\tno match French schooner Tara sets sail on scientific mission to study climate change \t ['unknown intent']\n",
      "\tno match How climate change could benefit Russia \t ['covid-19 is not real']\n",
      "\tno match Monetary Expansion Yielding Diminishing Returns And An Environment Very Friendly To Gold \t ['unknown intent']\n",
      "\tno match Even under Trump, America did a better job at tackling the climate emergency than Canada did, says Green Party Leader Annamie Paul \t ['unknown intent']\n",
      "\tno match Pfizer vaccine has caused the death of six people \t ['unknown intent']\n",
      "\tno match Trump urged to send Paris climate plan to senate to block Biden \t ['unknown intent']\n",
      "\tno match Should the price of meat reflect its impact on the environment? \t ['unknown intent']\n",
      "\tno match Lidocaine is effective in the treatment of COVID-19. \t ['unknown intent']\n",
      "\tno match Ancient whale skeleton found in Thailand holds clues to climate change \t ['covid-19 is not real']\n",
      "text-davinci-002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-ul2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-babbage-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-small\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-davinci-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-xl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-large\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-davinci-003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-xxl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-curie-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19539/3584148622.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    }
   ],
   "source": [
    "for fn in glob(\"css_data/mrf/answer-explanation*\"):\n",
    "    model = \"-\".join(fn.split(\"-\")[2:])\n",
    "    print(model)\n",
    "    with open(f\"css_data/mrf/prompts.json-explanation-{model}\", \"r\") as f:\n",
    "        prompts = json.load(f)\n",
    "        \n",
    "    df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n",
    "\n",
    "    mrf_test = []\n",
    "    for _, row in df.iterrows():\n",
    "        if type(row[\"Generated\"])==str:\n",
    "            pred = row[\"Generated\"].replace(\"&\", \"\")\n",
    "            headline = \"\"\n",
    "            if str(row[\"idx\"]) in prompts:\n",
    "                headline = prompts[str(row[\"idx\"])].split('\\n')[0].strip()\n",
    "            \n",
    "            lbl = row[\"writer_intent\"]\n",
    "            consider = mrf[mrf[\"headline\"]==headline].copy()\n",
    "            \n",
    "            if (not len(consider)) or (consider[\"writer_intent\"].iloc[0]!=lbl):\n",
    "                consider = mrf[mrf[\"writer_intent\"]==lbl].copy()\n",
    "            \n",
    "            consider['Generated'] = pred\n",
    "            consider['Model'] = model\n",
    "            consider['Task'] = \"mrf\"\n",
    "            consider['misinfo'] = consider['gold_label']\n",
    "            \n",
    "            if len(consider)==1:\n",
    "                if consider[\"writer_intent\"].iloc[0]==lbl:\n",
    "                    mrf_test.append(consider)\n",
    "                else:\n",
    "                    print(\"Mismatch\", model, row[\"idx\"])\n",
    "                    break\n",
    "            elif len(consider)>1:\n",
    "                print(\"\\tno match\", headline, '\\t', lbl)\n",
    "            else:\n",
    "                pass\n",
    "    if len(mrf_test):\n",
    "        hit_df = pd.concat(mrf_test)[['headline', 'writer_intent', 'misinfo', 'Generated', 'Model', 'Task']].sample(frac=1, random_state=7).copy()\n",
    "        hit_df.to_csv(f'hit/input/mrf/mrf_{model}.csv', index=False)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f65fe53",
   "metadata": {},
   "outputs": [],
   "source": [
    "q = \"'Buddha would be green': Dalai Lama calls for urgent climate action\"\n",
    "mrf[mrf['headline']==q]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "47ed18f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in df.iterrows():\n",
    "    pred = row[\"Generated\"].replace(\"&\", \"\")\n",
    "    lbl = prompts[str(row[\"idx\"])].split('\\n')[0].strip()\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8e3b3ef3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Some people are reporting delays in getting the flu vaccine in Texas',\n",
       " 'Texas sees delays amid push for faster vaccine rollout')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred, lbl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a149c7bd",
   "metadata": {},
   "source": [
    "# FLUTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41c1f892",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os, re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d9c2a19a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_premise_hypothesis(txt):\n",
    "    premises = re.findall(r\"premise: (.*?)(?:\\n|$|hypothesis)\", txt)\n",
    "    hypotheses = re.findall(r\"hypothesis: (.*?)(?:\\n|$|premise)\", txt)\n",
    "    premise, hypothesis = \"\", \"\"\n",
    "    if len(premises):\n",
    "        premise = premises[0]\n",
    "    if len(hypotheses):\n",
    "        hypothesis = hypotheses[0]\n",
    "    return premise.strip(), hypothesis.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "17cef6d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "TASK = \"css_data/flute\"\n",
    "baseline = f\"{TASK}/T5-finetune-flute_predict.json\"\n",
    "with open(baseline, \"r\") as infile:\n",
    "    base_json = json.load(infile)\n",
    "with open(\"css_data/flute/flute-explanation.json\", \"r\") as f:\n",
    "    flute = pd.DataFrame.from_dict(json.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "26bc2632",
   "metadata": {},
   "outputs": [],
   "source": [
    "premises = []\n",
    "hypotheses = []\n",
    "for c in flute.context.values:\n",
    "    p, h = get_premise_hypothesis(c)\n",
    "    premises.append(p)\n",
    "    hypotheses.append(h)\n",
    "flute[\"premise\"] = premises\n",
    "flute[\"hypothesis\"] = hypotheses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c482ab2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "flute_test = []\n",
    "for idx, lbl in enumerate(base_json['labels']):\n",
    "    pred = base_json['predictions'][idx]\n",
    "    pred_label, pred_expl = re.split(r\"[&]+\", pred)\n",
    "    lbl_label, lbl_expl = re.split(r\"[&]+\", lbl)\n",
    "    \n",
    "    consider = flute[flute[\"additional_labels\"]==lbl_expl].copy()\n",
    "    consider['Generated'] = pred_expl\n",
    "    consider['Generated_Label'] = pred_label\n",
    "    consider['Model'] = \"baseline\"\n",
    "    consider['Task'] = \"flute\"\n",
    "    if len(consider)>0:\n",
    "        flute_test.append(consider.iloc[0:1])\n",
    "    else:\n",
    "        print(\"no match\", lbl)\n",
    "    #sbic_test.append(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5329f94e",
   "metadata": {},
   "outputs": [],
   "source": [
    "hit_df = pd.concat(flute_test)[['premise', 'hypothesis', 'labels', 'additional_labels', 'Generated', 'Generated_Label', 'Model', 'Task']].sample(frac=1, random_state=7).copy()\n",
    "hit_df.to_csv('hit/input/flute/flute_baseline.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7d1e5af0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-xl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-small\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-curie-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-davinci-003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chatgpt\n",
      "flan-t5-large\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n",
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-ada-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-babbage-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-davinci-002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-davinci-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19601/1814458700.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n"
     ]
    }
   ],
   "source": [
    "for fn in glob(\"css_data/flute/answer-explain*\"):\n",
    "    model = \"-\".join(fn.split(\"-\")[2:])\n",
    "    print(model)\n",
    "    with open(f\"css_data/flute/prompts.json-explanation-{model}\", \"r\") as f:\n",
    "        prompts = json.load(f)\n",
    "        \n",
    "    df = pd.read_csv(fn, sep='\\t', names=['idx', 'writer_intent', 'Generated'], error_bad_lines=False)\n",
    "\n",
    "    flute_test = []\n",
    "    for _, row in df.iterrows():\n",
    "        if type(row[\"Generated\"])==str:\n",
    "            pred = row[\"Generated\"].replace(\"&\", \"\")\n",
    "            if str(row[\"idx\"]) in prompts:\n",
    "                prompt = prompts[str(row[\"idx\"])]\n",
    "                p, h = get_premise_hypothesis(prompt)\n",
    "                consider = flute[(flute[\"premise\"]==p) & (flute[\"hypothesis\"]==h)].copy() \n",
    "\n",
    "                consider['Generated'] = pred\n",
    "                consider['Model'] = model\n",
    "                consider['Task'] = \"flute\"\n",
    "\n",
    "                if len(consider)>0:\n",
    "                    flute_test.append(consider.iloc[0:1])\n",
    "                else:\n",
    "                    print(\"\\tno match\", p, '\\t', h)\n",
    "            else:\n",
    "                print(\"Misaligned\", model)\n",
    "                break\n",
    "    if len(flute_test):\n",
    "        hit_df = pd.concat(flute_test)[['premise', 'hypothesis', 'labels', 'additional_labels', 'Generated', 'Model', 'Task']].sample(frac=1, random_state=7).copy()\n",
    "        hit_df.to_csv(f'hit/input/flute/flute_{model}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0d0fa725",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('css_data/flute/prompts.json-explanation-text-davinci-003', 'r') as f: \n",
    "    dv3 = json.load(f)\n",
    "df = pd.read_csv(\"hit/input/flute/flute_text-davinci-003.csv\")\n",
    "#df[\"Gen_DV3\"] = df[\"Generated\"]\n",
    "df[\"Generated\"] = [None for x in df.Generated.values]\n",
    "df[\"Model\"] = \"chatgpt\"\n",
    "rerun = pd.read_csv('css_data/flute/answer-explanation-chatgpt-rerun', sep='\\t', names=[\"idx\", \"Generated\"])\n",
    "for _, row in rerun.iterrows():\n",
    "    prompt = dv3[str(row[\"idx\"])]\n",
    "    p, h = get_premise_hypothesis(prompt)\n",
    "    idx = df[(df[\"premise\"]==p) & (df[\"hypothesis\"]==h)]\n",
    "    if len(idx):\n",
    "        df.loc[idx.index[0]][\"Generated\"] = row[\"Generated\"]\n",
    "df[~df[\"Generated\"].isna()].to_csv(\"hit/input/flute/flute_chatgpt.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d32156ce",
   "metadata": {},
   "source": [
    "## Positive Reframing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "668d99cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os, re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4a3e765d",
   "metadata": {},
   "outputs": [],
   "source": [
    "prf = pd.read_csv('css_data/positive_reframing/positive_reframing.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1456597c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>reframed_text</th>\n",
       "      <th>strategy</th>\n",
       "      <th>original_with_label</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Always stressing and thinking about loads of t...</td>\n",
       "      <td>Loads of things on my mind, I need to make a l...</td>\n",
       "      <td>['growth', 'neutralizing']</td>\n",
       "      <td>Always stressing and thinking about loads of t...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The list of things I have to do today is forev...</td>\n",
       "      <td>Today I have a lot to do. Time for productivit...</td>\n",
       "      <td>['growth', 'optimism']</td>\n",
       "      <td>The list of things I have to do today is forev...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>If I don't have a mental breakdown before the ...</td>\n",
       "      <td>I'm going to look after my mental health over ...</td>\n",
       "      <td>['growth', 'neutralizing', 'optimism']</td>\n",
       "      <td>If I don't have a mental breakdown before the ...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I just caught myself taking extra deep breaths...</td>\n",
       "      <td>Trying to keep breathing. Extra deep</td>\n",
       "      <td>['neutralizing']</td>\n",
       "      <td>I just caught myself taking extra deep breaths...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Worst night ever. Just one bad thing after ano...</td>\n",
       "      <td>I had a tough night but it's over and the holi...</td>\n",
       "      <td>['impermanence']</td>\n",
       "      <td>Worst night ever. Just one bad thing after ano...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8344</th>\n",
       "      <td>Anybody would think I'm 13 again and going thr...</td>\n",
       "      <td>People would think I am going through puberty ...</td>\n",
       "      <td>['neutralizing', 'optimism']</td>\n",
       "      <td>Anybody would think I'm 13 again and going thr...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8345</th>\n",
       "      <td>How come it seems the spring semester is harde...</td>\n",
       "      <td>Even though spring semester seems harder than ...</td>\n",
       "      <td>['neutralizing', 'optimism']</td>\n",
       "      <td>How come it seems the spring semester is harde...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8346</th>\n",
       "      <td>The snow and ice is bringing out the crazy in ...</td>\n",
       "      <td>The snow and ice calm me. I don't see any bad ...</td>\n",
       "      <td>['neutralizing']</td>\n",
       "      <td>The snow and ice is bringing out the crazy in ...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8347</th>\n",
       "      <td>The amount of stuff I need to sort out before ...</td>\n",
       "      <td>I can't wait to finish settling my affairs and...</td>\n",
       "      <td>['impermanence']</td>\n",
       "      <td>The amount of stuff I need to sort out before ...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8348</th>\n",
       "      <td>Ok.lets talk about thiswhat is gonna happen wi...</td>\n",
       "      <td>Weather can not always be predicted and that's...</td>\n",
       "      <td>['impermanence']</td>\n",
       "      <td>Ok.lets talk about thiswhat is gonna happen wi...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8349 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          original_text  \\\n",
       "0     Always stressing and thinking about loads of t...   \n",
       "1     The list of things I have to do today is forev...   \n",
       "2     If I don't have a mental breakdown before the ...   \n",
       "3     I just caught myself taking extra deep breaths...   \n",
       "4     Worst night ever. Just one bad thing after ano...   \n",
       "...                                                 ...   \n",
       "8344  Anybody would think I'm 13 again and going thr...   \n",
       "8345  How come it seems the spring semester is harde...   \n",
       "8346  The snow and ice is bringing out the crazy in ...   \n",
       "8347  The amount of stuff I need to sort out before ...   \n",
       "8348  Ok.lets talk about thiswhat is gonna happen wi...   \n",
       "\n",
       "                                          reframed_text  \\\n",
       "0     Loads of things on my mind, I need to make a l...   \n",
       "1     Today I have a lot to do. Time for productivit...   \n",
       "2     I'm going to look after my mental health over ...   \n",
       "3                  Trying to keep breathing. Extra deep   \n",
       "4     I had a tough night but it's over and the holi...   \n",
       "...                                                 ...   \n",
       "8344  People would think I am going through puberty ...   \n",
       "8345  Even though spring semester seems harder than ...   \n",
       "8346  The snow and ice calm me. I don't see any bad ...   \n",
       "8347  I can't wait to finish settling my affairs and...   \n",
       "8348  Weather can not always be predicted and that's...   \n",
       "\n",
       "                                    strategy  \\\n",
       "0                 ['growth', 'neutralizing']   \n",
       "1                     ['growth', 'optimism']   \n",
       "2     ['growth', 'neutralizing', 'optimism']   \n",
       "3                           ['neutralizing']   \n",
       "4                           ['impermanence']   \n",
       "...                                      ...   \n",
       "8344            ['neutralizing', 'optimism']   \n",
       "8345            ['neutralizing', 'optimism']   \n",
       "8346                        ['neutralizing']   \n",
       "8347                        ['impermanence']   \n",
       "8348                        ['impermanence']   \n",
       "\n",
       "                                    original_with_label  split  \n",
       "0     Always stressing and thinking about loads of t...   test  \n",
       "1     The list of things I have to do today is forev...   test  \n",
       "2     If I don't have a mental breakdown before the ...   test  \n",
       "3     I just caught myself taking extra deep breaths...   test  \n",
       "4     Worst night ever. Just one bad thing after ano...   test  \n",
       "...                                                 ...    ...  \n",
       "8344  Anybody would think I'm 13 again and going thr...  train  \n",
       "8345  How come it seems the spring semester is harde...  train  \n",
       "8346  The snow and ice is bringing out the crazy in ...  train  \n",
       "8347  The amount of stuff I need to sort out before ...  train  \n",
       "8348  Ok.lets talk about thiswhat is gonna happen wi...  train  \n",
       "\n",
       "[8349 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3a705f82",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-ada-001\n",
      "Misaligned idx                                                             25\n",
      "reframed_text    Only halfway through the work day and I've com...\n",
      "Generated                                                      NaN\n",
      "Name: 25, dtype: object\n",
      "Misaligned idx                                                             27\n",
      "reframed_text    Essential workers are finding it hard at the m...\n",
      "Generated                                                      NaN\n",
      "Name: 27, dtype: object\n",
      "Misaligned idx                                                             49\n",
      "reframed_text    I'm so thankful to be offered a promotion at m...\n",
      "Generated                                                      NaN\n",
      "Name: 49, dtype: object\n",
      "Misaligned idx                                                             25\n",
      "reframed_text    Only halfway through the work day and I've com...\n",
      "Generated                                                      NaN\n",
      "Name: 63, dtype: object\n",
      "Misaligned idx                                                             27\n",
      "reframed_text    Essential workers are finding it hard at the m...\n",
      "Generated                                                      NaN\n",
      "Name: 64, dtype: object\n",
      "Misaligned idx                                                             49\n",
      "reframed_text    I'm so thankful to be offered a promotion at m...\n",
      "Generated                                                      NaN\n",
      "Name: 65, dtype: object\n",
      "Misaligned idx                                                             75\n",
      "reframed_text    Going to treat myself and have a relaxing time...\n",
      "Generated                                                      NaN\n",
      "Name: 78, dtype: object\n",
      "Misaligned idx                                                             76\n",
      "reframed_text    Even though it's hard to study in my room beca...\n",
      "Generated                                                      NaN\n",
      "Name: 79, dtype: object\n",
      "Misaligned idx                                                             25\n",
      "reframed_text    Only halfway through the work day and I've com...\n",
      "Generated                                                      NaN\n",
      "Name: 98, dtype: object\n",
      "Misaligned idx                                                             27\n",
      "reframed_text    Essential workers are finding it hard at the m...\n",
      "Generated                                                      NaN\n",
      "Name: 99, dtype: object\n",
      "Misaligned idx                                                             49\n",
      "reframed_text    I'm so thankful to be offered a promotion at m...\n",
      "Generated                                                      NaN\n",
      "Name: 100, dtype: object\n",
      "Misaligned idx                                                             75\n",
      "reframed_text    Going to treat myself and have a relaxing time...\n",
      "Generated                                                      NaN\n",
      "Name: 101, dtype: object\n",
      "Misaligned idx                                                             76\n",
      "reframed_text    Even though it's hard to study in my room beca...\n",
      "Generated                                                      NaN\n",
      "Name: 102, dtype: object\n",
      "Misaligned idx                                                             25\n",
      "reframed_text    Only halfway through the work day and I've com...\n",
      "Generated                                                      NaN\n",
      "Name: 103, dtype: object\n",
      "Misaligned idx                                                             27\n",
      "reframed_text    Essential workers are finding it hard at the m...\n",
      "Generated                                                      NaN\n",
      "Name: 104, dtype: object\n",
      "Misaligned idx                                                             49\n",
      "reframed_text    I'm so thankful to be offered a promotion at m...\n",
      "Generated                                                      NaN\n",
      "Name: 105, dtype: object\n",
      "Misaligned idx                                                             75\n",
      "reframed_text    Going to treat myself and have a relaxing time...\n",
      "Generated                                                      NaN\n",
      "Name: 106, dtype: object\n",
      "Misaligned idx                                                             76\n",
      "reframed_text    Even though it's hard to study in my room beca...\n",
      "Generated                                                      NaN\n",
      "Name: 107, dtype: object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Misaligned idx                                                             25\n",
      "reframed_text    Only halfway through the work day and I've com...\n",
      "Generated                                                      NaN\n",
      "Name: 159, dtype: object\n",
      "Misaligned idx                                                             27\n",
      "reframed_text    Essential workers are finding it hard at the m...\n",
      "Generated                                                      NaN\n",
      "Name: 160, dtype: object\n",
      "Misaligned idx                                                             49\n",
      "reframed_text    I'm so thankful to be offered a promotion at m...\n",
      "Generated                                                      NaN\n",
      "Name: 161, dtype: object\n",
      "Misaligned idx                                                             75\n",
      "reframed_text    Going to treat myself and have a relaxing time...\n",
      "Generated                                                      NaN\n",
      "Name: 162, dtype: object\n",
      "Misaligned idx                                                             76\n",
      "reframed_text    Even though it's hard to study in my room beca...\n",
      "Generated                                                      NaN\n",
      "Name: 163, dtype: object\n",
      "Misaligned idx                                                            146\n",
      "reframed_text    It has been a year after closing my aunty, and...\n",
      "Generated                                                      NaN\n",
      "Name: 164, dtype: object\n",
      "Misaligned idx                                                            177\n",
      "reframed_text    I'm pretty sure I will get to take the bus to ...\n",
      "Generated                                                      NaN\n",
      "Name: 195, dtype: object\n",
      "Misaligned idx                                                            233\n",
      "reframed_text    Even though I've been on hold trying to get in...\n",
      "Generated                                                      NaN\n",
      "Name: 251, dtype: object\n",
      "Misaligned idx                                                            235\n",
      "reframed_text    I'm glad I took exams to find out soon how to ...\n",
      "Generated                                                      NaN\n",
      "Name: 253, dtype: object\n",
      "Misaligned idx                                                            239\n",
      "reframed_text    Even though I'm stressed about school, I'm jus...\n",
      "Generated                                                      NaN\n",
      "Name: 257, dtype: object\n",
      "Misaligned idx                                                            264\n",
      "reframed_text    I have a lot of things to do, but I'm going to...\n",
      "Generated                                                      NaN\n",
      "Name: 282, dtype: object\n",
      "Misaligned idx                                                            305\n",
      "reframed_text    I worry too much about not losing my commitmen...\n",
      "Generated                                                      NaN\n",
      "Name: 323, dtype: object\n",
      "Misaligned idx                                                            379\n",
      "reframed_text    Even though I have to stay up late packing, I'...\n",
      "Generated                                                      NaN\n",
      "Name: 397, dtype: object\n",
      "Misaligned idx                                                            411\n",
      "reframed_text    Sharing time with my brother is valuable, even...\n",
      "Generated                                                      NaN\n",
      "Name: 429, dtype: object\n",
      "Misaligned idx                                                            422\n",
      "reframed_text    I'm tired and cold, but I'm determined to fini...\n",
      "Generated                                                      NaN\n",
      "Name: 440, dtype: object\n",
      "Misaligned idx                                                            424\n",
      "reframed_text    I put my shorts on backwards and the wrong key...\n",
      "Generated                                                      NaN\n",
      "Name: 442, dtype: object\n",
      "Misaligned idx                                                            440\n",
      "reframed_text    Been feeling quite the past few days have to d...\n",
      "Generated                                                      NaN\n",
      "Name: 458, dtype: object\n",
      "text-babbage-001\n",
      "text-davinci-003\n",
      "flan-t5-large\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n",
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n",
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-small\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-xl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-xxl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-curie-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-davinci-002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-ul2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flan-t5-base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chatgpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text-davinci-001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_613/1881835767.py:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n"
     ]
    }
   ],
   "source": [
    "for fn in glob(\"css_data/positive_reframing/answer-*\"):\n",
    "    model = \"-\".join(fn.split(\"-\")[1:])\n",
    "    print(model)\n",
    "    with open(f\"css_data/positive_reframing/prompts.json-{model}\", \"r\") as f:\n",
    "        prompts = json.load(f)\n",
    "        \n",
    "    df = pd.read_csv(fn, sep='\\t', names=['idx', 'reframed_text', 'Generated'], error_bad_lines=False)\n",
    "\n",
    "    reframing_test = []\n",
    "    for _, row in df.iterrows():\n",
    "        \n",
    "        if type(row[\"Generated\"])==str:\n",
    "            pred = row[\"Generated\"].replace(\"&\", \"\")\n",
    "            \n",
    "            lbl = row[\"reframed_text\"]\n",
    "            consider = prf[prf[\"reframed_text\"]==lbl].copy()\n",
    "            \n",
    "            consider['Generated'] = pred\n",
    "            consider['Model'] = model\n",
    "            consider['Task'] = \"positive_reframing\"\n",
    "            \n",
    "            if len(consider)>0:\n",
    "                reframing_test.append(consider.iloc[0:1])\n",
    "            else:\n",
    "                print(\"\\tno match\", lbl)\n",
    "        else:\n",
    "            print(\"Misaligned\", row)\n",
    "    if len(reframing_test):\n",
    "        hit_df = pd.concat(reframing_test)[['original_text', 'reframed_text', \"strategy\", 'Generated', 'Model', 'Task']].sample(frac=1, random_state=7).copy()\n",
    "        hit_df.to_csv(f'hit/input/positive_reframing/positive_reframing_{model}.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f6f7f25",
   "metadata": {},
   "source": [
    "## Combine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9896be69",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os, re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "from ast import literal_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fb544e9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def build_comparison_rows(df, txt=\"post\", txt2=None, gold=\"targetStereotype\", secondary=\"targetMinority\", \n",
    "#                           n=4, literal=True, random_state=7):\n",
    "    \n",
    "#     gold_df = {}\n",
    "#     g = df.iloc[0][gold]\n",
    "    \n",
    "#     if literal:\n",
    "#         g = literal_eval(g)\n",
    "#         for i, t in enumerate(g):\n",
    "#             gold_df[i] = {\n",
    "#                 txt: df.iloc[0][txt],\n",
    "#                 secondary: df.iloc[0][secondary],\n",
    "#                 \"Generated\": t,\n",
    "#                 \"Model\": \"human\",\n",
    "#                 \"Task\": df.iloc[0][\"Task\"]\n",
    "#             }\n",
    "#             if txt2:\n",
    "#                 gold_df[i][txt2] = df.iloc[0][txt2]\n",
    "#     else:\n",
    "#         gold_df = {0: {\n",
    "#                 txt: df.iloc[0][txt],\n",
    "#                 secondary: df.iloc[0][secondary],\n",
    "#                 \"Generated\": g,\n",
    "#                 \"Model\": \"human\",\n",
    "#                 \"Task\": df.iloc[0][\"Task\"]   \n",
    "#         }}\n",
    "#         if txt2:\n",
    "#             gold_df[0][txt2] = df.iloc[0][txt2]\n",
    "#     gold_df = pd.DataFrame().from_dict(gold_df).T\n",
    "#     comb = pd.concat([gold_df, df])\n",
    "    \n",
    "#     rand = comb.sample(frac=1, random_state=random_state)\n",
    "#     rows = {}\n",
    "#     for i in range(0, len(rand)-4, n):\n",
    "#         row = {}\n",
    "#         sub = rand.iloc[i:i+4]\n",
    "#         j = 1\n",
    "#         for _, r in sub.iterrows():\n",
    "#             row[txt] = r[txt]\n",
    "#             if txt2:\n",
    "#                 row[txt2] = r[txt2]\n",
    "#             row[secondary] = r[secondary]\n",
    "#             row[f\"Generated_{j}\"] = r[\"Generated\"]\n",
    "#             row[f\"Model_{j}\"] = r[\"Model\"]\n",
    "#             row[\"Task\"] = r[\"Task\"]\n",
    "#             j+=1\n",
    "#         row[gold] = df.iloc[0][gold]\n",
    "#         rows[i] = row\n",
    "        \n",
    "#     return pd.DataFrame().from_dict(rows).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b8f07c49",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_comparison_rows(df, txt=\"post\", txt2=None, gold=\"targetStereotype\", secondary=\"targetMinority\", \n",
    "                          n=4, literal=True, random_state=7,\n",
    "                          MODELSET={\"human\", \"baseline\", \"text-davinci-003\", \"chatgpt\"}\n",
    "                         ):\n",
    "    \n",
    "    gold_df = {}\n",
    "    g = df.iloc[0][gold]\n",
    "    \n",
    "    if literal:\n",
    "        g = literal_eval(g)\n",
    "        for i, t in enumerate(g):\n",
    "            gold_df[i] = {\n",
    "                txt: df.iloc[0][txt],\n",
    "                secondary: df.iloc[0][secondary],\n",
    "                \"Generated\": t,\n",
    "                \"Model\": \"human\",\n",
    "                \"Task\": df.iloc[0][\"Task\"]\n",
    "            }\n",
    "            if txt2:\n",
    "                gold_df[i][txt2] = df.iloc[0][txt2]\n",
    "            break\n",
    "    else:\n",
    "        gold_df = {0: {\n",
    "                txt: df.iloc[0][txt],\n",
    "                secondary: df.iloc[0][secondary],\n",
    "                \"Generated\": g,\n",
    "                \"Model\": \"human\",\n",
    "                \"Task\": df.iloc[0][\"Task\"]   \n",
    "        }}\n",
    "        if txt2:\n",
    "            gold_df[0][txt2] = df.iloc[0][txt2]\n",
    "    gold_df = pd.DataFrame().from_dict(gold_df).T\n",
    "    comb = pd.concat([gold_df, df])\n",
    "    \n",
    "    #print(set(comb.Model.values))\n",
    "    #comb_alt = comb[[model in {\"baseline\", \"flan-ul2\", \"text-davinci-003\", \"chatgpt\"} for model in comb.Model.values]]\n",
    "    comb = comb[[model in MODELSET for model in comb.Model.values]]\n",
    "    \n",
    "    \n",
    "    rand = comb.sample(frac=1)\n",
    "    row = {}\n",
    "    for j in range(0, n):\n",
    "        try:\n",
    "            r = rand.iloc[j]\n",
    "        except:\n",
    "            print(j, \"out of position\")\n",
    "            #continue\n",
    "            return pd.DataFrame()\n",
    "        \n",
    "        row[txt] = r[txt]\n",
    "        if txt2:\n",
    "            row[txt2] = r[txt2]\n",
    "        row[secondary] = r[secondary]\n",
    "        row[f\"Generated_{j+1}\"] = r[\"Generated\"]\n",
    "        row[f\"Model_{j+1}\"] = r[\"Model\"]\n",
    "        row[\"Task\"] = r[\"Task\"]\n",
    "        row[gold] = df.iloc[0][gold]\n",
    "    \n",
    "    return pd.DataFrame().from_dict({0: row}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "04bb06fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "TASK = 'sbic'\n",
    "txt=\"post\"\n",
    "gold=\"targetStereotype\"\n",
    "secondary=\"targetMinority\"\n",
    "n=4\n",
    "literal=True\n",
    "random_state=7\n",
    "\n",
    "comb = []\n",
    "for fn in glob(f\"hit/input/{TASK}/*.csv\"):\n",
    "    comb.append(pd.read_csv(fn))\n",
    "df = pd.concat(comb)\n",
    "\n",
    "comparison_rows = []\n",
    "for lbl in set(df[txt]):\n",
    "    consider = df[df[txt]==lbl].copy()\n",
    "    comparison_rows.append(build_comparison_rows(consider,\n",
    "                                                 txt=txt,\n",
    "                                                 gold=gold,\n",
    "                                                 secondary=secondary,\n",
    "                                                 n=n,\n",
    "                                                 literal=literal,\n",
    "                                                 random_state=random_state,\n",
    "                                                 MODELSET={\"human\", \"text-curie-001\", \"text-babbage-001\", \"text-ada-001\"}\n",
    "                                                ))\n",
    "    \n",
    "comparison_df = pd.concat(comparison_rows)\n",
    "comparison_df.sample(n=66, random_state=random_state).to_csv(f\"hit/input/comparison_balanced_{TASK}_ada_babbage_curie.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9cb59fc7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Generated_1</th>\n",
       "      <th>Generated_2</th>\n",
       "      <th>Generated_3</th>\n",
       "      <th>Generated_4</th>\n",
       "      <th>Model_1</th>\n",
       "      <th>Model_2</th>\n",
       "      <th>Model_3</th>\n",
       "      <th>Model_4</th>\n",
       "      <th>Task</th>\n",
       "      <th>post</th>\n",
       "      <th>targetMinority</th>\n",
       "      <th>targetStereotype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>Pizza is a type of food that is made from doug...</td>\n",
       "      <td>jewish are boring people</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>human</td>\n",
       "      <td>sbic</td>\n",
       "      <td>What's the difference between pizza and Jews? ...</td>\n",
       "      <td>[\"jewish folks\"]</td>\n",
       "      <td>[\"jewish are boring people\", \"jewish people ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The group is going to die</td>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>indians are dirty</td>\n",
       "      <td>The train is heading towards the individual</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>Aan indian was taking a dump and died. Strange...</td>\n",
       "      <td>[\"India\"]</td>\n",
       "      <td>[\"indians are dirty\", \"indians have bad hygiene\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>makes light of holocaust</td>\n",
       "      <td>The difference between a Boy Scout and a Jew i...</td>\n",
       "      <td>A Boy Scout comes back from backpacking</td>\n",
       "      <td>A Jew is someone who is from a certain religio...</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>What’s the difference between a Jew and a Boy ...</td>\n",
       "      <td>[\"jewish folks\"]</td>\n",
       "      <td>[\"makes light of holocaust\", \"refers to holoca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>The sentence communicates a stereotype that Mu...</td>\n",
       "      <td>muslim folks are subhuman/objects or animals</td>\n",
       "      <td>Muslims are typically seen as being religious ...</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>human</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>How do you dissect a Muslim? Pretty sure the b...</td>\n",
       "      <td>[\"muslim folks\"]</td>\n",
       "      <td>[\"muslim folks are subhuman/objects or animals...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>are not equal</td>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>This sentence communicates that the people who...</td>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>human</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>I think its wrong to refer to a negro as \"boy\"...</td>\n",
       "      <td>[\"black folks\"]</td>\n",
       "      <td>[\"are not equal\", \"are not human\", \"blacks are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A black man left hanging is a lynching victim</td>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>lynching victim</td>\n",
       "      <td>allusion to lynching</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>human</td>\n",
       "      <td>sbic</td>\n",
       "      <td>What do you call a black man left hanging? a l...</td>\n",
       "      <td>[\"black folks\", \"murder victims\"]</td>\n",
       "      <td>[\"allusion to lynching\", \"black folks are infe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>african folks don't deserve to eat</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>human</td>\n",
       "      <td>sbic</td>\n",
       "      <td>we should not help poor african children\\nthey...</td>\n",
       "      <td>[\"African folks\"]</td>\n",
       "      <td>[\"african folks don't deserve to eat\", \"africa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The stereotype communicated by \"nigger mortis\"...</td>\n",
       "      <td>black people are criminals</td>\n",
       "      <td>The stereotype that black people die from natu...</td>\n",
       "      <td>Nigger mortis is the process of getting into a...</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>human</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>What happens when a black person dies? Nigger ...</td>\n",
       "      <td>[\"black folks\"]</td>\n",
       "      <td>[\"black people are criminals\", \"black people a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>refers to women as bitches</td>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>Bitches really be mad when u can just stop spe...</td>\n",
       "      <td>[\"women\"]</td>\n",
       "      <td>[\"refers to women as bitches\", \"women are clin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jewish folks got gassed</td>\n",
       "      <td>The Jews were so gassed during World War II th...</td>\n",
       "      <td>The Jews were so gassed that they no longer ex...</td>\n",
       "      <td>The Jews were stereotype that they would react...</td>\n",
       "      <td>human</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>How did the Jews react when WW II was over? Th...</td>\n",
       "      <td>[\"jewish folks\"]</td>\n",
       "      <td>[\"jewish folks got gassed\", \"jewish people jok...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>488 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Generated_1  \\\n",
       "0   The stereotype that this sentence communicates...   \n",
       "0                           The group is going to die   \n",
       "0                            makes light of holocaust   \n",
       "0   The stereotype communicated by this sentence i...   \n",
       "0                                       are not equal   \n",
       "..                                                ...   \n",
       "0       A black man left hanging is a lynching victim   \n",
       "0   The stereotype communicated by this sentence i...   \n",
       "0   The stereotype communicated by \"nigger mortis\"...   \n",
       "0                          refers to women as bitches   \n",
       "0                             jewish folks got gassed   \n",
       "\n",
       "                                          Generated_2  \\\n",
       "0   The stereotype communicated by this sentence i...   \n",
       "0   The stereotype communicated by this sentence i...   \n",
       "0   The difference between a Boy Scout and a Jew i...   \n",
       "0   The sentence communicates a stereotype that Mu...   \n",
       "0   The stereotype that this sentence communicates...   \n",
       "..                                                ...   \n",
       "0   The stereotype communicated by this sentence i...   \n",
       "0   The stereotype that this sentence communicates...   \n",
       "0                          black people are criminals   \n",
       "0   The stereotype that this sentence communicates...   \n",
       "0   The Jews were so gassed during World War II th...   \n",
       "\n",
       "                                          Generated_3  \\\n",
       "0   Pizza is a type of food that is made from doug...   \n",
       "0                                   indians are dirty   \n",
       "0             A Boy Scout comes back from backpacking   \n",
       "0        muslim folks are subhuman/objects or animals   \n",
       "0   This sentence communicates that the people who...   \n",
       "..                                                ...   \n",
       "0                                     lynching victim   \n",
       "0   The stereotype that this sentence communicates...   \n",
       "0   The stereotype that black people die from natu...   \n",
       "0   The stereotype communicated by this sentence i...   \n",
       "0   The Jews were so gassed that they no longer ex...   \n",
       "\n",
       "                                          Generated_4           Model_1  \\\n",
       "0                            jewish are boring people      text-ada-001   \n",
       "0         The train is heading towards the individual  text-babbage-001   \n",
       "0   A Jew is someone who is from a certain religio...             human   \n",
       "0   Muslims are typically seen as being religious ...    text-curie-001   \n",
       "0   The stereotype communicated by this sentence i...             human   \n",
       "..                                                ...               ...   \n",
       "0                                allusion to lynching      text-ada-001   \n",
       "0                  african folks don't deserve to eat    text-curie-001   \n",
       "0   Nigger mortis is the process of getting into a...    text-curie-001   \n",
       "0   The stereotype that this sentence communicates...             human   \n",
       "0   The Jews were stereotype that they would react...             human   \n",
       "\n",
       "             Model_2           Model_3           Model_4  Task  \\\n",
       "0     text-curie-001  text-babbage-001             human  sbic   \n",
       "0     text-curie-001             human      text-ada-001  sbic   \n",
       "0       text-ada-001    text-curie-001  text-babbage-001  sbic   \n",
       "0       text-ada-001             human  text-babbage-001  sbic   \n",
       "0   text-babbage-001      text-ada-001    text-curie-001  sbic   \n",
       "..               ...               ...               ...   ...   \n",
       "0     text-curie-001  text-babbage-001             human  sbic   \n",
       "0   text-babbage-001      text-ada-001             human  sbic   \n",
       "0              human  text-babbage-001      text-ada-001  sbic   \n",
       "0       text-ada-001    text-curie-001  text-babbage-001  sbic   \n",
       "0   text-babbage-001    text-curie-001      text-ada-001  sbic   \n",
       "\n",
       "                                                 post  \\\n",
       "0   What's the difference between pizza and Jews? ...   \n",
       "0   Aan indian was taking a dump and died. Strange...   \n",
       "0   What’s the difference between a Jew and a Boy ...   \n",
       "0   How do you dissect a Muslim? Pretty sure the b...   \n",
       "0   I think its wrong to refer to a negro as \"boy\"...   \n",
       "..                                                ...   \n",
       "0   What do you call a black man left hanging? a l...   \n",
       "0   we should not help poor african children\\nthey...   \n",
       "0   What happens when a black person dies? Nigger ...   \n",
       "0   Bitches really be mad when u can just stop spe...   \n",
       "0   How did the Jews react when WW II was over? Th...   \n",
       "\n",
       "                       targetMinority  \\\n",
       "0                    [\"jewish folks\"]   \n",
       "0                           [\"India\"]   \n",
       "0                    [\"jewish folks\"]   \n",
       "0                    [\"muslim folks\"]   \n",
       "0                     [\"black folks\"]   \n",
       "..                                ...   \n",
       "0   [\"black folks\", \"murder victims\"]   \n",
       "0                   [\"African folks\"]   \n",
       "0                     [\"black folks\"]   \n",
       "0                           [\"women\"]   \n",
       "0                    [\"jewish folks\"]   \n",
       "\n",
       "                                     targetStereotype  \n",
       "0   [\"jewish are boring people\", \"jewish people ar...  \n",
       "0   [\"indians are dirty\", \"indians have bad hygiene\"]  \n",
       "0   [\"makes light of holocaust\", \"refers to holoca...  \n",
       "0   [\"muslim folks are subhuman/objects or animals...  \n",
       "0   [\"are not equal\", \"are not human\", \"blacks are...  \n",
       "..                                                ...  \n",
       "0   [\"allusion to lynching\", \"black folks are infe...  \n",
       "0   [\"african folks don't deserve to eat\", \"africa...  \n",
       "0   [\"black people are criminals\", \"black people a...  \n",
       "0   [\"refers to women as bitches\", \"women are clin...  \n",
       "0   [\"jewish folks got gassed\", \"jewish people jok...  \n",
       "\n",
       "[488 rows x 12 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "75e33ce5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "3 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n"
     ]
    }
   ],
   "source": [
    "TASK = 'mrf'\n",
    "txt=\"headline\"\n",
    "gold=\"writer_intent\"\n",
    "secondary=\"misinfo\"\n",
    "n=4\n",
    "literal=True\n",
    "random_state=7\n",
    "\n",
    "comb = []\n",
    "for fn in glob(f\"hit/input/{TASK}/*.csv\"):\n",
    "    comb.append(pd.read_csv(fn))\n",
    "df = pd.concat(comb)\n",
    "\n",
    "comparison_rows = []\n",
    "for lbl in set(df[txt]):\n",
    "    consider = df[df[txt]==lbl].copy()\n",
    "    cr = build_comparison_rows(consider,\n",
    "                                 txt=txt,\n",
    "                                 gold=gold,\n",
    "                                 secondary=secondary,\n",
    "                                 n=n,\n",
    "                                 literal=literal,\n",
    "                                 random_state=random_state,\n",
    "                                 MODELSET={\"human\", \"baseline\", \"text-davinci-002\", \"text-davinci-001\"}\n",
    "                                )\n",
    "    if len(cr):\n",
    "        comparison_rows.append(cr)\n",
    "    \n",
    "comparison_df = pd.concat(comparison_rows)\n",
    "comparison_df.sample(n=66, random_state=random_state).to_csv(f\"hit/input/comparison_balanced_{TASK}_davinci_001_002.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4ba1310d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Generated_1</th>\n",
       "      <th>Generated_2</th>\n",
       "      <th>Generated_3</th>\n",
       "      <th>Generated_4</th>\n",
       "      <th>Model_1</th>\n",
       "      <th>Model_2</th>\n",
       "      <th>Model_3</th>\n",
       "      <th>Model_4</th>\n",
       "      <th>Task</th>\n",
       "      <th>headline</th>\n",
       "      <th>misinfo</th>\n",
       "      <th>writer_intent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>'climate change poses the most profound long-t...</td>\n",
       "      <td>Climate change is a more pressing issue than t...</td>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>ceo of uk's national health service says covid...</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>human</td>\n",
       "      <td>mrf</td>\n",
       "      <td>CEO Of UK's National Health Service: COVID-19 ...</td>\n",
       "      <td>misinfo</td>\n",
       "      <td>[\"ceo of uk's national health service says cov...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The implied message of the headline is that th...</td>\n",
       "      <td>The UK minister's first visit to Russia since ...</td>\n",
       "      <td>'nassau and climate change on agenda in first ...</td>\n",
       "      <td>government officials are not on the same page</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>baseline</td>\n",
       "      <td>human</td>\n",
       "      <td>mrf</td>\n",
       "      <td>Navalny and climate on agenda in first Russia ...</td>\n",
       "      <td>real</td>\n",
       "      <td>['government officials are not on the same pag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>International shipping is harming the environment</td>\n",
       "      <td>Some masks are better than others for climate</td>\n",
       "      <td>consumerism is causing climate change</td>\n",
       "      <td>'international shipping is killing the climate'</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>human</td>\n",
       "      <td>baseline</td>\n",
       "      <td>mrf</td>\n",
       "      <td>International shipping is killing the climate</td>\n",
       "      <td>real</td>\n",
       "      <td>['consumerism is causing climate change']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vaccine was deadly during trial</td>\n",
       "      <td>The vaccine is not safe</td>\n",
       "      <td>'vaccines are dangerous'</td>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>human</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>mrf</td>\n",
       "      <td>A doctor who was volunteering in clinical test...</td>\n",
       "      <td>misinfo</td>\n",
       "      <td>['vaccine was deadly during trial', 'people sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>The only way to end Australia's climate crimin...</td>\n",
       "      <td>only global tariffs will help australia on cli...</td>\n",
       "      <td>'the world is going to have to pay more'</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>baseline</td>\n",
       "      <td>mrf</td>\n",
       "      <td>Global tariffs the only way to end Australia's...</td>\n",
       "      <td>real</td>\n",
       "      <td>['only global tariffs will help australia on c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Covid-19 is serious and can lead to multiple h...</td>\n",
       "      <td>'he was hospitalized for covid-19.'</td>\n",
       "      <td>people can get covid and repeatedly go into th...</td>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>baseline</td>\n",
       "      <td>human</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>mrf</td>\n",
       "      <td>He Was Hospitalized for Covid-19. Then Hospita...</td>\n",
       "      <td>real</td>\n",
       "      <td>['people can get covid and repeatedly go into ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The first winners of the $10 billion climate-c...</td>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>a few environmental groups received large gran...</td>\n",
       "      <td>'the bezos administration is announcing the fi...</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>human</td>\n",
       "      <td>baseline</td>\n",
       "      <td>mrf</td>\n",
       "      <td>Bezos announces first winners of $10 billion c...</td>\n",
       "      <td>real</td>\n",
       "      <td>['a few environmental groups received large gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the uk will be shutting down again because of ...</td>\n",
       "      <td>'the government is preparing for a second lock...</td>\n",
       "      <td>The UK is preparing for a second lockdown, imp...</td>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>human</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>mrf</td>\n",
       "      <td>UK Coronavirus LIVE: England prepares for seco...</td>\n",
       "      <td>real</td>\n",
       "      <td>['the uk will be shutting down again because o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The vaccine may have negative side effects for...</td>\n",
       "      <td>'covid vaccine is female sterilization'</td>\n",
       "      <td>the covid-19 vaccine causes female sterilization</td>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>baseline</td>\n",
       "      <td>human</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>mrf</td>\n",
       "      <td>Head of Pfizer research said that covid vaccin...</td>\n",
       "      <td>misinfo</td>\n",
       "      <td>['the covid-19 vaccine causes female steriliza...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The vaccine may not be effective</td>\n",
       "      <td>Some masks are better than others</td>\n",
       "      <td>vaccine trials are not safe</td>\n",
       "      <td>'coronavac's clinical trials are stopped in Peru'</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>human</td>\n",
       "      <td>baseline</td>\n",
       "      <td>mrf</td>\n",
       "      <td>CoronaVac's clinical trials are stopped in Peru</td>\n",
       "      <td>misinfo</td>\n",
       "      <td>['vaccine trials are not safe', 'the vaccine i...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>478 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Generated_1  \\\n",
       "0   'climate change poses the most profound long-t...   \n",
       "0   The implied message of the headline is that th...   \n",
       "0   International shipping is harming the environment   \n",
       "0                     vaccine was deadly during trial   \n",
       "0                   Some masks are better than others   \n",
       "..                                                ...   \n",
       "0   Covid-19 is serious and can lead to multiple h...   \n",
       "0   The first winners of the $10 billion climate-c...   \n",
       "0   the uk will be shutting down again because of ...   \n",
       "0   The vaccine may have negative side effects for...   \n",
       "0                    The vaccine may not be effective   \n",
       "\n",
       "                                          Generated_2  \\\n",
       "0   Climate change is a more pressing issue than t...   \n",
       "0   The UK minister's first visit to Russia since ...   \n",
       "0       Some masks are better than others for climate   \n",
       "0                             The vaccine is not safe   \n",
       "0   The only way to end Australia's climate crimin...   \n",
       "..                                                ...   \n",
       "0                 'he was hospitalized for covid-19.'   \n",
       "0                   Some masks are better than others   \n",
       "0   'the government is preparing for a second lock...   \n",
       "0             'covid vaccine is female sterilization'   \n",
       "0                   Some masks are better than others   \n",
       "\n",
       "                                          Generated_3  \\\n",
       "0                   Some masks are better than others   \n",
       "0   'nassau and climate change on agenda in first ...   \n",
       "0               consumerism is causing climate change   \n",
       "0                            'vaccines are dangerous'   \n",
       "0   only global tariffs will help australia on cli...   \n",
       "..                                                ...   \n",
       "0   people can get covid and repeatedly go into th...   \n",
       "0   a few environmental groups received large gran...   \n",
       "0   The UK is preparing for a second lockdown, imp...   \n",
       "0    the covid-19 vaccine causes female sterilization   \n",
       "0                         vaccine trials are not safe   \n",
       "\n",
       "                                          Generated_4           Model_1  \\\n",
       "0   ceo of uk's national health service says covid...          baseline   \n",
       "0       government officials are not on the same page  text-davinci-001   \n",
       "0     'international shipping is killing the climate'  text-davinci-002   \n",
       "0                   Some masks are better than others             human   \n",
       "0            'the world is going to have to pay more'  text-davinci-001   \n",
       "..                                                ...               ...   \n",
       "0                   Some masks are better than others  text-davinci-002   \n",
       "0   'the bezos administration is announcing the fi...  text-davinci-002   \n",
       "0                   Some masks are better than others             human   \n",
       "0                   Some masks are better than others  text-davinci-002   \n",
       "0   'coronavac's clinical trials are stopped in Peru'  text-davinci-002   \n",
       "\n",
       "             Model_2           Model_3           Model_4 Task  \\\n",
       "0   text-davinci-002  text-davinci-001             human  mrf   \n",
       "0   text-davinci-002          baseline             human  mrf   \n",
       "0   text-davinci-001             human          baseline  mrf   \n",
       "0   text-davinci-002          baseline  text-davinci-001  mrf   \n",
       "0   text-davinci-002             human          baseline  mrf   \n",
       "..               ...               ...               ...  ...   \n",
       "0           baseline             human  text-davinci-001  mrf   \n",
       "0   text-davinci-001             human          baseline  mrf   \n",
       "0           baseline  text-davinci-002  text-davinci-001  mrf   \n",
       "0           baseline             human  text-davinci-001  mrf   \n",
       "0   text-davinci-001             human          baseline  mrf   \n",
       "\n",
       "                                             headline  misinfo  \\\n",
       "0   CEO Of UK's National Health Service: COVID-19 ...  misinfo   \n",
       "0   Navalny and climate on agenda in first Russia ...     real   \n",
       "0       International shipping is killing the climate     real   \n",
       "0   A doctor who was volunteering in clinical test...  misinfo   \n",
       "0   Global tariffs the only way to end Australia's...     real   \n",
       "..                                                ...      ...   \n",
       "0   He Was Hospitalized for Covid-19. Then Hospita...     real   \n",
       "0   Bezos announces first winners of $10 billion c...     real   \n",
       "0   UK Coronavirus LIVE: England prepares for seco...     real   \n",
       "0   Head of Pfizer research said that covid vaccin...  misinfo   \n",
       "0     CoronaVac's clinical trials are stopped in Peru  misinfo   \n",
       "\n",
       "                                        writer_intent  \n",
       "0   [\"ceo of uk's national health service says cov...  \n",
       "0   ['government officials are not on the same pag...  \n",
       "0           ['consumerism is causing climate change']  \n",
       "0   ['vaccine was deadly during trial', 'people sh...  \n",
       "0   ['only global tariffs will help australia on c...  \n",
       "..                                                ...  \n",
       "0   ['people can get covid and repeatedly go into ...  \n",
       "0   ['a few environmental groups received large gr...  \n",
       "0   ['the uk will be shutting down again because o...  \n",
       "0   ['the covid-19 vaccine causes female steriliza...  \n",
       "0   ['vaccine trials are not safe', 'the vaccine i...  \n",
       "\n",
       "[478 rows x 12 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f5c840fd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n",
      "1 out of position\n"
     ]
    }
   ],
   "source": [
    "TASK = 'flute'\n",
    "txt=\"premise\"\n",
    "txt2=\"hypothesis\"\n",
    "gold=\"additional_labels\"\n",
    "secondary=\"labels\"\n",
    "n=4\n",
    "literal=False\n",
    "random_state=7\n",
    "\n",
    "comb = []\n",
    "for fn in glob(f\"hit/input/{TASK}/*.csv\"):\n",
    "    if 'flan' in fn:\n",
    "        continue\n",
    "    comb.append(pd.read_csv(fn))\n",
    "df = pd.concat(comb)\n",
    "\n",
    "comparison_rows = []\n",
    "phs = set([(row['premise'],row['hypothesis']) for _, row in df.iterrows()])\n",
    "for p,h in phs:\n",
    "    consider = df[(df['premise']==p) & (df['hypothesis']==h)].copy()\n",
    "    cr = build_comparison_rows(consider,\n",
    "                                 txt=txt,\n",
    "                                 txt2=txt2,\n",
    "                                 gold=gold,\n",
    "                                 secondary=secondary,\n",
    "                                 n=n,\n",
    "                                 literal=literal,\n",
    "                                 random_state=random_state,\n",
    "                                MODELSET={\"human\", \"text-curie-001\", \"text-babbage-001\", \"text-ada-001\"}\n",
    "                                )\n",
    "    if len(cr):\n",
    "        comparison_rows.append(cr)\n",
    "    \n",
    "comparison_df = pd.concat(comparison_rows)\n",
    "comparison_df.sample(frac=1, random_state=random_state).to_csv(f\"hit/input/comparison_balanced_{TASK}_ada_babbage_curie.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "109be0ed",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Generated_1</th>\n",
       "      <th>Generated_2</th>\n",
       "      <th>Generated_3</th>\n",
       "      <th>Generated_4</th>\n",
       "      <th>Model_1</th>\n",
       "      <th>Model_2</th>\n",
       "      <th>Model_3</th>\n",
       "      <th>Model_4</th>\n",
       "      <th>Task</th>\n",
       "      <th>additional_labels</th>\n",
       "      <th>hypothesis</th>\n",
       "      <th>labels</th>\n",
       "      <th>premise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A melting pat of butter is a slippery and slip...</td>\n",
       "      <td>The figurative language in the hypothesis is t...</td>\n",
       "      <td>The figurative language in the hypothesis is a...</td>\n",
       "      <td>A pat of butter melting would become lazy and ...</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>flute</td>\n",
       "      <td>A pat of butter melting would become lazy and ...</td>\n",
       "      <td>His voice was like a melting pat of butter.</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>His voice was lazy and slippery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The figurative language in the hypothesis is a...</td>\n",
       "      <td>The figurative language in the hypothesis is t...</td>\n",
       "      <td>To throw something at someone is to be sarcast...</td>\n",
       "      <td>Monkeys foodfighting with bananas is hilarious...</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>baseline</td>\n",
       "      <td>human</td>\n",
       "      <td>flute</td>\n",
       "      <td>Monkeys foodfighting with bananas is hilarious...</td>\n",
       "      <td>The shade and sarcasm they throw at each other...</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>The way they shade and sarcasm each other is h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A rare stone in a museum is carefully guarded ...</td>\n",
       "      <td>A rare stone is a very rare object and so bein...</td>\n",
       "      <td>The figurative language in the hypothesis is t...</td>\n",
       "      <td>The hypothesis is a metaphor meaning that the ...</td>\n",
       "      <td>human</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>flute</td>\n",
       "      <td>A rare stone in a museum is carefully guarded ...</td>\n",
       "      <td>She is like a rare stone in a museum surrounde...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>She is so easily accessible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A hundred million fireflies dancing around jus...</td>\n",
       "      <td>The figurative language in the hypothesis is t...</td>\n",
       "      <td>A hundred million fireflies dancing would be a...</td>\n",
       "      <td>The hypothesis is a metaphor for how beautiful...</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>human</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>flute</td>\n",
       "      <td>A hundred million fireflies dancing would be a...</td>\n",
       "      <td>Its like a hundred million fireflies dancing a...</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>It is beautiful</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The hypothesis states that the person felt \"te...</td>\n",
       "      <td>The hypothesis is saying that the person felt ...</td>\n",
       "      <td>Losing a job is a very stressful experience an...</td>\n",
       "      <td>Losing a job is often a very traumatic experie...</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>baseline</td>\n",
       "      <td>human</td>\n",
       "      <td>flute</td>\n",
       "      <td>Losing a job is often a very traumatic experie...</td>\n",
       "      <td>I felt terrible when I found out that I was go...</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>I felt pretty down when I found out that I was...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The hypothesis is a metaphor for how life is c...</td>\n",
       "      <td>An endless road is a long, long, and long road...</td>\n",
       "      <td>An endless road has no end, so it is not tempo...</td>\n",
       "      <td>The figurative language in the hypothesis is t...</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>baseline</td>\n",
       "      <td>human</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>An endless road has no end, so it is not tempo...</td>\n",
       "      <td>We're starting to discover life is like an end...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>We're starting to realize that life is fleetin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The hypothesis restates the premise using the ...</td>\n",
       "      <td>To make two people means to make them feel the...</td>\n",
       "      <td>The idiom \"that makes two of us\" means \"the sa...</td>\n",
       "      <td>The hypothesis is saying that the premise and ...</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>baseline</td>\n",
       "      <td>human</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>The idiom \"that makes two of us\" means \"the sa...</td>\n",
       "      <td>that makes two of us.</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>I feel exactly the same way.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jobs for the boys refers to giving paid employ...</td>\n",
       "      <td>The hypothesis is saying that giving paid empl...</td>\n",
       "      <td>Jobs for the boys are not paid jobs and so the...</td>\n",
       "      <td>The hypothesis is a restatement of the premise...</td>\n",
       "      <td>human</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>flute</td>\n",
       "      <td>Jobs for the boys refers to giving paid employ...</td>\n",
       "      <td>But this isn't a case of jobs for the boys.</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>But this isn't a case of giving paid employmen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>To be rich means to be rich, but in this conte...</td>\n",
       "      <td>The hypothesis is saying that the person who s...</td>\n",
       "      <td>The hypothesis contradicts the premise because...</td>\n",
       "      <td>Stealing anything, regardless of how small it ...</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>flute</td>\n",
       "      <td>Stealing anything, regardless of how small it ...</td>\n",
       "      <td>I went to the store and stole a small bunch of...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>I went to the store and took a small bunch of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>To put up with someone or something means to e...</td>\n",
       "      <td>To put up with something means to reject or pr...</td>\n",
       "      <td>The figurative language in the hypothesis is \"...</td>\n",
       "      <td>The hypothesis states that the protagonist did...</td>\n",
       "      <td>human</td>\n",
       "      <td>baseline</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>flute</td>\n",
       "      <td>To put up with someone or something means to e...</td>\n",
       "      <td>I didn't like him enough to put up with that s...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>I didn't like him enough to reject or protest ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>476 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Generated_1  \\\n",
       "0   A melting pat of butter is a slippery and slip...   \n",
       "0   The figurative language in the hypothesis is a...   \n",
       "0   A rare stone in a museum is carefully guarded ...   \n",
       "0   A hundred million fireflies dancing around jus...   \n",
       "0   The hypothesis states that the person felt \"te...   \n",
       "..                                                ...   \n",
       "0   The hypothesis is a metaphor for how life is c...   \n",
       "0   The hypothesis restates the premise using the ...   \n",
       "0   Jobs for the boys refers to giving paid employ...   \n",
       "0   To be rich means to be rich, but in this conte...   \n",
       "0   To put up with someone or something means to e...   \n",
       "\n",
       "                                          Generated_2  \\\n",
       "0   The figurative language in the hypothesis is t...   \n",
       "0   The figurative language in the hypothesis is t...   \n",
       "0   A rare stone is a very rare object and so bein...   \n",
       "0   The figurative language in the hypothesis is t...   \n",
       "0   The hypothesis is saying that the person felt ...   \n",
       "..                                                ...   \n",
       "0   An endless road is a long, long, and long road...   \n",
       "0   To make two people means to make them feel the...   \n",
       "0   The hypothesis is saying that giving paid empl...   \n",
       "0   The hypothesis is saying that the person who s...   \n",
       "0   To put up with something means to reject or pr...   \n",
       "\n",
       "                                          Generated_3  \\\n",
       "0   The figurative language in the hypothesis is a...   \n",
       "0   To throw something at someone is to be sarcast...   \n",
       "0   The figurative language in the hypothesis is t...   \n",
       "0   A hundred million fireflies dancing would be a...   \n",
       "0   Losing a job is a very stressful experience an...   \n",
       "..                                                ...   \n",
       "0   An endless road has no end, so it is not tempo...   \n",
       "0   The idiom \"that makes two of us\" means \"the sa...   \n",
       "0   Jobs for the boys are not paid jobs and so the...   \n",
       "0   The hypothesis contradicts the premise because...   \n",
       "0   The figurative language in the hypothesis is \"...   \n",
       "\n",
       "                                          Generated_4           Model_1  \\\n",
       "0   A pat of butter melting would become lazy and ...          baseline   \n",
       "0   Monkeys foodfighting with bananas is hilarious...  text-davinci-002   \n",
       "0   The hypothesis is a metaphor meaning that the ...             human   \n",
       "0   The hypothesis is a metaphor for how beautiful...          baseline   \n",
       "0   Losing a job is often a very traumatic experie...  text-davinci-002   \n",
       "..                                                ...               ...   \n",
       "0   The figurative language in the hypothesis is t...  text-davinci-002   \n",
       "0   The hypothesis is saying that the premise and ...  text-davinci-002   \n",
       "0   The hypothesis is a restatement of the premise...             human   \n",
       "0   Stealing anything, regardless of how small it ...          baseline   \n",
       "0   The hypothesis states that the protagonist did...             human   \n",
       "\n",
       "             Model_2           Model_3           Model_4   Task  \\\n",
       "0   text-davinci-001  text-davinci-002             human  flute   \n",
       "0   text-davinci-001          baseline             human  flute   \n",
       "0           baseline  text-davinci-001  text-davinci-002  flute   \n",
       "0   text-davinci-001             human  text-davinci-002  flute   \n",
       "0   text-davinci-001          baseline             human  flute   \n",
       "..               ...               ...               ...    ...   \n",
       "0           baseline             human  text-davinci-001  flute   \n",
       "0           baseline             human  text-davinci-001  flute   \n",
       "0   text-davinci-001          baseline  text-davinci-002  flute   \n",
       "0   text-davinci-001  text-davinci-002             human  flute   \n",
       "0           baseline  text-davinci-001  text-davinci-002  flute   \n",
       "\n",
       "                                    additional_labels  \\\n",
       "0   A pat of butter melting would become lazy and ...   \n",
       "0   Monkeys foodfighting with bananas is hilarious...   \n",
       "0   A rare stone in a museum is carefully guarded ...   \n",
       "0   A hundred million fireflies dancing would be a...   \n",
       "0   Losing a job is often a very traumatic experie...   \n",
       "..                                                ...   \n",
       "0   An endless road has no end, so it is not tempo...   \n",
       "0   The idiom \"that makes two of us\" means \"the sa...   \n",
       "0   Jobs for the boys refers to giving paid employ...   \n",
       "0   Stealing anything, regardless of how small it ...   \n",
       "0   To put up with someone or something means to e...   \n",
       "\n",
       "                                           hypothesis         labels  \\\n",
       "0         His voice was like a melting pat of butter.     Entailment   \n",
       "0   The shade and sarcasm they throw at each other...     Entailment   \n",
       "0   She is like a rare stone in a museum surrounde...  Contradiction   \n",
       "0   Its like a hundred million fireflies dancing a...     Entailment   \n",
       "0   I felt terrible when I found out that I was go...     Entailment   \n",
       "..                                                ...            ...   \n",
       "0   We're starting to discover life is like an end...  Contradiction   \n",
       "0                               that makes two of us.     Entailment   \n",
       "0         But this isn't a case of jobs for the boys.     Entailment   \n",
       "0   I went to the store and stole a small bunch of...  Contradiction   \n",
       "0   I didn't like him enough to put up with that s...  Contradiction   \n",
       "\n",
       "                                              premise  \n",
       "0                     His voice was lazy and slippery  \n",
       "0   The way they shade and sarcasm each other is h...  \n",
       "0                         She is so easily accessible  \n",
       "0                                     It is beautiful  \n",
       "0   I felt pretty down when I found out that I was...  \n",
       "..                                                ...  \n",
       "0   We're starting to realize that life is fleetin...  \n",
       "0                        I feel exactly the same way.  \n",
       "0   But this isn't a case of giving paid employmen...  \n",
       "0   I went to the store and took a small bunch of ...  \n",
       "0   I didn't like him enough to reject or protest ...  \n",
       "\n",
       "[476 rows x 13 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "33633b34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n",
      "3 out of position\n"
     ]
    }
   ],
   "source": [
    "TASK = 'positive_reframing'\n",
    "txt=\"original_text\"\n",
    "gold=\"reframed_text\"\n",
    "secondary=\"strategy\"\n",
    "n=4\n",
    "literal=False\n",
    "random_state=7\n",
    "\n",
    "comb = []\n",
    "for fn in glob(f\"hit/input/{TASK}/*.csv\"):\n",
    "    comb.append(pd.read_csv(fn))\n",
    "df = pd.concat(comb)\n",
    "\n",
    "comparison_rows = []\n",
    "for lbl in set(df[txt]):\n",
    "    consider = df[df[txt]==lbl].copy()\n",
    "    cr = build_comparison_rows(consider,\n",
    "                                 txt=txt,\n",
    "                                 gold=gold,\n",
    "                                 secondary=secondary,\n",
    "                                 n=n,\n",
    "                                 literal=literal,\n",
    "                                 random_state=random_state,\n",
    "                                 MODELSET={\"human\", \"text-curie-001\", \"text-babbage-001\", \"text-ada-001\"}\n",
    "                                )\n",
    "    \n",
    "    if len(cr):\n",
    "        comparison_rows.append(cr)\n",
    "    \n",
    "comparison_df = pd.concat(comparison_rows)\n",
    "comparison_df.sample(frac=1, random_state=random_state).to_csv(f\"hit/input/comparison_balanced_{TASK}.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "74e36d5f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Generated_1</th>\n",
       "      <th>Generated_2</th>\n",
       "      <th>Generated_3</th>\n",
       "      <th>Generated_4</th>\n",
       "      <th>Model_1</th>\n",
       "      <th>Model_2</th>\n",
       "      <th>Model_3</th>\n",
       "      <th>Model_4</th>\n",
       "      <th>Task</th>\n",
       "      <th>additional_labels</th>\n",
       "      <th>hypothesis</th>\n",
       "      <th>labels</th>\n",
       "      <th>premise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The hypothesis is that the less fortunate have...</td>\n",
       "      <td>The hypothesis is an entailment of the premise...</td>\n",
       "      <td>Airing one's opinion means to speak out about ...</td>\n",
       "      <td>The relationship between the hypothesis and th...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>Airing one's opinion means to speak out about ...</td>\n",
       "      <td>She aired her opinions on welfare.</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>She openly give her point of view on the less ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The hypothesis is that the pain is coming from...</td>\n",
       "      <td>The hypothesis contradicts the premise because...</td>\n",
       "      <td>Experiencing back pain is never a good thing a...</td>\n",
       "      <td>The relationship between the premise and the h...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>Experiencing back pain is never a good thing a...</td>\n",
       "      <td>I've been having this back pain for a while no...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>My back has been having sharp random pains thr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The hypothesis is that Mella orders while bobb...</td>\n",
       "      <td>The figurative language in the hypothesis is t...</td>\n",
       "      <td>No sweat means not a problem or difficulty, bu...</td>\n",
       "      <td>The relationship between the premise and the h...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>No sweat means not a problem or difficulty, bu...</td>\n",
       "      <td>Mella orders, while bobbing her skull up and d...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>Mella orders, while bobbing her skull up and d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hypothesis suggests that the smile was a s...</td>\n",
       "      <td>The figurative language in the hypothesis is t...</td>\n",
       "      <td>An angel is a being of divine nature, and they...</td>\n",
       "      <td>The relationship between the premise and the h...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>An angel is a being of divine nature, and they...</td>\n",
       "      <td>I looked down at him, and he smiled at me like...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>He smiled treacherously up at me.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The hypothesis suggests that the protagonist f...</td>\n",
       "      <td>The hypothesis states that the person feels te...</td>\n",
       "      <td>It is natural to feel bad when you accidentall...</td>\n",
       "      <td>The relationship between the premise and the h...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>It is natural to feel bad when you accidentall...</td>\n",
       "      <td>I felt terrible when I accidentally bumped int...</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>I accidentally bumped into someone at the mall...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>The hypothesis implies that the flowers stoppe...</td>\n",
       "      <td>The hypothesis describes the flowers as if the...</td>\n",
       "      <td>To dance means to move nimbly or gracefully, w...</td>\n",
       "      <td>The flowers dance in the gentle breeze</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>To dance means to move nimbly or gracefully, w...</td>\n",
       "      <td>The flowers danced in the gentle breeze.</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>The flowers stopped in the gentle breeze.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>The hypothesis suggests that the gaze is like ...</td>\n",
       "      <td>The hypothesis describes the premise using fig...</td>\n",
       "      <td>A strong flame is usually associated with bein...</td>\n",
       "      <td>The hypothesis uses a figurative language to d...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>A strong flame is usually associated with bein...</td>\n",
       "      <td>His gaze was like a steady, strong flame .</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>His gaze was neither blazing nor compassionate.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>The hypothesis suggests that the best friend w...</td>\n",
       "      <td>The hypothesis is hyperbolic and is not meant ...</td>\n",
       "      <td>The beauty of Tahiti is often mentioned and th...</td>\n",
       "      <td>The hypothesis uses a metaphor when it says \"M...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>The beauty of Tahiti is often mentioned and th...</td>\n",
       "      <td>Tahiti is the most beautiful place on Earth an...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>My best friend is going to Tahiti with his fam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>The hypothesis is that people around the apart...</td>\n",
       "      <td>The hypothesis is a metaphor for how the peopl...</td>\n",
       "      <td>Cigarette butts are often unsightly and produc...</td>\n",
       "      <td>The relationship between the premise and the h...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>Cigarette butts are often unsightly and produc...</td>\n",
       "      <td>It's not only beautiful but also cool how peop...</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>I'm so mad at people around my apartment just ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>The hypothesis suggests that the sister gets t...</td>\n",
       "      <td>The hypothesis is a metaphor for how the perso...</td>\n",
       "      <td>It's unfair that one person gets to have more ...</td>\n",
       "      <td>The relationship between the premise and the h...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>human</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>flute</td>\n",
       "      <td>It's unfair that one person gets to have more ...</td>\n",
       "      <td>I hate that my sister gets to stay out as late...</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>My sister gets to stay out as late as she want...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>498 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Generated_1  \\\n",
       "0    The hypothesis is that the less fortunate have...   \n",
       "1    The hypothesis is that the pain is coming from...   \n",
       "2    The hypothesis is that Mella orders while bobb...   \n",
       "3    The hypothesis suggests that the smile was a s...   \n",
       "4    The hypothesis suggests that the protagonist f...   \n",
       "..                                                 ...   \n",
       "493  The hypothesis implies that the flowers stoppe...   \n",
       "494  The hypothesis suggests that the gaze is like ...   \n",
       "495  The hypothesis suggests that the best friend w...   \n",
       "496  The hypothesis is that people around the apart...   \n",
       "497  The hypothesis suggests that the sister gets t...   \n",
       "\n",
       "                                           Generated_2  \\\n",
       "0    The hypothesis is an entailment of the premise...   \n",
       "1    The hypothesis contradicts the premise because...   \n",
       "2    The figurative language in the hypothesis is t...   \n",
       "3    The figurative language in the hypothesis is t...   \n",
       "4    The hypothesis states that the person feels te...   \n",
       "..                                                 ...   \n",
       "493  The hypothesis describes the flowers as if the...   \n",
       "494  The hypothesis describes the premise using fig...   \n",
       "495  The hypothesis is hyperbolic and is not meant ...   \n",
       "496  The hypothesis is a metaphor for how the peopl...   \n",
       "497  The hypothesis is a metaphor for how the perso...   \n",
       "\n",
       "                                           Generated_3  \\\n",
       "0    Airing one's opinion means to speak out about ...   \n",
       "1    Experiencing back pain is never a good thing a...   \n",
       "2    No sweat means not a problem or difficulty, bu...   \n",
       "3    An angel is a being of divine nature, and they...   \n",
       "4    It is natural to feel bad when you accidentall...   \n",
       "..                                                 ...   \n",
       "493  To dance means to move nimbly or gracefully, w...   \n",
       "494  A strong flame is usually associated with bein...   \n",
       "495  The beauty of Tahiti is often mentioned and th...   \n",
       "496  Cigarette butts are often unsightly and produc...   \n",
       "497  It's unfair that one person gets to have more ...   \n",
       "\n",
       "                                           Generated_4           Model_1  \\\n",
       "0    The relationship between the hypothesis and th...  text-babbage-001   \n",
       "1    The relationship between the premise and the h...  text-babbage-001   \n",
       "2    The relationship between the premise and the h...  text-babbage-001   \n",
       "3    The relationship between the premise and the h...  text-babbage-001   \n",
       "4    The relationship between the premise and the h...  text-babbage-001   \n",
       "..                                                 ...               ...   \n",
       "493             The flowers dance in the gentle breeze  text-babbage-001   \n",
       "494  The hypothesis uses a figurative language to d...  text-babbage-001   \n",
       "495  The hypothesis uses a metaphor when it says \"M...  text-babbage-001   \n",
       "496  The relationship between the premise and the h...  text-babbage-001   \n",
       "497  The relationship between the premise and the h...  text-babbage-001   \n",
       "\n",
       "              Model_2 Model_3       Model_4   Task  \\\n",
       "0    text-davinci-002   human  text-ada-001  flute   \n",
       "1    text-davinci-002   human  text-ada-001  flute   \n",
       "2    text-davinci-002   human  text-ada-001  flute   \n",
       "3    text-davinci-002   human  text-ada-001  flute   \n",
       "4    text-davinci-002   human  text-ada-001  flute   \n",
       "..                ...     ...           ...    ...   \n",
       "493  text-davinci-002   human  text-ada-001  flute   \n",
       "494  text-davinci-002   human  text-ada-001  flute   \n",
       "495  text-davinci-002   human  text-ada-001  flute   \n",
       "496  text-davinci-002   human  text-ada-001  flute   \n",
       "497  text-davinci-002   human  text-ada-001  flute   \n",
       "\n",
       "                                     additional_labels  \\\n",
       "0    Airing one's opinion means to speak out about ...   \n",
       "1    Experiencing back pain is never a good thing a...   \n",
       "2    No sweat means not a problem or difficulty, bu...   \n",
       "3    An angel is a being of divine nature, and they...   \n",
       "4    It is natural to feel bad when you accidentall...   \n",
       "..                                                 ...   \n",
       "493  To dance means to move nimbly or gracefully, w...   \n",
       "494  A strong flame is usually associated with bein...   \n",
       "495  The beauty of Tahiti is often mentioned and th...   \n",
       "496  Cigarette butts are often unsightly and produc...   \n",
       "497  It's unfair that one person gets to have more ...   \n",
       "\n",
       "                                            hypothesis         labels  \\\n",
       "0                   She aired her opinions on welfare.     Entailment   \n",
       "1    I've been having this back pain for a while no...  Contradiction   \n",
       "2    Mella orders, while bobbing her skull up and d...  Contradiction   \n",
       "3    I looked down at him, and he smiled at me like...  Contradiction   \n",
       "4    I felt terrible when I accidentally bumped int...     Entailment   \n",
       "..                                                 ...            ...   \n",
       "493           The flowers danced in the gentle breeze.  Contradiction   \n",
       "494         His gaze was like a steady, strong flame .  Contradiction   \n",
       "495  Tahiti is the most beautiful place on Earth an...  Contradiction   \n",
       "496  It's not only beautiful but also cool how peop...  Contradiction   \n",
       "497  I hate that my sister gets to stay out as late...     Entailment   \n",
       "\n",
       "                                               premise  \n",
       "0    She openly give her point of view on the less ...  \n",
       "1    My back has been having sharp random pains thr...  \n",
       "2    Mella orders, while bobbing her skull up and d...  \n",
       "3                    He smiled treacherously up at me.  \n",
       "4    I accidentally bumped into someone at the mall...  \n",
       "..                                                 ...  \n",
       "493          The flowers stopped in the gentle breeze.  \n",
       "494    His gaze was neither blazing nor compassionate.  \n",
       "495  My best friend is going to Tahiti with his fam...  \n",
       "496  I'm so mad at people around my apartment just ...  \n",
       "497  My sister gets to stay out as late as she want...  \n",
       "\n",
       "[498 rows x 13 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TASK = 'flute'\n",
    "pd.read_csv(f\"hit/input/comparison_{TASK}.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "561db569",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post</th>\n",
       "      <th>targetMinority</th>\n",
       "      <th>Generated_1</th>\n",
       "      <th>Model_1</th>\n",
       "      <th>Task</th>\n",
       "      <th>Generated_2</th>\n",
       "      <th>Model_2</th>\n",
       "      <th>Generated_3</th>\n",
       "      <th>Model_3</th>\n",
       "      <th>Generated_4</th>\n",
       "      <th>Model_4</th>\n",
       "      <th>targetStereotype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What was the hardest part of the holocaust? My...</td>\n",
       "      <td>[\"Holocaust victims\", \"jewish folks\"]</td>\n",
       "      <td>a group of people</td>\n",
       "      <td>flan-t5-small</td>\n",
       "      <td>sbic</td>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>I'm sorry, I cannot provide an explanation for...</td>\n",
       "      <td>chatgpt</td>\n",
       "      <td>Jews are often viewed as victims of the Holocaust</td>\n",
       "      <td>text-davinci-003</td>\n",
       "      <td>[\"jewish people are inferior\", \"jewish people ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What was the hardest part of the holocaust? My...</td>\n",
       "      <td>[\"Holocaust victims\", \"jewish folks\"]</td>\n",
       "      <td>This sentence communicates the stereotype that...</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>sbic</td>\n",
       "      <td>trivializes harm to victims</td>\n",
       "      <td>human</td>\n",
       "      <td>jewish people are unwanted</td>\n",
       "      <td>human</td>\n",
       "      <td>jewish people are inferior</td>\n",
       "      <td>human</td>\n",
       "      <td>[\"jewish people are inferior\", \"jewish people ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What was the hardest part of the holocaust? My...</td>\n",
       "      <td>[\"Holocaust victims\", \"jewish folks\"]</td>\n",
       "      <td>The sentence communicates that the Holocaust w...</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>The hardest part of the holocaust was the geno...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>The hardest part of the holocaust was the lack...</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>[GROUP] are sexist.</td>\n",
       "      <td>flan-ul2</td>\n",
       "      <td>[\"jewish people are inferior\", \"jewish people ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What was the hardest part of the holocaust? My...</td>\n",
       "      <td>[\"Holocaust victims\", \"jewish folks\"]</td>\n",
       "      <td>Nazi</td>\n",
       "      <td>flan-t5-large</td>\n",
       "      <td>sbic</td>\n",
       "      <td>a dick</td>\n",
       "      <td>flan-t5-base</td>\n",
       "      <td>are dick</td>\n",
       "      <td>baseline</td>\n",
       "      <td>Jews are a hateful people</td>\n",
       "      <td>flan-t5-xxl</td>\n",
       "      <td>[\"jewish people are inferior\", \"jewish people ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A Cop Told A Suspect To Put His Hands Up, And ...</td>\n",
       "      <td>[\"black folks\"]</td>\n",
       "      <td>A cop told a suspect to put his hands up, and ...</td>\n",
       "      <td>flan-t5-small</td>\n",
       "      <td>sbic</td>\n",
       "      <td>This sentence communicates the stereotype that...</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>[GROUP] are a group of people</td>\n",
       "      <td>flan-t5-xl</td>\n",
       "      <td>The sentence \"Cop told suspect to put his hand...</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>[\"black folks are inferior\", \"black folks dese...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1912</th>\n",
       "      <td>we should not help poor african children\\nthey...</td>\n",
       "      <td>[\"African folks\"]</td>\n",
       "      <td>The stereotype communicated by this sentence i...</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>This sentence communicates the stereotype that...</td>\n",
       "      <td>text-davinci-003</td>\n",
       "      <td>[GROUP] are funny.</td>\n",
       "      <td>flan-ul2</td>\n",
       "      <td>poor</td>\n",
       "      <td>flan-t5-xxl</td>\n",
       "      <td>[\"african folks don't deserve to eat\", \"africa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1913</th>\n",
       "      <td>What do you call a woman who speaks their mind...</td>\n",
       "      <td>[\"women\"]</td>\n",
       "      <td>shouldn't speak</td>\n",
       "      <td>human</td>\n",
       "      <td>sbic</td>\n",
       "      <td>women are c*nts</td>\n",
       "      <td>human</td>\n",
       "      <td>a woman who speaks her mind</td>\n",
       "      <td>flan-t5-large</td>\n",
       "      <td>This sentence communicates the stereotype that...</td>\n",
       "      <td>chatgpt</td>\n",
       "      <td>[\"end up dead\", \"shouldn't speak\", \"women are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1914</th>\n",
       "      <td>What do you call a woman who speaks their mind...</td>\n",
       "      <td>[\"women\"]</td>\n",
       "      <td>end up dead</td>\n",
       "      <td>human</td>\n",
       "      <td>sbic</td>\n",
       "      <td>[GROUP] are a group of people who are buried i...</td>\n",
       "      <td>flan-t5-xl</td>\n",
       "      <td>a woman who speaks their mind is a cunt</td>\n",
       "      <td>flan-t5-xxl</td>\n",
       "      <td>Women are often seen as being too outspoken or...</td>\n",
       "      <td>text-davinci-003</td>\n",
       "      <td>[\"end up dead\", \"shouldn't speak\", \"women are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1915</th>\n",
       "      <td>What do you call a woman who speaks their mind...</td>\n",
       "      <td>[\"women\"]</td>\n",
       "      <td>women are sex slaves</td>\n",
       "      <td>baseline</td>\n",
       "      <td>sbic</td>\n",
       "      <td>women</td>\n",
       "      <td>flan-t5-base</td>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>text-davinci-001</td>\n",
       "      <td>The speaker is an individual who is passionate...</td>\n",
       "      <td>text-ada-001</td>\n",
       "      <td>[\"end up dead\", \"shouldn't speak\", \"women are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1916</th>\n",
       "      <td>What do you call a woman who speaks their mind...</td>\n",
       "      <td>[\"women\"]</td>\n",
       "      <td>The stereotype that this sentence communicates...</td>\n",
       "      <td>text-babbage-001</td>\n",
       "      <td>sbic</td>\n",
       "      <td>sexist</td>\n",
       "      <td>flan-ul2</td>\n",
       "      <td>This sentence communicates the stereotype that...</td>\n",
       "      <td>text-davinci-002</td>\n",
       "      <td>This sentence communicates that the woman in q...</td>\n",
       "      <td>text-curie-001</td>\n",
       "      <td>[\"end up dead\", \"shouldn't speak\", \"women are ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1917 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   post  \\\n",
       "0     What was the hardest part of the holocaust? My...   \n",
       "1     What was the hardest part of the holocaust? My...   \n",
       "2     What was the hardest part of the holocaust? My...   \n",
       "3     What was the hardest part of the holocaust? My...   \n",
       "4     A Cop Told A Suspect To Put His Hands Up, And ...   \n",
       "...                                                 ...   \n",
       "1912  we should not help poor african children\\nthey...   \n",
       "1913  What do you call a woman who speaks their mind...   \n",
       "1914  What do you call a woman who speaks their mind...   \n",
       "1915  What do you call a woman who speaks their mind...   \n",
       "1916  What do you call a woman who speaks their mind...   \n",
       "\n",
       "                             targetMinority  \\\n",
       "0     [\"Holocaust victims\", \"jewish folks\"]   \n",
       "1     [\"Holocaust victims\", \"jewish folks\"]   \n",
       "2     [\"Holocaust victims\", \"jewish folks\"]   \n",
       "3     [\"Holocaust victims\", \"jewish folks\"]   \n",
       "4                           [\"black folks\"]   \n",
       "...                                     ...   \n",
       "1912                      [\"African folks\"]   \n",
       "1913                              [\"women\"]   \n",
       "1914                              [\"women\"]   \n",
       "1915                              [\"women\"]   \n",
       "1916                              [\"women\"]   \n",
       "\n",
       "                                            Generated_1           Model_1  \\\n",
       "0                                     a group of people     flan-t5-small   \n",
       "1     This sentence communicates the stereotype that...  text-davinci-002   \n",
       "2     The sentence communicates that the Holocaust w...    text-curie-001   \n",
       "3                                                  Nazi     flan-t5-large   \n",
       "4     A cop told a suspect to put his hands up, and ...     flan-t5-small   \n",
       "...                                                 ...               ...   \n",
       "1912  The stereotype communicated by this sentence i...    text-curie-001   \n",
       "1913                                    shouldn't speak             human   \n",
       "1914                                        end up dead             human   \n",
       "1915                               women are sex slaves          baseline   \n",
       "1916  The stereotype that this sentence communicates...  text-babbage-001   \n",
       "\n",
       "      Task                                        Generated_2  \\\n",
       "0     sbic  The stereotype communicated by this sentence i...   \n",
       "1     sbic                        trivializes harm to victims   \n",
       "2     sbic  The hardest part of the holocaust was the geno...   \n",
       "3     sbic                                             a dick   \n",
       "4     sbic  This sentence communicates the stereotype that...   \n",
       "...    ...                                                ...   \n",
       "1912  sbic  This sentence communicates the stereotype that...   \n",
       "1913  sbic                                    women are c*nts   \n",
       "1914  sbic  [GROUP] are a group of people who are buried i...   \n",
       "1915  sbic                                              women   \n",
       "1916  sbic                                             sexist   \n",
       "\n",
       "               Model_2                                        Generated_3  \\\n",
       "0     text-davinci-001  I'm sorry, I cannot provide an explanation for...   \n",
       "1                human                         jewish people are unwanted   \n",
       "2     text-babbage-001  The hardest part of the holocaust was the lack...   \n",
       "3         flan-t5-base                                           are dick   \n",
       "4     text-davinci-001                      [GROUP] are a group of people   \n",
       "...                ...                                                ...   \n",
       "1912  text-davinci-003                                 [GROUP] are funny.   \n",
       "1913             human                        a woman who speaks her mind   \n",
       "1914        flan-t5-xl            a woman who speaks their mind is a cunt   \n",
       "1915      flan-t5-base  The stereotype that this sentence communicates...   \n",
       "1916          flan-ul2  This sentence communicates the stereotype that...   \n",
       "\n",
       "               Model_3                                        Generated_4  \\\n",
       "0              chatgpt  Jews are often viewed as victims of the Holocaust   \n",
       "1                human                         jewish people are inferior   \n",
       "2         text-ada-001                                [GROUP] are sexist.   \n",
       "3             baseline                          Jews are a hateful people   \n",
       "4           flan-t5-xl  The sentence \"Cop told suspect to put his hand...   \n",
       "...                ...                                                ...   \n",
       "1912          flan-ul2                                               poor   \n",
       "1913     flan-t5-large  This sentence communicates the stereotype that...   \n",
       "1914       flan-t5-xxl  Women are often seen as being too outspoken or...   \n",
       "1915  text-davinci-001  The speaker is an individual who is passionate...   \n",
       "1916  text-davinci-002  This sentence communicates that the woman in q...   \n",
       "\n",
       "               Model_4                                   targetStereotype  \n",
       "0     text-davinci-003  [\"jewish people are inferior\", \"jewish people ...  \n",
       "1                human  [\"jewish people are inferior\", \"jewish people ...  \n",
       "2             flan-ul2  [\"jewish people are inferior\", \"jewish people ...  \n",
       "3          flan-t5-xxl  [\"jewish people are inferior\", \"jewish people ...  \n",
       "4         text-ada-001  [\"black folks are inferior\", \"black folks dese...  \n",
       "...                ...                                                ...  \n",
       "1912       flan-t5-xxl  [\"african folks don't deserve to eat\", \"africa...  \n",
       "1913           chatgpt  [\"end up dead\", \"shouldn't speak\", \"women are ...  \n",
       "1914  text-davinci-003  [\"end up dead\", \"shouldn't speak\", \"women are ...  \n",
       "1915      text-ada-001  [\"end up dead\", \"shouldn't speak\", \"women are ...  \n",
       "1916    text-curie-001  [\"end up dead\", \"shouldn't speak\", \"women are ...  \n",
       "\n",
       "[1917 rows x 12 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TASK = 'sbic'\n",
    "pd.read_csv(f\"hit/input/comparison_{TASK}.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d28d2c4b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}