02-Quality.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_parquet(\"data/args.parquet\").drop(\"conclusion\", axis=1)\n",
    "\n",
    "IDS = df[\"id\"].tolist()\n",
    "SAMPLES = df[\"premise\"].tolist()\n",
    "\n",
    "del(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/webis-argquality20-full.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(\n",
    "    lowercase=True, \n",
    "    stop_words=\"english\",\n",
    "    max_features = 2**18\n",
    ")\n",
    "SAMPLES = vectorizer.fit_transform(SAMPLES)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Machine for Argument Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'F1-score: 0.88'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "X = df[\"Premise\"].astype(str).tolist()\n",
    "Y = (df[\"Combined Quality\"].astype(float) != -4.0).tolist()\n",
    "\n",
    "X = vectorizer.transform(X)\n",
    "\n",
    "classificator = SVC()\n",
    "classificator.fit(X,Y)\n",
    "\n",
    "scores = cross_val_score(classificator, X, Y, cv=10, scoring='f1')\n",
    "\"F1-score: {}\".format(scores.mean().round(4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Regression for Quality Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'MSE: 0.1412'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVR\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "X = df.loc[df[\"Combined Quality\"] != -4.0, \"Premise\"].astype(str).values\n",
    "Y = df.loc[df[\"Combined Quality\"] != -4.0, \"Combined Quality\"].astype(float).values\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "Y = scaler.fit_transform(Y.reshape(-1, 1)).ravel()\n",
    "X = vectorizer.transform(X).todense()\n",
    "\n",
    "regressor = LinearSVR(max_iter=10_000)\n",
    "regressor.fit(X, Y)\n",
    "\n",
    "scores = cross_val_score(regressor, X, Y, cv=10)\n",
    "\"MSE: {}\".format(scores.mean().round(4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting Scores for the args.me dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "PREDICTIONS = classificator.predict(SAMPLES)\n",
    "QUALITY = regressor.predict(SAMPLES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(zip(IDS, PREDICTIONS, QUALITY), columns=[\"id\", \"is_argument\", \"quality\"])\n",
    "df.loc[(df.is_argument == False),\"quality\"] = 0\n",
    "df = df.drop(\"is_argument\", axis=1)\n",
    "df.to_parquet(\"data/quality.parquet\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}