src/notebooks/lm-prep-cycle1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "22d2fa2e-8d0d-40e5-b438-d08f6e8fafaa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import operator\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from langdetect import detect\n",
    "\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "from data_input.load_data import load_corpus\n",
    "\n",
    "import readability\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3bd538c6-ac90-4158-8bb6-617e60f555e2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# load the data for the three evaluation cycles\n",
    "corpus1 = load_corpus(\"../../data/data_cycle1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5749b4d9-84bc-485b-8031-292373b53b2d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32387, 14)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "267603d0-268a-462d-ad5d-a0593a7c39c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# filter German language arguments\n",
    "def detect_language(text):\n",
    "    try:\n",
    "        return detect(text)\n",
    "    except:\n",
    "        return None\n",
    "\n",
    "def detect_languages_in_dataset(argu_corpus):\n",
    "    argu_corpus['language'] = argu_corpus['argument'].apply(detect_language)\n",
    "    return argu_corpus\n",
    "\n",
    "result_langu_df = detect_languages_in_dataset(corpus1)\n",
    "corpus_de = result_langu_df[result_langu_df['language'] == 'de']\n",
    "corpus_de = corpus_de.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "25f14a4e-48f2-4889-9c31-abf93a8cc6a2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22353, 15)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_de.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3a510d06-3a85-476a-bbf7-8aec857a0355",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>argument_id</th>\n",
       "      <th>argument</th>\n",
       "      <th>stance</th>\n",
       "      <th>topic</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>residence</th>\n",
       "      <th>civil_status</th>\n",
       "      <th>denomination</th>\n",
       "      <th>education</th>\n",
       "      <th>political_spectrum</th>\n",
       "      <th>important_political_issues</th>\n",
       "      <th>rile</th>\n",
       "      <th>galtan</th>\n",
       "      <th>language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201900</td>\n",
       "      <td>Das Schweizer Volk hat die MEI angenommen und ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Immigration</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>18-34</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Christ-katholisch</td>\n",
       "      <td>Fachhochschule</td>\n",
       "      <td>Mitte und Konservativ-Liberal</td>\n",
       "      <td>[Liberale Wirtschaftspolitik, Restriktive Migr...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>201901</td>\n",
       "      <td>Eine Legalisierung von Cannabis entlasten die ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Society</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>18-34</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Christ-katholisch</td>\n",
       "      <td>Fachhochschule</td>\n",
       "      <td>Mitte und Konservativ-Liberal</td>\n",
       "      <td>[Liberale Wirtschaftspolitik, Restriktive Migr...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>201902</td>\n",
       "      <td>Durch die Förderung der familienergänzenden Be...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Welfare</td>\n",
       "      <td>Weiblich</td>\n",
       "      <td>35-49</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Nicht bekannt</td>\n",
       "      <td>Universität</td>\n",
       "      <td>Mitte und Konservativ</td>\n",
       "      <td>[Offene Aussenpolitik, Liberale Wirtschaftspol...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>201903</td>\n",
       "      <td>Ich ziehe eine Elternzeit vor. Die Zeit nach d...</td>\n",
       "      <td>AGAINST</td>\n",
       "      <td>Welfare</td>\n",
       "      <td>Weiblich</td>\n",
       "      <td>35-49</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Nicht bekannt</td>\n",
       "      <td>Universität</td>\n",
       "      <td>Mitte und Konservativ</td>\n",
       "      <td>[Offene Aussenpolitik, Liberale Wirtschaftspol...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>201904</td>\n",
       "      <td>Unser Asylrecht muss konsequent angewendet wer...</td>\n",
       "      <td>AGAINST</td>\n",
       "      <td>Immigration</td>\n",
       "      <td>Weiblich</td>\n",
       "      <td>35-49</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Nicht bekannt</td>\n",
       "      <td>Universität</td>\n",
       "      <td>Mitte und Konservativ</td>\n",
       "      <td>[Offene Aussenpolitik, Liberale Wirtschaftspol...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22348</th>\n",
       "      <td>2019031140</td>\n",
       "      <td>Das Stimmrechtsalter ab 16 könnte prinzipiell ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Political System</td>\n",
       "      <td>Weiblich</td>\n",
       "      <td>35-49</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Universität</td>\n",
       "      <td>Links und Konservativ-Liberal</td>\n",
       "      <td>[Ausgebauter Umweltschutz, Ausgebauter Sozials...</td>\n",
       "      <td>Links</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22349</th>\n",
       "      <td>2019031147</td>\n",
       "      <td>Die heutige Technologie erlangt eigentlich E-V...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Political System</td>\n",
       "      <td>Weiblich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Römisch-katholisch</td>\n",
       "      <td>Universität</td>\n",
       "      <td>Mitte und Konservativ-Liberal</td>\n",
       "      <td>[Ausgebauter Umweltschutz, Ausgebauter Sozials...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22350</th>\n",
       "      <td>2019031148</td>\n",
       "      <td>Nur Volljaehrige sollen weiter stimmrechtig bl...</td>\n",
       "      <td>AGAINST</td>\n",
       "      <td>Political System</td>\n",
       "      <td>Weiblich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Römisch-katholisch</td>\n",
       "      <td>Universität</td>\n",
       "      <td>Mitte und Konservativ-Liberal</td>\n",
       "      <td>[Ausgebauter Umweltschutz, Ausgebauter Sozials...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22351</th>\n",
       "      <td>2019031162</td>\n",
       "      <td>La transparence est une vertue.\\n\\nTransparenz...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Political System</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>35-49</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Römisch-katholisch</td>\n",
       "      <td>Nicht bekannt</td>\n",
       "      <td>Links und Konservativ-Liberal</td>\n",
       "      <td>[Offene Aussenpolitik, Law &amp; Order, Ausgebaute...</td>\n",
       "      <td>Links</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22352</th>\n",
       "      <td>2019031452</td>\n",
       "      <td>Privatpersonen mit kleinen Spendenbeträgen sol...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Political System</td>\n",
       "      <td>Weiblich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Nicht bekannt</td>\n",
       "      <td>Mitte und Konservativ</td>\n",
       "      <td>[Offene Aussenpolitik, Liberale Wirtschaftspol...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ</td>\n",
       "      <td>de</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22353 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       argument_id                                           argument  \\\n",
       "0           201900  Das Schweizer Volk hat die MEI angenommen und ...   \n",
       "1           201901  Eine Legalisierung von Cannabis entlasten die ...   \n",
       "2           201902  Durch die Förderung der familienergänzenden Be...   \n",
       "3           201903  Ich ziehe eine Elternzeit vor. Die Zeit nach d...   \n",
       "4           201904  Unser Asylrecht muss konsequent angewendet wer...   \n",
       "...            ...                                                ...   \n",
       "22348   2019031140  Das Stimmrechtsalter ab 16 könnte prinzipiell ...   \n",
       "22349   2019031147  Die heutige Technologie erlangt eigentlich E-V...   \n",
       "22350   2019031148  Nur Volljaehrige sollen weiter stimmrechtig bl...   \n",
       "22351   2019031162  La transparence est une vertue.\\n\\nTransparenz...   \n",
       "22352   2019031452  Privatpersonen mit kleinen Spendenbeträgen sol...   \n",
       "\n",
       "        stance             topic    gender    age residence civil_status  \\\n",
       "0        FAVOR       Immigration  Männlich  18-34      Land        Ledig   \n",
       "1        FAVOR           Society  Männlich  18-34      Land        Ledig   \n",
       "2        FAVOR           Welfare  Weiblich  35-49      Land        Ledig   \n",
       "3      AGAINST           Welfare  Weiblich  35-49      Land        Ledig   \n",
       "4      AGAINST       Immigration  Weiblich  35-49      Land        Ledig   \n",
       "...        ...               ...       ...    ...       ...          ...   \n",
       "22348    FAVOR  Political System  Weiblich  35-49      Land  Verheiratet   \n",
       "22349    FAVOR  Political System  Weiblich  50-64      Land        Ledig   \n",
       "22350  AGAINST  Political System  Weiblich  50-64      Land        Ledig   \n",
       "22351    FAVOR  Political System  Männlich  35-49      Land  Verheiratet   \n",
       "22352    FAVOR  Political System  Weiblich  50-64      Land  Verheiratet   \n",
       "\n",
       "                               denomination       education  \\\n",
       "0                         Christ-katholisch  Fachhochschule   \n",
       "1                         Christ-katholisch  Fachhochschule   \n",
       "2                             Nicht bekannt     Universität   \n",
       "3                             Nicht bekannt     Universität   \n",
       "4                             Nicht bekannt     Universität   \n",
       "...                                     ...             ...   \n",
       "22348  Evangelischreformiert/protestantisch     Universität   \n",
       "22349                    Römisch-katholisch     Universität   \n",
       "22350                    Römisch-katholisch     Universität   \n",
       "22351                    Römisch-katholisch   Nicht bekannt   \n",
       "22352  Evangelischreformiert/protestantisch   Nicht bekannt   \n",
       "\n",
       "                  political_spectrum  \\\n",
       "0      Mitte und Konservativ-Liberal   \n",
       "1      Mitte und Konservativ-Liberal   \n",
       "2              Mitte und Konservativ   \n",
       "3              Mitte und Konservativ   \n",
       "4              Mitte und Konservativ   \n",
       "...                              ...   \n",
       "22348  Links und Konservativ-Liberal   \n",
       "22349  Mitte und Konservativ-Liberal   \n",
       "22350  Mitte und Konservativ-Liberal   \n",
       "22351  Links und Konservativ-Liberal   \n",
       "22352          Mitte und Konservativ   \n",
       "\n",
       "                              important_political_issues   rile  \\\n",
       "0      [Liberale Wirtschaftspolitik, Restriktive Migr...  Mitte   \n",
       "1      [Liberale Wirtschaftspolitik, Restriktive Migr...  Mitte   \n",
       "2      [Offene Aussenpolitik, Liberale Wirtschaftspol...  Mitte   \n",
       "3      [Offene Aussenpolitik, Liberale Wirtschaftspol...  Mitte   \n",
       "4      [Offene Aussenpolitik, Liberale Wirtschaftspol...  Mitte   \n",
       "...                                                  ...    ...   \n",
       "22348  [Ausgebauter Umweltschutz, Ausgebauter Sozials...  Links   \n",
       "22349  [Ausgebauter Umweltschutz, Ausgebauter Sozials...  Mitte   \n",
       "22350  [Ausgebauter Umweltschutz, Ausgebauter Sozials...  Mitte   \n",
       "22351  [Offene Aussenpolitik, Law & Order, Ausgebaute...  Links   \n",
       "22352  [Offene Aussenpolitik, Liberale Wirtschaftspol...  Mitte   \n",
       "\n",
       "                    galtan language  \n",
       "0      Konservativ-Liberal       de  \n",
       "1      Konservativ-Liberal       de  \n",
       "2              Konservativ       de  \n",
       "3              Konservativ       de  \n",
       "4              Konservativ       de  \n",
       "...                    ...      ...  \n",
       "22348  Konservativ-Liberal       de  \n",
       "22349  Konservativ-Liberal       de  \n",
       "22350  Konservativ-Liberal       de  \n",
       "22351  Konservativ-Liberal       de  \n",
       "22352          Konservativ       de  \n",
       "\n",
       "[22353 rows x 15 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_de"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e44d56ed-72ce-41c4-8e05-fe2cf059430a",
   "metadata": {},
   "source": [
    "# Preprocessing\n",
    "#### Clean data to improve meaningfulness of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4118c9f3-3303-4bbe-9bd1-21b2e5ec763f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ind_variables1 = [\"gender\", \"age\", \"residence\", \"civil_status\", \"denomination\", \"education\", \n",
    "                  \"rile\", \"galtan\", \"important_political_issues\"]\n",
    "ind_variables2 = [\"stance\", \"topic\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e344becc-869e-48df-ab20-911343f633bb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[gender\n",
       " Männlich    14308\n",
       " Weiblich     8045\n",
       " Name: count, dtype: int64,\n",
       " age\n",
       " 18-34    7467\n",
       " 50-64    7215\n",
       " 35-49    6393\n",
       " 65+      1278\n",
       " Name: count, dtype: int64,\n",
       " residence\n",
       " Land     20257\n",
       " Stadt     2096\n",
       " Name: count, dtype: int64,\n",
       " civil_status\n",
       " Verheiratet                   8190\n",
       " Nicht bekannt                 7566\n",
       " Ledig                         5294\n",
       " Geschieden                     575\n",
       " Konkubinat                     395\n",
       " Verwitwet                      234\n",
       " Eingetragene Partnerschaft      52\n",
       " Getrennt                        46\n",
       " Aufgelöste Partnerschaft         1\n",
       " Name: count, dtype: int64,\n",
       " denomination\n",
       " Nicht bekannt                             9018\n",
       " Evangelischreformiert/protestantisch      5484\n",
       " Römisch-katholisch                        4221\n",
       " Konfessionslos                            2993\n",
       " Andere christliche Gemeinschaften          506\n",
       " Christ-katholisch                           69\n",
       " Christlich-orthodox                         22\n",
       " Andere Kirchen/Religionsgemeinschaften      17\n",
       " Jüdische Gemeinschaften                     14\n",
       " Islamische Gemeinschaften                    9\n",
       " Name: count, dtype: int64,\n",
       " education\n",
       " Universität                  7007\n",
       " Nicht bekannt                6207\n",
       " Fachhochschule               2926\n",
       " Berufslehre/Berufsschule     1737\n",
       " Gymnasium/Seminar            1223\n",
       " Höhere Berufsausbildung      1097\n",
       " Höhere Fachschule             878\n",
       " Berufsmatura                  372\n",
       " Anderer Bildungsabschluss     235\n",
       " Sekundarschule                220\n",
       " Handelsschule                 168\n",
       " Diplommittelschule            157\n",
       " Primarschule/Realschule       126\n",
       " Name: count, dtype: int64,\n",
       " rile\n",
       " Links     12057\n",
       " Mitte      8733\n",
       " Rechts     1563\n",
       " Name: count, dtype: int64,\n",
       " galtan\n",
       " Konservativ-Liberal    14378\n",
       " Konservativ             6621\n",
       " Liberal                 1354\n",
       " Name: count, dtype: int64,\n",
       " important_political_issues\n",
       " [Offene Aussenpolitik, Ausgebauter Umweltschutz, Ausgebauter Sozialstaat, Liberale Gesellschaft]                    4109\n",
       " [Offene Aussenpolitik, Ausgebauter Umweltschutz, Ausgebauter Sozialstaat]                                           2258\n",
       " [Ausgebauter Umweltschutz, Ausgebauter Sozialstaat, Liberale Gesellschaft]                                          1907\n",
       " [Offene Aussenpolitik, Liberale Wirtschaftspolitik, Liberale Gesellschaft]                                          1217\n",
       " [Offene Aussenpolitik, Liberale Wirtschaftspolitik, Ausgebauter Umweltschutz, Liberale Gesellschaft]                1029\n",
       "                                                                                                                     ... \n",
       " [Offene Aussenpolitik, Restriktive Migrationspolitik, Ausgebauter Umweltschutz]                                        1\n",
       " [Law & Order, Ausgebauter Umweltschutz, Liberale Gesellschaft]                                                         1\n",
       " [Offene Aussenpolitik, Ausgebauter Sozialstaat]                                                                        1\n",
       " [Liberale Wirtschaftspolitik, Ausgebauter Umweltschutz, Ausgebauter Sozialstaat]                                       1\n",
       " [Offene Aussenpolitik, Liberale Wirtschaftspolitik, Law & Order, Ausgebauter Sozialstaat, Liberale Gesellschaft]       1\n",
       " Name: count, Length: 115, dtype: int64]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check sparsity\n",
    "[corpus_de[col].value_counts() for col in ind_variables1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "50592891-a03f-4309-94da-29216f8aed0f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Summarize groups\n",
    "corpus_de.loc[corpus_de[\"education\"] == \"Höhere Fachschule\", \n",
    "              \"education\"] = \"Höhere Berufsausbildung\"\n",
    "corpus_de.loc[corpus_de[\"education\"] == \"Handelsschule\", \n",
    "              \"education\"] = \"Höhere Berufsausbildung\"\n",
    "corpus_de.loc[corpus_de[\"education\"] == \"Berufsmatura\", \n",
    "              \"education\"] = \"Berufsmatura/Diplommittelschule\"\n",
    "corpus_de.loc[corpus_de[\"education\"] == \"Diplommittelschule\", \n",
    "              \"education\"] = \"Berufsmatura/Diplommittelschule\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bbd95099-5bf2-426e-b3ec-d355cbe2d075",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Drop groups with rare occurences (<50)\n",
    "corpus_de = corpus_de.drop(corpus_de[\n",
    "    corpus_de[\"civil_status\"].isin([\"Getrennt\"])].index)\n",
    "corpus_de = corpus_de.drop(corpus_de[\n",
    "    corpus_de[\"civil_status\"].isin([\"Aufgelöste Partnerschaft\"])].index)\n",
    "\n",
    "corpus_de = corpus_de.drop(corpus_de[\n",
    "    corpus_de[\"denomination\"].isin([\"Christlich-orthodox\"])].index)\n",
    "corpus_de = corpus_de.drop(corpus_de[\n",
    "    corpus_de[\"denomination\"].isin([\"Andere Kirchen/Religionsgemeinschaften\"])].index)\n",
    "corpus_de = corpus_de.drop(corpus_de[\n",
    "    corpus_de[\"denomination\"].isin([\"Jüdische Gemeinschaften\"])].index)\n",
    "corpus_de = corpus_de.drop(corpus_de[\n",
    "    corpus_de[\"denomination\"].isin([\"Islamische Gemeinschaften\"])].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fcee20f1-6462-428e-af62-a5faf6585825",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Drop rows with missing values\n",
    "corpus_de = corpus_de.drop(corpus_de[corpus_de[\"civil_status\"].isin([\"Nicht bekannt\"])].index)\n",
    "corpus_de = corpus_de.drop(corpus_de[corpus_de[\"denomination\"].isin([\"Nicht bekannt\"])].index)\n",
    "corpus_de = corpus_de.drop(corpus_de[corpus_de[\"education\"].isin([\"Nicht bekannt\"])].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b55351f5-640a-41b2-84bd-b9e47b728907",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[gender\n",
       " Männlich    7474\n",
       " Weiblich    3815\n",
       " Name: count, dtype: int64,\n",
       " age\n",
       " 50-64    3830\n",
       " 35-49    3563\n",
       " 18-34    3320\n",
       " 65+       576\n",
       " Name: count, dtype: int64,\n",
       " residence\n",
       " Land     10301\n",
       " Stadt      988\n",
       " Name: count, dtype: int64,\n",
       " civil_status\n",
       " Verheiratet                   6417\n",
       " Ledig                         3904\n",
       " Geschieden                     417\n",
       " Konkubinat                     369\n",
       " Verwitwet                      130\n",
       " Eingetragene Partnerschaft      52\n",
       " Name: count, dtype: int64,\n",
       " denomination\n",
       " Evangelischreformiert/protestantisch    4801\n",
       " Römisch-katholisch                      3438\n",
       " Konfessionslos                          2496\n",
       " Andere christliche Gemeinschaften        490\n",
       " Christ-katholisch                         64\n",
       " Name: count, dtype: int64,\n",
       " education\n",
       " Universität                        4903\n",
       " Fachhochschule                     2007\n",
       " Höhere Berufsausbildung            1488\n",
       " Berufslehre/Berufsschule           1288\n",
       " Gymnasium/Seminar                   874\n",
       " Berufsmatura/Diplommittelschule     347\n",
       " Anderer Bildungsabschluss           148\n",
       " Sekundarschule                      132\n",
       " Primarschule/Realschule             102\n",
       " Name: count, dtype: int64,\n",
       " rile\n",
       " Links     5667\n",
       " Mitte     4876\n",
       " Rechts     746\n",
       " Name: count, dtype: int64,\n",
       " galtan\n",
       " Konservativ-Liberal    7140\n",
       " Konservativ            3563\n",
       " Liberal                 586\n",
       " Name: count, dtype: int64,\n",
       " important_political_issues\n",
       " [Offene Aussenpolitik, Ausgebauter Umweltschutz, Ausgebauter Sozialstaat, Liberale Gesellschaft]                    1795\n",
       " [Offene Aussenpolitik, Ausgebauter Umweltschutz, Ausgebauter Sozialstaat]                                           1178\n",
       " [Offene Aussenpolitik, Liberale Wirtschaftspolitik, Liberale Gesellschaft]                                           701\n",
       " [Ausgebauter Umweltschutz, Ausgebauter Sozialstaat, Liberale Gesellschaft]                                           689\n",
       " [Offene Aussenpolitik, Liberale Wirtschaftspolitik, Ausgebauter Umweltschutz, Liberale Gesellschaft]                 555\n",
       "                                                                                                                     ... \n",
       " [Restriktive Finanzpolitik, Law & Order]                                                                               1\n",
       " [Offene Aussenpolitik]                                                                                                 1\n",
       " [Liberale Wirtschaftspolitik, Law & Order, Ausgebauter Umweltschutz]                                                   1\n",
       " [Offene Aussenpolitik, Liberale Gesellschaft]                                                                          1\n",
       " [Offene Aussenpolitik, Liberale Wirtschaftspolitik, Law & Order, Ausgebauter Sozialstaat, Liberale Gesellschaft]       1\n",
       " Name: count, Length: 88, dtype: int64]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[corpus_de[col].value_counts() for col in ind_variables1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5e094898-0c79-48cb-b8eb-fb3e3024471d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11289, 15)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_de.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70b9fe49-6f41-423a-86bb-ba1d64b142f7",
   "metadata": {},
   "source": [
    "# Feature Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60241dff-b5bb-416f-aa83-87d0863384b5",
   "metadata": {},
   "source": [
    "### Surface features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2a7c1fe4-9254-4877-9561-6cd4c31db4df",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "corpus_de['FleschReadingEase'] = corpus_de['argument'].apply(lambda x: readability.getmeasures(x, lang='de')['readability grades']['FleschReadingEase'])\n",
    "corpus_de['GunningFogIndex'] = corpus_de['argument'].apply(lambda x: readability.getmeasures(x, lang='de')['readability grades']['GunningFogIndex'])\n",
    "\n",
    "corpus_de['characters_per_word'] = corpus_de['argument'].apply(lambda x: readability.getmeasures(x, lang='de')['sentence info']['characters_per_word'])\n",
    "corpus_de['words_per_sentence'] = corpus_de['argument'].apply(lambda x: readability.getmeasures(x, lang='de')['sentence info']['words_per_sentence'])\n",
    "corpus_de['type_token_ratio'] = corpus_de['argument'].apply(lambda x: readability.getmeasures(x, lang='de')['sentence info']['type_token_ratio'])\n",
    "corpus_de['long_words'] = corpus_de['argument'].apply(lambda x: readability.getmeasures(x, lang='de')['sentence info']['long_words'])\n",
    "corpus_de['complex_words'] = corpus_de['argument'].apply(lambda x: readability.getmeasures(x, lang='de')['sentence info']['complex_words'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "246df0b2-4a23-45e2-9d0a-33548506ecfa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>argument_id</th>\n",
       "      <th>argument</th>\n",
       "      <th>stance</th>\n",
       "      <th>topic</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>residence</th>\n",
       "      <th>civil_status</th>\n",
       "      <th>denomination</th>\n",
       "      <th>education</th>\n",
       "      <th>...</th>\n",
       "      <th>rile</th>\n",
       "      <th>galtan</th>\n",
       "      <th>language</th>\n",
       "      <th>FleschReadingEase</th>\n",
       "      <th>GunningFogIndex</th>\n",
       "      <th>characters_per_word</th>\n",
       "      <th>words_per_sentence</th>\n",
       "      <th>type_token_ratio</th>\n",
       "      <th>long_words</th>\n",
       "      <th>complex_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201900</td>\n",
       "      <td>Das Schweizer Volk hat die MEI angenommen und ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Immigration</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>18-34</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Christ-katholisch</td>\n",
       "      <td>Fachhochschule</td>\n",
       "      <td>...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "      <td>25.703448</td>\n",
       "      <td>15.737931</td>\n",
       "      <td>5.758621</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.896552</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>201901</td>\n",
       "      <td>Eine Legalisierung von Cannabis entlasten die ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Society</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>18-34</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Christ-katholisch</td>\n",
       "      <td>Fachhochschule</td>\n",
       "      <td>...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "      <td>7.991154</td>\n",
       "      <td>13.476923</td>\n",
       "      <td>6.576923</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2019054</td>\n",
       "      <td>Die jetzige finanzielle Situation der Pensions...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Welfare</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Höhere Berufsausbildung</td>\n",
       "      <td>...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "      <td>26.470000</td>\n",
       "      <td>8.036364</td>\n",
       "      <td>6.545455</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2019055</td>\n",
       "      <td>Dort wo die Leistungen nicht angebracht sind, ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Welfare</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Höhere Berufsausbildung</td>\n",
       "      <td>...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "      <td>44.405000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2019056</td>\n",
       "      <td>Wünschenswert, aber eine Herkulesaufgabe für d...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Education</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Höhere Berufsausbildung</td>\n",
       "      <td>...</td>\n",
       "      <td>Mitte</td>\n",
       "      <td>Konservativ-Liberal</td>\n",
       "      <td>de</td>\n",
       "      <td>-34.129231</td>\n",
       "      <td>11.353846</td>\n",
       "      <td>7.923077</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    argument_id                                           argument stance  \\\n",
       "0        201900  Das Schweizer Volk hat die MEI angenommen und ...  FAVOR   \n",
       "1        201901  Eine Legalisierung von Cannabis entlasten die ...  FAVOR   \n",
       "33      2019054  Die jetzige finanzielle Situation der Pensions...  FAVOR   \n",
       "34      2019055  Dort wo die Leistungen nicht angebracht sind, ...  FAVOR   \n",
       "35      2019056  Wünschenswert, aber eine Herkulesaufgabe für d...  FAVOR   \n",
       "\n",
       "          topic    gender    age residence civil_status  \\\n",
       "0   Immigration  Männlich  18-34      Land        Ledig   \n",
       "1       Society  Männlich  18-34      Land        Ledig   \n",
       "33      Welfare  Männlich  50-64      Land  Verheiratet   \n",
       "34      Welfare  Männlich  50-64      Land  Verheiratet   \n",
       "35    Education  Männlich  50-64      Land  Verheiratet   \n",
       "\n",
       "                            denomination                education  ...   rile  \\\n",
       "0                      Christ-katholisch           Fachhochschule  ...  Mitte   \n",
       "1                      Christ-katholisch           Fachhochschule  ...  Mitte   \n",
       "33  Evangelischreformiert/protestantisch  Höhere Berufsausbildung  ...  Mitte   \n",
       "34  Evangelischreformiert/protestantisch  Höhere Berufsausbildung  ...  Mitte   \n",
       "35  Evangelischreformiert/protestantisch  Höhere Berufsausbildung  ...  Mitte   \n",
       "\n",
       "                 galtan language FleschReadingEase GunningFogIndex  \\\n",
       "0   Konservativ-Liberal       de         25.703448       15.737931   \n",
       "1   Konservativ-Liberal       de          7.991154       13.476923   \n",
       "33  Konservativ-Liberal       de         26.470000        8.036364   \n",
       "34  Konservativ-Liberal       de         44.405000       12.000000   \n",
       "35  Konservativ-Liberal       de        -34.129231       11.353846   \n",
       "\n",
       "    characters_per_word  words_per_sentence  type_token_ratio  long_words  \\\n",
       "0              5.758621                29.0          0.896552           9   \n",
       "1              6.576923                26.0          0.923077          11   \n",
       "33             6.545455                11.0          1.000000           6   \n",
       "34             6.000000                10.0          1.000000           4   \n",
       "35             7.923077                13.0          0.923077           7   \n",
       "\n",
       "    complex_words  \n",
       "0               3  \n",
       "1               2  \n",
       "33              1  \n",
       "34              2  \n",
       "35              2  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_de.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "839842cb-f297-45dd-ad0b-789b78e09b92",
   "metadata": {},
   "source": [
    "#### Mean and standard deviation for the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "63ff18ce-905d-4484-9b6b-98dfdfa9286a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "surface_features = ['FleschReadingEase', 'GunningFogIndex', 'characters_per_word', \n",
    "                             'words_per_sentence', 'type_token_ratio', 'long_words', 'complex_words']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "dfa269e9-27f6-4140-966c-f30118540233",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FleschReadingEase 19.15 -177.92 103.7 27.65\n",
      "GunningFogIndex 14.33 1.4 36.4 6.23\n",
      "characters_per_word 6.41 3.75 14.25 0.96\n",
      "words_per_sentence 23.36 3.5 85.0 14.26\n",
      "type_token_ratio 0.95 0.64 1.0 0.06\n",
      "long_words 8.89 0 32 5.63\n",
      "complex_words 2.95 0 15 2.28\n"
     ]
    }
   ],
   "source": [
    "for feat in surface_features:\n",
    "    print(feat, round(corpus_de[feat].mean(),2), round(corpus_de[feat].min(),2), \n",
    "          round(corpus_de[feat].max(),2), round(corpus_de[feat].std(),2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21f5e0c2-7fb7-4c46-bd39-1d5dd678a2ee",
   "metadata": {},
   "source": [
    "### Syntactic features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e2841a2e-7afb-487f-a904-ae0205caae54",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#ADJ: adjective\n",
    "#ADP: adposition\n",
    "#ADV: adverb\n",
    "#AUX: auxiliary\n",
    "#CCONJ: coordinating conjunction\n",
    "#DET: determiner\n",
    "#INTJ: interjection\n",
    "#NOUN: noun\n",
    "#NUM: numeral\n",
    "#PART: particle\n",
    "#PRON: pronoun\n",
    "#PROPN: proper noun\n",
    "#PUNCT: punctuation\n",
    "#SCONJ: subordinating conjunction\n",
    "#SYM: symbol\n",
    "#VERB: verb\n",
    "#X: other\n",
    "\n",
    "upos_tags = ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ', 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', \n",
    "             'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']\n",
    "\n",
    "nlp = spacy.load(\"de_core_news_sm\")\n",
    "\n",
    "\n",
    "def pos_features(text):\n",
    "    doc = nlp(text)\n",
    "    doc_pos = [token.pos_ for token in doc]\n",
    "    doc_length = len(doc_pos)\n",
    "    counts = {tag: doc_pos.count(tag)/doc_length for tag in upos_tags} # proportion of pos tags in text\n",
    "    return counts\n",
    "\n",
    "\n",
    "def entity_feature(text):\n",
    "    doc = nlp(text)\n",
    "    doc_ent = [token.ent_iob_ for token in doc]\n",
    "    doc_length = len(doc_ent)\n",
    "    count = (doc_ent.count('B')+doc_ent.count('I'))/doc_length # proportion of entities in text\n",
    "    return count\n",
    "\n",
    "\n",
    "def morph_features(text):\n",
    "    doc = nlp(text)\n",
    "    doc_morph = [token.morph for token in doc]\n",
    "    doc_length = len(doc_morph)\n",
    "    tense = sum([1 for token in doc_morph if \"Tense=Pres\" in token])/doc_length\n",
    "    mood = sum([1 for token in doc_morph if \"Mood=Imp\" in token])/doc_length\n",
    "    person = sum([1 for token in doc_morph if \"Person=1\" in token])/doc_length\n",
    "    return {\"pres_tense\": tense, \"imperative\": mood, \"first_person\": person}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "abf0d5c1-6f8a-400e-9e59-8f06dc5400fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# add column with pos tags (as dict)\n",
    "corpus_de['POS'] = corpus_de['argument'].apply(lambda x: pos_features(x))\n",
    "# pos dict to single columns\n",
    "corpus_de = pd.concat([corpus_de, corpus_de['POS'].apply(pd.Series)], axis=1)\n",
    "corpus_de = corpus_de.drop('POS', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d651a179-3857-4d46-8342-468912a92e8f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# add column with entity ratio\n",
    "corpus_de['Entities'] = corpus_de['argument'].apply(lambda x: entity_feature(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "dcb42eb4-e7ed-4c9d-a16b-ca138b6d469b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# add column with morphology features\n",
    "corpus_de['Morph'] = corpus_de['argument'].apply(lambda x: morph_features(x))\n",
    "# morph dict to single columns\n",
    "corpus_de = pd.concat([corpus_de, corpus_de['Morph'].apply(pd.Series)], axis=1)\n",
    "corpus_de = corpus_de.drop('Morph', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a5ea3011-cd6b-407c-8c3e-679ed2fc49c8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>argument_id</th>\n",
       "      <th>argument</th>\n",
       "      <th>stance</th>\n",
       "      <th>topic</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>residence</th>\n",
       "      <th>civil_status</th>\n",
       "      <th>denomination</th>\n",
       "      <th>education</th>\n",
       "      <th>...</th>\n",
       "      <th>PROPN</th>\n",
       "      <th>PUNCT</th>\n",
       "      <th>SCONJ</th>\n",
       "      <th>SYM</th>\n",
       "      <th>VERB</th>\n",
       "      <th>X</th>\n",
       "      <th>Entities</th>\n",
       "      <th>pres_tense</th>\n",
       "      <th>imperative</th>\n",
       "      <th>first_person</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201900</td>\n",
       "      <td>Das Schweizer Volk hat die MEI angenommen und ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Immigration</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>18-34</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Christ-katholisch</td>\n",
       "      <td>Fachhochschule</td>\n",
       "      <td>...</td>\n",
       "      <td>0.03125</td>\n",
       "      <td>0.093750</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.09375</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>201901</td>\n",
       "      <td>Eine Legalisierung von Cannabis entlasten die ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Society</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>18-34</td>\n",
       "      <td>Land</td>\n",
       "      <td>Ledig</td>\n",
       "      <td>Christ-katholisch</td>\n",
       "      <td>Fachhochschule</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2019054</td>\n",
       "      <td>Die jetzige finanzielle Situation der Pensions...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Welfare</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Höhere Berufsausbildung</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2019055</td>\n",
       "      <td>Dort wo die Leistungen nicht angebracht sind, ...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Welfare</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Höhere Berufsausbildung</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2019056</td>\n",
       "      <td>Wünschenswert, aber eine Herkulesaufgabe für d...</td>\n",
       "      <td>FAVOR</td>\n",
       "      <td>Education</td>\n",
       "      <td>Männlich</td>\n",
       "      <td>50-64</td>\n",
       "      <td>Land</td>\n",
       "      <td>Verheiratet</td>\n",
       "      <td>Evangelischreformiert/protestantisch</td>\n",
       "      <td>Höhere Berufsausbildung</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06250</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.06250</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    argument_id                                           argument stance  \\\n",
       "0        201900  Das Schweizer Volk hat die MEI angenommen und ...  FAVOR   \n",
       "1        201901  Eine Legalisierung von Cannabis entlasten die ...  FAVOR   \n",
       "33      2019054  Die jetzige finanzielle Situation der Pensions...  FAVOR   \n",
       "34      2019055  Dort wo die Leistungen nicht angebracht sind, ...  FAVOR   \n",
       "35      2019056  Wünschenswert, aber eine Herkulesaufgabe für d...  FAVOR   \n",
       "\n",
       "          topic    gender    age residence civil_status  \\\n",
       "0   Immigration  Männlich  18-34      Land        Ledig   \n",
       "1       Society  Männlich  18-34      Land        Ledig   \n",
       "33      Welfare  Männlich  50-64      Land  Verheiratet   \n",
       "34      Welfare  Männlich  50-64      Land  Verheiratet   \n",
       "35    Education  Männlich  50-64      Land  Verheiratet   \n",
       "\n",
       "                            denomination                education  ...  \\\n",
       "0                      Christ-katholisch           Fachhochschule  ...   \n",
       "1                      Christ-katholisch           Fachhochschule  ...   \n",
       "33  Evangelischreformiert/protestantisch  Höhere Berufsausbildung  ...   \n",
       "34  Evangelischreformiert/protestantisch  Höhere Berufsausbildung  ...   \n",
       "35  Evangelischreformiert/protestantisch  Höhere Berufsausbildung  ...   \n",
       "\n",
       "      PROPN     PUNCT     SCONJ  SYM      VERB    X  Entities  pres_tense  \\\n",
       "0   0.03125  0.093750  0.000000  0.0  0.125000  0.0   0.09375    0.062500   \n",
       "1   0.00000  0.133333  0.033333  0.0  0.166667  0.0   0.00000    0.166667   \n",
       "33  0.00000  0.083333  0.000000  0.0  0.083333  0.0   0.00000    0.083333   \n",
       "34  0.00000  0.166667  0.083333  0.0  0.083333  0.0   0.00000    0.083333   \n",
       "35  0.06250  0.187500  0.000000  0.0  0.062500  0.0   0.06250    0.062500   \n",
       "\n",
       "    imperative  first_person  \n",
       "0          0.0      0.000000  \n",
       "1          0.0      0.000000  \n",
       "33         0.0      0.083333  \n",
       "34         0.0      0.166667  \n",
       "35         0.0      0.000000  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_de.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c2e4f3c-9862-4977-81b6-0700b8cf34fc",
   "metadata": {},
   "source": [
    "#### Mean and standard deviation for the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f8f2f45b-c153-4e05-9e5e-c133b645b0ca",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "syntactic_features = ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ', 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', \n",
    "                    'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X', 'Entities', 'pres_tense',\n",
    "                    'imperative', 'first_person']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e4789140-ea49-4b8d-bd03-0bd4dd219757",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ADJ 0.06 0.0 0.43 0.05\n",
      "ADP 0.08 0.0 0.43 0.05\n",
      "ADV 0.11 0.0 0.62 0.07\n",
      "AUX 0.08 0.0 0.38 0.05\n",
      "CCONJ 0.03 0.0 0.23 0.03\n",
      "DET 0.12 0.0 0.36 0.06\n",
      "INTJ 0.0 0.0 0.0 0.0\n",
      "NOUN 0.21 0.0 0.8 0.07\n",
      "NUM 0.0 0.0 0.25 0.02\n",
      "PART 0.02 0.0 0.27 0.03\n",
      "PRON 0.05 0.0 0.4 0.05\n",
      "PROPN 0.01 0.0 0.38 0.03\n",
      "PUNCT 0.12 0.0 0.5 0.05\n",
      "SCONJ 0.01 0.0 0.17 0.02\n",
      "SYM 0.0 0.0 0.0 0.0\n",
      "VERB 0.09 0.0 0.5 0.05\n",
      "X 0.0 0.0 0.22 0.01\n",
      "Entities 0.04 0.0 0.62 0.05\n",
      "pres_tense 0.08 0.0 0.3 0.04\n",
      "imperative 0.0 0.0 0.11 0.0\n",
      "first_person 0.02 0.0 0.5 0.05\n"
     ]
    }
   ],
   "source": [
    "for feat in syntactic_features:\n",
    "    print(feat, round(corpus_de[feat].mean(),2), round(corpus_de[feat].min(),2), \n",
    "          round(corpus_de[feat].max(),2), round(corpus_de[feat].std(),2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13f2e082-3e44-44df-9593-2e6d49a2a1fb",
   "metadata": {},
   "source": [
    "#### Save the prepared corpus for OLS regression\n",
    "\n",
    "The OLS regression is conducted in the R script."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3ad1f225-2515-43f2-ae86-e924db141f55",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11289, 43)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_de.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6ae424f3-f6b5-450e-981c-0956dcf8e7e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_de.to_csv('corpus_de_ols_1_nondummy_nomv.csv', sep=\"\\t\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5697c68-882d-4589-8210-f3dbbadaf631",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}