00-00-convert_nyt_to_json.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert NYT to json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import\n",
    "import tarfile\n",
    "import os\n",
    "import shutil\n",
    "import glob\n",
    "import xml.etree.ElementTree as ET\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract a zip file, return the output folder name\n",
    "def extract_zip(zip_name):\n",
    "  # output folder name\n",
    "  output_folder = zip_name.split(\".\")[-2]\n",
    "  # remove old output folder\n",
    "  if os.path.exists(output_folder):\n",
    "    shutil.rmtree(output_folder)\n",
    "  # Extract zip\n",
    "  nytzipf = tarfile.open(zip_name)\n",
    "  nytzipf.extractall(output_folder)\n",
    "  nytzipf.close()\n",
    "  return output_folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'data/nyt_corpus_LDC2008T19.tgz'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m~/tmp/ipykernel_671227/1467151236.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Extract zip\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmain_folder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_zip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"data\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"nyt_corpus_LDC2008T19.tgz\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/tmp/ipykernel_671227/3126422280.py\u001b[0m in \u001b[0;36mextract_zip\u001b[0;34m(zip_name)\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrmtree\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m   \u001b[0;31m# Extract zip\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m   \u001b[0mnytzipf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m   \u001b[0mnytzipf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m   \u001b[0mnytzipf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(cls, name, mode, fileobj, bufsize, **kwargs)\u001b[0m\n\u001b[1;32m   1618\u001b[0m                     \u001b[0msaved_pos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfileobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1619\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1620\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1621\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mReadError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCompressionError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1622\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mfileobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mgzopen\u001b[0;34m(cls, name, mode, fileobj, compresslevel, **kwargs)\u001b[0m\n\u001b[1;32m   1682\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1683\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1684\u001b[0;31m             \u001b[0mfileobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGzipFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"b\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompresslevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1685\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1686\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mfileobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/gzip.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, filename, mode, compresslevel, fileobj, mtime)\u001b[0m\n\u001b[1;32m    171\u001b[0m             \u001b[0mmode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m'b'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    172\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfileobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m             \u001b[0mfileobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmyfileobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuiltins\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    174\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfilename\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m             \u001b[0mfilename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfileobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'name'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/nyt_corpus_LDC2008T19.tgz'"
     ]
    }
   ],
   "source": [
    "# Extract zip\n",
    "main_folder = extract_zip(os.path.join(\"data\", \"nyt_corpus_LDC2008T19.tgz\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data folder\n",
    "data_folder = os.path.join(main_folder, \"nyt_corpus\", \"data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of all sub zip data/year/month.zip (data/YYYY/MM.zip)\n",
    "zip_files = glob.glob(os.path.join(data_folder, \"*\", \"*.tgz\"))\n",
    "zip_files.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# text and abstract lists\n",
    "docs = []\n",
    "summaries = []\n",
    "paths = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For each zip\n",
    "for zip_file in zip_files:\n",
    "  month_f = extract_zip(zip_file)\n",
    "  xmls = glob.glob(os.path.join(month_f, \"*\", \"*\", \"*.xml\"))\n",
    "  xmls.sort()\n",
    "  for xml in xmls:\n",
    "    tree = ET.parse(xml)\n",
    "    root = tree.getroot()\n",
    "    abstract = root.find(\".//abstract\")\n",
    "    if abstract is not None:\n",
    "      full_text_balise = root.find('.//block[@class=\"full_text\"]')\n",
    "      if full_text_balise is not None:\n",
    "        paths.append(xml)\n",
    "        summaries.append(ET.tostring(abstract, encoding=\"utf-8\").decode(\"utf-8\"))\n",
    "        docs.append(ET.tostring(full_text_balise, encoding=\"utf-8\").decode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "654872"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of entries\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save panda dataframe to json\n",
    "df = pd.DataFrame(data={\"paths\": paths, \"docs\": docs, \"summaries\": summaries})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "re_html = re.compile(r'<[^>]+>')\n",
    "df[\"docs\"] = df[\"docs\"].apply(lambda x : re_html.sub('', x))\n",
    "df[\"summaries\"] = df[\"summaries\"].apply(lambda x : re_html.sub('', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json(main_folder + \".json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6d7f428a150b92572ac46240b6d7ae68586908362b054f21341550673eeb77dc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}