bt-classification-dbpedia.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b306adea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loads RDD\n",
    "s3_bucket= # s3 bucket name\n",
    "s3_key = \"fast-ai-nlp/dbpedia_csv/train.csv\"\n",
    "dbpedia_csv_train = f\"s3a://{s3_bucket}/{s3_key}\"\n",
    "train_rdd = sc.textFile(dbpedia_csv_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01127a1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split lines into columns; change split() argument depending on deliminiter\n",
    "parts = train_rdd.map(lambda l: l.split(','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28c62a28",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = parts.map(lambda p: f\"__label__{p[0]} {p[2]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94532cd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "771474cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized = sentences.map(lambda x: x.replace('\"', '').replace(\".\", \" . \").replace(\",\", \" , \").replace(\"-\", \" - \").replace(\"'\", \"\").replace(\"(\", \" ( \").replace(\")\", \" ) \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeea79f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_rdd_1 = tokenized.repartition(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "090bd02a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sc._jsc.hadoopConfiguration().set(\"mapred.output.committer.class\", \"org.apache.hadoop.mapred.DirectFileOutputCommitter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f38f437",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_out_url = f\"s3a://{s3_bucket}/blazing-text/classification/dbpedia/train\"\n",
    "print(f\"Save: {s3_out_url}\")\n",
    "tokenized_rdd_1.saveAsTextFile(s3_out_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a60913ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loads RDD\n",
    "s3_key = \"fast-ai-nlp/dbpedia_csv/test.csv\"\n",
    "dbpedia_csv_validation = f\"s3a://{s3_bucket}/{s3_key}\"\n",
    "validation_rdd = sc.textFile(dbpedia_csv_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9f58e83",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split lines into columns; change split() argument depending on deliminiter\n",
    "parts_validation = validation_rdd.map(lambda l: l.split(','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5eccfd93",
   "metadata": {},
   "outputs": [],
   "source": [
    "v_sentences = parts_validation.map(lambda p: f\"__label__{p[0]} {p[2]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b05175a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "v_sentences.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "057fc98e",
   "metadata": {},
   "outputs": [],
   "source": [
    "v_tokenized = v_sentences.map(lambda x: x.replace('\"', '').replace(\".\", \" . \").replace(\",\", \" , \").replace(\"-\", \" - \").replace(\"'\", \"\").replace(\"(\", \" ( \").replace(\")\", \" ) \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bea1d85",
   "metadata": {},
   "outputs": [],
   "source": [
    "v_tokenized_rdd_1 = v_tokenized.repartition(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b67fc8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_validation_url = f\"s3a://{s3_bucket}/blazing-text/classification/dbpedia/validation\"\n",
    "print(f\"Save: {s3_validation_url}\")\n",
    "v_tokenized_rdd_1.saveAsTextFile(s3_validation_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d799179a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Sparkmagic (PySpark)",
   "language": "python",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}