dataset_cleaning_for_DL_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'track_id', u'tags', u'song_id', u'title', u'preview_url',\n",
      "       u'preview_info', u'genres', u'preview_file'],\n",
      "      dtype='object')\n",
      "Index([u'track_id', u'tags', u'song_id', u'title', u'preview_url',\n",
      "       u'preview_info', u'genres', u'preview_file', u'moods', u'tempos',\n",
      "       u'gracenote'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "df1 = pandas.read_csv('./dataset/selected_song_based_on_tags.csv', sep='\\t')\n",
    "df2 = pandas.read_csv('./dataset/selected_song_based_on_tags (copy).csv', sep='\\t')\n",
    "\n",
    "print df1.columns\n",
    "print df2.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "songs with preview files: 2846\n",
      "songs with mood data: 3248\n"
     ]
    }
   ],
   "source": [
    "print 'songs with preview files: ' + str(len(df1[df1['preview_file']!='not found']))\n",
    "print 'songs with mood data: ' + str(len(df2[df2['moods']!='not found']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'track_id', u'song_id', u'preview_info', u'preview_file'], dtype='object')\n",
      "Index([u'track_id', u'song_id', u'title', u'preview_file'], dtype='object')\n",
      "Index([u'track_id', u'genres', u'moods', u'tempos'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "minimized_df1 = df1.drop(['title', 'tags', 'genres', 'preview_url'], axis=1)\n",
    "print (minimized_df1.columns)\n",
    "minimized_df1.columns = ['track_id', 'song_id', 'title', 'preview_file']\n",
    "print (minimized_df1.columns)\n",
    "\n",
    "minimized_df2 = df2.drop(['tags', 'song_id', 'title', 'preview_url', 'preview_file', 'preview_info', 'gracenote'], axis=1)\n",
    "print (minimized_df2.columns)\n",
    "\n",
    "# use_df = pandas.merge(minimized_df1, minimized_df2, on='track_id', how='inner')\n",
    "# minimized_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_cell(content):\n",
    "    char_to_delete = ['[', ']', \"'\", '\"']\n",
    "\n",
    "    for i in char_to_delete:\n",
    "#         print i\n",
    "        content=content.replace(i, '')\n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3249\n",
      "2846\n",
      "TRAAAAW128F429D538-mzm.jmksdiul.aac.p.m4a\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_id</th>\n",
       "      <th>song_id</th>\n",
       "      <th>title</th>\n",
       "      <th>preview_file</th>\n",
       "      <th>genres</th>\n",
       "      <th>moods</th>\n",
       "      <th>tempos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TRAAAAW128F429D538</td>\n",
       "      <td>SOMZWCG12A8C13C480</td>\n",
       "      <td>I Didn't Mean To - Casual</td>\n",
       "      <td>TRAAAAW128F429D538-mzm.jmksdiul.aac.p.m4a</td>\n",
       "      <td>['Urban', 'Western Hip-Hop/Rap', 'Conscious Hi...</td>\n",
       "      <td>['Cool', 'Wary / Defiant']</td>\n",
       "      <td>['Slow Tempo', 'Slow', '40s']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRAAABD128F429CF47</td>\n",
       "      <td>SOCIWDW12A8C13D406</td>\n",
       "      <td>The Letter - The Box Tops</td>\n",
       "      <td>TRAAABD128F429CF47-mzm.maejowgk.aac.p.m4a</td>\n",
       "      <td>['Rock', \"60's Rock\", \"Early 60's Pop-Rock\"]</td>\n",
       "      <td>['Upbeat', 'Carefree Pop']</td>\n",
       "      <td>['Medium Tempo', 'Medium Fast', '120s']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TRAAAEF128F4273421</td>\n",
       "      <td>SONHOTT12A8C13493C</td>\n",
       "      <td>Something Girls - Adam Ant</td>\n",
       "      <td>TRAAAEF128F4273421-mzm.wvrnqeth.aac.p.m4a</td>\n",
       "      <td>['Alternative &amp; Punk', 'New Wave Pop', 'New Wa...</td>\n",
       "      <td>['Yearning', 'Bittersweet Pop']</td>\n",
       "      <td>['Medium Tempo', 'Medium Fast', '120s']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TRAAAFD128F92F423A</td>\n",
       "      <td>SOFSOCN12A8C143F5D</td>\n",
       "      <td>Face the Ashes - Gob</td>\n",
       "      <td>TRAAAFD128F92F423A-mzaf_7872227378043099108.pl...</td>\n",
       "      <td>['Alternative &amp; Punk', 'Punk', 'Pop Punk']</td>\n",
       "      <td>['Defiant', 'Heavy Brooding']</td>\n",
       "      <td>['Medium Tempo', 'Medium Fast', '120s']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TRAABLR128F423B7E3</td>\n",
       "      <td>SOHUOAP12A8AE488E9</td>\n",
       "      <td>Floating - Blue Rodeo</td>\n",
       "      <td>TRAABLR128F423B7E3-mzi.hrspqvdo.aac.p.m4a</td>\n",
       "      <td>['Rock', 'Alternative Roots', 'Alt Country']</td>\n",
       "      <td>['Sensual', 'Sensual Groove']</td>\n",
       "      <td>['Medium Tempo', 'Medium Slow', '50s']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             track_id             song_id                       title  \\\n",
       "0  TRAAAAW128F429D538  SOMZWCG12A8C13C480   I Didn't Mean To - Casual   \n",
       "1  TRAAABD128F429CF47  SOCIWDW12A8C13D406   The Letter - The Box Tops   \n",
       "2  TRAAAEF128F4273421  SONHOTT12A8C13493C  Something Girls - Adam Ant   \n",
       "3  TRAAAFD128F92F423A  SOFSOCN12A8C143F5D        Face the Ashes - Gob   \n",
       "4  TRAABLR128F423B7E3  SOHUOAP12A8AE488E9       Floating - Blue Rodeo   \n",
       "\n",
       "                                        preview_file  \\\n",
       "0          TRAAAAW128F429D538-mzm.jmksdiul.aac.p.m4a   \n",
       "1          TRAAABD128F429CF47-mzm.maejowgk.aac.p.m4a   \n",
       "2          TRAAAEF128F4273421-mzm.wvrnqeth.aac.p.m4a   \n",
       "3  TRAAAFD128F92F423A-mzaf_7872227378043099108.pl...   \n",
       "4          TRAABLR128F423B7E3-mzi.hrspqvdo.aac.p.m4a   \n",
       "\n",
       "                                              genres  \\\n",
       "0  ['Urban', 'Western Hip-Hop/Rap', 'Conscious Hi...   \n",
       "1       ['Rock', \"60's Rock\", \"Early 60's Pop-Rock\"]   \n",
       "2  ['Alternative & Punk', 'New Wave Pop', 'New Wa...   \n",
       "3         ['Alternative & Punk', 'Punk', 'Pop Punk']   \n",
       "4       ['Rock', 'Alternative Roots', 'Alt Country']   \n",
       "\n",
       "                             moods                                   tempos  \n",
       "0       ['Cool', 'Wary / Defiant']            ['Slow Tempo', 'Slow', '40s']  \n",
       "1       ['Upbeat', 'Carefree Pop']  ['Medium Tempo', 'Medium Fast', '120s']  \n",
       "2  ['Yearning', 'Bittersweet Pop']  ['Medium Tempo', 'Medium Fast', '120s']  \n",
       "3    ['Defiant', 'Heavy Brooding']  ['Medium Tempo', 'Medium Fast', '120s']  \n",
       "4    ['Sensual', 'Sensual Groove']   ['Medium Tempo', 'Medium Slow', '50s']  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "use_df = pandas.merge(minimized_df1, minimized_df2, on='track_id', how='inner')\n",
    "print(len(use_df))\n",
    "use_df = use_df[(use_df['title'] != 'not found') & (use_df['preview_file'] != 'not found')  & (use_df['genres'] != 'not found') & (use_df['moods'] != 'not found') & (use_df['tempos'] != 'not found')]\n",
    "print(len(use_df))\n",
    "a = use_df['preview_file'][0]\n",
    "print(a[24:])\n",
    "\n",
    "def fixPreviewFile(content):\n",
    "    return content[24:]\n",
    "\n",
    "use_df['preview_file'] = use_df['preview_file'].apply(fixPreviewFile)\n",
    "\n",
    "use_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_id</th>\n",
       "      <th>song_id</th>\n",
       "      <th>title</th>\n",
       "      <th>preview_file</th>\n",
       "      <th>genres</th>\n",
       "      <th>moods</th>\n",
       "      <th>tempos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TRAAAAW128F429D538</td>\n",
       "      <td>SOMZWCG12A8C13C480</td>\n",
       "      <td>I Didn't Mean To - Casual</td>\n",
       "      <td>TRAAAAW128F429D538-mzm.jmksdiul.aac.p.m4a</td>\n",
       "      <td>Urban, Western Hip-Hop/Rap, Conscious Hip-Hop/Rap</td>\n",
       "      <td>Cool, Wary / Defiant</td>\n",
       "      <td>Slow Tempo, Slow, 40s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRAAABD128F429CF47</td>\n",
       "      <td>SOCIWDW12A8C13D406</td>\n",
       "      <td>The Letter - The Box Tops</td>\n",
       "      <td>TRAAABD128F429CF47-mzm.maejowgk.aac.p.m4a</td>\n",
       "      <td>Rock, 60s Rock, Early 60s Pop-Rock</td>\n",
       "      <td>Upbeat, Carefree Pop</td>\n",
       "      <td>Medium Tempo, Medium Fast, 120s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TRAAAEF128F4273421</td>\n",
       "      <td>SONHOTT12A8C13493C</td>\n",
       "      <td>Something Girls - Adam Ant</td>\n",
       "      <td>TRAAAEF128F4273421-mzm.wvrnqeth.aac.p.m4a</td>\n",
       "      <td>Alternative &amp; Punk, New Wave Pop, New Wave Pop</td>\n",
       "      <td>Yearning, Bittersweet Pop</td>\n",
       "      <td>Medium Tempo, Medium Fast, 120s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TRAAAFD128F92F423A</td>\n",
       "      <td>SOFSOCN12A8C143F5D</td>\n",
       "      <td>Face the Ashes - Gob</td>\n",
       "      <td>TRAAAFD128F92F423A-mzaf_7872227378043099108.pl...</td>\n",
       "      <td>Alternative &amp; Punk, Punk, Pop Punk</td>\n",
       "      <td>Defiant, Heavy Brooding</td>\n",
       "      <td>Medium Tempo, Medium Fast, 120s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TRAABLR128F423B7E3</td>\n",
       "      <td>SOHUOAP12A8AE488E9</td>\n",
       "      <td>Floating - Blue Rodeo</td>\n",
       "      <td>TRAABLR128F423B7E3-mzi.hrspqvdo.aac.p.m4a</td>\n",
       "      <td>Rock, Alternative Roots, Alt Country</td>\n",
       "      <td>Sensual, Sensual Groove</td>\n",
       "      <td>Medium Tempo, Medium Slow, 50s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             track_id             song_id                       title  \\\n",
       "0  TRAAAAW128F429D538  SOMZWCG12A8C13C480   I Didn't Mean To - Casual   \n",
       "1  TRAAABD128F429CF47  SOCIWDW12A8C13D406   The Letter - The Box Tops   \n",
       "2  TRAAAEF128F4273421  SONHOTT12A8C13493C  Something Girls - Adam Ant   \n",
       "3  TRAAAFD128F92F423A  SOFSOCN12A8C143F5D        Face the Ashes - Gob   \n",
       "4  TRAABLR128F423B7E3  SOHUOAP12A8AE488E9       Floating - Blue Rodeo   \n",
       "\n",
       "                                        preview_file  \\\n",
       "0          TRAAAAW128F429D538-mzm.jmksdiul.aac.p.m4a   \n",
       "1          TRAAABD128F429CF47-mzm.maejowgk.aac.p.m4a   \n",
       "2          TRAAAEF128F4273421-mzm.wvrnqeth.aac.p.m4a   \n",
       "3  TRAAAFD128F92F423A-mzaf_7872227378043099108.pl...   \n",
       "4          TRAABLR128F423B7E3-mzi.hrspqvdo.aac.p.m4a   \n",
       "\n",
       "                                              genres  \\\n",
       "0  Urban, Western Hip-Hop/Rap, Conscious Hip-Hop/Rap   \n",
       "1                 Rock, 60s Rock, Early 60s Pop-Rock   \n",
       "2     Alternative & Punk, New Wave Pop, New Wave Pop   \n",
       "3                 Alternative & Punk, Punk, Pop Punk   \n",
       "4               Rock, Alternative Roots, Alt Country   \n",
       "\n",
       "                       moods                           tempos  \n",
       "0       Cool, Wary / Defiant            Slow Tempo, Slow, 40s  \n",
       "1       Upbeat, Carefree Pop  Medium Tempo, Medium Fast, 120s  \n",
       "2  Yearning, Bittersweet Pop  Medium Tempo, Medium Fast, 120s  \n",
       "3    Defiant, Heavy Brooding  Medium Tempo, Medium Fast, 120s  \n",
       "4    Sensual, Sensual Groove   Medium Tempo, Medium Slow, 50s  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df = use_df\n",
    "cleaned_df['genres'] = cleaned_df['genres'].apply(clean_cell)\n",
    "cleaned_df['moods'] = cleaned_df['moods'].apply(clean_cell)\n",
    "cleaned_df['tempos'] = cleaned_df['tempos'].apply(clean_cell)\n",
    "\n",
    "cleaned_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres_df = pandas.DataFrame()\n",
    "\n",
    "moods_df = pandas.DataFrame()\n",
    "\n",
    "tempos_df = pandas.DataFrame()\n",
    "\n",
    "def df_splitter(orig_df, axis, label_concern):\n",
    "    temp_df = pandas.DataFrame()\n",
    "    for idx, row in orig_df.iterrows():\n",
    "        items = row[label_concern].split(',')\n",
    "        for i in items:\n",
    "            temp_df = temp_df.append([[row[axis], i.strip()]])\n",
    "            \n",
    "    return temp_df\n",
    "\n",
    "genres_df = df_splitter(cleaned_df, 'track_id', 'genres')\n",
    "moods_df = df_splitter(cleaned_df, 'track_id', 'moods')\n",
    "tempos_df = df_splitter(cleaned_df, 'track_id', 'tempos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "370\n",
      "Index([u'Alternative & Punk', u'Rock', u'Traditional', u'Urban', u'Pop',\n",
      "       u'Other', u'Western Hip-Hop/Rap', u'Metal', u'Western Pop',\n",
      "       u'Electronica',\n",
      "       ...\n",
      "       u'Merengue', u'Memphis Blues', u'Latin Metal', u'Japanese Pop',\n",
      "       u'J-Pop', u'Indian Subcontinent Pop', u'Grindcore',\n",
      "       u'General Spoken Word', u'General Indian Pop', u'Gospel Small Group'],\n",
      "      dtype='object', name=u'genre', length=370)\n",
      "Index([u'Alternative & Punk', u'Rock', u'Traditional', u'Urban', u'Pop',\n",
      "       u'Other', u'Western Hip-Hop/Rap', u'Metal', u'Western Pop',\n",
      "       u'Electronica', u'Punk', u'Indie Rock', u'Alternative', u'70s Rock',\n",
      "       u'Adult Alternative Rock', u'Electric Blues', u'Country', u'Jazz',\n",
      "       u'Contemporary R&B/Soul', u'Alternative Rock', u'Acoustic Blues',\n",
      "       u'60s Rock', u'Classic Country', u'Emo & Hardcore', u'Mainstream Rock',\n",
      "       u'Classic R&B/Soul', u'Synth Pop', u'General Mainstream Rock',\n",
      "       u'Pop Punk', u'Adult Alternative Pop', u'Black Metal',\n",
      "       u'Old School Hip-Hop/Rap', u'Latin Pop', u'General Latin Pop',\n",
      "       u'Brit Rock', u'New Wave Pop', u'Classic Hard Rock',\n",
      "       u'Adult Contemporary', u'East Coast Rap', u'European Pop',\n",
      "       u'Latin Rock', u'Hard Rock', u'Religious'],\n",
      "      dtype='object', name=u'genre')\n",
      "8572\n",
      "5639\n",
      "2801\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genre_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Alternative &amp; Punk</th>\n",
       "      <td>630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rock</th>\n",
       "      <td>609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Traditional</th>\n",
       "      <td>395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Urban</th>\n",
       "      <td>378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pop</th>\n",
       "      <td>272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other</th>\n",
       "      <td>245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Western Hip-Hop/Rap</th>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Metal</th>\n",
       "      <td>210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Western Pop</th>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Electronica</th>\n",
       "      <td>173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Punk</th>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Indie Rock</th>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Alternative</th>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70s Rock</th>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Adult Alternative Rock</th>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Electric Blues</th>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Country</th>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jazz</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Contemporary R&amp;B/Soul</th>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Alternative Rock</th>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Acoustic Blues</th>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60s Rock</th>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Classic Country</th>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Emo &amp; Hardcore</th>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mainstream Rock</th>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Classic R&amp;B/Soul</th>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Synth Pop</th>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>General Mainstream Rock</th>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pop Punk</th>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Adult Alternative Pop</th>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Black Metal</th>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Old School Hip-Hop/Rap</th>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Latin Pop</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>General Latin Pop</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brit Rock</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New Wave Pop</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Classic Hard Rock</th>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Adult Contemporary</th>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>East Coast Rap</th>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>European Pop</th>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Latin Rock</th>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hard Rock</th>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Religious</th>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         genre_count\n",
       "genre                               \n",
       "Alternative & Punk               630\n",
       "Rock                             609\n",
       "Traditional                      395\n",
       "Urban                            378\n",
       "Pop                              272\n",
       "Other                            245\n",
       "Western Hip-Hop/Rap              227\n",
       "Metal                            210\n",
       "Western Pop                      199\n",
       "Electronica                      173\n",
       "Punk                             160\n",
       "Indie Rock                       133\n",
       "Alternative                      133\n",
       "70s Rock                         125\n",
       "Adult Alternative Rock           106\n",
       "Electric Blues                   104\n",
       "Country                          101\n",
       "Jazz                              99\n",
       "Contemporary R&B/Soul             92\n",
       "Alternative Rock                  80\n",
       "Acoustic Blues                    70\n",
       "60s Rock                          69\n",
       "Classic Country                   67\n",
       "Emo & Hardcore                    59\n",
       "Mainstream Rock                   59\n",
       "Classic R&B/Soul                  58\n",
       "Synth Pop                         54\n",
       "General Mainstream Rock           53\n",
       "Pop Punk                          53\n",
       "Adult Alternative Pop             52\n",
       "Black Metal                       49\n",
       "Old School Hip-Hop/Rap            48\n",
       "Latin Pop                         46\n",
       "General Latin Pop                 46\n",
       "Brit Rock                         46\n",
       "New Wave Pop                      46\n",
       "Classic Hard Rock                 43\n",
       "Adult Contemporary                43\n",
       "East Coast Rap                    43\n",
       "European Pop                      42\n",
       "Latin Rock                        41\n",
       "Hard Rock                         41\n",
       "Religious                         40"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genres_df.columns = ['track_id', 'genre']  \n",
    "grouped_genres = genres_df.groupby('genre').agg({'genre': 'count'}).rename(columns={'genre': 'genre_count'}).sort_values('genre_count', ascending=False)\n",
    "print(len(grouped_genres))\n",
    "print grouped_genres.index\n",
    "# for i in grouped_genres.index:\n",
    "#     print i\n",
    "selected_genre = grouped_genres.head(43)\n",
    "print(selected_genre.index)\n",
    "selected_genre_df = genres_df[genres_df['genre'].isin(list(selected_genre.index))].reset_index(drop=True)\n",
    "print len(genres_df)\n",
    "print len(selected_genre_df)\n",
    "print len(selected_genre_df['track_id'].unique())\n",
    "selected_genre.head(43)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "125\n",
      "Index([u'Defiant', u'Empowering', u'Cool', u'Aggressive', u'Sensual',\n",
      "       u'Brooding', u'Excited', u'Rowdy', u'Urgent', u'Energizing',\n",
      "       ...\n",
      "       u'Mysterious / Dreamy', u'Romantic / Lyrical', u'Thrilling',\n",
      "       u'Hypnotic Rhythm', u'Solemn / Spiritual', u'Dark Cosmic',\n",
      "       u'Intimate Bittersweet', u'Invigorating / Joyous',\n",
      "       u'Awakening / Stately', u'Melodramatic'],\n",
      "      dtype='object', name=u'mood', length=125)\n",
      "           mood_count\n",
      "mood                 \n",
      "happiness        2073\n",
      "anger            1424\n",
      "sadness           999\n",
      "neutral           996\n",
      "                   83\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mood_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mood</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Defiant</th>\n",
       "      <td>268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Empowering</th>\n",
       "      <td>248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cool</th>\n",
       "      <td>217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Aggressive</th>\n",
       "      <td>207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sensual</th>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            mood_count\n",
       "mood                  \n",
       "Defiant            268\n",
       "Empowering         248\n",
       "Cool               217\n",
       "Aggressive         207\n",
       "Sensual            171"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "moods_df.columns = ['track_id', 'mood']\n",
    "grouped_moods = moods_df.groupby('mood').agg({'mood': 'count'}).sort_values('mood', ascending=False)\n",
    "print(len(grouped_moods))\n",
    "print grouped_moods.index\n",
    "# for i in grouped_moods.index:\n",
    "#     print i\n",
    "\n",
    "happiness_file = open('./dataset/deep_learning/mood_class/happiness.txt', 'rU')\n",
    "anger_file = open('./dataset/deep_learning/mood_class/anger.txt', 'rU')\n",
    "sadness_file = open('./dataset/deep_learning/mood_class/sadness.txt', 'rU')\n",
    "neutral_file = open('./dataset/deep_learning/mood_class/neutral.txt', 'rU')\n",
    "\n",
    "def iterateContent(orig_file):\n",
    "    temp = []\n",
    "    for line in orig_file.readlines():\n",
    "        temp.append(line.strip())\n",
    "    return temp\n",
    "\n",
    "selected_moods_df = moods_df.copy()\n",
    "\n",
    "happiness = iterateContent(happiness_file)\n",
    "anger = iterateContent(anger_file)\n",
    "sadness = iterateContent(sadness_file)\n",
    "neutral = iterateContent(neutral_file)\n",
    "\n",
    "selected_moods_df.loc[selected_moods_df['mood'].isin(happiness), 'mood'] = 'happiness'\n",
    "selected_moods_df.loc[selected_moods_df['mood'].isin(anger), 'mood'] = 'anger'\n",
    "selected_moods_df.loc[selected_moods_df['mood'].isin(sadness), 'mood'] = 'sadness'\n",
    "selected_moods_df.loc[selected_moods_df['mood'].isin(neutral), 'mood'] = 'neutral'\n",
    "selected_moods_df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "print selected_moods_df.groupby('mood').agg({'mood': 'count'}).rename(columns={'mood': 'mood_count'}).sort_values('mood_count', ascending=False).head()\n",
    "moods_df.groupby('mood').agg({'mood': 'count'}).rename(columns={'mood': 'mood_count'}).sort_values('mood_count', ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37\n",
      "Index([u'Medium Tempo', u'Medium Fast', u'Medium', u'Fast Tempo', u'Fast',\n",
      "       u'Medium Slow', u'Slow Tempo', u'60s', u'80s', u'90s', u'120s', u'40s',\n",
      "       u'Slow', u'70s', u'50s', u'100s', u'110s', u'130s', u'140s',\n",
      "       u'Very Slow', u'30s', u'150s', u'', u'Very Fast', u'160s', u'180s',\n",
      "       u'190s', u'170s', u'0', u'Static', u'220s', u'250s', u'210s', u'20s',\n",
      "       u'200s', u'240s', u'230s'],\n",
      "      dtype='object', name=u'tempo')\n",
      "8372\n",
      "2846\n",
      "2846\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tempo_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tempo</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>medium</th>\n",
       "      <td>5862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fast</th>\n",
       "      <td>1368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slow</th>\n",
       "      <td>1059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        tempo_count\n",
       "tempo              \n",
       "medium         5862\n",
       "fast           1368\n",
       "slow           1059\n",
       "                 83"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tempos_df.columns = ['track_id', 'tempo']\n",
    "grouped_tempos = tempos_df.groupby('tempo').agg({'tempo': 'count'}).sort_values('tempo', ascending=False)\n",
    "print(len(grouped_tempos))\n",
    "print(grouped_tempos.index)\n",
    "# for i in grouped_tempos.index:\n",
    "#     print i\n",
    "\n",
    "slow_file = open('./dataset/deep_learning/tempo_class/slow.txt', 'rU')\n",
    "medium_file = open('./dataset/deep_learning/tempo_class/medium.txt', 'rU')\n",
    "fast_file = open('./dataset/deep_learning/tempo_class/fast.txt', 'rU')\n",
    "\n",
    "def iterateContent(orig_file):\n",
    "    temp = []\n",
    "    for line in orig_file.readlines():\n",
    "        temp.append(line.strip())\n",
    "    return temp\n",
    "\n",
    "selected_tempos_df = tempos_df.copy()\n",
    "\n",
    "slow = iterateContent(slow_file)\n",
    "medium = iterateContent(medium_file)\n",
    "fast = iterateContent(fast_file)\n",
    "\n",
    "selected_tempos_df.loc[selected_tempos_df['tempo'].isin(slow), 'tempo'] = 'slow'\n",
    "selected_tempos_df.loc[selected_tempos_df['tempo'].isin(medium), 'tempo'] = 'medium'\n",
    "selected_tempos_df.loc[selected_tempos_df['tempo'].isin(fast), 'tempo'] = 'fast'\n",
    "\n",
    "print len(selected_tempos_df)\n",
    "print len(selected_tempos_df['track_id'].unique())\n",
    "print len(selected_tempos_df.drop_duplicates())\n",
    "\n",
    "selected_tempos_df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# grouped_tempos\n",
    "selected_tempos_df.groupby('tempo').agg({'tempo': 'count'}).rename(columns={'tempo': 'tempo_count'}).sort_values('tempo_count', ascending=False).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## creating genre label maps of the songs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_id</th>\n",
       "      <th>Alternative &amp; Punk</th>\n",
       "      <th>Rock</th>\n",
       "      <th>Traditional</th>\n",
       "      <th>Urban</th>\n",
       "      <th>Pop</th>\n",
       "      <th>Other</th>\n",
       "      <th>Western Hip-Hop/Rap</th>\n",
       "      <th>Metal</th>\n",
       "      <th>Western Pop</th>\n",
       "      <th>...</th>\n",
       "      <th>General Latin Pop</th>\n",
       "      <th>Brit Rock</th>\n",
       "      <th>New Wave Pop</th>\n",
       "      <th>Classic Hard Rock</th>\n",
       "      <th>Adult Contemporary</th>\n",
       "      <th>East Coast Rap</th>\n",
       "      <th>European Pop</th>\n",
       "      <th>Latin Rock</th>\n",
       "      <th>Hard Rock</th>\n",
       "      <th>Religious</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TRAAAAW128F429D538</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRAAABD128F429CF47</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TRAAAEF128F4273421</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TRAAAFD128F92F423A</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TRAABLR128F423B7E3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 44 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             track_id  Alternative & Punk  Rock  Traditional  Urban  Pop  \\\n",
       "0  TRAAAAW128F429D538                 0.0   0.0          0.0    1.0  0.0   \n",
       "1  TRAAABD128F429CF47                 0.0   1.0          0.0    0.0  0.0   \n",
       "2  TRAAAEF128F4273421                 1.0   0.0          0.0    0.0  0.0   \n",
       "3  TRAAAFD128F92F423A                 1.0   0.0          0.0    0.0  0.0   \n",
       "4  TRAABLR128F423B7E3                 0.0   1.0          0.0    0.0  0.0   \n",
       "\n",
       "   Other  Western Hip-Hop/Rap  Metal  Western Pop    ...      \\\n",
       "0    0.0                  1.0    0.0          0.0    ...       \n",
       "1    0.0                  0.0    0.0          0.0    ...       \n",
       "2    0.0                  0.0    0.0          0.0    ...       \n",
       "3    0.0                  0.0    0.0          0.0    ...       \n",
       "4    0.0                  0.0    0.0          0.0    ...       \n",
       "\n",
       "   General Latin Pop  Brit Rock  New Wave Pop  Classic Hard Rock  \\\n",
       "0                0.0        0.0           0.0                0.0   \n",
       "1                0.0        0.0           0.0                0.0   \n",
       "2                0.0        0.0           1.0                0.0   \n",
       "3                0.0        0.0           0.0                0.0   \n",
       "4                0.0        0.0           0.0                0.0   \n",
       "\n",
       "   Adult Contemporary  East Coast Rap  European Pop  Latin Rock  Hard Rock  \\\n",
       "0                 0.0             0.0           0.0         0.0        0.0   \n",
       "1                 0.0             0.0           0.0         0.0        0.0   \n",
       "2                 0.0             0.0           0.0         0.0        0.0   \n",
       "3                 0.0             0.0           0.0         0.0        0.0   \n",
       "4                 0.0             0.0           0.0         0.0        0.0   \n",
       "\n",
       "   Religious  \n",
       "0        0.0  \n",
       "1        0.0  \n",
       "2        0.0  \n",
       "3        0.0  \n",
       "4        0.0  \n",
       "\n",
       "[5 rows x 44 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "song_data_df = use_df.drop(['genres', 'moods', 'tempos'], axis=1)\n",
    "genre_label_map_df = pandas.DataFrame(columns = ['track_id'] + list(selected_genre.index))\n",
    "\n",
    "for idx,row in selected_genre_df.iterrows():\n",
    "#     print genre_label_map_df[genre_label_map_df['track_id'] == row['track_id']].empty\n",
    "    a = row['genre']\n",
    "    if genre_label_map_df[genre_label_map_df['track_id'] == row['track_id']].empty:\n",
    "        genre_label_map_df = genre_label_map_df.append({\n",
    "            'track_id': row['track_id'],\n",
    "            row['genre']: 1\n",
    "        }, ignore_index=True).fillna(0)\n",
    "    else:\n",
    "#         print idx\n",
    "#         print row['track_id']\n",
    "#         print genre_label_map_df.head()\n",
    "#         print genre_label_map_df.iloc[0]\n",
    "#         genre_label_map_df.iloc[genre_label_map_df['track_id'] == row['track_id']]+=1\n",
    "#         print genre_label_map_df.loc[genre_label_map_df['track_id'] == row['track_id']][row['genre']]\n",
    "        genre_label_map_df.loc[genre_label_map_df['track_id'] == row['track_id'], [row['genre']]]=1\n",
    "#         print genre_label_map_df.loc[genre_label_map_df['track_id'] == row['track_id']][row['genre']]\n",
    "#         break\n",
    "\n",
    "genre_label_map_df.head()\n",
    "\n",
    "# display = True\n",
    "\n",
    "# def labelCounter1(series):\n",
    "#     count = 0\n",
    "#     global display\n",
    "#     if display == True:\n",
    "#         display=False\n",
    "#         print series\n",
    "#     for i in series:\n",
    "#         if i == 'Urban':\n",
    "#             count+=1\n",
    "#     return count\n",
    "\n",
    "# def labelCounter2(series):\n",
    "#     count = 0\n",
    "#     for i in series:\n",
    "#         if i == 'Rock':\n",
    "#             count+=1\n",
    "#     return count\n",
    "\n",
    "# genre_agg = {}\n",
    "\n",
    "# # for genre in list(selected_genre.index):\n",
    "# for genre in ['Urban','Rock']:\n",
    "#     print genre\n",
    "#     def labelCounter(series):\n",
    "#         count = 0\n",
    "#         global display\n",
    "#         if display == True:\n",
    "#             print series\n",
    "#             print genre\n",
    "#         for i in series:\n",
    "#             if i == genre:\n",
    "#                 if display == True:\n",
    "#                     print '+1'\n",
    "#                 count+=1\n",
    "#         display=False\n",
    "#         return count\n",
    "#     genre_agg[genre] = labelCounter\n",
    "\n",
    "# use_genre_songs_df = selected_genre_df.groupby('track_id').agg({\n",
    "# #     'genre': {\n",
    "# #         'Urban': labelCounter1,\n",
    "# #         'Rock': labelCounter2\n",
    "# #         }\n",
    "#     'genre': genre_agg\n",
    "#     })\n",
    "    \n",
    "# print len(use_genre_songs_df[use_genre_songs_df[('genre', 'Urban')] == 1])\n",
    "# use_genre_songs_df.sort_values([('genre', 'Urban')], ascending=False).head()\n",
    "# # use_genre_songs_df.iloc[630]\n",
    "# # use_genre_songs_df.iloc[use_genre_songs_df.index.get_level_values('track_id') == 'TRAAAEF128F4273421']\n",
    "# # use_genre_songs_df.columns\n",
    "# # selected_genre_df[selected_genre_df['genre'] == 'Alternative & Punk']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import inspect\n",
    "# selected_genre_df[selected_genre_df['track_id']=='TRAAAEF128F4273421']\n",
    "# selected_genre_df[selected_genre_df['genre']=='Urban']\n",
    "# selected_genre_df.iloc[0]\n",
    "\n",
    "# use_genre_songs_df.iloc[227]\n",
    "# use_genre_songs_df.iloc[use_genre_songs_df.index.get_level_values('track_id') == 'TRAAAAW128F429D538']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## creating moods label maps of the songs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_id</th>\n",
       "      <th>happiness</th>\n",
       "      <th>sadness</th>\n",
       "      <th>anger</th>\n",
       "      <th>neutral</th>\n",
       "      <th></th>\n",
       "      <th>Other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TRAAAAW128F429D538</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRAAABD128F429CF47</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TRAAAEF128F4273421</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TRAAAFD128F92F423A</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TRAABLR128F423B7E3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             track_id  happiness  sadness  anger  neutral       Other\n",
       "0  TRAAAAW128F429D538        0.0      0.0    2.0      1.0  0.0    0.0\n",
       "1  TRAAABD128F429CF47        1.0      0.0    0.0      2.0  0.0    0.0\n",
       "2  TRAAAEF128F4273421        0.0      1.0    0.0      2.0  0.0    0.0\n",
       "3  TRAAAFD128F92F423A        0.0      2.0    1.0      0.0  0.0    0.0\n",
       "4  TRAABLR128F423B7E3        0.0      0.0    0.0      2.0  0.0    0.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unnormalized_mood_label_map_df = pandas.DataFrame(columns = ['track_id'] + ['happiness', 'sadness', 'anger', 'neutral'])\n",
    "\n",
    "for idx,row in selected_moods_df.iterrows():\n",
    "#     print unnormalized_mood_label_map_df[unnormalized_mood_label_map_df['track_id'] == row['track_id']].empty\n",
    "    a = row['mood']\n",
    "    if unnormalized_mood_label_map_df[unnormalized_mood_label_map_df['track_id'] == row['track_id']].empty:\n",
    "        unnormalized_mood_label_map_df = unnormalized_mood_label_map_df.append({\n",
    "            'track_id': row['track_id'],\n",
    "            row['mood']: 1\n",
    "        }, ignore_index=True).fillna(0)\n",
    "    else:\n",
    "        unnormalized_mood_label_map_df.loc[unnormalized_mood_label_map_df['track_id'] == row['track_id'], [row['mood']]]=1\n",
    "        unnormalized_mood_label_map_df.loc[unnormalized_mood_label_map_df['track_id'] == row['track_id'], [row['mood']]]+=1\n",
    "        \n",
    "\n",
    "unnormalized_mood_label_map_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_id</th>\n",
       "      <th>mood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>TRAABLR128F423B7E3</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>TRAABLR128F423B7E3</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             track_id     mood\n",
       "8  TRAABLR128F423B7E3  neutral\n",
       "9  TRAABLR128F423B7E3  neutral"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_moods_df[selected_moods_df['track_id'] == 'TRAABLR128F423B7E3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "mood_label_map_df = unnormalized_mood_label_map_df\n",
    "# for idx,row in mood_label_map_df.iterrows():\n",
    "# #     print row\n",
    "#     total_mood = row['happiness'] + row['sadness'] + row['anger'] + row['neutral']\n",
    "# #     print total_mood\n",
    "# #     print mood_label_map_df.happiness.iloc[idx]\n",
    "# #     print mood_label_map_df.happiness.iloc[idx]\n",
    "#     mood_label_map_df.happiness.iloc[idx] /= total_mood\n",
    "#     mood_label_map_df.sadness.iloc[idx] /= total_mood\n",
    "#     mood_label_map_df.anger.iloc[idx] /= total_mood\n",
    "#     mood_label_map_df.neutral.iloc[idx] /= total_mood\n",
    "# #     break\n",
    "# mood_label_map_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## creating tempo label maps of the songs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tempo_label_map_df = pandas.DataFrame(columns = ['track_id'] + ['slow', 'medium', 'fast'])\n",
    "\n",
    "# for idx,row in selected_tempos_df.iterrows():\n",
    "# #     print tempo_label_map_df[tempo_label_map_df['track_id'] == row['track_id']].empty\n",
    "#     a = row['tempo']\n",
    "#     if tempo_label_map_df[tempo_label_map_df['track_id'] == row['track_id']].empty:\n",
    "#         tempo_label_map_df = tempo_label_map_df.append({\n",
    "#             'track_id': row['track_id'],\n",
    "#             row['tempo']: 1\n",
    "#         }, ignore_index=True).fillna(0)\n",
    "#     else:\n",
    "#         tempo_label_map_df.loc[tempo_label_map_df['track_id'] == row['track_id'], [row['tempo']]]=1\n",
    "\n",
    "# tempo_label_map_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2763\n",
      "2746\n",
      "2746\n"
     ]
    }
   ],
   "source": [
    "print len(mood_label_map_df[mood_label_map_df['']==0])\n",
    "mood_label_map_df = mood_label_map_df[mood_label_map_df['']==0]\n",
    "print len(mood_label_map_df[mood_label_map_df['Other']==0])\n",
    "mood_label_map_df = mood_label_map_df[mood_label_map_df['Other']==0]\n",
    "print len(mood_label_map_df[mood_label_map_df['Other']==0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "''",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-3116293a13c3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtempo_label_map_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtempo_label_map_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtempo_label_map_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtempo_label_map_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtempo_label_map_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtempo_label_map_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtempo_label_map_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2137\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2138\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2139\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2141\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_getitem_column\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2144\u001b[0m         \u001b[0;31m# get column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2145\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2146\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2148\u001b[0m         \u001b[0;31m# duplicate columns & possible reduce dimensionality\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/pandas/core/generic.pyc\u001b[0m in \u001b[0;36m_get_item_cache\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m   1840\u001b[0m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1841\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1842\u001b[0;31m             \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1843\u001b[0m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_box_item_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1844\u001b[0m             \u001b[0mcache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, item, fastpath)\u001b[0m\n\u001b[1;32m   3841\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3842\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3843\u001b[0;31m                 \u001b[0mloc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3844\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3845\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0misna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/pandas/core/indexes/base.pyc\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2525\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2526\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2527\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2529\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: ''"
     ]
    }
   ],
   "source": [
    "print len(tempo_label_map_df[tempo_label_map_df['']==0])\n",
    "tempo_label_map_df = tempo_label_map_df[tempo_label_map_df['']==0]\n",
    "print len(tempo_label_map_df[tempo_label_map_df['']==0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the new synthesized datasets\n",
    "label_map_df = pandas.merge(song_data_df, genre_label_map_df, on='track_id', how='inner')\n",
    "print label_map_df.columns\n",
    "mood_label_map_df = mood_label_map_df.drop(['Other', ''], axis=1)\n",
    "label_map_df = pandas.merge(label_map_df, mood_label_map_df, on='track_id', how='inner')\n",
    "print label_map_df.columns\n",
    "tempo_label_map_df = tempo_label_map_df.drop([''], axis=1)\n",
    "label_map_df = pandas.merge(label_map_df, tempo_label_map_df, on='track_id', how='inner')\n",
    "print label_map_df.columns\n",
    "label_map_df.reset_index(inplace=True, drop=True)\n",
    "print label_map_df.columns\n",
    "\n",
    "print len(label_map_df)\n",
    "label_map_df.head()\n",
    "\n",
    "# label_map_df.to_csv('./dataset/deep_learning/label_map(fraction_mood).csv', sep='\\t', encoding='utf-8', index=False)\n",
    "# label_map_df.to_csv('./dataset/deep_learning/label_map(boolean_mood).csv', sep='\\t', encoding='utf-8', index=False)\n",
    "\n",
    "use_df.to_csv('./dataset/deep_learning/combined_data.csv', sep='\\t', encoding='utf-8', index=False)\n",
    "song_data_df.to_csv('./dataset/deep_learning/song_data.csv', sep='\\t', encoding='utf-8', index=False)\n",
    "genres_df.to_csv('./dataset/deep_learning/genre_song_pair.csv', sep='\\t', encoding='utf-8', index=False)\n",
    "moods_df.to_csv('./dataset/deep_learning/mood_song_pair.csv', sep='\\t', encoding='utf-8', index=False)\n",
    "tempos_df.to_csv('./dataset/deep_learning/tempo_song_data.csv', sep='\\t', encoding='utf-8', index=False)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}