prepare_hf_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4852090-af27-4f4a-85fe-a13ec39ab2c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.decomposition import PCA\n",
    "from umap import UMAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "af9a54ee-4a13-4b47-97b8-f7d03d8bbca6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json(\"/home/daniel/code/bengali-ai/bengali_ai_features_scalar.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fdebbd86-3c03-4329-b8dc-87c90a113c7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>split</th>\n",
       "      <th>path</th>\n",
       "      <th>audio_rms_mean</th>\n",
       "      <th>audio_rms_max</th>\n",
       "      <th>audio_rms_std</th>\n",
       "      <th>audio_spectral_flatness_mean</th>\n",
       "      <th>audio_length_s</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000005f3362c</td>\n",
       "      <td>ও বলেছে আপনার ঠিকানা!</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.095909</td>\n",
       "      <td>0.237279</td>\n",
       "      <td>0.066077</td>\n",
       "      <td>0.000890</td>\n",
       "      <td>1.116009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00001dddd002</td>\n",
       "      <td>কোন মহান রাষ্ট্রের নাগরিক হতে চাও?</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.021247</td>\n",
       "      <td>0.078481</td>\n",
       "      <td>0.018737</td>\n",
       "      <td>0.001006</td>\n",
       "      <td>2.448027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00001e0bc131</td>\n",
       "      <td>আমি তোমার কষ্টটা বুঝছি, কিন্তু এটা সঠিক পথ না।</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.035955</td>\n",
       "      <td>0.096530</td>\n",
       "      <td>0.026701</td>\n",
       "      <td>0.023012</td>\n",
       "      <td>4.716009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000024b3d810</td>\n",
       "      <td>নাচ শেষ হওয়ার পর সকলে শরীর ধুয়ে একসঙ্গে ভোজন...</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.027113</td>\n",
       "      <td>0.226049</td>\n",
       "      <td>0.040667</td>\n",
       "      <td>0.022640</td>\n",
       "      <td>7.452018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000028220ab3</td>\n",
       "      <td>হুমম, ওহ হেই, দেখো।</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.118137</td>\n",
       "      <td>0.241331</td>\n",
       "      <td>0.079757</td>\n",
       "      <td>0.021770</td>\n",
       "      <td>2.160000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963631</th>\n",
       "      <td>ffffd07108b7</td>\n",
       "      <td>আপনার সাথে কথা বলতে চাই।</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.107309</td>\n",
       "      <td>0.282542</td>\n",
       "      <td>0.085328</td>\n",
       "      <td>0.000881</td>\n",
       "      <td>2.340000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963632</th>\n",
       "      <td>ffffde37678a</td>\n",
       "      <td>সুতরাং পরের দিন আর-একটা ছবি না লইয়া চিত্রকর ছা...</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.052331</td>\n",
       "      <td>0.229947</td>\n",
       "      <td>0.054987</td>\n",
       "      <td>0.019101</td>\n",
       "      <td>4.608027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963633</th>\n",
       "      <td>ffffe1b5f095</td>\n",
       "      <td>সামাজিক কর্মকাণ্ডসমিতিতে গিয়ে দেখা যায়, শিল্পী...</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.070625</td>\n",
       "      <td>0.182665</td>\n",
       "      <td>0.044776</td>\n",
       "      <td>0.004284</td>\n",
       "      <td>5.688027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963634</th>\n",
       "      <td>ffffec31636e</td>\n",
       "      <td>গুগল ম্যাপসের সাহায্যে খুঁজে পাওয়া যাবে কোন জা...</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.079898</td>\n",
       "      <td>0.295988</td>\n",
       "      <td>0.072535</td>\n",
       "      <td>0.006693</td>\n",
       "      <td>5.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963635</th>\n",
       "      <td>fffff9f38566</td>\n",
       "      <td>তোমরা আমাকে কী নাম ধরে ডাকবে?</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/daniel/data/bengaliai/bengaliai-speech/t...</td>\n",
       "      <td>0.090154</td>\n",
       "      <td>0.236889</td>\n",
       "      <td>0.070949</td>\n",
       "      <td>0.002490</td>\n",
       "      <td>2.916009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>963636 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id                                           sentence  \\\n",
       "0       000005f3362c                              ও বলেছে আপনার ঠিকানা!   \n",
       "1       00001dddd002                 কোন মহান রাষ্ট্রের নাগরিক হতে চাও?   \n",
       "2       00001e0bc131     আমি তোমার কষ্টটা বুঝছি, কিন্তু এটা সঠিক পথ না।   \n",
       "3       000024b3d810  নাচ শেষ হওয়ার পর সকলে শরীর ধুয়ে একসঙ্গে ভোজন...   \n",
       "4       000028220ab3                                হুমম, ওহ হেই, দেখো।   \n",
       "...              ...                                                ...   \n",
       "963631  ffffd07108b7                           আপনার সাথে কথা বলতে চাই।   \n",
       "963632  ffffde37678a  সুতরাং পরের দিন আর-একটা ছবি না লইয়া চিত্রকর ছা...   \n",
       "963633  ffffe1b5f095  সামাজিক কর্মকাণ্ডসমিতিতে গিয়ে দেখা যায়, শিল্পী...   \n",
       "963634  ffffec31636e  গুগল ম্যাপসের সাহায্যে খুঁজে পাওয়া যাবে কোন জা...   \n",
       "963635  fffff9f38566                      তোমরা আমাকে কী নাম ধরে ডাকবে?   \n",
       "\n",
       "        split                                               path  \\\n",
       "0       train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "1       train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "2       train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "3       train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "4       train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "...       ...                                                ...   \n",
       "963631  train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "963632  train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "963633  train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "963634  train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "963635  train  /home/daniel/data/bengaliai/bengaliai-speech/t...   \n",
       "\n",
       "        audio_rms_mean  audio_rms_max  audio_rms_std  \\\n",
       "0             0.095909       0.237279       0.066077   \n",
       "1             0.021247       0.078481       0.018737   \n",
       "2             0.035955       0.096530       0.026701   \n",
       "3             0.027113       0.226049       0.040667   \n",
       "4             0.118137       0.241331       0.079757   \n",
       "...                ...            ...            ...   \n",
       "963631        0.107309       0.282542       0.085328   \n",
       "963632        0.052331       0.229947       0.054987   \n",
       "963633        0.070625       0.182665       0.044776   \n",
       "963634        0.079898       0.295988       0.072535   \n",
       "963635        0.090154       0.236889       0.070949   \n",
       "\n",
       "        audio_spectral_flatness_mean  audio_length_s  \n",
       "0                           0.000890        1.116009  \n",
       "1                           0.001006        2.448027  \n",
       "2                           0.023012        4.716009  \n",
       "3                           0.022640        7.452018  \n",
       "4                           0.021770        2.160000  \n",
       "...                              ...             ...  \n",
       "963631                      0.000881        2.340000  \n",
       "963632                      0.019101        4.608027  \n",
       "963633                      0.004284        5.688027  \n",
       "963634                      0.006693        5.400000  \n",
       "963635                      0.002490        2.916009  \n",
       "\n",
       "[963636 rows x 9 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e5415c5e-0024-4e9a-9388-a7fafaa7abdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'sentence', 'split', 'path', 'audio_rms_mean', 'audio_rms_max',\n",
       "       'audio_rms_std', 'audio_spectral_flatness_mean', 'audio_length_s'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cd5c4a85-5979-4af0-b9e7-0428e60bca75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(963636, 384)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "text_pkl_files = sorted(list(Path(\"/home/daniel/code/bengali-ai\").glob(\"*_text_embeddings.pkl\")))\n",
    "\n",
    "all_text_embeddings = []\n",
    "\n",
    "for text_pkl_file in text_pkl_files:\n",
    "    with open(text_pkl_file, \"rb\") as f:\n",
    "        embs = pickle.load(f)\n",
    "    all_text_embeddings.append(embs)\n",
    "\n",
    "text_embeddings = np.concatenate(all_text_embeddings, axis=0)\n",
    "\n",
    "print(text_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7d70f705-4547-4024-a8cd-334e5fd1c5ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying num dimensions 135\n",
      "Explained variance is 0.9792220090316912\n",
      "Trying num dimensions 136\n",
      "Explained variance is 0.9795272492117195\n",
      "Trying num dimensions 137\n",
      "Explained variance is 0.9798220671288702\n",
      "Trying num dimensions 138\n",
      "Explained variance is 0.9800726631260227\n",
      "Found num dimensions 138\n"
     ]
    }
   ],
   "source": [
    "for num_dimensions in range(138, text_embeddings.shape[1]):\n",
    "    print(f\"Trying num dimensions {num_dimensions}\")\n",
    "    pca = PCA(n_components=num_dimensions)\n",
    "    pca.fit(text_embeddings)\n",
    "    explained_variance = sum(pca.explained_variance_ratio_)\n",
    "    print(f\"Explained variance is {explained_variance}\")\n",
    "    if explained_variance >= 0.98:\n",
    "        print(f\"Found num dimensions {num_dimensions}\")\n",
    "        break\n",
    "\n",
    "df[\"text_embedding\"] = [e for e in pca.transform(text_embeddings)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8f2ce9b8-fdc8-4d8c-8aff-b71eaf259c20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/daniel/code/bengali-ai/000_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/001_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/002_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/003_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/004_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/005_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/006_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/007_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/008_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/009_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/010_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/011_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/012_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/013_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/014_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/015_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/016_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/017_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/018_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/019_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/020_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/021_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/022_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/023_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/024_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/025_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/026_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/027_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/028_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/029_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/030_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/031_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/032_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/033_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/034_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/035_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/036_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/037_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/038_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/039_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/040_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/041_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/042_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/043_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/044_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/045_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/046_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/047_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/048_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/049_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/050_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/051_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/052_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/053_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/054_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/055_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/056_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/057_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/058_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/059_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/060_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/061_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/062_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/063_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/064_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/065_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/066_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/067_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/068_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/069_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/070_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/071_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/072_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/073_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/074_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/075_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/076_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/077_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/078_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/079_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/080_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/081_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/082_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/083_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/084_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/085_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/086_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/087_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/088_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/089_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/090_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/091_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/092_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/093_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/094_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/095_audio_embeddings.pkl\n",
      "/home/daniel/code/bengali-ai/096_audio_embeddings.pkl\n",
      "0.9500902379722386\n"
     ]
    }
   ],
   "source": [
    "from sklearn.decomposition import IncrementalPCA\n",
    "audio_pkl_files = sorted(list(Path(\"/home/daniel/code/bengali-ai\").glob(\"*_audio_embeddings.pkl\")))\n",
    "\n",
    "inc_pca = IncrementalPCA(n_components=297)\n",
    "\n",
    "for audio_pkl_file in audio_pkl_files:\n",
    "    print(audio_pkl_file)\n",
    "    with open(audio_pkl_file, \"rb\") as f:\n",
    "        embs = pickle.load(f)\n",
    "    inc_pca.partial_fit(embs)\n",
    "\n",
    "print(sum(inc_pca.explained_variance_ratio_))\n",
    "\n",
    "all_audio_embeddings = []\n",
    "\n",
    "for audio_pkl_file in audio_pkl_files:\n",
    "    with open(audio_pkl_file, \"rb\") as f:\n",
    "        embs = pickle.load(f)\n",
    "    all_audio_embeddings.append(inc_pca.transform(embs))\n",
    "\n",
    "audio_embeddings = np.concatenate(all_audio_embeddings, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cfb37c79-5bb2-459c-bb5d-d6fd937cc8c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"audio_embedding\"] = [e.astype(np.float32) for e in audio_embeddings]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7288240c-afba-400b-8511-a08693ffc43a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"audio\"] = df[\"path\"].str[45:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dcbbf0bd-ca97-44d6-af8e-4cdeecf97047",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(columns=[\"path\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7febbdcf-7d36-43c9-9c1d-1bef069525e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>split</th>\n",
       "      <th>audio_rms_mean</th>\n",
       "      <th>audio_rms_max</th>\n",
       "      <th>audio_rms_std</th>\n",
       "      <th>audio_spectral_flatness_mean</th>\n",
       "      <th>audio_length_s</th>\n",
       "      <th>text_embedding</th>\n",
       "      <th>audio_embedding</th>\n",
       "      <th>audio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000005f3362c</td>\n",
       "      <td>ও বলেছে আপনার ঠিকানা!</td>\n",
       "      <td>train</td>\n",
       "      <td>0.095909</td>\n",
       "      <td>0.237279</td>\n",
       "      <td>0.066077</td>\n",
       "      <td>0.000890</td>\n",
       "      <td>1.116009</td>\n",
       "      <td>[-0.38994306, -0.097972825, -0.0056979237, 0.1...</td>\n",
       "      <td>[7.2484603, -7.543934, -4.7032595, -8.655085, ...</td>\n",
       "      <td>train_mp3s/000005f3362c.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00001dddd002</td>\n",
       "      <td>কোন মহান রাষ্ট্রের নাগরিক হতে চাও?</td>\n",
       "      <td>train</td>\n",
       "      <td>0.021247</td>\n",
       "      <td>0.078481</td>\n",
       "      <td>0.018737</td>\n",
       "      <td>0.001006</td>\n",
       "      <td>2.448027</td>\n",
       "      <td>[-0.1084951, -0.07227868, 0.053285215, -0.1264...</td>\n",
       "      <td>[11.617512, -0.12815115, -0.48695448, -2.82126...</td>\n",
       "      <td>train_mp3s/00001dddd002.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00001e0bc131</td>\n",
       "      <td>আমি তোমার কষ্টটা বুঝছি, কিন্তু এটা সঠিক পথ না।</td>\n",
       "      <td>train</td>\n",
       "      <td>0.035955</td>\n",
       "      <td>0.096530</td>\n",
       "      <td>0.026701</td>\n",
       "      <td>0.023012</td>\n",
       "      <td>4.716009</td>\n",
       "      <td>[-0.1496445, -0.06911754, 0.11506425, -0.15017...</td>\n",
       "      <td>[0.13251832, 7.5647473, -4.9663234, -0.6222096...</td>\n",
       "      <td>train_mp3s/00001e0bc131.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000024b3d810</td>\n",
       "      <td>নাচ শেষ হওয়ার পর সকলে শরীর ধুয়ে একসঙ্গে ভোজন...</td>\n",
       "      <td>train</td>\n",
       "      <td>0.027113</td>\n",
       "      <td>0.226049</td>\n",
       "      <td>0.040667</td>\n",
       "      <td>0.022640</td>\n",
       "      <td>7.452018</td>\n",
       "      <td>[0.004609096, -0.010123715, -0.054717332, -0.0...</td>\n",
       "      <td>[-13.47718, 3.0267537, -4.3793607, 0.48654944,...</td>\n",
       "      <td>train_mp3s/000024b3d810.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000028220ab3</td>\n",
       "      <td>হুমম, ওহ হেই, দেখো।</td>\n",
       "      <td>train</td>\n",
       "      <td>0.118137</td>\n",
       "      <td>0.241331</td>\n",
       "      <td>0.079757</td>\n",
       "      <td>0.021770</td>\n",
       "      <td>2.160000</td>\n",
       "      <td>[-0.36782098, 0.13273844, -0.16590148, 0.03617...</td>\n",
       "      <td>[-7.9069605, -4.8090887, -7.302672, -2.4339087...</td>\n",
       "      <td>train_mp3s/000028220ab3.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963631</th>\n",
       "      <td>ffffd07108b7</td>\n",
       "      <td>আপনার সাথে কথা বলতে চাই।</td>\n",
       "      <td>train</td>\n",
       "      <td>0.107309</td>\n",
       "      <td>0.282542</td>\n",
       "      <td>0.085328</td>\n",
       "      <td>0.000881</td>\n",
       "      <td>2.340000</td>\n",
       "      <td>[-0.21112299, -0.131941, 0.12897818, -0.110035...</td>\n",
       "      <td>[5.4855533, 5.2816706, -3.1930263, 7.416063, 2...</td>\n",
       "      <td>train_mp3s/ffffd07108b7.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963632</th>\n",
       "      <td>ffffde37678a</td>\n",
       "      <td>সুতরাং পরের দিন আর-একটা ছবি না লইয়া চিত্রকর ছা...</td>\n",
       "      <td>train</td>\n",
       "      <td>0.052331</td>\n",
       "      <td>0.229947</td>\n",
       "      <td>0.054987</td>\n",
       "      <td>0.019101</td>\n",
       "      <td>4.608027</td>\n",
       "      <td>[0.16658908, -0.026953176, 0.060520217, -0.065...</td>\n",
       "      <td>[5.314484, 5.8587527, 2.7001863, 6.3718195, 3....</td>\n",
       "      <td>train_mp3s/ffffde37678a.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963633</th>\n",
       "      <td>ffffe1b5f095</td>\n",
       "      <td>সামাজিক কর্মকাণ্ডসমিতিতে গিয়ে দেখা যায়, শিল্পী...</td>\n",
       "      <td>train</td>\n",
       "      <td>0.070625</td>\n",
       "      <td>0.182665</td>\n",
       "      <td>0.044776</td>\n",
       "      <td>0.004284</td>\n",
       "      <td>5.688027</td>\n",
       "      <td>[0.2818417, -0.23723033, -0.018151812, 0.10785...</td>\n",
       "      <td>[2.5140448, -1.4130806, 4.5127854, 2.3490546, ...</td>\n",
       "      <td>train_mp3s/ffffe1b5f095.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963634</th>\n",
       "      <td>ffffec31636e</td>\n",
       "      <td>গুগল ম্যাপসের সাহায্যে খুঁজে পাওয়া যাবে কোন জা...</td>\n",
       "      <td>train</td>\n",
       "      <td>0.079898</td>\n",
       "      <td>0.295988</td>\n",
       "      <td>0.072535</td>\n",
       "      <td>0.006693</td>\n",
       "      <td>5.400000</td>\n",
       "      <td>[0.25751734, -0.099529, -0.060007177, -0.13138...</td>\n",
       "      <td>[-5.8520145, -0.5586258, -4.825342, 1.7372414,...</td>\n",
       "      <td>train_mp3s/ffffec31636e.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963635</th>\n",
       "      <td>fffff9f38566</td>\n",
       "      <td>তোমরা আমাকে কী নাম ধরে ডাকবে?</td>\n",
       "      <td>train</td>\n",
       "      <td>0.090154</td>\n",
       "      <td>0.236889</td>\n",
       "      <td>0.070949</td>\n",
       "      <td>0.002490</td>\n",
       "      <td>2.916009</td>\n",
       "      <td>[-0.20965089, -0.0033394452, 0.03930898, -0.15...</td>\n",
       "      <td>[-2.3383148, -5.4779186, 6.585282, -4.269415, ...</td>\n",
       "      <td>train_mp3s/fffff9f38566.mp3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>963636 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id                                           sentence  \\\n",
       "0       000005f3362c                              ও বলেছে আপনার ঠিকানা!   \n",
       "1       00001dddd002                 কোন মহান রাষ্ট্রের নাগরিক হতে চাও?   \n",
       "2       00001e0bc131     আমি তোমার কষ্টটা বুঝছি, কিন্তু এটা সঠিক পথ না।   \n",
       "3       000024b3d810  নাচ শেষ হওয়ার পর সকলে শরীর ধুয়ে একসঙ্গে ভোজন...   \n",
       "4       000028220ab3                                হুমম, ওহ হেই, দেখো।   \n",
       "...              ...                                                ...   \n",
       "963631  ffffd07108b7                           আপনার সাথে কথা বলতে চাই।   \n",
       "963632  ffffde37678a  সুতরাং পরের দিন আর-একটা ছবি না লইয়া চিত্রকর ছা...   \n",
       "963633  ffffe1b5f095  সামাজিক কর্মকাণ্ডসমিতিতে গিয়ে দেখা যায়, শিল্পী...   \n",
       "963634  ffffec31636e  গুগল ম্যাপসের সাহায্যে খুঁজে পাওয়া যাবে কোন জা...   \n",
       "963635  fffff9f38566                      তোমরা আমাকে কী নাম ধরে ডাকবে?   \n",
       "\n",
       "        split  audio_rms_mean  audio_rms_max  audio_rms_std  \\\n",
       "0       train        0.095909       0.237279       0.066077   \n",
       "1       train        0.021247       0.078481       0.018737   \n",
       "2       train        0.035955       0.096530       0.026701   \n",
       "3       train        0.027113       0.226049       0.040667   \n",
       "4       train        0.118137       0.241331       0.079757   \n",
       "...       ...             ...            ...            ...   \n",
       "963631  train        0.107309       0.282542       0.085328   \n",
       "963632  train        0.052331       0.229947       0.054987   \n",
       "963633  train        0.070625       0.182665       0.044776   \n",
       "963634  train        0.079898       0.295988       0.072535   \n",
       "963635  train        0.090154       0.236889       0.070949   \n",
       "\n",
       "        audio_spectral_flatness_mean  audio_length_s  \\\n",
       "0                           0.000890        1.116009   \n",
       "1                           0.001006        2.448027   \n",
       "2                           0.023012        4.716009   \n",
       "3                           0.022640        7.452018   \n",
       "4                           0.021770        2.160000   \n",
       "...                              ...             ...   \n",
       "963631                      0.000881        2.340000   \n",
       "963632                      0.019101        4.608027   \n",
       "963633                      0.004284        5.688027   \n",
       "963634                      0.006693        5.400000   \n",
       "963635                      0.002490        2.916009   \n",
       "\n",
       "                                           text_embedding  \\\n",
       "0       [-0.38994306, -0.097972825, -0.0056979237, 0.1...   \n",
       "1       [-0.1084951, -0.07227868, 0.053285215, -0.1264...   \n",
       "2       [-0.1496445, -0.06911754, 0.11506425, -0.15017...   \n",
       "3       [0.004609096, -0.010123715, -0.054717332, -0.0...   \n",
       "4       [-0.36782098, 0.13273844, -0.16590148, 0.03617...   \n",
       "...                                                   ...   \n",
       "963631  [-0.21112299, -0.131941, 0.12897818, -0.110035...   \n",
       "963632  [0.16658908, -0.026953176, 0.060520217, -0.065...   \n",
       "963633  [0.2818417, -0.23723033, -0.018151812, 0.10785...   \n",
       "963634  [0.25751734, -0.099529, -0.060007177, -0.13138...   \n",
       "963635  [-0.20965089, -0.0033394452, 0.03930898, -0.15...   \n",
       "\n",
       "                                          audio_embedding  \\\n",
       "0       [7.2484603, -7.543934, -4.7032595, -8.655085, ...   \n",
       "1       [11.617512, -0.12815115, -0.48695448, -2.82126...   \n",
       "2       [0.13251832, 7.5647473, -4.9663234, -0.6222096...   \n",
       "3       [-13.47718, 3.0267537, -4.3793607, 0.48654944,...   \n",
       "4       [-7.9069605, -4.8090887, -7.302672, -2.4339087...   \n",
       "...                                                   ...   \n",
       "963631  [5.4855533, 5.2816706, -3.1930263, 7.416063, 2...   \n",
       "963632  [5.314484, 5.8587527, 2.7001863, 6.3718195, 3....   \n",
       "963633  [2.5140448, -1.4130806, 4.5127854, 2.3490546, ...   \n",
       "963634  [-5.8520145, -0.5586258, -4.825342, 1.7372414,...   \n",
       "963635  [-2.3383148, -5.4779186, 6.585282, -4.269415, ...   \n",
       "\n",
       "                              audio  \n",
       "0       train_mp3s/000005f3362c.mp3  \n",
       "1       train_mp3s/00001dddd002.mp3  \n",
       "2       train_mp3s/00001e0bc131.mp3  \n",
       "3       train_mp3s/000024b3d810.mp3  \n",
       "4       train_mp3s/000028220ab3.mp3  \n",
       "...                             ...  \n",
       "963631  train_mp3s/ffffd07108b7.mp3  \n",
       "963632  train_mp3s/ffffde37678a.mp3  \n",
       "963633  train_mp3s/ffffe1b5f095.mp3  \n",
       "963634  train_mp3s/ffffec31636e.mp3  \n",
       "963635  train_mp3s/fffff9f38566.mp3  \n",
       "\n",
       "[963636 rows x 11 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ee4836f8-58fb-4e80-8cf5-6313150b27a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_parquet(\"bengaliai.parquet\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}