fix tutorials only as needs.

Signed-off-by: Xuesong Yang <[email protected]>
NVIDIA · Mar 10, 2023 · 233322c · 233322c
1 parent 48af0e5
commit 233322c
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 49 deletions.
diff --git a/tutorials/asr/ASR_CTC_Language_Finetuning.ipynb b/tutorials/asr/ASR_CTC_Language_Finetuning.ipynb
@@ -268,28 +268,17 @@
       "source": [
         "# Manifest Utils\n",
         "from tqdm.auto import tqdm\n",
+        "from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest\n",
         "import json\n",
         "\n",
-        "def read_manifest(path):\n",
-        "    manifest = []\n",
-        "    with open(path, 'r') as f:\n",
-        "        for line in tqdm(f, desc=\"Reading manifest data\"):\n",
-        "            line = line.replace(\"\\n\", \"\")\n",
-        "            data = json.loads(line)\n",
-        "            manifest.append(data)\n",
-        "    return manifest\n",
-        "\n",
         "\n",
         "def write_processed_manifest(data, original_path):\n",
         "    original_manifest_name = os.path.basename(original_path)\n",
         "    new_manifest_name = original_manifest_name.replace(\".json\", \"_processed.json\")\n",
         "\n",
         "    manifest_dir = os.path.split(original_path)[0]\n",
         "    filepath = os.path.join(manifest_dir, new_manifest_name)\n",
-        "    with open(filepath, 'w') as f:\n",
-        "        for datum in tqdm(data, desc=\"Writing manifest data\"):\n",
-        "            datum = json.dumps(datum)\n",
-        "            f.write(f\"{datum}\\n\")\n",
+        "    write_manifest(filepath, data)\n",
         "    print(f\"Finished writing manifest: {filepath}\")\n",
         "    return filepath"
       ],
@@ -2164,4 +2153,5 @@
       ]
     }
   ]
-}
+}
+
diff --git a/tutorials/asr/Buffered_Transducer_Inference.ipynb b/tutorials/asr/Buffered_Transducer_Inference.ipynb
@@ -1408,23 +1408,6 @@
     "```"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "5GZ2smcq4IZE"
-   },
-   "outputs": [],
-   "source": [
-    "def read_manifest(manifest_path):\n",
-    "  manifest_data = []\n",
-    "  with open(manifest_path, 'r') as f:\n",
-    "    for line in f:\n",
-    "      data = json.loads(line)\n",
-    "      manifest_data.append(data)\n",
-    "  return manifest_data"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1433,6 +1416,8 @@
    },
    "outputs": [],
    "source": [
+    "from nemo.collections.asr.parts.utils.manifest_utils import read_manifest\n",
+    "\n",
     "manifest_data = read_manifest(manifest)\n",
     "print(f\"Read {len(manifest_data)} samples from manifest {manifest}\")"
    ]

diff --git a/tutorials/asr/Buffered_Transducer_Inference_with_LCS_Merge.ipynb b/tutorials/asr/Buffered_Transducer_Inference_with_LCS_Merge.ipynb
@@ -841,15 +841,7 @@
         "#@title Manifest helper\n",
         "import json\n",
         "import numpy as np\n",
-        "\n",
-        "\n",
-        "def read_manifest(manifest_path):\n",
-        "  manifest_data = []\n",
-        "  with open(manifest_path, 'r') as f:\n",
-        "    for line in f:\n",
-        "      data = json.loads(line)\n",
-        "      manifest_data.append(data)\n",
-        "  return manifest_data\n",
+        "from nemo.collections.asr.parts.utils.manifest_utils import read_manifest\n",
         "\n",
         "\n",
         "def subset_manifest(manifest, num_samples):\n",
@@ -1299,4 +1291,5 @@
       }
     }
   ]
-}
+}
+
diff --git a/tutorials/asr/asr_adapters/ASR_with_Adapters.ipynb b/tutorials/asr/asr_adapters/ASR_with_Adapters.ipynb
@@ -418,13 +418,7 @@
       "source": [
         "# utility method\n",
         "import json\n",
-        "\n",
-        "def read_manifest(manifest_filepath):\n",
-        "  data = []\n",
-        "  with open(manifest_filepath, 'r', encoding='utf8') as f:\n",
-        "    for line in f:\n",
-        "      data.append(json.loads(line))\n",
-        "  return data"
+        "from nemo.collections.asr.parts.utils.manifest_utils import read_manifest\n",
       ],
       "metadata": {
         "id": "tJBriqr3tQV7"
@@ -1313,4 +1307,5 @@
       }
     }
   ]
-}
+}
+