Merge branch 'main' of github.com:NVIDIA/NeMo into megatron_perceiver

Jenkinsfile

@@ -317,6 +317,20 @@ pipeline {
           }
         }
 
+        stage('L2: Speech Pre-training - Wav2Vec') {
+          steps {
+            sh 'python examples/asr/speech_pretraining/speech_pre_training.py \
+            --config-path="../conf/ssl/wav2vec/" --config-name="wav2vec_ci" \
+            model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \
+            model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \
+            trainer.devices=[1] \
+            trainer.accelerator="gpu" \
+            +trainer.fast_dev_run=True \
+            exp_manager.exp_dir=examples/asr/speech_pre_training_results'
+            sh 'rm -rf examples/asr/speech_pre_training_results'
+          }
+        }
+
         stage('L2: Speech to Text WPE - Conformer') {
           steps {
             sh 'python examples/asr/asr_ctc/speech_to_text_ctc_bpe.py \

docs/source/_static/css/custom.css

@@ -58,8 +58,19 @@ a:visited
 	margin-left: unset;
 }
 
+section {
+	overflow-x: auto;
+}
 
 /* ----------------------------------------------TABLES--------------------------------------- */
+section table {
+	overflow-x: auto;
+	display: block;
+}
+
+table {
+	font-size: small;
+}
 
 /* Table head Color */
 thead td

docs/source/asr/data/scores/ca/quartznet15x5_ca.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (ca)
+stt_ca_quartznet15x5,ca,6.0

docs/source/asr/data/scores/de/citrinet_de.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (de),MCV Dev-Set v7.0 (de),MCV Test-Set v7.0 (de),MLS Dev (en),MLS Test (en),VoxPopuli Dev (de),VoxPopuli Test (de)
+stt_de_citrinet_1024,de,,6.63,7.59,4.06,5.07,12.33,10.02

docs/source/asr/data/scores/de/conformer_de.csv

@@ -0,0 +1,3 @@
+Model Name,Language,MCV Dev-Set (v??) (de),MCV Dev-Set v7.0 (de),MCV Test-Set v7.0 (de),MLS Dev (en),MLS Test (en),VoxPopuli Dev (de),VoxPopuli Test (de)
+stt_de_conformer_ctc_large,de,,5.84,6.68,3.85,4.63,12.56,10.51
+stt_de_conformer_transducer_large,de,,4.75,5.36,3.46,4.19,11.21,9.14

docs/source/asr/data/scores/de/contextnet_de.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (de),MCV Dev-Set v7.0 (de),MCV Test-Set v7.0 (de),MLS Dev (en),MLS Test (en),VoxPopuli Dev (de),VoxPopuli Test (de)
+stt_de_contextnet_1024,de,,4.76,5.5,3.53,4.2,11.32,9.4

docs/source/asr/data/scores/de/quartznet15x5_de.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (de),MCV Dev-Set v7.0 (de),MCV Test-Set v7.0 (de),MLS Dev (en),MLS Test (en),VoxPopuli Dev (de),VoxPopuli Test (de)
+stt_de_quartznet15x5,de,11.78,,,,,,

docs/source/asr/data/scores/en/citrinet_en.csv

@@ -0,0 +1,7 @@
+Model Name,Language,Librispeech Dev-Clean,Librispeech Dev-Other,Librispeech Test-Clean,Librispeech Test-Other,MCV Test-Set v8.0 (en),MLS Dev (en),MLS Test (en),NSC Part1,NSC Part6,Peoples Speech Test v1,SLR 83 Test,WSJ Dev 93,WSJ Eval 92
+stt_en_citrinet_256,en,4.2 % WER,10.7 % WER,4.4 % WER,10.7 % WER,,,,,,,,,
+stt_en_citrinet_512,en,3.7 % WER,8.9 % WER,3.7 % WER,8.9 % WER,,,,,,,,,
+stt_en_citrinet_1024,en,3.7 % WER,8.3 % WER,3.6 % WER,7.9 % WER,,,,,,,,,
+stt_en_citrinet_256_gamma_0_25,en,4.7 %,10.6 %,4.8 %,10.7 %,,,,8.3 %,,,,5.8 %,3.6 %
+stt_en_citrinet_512_gamma_0_25,en,4.0 %,9.0 %,3.9 %,9.0 %,,,,6.9 %,,,,4.4 %,3.6 %
+stt_en_citrinet_1024_gamma_0_25,en,3.4 %,7.7 %,3.4 %,7.6 %,,,,6.2 %,,,,4.0 %,2.5 %

docs/source/asr/data/scores/en/conformer_en.csv

@@ -0,0 +1,14 @@
+Model Name,Language,Librispeech Dev-Clean,Librispeech Dev-Other,Librispeech Test-Clean,Librispeech Test-Other,MCV Test-Set v8.0 (en),MLS Dev (en),MLS Test (en),NSC Part1,NSC Part6,Peoples Speech Test v1,SLR 83 Test,WSJ Dev 93,WSJ Eval 92
+stt_en_conformer_ctc_small,en,3.6,8.1,3.7,8.1,,,,,,,,,
+stt_en_conformer_ctc_medium,en,2.5,5.8,2.6,5.9,,,,,,,,,
+stt_en_conformer_ctc_large,en,2.0,4.4,2.1,4.3,,,,,,,,,
+stt_en_conformer_ctc_xlarge,en,1.77 %,3.79 %,2.00 %,3.74 %,7.88 %,,5.99 %,,6.44 %,22.90 %,5.50 %,2.36 %,
+stt_en_conformer_ctc_small_ls,en,3.3,8.8,3.4,8.8,,,,,,,,,
+stt_en_conformer_ctc_medium_ls,en,2.7,7.4,3.0,7.3,,,,,,,,,
+stt_en_conformer_ctc_large_ls,en,2.4,6.2,2.7,6.0,,,,,,,,,
+stt_en_conformer_transducer_small,en,2.8,6.6,2.5,6.6,,,,,,,,,
+stt_en_conformer_transducer_medium,en,2.0,4.6,2.1,4.7,,,,,,,,,
+stt_en_conformer_transducer_large,en,1.5,3.5,1.7,3.6,,,,,,,,,
+stt_en_conformer_transducer_large_ls,en,2.1,5.0,2.3,5.1,,,,,,,,,
+stt_en_conformer_transducer_xlarge,en,1.48 %,2.95 %,1.62 %,3.01 %,6.46 %,4.59 %,5.32 %,5.70 %,6.47 %,21.32 %,,2.05 %,1.17 %
+stt_en_conformer_transducer_xxlarge,en,1.52 %,3.09 %,1.72 %,3.14 %,,5.29 %,5.85 %,6.64 %,,,,2.42 %,1.49 %

docs/source/asr/data/scores/en/contextnet_en.csv

@@ -0,0 +1,7 @@
+Model Name,Language,Librispeech Dev-Clean,Librispeech Dev-Other,Librispeech Test-Clean,Librispeech Test-Other,MCV Test-Set v8.0 (en),MLS Dev (en),MLS Test (en),NSC Part1,NSC Part6,Peoples Speech Test v1,SLR 83 Test,WSJ Dev 93,WSJ Eval 92
+stt_en_contextnet_256,en,3.3 %,7.9 %,3.3 %,8.0 %,,9.7 %,11.0 %,7.1 %,,,,4.6 %,3.2 %
+stt_en_contextnet_512,en,2.0 %,4.8 %,2.2 %,5.0 %,,6.6 %,7.3 %,5.9 %,,,,2.8 %,1.4 %
+stt_en_contextnet_1024,en,1.7 %,3.8 %,1.9 %,4.0 %,7.9 %,,5.9 %,5.2 %,6.5 %,21.7 %,4.7 %,2.3 %,1.3 %
+stt_en_contextnet_256_mls,en,,9.0 %,,9.2 %,,9.4 %,10.9 %,,,,,,
+stt_en_contextnet_512_mls,en,,5.2 %,,5.2 %,,5.6 %,6.6 %,,,,,,
+stt_en_contextnet_1024_mls,en,,4.1 %,,4.2 %,,4.6 %,5.6 %,,,,,,

docs/source/asr/data/scores/en/jasper10x5dr_en.csv

@@ -0,0 +1,2 @@
+Model Name,Language,Librispeech Dev-Clean,Librispeech Dev-Other,Librispeech Test-Clean,Librispeech Test-Other,MCV Test-Set v8.0 (en),MLS Dev (en),MLS Test (en),NSC Part1,NSC Part6,Peoples Speech Test v1,SLR 83 Test,WSJ Dev 93,WSJ Eval 92
+stt_en_jasper10x5dr,en,3.74,10.21,,,,,,,,,,,

docs/source/asr/data/scores/en/quartznet15x5_en.csv

@@ -0,0 +1,2 @@
+Model Name,Language,Librispeech Dev-Clean,Librispeech Dev-Other,Librispeech Test-Clean,Librispeech Test-Other,MCV Test-Set v8.0 (en),MLS Dev (en),MLS Test (en),NSC Part1,NSC Part6,Peoples Speech Test v1,SLR 83 Test,WSJ Dev 93,WSJ Eval 92
+stt_en_quartznet15x5,en,4.38,11.3,,,,,,,,,,,

docs/source/asr/data/scores/enes/conformer_enes.csv

@@ -0,0 +1,3 @@
+Model Name,Language,Fisher-Dev-Es,Librispeech Dev-Clean,Librispeech Dev-Other,Librispeech Test-Clean,Librispeech Test-Other,MCV Dev-Set v7.0 (en),MLS Dev (es),VoxPopuli Dev (es)
+stt_enes_conformer_ctc_large,enes,16.7 %,2.2 %,5.5 %,2.6 %,5.5 %,5.8 %,3.5 %,5.7 %
+stt_enes_conformer_transducer_large,enes,16.2 %,2.0 %,4.6 %,2.2 %,4.6 %,5.0 %,3.3 %,5.3 %

docs/source/asr/data/scores/enes/contextnet_enes.csv

@@ -0,0 +1,2 @@
+Model Name,Language,Fisher-Dev-Es,Librispeech Dev-Clean,Librispeech Dev-Other,Librispeech Test-Clean,Librispeech Test-Other,MCV Dev-Set v7.0 (en),MLS Dev (es),VoxPopuli Dev (es)
+stt_enes_contextnet_large,enes,14.8 %,2.2 %,5.6 %,2.3 %,5.5 %,4.7 %,3.0 %,5.0 %

docs/source/asr/data/scores/es/citrinet_es.csv

@@ -0,0 +1,3 @@
+Model Name,Language,Call Home Dev Test (es),Call Home Eval Test (es),Call Home Train (es),Fisher Dev Set (es),Fisher Test Set (es),MCV Dev-Set (v??) (es),MCV Dev-Set v7.0 (es),MCV Test-Set (v??) (es),MCV Test-Set v7.0 (es),MLS Dev (en),MLS Test (en),VoxPopuli Dev (es),VoxPopuli Test (es)
+stt_es_citrinet_512,es,,,,,,9.1 % WER,,10.3 % WER,,4.9 % WER,5.2 % WER,,
+stt_es_citrinet_1024_gamma_0_25,es,19.9 %,21.3 %,19.1 %,15.8 %,15.9 %,,6.1 %,,6.8 %,3.5 %,4.1 %,5.6 %,7.0 %

docs/source/asr/data/scores/es/conformer_es.csv

@@ -0,0 +1,3 @@
+Model Name,Language,Call Home Dev Test (es),Call Home Eval Test (es),Call Home Train (es),Fisher Dev Set (es),Fisher Test Set (es),MCV Dev-Set (v??) (es),MCV Dev-Set v7.0 (es),MCV Test-Set (v??) (es),MCV Test-Set v7.0 (es),MLS Dev (en),MLS Test (en),VoxPopuli Dev (es),VoxPopuli Test (es)
+stt_es_conformer_ctc_large,es,23.7 %,25.3 %,22.4 %,18.3 %,18.5 %,,6.3 %,,6.9 %,4.3 %,4.2 %,6.1 %,7.5 %
+stt_es_conformer_transducer_large,es,18.0 %,19.4 %,17.2 %,14.7 %,14.8 %,,4.6 %,,5.2 %,2.7 %,3.2 %,4.7 %,6.0 %

docs/source/asr/data/scores/es/contextnet_es.csv

@@ -0,0 +1,2 @@
+Model Name,Language,Call Home Dev Test (es),Call Home Eval Test (es),Call Home Train (es),Fisher Dev Set (es),Fisher Test Set (es),MCV Dev-Set (v??) (es),MCV Dev-Set v7.0 (es),MCV Test-Set (v??) (es),MCV Test-Set v7.0 (es),MLS Dev (en),MLS Test (en),VoxPopuli Dev (es),VoxPopuli Test (es)
+stt_es_contextnet_1024,es,19.1 %,20.7 %,18.2 %,15.3 %,15.1 %,,4.8 %,,5.2 %,3.1 %,3.5 %,5.1 %,6.2 %

docs/source/asr/data/scores/es/quartznet15x5_es.csv

@@ -0,0 +1,2 @@
+Model Name,Language,Call Home Dev Test (es),Call Home Eval Test (es),Call Home Train (es),Fisher Dev Set (es),Fisher Test Set (es),MCV Dev-Set (v??) (es),MCV Dev-Set v7.0 (es),MCV Test-Set (v??) (es),MCV Test-Set v7.0 (es),MLS Dev (en),MLS Test (en),VoxPopuli Dev (es),VoxPopuli Test (es)
+stt_es_quartznet15x5,es,,,,,,12.97,,,,,,,

docs/source/asr/data/scores/fr/citrinet_fr.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (fr),MCV Dev-Set v7.0 (fr),MCV Dev-Set v7.0 (fr) (No Hyphen),MCV Test-Set v7.0 (fr),MCV Test-Set v7.0 (fr) (No Hyphen),MLS Dev (en),MLS Dev (en) (No Hyphen),MLS Test (en),MLS Test (en) (No Hyphen)
+stt_fr_citrinet_1024_gamma_0_25,fr,,10.76,9.90,12.20,11.11,6.66,6.19,5.53,5.12

docs/source/asr/data/scores/fr/conformer_fr.csv

@@ -0,0 +1,3 @@
+Model Name,Language,MCV Dev-Set (v??) (fr),MCV Dev-Set v7.0 (fr),MCV Dev-Set v7.0 (fr) (No Hyphen),MCV Test-Set v7.0 (fr),MCV Test-Set v7.0 (fr) (No Hyphen),MLS Dev (en),MLS Dev (en) (No Hyphen),MLS Test (en),MLS Test (en) (No Hyphen)
+stt_fr_conformer_ctc_large,fr,,8.35,7.88,9.63,9.01,5.88,5.90,4.91,4.63
+stt_fr_conformer_transducer_large,fr,,6.85,,7.95,,5.05,,4.10,

docs/source/asr/data/scores/fr/contextnet_fr.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (fr),MCV Dev-Set v7.0 (fr),MCV Dev-Set v7.0 (fr) (No Hyphen),MCV Test-Set v7.0 (fr),MCV Test-Set v7.0 (fr) (No Hyphen),MLS Dev (en),MLS Dev (en) (No Hyphen),MLS Test (en),MLS Test (en) (No Hyphen)
+stt_fr_contextnet_1024,fr,,8.32,,9.42,,6.02,,5.01,

docs/source/asr/data/scores/fr/quartznet15x5_fr.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (fr),MCV Dev-Set v7.0 (fr),MCV Dev-Set v7.0 (fr) (No Hyphen),MCV Test-Set v7.0 (fr),MCV Test-Set v7.0 (fr) (No Hyphen),MLS Dev (en),MLS Dev (en) (No Hyphen),MLS Test (en),MLS Test (en) (No Hyphen)
+stt_fr_quartznet15x5,fr,14.01,,,,,,,,

docs/source/asr/data/scores/it/quartznet15x5_it.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (it)
+stt_it_quartznet15x5,it,15.22

docs/source/asr/data/scores/pl/quartznet15x5_pl.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (pl)
+stt_pl_quartznet15x5,pl,14

docs/source/asr/data/scores/ru/quartznet15x5_ru.csv

@@ -0,0 +1,2 @@
+Model Name,Language,MCV Dev-Set (v??) (ru)
+stt_ru_quartznet15x5,ru,16.23

docs/source/asr/data/scores/zh/citrinet_zh.csv

@@ -0,0 +1,3 @@
+Model Name,Language,AIShell Dev-Android v2,AIShell Dev-Ios v1,AIShell Dev-Ios v2,AIShell Dev-Mic v2,AIShell Test-Android v2,AIShell Test-Ios v1,AIShell Test-Ios v2,AIShell Test-Mic v2
+stt_zh_citrinet_512,zh,,6.25%,,,,6.44%,,
+stt_zh_citrinet_1024_gamma_0_25,zh,5.2 %,,4.8 %,5.2 %,5.5 %,,5.1 %,5.5 %

docs/source/asr/data/scores/zh/conformer_zh.csv

@@ -0,0 +1,2 @@
+Model Name,Language,AIShell Dev-Android v2,AIShell Dev-Ios v1,AIShell Dev-Ios v2,AIShell Dev-Mic v2,AIShell Test-Android v2,AIShell Test-Ios v1,AIShell Test-Ios v2,AIShell Test-Mic v2
+stt_zh_conformer_transducer_large,zh,3.4,,3.2,3.4,3.4,,3.2,3.4

docs/source/asr/intro.rst

@@ -37,6 +37,7 @@ The full documentation tree is as follows:
    datasets
    asr_language_modeling
    results
+   scores
    configs
    api
    resources

docs/source/asr/scores.rst

@@ -0,0 +1,211 @@
+..
+  AUTOGENERATED DOC: DO NOT EDIT MANUALLY !
+
+Scores
+------
+
+EN
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/en/citrinet_en.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/en/conformer_en.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/en/contextnet_en.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/en/jasper10x5dr_en.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/en/quartznet15x5_en.csv
+
+--------------------
+
+CA
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/ca/quartznet15x5_ca.csv
+
+--------------------
+
+DE
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/de/citrinet_de.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/de/conformer_de.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/de/contextnet_de.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/de/quartznet15x5_de.csv
+
+--------------------
+
+ENES
+^^^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/enes/conformer_enes.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/enes/contextnet_enes.csv
+
+--------------------
+
+ES
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/es/citrinet_es.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/es/conformer_es.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/es/contextnet_es.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/es/quartznet15x5_es.csv
+
+--------------------
+
+FR
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/fr/citrinet_fr.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/fr/conformer_fr.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/fr/contextnet_fr.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/fr/quartznet15x5_fr.csv
+
+--------------------
+
+IT
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/it/quartznet15x5_it.csv
+
+--------------------
+
+PL
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/pl/quartznet15x5_pl.csv
+
+--------------------
+
+RU
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/ru/quartznet15x5_ru.csv
+
+--------------------
+
+ZH
+^^
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/zh/citrinet_zh.csv
+
+--------------------
+
+.. csv-table::
+    :header-rows: 1
+    :align: left
+    :file: data/scores/zh/conformer_zh.csv
+
+--------------------
+

docs/source/starthere/tutorials.rst

@@ -129,7 +129,7 @@ To run a tutorial:
      - `Punctuation and Capitalization <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/nlp/Punctuation_and_Capitalization.ipynb>`_
    * - NLP
      - Entity Linking
-     - `Entity Linking <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Entity_Linking_Medical.ipynb>`_
+     - `Entity Linking <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/nlp/Entity_Linking_Medical.ipynb>`_
    * - NLP
      - Named Entity Recognition - BioMegatron
      - `Named Entity Recognition - BioMegatron <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/nlp/Token_Classification-BioMegatron.ipynb>`_
@@ -138,7 +138,7 @@ To run a tutorial:
      - `Relation Extraction - BioMegatron <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/nlp/Relation_Extraction-BioMegatron.ipynb>`_
    * - NLP
      - P-Tuning/Prompt-Tuning
-     - `P-Tuning/Prompt-Tuning <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Multitask_Prompt_and_PTuning.ipynb>`_
+     - `P-Tuning/Prompt-Tuning <https://github.com/NVIDIA/NeMo/blob/stable/tutorials/nlp/Multitask_Prompt_and_PTuning.ipynb>`_
    * - TTS
      - Speech Synthesis
      - `TTS Inference <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/tts/Inference_ModelSelect.ipynb>`_