Safeguard nemo_text_processing installation on ARM (#7485) (#7619)

* safeguard nemo_text_processing installing * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update check --------- Signed-off-by: Jason <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
NVIDIA · Oct 5, 2023 · 4381c58 · 4381c58
1 parent 70762de
commit 4381c58
Show file tree

Hide file tree

Showing 20 changed files with 199 additions and 67 deletions.
diff --git a/examples/nlp/duplex_text_normalization/duplex_text_normalization_infer.py b/examples/nlp/duplex_text_normalization/duplex_text_normalization_infer.py
@@ -50,7 +50,6 @@
 from typing import List
 
 from helpers import DECODER_MODEL, TAGGER_MODEL, instantiate_model_and_trainer
-from nemo_text_processing.text_normalization.data_loader_utils import post_process_punct
 from nn_wfst.en.electronic.normalize import ElectronicNormalizer
 from nn_wfst.en.whitelist.normalize import WhitelistNormalizer
 from omegaconf import DictConfig, OmegaConf
@@ -60,6 +59,15 @@
 from nemo.core.config import hydra_runner
 from nemo.utils import logging
 
+try:
+    from nemo_text_processing.text_normalization.data_loader_utils import post_process_punct
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
 
 @hydra_runner(config_path="conf", config_name="duplex_tn_config")
 def main(cfg: DictConfig) -> None:

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/normalize.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/normalize.py
@@ -12,16 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nemo_text_processing.text_normalization.normalize import Normalizer
-from nemo_text_processing.text_normalization.token_parser import TokenParser
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+    from nemo_text_processing.text_normalization.token_parser import TokenParser
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor
 
 
 class ElectronicNormalizer(Normalizer):
     """
     Normalizer for ELECTRONIC.
-    
+
     Args:
         input_case: accepting either "lower_cased" or "cased" input.
         lang: language

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/tokenize_and_classify.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/tokenize_and_classify.py
@@ -15,28 +15,35 @@
 
 import os
 
-import pynini
-from nemo_text_processing.text_normalization.en.graph_utils import (
-    NEMO_WHITE_SPACE,
-    GraphFst,
-    delete_extra_space,
-    delete_space,
-    generator_main,
-)
-from nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst
-from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
-from nemo_text_processing.text_normalization.en.taggers.word import WordFst
-from pynini.lib import pynutil
+try:
+    import pynini
+    from nemo_text_processing.text_normalization.en.graph_utils import (
+        NEMO_WHITE_SPACE,
+        GraphFst,
+        delete_extra_space,
+        delete_space,
+        generator_main,
+    )
+    from nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst
+    from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
+    from nemo_text_processing.text_normalization.en.taggers.word import WordFst
+    from pynini.lib import pynutil
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 from nemo.utils import logging
 
 
 class ClassifyFst(GraphFst):
     """
     Final class that composes all other classification grammars. This class can process an entire sentence including punctuation.
-    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. 
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
     More details to deployment at NeMo/tools/text_processing_deployment.
-    
+
     Args:
         input_case: accepting either "lower_cased" or "cased" input.
         deterministic: if True will provide a single transduction option,

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/verbalize.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/verbalize.py
@@ -12,15 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from nemo_text_processing.text_normalization.en.graph_utils import GraphFst
-from nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst
+try:
+    from nemo_text_processing.text_normalization.en.graph_utils import GraphFst
+    from nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 
 class VerbalizeFst(GraphFst):
     """
     Composes other verbalizer grammars.
-    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. 
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
     More details to deployment at NeMo/tools/text_processing_deployment.
 
     Args:

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/verbalize_final.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/verbalize_final.py
@@ -12,12 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-import pynini
-from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space
-from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst
-from nn_wfst.en.electronic.verbalize import VerbalizeFst
-from pynini.lib import pynutil
+try:
+    import pynini
+    from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space
+    from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst
+    from nn_wfst.en.electronic.verbalize import VerbalizeFst
+    from pynini.lib import pynutil
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 
 class VerbalizeFinalFst(GraphFst):

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/normalize.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/normalize.py
@@ -12,16 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nemo_text_processing.text_normalization.normalize import Normalizer
-from nemo_text_processing.text_normalization.token_parser import TokenParser
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+    from nemo_text_processing.text_normalization.token_parser import TokenParser
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor
 
 
 class WhitelistNormalizer(Normalizer):
     """
     Normalizer for WHITELIST.
-    
+
     Args:
         input_case: accepting either "lower_cased" or "cased" input.
         lang: language

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/tokenize_and_classify.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/tokenize_and_classify.py
@@ -15,28 +15,35 @@
 
 import os
 
-import pynini
-from nemo_text_processing.text_normalization.en.graph_utils import (
-    NEMO_WHITE_SPACE,
-    GraphFst,
-    delete_extra_space,
-    delete_space,
-    generator_main,
-)
-from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
-from nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst
-from nemo_text_processing.text_normalization.en.taggers.word import WordFst
-from pynini.lib import pynutil
+try:
+    import pynini
+    from nemo_text_processing.text_normalization.en.graph_utils import (
+        NEMO_WHITE_SPACE,
+        GraphFst,
+        delete_extra_space,
+        delete_space,
+        generator_main,
+    )
+    from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
+    from nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst
+    from nemo_text_processing.text_normalization.en.taggers.word import WordFst
+    from pynini.lib import pynutil
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 from nemo.utils import logging
 
 
 class ClassifyFst(GraphFst):
     """
     Final class that composes all other classification grammars. This class can process an entire sentence including punctuation.
-    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. 
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
     More details to deployment at NeMo/tools/text_processing_deployment.
-    
+
     Args:
         input_case: accepting either "lower_cased" or "cased" input.
         deterministic: if True will provide a single transduction option,

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/verbalize.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/verbalize.py
@@ -12,15 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from nemo_text_processing.text_normalization.en.graph_utils import GraphFst
-from nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
+try:
+    from nemo_text_processing.text_normalization.en.graph_utils import GraphFst
+    from nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 
 class VerbalizeFst(GraphFst):
     """
     Composes other verbalizer grammars.
-    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. 
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
     More details to deployment at NeMo/tools/text_processing_deployment.
 
     Args:

diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/verbalize_final.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/verbalize_final.py
@@ -13,11 +13,18 @@
 # limitations under the License.
 
 
-import pynini
-from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space
-from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst
-from nn_wfst.en.electronic.verbalize import VerbalizeFst
-from pynini.lib import pynutil
+try:
+    import pynini
+    from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space
+    from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst
+    from nn_wfst.en.electronic.verbalize import VerbalizeFst
+    from pynini.lib import pynutil
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
 
 
 class VerbalizeFinalFst(GraphFst):

diff --git a/requirements/requirements_tts.txt b/requirements/requirements_tts.txt
@@ -4,7 +4,8 @@ jieba
 kornia
 librosa
 matplotlib
-nemo_text_processing
+# pynini does not currently support aarch, disable nemo_text_processing for now
+nemo_text_processing; 'arm' not in platform_machine and 'aarch' not in platform_machine
 nltk
 pandas
 pypinyin

diff --git a/scripts/dataset_processing/tts/hui_acg/get_data.py b/scripts/dataset_processing/tts/hui_acg/get_data.py
@@ -21,9 +21,17 @@
 
 import pandas as pd
 from joblib import Parallel, delayed
-from nemo_text_processing.text_normalization.normalize import Normalizer
 from tqdm import tqdm
 
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
 from nemo.utils import logging
 
 # full corpus.

diff --git a/scripts/dataset_processing/tts/ljspeech/get_data.py b/scripts/dataset_processing/tts/ljspeech/get_data.py
@@ -20,9 +20,17 @@
 
 import sox
 import wget
-from nemo_text_processing.text_normalization.normalize import Normalizer
 from tqdm import tqdm
 
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
 
 def get_args():
     parser = argparse.ArgumentParser(description='Download LJSpeech and create manifests with predefined split')

diff --git a/scripts/dataset_processing/tts/preprocess_text.py b/scripts/dataset_processing/tts/preprocess_text.py
@@ -32,10 +32,18 @@
 
 from hydra.utils import instantiate
 from joblib import Parallel, delayed
-from nemo_text_processing.text_normalization.normalize import Normalizer
 from omegaconf import OmegaConf
 from tqdm import tqdm
 
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
 from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
 
 

diff --git a/scripts/dataset_processing/tts/sfbilingual/get_data.py b/scripts/dataset_processing/tts/sfbilingual/get_data.py
@@ -20,9 +20,17 @@
 from pathlib import Path
 
 import numpy as np
-from nemo_text_processing.text_normalization.normalize import Normalizer
 from opencc import OpenCC
 
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
 
 def get_args():
     parser = argparse.ArgumentParser(

diff --git a/scripts/dataset_processing/tts/thorsten_neutral/get_data.py b/scripts/dataset_processing/tts/thorsten_neutral/get_data.py
@@ -32,9 +32,17 @@
 from pathlib import Path
 
 from joblib import Parallel, delayed
-from nemo_text_processing.text_normalization.normalize import Normalizer
 from tqdm import tqdm
 
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
 from nemo.utils import logging
 
 # Thorsten Müller published two neural voice datasets, 21.02 and 22.10.