From 9df5d7bbe5f70f181b98013458e1e47944b20ca9 Mon Sep 17 00:00:00 2001 From: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:31:54 -0800 Subject: [PATCH] Content Type Classifier (#361) * add initial changes Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * add python scripts Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * edit docs Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * update max_len Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * update readmes Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * update readmes Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * rename to content type classifier Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * edit docstring Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * update arg to output_path Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * add vibhu's suggestions Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> * edit docstrings Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> --------- Signed-off-by: Sarah Yurick <sarahyurick@gmail.com> Signed-off-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> --- README.md | 2 +- docs/user-guide/api/classifiers.rst | 3 + docs/user-guide/cpuvsgpu.rst | 1 + .../distributeddataclassification.rst | 26 +++- examples/classifiers/README.md | 1 + examples/classifiers/content_type_example.py | 65 +++++++++ nemo_curator/classifiers/__init__.py | 2 + nemo_curator/classifiers/content_type.py | 138 ++++++++++++++++++ nemo_curator/classifiers/domain.py | 4 +- nemo_curator/classifiers/fineweb_edu.py | 7 +- nemo_curator/classifiers/quality.py | 6 +- nemo_curator/scripts/classifiers/README.md | 20 +++ .../content_type_classifier_inference.py | 112 ++++++++++++++ pyproject.toml | 1 + 14 files changed, 377 insertions(+), 11 deletions(-) create mode 100644 examples/classifiers/content_type_example.py create mode 100644 nemo_curator/classifiers/content_type.py create mode 100644 nemo_curator/scripts/classifiers/content_type_classifier_inference.py diff --git a/README.md b/README.md index 84f370c5f..6f00331ca 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ All of our text pipelines have great multilingual support. - [Heuristic Filtering](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/qualityfiltering.html) - Classifier Filtering - [fastText](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/qualityfiltering.html) - - GPU-Accelerated models: [Domain (English and multilingual), Quality, Safety, and Educational Content Classification](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/distributeddataclassification.html) + - GPU-Accelerated models: [Domain (English and multilingual), Quality, Safety, Educational Content, and Content Type Classification](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/distributeddataclassification.html) - **GPU-Accelerated Deduplication** - [Exact Deduplication](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/gpudeduplication.html) - [Fuzzy Deduplication](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/gpudeduplication.html) via MinHash Locality Sensitive Hashing diff --git a/docs/user-guide/api/classifiers.rst b/docs/user-guide/api/classifiers.rst index d07b74d05..856680519 100644 --- a/docs/user-guide/api/classifiers.rst +++ b/docs/user-guide/api/classifiers.rst @@ -19,3 +19,6 @@ Classifiers .. autoclass:: nemo_curator.classifiers.InstructionDataGuardClassifier :members: + +.. autoclass:: nemo_curator.classifiers.ContentTypeClassifier + :members: diff --git a/docs/user-guide/cpuvsgpu.rst b/docs/user-guide/cpuvsgpu.rst index cb875ba67..6b1b38a28 100644 --- a/docs/user-guide/cpuvsgpu.rst +++ b/docs/user-guide/cpuvsgpu.rst @@ -71,6 +71,7 @@ The following NeMo Curator modules are GPU based. * Quality Classification * AEGIS and Instruction-Data-Guard Safety Models * FineWeb Educational Content Classification + * Content Type Classification GPU modules store the ``DocumentDataset`` using a ``cudf`` backend instead of a ``pandas`` one. To read a dataset into GPU memory, one could use the following function call. diff --git a/docs/user-guide/distributeddataclassification.rst b/docs/user-guide/distributeddataclassification.rst index 70362ad27..fd4b33565 100644 --- a/docs/user-guide/distributeddataclassification.rst +++ b/docs/user-guide/distributeddataclassification.rst @@ -15,7 +15,7 @@ NeMo Curator provides a module to help users run inference with pre-trained mode This is achieved by chunking the datasets across multiple computing nodes, each equipped with multiple GPUs, to accelerate the classification task in a distributed manner. Since the classification of a single text document is independent of other documents within the dataset, we can distribute the workload across multiple nodes and GPUs to perform parallel processing. -Domain (English and multilingual), quality, content safety, and educational content models are tasks we include as examples within our module. +Domain (English and multilingual), quality, content safety, educational content, and content type models are tasks we include as examples within our module. Here, we summarize why each is useful for training an LLM: @@ -31,6 +31,8 @@ Here, we summarize why each is useful for training an LLM: - The **FineWeb Educational Content Classifier** focuses on identifying and prioritizing educational material within datasets. This classifier is especially useful for training LLMs on specialized educational content, which can improve their performance on knowledge-intensive tasks. Models trained on high-quality educational content demonstrate enhanced capabilities on academic benchmarks such as MMLU and ARC, showcasing the classifier's impact on improving the knowledge-intensive task performance of LLMs. +- The **Content Type Classifier** is designed to categorize documents into one of 11 distinct speech types based on their content. It analyzes and understands the nuances of textual information, enabling accurate classification across a diverse range of content types. + ----------------------------------------- Usage ----------------------------------------- @@ -232,6 +234,28 @@ For example, to create a dataset with only highly educational content (scores 4 high_edu_dataset = result_dataset[result_dataset["fineweb-edu-score-int"] >= 4] high_edu_dataset.to_json("high_educational_content/") +Content Type Classifier +^^^^^^^^^^^^^^^^^^^^^^^ + +The Content Type Classifier is used to categorize speech types based on their content. It analyzes and understands the nuances of textual information, enabling accurate classification across a diverse range of content types. + +Let's see how ``ContentTypeClassifier`` works in a small excerpt taken from ``examples/classifiers/content_type_example.py``: + +.. code-block:: python + + from nemo_curator.classifiers import ContentTypeClassifier + + files = get_all_files_paths_under("books_dataset/") + input_dataset = DocumentDataset.read_json(files, backend="cudf") + + content_type_classifier = ContentTypeClassifier(filter_by=["Blogs", "News"]) + result_dataset = content_type_classifier(dataset=input_dataset) + + result_dataset.to_json("blogs_and_news/") + +In this example, the content type classifier is obtained directly from `Hugging Face <https://huggingface.co/nvidia/content-type-classifier-deberta>`_. +It filters the input dataset to include only documents classified as "Blogs" or "News". + ----------------------------------------- CrossFit Integration ----------------------------------------- diff --git a/examples/classifiers/README.md b/examples/classifiers/README.md index f244ef02f..5746e98c3 100644 --- a/examples/classifiers/README.md +++ b/examples/classifiers/README.md @@ -8,6 +8,7 @@ The Python scripts in this directory demonstrate how to run classification on yo - AEGIS Safety Models - Instruction-Data-Guard Model - FineWeb Educational Content Classifier +- Content Type Classifier For more information about these classifiers, please see NeMo Curator's [Distributed Data Classification documentation](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/distributeddataclassification.html). diff --git a/examples/classifiers/content_type_example.py b/examples/classifiers/content_type_example.py new file mode 100644 index 000000000..327655604 --- /dev/null +++ b/examples/classifiers/content_type_example.py @@ -0,0 +1,65 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import time + +from nemo_curator.classifiers import ContentTypeClassifier +from nemo_curator.datasets import DocumentDataset +from nemo_curator.utils.distributed_utils import get_client +from nemo_curator.utils.script_utils import ArgumentHelper + + +def main(args): + global_st = time.time() + + # Input can be a string or list + input_file_path = "/path/to/data" + output_file_path = "./" + + client_args = ArgumentHelper.parse_client_args(args) + client_args["cluster_type"] = "gpu" + client = get_client(**client_args) + + input_dataset = DocumentDataset.read_json( + input_file_path, backend="cudf", add_filename=True + ) + + content_type_classifier = ContentTypeClassifier(filter_by=["Blogs", "News"]) + result_dataset = content_type_classifier(dataset=input_dataset) + + result_dataset.to_json(output_path=output_file_path, write_to_filename=True) + + global_et = time.time() + print( + f"Total time taken for content type classifier inference: {global_et-global_st} s", + flush=True, + ) + + client.close() + + +def attach_args( + parser=argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ), +): + argumentHelper = ArgumentHelper(parser) + argumentHelper.add_distributed_classifier_cluster_args() + + return argumentHelper.parser + + +if __name__ == "__main__": + main(attach_args().parse_args()) diff --git a/nemo_curator/classifiers/__init__.py b/nemo_curator/classifiers/__init__.py index f48cda6b0..8ddcf6e25 100644 --- a/nemo_curator/classifiers/__init__.py +++ b/nemo_curator/classifiers/__init__.py @@ -16,6 +16,7 @@ os.environ["RAPIDS_NO_INITIALIZE"] = "1" from .aegis import AegisClassifier, InstructionDataGuardClassifier +from .content_type import ContentTypeClassifier from .domain import DomainClassifier, MultilingualDomainClassifier from .fineweb_edu import FineWebEduClassifier from .quality import QualityClassifier @@ -27,4 +28,5 @@ "AegisClassifier", "InstructionDataGuardClassifier", "FineWebEduClassifier", + "ContentTypeClassifier", ] diff --git a/nemo_curator/classifiers/content_type.py b/nemo_curator/classifiers/content_type.py new file mode 100644 index 000000000..617d51726 --- /dev/null +++ b/nemo_curator/classifiers/content_type.py @@ -0,0 +1,138 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from dataclasses import dataclass +from typing import List, Optional + +os.environ["RAPIDS_NO_INITIALIZE"] = "1" +from crossfit.backend.torch.hf.model import HFModel +from transformers import AutoConfig, AutoTokenizer + +from nemo_curator.classifiers.base import ( + DistributedDataClassifier, + HFDeberta, + _get_suggest_memory_for_classifier, + _run_classifier_helper, +) +from nemo_curator.datasets import DocumentDataset + +CONTENT_TYPE_IDENTIFIER = "nvidia/content-type-classifier-deberta" + + +@dataclass +class ContentTypeModelConfig: + model: str = "microsoft/deberta-v3-base" + fc_dropout: float = 0.2 + max_len: int = 1024 + + +class ContentTypeModel(HFModel): + def __init__( + self, + config: ContentTypeModelConfig, + autocast: bool = False, + max_mem_gb: Optional[int] = None, + ): + self.config = config + self.autocast = autocast + if max_mem_gb is None: + max_mem_gb = _get_suggest_memory_for_classifier() + + super().__init__(self.config.model, max_mem_gb=max_mem_gb) + + def load_model(self, device: str = "cuda"): + model = HFDeberta.from_pretrained(CONTENT_TYPE_IDENTIFIER) + model.set_autocast(self.autocast) + model = model.to(device) + return model.eval() + + def load_tokenizer(self): + return AutoTokenizer.from_pretrained(CONTENT_TYPE_IDENTIFIER) + + def load_config(self): + return AutoConfig.from_pretrained(CONTENT_TYPE_IDENTIFIER) + + +class ContentTypeClassifier(DistributedDataClassifier): + """ + ContentTypeClassifier is a text classification model designed to categorize documents into one of 11 distinct speech types based on their content. + It analyzes and understands the nuances of textual information, enabling accurate classification across a diverse range of content types. + The pretrained model used by this class can be found on Hugging Face here: https://huggingface.co/nvidia/content-type-classifier-deberta. + This classifier is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large datasets. + + Attributes: + filter_by (list[str], optional): The classes to filter the dataset by. + If None, all classes will be included. Defaults to None. + batch_size (int): The number of samples per batch for inference. Defaults to 256. + text_field (str): The field in the dataset that should be classified. + pred_column (str): The column name where predictions will be stored. Defaults to "content_pred". + prob_column (str, optional): The column name where prediction probabilities will be stored. Defaults to None. + max_chars (int): The maximum number of characters in each document to consider for classification. Defaults to 5000. + device_type (str): The type of device to use for inference, either "cuda" or "cpu". Defaults to "cuda". + autocast (bool): Whether to use mixed precision for faster inference. Defaults to True. + max_mem_gb (int, optional): The maximum amount of memory in GB to allocate for the model. If None, + it defaults to the available GPU memory minus 4 GB. + + """ + + def __init__( + self, + filter_by: Optional[List[str]] = None, + batch_size: int = 256, + text_field: str = "text", + pred_column: str = "content_pred", + prob_column: Optional[str] = None, + max_chars: int = 5000, + device_type: str = "cuda", + autocast: bool = True, + max_mem_gb: Optional[int] = None, + ): + config = AutoConfig.from_pretrained(CONTENT_TYPE_IDENTIFIER) + + self.text_field = text_field + self.prob_column = prob_column + self.labels = list(config.label2id.keys()) + self.labels.sort(key=lambda x: config.label2id[x]) + self.out_dim = len(self.labels) + + model = ContentTypeModel( + config=ContentTypeModelConfig, autocast=autocast, max_mem_gb=max_mem_gb + ) + + super().__init__( + model=model, + labels=self.labels, + filter_by=filter_by, + batch_size=batch_size, + out_dim=self.out_dim, + pred_column=pred_column, + max_chars=max_chars, + device_type=device_type, + autocast=autocast, + ) + + def _run_classifier(self, dataset: DocumentDataset) -> DocumentDataset: + print("Starting content type classifier inference", flush=True) + df = dataset.df + df = _run_classifier_helper( + df=df, + model=self.model, + labels=self.labels, + max_chars=self.max_chars, + batch_size=self.batch_size, + label_col=self.pred_column, + text_field=self.text_field, + prob_col=self.prob_column, + ) + return DocumentDataset(df) diff --git a/nemo_curator/classifiers/domain.py b/nemo_curator/classifiers/domain.py index 57e27e8be..50e0d1cdf 100644 --- a/nemo_curator/classifiers/domain.py +++ b/nemo_curator/classifiers/domain.py @@ -148,7 +148,7 @@ class DomainClassifier(_DomainClassifier): """ DomainClassifier is a specialized classifier designed for English text domain classification tasks, utilizing the NVIDIA Domain Classifier (https://huggingface.co/nvidia/domain-classifier) model. - This class is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large datasets. + This classifier is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large datasets. Attributes: filter_by (list[str], optional): The classes to filter the dataset by. @@ -196,7 +196,7 @@ class MultilingualDomainClassifier(_DomainClassifier): MultilingualDomainClassifier is a specialized classifier designed for domain classification tasks, utilizing the NVIDIA Multilingual Domain Classifier (https://huggingface.co/nvidia/multilingual-domain-classifier) model. It supports domain classification across 52 languages. - This class is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large datasets. + This classifier is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large datasets. Attributes: filter_by (list[str], optional): The classes to filter the dataset by. diff --git a/nemo_curator/classifiers/fineweb_edu.py b/nemo_curator/classifiers/fineweb_edu.py index 932bfc8c8..01799c7ab 100644 --- a/nemo_curator/classifiers/fineweb_edu.py +++ b/nemo_curator/classifiers/fineweb_edu.py @@ -69,10 +69,9 @@ def load_config(self): class FineWebEduClassifier(DistributedDataClassifier): """ - FineWebEduClassifier is a specialized classifier designed for educational content assessment, utilizing the - Hugging Face FineWeb EDU Classifier model (https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier). - This class is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference - on large text datasets. + FineWebEduClassifier is a specialized classifier designed for educational content assessment, + utilizing the Hugging Face FineWeb EDU Classifier model (https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier). + This classifier is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large text datasets. Attributes: batch_size (int): The number of samples per batch for inference. Defaults to 256. diff --git a/nemo_curator/classifiers/quality.py b/nemo_curator/classifiers/quality.py index 90db1de55..31542b721 100644 --- a/nemo_curator/classifiers/quality.py +++ b/nemo_curator/classifiers/quality.py @@ -65,9 +65,9 @@ def load_config(self): class QualityClassifier(DistributedDataClassifier): """ - QualityClassifier is a specialized classifier designed for quality assessment tasks, utilizing the - NVIDIA Quality Classifier model (https://huggingface.co/nvidia/quality-classifier-deberta). This class is - optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large datasets. + QualityClassifier is a specialized classifier designed for quality assessment tasks, + utilizing the NVIDIA Quality Classifier model (https://huggingface.co/nvidia/quality-classifier-deberta). + This classifier is optimized for running on multi-node, multi-GPU setups to enable fast and efficient inference on large datasets. Attributes: filter_by (list[str], optional): The classes to filter the dataset by. If None, all classes will be included. Defaults to None. diff --git a/nemo_curator/scripts/classifiers/README.md b/nemo_curator/scripts/classifiers/README.md index 490ed032e..8a501b3ed 100644 --- a/nemo_curator/scripts/classifiers/README.md +++ b/nemo_curator/scripts/classifiers/README.md @@ -8,6 +8,7 @@ The Python scripts in this directory demonstrate how to run classification on yo - AEGIS Safety Models - Instruction-Data-Guard Model - FineWeb Educational Content Classifier +- Content Type Classifier For more information about these classifiers, please see NeMo Curator's [Distributed Data Classification documentation](https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/distributeddataclassification.html). @@ -136,3 +137,22 @@ fineweb_edu_classifier_inference \ ``` Additional arguments may be added for customizing a Dask cluster and client. Run `fineweb_edu_classifier_inference --help` for more information. + +#### Content type classifier inference + +```bash +# same as `python content_type_classifier_inference.py` +content_type_classifier_inference \ + --input-data-dir /path/to/data/directory \ + --output-data-dir /path/to/output/directory \ + --input-file-type "jsonl" \ + --input-file-extension "jsonl" \ + --output-file-type "jsonl" \ + --input-text-field "text" \ + --batch-size 64 \ + --autocast \ + --max-chars 5000 \ + --device "gpu" +``` + +Additional arguments may be added for customizing a Dask cluster and client. Run `content_type_classifier_inference --help` for more information. diff --git a/nemo_curator/scripts/classifiers/content_type_classifier_inference.py b/nemo_curator/scripts/classifiers/content_type_classifier_inference.py new file mode 100644 index 000000000..57c25c471 --- /dev/null +++ b/nemo_curator/scripts/classifiers/content_type_classifier_inference.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import warnings + +os.environ["RAPIDS_NO_INITIALIZE"] = "1" + +from nemo_curator.classifiers import ContentTypeClassifier +from nemo_curator.datasets import DocumentDataset + +# Get relevant args +from nemo_curator.utils.distributed_utils import get_client, read_data, write_to_disk +from nemo_curator.utils.file_utils import get_remaining_files +from nemo_curator.utils.script_utils import ArgumentHelper + +warnings.filterwarnings("ignore") + + +def main(): + args = ArgumentHelper.parse_distributed_classifier_args( + description="Run content type classifier inference." + ).parse_args() + print(f"Arguments parsed = {args}", flush=True) + client_args = ArgumentHelper.parse_client_args(args) + client_args["cluster_type"] = "gpu" + client = get_client(**client_args) + print("Starting content type classifier inference", flush=True) + global_st = time.time() + files_per_run = len(client.scheduler_info()["workers"]) * 2 + + if not os.path.exists(args.output_data_dir): + os.makedirs(args.output_data_dir) + + # Some times jsonl files are stored as .json + # So to handle that case we can pass the input_file_extension + if args.input_file_extension is not None: + input_file_extension = args.input_file_extension + else: + input_file_extension = args.input_file_type + + input_files = get_remaining_files( + args.input_data_dir, args.output_data_dir, input_file_extension + ) + print(f"Total input files {len(input_files)}", flush=True) + + if args.input_file_type == "pickle": + add_filename = False + else: + add_filename = True + + content_type_classifier = ContentTypeClassifier( + text_field=args.input_text_field, + max_chars=args.max_chars, + batch_size=args.batch_size, + autocast=args.autocast, + max_mem_gb=args.max_mem_gb_classifier, + ) + + for file_batch_id, i in enumerate(range(0, len(input_files), files_per_run)): + batch_st = time.time() + current_batch_files = input_files[i : i + files_per_run] + print( + f"File Batch ID {file_batch_id}: total input files {len(current_batch_files)}", + flush=True, + ) + df = read_data( + input_files=current_batch_files, + file_type=args.input_file_type, + add_filename=add_filename, + ) + df = content_type_classifier(DocumentDataset(df)).df + print(f"Total input Dask DataFrame partitions {df.npartitions}", flush=True) + + write_to_disk( + df=df, + output_path=args.output_data_dir, + write_to_filename=add_filename, + output_type=args.output_file_type, + ) + batch_et = time.time() + print( + f"File Batch ID {file_batch_id}: completed in {batch_et-batch_st} seconds", + flush=True, + ) + + global_et = time.time() + print( + f"Total time taken for content type classifier inference: {global_et-global_st} s", + flush=True, + ) + client.close() + + +def console_script(): + main() + + +if __name__ == "__main__": + console_script() diff --git a/pyproject.toml b/pyproject.toml index ab85bf2eb..87dce1a52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,6 +149,7 @@ domain_classifier_inference = "nemo_curator.scripts.classifiers.domain_classifie quality_classifier_inference = "nemo_curator.scripts.classifiers.quality_classifier_inference:console_script" aegis_classifier_inference = "nemo_curator.scripts.classifiers.aegis_classifier_inference:console_script" fineweb_edu_classifier_inference = "nemo_curator.scripts.classifiers.fineweb_edu_classifier_inference:console_script" +content_type_classifier_inference = "nemo_curator.scripts.classifiers.content_type_classifier_inference:console_script" instruction_data_guard_classifier_inference = "nemo_curator.scripts.classifiers.instruction_data_guard_classifier_inference:console_script" multilingual_domain_classifier_inference = "nemo_curator.scripts.classifiers.multilingual_domain_classifier_inference:console_script" verify_classification_results = "nemo_curator.scripts.verify_classification_results:console_script"