From bb3c712f959317bc50b4b7ea1fa719359f8108a6 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 11 Feb 2026 10:58:38 -0500 Subject: [PATCH 1/6] Fix: fasttext predict call for numpy>2 (#1482) Signed-off-by: Ayush Dattagupta Signed-off-by: Omkar Kabde --- nemo_curator/stages/text/filters/fasttext_filter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nemo_curator/stages/text/filters/fasttext_filter.py b/nemo_curator/stages/text/filters/fasttext_filter.py index 75e6a070cc..f092a038e2 100644 --- a/nemo_curator/stages/text/filters/fasttext_filter.py +++ b/nemo_curator/stages/text/filters/fasttext_filter.py @@ -44,9 +44,9 @@ def score_document(self, text: str) -> float: model = self._fasttext_quality_filter_model text = text.replace("\n", " ").replace("__label__", " ") - pred = model.predict(text) - document_score = pred[1][0] - if pred[0][0] != self._label: + label, score = model.predict([text]) + document_score = score[0][0].item() + if label[0][0] != self._label: document_score = 1 - document_score return document_score @@ -78,9 +78,9 @@ def score_document(self, text: str) -> list[float | str]: model = self._fasttext_langid_model pp = text.strip().replace("\n", " ") - label, score = model.predict(pp, k=1) - score = score[0] - lang_code = label[0][-2:].upper() + label, score = model.predict([pp], k=1) + score = score[0][0].item() + lang_code = label[0][0][-2:].upper() # Need to convert it to a string to allow backend conversions return str([score, lang_code]) From 1cd45d4661f879c7c67b34afeec5d08c9d7e01d2 Mon Sep 17 00:00:00 2001 From: Abhinav Garg Date: Wed, 11 Feb 2026 08:32:09 -0800 Subject: [PATCH 2/6] =?UTF-8?q?Update=20transformers=20dependency=20to=20e?= =?UTF-8?q?xact=20version=204.55.2=20in=20pyproject.t=E2=80=A6=20(#1471)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update transformers dependency to exact version 4.55.2 in pyproject.toml and uv.lock to prevent import failures in Cosmos Embed. Downgrade tokenizers version to 0.21.4 for compatibility. Signed-off-by: Abhinav Garg * Update transformers dependency in pyproject.toml and uv.lock to allow versions up to 4.55.2, ensuring compatibility with Cosmos Embed imports. Signed-off-by: Abhinav Garg --------- Signed-off-by: Abhinav Garg Co-authored-by: Ayush Dattagupta Signed-off-by: Omkar Kabde --- pyproject.toml | 2 +- uv.lock | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 32b3fd8ef8..d07066fe52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,7 +202,7 @@ override-dependencies = [ "numpy>=2.0.0,<=2.2.0", # Override nemo-toolkits constraint of <2.0.0, upperbounds for Numba compatibility "protobuf>=5.29.5", # Override nemo-toolkits constraint of ~=5.29.5 "setuptools>=80.10.1", # Override setuptools range in other dependencies to address CVE GHSA-58pv-8j8x-9vj2 - "transformers>=4.55.2", + "transformers<=4.55.2", # Else Cosmos Embed imports fail ] diff --git a/uv.lock b/uv.lock index bd39e6afc9..0e48b348e0 100644 --- a/uv.lock +++ b/uv.lock @@ -51,7 +51,7 @@ overrides = [ { name = "numpy", specifier = ">=2.0.0,<=2.2.0" }, { name = "protobuf", specifier = ">=5.29.5" }, { name = "setuptools", specifier = ">=80.10.1" }, - { name = "transformers", specifier = ">=4.55.2" }, + { name = "transformers", specifier = "<=4.55.2" }, ] [[package]] @@ -7714,27 +7714,27 @@ wheels = [ [[package]] name = "tokenizers" -version = "0.22.1" +version = "0.21.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/46/fb6854cec3278fbfa4a75b50232c77622bc517ac886156e6afbfa4d8fc6e/tokenizers-0.22.1.tar.gz", hash = "sha256:61de6522785310a309b3407bac22d99c4db5dba349935e99e4d15ea2226af2d9", size = 363123, upload-time = "2025-09-19T09:49:23.424Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/2f/402986d0823f8d7ca139d969af2917fefaa9b947d1fb32f6168c509f2492/tokenizers-0.21.4.tar.gz", hash = "sha256:fa23f85fbc9a02ec5c6978da172cdcbac23498c3ca9f3645c5c68740ac007880", size = 351253, upload-time = "2025-07-28T15:48:54.325Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/33/f4b2d94ada7ab297328fc671fed209368ddb82f965ec2224eb1892674c3a/tokenizers-0.22.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59fdb013df17455e5f950b4b834a7b3ee2e0271e6378ccb33aa74d178b513c73", size = 3069318, upload-time = "2025-09-19T09:49:11.848Z" }, - { url = "https://files.pythonhosted.org/packages/1c/58/2aa8c874d02b974990e89ff95826a4852a8b2a273c7d1b4411cdd45a4565/tokenizers-0.22.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d4e484f7b0827021ac5f9f71d4794aaef62b979ab7608593da22b1d2e3c4edc", size = 2926478, upload-time = "2025-09-19T09:49:09.759Z" }, - { url = "https://files.pythonhosted.org/packages/1e/3b/55e64befa1e7bfea963cf4b787b2cea1011362c4193f5477047532ce127e/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d2962dd28bc67c1f205ab180578a78eef89ac60ca7ef7cbe9635a46a56422a", size = 3256994, upload-time = "2025-09-19T09:48:56.701Z" }, - { url = "https://files.pythonhosted.org/packages/71/0b/fbfecf42f67d9b7b80fde4aabb2b3110a97fac6585c9470b5bff103a80cb/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38201f15cdb1f8a6843e6563e6e79f4abd053394992b9bbdf5213ea3469b4ae7", size = 3153141, upload-time = "2025-09-19T09:48:59.749Z" }, - { url = "https://files.pythonhosted.org/packages/17/a9/b38f4e74e0817af8f8ef925507c63c6ae8171e3c4cb2d5d4624bf58fca69/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1cbe5454c9a15df1b3443c726063d930c16f047a3cc724b9e6e1a91140e5a21", size = 3508049, upload-time = "2025-09-19T09:49:05.868Z" }, - { url = "https://files.pythonhosted.org/packages/d2/48/dd2b3dac46bb9134a88e35d72e1aa4869579eacc1a27238f1577270773ff/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7d094ae6312d69cc2a872b54b91b309f4f6fbce871ef28eb27b52a98e4d0214", size = 3710730, upload-time = "2025-09-19T09:49:01.832Z" }, - { url = "https://files.pythonhosted.org/packages/93/0e/ccabc8d16ae4ba84a55d41345207c1e2ea88784651a5a487547d80851398/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afd7594a56656ace95cdd6df4cca2e4059d294c5cfb1679c57824b605556cb2f", size = 3412560, upload-time = "2025-09-19T09:49:03.867Z" }, - { url = "https://files.pythonhosted.org/packages/d0/c6/dc3a0db5a6766416c32c034286d7c2d406da1f498e4de04ab1b8959edd00/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ef6063d7a84994129732b47e7915e8710f27f99f3a3260b8a38fc7ccd083f4", size = 3250221, upload-time = "2025-09-19T09:49:07.664Z" }, - { url = "https://files.pythonhosted.org/packages/d7/a6/2c8486eef79671601ff57b093889a345dd3d576713ef047776015dc66de7/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ba0a64f450b9ef412c98f6bcd2a50c6df6e2443b560024a09fa6a03189726879", size = 9345569, upload-time = "2025-09-19T09:49:14.214Z" }, - { url = "https://files.pythonhosted.org/packages/6b/16/32ce667f14c35537f5f605fe9bea3e415ea1b0a646389d2295ec348d5657/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:331d6d149fa9c7d632cde4490fb8bbb12337fa3a0232e77892be656464f4b446", size = 9271599, upload-time = "2025-09-19T09:49:16.639Z" }, - { url = "https://files.pythonhosted.org/packages/51/7c/a5f7898a3f6baa3fc2685c705e04c98c1094c523051c805cdd9306b8f87e/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:607989f2ea68a46cb1dfbaf3e3aabdf3f21d8748312dbeb6263d1b3b66c5010a", size = 9533862, upload-time = "2025-09-19T09:49:19.146Z" }, - { url = "https://files.pythonhosted.org/packages/36/65/7e75caea90bc73c1dd8d40438adf1a7bc26af3b8d0a6705ea190462506e1/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a0f307d490295717726598ef6fa4f24af9d484809223bbc253b201c740a06390", size = 9681250, upload-time = "2025-09-19T09:49:21.501Z" }, - { url = "https://files.pythonhosted.org/packages/30/2c/959dddef581b46e6209da82df3b78471e96260e2bc463f89d23b1bf0e52a/tokenizers-0.22.1-cp39-abi3-win32.whl", hash = "sha256:b5120eed1442765cd90b903bb6cfef781fd8fe64e34ccaecbae4c619b7b12a82", size = 2472003, upload-time = "2025-09-19T09:49:27.089Z" }, - { url = "https://files.pythonhosted.org/packages/b3/46/e33a8c93907b631a99377ef4c5f817ab453d0b34f93529421f42ff559671/tokenizers-0.22.1-cp39-abi3-win_amd64.whl", hash = "sha256:65fd6e3fb11ca1e78a6a93602490f134d1fdeb13bcef99389d5102ea318ed138", size = 2674684, upload-time = "2025-09-19T09:49:24.953Z" }, + { url = "https://files.pythonhosted.org/packages/98/c6/fdb6f72bf6454f52eb4a2510be7fb0f614e541a2554d6210e370d85efff4/tokenizers-0.21.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ccc10a7c3bcefe0f242867dc914fc1226ee44321eb618cfe3019b5df3400133", size = 2863987, upload-time = "2025-07-28T15:48:44.877Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a6/28975479e35ddc751dc1ddc97b9b69bf7fcf074db31548aab37f8116674c/tokenizers-0.21.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e2f601a8e0cd5be5cc7506b20a79112370b9b3e9cb5f13f68ab11acd6ca7d60", size = 2732457, upload-time = "2025-07-28T15:48:43.265Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8f/24f39d7b5c726b7b0be95dca04f344df278a3fe3a4deb15a975d194cbb32/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39b376f5a1aee67b4d29032ee85511bbd1b99007ec735f7f35c8a2eb104eade5", size = 3012624, upload-time = "2025-07-28T13:22:43.895Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/26358925717687a58cb74d7a508de96649544fad5778f0cd9827398dc499/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2107ad649e2cda4488d41dfd031469e9da3fcbfd6183e74e4958fa729ffbf9c6", size = 2939681, upload-time = "2025-07-28T13:22:47.499Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/cc300fea5db2ab5ddc2c8aea5757a27b89c84469899710c3aeddc1d39801/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c73012da95afafdf235ba80047699df4384fdc481527448a078ffd00e45a7d9", size = 3247445, upload-time = "2025-07-28T15:48:39.711Z" }, + { url = "https://files.pythonhosted.org/packages/be/bf/98cb4b9c3c4afd8be89cfa6423704337dc20b73eb4180397a6e0d456c334/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f23186c40395fc390d27f519679a58023f368a0aad234af145e0f39ad1212732", size = 3428014, upload-time = "2025-07-28T13:22:49.569Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/96c1cc780e6ca7f01a57c13235dd05b7bc1c0f3588512ebe9d1331b5f5ae/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc88bb34e23a54cc42713d6d98af5f1bf79c07653d24fe984d2d695ba2c922a2", size = 3193197, upload-time = "2025-07-28T13:22:51.471Z" }, + { url = "https://files.pythonhosted.org/packages/f2/90/273b6c7ec78af547694eddeea9e05de771278bd20476525ab930cecaf7d8/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51b7eabb104f46c1c50b486520555715457ae833d5aee9ff6ae853d1130506ff", size = 3115426, upload-time = "2025-07-28T15:48:41.439Z" }, + { url = "https://files.pythonhosted.org/packages/91/43/c640d5a07e95f1cf9d2c92501f20a25f179ac53a4f71e1489a3dcfcc67ee/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:714b05b2e1af1288bd1bc56ce496c4cebb64a20d158ee802887757791191e6e2", size = 9089127, upload-time = "2025-07-28T15:48:46.472Z" }, + { url = "https://files.pythonhosted.org/packages/44/a1/dd23edd6271d4dca788e5200a807b49ec3e6987815cd9d0a07ad9c96c7c2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1340ff877ceedfa937544b7d79f5b7becf33a4cfb58f89b3b49927004ef66f78", size = 9055243, upload-time = "2025-07-28T15:48:48.539Z" }, + { url = "https://files.pythonhosted.org/packages/21/2b/b410d6e9021c4b7ddb57248304dc817c4d4970b73b6ee343674914701197/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3c1f4317576e465ac9ef0d165b247825a2a4078bcd01cba6b54b867bdf9fdd8b", size = 9298237, upload-time = "2025-07-28T15:48:50.443Z" }, + { url = "https://files.pythonhosted.org/packages/b7/0a/42348c995c67e2e6e5c89ffb9cfd68507cbaeb84ff39c49ee6e0a6dd0fd2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c212aa4e45ec0bb5274b16b6f31dd3f1c41944025c2358faaa5782c754e84c24", size = 9461980, upload-time = "2025-07-28T15:48:52.325Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d3/dacccd834404cd71b5c334882f3ba40331ad2120e69ded32cf5fda9a7436/tokenizers-0.21.4-cp39-abi3-win32.whl", hash = "sha256:6c42a930bc5f4c47f4ea775c91de47d27910881902b0f20e4990ebe045a415d0", size = 2329871, upload-time = "2025-07-28T15:48:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/41/f2/fd673d979185f5dcbac4be7d09461cbb99751554ffb6718d0013af8604cb/tokenizers-0.21.4-cp39-abi3-win_amd64.whl", hash = "sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597", size = 2507568, upload-time = "2025-07-28T15:48:55.456Z" }, ] [[package]] @@ -8088,7 +8088,7 @@ wheels = [ [[package]] name = "transformers" -version = "4.57.3" +version = "4.55.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, @@ -8102,9 +8102,9 @@ dependencies = [ { name = "tokenizers" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dd/70/d42a739e8dfde3d92bb2fff5819cbf331fe9657323221e79415cd5eb65ee/transformers-4.57.3.tar.gz", hash = "sha256:df4945029aaddd7c09eec5cad851f30662f8bd1746721b34cc031d70c65afebc", size = 10139680, upload-time = "2025-11-25T15:51:30.139Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/a5/d8b8a1f3a051daeb5f11253bb69fc241f193d1c0566e299210ed9220ff4e/transformers-4.55.2.tar.gz", hash = "sha256:a45ec60c03474fd67adbce5c434685051b7608b3f4f167c25aa6aeb1cad16d4f", size = 9571466, upload-time = "2025-08-13T18:25:43.767Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/6b/2f416568b3c4c91c96e5a365d164f8a4a4a88030aa8ab4644181fdadce97/transformers-4.57.3-py3-none-any.whl", hash = "sha256:c77d353a4851b1880191603d36acb313411d3577f6e2897814f333841f7003f4", size = 11993463, upload-time = "2025-11-25T15:51:26.493Z" }, + { url = "https://files.pythonhosted.org/packages/db/5a/022ac010bedfb5119734cf9d743cf1d830cb4c604f53bb1552216f4344dc/transformers-4.55.2-py3-none-any.whl", hash = "sha256:097e3c2e2c0c9681db3da9d748d8f9d6a724c644514673d0030e8c5a1109f1f1", size = 11269748, upload-time = "2025-08-13T18:25:40.394Z" }, ] [[package]] From e715abaa8b993c66ee5f847977144b28a109aa33 Mon Sep 17 00:00:00 2001 From: Omkar Kabde Date: Wed, 11 Feb 2026 23:57:54 +0530 Subject: [PATCH 3/6] remove property and add check Signed-off-by: Omkar Kabde --- nemo_curator/stages/base.py | 9 +++++++++ nemo_curator/stages/synthetic/nemotron_cc/base.py | 5 +---- .../stages/synthetic/nemotron_cc/nemotron_cc.py | 10 ++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/nemo_curator/stages/base.py b/nemo_curator/stages/base.py index e0ae445a1b..3c55f15c98 100644 --- a/nemo_curator/stages/base.py +++ b/nemo_curator/stages/base.py @@ -112,6 +112,15 @@ def __init_subclass__(cls, **kwargs): msg = f"{cls.__name__} must not override '_batch_size'" raise TypeError(msg) + for attr in ("name", "resources", "batch_size"): + if isinstance(cls.__dict__.get(attr), property): + msg = ( + f"{cls.__name__} must not define '{attr}' as a @property. " + f"Use a plain class attribute or dataclass field instead, " + f"so that ProcessingStage.with_() can override it." + ) + raise TypeError(msg) + def num_workers(self) -> int | None: """Number of workers required. If None, then executor will determine the number of workers.""" return None diff --git a/nemo_curator/stages/synthetic/nemotron_cc/base.py b/nemo_curator/stages/synthetic/nemotron_cc/base.py index 6d93ebb394..bc929d3254 100644 --- a/nemo_curator/stages/synthetic/nemotron_cc/base.py +++ b/nemo_curator/stages/synthetic/nemotron_cc/base.py @@ -41,10 +41,7 @@ class BaseSyntheticStage(ProcessingStage[DocumentBatch, DocumentBatch]): client: AsyncLLMClient | LLMClient = None model_name: str = None generation_config: GenerationConfig | None = None - - @property - def name(self) -> str: - return "NemotronCCBaseStage" + name: str = "NemotronCCBaseStage" def __post_init__(self) -> None: self.is_async_client = isinstance(self.client, AsyncLLMClient) diff --git a/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py b/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py index 8964c4c972..c9f17fb838 100644 --- a/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py +++ b/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py @@ -64,10 +64,7 @@ class DiverseQAPostProcessingStage(ProcessingStage[DocumentBatch, DocumentBatch] tokenizer: AutoTokenizer | None = None prefix: str = "Here are the questions and answers based on the provided text:" max_num_pairs: int = 10 - - @property - def name(self) -> str: - return "DiverseQAPostProcessing" + name: str = "DiverseQAPostProcessing" def process(self, batch: DocumentBatch) -> DocumentBatch: df = batch.to_pandas() @@ -149,10 +146,7 @@ class KnowledgeListPostProcessingStage(ProcessingStage[DocumentBatch, DocumentBa """ input_field: str = "knowledge_list" - - @property - def name(self) -> str: - return "KnowledgeListPostProcessing" + name: str = "KnowledgeListPostProcessing" def process(self, batch: DocumentBatch) -> DocumentBatch: df = batch.to_pandas() From d0ab9981630b53b18bbd8a0982bd778429a85ecd Mon Sep 17 00:00:00 2001 From: Omkar Kabde Date: Thu, 12 Feb 2026 01:13:54 +0530 Subject: [PATCH 4/6] add tests Signed-off-by: Omkar Kabde --- tests/stages/common/test_base.py | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/stages/common/test_base.py b/tests/stages/common/test_base.py index e1ee36cd31..d70a837e6c 100644 --- a/tests/stages/common/test_base.py +++ b/tests/stages/common/test_base.py @@ -340,6 +340,68 @@ def inputs(self) -> tuple[list[str], list[str]]: def outputs(self) -> tuple[list[str], list[str]]: return [], [] + def test_name_property_decorator(self): + """Test that ProcessingStage raises an error if a derived class defines 'name' as a @property.""" + with pytest.raises(TypeError, match="must not define 'name' as a @property"): + + class MockStagePropertyName(ProcessingStage[MockTask, MockTask]): + @property + def name(self) -> str: + return "PropertyName" + + def process(self, task: MockTask) -> MockTask: + return task + + def test_resources_property_decorator(self): + """Test that ProcessingStage raises an error if a derived class defines 'resources' as a @property.""" + with pytest.raises(TypeError, match="must not define 'resources' as a @property"): + + class MockStagePropertyResources(ProcessingStage[MockTask, MockTask]): + name = "MockStagePropertyResources" + + @property + def resources(self) -> Resources: + return Resources(cpus=1.0) + + def process(self, task: MockTask) -> MockTask: + return task + + def test_batch_size_property_decorator(self): + """Test that ProcessingStage raises an error if a derived class defines 'batch_size' as a @property.""" + with pytest.raises(TypeError, match="must not define 'batch_size' as a @property"): + + class MockStagePropertyBatchSize(ProcessingStage[MockTask, MockTask]): + name = "MockStagePropertyBatchSize" + + @property + def batch_size(self) -> int: + return 1 + + def process(self, task: MockTask) -> MockTask: + return task + + def test_plain_attribute_override_allowed(self): + """Test that plain attribute overrides for name, resources, and batch_size are still allowed.""" + + class PlainAttrStage(ProcessingStage[MockTask, MockTask]): + name = "PlainAttrStage" + resources = Resources(cpus=4.0) + batch_size = 8 + + def process(self, task: MockTask) -> MockTask: + return task + + stage = PlainAttrStage() + assert stage.name == "PlainAttrStage" + assert stage.resources == Resources(cpus=4.0) + assert stage.batch_size == 8 + + # with_() should work fine on plain attributes + modified = stage.with_(name="Modified", resources=Resources(cpus=2.0), batch_size=4) + assert modified.name == "Modified" + assert modified.resources == Resources(cpus=2.0) + assert modified.batch_size == 4 + def test_nested_class_inheritance(self): """Test that nested class inheritance raises an error if a derived class overrides the _name, _resources, or _batch_size property.""" with pytest.raises(TypeError, match="MockStageNestedOverriddenName must not override '_name'"): From ac5b0e13260b6b4b7e5a14fb164c09a551565788 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Wed, 11 Feb 2026 14:19:46 -0500 Subject: [PATCH 5/6] docs: vdr feedback (#1477) * docs: vdr feedback Signed-off-by: Lawrence Lane * Update docs/admin/installation.md Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> Signed-off-by: Lawrence Lane * Update docs/admin/installation.md Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> Signed-off-by: Lawrence Lane * Update docs/admin/installation.md Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> Signed-off-by: Lawrence Lane * Update docs/curate-text/process-data/quality-assessment/distributed-classifier.md Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> Signed-off-by: Lawrence Lane * feedback Signed-off-by: Lawrence Lane * feedback Signed-off-by: Lawrence Lane * re order sidebar Signed-off-by: Lawrence Lane * release notes draft Signed-off-by: Lawrence Lane * feedback Signed-off-by: Lawrence Lane * remove more internvid content Signed-off-by: Lawrence Lane * release note fix Signed-off-by: Lawrence Lane * Update docs/curate-video/tutorials/split-dedup.md Signed-off-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> --------- Signed-off-by: Lawrence Lane Signed-off-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com> Signed-off-by: Omkar Kabde --- docs/about/release-notes/index.md | 307 +++++++----------- docs/admin/installation.md | 27 +- docs/broken_links_false_positives.json | 1 + .../process-data/deduplication/fuzzy.md | 4 + .../process-data/deduplication/semdedup.md | 4 + .../distributed-classifier.md | 36 +- .../process-data/captions-preview.md | 2 +- docs/curate-video/process-data/clipping.md | 4 +- docs/curate-video/process-data/dedup.md | 8 +- docs/curate-video/process-data/embeddings.md | 2 +- docs/curate-video/process-data/filtering.md | 4 +- .../process-data/frame-extraction.md | 4 +- docs/curate-video/process-data/index.md | 2 +- docs/curate-video/save-export.md | 9 +- docs/curate-video/tutorials/beginner.md | 4 +- docs/curate-video/tutorials/split-dedup.md | 16 +- docs/get-started/audio.md | 2 +- docs/get-started/image.md | 22 +- docs/get-started/text.md | 10 +- docs/get-started/video.md | 23 +- docs/index.md | 16 +- docs/project.json | 2 +- .../infrastructure/container-environments.md | 4 +- tutorials/quickstart.py | 6 +- .../llama-nemotron-data-curation/README.md | 29 +- 25 files changed, 285 insertions(+), 263 deletions(-) create mode 100644 docs/broken_links_false_positives.json diff --git a/docs/about/release-notes/index.md b/docs/about/release-notes/index.md index 2e2ded5f08..a660ad933f 100644 --- a/docs/about/release-notes/index.md +++ b/docs/about/release-notes/index.md @@ -12,215 +12,152 @@ modality: "universal" # NeMo Curator Release Notes: {{ current_release }} -## Synthetic Data Generation +## What's New in 26.02 -New Ray-based synthetic data generation capabilities for creating and augmenting training data using LLMs: - -- **LLM Client Infrastructure**: OpenAI-compatible async/sync clients with automatic rate limiting, retry logic, and exponential backoff -- **Multilingual Q&A Generation**: Generate synthetic Q&A pairs across multiple languages using customizable prompts -- **Nemotron-CC Pipelines**: Advanced text transformation and knowledge extraction workflows: - - **Wikipedia Paraphrasing**: Improve low-quality text by rewriting in Wikipedia-style prose - - **Diverse QA**: Generate diverse question-answer pairs for reading comprehension training - - **Distill**: Create condensed, information-dense paraphrases preserving key concepts - - **Extract Knowledge**: Extract factual content as textbook-style passages - - **Knowledge List**: Extract structured fact lists from documents - -Learn more in the [Synthetic Data Generation documentation](../../curate-text/synthetic/index.md). - - ```{list-table} Available Installation Extras - :header-rows: 1 - :widths: 25 35 40 - - * - Extra - - Installation Command - - Description - * - **All Modalities** - - `nemo-curator[all]` - - Complete installation with all modalities and GPU support - * - **Text Curation** - - `nemo-curator[text_cuda12]` - - GPU-accelerated text processing with RAPIDS - * - **Image Curation** - - `nemo-curator[image_cuda12]` - - Image processing with NVIDIA DALI - * - **Audio Curation** - - `nemo-curator[audio_cuda12]` - - Speech recognition with NeMo ASR models - * - **Video Curation** - - `nemo-curator[video_cuda12]` - - Video processing with GPU acceleration - * - **Basic GPU** - - `nemo-curator[cuda12]` - - CUDA utilities without modality-specific dependencies - ``` - - All GPU installations require the NVIDIA PyPI index: - ```bash - uv pip install https://pypi.nvidia.com nemo-curator[EXTRA] - ``` - -## New Modalities - -### Video - -NeMo Curator now supports comprehensive [video data curation](../../curate-video/index.md) with distributed processing capabilities: - -- **Video splitting**: [Fixed-stride](../../curate-video/process-data/clipping.md) and [scene-change detection (TransNetV2)](../../curate-video/process-data/clipping.md) for clip extraction -- **Semantic deduplication**: [K-means clustering and pairwise similarity](../../curate-video/process-data/dedup.md) for near-duplicate clip removal -- **Content filtering**: [Motion-based filtering](../../curate-video/process-data/filtering.md) and [aesthetic filtering](../../curate-video/process-data/filtering.md) for quality improvement -- **Embedding generation**: Cosmos-Embed1 models for clip-level embeddings -- **Enhanced captioning**: [VL-based caption generation with optional LLM-based rewriting](../../curate-video/process-data/captions-preview.md) (Qwen-VL and Qwen-LM supported) for detailed video descriptions -- **Ray-based distributed architecture**: Scalable video processing with [autoscaling support](../concepts/video/architecture.md) - -### Audio - -New [audio curation capabilities](../../curate-audio/index.md) for speech data processing: - -- **ASR inference**: [Automatic speech recognition](../../curate-audio/process-data/asr-inference/index.md) using NeMo Framework pretrained models -- **Quality assessment**: [Word Error Rate (WER) and Character Error Rate (CER)](../../curate-audio/process-data/quality-assessment/index.md) calculation -- **Speech metrics**: [Duration analysis and speech rate metrics](../../curate-audio/process-data/audio-analysis/index.md) (words/characters per second) -- **Text integration**: Seamless integration with [text curation workflows](../../curate-audio/process-data/text-integration/index.md) via `AudioToDocumentStage` -- **Manifest support**: JSONL manifest format for audio file management - -## Modality Refactors - -### Text - -- **Ray backend migration**: Complete transition from Dask to Ray for distributed [text processing](../../curate-text/index.md) -- **Improved model-based classifier throughput**: Better overlapping of compute between tokenization and inference through [length-based sequence sorting](../../curate-text/process-data/quality-assessment/distributed-classifier.md) for optimal GPU memory utilization -- **Task-centric architecture**: New `Task`-based processing model for finer-grained control -- **Pipeline redesign**: Updated `ProcessingStage` and `Pipeline` architecture with resource specification - -### Image - -- **Pipeline-based architecture**: Transitioned from legacy `ImageTextPairDataset` to modern [stage-based processing](../../curate-images/index.md) with `ImageReaderStage`, `ImageEmbeddingStage`, and filter stages -- **DALI-based image loading**: New `ImageReaderStage` uses NVIDIA DALI for high-performance WebDataset tar shard processing with GPU/CPU fallback -- **Modular processing stages**: Separate stages for [embedding generation](../../curate-images/process-data/embeddings/index.md), [aesthetic filtering](../../curate-images/process-data/filters/aesthetic.md), and [NSFW filtering](../../curate-images/process-data/filters/nsfw.md) -- **Task-based data flow**: Images processed as `ImageBatch` tasks containing `ImageObject` instances with metadata, embeddings, and classification scores - -Learn more about [image curation](../../curate-images/index.md). - -## Deduplication Improvements - -Enhanced deduplication capabilities across all modalities with improved performance and flexibility: - -- **Exact and Fuzzy deduplication**: Updated [rapidsmpf-based shuffle backend](../../reference/infrastructure/gpu-processing.md) for more efficient GPU-to-GPU data transfer and better spilling capabilities -- **Semantic deduplication**: Support for deduplicating [text](../../curate-text/process-data/deduplication/semdedup.md) and [video](../../curate-video/process-data/dedup.md) datasets using unified embedding-based workflows -- **New ranking strategies**: Added `RankingStrategy` which allows you to rank elements within cluster centers to decide which point to prioritize during duplicate removal, supporting [metadata-based ranking](../../curate-text/process-data/deduplication/semdedup.md) to prioritize specific datasets or inputs - -## Core Refactors - -The architecture refactor introduces a layered system with unified interfaces and multiple execution backends: - -```{mermaid} -graph LR - subgraph "User Layer" - P[Pipeline] - S1[ProcessingStage X→Y] - S2[ProcessingStage Y→Z] - S3[ProcessingStage Z→W] - R[Resources
CPU/GPU/NVDEC/NVENC] - end - - subgraph "Orchestration Layer" - BE[BaseExecutor Interface] - end - - subgraph "Backend Layer" - XE[XennaExecutor] - RAP[RayActorPoolExecutor] - RDE[RayDataExecutor] - end - - subgraph "Adaptation Layer" - XA[Xenna Adapter] - RAPA[Ray Actor Adapter] - RDA[Ray Data Adapter] - end - - subgraph "Execution Layer" - X[Cosmos-Xenna
Streaming/Batch] - RAY1[Ray Actor Pool
Load Balancing] - RAY2[Ray Data API
Dataset Processing] - end - - P --> S1 - P --> S2 - P --> S3 - S1 -.-> R - S2 -.-> R - S3 -.-> R - - P --> BE - BE --> XE - BE --> RAP - BE --> RDE - - XE --> XA - RAP --> RAPA - RDE --> RDA - - XA --> X - RAPA --> RAY1 - RDA --> RAY2 - - style P fill:#E6F3FF - style BE fill:#F0F8FF +### Benchmarking Infrastructure + +New comprehensive benchmarking framework for performance monitoring and optimization: + +- **End-to-End Pipeline Benchmarking**: Automated benchmarks for all curation modalities (text, image, video, audio) +- **Performance Tracking**: Integration with MLflow for metrics tracking and Slack for notifications +- **Nightly Benchmarks**: Continuous performance monitoring across: + - Text pipelines: exact deduplication, fuzzy deduplication, semantic deduplication, score filters, modifiers + - Image curation workflows with DALI-based processing + - Video processing pipelines with scene detection and semantic deduplication + - Audio ASR inference and quality assessment +- **Grafana Dashboards**: Real-time monitoring of pipeline performance and resource utilization + +### Ray Actor Pool Executor Improvements + +Enhanced features for the experimental Ray Actor Pool execution backend: + +- **Progress Bars**: New visual feedback for long-running actor pool operations, making it easier to monitor pipeline execution +- **Improved Load Balancing**: Better worker distribution and task scheduling +- **Enhanced Stability**: Continued refinements to the experimental executor + +Learn more in the [Execution Backends documentation](../../reference/infrastructure/execution-backends.md). + +### Enhanced Embedding Generation + +Expanded embedding support with new model integrations: + +- **vLLM Integration**: High-performance LLM-based embedding generation with automatic batching +- **Sentence Transformers**: Support for popular sentence embedding models +- **Unified API**: Consistent embedding interface across text, image, and video modalities + +### YAML Configuration Support + +Declarative pipeline configuration for text curation workflows: + +- **YAML-Based Pipelines**: Define entire curation pipelines in YAML configuration files +- **Pre-Built Configurations**: Ready-to-use configs for common workflows: + - Code filtering, exact/fuzzy/semantic deduplication + - Heuristic filtering (English and non-English) + - FastText language identification +- **Reproducible Workflows**: Version-controlled pipeline definitions for consistent results + +Example: +```bash +python -m nemo_curator.config.run --config_file heuristic_filter_english_pipeline.yaml ``` -### Pipelines +### Workflow Results API + +New API for tracking and analyzing pipeline execution: + +- **WorkflowRunResult**: Structured results object capturing execution metrics +- **Performance Metrics**: Automatic tracking of processing time, throughput, and resource usage +- **Better Debugging**: Detailed logs and error reporting for failed stages + +## Improvements from 25.09 -- **New Pipeline API**: Ray-based pipeline execution with `BaseExecutor` interface -- **Multiple backends**: Support for [Xenna, Ray Actor Pool, and Ray Data execution backends](../../reference/infrastructure/execution-backends.md) -- **Resource specification**: Configurable CPU and GPU memory requirements per stage -- **Stage composition**: Improved stage validation and execution orchestration +### Video Curation -### Stages +- **Model Updates**: Removed InternVideo2 dependency; updated to more performant alternatives +- **vLLM 0.14.1**: Upgraded for better video captioning compatibility and performance +- **FFmpeg 8.0.1**: Latest FFmpeg with improved codec support and performance +- **Enhanced Tutorials**: Improved video processing examples with real-world scenarios -- **ProcessingStage redesign**: Generic `ProcessingStage[X, Y]` base class with type safety -- **Resource requirements**: Built-in resource specification for CPU and GPU memory -- **Backend adapters**: Stage adaptation layer for different Ray orchestration systems -- **Input/output validation**: Enhanced type checking and data validation +### Audio Curation -## Tutorials +- **Enhanced Documentation**: Comprehensive ASR inference and quality assessment guides +- **Improved WER Filtering**: Better guidance for Word Error Rate filtering thresholds +- **Manifest Handling**: More robust JSONL manifest processing for large audio datasets -- **Text tutorials**: Updated all [text curation tutorials](https://github.com/NVIDIA-NeMo/Curator/tree/main/tutorials/text) to use new Ray-based API -- **Image tutorials**: Migrated [image processing tutorials](https://github.com/NVIDIA-NeMo/Curator/tree/main/tutorials/image) to unified backend -- **Audio tutorials**: New [audio curation tutorials](https://github.com/NVIDIA-NeMo/Curator/tree/main/tutorials/audio) -- **Video tutorials**: New [video processing tutorials](https://github.com/NVIDIA-NeMo/Curator/tree/main/tutorials/video) +### Image Curation -For all tutorial content, refer to the [tutorials directory](https://github.com/NVIDIA-NeMo/Curator/tree/main/tutorials) in the NeMo Curator GitHub repository. +- **Optimized Batch Sizes**: Reduced default batch sizes for better CPU memory usage (batch_size=50, num_threads=4) +- **Memory Guidance**: Added troubleshooting documentation for out-of-memory errors +- **Tutorial Improvements**: Updated examples optimized for typical GPU configurations -## Known Limitations +### Text Curation -> (Pending Refactor in Future Release) +- **ID Field Standardization**: Unified ID naming conventions across all deduplication workflows +- **Performance Optimizations**: Fused document iterate and extract stages for reduced overhead +- **Better Memory Management**: Improved handling of large-scale semantic deduplication +- **Small Cluster Warnings**: Automatic warnings when n_clusters is too small for effective deduplication +- **FilePartitioning Improvements**: One worker per partition for better parallelization -### Generation +### Deduplication Enhancements -- **Synthetic data generation**: Synthetic text generation features are being refactored for Ray compatibility -- **Hard negative mining**: Retrieval-based data generation workflows under development +- **Cloud Storage Support**: Fixed ParquetReader/Writer and pairwise I/O for S3, GCS, and Azure Blob +- **Non-Blocking ID Generation**: Improved ID generator performance for large datasets +- **Empty Batch Handling**: Better error handling for filters processing empty data batches -### PII +## Dependency Updates -- **PII processing**: Personal Identifiable Information removal tools are being updated for Ray backend -- **Privacy workflows**: Enhanced privacy-preserving data curation capabilities in development +- **Transformers**: Pinned to 4.55.2 for stability and compatibility +- **vLLM**: Updated to 0.14.1 with video pipeline compatibility fixes +- **FFmpeg**: Upgraded to 8.0.1 for enhanced multimedia processing +- **Security Patches**: + - Addressed CVEs in aiohttp, urllib3, python-multipart, setuptools + - Removed vulnerable thirdparty aiohttp file from Ray + - Updated to secure dependency versions -### Blending & Shuffling +## Bug Fixes -- **Data blending**: Multi-source dataset blending functionality being refactored -- **Dataset shuffling**: Large-scale data shuffling operations under development +- Fixed fasttext predict call compatibility with numpy>2 +- Fixed broken NeMo Framework documentation links +- Fixed MegatronTokenizerWriter to download only necessary tokenizer files +- Fixed ID generator blocking issues for large-scale processing +- Fixed vLLM API compatibility with video captioning pipeline +- Fixed Gliner tutorial examples and SDG workflow bugs +- Improved semantic deduplication unit test reliability -## Docs Refactor +## Infrastructure & Developer Experience -- **Local preview capability**: Improved documentation build system with local preview support -- **Modality-specific guides**: Comprehensive documentation for each supported modality ([text](../../curate-text/index.md), [image](../../curate-images/index.md), [audio](../../curate-audio/index.md), [video](../../curate-video/index.md)) -- **API reference**: Complete [API documentation](../../apidocs/index.rst) with type annotations and examples +- **Secrets Detection**: Automated secret scanning in CI/CD workflows +- **Dependabot Integration**: Automatic dependency update pull requests +- **Enhanced Install Tests**: Comprehensive installation validation across environments +- **AWS Runner Support**: CI/CD execution on AWS infrastructure +- **Docker Optimization**: Improved layer caching and build times with uv +- **Code Linting**: Standardized code quality checks with markdownlint and pre-commit hooks +- **Cursor Rules**: Development guidelines and patterns for IDE assistance + +## Breaking Changes + +- **InternVideo2 Removed**: Video pipelines must use alternative embedding models (Cosmos-Embed1) +- **ID Field Standardization**: Custom deduplication workflows may need updates to use standardized ID field names + +## Documentation Improvements + +- **Heuristic Filter Guide**: Comprehensive documentation for language-specific filtering strategies +- **Distributed Classifier**: Enhanced GPU memory optimization guidance with length-based sequence sorting +- **Installation Guide**: Clearer instructions with troubleshooting for common issues +- **Memory Management**: New guidance for handling CPU/GPU memory constraints +- **AWS Integration**: Updated tutorials with correct AWS credentials setup --- ## What's Next -The next release will focus on code curation and math curation. +Future releases will focus on: + +- **Code Curation**: Specialized pipelines for curating code datasets +- **Math Curation**: Mathematical reasoning and problem-solving data curation +- **Generation Features**: Completing the Ray refactor for synthetic data generation +- **PII Processing**: Enhanced privacy-preserving data curation with Ray backend +- **Blending & Shuffling**: Large-scale multi-source dataset blending and shuffling operations ```{toctree} :hidden: diff --git a/docs/admin/installation.md b/docs/admin/installation.md index e9796e8fa0..b779a6af67 100644 --- a/docs/admin/installation.md +++ b/docs/admin/installation.md @@ -18,7 +18,7 @@ This guide covers installing NeMo Curator with support for **all modalities** an ### System Requirements -For comprehensive system requirements and production deployment specifications, see [Production Deployment Requirements](deployment/requirements.md). +For comprehensive system requirements and production deployment specifications, refer to [Production Deployment Requirements](deployment/requirements.md). **Quick Start Requirements:** @@ -26,6 +26,7 @@ For comprehensive system requirements and production deployment specifications, - **Python**: 3.10, 3.11, or 3.12 - **Memory**: 16GB+ RAM for basic text processing - **GPU** (optional): NVIDIA GPU with 16GB+ VRAM for acceleration +- **CUDA 12** (required for `audio_cuda12`, `video_cuda12`, `image_cuda12`, and `text_cuda12` extras) ### Development vs Production @@ -41,9 +42,13 @@ For comprehensive system requirements and production deployment specifications, Choose one of the following installation methods based on your needs: +:::{tip} +**Docker is the recommended installation method** for video and audio workflows. The NeMo Curator container includes FFmpeg (with NVENC support) pre-configured, avoiding manual dependency setup. Refer to the [Container Installation](#container-installation) tab below. +::: + ::::{tab-set} -:::{tab-item} PyPI Installation (Recommended) +:::{tab-item} PyPI Installation Install NeMo Curator from the Python Package Index using `uv` for proper dependency resolution. @@ -89,9 +94,9 @@ uv sync --all-extras --all-groups ::: -:::{tab-item} Container Installation +:::{tab-item} Container Installation (Recommended for Video/Audio) -NeMo Curator is available as a standalone container on NGC: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo-curator. The container includes NeMo Curator with all dependencies pre-installed. You can run it with: +NeMo Curator is available as a standalone container on NGC: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo-curator. The container includes NeMo Curator with all dependencies pre-installed, including FFmpeg with NVENC support. ```bash # Pull the container from NGC @@ -101,6 +106,14 @@ docker pull nvcr.io/nvidia/nemo-curator:{{ container_version }} docker run --gpus all -it --rm nvcr.io/nvidia/nemo-curator:{{ container_version }} ``` +```{important} +After entering the container, activate the virtual environment before running any NeMo Curator commands: + + source /opt/venv/env.sh + +The container uses a virtual environment at `/opt/venv`. If you see `No module named nemo_curator`, the environment has not been activated. +``` + Alternatively, you can build the NeMo Curator container locally using the provided Dockerfile: ```bash @@ -115,7 +128,7 @@ docker run --gpus all -it --rm nemo-curator:latest **Benefits:** -- Pre-configured environment with all dependencies +- Pre-configured environment with all dependencies (FFmpeg, CUDA libraries) - Consistent runtime across different systems - Ideal for production deployments @@ -157,6 +170,10 @@ If encoders are missing, reinstall `FFmpeg` with the required options or use the ::: :::: +```{note} +**FFmpeg build requires CUDA toolkit (nvcc):** If you encounter `ERROR: failed checking for nvcc` during FFmpeg installation, ensure that the CUDA toolkit is installed and `nvcc` is available on your `PATH`. You can verify with `nvcc --version`. If using the NeMo Curator container, FFmpeg is pre-installed with NVENC support. +``` + --- ## Package Extras diff --git a/docs/broken_links_false_positives.json b/docs/broken_links_false_positives.json new file mode 100644 index 0000000000..8567d468c0 --- /dev/null +++ b/docs/broken_links_false_positives.json @@ -0,0 +1 @@ +{"filename": "get-started/text.md", "lineno": 119, "status": "broken", "code": 0, "uri": "https://huggingface.co/settings/tokens", "info": "unauthorized"} diff --git a/docs/curate-text/process-data/deduplication/fuzzy.md b/docs/curate-text/process-data/deduplication/fuzzy.md index fb5e84c451..90e645832f 100644 --- a/docs/curate-text/process-data/deduplication/fuzzy.md +++ b/docs/curate-text/process-data/deduplication/fuzzy.md @@ -34,6 +34,10 @@ Ideal for detecting documents with minor differences such as formatting changes, - Ray cluster with GPU support (required for distributed processing) - Stable document identifiers for removal (either existing IDs or IDs generated by the workflow and removal stages) +```{note} +**Running in Docker**: When running fuzzy deduplication inside the NeMo Curator container, ensure the container is started with `--gpus all` so that Ray workers can access the GPU. Without GPU access, you may see `CUDARuntimeError` or `AttributeError: 'CUDARuntimeError' object has no attribute 'msg'`. Also activate the virtual environment with `source /opt/venv/env.sh` after entering the container. +``` + ## Quick Start Get started with fuzzy deduplication using the following example of identifying duplicates, then remove them: diff --git a/docs/curate-text/process-data/deduplication/semdedup.md b/docs/curate-text/process-data/deduplication/semdedup.md index 5d14ea1c1c..cbcb581f16 100644 --- a/docs/curate-text/process-data/deduplication/semdedup.md +++ b/docs/curate-text/process-data/deduplication/semdedup.md @@ -42,6 +42,10 @@ Based on [SemDeDup: Data-efficient learning at web-scale through semantic dedupl - GPU acceleration (required for embedding generation and clustering) - Stable document identifiers for removal (either existing IDs or IDs managed by the workflow and removal stages) +```{note} +**Running in Docker**: When running semantic deduplication inside the NeMo Curator container, ensure the container is started with `--gpus all` so that CUDA GPUs are available. Without this flag, you will see `RuntimeError: No CUDA GPUs are available`. Also activate the virtual environment with `source /opt/venv/env.sh` after entering the container. +``` + ## Quick Start Get started with semantic deduplication using the following example of identifying duplicates, then remove them in one step: diff --git a/docs/curate-text/process-data/quality-assessment/distributed-classifier.md b/docs/curate-text/process-data/quality-assessment/distributed-classifier.md index d9f92280e8..f3e4abcde1 100644 --- a/docs/curate-text/process-data/quality-assessment/distributed-classifier.md +++ b/docs/curate-text/process-data/quality-assessment/distributed-classifier.md @@ -20,7 +20,7 @@ The distributed data classification in NeMo Curator works by: 1. **Parallel Processing**: Chunking datasets across multiple computing nodes and GPUs to accelerate classification 2. **Pre-trained Models**: Using specialized models for different classification tasks -3. **Batched Inference**: Optimizing throughput with intelligent batching via CrossFit integration +3. **Batched Inference**: Optimizing throughput with intelligent batching 4. **Consistent API**: Providing a unified interface through the `DistributedDataClassifier` base class The `DistributedDataClassifier` is designed to run on GPU clusters with minimal code changes regardless of which specific classifier you're using. All classifiers support filtering based on classification results and storing prediction scores as metadata. @@ -29,6 +29,16 @@ The `DistributedDataClassifier` is designed to run on GPU clusters with minimal Distributed classification requires GPU acceleration and is not supported for CPU-only processing. As long as GPU resources are available and NeMo Curator is correctly installed, GPU acceleration is handled automatically. ::: +```{tip} +**Running the tutorial notebooks**: The classification tutorial notebooks require the `text_cuda12` or `all` installation extra to include all relevant dependencies. If you encounter `ModuleNotFoundError`, reinstall with the appropriate extra: + + uv pip install "nemo-curator[text_cuda12]" + +When using classifiers that download from Hugging Face (such as Aegis and InstructionDataGuard), set your `HF_TOKEN` environment variable to avoid rate limiting: + + export HF_TOKEN="your_token_here" +``` + --- ## Usage @@ -39,16 +49,16 @@ NVIDIA NeMo Curator provides a base class `DistributedDataClassifier` that can b | Classifier | Purpose | Model Location | Key Parameters | Requirements | |---|---|---|---|---| -| DomainClassifier | Categorize English text by domain | [nvidia/domain-classifier](https://huggingface.co/nvidia/domain-classifier) | `filter_by`, `text_field` | None | -| MultilingualDomainClassifier | Categorize text in 52 languages by domain | [nvidia/multilingual-domain-classifier](https://huggingface.co/nvidia/multilingual-domain-classifier) | `filter_by`, `text_field` | None | -| QualityClassifier | Assess document quality | [nvidia/quality-classifier-deberta](https://huggingface.co/nvidia/quality-classifier-deberta) | `filter_by`, `text_field` | None | -| AegisClassifier | Detect unsafe content | [nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0](https://huggingface.co/nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0) | `aegis_variant`, `filter_by` | HuggingFace token | -| InstructionDataGuardClassifier | Detect poisoning attacks | [nvidia/instruction-data-guard](https://huggingface.co/nvidia/instruction-data-guard) | `text_field`, `label_field` | HuggingFace token | -| FineWebEduClassifier | Score educational value | [HuggingFaceFW/fineweb-edu-classifier](https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier) | `label_field`, `int_field` | None | -| FineWebMixtralEduClassifier | Score educational value (Mixtral annotations) | [nvidia/nemocurator-fineweb-mixtral-edu-classifier](https://huggingface.co/nvidia/nemocurator-fineweb-mixtral-edu-classifier) | `label_field`, `int_field`, `model_inference_batch_size=1024` | None | -| FineWebNemotronEduClassifier | Score educational value (Nemotron annotations) | [nvidia/nemocurator-fineweb-nemotron-4-edu-classifier](https://huggingface.co/nvidia/nemocurator-fineweb-nemotron-4-edu-classifier) | `label_field`, `int_field`, `model_inference_batch_size=1024` | None | -| ContentTypeClassifier | Categorize by speech type | [nvidia/content-type-classifier-deberta](https://huggingface.co/nvidia/content-type-classifier-deberta) | `filter_by`, `text_field` | None | -| PromptTaskComplexityClassifier | Classify prompt tasks and complexity | [nvidia/prompt-task-and-complexity-classifier](https://huggingface.co/nvidia/prompt-task-and-complexity-classifier) | `text_field` | None | +| DomainClassifier | Assigns one of 26 domain labels (such as "Sports," "Science," "News") to English text | [nvidia/domain-classifier](https://huggingface.co/nvidia/domain-classifier) | `filter_by`, `text_field` | None | +| MultilingualDomainClassifier | Assigns domain labels to text in 52 languages; same labels as DomainClassifier | [nvidia/multilingual-domain-classifier](https://huggingface.co/nvidia/multilingual-domain-classifier) | `filter_by`, `text_field` | None | +| QualityClassifier | Rates document quality as "Low," "Medium," or "High" using a DeBERTa model | [nvidia/quality-classifier-deberta](https://huggingface.co/nvidia/quality-classifier-deberta) | `filter_by`, `text_field` | None | +| AegisClassifier | Detects unsafe content across 13 risk categories (violence, hate speech, and others) using LlamaGuard | [nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0](https://huggingface.co/nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0) | `aegis_variant`, `filter_by` | HuggingFace token | +| InstructionDataGuardClassifier | Identifies LLM poisoning attacks in instruction-response pairs | [nvidia/instruction-data-guard](https://huggingface.co/nvidia/instruction-data-guard) | `text_field`, `label_field` | HuggingFace token | +| FineWebEduClassifier | Scores educational value from 0 to 5 (0=spam, 5=scholarly) for training data selection | [HuggingFaceFW/fineweb-edu-classifier](https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier) | `label_field`, `int_field` | None | +| FineWebMixtralEduClassifier | Scores educational value from 0 to 5 using Mixtral 8x22B annotation data | [nvidia/nemocurator-fineweb-mixtral-edu-classifier](https://huggingface.co/nvidia/nemocurator-fineweb-mixtral-edu-classifier) | `label_field`, `int_field`, `model_inference_batch_size=1024` | None | +| FineWebNemotronEduClassifier | Scores educational value from 0 to 5 using Nemotron-4-340B annotation data | [nvidia/nemocurator-fineweb-nemotron-4-edu-classifier](https://huggingface.co/nvidia/nemocurator-fineweb-nemotron-4-edu-classifier) | `label_field`, `int_field`, `model_inference_batch_size=1024` | None | +| ContentTypeClassifier | Categorizes text into 11 speech types (such as "Blogs," "News," "Academic") | [nvidia/content-type-classifier-deberta](https://huggingface.co/nvidia/content-type-classifier-deberta) | `filter_by`, `text_field` | None | +| PromptTaskComplexityClassifier | Labels prompts by task type (such as QA and summarization) and complexity dimensions | [nvidia/prompt-task-and-complexity-classifier](https://huggingface.co/nvidia/prompt-task-and-complexity-classifier) | `text_field` | None | ### Domain Classifier @@ -365,6 +375,10 @@ pipeline.add_stage(writer) results = pipeline.run() # Uses XennaExecutor by default ``` +## Custom Model Integration + +You can integrate your own classification models by extending `DistributedDataClassifier`. Refer to the [Text Classifiers README](https://github.com/NVIDIA-NeMo/Curator/tree/main/nemo_curator/stages/text/classifiers#text-classifiers) for implementation details and examples. + ## Performance Optimization NVIDIA NeMo Curator's distributed classifiers are optimized for high-throughput processing through several key features: diff --git a/docs/curate-video/process-data/captions-preview.md b/docs/curate-video/process-data/captions-preview.md index 0f32b03460..27674236c1 100644 --- a/docs/curate-video/process-data/captions-preview.md +++ b/docs/curate-video/process-data/captions-preview.md @@ -68,7 +68,7 @@ pipe.run() :::{tab-item} Script Flags ```bash -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --generate-captions \ --captioning-algorithm qwen \ diff --git a/docs/curate-video/process-data/clipping.md b/docs/curate-video/process-data/clipping.md index 86adceec77..d34045778b 100644 --- a/docs/curate-video/process-data/clipping.md +++ b/docs/curate-video/process-data/clipping.md @@ -84,7 +84,7 @@ pipe.run() ```bash # Fixed stride -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --splitting-algorithm fixed_stride \ --fixed-stride-split-duration 10.0 \ @@ -92,7 +92,7 @@ python -m nemo_curator.examples.video.video_split_clip_example \ --limit-clips 0 # TransNetV2 -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --splitting-algorithm transnetv2 \ --transnetv2-frame-decoder-mode pynvc \ diff --git a/docs/curate-video/process-data/dedup.md b/docs/curate-video/process-data/dedup.md index 6cba46e520..f343425cb9 100644 --- a/docs/curate-video/process-data/dedup.md +++ b/docs/curate-video/process-data/dedup.md @@ -15,7 +15,7 @@ modality: "video-only" Use clip-level embeddings to identify near-duplicate video clips so your dataset remains compact, diverse, and efficient to train on. ## Before You Start -- Make sure you have embeddings which are written by the [`ClipWriterStage`](video-save-export) under `iv2_embd_parquet/` or `ce1_embd_parquet/`. For a runnable workflow, refer to the [Split and Remove Duplicates Workflow](video-tutorials-split-dedup). The embeddings must be in parquet files containing the columns `id` and `embedding`. +- Make sure you have embeddings which are written by the [`ClipWriterStage`](video-save-export) under `ce1_embd_parquet/`. For a runnable workflow, refer to the [Split and Remove Duplicates Workflow](video-tutorials-split-dedup). The embeddings must be in parquet files containing the columns `id` and `embedding`. - Verify local paths or configure S3-compatible credentials. Provide `storage_options` in read/write keyword arguments when reading or writing cloud paths. @@ -24,7 +24,7 @@ Use clip-level embeddings to identify near-duplicate video clips so your dataset Duplicate identification operates on clip-level embeddings produced during processing: 1. **Inputs** - - Parquet batches from `ClipWriterStage` under `iv2_embd_parquet/` or `ce1_embd_parquet/` + - Parquet batches from `ClipWriterStage` under `ce1_embd_parquet/` - Columns: `id`, `embedding` 2. **Outputs** @@ -50,13 +50,13 @@ from nemo_curator.stages.deduplication.semantic.ranking import RankingStrategy from nemo_curator.backends.xenna import XennaExecutor workflow = SemanticDeduplicationWorkflow( - input_path="/path/to/embeddings/", # e.g., iv2_embd_parquet/ or ce1_embd_parquet/ + input_path="/path/to/embeddings/", # e.g., ce1_embd_parquet/ output_path="/path/to/duplicates/", cache_path="/path/to/cache/", # Optional: defaults to output_path n_clusters=1000, id_field="id", embedding_field="embedding", - embedding_dim=512, # 512 for InternVideo2, varies for Cosmos-Embed1 + embedding_dim=768, # Embedding dimension (768 for Cosmos-Embed1, varies by model) input_filetype="parquet", eps=0.1, # Similarity threshold: cosine_sim >= 1.0 - eps identifies duplicates ranking_strategy=RankingStrategy.metadata_based( diff --git a/docs/curate-video/process-data/embeddings.md b/docs/curate-video/process-data/embeddings.md index 39b9935a6c..fe9c083435 100644 --- a/docs/curate-video/process-data/embeddings.md +++ b/docs/curate-video/process-data/embeddings.md @@ -65,7 +65,7 @@ pipe.run() ```bash # Cosmos-Embed1 (224p) -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --generate-embeddings \ --embedding-algorithm cosmos-embed1-224p \ diff --git a/docs/curate-video/process-data/filtering.md b/docs/curate-video/process-data/filtering.md index 8219171079..1c4b567f03 100644 --- a/docs/curate-video/process-data/filtering.md +++ b/docs/curate-video/process-data/filtering.md @@ -83,7 +83,7 @@ pipe.run() ```bash # Motion filtering -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --motion-filter enable \ --motion-decode-target-fps 2.0 \ @@ -95,7 +95,7 @@ python -m nemo_curator.examples.video.video_split_clip_example \ --motion-score-gpus-per-worker 0.5 # Aesthetic filtering -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --aesthetic-threshold 3.5 \ --aesthetic-reduction min \ diff --git a/docs/curate-video/process-data/frame-extraction.md b/docs/curate-video/process-data/frame-extraction.md index 30d9c3bc5f..bb46562b32 100644 --- a/docs/curate-video/process-data/frame-extraction.md +++ b/docs/curate-video/process-data/frame-extraction.md @@ -66,13 +66,13 @@ pipe.run() ```bash # Clip frames implicitly when generating embeddings or aesthetics -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --generate-embeddings \ --clip-extraction-target-res -1 # Full-video frames for TransNetV2 scene change -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ ... \ --splitting-algorithm transnetv2 \ --transnetv2-frame-decoder-mode pynvc diff --git a/docs/curate-video/process-data/index.md b/docs/curate-video/process-data/index.md index 35f3844129..6e49781525 100644 --- a/docs/curate-video/process-data/index.md +++ b/docs/curate-video/process-data/index.md @@ -124,7 +124,7 @@ pipeline.add_stage( ) ``` -Path helpers are available to resolve common locations (such as `clips/`, `filtered_clips/`, `previews/`, `metas/v0/`, and `iv2_embd_parquet/`). +Path helpers are available to resolve common locations (such as `clips/`, `filtered_clips/`, `previews/`, `metas/v0/`, and `ce1_embd_parquet/`). ```{toctree} :maxdepth: 2 diff --git a/docs/curate-video/save-export.md b/docs/curate-video/save-export.md index ac0d72ef69..f120055400 100644 --- a/docs/curate-video/save-export.md +++ b/docs/curate-video/save-export.md @@ -95,8 +95,8 @@ The writer produces these directories under `output_path`: - `filtered_clips/`: Media for filtered-out clips. - `previews/`: Preview images (`.webp`). - `metas/v0/`: Per-clip metadata (`.json`). -- `iv2_embd/`, `ce1_embd/`: Per-clip embeddings (`.pickle`). -- `iv2_embd_parquet/`, `ce1_embd_parquet/`: Parquet batches with columns `id` and `embedding`. +- `ce1_embd/`: Per-clip embeddings (`.pickle`). +- `ce1_embd_parquet/`: Parquet batches with columns `id` and `embedding`. - `processed_videos/`, `processed_clip_chunks/`: Video-level metadata and per-chunk statistics. ### Per-Clip Metadata @@ -132,8 +132,8 @@ Each clip writes a JSON file under `metas/v0/` with clip- and window-level field ### Embeddings and Parquet outputs -- When embeddings exist, the stage writes per-clip `.pickle` files under `iv2_embd/` or `ce1_embd/`. -- The stage also batches embeddings per clip chunk into Parquet files under `iv2_embd_parquet/` or `ce1_embd_parquet/` with columns `id` and `embedding` and writes those files to disk. +- When embeddings exist, the stage writes per-clip `.pickle` files under `ce1_embd/`. +- The stage also batches embeddings per clip chunk into Parquet files under `ce1_embd_parquet/` with columns `id` and `embedding` and writes those files to disk. ## Helpers @@ -150,7 +150,6 @@ clips_dir = ClipWriterStage.get_output_path_clips(OUT) filtered_clips_dir = ClipWriterStage.get_output_path_clips(OUT, filtered=True) previews_dir = ClipWriterStage.get_output_path_previews(OUT) metas_dir = ClipWriterStage.get_output_path_metas(OUT, "v0") -iv2_parquet_dir = ClipWriterStage.get_output_path_iv2_embd_parquet(OUT) ce1_parquet_dir = ClipWriterStage.get_output_path_ce1_embd_parquet(OUT) processed_videos_dir = ClipWriterStage.get_output_path_processed_videos(OUT) processed_chunks_dir = ClipWriterStage.get_output_path_processed_clip_chunks(OUT) diff --git a/docs/curate-video/tutorials/beginner.md b/docs/curate-video/tutorials/beginner.md index 6d4ae62cc7..acda9d4999 100644 --- a/docs/curate-video/tutorials/beginner.md +++ b/docs/curate-video/tutorials/beginner.md @@ -230,10 +230,10 @@ pipeline.add_stage( When using the example pipeline module, configure the writer-related flags: ```bash -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ --video-dir "$VIDEO_DIR" \ --model-dir "$MODEL_DIR" \ - --output-clip-path "$OUT_DIR" \ + --output-path "$OUT_DIR" \ --no-upload-clips # optional: do not write mp4s --dry-run # optional: write nothing, validate only --generate-embeddings # optional: enable embedding outputs diff --git a/docs/curate-video/tutorials/split-dedup.md b/docs/curate-video/tutorials/split-dedup.md index 7a8edc2ead..2147d0bddd 100644 --- a/docs/curate-video/tutorials/split-dedup.md +++ b/docs/curate-video/tutorials/split-dedup.md @@ -31,10 +31,10 @@ Learn how to run the splitting pipeline to generate clips and embeddings, then r Run the splitting example. Set `VIDEO_DIR`, `OUT_DIR`, and `MODEL_DIR` first. ```bash -python -m nemo_curator.examples.video.video_split_clip_example \ +python tutorials/video/getting-started/video_split_clip_example.py \ --video-dir "$VIDEO_DIR" \ --model-dir "$MODEL_DIR" \ - --output-clip-path "$OUT_DIR" \ + --output-path "$OUT_DIR" \ --splitting-algorithm fixed_stride \ --fixed-stride-split-duration 10.0 \ --embedding-algorithm cosmos-embed1-224p \ @@ -49,7 +49,7 @@ Writer-related flags you can add: --dry-run # Write nothing; validate only ``` -The pipeline writes embeddings under `$OUT_DIR/iv2_embd_parquet/` (or `ce1_embd_parquet/` if you use Cosmos-Embed1). +The pipeline writes embeddings under `$OUT_DIR/ce1_embd_parquet/` when using Cosmos-Embed1. ### Embedding Format Example @@ -64,7 +64,7 @@ The pipeline writes embeddings to Parquet with two columns: ```text $OUT_DIR/ - iv2_embd_parquet/ + ce1_embd_parquet/ 1a2b3c4d-....parquet 5e6f7g8h-....parquet ``` @@ -93,7 +93,7 @@ embedding: list # length = 768 for Cosmos-Embed1 ```python import pyarrow.parquet as pq -table = pq.read_table(f"{OUT_DIR}/iv2_embd_parquet") +table = pq.read_table(f"{OUT_DIR}/ce1_embd_parquet") df = table.to_pandas() print(df.head()) # columns: id, embedding (list[float]) ``` @@ -113,7 +113,7 @@ from nemo_curator.pipeline import Pipeline from nemo_curator.stages.deduplication.semantic.kmeans import KMeansStage from nemo_curator.stages.deduplication.semantic.pairwise import PairwiseStage -INPUT_PARQUET = f"{OUT_DIR}/iv2_embd_parquet" # or s3://... +INPUT_PARQUET = f"{OUT_DIR}/ce1_embd_parquet" # or s3://... OUTPUT_DIR = f"{OUT_DIR}/semantic_dedup" pipe = Pipeline(name="video_semantic_dedup", description="K-means + pairwise duplicate removal") @@ -175,7 +175,7 @@ Video-specific pointers: - Use `ClipWriterStage` path helpers to locate outputs: `nemo_curator/stages/video/io/clip_writer.py`. - Processed videos: `get_output_path_processed_videos(OUT_DIR)` - Clip chunks and previews: `get_output_path_processed_clip_chunks(OUT_DIR)`, `get_output_path_previews(OUT_DIR)` - - Embeddings parquet: `${OUT_DIR}/iv2_embd_parquet` (or `${OUT_DIR}/ce1_embd_parquet`) + - Embeddings parquet: `${OUT_DIR}/ce1_embd_parquet` ### Example Export @@ -188,7 +188,7 @@ from glob import glob OUT_DIR = os.environ["OUT_DIR"] clips_dir = os.path.join(OUT_DIR, "clips") # adjust if filtering path used -meta_parquet = os.path.join(OUT_DIR, "iv2_embd_parquet") +meta_parquet = os.path.join(OUT_DIR, "ce1_embd_parquet") def iter_clips(path): for p in glob(os.path.join(path, "**", "*.mp4"), recursive=True): diff --git a/docs/get-started/audio.md b/docs/get-started/audio.md index 0f4bd5d057..fa28713481 100644 --- a/docs/get-started/audio.md +++ b/docs/get-started/audio.md @@ -182,7 +182,7 @@ You can also run the pipeline using the downloaded configuration: ```bash cd ~/nemo_curator -python -m nemo_curator.examples.audio.fleurs.run \ +python tutorials/audio/fleurs/run.py \ --config-path ~/nemo_curator/configs \ --config-name fleurs_pipeline.yaml \ raw_data_dir=~/nemo_curator/audio_data diff --git a/docs/get-started/image.md b/docs/get-started/image.md index 588028621b..a739479e25 100644 --- a/docs/get-started/image.md +++ b/docs/get-started/image.md @@ -114,6 +114,24 @@ For this example, you'll need: Here's a simple example to get started with NeMo Curator's image curation pipeline: +:::{note} +**CPU Memory Considerations** + +Image loading and decoding happens in CPU memory before GPU processing. If you encounter out-of-memory errors during the `ImageReaderStage`, reduce: +- `batch_size`: Number of images per batch (reduce to 32-50 for systems with limited RAM) +- `num_threads`: Parallel decoding threads (reduce to 4 for systems with limited RAM) +- `num_cpus`: Ray Client CPU allocation (reduce to 8-16 for systems with limited RAM) + +The example below uses conservative defaults suitable for most systems. For high-memory systems, you can increase these values for better performance. + +To configure Ray with limited CPU resources: +```python +from nemo_curator.core.client import RayClient +ray_client = RayClient(num_cpus=8) # Adjust based on available CPU cores +ray_client.start() +``` +::: + ```python from nemo_curator.pipeline import Pipeline from nemo_curator.backends.xenna import XennaExecutor @@ -136,9 +154,9 @@ pipeline.add_stage(FilePartitioningStage( # Stage 2: Read images from tar files using DALI pipeline.add_stage(ImageReaderStage( - batch_size=100, + batch_size=50, verbose=True, - num_threads=8, + num_threads=4, num_gpus_per_worker=0.25, )) diff --git a/docs/get-started/text.md b/docs/get-started/text.md index 9b7258f60b..49b801611f 100644 --- a/docs/get-started/text.md +++ b/docs/get-started/text.md @@ -108,7 +108,15 @@ mkdir -p ~/nemo_curator/data/curated ``` ```{note} -For this example, you'll need sample JSONL files in `~/nemo_curator/data/sample/`. Each line should be a JSON object with at least `text` and `id` fields. You can create test data or refer to {ref}`Read Existing Data ` and {ref}`Data Loading ` for information on downloading data. +For this example, you need sample JSONL files in `~/nemo_curator/data/sample/`. Each line should be a JSON object with at least `text` and `id` fields. You can create test data or refer to {ref}`Read Existing Data ` and {ref}`Data Loading ` for information on downloading data. +``` + +```{tip} +**Set your HuggingFace token** to avoid rate limiting when downloading models or datasets: + + export HF_TOKEN="your_token_here" + +Without a token, repeated downloads from Hugging Face may result in `429 Client Error` (rate limiting). Get a free token at [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens). ``` ## Basic Text Curation Example diff --git a/docs/get-started/video.md b/docs/get-started/video.md index 6354c1551d..74cac57df4 100644 --- a/docs/get-started/video.md +++ b/docs/get-started/video.md @@ -26,7 +26,7 @@ This quickstart guide demonstrates how to: 4. **Process videos** through a complete splitting and embedding pipeline 5. **Generate outputs** ready for duplicate removal, captioning, and model training -**What you'll build:** A video processing pipeline that: +**What you build:** A video processing pipeline that: - Splits videos into 10-second clips using fixed stride or scene detection - Generates clip-level embeddings for similarity search and deduplication - Optionally creates captions and preview images @@ -63,7 +63,7 @@ To use NeMo Curator's video curation capabilities, ensure your system meets thes - CPU encoders: `libopenh264` or `libx264` (fallback options) :::{tip} -If you don't have `uv` installed, refer to the [Installation Guide](../admin/installation.md) for setup instructions, or install it quickly with: +If `uv` is not installed, refer to the [Installation Guide](../admin/installation.md) for setup instructions, or install it quickly with: ```bash curl -LsSf https://astral.sh/uv/0.8.22/install.sh | sh @@ -183,7 +183,7 @@ NeMo Curator supports two embedding model families: - [cosmos-embed1-336p on Hugging Face](https://huggingface.co/nvidia/cosmos-embed1-336p) - [cosmos-embed1-448p on Hugging Face](https://huggingface.co/nvidia/cosmos-embed1-448p) -For this quickstart, we're going to set up support for **Cosmos-Embed1-224p**. +For this quickstart, the following steps set up support for **Cosmos-Embed1-224p**. ### Prepare Model Weights @@ -211,7 +211,7 @@ Organize input videos and output locations before running the pipeline. MODEL_DIR=/path/to/models ``` -- **S3**: For cloud storage (AWS S3, MinIO, etc.). Configure credentials in `~/.aws/credentials` and use `s3://` paths for `--video-dir` and `--output-clip-path`. +- **S3**: For cloud storage (AWS S3, MinIO, etc.). Configure credentials in `~/.aws/credentials` and use `s3://` paths for `--video-dir` and `--output-path`. **S3 usage notes:** - Input videos can be read from S3 paths @@ -227,7 +227,7 @@ Use the example script from https://github.com/NVIDIA-NeMo/Curator/tree/main/tut python tutorials/video/getting-started/video_split_clip_example.py \ --video-dir "$DATA_DIR" \ --model-dir "$MODEL_DIR" \ - --output-clip-path "$OUT_DIR" \ + --output-path "$OUT_DIR" \ --splitting-algorithm fixed_stride \ --fixed-stride-split-duration 10.0 \ --embedding-algorithm cosmos-embed1-224p \ @@ -242,6 +242,19 @@ python tutorials/video/getting-started/video_split_clip_example.py \ 4. Encodes clips using libopenh264 codec 5. Writes output clips and metadata to `$OUT_DIR` +```{tip} +**Using a config file**: The example script accepts many command-line arguments. For complex configurations, you can store arguments in a file and pass them with the `@` prefix: + + echo '--video-dir /data/videos + --output-path /data/output + --splitting-algorithm fixed_stride + --fixed-stride-split-duration 10.0 + --embedding-algorithm cosmos-embed1-224p + --transcode-encoder libopenh264' > my_config.txt + + python tutorials/video/getting-started/video_split_clip_example.py @my_config.txt +``` + ### Configuration Options Reference | Option | Values | Description | diff --git a/docs/index.md b/docs/index.md index fc4390ebf3..62f4581f16 100644 --- a/docs/index.md +++ b/docs/index.md @@ -215,6 +215,14 @@ Video Curation Quickstart Audio Curation Quickstart :::: +::::{toctree} +:hidden: +:caption: Setup & Deployment +:maxdepth: 2 +admin/index.md +Install Curator +:::: + ::::{toctree} :hidden: :caption: Curate Text @@ -261,14 +269,6 @@ Process Data Save & Export :::: -::::{toctree} -:hidden: -:caption: Setup & Deployment -:maxdepth: 2 -admin/index.md -Install Curator -:::: - ::::{toctree} :hidden: :caption: Reference diff --git a/docs/project.json b/docs/project.json index f719e487e2..496251b8c2 100644 --- a/docs/project.json +++ b/docs/project.json @@ -1 +1 @@ -{"name": "nemo-curator", "version": "25.09"} \ No newline at end of file +{"name": "nemo-curator", "version": "26.02"} \ No newline at end of file diff --git a/docs/reference/infrastructure/container-environments.md b/docs/reference/infrastructure/container-environments.md index 0dda6dbbce..a13e7fccaf 100644 --- a/docs/reference/infrastructure/container-environments.md +++ b/docs/reference/infrastructure/container-environments.md @@ -37,7 +37,7 @@ NeMo Curator provides official Docker containers with all dependencies pre-insta The primary container includes comprehensive support for all curation modalities: -**Container registry:** `nvcr.io/nvidia/nemo-curator:25.09` +**Container registry:** `nvcr.io/nvidia/nemo-curator:{{ container_version }}` **Supported modalities:** - ✅ Text curation (CPU/GPU) @@ -76,7 +76,7 @@ The primary container includes comprehensive support for all curation modalities * - Installation - NeMo Curator installed with all optional dependencies (`[all]` extras) using uv with NVIDIA index * - Environment Path - - Virtual environment activated by default: `/opt/venv/bin:$PATH` + - Virtual environment at `/opt/venv`. Activate with `source /opt/venv/env.sh` after entering the container. ``` --- diff --git a/tutorials/quickstart.py b/tutorials/quickstart.py index aa6b7fe271..5cf2ade2fa 100644 --- a/tutorials/quickstart.py +++ b/tutorials/quickstart.py @@ -39,9 +39,9 @@ from nemo_curator.tasks import Task, _EmptyTask SAMPLE_SENTENCES = [ - "I love this product", - "I hate this product", - "I'm neutral about this product", + "I love this product, it works great", + "I hate this product, it broke immediately", + "This product is okay but nothing special", ] diff --git a/tutorials/text/llama-nemotron-data-curation/README.md b/tutorials/text/llama-nemotron-data-curation/README.md index f70f128767..28b8d5a2da 100644 --- a/tutorials/text/llama-nemotron-data-curation/README.md +++ b/tutorials/text/llama-nemotron-data-curation/README.md @@ -4,7 +4,7 @@ The [Llama Nemotron Post-Training Dataset](https://huggingface.co/datasets/nvidi Organized into distinct subsets for supervised fine-tuning (SFT) or reinforcement learning (RL), it encompasses samples from various problem domains. All samples are in JSON lines (JSONL) format and contain metadata such as license type, source model, as well as the [Llama Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/llama-nemotron/) model(s) trained with that sample. -Each sample consists of a prompt, along with an expected response with detailed chain-of-thought (CoT) reasoning traces followed by responses (i.e., "reasoning on"), as well as samples with direct responses (i.e., "reasoning off"). +Each sample consists of a prompt, along with an expected response with detailed chain-of-thought (CoT) reasoning traces followed by responses (that is, "reasoning on"), as well as samples with direct responses (that is, "reasoning off"). Here is an example of what a sample from the dataset may look like: ```bash @@ -42,7 +42,7 @@ Setup requirements: - Hardware: This tutorial can be run entirely on CPU workers - Recommended environment: This tutorial was developed and tested with a Conda environment -Please refer to NeMo Curator's [documentation](https://docs.nvidia.com/nemo/curator/latest/) for instructions on how to download NeMo Curator via PyPI, source, or Docker. +Refer to the NeMo Curator [documentation](https://docs.nvidia.com/nemo/curator/latest/) for instructions on how to download NeMo Curator through PyPI, source, or Docker. ## Prerequisites @@ -58,7 +58,7 @@ git lfs install git clone https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset ``` -Alternatively, the dataset can be downloaded via Python: +Alternatively, the dataset can be downloaded using Python: ```python from huggingface_hub import snapshot_download @@ -71,7 +71,7 @@ snapshot_download( ) ``` -Please ensure that the above dataset was downloaded correctly. You can check: +Ensure that the dataset was downloaded correctly. You can verify with the following commands: ```bash $ ls /path/to/Llama-Nemotron-Post-Training-Dataset/SFT @@ -89,7 +89,7 @@ The tokenizer used by this tutorial is called [meta-llama/Llama-3.1-8B-Instruct] 1. Visit https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct 2. Click "Access request" 3. Fill out the form and wait for approval -4. Once approved, log in to your Hugging Face account via the Hugging Face CLI. In the terminal, this can be done via `huggingface-cli login` +4. After approval, log in to your Hugging Face account using the Hugging Face CLI. In the terminal, run `huggingface-cli login` ### Download FastText language identification model @@ -130,20 +130,27 @@ The above script applies basic filtering to the input dataset: - Only take samples used for Nemotron Nano training - Remove empty and malformed samples - Remove non-English samples -- Remove samples with total length (system prompt, input, and output responses) longer than 16k tokens (with chat template applied via the tokenizer) -- Remove samples with output responses longer than 8k tokens (with chat template applied via the tokenizer) +- Remove samples with total length (system prompt, input, and output responses) longer than 16k tokens (with chat template applied using the tokenizer) +- Remove samples with output responses longer than 8k tokens (with chat template applied using the tokenizer) - Only keep columns specified by the `--keep-columns` parameter. We recommend keeping the "input", "output", and "completion_token_count" columns (the "completion_token_count" column always needs to be kept, so that we can sort the samples) -After filtering, it sorts all samples by completion (output response) length, then "interleaves" thinking ON/thinking OFF for curriculum learning. The idea here is to sort the samples in increasing order of difficulty, using the completion token count as a measure of sample difficulty. By default, we interleave one record at a time (i.e., 1 thinking ON sample, then 1 thinking OFF sample, then 1 thinking ON sample, etc.). The user may pass `--chunk-size` followed by an integer to interleave 10 records at a time (10 thinking ON samples, then 10 thinking OFF samples, then 10 thinking ON samples, and so on), 100 records at a time, etc. as desired. We interleave samples from the "reasoning on" and "reasoning off" buckets to gradually introduce complexity. +After filtering, it sorts all samples by completion (output response) length, then "interleaves" thinking ON/thinking OFF for curriculum learning. The idea here is to sort the samples in increasing order of difficulty, using the completion token count as a measure of sample difficulty. By default, we interleave one record at a time (that is, one thinking ON sample, then one thinking OFF sample, then one thinking ON sample, and so on). The user may pass `--chunk-size` followed by an integer to interleave 10 records at a time (10 thinking ON samples, then 10 thinking OFF samples, then 10 thinking ON samples, and so on), 100 records at a time, and so on as desired. We interleave samples from the "reasoning on" and "reasoning off" buckets to gradually introduce complexity. + +## System Requirements + +- **Memory**: This tutorial can be CPU-only but is memory-intensive. For smaller memory systems, use `--filename-filter` to select a subset of the data. +- **CPU allocation**: The `--num-cpus` parameter controls parallelism. Each CPU worker processes data in parallel, so more CPUs means more memory usage. Start with a conservative value and increase gradually. ## Debugging Out of Memory Errors -If you are running into out of memory (OOM) errors, there are a couple of approaches you can try. One is to avoid very large partitions of data. By default, the JSONL data is read with a blocksize of 100 MB per partition. To customize the file reading logic, the user may specify `--json-blocksize "100mb"` with any string representation for the partition size in MB (e.g., "100mb", "256mb"). +If you encounter out-of-memory (OOM) errors: -It can be useful to play around with the `--num-cpus` parameter as well. The goal is to maximize it for improved performance without oversubscribing your available hardware. +1. **Reduce partition size**: Lower the blocksize to reduce per-partition memory. Set `--json-blocksize "50mb"` (default is "100mb"). +2. **Reduce CPU count**: Lower `--num-cpus` to reduce parallel memory pressure rather than using all available cores. +3. **Subset the data**: Use `--filename-filter` to process only specific subsets (such as `--filename-filter "chat"`). ## Next Steps -To see how to train a reasoning model with the resulting dataset, please refer to this NeMo tutorial: [Train Your Own Reasoning Model in 48 Hours on a Single GPU](https://github.com/NVIDIA/NeMo/tree/main/tutorials/llm/reasoning). +To see how to train a reasoning model with the resulting dataset, refer to this NeMo tutorial: [Train Your Own Reasoning Model in 48 Hours on a Single GPU](https://github.com/NVIDIA/NeMo/tree/main/tutorials/llm/reasoning). The NeMo tutorial expects the `/path/to/curated-data/training.jsonl` file generated by this tutorial as input. From 370c784fb9b32aaf735190c987500c782dc71ec3 Mon Sep 17 00:00:00 2001 From: Omkar Kabde Date: Thu, 12 Feb 2026 01:56:49 +0530 Subject: [PATCH 6/6] remove override test Signed-off-by: Omkar Kabde --- tests/stages/common/test_base.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/stages/common/test_base.py b/tests/stages/common/test_base.py index d70a837e6c..937bc3d2f5 100644 --- a/tests/stages/common/test_base.py +++ b/tests/stages/common/test_base.py @@ -380,28 +380,6 @@ def batch_size(self) -> int: def process(self, task: MockTask) -> MockTask: return task - def test_plain_attribute_override_allowed(self): - """Test that plain attribute overrides for name, resources, and batch_size are still allowed.""" - - class PlainAttrStage(ProcessingStage[MockTask, MockTask]): - name = "PlainAttrStage" - resources = Resources(cpus=4.0) - batch_size = 8 - - def process(self, task: MockTask) -> MockTask: - return task - - stage = PlainAttrStage() - assert stage.name == "PlainAttrStage" - assert stage.resources == Resources(cpus=4.0) - assert stage.batch_size == 8 - - # with_() should work fine on plain attributes - modified = stage.with_(name="Modified", resources=Resources(cpus=2.0), batch_size=4) - assert modified.name == "Modified" - assert modified.resources == Resources(cpus=2.0) - assert modified.batch_size == 4 - def test_nested_class_inheritance(self): """Test that nested class inheritance raises an error if a derived class overrides the _name, _resources, or _batch_size property.""" with pytest.raises(TypeError, match="MockStageNestedOverriddenName must not override '_name'"):