From 26790acf8f42e6074780ca61d507f740fc002217 Mon Sep 17 00:00:00 2001 From: Ryan Wolf Date: Fri, 12 Apr 2024 09:32:28 -0700 Subject: [PATCH 1/5] Move tokenizer import Signed-off-by: Ryan Wolf --- nemo_curator/filters/code.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo_curator/filters/code.py b/nemo_curator/filters/code.py index 9a209ec47c..56e4ea69ae 100644 --- a/nemo_curator/filters/code.py +++ b/nemo_curator/filters/code.py @@ -18,7 +18,6 @@ import numpy as np from bs4 import BeautifulSoup from comment_parser import comment_parser -from nemo.collections.common.tokenizers import SentencePieceTokenizer from nemo_curator.filters.doc_filter import DocumentFilter, import_filter from nemo_curator.utils.constants import regex_alpha, regex_alphanum @@ -104,6 +103,8 @@ def keep_document(self, score): class TokenizerFertilityFilter(DocumentFilter): def __init__(self, path_to_tokenizer=None, min_char_to_token_ratio=2.5): + from nemo.collections.common.tokenizers import SentencePieceTokenizer + if path_to_tokenizer is None: raise ValueError( "Must provide a valid path to a SentencePiece " "tokenizer" From c4ff332fade14a7e8e04649960494804f7d2f2ef Mon Sep 17 00:00:00 2001 From: Ryan Wolf Date: Wed, 17 Apr 2024 09:37:39 -0700 Subject: [PATCH 2/5] Reduce inductor threads Signed-off-by: Ryan Wolf --- nemo_curator/filters/code.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nemo_curator/filters/code.py b/nemo_curator/filters/code.py index 56e4ea69ae..07829b2436 100644 --- a/nemo_curator/filters/code.py +++ b/nemo_curator/filters/code.py @@ -103,6 +103,9 @@ def keep_document(self, score): class TokenizerFertilityFilter(DocumentFilter): def __init__(self, path_to_tokenizer=None, min_char_to_token_ratio=2.5): + import os + + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = 1 from nemo.collections.common.tokenizers import SentencePieceTokenizer if path_to_tokenizer is None: From aa36931c53b523aa712057dd81aec48241ce5107 Mon Sep 17 00:00:00 2001 From: Ryan Wolf Date: Wed, 17 Apr 2024 10:25:44 -0700 Subject: [PATCH 3/5] Change env int to string Signed-off-by: Ryan Wolf --- nemo_curator/filters/code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_curator/filters/code.py b/nemo_curator/filters/code.py index 07829b2436..d193817900 100644 --- a/nemo_curator/filters/code.py +++ b/nemo_curator/filters/code.py @@ -105,7 +105,7 @@ class TokenizerFertilityFilter(DocumentFilter): def __init__(self, path_to_tokenizer=None, min_char_to_token_ratio=2.5): import os - os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = 1 + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" from nemo.collections.common.tokenizers import SentencePieceTokenizer if path_to_tokenizer is None: From 35929779b19fd5cdd6ecef8e14422486f085df9d Mon Sep 17 00:00:00 2001 From: Ryan Wolf Date: Wed, 17 Apr 2024 11:09:52 -0700 Subject: [PATCH 4/5] Change location of env var Signed-off-by: Ryan Wolf --- nemo_curator/filters/code.py | 3 --- nemo_curator/modules/__init__.py | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo_curator/filters/code.py b/nemo_curator/filters/code.py index d193817900..56e4ea69ae 100644 --- a/nemo_curator/filters/code.py +++ b/nemo_curator/filters/code.py @@ -103,9 +103,6 @@ def keep_document(self, score): class TokenizerFertilityFilter(DocumentFilter): def __init__(self, path_to_tokenizer=None, min_char_to_token_ratio=2.5): - import os - - os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" from nemo.collections.common.tokenizers import SentencePieceTokenizer if path_to_tokenizer is None: diff --git a/nemo_curator/modules/__init__.py b/nemo_curator/modules/__init__.py index d845441f31..7be9c6822a 100644 --- a/nemo_curator/modules/__init__.py +++ b/nemo_curator/modules/__init__.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + +os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" from .add_id import AddId from .exact_dedup import ExactDuplicates From af800eb1f56eb045c37e5a47c312513f6e6e4518 Mon Sep 17 00:00:00 2001 From: Ryan Wolf Date: Mon, 22 Apr 2024 12:02:26 -0700 Subject: [PATCH 5/5] Add comment linking issue Signed-off-by: Ryan Wolf --- nemo_curator/modules/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nemo_curator/modules/__init__.py b/nemo_curator/modules/__init__.py index 7be9c6822a..d7c099803f 100644 --- a/nemo_curator/modules/__init__.py +++ b/nemo_curator/modules/__init__.py @@ -13,6 +13,10 @@ # limitations under the License. import os +# Disables multiprocessing in torch.compile calls. +# Without this, Dasks multiprocessing combined with PyTorch's +# gives errors like "daemonic processes are not allowed to have children" +# See https://github.com/NVIDIA/NeMo-Curator/issues/31 os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" from .add_id import AddId