diff --git a/nemo_curator/stages/text/filters/fasttext_filter.py b/nemo_curator/stages/text/filters/fasttext_filter.py index 75e6a070cc..f092a038e2 100644 --- a/nemo_curator/stages/text/filters/fasttext_filter.py +++ b/nemo_curator/stages/text/filters/fasttext_filter.py @@ -44,9 +44,9 @@ def score_document(self, text: str) -> float: model = self._fasttext_quality_filter_model text = text.replace("\n", " ").replace("__label__", " ") - pred = model.predict(text) - document_score = pred[1][0] - if pred[0][0] != self._label: + label, score = model.predict([text]) + document_score = score[0][0].item() + if label[0][0] != self._label: document_score = 1 - document_score return document_score @@ -78,9 +78,9 @@ def score_document(self, text: str) -> list[float | str]: model = self._fasttext_langid_model pp = text.strip().replace("\n", " ") - label, score = model.predict(pp, k=1) - score = score[0] - lang_code = label[0][-2:].upper() + label, score = model.predict([pp], k=1) + score = score[0][0].item() + lang_code = label[0][0][-2:].upper() # Need to convert it to a string to allow backend conversions return str([score, lang_code])