remove legacy tokenizer system

src/megatron/bridge/data/builders/finetuning_dataset.py

@@ -22,7 +22,6 @@
 
 from megatron.bridge.data.datasets.packed_sequence import PackedSequenceSpecs
 from megatron.bridge.data.datasets.sft import create_sft_dataset
-from megatron.bridge.training.tokenizers.tokenizer import _HuggingFaceTokenizer
 from megatron.bridge.utils.common_utils import get_rank_safe, print_rank_0
 
 
@@ -321,21 +320,13 @@ def test_path(self) -> Path:
     def _extract_tokenizer_model_name(self) -> str:
         """Automatically get the model name from model path."""
         # Legacy tokenizer compatibility
-        if getattr(self.tokenizer, "legacy", False):
-            tokenizer_cls = _HuggingFaceTokenizer
-            tokenizer_instance = self.tokenizer
-        else:
-            tokenizer_cls = HuggingFaceTokenizer
-            tokenizer_instance = self.tokenizer._tokenizer
+        tokenizer_cls = HuggingFaceTokenizer
+        tokenizer_instance = self.tokenizer._tokenizer
 
         if self.packed_sequence_specs and self.packed_sequence_specs.tokenizer_model_name is not None:
             return self.packed_sequence_specs.tokenizer_model_name
         elif isinstance(tokenizer_instance, tokenizer_cls):
-            # Legacy tokenizer compatibility
-            if getattr(self.tokenizer, "legacy", False):
-                name = self.tokenizer._tokenizer.name_or_path
-            else:
-                name = self.tokenizer.path
+            name = self.tokenizer.path
 
             if name.endswith("context/nemo_tokenizer"):
                 # NEMO_HOME/hf_org/hf_model/context/nemo_tokenizer => hf_org--hf_model