From fb441732cc6cb6a4b08fc29fc266533f838268b5 Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Wed, 31 Jul 2024 22:34:28 +0530 Subject: [PATCH 01/10] feat: support pretokenized datasets Signed-off-by: Mehant Kammakomati --- tests/test_sft_trainer.py | 49 ++++++++++++++++++++ tuning/config/configs.py | 2 + tuning/sft_trainer.py | 9 ++++ tuning/utils/preprocessing_utils.py | 71 +++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+) diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py index b01c216c4b..e5af9ec662 100644 --- a/tests/test_sft_trainer.py +++ b/tests/test_sft_trainer.py @@ -22,7 +22,9 @@ import tempfile # Third Party +from datasets import load_dataset from datasets.exceptions import DatasetGenerationError +from transformers import AutoTokenizer from transformers.trainer_callback import TrainerCallback import pytest import torch @@ -416,6 +418,53 @@ def test_run_causallm_ft_and_inference(): _test_run_inference(tempdir=tempdir) +def test_run_causallm_ft_pretokenized(): + """Check if we can bootstrap and finetune causallm models using pretokenized data""" + with tempfile.TemporaryDirectory() as tempdir: + data_formatting_args = copy.deepcopy(DATA_ARGS) + tokenized_data_path = os.path.join(tempdir, "tokenized_data.json") + + # below args not needed for pretokenized data + data_formatting_args.data_formatter_template = None + data_formatting_args.dataset_text_field = None + data_formatting_args.response_template = None + + # load, tokenize the data, and save it to a file + tokenizer = AutoTokenizer.from_pretrained(MODEL_ARGS.model_name_or_path) + loaded_data = load_dataset( + "json", data_files=data_formatting_args.training_data_path, split="train" + ) + + loaded_data = loaded_data.map( + lambda sample: {"input_ids": tokenizer.encode(sample["output"])} + ) + loaded_data = loaded_data.map(lambda sample: {"labels": sample["input_ids"]}) + + loaded_data.to_json(tokenized_data_path) + + # update the training data path to tokenized data + data_formatting_args.training_data_path = tokenized_data_path + + train_args = copy.deepcopy(TRAIN_ARGS) + train_args.output_dir = tempdir + + sft_trainer.train(MODEL_ARGS, data_formatting_args, train_args) + + # validate full ft configs + _validate_training(tempdir) + checkpoint_path = _get_checkpoint_path(tempdir) + + # Load the model + loaded_model = TunedCausalLM.load(checkpoint_path, MODEL_NAME) + + # Run inference on the text + output_inference = loaded_model.run( + "### Text: @NortonSupport Thanks much.\n\n### Label:", max_new_tokens=50 + ) + assert len(output_inference) > 0 + assert "### Text: @NortonSupport Thanks much.\n\n### Label:" in output_inference + + ############################# Helper functions ############################# def _test_run_causallm_ft(training_args, model_args, data_args, tempdir): train_args = copy.deepcopy(training_args) diff --git a/tuning/config/configs.py b/tuning/config/configs.py index 92fb4f8f88..8cb8229c6e 100644 --- a/tuning/config/configs.py +++ b/tuning/config/configs.py @@ -32,6 +32,8 @@ DEFAULT_BOS_TOKEN = "" DEFAULT_UNK_TOKEN = "" +PADDING_STRATEGY_LONGEST = "longest" + @dataclass class ModelArguments: diff --git a/tuning/sft_trainer.py b/tuning/sft_trainer.py index d889c67e7a..edf1734719 100644 --- a/tuning/sft_trainer.py +++ b/tuning/sft_trainer.py @@ -268,13 +268,22 @@ def train( formatted_train_dataset, formatted_validation_dataset, dataset_text_field, +<<<<<<< HEAD ) = format_dataset(data_args, tokenizer, max_seq_length) +======= + ) = format_dataset(data_args, tokenizer) +>>>>>>> d48ea3c (feat: support pretokenized datasets) data_collator = get_data_collator( packing, data_args.response_template, tokenizer, +<<<<<<< HEAD formatted_train_dataset, max_seq_length, +======= + max_seq_length, + data_args.training_data_path, +>>>>>>> d48ea3c (feat: support pretokenized datasets) ) if framework is not None and framework.requires_agumentation: diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index e3dbdb37d3..6680945723 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -33,8 +33,55 @@ JSON_OUTPUT_KEY = "output" +# check if the provided dataset is pretokenized or not +# the check is taken from trl +# https://github.com/huggingface/trl/blob/ddf4c8dc3ecf6d9ee2b24f94c62182ffd682c808/trl/trainer/sft_trainer.py#L498-L509 +def is_pretokenized_dataset(data_path: str): + if data_path: + # load one sample from the dataset in order to inspect columns + dataset = datasets.load_dataset("json", data_files=data_path, split="train[:1]") + return ("input_ids" in dataset.column_names) and ( + "labels" in dataset.column_names + ) + return False + + def validate_data_args(data_args: configs.DataArguments, packing: bool): + is_train_data_pretokenized = is_pretokenized_dataset( + data_path=data_args.training_data_path + ) + is_eval_data_pretokenized = is_pretokenized_dataset( + data_path=data_args.validation_data_path + ) + + # if the provided train dataset is pretokenized + # however user provides formatting flags, error out + if is_train_data_pretokenized: + if ( + data_args.response_template + or data_args.data_formatter_template + or data_args.dataset_text_field + ): + raise ValueError( + "fields response_template, data_formatter_template, and dataset_text_field \ + are not applicable for pretokenized datasets" + ) + + # if the train dataset is pretokenized + # ensure validation dataset is pretokenized otherwise error out + if data_args.validation_data_path and not is_eval_data_pretokenized: + raise ValueError( + "validation data should be pretokenized to be used \ + along with pretokenized train data" + ) + + # packing wont be available for pretokenized datasets in trl library + # see: https://github.com/huggingface/trl/issues/1848 + if packing: + logger.warning("packing will not be used when datasets are pretokenized") + return + assert isinstance( data_args.training_data_path, str ), "Training data path has to be set and str" @@ -109,6 +156,15 @@ def get_data_collator( Callable Callable collator to be leveraged by the trainer. """ + is_train_data_pretokenized = is_pretokenized_dataset(data_path=training_data_path) + + if is_train_data_pretokenized: + return DataCollatorForSeq2Seq( + tokenizer=tokenizer, + padding=configs.PADDING_STRATEGY_LONGEST, + max_length=max_sequence_length, + return_tensors="pt", + ) if not packing: # TODO: near term - how response template ids are parsed out needs to be cleaned. # The [2:] here applies if response template has \n prefix, it is needed to strip \n, @@ -151,6 +207,21 @@ def format_dataset( tuple containing train_dataset, eval_dataset and dataset_text_field """ eval_dataset = None + is_train_data_pretokenized = is_pretokenized_dataset( + data_path=data_args.training_data_path + ) + + if is_train_data_pretokenized: + train_dataset = datasets.load_dataset( + "json", data_files=data_args.training_data_path, split="train" + ) + if data_args.validation_data_path: + eval_dataset = datasets.load_dataset( + "json", data_files=data_args.validation_data_path, split="train" + ) + # dataset_text_field is irrelevant to pretokenized datasets + return train_dataset, eval_dataset, None + dataset_text_field = data_args.dataset_text_field if data_args.data_formatter_template or dataset_text_field: if dataset_text_field is None: From b369878e911b84e1f3fd0f00e84b876517c0215a Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Thu, 1 Aug 2024 07:22:05 +0530 Subject: [PATCH 02/10] fix: rebase with upstream and review commits Signed-off-by: Mehant Kammakomati --- tuning/sft_trainer.py | 9 -------- tuning/utils/preprocessing_utils.py | 32 ++++++++++++++++------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/tuning/sft_trainer.py b/tuning/sft_trainer.py index edf1734719..d889c67e7a 100644 --- a/tuning/sft_trainer.py +++ b/tuning/sft_trainer.py @@ -268,22 +268,13 @@ def train( formatted_train_dataset, formatted_validation_dataset, dataset_text_field, -<<<<<<< HEAD ) = format_dataset(data_args, tokenizer, max_seq_length) -======= - ) = format_dataset(data_args, tokenizer) ->>>>>>> d48ea3c (feat: support pretokenized datasets) data_collator = get_data_collator( packing, data_args.response_template, tokenizer, -<<<<<<< HEAD formatted_train_dataset, max_seq_length, -======= - max_seq_length, - data_args.training_data_path, ->>>>>>> d48ea3c (feat: support pretokenized datasets) ) if framework is not None and framework.requires_agumentation: diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 6680945723..6e633107cc 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # Standard -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, Optional, Union import json # Third Party -from datasets import Dataset +from datasets import Dataset, IterableDataset +from datasets.exceptions import DatasetGenerationError from transformers import AutoTokenizer, DataCollatorForSeq2Seq from transformers.utils import logging from trl import DataCollatorForCompletionOnlyLM @@ -36,14 +37,16 @@ # check if the provided dataset is pretokenized or not # the check is taken from trl # https://github.com/huggingface/trl/blob/ddf4c8dc3ecf6d9ee2b24f94c62182ffd682c808/trl/trainer/sft_trainer.py#L498-L509 -def is_pretokenized_dataset(data_path: str): - if data_path: - # load one sample from the dataset in order to inspect columns - dataset = datasets.load_dataset("json", data_files=data_path, split="train[:1]") - return ("input_ids" in dataset.column_names) and ( - "labels" in dataset.column_names - ) - return False +def is_pretokenized_dataset(data: Union[str, Dataset, IterableDataset]): + if not data: + return False + if isinstance(data, str): + try: + data = datasets.load_dataset("json", data_files=data, split="train[:1]") + except DatasetGenerationError as e: + raise ValueError(f"failed to load the provided dataset. {e}") + + return ("input_ids" in data.column_names) and ("labels" in data.column_names) def validate_data_args(data_args: configs.DataArguments, packing: bool): @@ -63,7 +66,7 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): or data_args.data_formatter_template or data_args.dataset_text_field ): - raise ValueError( + logger.warning( "fields response_template, data_formatter_template, and dataset_text_field \ are not applicable for pretokenized datasets" ) @@ -156,14 +159,15 @@ def get_data_collator( Callable Callable collator to be leveraged by the trainer. """ - is_train_data_pretokenized = is_pretokenized_dataset(data_path=training_data_path) + is_train_data_pretokenized = is_pretokenized_dataset( + data_path=formatted_train_dataset + ) if is_train_data_pretokenized: return DataCollatorForSeq2Seq( tokenizer=tokenizer, padding=configs.PADDING_STRATEGY_LONGEST, - max_length=max_sequence_length, - return_tensors="pt", + max_length=max_seq_length, ) if not packing: # TODO: near term - how response template ids are parsed out needs to be cleaned. From a7c3869549d42fd3654395e899bed8fcedece632 Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Thu, 1 Aug 2024 07:27:08 +0530 Subject: [PATCH 03/10] fix: rebase with upstream and review commits Signed-off-by: Mehant Kammakomati --- tuning/utils/preprocessing_utils.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 6e633107cc..8395fc6f87 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -44,19 +44,15 @@ def is_pretokenized_dataset(data: Union[str, Dataset, IterableDataset]): try: data = datasets.load_dataset("json", data_files=data, split="train[:1]") except DatasetGenerationError as e: - raise ValueError(f"failed to load the provided dataset. {e}") + raise ValueError("failed to load the provided dataset") from e return ("input_ids" in data.column_names) and ("labels" in data.column_names) def validate_data_args(data_args: configs.DataArguments, packing: bool): - is_train_data_pretokenized = is_pretokenized_dataset( - data_path=data_args.training_data_path - ) - is_eval_data_pretokenized = is_pretokenized_dataset( - data_path=data_args.validation_data_path - ) + is_train_data_pretokenized = is_pretokenized_dataset(data_args.training_data_path) + is_eval_data_pretokenized = is_pretokenized_dataset(data_args.validation_data_path) # if the provided train dataset is pretokenized # however user provides formatting flags, error out @@ -159,9 +155,7 @@ def get_data_collator( Callable Callable collator to be leveraged by the trainer. """ - is_train_data_pretokenized = is_pretokenized_dataset( - data_path=formatted_train_dataset - ) + is_train_data_pretokenized = is_pretokenized_dataset(formatted_train_dataset) if is_train_data_pretokenized: return DataCollatorForSeq2Seq( @@ -211,9 +205,7 @@ def format_dataset( tuple containing train_dataset, eval_dataset and dataset_text_field """ eval_dataset = None - is_train_data_pretokenized = is_pretokenized_dataset( - data_path=data_args.training_data_path - ) + is_train_data_pretokenized = is_pretokenized_dataset(data_args.training_data_path) if is_train_data_pretokenized: train_dataset = datasets.load_dataset( From 533e53be6a9e3d16c4f9ed65ef7c00ab8545ede3 Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Thu, 1 Aug 2024 07:42:37 +0530 Subject: [PATCH 04/10] fix: rebase with upstream and review commits Signed-off-by: Mehant Kammakomati --- tests/utils/test_preprocessing_utils.py | 24 ++++++++++++++++++++++++ tuning/utils/preprocessing_utils.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 9d0e1519d5..f757490816 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -21,6 +21,7 @@ get_data_collator, get_formatted_dataset_with_single_sequence, get_preprocessed_dataset, + is_pretokenized_dataset, load_hf_dataset_from_jsonl_file, validate_data_args, ) @@ -175,6 +176,29 @@ def test_get_data_collator( assert isinstance(collator, expected_collator) +@pytest.mark.parametrize( + "data, result", + [ + (TWITTER_COMPLAINTS_DATA, False), + ( + Dataset.from_list( + [ + { + "input_ids": [9437, 29, 210], + "attention_mask": [1, 1, 1], + "labels": [1, 20, 30], + } + ] + ), + True, + ), + ], +) +def test_is_pretokenized_dat(data, result): + """Ensure that the correct collator type is fetched based on the data args""" + assert is_pretokenized_dataset(data=data) == result + + # Tests for validating data args # Invalid args return ValueError @pytest.mark.parametrize( diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 8395fc6f87..857bd80680 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -44,7 +44,7 @@ def is_pretokenized_dataset(data: Union[str, Dataset, IterableDataset]): try: data = datasets.load_dataset("json", data_files=data, split="train[:1]") except DatasetGenerationError as e: - raise ValueError("failed to load the provided dataset") from e + raise DatasetGenerationError("failed to load the provided dataset") from e return ("input_ids" in data.column_names) and ("labels" in data.column_names) From 6f1d99efb751e79efda1bd26520f836796b2f29f Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Wed, 31 Jul 2024 22:05:56 -0600 Subject: [PATCH 05/10] consolidate collator code Signed-off-by: Sukriti-Sharma4 --- tuning/config/configs.py | 3 --- tuning/utils/preprocessing_utils.py | 11 +---------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/tuning/config/configs.py b/tuning/config/configs.py index 8cb8229c6e..f42e6ca985 100644 --- a/tuning/config/configs.py +++ b/tuning/config/configs.py @@ -32,9 +32,6 @@ DEFAULT_BOS_TOKEN = "" DEFAULT_UNK_TOKEN = "" -PADDING_STRATEGY_LONGEST = "longest" - - @dataclass class ModelArguments: model_name_or_path: Optional[str] = field(default="facebook/opt-125m") diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 857bd80680..ebce42acc0 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -157,12 +157,6 @@ def get_data_collator( """ is_train_data_pretokenized = is_pretokenized_dataset(formatted_train_dataset) - if is_train_data_pretokenized: - return DataCollatorForSeq2Seq( - tokenizer=tokenizer, - padding=configs.PADDING_STRATEGY_LONGEST, - max_length=max_seq_length, - ) if not packing: # TODO: near term - how response template ids are parsed out needs to be cleaned. # The [2:] here applies if response template has \n prefix, it is needed to strip \n, @@ -179,10 +173,7 @@ def get_data_collator( ) # Note that this automatically pads labels with -100 # TODO check if this is sufficient for preprocessed - if ( - "attention_mask" in formatted_train_dataset.column_names - and "labels" in formatted_train_dataset.column_names - ): + if is_train_data_pretokenized: return DataCollatorForSeq2Seq( tokenizer=tokenizer, padding=True, max_length=max_seq_length ) From 61d4984f510ec7c456eb5470059e7af08e6b022d Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Wed, 31 Jul 2024 22:42:04 -0600 Subject: [PATCH 06/10] add valuerrors for incorrect args Signed-off-by: Sukriti-Sharma4 --- tuning/config/configs.py | 1 + tuning/utils/preprocessing_utils.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tuning/config/configs.py b/tuning/config/configs.py index f42e6ca985..92fb4f8f88 100644 --- a/tuning/config/configs.py +++ b/tuning/config/configs.py @@ -32,6 +32,7 @@ DEFAULT_BOS_TOKEN = "" DEFAULT_UNK_TOKEN = "" + @dataclass class ModelArguments: model_name_or_path: Optional[str] = field(default="facebook/opt-125m") diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index ebce42acc0..fef512fc43 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -51,9 +51,14 @@ def is_pretokenized_dataset(data: Union[str, Dataset, IterableDataset]): def validate_data_args(data_args: configs.DataArguments, packing: bool): + assert isinstance( + data_args.training_data_path, str + ), "Training data path has to be set and str" + is_train_data_pretokenized = is_pretokenized_dataset(data_args.training_data_path) is_eval_data_pretokenized = is_pretokenized_dataset(data_args.validation_data_path) + ### Data format 1 # if the provided train dataset is pretokenized # however user provides formatting flags, error out if is_train_data_pretokenized: @@ -62,7 +67,7 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): or data_args.data_formatter_template or data_args.dataset_text_field ): - logger.warning( + raise ValueError( "fields response_template, data_formatter_template, and dataset_text_field \ are not applicable for pretokenized datasets" ) @@ -78,13 +83,10 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): # packing wont be available for pretokenized datasets in trl library # see: https://github.com/huggingface/trl/issues/1848 if packing: - logger.warning("packing will not be used when datasets are pretokenized") + raise ValueError("packing will not be used when datasets are pretokenized") return - assert isinstance( - data_args.training_data_path, str - ), "Training data path has to be set and str" - + ### Data format 2 # Dataset containing single sequence needs a response template for masking if data_args.dataset_text_field or data_args.data_formatter_template: if data_args.response_template is None: @@ -110,6 +112,7 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): but are mutually exclusive options" ) + ### Data format 3 # If not single sequence, JSON should contain input/output fields if not (data_args.dataset_text_field or data_args.data_formatter_template): json_dataset = datasets.load_dataset( @@ -125,8 +128,6 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): "JSON should contain output field if no dataset_text_field or \ data_formatter_template specified" ) - # TODO(s) In future support - # Allow pretokenized Dataset besides JSON. def get_data_collator( From 3c30cae2574eb8a1098a19e03d64f4db65fd041a Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Thu, 1 Aug 2024 10:59:00 +0530 Subject: [PATCH 07/10] feat: add unit tests for validate_data_args and format_dataset Signed-off-by: Mehant Kammakomati --- tests/data/__init__.py | 3 + ...s_tokenized_with_maykeye_tinyllama_v0.json | 10 ++ tests/test_sft_trainer.py | 16 +-- tests/utils/test_preprocessing_utils.py | 103 ++++++++++++++++++ 4 files changed, 118 insertions(+), 14 deletions(-) create mode 100644 tests/data/twitter_complaints_tokenized_with_maykeye_tinyllama_v0.json diff --git a/tests/data/__init__.py b/tests/data/__init__.py index cf88ece96a..7dcf49757d 100644 --- a/tests/data/__init__.py +++ b/tests/data/__init__.py @@ -24,6 +24,9 @@ DATA_DIR, "twitter_complaints_input_output.json" ) TWITTER_COMPLAINTS_JSON_FORMAT = os.path.join(DATA_DIR, "twitter_complaints_json.json") +TWITTER_COMPLAINTS_TOKENIZED = os.path.join( + DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.json" +) EMPTY_DATA = os.path.join(DATA_DIR, "empty_data.json") MALFORMATTED_DATA = os.path.join(DATA_DIR, "malformatted_data.json") MODEL_NAME = "Maykeye/TinyLLama-v0" diff --git a/tests/data/twitter_complaints_tokenized_with_maykeye_tinyllama_v0.json b/tests/data/twitter_complaints_tokenized_with_maykeye_tinyllama_v0.json new file mode 100644 index 0000000000..1d56770e35 --- /dev/null +++ b/tests/data/twitter_complaints_tokenized_with_maykeye_tinyllama_v0.json @@ -0,0 +1,10 @@ +{"Tweet text":"@HMRCcustomers No this is my first job","ID":0,"Label":2,"text_label":"no complaint","output":"### Text: @HMRCcustomers No this is my first job\n\n### Label: no complaint","input_ids":[1,16121,9211,31871,1662,31866,31856,7416,17632,369,1398,433,322,629,712,1784,13,13,8458,31922,21597,31871,697,9566],"labels":[1,16121,9211,31871,1662,31866,31856,7416,17632,369,1398,433,322,629,712,1784,13,13,8458,31922,21597,31871,697,9566]} +{"Tweet text":"@KristaMariePark Thank you for your interest! If you decide to cancel, you can call Customer Care at 1-800-NYTIMES.","ID":1,"Label":2,"text_label":"no complaint","output":"### Text: @KristaMariePark Thank you for your interest! If you decide to cancel, you can call Customer Care at 1-800-NYTIMES.\n\n### Label: no complaint","input_ids":[1,16121,9211,31871,1662,31892,1260,31825,11273,503,31857,632,5284,365,329,553,1280,31905,960,365,6194,289,11025,31844,365,473,987,12207,4218,389,31822,31853,31854,31886,31852,31852,31854,11300,31847,3873,1507,31843,13,13,8458,31922,21597,31871,697,9566],"labels":[1,16121,9211,31871,1662,31892,1260,31825,11273,503,31857,632,5284,365,329,553,1280,31905,960,365,6194,289,11025,31844,365,473,987,12207,4218,389,31822,31853,31854,31886,31852,31852,31854,11300,31847,3873,1507,31843,13,13,8458,31922,21597,31871,697,9566]} +{"Tweet text":"If I can't get my 3rd pair of @beatsbydre powerbeats to work today I'm doneski man. This is a slap in my balls. Your next @Bose @BoseService","ID":2,"Label":1,"text_label":"complaint","output":"### Text: If I can't get my 3rd pair of @beatsbydre powerbeats to work today I'm doneski man. This is a slap in my balls. Your next @Bose @BoseService\n\n### Label: complaint","input_ids":[1,16121,9211,31871,960,312,473,31876,31824,685,629,31822,31878,4449,5861,287,1662,1299,1574,1590,31833,263,1360,1299,1574,289,623,31822,31824,16346,312,31876,31836,994,277,3560,567,31843,672,322,260,29458,288,629,14881,31843,2628,1423,1662,31858,601,1662,31858,601,8378,13,13,8458,31922,21597,31871,9566],"labels":[1,16121,9211,31871,960,312,473,31876,31824,685,629,31822,31878,4449,5861,287,1662,1299,1574,1590,31833,263,1360,1299,1574,289,623,31822,31824,16346,312,31876,31836,994,277,3560,567,31843,672,322,260,29458,288,629,14881,31843,2628,1423,1662,31858,601,1662,31858,601,8378,13,13,8458,31922,21597,31871,9566]} +{"Tweet text":"@EE On Rosneath Arial having good upload and download speeds but terrible latency 200ms. Why is this.","ID":3,"Label":1,"text_label":"complaint","output":"### Text: @EE On Rosneath Arial having good upload and download speeds but terrible latency 200ms. Why is this.\n\n### Label: complaint","input_ids":[1,16121,9211,31871,1662,7766,1078,8123,17561,308,3456,1833,975,10849,291,4372,15379,504,10011,2368,1512,31822,31855,31852,31852,1243,31843,3007,322,433,31843,13,13,8458,31922,21597,31871,9566],"labels":[1,16121,9211,31871,1662,7766,1078,8123,17561,308,3456,1833,975,10849,291,4372,15379,504,10011,2368,1512,31822,31855,31852,31852,1243,31843,3007,322,433,31843,13,13,8458,31922,21597,31871,9566]} +{"Tweet text":"Couples wallpaper, so cute. :) #BrothersAtHome","ID":4,"Label":2,"text_label":"no complaint","output":"### Text: Couples wallpaper, so cute. :) #BrothersAtHome\n\n### Label: no complaint","input_ids":[1,16121,9211,31871,12371,2208,26657,31844,560,14138,31843,21994,1257,24870,496,31829,8198,19057,13,13,8458,31922,21597,31871,697,9566],"labels":[1,16121,9211,31871,12371,2208,26657,31844,560,14138,31843,21994,1257,24870,496,31829,8198,19057,13,13,8458,31922,21597,31871,697,9566]} +{"Tweet text":"@mckelldogs This might just be me, but-- eyedrops? Artificial tears are so useful when you're sleep-deprived and sp\u2026 https:\/\/t.co\/WRtNsokblG","ID":5,"Label":2,"text_label":"no complaint","output":"### Text: @mckelldogs This might just be me, but-- eyedrops? Artificial tears are so useful when you're sleep-deprived and sp\u2026 https:\/\/t.co\/WRtNsokblG\n\n### Label: no complaint","input_ids":[1,16121,9211,31871,1662,31836,651,307,395,13094,672,1467,701,333,515,31844,504,1097,2266,282,305,781,31902,21626,31822,31824,5540,397,560,5253,662,365,31876,263,4985,31854,8903,16801,291,612,31925,2011,1129,31824,31843,1358,31873,19919,31824,31865,31829,469,2131,31874,13,13,8458,31922,21597,31871,697,9566],"labels":[1,16121,9211,31871,1662,31836,651,307,395,13094,672,1467,701,333,515,31844,504,1097,2266,282,305,781,31902,21626,31822,31824,5540,397,560,5253,662,365,31876,263,4985,31854,8903,16801,291,612,31925,2011,1129,31824,31843,1358,31873,19919,31824,31865,31829,469,2131,31874,13,13,8458,31922,21597,31871,697,9566]} +{"Tweet text":"@Yelp can we get the exact calculations for a business rating (for example if its 4 stars but actually 4.2) or do we use a 3rd party site?","ID":6,"Label":2,"text_label":"no complaint","output":"### Text: @Yelp can we get the exact calculations for a business rating (for example if its 4 stars but actually 4.2) or do we use a 3rd party site?\n\n### Label: no complaint","input_ids":[1,16121,9211,31871,1662,31900,307,31837,473,382,685,266,3195,17532,329,260,1173,9363,352,1671,1881,646,619,31822,31882,5556,504,2091,31822,31882,31843,31855,31861,405,499,382,863,260,31822,31878,4449,2540,2042,31902,13,13,8458,31922,21597,31871,697,9566],"labels":[1,16121,9211,31871,1662,31900,307,31837,473,382,685,266,3195,17532,329,260,1173,9363,352,1671,1881,646,619,31822,31882,5556,504,2091,31822,31882,31843,31855,31861,405,499,382,863,260,31822,31878,4449,2540,2042,31902,13,13,8458,31922,21597,31871,697,9566]} +{"Tweet text":"@nationalgridus I have no water and the bill is current and paid. Can you do something about this?","ID":7,"Label":1,"text_label":"complaint","output":"### Text: @nationalgridus I have no water and the bill is current and paid. Can you do something about this?\n\n### Label: complaint","input_ids":[1,16121,9211,31871,1662,14390,16373,337,312,435,697,1579,291,266,3925,322,1434,291,3877,31843,1456,365,499,1419,562,433,31902,13,13,8458,31922,21597,31871,9566],"labels":[1,16121,9211,31871,1662,14390,16373,337,312,435,697,1579,291,266,3925,322,1434,291,3877,31843,1456,365,499,1419,562,433,31902,13,13,8458,31922,21597,31871,9566]} +{"Tweet text":"Never shopping at @MACcosmetics again. Every time I go in there, their employees are super rude\/condescending. I'll take my $$ to @Sephora","ID":8,"Label":1,"text_label":"complaint","output":"### Text: Never shopping at @MACcosmetics again. Every time I go in there, their employees are super rude\/condescending. I'll take my $$ to @Sephora\n\n### Label: complaint","input_ids":[1,16121,9211,31871,7265,7550,389,1662,31856,2226,11596,27771,898,31843,3259,647,312,498,288,635,31844,518,3822,397,2168,28910,31873,13627,4107,1708,31843,312,31876,608,1090,629,10279,289,1662,29966,31831,5605,13,13,8458,31922,21597,31871,9566],"labels":[1,16121,9211,31871,7265,7550,389,1662,31856,2226,11596,27771,898,31843,3259,647,312,498,288,635,31844,518,3822,397,2168,28910,31873,13627,4107,1708,31843,312,31876,608,1090,629,10279,289,1662,29966,31831,5605,13,13,8458,31922,21597,31871,9566]} +{"Tweet text":"@JenniferTilly Merry Christmas to as well. You get more stunning every year \ufffd\ufffd","ID":9,"Label":2,"text_label":"no complaint","output":"### Text: @JenniferTilly Merry Christmas to as well. You get more stunning every year \ufffd\ufffd\n\n### Label: no complaint","input_ids":[1,16121,9211,31871,1662,31884,1450,7064,31847,6538,30894,4472,289,362,828,31843,864,685,541,9932,843,584,18694,31986,13,13,8458,31922,21597,31871,697,9566],"labels":[1,16121,9211,31871,1662,31884,1450,7064,31847,6538,30894,4472,289,362,828,31843,864,685,541,9932,843,584,18694,31986,13,13,8458,31922,21597,31871,697,9566]} diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py index e5af9ec662..1c73d1e602 100644 --- a/tests/test_sft_trainer.py +++ b/tests/test_sft_trainer.py @@ -39,6 +39,7 @@ TWITTER_COMPLAINTS_DATA, TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, TWITTER_COMPLAINTS_JSON_FORMAT, + TWITTER_COMPLAINTS_TOKENIZED, ) # Local @@ -429,21 +430,8 @@ def test_run_causallm_ft_pretokenized(): data_formatting_args.dataset_text_field = None data_formatting_args.response_template = None - # load, tokenize the data, and save it to a file - tokenizer = AutoTokenizer.from_pretrained(MODEL_ARGS.model_name_or_path) - loaded_data = load_dataset( - "json", data_files=data_formatting_args.training_data_path, split="train" - ) - - loaded_data = loaded_data.map( - lambda sample: {"input_ids": tokenizer.encode(sample["output"])} - ) - loaded_data = loaded_data.map(lambda sample: {"labels": sample["input_ids"]}) - - loaded_data.to_json(tokenized_data_path) - # update the training data path to tokenized data - data_formatting_args.training_data_path = tokenized_data_path + data_formatting_args.training_data_path = TWITTER_COMPLAINTS_TOKENIZED train_args = copy.deepcopy(TRAIN_ARGS) train_args.output_dir = tempdir diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index f757490816..ddda0fbb20 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -11,6 +11,7 @@ MODEL_NAME, TWITTER_COMPLAINTS_DATA, TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, + TWITTER_COMPLAINTS_TOKENIZED, ) # Local @@ -235,6 +236,54 @@ def test_is_pretokenized_dat(data, result): ), False, ), + # pretokenized dataset with dataset_text_field + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + dataset_text_field="output", + ), + False, + ), + # pretokenized dataset with data formatter + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + data_formatter_template="### Input: {{input}} \n\n### Response: {{output}}", + ), + False, + ), + # pretokenized dataset with response template + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + response_template="\n### Label:", + ), + False, + ), + # pretokenized training dataset with validation data not pretokenized + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + validation_data_path=TWITTER_COMPLAINTS_DATA, + ), + False, + ), + # pretokenized data with dataset_text_field and response template + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + response_template="\n### Label:", + dataset_text_field="output", + ), + False, + ), + # pretokenized data with packing to True + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + ), + True, + ), ], ) def test_validate_args(data_args, packing): @@ -243,6 +292,31 @@ def test_validate_args(data_args, packing): validate_data_args(data_args, packing) +@pytest.mark.parametrize( + "data_args, packing", + [ + # pretokenized train dataset and no validation dataset passed + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + ), + False, + ), + # pretokenized train and validation datasets + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + validation_data_path=TWITTER_COMPLAINTS_TOKENIZED, + ), + False, + ), + ], +) +def test_validate_args(data_args, packing): + """Ensure that supported data args do not error out when passing pretokenized datasets""" + validate_data_args(data_args, packing) + + @pytest.mark.parametrize( "data_path, dataset_text_field, data_formatter_template", [ @@ -310,3 +384,32 @@ def test_format_dataset(data_args): else: assert dataset_text_field in train_set.column_names assert dataset_text_field in eval_set.column_names + + +@pytest.mark.parametrize( + "data_args", + [ + # pretokenized train and validation datasets + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + validation_data_path=TWITTER_COMPLAINTS_TOKENIZED, + ) + ), + # pretokenized train datasets + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_TOKENIZED, + ) + ), + ], +) +def test_format_dataset_pretokenized(data_args): + """Ensure that pretokenized datasets are loaded and returned as is""" + train_set, eval_set, _ = format_dataset(data_args, None, max_seq_length=1024) + assert isinstance(train_set, Dataset) + assert isinstance(eval_set, Dataset) + + column_names = set(["input_ids", "labels"]) + assert set(eval_set.column_names) == column_names + assert set(train_set.column_names) == column_names From 7b4b3c8a7f381a48ed6654a15931b4ee0688d0f0 Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Thu, 1 Aug 2024 11:01:09 +0530 Subject: [PATCH 08/10] feat: add unit tests for validate_data_args and format_dataset Signed-off-by: Mehant Kammakomati --- tests/test_sft_trainer.py | 3 --- tests/utils/test_preprocessing_utils.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py index 1c73d1e602..9df7eeee8c 100644 --- a/tests/test_sft_trainer.py +++ b/tests/test_sft_trainer.py @@ -22,9 +22,7 @@ import tempfile # Third Party -from datasets import load_dataset from datasets.exceptions import DatasetGenerationError -from transformers import AutoTokenizer from transformers.trainer_callback import TrainerCallback import pytest import torch @@ -423,7 +421,6 @@ def test_run_causallm_ft_pretokenized(): """Check if we can bootstrap and finetune causallm models using pretokenized data""" with tempfile.TemporaryDirectory() as tempdir: data_formatting_args = copy.deepcopy(DATA_ARGS) - tokenized_data_path = os.path.join(tempdir, "tokenized_data.json") # below args not needed for pretokenized data data_formatting_args.data_formatter_template = None diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index ddda0fbb20..4da9d9f55a 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -312,7 +312,7 @@ def test_validate_args(data_args, packing): ), ], ) -def test_validate_args(data_args, packing): +def test_validate_args_pretokenized(data_args, packing): """Ensure that supported data args do not error out when passing pretokenized datasets""" validate_data_args(data_args, packing) From 6e7fccf3ed4f4516fc737bfae82fbd659c4cfa3e Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Thu, 1 Aug 2024 11:07:45 +0530 Subject: [PATCH 09/10] feat: add unit tests for validate_data_args and format_dataset Signed-off-by: Mehant Kammakomati --- tests/utils/test_preprocessing_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 4da9d9f55a..0063649d42 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -410,6 +410,6 @@ def test_format_dataset_pretokenized(data_args): assert isinstance(train_set, Dataset) assert isinstance(eval_set, Dataset) - column_names = set(["input_ids", "labels"]) - assert set(eval_set.column_names) == column_names - assert set(train_set.column_names) == column_names + assert set(["input_ids", "labels"]).issubset(set(train_set.column_names)) + if eval_set: + assert set(["input_ids", "labels"]).issubset(set(eval_set.column_names)) From 4f396390f43a1f08257b1299962c2a616f19361e Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati Date: Thu, 1 Aug 2024 11:12:09 +0530 Subject: [PATCH 10/10] feat: add unit tests for validate_data_args and format_dataset Signed-off-by: Mehant Kammakomati --- tests/utils/test_preprocessing_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 0063649d42..770d999ce1 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -408,7 +408,8 @@ def test_format_dataset_pretokenized(data_args): """Ensure that pretokenized datasets are loaded and returned as is""" train_set, eval_set, _ = format_dataset(data_args, None, max_seq_length=1024) assert isinstance(train_set, Dataset) - assert isinstance(eval_set, Dataset) + if eval_set: + assert isinstance(eval_set, Dataset) assert set(["input_ids", "labels"]).issubset(set(train_set.column_names)) if eval_set: