From 3f13d8e5acf8e818cf1862dd5f5c0369198787a9 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Mon, 22 Jul 2024 21:19:53 -0600 Subject: [PATCH 01/15] refactor code to preprocess datasets Co-authored-by: Alex-Brooks Signed-off-by: Sukriti-Sharma4 --- tests/utils/test_data_utils.py | 23 ++++--- tests/utils/test_preprocessing_utils.py | 56 +++++++++++++++++ tuning/sft_trainer.py | 61 ++++-------------- tuning/utils/data_utils.py | 12 ++-- tuning/utils/preprocessing_utils.py | 82 ++++++++++++++++++++----- 5 files changed, 158 insertions(+), 76 deletions(-) diff --git a/tests/utils/test_data_utils.py b/tests/utils/test_data_utils.py index 471f28590b..f027d9fdca 100644 --- a/tests/utils/test_data_utils.py +++ b/tests/utils/test_data_utils.py @@ -34,12 +34,13 @@ def test_apply_custom_formatting_template(): "### Input: @HMRCcustomers No this is my first job" + " \n\n ### Response: no complaint" ) - formatted_dataset, dataset_text_field = data_utils.apply_custom_formatting_template( - json_dataset, template + formatted_dataset_field = "formatted_data_field" + formatted_dataset = data_utils.apply_custom_formatting_template( + json_dataset, template, formatted_dataset_field ) # a new dataset_text_field is created in Dataset - assert dataset_text_field in formatted_dataset["train"][0] - assert formatted_dataset["train"][0][dataset_text_field] == expected_response + assert formatted_dataset_field in formatted_dataset["train"][0] + assert formatted_dataset["train"][0][formatted_dataset_field] == expected_response def test_apply_custom_formatting_template_adds_eos_token(): @@ -50,17 +51,21 @@ def test_apply_custom_formatting_template_adds_eos_token(): "### Input: @HMRCcustomers No this is my first job" + " \n\n ### Response: no complaintEOS" ) - formatted_dataset, dataset_text_field = data_utils.apply_custom_formatting_template( - json_dataset, template, "EOS" + formatted_dataset_field = "formatted_data_field" + formatted_dataset = data_utils.apply_custom_formatting_template( + json_dataset, template, formatted_dataset_field, "EOS" ) # a new dataset_text_field is created in Dataset - assert dataset_text_field in formatted_dataset["train"][0] - assert formatted_dataset["train"][0][dataset_text_field] == expected_response + assert formatted_dataset_field in formatted_dataset["train"][0] + assert formatted_dataset["train"][0][formatted_dataset_field] == expected_response def test_apply_custom_formatting_template_gives_error_with_wrong_keys(): """Tests that the formatting function will throw error if wrong keys are passed to template""" json_dataset = datasets.load_dataset("json", data_files=TWITTER_COMPLAINTS_DATA) template = "### Input: {{not found}} \n\n ### Response: {{text_label}}" + formatted_dataset_field = "formatted_data_field" with pytest.raises(KeyError): - data_utils.apply_custom_formatting_template(json_dataset, template, "EOS") + data_utils.apply_custom_formatting_template( + json_dataset, template, formatted_dataset_field, "EOS" + ) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 7a807da992..934b087b8c 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -16,7 +16,9 @@ from tuning.config import configs from tuning.utils.preprocessing_utils import ( combine_sequence, + format_dataset, get_data_trainer_kwargs, + get_formatted_dataset_with_single_sequence, get_preprocessed_dataset, load_hf_dataset_from_jsonl_file, validate_data_args, @@ -207,3 +209,57 @@ def test_get_trainer_kwargs_with_custom_masking(use_validation_data): def test_validate_args(data_args, packing): with pytest.raises(ValueError): validate_data_args(data_args, packing) + + +@pytest.mark.parametrize( + "data_path, dataset_text_field, data_formatter_template", + [ + (TWITTER_COMPLAINTS_DATA, "output", None), + ( + TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, + "formatted_field", + "### Text:{{input}} \n\n### Label: {{output}}", + ), + ], +) +def test_get_formatted_dataset_with_single_sequence( + data_path, dataset_text_field, data_formatter_template +): + tokenizer = AutoTokenizer.from_pretrained("Maykeye/TinyLLama-v0") + formatted_dataset = get_formatted_dataset_with_single_sequence( + data_path, dataset_text_field, tokenizer, data_formatter_template + ) + assert isinstance(formatted_dataset, Dataset) + assert dataset_text_field in formatted_dataset.column_names + + +@pytest.mark.parametrize( + "data_args", + [ + # single sequence and response template + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_DATA, + validation_data_path=TWITTER_COMPLAINTS_DATA, + dataset_text_field="output", + response_template="\n### Label:", + ) + ), + # data formatter template with input/output JSON + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, + validation_data_path=TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, + dataset_text_field="formatted_field", + data_formatter_template="### Text:{{input}} \n\n### Label: {{output}}", + ) + ), + ], +) +def test_format_dataset(data_args): + tokenizer = AutoTokenizer.from_pretrained("Maykeye/TinyLLama-v0") + train_set, eval_set, dataset_text_field = format_dataset(data_args, tokenizer) + assert isinstance(train_set, Dataset) + assert isinstance(eval_set, Dataset) + assert dataset_text_field in train_set.column_names + assert dataset_text_field in eval_set.column_names \ No newline at end of file diff --git a/tuning/sft_trainer.py b/tuning/sft_trainer.py index 6e7f2eb679..0e360ad4f5 100644 --- a/tuning/sft_trainer.py +++ b/tuning/sft_trainer.py @@ -35,7 +35,6 @@ ) from transformers.utils import is_accelerate_available, logging from trl import SFTConfig, SFTTrainer -import datasets import fire import transformers @@ -56,13 +55,16 @@ from tuning.trainercontroller import TrainerControllerCallback from tuning.utils.config_utils import get_hf_peft_config, get_json_config from tuning.utils.data_type_utils import get_torch_dtype -from tuning.utils.data_utils import apply_custom_formatting_template from tuning.utils.error_logging import ( INTERNAL_ERROR_EXIT_CODE, USER_ERROR_EXIT_CODE, write_termination_log, ) -from tuning.utils.preprocessing_utils import get_data_collator, validate_data_args +from tuning.utils.preprocessing_utils import ( + format_dataset, + get_data_collator, + validate_data_args, +) def train( @@ -261,52 +263,13 @@ def train( # Validate if data args are set properly validate_data_args(data_args, packing) - data_collator = get_data_collator(packing, data_args.response_template, tokenizer) - - # load the data by parsing JSON - ### TODO: all the jSON file formatting will be moved to a separate function - data_files = {"train": data_args.training_data_path} - if data_args.validation_data_path: - data_files["validation"] = data_args.validation_data_path - format_dataset = lambda example: { # pylint: disable=unnecessary-lambda-assignment - f"{data_args.dataset_text_field}": example[f"{data_args.dataset_text_field}"] - + tokenizer.eos_token - } - - json_dataset = datasets.load_dataset("json", data_files=data_files) - if data_args.data_formatter_template: - ( - formatted_train_dataset, - data_args.dataset_text_field, - ) = apply_custom_formatting_template( - json_dataset["train"], - data_args.data_formatter_template, - tokenizer.eos_token, - ) - else: - formatted_train_dataset = json_dataset["train"].map(format_dataset) - logger.info("Training dataset length is %s", len(formatted_train_dataset)) - - formatted_validation_dataset = None - if data_args.validation_data_path: - if data_args.data_formatter_template: - ( - formatted_validation_dataset, - data_args.dataset_text_field, - ) = apply_custom_formatting_template( - json_dataset["validation"], - data_args.data_formatter_template, - tokenizer.eos_token, - ) - else: - formatted_validation_dataset = json_dataset["validation"].map( - format_dataset - ) - logger.info( - "Validation dataset length is %s", len(formatted_validation_dataset) - ) - ### JSON file formatting ends here + ( + formatted_train_dataset, + formatted_validation_dataset, + dataset_text_field, + ) = format_dataset(data_args, tokenizer) + data_collator = get_data_collator(packing, data_args.response_template, tokenizer) if framework is not None and framework.requires_agumentation: model, (peft_config,) = framework.augmentation( @@ -337,7 +300,7 @@ def train( eval_dataset=formatted_validation_dataset, packing=packing, data_collator=data_collator, - dataset_text_field=data_args.dataset_text_field, + dataset_text_field=dataset_text_field, args=training_args, max_seq_length=max_seq_length, callbacks=trainer_callbacks, diff --git a/tuning/utils/data_utils.py b/tuning/utils/data_utils.py index 3e67cc56ff..6865d86c42 100644 --- a/tuning/utils/data_utils.py +++ b/tuning/utils/data_utils.py @@ -2,21 +2,25 @@ import re -def apply_custom_formatting_template(dataset, template, eos_token=""): +def apply_custom_formatting_template( + dataset, template, formatted_dataset_field, eos_token="" +): """Function to format datasets with Alpaca style / other templates. Args: dataset: the HF Dataset element loaded from a JSON or DatasetDict object. template: Template to format data with. Features of Dataset should be referred to by {{key}} + formatted_dataset_field: Dataset_text_field eos_token: string EOS token to be appended while formatting data to a single sequence. Defaults to empty Returns: - Formatted HF Dataset, dataset_field name that contains formatted data. + Formatted HF Dataset """ - formatted_dataset_field = "formatted_data_field" template += eos_token + assert formatted_dataset_field is not None + def formatter(element): def replace_text(match_obj): captured_groups = match_obj.groups() @@ -37,4 +41,4 @@ def replace_text(match_obj): ) } - return dataset.map(formatter), formatted_dataset_field + return dataset.map(formatter) diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 545e163522..ba7080b24f 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -18,11 +18,15 @@ # Third Party from datasets import Dataset from transformers import AutoTokenizer, DataCollatorForSeq2Seq +from transformers.utils import logging from trl import DataCollatorForCompletionOnlyLM import datasets # Local from tuning.config import configs +from tuning.utils.data_utils import apply_custom_formatting_template + +logger = logging.get_logger("sft_trainer_preprocessing") def validate_data_args(data_args: configs.DataArguments, packing: bool): @@ -112,6 +116,39 @@ def get_data_collator( # 2. add anything needed for preprocessed input +def format_dataset(data_args: configs.DataArguments, tokenizer: AutoTokenizer): + """ + Args: + data_args: tuning.config.configs.DataArguments + tokenizer: AutoTokenizer + Returns: + Tuple(Dataset, Dataset, str) + tuple containing train_dataset, eval_dataset and dataset_text_field + """ + eval_dataset = None + dataset_text_field = data_args.dataset_text_field + if data_args.data_formatter_template or dataset_text_field: + if dataset_text_field is None: + dataset_text_field = "new_formatted_field" + train_dataset = get_formatted_dataset_with_single_sequence( + data_args.training_data_path, + dataset_text_field, + tokenizer, + data_args.data_formatter_template, + ) + logger.info("Training dataset length is %s", len(train_dataset)) + if data_args.validation_data_path: + (eval_dataset) = get_formatted_dataset_with_single_sequence( + data_args.validation_data_path, + dataset_text_field, + tokenizer, + data_args.data_formatter_template, + ) + logger.info("Validation dataset length is %s", len(eval_dataset)) + # TODO: add a else here for preprocessing + return train_dataset, eval_dataset, dataset_text_field + + ################################################################################### ### The functions below are not yet used. Iterative development towards new features @@ -222,13 +259,11 @@ def get_data_trainer_kwargs( output_field_name="output", ) else: - # Collator is a DataCollatorForCompletionOnlyLM or None; - # Load it as JSON and apply our normal preprocessing logic - train_dataset = get_formatted_dataset( + train_dataset = get_formatted_dataset_with_single_sequence( training_data_path, dataset_text_field, tokenizer ) if validation_data_path: - eval_dataset = get_formatted_dataset( + eval_dataset = get_formatted_dataset_with_single_sequence( validation_data_path, dataset_text_field, tokenizer ) @@ -238,8 +273,11 @@ def get_data_trainer_kwargs( return data_kwargs -def get_formatted_dataset( - data_path: str, dataset_text_field: str, tokenizer: AutoTokenizer +def get_formatted_dataset_with_single_sequence( + data_path: str, + dataset_text_field: str, + tokenizer: AutoTokenizer, + data_formatter_template: Optional[str] = None ) -> Dataset: """Applies formatting to the loaded dataset instance; does NOT pretokenize data. @@ -247,22 +285,38 @@ def get_formatted_dataset( data_path: str Path to the file to be loaded. dataset_text_field: str - Dataset text field fto be used for formatting by TRL. + Dataset text field to be used for formatting. + If data_formatter_template specified, \ + this will be the new field creating single sequence. tokenizer: AutoTokenizer Loaded tokenizer object to be used by the collator. + data_formatter_template: str + Template to apply to create single sequence and store it in dataset_text_field Returns: Dataset HF Dataset with formatted [str] data. """ - format_dataset = lambda example: { # pylint: disable=unnecessary-lambda-assignment - f"{dataset_text_field}": example[f"{dataset_text_field}"] + tokenizer.eos_token - } - json_dataset = datasets.load_dataset("json", data_files=data_path) - return json_dataset.map(format_dataset)[ - "train" - ] # HACK - for now, we just do both datasets separately; train is the default split + json_dataset = datasets.load_dataset("json", data_files=data_path) + format_dataset_EOS = ( + lambda example: { # pylint: disable=unnecessary-lambda-assignment + f"{dataset_text_field}": example[f"{dataset_text_field}"] + + tokenizer.eos_token + } + ) + if data_formatter_template: + formatted_train_dataset = apply_custom_formatting_template( + json_dataset["train"], + data_formatter_template, + dataset_text_field, + tokenizer.eos_token, + ) + else: + formatted_train_dataset = json_dataset.map(format_dataset_EOS)[ + "train" + ] # HACK - for now, we just do both datasets separately; train is the default split + return formatted_train_dataset def get_preprocessed_dataset( data_path: str, From 926eac2f8a10d0c4b14629cb7b26872d6be9d89e Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Mon, 22 Jul 2024 21:32:32 -0600 Subject: [PATCH 02/15] fix formatting Co-authored-by: Alex-Brooks Signed-off-by: Sukriti-Sharma4 --- tests/utils/test_preprocessing_utils.py | 2 +- tuning/utils/preprocessing_utils.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 934b087b8c..d65b80b489 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -262,4 +262,4 @@ def test_format_dataset(data_args): assert isinstance(train_set, Dataset) assert isinstance(eval_set, Dataset) assert dataset_text_field in train_set.column_names - assert dataset_text_field in eval_set.column_names \ No newline at end of file + assert dataset_text_field in eval_set.column_names diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index ba7080b24f..3bbbd40957 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -277,7 +277,7 @@ def get_formatted_dataset_with_single_sequence( data_path: str, dataset_text_field: str, tokenizer: AutoTokenizer, - data_formatter_template: Optional[str] = None + data_formatter_template: Optional[str] = None, ) -> Dataset: """Applies formatting to the loaded dataset instance; does NOT pretokenize data. @@ -318,6 +318,7 @@ def get_formatted_dataset_with_single_sequence( ] # HACK - for now, we just do both datasets separately; train is the default split return formatted_train_dataset + def get_preprocessed_dataset( data_path: str, tokenizer: AutoTokenizer, From 5b9d8720e0c632a90605e33ad8b74c166f2f5f27 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Mon, 22 Jul 2024 22:54:34 -0600 Subject: [PATCH 03/15] allow input/output in validate args Co-authored-by: Alex-Brooks Signed-off-by: Sukriti-Sharma4 --- tests/utils/test_preprocessing_utils.py | 15 ++++++++++ tuning/utils/preprocessing_utils.py | 40 +++++++++++++++---------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index d65b80b489..d9f621d81d 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -196,6 +196,14 @@ def test_get_trainer_kwargs_with_custom_masking(use_validation_data): ), False, ), + # data formatter with no response template + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_DATA, + data_formatter_template="### Input: {{input}} \n\n### Response: {{output}}", + ), + False, + ), # response template with no dataset_text_field or formatter ( configs.DataArguments( @@ -204,6 +212,13 @@ def test_get_trainer_kwargs_with_custom_masking(use_validation_data): ), False, ), + # JSON without input / output for no single sequence arguments + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_DATA, + ), + False, + ), ], ) def test_validate_args(data_args, packing): diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 3bbbd40957..2dd664f66f 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -36,20 +36,14 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): ), "Training data path has to be set and str" # Dataset containing single sequence needs a response template for masking - if data_args.response_template is None and data_args.dataset_text_field is not None: - if packing is False: - raise ValueError( - "Since dataset_text_field is provided and packing is disabled, \ - needs a corresponding response template for masking" - ) - - # Currently if packing is false, we require a response_template. This may change in future. - if packing is False: + if data_args.dataset_text_field or data_args.data_formatter_template: if data_args.response_template is None: - raise ValueError( - "Response template is None, needs to be set for training \ - with packing disabled." - ) + if packing is False: + raise ValueError( + "Since dataset_text_field or data_formatter_template \ + is provided and packing is disabled, \ + needs a corresponding response template for masking" + ) if data_args.response_template: # To use Response template, pass datasets with single sequence instances \ @@ -65,10 +59,24 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): "dataset_text_field and data_formatter_template are both set,\ but are mutually exclusive options" ) - # TODO(s) In future seupport two more formats: - # 1. Allow no response template, and JSON with input/output fields and mask input - # 2. Allow pretokenized Dataset besides JSON. + # If not single sequence, JSON should contain input/output fields + if not (data_args.dataset_text_field or data_args.data_formatter_template): + json_dataset = datasets.load_dataset( + "json", data_files=data_args.training_data_path + ) + if "input" not in json_dataset.column_names: + raise ValueError( + "JSON should contain input field if no dataset_text_field or \ + data_formatter_template specified" + ) + if "output" not in json_dataset.column_names: + raise ValueError( + "JSON should contain output field if no dataset_text_field or \ + data_formatter_template specified" + ) + # TODO(s) In future support + # Allow pretokenized Dataset besides JSON. def get_data_collator( From 22cd6601b88ba0acc81202317777e6fc7a5cb187 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Mon, 22 Jul 2024 23:48:01 -0600 Subject: [PATCH 04/15] format input/output JSON and mask Co-authored-by: Alex-Brooks Signed-off-by: Sukriti-Sharma4 --- tests/utils/test_preprocessing_utils.py | 20 +++++++++++++++++--- tuning/sft_trainer.py | 2 +- tuning/utils/preprocessing_utils.py | 23 +++++++++++++++++++++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index d9f621d81d..595e0fe1dd 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -269,12 +269,26 @@ def test_get_formatted_dataset_with_single_sequence( data_formatter_template="### Text:{{input}} \n\n### Label: {{output}}", ) ), + # input/output JSON with masking on input + ( + configs.DataArguments( + training_data_path=TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, + validation_data_path=TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, + ) + ), ], ) def test_format_dataset(data_args): tokenizer = AutoTokenizer.from_pretrained("Maykeye/TinyLLama-v0") - train_set, eval_set, dataset_text_field = format_dataset(data_args, tokenizer) + train_set, eval_set, dataset_text_field = format_dataset( + data_args, tokenizer, max_seq_length=1024 + ) assert isinstance(train_set, Dataset) assert isinstance(eval_set, Dataset) - assert dataset_text_field in train_set.column_names - assert dataset_text_field in eval_set.column_names + if dataset_text_field is None: + column_names = set(["input_ids", "attention_mask", "labels"]) + assert set(eval_set.column_names) == column_names + assert set(train_set.column_names) == column_names + else: + assert dataset_text_field in train_set.column_names + assert dataset_text_field in eval_set.column_names diff --git a/tuning/sft_trainer.py b/tuning/sft_trainer.py index 0e360ad4f5..e64218945f 100644 --- a/tuning/sft_trainer.py +++ b/tuning/sft_trainer.py @@ -268,7 +268,7 @@ def train( formatted_train_dataset, formatted_validation_dataset, dataset_text_field, - ) = format_dataset(data_args, tokenizer) + ) = format_dataset(data_args, tokenizer, max_seq_length) data_collator = get_data_collator(packing, data_args.response_template, tokenizer) if framework is not None and framework.requires_agumentation: diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 2dd664f66f..4d11d618cc 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -124,7 +124,9 @@ def get_data_collator( # 2. add anything needed for preprocessed input -def format_dataset(data_args: configs.DataArguments, tokenizer: AutoTokenizer): +def format_dataset( + data_args: configs.DataArguments, tokenizer: AutoTokenizer, max_seq_length: int +): """ Args: data_args: tuning.config.configs.DataArguments @@ -153,7 +155,24 @@ def format_dataset(data_args: configs.DataArguments, tokenizer: AutoTokenizer): data_args.data_formatter_template, ) logger.info("Validation dataset length is %s", len(eval_dataset)) - # TODO: add a else here for preprocessing + else: + # This is for JSON containing input/output fields + train_dataset = get_preprocessed_dataset( + data_args.training_data_path, + tokenizer, + max_seq_length, + input_field_name="input", + output_field_name="output", + ) + if data_args.validation_data_path: + eval_dataset = get_preprocessed_dataset( + data_args.validation_data_path, + tokenizer, + max_seq_length, + input_field_name="input", + output_field_name="output", + ) + return train_dataset, eval_dataset, dataset_text_field From 630191a771bc1aa28a0b38d157770c20faa9aa5b Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Tue, 23 Jul 2024 22:41:23 -0600 Subject: [PATCH 05/15] function to return suitable collator Co-authored-by: Alex-Brooks Signed-off-by: Sukriti-Sharma4 --- tests/utils/test_preprocessing_utils.py | 49 +++++++++++++++++++++++++ tuning/sft_trainer.py | 8 +++- tuning/utils/preprocessing_utils.py | 26 ++++++++----- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 595e0fe1dd..2aa6c56986 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -17,6 +17,7 @@ from tuning.utils.preprocessing_utils import ( combine_sequence, format_dataset, + get_data_collator, get_data_trainer_kwargs, get_formatted_dataset_with_single_sequence, get_preprocessed_dataset, @@ -151,6 +152,54 @@ def test_get_trainer_kwargs_with_response_template_and_text_field( assert set(trainer_kwargs["train_dataset"].column_names) == column_names +@pytest.mark.parametrize( + "packing, response_template, formatted_train_dataset, max_seq_length, expected_collator", + [ + ( + False, + "\n### Label:", + load_hf_dataset_from_jsonl_file( + TWITTER_COMPLAINTS_DATA, + input_field_name="Tweet text", + output_field_name="text_label", + ), + 1024, + DataCollatorForCompletionOnlyLM, + ), + ( + False, + None, + Dataset.from_list( + [ + { + "input_ids": [9437, 29, 210], + "attention_mask": [1, 1, 1], + "labels": [1, 20, 30], + } + ] + ), + 1024, + DataCollatorForSeq2Seq, + ), + ], +) +def test_get_data_collator( + packing, + response_template, + formatted_train_dataset, + max_seq_length, + expected_collator, +): + collator = get_data_collator( + packing, + response_template, + AutoTokenizer.from_pretrained("Maykeye/TinyLLama-v0"), + formatted_train_dataset, + max_seq_length, + ) + assert isinstance(collator, expected_collator) + + @pytest.mark.parametrize("use_validation_data", [True, False]) def test_get_trainer_kwargs_with_custom_masking(use_validation_data): training_data_path = TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT diff --git a/tuning/sft_trainer.py b/tuning/sft_trainer.py index e64218945f..308dd2ff3c 100644 --- a/tuning/sft_trainer.py +++ b/tuning/sft_trainer.py @@ -269,7 +269,13 @@ def train( formatted_validation_dataset, dataset_text_field, ) = format_dataset(data_args, tokenizer, max_seq_length) - data_collator = get_data_collator(packing, data_args.response_template, tokenizer) + data_collator = get_data_collator( + packing, + data_args.response_template, + tokenizer, + formatted_train_dataset, + max_seq_length, + ) if framework is not None and framework.requires_agumentation: model, (peft_config,) = framework.augmentation( diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 4d11d618cc..767cae2c67 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -83,6 +83,8 @@ def get_data_collator( packing: bool, response_template: Optional[str], tokenizer: AutoTokenizer, + formatted_train_dataset: Dataset, + max_seq_length: int, ) -> Callable: """Create and return the the appropriate collator type based on the configuration for packing, response_template, and dataset_text_field. @@ -94,6 +96,10 @@ def get_data_collator( Response template to be used for formatting by TRL. tokenizer: AutoTokenizer Loaded tokenizer object to be used by the collator. + formatted_train_dataset: Dataset + Train Dataset formatted for tuning + max_seq_length: int + Max sequence length expected Returns: Callable @@ -113,15 +119,15 @@ def get_data_collator( tokenizer=tokenizer, ignore_index=configs.IGNORE_INDEX, ) - # TO DO with future changes, - # 1. Support no packing and seq2seq colator without response template - # # if dataset_text_field is None and response_template is None: - # # Use the seq2seq data collator; - # # Note that this automatically pads labels with -100 - # return DataCollatorForSeq2Seq( - # tokenizer=tokenizer, padding=True, max_length=max_sequence_length - # ) - # 2. add anything needed for preprocessed input + # Note that this automatically pads labels with -100 + # TODO check if this is sufficient for preprocessed + if ( + "attention_mask" in formatted_train_dataset.column_names + and "labels" in formatted_train_dataset.column_names + ): + return DataCollatorForSeq2Seq( + tokenizer=tokenizer, padding=True, max_length=max_seq_length + ) def format_dataset( @@ -131,6 +137,8 @@ def format_dataset( Args: data_args: tuning.config.configs.DataArguments tokenizer: AutoTokenizer + max_seq_length: int + Max sequence length expected Returns: Tuple(Dataset, Dataset, str) tuple containing train_dataset, eval_dataset and dataset_text_field From 3ea39048fbb5ce547f08e148521c6edbd1800ea6 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Tue, 23 Jul 2024 23:09:53 -0600 Subject: [PATCH 06/15] add tests for SFT Trainer input/output format Co-authored-by: Alex-Brooks Signed-off-by: Sukriti-Sharma4 --- tests/test_sft_trainer.py | 56 +++++++++++++++++++++++++++++ tuning/utils/preprocessing_utils.py | 4 +-- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py index 57ff216cbe..a28de0bdf4 100644 --- a/tests/test_sft_trainer.py +++ b/tests/test_sft_trainer.py @@ -34,6 +34,7 @@ EMPTY_DATA, MALFORMATTED_DATA, TWITTER_COMPLAINTS_DATA, + TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT, TWITTER_COMPLAINTS_JSON_FORMAT, ) @@ -724,3 +725,58 @@ def test_run_with_good_experimental_metadata(): additional_callbacks=[TrainerCallback()], exp_metadata=metadata, ) + + +### Tests for pretokenized data +def test_pretokenized_dataset(): + """Ensure that we can provide a pretokenized dataset with input/output format.""" + with tempfile.TemporaryDirectory() as tempdir: + train_args = copy.deepcopy(TRAIN_ARGS) + train_args.output_dir = tempdir + data_args = copy.deepcopy(DATA_ARGS) + data_args.dataset_text_field = None + data_args.response_template = None + data_args.training_data_path = TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT + sft_trainer.train(MODEL_ARGS, data_args, train_args, PEFT_PT_ARGS) + _validate_training(tempdir) + + +@pytest.mark.parametrize( + "dataset_text_field,response_template", + [ + ("foo", None), + (None, "bar"), + ], +) +def test_pretokenized_dataset_bad_args(dataset_text_field, response_template): + """Ensure that we can't provide only dataset text field / response template for pretok data.""" + with tempfile.TemporaryDirectory() as tempdir: + train_args = copy.deepcopy(TRAIN_ARGS) + train_args.output_dir = tempdir + + data_args = copy.deepcopy(DATA_ARGS) + data_args.dataset_text_field = dataset_text_field + data_args.response_template = response_template + data_args.training_data_path = TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT + # We should raise an error since we should not have a dataset text + # field or a response template if we're pretokenized data + with pytest.raises(ValueError): + sft_trainer.train(MODEL_ARGS, data_args, train_args, PEFT_PT_ARGS) + + +def test_pretokenized_dataset_wrong_format(): + """Ensure that we fail to generate data if the data is in the wrong format.""" + with tempfile.TemporaryDirectory() as tempdir: + train_args = copy.deepcopy(TRAIN_ARGS) + train_args.output_dir = tempdir + + data_args = copy.deepcopy(DATA_ARGS) + data_args.dataset_text_field = None + data_args.response_template = None + data_args.training_data_path = TWITTER_COMPLAINTS_DATA + + # It would be best to handle this in a way that is more understandable; we might + # need to add directly validation prior to the dataset generation since datasets + # is essentially swallowing a KeyError here. + with pytest.raises(ValueError): + sft_trainer.train(MODEL_ARGS, data_args, train_args, PEFT_PT_ARGS) diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 767cae2c67..806194163f 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -65,12 +65,12 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): json_dataset = datasets.load_dataset( "json", data_files=data_args.training_data_path ) - if "input" not in json_dataset.column_names: + if "input" not in json_dataset["train"].column_names: raise ValueError( "JSON should contain input field if no dataset_text_field or \ data_formatter_template specified" ) - if "output" not in json_dataset.column_names: + if "output" not in json_dataset["train"].column_names: raise ValueError( "JSON should contain output field if no dataset_text_field or \ data_formatter_template specified" From 815f4d2a6e77cc68e98412f61b97f58dfbb664f8 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Tue, 23 Jul 2024 23:14:43 -0600 Subject: [PATCH 07/15] remove unused functions Co-authored-by: Alex-Brooks Signed-off-by: Sukriti-Sharma4 --- tests/utils/test_preprocessing_utils.py | 77 --------------- tuning/utils/preprocessing_utils.py | 124 ------------------------ 2 files changed, 201 deletions(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 2aa6c56986..dd1a871902 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -18,7 +18,6 @@ combine_sequence, format_dataset, get_data_collator, - get_data_trainer_kwargs, get_formatted_dataset_with_single_sequence, get_preprocessed_dataset, load_hf_dataset_from_jsonl_file, @@ -108,50 +107,6 @@ def test_get_preprocessed_dataset(max_sequence_length): assert key_lengths.pop() <= max_sequence_length -# Tests for fetching train args -@pytest.mark.parametrize( - "use_validation_data, collator_type, packing", - [ - (True, None, True), - (False, None, True), - (True, DataCollatorForCompletionOnlyLM, False), - (False, DataCollatorForCompletionOnlyLM, False), - ], -) -def test_get_trainer_kwargs_with_response_template_and_text_field( - use_validation_data, collator_type, packing -): - training_data_path = TWITTER_COMPLAINTS_DATA - validation_data_path = training_data_path if use_validation_data else None - # Expected columns in the raw loaded dataset for the twitter data - column_names = set(["Tweet text", "ID", "Label", "text_label", "output"]) - trainer_kwargs = get_data_trainer_kwargs( - training_data_path=training_data_path, - validation_data_path=validation_data_path, - packing=packing, - response_template="\n### Label:", - max_sequence_length=100, - tokenizer=AutoTokenizer.from_pretrained("Maykeye/TinyLLama-v0"), - dataset_text_field="output", - ) - assert len(trainer_kwargs) == 3 - # If we are packing, we should not have a data collator - if collator_type is None: - assert trainer_kwargs["data_collator"] is None - else: - assert isinstance(trainer_kwargs["data_collator"], collator_type) - - # We should only have a validation dataset if one is present - if validation_data_path is None: - assert trainer_kwargs["eval_dataset"] is None - else: - assert isinstance(trainer_kwargs["eval_dataset"], Dataset) - assert set(trainer_kwargs["eval_dataset"].column_names) == column_names - - assert isinstance(trainer_kwargs["train_dataset"], Dataset) - assert set(trainer_kwargs["train_dataset"].column_names) == column_names - - @pytest.mark.parametrize( "packing, response_template, formatted_train_dataset, max_seq_length, expected_collator", [ @@ -200,38 +155,6 @@ def test_get_data_collator( assert isinstance(collator, expected_collator) -@pytest.mark.parametrize("use_validation_data", [True, False]) -def test_get_trainer_kwargs_with_custom_masking(use_validation_data): - training_data_path = TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT - validation_data_path = training_data_path if use_validation_data else None - # Expected columns in the raw loaded dataset for the twitter data - column_names = set(["input_ids", "attention_mask", "labels"]) - trainer_kwargs = get_data_trainer_kwargs( - training_data_path=training_data_path, - validation_data_path=validation_data_path, - packing=False, - response_template=None, - max_sequence_length=100, - tokenizer=AutoTokenizer.from_pretrained("Maykeye/TinyLLama-v0"), - dataset_text_field=None, - ) - assert len(trainer_kwargs) == 4 - # If we are packing, we should not have a data collator - assert isinstance(trainer_kwargs["data_collator"], DataCollatorForSeq2Seq) - - # We should only have a validation dataset if one is present - if validation_data_path is None: - assert trainer_kwargs["eval_dataset"] is None - else: - assert isinstance(trainer_kwargs["eval_dataset"], Dataset) - assert set(trainer_kwargs["eval_dataset"].column_names) == column_names - - assert isinstance(trainer_kwargs["train_dataset"], Dataset) - assert set(trainer_kwargs["train_dataset"].column_names) == column_names - # Needed to sidestep TRL validation - assert trainer_kwargs["formatting_func"] is not None - - # Tests for validating data args # Invalid args return ValueError @pytest.mark.parametrize( diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 806194163f..e1448f13c2 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -184,130 +184,6 @@ def format_dataset( return train_dataset, eval_dataset, dataset_text_field -################################################################################### -### The functions below are not yet used. Iterative development towards new features - - -def get_data_collator_temp( - packing: bool, - dataset_text_field: Optional[str], - response_template: Optional[str], - max_sequence_length: int, - tokenizer: AutoTokenizer, -) -> Callable: - """Create and return the the appropriate collator type based on the configuration for packing, - response_template, and dataset_text_field. - - Args: - packing: bool - Whether or not we should apply packing or not. - dataset_text_field: Optional[str] - Dataset text field fto be used for formatting by TRL. - response_template: Optional[str] - Response template to be used for formatting by TRL. - max_sequence_length: int - Max sequence length to be used for sequence tokenization. - tokenizer: AutoTokenizer - Loaded tokenizer object to be used by the collator. - - Returns: - Callable - Callable collator to be leveraged by the trainer. - """ - if not packing: - if dataset_text_field is None and response_template is None: - # Use the seq2seq data collator; note that this automatically pads labels with -100 - return DataCollatorForSeq2Seq( - tokenizer=tokenizer, padding=True, max_length=max_sequence_length - ) - # TODO: near term - how response template ids are parsed out needs to be cleaned. - # The [2:] here applies if response template has \n prefix, it is needed to strip \n, - # otherwise template is not found. We will create issue to clean this out after we discuss - # data formats and collators we will support. - response_template_ids = tokenizer.encode( - response_template, add_special_tokens=False - )[2:] - return DataCollatorForCompletionOnlyLM( - response_template=response_template_ids, - tokenizer=tokenizer, - ignore_index=configs.IGNORE_INDEX, - ) - - -def get_data_trainer_kwargs( - training_data_path: str, - validation_data_path: str, - packing: bool, - response_template: Optional[str], - max_sequence_length: int, - tokenizer: AutoTokenizer, - dataset_text_field: Optional[str], -) -> Dict[str, Any]: - """Get trainer args related to data / processing. At the moment, this consists of: - - the training dataset - - the evaluation dataset - - the data collator - - Maybe a formatting a function [only for a special case for validation] - The result can be kwarg expanded into the trainer initialization. - - Args: - training_data_path: str - Path to the training data. - validation_data_path: str - Path to the validation data. - packing: bool - Whether or not we should apply packing or not. - response_template: Optional[str] - Response template to be used for formatting by TRL. - max_sequence_length: int - Max sequence length to be used for sequence tokenization. - tokenizer: AutoTokenizer - Loaded tokenizer object to be used by the collator. - dataset_text_field: Optional[str] - Dataset text field fto be used for formatting by TRL. - - Returns: - Dict[str, Any] - Data related kwargs to be used by the SFT Trainer. - """ - data_collator = get_data_collator_temp( - packing, dataset_text_field, response_template, max_sequence_length, tokenizer - ) - eval_dataset = None - data_kwargs = {} - if isinstance(data_collator, DataCollatorForSeq2Seq): - # HACK: This function is never called, but is needed to sidestep TRL's internal validation. - data_kwargs["formatting_func"] = lambda x: x - train_dataset = get_preprocessed_dataset( - training_data_path, - tokenizer, - max_sequence_length, - input_field_name="input", - output_field_name="output", - ) - if validation_data_path: - eval_dataset = get_preprocessed_dataset( - validation_data_path, - tokenizer, - max_sequence_length, - input_field_name="input", - output_field_name="output", - ) - else: - train_dataset = get_formatted_dataset_with_single_sequence( - training_data_path, dataset_text_field, tokenizer - ) - if validation_data_path: - eval_dataset = get_formatted_dataset_with_single_sequence( - validation_data_path, dataset_text_field, tokenizer - ) - - data_kwargs["data_collator"] = data_collator - data_kwargs["train_dataset"] = train_dataset - data_kwargs["eval_dataset"] = eval_dataset - return data_kwargs - - def get_formatted_dataset_with_single_sequence( data_path: str, dataset_text_field: str, From a06e2f56c69dd42d2e4f11c85a37468186cf4638 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Sun, 28 Jul 2024 23:13:54 -0600 Subject: [PATCH 08/15] add eos token to input/output format Signed-off-by: Sukriti-Sharma4 --- tests/test_sft_trainer.py | 4 ++-- tests/utils/test_preprocessing_utils.py | 18 ++++++++++++++++++ tuning/utils/preprocessing_utils.py | 11 +++++++---- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py index d8f512cfde..b01c216c4b 100644 --- a/tests/test_sft_trainer.py +++ b/tests/test_sft_trainer.py @@ -759,7 +759,7 @@ def test_pretokenized_dataset_bad_args(dataset_text_field, response_template): data_args.response_template = response_template data_args.training_data_path = TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT # We should raise an error since we should not have a dataset text - # field or a response template if we're pretokenized data + # field or a response template if we have pretokenized data with pytest.raises(ValueError): sft_trainer.train(MODEL_ARGS, data_args, train_args, PEFT_PT_ARGS) @@ -776,7 +776,7 @@ def test_pretokenized_dataset_wrong_format(): data_args.training_data_path = TWITTER_COMPLAINTS_DATA # It would be best to handle this in a way that is more understandable; we might - # need to add directly validation prior to the dataset generation since datasets + # need to directly add validation prior to the dataset generation since datasets # is essentially swallowing a KeyError here. with pytest.raises(ValueError): sft_trainer.train(MODEL_ARGS, data_args, train_args, PEFT_PT_ARGS) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 1801829d2e..581e3a3e6f 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -42,6 +42,24 @@ def test_combine_sequence(input_element, output_element, expected_res): assert comb_seq == expected_res +@pytest.mark.parametrize( + "input_element,output_element,expected_res", + [ + ("foo ", "bar", "foo bar"), + ("foo\n", "bar", "foo\nbar"), + ("foo\t", "bar", "foo\tbar"), + ("foo", "bar", "foo bar"), + ], +) +def test_combine_sequence_adds_eos(input_element, output_element, expected_res): + """Ensure that input / output elements are combined with correct whitespace handling.""" + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + comb_seq = combine_sequence(input_element, output_element, tokenizer.eos_token) + expected_res += tokenizer.eos_token + assert isinstance(comb_seq, str) + assert comb_seq == expected_res + + # Tests for loading the dataset from disk def test_load_hf_dataset_from_jsonl_file(): input_field_name = "Tweet text" diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 179879c73b..fa88bcdb0c 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -307,7 +307,7 @@ def get_jsonl_object(): ### Utils for custom masking / manipulating input / output strs, etc -def combine_sequence(input_element: str, output_element: str): +def combine_sequence(input_element: str, output_element: str, eos_token: str = ""): """Combines / concatenates input & output element. Args: @@ -315,6 +315,9 @@ def combine_sequence(input_element: str, output_element: str): Input component of the combined sequence. output_element: str Output component of the combined sequence. + eos_token: str + EOS token associated with the tokenizer. \ + If passed, it will be concatenated at end Returns: str @@ -323,8 +326,8 @@ def combine_sequence(input_element: str, output_element: str): if not input_element.endswith((" ", "\n", "\t")) and not output_element.startswith( (" ", "\n", "\t") ): - return input_element + " " + output_element - return input_element + output_element + return input_element + " " + output_element + eos_token + return input_element + output_element + eos_token def preprocess_and_tokenize( @@ -356,7 +359,7 @@ def preprocess_and_tokenize( Dictionary containing the input IDs/labels/attention mask for this record. """ combined_seq = combine_sequence( - element[input_field_name], element[output_field_name] + element[input_field_name], element[output_field_name], tokenizer ) tokenized_comb_seqs = tokenizer( From a5f01c4f7d9a82f156d2a8930d7a73704fd60bc6 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Sun, 28 Jul 2024 23:27:55 -0600 Subject: [PATCH 09/15] fix tests Signed-off-by: Sukriti-Sharma4 --- tuning/utils/preprocessing_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index fa88bcdb0c..5657bb96d1 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -359,7 +359,7 @@ def preprocess_and_tokenize( Dictionary containing the input IDs/labels/attention mask for this record. """ combined_seq = combine_sequence( - element[input_field_name], element[output_field_name], tokenizer + element[input_field_name], element[output_field_name], tokenizer.eos_token ) tokenized_comb_seqs = tokenizer( From 007e3e5fb0e19d2db85cbc4464838b4234fbfd88 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Mon, 29 Jul 2024 23:36:40 -0600 Subject: [PATCH 10/15] improve docstrings Signed-off-by: Sukriti-Sharma4 --- tests/utils/test_preprocessing_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_preprocessing_utils.py b/tests/utils/test_preprocessing_utils.py index 581e3a3e6f..9d0e1519d5 100644 --- a/tests/utils/test_preprocessing_utils.py +++ b/tests/utils/test_preprocessing_utils.py @@ -164,6 +164,7 @@ def test_get_data_collator( max_seq_length, expected_collator, ): + """Ensure that the correct collator type is fetched based on the data args""" collator = get_data_collator( packing, response_template, @@ -213,6 +214,7 @@ def test_get_data_collator( ], ) def test_validate_args(data_args, packing): + """Ensure that respective errors are thrown for incorrect data arguments""" with pytest.raises(ValueError): validate_data_args(data_args, packing) @@ -270,7 +272,8 @@ def test_get_formatted_dataset_with_single_sequence( ], ) def test_format_dataset(data_args): - tokenizer = AutoTokenizer.from_pretrained("Maykeye/TinyLLama-v0") + """Ensure that the train/eval data are properly formatted based on the data args / text field""" + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) train_set, eval_set, dataset_text_field = format_dataset( data_args, tokenizer, max_seq_length=1024 ) From 6faa4e5347e03d7cea6b2a2ec3321672318161f7 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Mon, 29 Jul 2024 23:55:08 -0600 Subject: [PATCH 11/15] keeping JSON keys constant Signed-off-by: Sukriti-Sharma4 --- tuning/utils/preprocessing_utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 5657bb96d1..2598b70306 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -28,6 +28,9 @@ logger = logging.get_logger("sft_trainer_preprocessing") +# In future we may make the fields configurable +JSON_INPUT_KEY = "input" +JSON_OUTPUT_KEY = "output" def validate_data_args(data_args: configs.DataArguments, packing: bool): @@ -65,12 +68,12 @@ def validate_data_args(data_args: configs.DataArguments, packing: bool): json_dataset = datasets.load_dataset( "json", data_files=data_args.training_data_path ) - if "input" not in json_dataset["train"].column_names: + if JSON_INPUT_KEY not in json_dataset["train"].column_names: raise ValueError( "JSON should contain input field if no dataset_text_field or \ data_formatter_template specified" ) - if "output" not in json_dataset["train"].column_names: + if JSON_OUTPUT_KEY not in json_dataset["train"].column_names: raise ValueError( "JSON should contain output field if no dataset_text_field or \ data_formatter_template specified" @@ -172,16 +175,16 @@ def format_dataset( data_args.training_data_path, tokenizer, max_seq_length, - input_field_name="input", - output_field_name="output", + input_field_name=JSON_INPUT_KEY, + output_field_name=JSON_OUTPUT_KEY, ) if data_args.validation_data_path: eval_dataset = get_preprocessed_dataset( data_args.validation_data_path, tokenizer, max_seq_length, - input_field_name="input", - output_field_name="output", + input_field_name=JSON_INPUT_KEY, + output_field_name=JSON_OUTPUT_KEY, ) return train_dataset, eval_dataset, dataset_text_field From 5af2ff0e709fe018a1b243bea490aaa46cbb192e Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Wed, 31 Jul 2024 16:40:59 -0600 Subject: [PATCH 12/15] support for input/output format Signed-off-by: Sukriti-Sharma4 --- README.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ac65f1c24f..c1ed986e5a 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,11 @@ pip install fms-hf-tuning[fms-accel] `fms-acceleration` is a collection of plugins that packages that accelerate fine-tuning / training of large models, as part of the `fms-hf-tuning` suite. For more details on see [this section below](#fms-acceleration). ## Data format -We support two data formats: +We support the following data formats: -1. #### Pre-process the JSON/JSONL dataset +1. #### JSON formats with a single sequence and a specified response_template to use for masking on completion. + +#### 1.1 Pre-process the JSON/JSONL dataset Pre-process the JSON/JSONL dataset to contain a single sequence of each data instance containing input + Response. The trainer is configured to expect a response template as a string. For example, if one wants to prepare the `alpaca` format data to feed into this trainer, it is quite easy and can be done with the following code. ```python @@ -87,7 +89,7 @@ The same way can be applied to any dataset, with more info can be found [here](h Once the JSON is converted using the formatting function, pass the `dataset_text_field` containing the single sequence to the trainer. -2. #### Format JSON/JSONL on the fly +#### 1.2 Format JSON/JSONL on the fly Pass a JSON/JSONL and a `data_formatter_template` to use the formatting function on the fly while tuning. The template should specify fields of JSON with `{{field}}`. While tuning, the data will be converted to a single sequence using the template. JSON fields can contain alpha-numeric characters, spaces and the following special symbols - "." , "_", "-". @@ -101,8 +103,19 @@ data_formatter_template: `### Input: {{input}} \n\n##Label: {{output}}` Formatting will happen on the fly while tuning. The keys in template should match fields in JSON file. The `response template` corresponding to the above template will need to be supplied. in this case, `response template` = `\n## Label:`. +##### In conclusion, if using the reponse_template and single sequence, either the `data_formatter_template` argument or `dataset_text_field` needs to be supplied to the trainer. + +2. #### JSONL with input and output fields (no response template) + + Pass a JSONL containing fields "input" with source text and "output" with class labels. Pre-format the input as you see fit. The output field will simply be concatenated to the end of input to create single sequence, and input will be masked. -##### In conclusion, either the `data_formatter_template` argument or `dataset_text_field` needs to be supplied to the trainer. + The "input" and "output" field names are mandatory and cannot be changed. + +Example: Train.json +`{ + "input": "### Input: Colorado is a state in USA ### Output:", + "output": "USA : Location" +}` ## Supported Models From 7d1c9e331a01e3a12361433ea2613b065f8aae43 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Wed, 31 Jul 2024 16:47:06 -0600 Subject: [PATCH 13/15] formatting fixes Signed-off-by: Sukriti-Sharma4 --- README.md | 4 ++-- tuning/utils/preprocessing_utils.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c1ed986e5a..d8d8c3cc22 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ pip install fms-hf-tuning[fms-accel] ## Data format We support the following data formats: -1. #### JSON formats with a single sequence and a specified response_template to use for masking on completion. +### 1. JSON formats with a single sequence and a specified response_template to use for masking on completion. #### 1.1 Pre-process the JSON/JSONL dataset Pre-process the JSON/JSONL dataset to contain a single sequence of each data instance containing input + Response. The trainer is configured to expect a response template as a string. For example, if one wants to prepare the `alpaca` format data to feed into this trainer, it is quite easy and can be done with the following code. @@ -105,7 +105,7 @@ Formatting will happen on the fly while tuning. The keys in template should matc ##### In conclusion, if using the reponse_template and single sequence, either the `data_formatter_template` argument or `dataset_text_field` needs to be supplied to the trainer. -2. #### JSONL with input and output fields (no response template) +### 2. JSONL with input and output fields (no response template) Pass a JSONL containing fields "input" with source text and "output" with class labels. Pre-format the input as you see fit. The output field will simply be concatenated to the end of input to create single sequence, and input will be masked. diff --git a/tuning/utils/preprocessing_utils.py b/tuning/utils/preprocessing_utils.py index 2598b70306..e3dbdb37d3 100644 --- a/tuning/utils/preprocessing_utils.py +++ b/tuning/utils/preprocessing_utils.py @@ -32,6 +32,7 @@ JSON_INPUT_KEY = "input" JSON_OUTPUT_KEY = "output" + def validate_data_args(data_args: configs.DataArguments, packing: bool): assert isinstance( From 264b556af6a4bc5403377c8341c2a63c3800e838 Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Wed, 31 Jul 2024 17:11:04 -0600 Subject: [PATCH 14/15] update rEADME formats Signed-off-by: Sukriti-Sharma4 --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d8d8c3cc22..5b0bbae858 100644 --- a/README.md +++ b/README.md @@ -111,11 +111,11 @@ Formatting will happen on the fly while tuning. The keys in template should matc The "input" and "output" field names are mandatory and cannot be changed. -Example: Train.json -`{ - "input": "### Input: Colorado is a state in USA ### Output:", - "output": "USA : Location" -}` +Example: Train.jsonl + +`{"input": "### Input: Colorado is a state in USA ### Output:", "output": "USA : Location"} +{"input": "### Input: Arizona is also a state in USA ### Output:", "output": "USA : Location"} +` ## Supported Models From 5a96655cbec0c800ce5b66223951da3796bb436a Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Wed, 31 Jul 2024 17:15:29 -0600 Subject: [PATCH 15/15] formatting README Signed-off-by: Sukriti-Sharma4 --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b0bbae858..a37abc1cfe 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,10 @@ Formatting will happen on the fly while tuning. The keys in template should matc Example: Train.jsonl -`{"input": "### Input: Colorado is a state in USA ### Output:", "output": "USA : Location"} +``` +{"input": "### Input: Colorado is a state in USA ### Output:", "output": "USA : Location"} {"input": "### Input: Arizona is also a state in USA ### Output:", "output": "USA : Location"} -` +``` ## Supported Models