From 3bad3ec537b268d5196ce577df89e048f13325d5 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Tue, 15 Nov 2022 11:25:18 +0000 Subject: [PATCH 1/8] test hub tf callback --- tests/test_modeling_tf_common.py | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 2f4c12257645..b641a57d16bc 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -35,6 +35,7 @@ from transformers import is_tf_available, is_torch_available from transformers.configuration_utils import PretrainedConfig from transformers.models.auto import get_values +from transformers.keras_callbacks import PushToHubCallback from transformers.testing_utils import ( # noqa: F401 TOKEN, USER, @@ -2344,6 +2345,11 @@ def tearDownClass(cls): except HTTPError: pass + try: + delete_repo(token=cls._token, repo_id="test-model-tf-callback") + except HTTPError: + pass + try: delete_repo(token=cls._token, repo_id="valid_org/test-model-tf-org") except HTTPError: @@ -2386,6 +2392,33 @@ def test_push_to_hub(self): models_equal = False self.assertTrue(models_equal) + def test_push_to_hub_callback(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = TFBertModel(config) + + logging.set_verbosity_info() + logger = logging.get_logger("transformers.utils.hub") + with CaptureLogger(logger) as cl: + with tempfile.TemporaryDirectory() as tmp_dir: + push_to_hub_callback = PushToHubCallback( + output_dir=tmp_dir, + hub_model_id="test-model-tf-callback", + hub_token=self._token, + ) + model.fit(model.dummy_inputs, epochs=1, callbacks=[push_to_hub_callback]) + logging.set_verbosity_warning() + # Check the model card was created and uploaded. + self.assertIn("Uploading README.md to __DUMMY_TRANSFORMERS_USER__/test-model-tf-callback", cl.out) + + new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf-callback") + models_equal = True + for p1, p2 in zip(model.weights, new_model.weights): + if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + models_equal = False + self.assertTrue(models_equal) + def test_push_to_hub_in_organization(self): config = BertConfig( vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 From 851a156108179c64c5cd8dc1df278ca5c95316fc Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Tue, 15 Nov 2022 11:26:48 +0000 Subject: [PATCH 2/8] make fixup --- tests/test_modeling_tf_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index b641a57d16bc..be8839b485e3 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -34,8 +34,8 @@ from requests.exceptions import HTTPError from transformers import is_tf_available, is_torch_available from transformers.configuration_utils import PretrainedConfig -from transformers.models.auto import get_values from transformers.keras_callbacks import PushToHubCallback +from transformers.models.auto import get_values from transformers.testing_utils import ( # noqa: F401 TOKEN, USER, From 9a972d5b49683ad7b57dd7f9798b784946594385 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Tue, 15 Nov 2022 11:41:08 +0000 Subject: [PATCH 3/8] move import --- tests/test_modeling_tf_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index be8839b485e3..56a6fc5e2314 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -34,7 +34,6 @@ from requests.exceptions import HTTPError from transformers import is_tf_available, is_torch_available from transformers.configuration_utils import PretrainedConfig -from transformers.keras_callbacks import PushToHubCallback from transformers.models.auto import get_values from transformers.testing_utils import ( # noqa: F401 TOKEN, @@ -79,6 +78,7 @@ TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING, TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, BertConfig, + PushToHubCallback, RagRetriever, TFAutoModel, TFAutoModelForSequenceClassification, From d1941f55b4d3e938cd9c2a534526143bf7fe365c Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 16 Nov 2022 09:55:24 +0000 Subject: [PATCH 4/8] create repo before cloning it --- src/transformers/keras_callbacks.py | 4 +++- src/transformers/models/blenderbot/tokenization_blenderbot.py | 1 + .../models/blenderbot/tokenization_blenderbot_fast.py | 1 + src/transformers/models/markuplm/tokenization_markuplm.py | 1 + .../models/markuplm/tokenization_markuplm_fast.py | 1 + src/transformers/models/tapex/tokenization_tapex.py | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/transformers/keras_callbacks.py b/src/transformers/keras_callbacks.py index 6c61433712e2..747f295ed1e0 100644 --- a/src/transformers/keras_callbacks.py +++ b/src/transformers/keras_callbacks.py @@ -9,7 +9,7 @@ from packaging.version import parse from tensorflow.keras.callbacks import Callback -from huggingface_hub import Repository +from huggingface_hub import Repository, create_repo from . import IntervalStrategy, PreTrainedTokenizerBase from .modelcard import TrainingSummary @@ -339,11 +339,13 @@ def __init__( self.output_dir = output_dir self.hub_model_id = hub_model_id + create_repo(self.hub_model_id, exists_ok=True) self.repo = Repository( str(self.output_dir), clone_from=self.hub_model_id, use_auth_token=hub_token if hub_token else True, ) + self.tokenizer = tokenizer self.last_job = None self.checkpoint = checkpoint diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot.py b/src/transformers/models/blenderbot/tokenization_blenderbot.py index ef0a24e06d9a..157d495a7af9 100644 --- a/src/transformers/models/blenderbot/tokenization_blenderbot.py +++ b/src/transformers/models/blenderbot/tokenization_blenderbot.py @@ -399,6 +399,7 @@ def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A Blenderbot sequence has the following format: - single sequence: ` X ` + Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py b/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py index 508c643f4d82..4e65294cb904 100644 --- a/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py +++ b/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py @@ -284,6 +284,7 @@ def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A Blenderbot sequence has the following format: - single sequence: ` X ` + Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added diff --git a/src/transformers/models/markuplm/tokenization_markuplm.py b/src/transformers/models/markuplm/tokenization_markuplm.py index faaebf3f268b..2c9e006858ca 100644 --- a/src/transformers/models/markuplm/tokenization_markuplm.py +++ b/src/transformers/models/markuplm/tokenization_markuplm.py @@ -428,6 +428,7 @@ def build_inputs_with_special_tokens( adding special tokens. A RoBERTa sequence has the following format: - single sequence: ` X ` - pair of sequences: ` A B ` + Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. diff --git a/src/transformers/models/markuplm/tokenization_markuplm_fast.py b/src/transformers/models/markuplm/tokenization_markuplm_fast.py index c55cb2b3e124..1531c5ca4bce 100644 --- a/src/transformers/models/markuplm/tokenization_markuplm_fast.py +++ b/src/transformers/models/markuplm/tokenization_markuplm_fast.py @@ -883,6 +883,7 @@ def build_inputs_with_special_tokens( adding special tokens. A RoBERTa sequence has the following format: - single sequence: ` X ` - pair of sequences: ` A B ` + Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. diff --git a/src/transformers/models/tapex/tokenization_tapex.py b/src/transformers/models/tapex/tokenization_tapex.py index e9994ff7b5cb..f862ba7e4f76 100644 --- a/src/transformers/models/tapex/tokenization_tapex.py +++ b/src/transformers/models/tapex/tokenization_tapex.py @@ -342,6 +342,7 @@ def build_inputs_with_special_tokens( adding special tokens. A TAPEX sequence has the following format: - single sequence: ` X ` - pair of sequences: ` A B ` + Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. From 942415fd4f9d21916333efd62e9eea51f6f6022a Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 16 Nov 2022 10:07:20 +0000 Subject: [PATCH 5/8] functional TF code --- tests/test_modeling_tf_common.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 56a6fc5e2314..a61a546d8f67 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -82,6 +82,7 @@ RagRetriever, TFAutoModel, TFAutoModelForSequenceClassification, + TFBertForMaskedLM, TFBertModel, TFRagModel, TFSharedEmbeddings, @@ -2396,7 +2397,8 @@ def test_push_to_hub_callback(self): config = BertConfig( vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 ) - model = TFBertModel(config) + model = TFBertForMaskedLM(config) + model.compile() logging.set_verbosity_info() logger = logging.get_logger("transformers.utils.hub") @@ -2407,12 +2409,12 @@ def test_push_to_hub_callback(self): hub_model_id="test-model-tf-callback", hub_token=self._token, ) - model.fit(model.dummy_inputs, epochs=1, callbacks=[push_to_hub_callback]) + model.fit(model.dummy_inputs, model.dummy_inputs, epochs=1, callbacks=[push_to_hub_callback]) logging.set_verbosity_warning() # Check the model card was created and uploaded. self.assertIn("Uploading README.md to __DUMMY_TRANSFORMERS_USER__/test-model-tf-callback", cl.out) - new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf-callback") + new_model = TFBertForMaskedLM.from_pretrained(f"{USER}/test-model-tf-callback") models_equal = True for p1, p2 in zip(model.weights, new_model.weights): if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: From 53458a0490c9022cc4a746d33ba64d77361274be Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 16 Nov 2022 11:17:50 +0000 Subject: [PATCH 6/8] correctly store last epoch --- src/transformers/keras_callbacks.py | 29 +++++++++++++++++------------ tests/test_modeling_tf_common.py | 22 ++++++++-------------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/transformers/keras_callbacks.py b/src/transformers/keras_callbacks.py index 747f295ed1e0..a74d3c6530ee 100644 --- a/src/transformers/keras_callbacks.py +++ b/src/transformers/keras_callbacks.py @@ -339,7 +339,7 @@ def __init__( self.output_dir = output_dir self.hub_model_id = hub_model_id - create_repo(self.hub_model_id, exists_ok=True) + create_repo(self.hub_model_id, exist_ok=True) self.repo = Repository( str(self.output_dir), clone_from=self.hub_model_id, @@ -396,17 +396,22 @@ def on_epoch_end(self, epoch, logs=None): ) def on_train_end(self, logs=None): + # Makes sure the latest version of the model is uploaded if self.last_job is not None and not self.last_job.is_done: - self.last_job._process.terminate() # Gotta go fast + logging.info("Pushing the last epoch to the Hub, this may take a while...") while not self.last_job.is_done: sleep(1) - self.model.save_pretrained(self.output_dir) - if self.tokenizer is not None: - self.tokenizer.save_pretrained(self.output_dir) - train_summary = TrainingSummary.from_keras( - model=self.model, model_name=self.hub_model_id, keras_history=self.training_history, **self.model_card_args - ) - model_card = train_summary.to_model_card() - with (self.output_dir / "README.md").open("w") as f: - f.write(model_card) - self.repo.push_to_hub(commit_message="End of training", blocking=True) + else: + self.model.save_pretrained(self.output_dir) + if self.tokenizer is not None: + self.tokenizer.save_pretrained(self.output_dir) + train_summary = TrainingSummary.from_keras( + model=self.model, + model_name=self.hub_model_id, + keras_history=self.training_history, + **self.model_card_args + ) + model_card = train_summary.to_model_card() + with (self.output_dir / "README.md").open("w") as f: + f.write(model_card) + self.repo.push_to_hub(commit_message="End of training", blocking=True) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index a61a546d8f67..4cf8707fefd8 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -2370,7 +2370,7 @@ def test_push_to_hub(self): model.push_to_hub("test-model-tf", use_auth_token=self._token) logging.set_verbosity_warning() # Check the model card was created and uploaded. - self.assertIn("Uploading README.md to __DUMMY_TRANSFORMERS_USER__/test-model-tf", cl.out) + self.assertIn("Uploading the following files to __DUMMY_TRANSFORMERS_USER__/test-model-tf", cl.out) new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") models_equal = True @@ -2400,19 +2400,13 @@ def test_push_to_hub_callback(self): model = TFBertForMaskedLM(config) model.compile() - logging.set_verbosity_info() - logger = logging.get_logger("transformers.utils.hub") - with CaptureLogger(logger) as cl: - with tempfile.TemporaryDirectory() as tmp_dir: - push_to_hub_callback = PushToHubCallback( - output_dir=tmp_dir, - hub_model_id="test-model-tf-callback", - hub_token=self._token, - ) - model.fit(model.dummy_inputs, model.dummy_inputs, epochs=1, callbacks=[push_to_hub_callback]) - logging.set_verbosity_warning() - # Check the model card was created and uploaded. - self.assertIn("Uploading README.md to __DUMMY_TRANSFORMERS_USER__/test-model-tf-callback", cl.out) + with tempfile.TemporaryDirectory() as tmp_dir: + push_to_hub_callback = PushToHubCallback( + output_dir=tmp_dir, + hub_model_id="test-model-tf-callback", + hub_token=self._token, + ) + model.fit(model.dummy_inputs, model.dummy_inputs, epochs=1, callbacks=[push_to_hub_callback]) new_model = TFBertForMaskedLM.from_pretrained(f"{USER}/test-model-tf-callback") models_equal = True From 56e580430feab2553b0cdd2d5256a1f90c8e11fe Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 16 Nov 2022 11:23:22 +0000 Subject: [PATCH 7/8] make fixup --- src/transformers/keras_callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/keras_callbacks.py b/src/transformers/keras_callbacks.py index a74d3c6530ee..7128f348c36e 100644 --- a/src/transformers/keras_callbacks.py +++ b/src/transformers/keras_callbacks.py @@ -409,7 +409,7 @@ def on_train_end(self, logs=None): model=self.model, model_name=self.hub_model_id, keras_history=self.training_history, - **self.model_card_args + **self.model_card_args, ) model_card = train_summary.to_model_card() with (self.output_dir / "README.md").open("w") as f: From 90629c9b781e803c181963fba848afd9dceb7e81 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Thu, 17 Nov 2022 12:03:49 +0000 Subject: [PATCH 8/8] better weight comparison --- tests/test_modeling_tf_common.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 4cf8707fefd8..f6a2d74342e4 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -2375,8 +2375,9 @@ def test_push_to_hub(self): new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") models_equal = True for p1, p2 in zip(model.weights, new_model.weights): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + if not tf.math.reduce_all(p1 == p2): models_equal = False + break self.assertTrue(models_equal) # Reset repo @@ -2389,8 +2390,9 @@ def test_push_to_hub(self): new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") models_equal = True for p1, p2 in zip(model.weights, new_model.weights): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + if not tf.math.reduce_all(p1 == p2): models_equal = False + break self.assertTrue(models_equal) def test_push_to_hub_callback(self): @@ -2411,8 +2413,9 @@ def test_push_to_hub_callback(self): new_model = TFBertForMaskedLM.from_pretrained(f"{USER}/test-model-tf-callback") models_equal = True for p1, p2 in zip(model.weights, new_model.weights): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + if not tf.math.reduce_all(p1 == p2): models_equal = False + break self.assertTrue(models_equal) def test_push_to_hub_in_organization(self): @@ -2428,8 +2431,9 @@ def test_push_to_hub_in_organization(self): new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") models_equal = True for p1, p2 in zip(model.weights, new_model.weights): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + if not tf.math.reduce_all(p1 == p2): models_equal = False + break self.assertTrue(models_equal) # Reset repo @@ -2444,6 +2448,7 @@ def test_push_to_hub_in_organization(self): new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") models_equal = True for p1, p2 in zip(model.weights, new_model.weights): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + if not tf.math.reduce_all(p1 == p2): models_equal = False + break self.assertTrue(models_equal)