From 39d3a35cbe1b0e43a7c269175ca5a46b0e0c61ca Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 17:00:17 +0100 Subject: [PATCH 1/8] yes! --- tests/test_modeling_common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 9bf351473074..1fe8f043dc0b 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -816,7 +816,10 @@ def recursive_check(batched_object, single_row_object, model_name, key): ), ) + set_model_tester_for_less_flaky_test(self) + config, batched_input = self.model_tester.prepare_config_and_inputs_for_common() + set_config_for_less_flaky_test(config) equivalence = get_tensor_equivalence_function(batched_input) for model_class in self.all_model_classes: @@ -827,6 +830,7 @@ def recursive_check(batched_object, single_row_object, model_name, key): config, batched_input = self.model_tester.prepare_config_and_inputs_for_model_class(model_class) batched_input_prepared = self._prepare_for_class(batched_input, model_class) model = model_class(config).to(torch_device).eval() + set_model_for_less_flaky_test(model) batch_size = self.model_tester.batch_size single_row_input = {} From 4a92e8aab0f8e06fa4e180c60e001081f720fb43 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 17:31:54 +0100 Subject: [PATCH 2/8] oh no!!! --- src/transformers/testing_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 6890a5368894..c80752a22217 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1421,6 +1421,11 @@ def set_model_tester_for_less_flaky_test(test_case): test_case.model_tester.text_config = copy.deepcopy(test_case.model_tester.text_config) test_case.model_tester.text_config["num_hidden_layers"] = 1 + # A few model class specific handling + + # For Albert + if hasattr(test_case.model_tester, "num_hidden_groups"): + test_case.model_tester.num_hidden_groups = test_case.model_tester.num_hidden_layers def set_config_for_less_flaky_test(config): target_attrs = [ From b87a811cf98919d0e3f155b222e5577629c9a9ea Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 17:40:12 +0100 Subject: [PATCH 3/8] oh no!!! --- src/transformers/testing_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index c80752a22217..8cf11d21a9ae 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1426,6 +1426,8 @@ def set_model_tester_for_less_flaky_test(test_case): # For Albert if hasattr(test_case.model_tester, "num_hidden_groups"): test_case.model_tester.num_hidden_groups = test_case.model_tester.num_hidden_layers + if hasattr(test_case.model_tester, "neck_hidden_sizes"): + test_case.model_tester.neck_hidden_sizes = test_case.model_tester.neck_hidden_sizes[:test_case.model_tester.num_hidden_layers] def set_config_for_less_flaky_test(config): target_attrs = [ From 9120a53189216b164260c96eaed2750ef5202de1 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 17:41:58 +0100 Subject: [PATCH 4/8] style --- src/transformers/testing_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 8cf11d21a9ae..c818595e49fc 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1427,7 +1427,10 @@ def set_model_tester_for_less_flaky_test(test_case): if hasattr(test_case.model_tester, "num_hidden_groups"): test_case.model_tester.num_hidden_groups = test_case.model_tester.num_hidden_layers if hasattr(test_case.model_tester, "neck_hidden_sizes"): - test_case.model_tester.neck_hidden_sizes = test_case.model_tester.neck_hidden_sizes[:test_case.model_tester.num_hidden_layers] + test_case.model_tester.neck_hidden_sizes = test_case.model_tester.neck_hidden_sizes[ + : test_case.model_tester.num_hidden_layers + ] + def set_config_for_less_flaky_test(config): target_attrs = [ From 2cd007b34bd3fccd62c7ff6b97325afbacab587d Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 17:55:58 +0100 Subject: [PATCH 5/8] oh no!!! --- src/transformers/testing_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index c818595e49fc..3519ec82f5ef 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1426,10 +1426,16 @@ def set_model_tester_for_less_flaky_test(test_case): # For Albert if hasattr(test_case.model_tester, "num_hidden_groups"): test_case.model_tester.num_hidden_groups = test_case.model_tester.num_hidden_layers + # For DPT if hasattr(test_case.model_tester, "neck_hidden_sizes"): test_case.model_tester.neck_hidden_sizes = test_case.model_tester.neck_hidden_sizes[ : test_case.model_tester.num_hidden_layers ] + # For DPT auto backbone + if hasattr(test_case.model_tester, "out_features"): + test_case.model_tester.out_features = test_case.model_tester.out_features[ + : test_case.model_tester.num_hidden_layers + ] def set_config_for_less_flaky_test(config): From de7664dc8fe1ba3d68f67a685af7d25f0b8159f5 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 18:36:11 +0100 Subject: [PATCH 6/8] oh no!!! --- src/transformers/testing_utils.py | 9 +++++---- tests/models/upernet/test_modeling_upernet.py | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 3519ec82f5ef..ec25e5b11774 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1432,10 +1432,11 @@ def set_model_tester_for_less_flaky_test(test_case): : test_case.model_tester.num_hidden_layers ] # For DPT auto backbone - if hasattr(test_case.model_tester, "out_features"): - test_case.model_tester.out_features = test_case.model_tester.out_features[ - : test_case.model_tester.num_hidden_layers - ] + if hasattr(test_case.model_tester, "num_hidden_layers"): + if hasattr(test_case.model_tester, "out_features"): + test_case.model_tester.out_features = test_case.model_tester.out_features[ + : test_case.model_tester.num_hidden_layers + ] def set_config_for_less_flaky_test(config): diff --git a/tests/models/upernet/test_modeling_upernet.py b/tests/models/upernet/test_modeling_upernet.py index 464061915e8b..43146a477931 100644 --- a/tests/models/upernet/test_modeling_upernet.py +++ b/tests/models/upernet/test_modeling_upernet.py @@ -82,7 +82,6 @@ def __init__( self.out_features = out_features self.num_labels = num_labels self.scope = scope - self.num_hidden_layers = num_stages def prepare_config_and_inputs(self): pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) From 1b42ba18b7f9e991e18fcd2786933aeaca715f71 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 19:05:33 +0100 Subject: [PATCH 7/8] oh no!!! --- src/transformers/testing_utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index ec25e5b11774..3b203e55d91f 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1410,10 +1410,12 @@ def assert_screenout(out, what): def set_model_tester_for_less_flaky_test(test_case): if hasattr(test_case.model_tester, "num_hidden_layers"): - test_case.model_tester.num_hidden_layers = 1 + if not (hasattr(test_case.model_tester, "out_features") or hasattr(test_case.model_tester, "out_indices")): + test_case.model_tester.num_hidden_layers = 1 if ( hasattr(test_case.model_tester, "vision_config") and "num_hidden_layers" in test_case.model_tester.vision_config + and (not (hasattr(test_case.model_tester.vision_config, "out_features") or hasattr(test_case.model_tester.vision_config, "out_indices"))) ): test_case.model_tester.vision_config = copy.deepcopy(test_case.model_tester.vision_config) test_case.model_tester.vision_config["num_hidden_layers"] = 1 @@ -1431,12 +1433,6 @@ def set_model_tester_for_less_flaky_test(test_case): test_case.model_tester.neck_hidden_sizes = test_case.model_tester.neck_hidden_sizes[ : test_case.model_tester.num_hidden_layers ] - # For DPT auto backbone - if hasattr(test_case.model_tester, "num_hidden_layers"): - if hasattr(test_case.model_tester, "out_features"): - test_case.model_tester.out_features = test_case.model_tester.out_features[ - : test_case.model_tester.num_hidden_layers - ] def set_config_for_less_flaky_test(config): From 9eb2ea6f83db3a1d7fa35f36eec7319d973163ed Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 8 Jan 2025 19:37:59 +0100 Subject: [PATCH 8/8] oh no!!! --- src/transformers/testing_utils.py | 36 ++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 3b203e55d91f..7876b22a2bb9 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1409,30 +1409,42 @@ def assert_screenout(out, what): def set_model_tester_for_less_flaky_test(test_case): - if hasattr(test_case.model_tester, "num_hidden_layers"): - if not (hasattr(test_case.model_tester, "out_features") or hasattr(test_case.model_tester, "out_indices")): - test_case.model_tester.num_hidden_layers = 1 + target_num_hidden_layers = 1 + # TODO (if possible): Avoid exceptional cases + exceptional_classes = [ + "ZambaModelTester", + "RwkvModelTester", + "AriaVisionText2TextModelTester", + "GPTNeoModelTester", + "DPTModelTester", + ] + if test_case.model_tester.__class__.__name__ in exceptional_classes: + target_num_hidden_layers = None + if hasattr(test_case.model_tester, "out_features") or hasattr(test_case.model_tester, "out_indices"): + target_num_hidden_layers = None + + if hasattr(test_case.model_tester, "num_hidden_layers") and target_num_hidden_layers is not None: + test_case.model_tester.num_hidden_layers = target_num_hidden_layers if ( hasattr(test_case.model_tester, "vision_config") and "num_hidden_layers" in test_case.model_tester.vision_config - and (not (hasattr(test_case.model_tester.vision_config, "out_features") or hasattr(test_case.model_tester.vision_config, "out_indices"))) + and target_num_hidden_layers is not None ): test_case.model_tester.vision_config = copy.deepcopy(test_case.model_tester.vision_config) - test_case.model_tester.vision_config["num_hidden_layers"] = 1 - if hasattr(test_case.model_tester, "text_config") and "num_hidden_layers" in test_case.model_tester.text_config: + test_case.model_tester.vision_config["num_hidden_layers"] = target_num_hidden_layers + if ( + hasattr(test_case.model_tester, "text_config") + and "num_hidden_layers" in test_case.model_tester.text_config + and target_num_hidden_layers is not None + ): test_case.model_tester.text_config = copy.deepcopy(test_case.model_tester.text_config) - test_case.model_tester.text_config["num_hidden_layers"] = 1 + test_case.model_tester.text_config["num_hidden_layers"] = target_num_hidden_layers # A few model class specific handling # For Albert if hasattr(test_case.model_tester, "num_hidden_groups"): test_case.model_tester.num_hidden_groups = test_case.model_tester.num_hidden_layers - # For DPT - if hasattr(test_case.model_tester, "neck_hidden_sizes"): - test_case.model_tester.neck_hidden_sizes = test_case.model_tester.neck_hidden_sizes[ - : test_case.model_tester.num_hidden_layers - ] def set_config_for_less_flaky_test(config):