From cc20eb8ebdbb44175cb2587fff9876f23ca73e34 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Wed, 1 Feb 2023 09:51:50 +0100 Subject: [PATCH 01/10] First draft --- .../models/detr/configuration_detr.py | 27 ++++++++++++++++++- .../models/detr/convert_detr_to_pytorch.py | 25 ++++++++++++----- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/detr/configuration_detr.py b/src/transformers/models/detr/configuration_detr.py index 430efc913b37..6678f7144877 100644 --- a/src/transformers/models/detr/configuration_detr.py +++ b/src/transformers/models/detr/configuration_detr.py @@ -14,8 +14,9 @@ # limitations under the License. """ DETR model configuration""" +import copy from collections import OrderedDict -from typing import Mapping +from typing import Dict, Mapping from packaging import version @@ -233,6 +234,30 @@ def num_attention_heads(self) -> int: def hidden_size(self) -> int: return self.d_model + @classmethod + def from_backbone_config(cls, backbone_config: PretrainedConfig, **kwargs): + """Instantiate a [`DetrConfig`] (or a derived class) from a pre-trained backbone model configuration. + Args: + backbone_config ([`PretrainedConfig`]): + The backbone configuration. + Returns: + [`Mask2FormerConfig`]: An instance of a configuration object + """ + return cls( + backbone_config=backbone_config, + **kwargs, + ) + + def to_dict(self) -> Dict[str, any]: + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output["backbone_config"] = self.backbone_config.to_dict() + output["model_type"] = self.__class__.model_type + return output + class DetrOnnxConfig(OnnxConfig): torch_onnx_minimum_version = version.parse("1.11") diff --git a/src/transformers/models/detr/convert_detr_to_pytorch.py b/src/transformers/models/detr/convert_detr_to_pytorch.py index 3ff2e38ac383..85c87928f173 100644 --- a/src/transformers/models/detr/convert_detr_to_pytorch.py +++ b/src/transformers/models/detr/convert_detr_to_pytorch.py @@ -33,12 +33,12 @@ def get_detr_config(model_name): - config = DetrConfig(use_timm_backbone=False) + config = DetrConfig(use_timm_backbone=False, backbone=None, dilation=None) # set backbone attributes - if "resnet50" in model_name: + if "resnet-50" in model_name: pass - elif "resnet101" in model_name: + elif "resnet-101" in model_name: config.backbone_config = ResNetConfig.from_pretrained("microsoft/resnet-101") else: raise ValueError("Model name should include either resnet50 or resnet101") @@ -286,7 +286,7 @@ def prepare_img(): @torch.no_grad() -def convert_detr_checkpoint(model_name, pytorch_dump_folder_path): +def convert_detr_checkpoint(model_name, pytorch_dump_folder_path=None, push_to_hub=False): """ Copy/paste/tweak model's weights to our DETR structure. """ @@ -295,8 +295,12 @@ def convert_detr_checkpoint(model_name, pytorch_dump_folder_path): config, is_panoptic = get_detr_config(model_name) # load original model from torch hub + model_name_to_original_name = { + "detr-resnet-50": "detr_resnet50", + "detr-resnet-101": "detr_resnet101", + } logger.info(f"Converting model {model_name}...") - detr = torch.hub.load("facebookresearch/detr", model_name, pretrained=True).eval() + detr = torch.hub.load("facebookresearch/detr", model_name_to_original_name[model_name], pretrained=True).eval() state_dict = detr.state_dict() # rename keys for src, dest in create_rename_keys(config): @@ -360,15 +364,22 @@ def convert_detr_checkpoint(model_name, pytorch_dump_folder_path): model.save_pretrained(pytorch_dump_folder_path) processor.save_pretrained(pytorch_dump_folder_path) + if push_to_hub: + # Upload model and image processor to the hub + logger.info("Uploading PyTorch model and image processor to the hub...") + model.push_to_hub(f"nielsr/{model_name}") + processor.push_to_hub(f"nielsr/{model_name}") + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( - "--model_name", default="detr_resnet50", type=str, help="Name of the DETR model you'd like to convert." + "--model_name", default="detr-resnet-50", type=str, help="Name of the DETR model you'd like to convert." ) parser.add_argument( "--pytorch_dump_folder_path", default=None, type=str, help="Path to the folder to output PyTorch model." ) + parser.add_argument("--push_to_hub", action="store_true", help="Whether to push the model to the hub or not.") args = parser.parse_args() - convert_detr_checkpoint(args.model_name, args.pytorch_dump_folder_path) + convert_detr_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub) From 3d6ac0443b94e40a32dd917c041e80202f5c0fb4 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Thu, 2 Feb 2023 14:09:32 +0100 Subject: [PATCH 02/10] Fix to_dict --- src/transformers/models/detr/configuration_detr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/detr/configuration_detr.py b/src/transformers/models/detr/configuration_detr.py index 6678f7144877..4edf79458a24 100644 --- a/src/transformers/models/detr/configuration_detr.py +++ b/src/transformers/models/detr/configuration_detr.py @@ -254,7 +254,8 @@ def to_dict(self) -> Dict[str, any]: `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, """ output = copy.deepcopy(self.__dict__) - output["backbone_config"] = self.backbone_config.to_dict() + if output["backbone_config"] is not None: + output["backbone_config"] = self.backbone_config.to_dict() output["model_type"] = self.__class__.model_type return output From 9cee8d378ec882ed477332fcf45b3230baaff083 Mon Sep 17 00:00:00 2001 From: Niels Date: Sat, 25 Feb 2023 18:34:43 +0100 Subject: [PATCH 03/10] Improve conversion script --- src/transformers/models/detr/convert_detr_to_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/detr/convert_detr_to_pytorch.py b/src/transformers/models/detr/convert_detr_to_pytorch.py index 85c87928f173..9f322404505b 100644 --- a/src/transformers/models/detr/convert_detr_to_pytorch.py +++ b/src/transformers/models/detr/convert_detr_to_pytorch.py @@ -33,7 +33,7 @@ def get_detr_config(model_name): - config = DetrConfig(use_timm_backbone=False, backbone=None, dilation=None) + config = DetrConfig(use_timm_backbone=False, backbone=None, dilation=None, use_pretrained_backbone=None) # set backbone attributes if "resnet-50" in model_name: From 5364802cb7edc097a90f04b01afcd8f3a14c13ae Mon Sep 17 00:00:00 2001 From: Niels Date: Mon, 27 Feb 2023 11:10:05 +0100 Subject: [PATCH 04/10] Update config --- src/transformers/models/detr/configuration_detr.py | 2 ++ .../models/detr/convert_detr_to_pytorch.py | 10 +++++----- .../configuration_table_transformer.py | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/detr/configuration_detr.py b/src/transformers/models/detr/configuration_detr.py index 4edf79458a24..15c9a635787f 100644 --- a/src/transformers/models/detr/configuration_detr.py +++ b/src/transformers/models/detr/configuration_detr.py @@ -188,6 +188,8 @@ def __init__( backbone_model_type = backbone_config.get("model_type") config_class = CONFIG_MAPPING[backbone_model_type] backbone_config = config_class.from_dict(backbone_config) + # set timm attributes to None + dilation, backbone, use_pretrained_backbone = None, None, None self.use_timm_backbone = use_timm_backbone self.backbone_config = backbone_config diff --git a/src/transformers/models/detr/convert_detr_to_pytorch.py b/src/transformers/models/detr/convert_detr_to_pytorch.py index 9f322404505b..fe9290f4d429 100644 --- a/src/transformers/models/detr/convert_detr_to_pytorch.py +++ b/src/transformers/models/detr/convert_detr_to_pytorch.py @@ -33,16 +33,16 @@ def get_detr_config(model_name): - config = DetrConfig(use_timm_backbone=False, backbone=None, dilation=None, use_pretrained_backbone=None) - - # set backbone attributes + # initialize config if "resnet-50" in model_name: - pass + backbone_config = ResNetConfig.from_pretrained("microsoft/resnet-50") elif "resnet-101" in model_name: - config.backbone_config = ResNetConfig.from_pretrained("microsoft/resnet-101") + backbone_config = ResNetConfig.from_pretrained("microsoft/resnet-101") else: raise ValueError("Model name should include either resnet50 or resnet101") + config = DetrConfig(use_timm_backbone=False, backbone_config=backbone_config) + # set label attributes is_panoptic = "panoptic" in model_name if is_panoptic: diff --git a/src/transformers/models/table_transformer/configuration_table_transformer.py b/src/transformers/models/table_transformer/configuration_table_transformer.py index d74424ce69fe..94213008c6b6 100644 --- a/src/transformers/models/table_transformer/configuration_table_transformer.py +++ b/src/transformers/models/table_transformer/configuration_table_transformer.py @@ -189,6 +189,8 @@ def __init__( backbone_model_type = backbone_config.get("model_type") config_class = CONFIG_MAPPING[backbone_model_type] backbone_config = config_class.from_dict(backbone_config) + # set timm attributes to None + dilation, backbone, use_pretrained_backbone = None, None, None self.use_timm_backbone = use_timm_backbone self.backbone_config = backbone_config From bc1baf3ffae46366a71261aac584afa22bb3dda1 Mon Sep 17 00:00:00 2001 From: Niels Date: Mon, 27 Feb 2023 14:23:18 +0100 Subject: [PATCH 05/10] Remove timm dependency --- src/transformers/__init__.py | 32 +++++++++---------- src/transformers/models/detr/__init__.py | 6 ++-- src/transformers/utils/dummy_pt_objects.py | 31 ++++++++++++++++++ .../utils/dummy_timm_and_vision_objects.py | 31 ------------------ 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index e721e01016b3..80a51656b755 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -885,15 +885,6 @@ "DeformableDetrPreTrainedModel", ] ) - _import_structure["models.detr"].extend( - [ - "DETR_PRETRAINED_MODEL_ARCHIVE_LIST", - "DetrForObjectDetection", - "DetrForSegmentation", - "DetrModel", - "DetrPreTrainedModel", - ] - ) _import_structure["models.table_transformer"].extend( [ "TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -1424,6 +1415,15 @@ "DetaPreTrainedModel", ] ) + _import_structure["models.detr"].extend( + [ + "DETR_PRETRAINED_MODEL_ARCHIVE_LIST", + "DetrForObjectDetection", + "DetrForSegmentation", + "DetrModel", + "DetrPreTrainedModel", + ] + ) _import_structure["models.dinat"].extend( [ "DINAT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -4417,13 +4417,6 @@ DeformableDetrModel, DeformableDetrPreTrainedModel, ) - from .models.detr import ( - DETR_PRETRAINED_MODEL_ARCHIVE_LIST, - DetrForObjectDetection, - DetrForSegmentation, - DetrModel, - DetrPreTrainedModel, - ) from .models.table_transformer import ( TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST, TableTransformerForObjectDetection, @@ -4862,6 +4855,13 @@ DetaModel, DetaPreTrainedModel, ) + from .models.detr import ( + DETR_PRETRAINED_MODEL_ARCHIVE_LIST, + DetrForObjectDetection, + DetrForSegmentation, + DetrModel, + DetrPreTrainedModel, + ) from .models.dinat import ( DINAT_PRETRAINED_MODEL_ARCHIVE_LIST, DinatBackbone, diff --git a/src/transformers/models/detr/__init__.py b/src/transformers/models/detr/__init__.py index 1dcda4cc171b..9cbaca9a5458 100644 --- a/src/transformers/models/detr/__init__.py +++ b/src/transformers/models/detr/__init__.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING -from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_timm_available, is_vision_available +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available _import_structure = {"configuration_detr": ["DETR_PRETRAINED_CONFIG_ARCHIVE_MAP", "DetrConfig", "DetrOnnxConfig"]} @@ -29,7 +29,7 @@ _import_structure["image_processing_detr"] = ["DetrImageProcessor"] try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass @@ -56,7 +56,7 @@ from .image_processing_detr import DetrImageProcessor try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index d06bf1aea03a..ba70d907f4b2 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -2135,6 +2135,37 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +DETR_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class DetrForObjectDetection(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class DetrForSegmentation(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class DetrModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class DetrPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + DINAT_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/src/transformers/utils/dummy_timm_and_vision_objects.py b/src/transformers/utils/dummy_timm_and_vision_objects.py index b4a28babfca0..1aef43e5cd1b 100644 --- a/src/transformers/utils/dummy_timm_and_vision_objects.py +++ b/src/transformers/utils/dummy_timm_and_vision_objects.py @@ -57,37 +57,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["timm", "vision"]) -DETR_PRETRAINED_MODEL_ARCHIVE_LIST = None - - -class DetrForObjectDetection(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class DetrForSegmentation(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class DetrModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class DetrPreTrainedModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = None From fff26977a6aad9f926b2d0bc47c3a33a84f8683f Mon Sep 17 00:00:00 2001 From: Niels Date: Mon, 27 Feb 2023 14:47:56 +0100 Subject: [PATCH 06/10] Fix dummies --- src/transformers/__init__.py | 107 +++++++----------- .../models/conditional_detr/__init__.py | 6 +- .../models/deformable_detr/__init__.py | 6 +- .../models/table_transformer/__init__.py | 6 +- src/transformers/utils/dummy_pt_objects.py | 79 +++++++++++++ 5 files changed, 132 insertions(+), 72 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 80a51656b755..c75c9b195f91 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -866,43 +866,6 @@ _import_structure["models.vit_hybrid"].extend(["ViTHybridImageProcessor"]) _import_structure["models.yolos"].extend(["YolosFeatureExtractor", "YolosImageProcessor"]) -# Timm-backed objects -try: - if not (is_timm_available() and is_vision_available()): - raise OptionalDependencyNotAvailable() -except OptionalDependencyNotAvailable: - from .utils import dummy_timm_and_vision_objects - - _import_structure["utils.dummy_timm_and_vision_objects"] = [ - name for name in dir(dummy_timm_and_vision_objects) if not name.startswith("_") - ] -else: - _import_structure["models.deformable_detr"].extend( - [ - "DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST", - "DeformableDetrForObjectDetection", - "DeformableDetrModel", - "DeformableDetrPreTrainedModel", - ] - ) - _import_structure["models.table_transformer"].extend( - [ - "TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST", - "TableTransformerForObjectDetection", - "TableTransformerModel", - "TableTransformerPreTrainedModel", - ] - ) - _import_structure["models.conditional_detr"].extend( - [ - "CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST", - "ConditionalDetrForObjectDetection", - "ConditionalDetrForSegmentation", - "ConditionalDetrModel", - "ConditionalDetrPreTrainedModel", - ] - ) - # PyTorch-backed objects try: @@ -1300,6 +1263,15 @@ "CodeGenPreTrainedModel", ] ) + _import_structure["models.conditional_detr"].extend( + [ + "CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST", + "ConditionalDetrForObjectDetection", + "ConditionalDetrForSegmentation", + "ConditionalDetrModel", + "ConditionalDetrPreTrainedModel", + ] + ) _import_structure["models.convbert"].extend( [ "CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -1397,6 +1369,14 @@ "DecisionTransformerPreTrainedModel", ] ) + _import_structure["models.deformable_detr"].extend( + [ + "DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST", + "DeformableDetrForObjectDetection", + "DeformableDetrModel", + "DeformableDetrPreTrainedModel", + ] + ) _import_structure["models.deit"].extend( [ "DEIT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -2372,6 +2352,14 @@ "load_tf_weights_in_t5", ] ) + _import_structure["models.table_transformer"].extend( + [ + "TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST", + "TableTransformerForObjectDetection", + "TableTransformerModel", + "TableTransformerPreTrainedModel", + ] + ) _import_structure["models.tapas"].extend( [ "TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -4398,32 +4386,6 @@ from .models.yolos import YolosFeatureExtractor, YolosImageProcessor # Modeling - try: - if not (is_timm_available() and is_vision_available()): - raise OptionalDependencyNotAvailable() - except OptionalDependencyNotAvailable: - from .utils.dummy_timm_and_vision_objects import * - else: - from .models.conditional_detr import ( - CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST, - ConditionalDetrForObjectDetection, - ConditionalDetrForSegmentation, - ConditionalDetrModel, - ConditionalDetrPreTrainedModel, - ) - from .models.deformable_detr import ( - DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST, - DeformableDetrForObjectDetection, - DeformableDetrModel, - DeformableDetrPreTrainedModel, - ) - from .models.table_transformer import ( - TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST, - TableTransformerForObjectDetection, - TableTransformerModel, - TableTransformerPreTrainedModel, - ) - try: if not is_torch_available(): raise OptionalDependencyNotAvailable() @@ -4760,6 +4722,13 @@ CodeGenModel, CodeGenPreTrainedModel, ) + from .models.conditional_detr import ( + CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST, + ConditionalDetrForObjectDetection, + ConditionalDetrForSegmentation, + ConditionalDetrModel, + ConditionalDetrPreTrainedModel, + ) from .models.convbert import ( CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST, ConvBertForMaskedLM, @@ -4841,6 +4810,12 @@ DecisionTransformerModel, DecisionTransformerPreTrainedModel, ) + from .models.deformable_detr import ( + DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST, + DeformableDetrForObjectDetection, + DeformableDetrModel, + DeformableDetrPreTrainedModel, + ) from .models.deit import ( DEIT_PRETRAINED_MODEL_ARCHIVE_LIST, DeiTForImageClassification, @@ -5626,6 +5601,12 @@ T5PreTrainedModel, load_tf_weights_in_t5, ) + from .models.table_transformer import ( + TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST, + TableTransformerForObjectDetection, + TableTransformerModel, + TableTransformerPreTrainedModel, + ) from .models.tapas import ( TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST, TapasForMaskedLM, diff --git a/src/transformers/models/conditional_detr/__init__.py b/src/transformers/models/conditional_detr/__init__.py index da5ed8f2f8ed..565323321160 100644 --- a/src/transformers/models/conditional_detr/__init__.py +++ b/src/transformers/models/conditional_detr/__init__.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING -from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_timm_available, is_vision_available +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available _import_structure = { @@ -35,7 +35,7 @@ _import_structure["image_processing_conditional_detr"] = ["ConditionalDetrImageProcessor"] try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass @@ -66,7 +66,7 @@ from .image_processing_conditional_detr import ConditionalDetrImageProcessor try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass diff --git a/src/transformers/models/deformable_detr/__init__.py b/src/transformers/models/deformable_detr/__init__.py index 6614bc5f9261..a560265f4bfc 100644 --- a/src/transformers/models/deformable_detr/__init__.py +++ b/src/transformers/models/deformable_detr/__init__.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING -from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_timm_available, is_vision_available +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available _import_structure = { @@ -31,7 +31,7 @@ _import_structure["image_processing_deformable_detr"] = ["DeformableDetrImageProcessor"] try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass @@ -57,7 +57,7 @@ from .image_processing_deformable_detr import DeformableDetrImageProcessor try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass diff --git a/src/transformers/models/table_transformer/__init__.py b/src/transformers/models/table_transformer/__init__.py index bacb6a810a1c..346bc9ef9caa 100644 --- a/src/transformers/models/table_transformer/__init__.py +++ b/src/transformers/models/table_transformer/__init__.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING -from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_timm_available +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available _import_structure = { @@ -26,7 +26,7 @@ } try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass @@ -47,7 +47,7 @@ ) try: - if not is_timm_available(): + if not is_torch_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: pass diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index ba70d907f4b2..5212ea250a02 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1661,6 +1661,37 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class ConditionalDetrForObjectDetection(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class ConditionalDetrForSegmentation(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class ConditionalDetrModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class ConditionalDetrPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None @@ -2073,6 +2104,30 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class DeformableDetrForObjectDetection(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class DeformableDetrModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class DeformableDetrPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + DEIT_PRETRAINED_MODEL_ARCHIVE_LIST = None @@ -6071,6 +6126,30 @@ def load_tf_weights_in_t5(*args, **kwargs): requires_backends(load_tf_weights_in_t5, ["torch"]) +TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class TableTransformerForObjectDetection(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class TableTransformerModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class TableTransformerPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = None From d4035672d73733baace4c35f810b9831a2582405 Mon Sep 17 00:00:00 2001 From: Niels Date: Thu, 2 Mar 2023 14:58:53 +0100 Subject: [PATCH 07/10] Fix typo, add integration test --- .../image_processing_conditional_detr.py | 4 +-- .../image_processing_deformable_detr.py | 4 +-- .../models/detr/image_processing_detr.py | 4 +-- tests/models/detr/test_modeling_detr.py | 33 +++++++++++++++++-- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index cc4a24cb05dd..a496f7787fb3 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -1101,12 +1101,12 @@ def preprocess( images (`ImageInput`): Image or batch of images to preprocess. annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): - List of annotations associated with the image or batch of images. If annotionation is for object + List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: - "image_id" (`int`): The image id. - "annotations" (`List[Dict]`): List of annotations for an image. Each annotation should be a dictionary. An image can have no annotations, in which case the list should be empty. - If annotionation is for segmentation, the annotations should be a dictionary with the following keys: + If annotation is for segmentation, the annotations should be a dictionary with the following keys: - "image_id" (`int`): The image id. - "segments_info" (`List[Dict]`): List of segments for an image. Each segment should be a dictionary. An image can have no segments, in which case the list should be empty. diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index 87e6ef508c6d..01ff54471904 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -1099,12 +1099,12 @@ def preprocess( images (`ImageInput`): Image or batch of images to preprocess. annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): - List of annotations associated with the image or batch of images. If annotionation is for object + List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: - "image_id" (`int`): The image id. - "annotations" (`List[Dict]`): List of annotations for an image. Each annotation should be a dictionary. An image can have no annotations, in which case the list should be empty. - If annotionation is for segmentation, the annotations should be a dictionary with the following keys: + If annotation is for segmentation, the annotations should be a dictionary with the following keys: - "image_id" (`int`): The image id. - "segments_info" (`List[Dict]`): List of segments for an image. Each segment should be a dictionary. An image can have no segments, in which case the list should be empty. diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index 6a0ace45b851..eaeae66c9654 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -1065,12 +1065,12 @@ def preprocess( images (`ImageInput`): Image or batch of images to preprocess. annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): - List of annotations associated with the image or batch of images. If annotionation is for object + List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: - "image_id" (`int`): The image id. - "annotations" (`List[Dict]`): List of annotations for an image. Each annotation should be a dictionary. An image can have no annotations, in which case the list should be empty. - If annotionation is for segmentation, the annotations should be a dictionary with the following keys: + If annotation is for segmentation, the annotations should be a dictionary with the following keys: - "image_id" (`int`): The image id. - "segments_info" (`List[Dict]`): List of segments for an image. Each segment should be a dictionary. An image can have no segments, in which case the list should be empty. diff --git a/tests/models/detr/test_modeling_detr.py b/tests/models/detr/test_modeling_detr.py index 6c7c512cc830..052b268d6411 100644 --- a/tests/models/detr/test_modeling_detr.py +++ b/tests/models/detr/test_modeling_detr.py @@ -20,7 +20,7 @@ import unittest from transformers import DetrConfig, is_timm_available, is_vision_available -from transformers.testing_utils import require_timm, require_vision, slow, torch_device +from transformers.testing_utils import require_timm, require_torch, require_vision, slow, torch_device from transformers.utils import cached_property from ...generation.test_utils import GenerationTesterMixin @@ -510,7 +510,7 @@ def prepare_img(): @require_timm @require_vision @slow -class DetrModelIntegrationTests(unittest.TestCase): +class DetrModelIntegrationTestsTimmBackbone(unittest.TestCase): @cached_property def default_feature_extractor(self): return DetrFeatureExtractor.from_pretrained("facebook/detr-resnet-50") if is_vision_available() else None @@ -626,3 +626,32 @@ def test_inference_panoptic_segmentation_head(self): self.assertTrue(torch.allclose(results["segmentation"][:3, :3], expected_slice_segmentation, atol=1e-4)) self.assertTrue(len(results["segments_info"]), expected_number_of_segments) self.assertDictEqual(results["segments_info"][0], expected_first_segment) + +@require_vision +@require_torch +@slow +class DetrModelIntegrationTests(unittest.TestCase): + @cached_property + def default_feature_extractor(self): + return ( + DetrFeatureExtractor.from_pretrained("facebook/detr-resnet-50", revision="no_timm") + if is_vision_available() + else None + ) + + def test_inference_no_head(self): + model = DetrModel.from_pretrained("facebook/detr-resnet-50", revision="no_timm").to(torch_device) + + feature_extractor = self.default_feature_extractor + image = prepare_img() + encoding = feature_extractor(images=image, return_tensors="pt").to(torch_device) + + with torch.no_grad(): + outputs = model(**encoding) + + expected_shape = torch.Size((1, 100, 256)) + assert outputs.last_hidden_state.shape == expected_shape + expected_slice = torch.tensor( + [[0.0616, -0.5146, -0.4032], [-0.7629, -0.4934, -1.7153], [-0.4768, -0.6403, -0.7826]] + ).to(torch_device) + self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=1e-4)) From 665c93d63710be36c9e1933b8ed652f26285d9bd Mon Sep 17 00:00:00 2001 From: Niels Date: Fri, 3 Mar 2023 09:06:59 +0100 Subject: [PATCH 08/10] Upload 101 model as well --- .../models/detr/convert_detr_to_pytorch.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/detr/convert_detr_to_pytorch.py b/src/transformers/models/detr/convert_detr_to_pytorch.py index fe9290f4d429..a52e592b945d 100644 --- a/src/transformers/models/detr/convert_detr_to_pytorch.py +++ b/src/transformers/models/detr/convert_detr_to_pytorch.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2022 The HuggingFace Inc. team. +# Copyright 2023 The HuggingFace Inc. team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -348,9 +348,6 @@ def convert_detr_checkpoint(model_name, pytorch_dump_folder_path=None, push_to_h original_outputs = detr(pixel_values) outputs = model(pixel_values) - print("Logits:", outputs.logits[0, :3, :3]) - print("Original logits:", original_outputs["pred_logits"][0, :3, :3]) - assert torch.allclose(outputs.logits, original_outputs["pred_logits"], atol=1e-3) assert torch.allclose(outputs.pred_boxes, original_outputs["pred_boxes"], atol=1e-3) if is_panoptic: @@ -375,7 +372,11 @@ def convert_detr_checkpoint(model_name, pytorch_dump_folder_path=None, push_to_h parser = argparse.ArgumentParser() parser.add_argument( - "--model_name", default="detr-resnet-50", type=str, help="Name of the DETR model you'd like to convert." + "--model_name", + default="detr-resnet-50", + type=str, + choices=["detr-resnet-50", "detr-resnet-101"], + help="Name of the DETR model you'd like to convert.", ) parser.add_argument( "--pytorch_dump_folder_path", default=None, type=str, help="Path to the folder to output PyTorch model." From adbdc29661f73e1f066bdc8e3aa87328bd8fe4bb Mon Sep 17 00:00:00 2001 From: Niels Date: Mon, 6 Mar 2023 14:02:29 +0100 Subject: [PATCH 09/10] Remove timm dummies --- .../models/detr/configuration_detr.py | 7 +- .../utils/dummy_timm_and_vision_objects.py | 81 ------------------- 2 files changed, 2 insertions(+), 86 deletions(-) delete mode 100644 src/transformers/utils/dummy_timm_and_vision_objects.py diff --git a/src/transformers/models/detr/configuration_detr.py b/src/transformers/models/detr/configuration_detr.py index 15c9a635787f..b3da5f86b016 100644 --- a/src/transformers/models/detr/configuration_detr.py +++ b/src/transformers/models/detr/configuration_detr.py @@ -243,12 +243,9 @@ def from_backbone_config(cls, backbone_config: PretrainedConfig, **kwargs): backbone_config ([`PretrainedConfig`]): The backbone configuration. Returns: - [`Mask2FormerConfig`]: An instance of a configuration object + [`DetrConfig`]: An instance of a configuration object """ - return cls( - backbone_config=backbone_config, - **kwargs, - ) + return cls(backbone_config=backbone_config, **kwargs) def to_dict(self) -> Dict[str, any]: """ diff --git a/src/transformers/utils/dummy_timm_and_vision_objects.py b/src/transformers/utils/dummy_timm_and_vision_objects.py deleted file mode 100644 index 1aef43e5cd1b..000000000000 --- a/src/transformers/utils/dummy_timm_and_vision_objects.py +++ /dev/null @@ -1,81 +0,0 @@ -# This file is autogenerated by the command `make fix-copies`, do not edit. -from ..utils import DummyObject, requires_backends - - -CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST = None - - -class ConditionalDetrForObjectDetection(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class ConditionalDetrForSegmentation(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class ConditionalDetrModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class ConditionalDetrPreTrainedModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST = None - - -class DeformableDetrForObjectDetection(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class DeformableDetrModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class DeformableDetrPreTrainedModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = None - - -class TableTransformerForObjectDetection(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class TableTransformerModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) - - -class TableTransformerPreTrainedModel(metaclass=DummyObject): - _backends = ["timm", "vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["timm", "vision"]) From f981818e8d725da41c26747dad0b209d893f64d7 Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 7 Mar 2023 20:29:43 +0100 Subject: [PATCH 10/10] Fix style --- tests/models/detr/test_modeling_detr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models/detr/test_modeling_detr.py b/tests/models/detr/test_modeling_detr.py index 052b268d6411..31d3db444ca0 100644 --- a/tests/models/detr/test_modeling_detr.py +++ b/tests/models/detr/test_modeling_detr.py @@ -627,6 +627,7 @@ def test_inference_panoptic_segmentation_head(self): self.assertTrue(len(results["segments_info"]), expected_number_of_segments) self.assertDictEqual(results["segments_info"][0], expected_first_segment) + @require_vision @require_torch @slow