Skip to content

[DETR and friends] Remove is_timm_available #21814

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sgugger merged 10 commits into from
Mar 7, 2023
Merged

[DETR and friends] Remove is_timm_available #21814

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
cc20eb8
First draft
Feb 1, 2023
3d6ac04
Fix to_dict
Feb 2, 2023
9cee8d3
Improve conversion script
NielsRogge Feb 25, 2023
5364802
Update config
NielsRogge Feb 27, 2023
bc1baf3
Remove timm dependency
NielsRogge Feb 27, 2023
fff2697
Fix dummies
NielsRogge Feb 27, 2023
d403567
Fix typo, add integration test
NielsRogge Mar 2, 2023
665c93d
Upload 101 model as well
NielsRogge Mar 3, 2023
adbdc29
Remove timm dummies
NielsRogge Mar 6, 2023
f981818
Fix style
NielsRogge Mar 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 60 additions & 79 deletions src/transformers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -866,52 +866,6 @@
_import_structure["models.vit_hybrid"].extend(["ViTHybridImageProcessor"])
_import_structure["models.yolos"].extend(["YolosFeatureExtractor", "YolosImageProcessor"])

# Timm-backed objects
try:
if not (is_timm_available() and is_vision_available()):
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
from .utils import dummy_timm_and_vision_objects

_import_structure["utils.dummy_timm_and_vision_objects"] = [
name for name in dir(dummy_timm_and_vision_objects) if not name.startswith("_")
]
else:
_import_structure["models.deformable_detr"].extend(
[
"DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST",
"DeformableDetrForObjectDetection",
"DeformableDetrModel",
"DeformableDetrPreTrainedModel",
]
)
_import_structure["models.detr"].extend(
[
"DETR_PRETRAINED_MODEL_ARCHIVE_LIST",
"DetrForObjectDetection",
"DetrForSegmentation",
"DetrModel",
"DetrPreTrainedModel",
]
)
_import_structure["models.table_transformer"].extend(
[
"TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
"TableTransformerForObjectDetection",
"TableTransformerModel",
"TableTransformerPreTrainedModel",
]
)
_import_structure["models.conditional_detr"].extend(
[
"CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST",
"ConditionalDetrForObjectDetection",
"ConditionalDetrForSegmentation",
"ConditionalDetrModel",
"ConditionalDetrPreTrainedModel",
]
)


# PyTorch-backed objects
try:
Expand Down Expand Up @@ -1309,6 +1263,15 @@
"CodeGenPreTrainedModel",
]
)
_import_structure["models.conditional_detr"].extend(
[
"CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST",
"ConditionalDetrForObjectDetection",
"ConditionalDetrForSegmentation",
"ConditionalDetrModel",
"ConditionalDetrPreTrainedModel",
]
)
_import_structure["models.convbert"].extend(
[
"CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
Expand Down Expand Up @@ -1406,6 +1369,14 @@
"DecisionTransformerPreTrainedModel",
]
)
_import_structure["models.deformable_detr"].extend(
[
"DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST",
"DeformableDetrForObjectDetection",
"DeformableDetrModel",
"DeformableDetrPreTrainedModel",
]
)
_import_structure["models.deit"].extend(
[
"DEIT_PRETRAINED_MODEL_ARCHIVE_LIST",
Expand All @@ -1424,6 +1395,15 @@
"DetaPreTrainedModel",
]
)
_import_structure["models.detr"].extend(
[
"DETR_PRETRAINED_MODEL_ARCHIVE_LIST",
"DetrForObjectDetection",
"DetrForSegmentation",
"DetrModel",
"DetrPreTrainedModel",
]
)
_import_structure["models.dinat"].extend(
[
"DINAT_PRETRAINED_MODEL_ARCHIVE_LIST",
Expand Down Expand Up @@ -2372,6 +2352,14 @@
"load_tf_weights_in_t5",
]
)
_import_structure["models.table_transformer"].extend(
[
"TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
"TableTransformerForObjectDetection",
"TableTransformerModel",
"TableTransformerPreTrainedModel",
]
)
_import_structure["models.tapas"].extend(
[
"TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST",
Expand Down Expand Up @@ -4398,39 +4386,6 @@
from .models.yolos import YolosFeatureExtractor, YolosImageProcessor

# Modeling
try:
if not (is_timm_available() and is_vision_available()):
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
from .utils.dummy_timm_and_vision_objects import *
else:
from .models.conditional_detr import (
CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
ConditionalDetrForObjectDetection,
ConditionalDetrForSegmentation,
ConditionalDetrModel,
ConditionalDetrPreTrainedModel,
)
from .models.deformable_detr import (
DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
DeformableDetrForObjectDetection,
DeformableDetrModel,
DeformableDetrPreTrainedModel,
)
from .models.detr import (
DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
DetrForObjectDetection,
DetrForSegmentation,
DetrModel,
DetrPreTrainedModel,
)
from .models.table_transformer import (
TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
TableTransformerForObjectDetection,
TableTransformerModel,
TableTransformerPreTrainedModel,
)

try:
if not is_torch_available():
raise OptionalDependencyNotAvailable()
Expand Down Expand Up @@ -4767,6 +4722,13 @@
CodeGenModel,
CodeGenPreTrainedModel,
)
from .models.conditional_detr import (
CONDITIONAL_DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
ConditionalDetrForObjectDetection,
ConditionalDetrForSegmentation,
ConditionalDetrModel,
ConditionalDetrPreTrainedModel,
)
from .models.convbert import (
CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
ConvBertForMaskedLM,
Expand Down Expand Up @@ -4848,6 +4810,12 @@
DecisionTransformerModel,
DecisionTransformerPreTrainedModel,
)
from .models.deformable_detr import (
DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
DeformableDetrForObjectDetection,
DeformableDetrModel,
DeformableDetrPreTrainedModel,
)
from .models.deit import (
DEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
DeiTForImageClassification,
Expand All @@ -4862,6 +4830,13 @@
DetaModel,
DetaPreTrainedModel,
)
from .models.detr import (
DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
DetrForObjectDetection,
DetrForSegmentation,
DetrModel,
DetrPreTrainedModel,
)
from .models.dinat import (
DINAT_PRETRAINED_MODEL_ARCHIVE_LIST,
DinatBackbone,
Expand Down Expand Up @@ -5626,6 +5601,12 @@
T5PreTrainedModel,
load_tf_weights_in_t5,
)
from .models.table_transformer import (
TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
TableTransformerForObjectDetection,
TableTransformerModel,
TableTransformerPreTrainedModel,
)
from .models.tapas import (
TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
TapasForMaskedLM,
Expand Down
6 changes: 3 additions & 3 deletions src/transformers/models/conditional_detr/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_timm_available, is_vision_available
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_import_structure = {
Expand All @@ -35,7 +35,7 @@
_import_structure["image_processing_conditional_detr"] = ["ConditionalDetrImageProcessor"]

try:
if not is_timm_available():
if not is_torch_available():
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
pass
Expand Down Expand Up @@ -66,7 +66,7 @@
from .image_processing_conditional_detr import ConditionalDetrImageProcessor

try:
if not is_timm_available():
if not is_torch_available():
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
pass
Expand Down
4 changes: 2 additions & 2 deletions src/transformers/models/conditional_detr/image_processing_conditional_detr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1101,12 +1101,12 @@ def preprocess(
images (`ImageInput`):
Image or batch of images to preprocess.
annotations (`AnnotationType` or `List[AnnotationType]`, *optional*):
List of annotations associated with the image or batch of images. If annotionation is for object
List of annotations associated with the image or batch of images. If annotation is for object
detection, the annotations should be a dictionary with the following keys:
- "image_id" (`int`): The image id.
- "annotations" (`List[Dict]`): List of annotations for an image. Each annotation should be a
dictionary. An image can have no annotations, in which case the list should be empty.
If annotionation is for segmentation, the annotations should be a dictionary with the following keys:
If annotation is for segmentation, the annotations should be a dictionary with the following keys:
- "image_id" (`int`): The image id.
- "segments_info" (`List[Dict]`): List of segments for an image. Each segment should be a dictionary.
An image can have no segments, in which case the list should be empty.
Expand Down
6 changes: 3 additions & 3 deletions src/transformers/models/deformable_detr/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_timm_available, is_vision_available
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_import_structure = {
Expand All @@ -31,7 +31,7 @@
_import_structure["image_processing_deformable_detr"] = ["DeformableDetrImageProcessor"]

try:
if not is_timm_available():
if not is_torch_available():
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
pass
Expand All @@ -57,7 +57,7 @@
from .image_processing_deformable_detr import DeformableDetrImageProcessor

try:
if not is_timm_available():
if not is_torch_available():
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
pass
Expand Down
4 changes: 2 additions & 2 deletions src/transformers/models/deformable_detr/image_processing_deformable_detr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1099,12 +1099,12 @@ def preprocess(
images (`ImageInput`):
Image or batch of images to preprocess.
annotations (`AnnotationType` or `List[AnnotationType]`, *optional*):
List of annotations associated with the image or batch of images. If annotionation is for object
List of annotations associated with the image or batch of images. If annotation is for object
detection, the annotations should be a dictionary with the following keys:
- "image_id" (`int`): The image id.
- "annotations" (`List[Dict]`): List of annotations for an image. Each annotation should be a
dictionary. An image can have no annotations, in which case the list should be empty.
If annotionation is for segmentation, the annotations should be a dictionary with the following keys:
If annotation is for segmentation, the annotations should be a dictionary with the following keys:
- "image_id" (`int`): The image id.
- "segments_info" (`List[Dict]`): List of segments for an image. Each segment should be a dictionary.
An image can have no segments, in which case the list should be empty.
Expand Down
6 changes: 3 additions & 3 deletions src/transformers/models/detr/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_timm_available, is_vision_available
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_import_structure = {"configuration_detr": ["DETR_PRETRAINED_CONFIG_ARCHIVE_MAP", "DetrConfig", "DetrOnnxConfig"]}
Expand All @@ -29,7 +29,7 @@
_import_structure["image_processing_detr"] = ["DetrImageProcessor"]

try:
if not is_timm_available():
if not is_torch_available():
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
pass
Expand All @@ -56,7 +56,7 @@
from .image_processing_detr import DetrImageProcessor

try:
if not is_timm_available():
if not is_torch_available():
raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
pass
Expand Down
27 changes: 26 additions & 1 deletion src/transformers/models/detr/configuration_detr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
# limitations under the License.
""" DETR model configuration"""

import copy
from collections import OrderedDict
from typing import Mapping
from typing import Dict, Mapping

from packaging import version

Expand Down Expand Up @@ -187,6 +188,8 @@ def __init__(
backbone_model_type = backbone_config.get("model_type")
config_class = CONFIG_MAPPING[backbone_model_type]
backbone_config = config_class.from_dict(backbone_config)
# set timm attributes to None
dilation, backbone, use_pretrained_backbone = None, None, None
Copy link
Copy Markdown
Contributor

@amyeroberts amyeroberts Mar 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I realise this is already in the design and not introduced in this PR - but it feels a bit funny to have arguments which configure the timm checkpoint to be at the model level e.g. dilation, whereas for a transformers backbone they're contained within their own config.

Copy link
Copy Markdown
Collaborator Author

@NielsRogge NielsRogge Mar 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, however note that this is a legacy way of using timm models as backbone.

In the future, one should always leverage the AutoBackbone class for frameworks that make use of several backbones

Copy link
Copy Markdown
Contributor

@amyeroberts amyeroberts Mar 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To manage the different configuration options, I'd introduce additional argument checks and have defaults which are None to prevent clashes between timm and transformer backbone expected behaviour. As a user, I'd find it confusing to pass in use_pretrained_backbone=False and then have a pretrained backbone loaded.

i.e. something like this:

    def __init__(
        self,
        use_timm_backbone=True,
        backbone_config=None,
        backbone=None,
        use_pretrained_backbone=None,
        dilation=None, 
    ):
    
    if use_timm_backbone:
        dilation = dilation if dilation is not None else False
        use_pretrained_backbone = use_pretrained_backbone if use_pretrained_backbone is not None else True
        backbone = backbone if backbone is not None else "resnet50"         

        if backbone_config is not None: 
            raise ValueError("Cannot specify backbone_config if use_timm_backbone is None")
        
    else:
        if None in [dilation, use_pretrained_backbone, backbone]:
            raise ValueError(
                "Cannot specify dilation, use_ptrained_backbone or backbone if use_timm_backbone is False"
            )

Copy link
Copy Markdown
Collaborator

@sgugger sgugger Mar 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed! Note that this can go in a followup PR.


self.use_timm_backbone = use_timm_backbone
self.backbone_config = backbone_config
Expand Down Expand Up @@ -233,6 +236,28 @@ def num_attention_heads(self) -> int:
def hidden_size(self) -> int:
return self.d_model

@classmethod
def from_backbone_config(cls, backbone_config: PretrainedConfig, **kwargs):
"""Instantiate a [`DetrConfig`] (or a derived class) from a pre-trained backbone model configuration.
Args:
backbone_config ([`PretrainedConfig`]):
The backbone configuration.
Returns:
[`DetrConfig`]: An instance of a configuration object
"""
return cls(backbone_config=backbone_config, **kwargs)

def to_dict(self) -> Dict[str, any]:
"""
Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. Returns:
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
"""
output = copy.deepcopy(self.__dict__)
if output["backbone_config"] is not None:
output["backbone_config"] = self.backbone_config.to_dict()
output["model_type"] = self.__class__.model_type
return output


class DetrOnnxConfig(OnnxConfig):
torch_onnx_minimum_version = version.parse("1.11")
Expand Down
Loading