From b69f43a543838be7592dbb958bfe5b2dda2346c1 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 2 Mar 2023 19:29:09 +0000 Subject: [PATCH 01/12] Initial commit --- .../contrastive-image-text/README.md | 102 ++++ .../contrastive-image-text/requirements.txt | 2 + .../contrastive-image-text/run_clip.py | 535 ++++++++++++++++++ 3 files changed, 639 insertions(+) create mode 100644 examples/tensorflow/contrastive-image-text/README.md create mode 100644 examples/tensorflow/contrastive-image-text/requirements.txt create mode 100644 examples/tensorflow/contrastive-image-text/run_clip.py diff --git a/examples/tensorflow/contrastive-image-text/README.md b/examples/tensorflow/contrastive-image-text/README.md new file mode 100644 index 000000000000..1b3d0eef9a59 --- /dev/null +++ b/examples/tensorflow/contrastive-image-text/README.md @@ -0,0 +1,102 @@ + + +# VisionTextDualEncoder and CLIP model training examples + +The following example showcases how to train a CLIP-like vision-text dual encoder model +using a pre-trained vision and text encoder. + +Such a model can be used for natural language image search and potentially zero-shot image classification. +The model is inspired by [CLIP](https://openai.com/blog/clip/), introduced by Alec Radford et al. +The idea is to train a vision encoder and a text encoder jointly to project the representation of images and their +captions into the same embedding space, such that the caption embeddings are located near the embeddings +of the images they describe. + +### Download COCO dataset (2017) +This example uses COCO dataset (2017) through a custom dataset script, which requires users to manually download the +COCO dataset before training. + +```bash +mkdir data +cd data +wget http://images.cocodataset.org/zips/train2017.zip +wget http://images.cocodataset.org/zips/val2017.zip +wget http://images.cocodataset.org/zips/test2017.zip +wget http://images.cocodataset.org/annotations/annotations_trainval2017.zip +wget http://images.cocodataset.org/annotations/image_info_test2017.zip +cd .. +``` + +Having downloaded COCO dataset manually you should be able to load with the `ydshieh/coc_dataset_script` dataset loading script: + +```py +import os +import datasets + +COCO_DIR = os.path.join(os.getcwd(), "data") +ds = datasets.load_dataset("ydshieh/coco_dataset_script", "2017", data_dir=COCO_DIR) +``` + +### Create a model from a vision encoder model and a text encoder model +Next, we create a [TFVisionTextDualEncoderModel](https://huggingface.co/docs/transformers/model_doc/vision-text-dual-encoder#tfvisiontextdualencoder). +The `TFVisionTextDualEncoderModel` class lets you load any vision and text encoder model to create a dual encoder. +Here is an example of how to load the model using pre-trained vision and text models. + +```python3 +from transformers import ( + TFVisionTextDualEncoderModel, + VisionTextDualEncoderProcessor, + AutoTokenizer, + AutoImageProcessor +) + +model = TFVisionTextDualEncoderModel.from_vision_text_pretrained( + "openai/clip-vit-base-patch32", "roberta-base" +) + +tokenizer = AutoTokenizer.from_pretrained("roberta-base") +image_processor = AutoImageProcessor.from_pretrained("openai/clip-vit-base-patch32") +processor = VisionTextDualEncoderProcessor(image_processor, tokenizer) + +# save the model and processor +model.save_pretrained("clip-roberta") +processor.save_pretrained("clip-roberta") +``` + +This loads both the text and vision encoders using pre-trained weights, the projection layers are randomly +initialized except for CLIP's vision model. If you use CLIP to initialize the vision model then the vision projection weights are also +loaded using the pre-trained weights. + +### Train the model +Finally, we can run the example script to train the model: + +```bash +python examples/tensorflow/contrastive-image-text/run_clip.py \ + --output_dir ./clip-roberta-finetuned \ + --model_name_or_path ./clip-roberta \ + --data_dir $PWD/data \ + --dataset_name ydshieh/coco_dataset_script \ + --dataset_config_name=2017 \ + --image_column image_path \ + --caption_column caption \ + --remove_unused_columns=False \ + --do_train --do_eval \ + --per_device_train_batch_size="64" \ + --per_device_eval_batch_size="64" \ + --learning_rate="5e-5" --warmup_steps="0" --weight_decay 0.1 \ + --overwrite_output_dir \ + --push_to_hub +``` diff --git a/examples/tensorflow/contrastive-image-text/requirements.txt b/examples/tensorflow/contrastive-image-text/requirements.txt new file mode 100644 index 000000000000..ef4bf188bff2 --- /dev/null +++ b/examples/tensorflow/contrastive-image-text/requirements.txt @@ -0,0 +1,2 @@ +tensorflow>=2.6.0 +datasets>=1.8.0 \ No newline at end of file diff --git a/examples/tensorflow/contrastive-image-text/run_clip.py b/examples/tensorflow/contrastive-image-text/run_clip.py new file mode 100644 index 000000000000..1c79ff3fc0cb --- /dev/null +++ b/examples/tensorflow/contrastive-image-text/run_clip.py @@ -0,0 +1,535 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2022 The HuggingFace Team All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Training a CLIP like dual encoder models using text and vision encoders in the library. + +The script can be used to train CLIP like models for languages other than English by using +a text encoder pre-trained in the desired language. Currently this script supports the following vision +and text models: +Vision models: ViT(https://huggingface.co/models?filter=vit), CLIP (https://huggingface.co/models?filter=clip) +Text models: BERT, ROBERTa (https://huggingface.co/models?filter=fill-mask) +""" + +import logging +import os +import sys +from dataclasses import dataclass, field +from typing import Optional + +from datasets import load_dataset +from PIL import Image + +import transformers +from transformers import ( + AutoImageProcessor, + TFAutoModel, + AutoTokenizer, + HfArgumentParser, + TFTrainingArguments, + set_seed, +) +from transformers.trainer_utils import get_last_checkpoint +from transformers.utils import check_min_version, send_example_telemetry +from transformers.utils.versions import require_version + + +logger = logging.getLogger(__name__) + +# Will error if the minimal version of Transformers is not installed. Remove at your own risks. +check_min_version("4.27.0.dev0") + +require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/contrastive-image-text/requirements.txt") + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}, + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + image_processor_name: str = field(default=None, metadata={"help": "Name or path of preprocessor config."}) + cache_dir: Optional[str] = field( + default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"} + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": ( + "Will use the token generated when running `huggingface-cli login` (necessary to use this script " + "with private models)." + ) + }, + ) + freeze_vision_model: bool = field( + default=False, metadata={"help": "Whether to freeze the vision model parameters or not."} + ) + freeze_text_model: bool = field( + default=False, metadata={"help": "Whether to freeze the text model parameters or not."} + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_name: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + data_dir: Optional[str] = field(default=None, metadata={"help": "The data directory containing input files."}) + image_column: Optional[str] = field( + default="image_path", + metadata={"help": "The name of the column in the datasets containing the full image file paths."}, + ) + caption_column: Optional[str] = field( + default="caption", + metadata={"help": "The name of the column in the datasets containing the image captions."}, + ) + train_file: Optional[str] = field( + default=None, metadata={"help": "The input training data file (a jsonlines file)."} + ) + validation_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input evaluation data file (a jsonlines file)."}, + ) + test_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input testing data file (a jsonlines file)."}, + ) + max_seq_length: Optional[int] = field( + default=128, + metadata={ + "help": ( + "The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded." + ) + }, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + ) + }, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + + def __post_init__(self): + if self.dataset_name is None and self.train_file is None and self.validation_file is None: + raise ValueError("Need either a dataset name or a training/validation file.") + else: + if self.train_file is not None: + extension = self.train_file.split(".")[-1] + assert extension in ["csv", "json"], "`train_file` should be a csv or a json file." + if self.validation_file is not None: + extension = self.validation_file.split(".")[-1] + assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file." + if self.validation_file is not None: + extension = self.validation_file.split(".")[-1] + assert extension == "json", "`validation_file` should be a json file." + + +dataset_name_mapping = { + "image_caption_dataset.py": ("image_path", "caption"), +} + + +# # We use tf.data for faster image pre-processing. +# class Transform(torch.nn.Module): +# def __init__(self, image_size, mean, std): +# super().__init__() +# self.transforms = torch.nn.Sequential( +# Resize([image_size], interpolation=InterpolationMode.BICUBIC), +# CenterCrop(image_size), +# ConvertImageDtype(torch.float), +# Normalize(mean, std), +# ) +# +# def forward(self, x) -> torch.Tensor: +# """`x` should be an instance of `PIL.Image.Image`""" +# with torch.no_grad(): +# x = self.transforms(x) +# return x +# +# +# def collate_fn(examples): +# pixel_values = torch.stack([example["pixel_values"] for example in examples]) +# input_ids = torch.tensor([example["input_ids"] for example in examples], dtype=torch.long) +# attention_mask = torch.tensor([example["attention_mask"] for example in examples], dtype=torch.long) +# return { +# "pixel_values": pixel_values, +# "input_ids": input_ids, +# "attention_mask": attention_mask, +# "return_loss": True, +# } + + +def main(): + # 1. Parse input arguments + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_clip", model_args, data_args) + + # 2. Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + if training_args.should_log: + # The default of training_args.log_level is passive, so we set log level at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" + ) + logger.info(f"Training/evaluation parameters {training_args}") + + # 3. Detecting last checkpoint and eventualy continue from last checkpoint + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # 4. Load dataset + # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below) + # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/ + # (the dataset will be downloaded automatically from the datasets Hub). + # + # For CSV/JSON files this script will use the first column for the full image path and the second column for the + # captions (unless you specify column names for this with the `image_column` and `caption_column` arguments). + # + if data_args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + cache_dir=model_args.cache_dir, + keep_in_memory=False, + data_dir=data_args.data_dir, + use_auth_token=True if model_args.use_auth_token else None, + ) + else: + data_files = {} + if data_args.train_file is not None: + data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] + if data_args.validation_file is not None: + data_files["validation"] = data_args.validation_file + extension = data_args.validation_file.split(".")[-1] + if data_args.test_file is not None: + data_files["test"] = data_args.test_file + extension = data_args.test_file.split(".")[-1] + dataset = load_dataset( + extension, + data_files=data_files, + cache_dir=model_args.cache_dir, + use_auth_token=True if model_args.use_auth_token else None, + ) + # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at + # https://huggingface.co/docs/datasets/loading_datasets.html. + + # 5. Load pretrained model, tokenizer, and image processor + if model_args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained( + model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer + ) + elif model_args.model_name_or_path: + tokenizer = AutoTokenizer.from_pretrained( + model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer + ) + else: + raise ValueError( + "You are instantiating a new tokenizer from scratch. This is not supported by this script." + "You can do it from another script, save it, and load it from here, using --tokenizer_name." + ) + + # Load image_processor, in this script we only use this to get the mean and std for normalization. + image_processor = AutoImageProcessor.from_pretrained( + model_args.image_processor_name or model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + + model = TFAutoModel.from_pretrained( + model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + config = model.config + + def _freeze_params(module): + for param in module.parameters(): + param.requires_grad = False + + if model_args.freeze_vision_model: + _freeze_params(model.vision_model) + + if model_args.freeze_text_model: + _freeze_params(model.text_model) + + # set seed for torch dataloaders + set_seed(training_args.seed) + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + if training_args.do_train: + column_names = dataset["train"].column_names + elif training_args.do_eval: + column_names = dataset["validation"].column_names + elif training_args.do_predict: + column_names = dataset["test"].column_names + else: + logger.info("There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.") + return + + # 6. Get the column names for input/target. + dataset_columns = dataset_name_mapping.get(data_args.dataset_name, None) + if data_args.image_column is None: + image_column = dataset_columns[0] if dataset_columns is not None else column_names[0] + else: + image_column = data_args.image_column + if image_column not in column_names: + raise ValueError( + f"--image_column' value '{data_args.image_column}' needs to be one of: {', '.join(column_names)}" + ) + if data_args.caption_column is None: + caption_column = dataset_columns[1] if dataset_columns is not None else column_names[1] + else: + caption_column = data_args.caption_column + if caption_column not in column_names: + raise ValueError( + f"--caption_column' value '{data_args.caption_column}' needs to be one of: {', '.join(column_names)}" + ) + + # # 7. Preprocessing the datasets. + # # Initialize torchvision transforms and jit it for faster processing. + # image_transformations = Transform( + # config.vision_config.image_size, image_processor.image_mean, image_processor.image_std + # ) + + # Preprocessing the datasets. + # We need to tokenize input captions and transform the images. + def tokenize_captions(examples): + captions = list(examples[caption_column]) + text_inputs = tokenizer(captions, max_length=data_args.max_seq_length, padding="max_length", truncation=True) + examples["input_ids"] = text_inputs.input_ids + examples["attention_mask"] = text_inputs.attention_mask + return examples + + def transform_images(examples): + images = [read_image(image_file, mode=ImageReadMode.RGB) for image_file in examples[image_column]] + examples["pixel_values"] = [image_transformations(image) for image in images] + return examples + + def filter_corrupt_images(examples): + """remove problematic images""" + valid_images = [] + for image_file in examples[image_column]: + try: + Image.open(image_file) + valid_images.append(True) + except Exception: + valid_images.append(False) + return valid_images + + if training_args.do_train: + if "train" not in dataset: + raise ValueError("--do_train requires a train dataset") + train_dataset = dataset["train"] + if data_args.max_train_samples is not None: + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) + + train_dataset = train_dataset.filter( + filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers + ) + train_dataset = train_dataset.map( + function=tokenize_captions, + batched=True, + remove_columns=[col for col in column_names if col != image_column], + num_proc=data_args.preprocessing_num_workers, + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on train dataset", + ) + + breakpoint() + + # Transform images on the fly as doing it on the whole dataset takes too much time. + train_dataset.set_transform(transform_images) + + if training_args.do_eval: + if "validation" not in dataset: + raise ValueError("--do_eval requires a train validation") + eval_dataset = dataset["validation"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) + + eval_dataset = eval_dataset.filter( + filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers + ) + eval_dataset = eval_dataset.map( + function=tokenize_captions, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=[col for col in column_names if col != image_column], + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on validation dataset", + ) + + # Transform images on the fly as doing it on the whole dataset takes too much time. + eval_dataset.set_transform(transform_images) + + if training_args.do_predict: + if "test" not in dataset: + raise ValueError("--do_predict requires a test dataset") + test_dataset = dataset["test"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(test_dataset), data_args.max_eval_samples) + test_dataset = test_dataset.select(range(max_eval_samples)) + + test_dataset = test_dataset.filter( + filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers + ) + test_dataset = test_dataset.map( + function=tokenize_captions, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=[col for col in column_names if col != image_column], + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on test dataset", + ) + + # Transform images on the fly as doing it on the whole dataset takes too much time. + test_dataset.set_transform(transform_images) + + # 8. Initalize our trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset if training_args.do_train else None, + eval_dataset=eval_dataset if training_args.do_eval else None, + data_collator=collate_fn, + ) + + # 9. Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + trainer.save_model() + trainer.log_metrics("train", train_result.metrics) + trainer.save_metrics("train", train_result.metrics) + trainer.save_state() + + # 10. Evaluation + if training_args.do_eval: + metrics = trainer.evaluate() + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + # 11. Write Training Stats and push to hub. + kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "contrastive-image-text-modeling"} + if data_args.dataset_name is not None: + kwargs["dataset_tags"] = data_args.dataset_name + if data_args.dataset_config_name is not None: + kwargs["dataset_args"] = data_args.dataset_config_name + kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}" + else: + kwargs["dataset"] = data_args.dataset_name + + if training_args.push_to_hub: + trainer.push_to_hub(**kwargs) + else: + trainer.create_model_card(**kwargs) + + +if __name__ == "__main__": + main() From 3516cd00b5632db84f97b8df772ac79f3ea74bc4 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 3 Mar 2023 14:13:48 +0000 Subject: [PATCH 02/12] stash commit --- .../contrastive-image-text/README.md | 2 +- .../contrastive-image-text/run_clip.py | 91 ++++++++++++++----- 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/examples/tensorflow/contrastive-image-text/README.md b/examples/tensorflow/contrastive-image-text/README.md index 1b3d0eef9a59..bd688c81fd5c 100644 --- a/examples/tensorflow/contrastive-image-text/README.md +++ b/examples/tensorflow/contrastive-image-text/README.md @@ -86,7 +86,7 @@ Finally, we can run the example script to train the model: ```bash python examples/tensorflow/contrastive-image-text/run_clip.py \ --output_dir ./clip-roberta-finetuned \ - --model_name_or_path ./clip-roberta \ + --model_name_or_path clip-roberta \ --data_dir $PWD/data \ --dataset_name ydshieh/coco_dataset_script \ --dataset_config_name=2017 \ diff --git a/examples/tensorflow/contrastive-image-text/run_clip.py b/examples/tensorflow/contrastive-image-text/run_clip.py index 1c79ff3fc0cb..bff31750d51f 100644 --- a/examples/tensorflow/contrastive-image-text/run_clip.py +++ b/examples/tensorflow/contrastive-image-text/run_clip.py @@ -36,12 +36,12 @@ from transformers import ( AutoImageProcessor, TFAutoModel, + Dual AutoTokenizer, HfArgumentParser, TFTrainingArguments, set_seed, ) -from transformers.trainer_utils import get_last_checkpoint from transformers.utils import check_min_version, send_example_telemetry from transformers.utils.versions import require_version @@ -62,6 +62,15 @@ class ModelArguments: model_name_or_path: str = field( metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}, + default=None + ) + image_model_name_or_path: str = field( + metadata={"help": "Path to pretrained image model or model identifier from huggingface.co/models"}, + default=None + ) + text_model_name_or_path: str = field( + metadata={"help": "Path to pretrained text model or model identifier from huggingface.co/models"}, + default=None ) config_name: Optional[str] = field( default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} @@ -229,9 +238,20 @@ def main(): else: model_args, data_args, training_args = parser.parse_args_into_dataclasses() + if model_args.model_name_or_path is not None: + if model_args.image_model_name_or_path is not None or model_args.text_model_name_or_path is not None: + raise ValueError("If using model_name_or_path, you cannot specify separate image/text model paths as well!") + + if model_args.image_model_name_or_path is not None or model_args.text_model_name_or_path is not None: + if model_args.model_name_or_path is not None: + raise ValueError("If using separate image/text model paths, you cannot specify model_name_or_path as well!") + if not (model_args.image_model_name_or_path is not None and model_args.text_model_name_or_path is not None): + raise ValueError("If using separate image/text model paths, you must specify both image_model_name_or_path " + "and text_model_name_or_path!") + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The - # information sent is the one passed as arguments along with your Python/PyTorch versions. - send_example_telemetry("run_clip", model_args, data_args) + # information sent is the one passed as arguments along with your Python/TensorFlow versions. + send_example_telemetry("run_clip", model_args, data_args, framework="tensorflow") # 2. Setup logging logging.basicConfig( @@ -325,31 +345,58 @@ def main(): "You can do it from another script, save it, and load it from here, using --tokenizer_name." ) - # Load image_processor, in this script we only use this to get the mean and std for normalization. - image_processor = AutoImageProcessor.from_pretrained( - model_args.image_processor_name or model_args.model_name_or_path, - cache_dir=model_args.cache_dir, - revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, - ) + if model_args.model_name_or_path: + # Load image_processor, in this script we only use this to get the mean and std for normalization. + image_processor = AutoImageProcessor.from_pretrained( + model_args.image_processor_name or model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) - model = TFAutoModel.from_pretrained( - model_args.model_name_or_path, - cache_dir=model_args.cache_dir, - revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, - ) - config = model.config + model = TFAutoModel.from_pretrained( + model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + config = model.config + else: + # Load image_processor, in this script we only use this to get the mean and std for normalization. + image_processor = AutoImageProcessor.from_pretrained( + model_args.image_processor_name or model_args.image_model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) - def _freeze_params(module): - for param in module.parameters(): - param.requires_grad = False + image_model = TFAutoModel.from_pretrained( + model_args.image_model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + + text_model = TFAutoModel.from_pretrained( + model_args.text_model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + + model = TFImageText.from_pretrained( + model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + config = model.config if model_args.freeze_vision_model: - _freeze_params(model.vision_model) + model.vision_model.trainable = False if model_args.freeze_text_model: - _freeze_params(model.text_model) + model.text_model.trainable = False # set seed for torch dataloaders set_seed(training_args.seed) From 35fe4b85473c21b316df7a55b961e9e4651827c3 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 3 Mar 2023 18:10:38 +0000 Subject: [PATCH 03/12] Add model checkpointing and pushing --- .../contrastive-image-text/run_clip.py | 279 +++++++++--------- .../modeling_tf_vision_text_dual_encoder.py | 2 + 2 files changed, 143 insertions(+), 138 deletions(-) diff --git a/examples/tensorflow/contrastive-image-text/run_clip.py b/examples/tensorflow/contrastive-image-text/run_clip.py index bff31750d51f..bd3f63f7c2ca 100644 --- a/examples/tensorflow/contrastive-image-text/run_clip.py +++ b/examples/tensorflow/contrastive-image-text/run_clip.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding=utf-8 -# Copyright 2022 The HuggingFace Team All rights reserved. +# Copyright 2023 The HuggingFace Team All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,17 +29,20 @@ from dataclasses import dataclass, field from typing import Optional +import tensorflow as tf from datasets import load_dataset from PIL import Image import transformers from transformers import ( AutoImageProcessor, - TFAutoModel, - Dual AutoTokenizer, HfArgumentParser, + PushToHubCallback, + TFAutoModel, TFTrainingArguments, + TFVisionTextDualEncoderModel, + create_optimizer, set_seed, ) from transformers.utils import check_min_version, send_example_telemetry @@ -61,16 +64,14 @@ class ModelArguments: """ model_name_or_path: str = field( - metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}, - default=None + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}, default=None ) - image_model_name_or_path: str = field( + vision_model_name_or_path: str = field( metadata={"help": "Path to pretrained image model or model identifier from huggingface.co/models"}, - default=None + default=None, ) text_model_name_or_path: str = field( - metadata={"help": "Path to pretrained text model or model identifier from huggingface.co/models"}, - default=None + metadata={"help": "Path to pretrained text model or model identifier from huggingface.co/models"}, default=None ) config_name: Optional[str] = field( default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} @@ -194,34 +195,39 @@ def __post_init__(self): } -# # We use tf.data for faster image pre-processing. -# class Transform(torch.nn.Module): -# def __init__(self, image_size, mean, std): -# super().__init__() -# self.transforms = torch.nn.Sequential( -# Resize([image_size], interpolation=InterpolationMode.BICUBIC), -# CenterCrop(image_size), -# ConvertImageDtype(torch.float), -# Normalize(mean, std), -# ) -# -# def forward(self, x) -> torch.Tensor: -# """`x` should be an instance of `PIL.Image.Image`""" -# with torch.no_grad(): -# x = self.transforms(x) -# return x -# -# -# def collate_fn(examples): -# pixel_values = torch.stack([example["pixel_values"] for example in examples]) -# input_ids = torch.tensor([example["input_ids"] for example in examples], dtype=torch.long) -# attention_mask = torch.tensor([example["attention_mask"] for example in examples], dtype=torch.long) -# return { -# "pixel_values": pixel_values, -# "input_ids": input_ids, -# "attention_mask": attention_mask, -# "return_loss": True, -# } +def crop_to_square(image): + height, width = tf.shape(image)[0], tf.shape(image)[1] + if height > width: + image = tf.image.crop_to_bounding_box(image, (height - width) // 2, 0, width, width) + elif width > height: + image = tf.image.crop_to_bounding_box(image, 0, (width - height) // 2, height, height) + return image + + +def load_as_tf_dataset(dataset, image_column, image_size, mean, std, batch_size, shuffle): + dataset = dataset.with_format("tensorflow")[:] # Load the dataset as tensor slices, but not the images yet! + tf_dataset = tf.data.Dataset.from_tensor_slices(dataset) + + def load_image(sample): + image_path = sample[image_column] + image = tf.io.read_file(image_path) + image = tf.image.decode_image(image, channels=3, expand_animations=False) + image = crop_to_square(image) + image = tf.image.resize(image, [image_size, image_size], method="bicubic", antialias=True) + image = image / 255.0 + image = (image - mean) / std + image = tf.transpose(image, perm=[2, 0, 1]) # Convert to channels-first + sample["pixel_values"] = image + del sample[image_column] + return sample + + if shuffle: + tf_dataset = tf_dataset.shuffle(len(tf_dataset)) + tf_dataset = tf_dataset.map(load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE) + tf_dataset = tf_dataset.batch(batch_size, drop_remainder=shuffle) + tf_dataset = tf_dataset.prefetch(tf.data.experimental.AUTOTUNE) + + return tf_dataset def main(): @@ -239,15 +245,21 @@ def main(): model_args, data_args, training_args = parser.parse_args_into_dataclasses() if model_args.model_name_or_path is not None: - if model_args.image_model_name_or_path is not None or model_args.text_model_name_or_path is not None: - raise ValueError("If using model_name_or_path, you cannot specify separate image/text model paths as well!") + if model_args.vision_model_name_or_path is not None or model_args.text_model_name_or_path is not None: + raise ValueError( + "If using model_name_or_path, you cannot specify separate image/text model paths as well!" + ) - if model_args.image_model_name_or_path is not None or model_args.text_model_name_or_path is not None: + if model_args.vision_model_name_or_path is not None or model_args.text_model_name_or_path is not None: if model_args.model_name_or_path is not None: - raise ValueError("If using separate image/text model paths, you cannot specify model_name_or_path as well!") - if not (model_args.image_model_name_or_path is not None and model_args.text_model_name_or_path is not None): - raise ValueError("If using separate image/text model paths, you must specify both image_model_name_or_path " - "and text_model_name_or_path!") + raise ValueError( + "If using separate image/text model paths, you cannot specify model_name_or_path as well!" + ) + if not (model_args.vision_model_name_or_path is not None and model_args.text_model_name_or_path is not None): + raise ValueError( + "If using separate image/text model paths, you must specify both vision_model_name_or_path " + "and text_model_name_or_path!" + ) # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The # information sent is the one passed as arguments along with your Python/TensorFlow versions. @@ -280,7 +292,6 @@ def main(): # 3. Detecting last checkpoint and eventualy continue from last checkpoint last_checkpoint = None if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: - last_checkpoint = get_last_checkpoint(training_args.output_dir) if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: raise ValueError( f"Output directory ({training_args.output_dir}) already exists and is not empty. " @@ -339,6 +350,10 @@ def main(): tokenizer = AutoTokenizer.from_pretrained( model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer ) + elif model_args.text_model_name_or_path: + tokenizer = AutoTokenizer.from_pretrained( + model_args.text_model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer + ) else: raise ValueError( "You are instantiating a new tokenizer from scratch. This is not supported by this script." @@ -360,37 +375,22 @@ def main(): revision=model_args.model_revision, use_auth_token=True if model_args.use_auth_token else None, ) - config = model.config else: # Load image_processor, in this script we only use this to get the mean and std for normalization. image_processor = AutoImageProcessor.from_pretrained( - model_args.image_processor_name or model_args.image_model_name_or_path, + model_args.image_processor_name or model_args.vision_model_name_or_path, cache_dir=model_args.cache_dir, revision=model_args.model_revision, use_auth_token=True if model_args.use_auth_token else None, ) - image_model = TFAutoModel.from_pretrained( - model_args.image_model_name_or_path, - cache_dir=model_args.cache_dir, - revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, - ) - - text_model = TFAutoModel.from_pretrained( - model_args.text_model_name_or_path, - cache_dir=model_args.cache_dir, - revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, - ) - - model = TFImageText.from_pretrained( - model_args.model_name_or_path, + model = TFVisionTextDualEncoderModel.from_vision_text_pretrained( + vision_model_name_or_path=model_args.vision_model_name_or_path, + text_model_name_or_path=model_args.text_model_name_or_path, cache_dir=model_args.cache_dir, - revision=model_args.model_revision, use_auth_token=True if model_args.use_auth_token else None, ) - config = model.config + config = model.config if model_args.freeze_vision_model: model.vision_model.trainable = False @@ -433,12 +433,7 @@ def main(): ) # # 7. Preprocessing the datasets. - # # Initialize torchvision transforms and jit it for faster processing. - # image_transformations = Transform( - # config.vision_config.image_size, image_processor.image_mean, image_processor.image_std - # ) - # Preprocessing the datasets. # We need to tokenize input captions and transform the images. def tokenize_captions(examples): captions = list(examples[caption_column]) @@ -447,11 +442,6 @@ def tokenize_captions(examples): examples["attention_mask"] = text_inputs.attention_mask return examples - def transform_images(examples): - images = [read_image(image_file, mode=ImageReadMode.RGB) for image_file in examples[image_column]] - examples["pixel_values"] = [image_transformations(image) for image in images] - return examples - def filter_corrupt_images(examples): """remove problematic images""" valid_images = [] @@ -483,10 +473,15 @@ def filter_corrupt_images(examples): desc="Running tokenizer on train dataset", ) - breakpoint() - - # Transform images on the fly as doing it on the whole dataset takes too much time. - train_dataset.set_transform(transform_images) + tf_train_dataset = load_as_tf_dataset( + dataset=train_dataset, + batch_size=training_args.per_device_train_batch_size, + image_column=image_column, + image_size=config.vision_config.image_size, + mean=image_processor.image_mean, + std=image_processor.image_std, + shuffle=True, + ) if training_args.do_eval: if "validation" not in dataset: @@ -508,74 +503,82 @@ def filter_corrupt_images(examples): desc="Running tokenizer on validation dataset", ) - # Transform images on the fly as doing it on the whole dataset takes too much time. - eval_dataset.set_transform(transform_images) - - if training_args.do_predict: - if "test" not in dataset: - raise ValueError("--do_predict requires a test dataset") - test_dataset = dataset["test"] - if data_args.max_eval_samples is not None: - max_eval_samples = min(len(test_dataset), data_args.max_eval_samples) - test_dataset = test_dataset.select(range(max_eval_samples)) - - test_dataset = test_dataset.filter( - filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers - ) - test_dataset = test_dataset.map( - function=tokenize_captions, - batched=True, - num_proc=data_args.preprocessing_num_workers, - remove_columns=[col for col in column_names if col != image_column], - load_from_cache_file=not data_args.overwrite_cache, - desc="Running tokenizer on test dataset", + tf_eval_dataset = load_as_tf_dataset( + dataset=eval_dataset, + batch_size=training_args.per_device_eval_batch_size, + image_column=image_column, + image_size=config.vision_config.image_size, + mean=image_processor.image_mean, + std=image_processor.image_std, + shuffle=False, ) - # Transform images on the fly as doing it on the whole dataset takes too much time. - test_dataset.set_transform(transform_images) - - # 8. Initalize our trainer - trainer = Trainer( - model=model, - args=training_args, - train_dataset=train_dataset if training_args.do_train else None, - eval_dataset=eval_dataset if training_args.do_eval else None, - data_collator=collate_fn, - ) - - # 9. Training - if training_args.do_train: - checkpoint = None - if training_args.resume_from_checkpoint is not None: - checkpoint = training_args.resume_from_checkpoint - elif last_checkpoint is not None: - checkpoint = last_checkpoint - train_result = trainer.train(resume_from_checkpoint=checkpoint) - trainer.save_model() - trainer.log_metrics("train", train_result.metrics) - trainer.save_metrics("train", train_result.metrics) - trainer.save_state() - - # 10. Evaluation - if training_args.do_eval: - metrics = trainer.evaluate() - trainer.log_metrics("eval", metrics) - trainer.save_metrics("eval", metrics) + # 8. Preparing push_to_hub and model card + push_to_hub_model_id = training_args.push_to_hub_model_id + model_name = model_args.model_name_or_path.split("/")[-1] + if not push_to_hub_model_id: + if data_args.dataset_name is not None: + push_to_hub_model_id = f"{model_name}-finetuned-{data_args.dataset_name}" + else: + push_to_hub_model_id = f"{model_name}-finetuned-token-classification" - # 11. Write Training Stats and push to hub. - kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "contrastive-image-text-modeling"} + model_card_kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "token-classification"} if data_args.dataset_name is not None: - kwargs["dataset_tags"] = data_args.dataset_name + model_card_kwargs["dataset_tags"] = data_args.dataset_name if data_args.dataset_config_name is not None: - kwargs["dataset_args"] = data_args.dataset_config_name - kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}" + model_card_kwargs["dataset_args"] = data_args.dataset_config_name + model_card_kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}" else: - kwargs["dataset"] = data_args.dataset_name + model_card_kwargs["dataset"] = data_args.dataset_name if training_args.push_to_hub: - trainer.push_to_hub(**kwargs) + callbacks = [ + PushToHubCallback( + output_dir=training_args.output_dir, + model_id=push_to_hub_model_id, + organization=training_args.push_to_hub_organization, + token=training_args.push_to_hub_token, + tokenizer=tokenizer, + **model_card_kwargs, + ) + ] else: - trainer.create_model_card(**kwargs) + callbacks = [] + + # # 9. Training + if training_args.do_train: + num_train_steps = int(len(tf_train_dataset) * int(training_args.num_train_epochs)) + if training_args.warmup_steps > 0: + num_warmup_steps = training_args.warmup_steps + elif training_args.warmup_ratio > 0: + num_warmup_steps = int(num_train_steps * training_args.warmup_ratio) + else: + num_warmup_steps = 0 + optimizer, lr_schedule = create_optimizer( + init_lr=training_args.learning_rate, + num_train_steps=num_train_steps, + num_warmup_steps=num_warmup_steps, + adam_beta1=training_args.adam_beta1, + adam_beta2=training_args.adam_beta2, + adam_epsilon=training_args.adam_epsilon, + weight_decay_rate=training_args.weight_decay, + adam_global_clipnorm=training_args.max_grad_norm, + ) + model.compile(optimizer=optimizer, jit_compile=training_args.xla) + + if not training_args.do_eval: + tf_eval_dataset = None + model.fit( + tf_train_dataset, + validation_data=tf_eval_dataset, + epochs=int(training_args.num_train_epochs), + callbacks=callbacks, + ) + + # # 10. Evaluation + + if training_args.do_eval and not training_args.do_train: + model.evaluate(tf_eval_dataset) if __name__ == "__main__": diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_tf_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_tf_vision_text_dual_encoder.py index 7efc3e8ae31c..a2211f245ec5 100644 --- a/src/transformers/models/vision_text_dual_encoder/modeling_tf_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_tf_vision_text_dual_encoder.py @@ -438,6 +438,8 @@ def call( loss = None if return_loss: loss = clip_loss(logits_per_text) + if loss.shape.rank == 0: + loss = tf.expand_dims(loss, 0) if not return_dict: output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs) From 4ac54c8f63c0b8db96e55fbfb440736d09f74842 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 3 Mar 2023 18:19:09 +0000 Subject: [PATCH 04/12] Fix model name inference --- .../tensorflow/contrastive-image-text/run_clip.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/tensorflow/contrastive-image-text/run_clip.py b/examples/tensorflow/contrastive-image-text/run_clip.py index bd3f63f7c2ca..dffbfc68bdb0 100644 --- a/examples/tensorflow/contrastive-image-text/run_clip.py +++ b/examples/tensorflow/contrastive-image-text/run_clip.py @@ -515,14 +515,19 @@ def filter_corrupt_images(examples): # 8. Preparing push_to_hub and model card push_to_hub_model_id = training_args.push_to_hub_model_id - model_name = model_args.model_name_or_path.split("/")[-1] + if model_args.model_name_or_path is not None: + model_name = model_args.model_name_or_path.split("/")[-1] + else: + vision_name = model_args.vision_model_name_or_path.split("/")[-1] + text_name = model_args.text_model_name_or_path.split("/")[-1] + model_name = f"{vision_name}-{text_name}" if not push_to_hub_model_id: if data_args.dataset_name is not None: push_to_hub_model_id = f"{model_name}-finetuned-{data_args.dataset_name}" else: - push_to_hub_model_id = f"{model_name}-finetuned-token-classification" + push_to_hub_model_id = f"{model_name}-finetuned-contrastive-image-text-modeling" - model_card_kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "token-classification"} + model_card_kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "contrastive-image-text-modeling"} if data_args.dataset_name is not None: model_card_kwargs["dataset_tags"] = data_args.dataset_name if data_args.dataset_config_name is not None: From a09e3873b84ee18b3ab9c7132a428b91e284dc6d Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 3 Mar 2023 18:25:53 +0000 Subject: [PATCH 05/12] Update README --- .../contrastive-image-text/README.md | 35 ++++--------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/examples/tensorflow/contrastive-image-text/README.md b/examples/tensorflow/contrastive-image-text/README.md index bd688c81fd5c..29f85fcf5846 100644 --- a/examples/tensorflow/contrastive-image-text/README.md +++ b/examples/tensorflow/contrastive-image-text/README.md @@ -51,34 +51,12 @@ ds = datasets.load_dataset("ydshieh/coco_dataset_script", "2017", data_dir=COCO_ ``` ### Create a model from a vision encoder model and a text encoder model -Next, we create a [TFVisionTextDualEncoderModel](https://huggingface.co/docs/transformers/model_doc/vision-text-dual-encoder#tfvisiontextdualencoder). -The `TFVisionTextDualEncoderModel` class lets you load any vision and text encoder model to create a dual encoder. -Here is an example of how to load the model using pre-trained vision and text models. +We can either load a CLIP-like vision-text dual encoder model from an existing dual encoder model, or +by using a pre-trained vision encoder model and a pre-trained text encoder model. -```python3 -from transformers import ( - TFVisionTextDualEncoderModel, - VisionTextDualEncoderProcessor, - AutoTokenizer, - AutoImageProcessor -) - -model = TFVisionTextDualEncoderModel.from_vision_text_pretrained( - "openai/clip-vit-base-patch32", "roberta-base" -) - -tokenizer = AutoTokenizer.from_pretrained("roberta-base") -image_processor = AutoImageProcessor.from_pretrained("openai/clip-vit-base-patch32") -processor = VisionTextDualEncoderProcessor(image_processor, tokenizer) - -# save the model and processor -model.save_pretrained("clip-roberta") -processor.save_pretrained("clip-roberta") -``` - -This loads both the text and vision encoders using pre-trained weights, the projection layers are randomly -initialized except for CLIP's vision model. If you use CLIP to initialize the vision model then the vision projection weights are also -loaded using the pre-trained weights. +If you wish to load an existing dual encoder model, please use the `--model_name_or_path` argument. If +you want to use a pre-trained vision encoder model and a pre-trained text encoder model, please use the +`--vision_model_name_or_path` and `--text_model_name_or_path` arguments instead. ### Train the model Finally, we can run the example script to train the model: @@ -86,7 +64,8 @@ Finally, we can run the example script to train the model: ```bash python examples/tensorflow/contrastive-image-text/run_clip.py \ --output_dir ./clip-roberta-finetuned \ - --model_name_or_path clip-roberta \ + --vision_model_name_or_path openai/clip-vit-base-patch32 \ + --text_model_name_or_path roberta-base \ --data_dir $PWD/data \ --dataset_name ydshieh/coco_dataset_script \ --dataset_config_name=2017 \ From 542600f14640bb5904605c9f638283db02eca9bc Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 3 Mar 2023 18:31:56 +0000 Subject: [PATCH 06/12] Update README --- examples/tensorflow/contrastive-image-text/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tensorflow/contrastive-image-text/README.md b/examples/tensorflow/contrastive-image-text/README.md index 29f85fcf5846..2a397f654aa3 100644 --- a/examples/tensorflow/contrastive-image-text/README.md +++ b/examples/tensorflow/contrastive-image-text/README.md @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. --> -# VisionTextDualEncoder and CLIP model training examples +# TFVisionTextDualEncoder and CLIP model training examples The following example showcases how to train a CLIP-like vision-text dual encoder model using a pre-trained vision and text encoder. @@ -55,7 +55,7 @@ We can either load a CLIP-like vision-text dual encoder model from an existing d by using a pre-trained vision encoder model and a pre-trained text encoder model. If you wish to load an existing dual encoder model, please use the `--model_name_or_path` argument. If -you want to use a pre-trained vision encoder model and a pre-trained text encoder model, please use the +you want to use separate pre-trained vision and text models, please use the `--vision_model_name_or_path` and `--text_model_name_or_path` arguments instead. ### Train the model From 811ffaf2f95d8999dd7ccd92e2e0e9e31541732b Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 3 Mar 2023 18:45:17 +0000 Subject: [PATCH 07/12] Remove a couple of Torch references --- examples/tensorflow/contrastive-image-text/run_clip.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/tensorflow/contrastive-image-text/run_clip.py b/examples/tensorflow/contrastive-image-text/run_clip.py index dffbfc68bdb0..c567f9ab909c 100644 --- a/examples/tensorflow/contrastive-image-text/run_clip.py +++ b/examples/tensorflow/contrastive-image-text/run_clip.py @@ -54,7 +54,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. check_min_version("4.27.0.dev0") -require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/contrastive-image-text/requirements.txt") +require_version("datasets>=1.8.0", "To fix: pip install -r examples/tensorflow/contrastive-image-text/requirements.txt") @dataclass @@ -398,9 +398,6 @@ def main(): if model_args.freeze_text_model: model.text_model.trainable = False - # set seed for torch dataloaders - set_seed(training_args.seed) - # Preprocessing the datasets. # We need to tokenize inputs and targets. if training_args.do_train: From 772b0600c3bfa33535ff8e6e8eb496b7da65585c Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 3 Mar 2023 18:46:18 +0000 Subject: [PATCH 08/12] Update copyright date --- examples/tensorflow/contrastive-image-text/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tensorflow/contrastive-image-text/README.md b/examples/tensorflow/contrastive-image-text/README.md index 2a397f654aa3..9e3a011fcb33 100644 --- a/examples/tensorflow/contrastive-image-text/README.md +++ b/examples/tensorflow/contrastive-image-text/README.md @@ -1,5 +1,5 @@