From 558190bbce35a369f6ba21e6e51f2c634133858f Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 2 Nov 2020 18:47:40 +0100 Subject: [PATCH 01/18] fix typo --- utils/check_dummies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check_dummies.py b/utils/check_dummies.py index 81adb4160222..c6fe90bc4330 100644 --- a/utils/check_dummies.py +++ b/utils/check_dummies.py @@ -165,7 +165,7 @@ def {0}(*args, **kwargs): def read_init(): - """ Read the init and exctracts PyTorch, TensorFlow, SentencePiece and Tokenizers objects. """ + """ Read the init and extracts PyTorch, TensorFlow, SentencePiece and Tokenizers objects. """ with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r", encoding="utf-8") as f: lines = f.readlines() From a10fb6cee14c8590023304f79398ba3fed0acd04 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 2 Nov 2020 18:49:39 +0100 Subject: [PATCH 02/18] rm use_cdn & references, and implement new hf_bucket_url --- model_cards/t5-11b-README.md | 4 +-- scripts/fsmt/convert-allenai-wmt16.sh | 4 --- scripts/fsmt/convert-allenai-wmt19.sh | 4 --- scripts/fsmt/convert-facebook-wmt19.sh | 4 --- src/transformers/configuration_utils.py | 3 +- ..._original_pytorch_checkpoint_to_pytorch.py | 4 --- src/transformers/file_utils.py | 34 ++++++++++--------- src/transformers/modelcard.py | 2 +- src/transformers/modeling_auto.py | 8 +++-- src/transformers/modeling_flax_utils.py | 3 +- src/transformers/modeling_tf_auto.py | 3 -- src/transformers/modeling_tf_utils.py | 5 --- src/transformers/modeling_utils.py | 7 ++-- src/transformers/tokenization_utils_base.py | 2 +- tests/test_modeling_tf_bert.py | 2 +- tests/test_modeling_tf_longformer.py | 4 +-- 16 files changed, 35 insertions(+), 58 deletions(-) diff --git a/model_cards/t5-11b-README.md b/model_cards/t5-11b-README.md index 7ddf3c2768fc..131667018caa 100644 --- a/model_cards/t5-11b-README.md +++ b/model_cards/t5-11b-README.md @@ -12,8 +12,8 @@ inference: false ## Disclaimer -Due do it's immense size, `t5-11b` requires some special treatment. -First, `t5-11b` should be loaded with flag `use_cdn` set to `False` as follows: +**Before `transformers` v3.5.0**, due do its immense size, `t5-11b` required some special treatment. +If you're using transformers `<= v3.4.0`, `t5-11b` should be loaded with flag `use_cdn` set to `False` as follows: ```python t5 = transformers.T5ForConditionalGeneration.from_pretrained('t5-11b', use_cdn = False) diff --git a/scripts/fsmt/convert-allenai-wmt16.sh b/scripts/fsmt/convert-allenai-wmt16.sh index 5f45c883e8b1..ee76a4df189f 100755 --- a/scripts/fsmt/convert-allenai-wmt16.sh +++ b/scripts/fsmt/convert-allenai-wmt16.sh @@ -56,7 +56,3 @@ cd - perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for ("wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json # add/remove files as needed -# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload -# So the only way to start using the new model sooner is either: -# 1. download it to a local path and use that path as model_name -# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere diff --git a/scripts/fsmt/convert-allenai-wmt19.sh b/scripts/fsmt/convert-allenai-wmt19.sh index 25f1fec94be3..7cd25e3cade0 100755 --- a/scripts/fsmt/convert-allenai-wmt19.sh +++ b/scripts/fsmt/convert-allenai-wmt19.sh @@ -44,7 +44,3 @@ cd - perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for ("wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json # add/remove files as needed -# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload -# So the only way to start using the new model sooner is either: -# 1. download it to a local path and use that path as model_name -# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere diff --git a/scripts/fsmt/convert-facebook-wmt19.sh b/scripts/fsmt/convert-facebook-wmt19.sh index 6edf51d2a454..f4f9a84b58f5 100755 --- a/scripts/fsmt/convert-facebook-wmt19.sh +++ b/scripts/fsmt/convert-facebook-wmt19.sh @@ -55,7 +55,3 @@ cd - perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for map { "wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json # add/remove files as needed -# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload -# So the only way to start using the new model sooner is either: -# 1. download it to a local path and use that path as model_name -# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index eb21fa203423..c03bc8892f3d 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -362,6 +362,7 @@ def get_config_dict(cls, pretrained_model_name_or_path: str, **kwargs) -> Tuple[ resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) if os.path.isdir(pretrained_model_name_or_path): config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) @@ -369,7 +370,7 @@ def get_config_dict(cls, pretrained_model_name_or_path: str, **kwargs) -> Tuple[ config_file = pretrained_model_name_or_path else: config_file = hf_bucket_url( - pretrained_model_name_or_path, filename=CONFIG_NAME, use_cdn=False, mirror=None + pretrained_model_name_or_path, filename=CONFIG_NAME, revision=revision, mirror=None ) try: diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 77247495520c..9bcf41f5db9c 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -248,10 +248,6 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder print("\nLast step is to upload the files to s3") print(f"cd {data_root}") print(f"transformers-cli upload {model_dir}") - print( - "Note: CDN caches files for up to 24h, so either use a local model path " - "or use `from_pretrained(mname, use_cdn=False)` to use the non-cached version." - ) if __name__ == "__main__": diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index d9f2ec0db686..2e6fab942936 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -217,6 +217,8 @@ S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co" +HUGGINGFACE_CO_PREFIX = "http://huggingface.test/{model_id}/resolve/{revision}/{filename}" +# HUGGINGFACE_CO_PREFIX = "https://moon-preprod.huggingface.co/{model_id}/resolve/{revision}/{filename}" PRESET_MIRROR_DICT = { "tuna": "https://mirrors.tuna.tsinghua.edu.cn/hugging-face-models", "bfsu": "https://mirrors.bfsu.edu.cn/hugging-face-models", @@ -825,31 +827,31 @@ def is_remote_url(url_or_filename): return parsed.scheme in ("http", "https") -def hf_bucket_url(model_id: str, filename: str, use_cdn=True, mirror=None) -> str: +def hf_bucket_url(model_id: str, filename: str, revision="main", mirror=None) -> str: """ - Resolve a model identifier, and a file name, to a HF-hosted url on either S3 or Cloudfront (a Content Delivery - Network, or CDN). + Resolve a model identifier, and a file name, to a huggingface.co-hosted url, potentially redirecting to Cloudfront + (a Content Delivery Network, or CDN). Cloudfront is replicated over the globe so downloads are way faster for the end user (and it also lowers our bandwidth costs). However, it is more aggressively cached by default, so may not always reflect the latest changes to the underlying file (default TTL is 24 hours). - In terms of client-side caching from this library, even though Cloudfront relays the ETags from S3, using one or + This is not an issue here however, because since migrating to git-based model versioning on huggingface.co, + we now store the files on S3/Cloudfront in a content-addressable way (i.e., the file name is its hash). + + TODO(update) In terms of client-side caching from this library, even though Cloudfront relays the ETags from S3, using one or the other (or switching from one to the other) will affect caching: cached files are not shared between the two because the cached file's name contains a hash of the url. """ - endpoint = ( - PRESET_MIRROR_DICT.get(mirror, mirror) - if mirror - else CLOUDFRONT_DISTRIB_PREFIX - if use_cdn - else S3_BUCKET_PREFIX - ) - legacy_format = "/" not in model_id - if legacy_format: - return f"{endpoint}/{model_id}-{filename}" - else: - return f"{endpoint}/{model_id}/{filename}" + if mirror: + endpoint = PRESET_MIRROR_DICT.get(mirror, mirror) + legacy_format = "/" not in model_id + if legacy_format: + return f"{endpoint}/{model_id}-{filename}" + else: + return f"{endpoint}/{model_id}/{filename}" + + return HUGGINGFACE_CO_PREFIX.format(model_id=model_id, revision=revision, filename=filename) def url_to_filename(url, etag=None): diff --git a/src/transformers/modelcard.py b/src/transformers/modelcard.py index da3f88f457d5..8760af7bf934 100644 --- a/src/transformers/modelcard.py +++ b/src/transformers/modelcard.py @@ -145,7 +145,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): model_card_file = pretrained_model_name_or_path else: model_card_file = hf_bucket_url( - pretrained_model_name_or_path, filename=MODEL_CARD_NAME, use_cdn=False, mirror=None + pretrained_model_name_or_path, filename=MODEL_CARD_NAME, mirror=None ) if find_from_standard_name or pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: diff --git a/src/transformers/modeling_auto.py b/src/transformers/modeling_auto.py index 3ec971325075..91b92d00df02 100644 --- a/src/transformers/modeling_auto.py +++ b/src/transformers/modeling_auto.py @@ -523,9 +523,11 @@ Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try downloading the model). - use_cdn(:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether or not to use Cloudfront (a Content Delivery Network, or CDN) when searching for the model on - our S3 (faster). Should be set to :obj:`False` for checkpoints larger than 20GB. + revision(:obj:`str`, `optional`, defaults to :obj:`main`): + Whether to pin to a specific model version (can be a branch name, a tag name, or a commit id). + We use a git-based model for storing models and other artefacts on huggingface.co, so ``revision`` + is any identifier allowed by git. + TODO(if agreed upon, duplicate this doc elsewhere.) kwargs (additional keyword arguments, `optional`): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., :obj:`output_attentions=True`). Behaves differently depending on whether a ``config`` is provided or diff --git a/src/transformers/modeling_flax_utils.py b/src/transformers/modeling_flax_utils.py index 6c9eb14ce40e..7087a334a7b4 100644 --- a/src/transformers/modeling_flax_utils.py +++ b/src/transformers/modeling_flax_utils.py @@ -107,7 +107,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): proxies = kwargs.pop("proxies", None) # output_loading_info = kwargs.pop("output_loading_info", False) local_files_only = kwargs.pop("local_files_only", False) - use_cdn = kwargs.pop("use_cdn", True) # Load config if we don't provide a configuration if not isinstance(config, PretrainedConfig): @@ -131,7 +130,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path else: - archive_file = hf_bucket_url(pretrained_model_name_or_path, filename=WEIGHTS_NAME, use_cdn=use_cdn) + archive_file = hf_bucket_url(pretrained_model_name_or_path, filename=WEIGHTS_NAME) # redirect to the cache, if necessary try: diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py index 6f0ea863a9ce..d05b887ec2e7 100644 --- a/src/transformers/modeling_tf_auto.py +++ b/src/transformers/modeling_tf_auto.py @@ -416,9 +416,6 @@ Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try downloading the model). - use_cdn(:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether or not to use Cloudfront (a Content Delivery Network, or CDN) when searching for the model on - our S3 (faster). Should be set to :obj:`False` for checkpoints larger than 20GB. kwargs (additional keyword arguments, `optional`): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., :obj:`output_attentions=True`). Behaves differently depending on whether a ``config`` is provided or diff --git a/src/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py index ab3523b8724e..2b0638ef3976 100644 --- a/src/transformers/modeling_tf_utils.py +++ b/src/transformers/modeling_tf_utils.py @@ -572,9 +572,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try doanloading the model). - use_cdn(:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether or not to use Cloudfront (a Content Delivery Network, or CDN) when searching for the model on - our S3 (faster). Should be set to :obj:`False` for checkpoints larger than 20GB. mirror(:obj:`str`, `optional`, defaults to :obj:`None`): Mirror source to accelerate downloads in China. If you are from China and have an accessibility problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. @@ -616,7 +613,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): proxies = kwargs.pop("proxies", None) output_loading_info = kwargs.pop("output_loading_info", False) local_files_only = kwargs.pop("local_files_only", False) - use_cdn = kwargs.pop("use_cdn", True) mirror = kwargs.pop("mirror", None) # Load config if we don't provide a configuration @@ -659,7 +655,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): archive_file = hf_bucket_url( pretrained_model_name_or_path, filename=(WEIGHTS_NAME if from_pt else TF2_WEIGHTS_NAME), - use_cdn=use_cdn, mirror=mirror, ) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index b7a87f99a179..26e0fc0c9a12 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -813,9 +813,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try doanloading the model). - use_cdn(:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether or not to use Cloudfront (a Content Delivery Network, or CDN) when searching for the model on - our S3 (faster). Should be set to :obj:`False` for checkpoints larger than 20GB. mirror(:obj:`str`, `optional`, defaults to :obj:`None`): Mirror source to accelerate downloads in China. If you are from China and have an accessibility problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. @@ -857,7 +854,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): proxies = kwargs.pop("proxies", None) output_loading_info = kwargs.pop("output_loading_info", False) local_files_only = kwargs.pop("local_files_only", False) - use_cdn = kwargs.pop("use_cdn", True) + revision = kwargs.pop("revision", None) mirror = kwargs.pop("mirror", None) # Load config if we don't provide a configuration @@ -909,7 +906,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): archive_file = hf_bucket_url( pretrained_model_name_or_path, filename=(TF2_WEIGHTS_NAME if from_tf else WEIGHTS_NAME), - use_cdn=use_cdn, + revision=revision, mirror=mirror, ) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 6a0156d608e3..51b87d30b7d1 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1601,7 +1601,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): full_file_name = None else: full_file_name = hf_bucket_url( - pretrained_model_name_or_path, filename=file_name, use_cdn=False, mirror=None + pretrained_model_name_or_path, filename=file_name, mirror=None ) vocab_files[file_id] = full_file_name diff --git a/tests/test_modeling_tf_bert.py b/tests/test_modeling_tf_bert.py index 7fbdb08c87c7..6fda686aea61 100644 --- a/tests/test_modeling_tf_bert.py +++ b/tests/test_modeling_tf_bert.py @@ -323,7 +323,7 @@ def test_model_from_pretrained(self): def test_custom_load_tf_weights(self): model, output_loading_info = TFBertForTokenClassification.from_pretrained( - "jplu/tiny-tf-bert-random", use_cdn=False, output_loading_info=True + "jplu/tiny-tf-bert-random", output_loading_info=True ) self.assertEqual(sorted(output_loading_info["unexpected_keys"]), ["mlm___cls", "nsp___cls"]) for layer in output_loading_info["missing_keys"]: diff --git a/tests/test_modeling_tf_longformer.py b/tests/test_modeling_tf_longformer.py index 0f07dc780f1d..6adaeef8b258 100644 --- a/tests/test_modeling_tf_longformer.py +++ b/tests/test_modeling_tf_longformer.py @@ -436,7 +436,7 @@ def test_chunk(self): tf.debugging.assert_near(chunked_hidden_states[0, 0, :, 0], expected_slice_along_chunk, rtol=1e-3) def test_layer_local_attn(self): - model = TFLongformerModel.from_pretrained("patrickvonplaten/longformer-random-tiny", use_cdn=False) + model = TFLongformerModel.from_pretrained("patrickvonplaten/longformer-random-tiny") layer = model.longformer.encoder.layer[0].attention.self_attention hidden_states = self._get_hidden_states() batch_size, seq_length, hidden_size = hidden_states.shape @@ -460,7 +460,7 @@ def test_layer_local_attn(self): tf.debugging.assert_near(output_hidden_states[0, 1], expected_slice, rtol=1e-3) def test_layer_global_attn(self): - model = TFLongformerModel.from_pretrained("patrickvonplaten/longformer-random-tiny", use_cdn=False) + model = TFLongformerModel.from_pretrained("patrickvonplaten/longformer-random-tiny") layer = model.longformer.encoder.layer[0].attention.self_attention hidden_states = self._get_hidden_states() From 1a037f0e096b00047f92dfa42625bfe58c275bcd Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 5 Nov 2020 10:45:06 +0100 Subject: [PATCH 03/18] I'm pretty sure we don't need to `read` this file --- src/transformers/file_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 2e6fab942936..53f8ed884ea7 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -1109,8 +1109,8 @@ def get_from_cache( incomplete_path = cache_path + ".incomplete" @contextmanager - def _resumable_file_manager(): - with open(incomplete_path, "a+b") as f: + def _resumable_file_manager() -> "io.BufferedWriter": + with open(incomplete_path, "ab") as f: yield f temp_file_manager = _resumable_file_manager From 0604e8303e4ad8a8c260c2ad3c4234e9a8c142dd Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 5 Nov 2020 10:45:50 +0100 Subject: [PATCH 04/18] same here --- src/transformers/file_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 53f8ed884ea7..9395faeb0cad 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -1119,7 +1119,7 @@ def _resumable_file_manager() -> "io.BufferedWriter": else: resume_size = 0 else: - temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) + temp_file_manager = partial(tempfile.NamedTemporaryFile, mode="wb", dir=cache_dir, delete=False) resume_size = 0 # Download to temporary file, then copy to cache dir once finished. From 2790c6ba3af5470d8f72da38cbd7a9e605a8723d Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 5 Nov 2020 13:17:12 +0100 Subject: [PATCH 05/18] [BIG] file_utils.networking: do not gobble up errors anymore --- src/transformers/configuration_utils.py | 5 +- src/transformers/file_utils.py | 99 ++++++++++++++------- src/transformers/modelcard.py | 6 +- src/transformers/modeling_auto.py | 7 +- src/transformers/modeling_tf_utils.py | 5 +- src/transformers/modeling_utils.py | 5 +- src/transformers/retrieval_rag.py | 2 - src/transformers/tokenization_utils_base.py | 53 ++++------- tests/test_file_utils.py | 38 ++++++++ 9 files changed, 133 insertions(+), 87 deletions(-) create mode 100644 tests/test_file_utils.py diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index c03bc8892f3d..b0e66adb1ac4 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -384,11 +384,10 @@ def get_config_dict(cls, pretrained_model_name_or_path: str, **kwargs) -> Tuple[ local_files_only=local_files_only, ) # Load config dict - if resolved_config_file is None: - raise EnvironmentError config_dict = cls._dict_from_json_file(resolved_config_file) - except EnvironmentError: + except EnvironmentError as err: + logger.error(err) msg = ( f"Can't load config for '{pretrained_model_name_or_path}'. Make sure that:\n\n" f"- '{pretrained_model_name_or_path}' is a correct model identifier listed on 'https://huggingface.co/models'\n\n" diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 9395faeb0cad..aad80119ff48 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -4,6 +4,7 @@ """ import fnmatch +import io import json import os import re @@ -17,7 +18,7 @@ from functools import partial, wraps from hashlib import sha256 from pathlib import Path -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, BinaryIO, Dict, Optional, Tuple, Union from urllib.parse import urlparse from zipfile import ZipFile, is_zipfile @@ -217,8 +218,8 @@ S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co" -HUGGINGFACE_CO_PREFIX = "http://huggingface.test/{model_id}/resolve/{revision}/{filename}" -# HUGGINGFACE_CO_PREFIX = "https://moon-preprod.huggingface.co/{model_id}/resolve/{revision}/{filename}" +# HUGGINGFACE_CO_PREFIX = "http://huggingface.test/{model_id}/resolve/{revision}/{filename}" +HUGGINGFACE_CO_PREFIX = "https://moon-preprod.huggingface.co/{model_id}/resolve/{revision}/{filename}" PRESET_MIRROR_DICT = { "tuna": "https://mirrors.tuna.tsinghua.edu.cn/hugging-face-models", "bfsu": "https://mirrors.bfsu.edu.cn/hugging-face-models", @@ -827,7 +828,7 @@ def is_remote_url(url_or_filename): return parsed.scheme in ("http", "https") -def hf_bucket_url(model_id: str, filename: str, revision="main", mirror=None) -> str: +def hf_bucket_url(model_id: str, filename: str, revision: Optional[str] = None, mirror=None) -> str: """ Resolve a model identifier, and a file name, to a huggingface.co-hosted url, potentially redirecting to Cloudfront (a Content Delivery Network, or CDN). @@ -836,12 +837,12 @@ def hf_bucket_url(model_id: str, filename: str, revision="main", mirror=None) -> bandwidth costs). However, it is more aggressively cached by default, so may not always reflect the latest changes to the underlying file (default TTL is 24 hours). - This is not an issue here however, because since migrating to git-based model versioning on huggingface.co, - we now store the files on S3/Cloudfront in a content-addressable way (i.e., the file name is its hash). + This is not an issue here however, because since migrating to git-based model versioning on huggingface.co, we now + store the files on S3/Cloudfront in a content-addressable way (i.e., the file name is its hash). - TODO(update) In terms of client-side caching from this library, even though Cloudfront relays the ETags from S3, using one or - the other (or switching from one to the other) will affect caching: cached files are not shared between the two - because the cached file's name contains a hash of the url. + TODO(update) In terms of client-side caching from this library, even though Cloudfront relays the ETags from S3, + using one or the other (or switching from one to the other) will affect caching: cached files are not shared + between the two because the cached file's name contains a hash of the url. """ if mirror: endpoint = PRESET_MIRROR_DICT.get(mirror, mirror) @@ -851,10 +852,12 @@ def hf_bucket_url(model_id: str, filename: str, revision="main", mirror=None) -> else: return f"{endpoint}/{model_id}/{filename}" + if revision is None: + revision = "main" return HUGGINGFACE_CO_PREFIX.format(model_id=model_id, revision=revision, filename=filename) -def url_to_filename(url, etag=None): +def url_to_filename(url: str, etag: Optional[str] = None) -> str: """ Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can @@ -862,13 +865,11 @@ def url_to_filename(url, etag=None): https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) """ url_bytes = url.encode("utf-8") - url_hash = sha256(url_bytes) - filename = url_hash.hexdigest() + filename = sha256(url_bytes).hexdigest() if etag: etag_bytes = etag.encode("utf-8") - etag_hash = sha256(etag_bytes) - filename += "." + etag_hash.hexdigest() + filename += "." + sha256(etag_bytes).hexdigest() if url.endswith(".h5"): filename += ".h5" @@ -929,8 +930,10 @@ def cached_path( re-extract the archive and override the folder where it was extracted. Return: - None in case of non-recoverable file (non-existent or inaccessible url + no cache on disk). Local path (string) - otherwise + Local path (string) of file or if networking is off, last version of file cached on disk. + + Raises: + In case of non-recoverable file (non-existent or inaccessible url + no cache on disk). """ if cache_dir is None: cache_dir = TRANSFORMERS_CACHE @@ -994,7 +997,10 @@ def cached_path( return output_path -def http_get(url, temp_file, proxies=None, resume_size=0, user_agent: Union[Dict, str, None] = None): +def http_user_agent(user_agent: Union[Dict, str, None] = None) -> str: + """ + Formats a user-agent string with basic info about a request. + """ ua = "transformers/{}; python/{}".format(__version__, sys.version.split()[0]) if is_torch_available(): ua += "; torch/{}".format(torch.__version__) @@ -1004,13 +1010,19 @@ def http_get(url, temp_file, proxies=None, resume_size=0, user_agent: Union[Dict ua += "; " + "; ".join("{}/{}".format(k, v) for k, v in user_agent.items()) elif isinstance(user_agent, str): ua += "; " + user_agent - headers = {"user-agent": ua} + return ua + + +def http_get(url: str, temp_file: BinaryIO, proxies=None, resume_size=0, user_agent: Union[Dict, str, None] = None): + """ + Donwload remote file. Do not gobble up errors. + """ + headers = {"user-agent": http_user_agent(user_agent)} if resume_size > 0: headers["Range"] = "bytes=%d-" % (resume_size,) - response = requests.get(url, stream=True, proxies=proxies, headers=headers) - if response.status_code == 416: # Range not satisfiable - return - content_length = response.headers.get("Content-Length") + r = requests.get(url, stream=True, proxies=proxies, headers=headers) + r.raise_for_status() + content_length = r.headers.get("Content-Length") total = resume_size + int(content_length) if content_length is not None else None progress = tqdm( unit="B", @@ -1020,7 +1032,7 @@ def http_get(url, temp_file, proxies=None, resume_size=0, user_agent: Union[Dict desc="Downloading", disable=bool(logging.get_verbosity() == logging.NOTSET), ) - for chunk in response.iter_content(chunk_size=1024): + for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks progress.update(len(chunk)) temp_file.write(chunk) @@ -1028,7 +1040,7 @@ def http_get(url, temp_file, proxies=None, resume_size=0, user_agent: Union[Dict def get_from_cache( - url, + url: str, cache_dir=None, force_download=False, proxies=None, @@ -1042,8 +1054,10 @@ def get_from_cache( path to the cached file. Return: - None in case of non-recoverable file (non-existent or inaccessible url + no cache on disk). Local path (string) - otherwise + Local path (string) of file or if networking is off, last version of file cached on disk. + + Raises: + In case of non-recoverable file (non-existent or inaccessible url + no cache on disk). """ if cache_dir is None: cache_dir = TRANSFORMERS_CACHE @@ -1052,13 +1066,28 @@ def get_from_cache( os.makedirs(cache_dir, exist_ok=True) + url_to_download = url etag = None if not local_files_only: try: - response = requests.head(url, allow_redirects=True, proxies=proxies, timeout=etag_timeout) - if response.status_code == 200: - etag = response.headers.get("ETag") - except (EnvironmentError, requests.exceptions.Timeout): + headers = {"user-agent": http_user_agent(user_agent)} + r = requests.head(url, headers=headers, allow_redirects=False, proxies=proxies, timeout=etag_timeout) + r.raise_for_status() + etag = r.headers.get("X-Linked-Etag") or r.headers.get("ETag") + # We favor a custom header indicating the etag of the linked resource, and + # we fallback to the regular etag header. + # If we don't have any of those, raise an error. + if etag is None: + raise OSError( + "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility." + ) + # In case of a redirect, + # save an extra redirect on the request.get call, + # and ensure we download the exact atomic version even if it changed + # between the HEAD and the GET (unlikely, but hey). + if 300 <= r.status_code <= 399: + url_to_download = r.headers["Location"] + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as err: # etag is already None pass @@ -1067,7 +1096,7 @@ def get_from_cache( # get cache path to put the file cache_path = os.path.join(cache_dir, filename) - # etag is None = we don't have a connection, or url doesn't exist, or is otherwise inaccessible. + # etag is None == we don't have a connection or we passed local_files_only. # try to get the last downloaded one if etag is None: if os.path.exists(cache_path): @@ -1090,7 +1119,11 @@ def get_from_cache( " disabled. To enable model look-ups and downloads online, set 'local_files_only'" " to False." ) - return None + else: + raise ValueError( + "Connection error, and we cannot find the requested files in the cached path." + " Please try again or make sure your Internet connection is on." + ) # From now on, etag is not None. if os.path.exists(cache_path) and not force_download: @@ -1127,7 +1160,7 @@ def _resumable_file_manager() -> "io.BufferedWriter": with temp_file_manager() as temp_file: logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) - http_get(url, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent) + http_get(url_to_download, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent) logger.info("storing %s in cache at %s", url, cache_path) os.replace(temp_file.name, cache_path) diff --git a/src/transformers/modelcard.py b/src/transformers/modelcard.py index 8760af7bf934..c0a3d1057824 100644 --- a/src/transformers/modelcard.py +++ b/src/transformers/modelcard.py @@ -144,9 +144,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): model_card_file = pretrained_model_name_or_path else: - model_card_file = hf_bucket_url( - pretrained_model_name_or_path, filename=MODEL_CARD_NAME, mirror=None - ) + model_card_file = hf_bucket_url(pretrained_model_name_or_path, filename=MODEL_CARD_NAME, mirror=None) if find_from_standard_name or pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: model_card_file = model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) @@ -156,8 +154,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): try: # Load from URL or cache if already cached resolved_model_card_file = cached_path(model_card_file, cache_dir=cache_dir, proxies=proxies) - if resolved_model_card_file is None: - raise EnvironmentError if resolved_model_card_file == model_card_file: logger.info("loading model card file {}".format(model_card_file)) else: diff --git a/src/transformers/modeling_auto.py b/src/transformers/modeling_auto.py index 91b92d00df02..798b2a1a2377 100644 --- a/src/transformers/modeling_auto.py +++ b/src/transformers/modeling_auto.py @@ -524,10 +524,9 @@ local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try downloading the model). revision(:obj:`str`, `optional`, defaults to :obj:`main`): - Whether to pin to a specific model version (can be a branch name, a tag name, or a commit id). - We use a git-based model for storing models and other artefacts on huggingface.co, so ``revision`` - is any identifier allowed by git. - TODO(if agreed upon, duplicate this doc elsewhere.) + Whether to pin to a specific model version (can be a branch name, a tag name, or a commit id). We use a + git-based model for storing models and other artefacts on huggingface.co, so ``revision`` is any + identifier allowed by git. TODO(if agreed upon, duplicate this doc elsewhere.) kwargs (additional keyword arguments, `optional`): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., :obj:`output_attentions=True`). Behaves differently depending on whether a ``config`` is provided or diff --git a/src/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py index 2b0638ef3976..c06911c6f6a1 100644 --- a/src/transformers/modeling_tf_utils.py +++ b/src/transformers/modeling_tf_utils.py @@ -668,9 +668,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): resume_download=resume_download, local_files_only=local_files_only, ) - if resolved_archive_file is None: - raise EnvironmentError - except EnvironmentError: + except EnvironmentError as err: + logger.error(err) msg = ( f"Can't load weights for '{pretrained_model_name_or_path}'. Make sure that:\n\n" f"- '{pretrained_model_name_or_path}' is a correct model identifier listed on 'https://huggingface.co/models'\n\n" diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 26e0fc0c9a12..8e656060925f 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -920,9 +920,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): resume_download=resume_download, local_files_only=local_files_only, ) - if resolved_archive_file is None: - raise EnvironmentError - except EnvironmentError: + except EnvironmentError as err: + logger.error(err) msg = ( f"Can't load weights for '{pretrained_model_name_or_path}'. Make sure that:\n\n" f"- '{pretrained_model_name_or_path}' is a correct model identifier listed on 'https://huggingface.co/models'\n\n" diff --git a/src/transformers/retrieval_rag.py b/src/transformers/retrieval_rag.py index 30e8c1c951e8..36c0d371a418 100644 --- a/src/transformers/retrieval_rag.py +++ b/src/transformers/retrieval_rag.py @@ -125,8 +125,6 @@ def _resolve_path(self, index_path, filename): try: # Load from URL or cache if already cached resolved_archive_file = cached_path(archive_file) - if resolved_archive_file is None: - raise EnvironmentError except EnvironmentError: msg = ( f"Can't load '{archive_file}'. Make sure that:\n\n" diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 51b87d30b7d1..689679c41d6a 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -29,6 +29,8 @@ import numpy as np +import requests + from .file_utils import ( add_end_docstrings, cached_path, @@ -1600,19 +1602,17 @@ def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) full_file_name = None else: - full_file_name = hf_bucket_url( - pretrained_model_name_or_path, filename=file_name, mirror=None - ) + full_file_name = hf_bucket_url(pretrained_model_name_or_path, filename=file_name, mirror=None) vocab_files[file_id] = full_file_name # Get files from url, cache, or disk depending on the case - try: - resolved_vocab_files = {} - for file_id, file_path in vocab_files.items(): - if file_path is None: - resolved_vocab_files[file_id] = None - else: + resolved_vocab_files = {} + for file_id, file_path in vocab_files.items(): + if file_path is None: + resolved_vocab_files[file_id] = None + else: + try: resolved_vocab_files[file_id] = cached_path( file_path, cache_dir=cache_dir, @@ -1621,34 +1621,19 @@ def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): resume_download=resume_download, local_files_only=local_files_only, ) - except EnvironmentError: - if pretrained_model_name_or_path in s3_models: - msg = "Couldn't reach server at '{}' to download vocabulary files." - else: - msg = ( - "Model name '{}' was not found in tokenizers model name list ({}). " - "We assumed '{}' was a path or url to a directory containing vocabulary files " - "named {}, but couldn't find such vocabulary files at this path or url.".format( - pretrained_model_name_or_path, - ", ".join(s3_models), - pretrained_model_name_or_path, - list(cls.vocab_files_names.values()), - ) - ) - - raise EnvironmentError(msg) + except requests.exceptions.HTTPError as err: + if "404 Client Error" in str(err): + logger.debug(err) + else: + raise err if all(full_file_name is None for full_file_name in resolved_vocab_files.values()): - raise EnvironmentError( - "Model name '{}' was not found in tokenizers model name list ({}). " - "We assumed '{}' was a path, a model identifier, or url to a directory containing vocabulary files " - "named {} but couldn't find such vocabulary files at this path or url.".format( - pretrained_model_name_or_path, - ", ".join(s3_models), - pretrained_model_name_or_path, - list(cls.vocab_files_names.values()), - ) + msg = ( + f"Can't load tokenizer for '{pretrained_model_name_or_path}'. Make sure that:\n\n" + f"- '{pretrained_model_name_or_path}' is a correct model identifier listed on 'https://huggingface.co/models'\n\n" + f"- or '{pretrained_model_name_or_path}' is the correct path to a directory containing relevant tokenizer files\n\n" ) + raise EnvironmentError(msg) for file_id, file_path in vocab_files.items(): if file_path == resolved_vocab_files[file_id]: diff --git a/tests/test_file_utils.py b/tests/test_file_utils.py new file mode 100644 index 000000000000..48ce23e33114 --- /dev/null +++ b/tests/test_file_utils.py @@ -0,0 +1,38 @@ +import unittest + +import requests +from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME, filename_to_url, get_from_cache, hf_bucket_url +from transformers.testing_utils import DUMMY_UNKWOWN_IDENTIFIER + + +MODEL_ID = DUMMY_UNKWOWN_IDENTIFIER +REVISION_ID = "main" +PINNED_SHA1 = "d9e9f15bc825e4b2c9249e9578f884bbcb5e3684" +PINNED_SHA256 = "4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3" + + +class GetFromCacheTests(unittest.TestCase): + def test_bogus_url(self): + # This lets us simulate no connection + # as the error raised is the same + # `ConnectionError` + url = "https://bogus" + with self.assertRaisesRegex(ValueError, "Connection error"): + _ = get_from_cache(url) + + def test_not_found(self): + url = hf_bucket_url(MODEL_ID, filename="missing.bin") + with self.assertRaisesRegex(requests.exceptions.HTTPError, "404 Client Error"): + _ = get_from_cache(url) + + def test_standard_object(self): + url = hf_bucket_url(MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID) + filepath = get_from_cache(url, force_download=True) + metadata = filename_to_url(filepath) + self.assertEqual(metadata, (url, f'"{PINNED_SHA1}"')) + + def test_lfs_object(self): + url = hf_bucket_url(MODEL_ID, filename=WEIGHTS_NAME, revision=REVISION_ID) + filepath = get_from_cache(url, force_download=True) + metadata = filename_to_url(filepath) + self.assertEqual(metadata, (url, f'"{PINNED_SHA256}"')) From d3c6382b3ef7e5eba19a83c86ae7788990d62e75 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 5 Nov 2020 13:35:16 +0100 Subject: [PATCH 06/18] =?UTF-8?q?Fix=20CI=20=F0=9F=98=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/transformers/file_utils.py | 2 +- src/transformers/tokenization_utils_base.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index aad80119ff48..753baba884ac 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -1087,7 +1087,7 @@ def get_from_cache( # between the HEAD and the GET (unlikely, but hey). if 300 <= r.status_code <= 399: url_to_download = r.headers["Location"] - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as err: + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): # etag is already None pass diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 689679c41d6a..85ebc7872afa 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1624,6 +1624,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): except requests.exceptions.HTTPError as err: if "404 Client Error" in str(err): logger.debug(err) + resolved_vocab_files[file_id] = None else: raise err From 604f7b3961bb93a8eb86ca5b685144815801cb27 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 12:37:42 +0100 Subject: [PATCH 07/18] Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/modeling_auto.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/modeling_auto.py b/src/transformers/modeling_auto.py index 798b2a1a2377..d6c78a906d67 100644 --- a/src/transformers/modeling_auto.py +++ b/src/transformers/modeling_auto.py @@ -523,10 +523,10 @@ Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try downloading the model). - revision(:obj:`str`, `optional`, defaults to :obj:`main`): - Whether to pin to a specific model version (can be a branch name, a tag name, or a commit id). We use a - git-based model for storing models and other artefacts on huggingface.co, so ``revision`` is any - identifier allowed by git. TODO(if agreed upon, duplicate this doc elsewhere.) + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based format for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. kwargs (additional keyword arguments, `optional`): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., :obj:`output_attentions=True`). Behaves differently depending on whether a ``config`` is provided or From d2143ae84c652c3e8c5790fd386c6ae80bf71ac4 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 12:52:11 +0100 Subject: [PATCH 08/18] Tiny doc tweak --- src/transformers/file_utils.py | 19 ++++++++++--------- src/transformers/modeling_auto.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 753baba884ac..842337380947 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -830,19 +830,20 @@ def is_remote_url(url_or_filename): def hf_bucket_url(model_id: str, filename: str, revision: Optional[str] = None, mirror=None) -> str: """ - Resolve a model identifier, and a file name, to a huggingface.co-hosted url, potentially redirecting to Cloudfront - (a Content Delivery Network, or CDN). + Resolve a model identifier, a file name, and an optional revision id, to a huggingface.co-hosted url, redirecting + to Cloudfront (a Content Delivery Network, or CDN) for large files. Cloudfront is replicated over the globe so downloads are way faster for the end user (and it also lowers our - bandwidth costs). However, it is more aggressively cached by default, so may not always reflect the latest changes - to the underlying file (default TTL is 24 hours). + bandwidth costs). - This is not an issue here however, because since migrating to git-based model versioning on huggingface.co, we now - store the files on S3/Cloudfront in a content-addressable way (i.e., the file name is its hash). + Cloudfront aggressively caches files by default (default TTL is 24 hours), however this is not an issue here + because we migrated to a git-based versioning system on huggingface.co, so we now store the files on S3/Cloudfront + in a content-addressable way (i.e., the file name is its hash). Using content-addressable filenames means cache + can't ever be stale. - TODO(update) In terms of client-side caching from this library, even though Cloudfront relays the ETags from S3, - using one or the other (or switching from one to the other) will affect caching: cached files are not shared - between the two because the cached file's name contains a hash of the url. + In terms of client-side caching from this library, we base our caching on the objects' ETag. An object' ETag is: + its sha1 if stored in git, or its sha256 if stored in git-lfs. Files cached locally from transformers before v3.5.0 + are not shared with those new files, because the cached file's name contains a hash of the url (which changed). """ if mirror: endpoint = PRESET_MIRROR_DICT.get(mirror, mirror) diff --git a/src/transformers/modeling_auto.py b/src/transformers/modeling_auto.py index d6c78a906d67..2638a8a67760 100644 --- a/src/transformers/modeling_auto.py +++ b/src/transformers/modeling_auto.py @@ -525,7 +525,7 @@ Whether or not to only look at local files (e.g., not try downloading the model). revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a - git-based format for storing models and other artifacts on huggingface.co, so ``revision`` can be any + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any identifier allowed by git. kwargs (additional keyword arguments, `optional`): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., From fe854127821e2edb80328ccedc1efca82d1e5d1f Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 13:11:23 +0100 Subject: [PATCH 09/18] Add doc + pass kwarg everywhere --- src/transformers/configuration_auto.py | 4 ++++ src/transformers/configuration_utils.py | 4 ++++ src/transformers/modeling_flax_utils.py | 21 ++++++++++----------- src/transformers/modeling_tf_auto.py | 4 ++++ src/transformers/modeling_tf_utils.py | 7 +++++++ src/transformers/modeling_utils.py | 5 +++++ src/transformers/tokenization_auto.py | 4 ++++ src/transformers/tokenization_utils_base.py | 9 ++++++++- 8 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/transformers/configuration_auto.py b/src/transformers/configuration_auto.py index 3e411ac37ec7..4689f38e72c4 100644 --- a/src/transformers/configuration_auto.py +++ b/src/transformers/configuration_auto.py @@ -289,6 +289,10 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): proxies (:obj:`Dict[str, str]`, `optional`): A dictionary of proxy servers to use by protocol or endpoint, e.g., :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. return_unused_kwargs (:obj:`bool`, `optional`, defaults to :obj:`False`): If :obj:`False`, then this function returns just the final configuration object. diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index b0e66adb1ac4..6934e02f512a 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -311,6 +311,10 @@ def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs) -> "Pretr proxies (:obj:`Dict[str, str]`, `optional`): A dictionary of proxy servers to use by protocol or endpoint, e.g., :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. return_unused_kwargs (:obj:`bool`, `optional`, defaults to :obj:`False`): If :obj:`False`, then this function returns just the final configuration object. diff --git a/src/transformers/modeling_flax_utils.py b/src/transformers/modeling_flax_utils.py index 7087a334a7b4..c8275e43942d 100644 --- a/src/transformers/modeling_flax_utils.py +++ b/src/transformers/modeling_flax_utils.py @@ -107,6 +107,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): proxies = kwargs.pop("proxies", None) # output_loading_info = kwargs.pop("output_loading_info", False) local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) # Load config if we don't provide a configuration if not isinstance(config, PretrainedConfig): @@ -120,6 +121,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, + revision=revision, **kwargs, ) else: @@ -130,7 +132,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path else: - archive_file = hf_bucket_url(pretrained_model_name_or_path, filename=WEIGHTS_NAME) + archive_file = hf_bucket_url(pretrained_model_name_or_path, filename=WEIGHTS_NAME, revision=revision) # redirect to the cache, if necessary try: @@ -142,16 +144,13 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): resume_download=resume_download, local_files_only=local_files_only, ) - except EnvironmentError: - if pretrained_model_name_or_path in cls.pretrained_model_archive_map: - msg = f"Couldn't reach server at '{archive_file}' to download pretrained weights." - else: - msg = ( - f"Model name '{pretrained_model_name_or_path}' " - f"was not found in model name list ({', '.join(cls.pretrained_model_archive_map.keys())}). " - f"We assumed '{archive_file}' was a path or url to model weight files but " - "couldn't find any such file at this path or url." - ) + except EnvironmentError as err: + logger.error(err) + msg = ( + f"Can't load weights for '{pretrained_model_name_or_path}'. Make sure that:\n\n" + f"- '{pretrained_model_name_or_path}' is a correct model identifier listed on 'https://huggingface.co/models'\n\n" + f"- or '{pretrained_model_name_or_path}' is the correct path to a directory containing a file named one of {TF2_WEIGHTS_NAME}, {WEIGHTS_NAME}.\n\n" + ) raise EnvironmentError(msg) if resolved_archive_file == archive_file: diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py index d05b887ec2e7..1f91b5c1236d 100644 --- a/src/transformers/modeling_tf_auto.py +++ b/src/transformers/modeling_tf_auto.py @@ -416,6 +416,10 @@ Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try downloading the model). + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. kwargs (additional keyword arguments, `optional`): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., :obj:`output_attentions=True`). Behaves differently depending on whether a ``config`` is provided or diff --git a/src/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py index c06911c6f6a1..487f04aeb327 100644 --- a/src/transformers/modeling_tf_utils.py +++ b/src/transformers/modeling_tf_utils.py @@ -572,6 +572,10 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try doanloading the model). + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. mirror(:obj:`str`, `optional`, defaults to :obj:`None`): Mirror source to accelerate downloads in China. If you are from China and have an accessibility problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. @@ -613,6 +617,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): proxies = kwargs.pop("proxies", None) output_loading_info = kwargs.pop("output_loading_info", False) local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) mirror = kwargs.pop("mirror", None) # Load config if we don't provide a configuration @@ -627,6 +632,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, + revision=revision, **kwargs, ) else: @@ -655,6 +661,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): archive_file = hf_bucket_url( pretrained_model_name_or_path, filename=(WEIGHTS_NAME if from_pt else TF2_WEIGHTS_NAME), + revision=revision, mirror=mirror, ) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 8e656060925f..e0ac5eee0de0 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -813,6 +813,10 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to only look at local files (e.g., not try doanloading the model). + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. mirror(:obj:`str`, `optional`, defaults to :obj:`None`): Mirror source to accelerate downloads in China. If you are from China and have an accessibility problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. @@ -869,6 +873,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, + revision=revision, **kwargs, ) else: diff --git a/src/transformers/tokenization_auto.py b/src/transformers/tokenization_auto.py index 9cadfdfb3690..93c9fbfe64a9 100644 --- a/src/transformers/tokenization_auto.py +++ b/src/transformers/tokenization_auto.py @@ -276,6 +276,10 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): proxies (:obj:`Dict[str, str]`, `optional`): A dictionary of proxy servers to use by protocol or endpoint, e.g., :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. use_fast (:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to try to load the fast version of the tokenizer. kwargs (additional keyword arguments, `optional`): diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 85ebc7872afa..9420e553822e 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1517,6 +1517,10 @@ def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): proxies (:obj:`Dict[str, str], `optional`): A dictionary of proxy servers to use by protocol or endpoint, e.g., :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any + identifier allowed by git. inputs (additional positional arguments, `optional`): Will be passed along to the Tokenizer ``__init__`` method. kwargs (additional keyword arguments, `optional`): @@ -1551,6 +1555,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) s3_models = list(cls.max_model_input_sizes.keys()) vocab_files = {} @@ -1602,7 +1607,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) full_file_name = None else: - full_file_name = hf_bucket_url(pretrained_model_name_or_path, filename=file_name, mirror=None) + full_file_name = hf_bucket_url( + pretrained_model_name_or_path, filename=file_name, revision=revision, mirror=None + ) vocab_files[file_id] = full_file_name From 8d3cd55f64af92a0b518c17eb1d4b689d71978e9 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 13:29:39 +0100 Subject: [PATCH 10/18] Add more tests and explain cc @sshleifer let me know if better Co-Authored-By: Sam Shleifer --- tests/test_file_utils.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/test_file_utils.py b/tests/test_file_utils.py index 48ce23e33114..2080d7871109 100644 --- a/tests/test_file_utils.py +++ b/tests/test_file_utils.py @@ -6,9 +6,19 @@ MODEL_ID = DUMMY_UNKWOWN_IDENTIFIER -REVISION_ID = "main" +# An actual model hosted on huggingface.co + +REVISION_ID_DEFAULT = "main" +# Default branch name +REVISION_ID_ONE_SPECIFIC_COMMIT = "432fec7515a022825ab3b79096838778e30aac17" +# One particular commit +REVISION_ID_INVALID = "aaaaaaa" +# This commit does not exist, so we should 404. + PINNED_SHA1 = "d9e9f15bc825e4b2c9249e9578f884bbcb5e3684" +# Sha-1 of config.json on the top of `main`, for checking purposes PINNED_SHA256 = "4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3" +# Sha-256 of pytorch_model.bin on the top of `main`, for checking purposes class GetFromCacheTests(unittest.TestCase): @@ -20,19 +30,34 @@ def test_bogus_url(self): with self.assertRaisesRegex(ValueError, "Connection error"): _ = get_from_cache(url) - def test_not_found(self): + def test_file_not_found(self): + # Valid revision (None) but missing file. url = hf_bucket_url(MODEL_ID, filename="missing.bin") with self.assertRaisesRegex(requests.exceptions.HTTPError, "404 Client Error"): _ = get_from_cache(url) + def test_revision_not_found(self): + # Valid file but missing revision + url = hf_bucket_url(MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_INVALID) + with self.assertRaisesRegex(requests.exceptions.HTTPError, "404 Client Error"): + _ = get_from_cache(url) + def test_standard_object(self): - url = hf_bucket_url(MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID) + url = hf_bucket_url(MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_DEFAULT) filepath = get_from_cache(url, force_download=True) metadata = filename_to_url(filepath) self.assertEqual(metadata, (url, f'"{PINNED_SHA1}"')) + def test_standard_object(self): + # Same object, but different revision + url = hf_bucket_url(MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_ONE_SPECIFIC_COMMIT) + filepath = get_from_cache(url, force_download=True) + metadata = filename_to_url(filepath) + self.assertNotEqual(metadata[1], f'"{PINNED_SHA1}"') + # Caution: check that the etag is *not* equal to the one from `test_standard_object` + def test_lfs_object(self): - url = hf_bucket_url(MODEL_ID, filename=WEIGHTS_NAME, revision=REVISION_ID) + url = hf_bucket_url(MODEL_ID, filename=WEIGHTS_NAME, revision=REVISION_ID_DEFAULT) filepath = get_from_cache(url, force_download=True) metadata = filename_to_url(filepath) self.assertEqual(metadata, (url, f'"{PINNED_SHA256}"')) From caa5aeba6dddec5d3059dd3037108021517debe0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 13:39:22 +0100 Subject: [PATCH 11/18] Also implement revision in pipelines In the case where we're passing a task name or a string model identifier --- src/transformers/pipelines.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/transformers/pipelines.py b/src/transformers/pipelines.py index 4ec85e1f4c19..f94498939e1b 100755 --- a/src/transformers/pipelines.py +++ b/src/transformers/pipelines.py @@ -86,7 +86,7 @@ logger = logging.get_logger(__name__) -def get_framework(model): +def get_framework(model, revision: Optional[str] = None): """ Select framework (TensorFlow or PyTorch) to use. @@ -103,14 +103,14 @@ def get_framework(model): ) if isinstance(model, str): if is_torch_available() and not is_tf_available(): - model = AutoModel.from_pretrained(model) + model = AutoModel.from_pretrained(model, revision=revision) elif is_tf_available() and not is_torch_available(): - model = TFAutoModel.from_pretrained(model) + model = TFAutoModel.from_pretrained(model, revision=revision) else: try: - model = AutoModel.from_pretrained(model) + model = AutoModel.from_pretrained(model, revision=revision) except OSError: - model = TFAutoModel.from_pretrained(model) + model = TFAutoModel.from_pretrained(model, revision=revision) framework = "tf" if model.__class__.__name__.startswith("TF") else "pt" return framework @@ -2730,6 +2730,7 @@ def pipeline( config: Optional[Union[str, PretrainedConfig]] = None, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, framework: Optional[str] = None, + revision: Optional[str] = None, use_fast: bool = False, **kwargs ) -> Pipeline: @@ -2784,6 +2785,10 @@ def pipeline( If no framework is specified, will default to the one currently installed. If no framework is specified and both frameworks are installed, will default to the framework of the :obj:`model`, or to PyTorch if no model is provided. + revision(:obj:`str`, `optional`, defaults to :obj:`"main"`): + When passing a task name or a string model identifier: The specific model version to use. It can be a + branch name, a tag name, or a commit id, since we use a git-based system for storing models and other + artifacts on huggingface.co, so ``revision`` can be any identifier allowed by git. use_fast (:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or not to use a Fast tokenizer if possible (a :class:`~transformers.PreTrainedTokenizerFast`). kwargs: @@ -2845,17 +2850,19 @@ def pipeline( if isinstance(tokenizer, tuple): # For tuple we have (tokenizer name, {kwargs}) use_fast = tokenizer[1].pop("use_fast", use_fast) - tokenizer = AutoTokenizer.from_pretrained(tokenizer[0], use_fast=use_fast, **tokenizer[1]) + tokenizer = AutoTokenizer.from_pretrained( + tokenizer[0], use_fast=use_fast, revision=revision, **tokenizer[1] + ) else: - tokenizer = AutoTokenizer.from_pretrained(tokenizer, use_fast=use_fast) + tokenizer = AutoTokenizer.from_pretrained(tokenizer, revision=revision, use_fast=use_fast) # Instantiate config if needed if isinstance(config, str): - config = AutoConfig.from_pretrained(config) + config = AutoConfig.from_pretrained(config, revision=revision) # Instantiate modelcard if needed if isinstance(modelcard, str): - modelcard = ModelCard.from_pretrained(modelcard) + modelcard = ModelCard.from_pretrained(modelcard, revision=revision) # Instantiate model if needed if isinstance(model, str): @@ -2873,7 +2880,7 @@ def pipeline( "Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. " "Trying to load the model with Tensorflow." ) - model = model_class.from_pretrained(model, config=config, **model_kwargs) + model = model_class.from_pretrained(model, config=config, revision=revision, **model_kwargs) if task == "translation" and model.config.task_specific_params: for key in model.config.task_specific_params: if key.startswith("translation"): From 7610640fd956f5df273ceb1f95858a55508835e7 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 13:47:52 +0100 Subject: [PATCH 12/18] =?UTF-8?q?Fix=20CI=20=F0=9F=98=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_file_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_file_utils.py b/tests/test_file_utils.py index 2080d7871109..65046f1e7dbe 100644 --- a/tests/test_file_utils.py +++ b/tests/test_file_utils.py @@ -48,7 +48,7 @@ def test_standard_object(self): metadata = filename_to_url(filepath) self.assertEqual(metadata, (url, f'"{PINNED_SHA1}"')) - def test_standard_object(self): + def test_standard_object_rev(self): # Same object, but different revision url = hf_bucket_url(MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_ONE_SPECIFIC_COMMIT) filepath = get_from_cache(url, force_download=True) From 35329c8e91997690884abfe357f382b193a8fa8b Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 14:05:53 +0100 Subject: [PATCH 13/18] Fix CI --- src/transformers/modeling_flax_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_flax_utils.py b/src/transformers/modeling_flax_utils.py index c8275e43942d..163bb4f2ef6e 100644 --- a/src/transformers/modeling_flax_utils.py +++ b/src/transformers/modeling_flax_utils.py @@ -149,7 +149,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): msg = ( f"Can't load weights for '{pretrained_model_name_or_path}'. Make sure that:\n\n" f"- '{pretrained_model_name_or_path}' is a correct model identifier listed on 'https://huggingface.co/models'\n\n" - f"- or '{pretrained_model_name_or_path}' is the correct path to a directory containing a file named one of {TF2_WEIGHTS_NAME}, {WEIGHTS_NAME}.\n\n" + f"- or '{pretrained_model_name_or_path}' is the correct path to a directory containing a file named {WEIGHTS_NAME}.\n\n" ) raise EnvironmentError(msg) From 87d9087652c72ba17322a0e313e2a92b3e88661f Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 16:43:15 +0100 Subject: [PATCH 14/18] [hf_api] new methods + command line implem --- src/transformers/commands/user.py | 157 +++++++++++++++++++++++++----- src/transformers/hf_api.py | 105 +++++++++++++++----- tests/test_hf_api.py | 27 ++--- 3 files changed, 230 insertions(+), 59 deletions(-) diff --git a/src/transformers/commands/user.py b/src/transformers/commands/user.py index fa4f6dafd87d..32ed016269c0 100644 --- a/src/transformers/commands/user.py +++ b/src/transformers/commands/user.py @@ -3,6 +3,7 @@ from argparse import ArgumentParser from getpass import getpass from typing import List, Union +import subprocess from requests.exceptions import HTTPError from transformers.commands import BaseTransformersCLICommand @@ -21,8 +22,8 @@ def register_subcommand(parser: ArgumentParser): whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args)) logout_parser = parser.add_parser("logout", help="Log out") logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) - # s3 - s3_parser = parser.add_parser("s3", help="{ls, rm} Commands to interact with the files you upload on S3.") + # s3_datasets (s3-based system) + s3_parser = parser.add_parser("s3_datasets", help="{ls, rm} Commands to interact with the files you upload on S3.") s3_subparsers = s3_parser.add_subparsers(help="s3 related commands") ls_parser = s3_subparsers.add_parser("ls") ls_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") @@ -31,10 +32,9 @@ def register_subcommand(parser: ArgumentParser): rm_parser.add_argument("filename", type=str, help="individual object filename to delete from S3.") rm_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") rm_parser.set_defaults(func=lambda args: DeleteObjCommand(args)) - # upload - upload_parser = parser.add_parser("upload", help="Upload a model to S3.") + upload_parser = s3_subparsers.add_parser("upload", help="Upload a file to S3.") upload_parser.add_argument( - "path", type=str, help="Local path of the model folder or individual file to upload." + "path", type=str, help="Local path of the folder or individual file to upload." ) upload_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") upload_parser.add_argument( @@ -42,6 +42,27 @@ def register_subcommand(parser: ArgumentParser): ) upload_parser.add_argument("-y", "--yes", action="store_true", help="Optional: answer Yes to the prompt") upload_parser.set_defaults(func=lambda args: UploadCommand(args)) + # deprecated model upload + upload_parser = parser.add_parser("upload", help=( + "Deprecated: used to be the way to upload a model to S3." + " We now use a git-based system for storing models and other artifacts." + " Use the `repo create` command instead." + )) + upload_parser.set_defaults(func=lambda args: DeprecatedUploadCommand(args)) + + # new system: git-based repo system + repo_parser = parser.add_parser("repo", help="{create, ls-files} Commands to interact with your huggingface.co repos.") + repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands") + ls_parser = repo_subparsers.add_parser("ls-files", help="List all your files on huggingface.co") + ls_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") + ls_parser.set_defaults(func=lambda args: ListReposObjsCommand(args)) + repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co") + repo_create_parser.add_argument( + "name", type=str, help="Name for your model's repo. Will be namespaced under your username to build the model id." + ) + repo_create_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") + repo_create_parser.add_argument("-y", "--yes", action="store_true", help="Optional: answer Yes to the prompt") + repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args)) class ANSI: @@ -51,6 +72,7 @@ class ANSI: _bold = "\u001b[1m" _red = "\u001b[31m" + _gray = "\u001b[90m" _reset = "\u001b[0m" @classmethod @@ -61,6 +83,28 @@ def bold(cls, s): def red(cls, s): return "{}{}{}".format(cls._bold + cls._red, s, cls._reset) + @classmethod + def gray(cls, s): + return "{}{}{}".format(cls._gray, s, cls._reset) + + +def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: + """ + Inspired by: + + - stackoverflow.com/a/8356620/593036 + - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + row_format = ("{{:{}}} " * len(headers)).format(*col_widths) + lines = [] + lines.append(row_format.format(*headers)) + lines.append(row_format.format(*["-" * w for w in col_widths])) + for row in rows: + lines.append(row_format.format(*row)) + return "\n".join(lines) + + class BaseUserCommand: def __init__(self, args): @@ -123,23 +167,8 @@ def run(self): print("Successfully logged out.") -class ListObjsCommand(BaseUserCommand): - def tabulate(self, rows: List[List[Union[str, int]]], headers: List[str]) -> str: - """ - Inspired by: - - - stackoverflow.com/a/8356620/593036 - - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data - """ - col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] - row_format = ("{{:{}}} " * len(headers)).format(*col_widths) - lines = [] - lines.append(row_format.format(*headers)) - lines.append(row_format.format(*["-" * w for w in col_widths])) - for row in rows: - lines.append(row_format.format(*row)) - return "\n".join(lines) +class ListObjsCommand(BaseUserCommand): def run(self): token = HfFolder.get_token() if token is None: @@ -155,7 +184,7 @@ def run(self): print("No shared file yet") exit() rows = [[obj.filename, obj.LastModified, obj.ETag, obj.Size] for obj in objs] - print(self.tabulate(rows, headers=["Filename", "LastModified", "ETag", "Size"])) + print(tabulate(rows, headers=["Filename", "LastModified", "ETag", "Size"])) class DeleteObjCommand(BaseUserCommand): @@ -173,6 +202,90 @@ def run(self): print("Done") +class ListReposObjsCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit(1) + try: + objs = self._api.list_repos_objs(token, organization=self.args.organization) + except HTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + exit(1) + if len(objs) == 0: + print("No shared file yet") + exit() + rows = [[obj.filename, obj.lastModified, obj.commit, obj.size] for obj in objs] + print(tabulate(rows, headers=["Filename", "LastModified", "Commit-Sha", "Size"])) + + +class RepoCreateCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit(1) + try: + stdout = subprocess.run(["git", "--version"], capture_output=True).stdout.decode("utf-8") + print(ANSI.gray(stdout.strip())) + except FileNotFoundError: + print("Looks like you do not have git installed, please install.") + + try: + stdout = subprocess.run(["git-lfs", "--version"], capture_output=True).stdout.decode("utf-8") + print(ANSI.gray(stdout.strip())) + except FileNotFoundError: + print(ANSI.red( + "Looks like you do not have git-lfs installed, please install." + " You can install from https://git-lfs.github.com/." + " Then run `git lfs install` (you only have to do this once)." + )) + print("") + + user, _ = self._api.whoami(token) + namespace = self.args.organization if self.args.organization is not None else user + + print( + "You are about to create {}".format( + ANSI.bold(namespace + "/" + self.args.name) + ) + ) + + if not self.args.yes: + choice = input("Proceed? [Y/n] ").lower() + if not (choice == "" or choice == "y" or choice == "yes"): + print("Abort") + exit() + try: + url = self._api.create_repo(token, name=self.args.name, organization=self.args.organization) + except HTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + exit(1) + print("\nYour repo now lives at:") + print(" {}".format(ANSI.bold(url))) + print( + "\nYou can clone it locally with the command below," + " and commit/push as usual." + ) + print(f"\n git clone {url}") + print("") + + +class DeprecatedUploadCommand(BaseUserCommand): + def run(self): + print( + ANSI.red( + "Deprecated: used to be the way to upload a model to S3." + " We now use a git-based system for storing models and other artifacts." + " Use the `repo create` command instead." + ) + ) + exit(1) + + class UploadCommand(BaseUserCommand): def walk_dir(self, rel_path): """ diff --git a/src/transformers/hf_api.py b/src/transformers/hf_api.py index c8a4def10b5c..622054e92257 100644 --- a/src/transformers/hf_api.py +++ b/src/transformers/hf_api.py @@ -24,12 +24,26 @@ import requests -ENDPOINT = "https://huggingface.co" +# ENDPOINT = "https://huggingface.co" +# ENDPOINT = "http://huggingface.test" +ENDPOINT = "https://moon-preprod.huggingface.co" + + +class RepoObj: + """ + HuggingFace git-based system, data structure that represents a file belonging to the current user. + """ + + def __init__(self, filename: str, lastModified: str, commit: str, size: int, **kwargs): + self.filename = filename + self.lastModified = lastModified + self.commit = commit + self.size = size class S3Obj: """ - Data structure that represents a file belonging to the current user. + HuggingFace S3-based system, data structure that represents a file belonging to the current user. """ def __init__(self, filename: str, LastModified: str, ETag: str, Size: int, **kwargs): @@ -46,38 +60,25 @@ def __init__(self, write: str, access: str, type: str, **kwargs): self.type = type # mime-type to send to S3. -class S3Object: +class ModelSibling: """ - Data structure that represents a public file accessible on our S3. + Data structure that represents a public file inside a model, accessible from huggingface.co """ - def __init__( - self, - key: str, # S3 object key - etag: str, - lastModified: str, - size: int, - rfilename: str, # filename relative to config.json - **kwargs - ): - self.key = key - self.etag = etag - self.lastModified = lastModified - self.size = size - self.rfilename = rfilename + def __init__(self, rfilename: str, **kwargs): + self.rfilename = rfilename # filename relative to the model root for k, v in kwargs.items(): setattr(self, k, v) class ModelInfo: """ - Info about a public model accessible from our S3. + Info about a public model accessible from huggingface.co """ def __init__( self, - modelId: str, # id of model - key: str, # S3 object key of config.json + modelId: Optional[str] = None, # id of model author: Optional[str] = None, downloads: Optional[int] = None, tags: List[str] = [], @@ -86,12 +87,11 @@ def __init__( **kwargs ): self.modelId = modelId - self.key = key self.author = author self.downloads = downloads self.tags = tags self.pipeline_tag = pipeline_tag - self.siblings = [S3Object(**x) for x in siblings] if siblings is not None else None + self.siblings = [ModelSibling(**x) for x in siblings] if siblings is not None else None for k, v in kwargs.items(): setattr(self, k, v) @@ -134,9 +134,11 @@ def logout(self, token: str) -> None: def presign(self, token: str, filename: str, organization: Optional[str] = None) -> PresignedUrl: """ + HuggingFace S3-based system, used for datasets and metrics. + Call HF API to get a presigned url to upload `filename` to S3. """ - path = "{}/api/presign".format(self.endpoint) + path = "{}/api/datasets/presign".format(self.endpoint) r = requests.post( path, headers={"authorization": "Bearer {}".format(token)}, @@ -148,6 +150,8 @@ def presign(self, token: str, filename: str, organization: Optional[str] = None) def presign_and_upload(self, token: str, filename: str, filepath: str, organization: Optional[str] = None) -> str: """ + HuggingFace S3-based system, used for datasets and metrics. + Get a presigned url, then upload file to S3. Outputs: url: Read-only url for the stored file on S3. @@ -169,9 +173,11 @@ def presign_and_upload(self, token: str, filename: str, filepath: str, organizat def list_objs(self, token: str, organization: Optional[str] = None) -> List[S3Obj]: """ + HuggingFace S3-based system, used for datasets and metrics. + Call HF API to list all stored files for user (or one of their organizations). """ - path = "{}/api/listObjs".format(self.endpoint) + path = "{}/api/datasets/listObjs".format(self.endpoint) params = {"organization": organization} if organization is not None else None r = requests.get(path, params=params, headers={"authorization": "Bearer {}".format(token)}) r.raise_for_status() @@ -180,9 +186,11 @@ def list_objs(self, token: str, organization: Optional[str] = None) -> List[S3Ob def delete_obj(self, token: str, filename: str, organization: Optional[str] = None): """ + HuggingFace S3-based system, used for datasets and metrics. + Call HF API to delete a file stored by user """ - path = "{}/api/deleteObj".format(self.endpoint) + path = "{}/api/datasets/deleteObj".format(self.endpoint) r = requests.delete( path, headers={"authorization": "Bearer {}".format(token)}, @@ -200,6 +208,51 @@ def model_list(self) -> List[ModelInfo]: d = r.json() return [ModelInfo(**x) for x in d] + def list_repos_objs(self, token: str, organization: Optional[str] = None) -> List[S3Obj]: + """ + HuggingFace git-based system, used for models. + + Call HF API to list all stored files for user (or one of their organizations). + """ + path = "{}/api/repos/ls".format(self.endpoint) + params = {"organization": organization} if organization is not None else None + r = requests.get(path, params=params, headers={"authorization": "Bearer {}".format(token)}) + r.raise_for_status() + d = r.json() + return [RepoObj(**x) for x in d] + + def create_repo(self, token: str, name: str, organization: Optional[str] = None) -> str: + """ + HuggingFace git-based system, used for models. + + Call HF API to create a whole repo. + """ + path = "{}/api/repos/create".format(self.endpoint) + r = requests.post( + path, + headers={"authorization": "Bearer {}".format(token)}, + json={"name": name, "organization": organization}, + ) + r.raise_for_status() + d = r.json() + return d["url"] + + def delete_repo(self, token: str, name: str, organization: Optional[str] = None): + """ + HuggingFace git-based system, used for models. + + Call HF API to delete a whole repo. + + CAUTION(this is irreversible). + """ + path = "{}/api/repos/delete".format(self.endpoint) + r = requests.delete( + path, + headers={"authorization": "Bearer {}".format(token)}, + json={"name": name, "organization": organization}, + ) + r.raise_for_status() + class TqdmProgressFileReader: """ diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index a68023cda33d..9f5ac6432a11 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -20,7 +20,7 @@ import requests from requests.exceptions import HTTPError -from transformers.hf_api import HfApi, HfFolder, ModelInfo, PresignedUrl, S3Obj +from transformers.hf_api import HfApi, HfFolder, ModelInfo, PresignedUrl, RepoObj, S3Obj USER = "__DUMMY_TRANSFORMERS_USER__" @@ -35,7 +35,10 @@ os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/empty.txt"), ), ] -ENDPOINT_STAGING = "https://moon-staging.huggingface.co" +REPO_NAME = "my-model-{}".format(int(time.time())) +# ENDPOINT_STAGING = "https://moon-staging.huggingface.co" +ENDPOINT_STAGING = "https://moon-preprod.huggingface.co" +# ENDPOINT_STAGING = "http://huggingface.test" class HfApiCommonTest(unittest.TestCase): @@ -78,15 +81,6 @@ def test_presign_valid_org(self): urls = self._api.presign(token=self._token, filename="nested/valid_org.txt", organization="valid_org") self.assertIsInstance(urls, PresignedUrl) - def test_presign_invalid(self): - try: - _ = self._api.presign(token=self._token, filename="non_nested.json") - except HTTPError as e: - self.assertIsNotNone(e.response.text) - self.assertTrue("Filename invalid" in e.response.text) - else: - self.fail("Expected an exception") - def test_presign(self): for FILE_KEY, FILE_PATH in FILES: urls = self._api.presign(token=self._token, filename=FILE_KEY) @@ -109,6 +103,17 @@ def test_list_objs(self): o = objs[-1] self.assertIsInstance(o, S3Obj) + def test_list_repos_objs(self): + objs = self._api.list_repos_objs(token=self._token) + self.assertIsInstance(objs, list) + if len(objs) > 0: + o = objs[-1] + self.assertIsInstance(o, RepoObj) + + def test_create_and_delete_repo(self): + self._api.create_repo(token=self._token, name=REPO_NAME) + self._api.delete_repo(token=self._token, name=REPO_NAME) + class HfApiPublicTest(unittest.TestCase): def test_staging_model_list(self): From 0e42b97e668a586e89297230fa46ed2d602effd9 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Nov 2020 16:50:16 +0100 Subject: [PATCH 15/18] make style --- src/transformers/commands/user.py | 64 +++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/transformers/commands/user.py b/src/transformers/commands/user.py index 32ed016269c0..77a194feaf90 100644 --- a/src/transformers/commands/user.py +++ b/src/transformers/commands/user.py @@ -1,9 +1,9 @@ import os +import subprocess import sys from argparse import ArgumentParser from getpass import getpass from typing import List, Union -import subprocess from requests.exceptions import HTTPError from transformers.commands import BaseTransformersCLICommand @@ -23,7 +23,9 @@ def register_subcommand(parser: ArgumentParser): logout_parser = parser.add_parser("logout", help="Log out") logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) # s3_datasets (s3-based system) - s3_parser = parser.add_parser("s3_datasets", help="{ls, rm} Commands to interact with the files you upload on S3.") + s3_parser = parser.add_parser( + "s3_datasets", help="{ls, rm} Commands to interact with the files you upload on S3." + ) s3_subparsers = s3_parser.add_subparsers(help="s3 related commands") ls_parser = s3_subparsers.add_parser("ls") ls_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") @@ -33,9 +35,7 @@ def register_subcommand(parser: ArgumentParser): rm_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") rm_parser.set_defaults(func=lambda args: DeleteObjCommand(args)) upload_parser = s3_subparsers.add_parser("upload", help="Upload a file to S3.") - upload_parser.add_argument( - "path", type=str, help="Local path of the folder or individual file to upload." - ) + upload_parser.add_argument("path", type=str, help="Local path of the folder or individual file to upload.") upload_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") upload_parser.add_argument( "--filename", type=str, default=None, help="Optional: override individual object filename on S3." @@ -43,22 +43,29 @@ def register_subcommand(parser: ArgumentParser): upload_parser.add_argument("-y", "--yes", action="store_true", help="Optional: answer Yes to the prompt") upload_parser.set_defaults(func=lambda args: UploadCommand(args)) # deprecated model upload - upload_parser = parser.add_parser("upload", help=( - "Deprecated: used to be the way to upload a model to S3." - " We now use a git-based system for storing models and other artifacts." - " Use the `repo create` command instead." - )) + upload_parser = parser.add_parser( + "upload", + help=( + "Deprecated: used to be the way to upload a model to S3." + " We now use a git-based system for storing models and other artifacts." + " Use the `repo create` command instead." + ), + ) upload_parser.set_defaults(func=lambda args: DeprecatedUploadCommand(args)) # new system: git-based repo system - repo_parser = parser.add_parser("repo", help="{create, ls-files} Commands to interact with your huggingface.co repos.") + repo_parser = parser.add_parser( + "repo", help="{create, ls-files} Commands to interact with your huggingface.co repos." + ) repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands") ls_parser = repo_subparsers.add_parser("ls-files", help="List all your files on huggingface.co") ls_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") ls_parser.set_defaults(func=lambda args: ListReposObjsCommand(args)) repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co") repo_create_parser.add_argument( - "name", type=str, help="Name for your model's repo. Will be namespaced under your username to build the model id." + "name", + type=str, + help="Name for your model's repo. Will be namespaced under your username to build the model id.", ) repo_create_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") repo_create_parser.add_argument("-y", "--yes", action="store_true", help="Optional: answer Yes to the prompt") @@ -105,7 +112,6 @@ def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: return "\n".join(lines) - class BaseUserCommand: def __init__(self, args): self.args = args @@ -167,7 +173,6 @@ def run(self): print("Successfully logged out.") - class ListObjsCommand(BaseUserCommand): def run(self): token = HfFolder.get_token() @@ -237,21 +242,19 @@ def run(self): stdout = subprocess.run(["git-lfs", "--version"], capture_output=True).stdout.decode("utf-8") print(ANSI.gray(stdout.strip())) except FileNotFoundError: - print(ANSI.red( - "Looks like you do not have git-lfs installed, please install." - " You can install from https://git-lfs.github.com/." - " Then run `git lfs install` (you only have to do this once)." - )) + print( + ANSI.red( + "Looks like you do not have git-lfs installed, please install." + " You can install from https://git-lfs.github.com/." + " Then run `git lfs install` (you only have to do this once)." + ) + ) print("") - + user, _ = self._api.whoami(token) namespace = self.args.organization if self.args.organization is not None else user - print( - "You are about to create {}".format( - ANSI.bold(namespace + "/" + self.args.name) - ) - ) + print("You are about to create {}".format(ANSI.bold(namespace + "/" + self.args.name))) if not self.args.yes: choice = input("Proceed? [Y/n] ").lower() @@ -266,10 +269,7 @@ def run(self): exit(1) print("\nYour repo now lives at:") print(" {}".format(ANSI.bold(url))) - print( - "\nYou can clone it locally with the command below," - " and commit/push as usual." - ) + print("\nYou can clone it locally with the command below," " and commit/push as usual.") print(f"\n git clone {url}") print("") @@ -278,9 +278,9 @@ class DeprecatedUploadCommand(BaseUserCommand): def run(self): print( ANSI.red( - "Deprecated: used to be the way to upload a model to S3." - " We now use a git-based system for storing models and other artifacts." - " Use the `repo create` command instead." + "Deprecated: used to be the way to upload a model to S3." + " We now use a git-based system for storing models and other artifacts." + " Use the `repo create` command instead." ) ) exit(1) From 836710078b186f784e46f91313ca309751dd8715 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Nov 2020 12:46:09 +0100 Subject: [PATCH 16/18] Final endpoints post-migration --- src/transformers/file_utils.py | 4 ++-- src/transformers/hf_api.py | 4 +--- tests/test_hf_api.py | 4 +--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 842337380947..f6b63fa8962f 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -218,8 +218,8 @@ S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co" -# HUGGINGFACE_CO_PREFIX = "http://huggingface.test/{model_id}/resolve/{revision}/{filename}" -HUGGINGFACE_CO_PREFIX = "https://moon-preprod.huggingface.co/{model_id}/resolve/{revision}/{filename}" +HUGGINGFACE_CO_PREFIX = "https://huggingface.co/{model_id}/resolve/{revision}/{filename}" + PRESET_MIRROR_DICT = { "tuna": "https://mirrors.tuna.tsinghua.edu.cn/hugging-face-models", "bfsu": "https://mirrors.bfsu.edu.cn/hugging-face-models", diff --git a/src/transformers/hf_api.py b/src/transformers/hf_api.py index 622054e92257..d5f997760830 100644 --- a/src/transformers/hf_api.py +++ b/src/transformers/hf_api.py @@ -24,9 +24,7 @@ import requests -# ENDPOINT = "https://huggingface.co" -# ENDPOINT = "http://huggingface.test" -ENDPOINT = "https://moon-preprod.huggingface.co" +ENDPOINT = "https://huggingface.co" class RepoObj: diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index 9f5ac6432a11..fed1fc42f11c 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -36,9 +36,7 @@ ), ] REPO_NAME = "my-model-{}".format(int(time.time())) -# ENDPOINT_STAGING = "https://moon-staging.huggingface.co" -ENDPOINT_STAGING = "https://moon-preprod.huggingface.co" -# ENDPOINT_STAGING = "http://huggingface.test" +ENDPOINT_STAGING = "https://moon-staging.huggingface.co" class HfApiCommonTest(unittest.TestCase): From 9749636658c1e58052949bfbe6ede691a69d6b58 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Nov 2020 12:54:30 +0100 Subject: [PATCH 17/18] Fix post-migration --- tests/test_file_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_file_utils.py b/tests/test_file_utils.py index 65046f1e7dbe..da5709888556 100644 --- a/tests/test_file_utils.py +++ b/tests/test_file_utils.py @@ -10,8 +10,8 @@ REVISION_ID_DEFAULT = "main" # Default branch name -REVISION_ID_ONE_SPECIFIC_COMMIT = "432fec7515a022825ab3b79096838778e30aac17" -# One particular commit +REVISION_ID_ONE_SPECIFIC_COMMIT = "f2c752cfc5c0ab6f4bdec59acea69eefbee381c2" +# One particular commit (not the top of `main`) REVISION_ID_INVALID = "aaaaaaa" # This commit does not exist, so we should 404. From f0bc41d44e3910ffbb89b6dfe8d2d7f79c9584a4 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Nov 2020 13:09:41 +0100 Subject: [PATCH 18/18] Py3.6 compat cc @stefan-it Thank you @stas00 --- src/transformers/commands/user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/commands/user.py b/src/transformers/commands/user.py index 77a194feaf90..9124f80a8559 100644 --- a/src/transformers/commands/user.py +++ b/src/transformers/commands/user.py @@ -233,13 +233,13 @@ def run(self): print("Not logged in") exit(1) try: - stdout = subprocess.run(["git", "--version"], capture_output=True).stdout.decode("utf-8") + stdout = subprocess.check_output(["git", "--version"]).decode("utf-8") print(ANSI.gray(stdout.strip())) except FileNotFoundError: print("Looks like you do not have git installed, please install.") try: - stdout = subprocess.run(["git-lfs", "--version"], capture_output=True).stdout.decode("utf-8") + stdout = subprocess.check_output(["git-lfs", "--version"]).decode("utf-8") print(ANSI.gray(stdout.strip())) except FileNotFoundError: print(