From 6ea2650fcff9b21ffd587c14022d190420ac98c7 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 3 Sep 2020 23:29:28 -0700 Subject: [PATCH 001/109] ready for PR --- config.py | 212 +++ docs/source/index.rst | 9 +- docs/source/model_doc/fsmt.rst | 53 + model_cards/stas/fsmt-wmt19-de-en/README.md | 87 ++ model_cards/stas/fsmt-wmt19-en-de/README.md | 87 ++ model_cards/stas/fsmt-wmt19-en-ru/README.md | 87 ++ model_cards/stas/fsmt-wmt19-ru-en/README.md | 87 ++ src/transformers/__init__.py | 8 + src/transformers/configuration_auto.py | 6 + src/transformers/configuration_fsmt.py | 224 +++ ..._original_pytorch_checkpoint_to_pytorch.py | 398 +++++ src/transformers/generation_utils.py | 16 +- src/transformers/modeling_auto.py | 5 + src/transformers/modeling_fsmt.py | 1341 +++++++++++++++++ src/transformers/modeling_utils.py | 1 + src/transformers/tokenization_auto.py | 3 + src/transformers/tokenization_fsmt.py | 567 +++++++ src/transformers/utils/logging.py | 38 + tests/conftest.py | 28 + tests/test_modeling_fsmt.py | 509 +++++++ tests/test_tokenization_fsmt.py | 148 ++ 21 files changed, 3902 insertions(+), 12 deletions(-) create mode 100644 config.py create mode 100644 docs/source/model_doc/fsmt.rst create mode 100644 model_cards/stas/fsmt-wmt19-de-en/README.md create mode 100644 model_cards/stas/fsmt-wmt19-en-de/README.md create mode 100644 model_cards/stas/fsmt-wmt19-en-ru/README.md create mode 100644 model_cards/stas/fsmt-wmt19-ru-en/README.md create mode 100644 src/transformers/configuration_fsmt.py create mode 100755 src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py create mode 100644 src/transformers/modeling_fsmt.py create mode 100644 src/transformers/tokenization_fsmt.py create mode 100644 tests/test_modeling_fsmt.py create mode 100644 tests/test_tokenization_fsmt.py diff --git a/config.py b/config.py new file mode 100644 index 000000000000..c8bf8193d432 --- /dev/null +++ b/config.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# coding: utf-8 + +import os, sys +sys.path.insert(0, f"{os.getcwd()}/src") + + +import torch +from pprint import pprint +import fairseq + + +def dump_state_keys(state_dict): print("\n".join(state_dict.keys())) + + +# # Baseline + +#checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt' +checkpoint_file='model1.pt' +ru2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.ru-en', checkpoint_file=checkpoint_file, tokenizer='moses', bpe='fastbpe') + + +# from fairseq import hub_utils +# #checkpoint_file = 'model1.pt:model2.pt:model3.pt:model4.pt' +# checkpoint_file = 'model1.pt' +# model_name_or_path = 'transformer.wmt19.ru-en' +# data_name_or_path = '.' +# cls = fairseq.model_parallel.models.transformer.ModelParallelTransformerModel +# models = cls.hub_models() +# kwargs = {'bpe': 'fastbpe', 'tokenizer': 'moses'} + +# ru2en = hub_utils.from_pretrained( +# model_name_or_path, +# checkpoint_file, +# data_name_or_path, +# archive_map=models, +# **kwargs +# ) + + + +model = ru2en.models[0] +model + + + +args = dict(vars(ru2en.args)) + + + +args["source_lang"] +args["encoder_embed_dim"] +args["decoder_embed_dim"] + +pprint(args) + + + +pprint(args.keys()) + + + +model_state_dict = model.state_dict() +#model_state_dict + + + +#model = dict(vars(model)) +#dump_state_keys(model_state_dict) + + + +#model.items() +model_state_dict["decoder.layers.5.fc2.bias"].shape.numel() +model_state_dict["decoder.layers.5.fc2.bias"].shape[0] + + + +# dump the state_dict attrs and their shape +#pprint([f"{' '.join(map(str, v.shape)):>12} {k}"for k,v in model_state_dict.items()]) + + +# renames/removal +from collections import OrderedDict + +rename_keys = [ +# ("model.encoder.embed_positions._float_tensor", "model.encoder.embed_positions.weight"), +# ("model.decoder.embed_positions._float_tensor", "model.decoder.embed_positions.weight"), +# ("", ""), +# ("", ""), +# ("", ""), +# ("", ""), +] + +def remove_ignore_keys_(model_state_dict): + ignore_keys = [ + "model.model", + "model.encoder.version", + "model.decoder.version", + "model.encoder_embed_tokens.weight", + "model.decoder_embed_tokens.weight", +# "model.encoder.embed_positions._float_tensor", # not storing model.encoder.embed_positions.weight +# "model.decoder.embed_positions._float_tensor", # not storing model.decoder.embed_positions.weight + ] + for k in ignore_keys: + model_state_dict.pop(k, None) + +def rename_key(dct, old, new): + val = dct.pop(old) + dct[new] = val + +#model_state_dict = chkpt["model"].copy() + +# rename keys to start with model. +model_state_dict_new = OrderedDict(("model."+k, v) for k, v in model_state_dict.items()) +# check: +#model_state_dict["model.encoder.layers.0.fc1.bias"] +#chkpt["model"]["encoder.layers.0.fc1.bias"] + +remove_ignore_keys_(model_state_dict_new) +for src, dest in rename_keys: + rename_key(model_state_dict_new, src, dest) + +model_state_dict_new["model.decoder.embed_tokens.weight"].shape + +# XXX: emulate non-existing layer - perhaps it'll be removed instead in the model - for now just a bias of 0's +model_state_dict_new["final_logits_bias"] = torch.zeros((1, model_state_dict_new["model.decoder.embed_tokens.weight"].shape[0])) + +model_state_dict_new["final_logits_bias"].shape + + +from transformers.modeling_fsmt import FSMTForConditionalGeneration +from transformers.configuration_fsmt import FSMTConfig + +#dump_state_keys(model_state_dict_new) +#model_state_dict_new["model.decoder.embed_tokens.weight"].shape + + + +# let's add dummy things so that load_state_dict doesn't complain +# (embed_positions): SinusoidalPositionalEmbedding(1024, 1024) +# XXX: these seem to be autogenerated on the fly, no need to store +#model_state_dict_new["model.encoder.embed_positions.weight"] = model_state_dict_new["model.decoder.embed_positions.weight"] = torch.zeros((args["decoder_input_dim"], args["decoder_input_dim"])) + +#model_state_dict_new["model.encoder.embed_positions.weight"].shape +#model_state_dict_new["model.decoder.embed_positions.weight"].shape + +# these too get autogenerated: +# "model.encoder_embed_tokens.weight", +# "model.decoder_embed_tokens.weight", + +# # encoder_emd_tok_dim +# args["src_vocab_size"] = 31232 +# args["tgt_vocab_size"] = 31640 +# +# model_state_dict_new["model.encoder_embed_tokens.weight"] = torch.zeros((args["src_vocab_size"], args["encoder_embed_dim"])) +# +# model_state_dict_new["model.decoder_embed_tokens.weight"] = torch.zeros((args["tgt_vocab_size"], args["decoder_embed_dim"])) +# +# model_state_dict_new["model.encoder_embed_tokens.weight"].shape +# model_state_dict_new["model.decoder_embed_tokens.weight"].shape + + +hf_checkpoint_name = "/code/huggingface/transformers-fair-wmt/data/fsmt-wmt19-ru-en/config.json" +config = FSMTConfig.from_pretrained(hf_checkpoint_name) +model_new = FSMTForConditionalGeneration(config).eval() +#state_dict = chkpt["model"] + +import torch +def compare_state_dicts(d1, d2, cmp_func=torch.equal): + ok = 1 + for k in sorted( set(d1.keys()) | set(d2.keys()) ): + if k in d1 and k in d2: + if cmp_func(d1[k], d2[k]): + pass + else: + ok = 0 + print(f"Key {k}: values mismatch: \n{d1[k]}\n{d2[k]}\n") + else: + ok = 0 + which = "1st" if k in d2 else "2nd" + print(f"{which} dict doesn't have key {k}\n") + if ok: + print('Models match') +#compare_state_dicts(model_new.state_dict(), model_state_dict_new) + +torch.save(model_state_dict_new, "/tmp/new.pt") + +# show missing or extraneous/mismatching keys (need to remap/change model to match) +# XXX: somehow this is the key for making the model work +# if I remove this - it stops working +# FSMTForConditionalGeneration probably loads some garbage +model_new.load_state_dict(model_state_dict_new) + +model_new + + +from transformers.tokenization_fsmt import FSMTTokenizer +tokenizer = FSMTTokenizer.from_pretrained('fsmt-wmt19-ru-en') + +model_new.eval() + +sentence = "Машинное обучение - это здорово! Ты молодец." + +input_ids = tokenizer.encode(sentence, return_tensors='pt') +print(input_ids) +outputs = model_new.generate(input_ids)#, num_beams=5) +print("Outputs") +print(outputs) +for output in outputs: + decoded = tokenizer.decode(output, skip_special_tokens=True) + print(decoded) diff --git a/docs/source/index.rst b/docs/source/index.rst index a2acc39466e9..027d8604d819 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,9 +3,9 @@ Transformers State-of-the-art Natural Language Processing for Pytorch and TensorFlow 2.0. -🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides general-purpose -architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural -Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between +🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides general-purpose +architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural +Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between TensorFlow 2.0 and PyTorch. This is the documentation of our repository `transformers `_. @@ -127,7 +127,7 @@ conversion utilities for the following models: 23. `Pegasus `_ (from Google) released with the paper `PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization `_ by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu. 24. `MBart `_ (from Facebook) released with the paper `Multilingual Denoising Pre-training for Neural Machine Translation `_ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, - Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. + Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. 25. `LXMERT `_ (from UNC Chapel Hill) released with the paper `LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering `_ by Hao Tan and Mohit Bansal. @@ -216,6 +216,7 @@ conversion utilities for the following models: model_doc/dpr model_doc/pegasus model_doc/mbart + model_doc/fsmt model_doc/lxmert internal/modeling_utils internal/tokenization_utils diff --git a/docs/source/model_doc/fsmt.rst b/docs/source/model_doc/fsmt.rst new file mode 100644 index 000000000000..c8af02bf931c --- /dev/null +++ b/docs/source/model_doc/fsmt.rst @@ -0,0 +1,53 @@ +FSMT +---------------------------------------------------- +**DISCLAIMER:** If you see something strange, +file a `Github Issue `__ and assign +@stas00. + +Overview +~~~~~~~~~~~~~~~~~~~~~ + +This model is a porting of a transformer model for translation, discussed in `this paper `__: + +Facebook FAIR's WMT19 News Translation Task Submission + +Nathan Ng, Kyra Yee, Alexei Baevski, Myle Ott, Michael Auli, Sergey Edunov + + This paper describes Facebook FAIR's submission to the WMT19 shared news translation task. We participate in two language pairs and four language directions, English <-> German and English <-> Russian. Following our submission from last year, our baseline systems are large BPE-based transformer models trained with the Fairseq sequence modeling toolkit which rely on sampled back-translations. This year we experiment with different bitext data filtering schemes, as well as with adding filtered back-translated data. We also ensemble and fine-tune our models on domain-specific data, then decode using noisy channel model reranking. Our submissions are ranked first in all four directions of the human evaluation campaign. On En->De, our system significantly outperforms other systems as well as human translations. This system improves upon our WMT'18 submission by 4.5 BLEU points. + +The Authors' code can be found `here `__. + + + +Implementation Notes +~~~~~~~~~~~~~~~~~~~~ + +- FSMT uses source and target vocab pair, that aren't combined into one. It doesn't share embed tokens either. Its tokenizer is very similar to `XLMTokenizer` and the main model is derived from `BartModel`. + + +FSMTForConditionalGeneration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.FSMTForConditionalGeneration + :members: forward + + +FSMTConfig +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.FSMTConfig + :members: + + +FSMTTokenizer +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.FSMTTokenizer + :members: + + +FSMTModel +~~~~~~~~~~~~~ + +.. autoclass:: transformers.FSMTModel + :members: forward diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md new file mode 100644 index 000000000000..c14e227c9a8d --- /dev/null +++ b/model_cards/stas/fsmt-wmt19-de-en/README.md @@ -0,0 +1,87 @@ + +--- +language: de, en +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# Model name + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for de-en. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) +* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) +* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) +* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "fsmt-wmt19-de-en" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +pair = ["de", "en"] +input = "Maschinelles Lernen ist großartig, oder? + +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Machine learning is great, isn't it? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +Fairseq reported score is [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) + +The porting of this model is still in progress, but so far we have the following BLEU score: 39.4278 + +The score was calculated using this code: + +```python +git clone https://github.com/huggingface/transformers +cd transformers +cd examples/seq2seq +export PAIR=de-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md new file mode 100644 index 000000000000..18762a8a792b --- /dev/null +++ b/model_cards/stas/fsmt-wmt19-en-de/README.md @@ -0,0 +1,87 @@ + +--- +language: en, de +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# Model name + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-de. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) +* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) +* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) +* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "fsmt-wmt19-en-de" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +pair = ["en", "de"] +input = "Machine learning is great, isn't it? + +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Maschinelles Lernen ist großartig, oder? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +Fairseq reported score is [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) + +The porting of this model is still in progress, but so far we have the following BLEU score: 41.0814 + +The score was calculated using this code: + +```python +git clone https://github.com/huggingface/transformers +cd transformers +cd examples/seq2seq +export PAIR=en-de +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md new file mode 100644 index 000000000000..0b01f4095022 --- /dev/null +++ b/model_cards/stas/fsmt-wmt19-en-ru/README.md @@ -0,0 +1,87 @@ + +--- +language: en, ru +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# Model name + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-ru. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) +* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) +* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) +* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "fsmt-wmt19-en-ru" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +pair = ["en", "ru"] +input = "Machine learning is great, isn't it? + +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Машинное обучение - это здорово, не так ли? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +Fairseq reported score is [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) + +The porting of this model is still in progress, but so far we have the following BLEU score: 31.2695 + +The score was calculated using this code: + +```python +git clone https://github.com/huggingface/transformers +cd transformers +cd examples/seq2seq +export PAIR=en-ru +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md new file mode 100644 index 000000000000..59793af59fbd --- /dev/null +++ b/model_cards/stas/fsmt-wmt19-ru-en/README.md @@ -0,0 +1,87 @@ + +--- +language: ru, en +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# Model name + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for ru-en. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) +* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) +* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) +* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "fsmt-wmt19-ru-en" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +pair = ["ru", "en"] +input = "Машинное обучение - это здорово, не так ли? + +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Machine learning is great, isn't it? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +Fairseq reported score is [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) + +The porting of this model is still in progress, but so far we have the following BLEU score: 38.8524 + +The score was calculated using this code: + +```python +git clone https://github.com/huggingface/transformers +cd transformers +cd examples/seq2seq +export PAIR=ru-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 624246b33839..bc0ae333ac8c 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -29,6 +29,7 @@ from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig from .configuration_encoder_decoder import EncoderDecoderConfig from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig +from .configuration_fsmt import FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP, FSMTConfig from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig @@ -155,6 +156,7 @@ ) from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast from .tokenization_flaubert import FlaubertTokenizer +from .tokenization_fsmt import FSMTTokenizer from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast from .tokenization_lxmert import LxmertTokenizer, LxmertTokenizerFast @@ -327,6 +329,12 @@ FlaubertModel, FlaubertWithLMHeadModel, ) + from .modeling_fsmt import ( + FSMT_PRETRAINED_MODEL_ARCHIVE_LIST, + FSMTForConditionalGeneration, + FSMTModel, + PretrainedFSMTModel, + ) from .modeling_gpt2 import ( GPT2_PRETRAINED_MODEL_ARCHIVE_LIST, GPT2DoubleHeadsModel, diff --git a/src/transformers/configuration_auto.py b/src/transformers/configuration_auto.py index 6dc1e5dd0d17..b6bf4cf1313a 100644 --- a/src/transformers/configuration_auto.py +++ b/src/transformers/configuration_auto.py @@ -26,6 +26,7 @@ from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig from .configuration_encoder_decoder import EncoderDecoderConfig from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig +from .configuration_fsmt import FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP, FSMTConfig from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig @@ -64,6 +65,7 @@ T5_PRETRAINED_CONFIG_ARCHIVE_MAP, XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP, ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, @@ -116,6 +118,10 @@ "bart", BartConfig, ), + ( + "fsmt", + FSMTConfig, + ), ( "reformer", ReformerConfig, diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py new file mode 100644 index 000000000000..b7cff46bde63 --- /dev/null +++ b/src/transformers/configuration_fsmt.py @@ -0,0 +1,224 @@ +# coding=utf-8 +# Copyright 2019-present, Facebook, Inc and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XLM configuration """ + + +import logging + +from .configuration_utils import PretrainedConfig +from .file_utils import add_start_docstrings_to_callable + + +logger = logging.getLogger(__name__) + +FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/config.json", + "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/config.json", + "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/config.json", + "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/config.json", +} + + +FSMT_CONFIG_ARGS_DOC = r""" + Args: + src_vocab_size (:obj:`int`, optional, defaults to None): + defines the different tokens that can be represented by `inputs_ids` passed to the forward + method in the encoder. + tgt_vocab_size (:obj:`int`, optional, defaults to None): + defines the different tokens that can be represented by `inputs_ids` passed to the forward + method in the decoder. + d_model (:obj:`int`, optional, defaults to 1024): + Dimensionality of the layers and the pooler layer. + encoder_layers (:obj:`int`, optional, defaults to 12): + Number of encoder layers, 16 for pegasus, 6 for bart-base and marian + decoder_layers (:obj:`int`, optional, defaults to 12): + Number of decoder layers, 16 for pegasus, 6 for bart-base and marian + encoder_attention_heads (:obj:`int`, optional, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + decoder_attention_heads (:obj:`int`, optional, defaults to 16): + Number of attention heads for each attention layer in the Transformer decoder. + decoder_ffn_dim (:obj:`int`, optional, defaults to 4096): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in decoder. + encoder_ffn_dim (:obj:`int`, optional, defaults to 4096): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in decoder. + activation_function (:obj:`str` or :obj:`function`, optional, defaults to "relu"): + The non-linear activation function (function or string) in the encoder and pooler. + If string, "gelu", "relu", "swish" and "gelu_new" are supported. + dropout (:obj:`float`, optional, defaults to 0.1): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (:obj:`float`, optional, defaults to 0.0): + The dropout ratio for the attention probabilities. + activation_dropout (:obj:`float`, optional, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + max_position_embeddings (:obj:`int`, optional, defaults to 1024): + The maximum sequence length that this model might ever be used with. + Typically set this to something large just in case (e.g., 512 or 1024 or 2048). + init_std (:obj:`float`, optional, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + add_bias_logits (:obj:`bool`, optional, defaults to False): + True for marian only. + normalize_before (:obj:`bool`, optional, defaults to False): + Call layernorm before attention ops. + normalize_embedding (:obj:`bool`, optional, defaults to False): + Call layernorm after embeddings. + static_position_embeddings (:obj:`bool`, optional, defaults to True): + Don't learn positional embeddings, use sinusoidal. + add_final_layer_norm (:obj:`bool`, optional, defaults to False): + Why not add another layernorm? + scale_embedding (:obj:`bool`, optional, defaults to True): + Scale embeddings by diving by sqrt(d_model). + bos_token_id (:obj:`int`, optional, defaults to 0) + Beginning of stream token id. + pad_token_id (:obj:`int`, optional, defaults to 1) + Padding token id. + eos_token_id (:obj:`int`, optional, defaults to 2) + End of stream token id. + encoder_layerdrop: (:obj:`float`, optional, defaults to 0.0): + Google "layerdrop arxiv", as its not explainable in one line. + decoder_layerdrop: (:obj:`float`, optional, defaults to 0.0): + Google "layerdrop arxiv", as its not explainable in one line. + is_encoder_decoder (:obj:`bool`, optional, defaults to True): + Whether this is an encoder/decoder model. + tie_word_embeddings (:obj:`bool`, optional, defaults to False): + Whether to tie input and output embeddings. +""" + +# Porting notes: +# this one is modeled after BartConfig +# +# Differences with BART: +# - src/tgt vocabs aren't shared +# - token embeddings aren't shared +# - needs a language pair +# - scale_embedding are True +# - normalize_embedding are False +# - static_position_embeddings are True +# +# some unused args were removed too + + +@add_start_docstrings_to_callable(FSMT_CONFIG_ARGS_DOC) +class FSMTConfig(PretrainedConfig): + r""" + Configuration class for FSMT. Parameters are renamed from the fairseq implementation + + Differences with BART: + - src/tgt vocabs aren't shared - token embeddings aren't shared + + """ + model_type = "fairseq" + + # update the defaults from config file + def __init__( + self, + src_vocab_size=None, + tgt_vocab_size=None, + activation_function="relu", + d_model=1024, + max_length=200, + num_beams=8, + max_position_embeddings=1024, + encoder_ffn_dim=4096, + encoder_layers=12, + encoder_attention_heads=16, + encoder_layerdrop=0.0, + decoder_ffn_dim=4096, + decoder_layers=12, + decoder_attention_heads=16, + decoder_layerdrop=0.0, + attention_dropout=0.0, + dropout=0.1, + activation_dropout=0.0, + init_std=0.02, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + add_bias_logits=False, + add_final_layer_norm=False, + is_encoder_decoder=True, + normalize_before=False, + normalize_embedding=False, + scale_embedding=True, + static_position_embeddings=True, + tie_word_embeddings=False, + **common_kwargs + ): + r""" + :class:`~transformers.FSMTConfig` is the configuration class for `FSMTModel`. + + Examples:: + + >>> from transformers import FSMTConfig, FSMTModel + + >>> config = FSMTConfig.from_pretrained('stas/fsmt-wmt19-en-ru') + >>> model = FSMTModel(config) + + """ + if "hidden_size" in common_kwargs: + raise ValueError("hidden size is called d_model") + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + is_encoder_decoder=is_encoder_decoder, + tie_word_embeddings=tie_word_embeddings, + **common_kwargs, + ) + self.src_vocab_size = src_vocab_size + self.tgt_vocab_size = tgt_vocab_size + self.d_model = d_model # encoder_embed_dim and decoder_embed_dim + self.max_length = max_length + self.num_beams = num_beams + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_layers = self.num_hidden_layers = encoder_layers + self.encoder_attention_heads = encoder_attention_heads + self.encoder_layerdrop = encoder_layerdrop + self.decoder_layerdrop = decoder_layerdrop + self.decoder_ffn_dim = decoder_ffn_dim + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.max_position_embeddings = max_position_embeddings + self.init_std = init_std # Normal(0, this parameter) + self.activation_function = activation_function + + # XXX: needed in generation_utils.py:382 + # alternatively need to setup config.decoder object + self.decoder_start_token_id = eos_token_id + + # Params introduced for Mbart + self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + self.normalize_embedding = normalize_embedding # True for mbart, False otherwise + self.normalize_before = normalize_before # combo of fairseq's encoder_ and decoder_normalize_before + self.add_final_layer_norm = add_final_layer_norm + + # Params introduced for Marian + self.add_bias_logits = add_bias_logits + self.static_position_embeddings = static_position_embeddings + + # 3 Types of Dropout + self.attention_dropout = attention_dropout + self.activation_dropout = activation_dropout + self.dropout = dropout + + # pos embedding offset + self.extra_pos_embeddings = self.pad_token_id + 1 + + @property + def num_attention_heads(self) -> int: + return self.encoder_attention_heads + + @property + def hidden_size(self) -> int: + return self.d_model diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py new file mode 100755 index 000000000000..8628b48c3508 --- /dev/null +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -0,0 +1,398 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +Convert fairseq transform wmt19 checkpoint. + +To convert run: +assuming the fairseq data is under data/wmt19.ru-en.ensemble, data/wmt19.en-ru.ensemble, etc + +export ROOT=/code/huggingface/transformers-fair-wmt +cd $ROOT +mkdir data + +# get data (run once) +wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-de.joined-dict.ensemble.tar.gz +wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.de-en.joined-dict.ensemble.tar.gz +wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-ru.ensemble.tar.gz +wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.ru-en.ensemble.tar.gz +tar -xvzf wmt19.en-de.joined-dict.ensemble.tar.gz +tar -xvzf wmt19.de-en.joined-dict.ensemble.tar.gz +tar -xvzf wmt19.en-ru.ensemble.tar.gz +tar -xvzf wmt19.ru-en.ensemble.tar.gz + + +# run conversions and uploads + +export PAIR=ru-en +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR + +export PAIR=en-ru +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR + +export PAIR=de-en +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR + +export PAIR=en-de +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR + + +# upload +cd data +transformers-cli upload fsmt-wmt19-ru-en +transformers-cli upload fsmt-wmt19-en-ru +transformers-cli upload fsmt-wmt19-de-en +transformers-cli upload fsmt-wmt19-en-de +cd - + +# force cache invalidation, which will now download the new models +PYTHONPATH="src" python -c 'from transformers import AutoModel; [AutoModel.from_pretrained("stas/fsmt-wmt19-"+p, use_cdn=False) for p in ["en-ru","ru-en","en-de","de-en"]]' + +# happy translations + +""" + +import argparse +import json +import logging +import os +import re +from collections import OrderedDict +from os.path import basename, dirname + +import fairseq +import torch +from fairseq import hub_utils +from fairseq.data.dictionary import Dictionary + +from transformers import WEIGHTS_NAME +from transformers.configuration_fsmt import FSMTConfig +from transformers.modeling_fsmt import FSMTForConditionalGeneration, get_authorized_missing_keys +from transformers.tokenization_fsmt import VOCAB_FILES_NAMES + + +logging.basicConfig(level=logging.INFO) + +DEBUG = 1 + +json_indent = 2 if DEBUG else None + + +def rewrite_dict_keys(d): + # (1) remove word breaking symbol, (2) add word ending symbol where the word is not broken up, + # e.g.: d = {'le@@': 5, 'tt@@': 6, 'er': 7} => {'le': 5, 'tt': 6, 'er': 7} + d2 = dict((re.sub(r"@@$", "", k), v) if k.endswith("@@") else (re.sub(r"$", "", k), v) for k, v in d.items()) + keep_keys = " ".split() + # restore the special tokens + for k in keep_keys: + del d2[f"{k}"] + d2[k] = d[k] # restore + return d2 + + +def write_model_card(model_card_dir, src_lang, tgt_lang): + + texts = { + "en": "Machine learning is great, isn't it?", + "ru": "Машинное обучение - это здорово, не так ли?", + "de": "Maschinelles Lernen ist großartig, oder?", + } + + # BLUE scores as follows: + # "pair": [fairseq, transformers] + scores = { + "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "31.2695"], + "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "38.8524"], + "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "39.4278"], + "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "41.0814"], + } + pair = f"{src_lang}-{tgt_lang}" + + readme = f""" +--- +language: {src_lang}, {tgt_lang} +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# FSMT + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) +* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) +* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) +* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "fsmt-wmt19-{src_lang}-{tgt_lang}" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +pair = ["{src_lang}", "{tgt_lang}"] +input = "{texts[src_lang]} + +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # {texts[tgt_lang]} + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +Fairseq reported score is { scores[pair][0] } + +The porting of this model is still in progress, but so far we have the following BLEU score: { scores[pair][1] } + +The score was calculated using this code: + +```python +git clone https://github.com/huggingface/transformers +cd transformers +cd examples/seq2seq +export PAIR={pair} +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + +""" + os.makedirs(model_card_dir, exist_ok=True) + path = os.path.join(model_card_dir, "README.md") + with open(path, "w", encoding="utf-8") as f: + f.write(readme) + + +def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder_path): + + # prep + assert os.path.exists(fsmt_checkpoint_path) + os.makedirs(pytorch_dump_folder_path, exist_ok=True) + print(f"Writing results to {pytorch_dump_folder_path}") + + # XXX: Need to work out the ensemble as fairseq does, for now using just one chkpt + # checkpoint_file = 'model1.pt:model2.pt:model3.pt:model4.pt' + checkpoint_file = "model1.pt" + # model_name_or_path = 'transformer.wmt19.ru-en' + data_name_or_path = "." + cls = fairseq.model_parallel.models.transformer.ModelParallelTransformerModel + models = cls.hub_models() + kwargs = {"bpe": "fastbpe", "tokenizer": "moses"} + + # note: there is some magic happening here, so can't use torch.load() directly on the model file + # see: load_state_dict() in fairseq_model.py + chkpt = hub_utils.from_pretrained( + fsmt_checkpoint_path, checkpoint_file, data_name_or_path, archive_map=models, **kwargs + ) + + args = dict(vars(chkpt["args"])) + + src_lang = args["source_lang"] + tgt_lang = args["target_lang"] + + data_root = dirname(pytorch_dump_folder_path) + model_dir = basename(pytorch_dump_folder_path) + proj_root = dirname(dirname(dirname(os.path.realpath(__file__)))) + + # dicts + src_dict_file = os.path.join(fsmt_checkpoint_path, f"dict.{src_lang}.txt") + tgt_dict_file = os.path.join(fsmt_checkpoint_path, f"dict.{tgt_lang}.txt") + + src_dict = Dictionary.load(src_dict_file) + src_vocab = rewrite_dict_keys(src_dict.indices) + src_vocab_size = len(src_vocab) + pytorch_vocab_file_src = os.path.join(pytorch_dump_folder_path, f"vocab-{src_lang}.json") + print(f"Generating {pytorch_vocab_file_src}") + with open(pytorch_vocab_file_src, "w", encoding="utf-8") as f: + f.write(json.dumps(src_vocab, ensure_ascii=False, indent=json_indent)) + + tgt_dict = Dictionary.load(tgt_dict_file) + tgt_vocab = rewrite_dict_keys(tgt_dict.indices) + tgt_vocab_size = len(tgt_vocab) + pytorch_vocab_file_tgt = os.path.join(pytorch_dump_folder_path, f"vocab-{tgt_lang}.json") + print(f"Generating {pytorch_vocab_file_tgt}") + with open(pytorch_vocab_file_tgt, "w", encoding="utf-8") as f: + f.write(json.dumps(tgt_vocab, ensure_ascii=False, indent=json_indent)) + + # merge_file (bpecodes) + merge_file = os.path.join(pytorch_dump_folder_path, VOCAB_FILES_NAMES["merges_file"]) + fairseq_merge_file = os.path.join(fsmt_checkpoint_path, "bpecodes") + with open(fairseq_merge_file, encoding="utf-8") as fin: + merges = fin.read() + merges = re.sub(r" \d+$", "", merges, 0, re.M) # remove frequency number + print(f"Generating {merge_file}") + with open(merge_file, "w", encoding="utf-8") as fout: + fout.write(merges) + + # config + fairseq_config_file = os.path.join(pytorch_dump_folder_path, "config.json") + + # XXX: need to compare with the other pre-trained models of this type and + # only set here what's different between them - the common settings go into + # config_fsmt + conf = { + "architectures": ["FSMTForConditionalGeneration"], + "model_type": "fsmt", + "activation_dropout": 0.0, + "activation_function": "relu", + "attention_dropout": args["attention_dropout"], + "d_model": args["decoder_embed_dim"], + "dropout": args["dropout"], + "init_std": 0.02, + "max_position_embeddings": 1024, # XXX: look up? + "num_hidden_layers": 6, # XXX: look up? + "src_vocab_size": src_vocab_size, + "tgt_vocab_size": tgt_vocab_size, + "langs": [src_lang, tgt_lang], + "encoder_attention_heads": args["encoder_attention_heads"], + "encoder_ffn_dim": args["encoder_ffn_embed_dim"], + "encoder_layerdrop": 0.0, + "encoder_layers": args["encoder_layers"], + "decoder_attention_heads": args["decoder_attention_heads"], + "decoder_ffn_dim": args["decoder_ffn_embed_dim"], + "decoder_layerdrop": 0.0, + "decoder_layers": args["decoder_layers"], + "bos_token_id": 0, + "pad_token_id": 1, + "eos_token_id": 2, + "id2label": {"0": "LABEL_0", "1": "LABEL_1", "2": "LABEL_2"}, # not needed? + "label2id": {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}, # not needed? + "add_bias_logits": False, + "add_final_layer_norm": False, + "is_encoder_decoder": True, + "normalize_before": False, + "normalize_embedding": False, + "scale_embedding": True, + "static_position_embeddings": True, + "tie_word_embeddings": False, + } + + print(f"Generating {fairseq_config_file}") + with open(fairseq_config_file, "w", encoding="utf-8") as f: + f.write(json.dumps(conf, ensure_ascii=False, indent=json_indent)) + + # model + model = chkpt["models"][0] + model_state_dict = model.state_dict() + + # rename keys to start with 'model.' + model_state_dict = OrderedDict(("model." + k, v) for k, v in model_state_dict.items()) + + # remove unneeded keys + ignore_keys = [ + "model.model", + "model.encoder.version", + "model.decoder.version", + "model.encoder_embed_tokens.weight", + "model.decoder_embed_tokens.weight", + ] + # let's save a lot of space, by not saving unneeded keys - lots of them! + ignore_keys.extend(get_authorized_missing_keys()) + for k in ignore_keys: + model_state_dict.pop(k, None) + + config = FSMTConfig.from_pretrained(pytorch_dump_folder_path) + model_new = FSMTForConditionalGeneration(config) + + # check that it loads ok + model_new.load_state_dict(model_state_dict, strict=False) + + # save + pytorch_weights_dump_path = os.path.join(pytorch_dump_folder_path, WEIGHTS_NAME) + print(f"Generating {pytorch_weights_dump_path}") + torch.save(model_state_dict, pytorch_weights_dump_path) + + # test that it's the same + test_state_dict = torch.load(pytorch_weights_dump_path) + # print(test_state_dict) + + def compare_state_dicts(d1, d2): + models_differ = 0 + for key_item_1, key_item_2 in zip(d1.items(), d2.items()): + if torch.equal(key_item_1[1], key_item_2[1]): + pass + else: + models_differ += 1 + if key_item_1[0] == key_item_2[0]: + print("Mismatch found at", key_item_1[0]) + else: + raise Exception + if models_differ == 0: + print("Models match perfectly! :)") + + compare_state_dicts(model_state_dict, test_state_dict) + + # model card + model_card_dir = os.path.join(proj_root, "model_cards", "stas", model_dir) + print(f"Generating model_card {src_lang}-{tgt_lang}") + write_model_card(model_card_dir, src_lang, tgt_lang) + + print("Conversion is done!") + print("\nLast step is to upload the files to s3") + print(f"cd {data_root}") + print(f"transformers-cli upload {model_dir}") + print(f"Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # Required parameters + parser.add_argument( + "--fsmt_checkpoint_path", default=None, type=str, required=True, help="Path to the official PyTorch dump dir." + ) + parser.add_argument( + "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model." + ) + args = parser.parse_args() + convert_fsmt_checkpoint_to_pytorch(args.fsmt_checkpoint_path, args.pytorch_dump_folder_path) diff --git a/src/transformers/generation_utils.py b/src/transformers/generation_utils.py index 638bb3b12e6d..edb1b17cc11c 100644 --- a/src/transformers/generation_utils.py +++ b/src/transformers/generation_utils.py @@ -361,12 +361,13 @@ def generate( # current position and vocab size if hasattr(self.config, "vocab_size"): vocab_size = self.config.vocab_size - elif ( - self.config.is_encoder_decoder - and hasattr(self.config, "decoder") - and hasattr(self.config.decoder, "vocab_size") - ): - vocab_size = self.config.decoder.vocab_size + elif self.config.is_encoder_decoder: + if hasattr(self.config, "tgt_vocab_size"): + vocab_size = self.config.tgt_vocab_size + elif hasattr(self.config, "decoder") and hasattr(self.config.decoder, "vocab_size"): + vocab_size = self.config.decoder.vocab_size + if vocab_size is None: + raise ValueError("vocab_size has to be defined") # set effective batch size and effective batch multiplier according to do_sample if do_sample: @@ -387,7 +388,6 @@ def generate( raise ValueError( "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation" ) - assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self) assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder) @@ -411,7 +411,7 @@ def generate( ) # shape: (batch_size * num_return_sequences * num_beams, cur_len) if self.config.is_encoder_decoder: - # create empty decoder_input_ids + # create empty decoder input_ids input_ids = torch.full( (effective_batch_size * num_beams, 1), decoder_start_token_id, diff --git a/src/transformers/modeling_auto.py b/src/transformers/modeling_auto.py index 7f85ce16dda2..f2b02ad0b477 100644 --- a/src/transformers/modeling_auto.py +++ b/src/transformers/modeling_auto.py @@ -29,6 +29,7 @@ ElectraConfig, EncoderDecoderConfig, FlaubertConfig, + FSMTConfig, GPT2Config, LongformerConfig, LxmertConfig, @@ -108,6 +109,7 @@ FlaubertModel, FlaubertWithLMHeadModel, ) +from .modeling_fsmt import FSMTForConditionalGeneration, FSMTModel from .modeling_gpt2 import GPT2LMHeadModel, GPT2Model from .modeling_longformer import ( LongformerForMaskedLM, @@ -198,6 +200,7 @@ (TransfoXLConfig, TransfoXLModel), (XLNetConfig, XLNetModel), (FlaubertConfig, FlaubertModel), + (FSMTConfig, FSMTModel), (XLMConfig, XLMModel), (CTRLConfig, CTRLModel), (ElectraConfig, ElectraModel), @@ -215,6 +218,7 @@ (CamembertConfig, CamembertForMaskedLM), (XLMRobertaConfig, XLMRobertaForMaskedLM), (BartConfig, BartForConditionalGeneration), + (FSMTConfig, FSMTForConditionalGeneration), (LongformerConfig, LongformerForMaskedLM), (RobertaConfig, RobertaForMaskedLM), (BertConfig, BertForPreTraining), @@ -301,6 +305,7 @@ (MarianConfig, MarianMTModel), (MBartConfig, MBartForConditionalGeneration), (BartConfig, BartForConditionalGeneration), + (FSMTConfig, FSMTForConditionalGeneration), (EncoderDecoderConfig, EncoderDecoderModel), ] ) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py new file mode 100644 index 000000000000..9035af504e10 --- /dev/null +++ b/src/transformers/modeling_fsmt.py @@ -0,0 +1,1341 @@ +# coding=utf-8 +# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Original implementation: https://github.com/pytorch/fairseq/tree/master/examples/wmt19 +# Authors: +# - @alexeib Alexei Baevski +# - @edunov Sergey Edunov +# - @michaelauli Michael Auli +# - @myleott Myle Ott +# - @nng555 Nathan Ng +# - David Grangier +# - Kyra Yee +# +# Paper: Facebook FAIR's WMT19 News Translation Task Submission https://arxiv.org/abs/1907.06616 +# +"""PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/""" + +import logging +import math +import random +import warnings +from typing import Any, Dict, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import Tensor, nn +from torch.nn import CrossEntropyLoss + +from .activations import ACT2FN +from .configuration_fsmt import FSMTConfig +from .file_utils import ( + add_code_sample_docstrings, + add_end_docstrings, + add_start_docstrings, + add_start_docstrings_to_callable, + replace_return_docstrings, +) +from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput +from .modeling_utils import PreTrainedModel + + +logger = logging.getLogger(__name__) + +_CONFIG_FOR_DOC = "FSMTConfig" +_TOKENIZER_FOR_DOC = "FSMTTokenizer" + + +FSMT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/" + "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/" + "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/" + "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/" +] + + +# See all FSMT models at https://huggingface.co/models?search=fsmt + + +# Porting notes: +# this one is modeled after BartModel* +# +# Currently only translation (fairseq also has weights for LM) +# +# fairseq provides weights for ru-en, en-ru and de-en, en-de pairs. All have been ported. +# - ru-en, en-ru use asymmetric vocab +# - de-en, en-de use a merged single vocab (but the code works as if they are separate) +# +# Differences with Bart: +# - not using bos token +# - 2 separate vocabs (src and target) +# - embed weights aren't tied +# - uses a model Ensemble (but that part isn't ported/implemented yet) - so we +# aren't getting as good of a BLEU score +# - uses a projection layer at the end of the decoder +# - doesn't use final_logits_bias +# - beam search: stops as soon as num_beams == len(hypos) (whereas transformers +# is not satisfied there and will continue searching until the next cycles +# aren't promising something better), comparing BLEU scores - the transformers +# algorithm is slightly superior, therefore using the latter. But if you want +# to match fairseq outputs, you need to pass ``early_stopping=True`` to ``generate()``. +# +# SinusoidalPositionalEmbedding is slightly different from Bart's - generates +# different embeddings. This implementation is copied verbatim from fairseq with +# some small changes to make it work here. +# +# Other changes: +# - doesn't support use_cache as Bart's version does +# +# TODO: +# - port model ensemble (fs uses 4 model checkpoints) +# - solve beam search discrepancies +# - There are keys in the state_dict that don't need to be saved, see the +# conversion script (get_authorized_missing_keys()), so need to ensure that if +# someone does further work with the weights they don't save those keys + +""" + +Here is how to compare BLEU scores against fairseq implementation: + +# Note: to match fairseq params you need to set num_beams=50 in +# `configuration_fsmt.py` and lower BS as it'll need more GPU memory + +cd examples/seq2seq + +# en-ru + +export PAIR=en-ru +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation + +# (fairseq BLEU: 36.4 http://matrix.statmt.org/matrix/output/1914?score_id=37605) + + + + +# ru-en + +export PAIR=ru-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation + +# (fairseq BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937) + + + + +# de-en + +export PAIR=de-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation + +# (fairseq BLEU: 42.3 http://matrix.statmt.org/matrix/output/1902?run_id=6750) + + + +# en-de + +export PAIR=en-de +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation + +# (fairseq BLEU: 43.1 http://matrix.statmt.org/matrix/output/1909?run_id=6862) + +""" + + +FSMT_START_DOCSTRING = r""" + + This model is a PyTorch `torch.nn.Module `_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matters related to general usage and behavior. + + Parameters: + config (:class:`~transformers.FSMTConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. + +""" +FSMT_GENERATION_EXAMPLE = r""" + Translation example:: + + from transformers import FSMTTokenizer, FSMTForConditionalGeneration + + mname = "stas/fsmt-wmt19-ru-en" + model = FSMTForConditionalGeneration.from_pretrained(mname) + tokenizer = FSMTTokenizer.from_pretrained(mname) + + src_text = "Машинное обучение - это здорово, не так ли?" + input_ids = tokenizer.encode(src_text, return_tensors='pt') + outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3) + for i, output in enumerate(outputs): + decoded = tokenizer.decode(output, skip_special_tokens=True) + print(f"{i}: {decoded}) + # 1: Machine learning is great, isn't it? ... + +""" + +FSMT_INPUTS_DOCSTRING = r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Use FSMTTokenizer.encode to produce them. + Padding will be ignored by default should you provide it. + Indices can be obtained using :class:`transformers.FSMTTokenizer.encode(text)`. + attention_mask (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): + Mask to avoid performing attention on padding token indices in input_ids. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + encoder_outputs (:obj:`tuple(tuple(torch.FloatTensor)`, `optional`, defaults to :obj:`None`): + Tuple consists of (`last_hidden_state`, `optional`: `hidden_states`, `optional`: `attentions`) + `last_hidden_state` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`) is a sequence of hidden-states at the output of the last layer of the encoder. + Used in the cross-attention of the decoder. + decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`, defaults to :obj:`None`): + Provide for translation and summarization training. By default, the model will create this tensor by shifting the input_ids right, following the paper. + decoder_attention_mask (:obj:`torch.BoolTensor` of shape :obj:`(batch_size, tgt_seq_len)`, `optional`, defaults to :obj:`None`): + Default behavior: generate a tensor that ignores pad tokens in decoder_input_ids. Causal mask will also be used by default. + If you want to change padding behavior, you should read :func:`~transformers.modeling_fairseqtranslator._prepare_decoder_inputs` and modify. + See diagram 1 in the paper for more info on the default strategy + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains pre-computed key and value hidden-states of the attention blocks. + Can be used to speed up decoding. + If ``past_key_values`` are used, the user can optionally input only the last + ``decoder_input_ids`` (those that don't have their past key value states given to this model) of shape + :obj:`(batch_size, 1)` instead of all ``decoder_input_ids`` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): + If `use_cache` is True, ``past_key_values`` are returned and can be used to speed up decoding (see + ``past_key_values``). + output_attentions (:obj:`bool`, `optional`, defaults to :obj:`None`): + If set to ``True``, the attentions tensors of all attention layers are returned. See ``attentions`` under returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`, defaults to :obj:`None`): + If set to ``True``, the hidden states of all layers are returned. See ``hidden_states`` under returned tensors for more detail. + return_dict (:obj:`bool`, `optional`, defaults to :obj:`None`): + If set to ``True``, the model will return a :class:`~transformers.file_utils.ModelOutput` instead of a + plain tuple. +""" + + +def invert_mask(attention_mask): + """Turns 1->0, 0->1, False->True, True-> False""" + assert attention_mask.dim() == 2 + return attention_mask.eq(0) + + +def _prepare_fsmt_decoder_inputs( + config, input_ids, decoder_input_ids=None, decoder_padding_mask=None, causal_mask_dtype=torch.float32 +): + """Prepare masks that ignore padding tokens in the decoder and a causal mask for the decoder if + none are provided. This mimics the default behavior in fairseq. To override it pass in masks. + Note: this is not called during generation + """ + pad_token_id = config.pad_token_id + if decoder_input_ids is None: + decoder_input_ids = shift_tokens_right(input_ids, pad_token_id) + bsz, tgt_len = decoder_input_ids.size() + if decoder_padding_mask is None: + decoder_padding_mask = make_padding_mask(decoder_input_ids, pad_token_id) + else: + decoder_padding_mask = invert_mask(decoder_padding_mask) + causal_mask = torch.triu(fill_with_neg_inf(torch.zeros(tgt_len, tgt_len)), 1).to( + dtype=causal_mask_dtype, device=decoder_input_ids.device + ) + return decoder_input_ids, decoder_padding_mask, causal_mask + + +class PretrainedFSMTModel(PreTrainedModel): + config_class = FSMTConfig + base_model_prefix = "model" + + def _init_weights(self, module): + std = self.config.init_std + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, SinusoidalPositionalEmbedding): + pass + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + @property + def dummy_inputs(self): + pad_token = self.config.pad_token_id + input_ids = torch.tensor([[0, 6, 10, 4, 2], [0, 8, 12, 2, pad_token]], device=self.device) + dummy_inputs = { + "attention_mask": input_ids.ne(pad_token), + "input_ids": input_ids, + } + return dummy_inputs + + +def _make_linear_from_emb(emb): + vocab_size, emb_size = emb.weight.shape + lin_layer = nn.Linear(vocab_size, emb_size, bias=False) + lin_layer.weight.data = emb.weight.data + return lin_layer + + +# Helper Functions, mostly for making masks +def _check_shapes(shape_1, shape2): + if shape_1 != shape2: + raise AssertionError("shape mismatch: {} != {}".format(shape_1, shape2)) + + +def shift_tokens_right(input_ids, pad_token_id): + """Shift input ids one token to the right, and wrap the last non pad token (usually ).""" + prev_output_tokens = input_ids.clone() + index_of_eos = (input_ids.ne(pad_token_id).sum(dim=1) - 1).unsqueeze(-1) + prev_output_tokens[:, 0] = input_ids.gather(1, index_of_eos).squeeze() + prev_output_tokens[:, 1:] = input_ids[:, :-1] + return prev_output_tokens + + +def make_padding_mask(input_ids, padding_idx=1): + """True for pad tokens""" + padding_mask = input_ids.eq(padding_idx) + if not padding_mask.any(): + padding_mask = None + return padding_mask + + +# Helper Modules + + +class EncoderLayer(nn.Module): + def __init__(self, config: FSMTConfig): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = SelfAttention( + self.embed_dim, + config.encoder_attention_heads, + dropout=config.attention_dropout, + ) + self.normalize_before = config.normalize_before + self.self_attn_layer_norm = LayerNorm(self.embed_dim) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) + self.final_layer_norm = LayerNorm(self.embed_dim) + + def forward(self, x, encoder_padding_mask, output_attentions=False): + """ + Args: + x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)` + encoder_padding_mask (ByteTensor): binary ByteTensor of shape + `(batch, src_len)` where padding elements are indicated by ``1``. + for t_tgt, t_src is excluded (or masked out), =0 means it is + included in attention + + Returns: + encoded output of shape `(seq_len, batch, embed_dim)` + """ + residual = x + if self.normalize_before: + x = self.self_attn_layer_norm(x) + x, attn_weights = self.self_attn( + query=x, key=x, key_padding_mask=encoder_padding_mask, output_attentions=output_attentions + ) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.self_attn_layer_norm(x) + + residual = x + if self.normalize_before: + x = self.final_layer_norm(x) + x = self.activation_fn(self.fc1(x)) + x = F.dropout(x, p=self.activation_dropout, training=self.training) + x = self.fc2(x) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.final_layer_norm(x) + return x, attn_weights + + +class FSMTEncoder(nn.Module): + """ + Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer + is a :class:`EncoderLayer`. + + Args: + config: FSMTConfig + """ + + def __init__(self, config: FSMTConfig, embed_tokens): + super().__init__() + + self.dropout = config.dropout + self.layerdrop = config.encoder_layerdrop + + embed_dim = embed_tokens.embedding_dim + self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0 + self.padding_idx = embed_tokens.padding_idx + self.max_source_positions = config.max_position_embeddings + + self.embed_tokens = embed_tokens + if config.static_position_embeddings: + # print(config.max_position_embeddings, embed_dim, self.padding_idx) + num_embeddings = config.src_vocab_size + self.embed_positions = SinusoidalPositionalEmbedding( + embed_dim, + self.padding_idx, + init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings + ) + else: + self.embed_positions = LearnedPositionalEmbedding( + config.max_position_embeddings, + embed_dim, + self.padding_idx, + config.extra_pos_embeddings, + ) + self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.encoder_layers)]) + self.layernorm_embedding = LayerNorm(embed_dim) if config.normalize_embedding else nn.Identity() + # mfairseqtranslator has one extra layer_norm + self.layer_norm = LayerNorm(config.d_model) if config.normalize_before else None + + def forward( + self, input_ids, attention_mask=None, output_attentions=False, output_hidden_states=False, return_dict=False + ): + """ + Args: + input_ids (LongTensor): tokens in the source language of shape + `(batch, src_len)` + attention_mask (torch.LongTensor): indicating which indices are padding tokens. + Returns: + BaseModelOutput or Tuple comprised of: + - **x** (Tensor): the last encoder layer's output of + shape `(src_len, batch, embed_dim)` + - **encoder_states** (tuple(torch.FloatTensor)): all intermediate + hidden states of shape `(src_len, batch, embed_dim)`. + Only populated if *output_hidden_states:* is True. + - **all_attentions** (tuple(torch.FloatTensor)): Attention weights for each layer. + During training might not be of length n_layers because of layer dropout. + """ + # check attention mask and invert + if attention_mask is not None: + attention_mask = invert_mask(attention_mask) + + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + embed_pos = self.embed_positions(input_ids) + x = inputs_embeds + embed_pos + x = self.layernorm_embedding(x) + x = F.dropout(x, p=self.dropout, training=self.training) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + encoder_states = [] if output_hidden_states else None + all_attentions = () if output_attentions else None + for encoder_layer in self.layers: + if output_hidden_states: + encoder_states.append(x) + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + dropout_probability = random.uniform(0, 1) + if self.training and (dropout_probability < self.layerdrop): # skip the layer + attn = None + else: + x, attn = encoder_layer(x, attention_mask, output_attentions=output_attentions) + + if output_attentions: + all_attentions = all_attentions + (attn,) + + if self.layer_norm: + x = self.layer_norm(x) + if output_hidden_states: + encoder_states.append(x) + # T x B x C -> B x T x C + encoder_states = tuple(hidden_state.transpose(0, 1) for hidden_state in encoder_states) + + # T x B x C -> B x T x C + x = x.transpose(0, 1) + + if not return_dict: + return tuple(v for v in [x, encoder_states, all_attentions] if v is not None) + return BaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions) + + +class DecoderLayer(nn.Module): + def __init__(self, config: FSMTConfig): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = SelfAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + ) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + self.normalize_before = config.normalize_before + + self.self_attn_layer_norm = LayerNorm(self.embed_dim) + self.encoder_attn = SelfAttention( + self.embed_dim, + config.decoder_attention_heads, + dropout=config.attention_dropout, + encoder_decoder_attention=True, + ) + self.encoder_attn_layer_norm = LayerNorm(self.embed_dim) + self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) + self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) + self.final_layer_norm = LayerNorm(self.embed_dim) + + def forward( + self, + x, + encoder_hidden_states, + encoder_attn_mask=None, + layer_state=None, + causal_mask=None, + decoder_padding_mask=None, + output_attentions=False, + ): + residual = x + + if layer_state is None: + layer_state = {} + if self.normalize_before: + x = self.self_attn_layer_norm(x) + # Self Attention + + x, self_attn_weights = self.self_attn( + query=x, + key=x, + layer_state=layer_state, # adds keys to layer state + key_padding_mask=decoder_padding_mask, + attn_mask=causal_mask, + output_attentions=output_attentions, + ) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.self_attn_layer_norm(x) + + # Cross attention + residual = x + assert self.encoder_attn.cache_key != self.self_attn.cache_key + if self.normalize_before: + x = self.encoder_attn_layer_norm(x) + x, _ = self.encoder_attn( + query=x, + key=encoder_hidden_states, + key_padding_mask=encoder_attn_mask, + layer_state=layer_state, # mutates layer state + ) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.encoder_attn_layer_norm(x) + + # Fully Connected + residual = x + if self.normalize_before: + x = self.final_layer_norm(x) + x = self.activation_fn(self.fc1(x)) + x = F.dropout(x, p=self.activation_dropout, training=self.training) + x = self.fc2(x) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.final_layer_norm(x) + return ( + x, + self_attn_weights, + layer_state, + ) # just self_attn weights for now, following t5, layer_state = cache for decoding + + +class FSMTDecoder(nn.Module): + """ + Transformer decoder consisting of *config.decoder_layers* layers. Each layer + is a :class:`DecoderLayer`. + Args: + config: FSMTConfig + embed_tokens (torch.nn.Embedding): output embedding + """ + + def __init__(self, config: FSMTConfig, embed_tokens: nn.Embedding): + super().__init__() + self.dropout = config.dropout + self.layerdrop = config.decoder_layerdrop + self.padding_idx = embed_tokens.padding_idx + self.max_target_positions = config.max_position_embeddings + self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 + self.embed_tokens = embed_tokens + embed_dim = embed_tokens.embedding_dim + if config.static_position_embeddings: + num_embeddings = config.tgt_vocab_size + # XXX: self.padding_idx and config.pad_token_id are the same? + self.embed_positions = SinusoidalPositionalEmbedding( + embed_dim, + self.padding_idx, + init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings + ) + else: + self.embed_positions = LearnedPositionalEmbedding( + config.max_position_embeddings, + config.d_model, + self.padding_idx, + config.extra_pos_embeddings, + ) + self.layers = nn.ModuleList( + [DecoderLayer(config) for _ in range(config.decoder_layers)] + ) # type: List[DecoderLayer] + self.layernorm_embedding = LayerNorm(config.d_model) if config.normalize_embedding else nn.Identity() + self.layer_norm = LayerNorm(config.d_model) if config.add_final_layer_norm else None + + # XXX: also add to init_weights + self.output_projection = nn.Linear( + self.embed_tokens.weight.shape[1], + self.embed_tokens.weight.shape[0], + bias=False, + ) + # self.output_projection.weight = self.embed_tokens.weight + # nn.init.normal_( + # self.output_projection.weight, mean=0, std=self.output_embed_dim ** -0.5 + # ) + + def forward( + self, + input_ids, + encoder_hidden_states, + encoder_padding_mask, + decoder_padding_mask, + decoder_causal_mask, + past_key_values=None, + use_cache=False, + output_attentions=False, + output_hidden_states=False, + return_dict=False, + **unused, + ): + """ + Includes several features from "Jointly Learning to Align and + Translate with Transformer Models" (Garg et al., EMNLP 2019). + + Args: + input_ids (LongTensor): previous decoder outputs of shape + `(batch, tgt_len)`, for teacher forcing + encoder_hidden_states: output from the encoder, used for + encoder-side attention + encoder_padding_mask: for ignoring pad tokens + past_key_values (dict or None): dictionary used for storing state during generation + + Returns: + BaseModelOutputWithPast or tuple: + - the decoder's features of shape `(batch, tgt_len, embed_dim)` + - the cache + - hidden states + - attentions + """ + if "decoder_cached_states" in unused: + warnings.warn( + "The `decoder_cached_states` argument is deprecated and will be removed in a future version, use `past_key_values` instead.", + FutureWarning, + ) + past_key_values = unused.pop("decoder_cached_states") + if "decoder_past_key_values" in unused: + warnings.warn( + "The `decoder_past_key_values` argument is deprecated and will be removed in a future version, use `past_key_values` instead.", + FutureWarning, + ) + past_key_values = unused.pop("decoder_past_key_values") + + # check attention mask and invert + if encoder_padding_mask is not None: + encoder_padding_mask = invert_mask(encoder_padding_mask) + + # embed positions + positions = self.embed_positions(input_ids) # , use_cache=use_cache) + + if use_cache: + input_ids = input_ids[:, -1:] + positions = positions[:, -1:] # happens after we embed them + # assert input_ids.ne(self.padding_idx).any() + + x = self.embed_tokens(input_ids) * self.embed_scale + x += positions + x = self.layernorm_embedding(x) + x = F.dropout(x, p=self.dropout, training=self.training) + + # Convert to FSMT output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim) + x = x.transpose(0, 1) + encoder_hidden_states = encoder_hidden_states.transpose(0, 1) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = [] + for idx, decoder_layer in enumerate(self.layers): + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + if output_hidden_states: + all_hidden_states += (x,) + dropout_probability = random.uniform(0, 1) + if self.training and (dropout_probability < self.layerdrop): + continue + + layer_state = past_key_values[idx] if past_key_values is not None else None + + x, layer_self_attn, layer_past = decoder_layer( + x, + encoder_hidden_states, + encoder_attn_mask=encoder_padding_mask, + decoder_padding_mask=decoder_padding_mask, + layer_state=layer_state, + causal_mask=decoder_causal_mask, + output_attentions=output_attentions, + ) + + if use_cache: + next_decoder_cache.append(layer_past.copy()) + + if self.layer_norm and (idx == len(self.layers) - 1): # last layer of mfairseqtranslator + x = self.layer_norm(x) + if output_attentions: + all_self_attns += (layer_self_attn,) + + # Convert to standard output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim) + if output_hidden_states: + all_hidden_states = tuple(hidden_state.transpose(0, 1) for hidden_state in all_hidden_states) + x = x.transpose(0, 1) + encoder_hidden_states = encoder_hidden_states.transpose(0, 1) + + # new XXX: not invoked? self.project_out_dim==None in fairseq + # but it then gets invoked later in x.output_layer() transformer.py:676 + x = self.output_projection(x) + + next_cache = next_decoder_cache if use_cache else None + + if not return_dict: + return tuple(v for v in [x, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=x, past_key_values=next_cache, hidden_states=all_hidden_states, attentions=all_self_attns + ) + + +def _reorder_buffer(attn_cache, new_order): + for k, input_buffer_k in attn_cache.items(): + if input_buffer_k is not None: + attn_cache[k] = input_buffer_k.index_select(0, new_order) + return attn_cache + + +class SelfAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + bias=True, + encoder_decoder_attention=False, # otherwise self_attention + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + self.scaling = self.head_dim ** -0.5 + + self.encoder_decoder_attention = encoder_decoder_attention + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.cache_key = "encoder_decoder" if self.encoder_decoder_attention else "self" + + def _shape(self, tensor, seq_len, bsz): + return tensor.contiguous().view(seq_len, bsz * self.num_heads, self.head_dim).transpose(0, 1) + + def forward( + self, + query, + key: Optional[Tensor], + key_padding_mask: Optional[Tensor] = None, + layer_state: Optional[Dict[str, Optional[Tensor]]] = None, + attn_mask: Optional[Tensor] = None, + output_attentions=False, + ) -> Tuple[Tensor, Optional[Tensor]]: + """Input shape: Time(SeqLen) x Batch x Channel""" + static_kv: bool = self.encoder_decoder_attention + tgt_len, bsz, embed_dim = query.size() + assert embed_dim == self.embed_dim + assert list(query.size()) == [tgt_len, bsz, embed_dim] + # get here for encoder decoder cause of static_kv + if layer_state is not None: # reuse k,v and encoder_padding_mask + saved_state = layer_state.get(self.cache_key, {}) + if "prev_key" in saved_state and static_kv: + # previous time steps are cached - no need to recompute key and value if they are static + key = None + else: + saved_state = None + layer_state = {} + + q = self.q_proj(query) * self.scaling + if static_kv: + if key is None: + k = v = None + else: + k = self.k_proj(key) + v = self.v_proj(key) + else: + k = self.k_proj(query) + v = self.v_proj(query) + + q = self._shape(q, tgt_len, bsz) + if k is not None: + k = self._shape(k, -1, bsz) + if v is not None: + v = self._shape(v, -1, bsz) + + if saved_state is not None: + k, v, key_padding_mask = self._use_saved_state(k, v, saved_state, key_padding_mask, static_kv, bsz) + + # Update cache + layer_state[self.cache_key] = { + "prev_key": k.view(bsz, self.num_heads, -1, self.head_dim), + "prev_value": v.view(bsz, self.num_heads, -1, self.head_dim), + "prev_key_padding_mask": key_padding_mask if not static_kv else None, + } + + assert k is not None + src_len = k.size(1) + attn_weights = torch.bmm(q, k.transpose(1, 2)) + assert attn_weights.size() == (bsz * self.num_heads, tgt_len, src_len) + + if attn_mask is not None: + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # This is part of a workaround to get around fork/join parallelism not supporting Optional types. + if key_padding_mask is not None and key_padding_mask.dim() == 0: + key_padding_mask = None + assert key_padding_mask is None or key_padding_mask.size()[:2] == ( + bsz, + src_len, + ) + + if key_padding_mask is not None: # don't attend to padding symbols + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + reshaped = key_padding_mask.unsqueeze(1).unsqueeze(2) + attn_weights = attn_weights.masked_fill(reshaped, float("-inf")) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + attn_weights = F.softmax(attn_weights, dim=-1) + attn_probs = F.dropout( + attn_weights, + p=self.dropout, + training=self.training, + ) + + assert v is not None + attn_output = torch.bmm(attn_probs, v) + assert attn_output.size() == (bsz * self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim) + attn_output = self.out_proj(attn_output) + if output_attentions: + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + else: + attn_weights = None + return attn_output, attn_weights + + def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz): + # saved states are stored with shape (bsz, num_heads, seq_len, head_dim) + if "prev_key" in saved_state: + _prev_key = saved_state["prev_key"] + assert _prev_key is not None + prev_key = _prev_key.view(bsz * self.num_heads, -1, self.head_dim) + if static_kv: + k = prev_key + else: + assert k is not None + k = torch.cat([prev_key, k], dim=1) + if "prev_value" in saved_state: + _prev_value = saved_state["prev_value"] + assert _prev_value is not None + prev_value = _prev_value.view(bsz * self.num_heads, -1, self.head_dim) + if static_kv: + v = prev_value + else: + assert v is not None + v = torch.cat([prev_value, v], dim=1) + assert k is not None and v is not None + prev_key_padding_mask: Optional[Tensor] = saved_state.get("prev_key_padding_mask", None) + if prev_key_padding_mask is not None: + if static_kv: + new_key_padding_mask = prev_key_padding_mask + else: + new_key_padding_mask = torch.cat([prev_key_padding_mask, key_padding_mask], dim=1) + else: + new_key_padding_mask = key_padding_mask + return k, v, new_key_padding_mask + + +# XXX: remove this and its references +class LearnedPositionalEmbedding(nn.Embedding): + """ + This module learns positional embeddings up to a fixed maximum size. + Padding ids are ignored by either offsetting based on padding_idx + or by setting padding_idx to None and ensuring that the appropriate + position ids are passed to the forward function. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, offset): + # FSMT is set up so that if padding_idx is specified then offset the embedding ids by 2 + # and adjust num_embeddings appropriately. Other models dont have this hack + self.offset = offset + assert padding_idx is not None + num_embeddings += offset + super().__init__(num_embeddings, embedding_dim, padding_idx=padding_idx) + + def forward(self, input_ids, use_cache=False): + """Input is expected to be of size [bsz x seqlen].""" + bsz, seq_len = input_ids.shape[:2] + if use_cache: + positions = input_ids.data.new(1, 1).fill_(seq_len - 1) # called before slicing + else: + # starts at 0, ends at 1-seq_len + positions = torch.arange(seq_len, dtype=torch.long, device=self.weight.device) + return super().forward(positions + self.offset) + + +def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True): + if torch.cuda.is_available(): + try: + from apex.normalization import FusedLayerNorm + + return FusedLayerNorm(normalized_shape, eps, elementwise_affine) + except ImportError: + pass + return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine) + + +def fill_with_neg_inf(t): + """FP16-compatible function that fills a input_ids with -inf.""" + return t.float().fill_(float("-inf")).type_as(t) + + +# Public API +def _get_shape(t): + return getattr(t, "shape", None) + + +# def output_projection(self): +# return nn.Linear( +# self.embed_tokens.weight.shape[1], +# self.embed_tokens.weight.shape[0], +# bias=False, +# ) + + +@add_start_docstrings( + "The bare FSMT Model outputting raw hidden-states without any specific head on top.", + FSMT_START_DOCSTRING, +) +class FSMTModel(PretrainedFSMTModel): + def __init__(self, config: FSMTConfig): + super().__init__(config) + + padding_idx = config.pad_token_id + encoder_embed_tokens = nn.Embedding(config.src_vocab_size, config.d_model, padding_idx) + decoder_embed_tokens = nn.Embedding(config.tgt_vocab_size, config.d_model, padding_idx) + + self.encoder = FSMTEncoder(config, encoder_embed_tokens) + self.decoder = FSMTDecoder(config, decoder_embed_tokens) + + self.init_weights() + + @add_start_docstrings_to_callable(FSMT_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + tokenizer_class=_TOKENIZER_FOR_DOC, + checkpoint="stas/fsmt-wmt19-ru-en", + output_type=BaseModelOutputWithPast, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids, + attention_mask=None, + decoder_input_ids=None, + encoder_outputs: Optional[Tuple] = None, + decoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs, + ): + if "decoder_past_key_values" in kwargs: + warnings.warn( + "The `decoder_past_key_values` argument is deprecated and will be removed in a future version, use `past_key_values` instead.", + FutureWarning, + ) + past_key_values = kwargs.pop("decoder_past_key_values") + + if decoder_input_ids is None: + use_cache = False + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # make masks if user doesn't supply + if not use_cache: + decoder_input_ids, decoder_padding_mask, causal_mask = _prepare_fsmt_decoder_inputs( + self.config, + input_ids, + decoder_input_ids=decoder_input_ids, + decoder_padding_mask=decoder_attention_mask, + causal_mask_dtype=self.decoder.embed_tokens.weight.dtype, + ) + else: + decoder_padding_mask, causal_mask = None, None + + assert decoder_input_ids is not None + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOuput when return_dict=False + elif return_dict and not isinstance(encoder_outputs, BaseModelOutput): + encoder_outputs = BaseModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + ) + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + decoder_outputs = self.decoder( + decoder_input_ids, + encoder_outputs[0], + attention_mask, + decoder_padding_mask, + decoder_causal_mask=causal_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return Seq2SeqModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + def get_input_embeddings(self): + return self.encoder.embed_tokens + + def set_input_embeddings(self, value): + self.encoder.embed_tokens = value # self.encoder_embed_tokens = value + + def get_output_embeddings(self): + return self.decoder.embed_tokens + # XXX: it was, but probably not needed here + # return _make_linear_from_emb(self.decoder.embed_tokens) # make it on the fly + + def set_output_embeddings(self, value): + self.decoder.embed_tokens = value # self.decoder_embed_tokens = value + + +def get_authorized_missing_keys(): + missing_keys = [r"encoder\.version", r"decoder\.version"] + + # these are 90 dict entries that aren't needed to be saved (they aren't in the original saved weights) + postfices = [fr"{x}.{y}" for x in ["k_proj", "v_proj", "q_proj"] for y in ["weight", "bias"]] + self_attn_keys = [ + fr"model.{x}.layers.{y}.self_attn.{z}" for x in ["encoder", "decoder"] for y in range(1, 6) for z in postfices + ] + encoder_attn_keys = [fr"model.decoder.layers.{y}.encoder_attn.{z}" for y in range(1, 6) for z in postfices] + missing_keys += self_attn_keys + encoder_attn_keys + return missing_keys + + +@add_start_docstrings( + "The FSMT Model with a language modeling head. Can be used for summarization.", FSMT_START_DOCSTRING +) +class FSMTForConditionalGeneration(PretrainedFSMTModel): + base_model_prefix = "model" + authorized_missing_keys = get_authorized_missing_keys() + + def __init__(self, config: FSMTConfig): + super().__init__(config) + base_model = FSMTModel(config) + self.model = base_model + + def resize_token_embeddings(self, new_num_tokens: int) -> nn.Embedding: + new_embeddings = super().resize_token_embeddings(new_num_tokens) + self.model.encoder.embed_tokens = new_embeddings + + new_embeddings = super().resize_token_embeddings(new_num_tokens) + self.model.decoder.embed_tokens = new_embeddings + + # XXX: this is not quite correct, as we have 2 different + # `new_embeddings`, and only one return value is expected. + return new_embeddings + + @add_start_docstrings_to_callable(FSMT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @add_end_docstrings(FSMT_GENERATION_EXAMPLE) + def forward( + self, + input_ids, + attention_mask=None, + encoder_outputs=None, + decoder_input_ids=None, + decoder_attention_mask=None, + past_key_values=None, + labels=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **unused, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): + Labels for computing the masked language modeling loss. + Indices should either be in ``[0, ..., config.vocab_size]`` or -100 (see ``input_ids`` docstring). + Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens + with labels in ``[0, ..., config.vocab_size]``. + + Returns: + + """ + if "lm_labels" in unused: + warnings.warn( + "The `lm_labels` argument is deprecated and will be removed in a future version, use `labels` instead.", + FutureWarning, + ) + labels = unused.pop("lm_labels") + if "decoder_cached_states" in unused: + warnings.warn( + "The `decoder_cached_states` argument is deprecated and will be removed in a future version, use `past_key_values` instead.", + FutureWarning, + ) + past_key_values = unused.pop("decoder_cached_states") + if "decoder_past_key_values" in unused: + warnings.warn( + "The `decoder_past_key_values` argument is deprecated and will be removed in a future version, use `past_key_values` instead.", + FutureWarning, + ) + past_key_values = unused.pop("decoder_past_key_values") + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if labels is not None: + use_cache = False + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + encoder_outputs=encoder_outputs, + decoder_attention_mask=decoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + lm_logits = outputs[0] + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # TODO(SS): do we need to ignore pad tokens in labels? + masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.tgt_vocab_size), labels.view(-1)) + + if not return_dict: + output = (lm_logits,) + outputs[1:] + return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output + + return Seq2SeqLMOutput( + loss=masked_lm_loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + def prepare_inputs_for_generation( + self, decoder_input_ids, past, attention_mask, use_cache, encoder_outputs, **kwargs + ): + return { + "input_ids": None, # encoder_outputs is defined. input_ids not needed + "encoder_outputs": encoder_outputs, + "past_key_values": past, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "use_cache": use_cache, # change this to avoid caching (presumably for debugging) + } + + def adjust_logits_during_generation(self, logits, cur_len, max_length): + if cur_len == max_length - 1 and self.config.eos_token_id is not None: + self._force_token_ids_generation(logits, self.config.eos_token_id) + return logits + + def _force_token_ids_generation(self, scores, token_ids) -> None: + """force one of token_ids to be generated by setting prob of all other tokens to 0""" + if isinstance(token_ids, int): + token_ids = [token_ids] + all_but_token_ids_mask = torch.tensor( + [x for x in range(self.config.tgt_vocab_size) if x not in token_ids], + dtype=torch.long, + device=next(self.parameters()).device, + ) + assert len(scores.shape) == 2, "scores should be of rank 2 with shape: [batch_size, vocab_size]" + scores[:, all_but_token_ids_mask] = -float("inf") + + @staticmethod + def _reorder_cache(past, beam_idx): + reordered_past = [] + for layer_past in past: + # get the correct batch idx from decoder layer's batch dim for cross and self-attn + layer_past_new = { + attn_key: _reorder_buffer(attn_cache, beam_idx) for attn_key, attn_cache in layer_past.items() + } + reordered_past.append(layer_past_new) + return reordered_past + + def get_encoder(self): + return self.model.encoder + + def get_output_embeddings(self): + return self.model.decoder.embed_tokens + # XXX: it was, but probably is not needed here + # return _make_linear_from_emb(self.decoder.embed_tokens) # make it on the fly + + +def make_positions(tensor, padding_idx: int): + """Replace non-padding symbols with their position numbers. + + Position numbers begin at padding_idx+1. Padding symbols are ignored. + """ + # The series of casts and type-conversions here are carefully + # balanced to both work with ONNX export and XLA. In particular XLA + # prefers ints, cumsum defaults to output longs, and ONNX doesn't know + # how to handle the dtype kwarg in cumsum. + mask = tensor.ne(padding_idx).int() + return (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + padding_idx + + +class SinusoidalPositionalEmbedding(nn.Module): + """This module produces sinusoidal positional embeddings of any length. + + Padding symbols are ignored. + """ + + def __init__(self, embedding_dim, padding_idx, init_size=1024): + super().__init__() + self.embedding_dim = embedding_dim + self.padding_idx = padding_idx + self.weights = SinusoidalPositionalEmbedding.get_embedding(init_size, embedding_dim, padding_idx) + self.register_buffer("_float_tensor", torch.zeros(1)) # used for getting the right device + self.max_positions = int(1e5) + + # XXX: bart uses s/num_embeddings/num_positions/, s/weights/weight/ - could make those match + @staticmethod + def get_embedding(num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None): + """Build sinusoidal embeddings. + + This matches the implementation in tensor2tensor, but differs slightly + from the description in Section 3.5 of "Attention Is All You Need". + """ + half_dim = embedding_dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb) + emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0) + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1) + if embedding_dim % 2 == 1: + # zero pad + emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1) + if padding_idx is not None: + emb[padding_idx, :] = 0 + return emb + + def forward( + self, + input, + incremental_state: Optional[Any] = None, + timestep: Optional[Tensor] = None, + positions: Optional[Any] = None, + ): + """Input is expected to be of size [bsz x seqlen].""" + # bspair = torch.onnx.operators.shape_as_tensor(input) + # bsz, seq_len = bspair[0], bspair[1] + bsz, seq_len = input.shape[:2] + max_pos = self.padding_idx + 1 + seq_len + if self.weights is None or max_pos > self.weights.size(0): + # recompute/expand embeddings if needed + self.weights = SinusoidalPositionalEmbedding.get_embedding(max_pos, self.embedding_dim, self.padding_idx) + self.weights = self.weights.to(self._float_tensor) + + if incremental_state is not None: + # positions is the same for every token when decoding a single step + pos = timestep.view(-1)[0] + 1 if timestep is not None else seq_len + return self.weights[self.padding_idx + pos, :].expand(bsz, 1, -1) + + positions = make_positions(input, self.padding_idx) + + return self.weights.index_select(0, positions.view(-1)).view(bsz, seq_len, -1).detach() diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 41c1de3eec60..ce89918ec0fb 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -528,6 +528,7 @@ def resize_token_embeddings(self, new_num_tokens: Optional[int] = None) -> torch return model_embeds # Update base model and current model config + # XXX: now we have src_vocab_size/tgt_vocab_size self.config.vocab_size = new_num_tokens base_model.vocab_size = new_num_tokens diff --git a/src/transformers/tokenization_auto.py b/src/transformers/tokenization_auto.py index 626e576759bc..bb1254712b51 100644 --- a/src/transformers/tokenization_auto.py +++ b/src/transformers/tokenization_auto.py @@ -27,6 +27,7 @@ DistilBertConfig, ElectraConfig, FlaubertConfig, + FSMTConfig, GPT2Config, LongformerConfig, LxmertConfig, @@ -54,6 +55,7 @@ from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast from .tokenization_flaubert import FlaubertTokenizer +from .tokenization_fsmt import FSMTTokenizer from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast from .tokenization_lxmert import LxmertTokenizer, LxmertTokenizerFast @@ -102,6 +104,7 @@ (FlaubertConfig, (FlaubertTokenizer, None)), (XLMConfig, (XLMTokenizer, None)), (CTRLConfig, (CTRLTokenizer, None)), + (FSMTConfig, (FSMTTokenizer, None)), ] ) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py new file mode 100644 index 000000000000..0c5bcd957e50 --- /dev/null +++ b/src/transformers/tokenization_fsmt.py @@ -0,0 +1,567 @@ +# coding=utf-8 +# Copyright 2019 The Open AI Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for XLM.""" + + +import json +import logging +import os +import re +import unicodedata +from typing import Dict, List, Optional + +import sacremoses as sm + +from .tokenization_utils import PreTrainedTokenizer + + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = { + "src_vocab_file": "vocab-src.json", + "tgt_vocab_file": "vocab-tgt.json", + "merges_file": "merges.txt", +} + +PRETRAINED_VOCAB_FILES_MAP = { + "src_vocab_file": { + "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-ru.json", + "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-en.json", + "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-de.json", + "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-en.json", + }, + "tgt_vocab_file": { + "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-en.json", + "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-ru.json", + "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-en.json", + "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-de.json", + }, + "merges_file": { + "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/merges.txt", + "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/merges.txt", + "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/merges.txt", + "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/merges.txt", + }, +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "stas/fsmt-wmt19-ru-en": 1024, + "stas/fsmt-wmt19-en-ru": 1024, + "stas/fsmt-wmt19-de-en": 1024, + "stas/fsmt-wmt19-en-de": 1024, +} + +PRETRAINED_INIT_CONFIGURATION = { + "stas/fsmt-wmt19-ru-en": { + "langs": ["ru", "en"], + }, + "stas/fsmt-wmt19-en-ru": { + "langs": ["en", "ru"], + }, + "stas/fsmt-wmt19-de-en": { + "langs": ["de", "en"], + }, + "stas/fsmt-wmt19-en-de": { + "langs": ["en", "de"], + }, +} + + +def get_pairs(word): + """ + Return set of symbol pairs in a word. + word is represented as tuple of symbols (symbols being variable-length strings) + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def replace_unicode_punct(text): + """ + Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl + """ + text = text.replace("，", ",") + text = re.sub(r"。\s*", ". ", text) + text = text.replace("、", ",") + text = text.replace("”", '"') + text = text.replace("“", '"') + text = text.replace("∶", ":") + text = text.replace("：", ":") + text = text.replace("？", "?") + text = text.replace("《", '"') + text = text.replace("》", '"') + text = text.replace("）", ")") + text = text.replace("！", "!") + text = text.replace("（", "(") + text = text.replace("；", ";") + text = text.replace("１", "1") + text = text.replace("」", '"') + text = text.replace("「", '"') + text = text.replace("０", "0") + text = text.replace("３", "3") + text = text.replace("２", "2") + text = text.replace("５", "5") + text = text.replace("６", "6") + text = text.replace("９", "9") + text = text.replace("７", "7") + text = text.replace("８", "8") + text = text.replace("４", "4") + text = re.sub(r"．\s*", ". ", text) + text = text.replace("～", "~") + text = text.replace("’", "'") + text = text.replace("…", "...") + text = text.replace("━", "-") + text = text.replace("〈", "<") + text = text.replace("〉", ">") + text = text.replace("【", "[") + text = text.replace("】", "]") + text = text.replace("％", "%") + return text + + +def remove_non_printing_char(text): + """ + Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl + """ + output = [] + for char in text: + cat = unicodedata.category(char) + if cat.startswith("C"): + continue + output.append(char) + return "".join(output) + + +# Porting notes: +# this one is modeled after XLMTokenizer +# +# added: +# - src_vocab_file, +# - tgt_vocab_file, +# - langs, + + +class FSMTTokenizer(PreTrainedTokenizer): + """ + BPE tokenizer for FSMT (fairseq transformer) + See: https://github.com/pytorch/fairseq/tree/master/examples/wmt19 + + + - Moses preprocessing & tokenization for most supported languages + - (optionally) lower case & normalize all inputs text + - argument ``special_tokens`` and function ``set_special_tokens``, can be used to add additional symbols \ + (ex: "__classify__") to a vocabulary + - `langs` defines a pair of languages + + This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the methods. Users + should refer to the superclass for more information regarding methods. + + Args: + langs (:obj:`List[str]`): + a list of two languages to translate from and to, e.g. ``["en", "ru"]``. + src_vocab_file (:obj:`string`): + Source language vocabulary file. + tgt_vocab_file (:obj:`string`): + Target language vocabulary file. + merges_file (:obj:`string`): + Merges file. + do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to lowercase the input when tokenizing. + remove_space (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to strip the text when tokenizing (removing excess spaces before and after the string). + keep_accents (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to keep accents when tokenizing. + unk_token (:obj:`string`, `optional`, defaults to ""): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + bos_token (:obj:`string`, `optional`, defaults to ""): + The beginning of sequence token that was used during pre-training. Can be used a sequence classifier token. + + .. note:: + + When building a sequence using special tokens, this is not the token that is used for the beginning + of sequence. The token used is the :obj:`cls_token`. + sep_token (:obj:`string`, `optional`, defaults to ""): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences + for sequence classification or for a text and a question for question answering. + It is also used as the last token of a sequence built with special tokens. + pad_token (:obj:`string`, `optional`, defaults to ""): + The token used for padding, for example when batching sequences of different lengths. + cls_token (:obj:`string`, `optional`, defaults to ""): + The classifier token which is used when doing sequence classification (classification of the whole + sequence instead of per-token classification). It is the first token of the sequence when built with + special tokens. + mask_token (:obj:`string`, `optional`, defaults to ""): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + additional_special_tokens (:obj:`List[str]`, `optional`, defaults to :obj:`["","","","","","","","","",""]`): + List of additional special tokens. + + + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__( + self, + langs=None, + src_vocab_file=None, + tgt_vocab_file=None, + merges_file=None, + unk_token="", + bos_token="", + sep_token="", + pad_token="", + cls_token="", + mask_token="", + additional_special_tokens=[ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ], + **kwargs + ): + super().__init__( + unk_token=unk_token, + bos_token=bos_token, + sep_token=sep_token, + pad_token=pad_token, + cls_token=cls_token, + mask_token=mask_token, + additional_special_tokens=additional_special_tokens, + **kwargs, + ) + + self.src_vocab_file = src_vocab_file + self.tgt_vocab_file = tgt_vocab_file + self.merges_file = merges_file + + # cache of sm.MosesPunctNormalizer instance + self.cache_moses_punct_normalizer = dict() + # cache of sm.MosesTokenizer instance + self.cache_moses_tokenizer = dict() + self.cache_moses_detokenizer = dict() + + if len(langs) != 2: + raise ValueError(f"langs arg needs to be a list of 2 langs, e.g. ['en', 'ru'], but got f{langs}") + self.src_lang, self.tgt_lang = langs[0], langs[1] + + with open(src_vocab_file, encoding="utf-8") as src_vocab_handle: + self.encoder = json.load(src_vocab_handle) + with open(tgt_vocab_file, encoding="utf-8") as tgt_vocab_handle: + tgt_vocab = json.load(tgt_vocab_handle) + self.decoder = {v: k for k, v in tgt_vocab.items()} + with open(merges_file, encoding="utf-8") as merges_handle: + merges = merges_handle.read().split("\n")[:-1] + merges = [tuple(merge.split()[:2]) for merge in merges] + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {} + + # hack override + def get_vocab(self) -> Dict[str, int]: + return self.get_src_vocab() + + # hack override + @property + def vocab_size(self) -> int: + return self.src_vocab_size + + def moses_punct_norm(self, text, lang): + if lang not in self.cache_moses_punct_normalizer: + punct_normalizer = sm.MosesPunctNormalizer(lang=lang) + self.cache_moses_punct_normalizer[lang] = punct_normalizer + else: + punct_normalizer = self.cache_moses_punct_normalizer[lang] + return punct_normalizer.normalize(text) + + def moses_tokenize(self, text, lang): + if lang not in self.cache_moses_tokenizer: + moses_tokenizer = sm.MosesTokenizer(lang=lang) + self.cache_moses_tokenizer[lang] = moses_tokenizer + else: + moses_tokenizer = self.cache_moses_tokenizer[lang] + return moses_tokenizer.tokenize(text, aggressive_dash_splits=True, return_str=False, escape=True) + + def moses_detokenize(self, tokens, lang): + if lang not in self.cache_moses_tokenizer: + moses_detokenizer = sm.MosesDetokenizer(lang=self.tgt_lang) + self.cache_moses_detokenizer[lang] = moses_detokenizer + else: + moses_detokenizer = self.cache_moses_detokenizer[lang] + return moses_detokenizer.detokenize(tokens) + + def moses_pipeline(self, text, lang): + text = replace_unicode_punct(text) + text = self.moses_punct_norm(text, lang) + text = remove_non_printing_char(text) + return text + + @property + def src_vocab_size(self): + return len(self.encoder) + + @property + def tgt_vocab_size(self): + return len(self.decoder) + + def get_src_vocab(self): + return dict(self.encoder, **self.added_tokens_encoder) + + def get_tgt_vocab(self): + return dict(self.decoder, **self.added_tokens_decoder) + + def bpe(self, token): + word = tuple(token[:-1]) + (token[-1] + "",) + if token in self.cache: + return self.cache[token] + pairs = get_pairs(word) + + if not pairs: + return token + "" + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + except ValueError: + new_word.extend(word[i:]) + break + else: + new_word.extend(word[i:j]) + i = j + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = " ".join(word) + if word == "\n ": + word = "\n" + self.cache[token] = word + return word + + def _tokenize(self, text, lang="en", bypass_tokenizer=False): + """ + Tokenize a string given language code using Moses. + + Details of tokenization: + - [sacremoses](https://github.com/alvations/sacremoses): port of Moses + - Install with `pip install sacremoses` + + Args: + - lang: ISO language code (default = 'en') (string). Languages should belong of the model supported languages. However, we don't enforce it. + - bypass_tokenizer: Allow users to preprocess and tokenize the sentences externally (default = False) (bool). If True, we only apply BPE. + + Returns: + List of tokens. + """ + # ignore `lang` which is currently isn't explicitly passed in tokenization_utils.py and always results in lang=en + # if lang != self.src_lang: + # raise ValueError(f"Expected lang={self.src_lang}, but got {lang}") + lang = self.src_lang + + if bypass_tokenizer: + text = text.split() + else: + text = self.moses_pipeline(text, lang=lang) + text = self.moses_tokenize(text, lang=lang) + + split_tokens = [] + for token in text: + if token: + split_tokens.extend([t for t in self.bpe(token).split(" ")]) + + return split_tokens + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.encoder.get(token, self.encoder.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.decoder.get(index, self.unk_token) + + def convert_tokens_to_string(self, tokens): + """ Converts a sequence of tokens (string) in a single string. """ + + # remove BPE + tokens = [t.replace(" ", "").replace("", " ") for t in tokens] + tokens = "".join(tokens).split() + # detokenize + text = self.moses_detokenize(tokens, self.tgt_lang) + return text + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A FAIRSEQ_TRANSFORMER sequence has the following format: + + - single sequence: `` X `` + - pair of sequences: `` A B `` + + Args: + token_ids_0 (:obj:`List[int]`): + List of IDs to which the special tokens will be added + token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`): + Optional second list of IDs for sequence pairs. + + Returns: + :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens. + + """ + sep = [self.sep_token_id] + + # no bos used in fairseq + if token_ids_1 is None: + return token_ids_0 + sep + return token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` methods. + + Args: + token_ids_0 (:obj:`List[int]`): + List of ids. + token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`): + Set to True if the token list is already formatted with special tokens for the model + + Returns: + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError( + "You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model." + ) + return list( + map( + lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, + token_ids_0, + ) + ) + # no bos used in fairseq + if token_ids_1 is not None: + return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + An FAIRSEQ_TRANSFORMER sequence pair mask has the following format: + + :: + + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + + if token_ids_1 is None, only returns the first portion of the mask (0s). + + Args: + token_ids_0 (:obj:`List[int]`): + List of ids. + token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`): + Optional second list of IDs for sequence pairs. + + Returns: + :obj:`List[int]`: List of `token type IDs <../glossary.html#token-type-ids>`_ according to the given + sequence(s). + """ + sep = [self.sep_token_id] + + # no bos used in fairseq + if token_ids_1 is None: + return len(token_ids_0 + sep) * [0] + return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + + def save_vocabulary(self, save_directory): + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (:obj:`str`): + The directory in which to save the vocabulary. + + Returns: + :obj:`Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + + src_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES["src_vocab_file"]) + tgt_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES["tgt_vocab_file"]) + merges_file = os.path.join(save_directory, VOCAB_FILES_NAMES["merges_file"]) + + with open(src_vocab_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.encoder, ensure_ascii=False)) + + with open(tgt_vocab_file, "w", encoding="utf-8") as f: + tgt_vocab = {v: k for k, v in self.decoder.items()} + f.write(json.dumps(tgt_vocab, ensure_ascii=False)) + + index = 0 + with open(merges_file, "w", encoding="utf-8") as writer: + for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + "Saving vocabulary to {}: BPE merge indices are not consecutive." + " Please check that the tokenizer is not corrupted!".format(merges_file) + ) + index = token_index + writer.write(" ".join(bpe_tokens) + "\n") + index += 1 + + return src_vocab_file, tgt_vocab_file, merges_file diff --git a/src/transformers/utils/logging.py b/src/transformers/utils/logging.py index 1987718ddb5b..c906e967a2ba 100644 --- a/src/transformers/utils/logging.py +++ b/src/transformers/utils/logging.py @@ -15,6 +15,7 @@ """ Logging utilities. """ import logging +import re import threading from logging import CRITICAL # NOQA from logging import DEBUG # NOQA @@ -182,3 +183,40 @@ def enable_propagation() -> None: _configure_library_root_logger() _get_library_root_logger().propagate = True + + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + + +def logging_levels_as_strings(): + return log_levels.keys() + + +def logging_level_str_to_code(level_str): + if level_str in log_levels: + return log_levels[level_str] + else: + raise ValueError(f"unknown level {level_str}, has to be one of: { log_levels.keys() }") + + +def set_global_logging_level(level=logging.ERROR, prefices=[""]): + """ + Override logging levels of different modules based on their name as a prefix. + It needs to be invoked after the modules have been loaded so that their loggers have been initialized. + + Args: + - level: desired level. e.g. logging.INFO. Optional. Default is logging.ERROR + - prefices: list of one or more str prefices to match (e.g. ["transformers", "torch"]). Optional. + Default is `[""]` to match all active loggers. + The match is a case-sensitive `module_name.startswith(prefix)` + """ + prefix_re = re.compile(fr'^(?:{ "|".join(prefices) })') + for name in logging.root.manager.loggerDict: + if re.match(prefix_re, name): + logging.getLogger(name).setLevel(level) diff --git a/tests/conftest.py b/tests/conftest.py index 0a83207cb5bb..efd7092fb971 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,36 @@ import sys from os.path import abspath, dirname, join +import pytest + # allow having multiple repository checkouts and not needing to remember to rerun # 'pip install -e .[dev]' when switching between checkouts and running tests. git_repo_path = abspath(join(dirname(dirname(__file__)), "src")) sys.path.insert(1, git_repo_path) + +# import local modules after fixing up sys.path +if 1: # flake be quiet + from transformers.utils.logging import ( + logging_level_str_to_code, + logging_levels_as_strings, + set_global_logging_level, + ) + + +def pytest_addoption(parser): + parser.addoption( + "--loglevel", + type=str, + default=False, + choices=logging_levels_as_strings(), + help="set global logger level before each test", + ) + + +@pytest.fixture(scope="session", autouse=True) +def run_this_before_each_test(request): + # set the loglevel for all loggers to the desired level + loglevel = request.config.getoption("--loglevel") + if loglevel: + set_global_logging_level(level=logging_level_str_to_code(loglevel)) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py new file mode 100644 index 000000000000..a53c6d89fcfd --- /dev/null +++ b/tests/test_modeling_fsmt.py @@ -0,0 +1,509 @@ +# coding=utf-8 +# Copyright 2020 Huggingface +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +import timeout_decorator # noqa + +from transformers import is_torch_available +from transformers.file_utils import cached_property +from transformers.testing_utils import require_torch, slow, torch_device + +from .test_configuration_common import ConfigTester +from .test_modeling_common import ModelTesterMixin, ids_tensor + + +if is_torch_available(): + import torch + + from transformers import FSMTConfig, FSMTForConditionalGeneration, FSMTModel, FSMTTokenizer + from transformers.modeling_fsmt import ( + SinusoidalPositionalEmbedding, + _prepare_fsmt_decoder_inputs, + invert_mask, + shift_tokens_right, + ) +PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.""" + + +@require_torch +class ModelTester: + def __init__( + self, + parent, + ): + self.parent = parent + self.src_vocab_size = 99 + self.tgt_vocab_size = 99 + self.langs = ["ru", "en"] + self.batch_size = 13 + self.seq_length = 7 + self.is_training = False + self.use_labels = False + self.hidden_size = 16 + self.num_hidden_layers = 2 + self.num_attention_heads = 4 + self.intermediate_size = 4 + self.hidden_act = "relu" + self.hidden_dropout_prob = 0.1 + self.attention_probs_dropout_prob = 0.1 + self.max_position_embeddings = 20 + self.bos_token_id = 0 + self.pad_token_id = 1 + self.eos_token_id = 2 + torch.manual_seed(0) + + # hack needed for modeling_common tests - despite not really having this attribute in this model + self.vocab_size = self.src_vocab_size + + def prepare_config_and_inputs_for_common(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.src_vocab_size).clamp( + 3, + ) + input_ids[:, -1] = 2 # Eos Token + + config = FSMTConfig( + vocab_size=self.src_vocab_size, # hack needed for common tests + src_vocab_size=self.src_vocab_size, + tgt_vocab_size=self.tgt_vocab_size, + langs=self.langs, + d_model=self.hidden_size, + encoder_layers=self.num_hidden_layers, + decoder_layers=self.num_hidden_layers, + encoder_attention_heads=self.num_attention_heads, + decoder_attention_heads=self.num_attention_heads, + encoder_ffn_dim=self.intermediate_size, + decoder_ffn_dim=self.intermediate_size, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + eos_token_id=self.eos_token_id, + bos_token_id=self.bos_token_id, + pad_token_id=self.pad_token_id, + ) + inputs_dict = prepare_fsmt_inputs_dict(config, input_ids) + return config, inputs_dict + + +def prepare_fsmt_inputs_dict( + config, + input_ids, + attention_mask=None, +): + if attention_mask is None: + attention_mask = input_ids.ne(config.pad_token_id) + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + } + + +@require_torch +class FSMTModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = (FSMTModel, FSMTForConditionalGeneration) if is_torch_available() else () + all_generative_model_classes = (FSMTForConditionalGeneration,) if is_torch_available() else () + is_encoder_decoder = True + # TODO(SS): fix the below in a separate PR + test_pruning = False + test_torchscript = True + test_head_masking = False + test_resize_embeddings = True # This requires inputs_dict['input_ids'] + test_missing_keys = False # because FSMTForConditionalGeneration and FSMTModel now have identical state_dict + + def setUp(self): + self.model_tester = ModelTester(self) + # XXX: hack to appease to all other models having vocab_size + self.config_tester = ConfigTester(self, config_class=FSMTConfig, vocab_size=99) + + def test_config(self): + self.config_tester.run_common_tests() + + # XXX: override test_model_common_attributes / different Embedding type + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + self.assertIsInstance(model.get_input_embeddings(), (torch.nn.Embedding)) + model.set_input_embeddings(torch.nn.Embedding(10, 10)) + x = model.get_output_embeddings() + self.assertTrue(x is None or isinstance(x, torch.nn.modules.sparse.Embedding)) + + def test_initialization_more(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = FSMTModel(config) + model.to(torch_device) + model.eval() + # test init + # self.assertTrue((model.encoder.embed_tokens.weight == model.shared.weight).all().item()) + + def _check_var(module): + """Check that we initialized various parameters from N(0, config.init_std).""" + self.assertAlmostEqual(torch.std(module.weight).item(), config.init_std, 2) + + _check_var(model.encoder.embed_tokens) + _check_var(model.encoder.layers[0].self_attn.k_proj) + _check_var(model.encoder.layers[0].fc1) + # XXX: different std for fairseq version of SinusoidalPositionalEmbedding + # self.assertAlmostEqual(torch.std(model.encoder.embed_positions.weights).item(), config.init_std, 2) + + def test_advanced_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.use_cache = False + inputs_dict["input_ids"][:, -2:] = config.pad_token_id + decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_fsmt_decoder_inputs( + config, inputs_dict["input_ids"] + ) + model = FSMTModel(config).to(torch_device).eval() + + decoder_features_with_created_mask = model(**inputs_dict)[0] + decoder_features_with_passed_mask = model( + decoder_attention_mask=invert_mask(decoder_attn_mask), decoder_input_ids=decoder_input_ids, **inputs_dict + )[0] + _assert_tensors_equal(decoder_features_with_passed_mask, decoder_features_with_created_mask) + useless_mask = torch.zeros_like(decoder_attn_mask) + decoder_features = model(decoder_attention_mask=useless_mask, **inputs_dict)[0] + self.assertTrue(isinstance(decoder_features, torch.Tensor)) # no hidden states or attentions + self.assertEqual( + decoder_features.size(), + (self.model_tester.batch_size, self.model_tester.seq_length, config.tgt_vocab_size), + ) + if decoder_attn_mask.min().item() < -1e3: # some tokens were masked + self.assertFalse((decoder_features_with_created_mask == decoder_features).all().item()) + + # Test different encoder attention masks + decoder_features_with_long_encoder_mask = model( + inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"].long() + )[0] + _assert_tensors_equal(decoder_features_with_long_encoder_mask, decoder_features_with_created_mask) + + def test_save_load_strict(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + for model_class in self.all_model_classes: + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True) + self.assertEqual(info["missing_keys"], []) + + @unittest.skip("Passing inputs_embeds not implemented for FSMT.") + def test_inputs_embeds(self): + pass + + @unittest.skip("model weights aren't tied in FSMT.") + def test_tie_model_weights(self): + pass + + # def test_auto_model(self): + # # XXX: add a tiny model to s3? + # model_name = "stas/fsmt-wmt19-ru-en-tiny" + # tiny = AutoModel.from_pretrained(model_name) # same vocab size + # tok = AutoTokenizer.from_pretrained(model_name) # same tokenizer + # inputs_dict = tok.batch_encode_plus(["Hello my friends"], return_tensors="pt") + + # with torch.no_grad(): + # tiny(**inputs_dict) + + +@require_torch +class FSMTHeadTests(unittest.TestCase): + src_vocab_size = 99 + tgt_vocab_size = 99 + langs = ["ru", "en"] + + def _get_config_and_data(self): + input_ids = torch.tensor( + [ + [71, 82, 18, 33, 46, 91, 2], + [68, 34, 26, 58, 30, 82, 2], + [5, 97, 17, 39, 94, 40, 2], + [76, 83, 94, 25, 70, 78, 2], + [87, 59, 41, 35, 48, 66, 2], + [55, 13, 16, 58, 5, 2, 1], # note padding + [64, 27, 31, 51, 12, 75, 2], + [52, 64, 86, 17, 83, 39, 2], + [48, 61, 9, 24, 71, 82, 2], + [26, 1, 60, 48, 22, 13, 2], + [21, 5, 62, 28, 14, 76, 2], + [45, 98, 37, 86, 59, 48, 2], + [70, 70, 50, 9, 28, 0, 2], + ], + dtype=torch.long, + device=torch_device, + ) + + batch_size = input_ids.shape[0] + config = FSMTConfig( + src_vocab_size=self.src_vocab_size, + tgt_vocab_size=self.tgt_vocab_size, + langs=self.langs, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + return_dict=True, + ) + return config, input_ids, batch_size + + def test_generate_beam_search(self): + input_ids = torch.Tensor([[71, 82, 2], [68, 34, 2]]).long().to(torch_device) + config = FSMTConfig( + src_vocab_size=self.src_vocab_size, + tgt_vocab_size=self.tgt_vocab_size, + langs=self.langs, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + ) + lm_model = FSMTForConditionalGeneration(config).to(torch_device) + lm_model.eval() + + max_length = 5 + new_input_ids = lm_model.generate( + input_ids.clone(), + do_sample=True, + num_return_sequences=1, + num_beams=2, + no_repeat_ngram_size=3, + max_length=max_length, + ) + self.assertEqual(new_input_ids.shape, (input_ids.shape[0], max_length)) + # TODO(SS): uneven length batches, empty inputs + + def test_shift_tokens_right(self): + input_ids = torch.Tensor([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]).long() + shifted = shift_tokens_right(input_ids, 1) + n_pad_before = input_ids.eq(1).float().sum() + n_pad_after = shifted.eq(1).float().sum() + self.assertEqual(shifted.shape, input_ids.shape) + self.assertEqual(n_pad_after, n_pad_before - 1) + self.assertTrue(torch.eq(shifted[:, 0], 2).all()) + + def test_generate_fp16(self): + config, input_ids, batch_size = self._get_config_and_data() + attention_mask = input_ids.ne(1).to(torch_device) + model = FSMTForConditionalGeneration(config).eval().to(torch_device) + if torch_device == "cuda": + model.half() + model.generate(input_ids, attention_mask=attention_mask) + model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) + + def test_dummy_inputs(self): + config, *_ = self._get_config_and_data() + model = FSMTForConditionalGeneration(config).eval().to(torch_device) + model(**model.dummy_inputs) + + def test_prepare_fsmt_decoder_inputs(self): + config, *_ = self._get_config_and_data() + input_ids = _long_tensor(([4, 4, 2])) + decoder_input_ids = _long_tensor([[26388, 2, config.pad_token_id]]) + ignore = float("-inf") + decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_fsmt_decoder_inputs( + config, input_ids, decoder_input_ids + ) + expected_causal_mask = torch.tensor( + [[0, ignore, ignore], [0, 0, ignore], [0, 0, 0]] # never attend to the final token, because its pad + ).to(input_ids.device) + self.assertEqual(decoder_attn_mask.size(), decoder_input_ids.size()) + self.assertTrue(torch.eq(expected_causal_mask, causal_mask).all()) + + def test_resize_tokens_embeddings_more(self): + config, input_ids, _ = self._get_config_and_data() + + def _get_embs(m): + return (m.get_input_embeddings().weight.data.clone(), m.get_output_embeddings().weight.data.clone()) + + model = FSMTForConditionalGeneration(config).eval().to(torch_device) + + # not equal in FSMT + # input, output = _get_embs(model) + # self.assertTrue(torch.eq(input, output).all(), msg=f"\n{input}\n{output}") + + new_src_vocab_size = 45 + model.resize_token_embeddings(new_src_vocab_size) + input_new, output_new = _get_embs(model) + self.assertEqual( + input_new.shape, + (new_src_vocab_size, config.d_model), + msg=f"input {input_new.shape}, {(new_src_vocab_size, config.d_model)}", + ) + self.assertEqual( + output_new.shape, + (new_src_vocab_size, config.d_model), + msg=f"output {input_new.shape}, {(new_src_vocab_size, config.d_model)}", + ) + self.assertTrue(torch.eq(input_new, output_new).all(), msg=f"{input_new}, {output_new}") + + +def _assert_tensors_equal(a, b, atol=1e-12, prefix=""): + """If tensors not close, or a and b arent both tensors, raise a nice Assertion error.""" + if a is None and b is None: + return True + try: + if torch.allclose(a, b, atol=atol): + return True + raise + except Exception: + msg = "{} != {}".format(a, b) + if prefix: + msg = prefix + ": " + msg + raise AssertionError(msg) + + +def _long_tensor(tok_lst): + return torch.tensor(tok_lst, dtype=torch.long, device=torch_device) + + +TOLERANCE = 1e-4 + + +@require_torch +class FSMTModelIntegrationTests(unittest.TestCase): + @cached_property + def default_tokenizer(self): + return FSMTTokenizer.from_pretrained("stas/fsmt-wmt19-ru-en") + + @slow + def test_inference_no_head(self): + tokenizer = self.default_tokenizer + model = FSMTModel.from_pretrained("stas/fsmt-wmt19-ru-en").to(torch_device) + + src_text = "My friend computer will translate this for me" + input_ids = tokenizer([src_text], return_tensors="pt")["input_ids"] + input_ids = _long_tensor(input_ids) + inputs_dict = prepare_fsmt_inputs_dict(model.config, input_ids) + with torch.no_grad(): + output = model(**inputs_dict)[0] + expected_shape = torch.Size((1, model.config.decoder_attention_heads, model.config.tgt_vocab_size)) + self.assertEqual(output.shape, expected_shape) + expected_slice = torch.tensor( + [[-3.1850, -3.1849, 2.9694], [-4.0242, -4.0242, 0.2494], [-3.4442, -3.4443, 0.3315]], device=torch_device + ) + print(output[:, :3, :3]) + self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=TOLERANCE)) + + # XXX: the rest of the tests were moved to tests/test_tokenization_bart.py - port from there into tokenization tests + + @slow + def test_translation(self): + text = { + "en": "Machine learning is great, isn't it?", + "ru": "Машинное обучение - это здорово, не так ли?", + "de": "Maschinelles Lernen ist großartig, oder?", + } + + pairs = [ + ["en", "ru"], + ["ru", "en"], + ["en", "de"], + ["de", "en"], + ] + + for src, tgt in pairs: + print(f"Testing {src} -> {tgt}") + mname = f"stas/fsmt-wmt19-{src}-{tgt}" + + src_sentence = text[src] + tgt_sentence = text[tgt] + + tokenizer = FSMTTokenizer.from_pretrained(mname) + model = FSMTForConditionalGeneration.from_pretrained(mname) + + input_ids = tokenizer.encode(src_sentence, return_tensors="pt") + outputs = model.generate(input_ids) + decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) + assert decoded == tgt_sentence, f"\n\ngot: {decoded}\nexp: {tgt_sentence}\n" + + +@require_torch +class TestSinusoidalPositionalEmbeddings(unittest.TestCase): + padding_idx = 1 + tolerance = 1e-4 + + def test_basic(self): + input_ids = torch.tensor([[4, 10]], dtype=torch.long, device=torch_device) + emb1 = SinusoidalPositionalEmbedding(embedding_dim=6, padding_idx=self.padding_idx, init_size=6).to( + torch_device + ) + emb = emb1(input_ids) + desired_weights = torch.tensor( + [ + [9.0930e-01, 1.9999e-02, 2.0000e-04, -4.1615e-01, 9.9980e-01, 1.0000e00], + [1.4112e-01, 2.9995e-02, 3.0000e-04, -9.8999e-01, 9.9955e-01, 1.0000e00], + ] + ) + self.assertTrue( + torch.allclose(emb[0], desired_weights, atol=self.tolerance), + msg=f"\nexp:\n{desired_weights}\ngot:\n{emb[0]}\n", + ) + + def test_odd_embed_dim(self): + # odd embedding_dim is allowed + SinusoidalPositionalEmbedding.get_embedding( + num_embeddings=4, embedding_dim=5, padding_idx=self.padding_idx + ).to(torch_device) + + # odd num_embeddings is allowed + SinusoidalPositionalEmbedding.get_embedding( + num_embeddings=5, embedding_dim=4, padding_idx=self.padding_idx + ).to(torch_device) + + @unittest.skip("different from marian (needs more research)") + def test_positional_emb_weights_against_marian(self): + + desired_weights = torch.tensor( + [ + [0, 0, 0, 0, 0], + [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374], + [0.90929741, 0.93651021, 0.95829457, 0.97505713, 0.98720258], + ] + ) + emb1 = SinusoidalPositionalEmbedding(init_size=512, embedding_dim=512, padding_idx=self.padding_idx).to( + torch_device + ) + weights = emb1.weights.data[:3, :5] + # XXX: only the 1st and 3rd lines match - this is testing against + # verbatim copy of SinusoidalPositionalEmbedding from fairseq + self.assertTrue( + torch.allclose(weights, desired_weights, atol=self.tolerance), + msg=f"\nexp:\n{desired_weights}\ngot:\n{weights}\n", + ) + + # test that forward pass is just a lookup, there is no ignore padding logic + input_ids = torch.tensor( + [[4, 10, self.padding_idx, self.padding_idx, self.padding_idx]], dtype=torch.long, device=torch_device + ) + no_cache_pad_zero = emb1(input_ids)[0] + # XXX: only the 1st line matches the 3rd + self.assertTrue( + torch.allclose(torch.tensor(desired_weights, device=torch_device), no_cache_pad_zero[:3, :5], atol=1e-3) + ) diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py new file mode 100644 index 000000000000..2f6deb9e35c2 --- /dev/null +++ b/tests/test_tokenization_fsmt.py @@ -0,0 +1,148 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import os +import unittest + +from transformers.file_utils import cached_property +from transformers.testing_utils import slow +from transformers.tokenization_fsmt import VOCAB_FILES_NAMES, FSMTTokenizer + +from .test_tokenization_common import TokenizerTesterMixin + + +class FSMTTokenizationTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = FSMTTokenizer + + def setUp(self): + super().setUp() + + # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt + vocab = [ + "l", + "o", + "w", + "e", + "r", + "s", + "t", + "i", + "d", + "n", + "w", + "r", + "t", + "lo", + "low", + "er", + "low", + "lowest", + "newer", + "wider", + "", + ] + vocab_tokens = dict(zip(vocab, range(len(vocab)))) + merges = ["l o 123", "lo w 1456", "e r 1789", ""] + + self.langs = ["en", "ru"] + config = { + "langs": self.langs, + } + + self.src_vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["src_vocab_file"]) + self.tgt_vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["tgt_vocab_file"]) + config_file = os.path.join(self.tmpdirname, "tokenizer_config.json") + self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"]) + with open(self.src_vocab_file, "w") as fp: + fp.write(json.dumps(vocab_tokens)) + # XXX: ru content + with open(self.tgt_vocab_file, "w") as fp: + fp.write(json.dumps(vocab_tokens)) + with open(self.merges_file, "w") as fp: + fp.write("\n".join(merges)) + with open(config_file, "w") as fp: + fp.write(json.dumps(config)) + + @cached_property + def tokenizer_ru_en(self): + return FSMTTokenizer.from_pretrained("stas/fsmt-wmt19-ru-en") + + @cached_property + def tokenizer_en_ru(self): + return FSMTTokenizer.from_pretrained("stas/fsmt-wmt19-en-ru") + + def test_full_tokenizer(self): + """ Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt """ + tokenizer = FSMTTokenizer(self.langs, self.src_vocab_file, self.tgt_vocab_file, self.merges_file) + + text = "lower" + bpe_tokens = ["low", "er"] + tokens = tokenizer.tokenize(text) + self.assertListEqual(tokens, bpe_tokens) + + input_tokens = tokens + [""] + input_bpe_tokens = [14, 15, 20] + self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) + + @slow + def test_sequence_builders(self): + tokenizer = self.tokenizer_ru_en + + text = tokenizer.encode("sequence builders", add_special_tokens=False) + text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False) + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + assert encoded_sentence == text + [2] + assert encoded_pair == text + [2] + text_2 + [2] + + @slow + def test_match_encode_decode(self): + tokenizer_enc = self.tokenizer_en_ru + tokenizer_dec = self.tokenizer_ru_en + + targets = [ + [ + "Here's a little song I wrote. Don't worry, be happy.", + [2470, 39, 11, 2349, 7222, 70, 5979, 7, 8450, 1050, 13160, 5, 26, 6445, 7, 2], + ], + ["This is it. No more. I'm done!", [132, 21, 37, 7, 1434, 86, 7, 70, 6476, 1305, 427, 2]], + ] + + # this data was added as different mismatches were found, to validate + # the targets (or create more inputs if problems are found) run: + # + # import torch + # for src_text, _ in targets: + # mname = "transformer.wmt19.en-ru" + # checkpoint_file = "model1.pt" + # model = torch.hub.load( + # "pytorch/fairseq", mname, checkpoint_file=checkpoint_file, tokenizer="moses", bpe="fastbpe" + # ) + # encoded = model.encode(src_text) + # print(f"""[\n"{src_text}",\n {encoded.tolist()}\n],""") + + for src_text, tgt_input_ids in targets: + input_ids = tokenizer_enc.encode(src_text, return_tensors="pt")[0].tolist() + print(input_ids) + print(tgt_input_ids) + self.assertListEqual(input_ids, tgt_input_ids) + + # and decode backward, using the reversed languages model + decoded_text = tokenizer_dec.decode(input_ids, skip_special_tokens=True) + self.assertEqual(decoded_text, src_text) From 825f71e50fe4db540a2770660c78afcfc80e7938 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 3 Sep 2020 23:33:47 -0700 Subject: [PATCH 002/109] cleanup --- config.py | 212 ------------------------------------------------------ 1 file changed, 212 deletions(-) delete mode 100644 config.py diff --git a/config.py b/config.py deleted file mode 100644 index c8bf8193d432..000000000000 --- a/config.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os, sys -sys.path.insert(0, f"{os.getcwd()}/src") - - -import torch -from pprint import pprint -import fairseq - - -def dump_state_keys(state_dict): print("\n".join(state_dict.keys())) - - -# # Baseline - -#checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt' -checkpoint_file='model1.pt' -ru2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.ru-en', checkpoint_file=checkpoint_file, tokenizer='moses', bpe='fastbpe') - - -# from fairseq import hub_utils -# #checkpoint_file = 'model1.pt:model2.pt:model3.pt:model4.pt' -# checkpoint_file = 'model1.pt' -# model_name_or_path = 'transformer.wmt19.ru-en' -# data_name_or_path = '.' -# cls = fairseq.model_parallel.models.transformer.ModelParallelTransformerModel -# models = cls.hub_models() -# kwargs = {'bpe': 'fastbpe', 'tokenizer': 'moses'} - -# ru2en = hub_utils.from_pretrained( -# model_name_or_path, -# checkpoint_file, -# data_name_or_path, -# archive_map=models, -# **kwargs -# ) - - - -model = ru2en.models[0] -model - - - -args = dict(vars(ru2en.args)) - - - -args["source_lang"] -args["encoder_embed_dim"] -args["decoder_embed_dim"] - -pprint(args) - - - -pprint(args.keys()) - - - -model_state_dict = model.state_dict() -#model_state_dict - - - -#model = dict(vars(model)) -#dump_state_keys(model_state_dict) - - - -#model.items() -model_state_dict["decoder.layers.5.fc2.bias"].shape.numel() -model_state_dict["decoder.layers.5.fc2.bias"].shape[0] - - - -# dump the state_dict attrs and their shape -#pprint([f"{' '.join(map(str, v.shape)):>12} {k}"for k,v in model_state_dict.items()]) - - -# renames/removal -from collections import OrderedDict - -rename_keys = [ -# ("model.encoder.embed_positions._float_tensor", "model.encoder.embed_positions.weight"), -# ("model.decoder.embed_positions._float_tensor", "model.decoder.embed_positions.weight"), -# ("", ""), -# ("", ""), -# ("", ""), -# ("", ""), -] - -def remove_ignore_keys_(model_state_dict): - ignore_keys = [ - "model.model", - "model.encoder.version", - "model.decoder.version", - "model.encoder_embed_tokens.weight", - "model.decoder_embed_tokens.weight", -# "model.encoder.embed_positions._float_tensor", # not storing model.encoder.embed_positions.weight -# "model.decoder.embed_positions._float_tensor", # not storing model.decoder.embed_positions.weight - ] - for k in ignore_keys: - model_state_dict.pop(k, None) - -def rename_key(dct, old, new): - val = dct.pop(old) - dct[new] = val - -#model_state_dict = chkpt["model"].copy() - -# rename keys to start with model. -model_state_dict_new = OrderedDict(("model."+k, v) for k, v in model_state_dict.items()) -# check: -#model_state_dict["model.encoder.layers.0.fc1.bias"] -#chkpt["model"]["encoder.layers.0.fc1.bias"] - -remove_ignore_keys_(model_state_dict_new) -for src, dest in rename_keys: - rename_key(model_state_dict_new, src, dest) - -model_state_dict_new["model.decoder.embed_tokens.weight"].shape - -# XXX: emulate non-existing layer - perhaps it'll be removed instead in the model - for now just a bias of 0's -model_state_dict_new["final_logits_bias"] = torch.zeros((1, model_state_dict_new["model.decoder.embed_tokens.weight"].shape[0])) - -model_state_dict_new["final_logits_bias"].shape - - -from transformers.modeling_fsmt import FSMTForConditionalGeneration -from transformers.configuration_fsmt import FSMTConfig - -#dump_state_keys(model_state_dict_new) -#model_state_dict_new["model.decoder.embed_tokens.weight"].shape - - - -# let's add dummy things so that load_state_dict doesn't complain -# (embed_positions): SinusoidalPositionalEmbedding(1024, 1024) -# XXX: these seem to be autogenerated on the fly, no need to store -#model_state_dict_new["model.encoder.embed_positions.weight"] = model_state_dict_new["model.decoder.embed_positions.weight"] = torch.zeros((args["decoder_input_dim"], args["decoder_input_dim"])) - -#model_state_dict_new["model.encoder.embed_positions.weight"].shape -#model_state_dict_new["model.decoder.embed_positions.weight"].shape - -# these too get autogenerated: -# "model.encoder_embed_tokens.weight", -# "model.decoder_embed_tokens.weight", - -# # encoder_emd_tok_dim -# args["src_vocab_size"] = 31232 -# args["tgt_vocab_size"] = 31640 -# -# model_state_dict_new["model.encoder_embed_tokens.weight"] = torch.zeros((args["src_vocab_size"], args["encoder_embed_dim"])) -# -# model_state_dict_new["model.decoder_embed_tokens.weight"] = torch.zeros((args["tgt_vocab_size"], args["decoder_embed_dim"])) -# -# model_state_dict_new["model.encoder_embed_tokens.weight"].shape -# model_state_dict_new["model.decoder_embed_tokens.weight"].shape - - -hf_checkpoint_name = "/code/huggingface/transformers-fair-wmt/data/fsmt-wmt19-ru-en/config.json" -config = FSMTConfig.from_pretrained(hf_checkpoint_name) -model_new = FSMTForConditionalGeneration(config).eval() -#state_dict = chkpt["model"] - -import torch -def compare_state_dicts(d1, d2, cmp_func=torch.equal): - ok = 1 - for k in sorted( set(d1.keys()) | set(d2.keys()) ): - if k in d1 and k in d2: - if cmp_func(d1[k], d2[k]): - pass - else: - ok = 0 - print(f"Key {k}: values mismatch: \n{d1[k]}\n{d2[k]}\n") - else: - ok = 0 - which = "1st" if k in d2 else "2nd" - print(f"{which} dict doesn't have key {k}\n") - if ok: - print('Models match') -#compare_state_dicts(model_new.state_dict(), model_state_dict_new) - -torch.save(model_state_dict_new, "/tmp/new.pt") - -# show missing or extraneous/mismatching keys (need to remap/change model to match) -# XXX: somehow this is the key for making the model work -# if I remove this - it stops working -# FSMTForConditionalGeneration probably loads some garbage -model_new.load_state_dict(model_state_dict_new) - -model_new - - -from transformers.tokenization_fsmt import FSMTTokenizer -tokenizer = FSMTTokenizer.from_pretrained('fsmt-wmt19-ru-en') - -model_new.eval() - -sentence = "Машинное обучение - это здорово! Ты молодец." - -input_ids = tokenizer.encode(sentence, return_tensors='pt') -print(input_ids) -outputs = model_new.generate(input_ids)#, num_beams=5) -print("Outputs") -print(outputs) -for output in outputs: - decoded = tokenizer.decode(output, skip_special_tokens=True) - print(decoded) From f05b7c43dffb6c15e7ff5734f1b14862591928f6 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 3 Sep 2020 23:38:38 -0700 Subject: [PATCH 003/109] correct FSMT_PRETRAINED_MODEL_ARCHIVE_LIST --- src/transformers/modeling_fsmt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 9035af504e10..066281f31b9a 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -58,10 +58,10 @@ FSMT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/" - "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/" - "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/" - "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/" + "stas/fsmt-wmt19-ru-en" + "stas/fsmt-wmt19-en-ru" + "stas/fsmt-wmt19-de-en" + "stas/fsmt-wmt19-en-de" ] From 08ffb0cf87c8ab011246017811e7a9c70ad6e5f3 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 3 Sep 2020 23:39:03 -0700 Subject: [PATCH 004/109] fix --- src/transformers/modeling_fsmt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 066281f31b9a..d6e9a5ea06af 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -58,10 +58,10 @@ FSMT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "stas/fsmt-wmt19-ru-en" - "stas/fsmt-wmt19-en-ru" - "stas/fsmt-wmt19-de-en" - "stas/fsmt-wmt19-en-de" + "stas/fsmt-wmt19-ru-en", + "stas/fsmt-wmt19-en-ru", + "stas/fsmt-wmt19-de-en", + "stas/fsmt-wmt19-en-de", ] From aab6348a0533c3469ec368c51e52415279c9291c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 3 Sep 2020 23:43:26 -0700 Subject: [PATCH 005/109] perfectionism --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 8628b48c3508..59e1fcdc101d 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -382,7 +382,7 @@ def compare_state_dicts(d1, d2): print("\nLast step is to upload the files to s3") print(f"cd {data_root}") print(f"transformers-cli upload {model_dir}") - print(f"Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") + print("Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") if __name__ == "__main__": From de7fdd39ea8df6d0fd7a8bc83de0972c8136ee29 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 00:11:16 -0700 Subject: [PATCH 006/109] revert change from another PR --- model_cards/stas/fsmt-wmt19-de-en/README.md | 4 +- model_cards/stas/fsmt-wmt19-en-de/README.md | 4 +- model_cards/stas/fsmt-wmt19-en-ru/README.md | 4 +- model_cards/stas/fsmt-wmt19-ru-en/README.md | 4 +- ..._original_pytorch_checkpoint_to_pytorch.py | 4 +- src/transformers/utils/logging.py | 38 ------------------- tests/conftest.py | 28 -------------- 7 files changed, 5 insertions(+), 81 deletions(-) diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md index c14e227c9a8d..c2983948d822 100644 --- a/model_cards/stas/fsmt-wmt19-de-en/README.md +++ b/model_cards/stas/fsmt-wmt19-de-en/README.md @@ -12,7 +12,7 @@ metrics: - http://www.statmt.org/wmt19/metrics-task.html --- -# Model name +# FSMT ## Model description @@ -40,9 +40,7 @@ mname = "fsmt-wmt19-de-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -pair = ["de", "en"] input = "Maschinelles Lernen ist großartig, oder? - input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md index 18762a8a792b..b0057366a5e3 100644 --- a/model_cards/stas/fsmt-wmt19-en-de/README.md +++ b/model_cards/stas/fsmt-wmt19-en-de/README.md @@ -12,7 +12,7 @@ metrics: - http://www.statmt.org/wmt19/metrics-task.html --- -# Model name +# FSMT ## Model description @@ -40,9 +40,7 @@ mname = "fsmt-wmt19-en-de" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -pair = ["en", "de"] input = "Machine learning is great, isn't it? - input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md index 0b01f4095022..90cadddec694 100644 --- a/model_cards/stas/fsmt-wmt19-en-ru/README.md +++ b/model_cards/stas/fsmt-wmt19-en-ru/README.md @@ -12,7 +12,7 @@ metrics: - http://www.statmt.org/wmt19/metrics-task.html --- -# Model name +# FSMT ## Model description @@ -40,9 +40,7 @@ mname = "fsmt-wmt19-en-ru" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -pair = ["en", "ru"] input = "Machine learning is great, isn't it? - input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md index 59793af59fbd..b0f10857ca7d 100644 --- a/model_cards/stas/fsmt-wmt19-ru-en/README.md +++ b/model_cards/stas/fsmt-wmt19-ru-en/README.md @@ -12,7 +12,7 @@ metrics: - http://www.statmt.org/wmt19/metrics-task.html --- -# Model name +# FSMT ## Model description @@ -40,9 +40,7 @@ mname = "fsmt-wmt19-ru-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -pair = ["ru", "en"] input = "Машинное обучение - это здорово, не так ли? - input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 59e1fcdc101d..ef45eaff3f82 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -162,9 +162,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -pair = ["{src_lang}", "{tgt_lang}"] input = "{texts[src_lang]} - input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) @@ -382,7 +380,7 @@ def compare_state_dicts(d1, d2): print("\nLast step is to upload the files to s3") print(f"cd {data_root}") print(f"transformers-cli upload {model_dir}") - print("Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") + print(f"Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") if __name__ == "__main__": diff --git a/src/transformers/utils/logging.py b/src/transformers/utils/logging.py index c906e967a2ba..1987718ddb5b 100644 --- a/src/transformers/utils/logging.py +++ b/src/transformers/utils/logging.py @@ -15,7 +15,6 @@ """ Logging utilities. """ import logging -import re import threading from logging import CRITICAL # NOQA from logging import DEBUG # NOQA @@ -183,40 +182,3 @@ def enable_propagation() -> None: _configure_library_root_logger() _get_library_root_logger().propagate = True - - -log_levels = { - "debug": logging.DEBUG, - "info": logging.INFO, - "warning": logging.WARNING, - "error": logging.ERROR, - "critical": logging.CRITICAL, -} - - -def logging_levels_as_strings(): - return log_levels.keys() - - -def logging_level_str_to_code(level_str): - if level_str in log_levels: - return log_levels[level_str] - else: - raise ValueError(f"unknown level {level_str}, has to be one of: { log_levels.keys() }") - - -def set_global_logging_level(level=logging.ERROR, prefices=[""]): - """ - Override logging levels of different modules based on their name as a prefix. - It needs to be invoked after the modules have been loaded so that their loggers have been initialized. - - Args: - - level: desired level. e.g. logging.INFO. Optional. Default is logging.ERROR - - prefices: list of one or more str prefices to match (e.g. ["transformers", "torch"]). Optional. - Default is `[""]` to match all active loggers. - The match is a case-sensitive `module_name.startswith(prefix)` - """ - prefix_re = re.compile(fr'^(?:{ "|".join(prefices) })') - for name in logging.root.manager.loggerDict: - if re.match(prefix_re, name): - logging.getLogger(name).setLevel(level) diff --git a/tests/conftest.py b/tests/conftest.py index efd7092fb971..0a83207cb5bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,36 +4,8 @@ import sys from os.path import abspath, dirname, join -import pytest - # allow having multiple repository checkouts and not needing to remember to rerun # 'pip install -e .[dev]' when switching between checkouts and running tests. git_repo_path = abspath(join(dirname(dirname(__file__)), "src")) sys.path.insert(1, git_repo_path) - -# import local modules after fixing up sys.path -if 1: # flake be quiet - from transformers.utils.logging import ( - logging_level_str_to_code, - logging_levels_as_strings, - set_global_logging_level, - ) - - -def pytest_addoption(parser): - parser.addoption( - "--loglevel", - type=str, - default=False, - choices=logging_levels_as_strings(), - help="set global logger level before each test", - ) - - -@pytest.fixture(scope="session", autouse=True) -def run_this_before_each_test(request): - # set the loglevel for all loggers to the desired level - loglevel = request.config.getoption("--loglevel") - if loglevel: - set_global_logging_level(level=logging_level_str_to_code(loglevel)) From 2bd939d5cdff4205e5bc09a4829d6ea156397152 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 00:16:22 -0700 Subject: [PATCH 007/109] odd, already committed this one --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index ef45eaff3f82..261dde7a9abb 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -380,7 +380,7 @@ def compare_state_dicts(d1, d2): print("\nLast step is to upload the files to s3") print(f"cd {data_root}") print(f"transformers-cli upload {model_dir}") - print(f"Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") + print("Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") if __name__ == "__main__": From 6db73648a62b44ca42df2d5c516c3f5ef5e9af1c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 12:49:30 -0700 Subject: [PATCH 008/109] non-interactive upload workaround --- ...convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 261dde7a9abb..9e3b66cc8687 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -51,10 +51,10 @@ # upload cd data -transformers-cli upload fsmt-wmt19-ru-en -transformers-cli upload fsmt-wmt19-en-ru -transformers-cli upload fsmt-wmt19-de-en -transformers-cli upload fsmt-wmt19-en-de +yes Y | transformers-cli upload fsmt-wmt19-ru-en +yes Y | transformers-cli upload fsmt-wmt19-en-ru +yes Y | transformers-cli upload fsmt-wmt19-de-en +yes Y | transformers-cli upload fsmt-wmt19-en-de cd - # force cache invalidation, which will now download the new models From 1e62879eb0524cdcf7d9fe92bec0374554407837 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 12:50:15 -0700 Subject: [PATCH 009/109] backup the failed experiment --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 9e3b66cc8687..04a0bad9137f 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -335,8 +335,9 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder "model.encoder_embed_tokens.weight", "model.decoder_embed_tokens.weight", ] + # XXX: this experiment isn't working - needs more investigation # let's save a lot of space, by not saving unneeded keys - lots of them! - ignore_keys.extend(get_authorized_missing_keys()) + # ignore_keys.extend(get_authorized_missing_keys()) for k in ignore_keys: model_state_dict.pop(k, None) From 7918e275a6446e04a1810988aaf98f8f940238a7 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 15:21:44 -0700 Subject: [PATCH 010/109] store langs in config --- src/transformers/configuration_fsmt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index b7cff46bde63..ea74cf07e2a9 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -123,6 +123,7 @@ class FSMTConfig(PretrainedConfig): # update the defaults from config file def __init__( self, + langs=None, src_vocab_size=None, tgt_vocab_size=None, activation_function="relu", @@ -176,6 +177,7 @@ def __init__( tie_word_embeddings=tie_word_embeddings, **common_kwargs, ) + self.langs = langs self.src_vocab_size = src_vocab_size self.tgt_vocab_size = tgt_vocab_size self.d_model = d_model # encoder_embed_dim and decoder_embed_dim From d17bf3dca8cc9f78efc55ecea47070e74482146b Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 15:27:47 -0700 Subject: [PATCH 011/109] workaround for localizing model path --- src/transformers/tokenization_fsmt.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 0c5bcd957e50..105bc668ed7f 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -78,6 +78,21 @@ }, } +# XXX: temp workaround to be able to run local models with run_eval.py, etc. +LOCALIZE=1 +if LOCALIZE: + old, new = ("stas/", "/code/huggingface/transformers-fair-wmt/data/") + + def localize(buf): return buf.replace(old, new) + + for d in [PRETRAINED_INIT_CONFIGURATION, PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES]: + for k, v in d.copy().items(): + d[localize(k)] = v + + for d in [PRETRAINED_VOCAB_FILES_MAP]: + for tk, tv in d.items(): + for k, v in tv.copy().items(): + tv[localize(k)] = v def get_pairs(word): """ @@ -268,9 +283,10 @@ def __init__( self.cache_moses_tokenizer = dict() self.cache_moses_detokenizer = dict() - if len(langs) != 2: + if langs and len(langs) == 2: + self.src_lang, self.tgt_lang = langs + else: raise ValueError(f"langs arg needs to be a list of 2 langs, e.g. ['en', 'ru'], but got f{langs}") - self.src_lang, self.tgt_lang = langs[0], langs[1] with open(src_vocab_file, encoding="utf-8") as src_vocab_handle: self.encoder = json.load(src_vocab_handle) From 9e15bdebc29d6c4d81814f8a3b6f23165e2351a6 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 15:31:30 -0700 Subject: [PATCH 012/109] doc clean up as in https://github.com/huggingface/transformers/pull/6956 --- src/transformers/configuration_fsmt.py | 16 ++++++++-------- src/transformers/modeling_fsmt.py | 18 +++++++++--------- src/transformers/tokenization_fsmt.py | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index ea74cf07e2a9..9a11c070e43a 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -67,17 +67,17 @@ Typically set this to something large just in case (e.g., 512 or 1024 or 2048). init_std (:obj:`float`, optional, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - add_bias_logits (:obj:`bool`, optional, defaults to False): + add_bias_logits (:obj:`bool`, optional, defaults to :obj:`False`): True for marian only. - normalize_before (:obj:`bool`, optional, defaults to False): + normalize_before (:obj:`bool`, optional, defaults to :obj:`False`): Call layernorm before attention ops. - normalize_embedding (:obj:`bool`, optional, defaults to False): + normalize_embedding (:obj:`bool`, optional, defaults to :obj:`False`): Call layernorm after embeddings. - static_position_embeddings (:obj:`bool`, optional, defaults to True): + static_position_embeddings (:obj:`bool`, optional, defaults to :obj:`True`): Don't learn positional embeddings, use sinusoidal. - add_final_layer_norm (:obj:`bool`, optional, defaults to False): + add_final_layer_norm (:obj:`bool`, optional, defaults to :obj:`False`): Why not add another layernorm? - scale_embedding (:obj:`bool`, optional, defaults to True): + scale_embedding (:obj:`bool`, optional, defaults to :obj:`True`): Scale embeddings by diving by sqrt(d_model). bos_token_id (:obj:`int`, optional, defaults to 0) Beginning of stream token id. @@ -89,9 +89,9 @@ Google "layerdrop arxiv", as its not explainable in one line. decoder_layerdrop: (:obj:`float`, optional, defaults to 0.0): Google "layerdrop arxiv", as its not explainable in one line. - is_encoder_decoder (:obj:`bool`, optional, defaults to True): + is_encoder_decoder (:obj:`bool`, optional, defaults to :obj:`True`): Whether this is an encoder/decoder model. - tie_word_embeddings (:obj:`bool`, optional, defaults to False): + tie_word_embeddings (:obj:`bool`, optional, defaults to :obj:`False`): Whether to tie input and output embeddings. """ diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index d6e9a5ea06af..8a74ca07afa7 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -217,17 +217,17 @@ Indices of input sequence tokens in the vocabulary. Use FSMTTokenizer.encode to produce them. Padding will be ignored by default should you provide it. Indices can be obtained using :class:`transformers.FSMTTokenizer.encode(text)`. - attention_mask (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): + attention_mask (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Mask to avoid performing attention on padding token indices in input_ids. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - encoder_outputs (:obj:`tuple(tuple(torch.FloatTensor)`, `optional`, defaults to :obj:`None`): + encoder_outputs (:obj:`tuple(tuple(torch.FloatTensor)`, `optional`): Tuple consists of (`last_hidden_state`, `optional`: `hidden_states`, `optional`: `attentions`) - `last_hidden_state` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`) is a sequence of hidden-states at the output of the last layer of the encoder. + `last_hidden_state` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`, defaults to :obj:`None`): + decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`): Provide for translation and summarization training. By default, the model will create this tensor by shifting the input_ids right, following the paper. - decoder_attention_mask (:obj:`torch.BoolTensor` of shape :obj:`(batch_size, tgt_seq_len)`, `optional`, defaults to :obj:`None`): + decoder_attention_mask (:obj:`torch.BoolTensor` of shape :obj:`(batch_size, tgt_seq_len)`, `optional`): Default behavior: generate a tensor that ignores pad tokens in decoder_input_ids. Causal mask will also be used by default. If you want to change padding behavior, you should read :func:`~transformers.modeling_fairseqtranslator._prepare_decoder_inputs` and modify. See diagram 1 in the paper for more info on the default strategy @@ -240,11 +240,11 @@ use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): If `use_cache` is True, ``past_key_values`` are returned and can be used to speed up decoding (see ``past_key_values``). - output_attentions (:obj:`bool`, `optional`, defaults to :obj:`None`): + output_attentions (:obj:`bool`, `optional`): If set to ``True``, the attentions tensors of all attention layers are returned. See ``attentions`` under returned tensors for more detail. - output_hidden_states (:obj:`bool`, `optional`, defaults to :obj:`None`): + output_hidden_states (:obj:`bool`, `optional`): If set to ``True``, the hidden states of all layers are returned. See ``hidden_states`` under returned tensors for more detail. - return_dict (:obj:`bool`, `optional`, defaults to :obj:`None`): + return_dict (:obj:`bool`, `optional`): If set to ``True``, the model will return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. """ @@ -1151,7 +1151,7 @@ def forward( **unused, ): r""" - labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Labels for computing the masked language modeling loss. Indices should either be in ``[0, ..., config.vocab_size]`` or -100 (see ``input_ids`` docstring). Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 105bc668ed7f..ce4de967a75d 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -461,7 +461,7 @@ def build_inputs_with_special_tokens( Args: token_ids_0 (:obj:`List[int]`): List of IDs to which the special tokens will be added - token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`): + token_ids_1 (:obj:`List[int]`, `optional`): Optional second list of IDs for sequence pairs. Returns: @@ -485,7 +485,7 @@ def get_special_tokens_mask( Args: token_ids_0 (:obj:`List[int]`): List of ids. - token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`): + token_ids_1 (:obj:`List[int]`, `optional`): Optional second list of IDs for sequence pairs. already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`): Set to True if the token list is already formatted with special tokens for the model @@ -528,7 +528,7 @@ def create_token_type_ids_from_sequences( Args: token_ids_0 (:obj:`List[int]`): List of ids. - token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`): + token_ids_1 (:obj:`List[int]`, `optional`): Optional second list of IDs for sequence pairs. Returns: From c8b16ba1f75d4eccf71e4f121f8182cea55c937a Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 15:33:03 -0700 Subject: [PATCH 013/109] style --- ...convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 2 +- src/transformers/tokenization_fsmt.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 04a0bad9137f..7892acbd3bcd 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -79,7 +79,7 @@ from transformers import WEIGHTS_NAME from transformers.configuration_fsmt import FSMTConfig -from transformers.modeling_fsmt import FSMTForConditionalGeneration, get_authorized_missing_keys +from transformers.modeling_fsmt import FSMTForConditionalGeneration # , get_authorized_missing_keys from transformers.tokenization_fsmt import VOCAB_FILES_NAMES diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index ce4de967a75d..d18d33274adb 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -79,11 +79,12 @@ } # XXX: temp workaround to be able to run local models with run_eval.py, etc. -LOCALIZE=1 +LOCALIZE = 1 if LOCALIZE: - old, new = ("stas/", "/code/huggingface/transformers-fair-wmt/data/") + old, new = ("stas/", "/code/huggingface/transformers-fair-wmt/data/") - def localize(buf): return buf.replace(old, new) + def localize(buf): + return buf.replace(old, new) for d in [PRETRAINED_INIT_CONFIGURATION, PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES]: for k, v in d.copy().items(): @@ -94,6 +95,7 @@ def localize(buf): return buf.replace(old, new) for k, v in tv.copy().items(): tv[localize(k)] = v + def get_pairs(word): """ Return set of symbol pairs in a word. From a95e04ae5322dbd88000e63ee9ce35f3aba1b90e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 15:35:55 -0700 Subject: [PATCH 014/109] back out debug mode --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 7892acbd3bcd..5c348cf64768 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -85,8 +85,7 @@ logging.basicConfig(level=logging.INFO) -DEBUG = 1 - +DEBUG = 0 json_indent = 2 if DEBUG else None From e126bdfb0f5e6ec97ddcdbf7993fccf385e0491a Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 15:52:26 -0700 Subject: [PATCH 015/109] document: run_eval.py --num_beams 10 --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 5c348cf64768..37db02e81891 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -193,11 +193,12 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO From 352f6764c4f13601cfd2cf02ca967d319c6862b6 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 16:03:25 -0700 Subject: [PATCH 016/109] remove unneeded constant --- src/transformers/__init__.py | 7 +------ src/transformers/modeling_fsmt.py | 9 --------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index bc0ae333ac8c..42b90e2f918a 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -329,12 +329,7 @@ FlaubertModel, FlaubertWithLMHeadModel, ) - from .modeling_fsmt import ( - FSMT_PRETRAINED_MODEL_ARCHIVE_LIST, - FSMTForConditionalGeneration, - FSMTModel, - PretrainedFSMTModel, - ) + from .modeling_fsmt import FSMTForConditionalGeneration, FSMTModel, PretrainedFSMTModel from .modeling_gpt2 import ( GPT2_PRETRAINED_MODEL_ARCHIVE_LIST, GPT2DoubleHeadsModel, diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 8a74ca07afa7..f90b9edb09be 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -56,15 +56,6 @@ _CONFIG_FOR_DOC = "FSMTConfig" _TOKENIZER_FOR_DOC = "FSMTTokenizer" - -FSMT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "stas/fsmt-wmt19-ru-en", - "stas/fsmt-wmt19-en-ru", - "stas/fsmt-wmt19-de-en", - "stas/fsmt-wmt19-en-de", -] - - # See all FSMT models at https://huggingface.co/models?search=fsmt From a2a2cca2ed2fe598a48cad673b737acbf381a636 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 16:09:07 -0700 Subject: [PATCH 017/109] typo --- src/transformers/configuration_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 9a11c070e43a..442e4c2b67f2 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" XLM configuration """ +""" FSMT configuration """ import logging From 0c23e74c4f81f3819e1b783f9d288a58da68d4d0 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 16:16:02 -0700 Subject: [PATCH 018/109] re-use bart's Attention --- src/transformers/modeling_fsmt.py | 159 +----------------------------- 1 file changed, 4 insertions(+), 155 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index f90b9edb09be..bf7c82c5063b 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -47,6 +47,7 @@ add_start_docstrings_to_callable, replace_return_docstrings, ) +from .modeling_bart import Attention from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel @@ -333,7 +334,7 @@ class EncoderLayer(nn.Module): def __init__(self, config: FSMTConfig): super().__init__() self.embed_dim = config.d_model - self.self_attn = SelfAttention( + self.self_attn = Attention( self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, @@ -489,7 +490,7 @@ class DecoderLayer(nn.Module): def __init__(self, config: FSMTConfig): super().__init__() self.embed_dim = config.d_model - self.self_attn = SelfAttention( + self.self_attn = Attention( embed_dim=self.embed_dim, num_heads=config.decoder_attention_heads, dropout=config.attention_dropout, @@ -500,7 +501,7 @@ def __init__(self, config: FSMTConfig): self.normalize_before = config.normalize_before self.self_attn_layer_norm = LayerNorm(self.embed_dim) - self.encoder_attn = SelfAttention( + self.encoder_attn = Attention( self.embed_dim, config.decoder_attention_heads, dropout=config.attention_dropout, @@ -751,158 +752,6 @@ def _reorder_buffer(attn_cache, new_order): return attn_cache -class SelfAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__( - self, - embed_dim, - num_heads, - dropout=0.0, - bias=True, - encoder_decoder_attention=False, # otherwise self_attention - ): - super().__init__() - self.embed_dim = embed_dim - self.num_heads = num_heads - self.dropout = dropout - self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" - self.scaling = self.head_dim ** -0.5 - - self.encoder_decoder_attention = encoder_decoder_attention - self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.cache_key = "encoder_decoder" if self.encoder_decoder_attention else "self" - - def _shape(self, tensor, seq_len, bsz): - return tensor.contiguous().view(seq_len, bsz * self.num_heads, self.head_dim).transpose(0, 1) - - def forward( - self, - query, - key: Optional[Tensor], - key_padding_mask: Optional[Tensor] = None, - layer_state: Optional[Dict[str, Optional[Tensor]]] = None, - attn_mask: Optional[Tensor] = None, - output_attentions=False, - ) -> Tuple[Tensor, Optional[Tensor]]: - """Input shape: Time(SeqLen) x Batch x Channel""" - static_kv: bool = self.encoder_decoder_attention - tgt_len, bsz, embed_dim = query.size() - assert embed_dim == self.embed_dim - assert list(query.size()) == [tgt_len, bsz, embed_dim] - # get here for encoder decoder cause of static_kv - if layer_state is not None: # reuse k,v and encoder_padding_mask - saved_state = layer_state.get(self.cache_key, {}) - if "prev_key" in saved_state and static_kv: - # previous time steps are cached - no need to recompute key and value if they are static - key = None - else: - saved_state = None - layer_state = {} - - q = self.q_proj(query) * self.scaling - if static_kv: - if key is None: - k = v = None - else: - k = self.k_proj(key) - v = self.v_proj(key) - else: - k = self.k_proj(query) - v = self.v_proj(query) - - q = self._shape(q, tgt_len, bsz) - if k is not None: - k = self._shape(k, -1, bsz) - if v is not None: - v = self._shape(v, -1, bsz) - - if saved_state is not None: - k, v, key_padding_mask = self._use_saved_state(k, v, saved_state, key_padding_mask, static_kv, bsz) - - # Update cache - layer_state[self.cache_key] = { - "prev_key": k.view(bsz, self.num_heads, -1, self.head_dim), - "prev_value": v.view(bsz, self.num_heads, -1, self.head_dim), - "prev_key_padding_mask": key_padding_mask if not static_kv else None, - } - - assert k is not None - src_len = k.size(1) - attn_weights = torch.bmm(q, k.transpose(1, 2)) - assert attn_weights.size() == (bsz * self.num_heads, tgt_len, src_len) - - if attn_mask is not None: - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_mask - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - # This is part of a workaround to get around fork/join parallelism not supporting Optional types. - if key_padding_mask is not None and key_padding_mask.dim() == 0: - key_padding_mask = None - assert key_padding_mask is None or key_padding_mask.size()[:2] == ( - bsz, - src_len, - ) - - if key_padding_mask is not None: # don't attend to padding symbols - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - reshaped = key_padding_mask.unsqueeze(1).unsqueeze(2) - attn_weights = attn_weights.masked_fill(reshaped, float("-inf")) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - attn_weights = F.softmax(attn_weights, dim=-1) - attn_probs = F.dropout( - attn_weights, - p=self.dropout, - training=self.training, - ) - - assert v is not None - attn_output = torch.bmm(attn_probs, v) - assert attn_output.size() == (bsz * self.num_heads, tgt_len, self.head_dim) - attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim) - attn_output = self.out_proj(attn_output) - if output_attentions: - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - else: - attn_weights = None - return attn_output, attn_weights - - def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz): - # saved states are stored with shape (bsz, num_heads, seq_len, head_dim) - if "prev_key" in saved_state: - _prev_key = saved_state["prev_key"] - assert _prev_key is not None - prev_key = _prev_key.view(bsz * self.num_heads, -1, self.head_dim) - if static_kv: - k = prev_key - else: - assert k is not None - k = torch.cat([prev_key, k], dim=1) - if "prev_value" in saved_state: - _prev_value = saved_state["prev_value"] - assert _prev_value is not None - prev_value = _prev_value.view(bsz * self.num_heads, -1, self.head_dim) - if static_kv: - v = prev_value - else: - assert v is not None - v = torch.cat([prev_value, v], dim=1) - assert k is not None and v is not None - prev_key_padding_mask: Optional[Tensor] = saved_state.get("prev_key_padding_mask", None) - if prev_key_padding_mask is not None: - if static_kv: - new_key_padding_mask = prev_key_padding_mask - else: - new_key_padding_mask = torch.cat([prev_key_padding_mask, key_padding_mask], dim=1) - else: - new_key_padding_mask = key_padding_mask - return k, v, new_key_padding_mask - - # XXX: remove this and its references class LearnedPositionalEmbedding(nn.Embedding): """ From ae8d10f4f687a6ff9f30ec2186b9aeca98d5e3e7 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 16:26:03 -0700 Subject: [PATCH 019/109] re-use EncoderLayer, DecoderLayer from bart --- src/transformers/modeling_fsmt.py | 153 +----------------------------- 1 file changed, 3 insertions(+), 150 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index bf7c82c5063b..0c39b090f4d2 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -47,7 +47,7 @@ add_start_docstrings_to_callable, replace_return_docstrings, ) -from .modeling_bart import Attention +from .modeling_bart import Attention, EncoderLayer, DecoderLayer from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel @@ -329,61 +329,6 @@ def make_padding_mask(input_ids, padding_idx=1): # Helper Modules - -class EncoderLayer(nn.Module): - def __init__(self, config: FSMTConfig): - super().__init__() - self.embed_dim = config.d_model - self.self_attn = Attention( - self.embed_dim, - config.encoder_attention_heads, - dropout=config.attention_dropout, - ) - self.normalize_before = config.normalize_before - self.self_attn_layer_norm = LayerNorm(self.embed_dim) - self.dropout = config.dropout - self.activation_fn = ACT2FN[config.activation_function] - self.activation_dropout = config.activation_dropout - self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) - self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) - self.final_layer_norm = LayerNorm(self.embed_dim) - - def forward(self, x, encoder_padding_mask, output_attentions=False): - """ - Args: - x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)` - encoder_padding_mask (ByteTensor): binary ByteTensor of shape - `(batch, src_len)` where padding elements are indicated by ``1``. - for t_tgt, t_src is excluded (or masked out), =0 means it is - included in attention - - Returns: - encoded output of shape `(seq_len, batch, embed_dim)` - """ - residual = x - if self.normalize_before: - x = self.self_attn_layer_norm(x) - x, attn_weights = self.self_attn( - query=x, key=x, key_padding_mask=encoder_padding_mask, output_attentions=output_attentions - ) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - if not self.normalize_before: - x = self.self_attn_layer_norm(x) - - residual = x - if self.normalize_before: - x = self.final_layer_norm(x) - x = self.activation_fn(self.fc1(x)) - x = F.dropout(x, p=self.activation_dropout, training=self.training) - x = self.fc2(x) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - if not self.normalize_before: - x = self.final_layer_norm(x) - return x, attn_weights - - class FSMTEncoder(nn.Module): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer @@ -422,7 +367,7 @@ def __init__(self, config: FSMTConfig, embed_tokens): ) self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.encoder_layers)]) self.layernorm_embedding = LayerNorm(embed_dim) if config.normalize_embedding else nn.Identity() - # mfairseqtranslator has one extra layer_norm + # mbart has one extra layer_norm self.layer_norm = LayerNorm(config.d_model) if config.normalize_before else None def forward( @@ -485,98 +430,6 @@ def forward( return tuple(v for v in [x, encoder_states, all_attentions] if v is not None) return BaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions) - -class DecoderLayer(nn.Module): - def __init__(self, config: FSMTConfig): - super().__init__() - self.embed_dim = config.d_model - self.self_attn = Attention( - embed_dim=self.embed_dim, - num_heads=config.decoder_attention_heads, - dropout=config.attention_dropout, - ) - self.dropout = config.dropout - self.activation_fn = ACT2FN[config.activation_function] - self.activation_dropout = config.activation_dropout - self.normalize_before = config.normalize_before - - self.self_attn_layer_norm = LayerNorm(self.embed_dim) - self.encoder_attn = Attention( - self.embed_dim, - config.decoder_attention_heads, - dropout=config.attention_dropout, - encoder_decoder_attention=True, - ) - self.encoder_attn_layer_norm = LayerNorm(self.embed_dim) - self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) - self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) - self.final_layer_norm = LayerNorm(self.embed_dim) - - def forward( - self, - x, - encoder_hidden_states, - encoder_attn_mask=None, - layer_state=None, - causal_mask=None, - decoder_padding_mask=None, - output_attentions=False, - ): - residual = x - - if layer_state is None: - layer_state = {} - if self.normalize_before: - x = self.self_attn_layer_norm(x) - # Self Attention - - x, self_attn_weights = self.self_attn( - query=x, - key=x, - layer_state=layer_state, # adds keys to layer state - key_padding_mask=decoder_padding_mask, - attn_mask=causal_mask, - output_attentions=output_attentions, - ) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - if not self.normalize_before: - x = self.self_attn_layer_norm(x) - - # Cross attention - residual = x - assert self.encoder_attn.cache_key != self.self_attn.cache_key - if self.normalize_before: - x = self.encoder_attn_layer_norm(x) - x, _ = self.encoder_attn( - query=x, - key=encoder_hidden_states, - key_padding_mask=encoder_attn_mask, - layer_state=layer_state, # mutates layer state - ) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - if not self.normalize_before: - x = self.encoder_attn_layer_norm(x) - - # Fully Connected - residual = x - if self.normalize_before: - x = self.final_layer_norm(x) - x = self.activation_fn(self.fc1(x)) - x = F.dropout(x, p=self.activation_dropout, training=self.training) - x = self.fc2(x) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - if not self.normalize_before: - x = self.final_layer_norm(x) - return ( - x, - self_attn_weights, - layer_state, - ) # just self_attn weights for now, following t5, layer_state = cache for decoding - - class FSMTDecoder(nn.Module): """ Transformer decoder consisting of *config.decoder_layers* layers. Each layer @@ -721,7 +574,7 @@ def forward( if use_cache: next_decoder_cache.append(layer_past.copy()) - if self.layer_norm and (idx == len(self.layers) - 1): # last layer of mfairseqtranslator + if self.layer_norm and (idx == len(self.layers) - 1): # last layer of mbart x = self.layer_norm(x) if output_attentions: all_self_attns += (layer_self_attn,) From 2795b067b3d99d6b69d4885a4c3ba89dc46c1e12 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 16:31:56 -0700 Subject: [PATCH 020/109] refactor --- src/transformers/modeling_fsmt.py | 4 ++- tests/test_modeling_fsmt.py | 54 ++++++++++++------------------- 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 0c39b090f4d2..e43cf9d47ea9 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -47,7 +47,7 @@ add_start_docstrings_to_callable, replace_return_docstrings, ) -from .modeling_bart import Attention, EncoderLayer, DecoderLayer +from .modeling_bart import Attention, DecoderLayer, EncoderLayer from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel @@ -329,6 +329,7 @@ def make_padding_mask(input_ids, padding_idx=1): # Helper Modules + class FSMTEncoder(nn.Module): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer @@ -430,6 +431,7 @@ def forward( return tuple(v for v in [x, encoder_states, all_attentions] if v is not None) return BaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions) + class FSMTDecoder(nn.Module): """ Transformer decoder consisting of *config.decoder_layers* layers. Each layer diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index a53c6d89fcfd..b5c98fe102f3 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -225,6 +225,25 @@ class FSMTHeadTests(unittest.TestCase): tgt_vocab_size = 99 langs = ["ru", "en"] + def _get_config(self): + return FSMTConfig( + src_vocab_size=self.src_vocab_size, + tgt_vocab_size=self.tgt_vocab_size, + langs=self.langs, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + return_dict=True, + ) + def _get_config_and_data(self): input_ids = torch.tensor( [ @@ -247,43 +266,12 @@ def _get_config_and_data(self): ) batch_size = input_ids.shape[0] - config = FSMTConfig( - src_vocab_size=self.src_vocab_size, - tgt_vocab_size=self.tgt_vocab_size, - langs=self.langs, - d_model=24, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=32, - decoder_ffn_dim=32, - max_position_embeddings=48, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - return_dict=True, - ) + config = self._get_config() return config, input_ids, batch_size def test_generate_beam_search(self): input_ids = torch.Tensor([[71, 82, 2], [68, 34, 2]]).long().to(torch_device) - config = FSMTConfig( - src_vocab_size=self.src_vocab_size, - tgt_vocab_size=self.tgt_vocab_size, - langs=self.langs, - d_model=24, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=32, - decoder_ffn_dim=32, - max_position_embeddings=48, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - ) + config = self._get_config() lm_model = FSMTForConditionalGeneration(config).to(torch_device) lm_model.eval() From 5931fe3571ba4b8d3866987eb0f4231dfacac2dc Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 16:36:38 -0700 Subject: [PATCH 021/109] send to cuda and fp16 --- tests/test_modeling_fsmt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index b5c98fe102f3..1304a112957d 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -424,7 +424,9 @@ def test_translation(self): tgt_sentence = text[tgt] tokenizer = FSMTTokenizer.from_pretrained(mname) - model = FSMTForConditionalGeneration.from_pretrained(mname) + model = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) + if torch_device == "cuda": + model.half() input_ids = tokenizer.encode(src_sentence, return_tensors="pt") outputs = model.generate(input_ids) From b027778f5cd0df9b55d7a328723471b3db2a33e5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 16:38:35 -0700 Subject: [PATCH 022/109] cleanup --- src/transformers/modeling_fsmt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index e43cf9d47ea9..be0ab50941e5 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -31,14 +31,13 @@ import math import random import warnings -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, List, Optional, Tuple import torch import torch.nn.functional as F from torch import Tensor, nn from torch.nn import CrossEntropyLoss -from .activations import ACT2FN from .configuration_fsmt import FSMTConfig from .file_utils import ( add_code_sample_docstrings, @@ -47,7 +46,7 @@ add_start_docstrings_to_callable, replace_return_docstrings, ) -from .modeling_bart import Attention, DecoderLayer, EncoderLayer +from .modeling_bart import DecoderLayer, EncoderLayer from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel From c35ef1c5c2dfc74f46f7890830ba0567a400fceb Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 4 Sep 2020 17:26:48 -0700 Subject: [PATCH 023/109] revert (moved to another PR) --- src/transformers/generation_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/generation_utils.py b/src/transformers/generation_utils.py index edb1b17cc11c..573c1c4927a3 100644 --- a/src/transformers/generation_utils.py +++ b/src/transformers/generation_utils.py @@ -411,7 +411,7 @@ def generate( ) # shape: (batch_size * num_return_sequences * num_beams, cur_len) if self.config.is_encoder_decoder: - # create empty decoder input_ids + # create empty decoder_input_ids input_ids = torch.full( (effective_batch_size * num_beams, 1), decoder_start_token_id, From f198df03bdfaaf300671daa365caaf3682f8fa36 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 5 Sep 2020 11:15:17 -0700 Subject: [PATCH 024/109] better error message --- src/transformers/tokenization_fsmt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index d18d33274adb..263fa784449b 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -288,7 +288,11 @@ def __init__( if langs and len(langs) == 2: self.src_lang, self.tgt_lang = langs else: - raise ValueError(f"langs arg needs to be a list of 2 langs, e.g. ['en', 'ru'], but got f{langs}") + raise ValueError( + f"arg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got {langs}. " + "Usually that means that tokenizer can't find a mapping for the given model path " + "in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer." + ) with open(src_vocab_file, encoding="utf-8") as src_vocab_handle: self.encoder = json.load(src_vocab_handle) From 239a8f1eb0dc224545c48b2a1c3e799e2a5a394e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 5 Sep 2020 13:49:49 -0700 Subject: [PATCH 025/109] document run_eval --num_beams --- model_cards/stas/fsmt-wmt19-de-en/README.md | 3 ++- model_cards/stas/fsmt-wmt19-en-de/README.md | 3 ++- model_cards/stas/fsmt-wmt19-en-ru/README.md | 3 ++- model_cards/stas/fsmt-wmt19-ru-en/README.md | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md index c2983948d822..d7cea62318c7 100644 --- a/model_cards/stas/fsmt-wmt19-de-en/README.md +++ b/model_cards/stas/fsmt-wmt19-de-en/README.md @@ -72,11 +72,12 @@ export PAIR=de-en export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md index b0057366a5e3..1aa2405f0bfa 100644 --- a/model_cards/stas/fsmt-wmt19-en-de/README.md +++ b/model_cards/stas/fsmt-wmt19-en-de/README.md @@ -72,11 +72,12 @@ export PAIR=en-de export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md index 90cadddec694..79101448633b 100644 --- a/model_cards/stas/fsmt-wmt19-en-ru/README.md +++ b/model_cards/stas/fsmt-wmt19-en-ru/README.md @@ -72,11 +72,12 @@ export PAIR=en-ru export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md index b0f10857ca7d..d4d2e690827a 100644 --- a/model_cards/stas/fsmt-wmt19-ru-en/README.md +++ b/model_cards/stas/fsmt-wmt19-ru-en/README.md @@ -72,11 +72,12 @@ export PAIR=ru-en export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO From fbdb96cea0ac0f9b322851f62fc9e1ea27d10db2 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 5 Sep 2020 13:51:40 -0700 Subject: [PATCH 026/109] solve the problem of tokenizer finding the right files when model is local --- ..._original_pytorch_checkpoint_to_pytorch.py | 55 ++++++++++++------- src/transformers/tokenization_fsmt.py | 33 +++-------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 37db02e81891..986de2e41b93 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -57,6 +57,10 @@ yes Y | transformers-cli upload fsmt-wmt19-en-de cd - +# if updating just small files and not the large models, here is a script to generate the right commands: +perl -le 'for $f (@ARGV) { print qq[yes Y | transformers-cli upload $_/$f --filename $_/$f] for map { "fsmt-wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json +# add/remove files as needed + # force cache invalidation, which will now download the new models PYTHONPATH="src" python -c 'from transformers import AutoModel; [AutoModel.from_pretrained("stas/fsmt-wmt19-"+p, use_cdn=False) for p in ["en-ru","ru-en","en-de","de-en"]]' @@ -81,6 +85,7 @@ from transformers.configuration_fsmt import FSMTConfig from transformers.modeling_fsmt import FSMTForConditionalGeneration # , get_authorized_missing_keys from transformers.tokenization_fsmt import VOCAB_FILES_NAMES +from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE logging.basicConfig(level=logging.INFO) @@ -250,36 +255,36 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder src_dict = Dictionary.load(src_dict_file) src_vocab = rewrite_dict_keys(src_dict.indices) src_vocab_size = len(src_vocab) - pytorch_vocab_file_src = os.path.join(pytorch_dump_folder_path, f"vocab-{src_lang}.json") - print(f"Generating {pytorch_vocab_file_src}") - with open(pytorch_vocab_file_src, "w", encoding="utf-8") as f: + src_vocab_file = os.path.join(pytorch_dump_folder_path, "vocab-src.json") + print(f"Generating {src_vocab_file}") + with open(src_vocab_file, "w", encoding="utf-8") as f: f.write(json.dumps(src_vocab, ensure_ascii=False, indent=json_indent)) tgt_dict = Dictionary.load(tgt_dict_file) tgt_vocab = rewrite_dict_keys(tgt_dict.indices) tgt_vocab_size = len(tgt_vocab) - pytorch_vocab_file_tgt = os.path.join(pytorch_dump_folder_path, f"vocab-{tgt_lang}.json") - print(f"Generating {pytorch_vocab_file_tgt}") - with open(pytorch_vocab_file_tgt, "w", encoding="utf-8") as f: + tgt_vocab_file = os.path.join(pytorch_dump_folder_path, "vocab-tgt.json") + print(f"Generating {tgt_vocab_file}") + with open(tgt_vocab_file, "w", encoding="utf-8") as f: f.write(json.dumps(tgt_vocab, ensure_ascii=False, indent=json_indent)) - # merge_file (bpecodes) - merge_file = os.path.join(pytorch_dump_folder_path, VOCAB_FILES_NAMES["merges_file"]) - fairseq_merge_file = os.path.join(fsmt_checkpoint_path, "bpecodes") - with open(fairseq_merge_file, encoding="utf-8") as fin: + # merges_file (bpecodes) + merges_file = os.path.join(pytorch_dump_folder_path, VOCAB_FILES_NAMES["merges_file"]) + fsmt_merges_file = os.path.join(fsmt_checkpoint_path, "bpecodes") + with open(fsmt_merges_file, encoding="utf-8") as fin: merges = fin.read() merges = re.sub(r" \d+$", "", merges, 0, re.M) # remove frequency number - print(f"Generating {merge_file}") - with open(merge_file, "w", encoding="utf-8") as fout: + print(f"Generating {merges_file}") + with open(merges_file, "w", encoding="utf-8") as fout: fout.write(merges) - # config - fairseq_config_file = os.path.join(pytorch_dump_folder_path, "config.json") + # model config + fsmt_model_config_file = os.path.join(pytorch_dump_folder_path, "config.json") # XXX: need to compare with the other pre-trained models of this type and # only set here what's different between them - the common settings go into - # config_fsmt - conf = { + # config_fsmt.py: FSMTConfig.__init__ + model_conf = { "architectures": ["FSMTForConditionalGeneration"], "model_type": "fsmt", "activation_dropout": 0.0, @@ -316,9 +321,21 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder "tie_word_embeddings": False, } - print(f"Generating {fairseq_config_file}") - with open(fairseq_config_file, "w", encoding="utf-8") as f: - f.write(json.dumps(conf, ensure_ascii=False, indent=json_indent)) + print(f"Generating {fsmt_model_config_file}") + with open(fsmt_model_config_file, "w", encoding="utf-8") as f: + f.write(json.dumps(model_conf, ensure_ascii=False, indent=json_indent)) + + # tokenizer config + fsmt_tokenizer_config_file = os.path.join(pytorch_dump_folder_path, TOKENIZER_CONFIG_FILE) + + tokenizer_conf = { + "langs": [src_lang, tgt_lang], + "model_max_length": 1024, + } + + print(f"Generating {fsmt_tokenizer_config_file}") + with open(fsmt_tokenizer_config_file, "w", encoding="utf-8") as f: + f.write(json.dumps(tokenizer_conf, ensure_ascii=False, indent=json_indent)) # model model = chkpt["models"][0] diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 263fa784449b..064cdb8b61be 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -37,16 +37,16 @@ PRETRAINED_VOCAB_FILES_MAP = { "src_vocab_file": { - "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-ru.json", - "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-en.json", - "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-de.json", - "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-en.json", + "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-src.json", + "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-src.json", + "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-src.json", + "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-src.json", }, "tgt_vocab_file": { - "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-en.json", - "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-ru.json", - "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-en.json", - "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-de.json", + "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-tgt.json", + "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-tgt.json", + "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-tgt.json", + "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-tgt.json", }, "merges_file": { "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/merges.txt", @@ -78,23 +78,6 @@ }, } -# XXX: temp workaround to be able to run local models with run_eval.py, etc. -LOCALIZE = 1 -if LOCALIZE: - old, new = ("stas/", "/code/huggingface/transformers-fair-wmt/data/") - - def localize(buf): - return buf.replace(old, new) - - for d in [PRETRAINED_INIT_CONFIGURATION, PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES]: - for k, v in d.copy().items(): - d[localize(k)] = v - - for d in [PRETRAINED_VOCAB_FILES_MAP]: - for tk, tv in d.items(): - for k, v in tv.copy().items(): - tv[localize(k)] = v - def get_pairs(word): """ From 6537979e84b14053572e971c3a0705327db308da Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 5 Sep 2020 19:54:53 -0700 Subject: [PATCH 027/109] polish, remove hardcoded config --- ..._original_pytorch_checkpoint_to_pytorch.py | 26 +++++++------------ src/transformers/modeling_fsmt.py | 18 +------------ 2 files changed, 11 insertions(+), 33 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 986de2e41b93..1bb5bc43cbaa 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -83,7 +83,7 @@ from transformers import WEIGHTS_NAME from transformers.configuration_fsmt import FSMTConfig -from transformers.modeling_fsmt import FSMTForConditionalGeneration # , get_authorized_missing_keys +from transformers.modeling_fsmt import FSMTForConditionalGeneration from transformers.tokenization_fsmt import VOCAB_FILES_NAMES from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE @@ -233,8 +233,10 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder models = cls.hub_models() kwargs = {"bpe": "fastbpe", "tokenizer": "moses"} - # note: there is some magic happening here, so can't use torch.load() directly on the model file - # see: load_state_dict() in fairseq_model.py + # note: since the model dump is old, fairseq has upgraded its model some + # time later, and it does a whole lot of rewrites and splits on the saved + # weights, therefore we can't use torch.load() directly on the model file. + # see: upgrade_state_dict(state_dict) in fairseq_model.py chkpt = hub_utils.from_pretrained( fsmt_checkpoint_path, checkpoint_file, data_name_or_path, archive_map=models, **kwargs ) @@ -281,36 +283,31 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder # model config fsmt_model_config_file = os.path.join(pytorch_dump_folder_path, "config.json") - # XXX: need to compare with the other pre-trained models of this type and - # only set here what's different between them - the common settings go into - # config_fsmt.py: FSMTConfig.__init__ model_conf = { "architectures": ["FSMTForConditionalGeneration"], "model_type": "fsmt", - "activation_dropout": 0.0, + "activation_dropout": args["activation_dropout"], "activation_function": "relu", "attention_dropout": args["attention_dropout"], "d_model": args["decoder_embed_dim"], "dropout": args["dropout"], "init_std": 0.02, - "max_position_embeddings": 1024, # XXX: look up? - "num_hidden_layers": 6, # XXX: look up? + "max_position_embeddings": args["max_source_positions"], + "num_hidden_layers": args["encoder_layers"], "src_vocab_size": src_vocab_size, "tgt_vocab_size": tgt_vocab_size, "langs": [src_lang, tgt_lang], "encoder_attention_heads": args["encoder_attention_heads"], "encoder_ffn_dim": args["encoder_ffn_embed_dim"], - "encoder_layerdrop": 0.0, + "encoder_layerdrop": args["encoder_layerdrop"], "encoder_layers": args["encoder_layers"], "decoder_attention_heads": args["decoder_attention_heads"], "decoder_ffn_dim": args["decoder_ffn_embed_dim"], - "decoder_layerdrop": 0.0, + "decoder_layerdrop": args["decoder_layerdrop"], "decoder_layers": args["decoder_layers"], "bos_token_id": 0, "pad_token_id": 1, "eos_token_id": 2, - "id2label": {"0": "LABEL_0", "1": "LABEL_1", "2": "LABEL_2"}, # not needed? - "label2id": {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}, # not needed? "add_bias_logits": False, "add_final_layer_norm": False, "is_encoder_decoder": True, @@ -352,9 +349,6 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder "model.encoder_embed_tokens.weight", "model.decoder_embed_tokens.weight", ] - # XXX: this experiment isn't working - needs more investigation - # let's save a lot of space, by not saving unneeded keys - lots of them! - # ignore_keys.extend(get_authorized_missing_keys()) for k in ignore_keys: model_state_dict.pop(k, None) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index be0ab50941e5..bf4bde655bb8 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -92,9 +92,6 @@ # TODO: # - port model ensemble (fs uses 4 model checkpoints) # - solve beam search discrepancies -# - There are keys in the state_dict that don't need to be saved, see the -# conversion script (get_authorized_missing_keys()), so need to ensure that if -# someone does further work with the weights they don't save those keys """ @@ -790,25 +787,12 @@ def set_output_embeddings(self, value): self.decoder.embed_tokens = value # self.decoder_embed_tokens = value -def get_authorized_missing_keys(): - missing_keys = [r"encoder\.version", r"decoder\.version"] - - # these are 90 dict entries that aren't needed to be saved (they aren't in the original saved weights) - postfices = [fr"{x}.{y}" for x in ["k_proj", "v_proj", "q_proj"] for y in ["weight", "bias"]] - self_attn_keys = [ - fr"model.{x}.layers.{y}.self_attn.{z}" for x in ["encoder", "decoder"] for y in range(1, 6) for z in postfices - ] - encoder_attn_keys = [fr"model.decoder.layers.{y}.encoder_attn.{z}" for y in range(1, 6) for z in postfices] - missing_keys += self_attn_keys + encoder_attn_keys - return missing_keys - - @add_start_docstrings( "The FSMT Model with a language modeling head. Can be used for summarization.", FSMT_START_DOCSTRING ) class FSMTForConditionalGeneration(PretrainedFSMTModel): base_model_prefix = "model" - authorized_missing_keys = get_authorized_missing_keys() + authorized_missing_keys = [r"encoder\.version", r"decoder\.version"] def __init__(self, config: FSMTConfig): super().__init__(config) From 819891139329811906da7a10c33fb94e12235e72 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 5 Sep 2020 20:04:28 -0700 Subject: [PATCH 028/109] add a note that the file is autogenerated to avoid losing changes --- model_cards/stas/fsmt-wmt19-de-en/README.md | 3 +++ model_cards/stas/fsmt-wmt19-en-de/README.md | 3 +++ model_cards/stas/fsmt-wmt19-en-ru/README.md | 3 +++ model_cards/stas/fsmt-wmt19-ru-en/README.md | 3 +++ .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 4 ++++ 5 files changed, 16 insertions(+) diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md index d7cea62318c7..49952e90950e 100644 --- a/model_cards/stas/fsmt-wmt19-de-en/README.md +++ b/model_cards/stas/fsmt-wmt19-de-en/README.md @@ -1,5 +1,8 @@ --- + + + language: de, en thumbnail: tags: diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md index 1aa2405f0bfa..8c823ebcfa1c 100644 --- a/model_cards/stas/fsmt-wmt19-en-de/README.md +++ b/model_cards/stas/fsmt-wmt19-en-de/README.md @@ -1,5 +1,8 @@ --- + + + language: en, de thumbnail: tags: diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md index 79101448633b..b32c83bebf26 100644 --- a/model_cards/stas/fsmt-wmt19-en-ru/README.md +++ b/model_cards/stas/fsmt-wmt19-en-ru/README.md @@ -1,5 +1,8 @@ --- + + + language: en, ru thumbnail: tags: diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md index d4d2e690827a..987d2353f6a8 100644 --- a/model_cards/stas/fsmt-wmt19-ru-en/README.md +++ b/model_cards/stas/fsmt-wmt19-ru-en/README.md @@ -1,5 +1,8 @@ --- + + + language: ru, en thumbnail: tags: diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 1bb5bc43cbaa..2daf7633f626 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -62,6 +62,7 @@ # add/remove files as needed # force cache invalidation, which will now download the new models +# XXX: this doesn't work: PYTHONPATH="src" python -c 'from transformers import AutoModel; [AutoModel.from_pretrained("stas/fsmt-wmt19-"+p, use_cdn=False) for p in ["en-ru","ru-en","en-de","de-en"]]' # happy translations @@ -126,6 +127,9 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): readme = f""" --- + + + language: {src_lang}, {tgt_lang} thumbnail: tags: From 0efea0f1984ffd2989a8981bc8a527d5320dd9e8 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 5 Sep 2020 20:25:20 -0700 Subject: [PATCH 029/109] prep for org change, remove unneeded code --- model_cards/stas/fsmt-wmt19-de-en/README.md | 2 +- model_cards/stas/fsmt-wmt19-en-de/README.md | 2 +- model_cards/stas/fsmt-wmt19-en-ru/README.md | 2 +- model_cards/stas/fsmt-wmt19-ru-en/README.md | 2 +- ..._original_pytorch_checkpoint_to_pytorch.py | 37 +++++-------------- 5 files changed, 14 insertions(+), 31 deletions(-) diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md index 49952e90950e..22cc338c17b5 100644 --- a/model_cards/stas/fsmt-wmt19-de-en/README.md +++ b/model_cards/stas/fsmt-wmt19-de-en/README.md @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "fsmt-wmt19-de-en" +mname = "stas/fsmt-wmt19-de-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md index 8c823ebcfa1c..4d0bac673a65 100644 --- a/model_cards/stas/fsmt-wmt19-en-de/README.md +++ b/model_cards/stas/fsmt-wmt19-en-de/README.md @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "fsmt-wmt19-en-de" +mname = "stas/fsmt-wmt19-en-de" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md index b32c83bebf26..2e389c8088d5 100644 --- a/model_cards/stas/fsmt-wmt19-en-ru/README.md +++ b/model_cards/stas/fsmt-wmt19-en-ru/README.md @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "fsmt-wmt19-en-ru" +mname = "stas/fsmt-wmt19-en-ru" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md index 987d2353f6a8..1a9cacb8eaa6 100644 --- a/model_cards/stas/fsmt-wmt19-ru-en/README.md +++ b/model_cards/stas/fsmt-wmt19-ru-en/README.md @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "fsmt-wmt19-ru-en" +mname = "stas/fsmt-wmt19-ru-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 2daf7633f626..c0b2accb7411 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -91,6 +91,9 @@ logging.basicConfig(level=logging.INFO) +ORG_NAME = "stas" # XXX: will become facebook + + DEBUG = 0 json_indent = 2 if DEBUG else None @@ -154,10 +157,10 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): All four models are available: -* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) -* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) -* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) -* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) +* [fsmt-wmt19-en-ru](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-en-ru) +* [fsmt-wmt19-ru-en](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-ru-en) +* [fsmt-wmt19-en-de](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-en-de) +* [fsmt-wmt19-de-en](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-de-en) ## Intended uses & limitations @@ -166,7 +169,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "fsmt-wmt19-{src_lang}-{tgt_lang}" +mname = "{ORG_NAME}/fsmt-wmt19-{src_lang}-{tgt_lang}" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) @@ -207,7 +210,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="../../src" python run_eval.py {ORG_NAME}/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO @@ -367,28 +370,8 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder print(f"Generating {pytorch_weights_dump_path}") torch.save(model_state_dict, pytorch_weights_dump_path) - # test that it's the same - test_state_dict = torch.load(pytorch_weights_dump_path) - # print(test_state_dict) - - def compare_state_dicts(d1, d2): - models_differ = 0 - for key_item_1, key_item_2 in zip(d1.items(), d2.items()): - if torch.equal(key_item_1[1], key_item_2[1]): - pass - else: - models_differ += 1 - if key_item_1[0] == key_item_2[0]: - print("Mismatch found at", key_item_1[0]) - else: - raise Exception - if models_differ == 0: - print("Models match perfectly! :)") - - compare_state_dicts(model_state_dict, test_state_dict) - # model card - model_card_dir = os.path.join(proj_root, "model_cards", "stas", model_dir) + model_card_dir = os.path.join(proj_root, "model_cards", ORG_NAME, model_dir) print(f"Generating model_card {src_lang}-{tgt_lang}") write_model_card(model_card_dir, src_lang, tgt_lang) From 3ce156cbc4499deeef4e62098c387e5193548eda Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 5 Sep 2020 23:50:36 -0700 Subject: [PATCH 030/109] switch to model4.pt, update scores --- model_cards/stas/fsmt-wmt19-de-en/README.md | 11 +++++---- model_cards/stas/fsmt-wmt19-en-de/README.md | 11 +++++---- model_cards/stas/fsmt-wmt19-en-ru/README.md | 11 +++++---- model_cards/stas/fsmt-wmt19-ru-en/README.md | 11 +++++---- ..._original_pytorch_checkpoint_to_pytorch.py | 23 +++++++++++-------- 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md index 22cc338c17b5..a7e4742821a5 100644 --- a/model_cards/stas/fsmt-wmt19-de-en/README.md +++ b/model_cards/stas/fsmt-wmt19-de-en/README.md @@ -61,16 +61,19 @@ Pretrained weights were left identical to the original model released by fairseq ## Eval results -Fairseq reported score is [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) +pair | fairseq | transformers +-------|---------|---------- +de-en | [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) | 41.18 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -The porting of this model is still in progress, but so far we have the following BLEU score: 39.4278 The score was calculated using this code: ```python git clone https://github.com/huggingface/transformers cd transformers -cd examples/seq2seq export PAIR=de-en export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR @@ -80,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md index 4d0bac673a65..a050ecf6cc75 100644 --- a/model_cards/stas/fsmt-wmt19-en-de/README.md +++ b/model_cards/stas/fsmt-wmt19-en-de/README.md @@ -61,16 +61,19 @@ Pretrained weights were left identical to the original model released by fairseq ## Eval results -Fairseq reported score is [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) +pair | fairseq | transformers +-------|---------|---------- +en-de | [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) | 42.79 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -The porting of this model is still in progress, but so far we have the following BLEU score: 41.0814 The score was calculated using this code: ```python git clone https://github.com/huggingface/transformers cd transformers -cd examples/seq2seq export PAIR=en-de export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR @@ -80,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md index 2e389c8088d5..6f2161e30c14 100644 --- a/model_cards/stas/fsmt-wmt19-en-ru/README.md +++ b/model_cards/stas/fsmt-wmt19-en-ru/README.md @@ -61,16 +61,19 @@ Pretrained weights were left identical to the original model released by fairseq ## Eval results -Fairseq reported score is [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) +pair | fairseq | transformers +-------|---------|---------- +en-ru | [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) | 33.29 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -The porting of this model is still in progress, but so far we have the following BLEU score: 31.2695 The score was calculated using this code: ```python git clone https://github.com/huggingface/transformers cd transformers -cd examples/seq2seq export PAIR=en-ru export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR @@ -80,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md index 1a9cacb8eaa6..bb44ed3b06ac 100644 --- a/model_cards/stas/fsmt-wmt19-ru-en/README.md +++ b/model_cards/stas/fsmt-wmt19-ru-en/README.md @@ -61,16 +61,19 @@ Pretrained weights were left identical to the original model released by fairseq ## Eval results -Fairseq reported score is [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) +pair | fairseq | transformers +-------|---------|---------- +ru-en | [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) | 38.93 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -The porting of this model is still in progress, but so far we have the following BLEU score: 38.8524 The score was calculated using this code: ```python git clone https://github.com/huggingface/transformers cd transformers -cd examples/seq2seq export PAIR=ru-en export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR @@ -80,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index c0b2accb7411..383ca9ab0af2 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -121,10 +121,10 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): # BLUE scores as follows: # "pair": [fairseq, transformers] scores = { - "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "31.2695"], - "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "38.8524"], - "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "39.4278"], - "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "41.0814"], + "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "33.29"], + "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "38.93"], + "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "41.18"], + "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "42.79"], } pair = f"{src_lang}-{tgt_lang}" @@ -191,16 +191,19 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): ## Eval results -Fairseq reported score is { scores[pair][0] } +pair | fairseq | transformers +-------|---------|---------- +{pair} | {scores[pair][0]} | {scores[pair][1]} + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -The porting of this model is still in progress, but so far we have the following BLEU score: { scores[pair][1] } The score was calculated using this code: ```python git clone https://github.com/huggingface/transformers cd transformers -cd examples/seq2seq export PAIR={pair} export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR @@ -210,7 +213,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py {ORG_NAME}/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py {ORG_NAME}/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO @@ -233,12 +236,13 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder # XXX: Need to work out the ensemble as fairseq does, for now using just one chkpt # checkpoint_file = 'model1.pt:model2.pt:model3.pt:model4.pt' - checkpoint_file = "model1.pt" + checkpoint_file = "model4.pt" # proved to give the highest BLEU score for each pair # model_name_or_path = 'transformer.wmt19.ru-en' data_name_or_path = "." cls = fairseq.model_parallel.models.transformer.ModelParallelTransformerModel models = cls.hub_models() kwargs = {"bpe": "fastbpe", "tokenizer": "moses"} + # print(f"using checkpoint {checkpoint_file}") # note: since the model dump is old, fairseq has upgraded its model some # time later, and it does a whole lot of rewrites and splits on the saved @@ -379,6 +383,7 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder print("\nLast step is to upload the files to s3") print(f"cd {data_root}") print(f"transformers-cli upload {model_dir}") + # XXX: this is invalid - waiting on issue to be resolved print("Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") From 9aae16b0f155e409f6b891bc29f5a4ba0e521b02 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 00:11:20 -0700 Subject: [PATCH 031/109] s/python/bash/ --- model_cards/stas/fsmt-wmt19-de-en/README.md | 2 +- model_cards/stas/fsmt-wmt19-en-de/README.md | 2 +- model_cards/stas/fsmt-wmt19-en-ru/README.md | 2 +- model_cards/stas/fsmt-wmt19-ru-en/README.md | 2 +- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md index a7e4742821a5..9da64b82f9b0 100644 --- a/model_cards/stas/fsmt-wmt19-de-en/README.md +++ b/model_cards/stas/fsmt-wmt19-de-en/README.md @@ -71,7 +71,7 @@ de-en | [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) | 41.18 The score was calculated using this code: -```python +```bash git clone https://github.com/huggingface/transformers cd transformers export PAIR=de-en diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md index a050ecf6cc75..211bbb90343e 100644 --- a/model_cards/stas/fsmt-wmt19-en-de/README.md +++ b/model_cards/stas/fsmt-wmt19-en-de/README.md @@ -71,7 +71,7 @@ en-de | [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) | 42.79 The score was calculated using this code: -```python +```bash git clone https://github.com/huggingface/transformers cd transformers export PAIR=en-de diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md index 6f2161e30c14..a73fed2130df 100644 --- a/model_cards/stas/fsmt-wmt19-en-ru/README.md +++ b/model_cards/stas/fsmt-wmt19-en-ru/README.md @@ -71,7 +71,7 @@ en-ru | [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) | 33.29 The score was calculated using this code: -```python +```bash git clone https://github.com/huggingface/transformers cd transformers export PAIR=en-ru diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md index bb44ed3b06ac..bd2678dd3cbe 100644 --- a/model_cards/stas/fsmt-wmt19-ru-en/README.md +++ b/model_cards/stas/fsmt-wmt19-ru-en/README.md @@ -71,7 +71,7 @@ ru-en | [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) | 38.93 The score was calculated using this code: -```python +```bash git clone https://github.com/huggingface/transformers cd transformers export PAIR=ru-en diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 383ca9ab0af2..631b30679cae 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -201,7 +201,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): The score was calculated using this code: -```python +```bash git clone https://github.com/huggingface/transformers cd transformers export PAIR={pair} From bdc88f02871ecb98bc7f48a8bc7370c1a23fcaba Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 20:35:28 -0700 Subject: [PATCH 032/109] missing init (but doesn't impact the finetuned model) --- src/transformers/modeling_fsmt.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index bf4bde655bb8..07aebfd4765d 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -467,16 +467,12 @@ def __init__(self, config: FSMTConfig, embed_tokens: nn.Embedding): self.layernorm_embedding = LayerNorm(config.d_model) if config.normalize_embedding else nn.Identity() self.layer_norm = LayerNorm(config.d_model) if config.add_final_layer_norm else None - # XXX: also add to init_weights self.output_projection = nn.Linear( self.embed_tokens.weight.shape[1], self.embed_tokens.weight.shape[0], bias=False, ) - # self.output_projection.weight = self.embed_tokens.weight - # nn.init.normal_( - # self.output_projection.weight, mean=0, std=self.output_embed_dim ** -0.5 - # ) + self.output_projection.weight = self.embed_tokens.weight def forward( self, From 38cc9c196772ce50f1eb27dbd057620beb2da35c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 20:36:03 -0700 Subject: [PATCH 033/109] cleanup --- src/transformers/modeling_fsmt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 07aebfd4765d..e6f6fea00fb5 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -776,8 +776,6 @@ def set_input_embeddings(self, value): def get_output_embeddings(self): return self.decoder.embed_tokens - # XXX: it was, but probably not needed here - # return _make_linear_from_emb(self.decoder.embed_tokens) # make it on the fly def set_output_embeddings(self, value): self.decoder.embed_tokens = value # self.decoder_embed_tokens = value From 226dad15ca6a9ef4e26178526e878e8fc5c85874 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 21:16:27 -0700 Subject: [PATCH 034/109] major refactor (reuse-bart) --- src/transformers/modeling_fsmt.py | 158 ++++-------------------------- tests/test_modeling_fsmt.py | 34 +------ 2 files changed, 23 insertions(+), 169 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index e6f6fea00fb5..5ccd41cda02c 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -46,7 +46,14 @@ add_start_docstrings_to_callable, replace_return_docstrings, ) -from .modeling_bart import DecoderLayer, EncoderLayer +from .modeling_bart import ( + DecoderLayer, + EncoderLayer, + LayerNorm, + _prepare_bart_decoder_inputs, + _reorder_buffer, + invert_mask, +) from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel @@ -97,43 +104,36 @@ Here is how to compare BLEU scores against fairseq implementation: -# Note: to match fairseq params you need to set num_beams=50 in -# `configuration_fsmt.py` and lower BS as it'll need more GPU memory - -cd examples/seq2seq - # en-ru export PAIR=en-ru export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 36.4 http://matrix.statmt.org/matrix/output/1914?score_id=37605) - - # ru-en export PAIR=ru-en export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation - -# (fairseq BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937) +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +# (fairseq BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937) # de-en @@ -142,11 +142,12 @@ export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 +export NUM_BEAMS=50 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 42.3 http://matrix.statmt.org/matrix/output/1902?run_id=6750) @@ -162,7 +163,7 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="../../src" python run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 43.1 http://matrix.statmt.org/matrix/output/1909?run_id=6862) @@ -171,8 +172,7 @@ FSMT_START_DOCSTRING = r""" - This model is a PyTorch `torch.nn.Module `_ sub-class. Use it as a regular PyTorch Module and - refer to the PyTorch documentation for all matters related to general usage and behavior. + This model is a PyTorch `torch.nn.Module `_ sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matters related to general usage and behavior. Parameters: config (:class:`~transformers.FSMTConfig`): Model configuration class with all the parameters of the model. @@ -238,33 +238,6 @@ """ -def invert_mask(attention_mask): - """Turns 1->0, 0->1, False->True, True-> False""" - assert attention_mask.dim() == 2 - return attention_mask.eq(0) - - -def _prepare_fsmt_decoder_inputs( - config, input_ids, decoder_input_ids=None, decoder_padding_mask=None, causal_mask_dtype=torch.float32 -): - """Prepare masks that ignore padding tokens in the decoder and a causal mask for the decoder if - none are provided. This mimics the default behavior in fairseq. To override it pass in masks. - Note: this is not called during generation - """ - pad_token_id = config.pad_token_id - if decoder_input_ids is None: - decoder_input_ids = shift_tokens_right(input_ids, pad_token_id) - bsz, tgt_len = decoder_input_ids.size() - if decoder_padding_mask is None: - decoder_padding_mask = make_padding_mask(decoder_input_ids, pad_token_id) - else: - decoder_padding_mask = invert_mask(decoder_padding_mask) - causal_mask = torch.triu(fill_with_neg_inf(torch.zeros(tgt_len, tgt_len)), 1).to( - dtype=causal_mask_dtype, device=decoder_input_ids.device - ) - return decoder_input_ids, decoder_padding_mask, causal_mask - - class PretrainedFSMTModel(PreTrainedModel): config_class = FSMTConfig base_model_prefix = "model" @@ -293,36 +266,6 @@ def dummy_inputs(self): return dummy_inputs -def _make_linear_from_emb(emb): - vocab_size, emb_size = emb.weight.shape - lin_layer = nn.Linear(vocab_size, emb_size, bias=False) - lin_layer.weight.data = emb.weight.data - return lin_layer - - -# Helper Functions, mostly for making masks -def _check_shapes(shape_1, shape2): - if shape_1 != shape2: - raise AssertionError("shape mismatch: {} != {}".format(shape_1, shape2)) - - -def shift_tokens_right(input_ids, pad_token_id): - """Shift input ids one token to the right, and wrap the last non pad token (usually ).""" - prev_output_tokens = input_ids.clone() - index_of_eos = (input_ids.ne(pad_token_id).sum(dim=1) - 1).unsqueeze(-1) - prev_output_tokens[:, 0] = input_ids.gather(1, index_of_eos).squeeze() - prev_output_tokens[:, 1:] = input_ids[:, :-1] - return prev_output_tokens - - -def make_padding_mask(input_ids, padding_idx=1): - """True for pad tokens""" - padding_mask = input_ids.eq(padding_idx) - if not padding_mask.any(): - padding_mask = None - return padding_mask - - # Helper Modules @@ -592,70 +535,11 @@ def forward( ) -def _reorder_buffer(attn_cache, new_order): - for k, input_buffer_k in attn_cache.items(): - if input_buffer_k is not None: - attn_cache[k] = input_buffer_k.index_select(0, new_order) - return attn_cache - - -# XXX: remove this and its references -class LearnedPositionalEmbedding(nn.Embedding): - """ - This module learns positional embeddings up to a fixed maximum size. - Padding ids are ignored by either offsetting based on padding_idx - or by setting padding_idx to None and ensuring that the appropriate - position ids are passed to the forward function. - """ - - def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, offset): - # FSMT is set up so that if padding_idx is specified then offset the embedding ids by 2 - # and adjust num_embeddings appropriately. Other models dont have this hack - self.offset = offset - assert padding_idx is not None - num_embeddings += offset - super().__init__(num_embeddings, embedding_dim, padding_idx=padding_idx) - - def forward(self, input_ids, use_cache=False): - """Input is expected to be of size [bsz x seqlen].""" - bsz, seq_len = input_ids.shape[:2] - if use_cache: - positions = input_ids.data.new(1, 1).fill_(seq_len - 1) # called before slicing - else: - # starts at 0, ends at 1-seq_len - positions = torch.arange(seq_len, dtype=torch.long, device=self.weight.device) - return super().forward(positions + self.offset) - - -def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True): - if torch.cuda.is_available(): - try: - from apex.normalization import FusedLayerNorm - - return FusedLayerNorm(normalized_shape, eps, elementwise_affine) - except ImportError: - pass - return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine) - - -def fill_with_neg_inf(t): - """FP16-compatible function that fills a input_ids with -inf.""" - return t.float().fill_(float("-inf")).type_as(t) - - # Public API def _get_shape(t): return getattr(t, "shape", None) -# def output_projection(self): -# return nn.Linear( -# self.embed_tokens.weight.shape[1], -# self.embed_tokens.weight.shape[0], -# bias=False, -# ) - - @add_start_docstrings( "The bare FSMT Model outputting raw hidden-states without any specific head on top.", FSMT_START_DOCSTRING, @@ -713,7 +597,7 @@ def forward( # make masks if user doesn't supply if not use_cache: - decoder_input_ids, decoder_padding_mask, causal_mask = _prepare_fsmt_decoder_inputs( + decoder_input_ids, decoder_padding_mask, causal_mask = _prepare_bart_decoder_inputs( self.config, input_ids, decoder_input_ids=decoder_input_ids, @@ -772,13 +656,13 @@ def get_input_embeddings(self): return self.encoder.embed_tokens def set_input_embeddings(self, value): - self.encoder.embed_tokens = value # self.encoder_embed_tokens = value + self.encoder.embed_tokens = value def get_output_embeddings(self): return self.decoder.embed_tokens def set_output_embeddings(self, value): - self.decoder.embed_tokens = value # self.decoder_embed_tokens = value + self.decoder.embed_tokens = value @add_start_docstrings( @@ -935,8 +819,6 @@ def get_encoder(self): def get_output_embeddings(self): return self.model.decoder.embed_tokens - # XXX: it was, but probably is not needed here - # return _make_linear_from_emb(self.decoder.embed_tokens) # make it on the fly def make_positions(tensor, padding_idx: int): diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 1304a112957d..66cd456907f9 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -30,13 +30,8 @@ import torch from transformers import FSMTConfig, FSMTForConditionalGeneration, FSMTModel, FSMTTokenizer - from transformers.modeling_fsmt import ( - SinusoidalPositionalEmbedding, - _prepare_fsmt_decoder_inputs, - invert_mask, - shift_tokens_right, - ) -PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.""" + from transformers.modeling_bart import _prepare_bart_decoder_inputs, invert_mask + from transformers.modeling_fsmt import SinusoidalPositionalEmbedding @require_torch @@ -164,7 +159,7 @@ def test_advanced_inputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() config.use_cache = False inputs_dict["input_ids"][:, -2:] = config.pad_token_id - decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_fsmt_decoder_inputs( + decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_bart_decoder_inputs( config, inputs_dict["input_ids"] ) model = FSMTModel(config).to(torch_device).eval() @@ -287,15 +282,6 @@ def test_generate_beam_search(self): self.assertEqual(new_input_ids.shape, (input_ids.shape[0], max_length)) # TODO(SS): uneven length batches, empty inputs - def test_shift_tokens_right(self): - input_ids = torch.Tensor([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]).long() - shifted = shift_tokens_right(input_ids, 1) - n_pad_before = input_ids.eq(1).float().sum() - n_pad_after = shifted.eq(1).float().sum() - self.assertEqual(shifted.shape, input_ids.shape) - self.assertEqual(n_pad_after, n_pad_before - 1) - self.assertTrue(torch.eq(shifted[:, 0], 2).all()) - def test_generate_fp16(self): config, input_ids, batch_size = self._get_config_and_data() attention_mask = input_ids.ne(1).to(torch_device) @@ -310,20 +296,6 @@ def test_dummy_inputs(self): model = FSMTForConditionalGeneration(config).eval().to(torch_device) model(**model.dummy_inputs) - def test_prepare_fsmt_decoder_inputs(self): - config, *_ = self._get_config_and_data() - input_ids = _long_tensor(([4, 4, 2])) - decoder_input_ids = _long_tensor([[26388, 2, config.pad_token_id]]) - ignore = float("-inf") - decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_fsmt_decoder_inputs( - config, input_ids, decoder_input_ids - ) - expected_causal_mask = torch.tensor( - [[0, ignore, ignore], [0, 0, ignore], [0, 0, 0]] # never attend to the final token, because its pad - ).to(input_ids.device) - self.assertEqual(decoder_attn_mask.size(), decoder_input_ids.size()) - self.assertTrue(torch.eq(expected_causal_mask, causal_mask).all()) - def test_resize_tokens_embeddings_more(self): config, input_ids, _ = self._get_config_and_data() From 25f53929b4732bb48547e48195a6b911039db7ba Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 21:19:39 -0700 Subject: [PATCH 035/109] new model, new expected weights --- tests/test_modeling_fsmt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 66cd456907f9..a402ed7e54a6 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -365,8 +365,11 @@ def test_inference_no_head(self): output = model(**inputs_dict)[0] expected_shape = torch.Size((1, model.config.decoder_attention_heads, model.config.tgt_vocab_size)) self.assertEqual(output.shape, expected_shape) + print(output[:, :3, :3]) + # expected numbers were generated when using just fairseq's model4.pt + # may have to adjust if switched to a different checkpoint expected_slice = torch.tensor( - [[-3.1850, -3.1849, 2.9694], [-4.0242, -4.0242, 0.2494], [-3.4442, -3.4443, 0.3315]], device=torch_device + [[-3.3069, -3.3069, 2.5253], [-4.2303, -4.2301, 0.8354], [-3.2488, -3.2487, 1.7397]], device=torch_device ) print(output[:, :3, :3]) self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=TOLERANCE)) From 486e06756c580f43a64966d95dc706e469fa4aa3 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 21:25:18 -0700 Subject: [PATCH 036/109] cleanup --- src/transformers/modeling_fsmt.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 5ccd41cda02c..8c33d6b1531b 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -391,7 +391,6 @@ def __init__(self, config: FSMTConfig, embed_tokens: nn.Embedding): embed_dim = embed_tokens.embedding_dim if config.static_position_embeddings: num_embeddings = config.tgt_vocab_size - # XXX: self.padding_idx and config.pad_token_id are the same? self.embed_positions = SinusoidalPositionalEmbedding( embed_dim, self.padding_idx, @@ -522,8 +521,6 @@ def forward( x = x.transpose(0, 1) encoder_hidden_states = encoder_hidden_states.transpose(0, 1) - # new XXX: not invoked? self.project_out_dim==None in fairseq - # but it then gets invoked later in x.output_layer() transformer.py:676 x = self.output_projection(x) next_cache = next_decoder_cache if use_cache else None From d60a1833210048397f82ed7a652614560d0913ad Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 21:35:09 -0700 Subject: [PATCH 037/109] cleanup --- src/transformers/modeling_fsmt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 8c33d6b1531b..393e00412284 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -50,6 +50,7 @@ DecoderLayer, EncoderLayer, LayerNorm, + LearnedPositionalEmbedding, _prepare_bart_decoder_inputs, _reorder_buffer, invert_mask, From a49409be550a2e35e2da7a67d1cb7ded2ce57134 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sun, 6 Sep 2020 21:36:14 -0700 Subject: [PATCH 038/109] full link --- src/transformers/modeling_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 393e00412284..7ce7ea22d189 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -25,7 +25,7 @@ # # Paper: Facebook FAIR's WMT19 News Translation Task Submission https://arxiv.org/abs/1907.06616 # -"""PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/""" +"""PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/tree/master/examples/wmt19""" import logging import math From 86ff5341484add785b5a98d8a7f09d9442663cf4 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 09:41:30 -0700 Subject: [PATCH 039/109] fix model type --- src/transformers/configuration_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 442e4c2b67f2..934a990672a2 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -118,7 +118,7 @@ class FSMTConfig(PretrainedConfig): - src/tgt vocabs aren't shared - token embeddings aren't shared """ - model_type = "fairseq" + model_type = "fsmt" # update the defaults from config file def __init__( From 46ac8f7f4fc47203bcbace183b263b44b3d530e6 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 09:45:34 -0700 Subject: [PATCH 040/109] merge porting notes --- src/transformers/configuration_fsmt.py | 14 -------------- src/transformers/modeling_fsmt.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 934a990672a2..5b33442029ba 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -95,20 +95,6 @@ Whether to tie input and output embeddings. """ -# Porting notes: -# this one is modeled after BartConfig -# -# Differences with BART: -# - src/tgt vocabs aren't shared -# - token embeddings aren't shared -# - needs a language pair -# - scale_embedding are True -# - normalize_embedding are False -# - static_position_embeddings are True -# -# some unused args were removed too - - @add_start_docstrings_to_callable(FSMT_CONFIG_ARGS_DOC) class FSMTConfig(PretrainedConfig): r""" diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 7ce7ea22d189..44b661ebfb46 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -97,6 +97,20 @@ # Other changes: # - doesn't support use_cache as Bart's version does # +# +# FSMTConfig changes with BartConfig +# +# Differences with BART: +# - src/tgt vocabs aren't shared +# - token embeddings aren't shared +# - needs a language pair +# - scale_embedding are True +# - normalize_embedding are False +# - static_position_embeddings are True +# +# some unused args were removed too +# +# # TODO: # - port model ensemble (fs uses 4 model checkpoints) # - solve beam search discrepancies From 9e46af1496b1b27f4e65e8cfd499e9e83fe6ad0f Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 10:01:10 -0700 Subject: [PATCH 041/109] style --- src/transformers/configuration_fsmt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 5b33442029ba..50fbffa0b284 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -95,6 +95,7 @@ Whether to tie input and output embeddings. """ + @add_start_docstrings_to_callable(FSMT_CONFIG_ARGS_DOC) class FSMTConfig(PretrainedConfig): r""" From 64a15ef575a7c3f84d9ae6bcfc59517f13cb8ae5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 10:31:07 -0700 Subject: [PATCH 042/109] cleanup --- src/transformers/generation_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/generation_utils.py b/src/transformers/generation_utils.py index 5587c0b7479c..053b13a34305 100644 --- a/src/transformers/generation_utils.py +++ b/src/transformers/generation_utils.py @@ -389,6 +389,7 @@ def generate( raise ValueError( "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation" ) + assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self) assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder) From 5300abd20ae604494c438e68e1967af66aee7edd Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 11:23:21 -0700 Subject: [PATCH 043/109] have to create a DecoderConfig object to handle vocab_size properly --- src/transformers/configuration_fsmt.py | 35 ++++++++++++++++++-------- src/transformers/generation_utils.py | 6 ++--- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 50fbffa0b284..69cd957362d7 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -15,6 +15,7 @@ """ FSMT configuration """ +import copy import logging from .configuration_utils import PretrainedConfig @@ -96,15 +97,19 @@ """ -@add_start_docstrings_to_callable(FSMT_CONFIG_ARGS_DOC) -class FSMTConfig(PretrainedConfig): - r""" - Configuration class for FSMT. Parameters are renamed from the fairseq implementation +class DecoderConfig(PretrainedConfig): + r""" Configuration class for FSMT's decoder specific things """ + model_type = "fsmt_decoder" + + def __init__(self, vocab_size=0, bos_token_id=0): + super().__init__() + self.vocab_size = vocab_size + self.bos_token_id = bos_token_id - Differences with BART: - - src/tgt vocabs aren't shared - token embeddings aren't shared - """ +@add_start_docstrings_to_callable(FSMT_CONFIG_ARGS_DOC) +class FSMTConfig(PretrainedConfig): + r""" Configuration class for FSMT.""" model_type = "fsmt" # update the defaults from config file @@ -182,9 +187,7 @@ def __init__( self.init_std = init_std # Normal(0, this parameter) self.activation_function = activation_function - # XXX: needed in generation_utils.py:382 - # alternatively need to setup config.decoder object - self.decoder_start_token_id = eos_token_id + self.decoder = DecoderConfig(vocab_size=tgt_vocab_size, bos_token_id=eos_token_id) # Params introduced for Mbart self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True @@ -211,3 +214,15 @@ def num_attention_heads(self) -> int: @property def hidden_size(self) -> int: return self.d_model + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default `to_dict()` from `PretrainedConfig`. + + Returns: + :obj:`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output["decoder"] = self.decoder.to_dict() + output["model_type"] = self.__class__.model_type + return output diff --git a/src/transformers/generation_utils.py b/src/transformers/generation_utils.py index 053b13a34305..4d2af8506b24 100644 --- a/src/transformers/generation_utils.py +++ b/src/transformers/generation_utils.py @@ -381,10 +381,10 @@ def generate( if self.config.is_encoder_decoder: if decoder_start_token_id is None: # see if BOS token can be used for decoder_start_token_id - if bos_token_id is not None: - decoder_start_token_id = bos_token_id - elif hasattr(self.config, "decoder") and hasattr(self.config.decoder, "bos_token_id"): + if hasattr(self.config, "decoder") and hasattr(self.config.decoder, "bos_token_id"): decoder_start_token_id = self.config.decoder.bos_token_id + elif bos_token_id is not None: + decoder_start_token_id = bos_token_id else: raise ValueError( "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation" From ef187b33676fbec6d74a05ef3cc99fdfbfcb2fe6 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 11:31:52 -0700 Subject: [PATCH 044/109] doc fix --- src/transformers/configuration_fsmt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 69cd957362d7..23b1124b4c2e 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -98,7 +98,9 @@ class DecoderConfig(PretrainedConfig): - r""" Configuration class for FSMT's decoder specific things """ + r""" + Configuration class for FSMT's decoder specific things. + """ model_type = "fsmt_decoder" def __init__(self, vocab_size=0, bos_token_id=0): @@ -109,7 +111,9 @@ def __init__(self, vocab_size=0, bos_token_id=0): @add_start_docstrings_to_callable(FSMT_CONFIG_ARGS_DOC) class FSMTConfig(PretrainedConfig): - r""" Configuration class for FSMT.""" + r""" + Configuration class for FSMT. + """ model_type = "fsmt" # update the defaults from config file From 3f9c4493b2cba07f914da38fe3dafa103aaf64f9 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 11:33:17 -0700 Subject: [PATCH 045/109] add note (not a public class) --- src/transformers/configuration_fsmt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 23b1124b4c2e..93118968ec4a 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -100,6 +100,7 @@ class DecoderConfig(PretrainedConfig): r""" Configuration class for FSMT's decoder specific things. + note: this is a private helper class """ model_type = "fsmt_decoder" From 2fb447fdee8943047d0d0a955851b989084659df Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 16:22:56 -0700 Subject: [PATCH 046/109] parametrize --- tests/test_modeling_fsmt.py | 46 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index a402ed7e54a6..e83d1bc08c71 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -18,6 +18,7 @@ import timeout_decorator # noqa +from parameterized import parameterized from transformers import is_torch_available from transformers.file_utils import cached_property from transformers.testing_utils import require_torch, slow, torch_device @@ -374,39 +375,38 @@ def test_inference_no_head(self): print(output[:, :3, :3]) self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=TOLERANCE)) - # XXX: the rest of the tests were moved to tests/test_tokenization_bart.py - port from there into tokenization tests - + @parameterized.expand( + [ + ["en-ru"], + ["ru-en"], + ["en-de"], + ["de-en"], + ] + ) @slow - def test_translation(self): + def test_translation(self, pair): text = { "en": "Machine learning is great, isn't it?", "ru": "Машинное обучение - это здорово, не так ли?", "de": "Maschinelles Lernen ist großartig, oder?", } - pairs = [ - ["en", "ru"], - ["ru", "en"], - ["en", "de"], - ["de", "en"], - ] - - for src, tgt in pairs: - print(f"Testing {src} -> {tgt}") - mname = f"stas/fsmt-wmt19-{src}-{tgt}" + src, tgt = pair.split("-") + print(f"Testing {src} -> {tgt}") + mname = f"stas/fsmt-wmt19-{pair}" - src_sentence = text[src] - tgt_sentence = text[tgt] + src_sentence = text[src] + tgt_sentence = text[tgt] - tokenizer = FSMTTokenizer.from_pretrained(mname) - model = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) - if torch_device == "cuda": - model.half() + tokenizer = FSMTTokenizer.from_pretrained(mname) + model = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) + if torch_device == "cuda": + model.half() - input_ids = tokenizer.encode(src_sentence, return_tensors="pt") - outputs = model.generate(input_ids) - decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) - assert decoded == tgt_sentence, f"\n\ngot: {decoded}\nexp: {tgt_sentence}\n" + input_ids = tokenizer.encode(src_sentence, return_tensors="pt") + outputs = model.generate(input_ids) + decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) + assert decoded == tgt_sentence, f"\n\ngot: {decoded}\nexp: {tgt_sentence}\n" @require_torch From fea70dd298bfe90cff4504d9c60dff9cb18a1cff Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 17:55:17 -0700 Subject: [PATCH 047/109] - add bleu scores integration tests --- tests/test_modeling_fsmt.py | 153 ++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index e83d1bc08c71..9e1f48225341 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -34,6 +34,15 @@ from transformers.modeling_bart import _prepare_bart_decoder_inputs, invert_mask from transformers.modeling_fsmt import SinusoidalPositionalEmbedding +import os +import sys + + +# XXX: make calculate_bleu accessible to integration tests? +examples_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "examples")) +sys.path.insert(0, examples_dir) +from seq2seq.utils import calculate_bleu # noqa + @require_torch class ModelTester: @@ -408,6 +417,150 @@ def test_translation(self, pair): decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) assert decoded == tgt_sentence, f"\n\ngot: {decoded}\nexp: {tgt_sentence}\n" + # BLEU eval data was generated using the following code: + # + # #!/bin/bash + # + # export OBJS=8 + # + # pairs=(ru-en en-ru en-de de-en) + # printf "data = {\n" + # for pair in "${pairs[@]}" + # do + # export PAIR=$pair + # printf " \"$PAIR\": {\n" + # printf " \"src\": [\n" + # sacrebleu -t wmt19 -l $PAIR --echo src | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' + # printf " ],\n" + # printf " \"tgt\": [\n" + # sacrebleu -t wmt19 -l $PAIR --echo ref | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' + # printf " ],\n" + # printf " },\n" + # done + # printf "}\n" + + bleu_data = { + "ru-en": { + "src": [ + """Названо число готовящихся к отправке в Донбасс новобранцев из Украины""", + """Официальный представитель Народной милиции самопровозглашенной Луганской Народной Республики (ЛНР) Андрей Марочко заявил, что зимой 2018-2019 года Украина направит в Донбасс не менее 3 тыс. новобранцев.""", + """По его словам, таким образом Киев планирует "хоть как-то доукомплектовать подразделения".""", + """\"Нежелание граждан Украины проходить службу в рядах ВС Украины, массовые увольнения привели к низкой укомплектованности подразделений", - рассказал Марочко, которого цитирует "РИА Новости".""", + """Он также не исключил, что реальные цифры призванных в армию украинцев могут быть увеличены в случае необходимости.""", + """В 2014-2017 годах Киев начал так называемую антитеррористическую операцию (АТО), которую позже сменили на операцию объединенных сил (ООС).""", + """Предполагалось, что эта мера приведет к усилению роли украинских силовиков в урегулировании ситуации.""", + """В конце августа 2018 года ситуация в Донбассе обострилась из-за убийства главы ДНР Александра Захарченко.""", + ], + "tgt": [ + """The number of new Ukrainian recruits ready to go to Donbass has become public""", + """Official representative of the peoples’ militia of the self-proclaimed Lugansk People’s Republic Andrey Marochko claimed that Ukrainian will send at least 3 thousand new recruits to Donbass in winter 2018-2019.""", + """This is how Kyiv tries “at least somehow to staff the units,” he said.""", + """“The unwillingness of Ukrainian citizens to serve in the Ukraine’s military forces, mass resignments lead to low understaffing,” said Marochko cited by RIA Novosti.""", + """Also, he doesn’t exclude that the real numbers of conscripts in the Ukrainian army can be raised is necessary.""", + """In 2014-2017, Kyiv started so-called antiterrorist operation, that ws later changed to the united forces operation.""", + """This measure was supposed to strengthen the role of the Ukrainian military in settling the situation.""", + """In the late August 2018, the situation in Donbass escalated as the DNR head Aleksandr Zakharchenko was killed.""", + ], + }, + "en-ru": { + "src": [ + """Welsh AMs worried about 'looking like muppets'""", + """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", + """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", + """AMs across the political spectrum are worried it could invite ridicule.""", + """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", + """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", + """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", + """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", + ], + "tgt": [ + """Члены Национальной ассамблеи Уэльса обеспокоены, что "выглядят как куклы\"""", + """Некоторые члены Национальной ассамблеи Уэльса в ужасе от предложения о том, что их наименование должно измениться на MPW (члены Парламента Уэльса).""", + """Этот вопрос был поднят в связи с планами по переименованию ассамблеи в Парламент Уэльса.""", + """Члены Национальной ассамблеи Уэльса всего политического спектра обеспокоены, что это может породить насмешки.""", + """Один из лейбористских членов Национальной ассамблеи Уэльса сказал, что его партия обеспокоена тем, что "это рифмуется с Twp и Pwp".""", + """Для читателей за предлами Уэльса: по-валлийски twp означает "глупый", а pwp означает "какашка".""", + """Член Национальной ассамблеи от Плайд сказал, что эта партия в целом "не счастлива" и предложил альтернативы.""", + """Представитель Консервативной партии Уэльса сказал, что его партия "открыта" к переименованию, но отметил, что между WMP и Muppet небольшая разница в произношении.""", + ], + }, + "en-de": { + "src": [ + """Welsh AMs worried about 'looking like muppets'""", + """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", + """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", + """AMs across the political spectrum are worried it could invite ridicule.""", + """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", + """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", + """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", + """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", + ], + "tgt": [ + """Walisische Ageordnete sorgen sich "wie Dödel auszusehen\"""", + """Es herrscht Bestürzung unter einigen Mitgliedern der Versammlung über einen Vorschlag, der ihren Titel zu MWPs (Mitglied der walisischen Parlament) ändern soll.""", + """Der Grund dafür waren Pläne, den Namen der Nationalversammlung in Walisisches Parlament zu ändern.""", + """Mitglieder aller Parteien der Nationalversammlung haben Bedenken, dass sie sich dadurch Spott aussetzen könnten.""", + """Ein Labour-Abgeordneter sagte, dass seine Gruppe "sich mit Twp und Pwp reimt".""", + """Hinweis für den Leser: „twp“ im Walisischen bedeutet „bescheuert“ und „pwp“ bedeutet „Kacke“.""", + """Ein Versammlungsmitglied von Plaid Cymru sagte, die Gruppe als Ganzes sei "nicht glücklich" und hat Alternativen vorgeschlagen.""", + """Ein walisischer Konservativer sagte, seine Gruppe wäre „offen“ für eine Namensänderung, wies aber darauf hin, dass es von „MWP“ (Mitglied des Walisischen Parlaments) nur ein kurzer verbaler Sprung zu „Muppet“ ist.""", + ], + }, + "de-en": { + "src": [ + """Schöne Münchnerin 2018: Schöne Münchnerin 2018 in Hvar: Neun Dates""", + """Von az, aktualisiert am 04.05.2018 um 11:11""", + """Ja, sie will...""", + """\"Schöne Münchnerin" 2018 werden!""", + """Am Nachmittag wartet erneut eine Überraschung auf unsere Kandidatinnen: sie werden das romantische Candlelight-Shooting vor der MY SOLARIS nicht alleine bestreiten, sondern an der Seite von Male-Model Fabian!""", + """Hvar - Flirten, kokettieren, verführen - keine einfachen Aufgaben für unsere Mädchen.""", + """Insbesondere dann, wenn in Deutschland ein Freund wartet.""", + """Dennoch liefern die neun "Schöne Münchnerin"-Kandidatinnen beim Shooting mit People-Fotograf Tuan ab und trotzen Wind, Gischt und Regen wie echte Profis.""", + ], + "tgt": [ + """The Beauty of Munich 2018: the Beauty of Munich 2018 in Hvar: Nine dates""", + """From A-Z, updated on 04/05/2018 at 11:11""", + """Yes, she wants to...""", + """to become "The Beauty of Munich" in 2018!""", + """In the afternoon there is another surprise waiting for our contestants: they will be competing for the romantic candlelight photo shoot at MY SOLARIS not alone, but together with a male-model Fabian!""", + """Hvar with its flirting, coquetting, and seduction is not an easy task for our girls.""", + """Especially when there is a boyfriend waiting in Germany.""", + """Despite dealing with wind, sprays and rain, the nine contestants of "The Beauty of Munich" behaved like real professionals at the photo shoot with People-photographer Tuan.""", + ], + }, + } + + @parameterized.expand( + [ + ["en-ru", 28.21], + ["ru-en", 23.49], + ["en-de", 22.11], + ["de-en", 29.31], + ] + ) + @slow + def test_bleu_scores(self, pair, min_bleu_score): + # note: this test is not testing the best performance since it only evals a small batch + # but it should be enough to detect a regression in the output quality + mname = f"stas/fsmt-wmt19-{pair}" + tokenizer = FSMTTokenizer.from_pretrained(mname) + model = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) + + src_sentences = self.bleu_data[pair]["src"] + tgt_sentences = self.bleu_data[pair]["tgt"] + + batch = tokenizer(src_sentences, return_tensors="pt", truncation=True, padding="longest").to(torch_device) + outputs = model.generate( + input_ids=batch.input_ids, + num_beams=8, + ) + decoded_sentences = tokenizer.batch_decode( + outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) + scores = calculate_bleu(decoded_sentences, tgt_sentences) + print(scores) + self.assertGreaterEqual(scores["bleu"], min_bleu_score) + @require_torch class TestSinusoidalPositionalEmbeddings(unittest.TestCase): From b934c0729e60b5b583bf4707b25d7a4947b535a8 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 18:17:58 -0700 Subject: [PATCH 048/109] skip test if sacrebleu is not installed --- tests/test_modeling_fsmt.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 9e1f48225341..697031a205fc 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -15,6 +15,7 @@ import tempfile import unittest +from unittest import skipIf import timeout_decorator # noqa @@ -27,6 +28,14 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor +try: + from sacrebleu import corpus_bleu + + sacrebleu_is_missing = False +except Exception: + sacrebleu_is_missing = True + + if is_torch_available(): import torch @@ -34,14 +43,10 @@ from transformers.modeling_bart import _prepare_bart_decoder_inputs, invert_mask from transformers.modeling_fsmt import SinusoidalPositionalEmbedding -import os -import sys - -# XXX: make calculate_bleu accessible to integration tests? -examples_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "examples")) -sys.path.insert(0, examples_dir) -from seq2seq.utils import calculate_bleu # noqa +def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: + """Uses sacrebleu's corpus_bleu implementation.""" + return {"bleu": round(corpus_bleu(output_lns, [refs_lns], **kwargs).score, 4)} @require_torch @@ -539,6 +544,7 @@ def test_translation(self, pair): ] ) @slow + @skipIf(sacrebleu_is_missing, "pip install sacrebleu") def test_bleu_scores(self, pair, min_bleu_score): # note: this test is not testing the best performance since it only evals a small batch # but it should be enough to detect a regression in the output quality From 8afc0c518fdb5ec83b87ac56004b4caaae22a38a Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 18:33:18 -0700 Subject: [PATCH 049/109] cache heavy models/tokenizers --- tests/test_modeling_fsmt.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 697031a205fc..7613a4a22617 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -363,14 +363,33 @@ def _long_tensor(tok_lst): @require_torch class FSMTModelIntegrationTests(unittest.TestCase): + tokenizers_cache = {} + models_cache = {} + @cached_property def default_tokenizer(self): - return FSMTTokenizer.from_pretrained("stas/fsmt-wmt19-ru-en") + return self.get_tokenizer("stas/fsmt-wmt19-ru-en") + + @cached_property + def default_model(self): + return self.get_model("stas/fsmt-wmt19-ru-en") + + def get_tokenizer(self, mname): + if mname not in self.tokenizers_cache: + self.tokenizers_cache[mname] = FSMTTokenizer.from_pretrained(mname) + return self.tokenizers_cache[mname] + + def get_model(self, mname): + if mname not in self.models_cache: + self.models_cache[mname] = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) + if torch_device == "cuda": + self.models_cache[mname].half() + return self.models_cache[mname] @slow def test_inference_no_head(self): tokenizer = self.default_tokenizer - model = FSMTModel.from_pretrained("stas/fsmt-wmt19-ru-en").to(torch_device) + model = self.default_model src_text = "My friend computer will translate this for me" input_ids = tokenizer([src_text], return_tensors="pt")["input_ids"] @@ -412,10 +431,8 @@ def test_translation(self, pair): src_sentence = text[src] tgt_sentence = text[tgt] - tokenizer = FSMTTokenizer.from_pretrained(mname) - model = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) - if torch_device == "cuda": - model.half() + tokenizer = self.get_tokenizer(mname) + model = self.get_model(mname) input_ids = tokenizer.encode(src_sentence, return_tensors="pt") outputs = model.generate(input_ids) @@ -549,8 +566,8 @@ def test_bleu_scores(self, pair, min_bleu_score): # note: this test is not testing the best performance since it only evals a small batch # but it should be enough to detect a regression in the output quality mname = f"stas/fsmt-wmt19-{pair}" - tokenizer = FSMTTokenizer.from_pretrained(mname) - model = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) + tokenizer = self.get_tokenizer(mname) + model = self.get_model(mname) src_sentences = self.bleu_data[pair]["src"] tgt_sentences = self.bleu_data[pair]["tgt"] From f734eff85ec2565c76e9b89c2bcb8f42e43abc80 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 18:57:29 -0700 Subject: [PATCH 050/109] some tweaks --- tests/test_modeling_fsmt.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 7613a4a22617..3e22093f1651 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -365,14 +365,15 @@ def _long_tensor(tok_lst): class FSMTModelIntegrationTests(unittest.TestCase): tokenizers_cache = {} models_cache = {} + default_mname = "stas/fsmt-wmt19-en-ru" @cached_property def default_tokenizer(self): - return self.get_tokenizer("stas/fsmt-wmt19-ru-en") + return self.get_tokenizer(self.default_mname) @cached_property def default_model(self): - return self.get_model("stas/fsmt-wmt19-ru-en") + return self.get_model(self.default_mname) def get_tokenizer(self, mname): if mname not in self.tokenizers_cache: @@ -389,7 +390,7 @@ def get_model(self, mname): @slow def test_inference_no_head(self): tokenizer = self.default_tokenizer - model = self.default_model + model = FSMTModel.from_pretrained(self.default_mname).to(torch_device) src_text = "My friend computer will translate this for me" input_ids = tokenizer([src_text], return_tensors="pt")["input_ids"] @@ -397,15 +398,13 @@ def test_inference_no_head(self): inputs_dict = prepare_fsmt_inputs_dict(model.config, input_ids) with torch.no_grad(): output = model(**inputs_dict)[0] - expected_shape = torch.Size((1, model.config.decoder_attention_heads, model.config.tgt_vocab_size)) + expected_shape = torch.Size((1, 10, model.config.tgt_vocab_size)) self.assertEqual(output.shape, expected_shape) - print(output[:, :3, :3]) - # expected numbers were generated when using just fairseq's model4.pt + # expected numbers were generated when en-ru model, using just fairseq's model4.pt # may have to adjust if switched to a different checkpoint expected_slice = torch.tensor( - [[-3.3069, -3.3069, 2.5253], [-4.2303, -4.2301, 0.8354], [-3.2488, -3.2487, 1.7397]], device=torch_device + [[-1.5753, -1.5753, 2.8975], [-0.9540, -0.9540, 1.0299], [-3.3131, -3.3131, 0.5219]] ) - print(output[:, :3, :3]) self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=TOLERANCE)) @parameterized.expand( From 753c77056adfd9e5938e2897deb1d5e508a87e77 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 19:25:25 -0700 Subject: [PATCH 051/109] remove tokens that aren't used --- src/transformers/tokenization_fsmt.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 064cdb8b61be..157b989dbc95 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -233,18 +233,6 @@ def __init__( pad_token="", cls_token="", mask_token="", - additional_special_tokens=[ - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - ], **kwargs ): super().__init__( @@ -254,7 +242,6 @@ def __init__( pad_token=pad_token, cls_token=cls_token, mask_token=mask_token, - additional_special_tokens=additional_special_tokens, **kwargs, ) From 07e98658dc17c895616db0b1f97d6c7ed3e1380c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 19:30:38 -0700 Subject: [PATCH 052/109] more purging --- src/transformers/tokenization_fsmt.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 157b989dbc95..3a89395fa486 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -162,7 +162,6 @@ class FSMTTokenizer(PreTrainedTokenizer): BPE tokenizer for FSMT (fairseq transformer) See: https://github.com/pytorch/fairseq/tree/master/examples/wmt19 - - Moses preprocessing & tokenization for most supported languages - (optionally) lower case & normalize all inputs text - argument ``special_tokens`` and function ``set_special_tokens``, can be used to add additional symbols \ @@ -183,10 +182,6 @@ class FSMTTokenizer(PreTrainedTokenizer): Merges file. do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): Whether to lowercase the input when tokenizing. - remove_space (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to strip the text when tokenizing (removing excess spaces before and after the string). - keep_accents (:obj:`bool`, `optional`, defaults to :obj:`False`): - Whether to keep accents when tokenizing. unk_token (:obj:`string`, `optional`, defaults to ""): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. @@ -203,16 +198,6 @@ class FSMTTokenizer(PreTrainedTokenizer): It is also used as the last token of a sequence built with special tokens. pad_token (:obj:`string`, `optional`, defaults to ""): The token used for padding, for example when batching sequences of different lengths. - cls_token (:obj:`string`, `optional`, defaults to ""): - The classifier token which is used when doing sequence classification (classification of the whole - sequence instead of per-token classification). It is the first token of the sequence when built with - special tokens. - mask_token (:obj:`string`, `optional`, defaults to ""): - The token used for masking values. This is the token used when training this model with masked language - modeling. This is the token which the model will try to predict. - additional_special_tokens (:obj:`List[str]`, `optional`, defaults to :obj:`["","","","","","","","","",""]`): - List of additional special tokens. - """ @@ -231,8 +216,6 @@ def __init__( bos_token="", sep_token="", pad_token="", - cls_token="", - mask_token="", **kwargs ): super().__init__( @@ -240,8 +223,6 @@ def __init__( bos_token=bos_token, sep_token=sep_token, pad_token=pad_token, - cls_token=cls_token, - mask_token=mask_token, **kwargs, ) From bb3f1a83ba5569c7c1b48e5e5681c2bb0d790ac4 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 19:33:43 -0700 Subject: [PATCH 053/109] simplify code --- src/transformers/tokenization_fsmt.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 3a89395fa486..80c1f01acf79 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -269,25 +269,21 @@ def moses_punct_norm(self, text, lang): if lang not in self.cache_moses_punct_normalizer: punct_normalizer = sm.MosesPunctNormalizer(lang=lang) self.cache_moses_punct_normalizer[lang] = punct_normalizer - else: - punct_normalizer = self.cache_moses_punct_normalizer[lang] - return punct_normalizer.normalize(text) + return self.cache_moses_punct_normalizer[lang].normalize(text) def moses_tokenize(self, text, lang): if lang not in self.cache_moses_tokenizer: moses_tokenizer = sm.MosesTokenizer(lang=lang) self.cache_moses_tokenizer[lang] = moses_tokenizer - else: - moses_tokenizer = self.cache_moses_tokenizer[lang] - return moses_tokenizer.tokenize(text, aggressive_dash_splits=True, return_str=False, escape=True) + return self.cache_moses_tokenizer[lang].tokenize( + text, aggressive_dash_splits=True, return_str=False, escape=True + ) def moses_detokenize(self, tokens, lang): if lang not in self.cache_moses_tokenizer: moses_detokenizer = sm.MosesDetokenizer(lang=self.tgt_lang) self.cache_moses_detokenizer[lang] = moses_detokenizer - else: - moses_detokenizer = self.cache_moses_detokenizer[lang] - return moses_detokenizer.detokenize(tokens) + return self.cache_moses_detokenizer[lang].detokenize(tokens) def moses_pipeline(self, text, lang): text = replace_unicode_punct(text) From 7ddf3ad183885aab1e253a1f8825100af294ffce Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 11:00:53 -0700 Subject: [PATCH 054/109] switch to using decoder_start_token_id --- src/transformers/configuration_fsmt.py | 2 ++ src/transformers/generation_utils.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 93118968ec4a..ad274069c51f 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -143,6 +143,7 @@ def __init__( pad_token_id=1, bos_token_id=0, eos_token_id=2, + decoder_start_token_id=2, add_bias_logits=False, add_final_layer_norm=False, is_encoder_decoder=True, @@ -170,6 +171,7 @@ def __init__( pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, + decoder_start_token_id=decoder_start_token_id, is_encoder_decoder=is_encoder_decoder, tie_word_embeddings=tie_word_embeddings, **common_kwargs, diff --git a/src/transformers/generation_utils.py b/src/transformers/generation_utils.py index 4d2af8506b24..053b13a34305 100644 --- a/src/transformers/generation_utils.py +++ b/src/transformers/generation_utils.py @@ -381,10 +381,10 @@ def generate( if self.config.is_encoder_decoder: if decoder_start_token_id is None: # see if BOS token can be used for decoder_start_token_id - if hasattr(self.config, "decoder") and hasattr(self.config.decoder, "bos_token_id"): - decoder_start_token_id = self.config.decoder.bos_token_id - elif bos_token_id is not None: + if bos_token_id is not None: decoder_start_token_id = bos_token_id + elif hasattr(self.config, "decoder") and hasattr(self.config.decoder, "bos_token_id"): + decoder_start_token_id = self.config.decoder.bos_token_id else: raise ValueError( "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation" From 600f056edc1e3ce66985924efed0dfc80772786a Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 11:11:47 -0700 Subject: [PATCH 055/109] add doc --- src/transformers/configuration_fsmt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index ad274069c51f..37aacf69d1dc 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -86,6 +86,8 @@ Padding token id. eos_token_id (:obj:`int`, optional, defaults to 2) End of stream token id. + decoder_start_token_id (:obj:`int`, `optional`): + This model starts decoding with `eos_token_id` encoder_layerdrop: (:obj:`float`, optional, defaults to 0.0): Google "layerdrop arxiv", as its not explainable in one line. decoder_layerdrop: (:obj:`float`, optional, defaults to 0.0): From 6a0552354c864d2a9d9f8fb0a0d7ac8294b535c5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 11:26:26 -0700 Subject: [PATCH 056/109] Revert "major refactor (reuse-bart)" This reverts commit 226dad15ca6a9ef4e26178526e878e8fc5c85874. --- src/transformers/modeling_fsmt.py | 131 +++++++++++++++++++++++++++--- tests/test_modeling_fsmt.py | 34 +++++++- 2 files changed, 150 insertions(+), 15 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 44b661ebfb46..d7e97aa168f7 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -46,15 +46,7 @@ add_start_docstrings_to_callable, replace_return_docstrings, ) -from .modeling_bart import ( - DecoderLayer, - EncoderLayer, - LayerNorm, - LearnedPositionalEmbedding, - _prepare_bart_decoder_inputs, - _reorder_buffer, - invert_mask, -) +from .modeling_bart import DecoderLayer, EncoderLayer from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel @@ -187,7 +179,8 @@ FSMT_START_DOCSTRING = r""" - This model is a PyTorch `torch.nn.Module `_ sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matters related to general usage and behavior. + This model is a PyTorch `torch.nn.Module `_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matters related to general usage and behavior. Parameters: config (:class:`~transformers.FSMTConfig`): Model configuration class with all the parameters of the model. @@ -253,6 +246,33 @@ """ +def invert_mask(attention_mask): + """Turns 1->0, 0->1, False->True, True-> False""" + assert attention_mask.dim() == 2 + return attention_mask.eq(0) + + +def _prepare_fsmt_decoder_inputs( + config, input_ids, decoder_input_ids=None, decoder_padding_mask=None, causal_mask_dtype=torch.float32 +): + """Prepare masks that ignore padding tokens in the decoder and a causal mask for the decoder if + none are provided. This mimics the default behavior in fairseq. To override it pass in masks. + Note: this is not called during generation + """ + pad_token_id = config.pad_token_id + if decoder_input_ids is None: + decoder_input_ids = shift_tokens_right(input_ids, pad_token_id) + bsz, tgt_len = decoder_input_ids.size() + if decoder_padding_mask is None: + decoder_padding_mask = make_padding_mask(decoder_input_ids, pad_token_id) + else: + decoder_padding_mask = invert_mask(decoder_padding_mask) + causal_mask = torch.triu(fill_with_neg_inf(torch.zeros(tgt_len, tgt_len)), 1).to( + dtype=causal_mask_dtype, device=decoder_input_ids.device + ) + return decoder_input_ids, decoder_padding_mask, causal_mask + + class PretrainedFSMTModel(PreTrainedModel): config_class = FSMTConfig base_model_prefix = "model" @@ -281,6 +301,36 @@ def dummy_inputs(self): return dummy_inputs +def _make_linear_from_emb(emb): + vocab_size, emb_size = emb.weight.shape + lin_layer = nn.Linear(vocab_size, emb_size, bias=False) + lin_layer.weight.data = emb.weight.data + return lin_layer + + +# Helper Functions, mostly for making masks +def _check_shapes(shape_1, shape2): + if shape_1 != shape2: + raise AssertionError("shape mismatch: {} != {}".format(shape_1, shape2)) + + +def shift_tokens_right(input_ids, pad_token_id): + """Shift input ids one token to the right, and wrap the last non pad token (usually ).""" + prev_output_tokens = input_ids.clone() + index_of_eos = (input_ids.ne(pad_token_id).sum(dim=1) - 1).unsqueeze(-1) + prev_output_tokens[:, 0] = input_ids.gather(1, index_of_eos).squeeze() + prev_output_tokens[:, 1:] = input_ids[:, :-1] + return prev_output_tokens + + +def make_padding_mask(input_ids, padding_idx=1): + """True for pad tokens""" + padding_mask = input_ids.eq(padding_idx) + if not padding_mask.any(): + padding_mask = None + return padding_mask + + # Helper Modules @@ -547,11 +597,70 @@ def forward( ) +def _reorder_buffer(attn_cache, new_order): + for k, input_buffer_k in attn_cache.items(): + if input_buffer_k is not None: + attn_cache[k] = input_buffer_k.index_select(0, new_order) + return attn_cache + + +# XXX: remove this and its references +class LearnedPositionalEmbedding(nn.Embedding): + """ + This module learns positional embeddings up to a fixed maximum size. + Padding ids are ignored by either offsetting based on padding_idx + or by setting padding_idx to None and ensuring that the appropriate + position ids are passed to the forward function. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, offset): + # FSMT is set up so that if padding_idx is specified then offset the embedding ids by 2 + # and adjust num_embeddings appropriately. Other models dont have this hack + self.offset = offset + assert padding_idx is not None + num_embeddings += offset + super().__init__(num_embeddings, embedding_dim, padding_idx=padding_idx) + + def forward(self, input_ids, use_cache=False): + """Input is expected to be of size [bsz x seqlen].""" + bsz, seq_len = input_ids.shape[:2] + if use_cache: + positions = input_ids.data.new(1, 1).fill_(seq_len - 1) # called before slicing + else: + # starts at 0, ends at 1-seq_len + positions = torch.arange(seq_len, dtype=torch.long, device=self.weight.device) + return super().forward(positions + self.offset) + + +def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True): + if torch.cuda.is_available(): + try: + from apex.normalization import FusedLayerNorm + + return FusedLayerNorm(normalized_shape, eps, elementwise_affine) + except ImportError: + pass + return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine) + + +def fill_with_neg_inf(t): + """FP16-compatible function that fills a input_ids with -inf.""" + return t.float().fill_(float("-inf")).type_as(t) + + # Public API def _get_shape(t): return getattr(t, "shape", None) +# def output_projection(self): +# return nn.Linear( +# self.embed_tokens.weight.shape[1], +# self.embed_tokens.weight.shape[0], +# bias=False, +# ) + + @add_start_docstrings( "The bare FSMT Model outputting raw hidden-states without any specific head on top.", FSMT_START_DOCSTRING, @@ -609,7 +718,7 @@ def forward( # make masks if user doesn't supply if not use_cache: - decoder_input_ids, decoder_padding_mask, causal_mask = _prepare_bart_decoder_inputs( + decoder_input_ids, decoder_padding_mask, causal_mask = _prepare_fsmt_decoder_inputs( self.config, input_ids, decoder_input_ids=decoder_input_ids, diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 3e22093f1651..69269d297aa3 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -40,9 +40,12 @@ import torch from transformers import FSMTConfig, FSMTForConditionalGeneration, FSMTModel, FSMTTokenizer - from transformers.modeling_bart import _prepare_bart_decoder_inputs, invert_mask - from transformers.modeling_fsmt import SinusoidalPositionalEmbedding - + from transformers.modeling_fsmt import ( + SinusoidalPositionalEmbedding, + _prepare_fsmt_decoder_inputs, + invert_mask, + shift_tokens_right, + ) def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: """Uses sacrebleu's corpus_bleu implementation.""" @@ -174,7 +177,7 @@ def test_advanced_inputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() config.use_cache = False inputs_dict["input_ids"][:, -2:] = config.pad_token_id - decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_bart_decoder_inputs( + decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_fsmt_decoder_inputs( config, inputs_dict["input_ids"] ) model = FSMTModel(config).to(torch_device).eval() @@ -297,6 +300,15 @@ def test_generate_beam_search(self): self.assertEqual(new_input_ids.shape, (input_ids.shape[0], max_length)) # TODO(SS): uneven length batches, empty inputs + def test_shift_tokens_right(self): + input_ids = torch.Tensor([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]).long() + shifted = shift_tokens_right(input_ids, 1) + n_pad_before = input_ids.eq(1).float().sum() + n_pad_after = shifted.eq(1).float().sum() + self.assertEqual(shifted.shape, input_ids.shape) + self.assertEqual(n_pad_after, n_pad_before - 1) + self.assertTrue(torch.eq(shifted[:, 0], 2).all()) + def test_generate_fp16(self): config, input_ids, batch_size = self._get_config_and_data() attention_mask = input_ids.ne(1).to(torch_device) @@ -311,6 +323,20 @@ def test_dummy_inputs(self): model = FSMTForConditionalGeneration(config).eval().to(torch_device) model(**model.dummy_inputs) + def test_prepare_fsmt_decoder_inputs(self): + config, *_ = self._get_config_and_data() + input_ids = _long_tensor(([4, 4, 2])) + decoder_input_ids = _long_tensor([[26388, 2, config.pad_token_id]]) + ignore = float("-inf") + decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_fsmt_decoder_inputs( + config, input_ids, decoder_input_ids + ) + expected_causal_mask = torch.tensor( + [[0, ignore, ignore], [0, 0, ignore], [0, 0, 0]] # never attend to the final token, because its pad + ).to(input_ids.device) + self.assertEqual(decoder_attn_mask.size(), decoder_input_ids.size()) + self.assertTrue(torch.eq(expected_causal_mask, causal_mask).all()) + def test_resize_tokens_embeddings_more(self): config, input_ids, _ = self._get_config_and_data() From 528d73a72bdb4c327b0f1c32a9b306331be20562 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 11:36:57 -0700 Subject: [PATCH 057/109] decouple from bart --- src/transformers/modeling_fsmt.py | 298 +++++++++++++++++++++++++++++- tests/test_modeling_fsmt.py | 1 + 2 files changed, 297 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index d7e97aa168f7..1fb5e8a54dd3 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -31,13 +31,14 @@ import math import random import warnings -from typing import Any, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import torch import torch.nn.functional as F from torch import Tensor, nn from torch.nn import CrossEntropyLoss +from .activations import ACT2FN from .configuration_fsmt import FSMTConfig from .file_utils import ( add_code_sample_docstrings, @@ -46,7 +47,6 @@ add_start_docstrings_to_callable, replace_return_docstrings, ) -from .modeling_bart import DecoderLayer, EncoderLayer from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel @@ -334,6 +334,56 @@ def make_padding_mask(input_ids, padding_idx=1): # Helper Modules +class EncoderLayer(nn.Module): + def __init__(self, config: FSMTConfig): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = Attention(self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout) + self.normalize_before = config.normalize_before + self.self_attn_layer_norm = LayerNorm(self.embed_dim) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) + self.final_layer_norm = LayerNorm(self.embed_dim) + + def forward(self, x, encoder_padding_mask, output_attentions=False): + """ + Args: + x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)` + encoder_padding_mask (ByteTensor): binary ByteTensor of shape + `(batch, src_len)` where padding elements are indicated by ``1``. + for t_tgt, t_src is excluded (or masked out), =0 means it is + included in attention + + Returns: + encoded output of shape `(seq_len, batch, embed_dim)` + """ + residual = x + if self.normalize_before: + x = self.self_attn_layer_norm(x) + x, attn_weights = self.self_attn( + query=x, key=x, key_padding_mask=encoder_padding_mask, output_attentions=output_attentions + ) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.self_attn_layer_norm(x) + + residual = x + if self.normalize_before: + x = self.final_layer_norm(x) + x = self.activation_fn(self.fc1(x)) + x = F.dropout(x, p=self.activation_dropout, training=self.training) + x = self.fc2(x) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.final_layer_norm(x) + return x, attn_weights + + class FSMTEncoder(nn.Module): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer @@ -436,6 +486,98 @@ def forward( return BaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions) +class DecoderLayer(nn.Module): + def __init__(self, config: FSMTConfig): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = Attention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + ) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + self.normalize_before = config.normalize_before + + self.self_attn_layer_norm = LayerNorm(self.embed_dim) + self.encoder_attn = Attention( + self.embed_dim, + config.decoder_attention_heads, + dropout=config.attention_dropout, + encoder_decoder_attention=True, + ) + self.encoder_attn_layer_norm = LayerNorm(self.embed_dim) + self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) + self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) + self.final_layer_norm = LayerNorm(self.embed_dim) + + def forward( + self, + x, + encoder_hidden_states, + encoder_attn_mask=None, + layer_state=None, + causal_mask=None, + decoder_padding_mask=None, + output_attentions=False, + ): + residual = x + + if layer_state is None: + layer_state = {} + if self.normalize_before: + x = self.self_attn_layer_norm(x) + # Self Attention + + x, self_attn_weights = self.self_attn( + query=x, + key=x, + layer_state=layer_state, # adds keys to layer state + key_padding_mask=decoder_padding_mask, + attn_mask=causal_mask, + output_attentions=output_attentions, + ) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.self_attn_layer_norm(x) + + # Cross attention + residual = x + assert self.encoder_attn.cache_key != self.self_attn.cache_key + if self.normalize_before: + x = self.encoder_attn_layer_norm(x) + x, _ = self.encoder_attn( + query=x, + key=encoder_hidden_states, + key_padding_mask=encoder_attn_mask, + layer_state=layer_state, # mutates layer state + ) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.encoder_attn_layer_norm(x) + + # Fully Connected + residual = x + if self.normalize_before: + x = self.final_layer_norm(x) + x = self.activation_fn(self.fc1(x)) + x = F.dropout(x, p=self.activation_dropout, training=self.training) + x = self.fc2(x) + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + if not self.normalize_before: + x = self.final_layer_norm(x) + return ( + x, + self_attn_weights, + layer_state, + ) # just self_attn weights for now, following t5, layer_state = cache for decoding + + class FSMTDecoder(nn.Module): """ Transformer decoder consisting of *config.decoder_layers* layers. Each layer @@ -604,6 +746,158 @@ def _reorder_buffer(attn_cache, new_order): return attn_cache +class Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + bias=True, + encoder_decoder_attention=False, # otherwise self_attention + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + self.scaling = self.head_dim ** -0.5 + + self.encoder_decoder_attention = encoder_decoder_attention + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.cache_key = "encoder_decoder" if self.encoder_decoder_attention else "self" + + def _shape(self, tensor, seq_len, bsz): + return tensor.contiguous().view(seq_len, bsz * self.num_heads, self.head_dim).transpose(0, 1) + + def forward( + self, + query, + key: Optional[Tensor], + key_padding_mask: Optional[Tensor] = None, + layer_state: Optional[Dict[str, Optional[Tensor]]] = None, + attn_mask: Optional[Tensor] = None, + output_attentions=False, + ) -> Tuple[Tensor, Optional[Tensor]]: + """Input shape: Time(SeqLen) x Batch x Channel""" + static_kv: bool = self.encoder_decoder_attention + tgt_len, bsz, embed_dim = query.size() + assert embed_dim == self.embed_dim + assert list(query.size()) == [tgt_len, bsz, embed_dim] + # get here for encoder decoder cause of static_kv + if layer_state is not None: # reuse k,v and encoder_padding_mask + saved_state = layer_state.get(self.cache_key, {}) + if "prev_key" in saved_state and static_kv: + # previous time steps are cached - no need to recompute key and value if they are static + key = None + else: + saved_state = None + layer_state = {} + + q = self.q_proj(query) * self.scaling + if static_kv: + if key is None: + k = v = None + else: + k = self.k_proj(key) + v = self.v_proj(key) + else: + k = self.k_proj(query) + v = self.v_proj(query) + + q = self._shape(q, tgt_len, bsz) + if k is not None: + k = self._shape(k, -1, bsz) + if v is not None: + v = self._shape(v, -1, bsz) + + if saved_state is not None: + k, v, key_padding_mask = self._use_saved_state(k, v, saved_state, key_padding_mask, static_kv, bsz) + + # Update cache + layer_state[self.cache_key] = { + "prev_key": k.view(bsz, self.num_heads, -1, self.head_dim), + "prev_value": v.view(bsz, self.num_heads, -1, self.head_dim), + "prev_key_padding_mask": key_padding_mask if not static_kv else None, + } + + assert k is not None + src_len = k.size(1) + attn_weights = torch.bmm(q, k.transpose(1, 2)) + assert attn_weights.size() == (bsz * self.num_heads, tgt_len, src_len) + + if attn_mask is not None: + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # This is part of a workaround to get around fork/join parallelism not supporting Optional types. + if key_padding_mask is not None and key_padding_mask.dim() == 0: + key_padding_mask = None + assert key_padding_mask is None or key_padding_mask.size()[:2] == ( + bsz, + src_len, + ) + + if key_padding_mask is not None: # don't attend to padding symbols + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + reshaped = key_padding_mask.unsqueeze(1).unsqueeze(2) + attn_weights = attn_weights.masked_fill(reshaped, float("-inf")) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + attn_weights = F.softmax(attn_weights, dim=-1) + attn_probs = F.dropout( + attn_weights, + p=self.dropout, + training=self.training, + ) + + assert v is not None + attn_output = torch.bmm(attn_probs, v) + assert attn_output.size() == (bsz * self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim) + attn_output = self.out_proj(attn_output) + if output_attentions: + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + else: + attn_weights = None + return attn_output, attn_weights + + def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz): + # saved states are stored with shape (bsz, num_heads, seq_len, head_dim) + if "prev_key" in saved_state: + _prev_key = saved_state["prev_key"] + assert _prev_key is not None + prev_key = _prev_key.view(bsz * self.num_heads, -1, self.head_dim) + if static_kv: + k = prev_key + else: + assert k is not None + k = torch.cat([prev_key, k], dim=1) + if "prev_value" in saved_state: + _prev_value = saved_state["prev_value"] + assert _prev_value is not None + prev_value = _prev_value.view(bsz * self.num_heads, -1, self.head_dim) + if static_kv: + v = prev_value + else: + assert v is not None + v = torch.cat([prev_value, v], dim=1) + assert k is not None and v is not None + prev_key_padding_mask: Optional[Tensor] = saved_state.get("prev_key_padding_mask", None) + if prev_key_padding_mask is not None: + if static_kv: + new_key_padding_mask = prev_key_padding_mask + else: + new_key_padding_mask = torch.cat([prev_key_padding_mask, key_padding_mask], dim=1) + else: + new_key_padding_mask = key_padding_mask + return k, v, new_key_padding_mask + + # XXX: remove this and its references class LearnedPositionalEmbedding(nn.Embedding): """ diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 69269d297aa3..994107207c0c 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -47,6 +47,7 @@ shift_tokens_right, ) + def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: """Uses sacrebleu's corpus_bleu implementation.""" return {"bleu": round(corpus_bleu(output_lns, [refs_lns], **kwargs).score, 4)} From 416fccf0db03b85a72184a2fb8c1c64235b5bfcf Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 11:47:21 -0700 Subject: [PATCH 058/109] remove unused code #1 --- src/transformers/configuration_fsmt.py | 14 ---- ..._original_pytorch_checkpoint_to_pytorch.py | 3 - src/transformers/modeling_fsmt.py | 74 ++++--------------- 3 files changed, 13 insertions(+), 78 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 37aacf69d1dc..f47244daa152 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -68,14 +68,8 @@ Typically set this to something large just in case (e.g., 512 or 1024 or 2048). init_std (:obj:`float`, optional, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - add_bias_logits (:obj:`bool`, optional, defaults to :obj:`False`): - True for marian only. normalize_before (:obj:`bool`, optional, defaults to :obj:`False`): Call layernorm before attention ops. - normalize_embedding (:obj:`bool`, optional, defaults to :obj:`False`): - Call layernorm after embeddings. - static_position_embeddings (:obj:`bool`, optional, defaults to :obj:`True`): - Don't learn positional embeddings, use sinusoidal. add_final_layer_norm (:obj:`bool`, optional, defaults to :obj:`False`): Why not add another layernorm? scale_embedding (:obj:`bool`, optional, defaults to :obj:`True`): @@ -146,13 +140,10 @@ def __init__( bos_token_id=0, eos_token_id=2, decoder_start_token_id=2, - add_bias_logits=False, add_final_layer_norm=False, is_encoder_decoder=True, normalize_before=False, - normalize_embedding=False, scale_embedding=True, - static_position_embeddings=True, tie_word_embeddings=False, **common_kwargs ): @@ -200,14 +191,9 @@ def __init__( # Params introduced for Mbart self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True - self.normalize_embedding = normalize_embedding # True for mbart, False otherwise self.normalize_before = normalize_before # combo of fairseq's encoder_ and decoder_normalize_before self.add_final_layer_norm = add_final_layer_norm - # Params introduced for Marian - self.add_bias_logits = add_bias_logits - self.static_position_embeddings = static_position_embeddings - # 3 Types of Dropout self.attention_dropout = attention_dropout self.activation_dropout = activation_dropout diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 631b30679cae..d88e4496f0b5 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -319,13 +319,10 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder "bos_token_id": 0, "pad_token_id": 1, "eos_token_id": 2, - "add_bias_logits": False, "add_final_layer_norm": False, "is_encoder_decoder": True, "normalize_before": False, - "normalize_embedding": False, "scale_embedding": True, - "static_position_embeddings": True, "tie_word_embeddings": False, } diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 1fb5e8a54dd3..5706c89023fd 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -97,8 +97,6 @@ # - token embeddings aren't shared # - needs a language pair # - scale_embedding are True -# - normalize_embedding are False -# - static_position_embeddings are True # # some unused args were removed too # @@ -405,23 +403,14 @@ def __init__(self, config: FSMTConfig, embed_tokens): self.max_source_positions = config.max_position_embeddings self.embed_tokens = embed_tokens - if config.static_position_embeddings: - # print(config.max_position_embeddings, embed_dim, self.padding_idx) - num_embeddings = config.src_vocab_size - self.embed_positions = SinusoidalPositionalEmbedding( - embed_dim, - self.padding_idx, - init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings - ) - else: - self.embed_positions = LearnedPositionalEmbedding( - config.max_position_embeddings, - embed_dim, - self.padding_idx, - config.extra_pos_embeddings, - ) + # print(config.max_position_embeddings, embed_dim, self.padding_idx) + num_embeddings = config.src_vocab_size + self.embed_positions = SinusoidalPositionalEmbedding( + embed_dim, + self.padding_idx, + init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings + ) self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.encoder_layers)]) - self.layernorm_embedding = LayerNorm(embed_dim) if config.normalize_embedding else nn.Identity() # mbart has one extra layer_norm self.layer_norm = LayerNorm(config.d_model) if config.normalize_before else None @@ -450,7 +439,6 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale embed_pos = self.embed_positions(input_ids) x = inputs_embeds + embed_pos - x = self.layernorm_embedding(x) x = F.dropout(x, p=self.dropout, training=self.training) # B x T x C -> T x B x C @@ -596,24 +584,15 @@ def __init__(self, config: FSMTConfig, embed_tokens: nn.Embedding): self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 self.embed_tokens = embed_tokens embed_dim = embed_tokens.embedding_dim - if config.static_position_embeddings: - num_embeddings = config.tgt_vocab_size - self.embed_positions = SinusoidalPositionalEmbedding( - embed_dim, - self.padding_idx, - init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings - ) - else: - self.embed_positions = LearnedPositionalEmbedding( - config.max_position_embeddings, - config.d_model, - self.padding_idx, - config.extra_pos_embeddings, - ) + num_embeddings = config.tgt_vocab_size + self.embed_positions = SinusoidalPositionalEmbedding( + embed_dim, + self.padding_idx, + init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings + ) self.layers = nn.ModuleList( [DecoderLayer(config) for _ in range(config.decoder_layers)] ) # type: List[DecoderLayer] - self.layernorm_embedding = LayerNorm(config.d_model) if config.normalize_embedding else nn.Identity() self.layer_norm = LayerNorm(config.d_model) if config.add_final_layer_norm else None self.output_projection = nn.Linear( @@ -683,7 +662,6 @@ def forward( x = self.embed_tokens(input_ids) * self.embed_scale x += positions - x = self.layernorm_embedding(x) x = F.dropout(x, p=self.dropout, training=self.training) # Convert to FSMT output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim) @@ -898,32 +876,6 @@ def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz): return k, v, new_key_padding_mask -# XXX: remove this and its references -class LearnedPositionalEmbedding(nn.Embedding): - """ - This module learns positional embeddings up to a fixed maximum size. - Padding ids are ignored by either offsetting based on padding_idx - or by setting padding_idx to None and ensuring that the appropriate - position ids are passed to the forward function. - """ - - def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, offset): - # FSMT is set up so that if padding_idx is specified then offset the embedding ids by 2 - # and adjust num_embeddings appropriately. Other models dont have this hack - self.offset = offset - assert padding_idx is not None - num_embeddings += offset - super().__init__(num_embeddings, embedding_dim, padding_idx=padding_idx) - - def forward(self, input_ids, use_cache=False): - """Input is expected to be of size [bsz x seqlen].""" - bsz, seq_len = input_ids.shape[:2] - if use_cache: - positions = input_ids.data.new(1, 1).fill_(seq_len - 1) # called before slicing - else: - # starts at 0, ends at 1-seq_len - positions = torch.arange(seq_len, dtype=torch.long, device=self.weight.device) - return super().forward(positions + self.offset) def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True): From 05f09bbc5120a70a23ce7418d4a799c5e51465e1 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 11:54:02 -0700 Subject: [PATCH 059/109] remove unused code #2 --- src/transformers/configuration_fsmt.py | 8 ----- ..._original_pytorch_checkpoint_to_pytorch.py | 2 -- src/transformers/modeling_fsmt.py | 36 ++++--------------- 3 files changed, 6 insertions(+), 40 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index f47244daa152..3490ca0e70ad 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -68,10 +68,6 @@ Typically set this to something large just in case (e.g., 512 or 1024 or 2048). init_std (:obj:`float`, optional, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - normalize_before (:obj:`bool`, optional, defaults to :obj:`False`): - Call layernorm before attention ops. - add_final_layer_norm (:obj:`bool`, optional, defaults to :obj:`False`): - Why not add another layernorm? scale_embedding (:obj:`bool`, optional, defaults to :obj:`True`): Scale embeddings by diving by sqrt(d_model). bos_token_id (:obj:`int`, optional, defaults to 0) @@ -140,9 +136,7 @@ def __init__( bos_token_id=0, eos_token_id=2, decoder_start_token_id=2, - add_final_layer_norm=False, is_encoder_decoder=True, - normalize_before=False, scale_embedding=True, tie_word_embeddings=False, **common_kwargs @@ -191,8 +185,6 @@ def __init__( # Params introduced for Mbart self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True - self.normalize_before = normalize_before # combo of fairseq's encoder_ and decoder_normalize_before - self.add_final_layer_norm = add_final_layer_norm # 3 Types of Dropout self.attention_dropout = attention_dropout diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index d88e4496f0b5..62ac3514f755 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -319,9 +319,7 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder "bos_token_id": 0, "pad_token_id": 1, "eos_token_id": 2, - "add_final_layer_norm": False, "is_encoder_decoder": True, - "normalize_before": False, "scale_embedding": True, "tie_word_embeddings": False, } diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 5706c89023fd..ede39e76512a 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -337,7 +337,6 @@ def __init__(self, config: FSMTConfig): super().__init__() self.embed_dim = config.d_model self.self_attn = Attention(self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout) - self.normalize_before = config.normalize_before self.self_attn_layer_norm = LayerNorm(self.embed_dim) self.dropout = config.dropout self.activation_fn = ACT2FN[config.activation_function] @@ -359,26 +358,20 @@ def forward(self, x, encoder_padding_mask, output_attentions=False): encoded output of shape `(seq_len, batch, embed_dim)` """ residual = x - if self.normalize_before: - x = self.self_attn_layer_norm(x) x, attn_weights = self.self_attn( query=x, key=x, key_padding_mask=encoder_padding_mask, output_attentions=output_attentions ) x = F.dropout(x, p=self.dropout, training=self.training) x = residual + x - if not self.normalize_before: - x = self.self_attn_layer_norm(x) + x = self.self_attn_layer_norm(x) residual = x - if self.normalize_before: - x = self.final_layer_norm(x) x = self.activation_fn(self.fc1(x)) x = F.dropout(x, p=self.activation_dropout, training=self.training) x = self.fc2(x) x = F.dropout(x, p=self.dropout, training=self.training) x = residual + x - if not self.normalize_before: - x = self.final_layer_norm(x) + x = self.final_layer_norm(x) return x, attn_weights @@ -411,8 +404,6 @@ def __init__(self, config: FSMTConfig, embed_tokens): init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings ) self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.encoder_layers)]) - # mbart has one extra layer_norm - self.layer_norm = LayerNorm(config.d_model) if config.normalize_before else None def forward( self, input_ids, attention_mask=None, output_attentions=False, output_hidden_states=False, return_dict=False @@ -459,8 +450,6 @@ def forward( if output_attentions: all_attentions = all_attentions + (attn,) - if self.layer_norm: - x = self.layer_norm(x) if output_hidden_states: encoder_states.append(x) # T x B x C -> B x T x C @@ -487,7 +476,6 @@ def __init__(self, config: FSMTConfig): self.dropout = config.dropout self.activation_fn = ACT2FN[config.activation_function] self.activation_dropout = config.activation_dropout - self.normalize_before = config.normalize_before self.self_attn_layer_norm = LayerNorm(self.embed_dim) self.encoder_attn = Attention( @@ -515,10 +503,8 @@ def forward( if layer_state is None: layer_state = {} - if self.normalize_before: - x = self.self_attn_layer_norm(x) - # Self Attention + # Self Attention x, self_attn_weights = self.self_attn( query=x, key=x, @@ -529,14 +515,11 @@ def forward( ) x = F.dropout(x, p=self.dropout, training=self.training) x = residual + x - if not self.normalize_before: - x = self.self_attn_layer_norm(x) + x = self.self_attn_layer_norm(x) # Cross attention residual = x assert self.encoder_attn.cache_key != self.self_attn.cache_key - if self.normalize_before: - x = self.encoder_attn_layer_norm(x) x, _ = self.encoder_attn( query=x, key=encoder_hidden_states, @@ -545,20 +528,16 @@ def forward( ) x = F.dropout(x, p=self.dropout, training=self.training) x = residual + x - if not self.normalize_before: - x = self.encoder_attn_layer_norm(x) + x = self.encoder_attn_layer_norm(x) # Fully Connected residual = x - if self.normalize_before: - x = self.final_layer_norm(x) x = self.activation_fn(self.fc1(x)) x = F.dropout(x, p=self.activation_dropout, training=self.training) x = self.fc2(x) x = F.dropout(x, p=self.dropout, training=self.training) x = residual + x - if not self.normalize_before: - x = self.final_layer_norm(x) + x = self.final_layer_norm(x) return ( x, self_attn_weights, @@ -593,7 +572,6 @@ def __init__(self, config: FSMTConfig, embed_tokens: nn.Embedding): self.layers = nn.ModuleList( [DecoderLayer(config) for _ in range(config.decoder_layers)] ) # type: List[DecoderLayer] - self.layer_norm = LayerNorm(config.d_model) if config.add_final_layer_norm else None self.output_projection = nn.Linear( self.embed_tokens.weight.shape[1], @@ -695,8 +673,6 @@ def forward( if use_cache: next_decoder_cache.append(layer_past.copy()) - if self.layer_norm and (idx == len(self.layers) - 1): # last layer of mbart - x = self.layer_norm(x) if output_attentions: all_self_attns += (layer_self_attn,) From 07c0e66c248f172d70901c34317dfccbc9d2c12c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 12:11:31 -0700 Subject: [PATCH 060/109] remove unused code #3 --- src/transformers/configuration_fsmt.py | 4 ---- src/transformers/modeling_fsmt.py | 30 +------------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 3490ca0e70ad..7bc058acf182 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -183,7 +183,6 @@ def __init__( self.decoder = DecoderConfig(vocab_size=tgt_vocab_size, bos_token_id=eos_token_id) - # Params introduced for Mbart self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True # 3 Types of Dropout @@ -191,9 +190,6 @@ def __init__( self.activation_dropout = activation_dropout self.dropout = dropout - # pos embedding offset - self.extra_pos_embeddings = self.pad_token_id + 1 - @property def num_attention_heads(self) -> int: return self.encoder_attention_heads diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index ede39e76512a..3349eded62d0 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -567,7 +567,7 @@ def __init__(self, config: FSMTConfig, embed_tokens: nn.Embedding): self.embed_positions = SinusoidalPositionalEmbedding( embed_dim, self.padding_idx, - init_size=num_embeddings + self.padding_idx + 1, # removed: config.max_position_embeddings + init_size=num_embeddings + self.padding_idx + 1, ) self.layers = nn.ModuleList( [DecoderLayer(config) for _ in range(config.decoder_layers)] @@ -852,8 +852,6 @@ def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz): return k, v, new_key_padding_mask - - def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True): if torch.cuda.is_available(): try: @@ -875,14 +873,6 @@ def _get_shape(t): return getattr(t, "shape", None) -# def output_projection(self): -# return nn.Linear( -# self.embed_tokens.weight.shape[1], -# self.embed_tokens.weight.shape[0], -# bias=False, -# ) - - @add_start_docstrings( "The bare FSMT Model outputting raw hidden-states without any specific head on top.", FSMT_START_DOCSTRING, @@ -1059,24 +1049,6 @@ def forward( Returns: """ - if "lm_labels" in unused: - warnings.warn( - "The `lm_labels` argument is deprecated and will be removed in a future version, use `labels` instead.", - FutureWarning, - ) - labels = unused.pop("lm_labels") - if "decoder_cached_states" in unused: - warnings.warn( - "The `decoder_cached_states` argument is deprecated and will be removed in a future version, use `past_key_values` instead.", - FutureWarning, - ) - past_key_values = unused.pop("decoder_cached_states") - if "decoder_past_key_values" in unused: - warnings.warn( - "The `decoder_past_key_values` argument is deprecated and will be removed in a future version, use `past_key_values` instead.", - FutureWarning, - ) - past_key_values = unused.pop("decoder_past_key_values") return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: From ab78042a0965952fc071152cee823988cdf3ff91 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 12:34:13 -0700 Subject: [PATCH 061/109] update instructions --- ...onvert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 62ac3514f755..cae381d788cf 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -58,12 +58,13 @@ cd - # if updating just small files and not the large models, here is a script to generate the right commands: -perl -le 'for $f (@ARGV) { print qq[yes Y | transformers-cli upload $_/$f --filename $_/$f] for map { "fsmt-wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json +perl -le 'for $f (@ARGV) { print qq[yes Y | transformers-cli upload $_/$f --filename $_/$f] for map { "fsmt-wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json # add/remove files as needed -# force cache invalidation, which will now download the new models -# XXX: this doesn't work: -PYTHONPATH="src" python -c 'from transformers import AutoModel; [AutoModel.from_pretrained("stas/fsmt-wmt19-"+p, use_cdn=False) for p in ["en-ru","ru-en","en-de","de-en"]]' +# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload +# So the only way to start using the new model sooner is either: +# 1. download it to a local path and use that path as model_name +# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere # happy translations From 97652745c014f19d737afbcda554cd8b938c82b5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 12:39:26 -0700 Subject: [PATCH 062/109] clean up --- src/transformers/modeling_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e3ec1fd1516a..964461d8d1cf 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -582,7 +582,6 @@ def resize_token_embeddings(self, new_num_tokens: Optional[int] = None) -> torch return model_embeds # Update base model and current model config - # XXX: now we have src_vocab_size/tgt_vocab_size self.config.vocab_size = new_num_tokens base_model.vocab_size = new_num_tokens From 661b7fd30d0625560e0e2fe7050eaac4f5f464b8 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 12:56:10 -0700 Subject: [PATCH 063/109] move bleu eval to examples --- examples/seq2seq/test_fsmt_bleu_score.py | 184 +++++++++++++++++++++++ src/transformers/modeling_fsmt.py | 9 +- tests/test_modeling_fsmt.py | 159 -------------------- 3 files changed, 190 insertions(+), 162 deletions(-) create mode 100644 examples/seq2seq/test_fsmt_bleu_score.py diff --git a/examples/seq2seq/test_fsmt_bleu_score.py b/examples/seq2seq/test_fsmt_bleu_score.py new file mode 100644 index 000000000000..d3ba075ecefb --- /dev/null +++ b/examples/seq2seq/test_fsmt_bleu_score.py @@ -0,0 +1,184 @@ +# coding=utf-8 +# Copyright 2020 Huggingface +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from sacrebleu import corpus_bleu + +from parameterized import parameterized +from transformers import FSMTForConditionalGeneration, FSMTTokenizer +from transformers.testing_utils import require_torch, slow, torch_device + + +def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: + """Uses sacrebleu's corpus_bleu implementation.""" + return {"bleu": round(corpus_bleu(output_lns, [refs_lns], **kwargs).score, 4)} + + +# BLEU eval data was generated using the following code: +# +# #!/bin/bash +# +# export OBJS=8 +# +# pairs=(ru-en en-ru en-de de-en) +# printf "data = {\n" +# for pair in "${pairs[@]}" +# do +# export PAIR=$pair +# printf " \"$PAIR\": {\n" +# printf " \"src\": [\n" +# sacrebleu -t wmt19 -l $PAIR --echo src | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' +# printf " ],\n" +# printf " \"tgt\": [\n" +# sacrebleu -t wmt19 -l $PAIR --echo ref | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' +# printf " ],\n" +# printf " },\n" +# done +# printf "}\n" + +bleu_data = { + "ru-en": { + "src": [ + """Названо число готовящихся к отправке в Донбасс новобранцев из Украины""", + """Официальный представитель Народной милиции самопровозглашенной Луганской Народной Республики (ЛНР) Андрей Марочко заявил, что зимой 2018-2019 года Украина направит в Донбасс не менее 3 тыс. новобранцев.""", + """По его словам, таким образом Киев планирует "хоть как-то доукомплектовать подразделения".""", + """\"Нежелание граждан Украины проходить службу в рядах ВС Украины, массовые увольнения привели к низкой укомплектованности подразделений", - рассказал Марочко, которого цитирует "РИА Новости".""", + """Он также не исключил, что реальные цифры призванных в армию украинцев могут быть увеличены в случае необходимости.""", + """В 2014-2017 годах Киев начал так называемую антитеррористическую операцию (АТО), которую позже сменили на операцию объединенных сил (ООС).""", + """Предполагалось, что эта мера приведет к усилению роли украинских силовиков в урегулировании ситуации.""", + """В конце августа 2018 года ситуация в Донбассе обострилась из-за убийства главы ДНР Александра Захарченко.""", + ], + "tgt": [ + """The number of new Ukrainian recruits ready to go to Donbass has become public""", + """Official representative of the peoples’ militia of the self-proclaimed Lugansk People’s Republic Andrey Marochko claimed that Ukrainian will send at least 3 thousand new recruits to Donbass in winter 2018-2019.""", + """This is how Kyiv tries “at least somehow to staff the units,” he said.""", + """“The unwillingness of Ukrainian citizens to serve in the Ukraine’s military forces, mass resignments lead to low understaffing,” said Marochko cited by RIA Novosti.""", + """Also, he doesn’t exclude that the real numbers of conscripts in the Ukrainian army can be raised is necessary.""", + """In 2014-2017, Kyiv started so-called antiterrorist operation, that ws later changed to the united forces operation.""", + """This measure was supposed to strengthen the role of the Ukrainian military in settling the situation.""", + """In the late August 2018, the situation in Donbass escalated as the DNR head Aleksandr Zakharchenko was killed.""", + ], + }, + "en-ru": { + "src": [ + """Welsh AMs worried about 'looking like muppets'""", + """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", + """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", + """AMs across the political spectrum are worried it could invite ridicule.""", + """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", + """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", + """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", + """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", + ], + "tgt": [ + """Члены Национальной ассамблеи Уэльса обеспокоены, что "выглядят как куклы\"""", + """Некоторые члены Национальной ассамблеи Уэльса в ужасе от предложения о том, что их наименование должно измениться на MPW (члены Парламента Уэльса).""", + """Этот вопрос был поднят в связи с планами по переименованию ассамблеи в Парламент Уэльса.""", + """Члены Национальной ассамблеи Уэльса всего политического спектра обеспокоены, что это может породить насмешки.""", + """Один из лейбористских членов Национальной ассамблеи Уэльса сказал, что его партия обеспокоена тем, что "это рифмуется с Twp и Pwp".""", + """Для читателей за предлами Уэльса: по-валлийски twp означает "глупый", а pwp означает "какашка".""", + """Член Национальной ассамблеи от Плайд сказал, что эта партия в целом "не счастлива" и предложил альтернативы.""", + """Представитель Консервативной партии Уэльса сказал, что его партия "открыта" к переименованию, но отметил, что между WMP и Muppet небольшая разница в произношении.""", + ], + }, + "en-de": { + "src": [ + """Welsh AMs worried about 'looking like muppets'""", + """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", + """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", + """AMs across the political spectrum are worried it could invite ridicule.""", + """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", + """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", + """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", + """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", + ], + "tgt": [ + """Walisische Ageordnete sorgen sich "wie Dödel auszusehen\"""", + """Es herrscht Bestürzung unter einigen Mitgliedern der Versammlung über einen Vorschlag, der ihren Titel zu MWPs (Mitglied der walisischen Parlament) ändern soll.""", + """Der Grund dafür waren Pläne, den Namen der Nationalversammlung in Walisisches Parlament zu ändern.""", + """Mitglieder aller Parteien der Nationalversammlung haben Bedenken, dass sie sich dadurch Spott aussetzen könnten.""", + """Ein Labour-Abgeordneter sagte, dass seine Gruppe "sich mit Twp und Pwp reimt".""", + """Hinweis für den Leser: „twp“ im Walisischen bedeutet „bescheuert“ und „pwp“ bedeutet „Kacke“.""", + """Ein Versammlungsmitglied von Plaid Cymru sagte, die Gruppe als Ganzes sei "nicht glücklich" und hat Alternativen vorgeschlagen.""", + """Ein walisischer Konservativer sagte, seine Gruppe wäre „offen“ für eine Namensänderung, wies aber darauf hin, dass es von „MWP“ (Mitglied des Walisischen Parlaments) nur ein kurzer verbaler Sprung zu „Muppet“ ist.""", + ], + }, + "de-en": { + "src": [ + """Schöne Münchnerin 2018: Schöne Münchnerin 2018 in Hvar: Neun Dates""", + """Von az, aktualisiert am 04.05.2018 um 11:11""", + """Ja, sie will...""", + """\"Schöne Münchnerin" 2018 werden!""", + """Am Nachmittag wartet erneut eine Überraschung auf unsere Kandidatinnen: sie werden das romantische Candlelight-Shooting vor der MY SOLARIS nicht alleine bestreiten, sondern an der Seite von Male-Model Fabian!""", + """Hvar - Flirten, kokettieren, verführen - keine einfachen Aufgaben für unsere Mädchen.""", + """Insbesondere dann, wenn in Deutschland ein Freund wartet.""", + """Dennoch liefern die neun "Schöne Münchnerin"-Kandidatinnen beim Shooting mit People-Fotograf Tuan ab und trotzen Wind, Gischt und Regen wie echte Profis.""", + ], + "tgt": [ + """The Beauty of Munich 2018: the Beauty of Munich 2018 in Hvar: Nine dates""", + """From A-Z, updated on 04/05/2018 at 11:11""", + """Yes, she wants to...""", + """to become "The Beauty of Munich" in 2018!""", + """In the afternoon there is another surprise waiting for our contestants: they will be competing for the romantic candlelight photo shoot at MY SOLARIS not alone, but together with a male-model Fabian!""", + """Hvar with its flirting, coquetting, and seduction is not an easy task for our girls.""", + """Especially when there is a boyfriend waiting in Germany.""", + """Despite dealing with wind, sprays and rain, the nine contestants of "The Beauty of Munich" behaved like real professionals at the photo shoot with People-photographer Tuan.""", + ], + }, +} + + +@require_torch +class ModelTester(unittest.TestCase): + def get_tokenizer(self, mname): + return FSMTTokenizer.from_pretrained(mname) + + def get_model(self, mname): + model = FSMTForConditionalGeneration.from_pretrained(mname).to(torch_device) + if torch_device == "cuda": + model.half() + return model + + @parameterized.expand( + [ + ["en-ru", 28.21], + ["ru-en", 23.49], + ["en-de", 22.11], + ["de-en", 29.31], + ] + ) + @slow + def test_bleu_scores(self, pair, min_bleu_score): + # note: this test is not testing the best performance since it only evals a small batch + # but it should be enough to detect a regression in the output quality + mname = f"stas/fsmt-wmt19-{pair}" + tokenizer = self.get_tokenizer(mname) + model = self.get_model(mname) + + src_sentences = bleu_data[pair]["src"] + tgt_sentences = bleu_data[pair]["tgt"] + + batch = tokenizer(src_sentences, return_tensors="pt", truncation=True, padding="longest").to(torch_device) + outputs = model.generate( + input_ids=batch.input_ids, + num_beams=8, + ) + decoded_sentences = tokenizer.batch_decode( + outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) + scores = calculate_bleu(decoded_sentences, tgt_sentences) + print(scores) + self.assertGreaterEqual(scores["bleu"], min_bleu_score) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 3349eded62d0..a4fc74f9622f 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -852,15 +852,18 @@ def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz): return k, v, new_key_padding_mask -def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True): +def get_layer_norm_func(): if torch.cuda.is_available(): try: from apex.normalization import FusedLayerNorm - return FusedLayerNorm(normalized_shape, eps, elementwise_affine) + return FusedLayerNorm except ImportError: pass - return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine) + return torch.nn.LayerNorm + + +LayerNorm = get_layer_norm_func() def fill_with_neg_inf(t): diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 994107207c0c..f9ac1c3ceb09 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -15,7 +15,6 @@ import tempfile import unittest -from unittest import skipIf import timeout_decorator # noqa @@ -28,14 +27,6 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor -try: - from sacrebleu import corpus_bleu - - sacrebleu_is_missing = False -except Exception: - sacrebleu_is_missing = True - - if is_torch_available(): import torch @@ -48,11 +39,6 @@ ) -def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: - """Uses sacrebleu's corpus_bleu implementation.""" - return {"bleu": round(corpus_bleu(output_lns, [refs_lns], **kwargs).score, 4)} - - @require_torch class ModelTester: def __init__( @@ -465,151 +451,6 @@ def test_translation(self, pair): decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) assert decoded == tgt_sentence, f"\n\ngot: {decoded}\nexp: {tgt_sentence}\n" - # BLEU eval data was generated using the following code: - # - # #!/bin/bash - # - # export OBJS=8 - # - # pairs=(ru-en en-ru en-de de-en) - # printf "data = {\n" - # for pair in "${pairs[@]}" - # do - # export PAIR=$pair - # printf " \"$PAIR\": {\n" - # printf " \"src\": [\n" - # sacrebleu -t wmt19 -l $PAIR --echo src | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' - # printf " ],\n" - # printf " \"tgt\": [\n" - # sacrebleu -t wmt19 -l $PAIR --echo ref | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' - # printf " ],\n" - # printf " },\n" - # done - # printf "}\n" - - bleu_data = { - "ru-en": { - "src": [ - """Названо число готовящихся к отправке в Донбасс новобранцев из Украины""", - """Официальный представитель Народной милиции самопровозглашенной Луганской Народной Республики (ЛНР) Андрей Марочко заявил, что зимой 2018-2019 года Украина направит в Донбасс не менее 3 тыс. новобранцев.""", - """По его словам, таким образом Киев планирует "хоть как-то доукомплектовать подразделения".""", - """\"Нежелание граждан Украины проходить службу в рядах ВС Украины, массовые увольнения привели к низкой укомплектованности подразделений", - рассказал Марочко, которого цитирует "РИА Новости".""", - """Он также не исключил, что реальные цифры призванных в армию украинцев могут быть увеличены в случае необходимости.""", - """В 2014-2017 годах Киев начал так называемую антитеррористическую операцию (АТО), которую позже сменили на операцию объединенных сил (ООС).""", - """Предполагалось, что эта мера приведет к усилению роли украинских силовиков в урегулировании ситуации.""", - """В конце августа 2018 года ситуация в Донбассе обострилась из-за убийства главы ДНР Александра Захарченко.""", - ], - "tgt": [ - """The number of new Ukrainian recruits ready to go to Donbass has become public""", - """Official representative of the peoples’ militia of the self-proclaimed Lugansk People’s Republic Andrey Marochko claimed that Ukrainian will send at least 3 thousand new recruits to Donbass in winter 2018-2019.""", - """This is how Kyiv tries “at least somehow to staff the units,” he said.""", - """“The unwillingness of Ukrainian citizens to serve in the Ukraine’s military forces, mass resignments lead to low understaffing,” said Marochko cited by RIA Novosti.""", - """Also, he doesn’t exclude that the real numbers of conscripts in the Ukrainian army can be raised is necessary.""", - """In 2014-2017, Kyiv started so-called antiterrorist operation, that ws later changed to the united forces operation.""", - """This measure was supposed to strengthen the role of the Ukrainian military in settling the situation.""", - """In the late August 2018, the situation in Donbass escalated as the DNR head Aleksandr Zakharchenko was killed.""", - ], - }, - "en-ru": { - "src": [ - """Welsh AMs worried about 'looking like muppets'""", - """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", - """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", - """AMs across the political spectrum are worried it could invite ridicule.""", - """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", - """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", - """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", - """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", - ], - "tgt": [ - """Члены Национальной ассамблеи Уэльса обеспокоены, что "выглядят как куклы\"""", - """Некоторые члены Национальной ассамблеи Уэльса в ужасе от предложения о том, что их наименование должно измениться на MPW (члены Парламента Уэльса).""", - """Этот вопрос был поднят в связи с планами по переименованию ассамблеи в Парламент Уэльса.""", - """Члены Национальной ассамблеи Уэльса всего политического спектра обеспокоены, что это может породить насмешки.""", - """Один из лейбористских членов Национальной ассамблеи Уэльса сказал, что его партия обеспокоена тем, что "это рифмуется с Twp и Pwp".""", - """Для читателей за предлами Уэльса: по-валлийски twp означает "глупый", а pwp означает "какашка".""", - """Член Национальной ассамблеи от Плайд сказал, что эта партия в целом "не счастлива" и предложил альтернативы.""", - """Представитель Консервативной партии Уэльса сказал, что его партия "открыта" к переименованию, но отметил, что между WMP и Muppet небольшая разница в произношении.""", - ], - }, - "en-de": { - "src": [ - """Welsh AMs worried about 'looking like muppets'""", - """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", - """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", - """AMs across the political spectrum are worried it could invite ridicule.""", - """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", - """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", - """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", - """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", - ], - "tgt": [ - """Walisische Ageordnete sorgen sich "wie Dödel auszusehen\"""", - """Es herrscht Bestürzung unter einigen Mitgliedern der Versammlung über einen Vorschlag, der ihren Titel zu MWPs (Mitglied der walisischen Parlament) ändern soll.""", - """Der Grund dafür waren Pläne, den Namen der Nationalversammlung in Walisisches Parlament zu ändern.""", - """Mitglieder aller Parteien der Nationalversammlung haben Bedenken, dass sie sich dadurch Spott aussetzen könnten.""", - """Ein Labour-Abgeordneter sagte, dass seine Gruppe "sich mit Twp und Pwp reimt".""", - """Hinweis für den Leser: „twp“ im Walisischen bedeutet „bescheuert“ und „pwp“ bedeutet „Kacke“.""", - """Ein Versammlungsmitglied von Plaid Cymru sagte, die Gruppe als Ganzes sei "nicht glücklich" und hat Alternativen vorgeschlagen.""", - """Ein walisischer Konservativer sagte, seine Gruppe wäre „offen“ für eine Namensänderung, wies aber darauf hin, dass es von „MWP“ (Mitglied des Walisischen Parlaments) nur ein kurzer verbaler Sprung zu „Muppet“ ist.""", - ], - }, - "de-en": { - "src": [ - """Schöne Münchnerin 2018: Schöne Münchnerin 2018 in Hvar: Neun Dates""", - """Von az, aktualisiert am 04.05.2018 um 11:11""", - """Ja, sie will...""", - """\"Schöne Münchnerin" 2018 werden!""", - """Am Nachmittag wartet erneut eine Überraschung auf unsere Kandidatinnen: sie werden das romantische Candlelight-Shooting vor der MY SOLARIS nicht alleine bestreiten, sondern an der Seite von Male-Model Fabian!""", - """Hvar - Flirten, kokettieren, verführen - keine einfachen Aufgaben für unsere Mädchen.""", - """Insbesondere dann, wenn in Deutschland ein Freund wartet.""", - """Dennoch liefern die neun "Schöne Münchnerin"-Kandidatinnen beim Shooting mit People-Fotograf Tuan ab und trotzen Wind, Gischt und Regen wie echte Profis.""", - ], - "tgt": [ - """The Beauty of Munich 2018: the Beauty of Munich 2018 in Hvar: Nine dates""", - """From A-Z, updated on 04/05/2018 at 11:11""", - """Yes, she wants to...""", - """to become "The Beauty of Munich" in 2018!""", - """In the afternoon there is another surprise waiting for our contestants: they will be competing for the romantic candlelight photo shoot at MY SOLARIS not alone, but together with a male-model Fabian!""", - """Hvar with its flirting, coquetting, and seduction is not an easy task for our girls.""", - """Especially when there is a boyfriend waiting in Germany.""", - """Despite dealing with wind, sprays and rain, the nine contestants of "The Beauty of Munich" behaved like real professionals at the photo shoot with People-photographer Tuan.""", - ], - }, - } - - @parameterized.expand( - [ - ["en-ru", 28.21], - ["ru-en", 23.49], - ["en-de", 22.11], - ["de-en", 29.31], - ] - ) - @slow - @skipIf(sacrebleu_is_missing, "pip install sacrebleu") - def test_bleu_scores(self, pair, min_bleu_score): - # note: this test is not testing the best performance since it only evals a small batch - # but it should be enough to detect a regression in the output quality - mname = f"stas/fsmt-wmt19-{pair}" - tokenizer = self.get_tokenizer(mname) - model = self.get_model(mname) - - src_sentences = self.bleu_data[pair]["src"] - tgt_sentences = self.bleu_data[pair]["tgt"] - - batch = tokenizer(src_sentences, return_tensors="pt", truncation=True, padding="longest").to(torch_device) - outputs = model.generate( - input_ids=batch.input_ids, - num_beams=8, - ) - decoded_sentences = tokenizer.batch_decode( - outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False - ) - scores = calculate_bleu(decoded_sentences, tgt_sentences) - print(scores) - self.assertGreaterEqual(scores["bleu"], min_bleu_score) - @require_torch class TestSinusoidalPositionalEmbeddings(unittest.TestCase): From 000a36b8a128602c6e61d329b6b6f0c97e137f6a Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 15:30:08 -0700 Subject: [PATCH 064/109] check import only once --- src/transformers/modeling_fsmt.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index a4fc74f9622f..1c9b4dde7e8c 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -244,6 +244,18 @@ """ +have_fused_layer_norm = False +if torch.cuda.is_available(): + try: + from apex.normalization import FusedLayerNorm + + have_fused_layer_norm = True + except ImportError: + pass + +LayerNorm = FusedLayerNorm if have_fused_layer_norm else torch.nn.LayerNorm + + def invert_mask(attention_mask): """Turns 1->0, 0->1, False->True, True-> False""" assert attention_mask.dim() == 2 @@ -852,20 +864,6 @@ def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz): return k, v, new_key_padding_mask -def get_layer_norm_func(): - if torch.cuda.is_available(): - try: - from apex.normalization import FusedLayerNorm - - return FusedLayerNorm - except ImportError: - pass - return torch.nn.LayerNorm - - -LayerNorm = get_layer_norm_func() - - def fill_with_neg_inf(t): """FP16-compatible function that fills a input_ids with -inf.""" return t.float().fill_(float("-inf")).type_as(t) From 3a69ca57653b018a631f36157901e11db2f298a9 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 15:30:26 -0700 Subject: [PATCH 065/109] move data+gen script into files --- .../seq2seq/test_data/fsmt/build-eval-data.sh | 25 ++++ .../seq2seq/test_data/fsmt/fsmt_val_data.yaml | 90 +++++++++++++ examples/seq2seq/test_fsmt_bleu_score.py | 119 +----------------- 3 files changed, 121 insertions(+), 113 deletions(-) create mode 100755 examples/seq2seq/test_data/fsmt/build-eval-data.sh create mode 100644 examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml diff --git a/examples/seq2seq/test_data/fsmt/build-eval-data.sh b/examples/seq2seq/test_data/fsmt/build-eval-data.sh new file mode 100755 index 000000000000..e15f43e1dca9 --- /dev/null +++ b/examples/seq2seq/test_data/fsmt/build-eval-data.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# generates bleu eval data +# ./build-eval-data.sh + +export OBJS=8 +pairs=(ru-en en-ru en-de de-en) + +( +printf "{\n" +for pair in "${pairs[@]}" +do + export PAIR=$pair + printf " \"$PAIR\": {\n" + printf " \"src\": [\n" + sacrebleu -t wmt19 -l $PAIR --echo src | head -$OBJS | perl -ne 'chomp; s#"#\\"#g; print qq[ "$_",\n]' + printf " ],\n" + printf " \"tgt\": [\n" + sacrebleu -t wmt19 -l $PAIR --echo ref | head -$OBJS | perl -ne 'chomp; s#"#\\"#g; print qq[ "$_",\n]' + printf " ],\n" + printf " },\n" +done +printf "}\n" + +) > fsmt_val_data.yaml diff --git a/examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml b/examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml new file mode 100644 index 000000000000..fe1c3881f38b --- /dev/null +++ b/examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml @@ -0,0 +1,90 @@ +{ + "ru-en": { + "src": [ + "Названо число готовящихся к отправке в Донбасс новобранцев из Украины", + "Официальный представитель Народной милиции самопровозглашенной Луганской Народной Республики (ЛНР) Андрей Марочко заявил, что зимой 2018-2019 года Украина направит в Донбасс не менее 3 тыс. новобранцев.", + "По его словам, таким образом Киев планирует \"хоть как-то доукомплектовать подразделения\".", + "\"Нежелание граждан Украины проходить службу в рядах ВС Украины, массовые увольнения привели к низкой укомплектованности подразделений\", - рассказал Марочко, которого цитирует \"РИА Новости\".", + "Он также не исключил, что реальные цифры призванных в армию украинцев могут быть увеличены в случае необходимости.", + "В 2014-2017 годах Киев начал так называемую антитеррористическую операцию (АТО), которую позже сменили на операцию объединенных сил (ООС).", + "Предполагалось, что эта мера приведет к усилению роли украинских силовиков в урегулировании ситуации.", + "В конце августа 2018 года ситуация в Донбассе обострилась из-за убийства главы ДНР Александра Захарченко.", + ], + "tgt": [ + "The number of new Ukrainian recruits ready to go to Donbass has become public", + "Official representative of the peoples’ militia of the self-proclaimed Lugansk People’s Republic Andrey Marochko claimed that Ukrainian will send at least 3 thousand new recruits to Donbass in winter 2018-2019.", + "This is how Kyiv tries “at least somehow to staff the units,” he said.", + "“The unwillingness of Ukrainian citizens to serve in the Ukraine’s military forces, mass resignments lead to low understaffing,” said Marochko cited by RIA Novosti.", + "Also, he doesn’t exclude that the real numbers of conscripts in the Ukrainian army can be raised is necessary.", + "In 2014-2017, Kyiv started so-called antiterrorist operation, that ws later changed to the united forces operation.", + "This measure was supposed to strengthen the role of the Ukrainian military in settling the situation.", + "In the late August 2018, the situation in Donbass escalated as the DNR head Aleksandr Zakharchenko was killed.", + ], + }, + "en-ru": { + "src": [ + "Welsh AMs worried about 'looking like muppets'", + "There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).", + "It has arisen because of plans to change the name of the assembly to the Welsh Parliament.", + "AMs across the political spectrum are worried it could invite ridicule.", + "One Labour AM said his group was concerned \"it rhymes with Twp and Pwp.\"", + "For readers outside of Wales: In Welsh twp means daft and pwp means poo.", + "A Plaid AM said the group as a whole was \"not happy\" and has suggested alternatives.", + "A Welsh Conservative said his group was \"open minded\" about the name change, but noted it was a short verbal hop from MWP to Muppet.", + ], + "tgt": [ + "Члены Национальной ассамблеи Уэльса обеспокоены, что \"выглядят как куклы\"", + "Некоторые члены Национальной ассамблеи Уэльса в ужасе от предложения о том, что их наименование должно измениться на MPW (члены Парламента Уэльса).", + "Этот вопрос был поднят в связи с планами по переименованию ассамблеи в Парламент Уэльса.", + "Члены Национальной ассамблеи Уэльса всего политического спектра обеспокоены, что это может породить насмешки.", + "Один из лейбористских членов Национальной ассамблеи Уэльса сказал, что его партия обеспокоена тем, что \"это рифмуется с Twp и Pwp\".", + "Для читателей за предлами Уэльса: по-валлийски twp означает \"глупый\", а pwp означает \"какашка\".", + "Член Национальной ассамблеи от Плайд сказал, что эта партия в целом \"не счастлива\" и предложил альтернативы.", + "Представитель Консервативной партии Уэльса сказал, что его партия \"открыта\" к переименованию, но отметил, что между WMP и Muppet небольшая разница в произношении.", + ], + }, + "en-de": { + "src": [ + "Welsh AMs worried about 'looking like muppets'", + "There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).", + "It has arisen because of plans to change the name of the assembly to the Welsh Parliament.", + "AMs across the political spectrum are worried it could invite ridicule.", + "One Labour AM said his group was concerned \"it rhymes with Twp and Pwp.\"", + "For readers outside of Wales: In Welsh twp means daft and pwp means poo.", + "A Plaid AM said the group as a whole was \"not happy\" and has suggested alternatives.", + "A Welsh Conservative said his group was \"open minded\" about the name change, but noted it was a short verbal hop from MWP to Muppet.", + ], + "tgt": [ + "Walisische Ageordnete sorgen sich \"wie Dödel auszusehen\"", + "Es herrscht Bestürzung unter einigen Mitgliedern der Versammlung über einen Vorschlag, der ihren Titel zu MWPs (Mitglied der walisischen Parlament) ändern soll.", + "Der Grund dafür waren Pläne, den Namen der Nationalversammlung in Walisisches Parlament zu ändern.", + "Mitglieder aller Parteien der Nationalversammlung haben Bedenken, dass sie sich dadurch Spott aussetzen könnten.", + "Ein Labour-Abgeordneter sagte, dass seine Gruppe \"sich mit Twp und Pwp reimt\".", + "Hinweis für den Leser: „twp“ im Walisischen bedeutet „bescheuert“ und „pwp“ bedeutet „Kacke“.", + "Ein Versammlungsmitglied von Plaid Cymru sagte, die Gruppe als Ganzes sei \"nicht glücklich\" und hat Alternativen vorgeschlagen.", + "Ein walisischer Konservativer sagte, seine Gruppe wäre „offen“ für eine Namensänderung, wies aber darauf hin, dass es von „MWP“ (Mitglied des Walisischen Parlaments) nur ein kurzer verbaler Sprung zu „Muppet“ ist.", + ], + }, + "de-en": { + "src": [ + "Schöne Münchnerin 2018: Schöne Münchnerin 2018 in Hvar: Neun Dates", + "Von az, aktualisiert am 04.05.2018 um 11:11", + "Ja, sie will...", + "\"Schöne Münchnerin\" 2018 werden!", + "Am Nachmittag wartet erneut eine Überraschung auf unsere Kandidatinnen: sie werden das romantische Candlelight-Shooting vor der MY SOLARIS nicht alleine bestreiten, sondern an der Seite von Male-Model Fabian!", + "Hvar - Flirten, kokettieren, verführen - keine einfachen Aufgaben für unsere Mädchen.", + "Insbesondere dann, wenn in Deutschland ein Freund wartet.", + "Dennoch liefern die neun \"Schöne Münchnerin\"-Kandidatinnen beim Shooting mit People-Fotograf Tuan ab und trotzen Wind, Gischt und Regen wie echte Profis.", + ], + "tgt": [ + "The Beauty of Munich 2018: the Beauty of Munich 2018 in Hvar: Nine dates", + "From A-Z, updated on 04/05/2018 at 11:11", + "Yes, she wants to...", + "to become \"The Beauty of Munich\" in 2018!", + "In the afternoon there is another surprise waiting for our contestants: they will be competing for the romantic candlelight photo shoot at MY SOLARIS not alone, but together with a male-model Fabian!", + "Hvar with its flirting, coquetting, and seduction is not an easy task for our girls.", + "Especially when there is a boyfriend waiting in Germany.", + "Despite dealing with wind, sprays and rain, the nine contestants of \"The Beauty of Munich\" behaved like real professionals at the photo shoot with People-photographer Tuan.", + ], + }, +} diff --git a/examples/seq2seq/test_fsmt_bleu_score.py b/examples/seq2seq/test_fsmt_bleu_score.py index d3ba075ecefb..42cd815450bf 100644 --- a/examples/seq2seq/test_fsmt_bleu_score.py +++ b/examples/seq2seq/test_fsmt_bleu_score.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import unittest from sacrebleu import corpus_bleu +import yaml from parameterized import parameterized from transformers import FSMTForConditionalGeneration, FSMTTokenizer -from transformers.testing_utils import require_torch, slow, torch_device +from transformers.testing_utils import get_tests_dir, require_torch, slow, torch_device def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: @@ -27,118 +29,9 @@ def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: return {"bleu": round(corpus_bleu(output_lns, [refs_lns], **kwargs).score, 4)} -# BLEU eval data was generated using the following code: -# -# #!/bin/bash -# -# export OBJS=8 -# -# pairs=(ru-en en-ru en-de de-en) -# printf "data = {\n" -# for pair in "${pairs[@]}" -# do -# export PAIR=$pair -# printf " \"$PAIR\": {\n" -# printf " \"src\": [\n" -# sacrebleu -t wmt19 -l $PAIR --echo src | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' -# printf " ],\n" -# printf " \"tgt\": [\n" -# sacrebleu -t wmt19 -l $PAIR --echo ref | head -$OBJS | perl -ne 'chomp; s#(^"|"$)#\\"#; print qq[ """$_""",\n]' -# printf " ],\n" -# printf " },\n" -# done -# printf "}\n" - -bleu_data = { - "ru-en": { - "src": [ - """Названо число готовящихся к отправке в Донбасс новобранцев из Украины""", - """Официальный представитель Народной милиции самопровозглашенной Луганской Народной Республики (ЛНР) Андрей Марочко заявил, что зимой 2018-2019 года Украина направит в Донбасс не менее 3 тыс. новобранцев.""", - """По его словам, таким образом Киев планирует "хоть как-то доукомплектовать подразделения".""", - """\"Нежелание граждан Украины проходить службу в рядах ВС Украины, массовые увольнения привели к низкой укомплектованности подразделений", - рассказал Марочко, которого цитирует "РИА Новости".""", - """Он также не исключил, что реальные цифры призванных в армию украинцев могут быть увеличены в случае необходимости.""", - """В 2014-2017 годах Киев начал так называемую антитеррористическую операцию (АТО), которую позже сменили на операцию объединенных сил (ООС).""", - """Предполагалось, что эта мера приведет к усилению роли украинских силовиков в урегулировании ситуации.""", - """В конце августа 2018 года ситуация в Донбассе обострилась из-за убийства главы ДНР Александра Захарченко.""", - ], - "tgt": [ - """The number of new Ukrainian recruits ready to go to Donbass has become public""", - """Official representative of the peoples’ militia of the self-proclaimed Lugansk People’s Republic Andrey Marochko claimed that Ukrainian will send at least 3 thousand new recruits to Donbass in winter 2018-2019.""", - """This is how Kyiv tries “at least somehow to staff the units,” he said.""", - """“The unwillingness of Ukrainian citizens to serve in the Ukraine’s military forces, mass resignments lead to low understaffing,” said Marochko cited by RIA Novosti.""", - """Also, he doesn’t exclude that the real numbers of conscripts in the Ukrainian army can be raised is necessary.""", - """In 2014-2017, Kyiv started so-called antiterrorist operation, that ws later changed to the united forces operation.""", - """This measure was supposed to strengthen the role of the Ukrainian military in settling the situation.""", - """In the late August 2018, the situation in Donbass escalated as the DNR head Aleksandr Zakharchenko was killed.""", - ], - }, - "en-ru": { - "src": [ - """Welsh AMs worried about 'looking like muppets'""", - """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", - """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", - """AMs across the political spectrum are worried it could invite ridicule.""", - """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", - """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", - """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", - """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", - ], - "tgt": [ - """Члены Национальной ассамблеи Уэльса обеспокоены, что "выглядят как куклы\"""", - """Некоторые члены Национальной ассамблеи Уэльса в ужасе от предложения о том, что их наименование должно измениться на MPW (члены Парламента Уэльса).""", - """Этот вопрос был поднят в связи с планами по переименованию ассамблеи в Парламент Уэльса.""", - """Члены Национальной ассамблеи Уэльса всего политического спектра обеспокоены, что это может породить насмешки.""", - """Один из лейбористских членов Национальной ассамблеи Уэльса сказал, что его партия обеспокоена тем, что "это рифмуется с Twp и Pwp".""", - """Для читателей за предлами Уэльса: по-валлийски twp означает "глупый", а pwp означает "какашка".""", - """Член Национальной ассамблеи от Плайд сказал, что эта партия в целом "не счастлива" и предложил альтернативы.""", - """Представитель Консервативной партии Уэльса сказал, что его партия "открыта" к переименованию, но отметил, что между WMP и Muppet небольшая разница в произношении.""", - ], - }, - "en-de": { - "src": [ - """Welsh AMs worried about 'looking like muppets'""", - """There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).""", - """It has arisen because of plans to change the name of the assembly to the Welsh Parliament.""", - """AMs across the political spectrum are worried it could invite ridicule.""", - """One Labour AM said his group was concerned "it rhymes with Twp and Pwp.\"""", - """For readers outside of Wales: In Welsh twp means daft and pwp means poo.""", - """A Plaid AM said the group as a whole was "not happy" and has suggested alternatives.""", - """A Welsh Conservative said his group was "open minded" about the name change, but noted it was a short verbal hop from MWP to Muppet.""", - ], - "tgt": [ - """Walisische Ageordnete sorgen sich "wie Dödel auszusehen\"""", - """Es herrscht Bestürzung unter einigen Mitgliedern der Versammlung über einen Vorschlag, der ihren Titel zu MWPs (Mitglied der walisischen Parlament) ändern soll.""", - """Der Grund dafür waren Pläne, den Namen der Nationalversammlung in Walisisches Parlament zu ändern.""", - """Mitglieder aller Parteien der Nationalversammlung haben Bedenken, dass sie sich dadurch Spott aussetzen könnten.""", - """Ein Labour-Abgeordneter sagte, dass seine Gruppe "sich mit Twp und Pwp reimt".""", - """Hinweis für den Leser: „twp“ im Walisischen bedeutet „bescheuert“ und „pwp“ bedeutet „Kacke“.""", - """Ein Versammlungsmitglied von Plaid Cymru sagte, die Gruppe als Ganzes sei "nicht glücklich" und hat Alternativen vorgeschlagen.""", - """Ein walisischer Konservativer sagte, seine Gruppe wäre „offen“ für eine Namensänderung, wies aber darauf hin, dass es von „MWP“ (Mitglied des Walisischen Parlaments) nur ein kurzer verbaler Sprung zu „Muppet“ ist.""", - ], - }, - "de-en": { - "src": [ - """Schöne Münchnerin 2018: Schöne Münchnerin 2018 in Hvar: Neun Dates""", - """Von az, aktualisiert am 04.05.2018 um 11:11""", - """Ja, sie will...""", - """\"Schöne Münchnerin" 2018 werden!""", - """Am Nachmittag wartet erneut eine Überraschung auf unsere Kandidatinnen: sie werden das romantische Candlelight-Shooting vor der MY SOLARIS nicht alleine bestreiten, sondern an der Seite von Male-Model Fabian!""", - """Hvar - Flirten, kokettieren, verführen - keine einfachen Aufgaben für unsere Mädchen.""", - """Insbesondere dann, wenn in Deutschland ein Freund wartet.""", - """Dennoch liefern die neun "Schöne Münchnerin"-Kandidatinnen beim Shooting mit People-Fotograf Tuan ab und trotzen Wind, Gischt und Regen wie echte Profis.""", - ], - "tgt": [ - """The Beauty of Munich 2018: the Beauty of Munich 2018 in Hvar: Nine dates""", - """From A-Z, updated on 04/05/2018 at 11:11""", - """Yes, she wants to...""", - """to become "The Beauty of Munich" in 2018!""", - """In the afternoon there is another surprise waiting for our contestants: they will be competing for the romantic candlelight photo shoot at MY SOLARIS not alone, but together with a male-model Fabian!""", - """Hvar with its flirting, coquetting, and seduction is not an easy task for our girls.""", - """Especially when there is a boyfriend waiting in Germany.""", - """Despite dealing with wind, sprays and rain, the nine contestants of "The Beauty of Munich" behaved like real professionals at the photo shoot with People-photographer Tuan.""", - ], - }, -} +filename = get_tests_dir() + "/test_data/fsmt/fsmt_val_data.yaml" +with io.open(filename, "r", encoding="utf-8") as f: + bleu_data = yaml.load(f) @require_torch From 08bbda6f14989e25d3ac2f5d1bc0e1e5dc95f26d Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 15:46:49 -0700 Subject: [PATCH 066/109] reuse via import --- examples/seq2seq/test_fsmt_bleu_score.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/seq2seq/test_fsmt_bleu_score.py b/examples/seq2seq/test_fsmt_bleu_score.py index 42cd815450bf..8981a05e4cdf 100644 --- a/examples/seq2seq/test_fsmt_bleu_score.py +++ b/examples/seq2seq/test_fsmt_bleu_score.py @@ -16,7 +16,11 @@ import io import unittest -from sacrebleu import corpus_bleu + +try: + from .utils import calculate_bleu +except ImportError: + from utils import calculate_bleu import yaml from parameterized import parameterized @@ -24,11 +28,6 @@ from transformers.testing_utils import get_tests_dir, require_torch, slow, torch_device -def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict: - """Uses sacrebleu's corpus_bleu implementation.""" - return {"bleu": round(corpus_bleu(output_lns, [refs_lns], **kwargs).score, 4)} - - filename = get_tests_dir() + "/test_data/fsmt/fsmt_val_data.yaml" with io.open(filename, "r", encoding="utf-8") as f: bleu_data = yaml.load(f) From 97975f5a3d04b9e253716a4b916c031829e46b82 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 15:47:15 -0700 Subject: [PATCH 067/109] take less space --- tests/test_tokenization_fsmt.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py index 2f6deb9e35c2..0760f6fb630a 100644 --- a/tests/test_tokenization_fsmt.py +++ b/tests/test_tokenization_fsmt.py @@ -124,18 +124,10 @@ def test_match_encode_decode(self): ["This is it. No more. I'm done!", [132, 21, 37, 7, 1434, 86, 7, 70, 6476, 1305, 427, 2]], ] - # this data was added as different mismatches were found, to validate - # the targets (or create more inputs if problems are found) run: - # - # import torch + # if data needs to be recreated, uncomment and run: # for src_text, _ in targets: - # mname = "transformer.wmt19.en-ru" - # checkpoint_file = "model1.pt" - # model = torch.hub.load( - # "pytorch/fairseq", mname, checkpoint_file=checkpoint_file, tokenizer="moses", bpe="fastbpe" - # ) - # encoded = model.encode(src_text) - # print(f"""[\n"{src_text}",\n {encoded.tolist()}\n],""") + # model = torch.hub.load("pytorch/fairseq", "transformer.wmt19.en-ru", checkpoint_file="model4.pt", tokenizer="moses", bpe="fastbpe") + # print(f"""[\n"{src_text}",\n {model.encode(src_text).tolist()}\n],""") for src_text, tgt_input_ids in targets: input_ids = tokenizer_enc.encode(src_text, return_tensors="pt")[0].tolist() From df2d8082de5b1769472bd8f8c21be147e375c031 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 16:05:45 -0700 Subject: [PATCH 068/109] add prepare_seq2seq_batch (auto-tested) --- src/transformers/tokenization_fsmt.py | 39 ++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 80c1f01acf79..eda1d663d6ba 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -24,7 +24,9 @@ import sacremoses as sm -from .tokenization_utils import PreTrainedTokenizer +from .file_utils import add_start_docstrings_to_callable +from .tokenization_utils import BatchEncoding, PreTrainedTokenizer +from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING logger = logging.getLogger(__name__) @@ -495,6 +497,41 @@ def create_token_type_ids_from_sequences( return len(token_ids_0 + sep) * [0] return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + @add_start_docstrings_to_callable(PREPARE_SEQ2SEQ_BATCH_DOCSTRING) + def prepare_seq2seq_batch( + self, + src_texts: List[str], + tgt_texts: Optional[List[str]] = None, + max_length: Optional[int] = None, + max_target_length: Optional[int] = None, + return_tensors: str = "pt", + truncation=True, + padding="longest", + **unused, + ) -> BatchEncoding: + """Prepare model inputs for translation. For best performance, translate one sentence at a time.""" + if "" in src_texts: + raise ValueError(f"found empty string in src_texts: {src_texts}") + tokenizer_kwargs = dict( + add_special_tokens=True, + return_tensors=return_tensors, + max_length=max_length, + truncation=truncation, + padding=padding, + ) + model_inputs: BatchEncoding = self(src_texts, **tokenizer_kwargs) + + if tgt_texts is None: + return model_inputs + if max_target_length is not None: + tokenizer_kwargs["max_length"] = max_target_length + + if max_target_length is not None: + tokenizer_kwargs["max_length"] = max_target_length + + model_inputs["labels"] = self(tgt_texts, **tokenizer_kwargs)["input_ids"] + return model_inputs + def save_vocabulary(self, save_directory): """ Save the vocabulary and special tokens file to a directory. From 4347db65cfec9fe9febcae3679bd3034c5bb267d Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 17:54:59 -0700 Subject: [PATCH 069/109] cleanup --- tests/test_tokenization_fsmt.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py index 0760f6fb630a..276e0d1d4ee1 100644 --- a/tests/test_tokenization_fsmt.py +++ b/tests/test_tokenization_fsmt.py @@ -124,15 +124,13 @@ def test_match_encode_decode(self): ["This is it. No more. I'm done!", [132, 21, 37, 7, 1434, 86, 7, 70, 6476, 1305, 427, 2]], ] - # if data needs to be recreated, uncomment and run: - # for src_text, _ in targets: - # model = torch.hub.load("pytorch/fairseq", "transformer.wmt19.en-ru", checkpoint_file="model4.pt", tokenizer="moses", bpe="fastbpe") - # print(f"""[\n"{src_text}",\n {model.encode(src_text).tolist()}\n],""") + # if data needs to be recreated or added, run: + # import torch + # model = torch.hub.load("pytorch/fairseq", "transformer.wmt19.en-ru", checkpoint_file="model4.pt", tokenizer="moses", bpe="fastbpe") + # for src_text, _ in targets: print(f"""[\n"{src_text}",\n {model.encode(src_text).tolist()}\n],""") for src_text, tgt_input_ids in targets: input_ids = tokenizer_enc.encode(src_text, return_tensors="pt")[0].tolist() - print(input_ids) - print(tgt_input_ids) self.assertListEqual(input_ids, tgt_input_ids) # and decode backward, using the reversed languages model From e82832c497a83089c4af566202d4687f741304f2 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 8 Sep 2020 20:25:51 -0700 Subject: [PATCH 070/109] recode test to use json instead of yaml --- .../seq2seq/test_data/fsmt/build-eval-data.py | 33 +++++++ .../seq2seq/test_data/fsmt/build-eval-data.sh | 25 ------ .../seq2seq/test_data/fsmt/fsmt_val_data.json | 90 +++++++++++++++++++ .../seq2seq/test_data/fsmt/fsmt_val_data.yaml | 90 ------------------- examples/seq2seq/test_fsmt_bleu_score.py | 9 +- 5 files changed, 128 insertions(+), 119 deletions(-) create mode 100755 examples/seq2seq/test_data/fsmt/build-eval-data.py delete mode 100755 examples/seq2seq/test_data/fsmt/build-eval-data.sh create mode 100644 examples/seq2seq/test_data/fsmt/fsmt_val_data.json delete mode 100644 examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml diff --git a/examples/seq2seq/test_data/fsmt/build-eval-data.py b/examples/seq2seq/test_data/fsmt/build-eval-data.py new file mode 100755 index 000000000000..46487c07ea84 --- /dev/null +++ b/examples/seq2seq/test_data/fsmt/build-eval-data.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import io +import json +import subprocess + + +pairs = [ + ["en", "ru"], + ["ru", "en"], + ["en", "de"], + ["de", "en"], +] + +n_objs = 8 + + +def get_all_data(pairs, n_objs): + text = {} + for src, tgt in pairs: + pair = f"{src}-{tgt}" + cmd = f"sacrebleu -t wmt19 -l {pair} --echo src".split() + src_lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines() + cmd = f"sacrebleu -t wmt19 -l {pair} --echo ref".split() + tgt_lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines() + text[pair] = {"src": src_lines[:n_objs], "tgt": tgt_lines[:n_objs]} + return text + + +text = get_all_data(pairs, n_objs) +filename = "./fsmt_val_data.json" +with io.open(filename, "w", encoding="utf-8") as f: + bleu_data = json.dump(text, f, indent=2, ensure_ascii=False) diff --git a/examples/seq2seq/test_data/fsmt/build-eval-data.sh b/examples/seq2seq/test_data/fsmt/build-eval-data.sh deleted file mode 100755 index e15f43e1dca9..000000000000 --- a/examples/seq2seq/test_data/fsmt/build-eval-data.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# generates bleu eval data -# ./build-eval-data.sh - -export OBJS=8 -pairs=(ru-en en-ru en-de de-en) - -( -printf "{\n" -for pair in "${pairs[@]}" -do - export PAIR=$pair - printf " \"$PAIR\": {\n" - printf " \"src\": [\n" - sacrebleu -t wmt19 -l $PAIR --echo src | head -$OBJS | perl -ne 'chomp; s#"#\\"#g; print qq[ "$_",\n]' - printf " ],\n" - printf " \"tgt\": [\n" - sacrebleu -t wmt19 -l $PAIR --echo ref | head -$OBJS | perl -ne 'chomp; s#"#\\"#g; print qq[ "$_",\n]' - printf " ],\n" - printf " },\n" -done -printf "}\n" - -) > fsmt_val_data.yaml diff --git a/examples/seq2seq/test_data/fsmt/fsmt_val_data.json b/examples/seq2seq/test_data/fsmt/fsmt_val_data.json new file mode 100644 index 000000000000..f38b30573331 --- /dev/null +++ b/examples/seq2seq/test_data/fsmt/fsmt_val_data.json @@ -0,0 +1,90 @@ +{ + "en-ru": { + "src": [ + "Welsh AMs worried about 'looking like muppets'", + "There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).", + "It has arisen because of plans to change the name of the assembly to the Welsh Parliament.", + "AMs across the political spectrum are worried it could invite ridicule.", + "One Labour AM said his group was concerned \"it rhymes with Twp and Pwp.\"", + "For readers outside of Wales: In Welsh twp means daft and pwp means poo.", + "A Plaid AM said the group as a whole was \"not happy\" and has suggested alternatives.", + "A Welsh Conservative said his group was \"open minded\" about the name change, but noted it was a short verbal hop from MWP to Muppet." + ], + "tgt": [ + "Члены Национальной ассамблеи Уэльса обеспокоены, что \"выглядят как куклы\"", + "Некоторые члены Национальной ассамблеи Уэльса в ужасе от предложения о том, что их наименование должно измениться на MPW (члены Парламента Уэльса).", + "Этот вопрос был поднят в связи с планами по переименованию ассамблеи в Парламент Уэльса.", + "Члены Национальной ассамблеи Уэльса всего политического спектра обеспокоены, что это может породить насмешки.", + "Один из лейбористских членов Национальной ассамблеи Уэльса сказал, что его партия обеспокоена тем, что \"это рифмуется с Twp и Pwp\".", + "Для читателей за предлами Уэльса: по-валлийски twp означает \"глупый\", а pwp означает \"какашка\".", + "Член Национальной ассамблеи от Плайд сказал, что эта партия в целом \"не счастлива\" и предложил альтернативы.", + "Представитель Консервативной партии Уэльса сказал, что его партия \"открыта\" к переименованию, но отметил, что между WMP и Muppet небольшая разница в произношении." + ] + }, + "ru-en": { + "src": [ + "Названо число готовящихся к отправке в Донбасс новобранцев из Украины", + "Официальный представитель Народной милиции самопровозглашенной Луганской Народной Республики (ЛНР) Андрей Марочко заявил, что зимой 2018-2019 года Украина направит в Донбасс не менее 3 тыс. новобранцев.", + "По его словам, таким образом Киев планирует \"хоть как-то доукомплектовать подразделения\".", + "\"Нежелание граждан Украины проходить службу в рядах ВС Украины, массовые увольнения привели к низкой укомплектованности подразделений\", - рассказал Марочко, которого цитирует \"РИА Новости\".", + "Он также не исключил, что реальные цифры призванных в армию украинцев могут быть увеличены в случае необходимости.", + "В 2014-2017 годах Киев начал так называемую антитеррористическую операцию (АТО), которую позже сменили на операцию объединенных сил (ООС).", + "Предполагалось, что эта мера приведет к усилению роли украинских силовиков в урегулировании ситуации.", + "В конце августа 2018 года ситуация в Донбассе обострилась из-за убийства главы ДНР Александра Захарченко." + ], + "tgt": [ + "The number of new Ukrainian recruits ready to go to Donbass has become public", + "Official representative of the peoples’ militia of the self-proclaimed Lugansk People’s Republic Andrey Marochko claimed that Ukrainian will send at least 3 thousand new recruits to Donbass in winter 2018-2019.", + "This is how Kyiv tries “at least somehow to staff the units,” he said.", + "“The unwillingness of Ukrainian citizens to serve in the Ukraine’s military forces, mass resignments lead to low understaffing,” said Marochko cited by RIA Novosti.", + "Also, he doesn’t exclude that the real numbers of conscripts in the Ukrainian army can be raised is necessary.", + "In 2014-2017, Kyiv started so-called antiterrorist operation, that ws later changed to the united forces operation.", + "This measure was supposed to strengthen the role of the Ukrainian military in settling the situation.", + "In the late August 2018, the situation in Donbass escalated as the DNR head Aleksandr Zakharchenko was killed." + ] + }, + "en-de": { + "src": [ + "Welsh AMs worried about 'looking like muppets'", + "There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).", + "It has arisen because of plans to change the name of the assembly to the Welsh Parliament.", + "AMs across the political spectrum are worried it could invite ridicule.", + "One Labour AM said his group was concerned \"it rhymes with Twp and Pwp.\"", + "For readers outside of Wales: In Welsh twp means daft and pwp means poo.", + "A Plaid AM said the group as a whole was \"not happy\" and has suggested alternatives.", + "A Welsh Conservative said his group was \"open minded\" about the name change, but noted it was a short verbal hop from MWP to Muppet." + ], + "tgt": [ + "Walisische Ageordnete sorgen sich \"wie Dödel auszusehen\"", + "Es herrscht Bestürzung unter einigen Mitgliedern der Versammlung über einen Vorschlag, der ihren Titel zu MWPs (Mitglied der walisischen Parlament) ändern soll.", + "Der Grund dafür waren Pläne, den Namen der Nationalversammlung in Walisisches Parlament zu ändern.", + "Mitglieder aller Parteien der Nationalversammlung haben Bedenken, dass sie sich dadurch Spott aussetzen könnten.", + "Ein Labour-Abgeordneter sagte, dass seine Gruppe \"sich mit Twp und Pwp reimt\".", + "Hinweis für den Leser: „twp“ im Walisischen bedeutet „bescheuert“ und „pwp“ bedeutet „Kacke“.", + "Ein Versammlungsmitglied von Plaid Cymru sagte, die Gruppe als Ganzes sei \"nicht glücklich\" und hat Alternativen vorgeschlagen.", + "Ein walisischer Konservativer sagte, seine Gruppe wäre „offen“ für eine Namensänderung, wies aber darauf hin, dass es von „MWP“ (Mitglied des Walisischen Parlaments) nur ein kurzer verbaler Sprung zu „Muppet“ ist." + ] + }, + "de-en": { + "src": [ + "Schöne Münchnerin 2018: Schöne Münchnerin 2018 in Hvar: Neun Dates", + "Von az, aktualisiert am 04.05.2018 um 11:11", + "Ja, sie will...", + "\"Schöne Münchnerin\" 2018 werden!", + "Am Nachmittag wartet erneut eine Überraschung auf unsere Kandidatinnen: sie werden das romantische Candlelight-Shooting vor der MY SOLARIS nicht alleine bestreiten, sondern an der Seite von Male-Model Fabian!", + "Hvar - Flirten, kokettieren, verführen - keine einfachen Aufgaben für unsere Mädchen.", + "Insbesondere dann, wenn in Deutschland ein Freund wartet.", + "Dennoch liefern die neun \"Schöne Münchnerin\"-Kandidatinnen beim Shooting mit People-Fotograf Tuan ab und trotzen Wind, Gischt und Regen wie echte Profis." + ], + "tgt": [ + "The Beauty of Munich 2018: the Beauty of Munich 2018 in Hvar: Nine dates", + "From A-Z, updated on 04/05/2018 at 11:11", + "Yes, she wants to...", + "to become \"The Beauty of Munich\" in 2018!", + "In the afternoon there is another surprise waiting for our contestants: they will be competing for the romantic candlelight photo shoot at MY SOLARIS not alone, but together with a male-model Fabian!", + "Hvar with its flirting, coquetting, and seduction is not an easy task for our girls.", + "Especially when there is a boyfriend waiting in Germany.", + "Despite dealing with wind, sprays and rain, the nine contestants of \"The Beauty of Munich\" behaved like real professionals at the photo shoot with People-photographer Tuan." + ] + } +} \ No newline at end of file diff --git a/examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml b/examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml deleted file mode 100644 index fe1c3881f38b..000000000000 --- a/examples/seq2seq/test_data/fsmt/fsmt_val_data.yaml +++ /dev/null @@ -1,90 +0,0 @@ -{ - "ru-en": { - "src": [ - "Названо число готовящихся к отправке в Донбасс новобранцев из Украины", - "Официальный представитель Народной милиции самопровозглашенной Луганской Народной Республики (ЛНР) Андрей Марочко заявил, что зимой 2018-2019 года Украина направит в Донбасс не менее 3 тыс. новобранцев.", - "По его словам, таким образом Киев планирует \"хоть как-то доукомплектовать подразделения\".", - "\"Нежелание граждан Украины проходить службу в рядах ВС Украины, массовые увольнения привели к низкой укомплектованности подразделений\", - рассказал Марочко, которого цитирует \"РИА Новости\".", - "Он также не исключил, что реальные цифры призванных в армию украинцев могут быть увеличены в случае необходимости.", - "В 2014-2017 годах Киев начал так называемую антитеррористическую операцию (АТО), которую позже сменили на операцию объединенных сил (ООС).", - "Предполагалось, что эта мера приведет к усилению роли украинских силовиков в урегулировании ситуации.", - "В конце августа 2018 года ситуация в Донбассе обострилась из-за убийства главы ДНР Александра Захарченко.", - ], - "tgt": [ - "The number of new Ukrainian recruits ready to go to Donbass has become public", - "Official representative of the peoples’ militia of the self-proclaimed Lugansk People’s Republic Andrey Marochko claimed that Ukrainian will send at least 3 thousand new recruits to Donbass in winter 2018-2019.", - "This is how Kyiv tries “at least somehow to staff the units,” he said.", - "“The unwillingness of Ukrainian citizens to serve in the Ukraine’s military forces, mass resignments lead to low understaffing,” said Marochko cited by RIA Novosti.", - "Also, he doesn’t exclude that the real numbers of conscripts in the Ukrainian army can be raised is necessary.", - "In 2014-2017, Kyiv started so-called antiterrorist operation, that ws later changed to the united forces operation.", - "This measure was supposed to strengthen the role of the Ukrainian military in settling the situation.", - "In the late August 2018, the situation in Donbass escalated as the DNR head Aleksandr Zakharchenko was killed.", - ], - }, - "en-ru": { - "src": [ - "Welsh AMs worried about 'looking like muppets'", - "There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).", - "It has arisen because of plans to change the name of the assembly to the Welsh Parliament.", - "AMs across the political spectrum are worried it could invite ridicule.", - "One Labour AM said his group was concerned \"it rhymes with Twp and Pwp.\"", - "For readers outside of Wales: In Welsh twp means daft and pwp means poo.", - "A Plaid AM said the group as a whole was \"not happy\" and has suggested alternatives.", - "A Welsh Conservative said his group was \"open minded\" about the name change, but noted it was a short verbal hop from MWP to Muppet.", - ], - "tgt": [ - "Члены Национальной ассамблеи Уэльса обеспокоены, что \"выглядят как куклы\"", - "Некоторые члены Национальной ассамблеи Уэльса в ужасе от предложения о том, что их наименование должно измениться на MPW (члены Парламента Уэльса).", - "Этот вопрос был поднят в связи с планами по переименованию ассамблеи в Парламент Уэльса.", - "Члены Национальной ассамблеи Уэльса всего политического спектра обеспокоены, что это может породить насмешки.", - "Один из лейбористских членов Национальной ассамблеи Уэльса сказал, что его партия обеспокоена тем, что \"это рифмуется с Twp и Pwp\".", - "Для читателей за предлами Уэльса: по-валлийски twp означает \"глупый\", а pwp означает \"какашка\".", - "Член Национальной ассамблеи от Плайд сказал, что эта партия в целом \"не счастлива\" и предложил альтернативы.", - "Представитель Консервативной партии Уэльса сказал, что его партия \"открыта\" к переименованию, но отметил, что между WMP и Muppet небольшая разница в произношении.", - ], - }, - "en-de": { - "src": [ - "Welsh AMs worried about 'looking like muppets'", - "There is consternation among some AMs at a suggestion their title should change to MWPs (Member of the Welsh Parliament).", - "It has arisen because of plans to change the name of the assembly to the Welsh Parliament.", - "AMs across the political spectrum are worried it could invite ridicule.", - "One Labour AM said his group was concerned \"it rhymes with Twp and Pwp.\"", - "For readers outside of Wales: In Welsh twp means daft and pwp means poo.", - "A Plaid AM said the group as a whole was \"not happy\" and has suggested alternatives.", - "A Welsh Conservative said his group was \"open minded\" about the name change, but noted it was a short verbal hop from MWP to Muppet.", - ], - "tgt": [ - "Walisische Ageordnete sorgen sich \"wie Dödel auszusehen\"", - "Es herrscht Bestürzung unter einigen Mitgliedern der Versammlung über einen Vorschlag, der ihren Titel zu MWPs (Mitglied der walisischen Parlament) ändern soll.", - "Der Grund dafür waren Pläne, den Namen der Nationalversammlung in Walisisches Parlament zu ändern.", - "Mitglieder aller Parteien der Nationalversammlung haben Bedenken, dass sie sich dadurch Spott aussetzen könnten.", - "Ein Labour-Abgeordneter sagte, dass seine Gruppe \"sich mit Twp und Pwp reimt\".", - "Hinweis für den Leser: „twp“ im Walisischen bedeutet „bescheuert“ und „pwp“ bedeutet „Kacke“.", - "Ein Versammlungsmitglied von Plaid Cymru sagte, die Gruppe als Ganzes sei \"nicht glücklich\" und hat Alternativen vorgeschlagen.", - "Ein walisischer Konservativer sagte, seine Gruppe wäre „offen“ für eine Namensänderung, wies aber darauf hin, dass es von „MWP“ (Mitglied des Walisischen Parlaments) nur ein kurzer verbaler Sprung zu „Muppet“ ist.", - ], - }, - "de-en": { - "src": [ - "Schöne Münchnerin 2018: Schöne Münchnerin 2018 in Hvar: Neun Dates", - "Von az, aktualisiert am 04.05.2018 um 11:11", - "Ja, sie will...", - "\"Schöne Münchnerin\" 2018 werden!", - "Am Nachmittag wartet erneut eine Überraschung auf unsere Kandidatinnen: sie werden das romantische Candlelight-Shooting vor der MY SOLARIS nicht alleine bestreiten, sondern an der Seite von Male-Model Fabian!", - "Hvar - Flirten, kokettieren, verführen - keine einfachen Aufgaben für unsere Mädchen.", - "Insbesondere dann, wenn in Deutschland ein Freund wartet.", - "Dennoch liefern die neun \"Schöne Münchnerin\"-Kandidatinnen beim Shooting mit People-Fotograf Tuan ab und trotzen Wind, Gischt und Regen wie echte Profis.", - ], - "tgt": [ - "The Beauty of Munich 2018: the Beauty of Munich 2018 in Hvar: Nine dates", - "From A-Z, updated on 04/05/2018 at 11:11", - "Yes, she wants to...", - "to become \"The Beauty of Munich\" in 2018!", - "In the afternoon there is another surprise waiting for our contestants: they will be competing for the romantic candlelight photo shoot at MY SOLARIS not alone, but together with a male-model Fabian!", - "Hvar with its flirting, coquetting, and seduction is not an easy task for our girls.", - "Especially when there is a boyfriend waiting in Germany.", - "Despite dealing with wind, sprays and rain, the nine contestants of \"The Beauty of Munich\" behaved like real professionals at the photo shoot with People-photographer Tuan.", - ], - }, -} diff --git a/examples/seq2seq/test_fsmt_bleu_score.py b/examples/seq2seq/test_fsmt_bleu_score.py index 8981a05e4cdf..ea1d46c6efb8 100644 --- a/examples/seq2seq/test_fsmt_bleu_score.py +++ b/examples/seq2seq/test_fsmt_bleu_score.py @@ -22,19 +22,20 @@ except ImportError: from utils import calculate_bleu -import yaml +import json + from parameterized import parameterized from transformers import FSMTForConditionalGeneration, FSMTTokenizer from transformers.testing_utils import get_tests_dir, require_torch, slow, torch_device -filename = get_tests_dir() + "/test_data/fsmt/fsmt_val_data.yaml" +filename = get_tests_dir() + "/test_data/fsmt/fsmt_val_data.json" with io.open(filename, "r", encoding="utf-8") as f: - bleu_data = yaml.load(f) + bleu_data = json.load(f) @require_torch -class ModelTester(unittest.TestCase): +class ModelEvalTester(unittest.TestCase): def get_tokenizer(self, mname): return FSMTTokenizer.from_pretrained(mname) From 15ebddcc4755d78a5dd35fbb607d9e0d594ff152 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 9 Sep 2020 09:41:54 -0700 Subject: [PATCH 071/109] ignore keys not needed --- src/transformers/modeling_fsmt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 1c9b4dde7e8c..3e987f9317d4 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -1004,7 +1004,6 @@ def set_output_embeddings(self, value): ) class FSMTForConditionalGeneration(PretrainedFSMTModel): base_model_prefix = "model" - authorized_missing_keys = [r"encoder\.version", r"decoder\.version"] def __init__(self, config: FSMTConfig): super().__init__(config) From 014aa1d6d2f2b566f293e8d74022bf5c2ae3dd87 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 10 Sep 2020 09:19:08 -0700 Subject: [PATCH 072/109] use the new -y in transformers-cli upload -y --- ...vert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index cae381d788cf..9b2196427ad7 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -51,14 +51,14 @@ # upload cd data -yes Y | transformers-cli upload fsmt-wmt19-ru-en -yes Y | transformers-cli upload fsmt-wmt19-en-ru -yes Y | transformers-cli upload fsmt-wmt19-de-en -yes Y | transformers-cli upload fsmt-wmt19-en-de +transformers-cli upload -y fsmt-wmt19-ru-en +transformers-cli upload -y fsmt-wmt19-en-ru +transformers-cli upload -y fsmt-wmt19-de-en +transformers-cli upload -y fsmt-wmt19-en-de cd - # if updating just small files and not the large models, here is a script to generate the right commands: -perl -le 'for $f (@ARGV) { print qq[yes Y | transformers-cli upload $_/$f --filename $_/$f] for map { "fsmt-wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json +perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for map { "fsmt-wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json # add/remove files as needed # Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload From 81c8dc59ad2e3f520d469962860af81d78739b8e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 10 Sep 2020 10:31:01 -0700 Subject: [PATCH 073/109] [xlm tok] config dict: fix str into int to match definition (#7034) --- src/transformers/tokenization_xlm.py | 304 +++++++++++++-------------- 1 file changed, 152 insertions(+), 152 deletions(-) diff --git a/src/transformers/tokenization_xlm.py b/src/transformers/tokenization_xlm.py index 7f6b2068c1ce..086a947bec9b 100644 --- a/src/transformers/tokenization_xlm.py +++ b/src/transformers/tokenization_xlm.py @@ -79,37 +79,37 @@ "xlm-mlm-en-2048": {"do_lowercase_and_remove_accent": True}, "xlm-mlm-ende-1024": { "do_lowercase_and_remove_accent": True, - "id2lang": {"0": "de", "1": "en"}, + "id2lang": {0: "de", 1: "en"}, "lang2id": {"de": 0, "en": 1}, }, "xlm-mlm-enfr-1024": { "do_lowercase_and_remove_accent": True, - "id2lang": {"0": "en", "1": "fr"}, + "id2lang": {0: "en", 1: "fr"}, "lang2id": {"en": 0, "fr": 1}, }, "xlm-mlm-enro-1024": { "do_lowercase_and_remove_accent": True, - "id2lang": {"0": "en", "1": "ro"}, + "id2lang": {0: "en", 1: "ro"}, "lang2id": {"en": 0, "ro": 1}, }, "xlm-mlm-tlm-xnli15-1024": { "do_lowercase_and_remove_accent": True, "id2lang": { - "0": "ar", - "1": "bg", - "2": "de", - "3": "el", - "4": "en", - "5": "es", - "6": "fr", - "7": "hi", - "8": "ru", - "9": "sw", - "10": "th", - "11": "tr", - "12": "ur", - "13": "vi", - "14": "zh", + 0: "ar", + 1: "bg", + 2: "de", + 3: "el", + 4: "en", + 5: "es", + 6: "fr", + 7: "hi", + 8: "ru", + 9: "sw", + 10: "th", + 11: "tr", + 12: "ur", + 13: "vi", + 14: "zh", }, "lang2id": { "ar": 0, @@ -132,21 +132,21 @@ "xlm-mlm-xnli15-1024": { "do_lowercase_and_remove_accent": True, "id2lang": { - "0": "ar", - "1": "bg", - "2": "de", - "3": "el", - "4": "en", - "5": "es", - "6": "fr", - "7": "hi", - "8": "ru", - "9": "sw", - "10": "th", - "11": "tr", - "12": "ur", - "13": "vi", - "14": "zh", + 0: "ar", + 1: "bg", + 2: "de", + 3: "el", + 4: "en", + 5: "es", + 6: "fr", + 7: "hi", + 8: "ru", + 9: "sw", + 10: "th", + 11: "tr", + 12: "ur", + 13: "vi", + 14: "zh", }, "lang2id": { "ar": 0, @@ -168,34 +168,34 @@ }, "xlm-clm-enfr-1024": { "do_lowercase_and_remove_accent": True, - "id2lang": {"0": "en", "1": "fr"}, + "id2lang": {0: "en", 1: "fr"}, "lang2id": {"en": 0, "fr": 1}, }, "xlm-clm-ende-1024": { "do_lowercase_and_remove_accent": True, - "id2lang": {"0": "de", "1": "en"}, + "id2lang": {0: "de", 1: "en"}, "lang2id": {"de": 0, "en": 1}, }, "xlm-mlm-17-1280": { "do_lowercase_and_remove_accent": False, "id2lang": { - "0": "ar", - "1": "de", - "2": "en", - "3": "es", - "4": "fr", - "5": "hi", - "6": "it", - "7": "ja", - "8": "ko", - "9": "nl", - "10": "pl", - "11": "pt", - "12": "ru", - "13": "sv", - "14": "tr", - "15": "vi", - "16": "zh", + 0: "ar", + 1: "de", + 2: "en", + 3: "es", + 4: "fr", + 5: "hi", + 6: "it", + 7: "ja", + 8: "ko", + 9: "nl", + 10: "pl", + 11: "pt", + 12: "ru", + 13: "sv", + 14: "tr", + 15: "vi", + 16: "zh", }, "lang2id": { "ar": 0, @@ -220,106 +220,106 @@ "xlm-mlm-100-1280": { "do_lowercase_and_remove_accent": False, "id2lang": { - "0": "af", - "1": "als", - "2": "am", - "3": "an", - "4": "ang", - "5": "ar", - "6": "arz", - "7": "ast", - "8": "az", - "9": "bar", - "10": "be", - "11": "bg", - "12": "bn", - "13": "br", - "14": "bs", - "15": "ca", - "16": "ceb", - "17": "ckb", - "18": "cs", - "19": "cy", - "20": "da", - "21": "de", - "22": "el", - "23": "en", - "24": "eo", - "25": "es", - "26": "et", - "27": "eu", - "28": "fa", - "29": "fi", - "30": "fr", - "31": "fy", - "32": "ga", - "33": "gan", - "34": "gl", - "35": "gu", - "36": "he", - "37": "hi", - "38": "hr", - "39": "hu", - "40": "hy", - "41": "ia", - "42": "id", - "43": "is", - "44": "it", - "45": "ja", - "46": "jv", - "47": "ka", - "48": "kk", - "49": "kn", - "50": "ko", - "51": "ku", - "52": "la", - "53": "lb", - "54": "lt", - "55": "lv", - "56": "mk", - "57": "ml", - "58": "mn", - "59": "mr", - "60": "ms", - "61": "my", - "62": "nds", - "63": "ne", - "64": "nl", - "65": "nn", - "66": "no", - "67": "oc", - "68": "pl", - "69": "pt", - "70": "ro", - "71": "ru", - "72": "scn", - "73": "sco", - "74": "sh", - "75": "si", - "76": "simple", - "77": "sk", - "78": "sl", - "79": "sq", - "80": "sr", - "81": "sv", - "82": "sw", - "83": "ta", - "84": "te", - "85": "th", - "86": "tl", - "87": "tr", - "88": "tt", - "89": "uk", - "90": "ur", - "91": "uz", - "92": "vi", - "93": "war", - "94": "wuu", - "95": "yi", - "96": "zh", - "97": "zh_classical", - "98": "zh_min_nan", - "99": "zh_yue", + 0: "af", + 1: "als", + 2: "am", + 3: "an", + 4: "ang", + 5: "ar", + 6: "arz", + 7: "ast", + 8: "az", + 9: "bar", + 10: "be", + 11: "bg", + 12: "bn", + 13: "br", + 14: "bs", + 15: "ca", + 16: "ceb", + 17: "ckb", + 18: "cs", + 19: "cy", + 20: "da", + 21: "de", + 22: "el", + 23: "en", + 24: "eo", + 25: "es", + 26: "et", + 27: "eu", + 28: "fa", + 29: "fi", + 30: "fr", + 31: "fy", + 32: "ga", + 33: "gan", + 34: "gl", + 35: "gu", + 36: "he", + 37: "hi", + 38: "hr", + 39: "hu", + 40: "hy", + 41: "ia", + 42: "id", + 43: "is", + 44: "it", + 45: "ja", + 46: "jv", + 47: "ka", + 48: "kk", + 49: "kn", + 50: "ko", + 51: "ku", + 52: "la", + 53: "lb", + 54: "lt", + 55: "lv", + 56: "mk", + 57: "ml", + 58: "mn", + 59: "mr", + 60: "ms", + 61: "my", + 62: "nds", + 63: "ne", + 64: "nl", + 65: "nn", + 66: "no", + 67: "oc", + 68: "pl", + 69: "pt", + 70: "ro", + 71: "ru", + 72: "scn", + 73: "sco", + 74: "sh", + 75: "si", + 76: "simple", + 77: "sk", + 78: "sl", + 79: "sq", + 80: "sr", + 81: "sv", + 82: "sw", + 83: "ta", + 84: "te", + 85: "th", + 86: "tl", + 87: "tr", + 88: "tt", + 89: "uk", + 90: "ur", + 91: "uz", + 92: "vi", + 93: "war", + 94: "wuu", + 95: "yi", + 96: "zh", + 97: "zh_classical", + 98: "zh_min_nan", + 99: "zh_yue", }, "lang2id": { "af": 0, From cb9d911bf5bc85df4ba463efa01812ee628601b2 Mon Sep 17 00:00:00 2001 From: Sam Shleifer Date: Thu, 10 Sep 2020 14:11:34 -0400 Subject: [PATCH 074/109] [s2s] --eval_max_generate_length (#7018) --- examples/seq2seq/distil_marian_enro_teacher.sh | 2 +- examples/seq2seq/distil_marian_no_teacher.sh | 2 +- examples/seq2seq/finetune.py | 16 ++++++++++++---- examples/seq2seq/test_seq2seq_examples.py | 1 + 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/seq2seq/distil_marian_enro_teacher.sh b/examples/seq2seq/distil_marian_enro_teacher.sh index 75ef07bc06bd..5c938a71604e 100755 --- a/examples/seq2seq/distil_marian_enro_teacher.sh +++ b/examples/seq2seq/distil_marian_enro_teacher.sh @@ -16,5 +16,5 @@ python distillation.py \ --train_batch_size=$BS --eval_batch_size=$BS \ --tokenizer_name Helsinki-NLP/opus-mt-en-ro \ --warmup_steps 500 --logger_name wandb \ - --fp16_opt_level O1 --task translation --normalize_hidden \ + --fp16_opt_level O1 --task translation --normalize_hidden --num_sanity_val_steps=0 \ "$@" diff --git a/examples/seq2seq/distil_marian_no_teacher.sh b/examples/seq2seq/distil_marian_no_teacher.sh index 66fdda1d17da..4a30628149df 100755 --- a/examples/seq2seq/distil_marian_no_teacher.sh +++ b/examples/seq2seq/distil_marian_no_teacher.sh @@ -13,5 +13,5 @@ python distillation.py \ --train_batch_size=$BS --eval_batch_size=$BS \ --tokenizer_name $m --model_name_or_path $m \ --warmup_steps 500 --sortish_sampler --logger_name wandb \ - --gpus 1 --fp16_opt_level=O1 --task translation \ + --gpus 1 --fp16_opt_level=O1 --task translation --num_sanity_val_steps=0 \ "$@" diff --git a/examples/seq2seq/finetune.py b/examples/seq2seq/finetune.py index ef0445e9007d..15e99bdb0c0b 100644 --- a/examples/seq2seq/finetune.py +++ b/examples/seq2seq/finetune.py @@ -11,7 +11,6 @@ import numpy as np import pytorch_lightning as pl import torch -from packaging import version from torch.utils.data import DataLoader from lightning_base import BaseTransformer, add_generic_args, generic_train @@ -94,6 +93,9 @@ def __init__(self, hparams, **kwargs): "val": self.hparams.val_max_target_length, "test": self.hparams.test_max_target_length, } + if self.hparams.sortish_sampler and self.hparams.gpus > 1: + self.hparams.sortish_sampler = False + warnings.warn("ignoring sortish_sampler as it is unsupported on multiple GPUs") assert self.target_lens["train"] <= self.target_lens["val"], f"target_lens: {self.target_lens}" assert self.target_lens["train"] <= self.target_lens["test"], f"target_lens: {self.target_lens}" @@ -114,6 +116,10 @@ def __init__(self, hparams, **kwargs): ) self.eval_beams = self.model.config.num_beams if self.hparams.eval_beams is None else self.hparams.eval_beams assert self.eval_beams >= 1, f"got self.eval_beams={self.eval_beams}. Need an integer > 1" + if self.hparams.eval_max_gen_length is not None: + self.eval_max_length = self.hparams.eval_max_gen_length + else: + self.eval_max_length = self.model.config.max_length self.val_metric = self.default_val_metric if self.hparams.val_metric is None else self.hparams.val_metric def freeze_embeds(self): @@ -209,12 +215,15 @@ def calc_generative_metrics(self, preds, target) -> Dict: def _generative_step(self, batch: dict) -> dict: t0 = time.time() + + # parser.add_argument('--eval_max_gen_length', type=int, default=None, help='never generate more than n tokens') generated_ids = self.model.generate( batch["input_ids"], attention_mask=batch["attention_mask"], use_cache=True, decoder_start_token_id=self.decoder_start_token_id, num_beams=self.eval_beams, + max_length=self.eval_max_length, ) gen_time = (time.time() - t0) / batch["input_ids"].shape[0] preds: List[str] = self.ids_to_clean_text(generated_ids) @@ -248,7 +257,7 @@ def get_dataloader(self, type_path: str, batch_size: int, shuffle: bool = False) dataset = self.get_dataset(type_path) sampler = None if self.hparams.sortish_sampler and type_path == "train": - assert self.hparams.gpus <= 1 # TODO: assert earlier + assert self.hparams.gpus <= 1 # this should never break because of the assertion in __init__ sampler = dataset.make_sortish_sampler(batch_size) shuffle = False @@ -321,6 +330,7 @@ def add_model_specific_args(parser, root_dir): parser.add_argument( "--val_metric", type=str, default=None, required=False, choices=["bleu", "rouge2", "loss", None] ) + parser.add_argument("--eval_max_gen_length", type=int, default=None, help="never generate more than n tokens") parser.add_argument("--save_top_k", type=int, default=1, required=False, help="How many checkpoints to save") parser.add_argument( "--early_stopping_patience", @@ -356,8 +366,6 @@ def main(args, model=None) -> SummarizationModule: model: SummarizationModule = SummarizationModule(args) else: model: SummarizationModule = TranslationModule(args) - if version.parse(torch.__version__) == version.parse("1.6") and args.fp16: - warnings.warn("FP16 only seems to work with torch 1.5+apex") dataset = Path(args.data_dir).name if ( args.logger_name == "default" diff --git a/examples/seq2seq/test_seq2seq_examples.py b/examples/seq2seq/test_seq2seq_examples.py index 3f4ff2a31d19..7acbbd7b5e8f 100644 --- a/examples/seq2seq/test_seq2seq_examples.py +++ b/examples/seq2seq/test_seq2seq_examples.py @@ -34,6 +34,7 @@ "supervise_forward": True, "normalize_hidden": True, "label_smoothing": 0.2, + "eval_max_gen_length": None, "eval_beams": 1, "val_metric": "loss", "save_top_k": 1, From 4d69131880bab624ede843ea54159580cd49b13a Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:51:08 -0400 Subject: [PATCH 075/109] Fix CI with change of name of nlp (#7054) * nlp -> datasets * More nlp -> datasets * Woopsie * More nlp -> datasets * One last --- .circleci/config.yml | 6 +++--- .github/workflows/self-push.yml | 2 +- .github/workflows/self-scheduled.yml | 2 +- examples/longform-qa/README.md | 2 +- examples/longform-qa/eli5_app.py | 6 +++--- examples/longform-qa/eli5_utils.py | 2 +- examples/requirements.txt | 2 +- examples/seq2seq/download_wmt.py | 10 +++++----- setup.cfg | 2 +- src/transformers/__init__.py | 2 +- src/transformers/file_utils.py | 10 +++++----- src/transformers/trainer.py | 30 ++++++++++++++-------------- tests/test_trainer.py | 8 ++++---- 13 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab730b4091b4..9f82220e6ac4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,7 +77,7 @@ jobs: - v0.3-torch_and_tf-{{ checksum "setup.py" }} - v0.3-{{ checksum "setup.py" }} - run: pip install --upgrade pip - - run: pip install git+https://github.com/huggingface/nlp + - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,tf-cpu,torch,testing] - run: pip install codecov pytest-cov - save_cache: @@ -104,7 +104,7 @@ jobs: - v0.3-torch-{{ checksum "setup.py" }} - v0.3-{{ checksum "setup.py" }} - run: pip install --upgrade pip - - run: pip install git+https://github.com/huggingface/nlp + - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,torch,testing] - save_cache: key: v0.3-torch-{{ checksum "setup.py" }} @@ -129,7 +129,7 @@ jobs: - v0.3-tf-{{ checksum "setup.py" }} - v0.3-{{ checksum "setup.py" }} - run: pip install --upgrade pip - - run: pip install git+https://github.com/huggingface/nlp + - run: pip install git+https://github.com/huggingface/datasets - run: pip install .[sklearn,tf-cpu,testing] - save_cache: key: v0.3-tf-{{ checksum "setup.py" }} diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index c855137f35ba..6e3f368cb762 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -46,7 +46,7 @@ jobs: pip install --upgrade pip pip install torch!=1.6.0 pip install .[sklearn,testing,onnxruntime] - pip install git+https://github.com/huggingface/nlp + pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks run: | diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 243ade6afe87..231fab7a953f 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -43,7 +43,7 @@ jobs: pip install --upgrade pip pip install torch!=1.6.0 pip install .[sklearn,testing,onnxruntime] - pip install git+https://github.com/huggingface/nlp + pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks run: | diff --git a/examples/longform-qa/README.md b/examples/longform-qa/README.md index 36f8c6c18bc1..888d5a782d4f 100644 --- a/examples/longform-qa/README.md +++ b/examples/longform-qa/README.md @@ -1,5 +1,5 @@ # Long Form Question Answering -This folder contains the code for the Long Form Question answering [demo](http://35.226.96.115:8080/) as well as methods to train and use a fully end-to-end Long Form Question Answering system using the [🤗transformers](https://github.com/huggingface/transformers) and [🤗nlp](https://github.com/huggingface/nlp) libraries. +This folder contains the code for the Long Form Question answering [demo](http://35.226.96.115:8080/) as well as methods to train and use a fully end-to-end Long Form Question Answering system using the [🤗transformers](https://github.com/huggingface/transformers) and [🤗datasets](https://github.com/huggingface/datasets) libraries. You can use these methods to train your own system by following along the associate [notebook](https://github.com/huggingface/notebooks/blob/master/longform-qa/Long_Form_Question_Answering_with_ELI5_and_Wikipedia.ipynb) or [blog post](https://yjernite.github.io/lfqa.html). diff --git a/examples/longform-qa/eli5_app.py b/examples/longform-qa/eli5_app.py index 66420b4c02a7..a57238df43a8 100644 --- a/examples/longform-qa/eli5_app.py +++ b/examples/longform-qa/eli5_app.py @@ -1,5 +1,5 @@ +import datasets import faiss -import nlp import numpy as np import streamlit as st import torch @@ -45,7 +45,7 @@ def load_models(): def load_indexes(): if LOAD_DENSE_INDEX: faiss_res = faiss.StandardGpuResources() - wiki40b_passages = nlp.load_dataset(path="wiki_snippets", name="wiki40b_en_100_0")["train"] + wiki40b_passages = datasets.load_dataset(path="wiki_snippets", name="wiki40b_en_100_0")["train"] wiki40b_passage_reps = np.memmap( "wiki40b_passages_reps_32_l-8_h-768_b-512-512.dat", dtype="float32", @@ -63,7 +63,7 @@ def load_indexes(): @st.cache(allow_output_mutation=True) def load_train_data(): - eli5 = nlp.load_dataset("eli5", name="LFQA_reddit") + eli5 = datasets.load_dataset("eli5", name="LFQA_reddit") eli5_train = eli5["train_eli5"] eli5_train_q_reps = np.memmap( "eli5_questions_reps.dat", dtype="float32", mode="r", shape=(eli5_train.num_rows, 128) diff --git a/examples/longform-qa/eli5_utils.py b/examples/longform-qa/eli5_utils.py index 5ef7ce0cc67f..95e1eaf6b66f 100644 --- a/examples/longform-qa/eli5_utils.py +++ b/examples/longform-qa/eli5_utils.py @@ -4,8 +4,8 @@ from random import choice, randint from time import time +import datasets # noqa: F401 import faiss # noqa: F401 -import nlp # noqa: F401 import numpy as np import pandas as pd import torch diff --git a/examples/requirements.txt b/examples/requirements.txt index 65d266f63c9e..615355ad3664 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -12,7 +12,7 @@ faiss streamlit elasticsearch pandas -nlp +datasets fire pytest conllu \ No newline at end of file diff --git a/examples/seq2seq/download_wmt.py b/examples/seq2seq/download_wmt.py index 294a489a841d..efe5ff0b9129 100644 --- a/examples/seq2seq/download_wmt.py +++ b/examples/seq2seq/download_wmt.py @@ -5,25 +5,25 @@ def download_wmt_dataset(src_lang="ro", tgt_lang="en", dataset="wmt16", save_dir=None) -> None: - """Download a dataset using the nlp package and save it to the format expected by finetune.py + """Download a dataset using the datasets package and save it to the format expected by finetune.py Format of save_dir: train.source, train.target, val.source, val.target, test.source, test.target. Args: src_lang: source language tgt_lang: target language - dataset: wmt16, wmt17, etc. wmt16 is a good start as it's small. To get the full list run `import nlp; print([d.id for d in nlp.list_datasets() if "wmt" in d.id])` + dataset: wmt16, wmt17, etc. wmt16 is a good start as it's small. To get the full list run `import datasets; print([d.id for d in datasets.list_datasets() if "wmt" in d.id])` save_dir: , where to save the datasets, defaults to f'{dataset}-{src_lang}-{tgt_lang}' Usage: >>> download_wmt_dataset('ro', 'en', dataset='wmt16') # saves to wmt16-ro-en """ try: - import nlp + import datasets except (ModuleNotFoundError, ImportError): - raise ImportError("run pip install nlp") + raise ImportError("run pip install datasets") pair = f"{src_lang}-{tgt_lang}" print(f"Converting {dataset}-{pair}") - ds = nlp.load_dataset(dataset, pair) + ds = datasets.load_dataset(dataset, pair) if save_dir is None: save_dir = f"{dataset}-{pair}" save_dir = Path(save_dir) diff --git a/setup.cfg b/setup.cfg index a51945d79fc5..b7d686bbd406 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,7 @@ known_first_party = transformers known_third_party = absl conllu + datasets elasticsearch fairseq faiss @@ -16,7 +17,6 @@ known_third_party = git h5py matplotlib - nlp nltk numpy packaging diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 496e0ee99872..652c01bb47de 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -84,7 +84,7 @@ add_start_docstrings, cached_path, is_apex_available, - is_nlp_available, + is_datasets_available, is_psutil_available, is_py3nvml_available, is_tf_available, diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 0a2100327b4a..beef7e833b5a 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -66,12 +66,12 @@ try: - import nlp # noqa: F401 + import datasets # noqa: F401 - _nlp_available = True + _datasets_available = True except ImportError: - _nlp_available = False + _datasets_available = False try: from torch.hub import _get_torch_home @@ -155,8 +155,8 @@ def is_torch_tpu_available(): return _torch_tpu_available -def is_nlp_available(): - return _nlp_available +def is_datasets_available(): + return _datasets_available def is_psutil_available(): diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index c1d1905e21e9..a981ff9f6d3a 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -20,7 +20,7 @@ from tqdm.auto import tqdm, trange from .data.data_collator import DataCollator, DataCollatorWithPadding, default_data_collator -from .file_utils import is_nlp_available, is_torch_tpu_available +from .file_utils import is_datasets_available, is_torch_tpu_available from .integrations import ( default_hp_search_backend, is_comet_available, @@ -65,8 +65,8 @@ _use_native_amp = True from torch.cuda.amp import autocast -if is_nlp_available(): - import nlp +if is_datasets_available(): + import datasets if is_torch_tpu_available(): import torch_xla.core.xla_model as xm @@ -179,10 +179,10 @@ class Trainer: :obj:`eval_dataset`. Will default to :func:`~transformers.default_data_collator` if no ``tokenizer`` is provided, an instance of :func:`~transformers.DataCollatorWithPadding` otherwise. train_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`): - The dataset to use for training. If it is an :obj:`nlp.Dataset`, columns not accepted by the + The dataset to use for training. If it is an :obj:`datasets.Dataset`, columns not accepted by the ``model.forward()`` method are automatically removed. eval_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`): - The dataset to use for evaluation. If it is an :obj:`nlp.Dataset`, columns not accepted by the + The dataset to use for evaluation. If it is an :obj:`datasets.Dataset`, columns not accepted by the ``model.forward()`` method are automatically removed. tokenizer (:class:`PreTrainedTokenizerBase`, `optional`): The tokenizer used to preprocess the data. If provided, will be used to automatically pad the inputs the @@ -280,10 +280,10 @@ def __init__( FutureWarning, ) - if is_nlp_available(): - if isinstance(train_dataset, nlp.Dataset): + if is_datasets_available(): + if isinstance(train_dataset, datasets.Dataset): self._remove_unused_columns(self.train_dataset, description="training") - if isinstance(eval_dataset, nlp.Dataset): + if isinstance(eval_dataset, datasets.Dataset): self._remove_unused_columns(self.eval_dataset, description="evaluation") self.global_step = None @@ -294,7 +294,7 @@ def __init__( self.hp_search_backend = None self.use_tune_checkpoints = False - def _remove_unused_columns(self, dataset: "nlp.Dataset", description: Optional[str] = None): + def _remove_unused_columns(self, dataset: "datasets.Dataset", description: Optional[str] = None): if not self.args.remove_unused_columns: return # Inspect model forward signature to keep only the arguments it accepts. @@ -364,12 +364,12 @@ def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoa Args: eval_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`): - If provided, will override :obj:`self.eval_dataset`. If it is an :obj:`nlp.Dataset`, columns not + If provided, will override :obj:`self.eval_dataset`. If it is an :obj:`datasets.Dataset`, columns not accepted by the ``model.forward()`` method are automatically removed. """ if eval_dataset is None and self.eval_dataset is None: raise ValueError("Trainer: evaluation requires an eval_dataset.") - elif eval_dataset is not None and is_nlp_available() and isinstance(eval_dataset, nlp.Dataset): + elif eval_dataset is not None and is_datasets_available() and isinstance(eval_dataset, datasets.Dataset): self._remove_unused_columns(eval_dataset, description="evaluation") eval_dataset = eval_dataset if eval_dataset is not None else self.eval_dataset eval_sampler = self._get_eval_sampler(eval_dataset) @@ -393,10 +393,10 @@ def get_test_dataloader(self, test_dataset: Dataset) -> DataLoader: Args: eval_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`): - The test dataset to use. If it is an :obj:`nlp.Dataset`, columns not accepted by the + The test dataset to use. If it is an :obj:`datasets.Dataset`, columns not accepted by the ``model.forward()`` method are automatically removed. """ - if is_nlp_available() and isinstance(test_dataset, nlp.Dataset): + if is_datasets_available() and isinstance(test_dataset, datasets.Dataset): self._remove_unused_columns(test_dataset, description="test") test_sampler = self._get_eval_sampler(test_dataset) @@ -1200,7 +1200,7 @@ def evaluate(self, eval_dataset: Optional[Dataset] = None) -> Dict[str, float]: Args: eval_dataset (:obj:`Dataset`, `optional`): - Pass a dataset if you wish to override :obj:`self.eval_dataset`. If it is an :obj:`nlp.Dataset`, + Pass a dataset if you wish to override :obj:`self.eval_dataset`. If it is an :obj:`datasets.Dataset`, columns not accepted by the ``model.forward()`` method are automatically removed. Returns: @@ -1227,7 +1227,7 @@ def predict(self, test_dataset: Dataset) -> PredictionOutput: Args: test_dataset (:obj:`Dataset`): - Dataset to run the predictions on. If it is an :obj:`nlp.Dataset`, columns not accepted by the + Dataset to run the predictions on. If it is an :obj:`datasets.Dataset`, columns not accepted by the ``model.forward()`` method are automatically removed. Returns: diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 034cc552f966..e6761599571f 100755 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -1,6 +1,6 @@ import unittest -import nlp +import datasets import numpy as np from transformers import AutoTokenizer, TrainingArguments, is_torch_available @@ -200,11 +200,11 @@ def test_predict(self): x = trainer.eval_dataset.x self.assertTrue(np.allclose(preds, 1.5 * x + 2.5)) - def test_trainer_with_nlp(self): + def test_trainer_with_datasets(self): np.random.seed(42) x = np.random.normal(size=(64,)).astype(np.float32) y = 2.0 * x + 3.0 + np.random.normal(scale=0.1, size=(64,)) - train_dataset = nlp.Dataset.from_dict({"input_x": x, "label": y}) + train_dataset = datasets.Dataset.from_dict({"input_x": x, "label": y}) # Base training. Should have the same results as test_reproducible_training model = RegressionModel() @@ -222,7 +222,7 @@ def test_trainer_with_nlp(self): # Adding one column not used by the model should have no impact z = np.random.normal(size=(64,)).astype(np.float32) - train_dataset = nlp.Dataset.from_dict({"input_x": x, "label": y, "extra": z}) + train_dataset = datasets.Dataset.from_dict({"input_x": x, "label": y, "extra": z}) model = RegressionModel() trainer = Trainer(model, args, train_dataset=train_dataset) trainer.train() From 5276f960b4148540a610ac7596c49dc77143de06 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 12 Sep 2020 13:29:29 -0700 Subject: [PATCH 076/109] extending to support allen_nlp wmt models - allow a specific checkpoint file to be passed - more arg settings - scripts for allen_nlp models --- ..._original_pytorch_checkpoint_to_pytorch.py | 175 +++++++++++++++--- 1 file changed, 154 insertions(+), 21 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 9b2196427ad7..a1dd455bb616 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -37,16 +37,16 @@ # run conversions and uploads export PAIR=ru-en -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR export PAIR=en-ru -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR export PAIR=de-en -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR export PAIR=en-de -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR # upload @@ -68,6 +68,132 @@ # happy translations + +###################################################################################### + +Convert fairseq transform wmt16 en-de checkpoints from https://github.com/jungokasai/deep-shallow + + +pip install gdown + +# get data (run once) + +cd data +gdown 'https://drive.google.com/uc?id=1x_G2cjvM1nW5hjAB8-vWxRqtQTlmIaQU' +gdown 'https://drive.google.com/uc?id=1oA2aqZlVNj5FarxBlNXEHpBS4lRetTzU' +gdown 'https://drive.google.com/uc?id=1Wup2D318QYBFPW_NKI1mfP_hXOfmUI9r' +tar -xvzf trans_ende_12-1_0.2.tar.gz +tar -xvzf trans_ende-dist_12-1_0.2.tar.gz +tar -xvzf trans_ende-dist_6-1_0.2.tar.gz + +gdown 'https://drive.google.com/uc?id=1mNufoynJ9-Zy1kJh2TA_lHm2squji0i9' +gdown 'https://drive.google.com/uc?id=1iO7um-HWoNoRKDtw27YUSgyeubn9uXqj' +tar -xvzf wmt16.en-de.deep-shallow.dist.tar.gz +tar -xvzf wmt16.en-de.deep-shallow.tar.gz + +cp wmt16.en-de.deep-shallow/data-bin/dict.*.txt trans_ende_12-1_0.2 +cp wmt16.en-de.deep-shallow.dist/data-bin/dict.*.txt trans_ende-dist_12-1_0.2 +cp wmt16.en-de.deep-shallow.dist/data-bin/dict.*.txt trans_ende-dist_6-1_0.2 +cp wmt16.en-de.deep-shallow/bpecodes trans_ende_12-1_0.2 +cp wmt16.en-de.deep-shallow.dist/bpecodes trans_ende-dist_12-1_0.2 +cp wmt16.en-de.deep-shallow.dist/bpecodes trans_ende-dist_6-1_0.2 + + +# another set wmt19-6-6-de-en +gdown 'https://drive.google.com/uc?id=1j6z9fYdlUyOYsh7KJoumRlr1yHczxR5T' +gdown 'https://drive.google.com/uc?id=1yT7ZjqfvUYOBXvMjeY8uGRHQFWoSo8Q5' +gdown 'https://drive.google.com/uc?id=15gAzHeRUCs-QV8vHeTReMPEh1j8excNE' +tar -xvzf wmt19.de-en.tar.gz +tar -xvzf wmt19_deen_base_dr0.1_1.tar.gz +tar -xvzf wmt19_deen_big_dr0.1_2.tar.gz +cp wmt19.de-en/data-bin/dict.*.txt wmt19_deen_base_dr0.1_1 +cp wmt19.de-en/data-bin/dict.*.txt wmt19_deen_big_dr0.1_2 + +cd - + + +# run conversions and uploads + +# wmt16-en-de set + +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/fsmt-wmt16-en-de-dist-12-1 + +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_6-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/fsmt-wmt16-en-de-dist-6-1 + +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/fsmt-wmt16-en-de-12-1 + + +# wmt19-de-en set + +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_base_dr0.1_1/checkpoint_best.pt --pytorch_dump_folder_path data/fsmt-wmt19-de-en-6-6-base + +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_big_dr0.1_2/checkpoint_best.pt --pytorch_dump_folder_path data/fsmt-wmt19-de-en-6-6-big + + + +# XXX: move into model card + +git clone https://github.com/huggingface/transformers +cd transformers +export PAIR=en-de +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=64 +export NUM_BEAMS=5 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target + +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt16-en-de-dist-12-1 +echo $PAIR $MODEL_PATH +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS + +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt16-en-de-dist-6-1 +echo $PAIR $MODEL_PATH +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS + +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt16-en-de-12-1 +echo $PAIR $MODEL_PATH +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS + +checkpoint_top5_average.pt: + +num_beams=5 + +chkpt file| top5_average | best | +----------|--------------|---------| +dist-12-1 | 29.9134 | 30.2591 | +dist-6-1 | 29.9837 | 29.3349 | +12-1 | 26.4008 | 24.1803 | + +checkpoint_best.pt + + +# wmt19-de-en set + +export PAIR=de-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=64 +export NUM_BEAMS=5 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target + +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt19-de-en-6-6-base +echo $PAIR $MODEL_PATH +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS + +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt19-de-en-6-6-big +echo $PAIR $MODEL_PATH +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS + + + + +``` + + """ import argparse @@ -94,9 +220,7 @@ ORG_NAME = "stas" # XXX: will become facebook - -DEBUG = 0 -json_indent = 2 if DEBUG else None +json_indent = 2 def rewrite_dict_keys(d): @@ -235,22 +359,22 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder os.makedirs(pytorch_dump_folder_path, exist_ok=True) print(f"Writing results to {pytorch_dump_folder_path}") - # XXX: Need to work out the ensemble as fairseq does, for now using just one chkpt - # checkpoint_file = 'model1.pt:model2.pt:model3.pt:model4.pt' - checkpoint_file = "model4.pt" # proved to give the highest BLEU score for each pair - # model_name_or_path = 'transformer.wmt19.ru-en' - data_name_or_path = "." + # handle various types of models + + checkpoint_file = basename(fsmt_checkpoint_path) + fsmt_folder_path = dirname(fsmt_checkpoint_path) + cls = fairseq.model_parallel.models.transformer.ModelParallelTransformerModel models = cls.hub_models() kwargs = {"bpe": "fastbpe", "tokenizer": "moses"} - # print(f"using checkpoint {checkpoint_file}") - + data_name_or_path = "." # note: since the model dump is old, fairseq has upgraded its model some # time later, and it does a whole lot of rewrites and splits on the saved # weights, therefore we can't use torch.load() directly on the model file. # see: upgrade_state_dict(state_dict) in fairseq_model.py + print(f"using checkpoint {checkpoint_file}") chkpt = hub_utils.from_pretrained( - fsmt_checkpoint_path, checkpoint_file, data_name_or_path, archive_map=models, **kwargs + fsmt_folder_path, checkpoint_file, data_name_or_path, archive_map=models, **kwargs ) args = dict(vars(chkpt["args"])) @@ -263,8 +387,8 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder proj_root = dirname(dirname(dirname(os.path.realpath(__file__)))) # dicts - src_dict_file = os.path.join(fsmt_checkpoint_path, f"dict.{src_lang}.txt") - tgt_dict_file = os.path.join(fsmt_checkpoint_path, f"dict.{tgt_lang}.txt") + src_dict_file = os.path.join(fsmt_folder_path, f"dict.{src_lang}.txt") + tgt_dict_file = os.path.join(fsmt_folder_path, f"dict.{tgt_lang}.txt") src_dict = Dictionary.load(src_dict_file) src_vocab = rewrite_dict_keys(src_dict.indices) @@ -284,7 +408,7 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder # merges_file (bpecodes) merges_file = os.path.join(pytorch_dump_folder_path, VOCAB_FILES_NAMES["merges_file"]) - fsmt_merges_file = os.path.join(fsmt_checkpoint_path, "bpecodes") + fsmt_merges_file = os.path.join(fsmt_folder_path, "bpecodes") with open(fsmt_merges_file, encoding="utf-8") as fin: merges = fin.read() merges = re.sub(r" \d+$", "", merges, 0, re.M) # remove frequency number @@ -295,6 +419,11 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder # model config fsmt_model_config_file = os.path.join(pytorch_dump_folder_path, "config.json") + # validate bpe/tokenizer config, as currently it's hardcoded to moses+fastbpe - + # may have to modify the tokenizer if a different type is used by a future model + assert args["bpe"] == "fastbpe", f"need to extend tokenizer to support bpe={args['bpe']}" + assert args["tokenizer"] == "moses", f"need to extend tokenizer to support bpe={args['tokenizer']}" + model_conf = { "architectures": ["FSMTForConditionalGeneration"], "model_type": "fsmt", @@ -321,8 +450,8 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder "pad_token_id": 1, "eos_token_id": 2, "is_encoder_decoder": True, - "scale_embedding": True, - "tie_word_embeddings": False, + "scale_embedding": not args["no_scale_embedding"], + "tie_word_embeddings": args["share_all_embeddings"], } print(f"Generating {fsmt_model_config_file}") @@ -387,7 +516,11 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder parser = argparse.ArgumentParser() # Required parameters parser.add_argument( - "--fsmt_checkpoint_path", default=None, type=str, required=True, help="Path to the official PyTorch dump dir." + "--fsmt_checkpoint_path", + default=None, + type=str, + required=True, + help="Path to the official PyTorch checkpoint file which is expected to reside in the dump dir with dicts, bpecodes, etc.", ) parser.add_argument( "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model." From e17a2f1d4a0747d61843a0beea2ac667e1d80f9b Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 12 Sep 2020 13:36:50 -0700 Subject: [PATCH 077/109] sync with changes --- src/transformers/configuration_auto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/configuration_auto.py b/src/transformers/configuration_auto.py index 253e4f918368..30d2ca9797f9 100644 --- a/src/transformers/configuration_auto.py +++ b/src/transformers/configuration_auto.py @@ -129,6 +129,7 @@ ("longformer", "Longformer"), ("roberta", "RoBERTa"), ("flaubert", "FlauBERT"), + ("fsmt", "FSMT"), ("bert", "BERT"), ("openai-gpt", "OpenAI GPT"), ("gpt2", "OpenAI GPT-2"), From 2babdf89a255755a602e78d508588b7daa96b1b6 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 12 Sep 2020 16:03:07 -0700 Subject: [PATCH 078/109] s/fsmt-wmt/wmt/ in model names --- src/transformers/configuration_fsmt.py | 10 ++-- ..._original_pytorch_checkpoint_to_pytorch.py | 59 +++++++++++-------- src/transformers/modeling_fsmt.py | 12 ++-- src/transformers/tokenization_fsmt.py | 40 ++++++------- 4 files changed, 65 insertions(+), 56 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 7bc058acf182..afd02c14527b 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -25,10 +25,10 @@ logger = logging.getLogger(__name__) FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/config.json", - "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/config.json", - "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/config.json", - "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/config.json", + "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/config.json", + "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/config.json", + "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/config.json", + "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/config.json", } @@ -148,7 +148,7 @@ def __init__( >>> from transformers import FSMTConfig, FSMTModel - >>> config = FSMTConfig.from_pretrained('stas/fsmt-wmt19-en-ru') + >>> config = FSMTConfig.from_pretrained('stas/wmt19-en-ru') >>> model = FSMTModel(config) """ diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index a1dd455bb616..f9b5b79fbe52 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -37,28 +37,28 @@ # run conversions and uploads export PAIR=ru-en -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR export PAIR=en-ru -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR export PAIR=de-en -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR export PAIR=en-de -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/fsmt-wmt19-$PAIR +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR # upload cd data -transformers-cli upload -y fsmt-wmt19-ru-en -transformers-cli upload -y fsmt-wmt19-en-ru -transformers-cli upload -y fsmt-wmt19-de-en -transformers-cli upload -y fsmt-wmt19-en-de +transformers-cli upload -y wmt19-ru-en +transformers-cli upload -y wmt19-en-ru +transformers-cli upload -y wmt19-de-en +transformers-cli upload -y wmt19-en-de cd - # if updating just small files and not the large models, here is a script to generate the right commands: -perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for map { "fsmt-wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json +perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for map { "wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json # add/remove files as needed # Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload @@ -116,20 +116,29 @@ # wmt16-en-de set -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/fsmt-wmt16-en-de-dist-12-1 +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-dist-12-1 -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_6-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/fsmt-wmt16-en-de-dist-6-1 +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_6-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-dist-6-1 -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/fsmt-wmt16-en-de-12-1 +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-12-1 # wmt19-de-en set -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_base_dr0.1_1/checkpoint_best.pt --pytorch_dump_folder_path data/fsmt-wmt19-de-en-6-6-base +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_base_dr0.1_1/checkpoint_best.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-base -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_big_dr0.1_2/checkpoint_best.pt --pytorch_dump_folder_path data/fsmt-wmt19-de-en-6-6-big +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_big_dr0.1_2/checkpoint_best.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-big +# upload +cd data +transformers-cli upload -y wmt16-en-de-dist-12-1 +transformers-cli upload -y wmt16-en-de-dist-6-1 +transformers-cli upload -y wmt16-en-de-12-1 +transformers-cli upload -y wmt19-de-en-6-6-base +transformers-cli upload -y wmt19-de-en-6-6-big +cd - + # XXX: move into model card @@ -144,15 +153,15 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt16-en-de-dist-12-1 +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt16-en-de-dist-12-1 echo $PAIR $MODEL_PATH PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt16-en-de-dist-6-1 +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt16-en-de-dist-6-1 echo $PAIR $MODEL_PATH PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt16-en-de-12-1 +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt16-en-de-12-1 echo $PAIR $MODEL_PATH PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS @@ -180,11 +189,11 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt19-de-en-6-6-base +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt19-de-en-6-6-base echo $PAIR $MODEL_PATH PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/fsmt-wmt19-de-en-6-6-big +MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt19-de-en-6-6-big echo $PAIR $MODEL_PATH PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS @@ -282,10 +291,10 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): All four models are available: -* [fsmt-wmt19-en-ru](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-en-ru) -* [fsmt-wmt19-ru-en](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-ru-en) -* [fsmt-wmt19-en-de](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-en-de) -* [fsmt-wmt19-de-en](https://huggingface.co/{ORG_NAME}/fsmt-wmt19-de-en) +* [wmt19-en-ru](https://huggingface.co/{ORG_NAME}/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/{ORG_NAME}/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/{ORG_NAME}/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/{ORG_NAME}/wmt19-de-en) ## Intended uses & limitations @@ -294,7 +303,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "{ORG_NAME}/fsmt-wmt19-{src_lang}-{tgt_lang}" +mname = "{ORG_NAME}/wmt19-{src_lang}-{tgt_lang}" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) @@ -338,7 +347,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py {ORG_NAME}/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py {ORG_NAME}/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` ## TODO diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 3e987f9317d4..1810ea8bb549 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -120,7 +120,7 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 36.4 http://matrix.statmt.org/matrix/output/1914?score_id=37605) @@ -135,7 +135,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937) @@ -152,7 +152,7 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 42.3 http://matrix.statmt.org/matrix/output/1902?run_id=6750) @@ -168,7 +168,7 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 43.1 http://matrix.statmt.org/matrix/output/1909?run_id=6862) @@ -191,7 +191,7 @@ from transformers import FSMTTokenizer, FSMTForConditionalGeneration - mname = "stas/fsmt-wmt19-ru-en" + mname = "stas/wmt19-ru-en" model = FSMTForConditionalGeneration.from_pretrained(mname) tokenizer = FSMTTokenizer.from_pretrained(mname) @@ -894,7 +894,7 @@ def __init__(self, config: FSMTConfig): @add_start_docstrings_to_callable(FSMT_INPUTS_DOCSTRING) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, - checkpoint="stas/fsmt-wmt19-ru-en", + checkpoint="stas/wmt19-ru-en", output_type=BaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, ) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index eda1d663d6ba..1499742df7f9 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -39,43 +39,43 @@ PRETRAINED_VOCAB_FILES_MAP = { "src_vocab_file": { - "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-src.json", - "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-src.json", - "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-src.json", - "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-src.json", + "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/vocab-src.json", + "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/vocab-src.json", + "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/vocab-src.json", + "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/vocab-src.json", }, "tgt_vocab_file": { - "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/vocab-tgt.json", - "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/vocab-tgt.json", - "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/vocab-tgt.json", - "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/vocab-tgt.json", + "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/vocab-tgt.json", + "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/vocab-tgt.json", + "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/vocab-tgt.json", + "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/vocab-tgt.json", }, "merges_file": { - "stas/fsmt-wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-ru-en/merges.txt", - "stas/fsmt-wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-ru/merges.txt", - "stas/fsmt-wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-de-en/merges.txt", - "stas/fsmt-wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/fsmt-wmt19-en-de/merges.txt", + "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/merges.txt", + "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/merges.txt", + "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/merges.txt", + "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/merges.txt", }, } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - "stas/fsmt-wmt19-ru-en": 1024, - "stas/fsmt-wmt19-en-ru": 1024, - "stas/fsmt-wmt19-de-en": 1024, - "stas/fsmt-wmt19-en-de": 1024, + "stas/wmt19-ru-en": 1024, + "stas/wmt19-en-ru": 1024, + "stas/wmt19-de-en": 1024, + "stas/wmt19-en-de": 1024, } PRETRAINED_INIT_CONFIGURATION = { - "stas/fsmt-wmt19-ru-en": { + "stas/wmt19-ru-en": { "langs": ["ru", "en"], }, - "stas/fsmt-wmt19-en-ru": { + "stas/wmt19-en-ru": { "langs": ["en", "ru"], }, - "stas/fsmt-wmt19-de-en": { + "stas/wmt19-de-en": { "langs": ["de", "en"], }, - "stas/fsmt-wmt19-en-de": { + "stas/wmt19-en-de": { "langs": ["en", "de"], }, } From adc808482702a207108a386e35af860193aea81e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 12 Sep 2020 16:04:28 -0700 Subject: [PATCH 079/109] s/fsmt-wmt/wmt/ in model names (p2) --- tests/test_modeling_fsmt.py | 6 +++--- tests/test_tokenization_fsmt.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index f9ac1c3ceb09..4b3d5fd7fe7e 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -210,7 +210,7 @@ def test_tie_model_weights(self): # def test_auto_model(self): # # XXX: add a tiny model to s3? - # model_name = "stas/fsmt-wmt19-ru-en-tiny" + # model_name = "stas/wmt19-ru-en-tiny" # tiny = AutoModel.from_pretrained(model_name) # same vocab size # tok = AutoTokenizer.from_pretrained(model_name) # same tokenizer # inputs_dict = tok.batch_encode_plus(["Hello my friends"], return_tensors="pt") @@ -378,7 +378,7 @@ def _long_tensor(tok_lst): class FSMTModelIntegrationTests(unittest.TestCase): tokenizers_cache = {} models_cache = {} - default_mname = "stas/fsmt-wmt19-en-ru" + default_mname = "stas/wmt19-en-ru" @cached_property def default_tokenizer(self): @@ -438,7 +438,7 @@ def test_translation(self, pair): src, tgt = pair.split("-") print(f"Testing {src} -> {tgt}") - mname = f"stas/fsmt-wmt19-{pair}" + mname = f"stas/wmt19-{pair}" src_sentence = text[src] tgt_sentence = text[tgt] diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py index 276e0d1d4ee1..63a6b8dbfd88 100644 --- a/tests/test_tokenization_fsmt.py +++ b/tests/test_tokenization_fsmt.py @@ -79,11 +79,11 @@ def setUp(self): @cached_property def tokenizer_ru_en(self): - return FSMTTokenizer.from_pretrained("stas/fsmt-wmt19-ru-en") + return FSMTTokenizer.from_pretrained("stas/wmt19-ru-en") @cached_property def tokenizer_en_ru(self): - return FSMTTokenizer.from_pretrained("stas/fsmt-wmt19-en-ru") + return FSMTTokenizer.from_pretrained("stas/wmt19-en-ru") def test_full_tokenizer(self): """ Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt """ From 3bc81154eb7761a6c020493f69ea8eb8dd1b4a40 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 12 Sep 2020 16:07:13 -0700 Subject: [PATCH 080/109] s/fsmt-wmt/wmt/ in model names (p3) --- examples/seq2seq/test_fsmt_bleu_score.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/seq2seq/test_fsmt_bleu_score.py b/examples/seq2seq/test_fsmt_bleu_score.py index ea1d46c6efb8..d7c0fa2ad199 100644 --- a/examples/seq2seq/test_fsmt_bleu_score.py +++ b/examples/seq2seq/test_fsmt_bleu_score.py @@ -57,7 +57,7 @@ def get_model(self, mname): def test_bleu_scores(self, pair, min_bleu_score): # note: this test is not testing the best performance since it only evals a small batch # but it should be enough to detect a regression in the output quality - mname = f"stas/fsmt-wmt19-{pair}" + mname = f"stas/wmt19-{pair}" tokenizer = self.get_tokenizer(mname) model = self.get_model(mname) From 101b544664b4bec3e2353a47f287d25964897a88 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 12 Sep 2020 22:31:29 -0700 Subject: [PATCH 081/109] switch to a better checkpoint --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index f9b5b79fbe52..183dc2ccbd3e 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -125,9 +125,11 @@ # wmt19-de-en set -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_base_dr0.1_1/checkpoint_best.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-base +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_base_dr0.1_1/checkpoint_last3_avg.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-base + +PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_big_dr0.1_2/checkpoint_last3_avg.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-big + -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_big_dr0.1_2/checkpoint_best.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-big # upload From 37917b08db9a6baabd5ec48cb87322343c9a153d Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 14 Sep 2020 21:30:42 -0700 Subject: [PATCH 082/109] typo --- src/transformers/tokenization_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 1499742df7f9..8b4ed2001f2f 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tokenization classes for XLM.""" +"""Tokenization classes for FSMT.""" import json From 12ccdbb1147467aadd9e0f550dc31fc315b742b5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 14 Sep 2020 21:31:13 -0700 Subject: [PATCH 083/109] make non-optional args such - adjust tests where possible or skip when there is no other choice --- src/transformers/configuration_fsmt.py | 54 +++++++++++++------------- tests/test_modeling_fsmt.py | 11 +++++- tests/test_tokenization_fsmt.py | 10 +++++ 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index afd02c14527b..95de5269ea2d 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -34,57 +34,59 @@ FSMT_CONFIG_ARGS_DOC = r""" Args: - src_vocab_size (:obj:`int`, optional, defaults to None): + langs (:obj:`List[str]`): + source language, target_language (e.g. ['en', 'ru']) + src_vocab_size (:obj:`int`): defines the different tokens that can be represented by `inputs_ids` passed to the forward method in the encoder. - tgt_vocab_size (:obj:`int`, optional, defaults to None): + tgt_vocab_size (:obj:`int`): defines the different tokens that can be represented by `inputs_ids` passed to the forward method in the decoder. - d_model (:obj:`int`, optional, defaults to 1024): + d_model (:obj:`int`, `optional`, defaults to 1024): Dimensionality of the layers and the pooler layer. - encoder_layers (:obj:`int`, optional, defaults to 12): + encoder_layers (:obj:`int`, `optional`, defaults to 12): Number of encoder layers, 16 for pegasus, 6 for bart-base and marian - decoder_layers (:obj:`int`, optional, defaults to 12): + decoder_layers (:obj:`int`, `optional`, defaults to 12): Number of decoder layers, 16 for pegasus, 6 for bart-base and marian - encoder_attention_heads (:obj:`int`, optional, defaults to 16): + encoder_attention_heads (:obj:`int`, `optional`, defaults to 16): Number of attention heads for each attention layer in the Transformer encoder. - decoder_attention_heads (:obj:`int`, optional, defaults to 16): + decoder_attention_heads (:obj:`int`, `optional`, defaults to 16): Number of attention heads for each attention layer in the Transformer decoder. - decoder_ffn_dim (:obj:`int`, optional, defaults to 4096): + decoder_ffn_dim (:obj:`int`, `optional`, defaults to 4096): Dimensionality of the "intermediate" (i.e., feed-forward) layer in decoder. - encoder_ffn_dim (:obj:`int`, optional, defaults to 4096): + encoder_ffn_dim (:obj:`int`, `optional`, defaults to 4096): Dimensionality of the "intermediate" (i.e., feed-forward) layer in decoder. - activation_function (:obj:`str` or :obj:`function`, optional, defaults to "relu"): + activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to "relu"): The non-linear activation function (function or string) in the encoder and pooler. If string, "gelu", "relu", "swish" and "gelu_new" are supported. - dropout (:obj:`float`, optional, defaults to 0.1): + dropout (:obj:`float`, `optional`, defaults to 0.1): The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. - attention_dropout (:obj:`float`, optional, defaults to 0.0): + attention_dropout (:obj:`float`, `optional`, defaults to 0.0): The dropout ratio for the attention probabilities. - activation_dropout (:obj:`float`, optional, defaults to 0.0): + activation_dropout (:obj:`float`, `optional`, defaults to 0.0): The dropout ratio for activations inside the fully connected layer. - max_position_embeddings (:obj:`int`, optional, defaults to 1024): + max_position_embeddings (:obj:`int`, `optional`, defaults to 1024): The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). - init_std (:obj:`float`, optional, defaults to 0.02): + init_std (:obj:`float`, `optional`, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - scale_embedding (:obj:`bool`, optional, defaults to :obj:`True`): + scale_embedding (:obj:`bool`, `optional`, defaults to :obj:`True`): Scale embeddings by diving by sqrt(d_model). - bos_token_id (:obj:`int`, optional, defaults to 0) + bos_token_id (:obj:`int`, `optional`, defaults to 0) Beginning of stream token id. - pad_token_id (:obj:`int`, optional, defaults to 1) + pad_token_id (:obj:`int`, `optional`, defaults to 1) Padding token id. - eos_token_id (:obj:`int`, optional, defaults to 2) + eos_token_id (:obj:`int`, `optional`, defaults to 2) End of stream token id. decoder_start_token_id (:obj:`int`, `optional`): This model starts decoding with `eos_token_id` - encoder_layerdrop: (:obj:`float`, optional, defaults to 0.0): + encoder_layerdrop: (:obj:`float`, `optional`, defaults to 0.0): Google "layerdrop arxiv", as its not explainable in one line. - decoder_layerdrop: (:obj:`float`, optional, defaults to 0.0): + decoder_layerdrop: (:obj:`float`, `optional`, defaults to 0.0): Google "layerdrop arxiv", as its not explainable in one line. - is_encoder_decoder (:obj:`bool`, optional, defaults to :obj:`True`): + is_encoder_decoder (:obj:`bool`, `optional`, defaults to :obj:`True`): Whether this is an encoder/decoder model. - tie_word_embeddings (:obj:`bool`, optional, defaults to :obj:`False`): + tie_word_embeddings (:obj:`bool`, `optional`, defaults to :obj:`False`): Whether to tie input and output embeddings. """ @@ -112,9 +114,9 @@ class FSMTConfig(PretrainedConfig): # update the defaults from config file def __init__( self, - langs=None, - src_vocab_size=None, - tgt_vocab_size=None, + langs, + src_vocab_size, + tgt_vocab_size, activation_function="relu", d_model=1024, max_length=200, diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 4b3d5fd7fe7e..aa1439c37968 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -125,8 +125,15 @@ class FSMTModelTest(ModelTesterMixin, unittest.TestCase): def setUp(self): self.model_tester = ModelTester(self) - # XXX: hack to appease to all other models having vocab_size - self.config_tester = ConfigTester(self, config_class=FSMTConfig, vocab_size=99) + self.langs = ["en", "ru"] + config = { + "langs": self.langs, + "src_vocab_size": 10, + "tgt_vocab_size": 20, + } + # XXX: hack to appease to all other models requiring `vocab_size` + config["vocab_size"] = 99 # no such thing in FSMT + self.config_tester = ConfigTester(self, config_class=FSMTConfig, **config) def test_config(self): self.config_tester.run_common_tests() diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py index 63a6b8dbfd88..eef13c828be6 100644 --- a/tests/test_tokenization_fsmt.py +++ b/tests/test_tokenization_fsmt.py @@ -61,6 +61,8 @@ def setUp(self): self.langs = ["en", "ru"] config = { "langs": self.langs, + "src_vocab_size": 10, + "tgt_vocab_size": 20, } self.src_vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["src_vocab_file"]) @@ -136,3 +138,11 @@ def test_match_encode_decode(self): # and decode backward, using the reversed languages model decoded_text = tokenizer_dec.decode(input_ids, skip_special_tokens=True) self.assertEqual(decoded_text, src_text) + + @unittest.skip("FSMTConfig.__init__ requires non-optional args") + def test_torch_encode_plus_sent_to_model(self): + pass + + @unittest.skip("FSMTConfig.__init__ requires non-optional args") + def test_np_encode_plus_sent_to_model(self): + pass From 2b3de7afb744d6900062963fb09660b0431ef03e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 14 Sep 2020 21:35:09 -0700 Subject: [PATCH 084/109] consistency --- src/transformers/configuration_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/configuration_auto.py b/src/transformers/configuration_auto.py index 2e961080d99a..9f429e3307f3 100644 --- a/src/transformers/configuration_auto.py +++ b/src/transformers/configuration_auto.py @@ -129,7 +129,7 @@ ("longformer", "Longformer"), ("roberta", "RoBERTa"), ("flaubert", "FlauBERT"), - ("fsmt", "FSMT"), + ("fsmt", "FairSeq Machine-Translation"), ("bert", "BERT"), ("openai-gpt", "OpenAI GPT"), ("gpt2", "OpenAI GPT-2"), From aeca7c6364b84d843c490fab8929cd61ac1ba453 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 14 Sep 2020 21:51:24 -0700 Subject: [PATCH 085/109] style --- tests/test_modeling_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index aa1439c37968..9382326fd0a9 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -132,7 +132,7 @@ def setUp(self): "tgt_vocab_size": 20, } # XXX: hack to appease to all other models requiring `vocab_size` - config["vocab_size"] = 99 # no such thing in FSMT + config["vocab_size"] = 99 # no such thing in FSMT self.config_tester = ConfigTester(self, config_class=FSMTConfig, **config) def test_config(self): From 4587b6ae033ef879712d984d914be1878aca154e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 14 Sep 2020 21:54:16 -0700 Subject: [PATCH 086/109] adjust header --- docs/source/model_doc/fsmt.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/source/model_doc/fsmt.rst b/docs/source/model_doc/fsmt.rst index c8af02bf931c..ea3b25d55af0 100644 --- a/docs/source/model_doc/fsmt.rst +++ b/docs/source/model_doc/fsmt.rst @@ -7,17 +7,13 @@ file a `Github Issue `__: +FSMT (FairSeq MachineTranslation) models were introduce in "Facebook FAIR's WMT19 News Translation Task Submission" __ by Nathan Ng, Kyra Yee, Alexei Baevski, Myle Ott, Michael Auli, Sergey Edunov. -Facebook FAIR's WMT19 News Translation Task Submission - -Nathan Ng, Kyra Yee, Alexei Baevski, Myle Ott, Michael Auli, Sergey Edunov +The abstract of the paper is the following: This paper describes Facebook FAIR's submission to the WMT19 shared news translation task. We participate in two language pairs and four language directions, English <-> German and English <-> Russian. Following our submission from last year, our baseline systems are large BPE-based transformer models trained with the Fairseq sequence modeling toolkit which rely on sampled back-translations. This year we experiment with different bitext data filtering schemes, as well as with adding filtered back-translated data. We also ensemble and fine-tune our models on domain-specific data, then decode using noisy channel model reranking. Our submissions are ranked first in all four directions of the human evaluation campaign. On En->De, our system significantly outperforms other systems as well as human translations. This system improves upon our WMT'18 submission by 4.5 BLEU points. -The Authors' code can be found `here `__. - - +The original code can be found here __. Implementation Notes ~~~~~~~~~~~~~~~~~~~~ From 1b70a5bab0fc857687b29950a4935a748acc914e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 10:40:02 -0700 Subject: [PATCH 087/109] cards moved (model rename) --- model_cards/stas/wmt19-de-en/README.md | 92 ++++++++++++++++++++++++++ model_cards/stas/wmt19-en-de/README.md | 92 ++++++++++++++++++++++++++ model_cards/stas/wmt19-en-ru/README.md | 92 ++++++++++++++++++++++++++ model_cards/stas/wmt19-ru-en/README.md | 92 ++++++++++++++++++++++++++ 4 files changed, 368 insertions(+) create mode 100644 model_cards/stas/wmt19-de-en/README.md create mode 100644 model_cards/stas/wmt19-en-de/README.md create mode 100644 model_cards/stas/wmt19-en-ru/README.md create mode 100644 model_cards/stas/wmt19-ru-en/README.md diff --git a/model_cards/stas/wmt19-de-en/README.md b/model_cards/stas/wmt19-de-en/README.md new file mode 100644 index 000000000000..29221eed0ca7 --- /dev/null +++ b/model_cards/stas/wmt19-de-en/README.md @@ -0,0 +1,92 @@ + +--- + + + +language: de, en +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# FSMT + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for de-en. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "stas/wmt19-de-en" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +input = "Maschinelles Lernen ist großartig, oder? +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Machine learning is great, isn't it? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +pair | fairseq | transformers +-------|---------|---------- +de-en | [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) | 41.18 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). + + +The score was calculated using this code: + +```bash +git clone https://github.com/huggingface/transformers +cd transformers +export PAIR=de-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +export NUM_BEAMS=50 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + diff --git a/model_cards/stas/wmt19-en-de/README.md b/model_cards/stas/wmt19-en-de/README.md new file mode 100644 index 000000000000..d411fd30982d --- /dev/null +++ b/model_cards/stas/wmt19-en-de/README.md @@ -0,0 +1,92 @@ + +--- + + + +language: en, de +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# FSMT + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-de. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "stas/wmt19-en-de" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +input = "Machine learning is great, isn't it? +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Maschinelles Lernen ist großartig, oder? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +pair | fairseq | transformers +-------|---------|---------- +en-de | [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) | 42.79 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). + + +The score was calculated using this code: + +```bash +git clone https://github.com/huggingface/transformers +cd transformers +export PAIR=en-de +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +export NUM_BEAMS=50 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + diff --git a/model_cards/stas/wmt19-en-ru/README.md b/model_cards/stas/wmt19-en-ru/README.md new file mode 100644 index 000000000000..fbbf0d93a04d --- /dev/null +++ b/model_cards/stas/wmt19-en-ru/README.md @@ -0,0 +1,92 @@ + +--- + + + +language: en, ru +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# FSMT + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-ru. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "stas/wmt19-en-ru" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +input = "Machine learning is great, isn't it? +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Машинное обучение - это здорово, не так ли? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +pair | fairseq | transformers +-------|---------|---------- +en-ru | [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) | 33.47 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). + + +The score was calculated using this code: + +```bash +git clone https://github.com/huggingface/transformers +cd transformers +export PAIR=en-ru +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +export NUM_BEAMS=50 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + diff --git a/model_cards/stas/wmt19-ru-en/README.md b/model_cards/stas/wmt19-ru-en/README.md new file mode 100644 index 000000000000..9042113d7744 --- /dev/null +++ b/model_cards/stas/wmt19-ru-en/README.md @@ -0,0 +1,92 @@ + +--- + + + +language: ru, en +thumbnail: +tags: +- translation +- wmt19 +license: Apache 2.0 +datasets: +- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) +metrics: +- http://www.statmt.org/wmt19/metrics-task.html +--- + +# FSMT + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for ru-en. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "stas/wmt19-ru-en" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +input = "Машинное обучение - это здорово, не так ли? +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Machine learning is great, isn't it? + +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) + +## Eval results + +pair | fairseq | transformers +-------|---------|---------- +ru-en | [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) | 39.20 + + +`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). + + +The score was calculated using this code: + +```bash +git clone https://github.com/huggingface/transformers +cd transformers +export PAIR=ru-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +export NUM_BEAMS=50 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +``` + +## TODO + +- port model ensemble (fairseq uses 4 model checkpoints) + From 062cfe016fc96558dd2908ce0446b6da4d3a2061 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 10:49:10 -0700 Subject: [PATCH 088/109] use best custom hparams --- src/transformers/configuration_fsmt.py | 19 +++++- ..._original_pytorch_checkpoint_to_pytorch.py | 58 +++++++++++++++---- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 95de5269ea2d..d2457a5001e3 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -88,6 +88,15 @@ Whether this is an encoder/decoder model. tie_word_embeddings (:obj:`bool`, `optional`, defaults to :obj:`False`): Whether to tie input and output embeddings. + num_beams (:obj:`int`, `optional`, defaults to 5) + Number of beams for beam search that will be used by default in the :obj:`generate` method + of the model. 1 means no beam search. + length_penalty (:obj:`float`, `optional`, defaults to 1) + Exponential penalty to the length that will be used by default in the :obj:`generate` method + of the model. + early_stopping (:obj:`bool`, `optional`, defaults to :obj:`False`) + Flag that will be used by default in the :obj:`generate` method of the model. Whether to stop + the beam search when at least ``num_beams`` sentences are finished per batch or not. """ @@ -120,7 +129,6 @@ def __init__( activation_function="relu", d_model=1024, max_length=200, - num_beams=8, max_position_embeddings=1024, encoder_ffn_dim=4096, encoder_layers=12, @@ -141,6 +149,9 @@ def __init__( is_encoder_decoder=True, scale_embedding=True, tie_word_embeddings=False, + num_beams=5, + length_penalty=1.0, + early_stopping=False, **common_kwargs ): r""" @@ -170,7 +181,7 @@ def __init__( self.tgt_vocab_size = tgt_vocab_size self.d_model = d_model # encoder_embed_dim and decoder_embed_dim self.max_length = max_length - self.num_beams = num_beams + self.encoder_ffn_dim = encoder_ffn_dim self.encoder_layers = self.num_hidden_layers = encoder_layers self.encoder_attention_heads = encoder_attention_heads @@ -183,6 +194,10 @@ def __init__( self.init_std = init_std # Normal(0, this parameter) self.activation_function = activation_function + self.num_beams = num_beams + self.length_penalty = length_penalty + self.early_stopping = early_stopping + self.decoder = DecoderConfig(vocab_size=tgt_vocab_size, bos_token_id=eos_token_id) self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 183dc2ccbd3e..2abd23f7eb55 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -142,6 +142,11 @@ cd - + +# if updating just small files and not the large models, here is a script to generate the right commands: +perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for ("wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1", "wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json +# add/remove files as needed + # XXX: move into model card git clone https://github.com/huggingface/transformers @@ -233,6 +238,26 @@ json_indent = 2 +# based on the results of a search on a range of `num_beams`, `length_penalty` and `early_stopping` +# values against wmt19 test data to obtain the best BLEU scores, we will use the following defaults: +# +# * `num_beams`: 5 (higher scores better, but requires more memory/is slower, can be adjusted by users) +# * `early_stopping`: `False` consistently scored better +# * `length_penalty` varied, so will assign the best one depending on the model +best_score_hparams = { + # fairseq: + "wmt19-ru-en": {"length_penalty": 1.1}, + "wmt19-en-ru": {"length_penalty": 1.15}, + "wmt19-en-de": {"length_penalty": 1.0}, + "wmt19-de-en": {"length_penalty": 1.1}, + # allen-nlp: + "wmt16-en-de-dist-12-1": {"length_penalty": 0.6}, + "wmt16-en-de-dist-6-1": {"length_penalty": 0.6}, + "wmt16-en-de-12-1": {"length_penalty": 0.8}, + "wmt19-de-en-6-6-base": {"length_penalty": 0.6}, + "wmt19-de-en-6-6-big": {"length_penalty": 0.6}, +} + def rewrite_dict_keys(d): # (1) remove word breaking symbol, (2) add word ending symbol where the word is not broken up, @@ -257,10 +282,10 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): # BLUE scores as follows: # "pair": [fairseq, transformers] scores = { - "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "33.29"], - "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "38.93"], - "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "41.18"], - "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "42.79"], + "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "39.20"], + "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "33.47"], + "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "42.83"], + "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "41.35"], } pair = f"{src_lang}-{tgt_lang}" @@ -323,7 +348,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): ## Training data -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). ## Eval results @@ -331,9 +356,9 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): -------|---------|---------- {pair} | {scores[pair][0]} | {scores[pair][1]} - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). - +The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: +- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). +- re-ranking The score was calculated using this code: @@ -344,13 +369,15 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 -export NUM_BEAMS=50 +export NUM_BEAMS=15 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py {ORG_NAME}/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` +note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. + ## TODO @@ -465,6 +492,14 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder "tie_word_embeddings": args["share_all_embeddings"], } + # good hparam defaults to start with + model_conf["num_beams"] = 5 + model_conf["early_stopping"] = False + if model_dir in best_score_hparams and "length_penalty" in best_score_hparams[model_dir]: + model_conf["length_penalty"] = best_score_hparams[model_dir]["length_penalty"] + else: + model_conf["length_penalty"] = 1.0 + print(f"Generating {fsmt_model_config_file}") with open(fsmt_model_config_file, "w", encoding="utf-8") as f: f.write(json.dumps(model_conf, ensure_ascii=False, indent=json_indent)) @@ -519,8 +554,9 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder print("\nLast step is to upload the files to s3") print(f"cd {data_root}") print(f"transformers-cli upload {model_dir}") - # XXX: this is invalid - waiting on issue to be resolved - print("Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to force redownload") + print( + "Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to use the non-cached version" + ) if __name__ == "__main__": From 5963a356534675f27ca98ef42ae2c413836cd8fe Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 10:49:38 -0700 Subject: [PATCH 089/109] update info --- model_cards/stas/wmt19-de-en/README.md | 14 ++++++++------ model_cards/stas/wmt19-en-de/README.md | 14 ++++++++------ model_cards/stas/wmt19-en-ru/README.md | 12 +++++++----- model_cards/stas/wmt19-ru-en/README.md | 12 +++++++----- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/model_cards/stas/wmt19-de-en/README.md b/model_cards/stas/wmt19-de-en/README.md index 29221eed0ca7..9f260bb0b23f 100644 --- a/model_cards/stas/wmt19-de-en/README.md +++ b/model_cards/stas/wmt19-de-en/README.md @@ -57,17 +57,17 @@ print(decoded) # Machine learning is great, isn't it? ## Training data -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). ## Eval results pair | fairseq | transformers -------|---------|---------- -de-en | [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) | 41.18 - - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). +de-en | [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) | 41.35 +The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: +- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). +- re-ranking The score was calculated using this code: @@ -78,13 +78,15 @@ export PAIR=de-en export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 -export NUM_BEAMS=50 +export NUM_BEAMS=15 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` +note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. + ## TODO diff --git a/model_cards/stas/wmt19-en-de/README.md b/model_cards/stas/wmt19-en-de/README.md index d411fd30982d..5b168f47ce6b 100644 --- a/model_cards/stas/wmt19-en-de/README.md +++ b/model_cards/stas/wmt19-en-de/README.md @@ -57,17 +57,17 @@ print(decoded) # Maschinelles Lernen ist großartig, oder? ## Training data -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). ## Eval results pair | fairseq | transformers -------|---------|---------- -en-de | [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) | 42.79 - - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). +en-de | [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) | 42.83 +The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: +- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). +- re-ranking The score was calculated using this code: @@ -78,13 +78,15 @@ export PAIR=en-de export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 -export NUM_BEAMS=50 +export NUM_BEAMS=15 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` +note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. + ## TODO diff --git a/model_cards/stas/wmt19-en-ru/README.md b/model_cards/stas/wmt19-en-ru/README.md index fbbf0d93a04d..611ab36297a2 100644 --- a/model_cards/stas/wmt19-en-ru/README.md +++ b/model_cards/stas/wmt19-en-ru/README.md @@ -57,7 +57,7 @@ print(decoded) # Машинное обучение - это здорово, не ## Training data -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). ## Eval results @@ -65,9 +65,9 @@ pair | fairseq | transformers -------|---------|---------- en-ru | [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) | 33.47 - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). - +The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: +- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). +- re-ranking The score was calculated using this code: @@ -78,13 +78,15 @@ export PAIR=en-ru export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 -export NUM_BEAMS=50 +export NUM_BEAMS=15 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` +note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. + ## TODO diff --git a/model_cards/stas/wmt19-ru-en/README.md b/model_cards/stas/wmt19-ru-en/README.md index 9042113d7744..b38349d98f21 100644 --- a/model_cards/stas/wmt19-ru-en/README.md +++ b/model_cards/stas/wmt19-ru-en/README.md @@ -57,7 +57,7 @@ print(decoded) # Machine learning is great, isn't it? ## Training data -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). ## Eval results @@ -65,9 +65,9 @@ pair | fairseq | transformers -------|---------|---------- ru-en | [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) | 39.20 - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). - +The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: +- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). +- re-ranking The score was calculated using this code: @@ -78,13 +78,15 @@ export PAIR=ru-en export DATA_DIR=data/$PAIR export SAVE_DIR=data/$PAIR export BS=8 -export NUM_BEAMS=50 +export NUM_BEAMS=15 mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` +note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. + ## TODO From a6bcd94aee3397cc4bc35f7cd45c851cf9fdf936 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 10:51:13 -0700 Subject: [PATCH 090/109] remove old cards --- model_cards/stas/fsmt-wmt19-de-en/README.md | 92 --------------------- model_cards/stas/fsmt-wmt19-en-de/README.md | 92 --------------------- model_cards/stas/fsmt-wmt19-en-ru/README.md | 92 --------------------- model_cards/stas/fsmt-wmt19-ru-en/README.md | 92 --------------------- 4 files changed, 368 deletions(-) delete mode 100644 model_cards/stas/fsmt-wmt19-de-en/README.md delete mode 100644 model_cards/stas/fsmt-wmt19-en-de/README.md delete mode 100644 model_cards/stas/fsmt-wmt19-en-ru/README.md delete mode 100644 model_cards/stas/fsmt-wmt19-ru-en/README.md diff --git a/model_cards/stas/fsmt-wmt19-de-en/README.md b/model_cards/stas/fsmt-wmt19-de-en/README.md deleted file mode 100644 index 9da64b82f9b0..000000000000 --- a/model_cards/stas/fsmt-wmt19-de-en/README.md +++ /dev/null @@ -1,92 +0,0 @@ - ---- - - - -language: de, en -thumbnail: -tags: -- translation -- wmt19 -license: Apache 2.0 -datasets: -- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) -metrics: -- http://www.statmt.org/wmt19/metrics-task.html ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for de-en. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) -* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) -* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) -* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers.tokenization_fsmt import FSMTTokenizer -from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/fsmt-wmt19-de-en" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Maschinelles Lernen ist großartig, oder? -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Machine learning is great, isn't it? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -de-en | [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) | 41.18 - - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). - - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=de-en -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=50 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - diff --git a/model_cards/stas/fsmt-wmt19-en-de/README.md b/model_cards/stas/fsmt-wmt19-en-de/README.md deleted file mode 100644 index 211bbb90343e..000000000000 --- a/model_cards/stas/fsmt-wmt19-en-de/README.md +++ /dev/null @@ -1,92 +0,0 @@ - ---- - - - -language: en, de -thumbnail: -tags: -- translation -- wmt19 -license: Apache 2.0 -datasets: -- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) -metrics: -- http://www.statmt.org/wmt19/metrics-task.html ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-de. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) -* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) -* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) -* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers.tokenization_fsmt import FSMTTokenizer -from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/fsmt-wmt19-en-de" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Machine learning is great, isn't it? -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Maschinelles Lernen ist großartig, oder? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -en-de | [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) | 42.79 - - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). - - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-de -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=50 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - diff --git a/model_cards/stas/fsmt-wmt19-en-ru/README.md b/model_cards/stas/fsmt-wmt19-en-ru/README.md deleted file mode 100644 index a73fed2130df..000000000000 --- a/model_cards/stas/fsmt-wmt19-en-ru/README.md +++ /dev/null @@ -1,92 +0,0 @@ - ---- - - - -language: en, ru -thumbnail: -tags: -- translation -- wmt19 -license: Apache 2.0 -datasets: -- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) -metrics: -- http://www.statmt.org/wmt19/metrics-task.html ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-ru. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) -* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) -* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) -* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers.tokenization_fsmt import FSMTTokenizer -from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/fsmt-wmt19-en-ru" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Machine learning is great, isn't it? -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Машинное обучение - это здорово, не так ли? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -en-ru | [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) | 33.29 - - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). - - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-ru -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=50 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - diff --git a/model_cards/stas/fsmt-wmt19-ru-en/README.md b/model_cards/stas/fsmt-wmt19-ru-en/README.md deleted file mode 100644 index bd2678dd3cbe..000000000000 --- a/model_cards/stas/fsmt-wmt19-ru-en/README.md +++ /dev/null @@ -1,92 +0,0 @@ - ---- - - - -language: ru, en -thumbnail: -tags: -- translation -- wmt19 -license: Apache 2.0 -datasets: -- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) -metrics: -- http://www.statmt.org/wmt19/metrics-task.html ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for ru-en. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [fsmt-wmt19-en-ru](https://huggingface.co/stas/fsmt-wmt19-en-ru) -* [fsmt-wmt19-ru-en](https://huggingface.co/stas/fsmt-wmt19-ru-en) -* [fsmt-wmt19-en-de](https://huggingface.co/stas/fsmt-wmt19-en-de) -* [fsmt-wmt19-de-en](https://huggingface.co/stas/fsmt-wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers.tokenization_fsmt import FSMTTokenizer -from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/fsmt-wmt19-ru-en" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Машинное обучение - это здорово, не так ли? -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Machine learning is great, isn't it? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616) - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -ru-en | [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) | 38.93 - - -`transformers`` currently doesn't support model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). - - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=ru-en -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=50 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/fsmt-wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - From 139edb14370bc6aa33c98b2af90064c5128666d1 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 10:53:11 -0700 Subject: [PATCH 091/109] cleanup --- tests/test_tokenization_fsmt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py index eef13c828be6..4eaa64d4a8cd 100644 --- a/tests/test_tokenization_fsmt.py +++ b/tests/test_tokenization_fsmt.py @@ -71,7 +71,6 @@ def setUp(self): self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"]) with open(self.src_vocab_file, "w") as fp: fp.write(json.dumps(vocab_tokens)) - # XXX: ru content with open(self.tgt_vocab_file, "w") as fp: fp.write(json.dumps(vocab_tokens)) with open(self.merges_file, "w") as fp: From a0dda2d8cf172cc8a2eac6e33129614b375dfb4a Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 11:17:52 -0700 Subject: [PATCH 092/109] s/stas/facebook/ --- examples/seq2seq/test_fsmt_bleu_score.py | 2 +- .../{stas => facebook}/wmt19-de-en/README.md | 12 +++--- .../{stas => facebook}/wmt19-en-de/README.md | 12 +++--- .../{stas => facebook}/wmt19-en-ru/README.md | 12 +++--- .../{stas => facebook}/wmt19-ru-en/README.md | 12 +++--- src/transformers/configuration_fsmt.py | 10 ++--- ..._original_pytorch_checkpoint_to_pytorch.py | 31 +++++++++----- src/transformers/modeling_fsmt.py | 12 +++--- src/transformers/tokenization_fsmt.py | 40 +++++++++---------- tests/test_modeling_fsmt.py | 6 +-- tests/test_tokenization_fsmt.py | 4 +- 11 files changed, 82 insertions(+), 71 deletions(-) rename model_cards/{stas => facebook}/wmt19-de-en/README.md (85%) rename model_cards/{stas => facebook}/wmt19-en-de/README.md (85%) rename model_cards/{stas => facebook}/wmt19-en-ru/README.md (85%) rename model_cards/{stas => facebook}/wmt19-ru-en/README.md (85%) diff --git a/examples/seq2seq/test_fsmt_bleu_score.py b/examples/seq2seq/test_fsmt_bleu_score.py index d7c0fa2ad199..3d4eb2915dfe 100644 --- a/examples/seq2seq/test_fsmt_bleu_score.py +++ b/examples/seq2seq/test_fsmt_bleu_score.py @@ -57,7 +57,7 @@ def get_model(self, mname): def test_bleu_scores(self, pair, min_bleu_score): # note: this test is not testing the best performance since it only evals a small batch # but it should be enough to detect a regression in the output quality - mname = f"stas/wmt19-{pair}" + mname = f"facebook/wmt19-{pair}" tokenizer = self.get_tokenizer(mname) model = self.get_model(mname) diff --git a/model_cards/stas/wmt19-de-en/README.md b/model_cards/facebook/wmt19-de-en/README.md similarity index 85% rename from model_cards/stas/wmt19-de-en/README.md rename to model_cards/facebook/wmt19-de-en/README.md index 9f260bb0b23f..681255b6cb72 100644 --- a/model_cards/stas/wmt19-de-en/README.md +++ b/model_cards/facebook/wmt19-de-en/README.md @@ -27,10 +27,10 @@ The abbreviation FSMT stands for FairSeqMachineTranslation All four models are available: -* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) +* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) ## Intended uses & limitations @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/wmt19-de-en" +mname = "facebook/wmt19-de-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) @@ -83,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. diff --git a/model_cards/stas/wmt19-en-de/README.md b/model_cards/facebook/wmt19-en-de/README.md similarity index 85% rename from model_cards/stas/wmt19-en-de/README.md rename to model_cards/facebook/wmt19-en-de/README.md index 5b168f47ce6b..600947e1f972 100644 --- a/model_cards/stas/wmt19-en-de/README.md +++ b/model_cards/facebook/wmt19-en-de/README.md @@ -27,10 +27,10 @@ The abbreviation FSMT stands for FairSeqMachineTranslation All four models are available: -* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) +* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) ## Intended uses & limitations @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/wmt19-en-de" +mname = "facebook/wmt19-en-de" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) @@ -83,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. diff --git a/model_cards/stas/wmt19-en-ru/README.md b/model_cards/facebook/wmt19-en-ru/README.md similarity index 85% rename from model_cards/stas/wmt19-en-ru/README.md rename to model_cards/facebook/wmt19-en-ru/README.md index 611ab36297a2..96c09cd7a5dc 100644 --- a/model_cards/stas/wmt19-en-ru/README.md +++ b/model_cards/facebook/wmt19-en-ru/README.md @@ -27,10 +27,10 @@ The abbreviation FSMT stands for FairSeqMachineTranslation All four models are available: -* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) +* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) ## Intended uses & limitations @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/wmt19-en-ru" +mname = "facebook/wmt19-en-ru" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) @@ -83,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. diff --git a/model_cards/stas/wmt19-ru-en/README.md b/model_cards/facebook/wmt19-ru-en/README.md similarity index 85% rename from model_cards/stas/wmt19-ru-en/README.md rename to model_cards/facebook/wmt19-ru-en/README.md index b38349d98f21..7938794c567d 100644 --- a/model_cards/stas/wmt19-ru-en/README.md +++ b/model_cards/facebook/wmt19-ru-en/README.md @@ -27,10 +27,10 @@ The abbreviation FSMT stands for FairSeqMachineTranslation All four models are available: -* [wmt19-en-ru](https://huggingface.co/stas/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/stas/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/stas/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/stas/wmt19-de-en) +* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) ## Intended uses & limitations @@ -39,7 +39,7 @@ All four models are available: ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "stas/wmt19-ru-en" +mname = "facebook/wmt19-ru-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) @@ -83,7 +83,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index d2457a5001e3..9459fe5d1974 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -25,10 +25,10 @@ logger = logging.getLogger(__name__) FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/config.json", - "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/config.json", - "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/config.json", - "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/config.json", + "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/config.json", + "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/config.json", + "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/config.json", + "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/config.json", } @@ -161,7 +161,7 @@ def __init__( >>> from transformers import FSMTConfig, FSMTModel - >>> config = FSMTConfig.from_pretrained('stas/wmt19-en-ru') + >>> config = FSMTConfig.from_pretrained('facebook/wmt19-en-ru') >>> model = FSMTModel(config) """ diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 2abd23f7eb55..a864de2f61e9 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -234,8 +234,6 @@ logging.basicConfig(level=logging.INFO) -ORG_NAME = "stas" # XXX: will become facebook - json_indent = 2 # based on the results of a search on a range of `num_beams`, `length_penalty` and `early_stopping` @@ -258,6 +256,18 @@ "wmt19-de-en-6-6-big": {"length_penalty": 0.6}, } +org_names = {} +for m in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]: + org_names[m] = "facebook" +for m in [ + "wmt16-en-de-dist-12-1", + "wmt16-en-de-dist-6-1", + "wmt16-en-de-12-1", + "wmt19-de-en-6-6-base", + "wmt19-de-en-6-6-big", +]: + org_names[m] = "allen_nlp" + def rewrite_dict_keys(d): # (1) remove word breaking symbol, (2) add word ending symbol where the word is not broken up, @@ -318,10 +328,10 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): All four models are available: -* [wmt19-en-ru](https://huggingface.co/{ORG_NAME}/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/{ORG_NAME}/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/{ORG_NAME}/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/{ORG_NAME}/wmt19-de-en) +* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) ## Intended uses & limitations @@ -330,7 +340,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): ```python from transformers.tokenization_fsmt import FSMTTokenizer from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "{ORG_NAME}/wmt19-{src_lang}-{tgt_lang}" +mname = "facebook/wmt19-{src_lang}-{tgt_lang}" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) @@ -374,7 +384,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang): sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py {ORG_NAME}/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS ``` note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. @@ -546,8 +556,9 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder torch.save(model_state_dict, pytorch_weights_dump_path) # model card - model_card_dir = os.path.join(proj_root, "model_cards", ORG_NAME, model_dir) - print(f"Generating model_card {src_lang}-{tgt_lang}") + org_name = org_names[model_dir] if model_dir in org_names else "stas" + model_card_dir = os.path.join(proj_root, "model_cards", org_name, model_dir) + print(f"Generating {model_card_dir}") write_model_card(model_card_dir, src_lang, tgt_lang) print("Conversion is done!") diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 1810ea8bb549..e5e406c6ffde 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -120,7 +120,7 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 36.4 http://matrix.statmt.org/matrix/output/1914?score_id=37605) @@ -135,7 +135,7 @@ mkdir -p $DATA_DIR sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937) @@ -152,7 +152,7 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 42.3 http://matrix.statmt.org/matrix/output/1902?run_id=6750) @@ -168,7 +168,7 @@ sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py stas/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS # (fairseq BLEU: 43.1 http://matrix.statmt.org/matrix/output/1909?run_id=6862) @@ -191,7 +191,7 @@ from transformers import FSMTTokenizer, FSMTForConditionalGeneration - mname = "stas/wmt19-ru-en" + mname = "facebook/wmt19-ru-en" model = FSMTForConditionalGeneration.from_pretrained(mname) tokenizer = FSMTTokenizer.from_pretrained(mname) @@ -894,7 +894,7 @@ def __init__(self, config: FSMTConfig): @add_start_docstrings_to_callable(FSMT_INPUTS_DOCSTRING) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, - checkpoint="stas/wmt19-ru-en", + checkpoint="facebook/wmt19-ru-en", output_type=BaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, ) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 8b4ed2001f2f..0e3a10af227a 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -39,43 +39,43 @@ PRETRAINED_VOCAB_FILES_MAP = { "src_vocab_file": { - "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/vocab-src.json", - "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/vocab-src.json", - "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/vocab-src.json", - "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/vocab-src.json", + "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/vocab-src.json", + "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/vocab-src.json", + "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/vocab-src.json", + "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/vocab-src.json", }, "tgt_vocab_file": { - "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/vocab-tgt.json", - "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/vocab-tgt.json", - "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/vocab-tgt.json", - "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/vocab-tgt.json", + "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/vocab-tgt.json", + "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/vocab-tgt.json", + "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/vocab-tgt.json", + "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/vocab-tgt.json", }, "merges_file": { - "stas/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-ru-en/merges.txt", - "stas/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-ru/merges.txt", - "stas/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-de-en/merges.txt", - "stas/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/stas/wmt19-en-de/merges.txt", + "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/merges.txt", + "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/merges.txt", + "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/merges.txt", + "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/merges.txt", }, } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - "stas/wmt19-ru-en": 1024, - "stas/wmt19-en-ru": 1024, - "stas/wmt19-de-en": 1024, - "stas/wmt19-en-de": 1024, + "facebook/wmt19-ru-en": 1024, + "facebook/wmt19-en-ru": 1024, + "facebook/wmt19-de-en": 1024, + "facebook/wmt19-en-de": 1024, } PRETRAINED_INIT_CONFIGURATION = { - "stas/wmt19-ru-en": { + "facebook/wmt19-ru-en": { "langs": ["ru", "en"], }, - "stas/wmt19-en-ru": { + "facebook/wmt19-en-ru": { "langs": ["en", "ru"], }, - "stas/wmt19-de-en": { + "facebook/wmt19-de-en": { "langs": ["de", "en"], }, - "stas/wmt19-en-de": { + "facebook/wmt19-en-de": { "langs": ["en", "de"], }, } diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index 9382326fd0a9..d6e0c3eb7df1 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -217,7 +217,7 @@ def test_tie_model_weights(self): # def test_auto_model(self): # # XXX: add a tiny model to s3? - # model_name = "stas/wmt19-ru-en-tiny" + # model_name = "facebook/wmt19-ru-en-tiny" # tiny = AutoModel.from_pretrained(model_name) # same vocab size # tok = AutoTokenizer.from_pretrained(model_name) # same tokenizer # inputs_dict = tok.batch_encode_plus(["Hello my friends"], return_tensors="pt") @@ -385,7 +385,7 @@ def _long_tensor(tok_lst): class FSMTModelIntegrationTests(unittest.TestCase): tokenizers_cache = {} models_cache = {} - default_mname = "stas/wmt19-en-ru" + default_mname = "facebook/wmt19-en-ru" @cached_property def default_tokenizer(self): @@ -445,7 +445,7 @@ def test_translation(self, pair): src, tgt = pair.split("-") print(f"Testing {src} -> {tgt}") - mname = f"stas/wmt19-{pair}" + mname = f"facebook/wmt19-{pair}" src_sentence = text[src] tgt_sentence = text[tgt] diff --git a/tests/test_tokenization_fsmt.py b/tests/test_tokenization_fsmt.py index 4eaa64d4a8cd..c3e08d566ad4 100644 --- a/tests/test_tokenization_fsmt.py +++ b/tests/test_tokenization_fsmt.py @@ -80,11 +80,11 @@ def setUp(self): @cached_property def tokenizer_ru_en(self): - return FSMTTokenizer.from_pretrained("stas/wmt19-ru-en") + return FSMTTokenizer.from_pretrained("facebook/wmt19-ru-en") @cached_property def tokenizer_en_ru(self): - return FSMTTokenizer.from_pretrained("stas/wmt19-en-ru") + return FSMTTokenizer.from_pretrained("facebook/wmt19-en-ru") def test_full_tokenizer(self): """ Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt """ From 7d3058e60cdaf785b9a8d5e0116f7db312e9b5eb Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 11:57:33 -0700 Subject: [PATCH 093/109] update scores --- examples/seq2seq/test_fsmt_bleu_score.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/seq2seq/test_fsmt_bleu_score.py b/examples/seq2seq/test_fsmt_bleu_score.py index 3d4eb2915dfe..95f475698f6f 100644 --- a/examples/seq2seq/test_fsmt_bleu_score.py +++ b/examples/seq2seq/test_fsmt_bleu_score.py @@ -47,10 +47,10 @@ def get_model(self, mname): @parameterized.expand( [ - ["en-ru", 28.21], - ["ru-en", 23.49], - ["en-de", 22.11], - ["de-en", 29.31], + ["en-ru", 26.0], + ["ru-en", 22.0], + ["en-de", 22.0], + ["de-en", 29.0], ] ) @slow From a5fb882354cb2ee282c18987e509d9ceb8f5b1e8 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 12:00:47 -0700 Subject: [PATCH 094/109] s/allen_nlp/allenai/ --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index a864de2f61e9..1340c25b96fb 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -248,7 +248,7 @@ "wmt19-en-ru": {"length_penalty": 1.15}, "wmt19-en-de": {"length_penalty": 1.0}, "wmt19-de-en": {"length_penalty": 1.1}, - # allen-nlp: + # allenai: "wmt16-en-de-dist-12-1": {"length_penalty": 0.6}, "wmt16-en-de-dist-6-1": {"length_penalty": 0.6}, "wmt16-en-de-12-1": {"length_penalty": 0.8}, @@ -266,7 +266,7 @@ "wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big", ]: - org_names[m] = "allen_nlp" + org_names[m] = "allenai" def rewrite_dict_keys(d): From 7f367379652830425424f17bed4bed05559d7fde Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 12:00:55 -0700 Subject: [PATCH 095/109] url maps aren't needed --- src/transformers/configuration_fsmt.py | 7 +--- src/transformers/tokenization_fsmt.py | 45 ++------------------------ 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 9459fe5d1974..23c491780e25 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -24,12 +24,7 @@ logger = logging.getLogger(__name__) -FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/config.json", - "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/config.json", - "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/config.json", - "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/config.json", -} +FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP = {} FSMT_CONFIG_ARGS_DOC = r""" diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 0e3a10af227a..3b6ff45987d9 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -37,48 +37,9 @@ "merges_file": "merges.txt", } -PRETRAINED_VOCAB_FILES_MAP = { - "src_vocab_file": { - "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/vocab-src.json", - "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/vocab-src.json", - "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/vocab-src.json", - "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/vocab-src.json", - }, - "tgt_vocab_file": { - "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/vocab-tgt.json", - "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/vocab-tgt.json", - "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/vocab-tgt.json", - "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/vocab-tgt.json", - }, - "merges_file": { - "facebook/wmt19-ru-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-ru-en/merges.txt", - "facebook/wmt19-en-ru": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-ru/merges.txt", - "facebook/wmt19-de-en": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-de-en/merges.txt", - "facebook/wmt19-en-de": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/wmt19-en-de/merges.txt", - }, -} - -PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - "facebook/wmt19-ru-en": 1024, - "facebook/wmt19-en-ru": 1024, - "facebook/wmt19-de-en": 1024, - "facebook/wmt19-en-de": 1024, -} - -PRETRAINED_INIT_CONFIGURATION = { - "facebook/wmt19-ru-en": { - "langs": ["ru", "en"], - }, - "facebook/wmt19-en-ru": { - "langs": ["en", "ru"], - }, - "facebook/wmt19-de-en": { - "langs": ["de", "en"], - }, - "facebook/wmt19-en-de": { - "langs": ["en", "de"], - }, -} +PRETRAINED_VOCAB_FILES_MAP = {} +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} +PRETRAINED_INIT_CONFIGURATION = {} def get_pairs(word): From f894fb9fe848344ac488f22d426dddc6ff7a4ad9 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 13:32:39 -0700 Subject: [PATCH 096/109] typo --- model_cards/facebook/wmt19-de-en/README.md | 2 +- model_cards/facebook/wmt19-en-de/README.md | 2 +- model_cards/facebook/wmt19-en-ru/README.md | 2 +- model_cards/facebook/wmt19-ru-en/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/model_cards/facebook/wmt19-de-en/README.md b/model_cards/facebook/wmt19-de-en/README.md index 681255b6cb72..4cbb36e2e7ac 100644 --- a/model_cards/facebook/wmt19-de-en/README.md +++ b/model_cards/facebook/wmt19-de-en/README.md @@ -43,7 +43,7 @@ mname = "facebook/wmt19-de-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -input = "Maschinelles Lernen ist großartig, oder? +input = "Maschinelles Lernen ist großartig, oder?" input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/model_cards/facebook/wmt19-en-de/README.md b/model_cards/facebook/wmt19-en-de/README.md index 600947e1f972..426082eaac3a 100644 --- a/model_cards/facebook/wmt19-en-de/README.md +++ b/model_cards/facebook/wmt19-en-de/README.md @@ -43,7 +43,7 @@ mname = "facebook/wmt19-en-de" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -input = "Machine learning is great, isn't it? +input = "Machine learning is great, isn't it?" input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/model_cards/facebook/wmt19-en-ru/README.md b/model_cards/facebook/wmt19-en-ru/README.md index 96c09cd7a5dc..26999e652539 100644 --- a/model_cards/facebook/wmt19-en-ru/README.md +++ b/model_cards/facebook/wmt19-en-ru/README.md @@ -43,7 +43,7 @@ mname = "facebook/wmt19-en-ru" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -input = "Machine learning is great, isn't it? +input = "Machine learning is great, isn't it?" input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/model_cards/facebook/wmt19-ru-en/README.md b/model_cards/facebook/wmt19-ru-en/README.md index 7938794c567d..a4071e4101a5 100644 --- a/model_cards/facebook/wmt19-ru-en/README.md +++ b/model_cards/facebook/wmt19-ru-en/README.md @@ -43,7 +43,7 @@ mname = "facebook/wmt19-ru-en" tokenizer = FSMTTokenizer.from_pretrained(mname) model = FSMTForConditionalGeneration.from_pretrained(mname) -input = "Машинное обучение - это здорово, не так ли? +input = "Машинное обучение - это здорово, не так ли?" input_ids = tokenizer.encode(input, return_tensors="pt") outputs = model.generate(input_ids) decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) From 99483e0739c3d67beae843b9aea500eadc7c351c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 16:03:24 -0700 Subject: [PATCH 097/109] move all the doc / build /eval generators to their own scripts --- ..._original_pytorch_checkpoint_to_pytorch.py | 332 +----------------- 1 file changed, 8 insertions(+), 324 deletions(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 1340c25b96fb..441e876fa124 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -12,205 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Convert fairseq transform wmt19 checkpoint. - -To convert run: -assuming the fairseq data is under data/wmt19.ru-en.ensemble, data/wmt19.en-ru.ensemble, etc - -export ROOT=/code/huggingface/transformers-fair-wmt -cd $ROOT -mkdir data - -# get data (run once) -wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-de.joined-dict.ensemble.tar.gz -wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.de-en.joined-dict.ensemble.tar.gz -wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-ru.ensemble.tar.gz -wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.ru-en.ensemble.tar.gz -tar -xvzf wmt19.en-de.joined-dict.ensemble.tar.gz -tar -xvzf wmt19.de-en.joined-dict.ensemble.tar.gz -tar -xvzf wmt19.en-ru.ensemble.tar.gz -tar -xvzf wmt19.ru-en.ensemble.tar.gz - - -# run conversions and uploads - -export PAIR=ru-en -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR - -export PAIR=en-ru -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR - -export PAIR=de-en -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR - -export PAIR=en-de -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR - - -# upload -cd data -transformers-cli upload -y wmt19-ru-en -transformers-cli upload -y wmt19-en-ru -transformers-cli upload -y wmt19-de-en -transformers-cli upload -y wmt19-en-de -cd - - -# if updating just small files and not the large models, here is a script to generate the right commands: -perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for map { "wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json -# add/remove files as needed - -# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload -# So the only way to start using the new model sooner is either: -# 1. download it to a local path and use that path as model_name -# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere - -# happy translations - - -###################################################################################### - -Convert fairseq transform wmt16 en-de checkpoints from https://github.com/jungokasai/deep-shallow - - -pip install gdown - -# get data (run once) - -cd data -gdown 'https://drive.google.com/uc?id=1x_G2cjvM1nW5hjAB8-vWxRqtQTlmIaQU' -gdown 'https://drive.google.com/uc?id=1oA2aqZlVNj5FarxBlNXEHpBS4lRetTzU' -gdown 'https://drive.google.com/uc?id=1Wup2D318QYBFPW_NKI1mfP_hXOfmUI9r' -tar -xvzf trans_ende_12-1_0.2.tar.gz -tar -xvzf trans_ende-dist_12-1_0.2.tar.gz -tar -xvzf trans_ende-dist_6-1_0.2.tar.gz - -gdown 'https://drive.google.com/uc?id=1mNufoynJ9-Zy1kJh2TA_lHm2squji0i9' -gdown 'https://drive.google.com/uc?id=1iO7um-HWoNoRKDtw27YUSgyeubn9uXqj' -tar -xvzf wmt16.en-de.deep-shallow.dist.tar.gz -tar -xvzf wmt16.en-de.deep-shallow.tar.gz - -cp wmt16.en-de.deep-shallow/data-bin/dict.*.txt trans_ende_12-1_0.2 -cp wmt16.en-de.deep-shallow.dist/data-bin/dict.*.txt trans_ende-dist_12-1_0.2 -cp wmt16.en-de.deep-shallow.dist/data-bin/dict.*.txt trans_ende-dist_6-1_0.2 -cp wmt16.en-de.deep-shallow/bpecodes trans_ende_12-1_0.2 -cp wmt16.en-de.deep-shallow.dist/bpecodes trans_ende-dist_12-1_0.2 -cp wmt16.en-de.deep-shallow.dist/bpecodes trans_ende-dist_6-1_0.2 - - -# another set wmt19-6-6-de-en -gdown 'https://drive.google.com/uc?id=1j6z9fYdlUyOYsh7KJoumRlr1yHczxR5T' -gdown 'https://drive.google.com/uc?id=1yT7ZjqfvUYOBXvMjeY8uGRHQFWoSo8Q5' -gdown 'https://drive.google.com/uc?id=15gAzHeRUCs-QV8vHeTReMPEh1j8excNE' -tar -xvzf wmt19.de-en.tar.gz -tar -xvzf wmt19_deen_base_dr0.1_1.tar.gz -tar -xvzf wmt19_deen_big_dr0.1_2.tar.gz -cp wmt19.de-en/data-bin/dict.*.txt wmt19_deen_base_dr0.1_1 -cp wmt19.de-en/data-bin/dict.*.txt wmt19_deen_big_dr0.1_2 - -cd - - - -# run conversions and uploads - -# wmt16-en-de set - -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-dist-12-1 - -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_6-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-dist-6-1 - -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-12-1 - - -# wmt19-de-en set - -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_base_dr0.1_1/checkpoint_last3_avg.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-base - -PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_big_dr0.1_2/checkpoint_last3_avg.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-big - - - - -# upload -cd data -transformers-cli upload -y wmt16-en-de-dist-12-1 -transformers-cli upload -y wmt16-en-de-dist-6-1 -transformers-cli upload -y wmt16-en-de-12-1 -transformers-cli upload -y wmt19-de-en-6-6-base -transformers-cli upload -y wmt19-de-en-6-6-big -cd - - - - -# if updating just small files and not the large models, here is a script to generate the right commands: -perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for ("wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1", "wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json -# add/remove files as needed - -# XXX: move into model card - -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-de -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=64 -export NUM_BEAMS=5 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target - -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt16-en-de-dist-12-1 -echo $PAIR $MODEL_PATH -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS - -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt16-en-de-dist-6-1 -echo $PAIR $MODEL_PATH -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS - -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt16-en-de-12-1 -echo $PAIR $MODEL_PATH -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS - -checkpoint_top5_average.pt: - -num_beams=5 - -chkpt file| top5_average | best | -----------|--------------|---------| -dist-12-1 | 29.9134 | 30.2591 | -dist-6-1 | 29.9837 | 29.3349 | -12-1 | 26.4008 | 24.1803 | - -checkpoint_best.pt - - -# wmt19-de-en set - -export PAIR=de-en -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=64 -export NUM_BEAMS=5 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target - -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt19-de-en-6-6-base -echo $PAIR $MODEL_PATH -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS - -MODEL_PATH=/code/huggingface/transformers-fair-wmt/data/wmt19-de-en-6-6-big -echo $PAIR $MODEL_PATH -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS - - - - -``` - - -""" +# Note: if you intend to run this script make sure you look under scripts/fsmt/ +# to locate the appropriate script to do the work correctly. There is a set of scripts to: +# - download and prepare data and run the conversion script +# - perform eval to get the best hparam into the config +# - generate model_cards - useful if you have multiple models from the same paper import argparse import json @@ -256,6 +63,7 @@ "wmt19-de-en-6-6-big": {"length_penalty": 0.6}, } +# this remaps the different models to their organization names org_names = {} for m in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]: org_names[m] = "facebook" @@ -281,125 +89,6 @@ def rewrite_dict_keys(d): return d2 -def write_model_card(model_card_dir, src_lang, tgt_lang): - - texts = { - "en": "Machine learning is great, isn't it?", - "ru": "Машинное обучение - это здорово, не так ли?", - "de": "Maschinelles Lernen ist großartig, oder?", - } - - # BLUE scores as follows: - # "pair": [fairseq, transformers] - scores = { - "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "39.20"], - "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "33.47"], - "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "42.83"], - "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "41.35"], - } - pair = f"{src_lang}-{tgt_lang}" - - readme = f""" ---- - - - -language: {src_lang}, {tgt_lang} -thumbnail: -tags: -- translation -- wmt19 -license: Apache 2.0 -datasets: -- http://www.statmt.org/wmt19/ ([test-set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)) -metrics: -- http://www.statmt.org/wmt19/metrics-task.html ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers.tokenization_fsmt import FSMTTokenizer -from transformers.modeling_fsmt import FSMTForConditionalGeneration -mname = "facebook/wmt19-{src_lang}-{tgt_lang}" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "{texts[src_lang]} -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # {texts[tgt_lang]} - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -{pair} | {scores[pair][0]} | {scores[pair][1]} - -The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: -- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -- re-ranking - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR={pair} -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=15 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` -note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. - - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - -""" - os.makedirs(model_card_dir, exist_ok=True) - path = os.path.join(model_card_dir, "README.md") - with open(path, "w", encoding="utf-8") as f: - f.write(readme) - - def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder_path): # prep @@ -555,18 +244,13 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder print(f"Generating {pytorch_weights_dump_path}") torch.save(model_state_dict, pytorch_weights_dump_path) - # model card - org_name = org_names[model_dir] if model_dir in org_names else "stas" - model_card_dir = os.path.join(proj_root, "model_cards", org_name, model_dir) - print(f"Generating {model_card_dir}") - write_model_card(model_card_dir, src_lang, tgt_lang) - print("Conversion is done!") print("\nLast step is to upload the files to s3") print(f"cd {data_root}") print(f"transformers-cli upload {model_dir}") print( - "Note: CDN caches files for up to 24h, so use `from_pretrained(mname, use_cdn=False)` to use the non-cached version" + "Note: CDN caches files for up to 24h, so either use a local model path " + "or use `from_pretrained(mname, use_cdn=False)` to use the non-cached version." ) From f9f4f83af1fff230ed7bdaf58d2e485e57092e23 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 15 Sep 2020 16:06:31 -0700 Subject: [PATCH 098/109] cleanup --- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 441e876fa124..0b2ce844e3b6 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -121,7 +121,6 @@ def convert_fsmt_checkpoint_to_pytorch(fsmt_checkpoint_path, pytorch_dump_folder data_root = dirname(pytorch_dump_folder_path) model_dir = basename(pytorch_dump_folder_path) - proj_root = dirname(dirname(dirname(os.path.realpath(__file__)))) # dicts src_dict_file = os.path.join(fsmt_folder_path, f"dict.{src_lang}.txt") From 361299b9de2f20ab125374485318bef59985738f Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 09:14:37 -0700 Subject: [PATCH 099/109] Apply suggestions from code review Co-authored-by: Lysandre Debut --- docs/source/model_doc/fsmt.rst | 2 +- src/transformers/configuration_fsmt.py | 4 ++-- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/model_doc/fsmt.rst b/docs/source/model_doc/fsmt.rst index ea3b25d55af0..ca808ef21fce 100644 --- a/docs/source/model_doc/fsmt.rst +++ b/docs/source/model_doc/fsmt.rst @@ -7,7 +7,7 @@ file a `Github Issue __ by Nathan Ng, Kyra Yee, Alexei Baevski, Myle Ott, Michael Auli, Sergey Edunov. +FSMT (FairSeq MachineTranslation) models were introduced in "Facebook FAIR's WMT19 News Translation Task Submission" __ by Nathan Ng, Kyra Yee, Alexei Baevski, Myle Ott, Michael Auli, Sergey Edunov. The abstract of the paper is the following: diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index 23c491780e25..cd8f13a3ee05 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -16,13 +16,13 @@ import copy -import logging +from .utils import logging from .configuration_utils import PretrainedConfig from .file_utils import add_start_docstrings_to_callable -logger = logging.getLogger(__name__) +logger = logging.get_logger(__name__) FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP = {} diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 0b2ce844e3b6..426f7a6347fc 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -21,7 +21,7 @@ import argparse import json -import logging +from .utils import logging import os import re from collections import OrderedDict @@ -39,7 +39,7 @@ from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE -logging.basicConfig(level=logging.INFO) +logging.set_verbosity_info() json_indent = 2 From 2f3da546524037dd3fabc9efeaa7e1a3c8e55aa8 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 09:20:03 -0700 Subject: [PATCH 100/109] Apply suggestions from code review Co-authored-by: Lysandre Debut --- src/transformers/modeling_fsmt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index e5e406c6ffde..a782cfc1983c 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -27,7 +27,7 @@ # """PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/tree/master/examples/wmt19""" -import logging +from .utils import logging import math import random import warnings @@ -51,7 +51,7 @@ from .modeling_utils import PreTrainedModel -logger = logging.getLogger(__name__) +logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "FSMTConfig" _TOKENIZER_FOR_DOC = "FSMTTokenizer" From d8591d8196b03988dba2e352d1089362f50f53ee Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 09:29:49 -0700 Subject: [PATCH 101/109] fix indent --- src/transformers/modeling_fsmt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index a782cfc1983c..29c6fca701bf 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -1040,11 +1040,11 @@ def forward( **unused, ): r""" - labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): - Labels for computing the masked language modeling loss. - Indices should either be in ``[0, ..., config.vocab_size]`` or -100 (see ``input_ids`` docstring). - Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens - with labels in ``[0, ..., config.vocab_size]``. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the masked language modeling loss. + Indices should either be in ``[0, ..., config.vocab_size]`` or -100 (see ``input_ids`` docstring). + Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens + with labels in ``[0, ..., config.vocab_size]``. Returns: From 78f81b20e80dc4c09d93782431dda16f6d8ee810 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 09:30:27 -0700 Subject: [PATCH 102/109] duplicated line --- src/transformers/tokenization_fsmt.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 3b6ff45987d9..3b594b1bf004 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -487,9 +487,6 @@ def prepare_seq2seq_batch( if max_target_length is not None: tokenizer_kwargs["max_length"] = max_target_length - if max_target_length is not None: - tokenizer_kwargs["max_length"] = max_target_length - model_inputs["labels"] = self(tgt_texts, **tokenizer_kwargs)["input_ids"] return model_inputs From dbfa7c6382f51c383bfdc0aa6cf951d179f433a5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 09:33:31 -0700 Subject: [PATCH 103/109] style --- src/transformers/configuration_fsmt.py | 2 +- .../convert_fsmt_original_pytorch_checkpoint_to_pytorch.py | 3 ++- src/transformers/modeling_fsmt.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/transformers/configuration_fsmt.py b/src/transformers/configuration_fsmt.py index cd8f13a3ee05..a9dd0efe4301 100644 --- a/src/transformers/configuration_fsmt.py +++ b/src/transformers/configuration_fsmt.py @@ -16,10 +16,10 @@ import copy -from .utils import logging from .configuration_utils import PretrainedConfig from .file_utils import add_start_docstrings_to_callable +from .utils import logging logger = logging.get_logger(__name__) diff --git a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py index 426f7a6347fc..c9d29a8f054b 100755 --- a/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py @@ -21,7 +21,6 @@ import argparse import json -from .utils import logging import os import re from collections import OrderedDict @@ -38,6 +37,8 @@ from transformers.tokenization_fsmt import VOCAB_FILES_NAMES from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE +from .utils import logging + logging.set_verbosity_info() diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 29c6fca701bf..992b1fdbb58d 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -27,7 +27,6 @@ # """PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/tree/master/examples/wmt19""" -from .utils import logging import math import random import warnings @@ -49,6 +48,7 @@ ) from .modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Seq2SeqLMOutput, Seq2SeqModelOutput from .modeling_utils import PreTrainedModel +from .utils import logging logger = logging.get_logger(__name__) From a5185cee48c30e62627cd072c5f2b91e6779fceb Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 10:27:10 -0700 Subject: [PATCH 104/109] use the correct add_start_docstrings --- src/transformers/tokenization_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 3b594b1bf004..18942fd2560c 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -458,7 +458,7 @@ def create_token_type_ids_from_sequences( return len(token_ids_0 + sep) * [0] return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] - @add_start_docstrings_to_callable(PREPARE_SEQ2SEQ_BATCH_DOCSTRING) + @add_start_docstrings(PREPARE_SEQ2SEQ_BATCH_DOCSTRING) def prepare_seq2seq_batch( self, src_texts: List[str], From a3eb3b43edf72ee2011ab4d2b35eb2dbb1d39590 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 10:29:15 -0700 Subject: [PATCH 105/109] oops --- src/transformers/tokenization_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 18942fd2560c..e9eaf3497090 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -24,7 +24,7 @@ import sacremoses as sm -from .file_utils import add_start_docstrings_to_callable +from .file_utils import add_start_docstrings from .tokenization_utils import BatchEncoding, PreTrainedTokenizer from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING From 9e13d10d4ead733a354ec17683e66470b9227251 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 13:08:26 -0700 Subject: [PATCH 106/109] resizing can't be done with the core approach, due to 2 dicts --- src/transformers/modeling_fsmt.py | 6 ++++-- tests/test_modeling_fsmt.py | 31 ++++--------------------------- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/src/transformers/modeling_fsmt.py b/src/transformers/modeling_fsmt.py index 992b1fdbb58d..a0c914cedef9 100644 --- a/src/transformers/modeling_fsmt.py +++ b/src/transformers/modeling_fsmt.py @@ -1017,8 +1017,10 @@ def resize_token_embeddings(self, new_num_tokens: int) -> nn.Embedding: new_embeddings = super().resize_token_embeddings(new_num_tokens) self.model.decoder.embed_tokens = new_embeddings - # XXX: this is not quite correct, as we have 2 different - # `new_embeddings`, and only one return value is expected. + # XXX: this is not quite correct, as we have 2 different `new_embeddings`, and + # only one return value is expected. Needs to be redesigned in the core to support dual dicts + raise NotImplementedError("this method needs re-thinking for models with 2 separate dictionaries") + return new_embeddings @add_start_docstrings_to_callable(FSMT_INPUTS_DOCSTRING) diff --git a/tests/test_modeling_fsmt.py b/tests/test_modeling_fsmt.py index d6e0c3eb7df1..4c56d76a7aac 100644 --- a/tests/test_modeling_fsmt.py +++ b/tests/test_modeling_fsmt.py @@ -207,6 +207,10 @@ def test_save_load_strict(self): model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True) self.assertEqual(info["missing_keys"], []) + @unittest.skip("can't be implemented for FSMT due to dual vocab.") + def test_resize_tokens_embeddings(self): + pass + @unittest.skip("Passing inputs_embeds not implemented for FSMT.") def test_inputs_embeds(self): pass @@ -331,33 +335,6 @@ def test_prepare_fsmt_decoder_inputs(self): self.assertEqual(decoder_attn_mask.size(), decoder_input_ids.size()) self.assertTrue(torch.eq(expected_causal_mask, causal_mask).all()) - def test_resize_tokens_embeddings_more(self): - config, input_ids, _ = self._get_config_and_data() - - def _get_embs(m): - return (m.get_input_embeddings().weight.data.clone(), m.get_output_embeddings().weight.data.clone()) - - model = FSMTForConditionalGeneration(config).eval().to(torch_device) - - # not equal in FSMT - # input, output = _get_embs(model) - # self.assertTrue(torch.eq(input, output).all(), msg=f"\n{input}\n{output}") - - new_src_vocab_size = 45 - model.resize_token_embeddings(new_src_vocab_size) - input_new, output_new = _get_embs(model) - self.assertEqual( - input_new.shape, - (new_src_vocab_size, config.d_model), - msg=f"input {input_new.shape}, {(new_src_vocab_size, config.d_model)}", - ) - self.assertEqual( - output_new.shape, - (new_src_vocab_size, config.d_model), - msg=f"output {input_new.shape}, {(new_src_vocab_size, config.d_model)}", - ) - self.assertTrue(torch.eq(input_new, output_new).all(), msg=f"{input_new}, {output_new}") - def _assert_tensors_equal(a, b, atol=1e-12, prefix=""): """If tensors not close, or a and b arent both tensors, raise a nice Assertion error.""" From cd0e95e5ae7f5c8e75049c3f649fb6116060191a Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 23:45:59 -0700 Subject: [PATCH 107/109] check that the arg is a list --- src/transformers/tokenization_fsmt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index e9eaf3497090..5c9ba039ec20 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -471,8 +471,12 @@ def prepare_seq2seq_batch( **unused, ) -> BatchEncoding: """Prepare model inputs for translation. For best performance, translate one sentence at a time.""" + + if type(src_texts) is not list: + raise ValueError(f"src_texts is expected to be a list") if "" in src_texts: raise ValueError(f"found empty string in src_texts: {src_texts}") + tokenizer_kwargs = dict( add_special_tokens=True, return_tensors=return_tensors, From 5b986cf18293d0d3747cdc57e0eeb195e5b0a6d2 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 16 Sep 2020 23:58:58 -0700 Subject: [PATCH 108/109] style --- src/transformers/tokenization_fsmt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 5c9ba039ec20..6eaadf31527b 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -471,12 +471,12 @@ def prepare_seq2seq_batch( **unused, ) -> BatchEncoding: """Prepare model inputs for translation. For best performance, translate one sentence at a time.""" - + if type(src_texts) is not list: raise ValueError(f"src_texts is expected to be a list") if "" in src_texts: raise ValueError(f"found empty string in src_texts: {src_texts}") - + tokenizer_kwargs = dict( add_special_tokens=True, return_tensors=return_tensors, From 1be40e38b68d1225e87527eddbf939804fd82dda Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 17 Sep 2020 01:11:11 -0700 Subject: [PATCH 109/109] style --- src/transformers/tokenization_fsmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_fsmt.py b/src/transformers/tokenization_fsmt.py index 6eaadf31527b..63f720c70a51 100644 --- a/src/transformers/tokenization_fsmt.py +++ b/src/transformers/tokenization_fsmt.py @@ -473,7 +473,7 @@ def prepare_seq2seq_batch( """Prepare model inputs for translation. For best performance, translate one sentence at a time.""" if type(src_texts) is not list: - raise ValueError(f"src_texts is expected to be a list") + raise ValueError("src_texts is expected to be a list") if "" in src_texts: raise ValueError(f"found empty string in src_texts: {src_texts}")