build/eval/gen-card scripts for fsmt

scripts/fsmt/convert-allenai-wmt16.sh

@@ -0,0 +1,62 @@
+#/usr/bin/env bash
+
+# this script acquires data and converts it to fsmt model
+# it covers:
+# - allenai/wmt16-en-de-dist-12-1
+# - allenai/wmt16-en-de-dist-6-1
+# - allenai/wmt16-en-de-12-1
+
+# this script needs to be run from the top level of the transformers repo
+if [ ! -d "src/transformers" ]; then
+    echo "Error: This script needs to be run from the top of the transformers repo"
+    exit 1
+fi
+
+mkdir data
+
+# get data (run once)
+
+cd data
+gdown 'https://drive.google.com/uc?id=1x_G2cjvM1nW5hjAB8-vWxRqtQTlmIaQU'
+gdown 'https://drive.google.com/uc?id=1oA2aqZlVNj5FarxBlNXEHpBS4lRetTzU'
+gdown 'https://drive.google.com/uc?id=1Wup2D318QYBFPW_NKI1mfP_hXOfmUI9r'
+tar -xvzf trans_ende_12-1_0.2.tar.gz
+tar -xvzf trans_ende-dist_12-1_0.2.tar.gz
+tar -xvzf trans_ende-dist_6-1_0.2.tar.gz
+gdown 'https://drive.google.com/uc?id=1mNufoynJ9-Zy1kJh2TA_lHm2squji0i9'
+gdown 'https://drive.google.com/uc?id=1iO7um-HWoNoRKDtw27YUSgyeubn9uXqj'
+tar -xvzf wmt16.en-de.deep-shallow.dist.tar.gz
+tar -xvzf wmt16.en-de.deep-shallow.tar.gz
+cp wmt16.en-de.deep-shallow/data-bin/dict.*.txt trans_ende_12-1_0.2
+cp wmt16.en-de.deep-shallow.dist/data-bin/dict.*.txt trans_ende-dist_12-1_0.2
+cp wmt16.en-de.deep-shallow.dist/data-bin/dict.*.txt trans_ende-dist_6-1_0.2
+cp wmt16.en-de.deep-shallow/bpecodes trans_ende_12-1_0.2
+cp wmt16.en-de.deep-shallow.dist/bpecodes trans_ende-dist_12-1_0.2
+cp wmt16.en-de.deep-shallow.dist/bpecodes trans_ende-dist_6-1_0.2
+cd -
+
+# run conversions and uploads
+
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-dist-12-1
+
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende-dist_6-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-dist-6-1
+
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/trans_ende_12-1_0.2/checkpoint_top5_average.pt --pytorch_dump_folder_path data/wmt16-en-de-12-1
+
+
+# upload
+cd data
+transformers-cli upload -y wmt16-en-de-dist-12-1
+transformers-cli upload -y wmt16-en-de-dist-6-1
+transformers-cli upload -y wmt16-en-de-12-1
+cd -
+
+
+# if updating just small files and not the large models, here is a script to generate the right commands:
+perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for ("wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json
+# add/remove files as needed
+
+# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload
+# So the only way to start using the new model sooner is either:
+# 1. download it to a local path and use that path as model_name
+# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere

scripts/fsmt/convert-allenai-wmt19.sh

@@ -0,0 +1,50 @@
+#/usr/bin/env bash
+
+# this script acquires data and converts it to fsmt model
+# it covers:
+# - allenai/wmt19-de-en-6-6-base
+# - allenai/wmt19-de-en-6-6-big
+
+# this script needs to be run from the top level of the transformers repo
+if [ ! -d "src/transformers" ]; then
+    echo "Error: This script needs to be run from the top of the transformers repo"
+    exit 1
+fi
+
+mkdir data
+
+# get data (run once)
+
+cd data
+gdown 'https://drive.google.com/uc?id=1j6z9fYdlUyOYsh7KJoumRlr1yHczxR5T'
+gdown 'https://drive.google.com/uc?id=1yT7ZjqfvUYOBXvMjeY8uGRHQFWoSo8Q5'
+gdown 'https://drive.google.com/uc?id=15gAzHeRUCs-QV8vHeTReMPEh1j8excNE'
+tar -xvzf wmt19.de-en.tar.gz
+tar -xvzf wmt19_deen_base_dr0.1_1.tar.gz
+tar -xvzf wmt19_deen_big_dr0.1_2.tar.gz
+cp wmt19.de-en/data-bin/dict.*.txt wmt19_deen_base_dr0.1_1
+cp wmt19.de-en/data-bin/dict.*.txt wmt19_deen_big_dr0.1_2
+cd -
+
+# run conversions and uploads
+
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_base_dr0.1_1/checkpoint_last3_avg.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-base
+
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19_deen_big_dr0.1_2/checkpoint_last3_avg.pt --pytorch_dump_folder_path data/wmt19-de-en-6-6-big
+
+
+# upload
+cd data
+transformers-cli upload -y wmt19-de-en-6-6-base
+transformers-cli upload -y wmt19-de-en-6-6-big
+cd -
+
+
+# if updating just small files and not the large models, here is a script to generate the right commands:
+perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for ("wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json
+# add/remove files as needed
+
+# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload
+# So the only way to start using the new model sooner is either:
+# 1. download it to a local path and use that path as model_name
+# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere

scripts/fsmt/convert-facebook-wmt19.sh

@@ -0,0 +1,61 @@
+#/usr/bin/env bash
+
+# this script acquires data and converts it to fsmt model
+# it covers:
+# - facebook/wmt19-ru-en
+# - facebook/wmt19-en-ru
+# - facebook/wmt19-de-en
+# - facebook/wmt19-en-de
+
+# this script needs to be run from the top level of the transformers repo
+if [ ! -d "src/transformers" ]; then
+    echo "Error: This script needs to be run from the top of the transformers repo"
+    exit 1
+fi
+
+mkdir data
+
+# get data (run once)
+
+cd data
+wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-de.joined-dict.ensemble.tar.gz
+wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.de-en.joined-dict.ensemble.tar.gz
+wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-ru.ensemble.tar.gz
+wget https://dl.fbaipublicfiles.com/fairseq/models/wmt19.ru-en.ensemble.tar.gz
+tar -xvzf wmt19.en-de.joined-dict.ensemble.tar.gz
+tar -xvzf wmt19.de-en.joined-dict.ensemble.tar.gz
+tar -xvzf wmt19.en-ru.ensemble.tar.gz
+tar -xvzf wmt19.ru-en.ensemble.tar.gz
+cd -
+
+# run conversions and uploads
+
+export PAIR=ru-en
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR
+
+export PAIR=en-ru
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR
+
+export PAIR=de-en
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR
+
+export PAIR=en-de
+PYTHONPATH="src" python src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py --fsmt_checkpoint_path data/wmt19.$PAIR.joined-dict.ensemble/model4.pt --pytorch_dump_folder_path data/wmt19-$PAIR
+
+
+# upload
+cd data
+transformers-cli upload -y wmt19-ru-en
+transformers-cli upload -y wmt19-en-ru
+transformers-cli upload -y wmt19-de-en
+transformers-cli upload -y wmt19-en-de
+cd -
+
+# if updating just small files and not the large models, here is a script to generate the right commands:
+perl -le 'for $f (@ARGV) { print qq[transformers-cli upload -y $_/$f --filename $_/$f] for map { "wmt19-$_" } ("en-ru", "ru-en", "de-en", "en-de")}' vocab-src.json vocab-tgt.json tokenizer_config.json config.json
+# add/remove files as needed
+
+# Caching note: Unfortunately due to CDN caching the uploaded model may be unavailable for up to 24hs after upload
+# So the only way to start using the new model sooner is either:
+# 1. download it to a local path and use that path as model_name
+# 2. make sure you use: from_pretrained(..., use_cdn=False) everywhere

scripts/fsmt/eval-allenai-wmt16.sh

@@ -0,0 +1,66 @@
+#/usr/bin/env bash
+
+# this script evals the following fsmt models
+# it covers:
+# - allenai/wmt16-en-de-dist-12-1
+# - allenai/wmt16-en-de-dist-6-1
+# - allenai/wmt16-en-de-12-1
+
+# this script needs to be run from the top level of the transformers repo
+if [ ! -d "src/transformers" ]; then
+    echo "Error: This script needs to be run from the top of the transformers repo"
+    exit 1
+fi
+
+# In these scripts you may have to lower BS if you get CUDA OOM (or increase it if you have a large GPU)
+
+### Normal eval ###
+
+export PAIR=en-de
+export DATA_DIR=data/$PAIR
+export SAVE_DIR=data/$PAIR
+export BS=64
+export NUM_BEAMS=5
+mkdir -p $DATA_DIR
+sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
+sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
+
+MODEL_PATH=allenai/wmt16-en-de-dist-12-1
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
+
+MODEL_PATH=allenai/wmt16-en-de-dist-6-1
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
+
+MODEL_PATH=allenai/wmt16-en-de-12-1
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
+
+
+
+### Searching hparams eval ###
+
+
+export PAIR=en-de
+export DATA_DIR=data/$PAIR
+export SAVE_DIR=data/$PAIR
+export BS=32
+export NUM_BEAMS=5
+mkdir -p $DATA_DIR
+sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
+sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
+
+MODEL_PATH=allenai/wmt16-en-de-dist-12-1
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:10:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1"
+
+
+MODEL_PATH=allenai/wmt16-en-de-dist-6-1
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:10:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1"
+
+
+MODEL_PATH=allenai/wmt16-en-de-12-1
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:10:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1"

scripts/fsmt/eval-allenai-wmt19.sh

@@ -0,0 +1,54 @@
+#/usr/bin/env bash
+
+# this script evals the following fsmt models
+# it covers:
+# - allenai/wmt19-de-en-6-6-base
+# - allenai/wmt19-de-en-6-6-big
+
+# this script needs to be run from the top level of the transformers repo
+if [ ! -d "src/transformers" ]; then
+    echo "Error: This script needs to be run from the top of the transformers repo"
+    exit 1
+fi
+
+# In these scripts you may have to lower BS if you get CUDA OOM (or increase it if you have a large GPU)
+
+### Normal eval ###
+
+export PAIR=de-en
+export DATA_DIR=data/$PAIR
+export SAVE_DIR=data/$PAIR
+export BS=64
+export NUM_BEAMS=5
+mkdir -p $DATA_DIR
+sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
+sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
+
+MODEL_PATH=allenai/wmt19-de-en-6-6-base
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
+
+MODEL_PATH=allenai/wmt19-de-en-6-6-big
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
+
+
+
+### Searching hparams eval ###
+
+export PAIR=de-en
+export DATA_DIR=data/$PAIR
+export SAVE_DIR=data/$PAIR
+export BS=16
+export NUM_BEAMS=5
+mkdir -p $DATA_DIR
+sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
+sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
+
+MODEL_PATH=allenai/wmt19-de-en-6-6-base
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:10:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1"
+
+MODEL_PATH=allenai/wmt19-de-en-6-6-big
+echo $PAIR $MODEL_PATH
+PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py $MODEL_PATH $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:10:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1"