Spaces:
Sleeping
Sleeping
| # Copyright 2020 The HuggingFace Team. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # this script evals the following fsmt models | |
| # it covers: | |
| # - facebook/wmt19-ru-en | |
| # - facebook/wmt19-en-ru | |
| # - facebook/wmt19-de-en | |
| # - facebook/wmt19-en-de | |
| # this script needs to be run from the top level of the transformers repo | |
| if [ ! -d "src/transformers" ]; then | |
| echo "Error: This script needs to be run from the top of the transformers repo" | |
| exit 1 | |
| fi | |
| # In these scripts you may have to lower BS if you get CUDA OOM (or increase it if you have a large GPU) | |
| ### a short estimate version for quick testing ### | |
| export PAIR=en-ru | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=8 | |
| export NUM_BEAMS=8 | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src | head -10 > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref | head -10 > $DATA_DIR/val.target | |
| echo $PAIR | |
| PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS | |
| ### Normal eval ### | |
| # ru-en | |
| export PAIR=ru-en | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=8 | |
| export NUM_BEAMS=50 | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS | |
| # (target BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937) | |
| # en-ru | |
| export PAIR=en-ru | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=8 | |
| export NUM_BEAMS=50 | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| echo $PAIR | |
| PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS | |
| # (target BLEU: 36.4 http://matrix.statmt.org/matrix/output/1914?score_id=37605) | |
| # en-de | |
| export PAIR=en-de | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=8 | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| echo $PAIR | |
| PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS | |
| # (target BLEU: 43.1 http://matrix.statmt.org/matrix/output/1909?run_id=6862) | |
| # de-en | |
| export PAIR=de-en | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=8 | |
| export NUM_BEAMS=50 | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| echo $PAIR | |
| PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS | |
| # (target BLEU: 42.3 http://matrix.statmt.org/matrix/output/1902?run_id=6750) | |
| ### Searching hparams eval ### | |
| # en-ru | |
| export PAIR=ru-en | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=32 | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| CUDA_VISIBLE_DEVICES="0" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1" | |
| # en-ru | |
| export PAIR=en-ru | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=16 | |
| mkdir -p $DATA_DIR | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| CUDA_VISIBLE_DEVICES="0" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false" | |
| # en-de | |
| export PAIR=en-de | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=16 | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| CUDA_VISIBLE_DEVICES="1" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false" | |
| # de-en | |
| export PAIR=de-en | |
| export DATA_DIR=data/$PAIR | |
| export SAVE_DIR=data/$PAIR | |
| export BS=16 | |
| mkdir -p $DATA_DIR | |
| mkdir -p $DATA_DIR | |
| sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source | |
| sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target | |
| CUDA_VISIBLE_DEVICES="1" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false" | |