mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-07 23:00:08 +06:00
149 lines
5.9 KiB
Bash
Executable File
149 lines
5.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# this script evals the following fsmt models
|
|
# it covers:
|
|
# - facebook/wmt19-ru-en
|
|
# - facebook/wmt19-en-ru
|
|
# - facebook/wmt19-de-en
|
|
# - facebook/wmt19-en-de
|
|
|
|
|
|
# this script needs to be run from the top level of the transformers repo
|
|
if [ ! -d "src/transformers" ]; then
|
|
echo "Error: This script needs to be run from the top of the transformers repo"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
# In these scripts you may have to lower BS if you get CUDA OOM (or increase it if you have a large GPU)
|
|
|
|
### a short estimate version for quick testing ###
|
|
|
|
export PAIR=en-ru
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=8
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src | head -10 > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref | head -10 > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
|
|
|
|
### Normal eval ###
|
|
|
|
# ru-en
|
|
|
|
export PAIR=ru-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=50
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
|
|
# (target BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937)
|
|
|
|
|
|
# en-ru
|
|
|
|
export PAIR=en-ru
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=50
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
# (target BLEU: 36.4 http://matrix.statmt.org/matrix/output/1914?score_id=37605)
|
|
|
|
|
|
|
|
# en-de
|
|
|
|
export PAIR=en-de
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
# (target BLEU: 43.1 http://matrix.statmt.org/matrix/output/1909?run_id=6862)
|
|
|
|
|
|
# de-en
|
|
|
|
export PAIR=de-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=50
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
# (target BLEU: 42.3 http://matrix.statmt.org/matrix/output/1902?run_id=6750)
|
|
|
|
|
|
### Searching hparams eval ###
|
|
|
|
# en-ru
|
|
|
|
export PAIR=ru-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=32
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="0" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1"
|
|
|
|
|
|
# en-ru
|
|
|
|
export PAIR=en-ru
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=16
|
|
mkdir -p $DATA_DIR
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="0" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false"
|
|
|
|
# en-de
|
|
|
|
export PAIR=en-de
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=16
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="1" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false"
|
|
|
|
# de-en
|
|
|
|
export PAIR=de-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=16
|
|
mkdir -p $DATA_DIR
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="1" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false"
|