File size: 1,200 Bytes
0f6478f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
# Install required package
pip install antlr4-python3-runtime==4.11 immutabledict langdetect nltk lm_eval
python -c "import nltk; nltk.download('punkt')"
MODEL_PATHS=(
huihui-ai/Qwen2.5-14B-Instruct-abliterated-v2
)
for MODEL_PATH in "${MODEL_PATHS[@]}"; do
MODEL_NAME=$(basename "$MODEL_PATH")
MODEL_DIR="./results/$MODEL_NAME"
mkdir -p "$MODEL_DIR"
MODEL_ARGS="trust_remote_code=True,pretrained=$MODEL_PATH,dtype=bfloat16"
BASE_COMMAND="accelerate launch -m lm_eval --model hf --model_args $MODEL_ARGS --batch_size 4 --fewshot_as_multiturn --apply_chat_template"
# IFEval
$BASE_COMMAND --tasks leaderboard_ifeval --fewshot_as_multiturn --output_path "$MODEL_DIR/ifeval"
# BBH (Big-Bench Hard)
$BASE_COMMAND --tasks leaderboard_bbh --num_fewshot 3 --fewshot_as_multiturn --output_path "$MODEL_DIR/bbh"
# GPQA
$BASE_COMMAND --tasks leaderboard_gpqa --fewshot_as_multiturn --output_path "$MODEL_DIR/gpqa"
# MMLU-Pro
$BASE_COMMAND --tasks leaderboard_mmlu_pro --num_fewshot 5 --fewshot_as_multiturn --output_path "$MODEL_DIR/mmlu_pro"
# TruthfulQA
$BASE_COMMAND --tasks truthfulqa_mc2 --fewshot_as_multiturn --output_path "$MODEL_DIR/truthfulqa"
done
|