eval_bloom_finnish_array.sh

#!/bin/bash
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH -p small-g
#SBATCH -t 24:00:00
#SBATCH --gpus-per-node=mi250:1
#SBATCH --exclusive=user
#SBATCH --hint=nomultithread
#SBATCH --account=project_462000241
#SBATCH -o logs/%j.out
#SBATCH -e logs/%j.err

source /pfs/lustrep2/scratch/project_462000241/muennighoff/lmevallatest/venv/bin/activate

cd /pfs/lustrep2/scratch/project_462000241/muennighoff/

export HF_DATASETS_OFFLINE=1
export HF_DATASETS_CACHE=/pfs/lustrep2/scratch/project_462000185/muennighoff/ds_cache

# Evaluation to run
CONFIGS=(
bigbench_analogies
bigbench_arithmetic_1_digit_addition
bigbench_arithmetic_1_digit_division
bigbench_arithmetic_1_digit_multiplication
bigbench_arithmetic_1_digit_subtraction
bigbench_arithmetic_2_digit_addition
bigbench_arithmetic_2_digit_division
bigbench_arithmetic_2_digit_multiplication
bigbench_arithmetic_2_digit_subtraction
bigbench_arithmetic_3_digit_addition
bigbench_arithmetic_3_digit_division
bigbench_arithmetic_3_digit_multiplication
bigbench_arithmetic_3_digit_subtraction
bigbench_arithmetic_4_digit_addition
bigbench_arithmetic_4_digit_division
bigbench_arithmetic_4_digit_multiplication
bigbench_arithmetic_4_digit_subtraction
bigbench_arithmetic_5_digit_addition
bigbench_arithmetic_5_digit_division
bigbench_arithmetic_5_digit_multiplication
bigbench_arithmetic_5_digit_subtraction
bigbench_cause_and_effect_one_sentence
bigbench_cause_and_effect_one_sentence_no_prompt
bigbench_cause_and_effect_two_sentences
bigbench_emotions
bigbench_empirical_judgments
bigbench_general_knowledge
bigbench_hhh_alignment_harmless
bigbench_hhh_alignment_helpful
bigbench_hhh_alignment_honest
bigbench_hhh_alignment_other
bigbench_misconceptions
bigbench_paraphrase
bigbench_sentence_ambiguity
bigbench_similarities_abstraction
)
CONFIGS=(
bigbench_intent_recognition
)
CONFIGSX=(
bigbench_arithmetic_1_digit_addition
bigbench_arithmetic_1_digit_division
bigbench_arithmetic_1_digit_multiplication
bigbench_arithmetic_1_digit_subtraction
bigbench_arithmetic_2_digit_addition
bigbench_arithmetic_2_digit_division
bigbench_arithmetic_2_digit_multiplication
bigbench_arithmetic_2_digit_subtraction
bigbench_arithmetic_3_digit_addition
bigbench_arithmetic_3_digit_division
bigbench_arithmetic_3_digit_multiplication
bigbench_arithmetic_3_digit_subtraction
bigbench_arithmetic_4_digit_addition
bigbench_arithmetic_4_digit_division
bigbench_arithmetic_4_digit_multiplication
bigbench_arithmetic_4_digit_subtraction
bigbench_arithmetic_5_digit_addition
bigbench_arithmetic_5_digit_division
bigbench_arithmetic_5_digit_multiplication
bigbench_arithmetic_5_digit_subtraction
)

CONFIGSX=(
bigbench_cause_and_effect_one_sentence
bigbench_cause_and_effect_one_sentence_no_prompt
bigbench_cause_and_effect_two_sentences
bigbench_emotions
bigbench_empirical_judgments
bigbench_general_knowledge
bigbench_hhh_alignment_harmless
bigbench_hhh_alignment_helpful
bigbench_hhh_alignment_honest
bigbench_hhh_alignment_other
)

# Iterate through all possible combinations of config and run the jobs
for ((i=0; i<${#CONFIGS[@]}; i++)); do
    eval_script="./eval_$i.slurm"
    OUTPUT_PATH=results/bloom_finnish_176B_${CONFIGS[$i]}_2shot.json
    cat <<EOT > $eval_script
#!/bin/bash
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH -p standard-g
#SBATCH -t 48:00:00
#SBATCH --gpus-per-node=mi250:8
#SBATCH --exclusive=user
#SBATCH --hint=nomultithread
#SBATCH --account=project_462000241
#SBATCH -o logs/%j.out
#SBATCH -e logs/%j.err

source /pfs/lustrep2/scratch/project_462000241/muennighoff/lmevallatest/venv/bin/activate

cd /pfs/lustrep2/scratch/project_462000241/muennighoff/lmevallatest/lm-evaluation-harness

export HF_DATASETS_OFFLINE=1
export HF_DATASETS_CACHE=/pfs/lustrep2/scratch/project_462000185/muennighoff/ds_cache

python main.py \
--model hf-causal-experimental \
--model_args pretrained=/pfs/lustrep2/scratch/project_462000185/muennighoff/bloom-finnish-176b \
--tasks ${CONFIGS[$i]} \
--device cuda:0 \
--output_path $OUTPUT_PATH \
--no_cache --num_fewshot 2
echo "END TIME: $(date)"
EOT
    # Submit the job
    sbatch $eval_script
    # Sleep for a bit to avoid hitting the job submission limit
    sleep 1
done


echo "END TIME: $(date)"