Eval commands
Run once per seed (1234, 2345, 3456). Final reported values are averages.
gsm8k_platinum_cot_llama
lm_eval \
--model local-chat-completions \
--tasks gsm8k_platinum_cot_llama \
--model_args model=Qwen/Qwen3.8-27B,max_length=69632,base_url=http://127.0.0.1:8002/v1/chat/completions,num_concurrent=32,max_retries=3,tokenized
_requests=False,tokenizer_backend=None,timeout=3600 \
--num_fewshot 0 \
--seed <SEED> \
--gen_kwargs do_sample=True,temperature=1.0,top_p=0.95,top_k=20,seed=<SEED>,max_gen_toks=32000 \
--apply_chat_template
ifeval
lm_eval \
--model local-chat-completions \
--tasks ifeval \
--model_args model=Qwen/Qwen3.8-27B,max_length=69632,base_url=http://127.0.0.1:8006/v1/chat/completions,num_concurrent=16,max_retrie
s=3,tokenized_requests=False,tokenizer_backend=None,timeout=3600 \
--num_fewshot 0 \
--seed <SEED> \
--gen_kwargs do_sample=True,temperature=1.0,top_p=0.95,top_k=20,seed=1234,max_gen_toks=32000 \
--apply_chat_template
aime25
lighteval endpoint litellm \
runs/20260818_110422_qwen3.8-27b-int4_full_ifeval-math-aime-gpqa/configs/litellm_aime25_seed1234.yaml \
'aime25|0' \
--save-details
math_500
lighteval endpoint litellm \
runs/20260818_110422_qwen3.8-27b-int4_full_ifeval-math-aime-gpqa/configs/litellm_math_500_seed1234.yaml \
'math_500|0' \
--save-details
GPQA Diamond
lighteval endpoint litellm \
runs/20260818_110422_qwen3.8-27b-int4_full_ifeval-math-aime-gpqa/configs/litellm_gpqa_diamond_seed1234.yaml \
'gpqa:diamond|0' \
--save-details