From 30a3e5af69b8d27af1d2ff350a6f69a61b605591 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Tue, 7 Oct 2025 13:36:15 -0400 Subject: [PATCH] [CI] Add Qwen3 MoE NVFP4 to Blackwell lm-eval (#26316) Signed-off-by: mgoin --- .buildkite/test-pipeline.yaml | 2 +- tests/evals/gsm8k/configs/Qwen3-30B-A3B-NVFP4.yaml | 6 ++++++ tests/evals/gsm8k/configs/models-blackwell.txt | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 tests/evals/gsm8k/configs/Qwen3-30B-A3B-NVFP4.yaml diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 76d2ce4a7aef..48dff31c14dc 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -868,7 +868,7 @@ steps: - pytest -s -v tests/quantization/test_blackwell_moe.py - label: Blackwell LM Eval Small Models - timeout_in_minutes: 75 + timeout_in_minutes: 120 gpu: b200 optional: true # run on nightlies source_file_dependencies: diff --git a/tests/evals/gsm8k/configs/Qwen3-30B-A3B-NVFP4.yaml b/tests/evals/gsm8k/configs/Qwen3-30B-A3B-NVFP4.yaml new file mode 100644 index 000000000000..6b7bdd1e65bb --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-30B-A3B-NVFP4.yaml @@ -0,0 +1,6 @@ +model_name: "nvidia/Qwen3-30B-A3B-FP4" +accuracy_threshold: 0.89 +num_questions: 1319 +num_fewshot: 5 +max_model_len: 4096 + diff --git a/tests/evals/gsm8k/configs/models-blackwell.txt b/tests/evals/gsm8k/configs/models-blackwell.txt index e577645d60d6..3c9b1084de7b 100644 --- a/tests/evals/gsm8k/configs/models-blackwell.txt +++ b/tests/evals/gsm8k/configs/models-blackwell.txt @@ -2,3 +2,4 @@ Qwen3-0.6B-FP8.yaml Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml Qwen1.5-MoE-W4A16-CT.yaml DeepSeek-V2-Lite-Instruct-FP8.yaml +Qwen3-30B-A3B-NVFP4.yaml