From f32229293e3c779004d9f90503b4375a3716879d Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Thu, 6 Nov 2025 22:19:07 +0800 Subject: [PATCH] Disable nm-testing models with issues in CI (#28206) Signed-off-by: mgoin --- ...Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml | 12 ------------ benchmarks/kernels/benchmark_grouped_gemm_cutlass.py | 4 ++-- benchmarks/kernels/benchmark_shapes.py | 4 ++-- 3 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 .buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml diff --git a/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml b/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml deleted file mode 100644 index 1bce7e7fdf146..0000000000000 --- a/.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# For vllm script, with -t option (tensor parallel size). -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise -b "auto" -l 1000 -f 5 -t 1 -model_name: "nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.595 - - name: "exact_match,flexible-extract" - value: 0.582 -limit: 1000 -num_fewshot: 5 diff --git a/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py index d525bd5faacf6..9b426d8d5f778 100644 --- a/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py +++ b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py @@ -16,8 +16,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe import ( from vllm.utils.argparse_utils import FlexibleArgumentParser DEFAULT_MODELS = [ - "nm-testing/Mixtral-8x7B-Instruct-v0.1", - "nm-testing/deepseekv2-lite", + "mistralai/Mixtral-8x7B-Instruct-v0.1", + "deepseek-ai/DeepSeek-V2-Lite", "ibm-granite/granite-3.0-1b-a400m", "ibm-granite/granite-3.0-3b-a800m", ] diff --git a/benchmarks/kernels/benchmark_shapes.py b/benchmarks/kernels/benchmark_shapes.py index 18c459c31d3f8..3e23c4cac059c 100644 --- a/benchmarks/kernels/benchmark_shapes.py +++ b/benchmarks/kernels/benchmark_shapes.py @@ -78,11 +78,11 @@ WEIGHT_SHAPES = { } WEIGHT_SHAPES_MOE = { - "nm-testing/Mixtral-8x7B-Instruct-v0.1": [ + "mistralai/Mixtral-8x7B-Instruct-v0.1": [ [8, 2, 4096, 28672], [8, 2, 14336, 4096], ], - "nm-testing/deepseekv2-lite": [ + "deepseek-ai/DeepSeek-V2-Lite": [ [64, 6, 2048, 1408], ], "ibm-granite/granite-3.0-1b-a400m": [