[CI] Split mteb test from Language Models Test (#24634)

Signed-off-by: wang.yuqi <noooop@126.com>

.buildkite/test-pipeline.yaml

@@ -642,6 +642,16 @@ steps:
   commands:
     - pytest -v -s models/language/pooling -m 'not core_model'
 
+- label: Language Models Test (MTEB)
+  timeout_in_minutes: 110
+  mirror_hardwares: [amdexperimental]
+  optional: true
+  source_file_dependencies:
+  - vllm/
+  - tests/models/language/pooling_mteb_test
+  commands:
+    - pytest -v -s models/language/pooling_mteb_test
+
 - label: Multi-Modal Processor Test # 44min
   timeout_in_minutes: 60
   source_file_dependencies:

tests/entrypoints/pooling/correctness/test_mteb_embed.py

@@ -4,10 +4,9 @@ import os
 
 import pytest
 
-from tests.models.language.pooling.mteb_utils import (MTEB_EMBED_TASKS,
-                                                      MTEB_EMBED_TOL,
-                                                      OpenAIClientMtebEncoder,
-                                                      run_mteb_embed_task)
+from tests.models.language.pooling_mteb_test.mteb_utils import (
+    MTEB_EMBED_TASKS, MTEB_EMBED_TOL, OpenAIClientMtebEncoder,
+    run_mteb_embed_task)
 from tests.utils import RemoteOpenAIServer
 
 os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"

tests/entrypoints/pooling/correctness/test_mteb_score.py

@@ -4,15 +4,9 @@ import os
 
 import pytest
 
-# yapf conflicts with isort for this block
-# yapf: disable
-from tests.models.language.pooling.mteb_utils import (MTEB_RERANK_LANGS,
-                                                      MTEB_RERANK_TASKS,
-                                                      MTEB_RERANK_TOL,
-                                                      RerankClientMtebEncoder,
-                                                      ScoreClientMtebEncoder,
-                                                      run_mteb_rerank)
-# yapf: enable
+from tests.models.language.pooling_mteb_test.mteb_utils import (
+    MTEB_RERANK_LANGS, MTEB_RERANK_TASKS, MTEB_RERANK_TOL,
+    RerankClientMtebEncoder, ScoreClientMtebEncoder, run_mteb_rerank)
 from tests.utils import RemoteOpenAIServer
 
 os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"

tests/models/language/pooling_mteb_test/test_baai.py

@@ -2,10 +2,12 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import pytest
 
-from ...utils import (CLSPoolingEmbedModelInfo, CLSPoolingRerankModelInfo,
-                      EmbedModelInfo, LASTPoolingEmbedModelInfo,
-                      RerankModelInfo)
-from .embed_utils import correctness_test_embed_models
+from tests.models.language.pooling.embed_utils import (
+    correctness_test_embed_models)
+from tests.models.utils import (CLSPoolingEmbedModelInfo,
+                                CLSPoolingRerankModelInfo, EmbedModelInfo,
+                                LASTPoolingEmbedModelInfo, RerankModelInfo)
+
 from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
 
 MODELS = [

tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py

@@ -7,9 +7,9 @@ import pytest
 import torch
 
 from tests.conftest import HfRunner
-
-from ...utils import LASTPoolingRerankModelInfo, RerankModelInfo
-from .mteb_utils import VllmMtebEncoder, mteb_test_rerank_models
+from tests.models.language.pooling_mteb_test.mteb_utils import (
+    VllmMtebEncoder, mteb_test_rerank_models)
+from tests.models.utils import LASTPoolingRerankModelInfo, RerankModelInfo
 
 RERANK_MODELS = [
     LASTPoolingRerankModelInfo("BAAI/bge-reranker-v2-gemma",

tests/models/language/pooling_mteb_test/test_cross_encoder.py

@@ -2,8 +2,9 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import pytest
 
-from ...utils import (CLSPoolingRerankModelInfo, LASTPoolingRerankModelInfo,
-                      RerankModelInfo)
+from tests.models.utils import (CLSPoolingRerankModelInfo,
+                                LASTPoolingRerankModelInfo, RerankModelInfo)
+
 from .mteb_utils import mteb_test_rerank_models
 
 RERANK_MODELS = [

tests/models/language/pooling_mteb_test/test_gte.py

@@ -3,10 +3,12 @@
 
 import pytest
 
-from ...utils import (CLSPoolingEmbedModelInfo, CLSPoolingRerankModelInfo,
-                      EmbedModelInfo, LASTPoolingEmbedModelInfo,
-                      RerankModelInfo)
-from .embed_utils import correctness_test_embed_models
+from tests.models.language.pooling.embed_utils import (
+    correctness_test_embed_models)
+from tests.models.utils import (CLSPoolingEmbedModelInfo,
+                                CLSPoolingRerankModelInfo, EmbedModelInfo,
+                                LASTPoolingEmbedModelInfo, RerankModelInfo)
+
 from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
 
 MODELS = [

tests/models/language/pooling_mteb_test/test_intfloat.py

@@ -2,8 +2,10 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import pytest
 
-from ...utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
-from .embed_utils import correctness_test_embed_models
+from tests.models.language.pooling.embed_utils import (
+    correctness_test_embed_models)
+from tests.models.utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
+
 from .mteb_utils import mteb_test_embed_models
 
 MODELS = [

tests/models/language/pooling_mteb_test/test_jina.py

@@ -4,12 +4,13 @@ from functools import partial
 
 import pytest
 
+from tests.models.language.pooling.embed_utils import (
+    check_embeddings_close, correctness_test_embed_models, matryoshka_fy)
+from tests.models.utils import (CLSPoolingEmbedModelInfo,
+                                CLSPoolingRerankModelInfo, EmbedModelInfo,
+                                RerankModelInfo)
 from vllm import PoolingParams
 
-from ...utils import (CLSPoolingEmbedModelInfo, CLSPoolingRerankModelInfo,
-                      EmbedModelInfo, RerankModelInfo)
-from .embed_utils import (check_embeddings_close,
-                          correctness_test_embed_models, matryoshka_fy)
 from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
 
 EMBEDDING_MODELS = [

tests/models/language/pooling_mteb_test/test_mxbai_rerank.py

@@ -6,8 +6,8 @@ import pytest
 import torch
 
 from tests.conftest import HfRunner
+from tests.models.utils import LASTPoolingRerankModelInfo, RerankModelInfo
 
-from ...utils import LASTPoolingRerankModelInfo, RerankModelInfo
 from .mteb_utils import mteb_test_rerank_models
 
 mxbai_rerank_hf_overrides = {

tests/models/language/pooling_mteb_test/test_nomic.py

@@ -3,8 +3,10 @@
 
 import pytest
 
-from ...utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
-from .embed_utils import correctness_test_embed_models
+from tests.models.language.pooling.embed_utils import (
+    correctness_test_embed_models)
+from tests.models.utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
+
 from .mteb_utils import mteb_test_embed_models
 
 MODELS = [

tests/models/language/pooling_mteb_test/test_qwen3_reranker.py

@@ -6,9 +6,9 @@ import pytest
 import torch
 
 from tests.conftest import HfRunner
+from tests.models.utils import LASTPoolingRerankModelInfo, RerankModelInfo
 from tests.utils import multi_gpu_test
 
-from ...utils import LASTPoolingRerankModelInfo, RerankModelInfo
 from .mteb_utils import mteb_test_rerank_models
 
 qwen3_reranker_hf_overrides = {

tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py

@@ -3,8 +3,10 @@
 
 import pytest
 
-from ...utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
-from .embed_utils import correctness_test_embed_models
+from tests.models.language.pooling.embed_utils import (
+    correctness_test_embed_models)
+from tests.models.utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
+
 from .mteb_utils import mteb_test_embed_models
 
 MODELS = [

tests/models/language/pooling_mteb_test/test_st_projector.py

@@ -2,8 +2,9 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import pytest
 
-from ...utils import (CLSPoolingEmbedModelInfo, EmbedModelInfo,
-                      LASTPoolingEmbedModelInfo)
+from tests.models.utils import (CLSPoolingEmbedModelInfo, EmbedModelInfo,
+                                LASTPoolingEmbedModelInfo)
+
 from .mteb_utils import mteb_test_embed_models
 
 # ST models with projector (Dense) layers