xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-08 16:07:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

SM120 / NVFP4: add device guard and runtime SM dispatch to cutlass_scaled_fp4_mm (#29711)

Browse Source 
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Hendrik Holtmann 2025-12-02 02:24:18 +01:00 committed by GitHub
parent 44822d7ff2
commit c0dfc89485
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
csrc/quantization/fp4/nvfp4_scaled_mm_entry.cu
View File

@ -15,6 +15,8 @@
*/ */
#include <torch/all.h> #include <torch/all.h>
#include <c10/cuda/CUDAGuard.h>
#include "cutlass_extensions/common.hpp"
#if defined ENABLE_NVFP4_SM100 && ENABLE_NVFP4_SM100 #if defined ENABLE_NVFP4_SM100 && ENABLE_NVFP4_SM100
void cutlass_scaled_fp4_mm_sm100a(torch::Tensor& D, torch::Tensor const& A, void cutlass_scaled_fp4_mm_sm100a(torch::Tensor& D, torch::Tensor const& A,
@ -32,23 +34,34 @@ void cutlass_scaled_fp4_mm_sm120a(torch::Tensor& D, torch::Tensor const& A,
torch::Tensor const& alpha); torch::Tensor const& alpha);
#endif #endif
void cutlass_scaled_fp4_mm(torch::Tensor& D, torch::Tensor const& A, void cutlass_scaled_fp4_mm(torch::Tensor& D, const torch::Tensor& A,
torch::Tensor const& B, torch::Tensor const& A_sf, const torch::Tensor& B, const torch::Tensor& A_sf,
torch::Tensor const& B_sf, const torch::Tensor& B_sf,
torch::Tensor const& alpha) { const torch::Tensor& alpha) {
#if defined ENABLE_NVFP4_SM100 && ENABLE_NVFP4_SM100 // Make sure we’re on A’s device.
return cutlass_scaled_fp4_mm_sm100a(D, A, B, A_sf, B_sf, alpha); const c10::cuda::OptionalCUDAGuard device_guard(device_of(A));
#elif defined ENABLE_NVFP4_SM120 && ENABLE_NVFP4_SM120 const int32_t sm = get_sm_version_num();
return cutlass_scaled_fp4_mm_sm120a(D, A, B, A_sf, B_sf, alpha);
#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
if (sm >= 100 && sm < 120) {
cutlass_scaled_fp4_mm_sm100a(D, A, B, A_sf, B_sf, alpha);
return;
}
#endif #endif
TORCH_CHECK_NOT_IMPLEMENTED(false,
"No compiled nvfp4 mm kernel, vLLM should " #if defined(ENABLE_NVFP4_SM120) && ENABLE_NVFP4_SM120
"be compiled using CUDA 12.8 and target " if (sm >= 120 && sm < 130) {
"compute capability 100 or above."); cutlass_scaled_fp4_mm_sm120a(D, A, B, A_sf, B_sf, alpha);
return;
}
#endif
TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 mm kernel for SM ", sm,
". Recompile with CUDA >= 12.8 and CC >= 100.");
} }
bool cutlass_scaled_mm_supports_fp4(int64_t cuda_device_capability) { bool cutlass_scaled_mm_supports_fp4(int64_t cuda_device_capability) {
int runtimeVersion; int runtimeVersion;
cudaRuntimeGetVersion(&runtimeVersion); cudaRuntimeGetVersion(&runtimeVersion);
return cuda_device_capability >= 100 && runtimeVersion >= 12080; return cuda_device_capability >= 100 && runtimeVersion >= 12080;
} }