diff --git a/CMakeLists.txt b/CMakeLists.txt
index c3719526c4ae..538f9adcb24e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -174,8 +174,10 @@ endif()
 #
 # Set nvcc fatbin compression.
 #
-if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND VLLM_GPU_LANG STREQUAL "CUDA")
-  list(APPEND VLLM_GPU_FLAGS "-Xfatbin" "-compress-all" "-compress-mode=size")
+if(VLLM_GPU_LANG STREQUAL "CUDA")
+  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8)
+    list(APPEND VLLM_GPU_FLAGS "-Xfatbin" "-compress-all" "-compress-mode=size")
+  endif()
 endif()