diff --git a/vllm/utils.py b/vllm/utils.py
index 17b97f393ff2..1db57bc50c83 100644
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -119,9 +119,11 @@ def is_hip() -> bool:
 
 @lru_cache(maxsize=None)
 def is_cpu() -> bool:
-    from importlib.metadata import version
-    is_cpu_flag = "cpu" in version("vllm")
-    return is_cpu_flag
+    from importlib.metadata import PackageNotFoundError, version
+    try:
+        return "cpu" in version("vllm")
+    except PackageNotFoundError:
+        return False
 
 
 @lru_cache(maxsize=None)