diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py
index cdae42fe4fce8..089d415ab5fb8 100644
--- a/vllm/compilation/backends.py
+++ b/vllm/compilation/backends.py
@@ -399,6 +399,7 @@ class VllmBackend:
         rank = vllm_config.parallel_config.rank
         dp_rank = vllm_config.parallel_config.data_parallel_rank
         local_cache_dir = os.path.join(cache_dir, f"rank_{rank}_{dp_rank}")
+        os.makedirs(local_cache_dir, exist_ok=True)
         self.compilation_config.local_cache_dir = local_cache_dir
 
         disable_cache = envs.VLLM_DISABLE_COMPILE_CACHE