diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 833581035a31..e6cfcad3d696 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -5,6 +5,7 @@ import copy import hashlib import json import os +import time from contextlib import contextmanager from dataclasses import field, replace from functools import lru_cache @@ -270,6 +271,9 @@ class VllmConfig: def __post_init__(self): """Verify configs are valid & consistent with each other.""" + # To give each torch profile run a unique instance name. + self.instance_id = f"{time.time_ns()}" + self.try_verify_and_update_config() if self.model_config is not None: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 4f4da73fba6e..119e474b1fca 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -79,6 +79,7 @@ class Worker(WorkerBase): # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace if envs.VLLM_TORCH_PROFILER_DIR: torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR + worker_name = f"{vllm_config.instance_id}-rank-{self.rank}" logger.info( "Profiling enabled. Traces will be saved to: %s", torch_profiler_trace_dir, @@ -101,7 +102,7 @@ class Worker(WorkerBase): with_stack=envs.VLLM_TORCH_PROFILER_WITH_STACK, with_flops=envs.VLLM_TORCH_PROFILER_WITH_FLOPS, on_trace_ready=torch.profiler.tensorboard_trace_handler( - torch_profiler_trace_dir, use_gzip=True + torch_profiler_trace_dir, worker_name=worker_name, use_gzip=True ), ) else: diff --git a/vllm/v1/worker/xpu_worker.py b/vllm/v1/worker/xpu_worker.py index a1e54628d9ed..31fa3f3bd6ac 100644 --- a/vllm/v1/worker/xpu_worker.py +++ b/vllm/v1/worker/xpu_worker.py @@ -39,6 +39,7 @@ class XPUWorker(Worker): # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace if envs.VLLM_TORCH_PROFILER_DIR: torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR + worker_name = f"{vllm_config.instance_id}-rank-{self.rank}" logger.info( "Profiling enabled. Traces will be saved to: %s", torch_profiler_trace_dir, @@ -61,7 +62,7 @@ class XPUWorker(Worker): with_stack=envs.VLLM_TORCH_PROFILER_WITH_STACK, with_flops=envs.VLLM_TORCH_PROFILER_WITH_FLOPS, on_trace_ready=torch.profiler.tensorboard_trace_handler( - torch_profiler_trace_dir, use_gzip=True + torch_profiler_trace_dir, worker_name=worker_name, use_gzip=True ), ) else: