xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-25 17:07:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix cpu usage and cache hit stats reporting on cpu environment (#18674)

Browse Source 
Signed-off-by: zzzyq <zhangyuqi94@gmail.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Yuqi Zhang 2025-05-25 02:36:06 -07:00 committed by GitHub
parent 279f854519
commit f2faac745d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
vllm/engine/llm_engine.py
View File

@ -1650,6 +1650,20 @@ class LLMEngine:
gpu_prefix_cache_hit_rate = self.scheduler[
0].get_prefix_cache_hit_rate(Device.GPU)
# Exchange the uasge and cache hit stats between gpu and cpu when
# running on cpu because the cpu_worker.py intentionally reports the
# number of cpu blocks as gpu blocks in favor of cache management.
if self.device_config.device_type == "cpu":
num_total_gpu, num_total_cpu = num_total_cpu, num_total_gpu
gpu_cache_usage_sys, cpu_cache_usage_sys = (
cpu_cache_usage_sys,
gpu_cache_usage_sys,
)
gpu_prefix_cache_hit_rate, cpu_prefix_cache_hit_rate = (
cpu_prefix_cache_hit_rate,
gpu_prefix_cache_hit_rate,
)
# Iteration stats
num_prompt_tokens_iter = 0
num_generation_tokens_iter = 0