xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-03-19 05:57:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

[V1][Minor] Print KV cache size in token counts (#13596)

Browse Source 
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon 2025-02-20 09:24:31 -08:00 committed by GitHub
parent 34aad515c8
commit d3ea50113c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
vllm/v1/core/kv_cache_utils.py
View File

@ -519,11 +519,13 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
"num_gpu_blocks_override=%d", num_blocks, num_gpu_blocks_override)
num_blocks = num_gpu_blocks_override
logger.info("# GPU blocks: %d", num_blocks)
max_concurrency = (num_blocks * vllm_config.cache_config.block_size /
vllm_config.model_config.max_model_len)
num_tokens = num_blocks * vllm_config.cache_config.block_size
num_tokens_str = f"{num_tokens:,}"
logger.info("GPU KV cache size: %s tokens", num_tokens_str)
max_model_len_str = f"{vllm_config.model_config.max_model_len:,}"
max_concurrency = num_tokens / vllm_config.model_config.max_model_len
logger.info("Maximum concurrency for %s tokens per request: %.2fx",
vllm_config.model_config.max_model_len, max_concurrency)
max_model_len_str, max_concurrency)
per_layer_size = page_size * num_blocks