xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-24 10:17:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

[MISC] rename interval to max_recent_requests (#14285)

Browse Source
This commit is contained in:
Ning Xie 2025-04-27 00:59:18 +08:00 committed by GitHub
parent 4d17e20310
commit fd11a325b8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/v1/core/test_kv_cache_utils.py
View File

@ -310,7 +310,7 @@ def test_metrics():
def stats(requests, queries, hits): def stats(requests, queries, hits):
return PrefixCacheStats(requests=requests, queries=queries, hits=hits) return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
metrics = PrefixCachingMetrics(interval=5) metrics = PrefixCachingMetrics(max_recent_requests=5)
assert metrics.hit_rate == 0.0 assert metrics.hit_rate == 0.0
metrics.observe(stats(1, 20, 9)) metrics.observe(stats(1, 20, 9))

12
vllm/v1/core/kv_cache_utils.py
View File

@ -47,15 +47,15 @@ NONE_HASH = int.from_bytes(os.urandom(32), byteorder="big") if os.getenv(
class PrefixCachingMetrics: class PrefixCachingMetrics:
"""Metrics for prefix caching with a hit rate of the most recent N requests. """Metrics for prefix caching with a hit rate of the max recent N requests.
Args: Args:
interval: The number of the most recent requests to aggregate. max_recent_requests: The number of the max recent requests to aggregate.
Defaults to 1000. Defaults to 1000.
""" """
def __init__(self, interval: int = 1000): def __init__(self, max_recent_requests: int = 1000):
self.interval = interval self.max_recent_requests = max_recent_requests
# The current aggregated values. # The current aggregated values.
self.aggregated_requests = 0 self.aggregated_requests = 0
self.aggregated_query_total = 0 self.aggregated_query_total = 0
@ -70,7 +70,7 @@ class PrefixCachingMetrics:
are being scheduled and are looking for computed blocks. are being scheduled and are looking for computed blocks.
When there are more than `interval` requests, the oldest set of When there are more than `interval` requests, the oldest set of
requestsare removed from the metrics. requests are removed from the metrics.
Args: Args:
stats: The prefix cache stats. stats: The prefix cache stats.
@ -87,7 +87,7 @@ class PrefixCachingMetrics:
self.aggregated_query_hit += stats.hits self.aggregated_query_hit += stats.hits
# Remove the oldest stats if the number of requests exceeds. # Remove the oldest stats if the number of requests exceeds.
if self.aggregated_requests > self.interval: if self.aggregated_requests > self.max_recent_requests:
old_requests, old_queries, old_hits = self.query_queue.popleft() old_requests, old_queries, old_hits = self.query_queue.popleft()
self.aggregated_requests -= old_requests self.aggregated_requests -= old_requests
self.aggregated_query_total -= old_queries self.aggregated_query_total -= old_queries