xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-24 11:36:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Core] Avoid metrics log noise when idle (#8868)

Browse Source 
Signed-off-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
Russell Bryant 2024-11-19 16:05:25 -05:00 committed by GitHub
parent 803f37eaaa
commit efa9084628
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
vllm/engine/metrics.py
View File

@ -421,6 +421,11 @@ def get_throughput(tracked_stats: List[int], now: float,
class LoggingStatLogger(StatLoggerBase): class LoggingStatLogger(StatLoggerBase):
"""LoggingStatLogger is used in LLMEngine to log to Stdout.""" """LoggingStatLogger is used in LLMEngine to log to Stdout."""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.last_prompt_throughput: Optional[float] = None
self.last_generation_throughput: Optional[float] = None
def log(self, stats: Stats) -> None: def log(self, stats: Stats) -> None:
"""Called by LLMEngine. """Called by LLMEngine.
Logs to Stdout every self.local_interval seconds.""" Logs to Stdout every self.local_interval seconds."""
@ -445,8 +450,14 @@ class LoggingStatLogger(StatLoggerBase):
now=stats.now, now=stats.now,
last_log=self.last_local_log) last_log=self.last_local_log)
# Log to stdout. log_fn = logger.info
logger.info( if not any((prompt_throughput, generation_throughput,
self.last_prompt_throughput,
self.last_generation_throughput)):
# Avoid log noise on an idle production system
log_fn = logger.debug
log_fn(
"Avg prompt throughput: %.1f tokens/s, " "Avg prompt throughput: %.1f tokens/s, "
"Avg generation throughput: %.1f tokens/s, " "Avg generation throughput: %.1f tokens/s, "
"Running: %d reqs, Swapped: %d reqs, " "Running: %d reqs, Swapped: %d reqs, "
@ -472,11 +483,16 @@ class LoggingStatLogger(StatLoggerBase):
self._format_spec_decode_metrics_str( self._format_spec_decode_metrics_str(
self.spec_decode_metrics)) self.spec_decode_metrics))
# Reset tracked stats for next interval. self._reset(stats, prompt_throughput, generation_throughput)
self.num_prompt_tokens = []
self.num_generation_tokens = [] def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
self.last_local_log = stats.now # Reset tracked stats for next interval.
self.spec_decode_metrics = None self.num_prompt_tokens = []
self.num_generation_tokens = []
self.last_local_log = stats.now
self.spec_decode_metrics = None
self.last_prompt_throughput = prompt_throughput
self.last_generation_throughput = generation_throughput
def _format_spec_decode_metrics_str( def _format_spec_decode_metrics_str(
self, metrics: "SpecDecodeWorkerMetrics") -> str: self, metrics: "SpecDecodeWorkerMetrics") -> str: