xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-24 07:37:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Add reset prefix cache for online serving (#22726)

Browse Source 
Signed-off-by: iAmir97 <Amir.balwel@embeddedllm.com>
Signed-off-by: iAmir97 <71513472+iAmir97@users.noreply.github.com>
Co-authored-by: iAmir97 <Amir.balwel@embeddedllm.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
iAmir97 2025-08-14 18:04:18 +07:00 committed by GitHub
parent f4efda821d
commit 7655dc3e45
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
vllm/engine/async_llm_engine.py
View File

@ -1092,6 +1092,7 @@ class AsyncLLMEngine(EngineClient):
self.engine.reset_prefix_cache(device) self.engine.reset_prefix_cache(device)
async def sleep(self, level: int = 1) -> None: async def sleep(self, level: int = 1) -> None:
await self.reset_prefix_cache()
self.engine.sleep(level) self.engine.sleep(level)
async def wake_up(self, tags: Optional[list[str]] = None) -> None: async def wake_up(self, tags: Optional[list[str]] = None) -> None:

1
vllm/v1/engine/async_llm.py
View File

@ -576,6 +576,7 @@ class AsyncLLM(EngineClient):
await self.engine_core.reset_prefix_cache_async() await self.engine_core.reset_prefix_cache_async()
async def sleep(self, level: int = 1) -> None: async def sleep(self, level: int = 1) -> None:
await self.reset_prefix_cache()
await self.engine_core.sleep_async(level) await self.engine_core.sleep_async(level)
async def wake_up(self, tags: Optional[list[str]] = None) -> None: async def wake_up(self, tags: Optional[list[str]] = None) -> None: