xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-09 11:37:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

add compile

Browse Source
This commit is contained in:
youkaichao 2024-07-26 19:29:36 -07:00
parent 55712941e5
commit 617fb893d5
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
vllm/worker/model_runner.py
View File

@ -787,6 +787,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
"provided. Defaulting to scaling factors of 1.0. " "provided. Defaulting to scaling factors of 1.0. "
"This may lead to less accurate results!") "This may lead to less accurate results!")
count = 0
def backend(gm, input):
nonlocal count
count += 1
print(count)
return gm.forward
self.model = torch.compile(self.model, backend=backend, fullgraph=True)
def save_sharded_state( def save_sharded_state(
self, self,
path: str, path: str,