xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-04-24 06:47:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[torch.compile] use size tuning for specific sizes (#10933)

Browse Source 
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao 2024-12-05 20:30:41 -08:00 committed by GitHub
parent 9743d64e4e
commit db87eb6c67
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/compilation/backends.py
View File

@ -43,6 +43,12 @@ def wrap_inductor(graph,
if additional_inductor_config is not None: if additional_inductor_config is not None:
current_config.update(additional_inductor_config) current_config.update(additional_inductor_config)
if isinstance(runtime_shape, int):
# for a specific batchsize, tuning triton kernel parameters
# can be beneficial
current_config["max_autotune"] = True
current_config["coordinate_descent_tuning"] = True
# inductor can inplace modify the graph, so we need to copy it # inductor can inplace modify the graph, so we need to copy it
# see https://github.com/pytorch/pytorch/issues/138980 # see https://github.com/pytorch/pytorch/issues/138980
graph = copy.deepcopy(graph) graph = copy.deepcopy(graph)