xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-09 22:25:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix eager mode performance (#2377)

Browse Source
This commit is contained in:
Woosuk Kwon 2024-01-08 10:11:06 -08:00 committed by GitHub
parent 05921a9a7a
commit c884819135
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/worker/model_runner.py
View File

@ -235,9 +235,11 @@ class ModelRunner:
input_block_tables[i, :len(block_table)] = block_table
block_tables = torch.tensor(input_block_tables, device="cuda")
else:
max_block_table_len = (max_context_len + self.block_size -
1) // self.block_size
block_tables = _make_tensor_with_pad(
block_tables,
max_len=max_context_len,
max_len=max_block_table_len,
pad=0,
dtype=torch.int,
device="cuda",