xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 06:35:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Convert before transpose (#1073)

Browse Source
This commit is contained in:
Woosuk Kwon 2023-09-18 11:51:48 -07:00 committed by GitHub
parent f029ef94d7
commit cc796b1358
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/model_executor/models/llama.py
View File

@ -43,8 +43,8 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
VocabParallelEmbedding) VocabParallelEmbedding)
from vllm.model_executor.quantization_utils import QuantizationConfig from vllm.model_executor.quantization_utils import QuantizationConfig
from vllm.model_executor.weight_utils import ( from vllm.model_executor.weight_utils import (
load_tensor_parallel_weights, load_padded_tensor_parallel_vocab, convert_pyslice_to_tensor, hf_model_weights_iterator,
hf_model_weights_iterator) load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
from vllm.sequence import SamplerOutput from vllm.sequence import SamplerOutput
KVCache = Tuple[torch.Tensor, torch.Tensor] KVCache = Tuple[torch.Tensor, torch.Tensor]
@ -337,6 +337,7 @@ class LlamaForCausalLM(nn.Module):
is_packed = self.quant_config.is_packed(name) is_packed = self.quant_config.is_packed(name)
is_transposed = self.quant_config.is_transposed(name) is_transposed = self.quant_config.is_transposed(name)
if is_transposed: if is_transposed:
loaded_weight = convert_pyslice_to_tensor(loaded_weight)
loaded_weight = loaded_weight.T loaded_weight = loaded_weight.T
is_attention_weight = False is_attention_weight = False