mirror of
https://git.datalinker.icu/vllm-project/vllm.git
synced 2026-07-17 15:57:13 +08:00
Revert the mixin changes
This commit is contained in:
parent
b03d1a04a8
commit
d525556a25
@ -820,34 +820,7 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
|
|||||||
return super()._parse_video_data(data)
|
return super()._parse_video_data(data)
|
||||||
|
|
||||||
|
|
||||||
class QwenVLSeriesProcessingInfoMixin:
|
class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||||
"""
|
|
||||||
Mixin that provides get_num_mm_encoder_tokens()
|
|
||||||
and get_num_mm_connector_tokens() methods for
|
|
||||||
QwenVL series models without affecting other multi-modal models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def get_num_mm_encoder_tokens(
|
|
||||||
self,
|
|
||||||
num_image_tokens: int,
|
|
||||||
) -> int:
|
|
||||||
hf_config = self.get_hf_config()
|
|
||||||
vision_config = hf_config.vision_config
|
|
||||||
merge_size = vision_config.spatial_merge_size
|
|
||||||
|
|
||||||
return num_image_tokens * merge_size**2
|
|
||||||
|
|
||||||
def get_num_mm_connector_tokens(
|
|
||||||
self,
|
|
||||||
num_vision_tokens: int,
|
|
||||||
) -> int:
|
|
||||||
hf_config = self.get_hf_config()
|
|
||||||
vision_config = hf_config.vision_config
|
|
||||||
merge_size = vision_config.spatial_merge_size
|
|
||||||
return num_vision_tokens // merge_size**2
|
|
||||||
|
|
||||||
|
|
||||||
class Qwen2VLProcessingInfo(QwenVLSeriesProcessingInfoMixin, BaseProcessingInfo):
|
|
||||||
def get_hf_config(self):
|
def get_hf_config(self):
|
||||||
return self.ctx.get_hf_config(Qwen2VLConfig)
|
return self.ctx.get_hf_config(Qwen2VLConfig)
|
||||||
|
|
||||||
@ -1131,6 +1104,25 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo])
|
|||||||
for modality in ("image", "video")
|
for modality in ("image", "video")
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def get_num_mm_encoder_tokens(
|
||||||
|
self,
|
||||||
|
num_image_tokens: int,
|
||||||
|
) -> int:
|
||||||
|
hf_config = self.get_hf_config()
|
||||||
|
vision_config = hf_config.vision_config
|
||||||
|
merge_size = vision_config.spatial_merge_size
|
||||||
|
|
||||||
|
return num_image_tokens * merge_size**2
|
||||||
|
|
||||||
|
def get_num_mm_connector_tokens(
|
||||||
|
self,
|
||||||
|
num_vision_tokens: int,
|
||||||
|
) -> int:
|
||||||
|
hf_config = self.get_hf_config()
|
||||||
|
vision_config = hf_config.vision_config
|
||||||
|
merge_size = vision_config.spatial_merge_size
|
||||||
|
return num_vision_tokens // merge_size**2
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
|
|||||||
@ -1412,6 +1412,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def get_num_mm_encoder_tokens(
|
||||||
|
self,
|
||||||
|
num_image_tokens: int,
|
||||||
|
) -> int:
|
||||||
|
"""
|
||||||
|
Implement this function to enable LoRA support
|
||||||
|
for the tower module of the multi-modal model
|
||||||
|
Given the number of image tokens, output the number of multi-modal encoder tokens
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def get_num_mm_connector_tokens(
|
||||||
|
self,
|
||||||
|
num_vision_tokens: int,
|
||||||
|
) -> int:
|
||||||
|
"""
|
||||||
|
Implement this function to enable LoRA support
|
||||||
|
for the connector module of the multi-modal model
|
||||||
|
Given the number of vision tokens, output the number of multi-modal connector tokens
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
def _bind_and_group_updates(
|
def _bind_and_group_updates(
|
||||||
self,
|
self,
|
||||||
prompt_updates: Sequence[PromptUpdate],
|
prompt_updates: Sequence[PromptUpdate],
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user