xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-10 09:57:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix Idefics3 bug (#10778)

Browse Source 
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jee Jee Li 2024-11-29 21:56:46 +08:00 committed by GitHub
parent c82b432d4a
commit 3132aac043
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 43 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
vllm/model_executor/models/idefics3.py
View File

@ -267,54 +267,56 @@ def input_processor_for_idefics3(ctx: InputContext,
n_images_in_text = [] n_images_in_text = []
text = inputs.get("prompt") text = inputs.get("prompt")
if text is not None: if text is None:
if isinstance(text, str): prompt_token_ids = inputs.get("prompt_token_ids", [])
text = [text] assert prompt_token_ids
elif not isinstance(text, list) and not isinstance(text[0], str): text = tokenizer.decode(prompt_token_ids)
raise ValueError("Invalid input text. Please provide a string, "
"or a list of strings")
fake_image_token = processor.fake_image_token.content if isinstance(text, str):
image_token = processor.image_token.content text = [text]
global_img_token = processor.global_image_tag elif not isinstance(text, list) and not isinstance(text[0], str):
raise ValueError("Invalid input text. Please provide a string, "
"or a list of strings")
prompt_strings = [] fake_image_token = processor.fake_image_token.content
for sample, sample_rows, sample_cols in zip(text, image_rows, image_token = processor.image_token.content
image_cols): global_img_token = processor.global_image_tag
n_images_in_text.append(sample.count(image_token))
# Replace the image token with fake tokens around the expanded prompt_strings = []
# image token sequence of length `image_seq_len` for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):
image_prompt_strings = [] n_images_in_text.append(sample.count(image_token))
for n_rows, n_cols in zip(sample_rows, sample_cols):
image_prompt_string = _get_image_prompt_string(
n_rows,
n_cols,
processor.image_seq_len,
image_token=image_token,
fake_token_around_image=fake_image_token,
global_img_token=global_img_token,
)
image_prompt_strings.append(image_prompt_string)
split_sample = sample.split(image_token) # Replace the image token with fake tokens around the expanded
if len(split_sample) == 0: # image token sequence of length `image_seq_len`
raise ValueError( image_prompt_strings = []
"The image token should be present in the text.") for n_rows, n_cols in zip(sample_rows, sample_cols):
image_prompt_string = _get_image_prompt_string(
n_rows,
n_cols,
processor.image_seq_len,
image_token=image_token,
fake_token_around_image=fake_image_token,
global_img_token=global_img_token,
)
image_prompt_strings.append(image_prompt_string)
# Place in the image prompt strings where the image tokens are split_sample = sample.split(image_token)
sample = split_sample[0] if len(split_sample) == 0:
for i, image_prompt_string in enumerate(image_prompt_strings): raise ValueError("The image token should be present in the text.")
sample += image_prompt_string + split_sample[i + 1]
prompt_strings.append(sample)
prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids # Place in the image prompt strings where the image tokens are
sample = split_sample[0]
for i, image_prompt_string in enumerate(image_prompt_strings):
sample += image_prompt_string + split_sample[i + 1]
prompt_strings.append(sample)
return token_inputs( prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
prompt_token_ids=prompt_token_ids,
prompt=prompt_strings[0], return token_inputs(
multi_modal_data=multi_modal_data, prompt_token_ids=prompt_token_ids,
) prompt=prompt_strings[0],
multi_modal_data=multi_modal_data,
)
def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int: def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int: