xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-14 14:41:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

get num_embeddings from params + cleanup + minimize diff + ruff

Browse Source
This commit is contained in:
Netanel Haber 2025-12-23 04:14:41 -08:00
parent 2a7ea9ba37
commit eac0271b0f
1 changed files with 150 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

298
vllm/model_executor/models/nano_nemotron_vl.py
View File

@ -88,7 +88,6 @@ Image.MAX_IMAGE_PIXELS = None # Disable the limit entirely
# Image.MAX_IMAGE_PIXELS = 300000000 # ~300M pixels # Image.MAX_IMAGE_PIXELS = 300000000 # ~300M pixels
# TODO(nhaber): get 2048 from config
# TODO(nhaber): does use_thumbnail=True work? # TODO(nhaber): does use_thumbnail=True work?
# TODO(nhaber): mixing images and videos will mess up the "text_prompt_length" calculation. # TODO(nhaber): mixing images and videos will mess up the "text_prompt_length" calculation.
@ -102,28 +101,20 @@ IMG_CONTEXT = "<image>"
DEFAULT_NUM_TILES = 12 DEFAULT_NUM_TILES = 12
@dataclass(kw_only=True, frozen=True) def num_image_token_per_tile(
class Dims: *, width: int, height: int, patch_size: int, downsample_ratio: int
height: int ) -> int:
width: int tile_size = math.sqrt((width // patch_size) * (height // patch_size))
num_tokens = int(tile_size**2 // (downsample_ratio**2))
CONV_MERGING = False # This is assumed to be False for now
PIXEL_SHUFFLE = True # This is assumed to be True for now
REDUCTION_FACTOR = 2 ** (PIXEL_SHUFFLE + CONV_MERGING)
def num_image_token_per_tile(*, tile_dims: Dims, patch_size: int, downsample_ratio: int) -> int:
tile_size = math.sqrt(tile_dims.width * tile_dims.height)
num_tokens = int(
(tile_size // patch_size) ** 2 * (downsample_ratio**2)
)
return num_tokens return num_tokens
def width_and_height_for_max_num_tokens_available( def width_and_height_for_max_num_tokens_available(
*, *,
target_num_tokens_post_shuffle: int, target_num_tokens_post_shuffle: int,
patch_size: int, patch_size: int,
) -> Dims: downsample_ratio: int,
) -> tuple[int, int]:
""" """
TODO(nhaber): optimize this so it squeezes closer to target number of tokens. TODO(nhaber): optimize this so it squeezes closer to target number of tokens.
Calculate image dimensions that produce approximately `target` tokens after Calculate image dimensions that produce approximately `target` tokens after
@ -133,14 +124,26 @@ def width_and_height_for_max_num_tokens_available(
need 4*B patches to get B tokens. need 4*B patches to get B tokens.
Examples: Examples:
>>> dims = width_and_height_for_max_num_tokens_available(B=8192, patch_size=16) >>> width, height = width_and_height_for_max_num_tokens_available(
>>> assert dims.width, dims.height == (2880, 2880) ... target_num_tokens_post_shuffle=8192,
>>> assert ((dims.width // 16) * (dims.height // 16) // 4) == 8100 # tokens after shuffle ... patch_size=16,
>>> assert num_image_token_per_tile(tile_dims=dims, patch_size=16, downsample_ratio=2) == 8100 ... downsample_ratio=2,
... )
>>> assert width, height == (2880, 2880)
>>> assert (width // 16) * (height // 16) // 2**2 == 8100 # tokens post-shuffle
>>> assert (
... num_image_token_per_tile(
... width=width, height=height, patch_size=16, downsample_ratio=2
... )
... == 8100
... )
""" """
side_pixels = math.isqrt(target_num_tokens_post_shuffle) * REDUCTION_FACTOR * patch_size side_pixels = (
math.isqrt(target_num_tokens_post_shuffle) * downsample_ratio * patch_size
)
assert isinstance(side_pixels, int) and side_pixels % patch_size == 0 assert isinstance(side_pixels, int) and side_pixels % patch_size == 0
return Dims(width=side_pixels, height=side_pixels) return side_pixels, side_pixels
@dataclass @dataclass
class DynamicResolutionParams: class DynamicResolutionParams:
@ -354,7 +357,7 @@ class BaseNanoNemotronVLProcessor(ABC):
self.max_num_tiles = max_num_tiles or DEFAULT_NUM_TILES self.max_num_tiles = max_num_tiles or DEFAULT_NUM_TILES
image_size: int = config.force_image_size image_size: int = config.force_image_size
self.patch_size: int = getattr(config, "patch_size", 16) self.patch_size: int = getattr(config, "patch_size", 16)
self.downsample_ratio: float = self.config.downsample_ratio # self.downsample_ratio: float = self.config.downsample_ratio
self.image_size = image_size self.image_size = image_size
self.use_thumbnail: bool = config.use_thumbnail self.use_thumbnail: bool = config.use_thumbnail
@ -392,9 +395,10 @@ class BaseNanoNemotronVLProcessor(ABC):
) )
return num_tiles * num_image_token_per_tile( return num_tiles * num_image_token_per_tile(
tile_dims=Dims(width=image_width, height=image_height), width=image_width,
height=image_height,
patch_size=self.patch_size, patch_size=self.patch_size,
downsample_ratio=self.downsample_ratio downsample_ratio=self.downsample_ratio,
) )
def _images_to_pixel_values_lst( def _images_to_pixel_values_lst(
@ -508,8 +512,9 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
super().__init__( super().__init__(
config=config, tokenizer=tokenizer, max_num_tiles=max_num_tiles, **kwargs config=config, tokenizer=tokenizer, max_num_tiles=max_num_tiles, **kwargs
) )
self.max_model_len = max_model_len
self._patch_size: int = getattr(config, "patch_size", 16)
self.max_model_len = max_model_len
self._min_num_patches = min_num_patches self._min_num_patches = min_num_patches
self._factor_max = factor_max self._factor_max = factor_max
self._pixel_shuffle = pixel_shuffle self._pixel_shuffle = pixel_shuffle
@ -518,47 +523,90 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
self._use_thumbnail = use_thumbnail self._use_thumbnail = use_thumbnail
self._thumbnail_size = thumbnail_size self._thumbnail_size = thumbnail_size
self._thumbnail_area_threshold = thumbnail_area_threshold self._thumbnail_area_threshold = thumbnail_area_threshold
self.norm_mean = torch.tensor(self.CLIP_PIXEL_MEAN).reshape(1, 3, 1, 1)
self.norm_std = torch.tensor(self.CLIP_PIXEL_STD).reshape(1, 3, 1, 1)
self._transform = T.Compose( self._transform = T.Compose(
[ [
T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
T.ToTensor(), # T.Lambda(lambda img: _fast_to_tensor(img)), T.ToTensor(), # T.Lambda(lambda img: _fast_to_tensor(img)),
# T.Normalize(mean=pixel_mean, std=pixel_std), - This is done down below with input_conditioner
] ]
) )
self._apply_data_augment = apply_data_augment self._apply_data_augment = apply_data_augment
reduction_factor = 1 / self.config.downsample_ratio
assert reduction_factor == 2.0, (
"I don't understand what's going on if this isn't 4"
)
self.downsample_ratio = int(reduction_factor) ** (pixel_shuffle + conv_merging)
assert self.downsample_ratio == 2, (
f"I don't understand what's going on if {self.downsample_ratio=} isn't 2"
)
self.norm_mean = torch.tensor(self.CLIP_PIXEL_MEAN).reshape(1, 3, 1, 1) def _get_num_embeddings(self, width: int, height: int) -> int:
self.norm_std = torch.tensor(self.CLIP_PIXEL_STD).reshape(1, 3, 1, 1) return num_image_token_per_tile(
self.downsample_ratio = 2 if pixel_shuffle else 1 width=width,
height=height,
patch_size=self._patch_size,
downsample_ratio=self.downsample_ratio,
)
def max_num_tokens_available(self, text_prompt_length: int) -> int:
return self.max_model_len - text_prompt_length - 4
def _images_to_pixel_values_lst(
self,
text_prompt_length: int,
images: list[Image.Image],
max_num_tiles: int,
) -> tuple[list[torch.Tensor], list[int]]:
num_tokens_available = self.max_num_tokens_available(text_prompt_length)
params_per_image = self.compute_params(images, num_tokens_available)
feature_sizes = []
images = []
for param in params_per_image:
for t in self.apply_params(param):
if t.ndim == 3:
t = t.unsqueeze(0)
images.append(t)
feature_sizes.append(param.num_embeddings)
print(f"{feature_sizes=}")
print(f"{params_per_image=}")
return images, feature_sizes
feature_size_cache: dict[ feature_size_cache: dict[
Image.Image, int Image.Image, int
] = {} # TODO(nhaber): Find a less silly way of doing this... Why can't this be a class variable? ] = {} # TODO(nhaber): Find a less silly way of doing this... Why can't this be an instance variable?
def apply_params(self, params: DynamicResolutionParams) -> torch.Tensor: def get_cached_feature_size(self, image: Image.Image) -> int:
feature_size = self.feature_size_cache[id(image)]
del self.feature_size_cache[id(image)]
return feature_size
def apply_params(self, params: DynamicResolutionParams) -> list[torch.Tensor]:
resized_img = params.media.resize( resized_img = params.media.resize(
( (
params.patch_size[0] * self.patch_size, params.patch_size[0] * self._patch_size,
params.patch_size[1] * self.patch_size, params.patch_size[1] * self._patch_size,
) )
) )
# processed_images = [resized_img] processed_images = [resized_img]
# # Add thumbnail if enabled and image area is below threshold # Add thumbnail if enabled and image area is below threshold
# if self._use_thumbnail: if self._use_thumbnail:
# # Calculate areas # Calculate areas
# resized_area = resized_img.size[0] * resized_img.size[1] resized_area = resized_img.size[0] * resized_img.size[1]
# thumbnail_area = self._thumbnail_size * self._thumbnail_size thumbnail_area = self._thumbnail_size * self._thumbnail_size
# area_ratio = resized_area / thumbnail_area area_ratio = resized_area / thumbnail_area
# # Only add thumbnail if resized image area is less than threshold % of # Only add thumbnail if resized image area is less than threshold % of thumbnail area
# # thumbnail area if area_ratio < self._thumbnail_area_threshold:
# if area_ratio < self._thumbnail_area_threshold: thumbnail_img = params.media.resize(
# thumbnail_img = params.media.resize( (self._thumbnail_size, self._thumbnail_size)
# (self._thumbnail_size, self._thumbnail_size) )
# ) processed_images.append(thumbnail_img)
# processed_images.append(thumbnail_img)
return self._transform(resized_img) return [self._transform(img) for img in processed_images]
def process_media( def process_media(
self, self,
@ -568,11 +616,11 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
tiling_augment_prob: float = 0.4, tiling_augment_prob: float = 0.4,
) -> tuple[DynamicResolutionParams, int]: ) -> tuple[DynamicResolutionParams, int]:
"""Process a single media item and return its parameters. """Process a single media item and return its parameters.
Args: Args:
media: The media item to process media: The media item to process
num_tokens_available: Number of tokens available for this media num_tokens_available: Number of tokens available for this media
data_augment: Whether to apply data augmentation to the image. Defaults to data_augment: Whether to apply data augmentation to the image. Defaults to False.
False.
Returns: Returns:
DynamicResolutionParams for the media DynamicResolutionParams for the media
""" """
@ -581,11 +629,9 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
"Dynamic resolution is only supported for image media" "Dynamic resolution is only supported for image media"
) )
orig_width, orig_height = media.width, media.height orig_width, orig_height = media.width, media.height
# TODO(nhaber): Ask Tyler - the round + 0.5 code is dangerous [banker's rounding], no?
closest_patch_height = math.ceil( closest_patch_height = round(orig_height / self._patch_size + 0.5)
orig_height / self.patch_size closest_patch_width = round(orig_width / self._patch_size + 0.5)
) # TODO(nhaber): Ask Tyler - the previous round + 0.5 code is dangerous [banker's rounding], no? If we flip this back to the round, the max_wh_fill_budget needs to do -1 for each of w;h to be safe
closest_patch_width = math.ceil(orig_width / self.patch_size)
patches = closest_patch_height * closest_patch_width patches = closest_patch_height * closest_patch_width
factor = min( factor = min(
@ -594,8 +640,7 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
target_patch_height = math.floor(factor * closest_patch_height) target_patch_height = math.floor(factor * closest_patch_height)
target_patch_width = math.floor(factor * closest_patch_width) target_patch_width = math.floor(factor * closest_patch_width)
# We only consider self._min_num_patches if it is greater than # We only consider self._min_num_patches if it is greater than current_num_tokens_available.
# current_num_tokens_available.
if ( if (
current_num_tokens_available > self._min_num_patches current_num_tokens_available > self._min_num_patches
and target_patch_height * target_patch_width < self._min_num_patches and target_patch_height * target_patch_width < self._min_num_patches
@ -608,20 +653,19 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
if ( if (
self._min_side is not None self._min_side is not None
and min(target_patch_width, target_patch_height) * self.patch_size and min(target_patch_width, target_patch_height) * self._patch_size
< self._min_side < self._min_side
): ):
if target_patch_width <= target_patch_height: if target_patch_width <= target_patch_height:
up_factor = self._min_side / (target_patch_width * self.patch_size) up_factor = self._min_side / (target_patch_width * self._patch_size)
new_patch_height = math.ceil(up_factor * target_patch_height) new_patch_height = math.ceil(up_factor * target_patch_height)
new_patch_width = math.ceil(up_factor * target_patch_width) new_patch_width = math.ceil(up_factor * target_patch_width)
if new_patch_height * new_patch_width > current_num_tokens_available: if new_patch_height * new_patch_width > current_num_tokens_available:
# If only one side can be min_side, make as big as possible at # If only one side can be min_side, make as big as possible at native aspect ratio while staying below max_patches
# native aspect ratio while staying below max_patches
if ( if (
max(current_num_tokens_available // new_patch_width, 1) max(current_num_tokens_available // new_patch_width, 1)
* self.patch_size * self._patch_size
< self._min_side < self._min_side
): ):
up_factor = math.sqrt( up_factor = math.sqrt(
@ -640,16 +684,15 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
target_patch_height = new_patch_height target_patch_height = new_patch_height
target_patch_width = new_patch_width target_patch_width = new_patch_width
else: else:
up_factor = self._min_side / (target_patch_height * self.patch_size) up_factor = self._min_side / (target_patch_height * self._patch_size)
new_patch_height = math.ceil(up_factor * target_patch_height) new_patch_height = math.ceil(up_factor * target_patch_height)
new_patch_width = math.ceil(up_factor * target_patch_width) new_patch_width = math.ceil(up_factor * target_patch_width)
if new_patch_height * new_patch_width > current_num_tokens_available: if new_patch_height * new_patch_width > current_num_tokens_available:
# If only one side can be min_side, make as big as possible at # If only one side can be min_side, make as big as possible at native aspect ratio while staying below max_patches
# native aspect ratio while staying below max_patches
if ( if (
max(current_num_tokens_available // new_patch_height, 1) max(current_num_tokens_available // new_patch_height, 1)
* self.patch_size * self._patch_size
< self._min_side < self._min_side
): ):
up_factor = math.sqrt( up_factor = math.sqrt(
@ -708,15 +751,10 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
target_patch_width, target_patch_height, current_num_tokens_available target_patch_width, target_patch_height, current_num_tokens_available
) )
assert isinstance(media, Image.Image), (
"Dynamic resolution is only supported for image media"
)
# Calculate embeddings for the main dynamic resolution image # Calculate embeddings for the main dynamic resolution image
num_embeddings_per_tile = num_image_token_per_tile( num_embeddings = self._get_num_embeddings(
tile_dims=Dims(width=target_patch_width, height=target_patch_height), target_patch_width * self._patch_size,
patch_size=self.patch_size, target_patch_height * self._patch_size,
downsample_ratio=self.downsample_ratio
) )
token_count = target_patch_width * target_patch_height token_count = target_patch_width * target_patch_height
@ -725,33 +763,30 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
num_tiles = 1 # Base dynamic resolution image num_tiles = 1 # Base dynamic resolution image
if self._use_thumbnail: if self._use_thumbnail:
# Calculate areas # Calculate areas
resized_area = (target_patch_width * self.patch_size) * ( resized_area = (target_patch_width * self._patch_size) * (
target_patch_height * self.patch_size target_patch_height * self._patch_size
) )
thumbnail_area = self._thumbnail_size * self._thumbnail_size thumbnail_area = self._thumbnail_size * self._thumbnail_size
area_ratio = resized_area / thumbnail_area area_ratio = resized_area / thumbnail_area
# Only add thumbnail if resized image area is less than threshold % of # Only add thumbnail if resized image area is less than threshold % of thumbnail area
# thumbnail area
if area_ratio < self._thumbnail_area_threshold: if area_ratio < self._thumbnail_area_threshold:
num_tiles += 1 # Add 1 for thumbnail num_tiles += 1 # Add 1 for thumbnail
# Add embeddings for thumbnail (thumbnail_size x thumbnail_size) # Add embeddings for thumbnail (thumbnail_size x thumbnail_size)
num_embeddings_per_tile += num_image_token_per_tile( num_embeddings += self._get_num_embeddings(
tile_dims=Dims(width=self._thumbnail_size, height=self._thumbnail_size), self._thumbnail_size, self._thumbnail_size
patch_size=self.patch_size,
downsample_ratio=self.downsample_ratio
) )
token_count += ( token_count += (
self._thumbnail_size self._thumbnail_size
// self.patch_size // self._patch_size
* self._thumbnail_size * self._thumbnail_size
// self.patch_size // self._patch_size
) )
return DynamicResolutionParams( return DynamicResolutionParams(
media=media, media=media,
num_tiles=num_tiles, num_tiles=num_tiles,
num_embeddings=num_embeddings_per_tile, num_embeddings=num_embeddings,
patch_size=(target_patch_width, target_patch_height), patch_size=(target_patch_width, target_patch_height),
), token_count ), token_count
@ -805,7 +840,7 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
media_list: list[Image.Image], media_list: list[Image.Image],
num_tokens_available: int | None = None, num_tokens_available: int | None = None,
data_augment: bool = False, data_augment: bool = False,
) -> tuple[list[DynamicResolutionParams], list[int]]: ) -> list[DynamicResolutionParams]:
"""Compute parameters for all media with iterative token budgeting. """Compute parameters for all media with iterative token budgeting.
Args: Args:
@ -821,26 +856,24 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
* (4 if self._pixel_shuffle else 1) * (4 if self._pixel_shuffle else 1)
* (4 if self._conv_merging else 1) * (4 if self._conv_merging else 1)
) )
# When the number of available token is too small, allow self._min_num_patches # When the number of available token is too small, allow self._min_num_patches per media and
# per media and let the sample be truncated. # let the sample be truncated.
num_tokens_available = max( num_tokens_available = max(
num_tokens_available, self._min_num_patches * len(media_list) num_tokens_available, self._min_num_patches * len(media_list)
) )
# Clip the number of tokens available per media to be between min and max # Clip the number of tokens available per media to be between min and max patches.
# patches.
num_tokens_available_per_media = [ num_tokens_available_per_media = [
max(num_tokens_available, self._min_num_patches) max(num_tokens_available, self._min_num_patches)
for _ in range(len(media_list)) for _ in range(len(media_list))
] ]
# In theory this could be a while True loop, but in case the process_media # In theory this could be a while True loop, but in case the process_media method slightly
# method slightly
# changes, I want to make sure we don't get stuck in an infinite loop. # changes, I want to make sure we don't get stuck in an infinite loop.
for _ in range(10): for _ in range(10):
# Step 1: Process each media with current token budget # Step 1: Process each media with current token budget
params: list[DynamicResolutionParams] = [] params = []
token_counts: list[int] = [] token_counts = []
for media, tokens_for_media in zip( for media, tokens_for_media in zip(
media_list, num_tokens_available_per_media media_list, num_tokens_available_per_media
@ -850,18 +883,14 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
) )
params.append(param) params.append(param)
token_counts.append(token_count) token_counts.append(token_count)
self.feature_size_cache[id(param.media)] = param.num_embeddings
# Step 2: Check if total tokens is within budget # Step 2: Check if total tokens is within budget
total_tokens = sum(token_counts) total_tokens = sum(token_counts)
if total_tokens <= num_tokens_available: if total_tokens <= num_tokens_available:
# We're within budget, return the params # We're within budget, return the params
# Convert from patch count to actual token count after downsampling return params
divisor = (4 if self._pixel_shuffle else 1) * (4 if self._conv_merging else 1)
adjusted_token_counts = [tc // divisor for tc in token_counts]
for param, feature_size in zip(params, adjusted_token_counts, strict=True):
self.feature_size_cache[id(param.media)] = feature_size
return params, adjusted_token_counts
# Step 3: We're over budget, need to scale down # Step 3: We're over budget, need to scale down
# Calculate scaling factor to get under budget # Calculate scaling factor to get under budget
@ -880,8 +909,8 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
for i in range(len(num_tokens_available_per_media)) for i in range(len(num_tokens_available_per_media))
] ]
) )
# If there was not scaling down, we're stuck just use min_num_patches per # If there was not scaling down, we're stuck just use min_num_patches per media, else
# media, else try with the scaled down num_tokens_available_per_media. # try with the scaled down num_tokens_available_per_media.
if not scaled_down: if not scaled_down:
num_tokens_available_per_media = [self._min_num_patches] * len( num_tokens_available_per_media = [self._min_num_patches] * len(
media_list media_list
@ -900,15 +929,15 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
) )
def rearrange_img(x): def rearrange_img(x):
py = x.shape[-2] // self.patch_size py = x.shape[-2] // self._patch_size
px = x.shape[-1] // self.patch_size px = x.shape[-1] // self._patch_size
x = einops.rearrange( x = einops.rearrange(
x, x,
"c (py yy) (px xx) -> (py px) (c yy xx)", "c (py yy) (px xx) -> (py px) (c yy xx)",
py=py, py=py,
yy=self.patch_size, yy=self._patch_size,
px=px, px=px,
xx=self.patch_size, xx=self._patch_size,
) )
return x return x
@ -941,34 +970,6 @@ class DynamicResolutionImageTiler(BaseNanoNemotronVLProcessor):
None, None,
) )
def max_num_tokens_available(self, text_prompt_length: int) -> int:
return self.max_model_len - text_prompt_length - 4
def _images_to_pixel_values_lst(
self,
text_prompt_length: int,
images: list[Image.Image],
max_num_tiles: int,
) -> tuple[list[torch.Tensor], list[int]]:
num_tokens_available = self.max_num_tokens_available(text_prompt_length)
params_per_image, feature_sizes = self.compute_params(
images, num_tokens_available
)
print(f"{feature_sizes=}")
print(f"{params_per_image=}")
images = []
for param in params_per_image:
t = self.apply_params(param)
if t.ndim == 3:
t = t.unsqueeze(0)
images.append(t)
return images, feature_sizes
def get_cached_feature_size(self, image: Image.Image) -> int:
feature_size = self.feature_size_cache[id(image)]
del self.feature_size_cache[id(image)]
return feature_size
class NanoNemotronVLProcessor(DynamicResolutionImageTiler): class NanoNemotronVLProcessor(DynamicResolutionImageTiler):
""" """
@ -1339,12 +1340,11 @@ class NanoNemotronVLProcessingInfo(BaseNanoNemotronVLProcessingInfo):
processor = self.get_hf_processor() # we get the CustomProcessor here processor = self.get_hf_processor() # we get the CustomProcessor here
max_image_tokens = self.get_max_image_tokens() * max_images max_image_tokens = self.get_max_image_tokens() * max_images
max_total_frames = ( max_total_frames = (seq_len - max_image_tokens) // num_image_token_per_tile(
seq_len - max_image_tokens width=256,
) // num_image_token_per_tile( height=256,
tile_dims=Dims(width=256, height=256), patch_size=processor._patch_size,
patch_size=processor.patch_size, downsample_ratio=processor.downsample_ratio,
downsample_ratio=processor.downsample_ratio
) # TODO(nhaber): get 256 dynamically ) # TODO(nhaber): get 256 dynamically
max_frames_per_video = max_total_frames // max(max_videos, 1) max_frames_per_video = max_total_frames // max(max_videos, 1)
return max(max_frames_per_video, 1) return max(max_frames_per_video, 1)
@ -1483,9 +1483,10 @@ class NanoNemotronVLMultiModalProcessor(
def get_video_replacement_internvl(item_idx: int): def get_video_replacement_internvl(item_idx: int):
feature_size = num_image_token_per_tile( feature_size = num_image_token_per_tile(
tile_dims=Dims(width=256, height=256), width=256,
patch_size=hf_processor.patch_size, height=256,
downsample_ratio=hf_processor.downsample_ratio patch_size=hf_processor._patch_size,
downsample_ratio=hf_processor.downsample_ratio,
) # TODO(nhaber): get 256 dynamically ) # TODO(nhaber): get 256 dynamically
video, metadata = mm_items["video"][item_idx] video, metadata = mm_items["video"][item_idx]
num_patches = video_num_patches[item_idx] num_patches = video_num_patches[item_idx]
@ -1550,17 +1551,18 @@ class NanoNemotronVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
num_images = mm_counts.get("image", 0) num_images = mm_counts.get("image", 0)
processor = self.info.get_hf_processor() processor = self.info.get_hf_processor()
B = processor.max_num_tokens_available(text_prompt_length=num_images) B = processor.max_num_tokens_available(text_prompt_length=num_images)
target_dims = width_and_height_for_max_num_tokens_available( target_width, target_height = width_and_height_for_max_num_tokens_available(
target_num_tokens_post_shuffle=B, target_num_tokens_post_shuffle=B,
patch_size=processor.patch_size, patch_size=processor._patch_size,
downsample_ratio=processor.downsample_ratio,
) )
image_overrides = mm_options.get("image") if mm_options else None image_overrides = mm_options.get("image") if mm_options else None
return { return {
"image": self._get_dummy_images( "image": self._get_dummy_images(
width=target_dims.width, width=target_width,
height=target_dims.height, height=target_height,
num_images=num_images, num_images=num_images,
overrides=image_overrides, overrides=image_overrides,
) )