From e42389f9d7a5e04aee3463b3e08bafdc86a9457b Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 26 Mar 2025 03:26:16 +0000 Subject: [PATCH 001/593] Transformers backend already supports V1 (#15463) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- tests/models/test_transformers.py | 22 +++++----------------- vllm/engine/arg_utils.py | 8 -------- vllm/model_executor/models/transformers.py | 2 ++ 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/tests/models/test_transformers.py b/tests/models/test_transformers.py index 243cb92ae2569..c45fc7e649ec8 100644 --- a/tests/models/test_transformers.py +++ b/tests/models/test_transformers.py @@ -3,8 +3,6 @@ Run `pytest tests/models/test_transformers.py`. """ -from contextlib import nullcontext - import pytest from ..conftest import HfRunner, VllmRunner @@ -42,7 +40,6 @@ def check_implementation( "model,model_impl", [ ("meta-llama/Llama-3.2-1B-Instruct", "transformers"), - ("openai-community/gpt2", "transformers"), ("ArthurZ/Ilama-3.2-1B", "auto"), # CUSTOM CODE ]) # trust_remote_code=True by default def test_models( @@ -52,20 +49,11 @@ def test_models( model: str, model_impl: str, ) -> None: - - maybe_raises = nullcontext() - if model == "openai-community/gpt2" and model_impl == "transformers": - # Model is not backend compatible - maybe_raises = pytest.raises( - ValueError, - match="The Transformers implementation.*not compatible with vLLM") - - with maybe_raises: - check_implementation(hf_runner, - vllm_runner, - example_prompts, - model, - model_impl=model_impl) + check_implementation(hf_runner, + vllm_runner, + example_prompts, + model, + model_impl=model_impl) @multi_gpu_test(num_gpus=2) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 65a1676c0637d..75ac326aaa3d6 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1613,14 +1613,6 @@ class EngineArgs: recommend_to_remove=False) return False - # No TransformersModel support so far. - if (model_config.model_impl == ModelImpl.TRANSFORMERS - or model_config.model_impl == "transformers"): - _raise_or_fallback( - feature_name=f"model_impl={model_config.model_impl}", - recommend_to_remove=False) - return False - # No Concurrent Partial Prefills so far. if (self.max_num_partial_prefills != EngineArgs.max_num_partial_prefills diff --git a/vllm/model_executor/models/transformers.py b/vllm/model_executor/models/transformers.py index fe6a9d7a4aa43..56ec00dcf222c 100644 --- a/vllm/model_executor/models/transformers.py +++ b/vllm/model_executor/models/transformers.py @@ -24,6 +24,7 @@ from transformers import AutoModel, PretrainedConfig, PreTrainedModel from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS from vllm.attention import Attention +from vllm.compilation.decorators import support_torch_compile from vllm.config import (CacheConfig, DeviceConfig, ModelConfig, ParallelConfig, VllmConfig) from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size @@ -109,6 +110,7 @@ def replace_linear_class( ) +@support_torch_compile class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): embedding_padding_modules = ["lm_head"] embedding_modules = ["embed_tokens" From 997c8811d6aadf92dc299e0c2a8d274117308880 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Wed, 26 Mar 2025 11:26:33 +0800 Subject: [PATCH 002/593] [Model] Support multi-image for Molmo (#15438) Signed-off-by: DarkLight1337 --- docs/source/models/supported_models.md | 2 +- .../vision_language/test_models.py | 2 +- vllm/model_executor/models/molmo.py | 57 +++++++++---------- vllm/model_executor/models/vision.py | 13 +++-- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/docs/source/models/supported_models.md b/docs/source/models/supported_models.md index 56ea8c5d8372b..f106195e10585 100644 --- a/docs/source/models/supported_models.md +++ b/docs/source/models/supported_models.md @@ -853,7 +853,7 @@ See [this page](#generative-models) for more information on how to use generativ * - * `MolmoForCausalLM` * Molmo - * T + I + * T + I+ * `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. * ✅︎ * ✅︎ diff --git a/tests/models/decoder_only/vision_language/test_models.py b/tests/models/decoder_only/vision_language/test_models.py index 94b61b6ae7803..d500ef5d8b805 100644 --- a/tests/models/decoder_only/vision_language/test_models.py +++ b/tests/models/decoder_only/vision_language/test_models.py @@ -431,7 +431,7 @@ VLM_TEST_SETTINGS = { ), "molmo": VLMTestInfo( models=["allenai/Molmo-7B-D-0924"], - test_type=(VLMTestType.IMAGE), + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), prompt_formatter=identity, max_model_len=4096, max_num_seqs=2, diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 3f0c644a5a866..146d48e522119 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -57,7 +57,7 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, merge_multimodal_embeddings) -from .vision import select_patch_features +from .vision import scatter_patch_features, select_patch_features # TODO: hard-coded for now. Consider making it configurable. VIT_LAYERS = [-2, -9] @@ -71,13 +71,13 @@ POOLING_SIZE = 2 class MolmoImageInputs(TypedDict): - images: Union[torch.Tensor, List[torch.Tensor]] + images: Union[torch.Tensor, list[torch.Tensor]] """Shape: `(batch_size, num_crops, num_patch, patch_dim)`""" - image_masks: Optional[Union[torch.Tensor, List[torch.Tensor]]] + image_masks: Optional[Union[torch.Tensor, list[torch.Tensor]]] """Shape: `(batch_size, num_crops, num_patch)`""" - feat_is_patch: Union[torch.Tensor, List[torch.Tensor]] + feat_is_patch: Union[torch.Tensor, list[torch.Tensor]] """ A boolean mask indicating which image features correspond to patch tokens. @@ -85,7 +85,7 @@ class MolmoImageInputs(TypedDict): Shape: `(batch_size, num_crops, num_patch)` """ - embed_is_patch: Union[torch.Tensor, List[torch.Tensor]] + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] """ A boolean mask indicating which image embeddings correspond to patch tokens. @@ -93,7 +93,7 @@ class MolmoImageInputs(TypedDict): Shape: `(batch_size, num_embeds)` """ - num_crops: torch.Tensor + num_crops: Union[torch.Tensor, list[torch.Tensor]] """Shape: `(batch_size, num_images)`""" @@ -1144,13 +1144,7 @@ class MolmoProcessorWrapper: image_input_idx = outputs.pop("image_input_idx", None) if image_input_idx is not None: - input_is_patch = input_ids == self.image_patch_id - image_input_idx_flat: torch.Tensor = image_input_idx.view(-1) - image_valid_flat = image_input_idx_flat >= 0 - feat_is_patch_flat = image_valid_flat.clone() - feat_is_patch_flat[image_valid_flat] = ( - input_is_patch[image_input_idx_flat[image_valid_flat]]) - feat_is_patch = feat_is_patch_flat.view(*image_input_idx.shape) + feat_is_patch = image_input_idx >= 0 input_is_embed = torch.isin( input_ids, @@ -1165,6 +1159,17 @@ class MolmoProcessorWrapper: embed_is_patch = embed_ids == self.image_patch_id assert embed_is_patch.sum() == feat_is_patch.sum() + # image_tokens = extra_joint + joint + # Both `extra_joint` and `joint` have `im_start_id` and `im_end_id` + embed_start = torch.nonzero(embed_ids == self.im_start_id)[::2, 0] + embed_end = torch.nonzero(embed_ids == self.im_end_id)[1::2, 0] + assert len(embed_start) == len(embed_end) == len(images) + + embed_is_patch = [ + embed_is_patch[start:end + 1] + for start, end in zip(embed_start, embed_end) + ] + tilings = [ self.select_tiling( image_width=image.size[0], @@ -1180,7 +1185,7 @@ class MolmoProcessorWrapper: outputs["num_crops"] = num_crops outputs["img_patch_id"] = self.image_patch_id - return BatchFeature(outputs, tensor_type=return_tensors) + return BatchFeature(outputs) class MolmoProcessingInfo(BaseProcessingInfo): @@ -1190,9 +1195,7 @@ class MolmoProcessingInfo(BaseProcessingInfo): return MolmoProcessorWrapper(processor) def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]: - # TODO: Investigate different `embed_is_patch` between cache/no-cache - # in multi-image case - return {"image": 1} + return {"image": None} def get_mm_max_tokens_per_item( self, @@ -1325,7 +1328,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): "image", num_crops), feat_is_patch=MultiModalFieldConfig.flat_from_sizes( "image", num_crops), - embed_is_patch=MultiModalFieldConfig.shared("image", num_images), + embed_is_patch=MultiModalFieldConfig.batched("image"), num_crops=MultiModalFieldConfig.batched("image"), img_patch_id=MultiModalFieldConfig.shared("image", num_images), ) @@ -1499,7 +1502,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, def _process_image_input( self, image_input: MolmoImageInputs, - ) -> Union[torch.Tensor, List[torch.Tensor]]: + ) -> Union[torch.Tensor, list[torch.Tensor]]: if isinstance(image_input["images"], list): # Call the vision backbone on the whole batch at once images_flat = flatten_bn(image_input["images"], concat=True) @@ -1530,7 +1533,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, feat_is_patch: torch.Tensor, # Shape: (num_crop, num_patch) num_crops: torch.Tensor, # Shape: (num_images,) embed_is_patch: torch.Tensor, # Shape: (num_embeds,) - ) -> list[torch.Tensor]: + ) -> tuple[torch.Tensor, ...]: """ Scatter the patch features into a contiguous tensor that corresponds to the embedding tokens defined by the multimodal processor. @@ -1565,16 +1568,12 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, feats_per_image = features.split(num_crops_per_image) f_is_patch_per_image = feat_is_patch.split(num_crops_per_image) - _, _, embed_dim = features.shape - (num_embeds, ) = embed_is_patch.shape + features = torch.cat([ + feats[f_is_patch] + for feats, f_is_patch in zip(feats_per_image, f_is_patch_per_image) + ]) - embeds_in_batch = list[torch.Tensor]() - for feats, f_is_patch in zip(feats_per_image, f_is_patch_per_image): - embeds = feats.new_full((num_embeds, embed_dim), torch.nan) - embeds[embed_is_patch] = feats[f_is_patch] - embeds_in_batch.append(embeds) - - return embeds_in_batch + return scatter_patch_features(features, embed_is_patch) def get_multimodal_embeddings( self, **kwargs: object) -> Optional[MultiModalEmbeddings]: diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py index 250b0ee3c2a1b..c91459398308e 100644 --- a/vllm/model_executor/models/vision.py +++ b/vllm/model_executor/models/vision.py @@ -155,7 +155,7 @@ def resolve_visual_encoder_outputs( def scatter_patch_features( features: torch.Tensor, - embed_is_patch: torch.Tensor, + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]], ) -> tuple[torch.Tensor, ...]: """ Scatter the patch features into a contiguous tensor that corresponds @@ -194,14 +194,19 @@ def scatter_patch_features( The resulting embedding tensor is: [ nan p1 p2 nan p3 p4 nan nan ] """ - num_images, num_embeds = embed_is_patch.shape - num_embeds_per_image = [num_embeds] * num_images + num_embeds_per_image = [ + e_is_patch.numel() for e_is_patch in embed_is_patch + ] + if isinstance(embed_is_patch, torch.Tensor): + embed_is_patch_flat = embed_is_patch.view(-1) + else: + embed_is_patch_flat = torch.cat(embed_is_patch) embeds_flat = features.new_full( (sum(num_embeds_per_image), features.shape[-1]), fill_value=torch.nan, ) - embeds_flat[embed_is_patch.view(-1)] = features.flatten(0, -2) + embeds_flat[embed_is_patch_flat] = features.flatten(0, -2) return embeds_flat.split(num_embeds_per_image) From 23114d33640175229a395b9ed1128c3a41ad65d9 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Tue, 25 Mar 2025 23:31:04 -0400 Subject: [PATCH 003/593] [Misc] Warn about v0 in benchmark_paged_attn.py (#15495) Signed-off-by: Tyler Michael Smith --- benchmarks/kernels/benchmark_paged_attention.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/kernels/benchmark_paged_attention.py b/benchmarks/kernels/benchmark_paged_attention.py index 48b351bc48141..2625239b08ef2 100644 --- a/benchmarks/kernels/benchmark_paged_attention.py +++ b/benchmarks/kernels/benchmark_paged_attention.py @@ -7,10 +7,13 @@ from typing import Optional import torch from vllm import _custom_ops as ops +from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser, create_kv_caches_with_random) +logger = init_logger(__name__) + NUM_BLOCKS = 128 * 1024 PARTITION_SIZE = 512 PARTITION_SIZE_ROCM = 256 @@ -193,6 +196,9 @@ def main( if __name__ == '__main__': + logger.warning("This script benchmarks the paged attention kernel. " + "By default this is no longer used in vLLM inference.") + parser = FlexibleArgumentParser( description="Benchmark the paged attention kernel.") parser.add_argument("--version", From 33437bc6e7af316fa9ce6b6e559501ca45d9cd45 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 25 Mar 2025 23:33:22 -0400 Subject: [PATCH 004/593] [BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results) (#15492) Signed-off-by: LucasWilkinson --- vllm/attention/ops/triton_merge_attn_states.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vllm/attention/ops/triton_merge_attn_states.py b/vllm/attention/ops/triton_merge_attn_states.py index 31545b607fecd..9671b933f47b9 100644 --- a/vllm/attention/ops/triton_merge_attn_states.py +++ b/vllm/attention/ops/triton_merge_attn_states.py @@ -54,6 +54,15 @@ def merge_attn_states_kernel( p_lse = tl.load(prefix_lse + head_idx * num_tokens + token_idx) s_lse = tl.load(suffix_lse + head_idx * num_tokens + token_idx) + + # FA2 and FA3 have different behavior for when the sum-exp is 0, this namely + # arises with 0 len seqlens. FA3 returns -inf here while FA2 returns inf. + # If we see an inf assume FA2 and convert inf to -inf for consistency + # and correctness. Inf generally doesn't make sense in this context outside + # of undefined-behavior/FA2-case, so I think this a safe assumption. + p_lse = float('-inf') if p_lse == float('inf') else p_lse + s_lse = float('-inf') if s_lse == float('inf') else s_lse + max_lse = tl.maximum(p_lse, s_lse) p_lse = p_lse - max_lse s_lse = s_lse - max_lse From 6c663dfd5e5b5ab4a1eb46391c2c65d1eff0218f Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Tue, 25 Mar 2025 20:33:45 -0700 Subject: [PATCH 005/593] [misc] LoRA - Skip LoRA kernels when not required (#15152) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath --- vllm/lora/ops/triton_ops/lora_expand.py | 13 +++- .../ops/triton_ops/lora_kernel_metadata.py | 42 ++++++++-- vllm/lora/ops/triton_ops/lora_shrink.py | 13 +++- vllm/worker/model_runner.py | 78 +++++++++++++------ 4 files changed, 113 insertions(+), 33 deletions(-) diff --git a/vllm/lora/ops/triton_ops/lora_expand.py b/vllm/lora/ops/triton_ops/lora_expand.py index b47e491ad7ed1..eacc6fb46ebd7 100644 --- a/vllm/lora/ops/triton_ops/lora_expand.py +++ b/vllm/lora/ops/triton_ops/lora_expand.py @@ -136,6 +136,7 @@ def _lora_expand( num_tokens_per_lora: torch.Tensor, # shape [max-loras + 1] lora_token_start_loc: torch.Tensor, # shape [max-loras + 2] lora_ids: torch.Tensor, # shape [max-loras + 1] + no_lora_flag_cpu: torch.Tensor, # shape [1] offset_start: int = 0, add_inputs: bool = False, ) -> None: @@ -157,11 +158,19 @@ def _lora_expand( identifies the the region in token_indices_sorted_by_lora_ids that LoRA lora_ids[i] should process. lora_ids (torch.Tensor): LoRA ids to process. + no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates + if there are any requests that require LoRA. offset_start (int, optional): Offset start for output_tensor. Defaults to 0. add_inputs (bool, optional): Whether to add the input tensor to the output tensor. Defaults to False. """ + + assert no_lora_flag_cpu.numel() == 1 + if no_lora_flag_cpu.item(): + # None of the inputs require LoRA. + return + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32] for weight in lora_b_weights: assert weight.dtype in [torch.float16, torch.bfloat16] @@ -170,6 +179,8 @@ def _lora_expand( assert output_tensor.is_contiguous() # metadata sanity check. + M = inputs.size(1) + assert token_lora_mapping.size(0) == M assert token_lora_mapping.size(0) == token_indices_sorted_by_lora_ids.size( 0) assert lora_ids.size(0) == num_tokens_per_lora.size(0) @@ -181,7 +192,6 @@ def _lora_expand( inputs.device) K = lora_b_weights[0].shape[-1] # K= rank - M = inputs.size(1) ADD_INPUTS = add_inputs MAX_LORAS = lora_ids.size(0) CAST_TYPE = False @@ -263,6 +273,7 @@ def _lora_expand_fake( num_tokens_per_lora: torch.Tensor, lora_token_start_loc: torch.Tensor, lora_ids: torch.Tensor, + no_lora_flag_cpu: torch.Tensor, offset_start: int = 0, add_inputs: bool = False, ) -> None: diff --git a/vllm/lora/ops/triton_ops/lora_kernel_metadata.py b/vllm/lora/ops/triton_ops/lora_kernel_metadata.py index 2add1177e84c8..1dcdfc814a891 100644 --- a/vllm/lora/ops/triton_ops/lora_kernel_metadata.py +++ b/vllm/lora/ops/triton_ops/lora_kernel_metadata.py @@ -17,6 +17,17 @@ class LoRAKernelMeta: num_tokens_per_lora: torch.Tensor lora_token_start_loc: torch.Tensor + # The V1 architecture uses the traced torch.compile graphs to execute + # a forward pass. Things to note about this process, + # 1. The tracing infers all python scalar datatype objects into a constant + # value. + # 2. The tracing cannot handle dynamic control flow. (dynamic control flow + # is an experimental feature in pytorch) + # 3. The internals of torch.ops functions are not traced. + # We disguise the "no_lora" flag as a cpu tensor and leverage point number 3 + # to early exit from inside the lora_expand / lora_shrink torch operation. + no_lora_flag_cpu: torch.Tensor + @staticmethod def make(max_loras: int, max_num_tokens: int, device: Union[torch.device, str]) -> "LoRAKernelMeta": @@ -47,17 +58,24 @@ class LoRAKernelMeta: lora_token_start_loc = torch.zeros(max_loras + 2, dtype=torch.int32, device=device) + + no_lora_flag_cpu = torch.tensor([False], + dtype=torch.bool, + device='cpu') + return LoRAKernelMeta( token_lora_mapping=token_lora_mapping, token_indices_sorted_by_lora_ids=token_indices_sorted_by_lora_ids, active_lora_ids=active_lora_ids, num_tokens_per_lora=num_tokens_per_lora, - lora_token_start_loc=lora_token_start_loc) + lora_token_start_loc=lora_token_start_loc, + no_lora_flag_cpu=no_lora_flag_cpu) def _reset(self): self.active_lora_ids.fill_(-1) self.num_tokens_per_lora.fill_(0) self.lora_token_start_loc.fill_(0) + self.no_lora_flag_cpu.fill_(False) def prepare_tensors(self, token_lora_mapping: torch.Tensor) -> None: """ @@ -70,6 +88,14 @@ class LoRAKernelMeta: self._reset() + # Check and record no-lora case. + no_lora = torch.all(token_lora_mapping == -1) + self.no_lora_flag_cpu[0] = no_lora + + if no_lora: + # Early exit. LoRA kernels will not be run. + return + num_tokens = token_lora_mapping.size(0) # copy token lora mapping @@ -100,7 +126,7 @@ class LoRAKernelMeta: def meta_args( self, token_nums: int ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, - torch.Tensor]: + torch.Tensor, torch.Tensor]: """ This function returns the kernel metadata required for the current forward pass execution of the kernel. The function returns all the @@ -111,7 +137,11 @@ class LoRAKernelMeta: token_nums (int): Number of input tokens in the current forward pass. """ - return (self.token_lora_mapping[:token_nums], - self.token_indices_sorted_by_lora_ids[:token_nums], - self.num_tokens_per_lora, self.lora_token_start_loc, - self.active_lora_ids) + return ( + self.token_lora_mapping[:token_nums], + self.token_indices_sorted_by_lora_ids[:token_nums], + self.num_tokens_per_lora, + self.lora_token_start_loc, + self.active_lora_ids, + self.no_lora_flag_cpu, + ) diff --git a/vllm/lora/ops/triton_ops/lora_shrink.py b/vllm/lora/ops/triton_ops/lora_shrink.py index a97c50c44f47a..82331939d859b 100644 --- a/vllm/lora/ops/triton_ops/lora_shrink.py +++ b/vllm/lora/ops/triton_ops/lora_shrink.py @@ -106,6 +106,7 @@ def _lora_shrink( num_tokens_per_lora: torch.Tensor, # shape [max-loras + 1] lora_token_start_loc: torch.Tensor, # shape [max-loras + 2] lora_ids: torch.Tensor, # shape [max-loras + 1] + no_lora_flag_cpu: torch.Tensor, # shape [1] scaling: float, ) -> None: """ @@ -126,8 +127,16 @@ def _lora_shrink( identifies the region in token_indices_sorted_by_lora_ids that LoRA lora_ids[i] should process. lora_ids (torch.Tensor): LoRA ids to process. + no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates + if there are any requests that require LoRA. scaling (float): Scaling factor. """ + + assert no_lora_flag_cpu.numel() == 1 + if no_lora_flag_cpu.item(): + # None of the inputs require LoRA. + return + assert inputs.dtype == lora_a_weights[0].dtype assert inputs.dtype in [torch.float16, torch.bfloat16] for weight in lora_a_weights: @@ -138,6 +147,8 @@ def _lora_shrink( assert output_tensor.is_contiguous() # metadata sanity check + M = inputs.size(0) + assert token_lora_mapping.size(0) == M assert token_lora_mapping.size(0) == token_indices_sorted_by_lora_ids.size( 0) assert lora_ids.size(0) == num_tokens_per_lora.size(0) @@ -146,7 +157,6 @@ def _lora_shrink( (lora_ptr_tensor, lora_strides_d0, lora_strides_d1, lora_strides_d2) = _get_lora_a_ptr(lora_a_weights, inputs.device) N, K = lora_a_weights[0].shape[-2:] # K=hidden_size,N=rank - M = inputs.size(0) NUM_SLICES = len(lora_a_weights) MAX_LORAS = lora_ids.size(0) @@ -218,6 +228,7 @@ def _lora_shrink_fake( num_tokens_per_lora: torch.Tensor, lora_token_start_loc: torch.Tensor, lora_ids: torch.Tensor, + no_lora_flag_cpu: torch.Tensor, scaling: float, ) -> None: return diff --git a/vllm/worker/model_runner.py b/vllm/worker/model_runner.py index 473bd901b5b23..edbafb48c9386 100644 --- a/vllm/worker/model_runner.py +++ b/vllm/worker/model_runner.py @@ -1242,6 +1242,29 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]): max_num_seqs = self.scheduler_config.max_num_seqs self._dummy_run(max_num_batched_tokens, max_num_seqs) + def _add_dummy_loras(self, num_loras: int) -> list[LoRARequest]: + assert num_loras > 0 + assert self.lora_manager is not None + + dummy_lora_requests: list[LoRARequest] = [] + with self.lora_manager.dummy_lora_cache(): + for idx in range(num_loras): + lora_id = idx + 1 + dummy_lora_request = LoRARequest( + lora_name=f"warmup_{lora_id}", + lora_int_id=lora_id, + lora_path="/not/a/real/path", + ) + self.lora_manager.add_dummy_lora(dummy_lora_request, + rank=LORA_WARMUP_RANK) + dummy_lora_requests.append(dummy_lora_request) + return dummy_lora_requests + + def _remove_dummy_loras(self): + # Remove dummy loras. + assert self.lora_manager is not None + self.remove_all_loras() + def _dummy_run(self, max_num_batched_tokens: int, max_num_seqs: int = 1) -> None: @@ -1251,28 +1274,20 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]): SamplingParams(top_p=0.99, top_k=self.vocab_size - 1) # This represents the maximum number of different requests - # that will have unique loras, an therefore the max amount of memory - # consumption create dummy lora request copies from the lora request - # passed in, which contains a lora from the lora warmup path. + # that will have unique loras, and therefore the max amount of + # memory consumption. Create dummy lora request copies from the + # lora request passed in, which contains a lora from the lora + # warmup path. dummy_lora_requests: List[LoRARequest] = [] dummy_lora_requests_per_seq: List[LoRARequest] = [] if self.lora_config: - assert self.lora_manager is not None - with self.lora_manager.dummy_lora_cache(): - for idx in range(self.lora_config.max_loras): - lora_id = idx + 1 - dummy_lora_request = LoRARequest( - lora_name=f"warmup_{lora_id}", - lora_int_id=lora_id, - lora_path="/not/a/real/path", - ) - self.lora_manager.add_dummy_lora(dummy_lora_request, - rank=LORA_WARMUP_RANK) - dummy_lora_requests.append(dummy_lora_request) - dummy_lora_requests_per_seq = [ - dummy_lora_requests[idx % len(dummy_lora_requests)] - for idx in range(max_num_seqs) - ] + dummy_lora_requests = self._add_dummy_loras( + self.lora_config.max_loras) + assert len(dummy_lora_requests) == self.lora_config.max_loras + dummy_lora_requests_per_seq = [ + dummy_lora_requests[idx % len(dummy_lora_requests)] + for idx in range(max_num_seqs) + ] # Profile memory usage with max_num_sequences sequences and the # total number of tokens equal to max_num_batched_tokens. @@ -1354,9 +1369,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]): self.execute_model(model_input, kv_caches, intermediate_tensors) torch.cuda.synchronize() if self.lora_config: - # Remove dummy loras. - assert self.lora_manager is not None - self.remove_all_loras() + self._remove_dummy_loras() + return def remove_all_loras(self): @@ -1479,6 +1493,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]): dtype=self.model_config.dtype, device=self.device) + dummy_lora_id: Optional[int] = None + dummy_lora_request: LoRARequest = [] + if self.lora_config: + # The goal is to capture the LoRA kernels in cuda graphs. + # for this purpose, as single dummy lora is sufficient. + dummy_lora_requests = self._add_dummy_loras(num_loras=1) + assert len(dummy_lora_requests) == 1 + dummy_lora_request = dummy_lora_requests[0] + dummy_lora_id = dummy_lora_request.lora_int_id + with self.attn_state.graph_capture(max_batch_size), graph_capture( self.device) as graph_capture_context: # NOTE: Capturing the largest batch size first may help reduce the @@ -1503,10 +1527,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]): attn_metadata.enable_kv_scales_calculation = False if self.lora_config: lora_mapping = LoRAMapping( - **dict(index_mapping=[0] * batch_size, - prompt_mapping=[0] * batch_size, + **dict(index_mapping=[dummy_lora_id] * batch_size, + prompt_mapping=[dummy_lora_id] * batch_size, is_prefill=False)) - self.set_active_loras(set(), lora_mapping) + self.set_active_loras(set([dummy_lora_request]), + lora_mapping) if self.prompt_adapter_config: prompt_adapter_mapping = PromptAdapterMapping( @@ -1562,6 +1587,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]): self.graph_runners[virtual_engine][batch_size] = ( graph_runner) + if self.lora_config: + self._remove_dummy_loras() + end_time = time.perf_counter() end_free_gpu_memory = torch.cuda.mem_get_info()[0] elapsed_time = end_time - start_time From 5aefd6ac3169b7b56023549cfa9614274d6e15f0 Mon Sep 17 00:00:00 2001 From: daniel-salib Date: Tue, 25 Mar 2025 22:29:54 -0700 Subject: [PATCH 006/593] Fix raw_request extraction in load_aware_call decorator (#15382) Signed-off-by: Daniel Salib --- vllm/entrypoints/utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/vllm/entrypoints/utils.py b/vllm/entrypoints/utils.py index 60cbb58af3d9a..773f52fa38f88 100644 --- a/vllm/entrypoints/utils.py +++ b/vllm/entrypoints/utils.py @@ -68,13 +68,20 @@ def decrement_server_load(request: Request): def load_aware_call(func): @functools.wraps(func) - async def wrapper(*args, raw_request: Request, **kwargs): + async def wrapper(*args, **kwargs): + raw_request = kwargs.get("raw_request", + args[1] if len(args) > 1 else None) + + if raw_request is None: + raise ValueError( + "raw_request required when server load tracking is enabled") + if not raw_request.app.state.enable_server_load_tracking: - return await func(*args, raw_request=raw_request, **kwargs) + return await func(*args, **kwargs) raw_request.app.state.server_load_metrics += 1 try: - response = await func(*args, raw_request=raw_request, **kwargs) + response = await func(*args, **kwargs) except Exception: raw_request.app.state.server_load_metrics -= 1 raise From 781d0562809b34f0c548cd354bbc01c861814f94 Mon Sep 17 00:00:00 2001 From: Bryan Lu <55512809+luyuzhe111@users.noreply.github.com> Date: Wed, 26 Mar 2025 01:24:07 -0700 Subject: [PATCH 007/593] [Feature] Enhance EAGLE Architecture with Proper RMS Norms (#14990) Signed-off-by: Bryan Lu Co-authored-by: Cyrus Leung --- vllm/config.py | 16 ++++++-- vllm/model_executor/models/eagle.py | 51 +++++++++++++++++++++--- vllm/transformers_utils/configs/eagle.py | 15 ++++++- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/vllm/config.py b/vllm/config.py index 87ede1e077b8a..6f2da6aa87136 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -800,10 +800,18 @@ class ModelConfig: @property def is_deepseek_mla(self) -> bool: - return (hasattr(self.hf_text_config, "model_type")) \ - and (self.hf_text_config.model_type in \ - ('deepseek_v2', 'deepseek_v3', 'deepseek_mtp'))\ - and (self.hf_text_config.kv_lora_rank is not None) + if not hasattr(self.hf_text_config, "model_type"): + return False + elif self.hf_text_config.model_type in \ + ('deepseek_v2', 'deepseek_v3', 'deepseek_mtp'): + return self.hf_text_config.kv_lora_rank is not None + elif self.hf_text_config.model_type == 'eagle': + # if the model is an EAGLE module, check for the + # underlying architecture + return self.hf_text_config.model.model_type in \ + ('deepseek_v2', 'deepseek_v3') \ + and self.hf_text_config.kv_lora_rank is not None + return False def get_head_size(self) -> int: # TODO remove hard code diff --git a/vllm/model_executor/models/eagle.py b/vllm/model_executor/models/eagle.py index 010e51a3b9f28..3e4a5040b7c89 100644 --- a/vllm/model_executor/models/eagle.py +++ b/vllm/model_executor/models/eagle.py @@ -7,6 +7,7 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.sampler import SamplerOutput from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -59,7 +60,15 @@ class EAGLE(nn.Module): truncated_vocab_size < vocab_size. To use this technique, one has to find the top-k most frequent tokens in target dataset and add that as a tensor in the draft checkpoint (using key token_map). Also, the draft config - needs to have truncated_vocab_size (=k) as an attribute.""" + needs to have truncated_vocab_size (=k) as an attribute. + 4. We allow an enhanced EAGLE architecture similar to the DeepSeek MTP + module with regards to the use of additional RMS norms. The original + EAGLE architecture 1) skips the pre-attention norm in its first + transformer block, and 2) skips the final output norm, both of which we + found to be suboptimal. We also add the support for separate norms + applying to both the token embedding and hidden states before projection + as in DeepSeek MTP, which we found to improve performance as well. + """ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -81,9 +90,22 @@ class EAGLE(nn.Module): # While weights and biases are generally not needed, # they are retained here to support certain unit tests # (e.g., spec_decode/e2e/test_eagle_correctness.py). - self.model.model.layers[0].input_layernorm = DummyInputLayerNorm( - weight=self.model.model.layers[0].input_layernorm.weight) - self.model.model.norm = DummyOutputNorm() + if not hasattr(self.config.model, + "skip_prenorm") or self.config.model.skip_prenorm: + self.model.model.layers[0].input_layernorm = DummyInputLayerNorm( + weight=self.model.model.layers[0].input_layernorm.weight) + + if not hasattr( + self.config.model, + "skip_output_norm") or self.config.model.skip_output_norm: + self.model.model.norm = DummyOutputNorm() + + self.add_para_norm = False + if hasattr(self.config.model, + "add_para_norm") and self.config.model.add_para_norm: + self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.add_para_norm = True self.orig_vocab_size = config.vocab_size self.truncated_vocab_size = config.truncated_vocab_size @@ -128,8 +150,17 @@ class EAGLE(nn.Module): if inputs_embeds is None: inputs_embeds = self.get_input_embeddings(input_ids) - inputs_embeds = self.fc( - torch.cat([inputs_embeds, previous_hidden_states], dim=-1)) + if self.add_para_norm: + inputs_embeds = torch.cat([ + self.enorm(inputs_embeds), + self.hnorm(previous_hidden_states) + ], + dim=-1) + else: + inputs_embeds = torch.cat([inputs_embeds, previous_hidden_states], + dim=-1) + + inputs_embeds = self.fc(inputs_embeds) inputs_embeds[positions == 0] = 0 # masking inputs at position=0 @@ -190,6 +221,14 @@ class EAGLE(nn.Module): else: logger.warning_once("Found bias in the loaded weights but " "the model config doesn't have bias.") + elif name.startswith("enorm.weight"): + weight_loader = getattr(self.enorm.weight, "weight_loader", + default_weight_loader) + weight_loader(self.enorm.weight, loaded_weight) + elif name.startswith("hnorm.weight"): + weight_loader = getattr(self.hnorm.weight, "weight_loader", + default_weight_loader) + weight_loader(self.hnorm.weight, loaded_weight) elif name.startswith("model.lm_head.") or name.startswith( "model.model."): model_weights[name.split("model.", 1)[-1]] = loaded_weight diff --git a/vllm/transformers_utils/configs/eagle.py b/vllm/transformers_utils/configs/eagle.py index b26aba66699fd..dd806061ff589 100644 --- a/vllm/transformers_utils/configs/eagle.py +++ b/vllm/transformers_utils/configs/eagle.py @@ -5,6 +5,8 @@ from typing import Optional, Union from transformers import AutoConfig, PretrainedConfig +from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekV2Config + class EAGLEConfig(PretrainedConfig): model_type = "eagle" @@ -14,8 +16,17 @@ class EAGLEConfig(PretrainedConfig): truncated_vocab_size: Optional[int] = None, **kwargs): - model_config = None if model is None else (AutoConfig.for_model( - **model) if isinstance(model, dict) else model) + model_config: Union[PretrainedConfig, DeepseekV2Config, None] + if isinstance(model, dict): + archs = model.get("architectures", []) + target_archs = ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"] + if any(target_arch in archs for target_arch in target_archs): + # AutoConfig does not support DeepSeek MoE models yet + model_config = DeepseekV2Config(**model) + else: + model_config = AutoConfig.for_model(**model) + else: + model_config = model for k, v in kwargs.items(): if k != "architectures" and k != "model_type" and hasattr( From 5ebf66748b8b67731972c389d879ca69c68dc2c4 Mon Sep 17 00:00:00 2001 From: vllmellm Date: Wed, 26 Mar 2025 16:30:30 +0800 Subject: [PATCH 008/593] [FEAT][ROCm] Integrate Fused MoE Kernels from AITER (#14967) Signed-off-by: vllmellm Signed-off-by: tjtanaa Co-authored-by: tjtanaa --- tests/kernels/test_moe.py | 25 ++- .../model_executor/test_enabled_custom_ops.py | 36 ++++ .../decoder_only/language/test_mistral.py | 41 +---- tests/quantization/test_fp8.py | 23 ++- vllm/envs.py | 15 ++ .../layers/fused_moe/fused_moe.py | 94 ++++++++--- vllm/model_executor/layers/fused_moe/layer.py | 14 +- .../layers/fused_moe/rocm_aiter_fused_moe.py | 157 ++++++++++++++++++ .../model_executor/layers/quantization/fp8.py | 52 ++++++ 9 files changed, 391 insertions(+), 66 deletions(-) create mode 100644 vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py diff --git a/tests/kernels/test_moe.py b/tests/kernels/test_moe.py index 653d2734afe89..3f4dd3cf0e5d7 100644 --- a/tests/kernels/test_moe.py +++ b/tests/kernels/test_moe.py @@ -3,7 +3,6 @@ Run `pytest tests/kernels/test_moe.py`. """ - import pytest import torch from torch.nn import Parameter @@ -216,11 +215,17 @@ def test_fused_moe_wn16(m: int, n: int, k: int, e: int, topk: int, @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) @pytest.mark.parametrize("padding", [True, False]) +@pytest.mark.parametrize( + "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]) @torch.inference_mode() -def test_mixtral_moe(dtype: torch.dtype, padding: bool): +def test_mixtral_moe(dtype: torch.dtype, padding: bool, use_rocm_aiter: bool, + monkeypatch): """Make sure our Mixtral MoE implementation agrees with the one from huggingface.""" + if use_rocm_aiter: + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + # Instantiate our and huggingface's MoE blocks config = MixtralConfig() hf_moe = MixtralSparseMoeBlock(config).to(dtype).to("cuda") @@ -268,10 +273,18 @@ def test_mixtral_moe(dtype: torch.dtype, padding: bool): torch.bfloat16: 1e-2, } - torch.testing.assert_close(hf_states.flatten(0, 1), - vllm_states, - rtol=mixtral_moe_tol[dtype], - atol=mixtral_moe_tol[dtype]) + if use_rocm_aiter: + # The values of rtol and atol are set based on the tests in ROCM AITER package. # noqa: E501 + # https://github.com/ROCm/aiter/blob/dfed377f4be7da96ca2d75ac0761f569676f7240/op_tests/test_moe.py#L174 # noqa: E501 + torch.testing.assert_close(hf_states.flatten(0, 1), + vllm_states, + rtol=0.01, + atol=100) + else: + torch.testing.assert_close(hf_states.flatten(0, 1), + vllm_states, + rtol=mixtral_moe_tol[dtype], + atol=mixtral_moe_tol[dtype]) @pytest.mark.parametrize("m", [1, 33, 64, 222]) diff --git a/tests/model_executor/test_enabled_custom_ops.py b/tests/model_executor/test_enabled_custom_ops.py index 24147b741278b..ac2e0f3542e78 100644 --- a/tests/model_executor/test_enabled_custom_ops.py +++ b/tests/model_executor/test_enabled_custom_ops.py @@ -7,6 +7,10 @@ from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.activation import (GeluAndMul, ReLUSquaredActivation, SiluAndMul) +from vllm.model_executor.layers.fused_moe.fused_moe import ( + dispatch_fused_experts_func, dispatch_topk_func, + torch_vllm_inplace_fused_experts, torch_vllm_outplace_fused_experts, + vllm_topk_softmax) from vllm.model_executor.layers.layernorm import ( RMSNorm, dispatch_cuda_rmsnorm_func, fused_add_rms_norm, rms_norm, rocm_aiter_fused_add_rms_norm, rocm_aiter_rms_norm) @@ -92,6 +96,38 @@ def test_enabled_ops_invalid(env: str): RMSNorm(1024).enabled() +@pytest.mark.parametrize("use_rocm_aiter", ["0", "1"]) +def test_topk_dispatch(use_rocm_aiter: str, monkeypatch): + monkeypatch.setenv("VLLM_ROCM_USE_AITER", use_rocm_aiter) + topk_func = dispatch_topk_func() + + if current_platform.is_rocm() and int(use_rocm_aiter): + from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + rocm_aiter_topk_softmax) + + assert topk_func == rocm_aiter_topk_softmax + else: + assert topk_func == vllm_topk_softmax + + +@pytest.mark.parametrize("use_rocm_aiter", ["0", "1"]) +@pytest.mark.parametrize("inplace", [True, False]) +def test_fused_experts_dispatch(use_rocm_aiter: str, inplace: bool, + monkeypatch): + + monkeypatch.setenv("VLLM_ROCM_USE_AITER", use_rocm_aiter) + fused_experts_func = dispatch_fused_experts_func(inplace) + if current_platform.is_rocm() and int(use_rocm_aiter): + from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + rocm_aiter_fused_experts) + + assert fused_experts_func == rocm_aiter_fused_experts + elif inplace: + assert fused_experts_func == torch_vllm_inplace_fused_experts + else: + assert fused_experts_func == torch_vllm_outplace_fused_experts + + @pytest.mark.parametrize("add_residual", [True, False]) @pytest.mark.parametrize("use_rocm_aiter", ["0", "1"]) @pytest.mark.parametrize("use_rocm_aiter_norm", ["0", "1"]) diff --git a/tests/models/decoder_only/language/test_mistral.py b/tests/models/decoder_only/language/test_mistral.py index 4c2055361d445..ec885386dd940 100644 --- a/tests/models/decoder_only/language/test_mistral.py +++ b/tests/models/decoder_only/language/test_mistral.py @@ -174,15 +174,8 @@ SAMPLE_JSON_SCHEMA = { @pytest.mark.parametrize("dtype", ["bfloat16"]) @pytest.mark.parametrize("max_tokens", [64]) @pytest.mark.parametrize("num_logprobs", [5]) -def test_models( - hf_runner, - vllm_runner, - example_prompts, - model: str, - dtype: str, - max_tokens: int, - num_logprobs: int, -) -> None: +def test_models(hf_runner, vllm_runner, example_prompts, model: str, + dtype: str, max_tokens: int, num_logprobs: int) -> None: # TODO(sang): Sliding window should be tested separately. with hf_runner(model, dtype=dtype) as hf_model: hf_outputs = hf_model.generate_greedy_logprobs_limit( @@ -206,14 +199,8 @@ def test_models( @pytest.mark.parametrize("dtype", ["bfloat16"]) @pytest.mark.parametrize("max_tokens", [64]) @pytest.mark.parametrize("num_logprobs", [5]) -def test_mistral_format( - vllm_runner, - example_prompts, - model: str, - dtype: str, - max_tokens: int, - num_logprobs: int, -) -> None: +def test_mistral_format(vllm_runner, example_prompts, model: str, dtype: str, + max_tokens: int, num_logprobs: int) -> None: with vllm_runner( model, dtype=dtype, @@ -244,11 +231,8 @@ def test_mistral_format( @pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS) @pytest.mark.parametrize("dtype", ["bfloat16"]) -def test_mistral_symbolic_languages( - vllm_runner, - model: str, - dtype: str, -) -> None: +def test_mistral_symbolic_languages(vllm_runner, model: str, + dtype: str) -> None: with vllm_runner(model, dtype=dtype, max_model_len=8192, @@ -266,11 +250,7 @@ def test_mistral_symbolic_languages( @pytest.mark.parametrize("dtype", ["bfloat16"]) @pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS) # v1 can't do func calling -def test_mistral_function_calling( - vllm_runner, - model: str, - dtype: str, -) -> None: +def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None: with vllm_runner(model, dtype=dtype, tokenizer_mode="mistral", @@ -301,11 +281,8 @@ def test_mistral_function_calling( @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("guided_backend", ["outlines", "lm-format-enforcer", "xgrammar"]) -def test_mistral_guided_decoding( - vllm_runner, - model: str, - guided_backend: str, -) -> None: +def test_mistral_guided_decoding(vllm_runner, model: str, + guided_backend: str) -> None: with vllm_runner(model, dtype='bfloat16', tokenizer_mode="mistral") as vllm_model: diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 19cf29d3e6591..e74e14a0dcb64 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -23,8 +23,14 @@ MODELS = [ reason="FP8 is not supported on this GPU type.") @pytest.mark.parametrize("model_id", MODELS) @pytest.mark.parametrize("force_marlin", [False, True]) +@pytest.mark.parametrize( + "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]) def test_model_load_and_run(vllm_runner, model_id: str, force_marlin: bool, - monkeypatch) -> None: + use_rocm_aiter: bool, monkeypatch) -> None: + + if use_rocm_aiter: + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + if force_marlin: monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1") @@ -47,7 +53,13 @@ KV_CACHE_MODELS = [ @pytest.mark.skipif(not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.") @pytest.mark.parametrize("model_id", KV_CACHE_MODELS) -def test_kv_cache_model_load_and_run(vllm_runner, model_id: str, monkeypatch): +@pytest.mark.parametrize( + "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]) +def test_kv_cache_model_load_and_run(vllm_runner, model_id: str, + use_rocm_aiter: bool, monkeypatch): + if use_rocm_aiter: + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + # vllm_runner.apply_model() relies on V0 internals. monkeypatch.setenv("VLLM_USE_V1", "0") with vllm_runner(model_id, kv_cache_dtype="fp8") as llm: @@ -86,8 +98,13 @@ def test_kv_cache_model_load_and_run(vllm_runner, model_id: str, monkeypatch): reason="FP8 is not supported on this GPU type.") @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) @pytest.mark.parametrize("force_marlin", [False, True]) +@pytest.mark.parametrize( + "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]) def test_load_fp16_model(vllm_runner, kv_cache_dtype: str, force_marlin: bool, - monkeypatch) -> None: + use_rocm_aiter: bool, monkeypatch) -> None: + if use_rocm_aiter: + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + # vllm_runner.apply_model() relies on V0 internals. monkeypatch.setenv("VLLM_USE_V1", "0") diff --git a/vllm/envs.py b/vllm/envs.py index b4305d9c8e22c..4c413006a6413 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -73,6 +73,8 @@ if TYPE_CHECKING: VLLM_DISABLED_KERNELS: list[str] = [] VLLM_USE_V1: bool = True VLLM_ROCM_USE_AITER: bool = False + VLLM_ROCM_USE_AITER_MOE: bool = True + VLLM_ROCM_USE_AITER_FP8_BLOCK_SCALED_MOE: bool = False VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_FP8_PADDING: bool = True VLLM_ROCM_MOE_PADDING: bool = True @@ -513,6 +515,19 @@ environment_variables: dict[str, Callable[[], Any]] = { lambda: (os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")), + # Whether to use aiter moe ops. + # By default is enabled. + "VLLM_ROCM_USE_AITER_MOE": + lambda: (os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in + ("true", "1")), + + # Whether to use aiter block scaled moe kernel. + # By default this is disabled. + "VLLM_ROCM_USE_AITER_FP8_BLOCK_SCALED_MOE": + lambda: + (os.getenv("VLLM_ROCM_USE_AITER_FP8_BLOCK_SCALED_MOE", "false").lower() in + ("true", "1")), + # use aiter rms norm op if aiter ops are enabled. "VLLM_ROCM_USE_AITER_RMSNORM": lambda: (os.getenv("VLLM_ROCM_USE_AITER_RMSNORM", "True").lower() in diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 4de020ff81c0e..97e915c60335a 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -17,6 +17,10 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import ( from vllm.platforms import current_platform from vllm.utils import direct_register_custom_op +from .rocm_aiter_fused_moe import (is_rocm_aiter_moe_enabled, + rocm_aiter_fused_experts, + rocm_aiter_topk_softmax) + logger = init_logger(__name__) @@ -1035,6 +1039,28 @@ def try_get_optimal_moe_config( return config +def vllm_topk_softmax(topk_weights: torch.Tensor, topk_indices: torch.Tensor, + token_expert_indices: torch.Tensor, + gating_output: torch.Tensor, + renormalize: bool) -> tuple[torch.Tensor, ...]: + ops.topk_softmax( + topk_weights, + topk_indices, + token_expert_indices, + gating_output, + ) + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + + return topk_weights, topk_indices + + +def dispatch_topk_func() -> Callable[..., tuple[torch.Tensor, ...]]: + if is_rocm_aiter_moe_enabled(): + return rocm_aiter_topk_softmax + return vllm_topk_softmax + + def fused_topk( hidden_states: torch.Tensor, gating_output: torch.Tensor, @@ -1059,17 +1085,14 @@ def fused_topk( dtype=torch.int32, device=hidden_states.device) - ops.topk_softmax( - topk_weights, - topk_ids, - token_expert_indicies, - gating_output.float(), # TODO(woosuk): Optimize this. - ) + gating_output_float = gating_output.float() # TODO(woosuk): Optimize this. + + topk_func = dispatch_topk_func() + topk_weights, topk_ids = topk_func(topk_weights, topk_ids, + token_expert_indicies, + gating_output_float, renormalize) + del token_expert_indicies # Not used. Will be used in the future. - - if renormalize: - topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) - return topk_weights, topk_ids @@ -1259,6 +1282,24 @@ direct_register_custom_op( ) +def torch_vllm_inplace_fused_experts(**kwargs) -> torch.Tensor: + torch.ops.vllm.inplace_fused_experts(**kwargs) + hidden_states = kwargs['hidden_states'] + return hidden_states + + +def torch_vllm_outplace_fused_experts(**kwargs) -> torch.Tensor: + return torch.ops.vllm.outplace_fused_experts(**kwargs) + + +def dispatch_fused_experts_func(inplace: bool) -> Callable[..., torch.Tensor]: + if is_rocm_aiter_moe_enabled(): + return rocm_aiter_fused_experts + if inplace: + return torch_vllm_inplace_fused_experts + return torch_vllm_outplace_fused_experts + + def fused_experts(hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor, @@ -1278,20 +1319,25 @@ def fused_experts(hidden_states: torch.Tensor, a1_scale: Optional[torch.Tensor] = None, a2_scale: Optional[torch.Tensor] = None, block_shape: Optional[List[int]] = None) -> torch.Tensor: - - if inplace: - torch.ops.vllm.inplace_fused_experts( - hidden_states, w1, w2, topk_weights, topk_ids, activation, - use_fp8_w8a8, use_int8_w8a16, use_int4_w4a16, global_num_experts, - expert_map, w1_scale, w2_scale, w1_zp, w2_zp, a1_scale, a2_scale, - block_shape) - return hidden_states - else: - return torch.ops.vllm.outplace_fused_experts( - hidden_states, w1, w2, topk_weights, topk_ids, activation, - use_fp8_w8a8, use_int8_w8a16, use_int4_w4a16, global_num_experts, - expert_map, w1_scale, w2_scale, w1_zp, w2_zp, a1_scale, a2_scale, - block_shape) + return dispatch_fused_experts_func(inplace)( + hidden_states=hidden_states, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=activation, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a16=use_int8_w8a16, + use_int4_w4a16=use_int4_w4a16, + global_num_experts=global_num_experts, + expert_map=expert_map, + w1_scale=w1_scale, + w2_scale=w2_scale, + w1_zp=w1_zp, + w2_zp=w2_zp, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape) def fused_experts_impl(hidden_states: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index bc134f676159e..b72f51aa52bfa 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -8,7 +8,7 @@ import torch import torch.nn.functional as F from torch.nn.parameter import UninitializedParameter -from vllm import envs +import vllm.envs as envs from vllm.config import get_current_vllm_config from vllm.distributed import (get_dp_group, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -16,6 +16,8 @@ from vllm.distributed import (get_dp_group, get_tensor_model_parallel_rank, from vllm.forward_context import ForwardContext, get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp +from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + is_rocm_aiter_moe_enabled, shuffle_weights) from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase) from vllm.model_executor.utils import set_weight_attrs @@ -118,6 +120,16 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): layer.w2_weight.data), requires_grad=False) + if is_rocm_aiter_moe_enabled(): + # reshaping weights is required for aiter moe kernel. + shuffled_w13, shuffled_w2 = shuffle_weights( + layer.w13_weight.data, layer.w2_weight.data) + + layer.w13_weight = torch.nn.Parameter(shuffled_w13, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, + requires_grad=False) + if current_platform.is_cpu(): if current_platform.get_cpu_architecture() == CpuArchEnum.X86: import intel_extension_for_pytorch as ipex diff --git a/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py b/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py new file mode 100644 index 0000000000000..c9bb676710a78 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +from typing import List, Optional + +import torch + +import vllm.envs as envs +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + per_token_group_quant_fp8) +from vllm.platforms import current_platform + + +def is_rocm_aiter_moe_enabled() -> bool: + return current_platform.is_rocm() \ + and envs.VLLM_ROCM_USE_AITER_MOE \ + and envs.VLLM_ROCM_USE_AITER \ + + +def is_rocm_aiter_block_scaled_moe_enabled() -> bool: + return is_rocm_aiter_moe_enabled() and \ + envs.VLLM_ROCM_USE_AITER_FP8_BLOCK_SCALED_MOE + + +def rocm_aiter_fused_experts( + *, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + use_fp8_w8a8: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, + expert_mask: Optional[torch.Tensor] = None, + **kwagrs # Ignore additional keyword arguments +) -> torch.Tensor: + + import aiter as rocm_aiter + import aiter.fused_moe_bf16_asm as rocm_aiter_asm_fmoe + + if envs.VLLM_ROCM_USE_AITER_FP8_BLOCK_SCALED_MOE and use_fp8_w8a8: + assert w1_scale is not None + assert w2_scale is not None + + local_E = E = w1.shape[0] + if expert_mask is not None: + E = expert_mask.numel() + + topk = topk_ids.shape[1] + model_dim = w1.shape[-1] + dtype = hidden_states.dtype + # The default block sizes are 128 in AITER. + if block_shape is None: + block_shape = [128, 128] + + scale_blk_k = block_shape[1] + + ( + sorted_token_ids, + sorted_weight_buf, + sorted_expert_ids, + num_valid_ids, + out_asm, + ) = rocm_aiter_asm_fmoe.moe_sorting_ck(topk_ids, + topk_weights, + E, + model_dim, + dtype, + expert_mask=expert_mask) + + a1, a1_scale = per_token_group_quant_fp8(hidden_states, scale_blk_k) + rocm_aiter.fmoe_fp8_blockscale_g1u1( + out_asm, + a1, + w1, + w2, + sorted_token_ids, + sorted_weight_buf, + sorted_expert_ids, + num_valid_ids, + topk, + w1_scale.view(local_E, -1), + w2_scale.view(local_E, -1), + a1_scale.t().contiguous(), + block_shape[0], + block_shape[1], + None, + ) + return out_asm + + elif use_fp8_w8a8: + return rocm_aiter_asm_fmoe.asm_moe(hidden_states=hidden_states, + w1=w1, + w2=w2, + topk_weight=topk_weights, + topk_ids=topk_ids, + fc1_scale=w1_scale, + fc2_scale=w2_scale, + fc1_smooth_scale=None, + fc2_smooth_scale=None, + a16=False) + + return rocm_aiter.ck_moe(hidden_states=hidden_states, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids) + + +def rocm_aiter_topk_softmax(topk_weights: torch.Tensor, + topk_indices: torch.Tensor, + token_expert_indices: torch.Tensor, + gating_output: torch.Tensor, + renormalize: bool) -> tuple[torch.Tensor, ...]: + import aiter as rocm_aiter + rocm_aiter.topk_softmax(topk_weights, topk_indices, token_expert_indices, + gating_output, renormalize) + + return topk_weights, topk_indices + + +def shuffle_weights(*tensors: torch.Tensor) -> tuple[torch.Tensor, ...]: + """ + Applies shuffle_weight function from AITER to each + input tensor and returns them. + + Args: + *tensors: Variable number of torch.Tensor objects. + + Returns: + A tuple of shuffled tensors. + """ + from aiter.ops.shuffle import shuffle_weight + + return tuple(shuffle_weight(tensor) for tensor in tensors) + + +def expand_weights(*tensors: torch.Tensor, + expansion_dims: list[int]) -> tuple[torch.Tensor, ...]: + """ + Expands the dimensions of input tensors. + + Args: + *tensors: A variable number of torch.Tensor objects. + expansion_dims: A list of expansion dimensions + corresponding to each tensor. + + Returns: + A tuple of tensors with expanded dimensions. + """ + + assert len(tensors) == len(expansion_dims), \ + "Number of tensors must match the number of expansion dimensions." + + return tuple( + tensor.unsqueeze(-1).unsqueeze(-1).expand((-1, dim, -1)) + for tensor, dim in zip(tensors, expansion_dims)) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index d92b0931a6ee0..bc17a569da2c3 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -13,6 +13,9 @@ from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported) +from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + expand_weights, is_rocm_aiter_block_scaled_moe_enabled, + is_rocm_aiter_moe_enabled, shuffle_weights) from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, UnquantizedLinearMethod) from vllm.model_executor.layers.quantization.base_config import ( @@ -554,6 +557,15 @@ class Fp8MoEMethod(FusedMoEMethodBase): layer.w2_weight = Parameter(w2_weight, requires_grad=False) layer.w2_weight_scale_inv = Parameter(w2_weight_scale_inv, requires_grad=False) + if is_rocm_aiter_block_scaled_moe_enabled(): + # reshaping weights is required for aiter moe kernel. + shuffled_w13, shuffled_w2 = shuffle_weights( + layer.w13_weight.data, layer.w2_weight.data) + + layer.w13_weight = torch.nn.Parameter(shuffled_w13, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, + requires_grad=False) return # If checkpoint is fp16, quantize in place. @@ -581,6 +593,26 @@ class Fp8MoEMethod(FusedMoEMethodBase): requires_grad=False) layer.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False) + if is_rocm_aiter_moe_enabled(): + # reshaping weights is required for aiter moe kernel. + w13_scales, w2_scales = expand_weights( + layer.w13_weight_scale.data, + layer.w2_weight_scale.data, + expansion_dims=[ + layer.w13_weight.shape[1], layer.w2_weight.shape[1] + ]) + layer.w13_weight_scale = torch.nn.Parameter( + w13_scales.contiguous(), requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter( + w2_scales.contiguous(), requires_grad=False) + + shuffled_w13, shuffled_w2 = shuffle_weights( + layer.w13_weight, layer.w2_weight) + + layer.w13_weight = torch.nn.Parameter(shuffled_w13, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, + requires_grad=False) return # If checkpoint is fp8, we need to handle that the @@ -648,6 +680,26 @@ class Fp8MoEMethod(FusedMoEMethodBase): dq_weight, max_w13_scales[expert_id]) start += shard_size + if is_rocm_aiter_moe_enabled(): + # reshaping weights is required for aiter moe kernel. + expansion_dims = [ + layer.w13_weight.shape[1], layer.w2_weight.shape[1] + ] + max_w13_scales, w2_scales = expand_weights( + max_w13_scales, + layer.w2_weight_scale.data, + expansion_dims=expansion_dims) + layer.w2_weight_scale = torch.nn.Parameter( + w2_scales.contiguous(), requires_grad=False) + + shuffled_w13, shuffled_w2 = shuffle_weights( + layer.w13_weight, layer.w2_weight) + + layer.w13_weight = torch.nn.Parameter(shuffled_w13, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, + requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, requires_grad=False) return From 99f536f83093dc22e4220b5bd0f8c63f9e86a406 Mon Sep 17 00:00:00 2001 From: wwl2755 Date: Wed, 26 Mar 2025 04:21:15 -0500 Subject: [PATCH 009/593] [Misc] Enhance warning information to user-defined chat template (#15408) Signed-off-by: wwl2755 --- tests/entrypoints/test_chat_utils.py | 10 +++---- vllm/entrypoints/chat_utils.py | 40 +++++++++++++++++---------- vllm/entrypoints/openai/api_server.py | 27 ++++++++++++++++-- 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 6efed990b1893..8cc51a5d73b3f 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -9,11 +9,11 @@ from transformers import __version__ as TRANSFORMERS_VERSION from vllm.assets.image import ImageAsset from vllm.config import ModelConfig -from vllm.entrypoints.chat_utils import (_resolve_hf_chat_template, - _try_extract_ast, load_chat_template, +from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template, parse_chat_messages, parse_chat_messages_futures, - resolve_chat_template_content_format) + resolve_chat_template_content_format, + resolve_hf_chat_template) from vllm.entrypoints.llm import apply_hf_chat_template from vllm.multimodal import MultiModalDataDict from vllm.multimodal.utils import encode_image_base64 @@ -747,7 +747,7 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools): }] if use_tools else None # Test detecting the tokenizer's chat_template - chat_template = _resolve_hf_chat_template( + chat_template = resolve_hf_chat_template( tokenizer, chat_template=None, tools=tools, @@ -781,7 +781,7 @@ def test_resolve_content_format_hf_defined(model, expected_format): tokenizer = tokenizer_group.tokenizer # Test detecting the tokenizer's chat_template - chat_template = _resolve_hf_chat_template( + chat_template = resolve_hf_chat_template( tokenizer, chat_template=None, tools=None, diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index d3613384590de..73a69d3037f7f 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -306,7 +306,24 @@ def _detect_content_format( return "openai" -def _resolve_hf_chat_template( +def resolve_mistral_chat_template( + chat_template: Optional[str], + **kwargs: Any, +) -> Optional[str]: + if chat_template is not None: + logger.warning_once( + "'chat_template' cannot be overridden for mistral tokenizer.") + if "add_generation_prompt" in kwargs: + logger.warning_once( + "'add_generation_prompt' is not supported for mistral tokenizer, " + "so it will be ignored.") + if "continue_final_message" in kwargs: + logger.warning_once( + "'continue_final_message' is not supported for mistral tokenizer, " + "so it will be ignored.") + return None + +def resolve_hf_chat_template( tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], chat_template: Optional[str], tools: Optional[list[dict[str, Any]]], @@ -352,7 +369,7 @@ def _resolve_chat_template_content_format( trust_remote_code: bool, ) -> _ChatTemplateContentFormat: if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)): - hf_chat_template = _resolve_hf_chat_template( + hf_chat_template = resolve_hf_chat_template( tokenizer, chat_template=chat_template, trust_remote_code=trust_remote_code, @@ -1140,7 +1157,7 @@ def apply_hf_chat_template( tokenize: bool = False, # Different from HF's default **kwargs: Any, ) -> str: - hf_chat_template = _resolve_hf_chat_template( + hf_chat_template = resolve_hf_chat_template( tokenizer, chat_template=chat_template, tools=tools, @@ -1169,17 +1186,12 @@ def apply_mistral_chat_template( tools: Optional[list[dict[str, Any]]], **kwargs: Any, ) -> list[int]: - if chat_template is not None: - logger.warning_once( - "'chat_template' cannot be overridden for mistral tokenizer.") - if "add_generation_prompt" in kwargs: - logger.warning_once( - "'add_generation_prompt' is not supported for mistral tokenizer, " - "so it will be ignored.") - if "continue_final_message" in kwargs: - logger.warning_once( - "'continue_final_message' is not supported for mistral tokenizer, " - "so it will be ignored.") + # The return value of resolve_mistral_chat_template is always None, + # and we won't use it. + resolve_mistral_chat_template( + chat_template=chat_template, + **kwargs, + ) return tokenizer.apply_chat_template( messages=messages, diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index f9b1d69a31d8c..374e43fb15341 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -35,7 +35,9 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore from vllm.engine.multiprocessing.client import MQLLMEngineClient from vllm.engine.multiprocessing.engine import run_mp_engine from vllm.engine.protocol import EngineClient -from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.chat_utils import (load_chat_template, + resolve_hf_chat_template, + resolve_mistral_chat_template) from vllm.entrypoints.launcher import serve_http from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.openai.cli_args import (make_arg_parser, @@ -84,6 +86,7 @@ from vllm.entrypoints.utils import load_aware_call, with_cancellation from vllm.logger import init_logger from vllm.transformers_utils.config import ( maybe_register_config_serialize_by_value) +from vllm.transformers_utils.tokenizer import MistralTokenizer from vllm.usage.usage_lib import UsageContext from vllm.utils import (Device, FlexibleArgumentParser, get_open_zmq_ipc_path, is_valid_ipv6_address, set_ulimit) @@ -883,8 +886,26 @@ async def init_app_state( resolved_chat_template = load_chat_template(args.chat_template) if resolved_chat_template is not None: - logger.info("Using supplied chat template:\n%s", - resolved_chat_template) + # Get the tokenizer to check official template + tokenizer = await engine_client.get_tokenizer() + + if isinstance(tokenizer, MistralTokenizer): + # The warning is logged in resolve_mistral_chat_template. + resolved_chat_template = resolve_mistral_chat_template( + chat_template=resolved_chat_template) + else: + hf_chat_template = resolve_hf_chat_template( + tokenizer, + chat_template=None, + tools=None, + trust_remote_code=model_config.trust_remote_code) + + if hf_chat_template != resolved_chat_template: + logger.warning( + "Using supplied chat template: %s\n" + "It is different from official chat template '%s'. " + "This discrepancy may lead to performance degradation.", + resolved_chat_template, args.model) state.openai_serving_models = OpenAIServingModels( engine_client=engine_client, From 4ec2cee000af209a9499e0696993834af4f45035 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Wed, 26 Mar 2025 18:12:47 +0800 Subject: [PATCH 010/593] [Misc] improve example script output (#15528) Signed-off-by: reidliu41 Co-authored-by: reidliu41 --- examples/offline_inference/basic/basic.py | 5 ++++- examples/offline_inference/basic/chat.py | 5 +++-- examples/offline_inference/basic/classify.py | 4 +++- examples/offline_inference/basic/embed.py | 4 +++- examples/offline_inference/basic/score.py | 4 +++- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/examples/offline_inference/basic/basic.py b/examples/offline_inference/basic/basic.py index a6e96c0bb4339..2ba5ec1192b19 100644 --- a/examples/offline_inference/basic/basic.py +++ b/examples/offline_inference/basic/basic.py @@ -18,7 +18,10 @@ llm = LLM(model="facebook/opt-125m") # that contain the prompt, generated text, and other information. outputs = llm.generate(prompts, sampling_params) # Print the outputs. +print("\nGenerated Outputs:\n" + "-" * 60) for output in outputs: prompt = output.prompt generated_text = output.outputs[0].text - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") \ No newline at end of file + print(f"Prompt: {prompt!r}") + print(f"Output: {generated_text!r}") + print("-" * 60) \ No newline at end of file diff --git a/examples/offline_inference/basic/chat.py b/examples/offline_inference/basic/chat.py index b2523e533a40a..2dea45f843cf3 100644 --- a/examples/offline_inference/basic/chat.py +++ b/examples/offline_inference/basic/chat.py @@ -27,12 +27,13 @@ def main(args: dict): sampling_params.top_k = top_k def print_outputs(outputs): + print("\nGenerated Outputs:\n" + "-" * 80) for output in outputs: prompt = output.prompt generated_text = output.outputs[0].text - print(f"Prompt: {prompt!r}") + print(f"Prompt: {prompt!r}\n") print(f"Generated text: {generated_text!r}") - print("-" * 80) + print("-" * 80) print("=" * 80) diff --git a/examples/offline_inference/basic/classify.py b/examples/offline_inference/basic/classify.py index 4ef949b4784de..72c29e4c77c30 100644 --- a/examples/offline_inference/basic/classify.py +++ b/examples/offline_inference/basic/classify.py @@ -23,12 +23,14 @@ def main(args: Namespace): outputs = model.classify(prompts) # Print the outputs. + print("\nGenerated Outputs:\n" + "-" * 60) for prompt, output in zip(prompts, outputs): probs = output.outputs.probs probs_trimmed = ((str(probs[:16])[:-1] + ", ...]") if len(probs) > 16 else probs) - print(f"Prompt: {prompt!r} | " + print(f"Prompt: {prompt!r} \n" f"Class Probabilities: {probs_trimmed} (size={len(probs)})") + print("-" * 60) if __name__ == "__main__": diff --git a/examples/offline_inference/basic/embed.py b/examples/offline_inference/basic/embed.py index f1655b6dbe111..0283909a2a84a 100644 --- a/examples/offline_inference/basic/embed.py +++ b/examples/offline_inference/basic/embed.py @@ -23,12 +23,14 @@ def main(args: Namespace): outputs = model.embed(prompts) # Print the outputs. + print("\nGenerated Outputs:\n" + "-" * 60) for prompt, output in zip(prompts, outputs): embeds = output.outputs.embedding embeds_trimmed = ((str(embeds[:16])[:-1] + ", ...]") if len(embeds) > 16 else embeds) - print(f"Prompt: {prompt!r} | " + print(f"Prompt: {prompt!r} \n" f"Embeddings: {embeds_trimmed} (size={len(embeds)})") + print("-" * 60) if __name__ == "__main__": diff --git a/examples/offline_inference/basic/score.py b/examples/offline_inference/basic/score.py index 2d21f1f0e3971..83b8253f4e257 100644 --- a/examples/offline_inference/basic/score.py +++ b/examples/offline_inference/basic/score.py @@ -22,9 +22,11 @@ def main(args: Namespace): outputs = model.score(text_1, texts_2) # Print the outputs. + print("\nGenerated Outputs:\n" + "-" * 60) for text_2, output in zip(texts_2, outputs): score = output.outputs.score - print(f"Pair: {[text_1, text_2]!r} | Score: {score}") + print(f"Pair: {[text_1, text_2]!r} \nScore: {score}") + print("-" * 60) if __name__ == "__main__": From cf5c8f1686d810883f27974fa4433f0f95c94cbe Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 26 Mar 2025 10:13:38 +0000 Subject: [PATCH 011/593] Separate base model from `TransformersModel` (#15467) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py --- docs/source/models/supported_models.md | 6 +- tests/distributed/test_pipeline_parallel.py | 2 +- tests/models/registry.py | 2 +- vllm/model_executor/model_loader/utils.py | 6 +- vllm/model_executor/models/registry.py | 4 +- vllm/model_executor/models/transformers.py | 149 +++++++++++++------- 6 files changed, 110 insertions(+), 59 deletions(-) diff --git a/docs/source/models/supported_models.md b/docs/source/models/supported_models.md index f106195e10585..8ff18a17d36c3 100644 --- a/docs/source/models/supported_models.md +++ b/docs/source/models/supported_models.md @@ -57,10 +57,10 @@ llm = LLM(model=..., task="generate") # Name or path of your model llm.apply_model(lambda model: print(type(model))) ``` -If it is `TransformersModel` then it means it's based on Transformers! +If it is `TransformersForCausalLM` then it means it's based on Transformers! :::{tip} -You can force the use of `TransformersModel` by setting `model_impl="transformers"` for or `--model-impl transformers` for the . +You can force the use of `TransformersForCausalLM` by setting `model_impl="transformers"` for or `--model-impl transformers` for the . ::: :::{note} @@ -119,7 +119,7 @@ Here is what happens in the background: 1. The config is loaded 2. `MyModel` Python class is loaded from the `auto_map`, and we check that the model `_supports_attention_backend`. -3. The `TransformersModel` backend is used. See , which leverage `self.config._attn_implementation = "vllm"`, thus the need to use `ALL_ATTENTION_FUNCTION`. +3. The `TransformersForCausalLM` backend is used. See , which leverage `self.config._attn_implementation = "vllm"`, thus the need to use `ALL_ATTENTION_FUNCTION`. To make your model compatible with tensor parallel, it needs: diff --git a/tests/distributed/test_pipeline_parallel.py b/tests/distributed/test_pipeline_parallel.py index e757db45c8cf5..751c4eb096ae0 100644 --- a/tests/distributed/test_pipeline_parallel.py +++ b/tests/distributed/test_pipeline_parallel.py @@ -175,7 +175,7 @@ TEXT_GENERATION_MODELS = { "inceptionai/jais-13b-chat": PPTestSettings.fast(), "ai21labs/Jamba-tiny-dev": PPTestSettings.fast(), "meta-llama/Llama-3.2-1B-Instruct": PPTestSettings.detailed(), - # Tests TransformersModel + # Tests TransformersForCausalLM "ArthurZ/Ilama-3.2-1B": PPTestSettings.fast(), "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(), "openbmb/MiniCPM3-4B": PPTestSettings.fast(), diff --git a/tests/models/registry.py b/tests/models/registry.py index 5c84e85aaa907..d7946b75b7978 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -319,7 +319,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { } _FALLBACK_MODEL = { - "TransformersModel": _HfExamplesInfo("ArthurZ/Ilama-3.2-1B", trust_remote_code=True), # noqa: E501 + "TransformersForCausalLM": _HfExamplesInfo("ArthurZ/Ilama-3.2-1B", trust_remote_code=True), # noqa: E501 } _EXAMPLE_MODELS = { diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index ce90614329725..a252c7f8e57bc 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -45,7 +45,7 @@ def is_transformers_impl_compatible( def resolve_transformers_fallback(model_config: ModelConfig, architectures: list[str]): for i, arch in enumerate(architectures): - if arch == "TransformersModel": + if arch == "TransformersForCausalLM": continue auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map", None) or dict() @@ -69,7 +69,7 @@ def resolve_transformers_fallback(model_config: ModelConfig, raise ValueError( f"The Transformers implementation of {arch} is not " "compatible with vLLM.") - architectures[i] = "TransformersModel" + architectures[i] = "TransformersForCausalLM" if model_config.model_impl == ModelImpl.AUTO: if not is_transformers_impl_compatible(arch, custom_model_module): raise ValueError( @@ -80,7 +80,7 @@ def resolve_transformers_fallback(model_config: ModelConfig, "%s has no vLLM implementation, falling back to Transformers " "implementation. Some features may not be supported and " "performance may not be optimal.", arch) - architectures[i] = "TransformersModel" + architectures[i] = "TransformersForCausalLM" return architectures diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 7c8e506713833..7797d9a2cc203 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -201,7 +201,7 @@ _SPECULATIVE_DECODING_MODELS = { } _FALLBACK_MODEL = { - "TransformersModel": ("transformers", "TransformersModel"), + "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"), } # yapf: enable @@ -425,7 +425,7 @@ class _ModelRegistry: # make sure Transformers fallback are put at the last if len(normalized_arch) != len(architectures): - normalized_arch.append("TransformersModel") + normalized_arch.append("TransformersForCausalLM") return normalized_arch def inspect_model_cls( diff --git a/vllm/model_executor/models/transformers.py b/vllm/model_executor/models/transformers.py index 56ec00dcf222c..6ea149506581c 100644 --- a/vllm/model_executor/models/transformers.py +++ b/vllm/model_executor/models/transformers.py @@ -43,7 +43,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant -from .utils import (PPMissingLayer, is_pp_missing_parameter, +from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper, + is_pp_missing_parameter, make_empty_intermediate_tensors_factory, maybe_prefix) logger = init_logger(__name__) @@ -110,13 +111,9 @@ def replace_linear_class( ) -@support_torch_compile -class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): - embedding_padding_modules = ["lm_head"] - embedding_modules = ["embed_tokens" - ] # TODO transformers will have a util to get it +class TransformersModel(nn.Module): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() logger.info("Using Transformers backend.") @@ -134,9 +131,6 @@ class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): self.parallel_config = parallel_config self.quant_config = quant_config - self.vocab_size = model_config.get_vocab_size() - self.unpadded_vocab_size = model_config.get_vocab_size() - self.pp_group = get_pp_group() self.pp_size = self.pp_group.world_size self.pp_rank = self.pp_group.rank_in_group @@ -144,13 +138,15 @@ class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): # Use meta device to delay allocating GPU tensors with torch.device("meta"): + # FIXME(Isotr0py): We need to refactor this part in the future to + # avoid registering an extra model layer, otherwise we will need a + # weights mapper to rename weights. self.model: PreTrainedModel = AutoModel.from_config( config, attn_implementation="vllm", torch_dtype=model_config.dtype, trust_remote_code=model_config.trust_remote_code, ) - prefix = self.model.base_model_prefix self.pipeline_parallel() self.tensor_parallel() @@ -168,32 +164,12 @@ class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): # Attention layers self.attention_instances = self.create_attention_instances() - # Output embeddings - if not isinstance(getattr(self, "lm_head", None), PPMissingLayer): - self.unpadded_vocab_size = config.vocab_size - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "lm_head"), - ) - if config.tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights( - self.model.get_input_embeddings()) - - logit_scale = getattr(config, "logit_scale", 1.0) - self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, - config.vocab_size, - logit_scale) - # Initialize buffers (e.g. rotary embedding inverse frequency) self.init_buffers(self.model) # Move remaining meta tensors to device (should happen last) self.meta_to_empty(self.model) - self.sampler = get_sampler() - self.make_empty_intermediate_tensors = ( make_empty_intermediate_tensors_factory(["hidden_states"], config.hidden_size)) @@ -248,9 +224,6 @@ class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): if not self.pp_group.is_last_rank: setattr(self.model, name, PPMissingLayer()) - if not self.pp_group.is_last_rank: - self.lm_head = PPMissingLayer() - def tensor_parallel(self): """ Apply the model's tensor parallelization plan. @@ -331,6 +304,9 @@ class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): for child in module.children(): self.meta_to_empty(child) + def get_input_embeddings(self) -> nn.Module: + return self.model.get_input_embeddings() + def forward( self, input_ids: Optional[torch.Tensor], @@ -361,21 +337,6 @@ class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): return hidden_states - def compute_logits( - self, - hidden_states: torch.Tensor, - sampling_metadata: SamplingMetadata, - ) -> Optional[torch.Tensor]: - logits = self.logits_processor(self.lm_head, hidden_states, - sampling_metadata) - return logits - - def sample(self, logits: torch.Tensor, - sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]: - - next_tokens = self.sampler(logits, sampling_metadata) - return next_tokens - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: params_dict = dict(self.named_parameters()) @@ -393,3 +354,93 @@ class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP): weight_loader(param, loaded_weight) loaded_params.add(name) return loaded_params + + +@support_torch_compile +class TransformersForCausalLM(nn.Module, SupportsQuant, SupportsLoRA, + SupportsPP): + embedding_padding_modules = ["lm_head"] + embedding_modules = ["embed_tokens" + ] # TODO transformers will have a util to get it + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config: PretrainedConfig = vllm_config.model_config.hf_config + quant_config: QuantizationConfig = vllm_config.quant_config + + self.config = config + + self.model = TransformersModel(vllm_config=vllm_config, prefix=prefix) + + if get_pp_group().is_last_rank: + self.unpadded_vocab_size = config.vocab_size + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + if config.tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights( + self.model.get_input_embeddings()) + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + logit_scale) + else: + self.lm_head = PPMissingLayer() + + self.sampler = get_sampler() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + # FIXME(Isotr0py): Don't use any weights mapper for Transformers fallback, + # this makes thing complicated. We need to remove this mapper after refactor + # `TransformersModel` in the future. + @property + def hf_to_vllm_mapper(self): + prefix_mapper = { + name: "model." + name + for name, _ in self.model.model.named_children() + } + return WeightsMapper( + orig_to_new_substr={"model.": "model.model."}, + orig_to_new_prefix=prefix_mapper, + ) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + model_output = self.model(input_ids, positions, intermediate_tensors, + inputs_embeds) + return model_output + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample(self, logits: torch.Tensor, + sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]: + + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) From 1aa162e030d62fcf476388ac77c141cb5b52957b Mon Sep 17 00:00:00 2001 From: cyyever Date: Wed, 26 Mar 2025 20:09:06 +0800 Subject: [PATCH 012/593] Apply torchfix (#15532) Signed-off-by: cyy --- vllm/attention/backends/rocm_flash_attn.py | 5 ++--- vllm/lora/models.py | 4 +++- vllm/model_executor/models/nemotron.py | 6 +++--- vllm/model_executor/models/phi4mm_utils.py | 9 ++++++--- vllm/multimodal/image.py | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/vllm/attention/backends/rocm_flash_attn.py b/vllm/attention/backends/rocm_flash_attn.py index c47202099ac60..34f5fedcf36e8 100644 --- a/vllm/attention/backends/rocm_flash_attn.py +++ b/vllm/attention/backends/rocm_flash_attn.py @@ -884,9 +884,8 @@ def _sdpa_attention( for i, seq_len in enumerate(seq_lens): end = start + seq_len - with torch.backends.cuda.sdp_kernel(enable_math=True, - enable_flash=False, - enable_mem_efficient=False): + with torch.nn.attention.sdpa_kernel( + torch.nn.attention.SDPBackend.MATH): sub_out = torch.nn.functional.scaled_dot_product_attention( query[:, start:end, :], key[:, start:end, :], diff --git a/vllm/lora/models.py b/vllm/lora/models.py index 22a45b60ca399..8164d919ca8b4 100644 --- a/vllm/lora/models.py +++ b/vllm/lora/models.py @@ -272,7 +272,9 @@ class LoRAModel(AdapterModel): f" target modules in {expected_lora_modules}" f" but received {unexpected_modules}." f" Please verify that the loaded LoRA module is correct") - tensors = torch.load(lora_bin_file_path, map_location=device) + tensors = torch.load(lora_bin_file_path, + map_location=device, + weights_only=True) else: raise ValueError(f"{lora_dir} doesn't contain tensors") diff --git a/vllm/model_executor/models/nemotron.py b/vllm/model_executor/models/nemotron.py index a2b4949496897..0ea296b2f93d1 100644 --- a/vllm/model_executor/models/nemotron.py +++ b/vllm/model_executor/models/nemotron.py @@ -63,8 +63,8 @@ def _cast_if_autocast_enabled(*args): if not torch.is_autocast_enabled(): return args else: - return torch.cuda.amp.autocast_mode._cast( - args, torch.get_autocast_gpu_dtype()) + return torch.amp.autocast_mode._cast( + args, device_type="cuda", dtype=torch.get_autocast_gpu_dtype()) class NemotronLayerNorm1P(nn.LayerNorm): @@ -89,7 +89,7 @@ class NemotronLayerNorm1P(nn.LayerNorm): residual = x args = _cast_if_autocast_enabled(x, self.normalized_shape, self.weight + 1, self.bias, self.eps) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x = torch.nn.functional.layer_norm(*args) return x if residual is None else (x, residual) diff --git a/vllm/model_executor/models/phi4mm_utils.py b/vllm/model_executor/models/phi4mm_utils.py index ca00207a9b6f7..9f08a1c4c6f5a 100644 --- a/vllm/model_executor/models/phi4mm_utils.py +++ b/vllm/model_executor/models/phi4mm_utils.py @@ -1766,9 +1766,12 @@ class MultiHeadedAttention(nn.Module): if mask.dtype != q.dtype: attn_mask = attn_mask.to(q.dtype) - with torch.backends.cuda.sdp_kernel(enable_flash=True, - enable_math=True, - enable_mem_efficient=True): + with torch.nn.attention.sdpa_kernel([ + torch.nn.attention.SDPBackend.FLASH_ATTENTION, + torch.nn.attention.SDPBackend.EFFICIENT_ATTENTION, + torch.nn.attention.SDPBackend.MATH, + torch.nn.attention.SDPBackend.CUDNN_ATTENTION, + ]): x = torch.nn.functional.scaled_dot_product_attention( q, k, diff --git a/vllm/multimodal/image.py b/vllm/multimodal/image.py index 255fac30bd78a..0c5a84c6508a1 100644 --- a/vllm/multimodal/image.py +++ b/vllm/multimodal/image.py @@ -149,7 +149,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]): return self.load_bytes(base64.b64decode(data)) def load_file(self, filepath: Path) -> torch.Tensor: - return torch.load(filepath) + return torch.load(filepath, weights_only=True) def encode_base64(self, media: torch.Tensor) -> str: return base64.b64encode(media.numpy()).decode('utf-8') From c091c0a58898b8a0a76e18bd6724732d80fcfc28 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 26 Mar 2025 14:26:48 +0000 Subject: [PATCH 013/593] Improve validation of TP in Transformers backend (#15540) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- vllm/model_executor/models/transformers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/models/transformers.py b/vllm/model_executor/models/transformers.py index 6ea149506581c..bdc390689104e 100644 --- a/vllm/model_executor/models/transformers.py +++ b/vllm/model_executor/models/transformers.py @@ -229,7 +229,10 @@ class TransformersModel(nn.Module): Apply the model's tensor parallelization plan. Currently only supports linear layers. """ - if self.tp_size > 1 and self.config.base_model_tp_plan is None: + if not self.model.supports_tp_plan: + if self.tp_size <= 1: + return + raise ValueError( f"{type(self.model)} does not support tensor parallel yet!") From 1711b929b6fadd02a7d66d936ec6ffd24e4c3b54 Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Wed, 26 Mar 2025 08:28:07 -0600 Subject: [PATCH 014/593] [Model] Add Reasoning Parser for Granite Models (#14202) Signed-off-by: Alex-Brooks Co-authored-by: Joe Runde --- docs/source/features/reasoning_outputs.md | 7 +- .../openai_chat_completion_with_reasoning.py | 1 + ...hat_completion_with_reasoning_streaming.py | 1 + .../test_granite_reasoning_parser.py | 349 +++++++++++++++++ vllm/engine/arg_utils.py | 2 +- .../openai/reasoning_parsers/__init__.py | 6 +- .../granite_reasoning_parser.py | 363 ++++++++++++++++++ .../guided_decoding/reasoner/__init__.py | 4 + 8 files changed, 730 insertions(+), 3 deletions(-) create mode 100644 tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py create mode 100644 vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py diff --git a/docs/source/features/reasoning_outputs.md b/docs/source/features/reasoning_outputs.md index 0b170aadc3443..879b16d4f7b50 100644 --- a/docs/source/features/reasoning_outputs.md +++ b/docs/source/features/reasoning_outputs.md @@ -4,7 +4,7 @@ vLLM offers support for reasoning models like [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1), which are designed to generate outputs containing both reasoning steps and final conclusions. -Reasoning models return a additional `reasoning_content` field in their outputs, which contains the reasoning steps that led to the final conclusion. This field is not present in the outputs of other models. +Reasoning models return an additional `reasoning_content` field in their outputs, which contains the reasoning steps that led to the final conclusion. This field is not present in the outputs of other models. ## Supported Models @@ -14,6 +14,9 @@ vLLM currently supports the following reasoning models: |--------------|-------------|------------------|-------------| | [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `guided_json`, `guided_regex` | ❌ | | [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `guided_json`, `guided_regex` | ✅ | +| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ | + +- IBM Granite 3.2 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`. ## Quickstart @@ -43,6 +46,7 @@ model = models.data[0].id # Round 1 messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}] +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}` response = client.chat.completions.create(model=model, messages=messages) reasoning_content = response.choices[0].message.reasoning_content @@ -97,6 +101,7 @@ models = client.models.list() model = models.data[0].id messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}] +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}` stream = client.chat.completions.create(model=model, messages=messages, stream=True) diff --git a/examples/online_serving/openai_chat_completion_with_reasoning.py b/examples/online_serving/openai_chat_completion_with_reasoning.py index b5dbed1205d35..e753cedcdc08d 100644 --- a/examples/online_serving/openai_chat_completion_with_reasoning.py +++ b/examples/online_serving/openai_chat_completion_with_reasoning.py @@ -31,6 +31,7 @@ model = models.data[0].id # Round 1 messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}] +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}` response = client.chat.completions.create(model=model, messages=messages) reasoning_content = response.choices[0].message.reasoning_content diff --git a/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py b/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py index fe4332576d438..cb13b0c614aa1 100644 --- a/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py +++ b/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py @@ -38,6 +38,7 @@ models = client.models.list() model = models.data[0].id messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}] +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}` stream = client.chat.completions.create(model=model, messages=messages, stream=True) diff --git a/tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py b/tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py new file mode 100644 index 0000000000000..84ac6600498b2 --- /dev/null +++ b/tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py @@ -0,0 +1,349 @@ +# SPDX-License-Identifier: Apache-2.0 +import pytest +from transformers import AutoTokenizer + +from tests.entrypoints.openai.reasoning_parsers.utils import ( + DeltaMessage, run_reasoning_extraction) +from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser, + ReasoningParserManager) + +parser_name = "granite" +START_REASONING = "Here is my thought process:" +START_RESPONSE = "Here is my response:" + +SIMPLE_REASONING = { + "output": + f"{START_REASONING}This is a reasoning section{START_RESPONSE}This is the rest", #noqa: E501 + "reasoning_content": "This is a reasoning section", + "content": "This is the rest", +} +COMPLETE_REASONING = { + "output": f"{START_REASONING}This is a reasoning section{START_RESPONSE}", + "reasoning_content": "This is a reasoning section", + "content": None, +} +NO_REASONING = { + "output": "This is content", + "reasoning_content": None, + "content": "This is content", +} +MULTIPLE_LINES = { + "output": + f"{START_REASONING}This\nThat{START_RESPONSE}This is the rest\nThat", + "reasoning_content": "This\nThat", + "content": "This is the rest\nThat", +} +REASONING_WITH_THINK = { + "output": + f"{START_REASONING}This is a reasoning section{START_RESPONSE}This is the rest", #noqa: E501 + "reasoning_content": "This is a reasoning section", + "content": "This is the rest", +} +COMPLETE_REASONING_WITH_THINK = { + "output": f"{START_REASONING}This is a reasoning section{START_RESPONSE}", + "reasoning_content": "This is a reasoning section", + "content": None, +} +MULTIPLE_LINES_WITH_THINK = { + "output": + f"{START_REASONING}This\nThat{START_RESPONSE}This is the rest\nThat", + "reasoning_content": "This\nThat", + "content": "This is the rest\nThat", +} + +TEST_CASES = [ + pytest.param( + False, + SIMPLE_REASONING, + id="simple_reasoning", + ), + pytest.param( + False, + COMPLETE_REASONING, + id="complete_reasoning", + ), + pytest.param( + False, + NO_REASONING, + id="no_reasoning", + ), + pytest.param( + False, + MULTIPLE_LINES, + id="multiple_lines", + ), + pytest.param( + False, + REASONING_WITH_THINK, + id="reasoning_with_think", + ), + pytest.param( + False, + COMPLETE_REASONING_WITH_THINK, + id="complete_reasoning_with_think", + ), + pytest.param( + False, + MULTIPLE_LINES_WITH_THINK, + id="multiple_lines_with_think", + ), + pytest.param( + True, + SIMPLE_REASONING, + id="simple_reasoning_streaming", + ), + pytest.param( + True, + COMPLETE_REASONING, + id="complete_reasoning_streaming", + ), + pytest.param( + True, + NO_REASONING, + id="no_reasoning_streaming", + ), + pytest.param( + True, + MULTIPLE_LINES, + id="multiple_lines_streaming", + ), + pytest.param( + True, + REASONING_WITH_THINK, + id="reasoning_with_think_streaming", + ), + pytest.param( + True, + COMPLETE_REASONING_WITH_THINK, + id="complete_reasoning_with_think_streaming", + ), + pytest.param( + True, + MULTIPLE_LINES_WITH_THINK, + id="multiple_lines_with_think_streaming", + ), +] + +# Global tokenizer initialization to avoid repeated loading +tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m") + + +@pytest.mark.parametrize("streaming, param_dict", TEST_CASES) +def test_reasoning( + streaming: bool, + param_dict: dict, +): + output = tokenizer.tokenize(param_dict["output"]) + # decode everything to tokens + output_tokens: list[str] = [ + tokenizer.convert_tokens_to_string([token]) for token in output + ] + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser( + parser_name)(tokenizer) + + reasoning, content = run_reasoning_extraction(parser, + output_tokens, + streaming=streaming) + + assert reasoning == param_dict["reasoning_content"] + assert content == param_dict["content"] + + +# Additional tests for verifying the correctness of granite streaming; this +# is complicated because granite uses multiple tokens to indicate when thinking +# is starting / when it's starting its response, so skipping special tokens +# is awkward. + +### Handling the start of reasoning +STREAMING_1 = { + "previous_text": None, + "current_text": "Here", + "delta_text": "Here", + "reasoning_content": None, + "content": None, +} +# When we fail, we should give what was previously being silenced first +STREAMING_2 = { + "previous_text": "Here is my thought", + "current_text": "Here is my thought failure", + "delta_text": " failure", + "reasoning_content": None, + "content": "Here is my thought failure", +} +# But then after the first one, we should only add the delta text to content +STREAMING_3 = { + "previous_text": "Here wrong", + "current_text": " words", + "delta_text": " Here wrong words", + "reasoning_content": None, + "content": " words", +} +# But then after the first one, we should only add the delta text to content +STREAMING_4 = { + "previous_text": "Here is my thought", + "current_text": "Here is my thought process:", + "delta_text": " process:", + "reasoning_content": None, + "content": None, +} +# Reasoning started successfully; parse reasoning content +STREAMING_5 = { + "previous_text": "Here is my thought process:", + "current_text": "Here is my thought process: foo", + "delta_text": " foo", + "reasoning_content": " foo", + "content": None, +} +# Response special sequence has started, but not finished. +STREAMING_6 = { + "previous_text": "Here is my thought process: foo", + "current_text": "Here is my thought process: foo Here is", + "delta_text": " Here is", + "reasoning_content": " ", + "content": None, +} +# Response special sequence started, but was broken; the reasoning +# content should be the content that was previously unused. +STREAMING_7 = { + "previous_text": "Here is my thought process: foo Here is", + "current_text": "Here is my thought process: foo Here is Here", + "delta_text": " Here", + "reasoning_content": "Here is ", + "content": None, +} +# Response special sequence is ongoing +STREAMING_8 = { + "previous_text": "Here is my thought process: foo Here is my response:", + "current_text": "Here is my thought process: foo Here is my response: bar", + "delta_text": " bar", + "reasoning_content": None, + "content": " bar", +} +# The delta text has everything; we should be able to correctly parse both +STREAMING_9 = { + "previous_text": None, + "current_text": "Here is my thought process: foo Here is my response: bar", + "delta_text": "Here is my thought process: foo Here is my response: bar", + "reasoning_content": " foo ", + "content": " bar", +} +## The Response is ongoing, and the delta mixes reasoning content / content +STREAMING_10 = { + "previous_text": "Here is my thought process: foo", + "current_text": + "Here is my thought process: foo bar Here is my response: baz", + "delta_text": " bar Here is my response: baz", + "reasoning_content": " bar ", + "content": " baz", +} +# The delta text starts a new substring that might be a response special seq +STREAMING_11 = { + "previous_text": + "Here is my thought process: This is a reasoning section ", + "current_text": + "Here is my thought process: This is a reasoning section Here", + "delta_text": "Here", + "reasoning_content": None, + "content": None, +} +# The delta text is finishing the response special seq +STREAMING_12 = { + "previous_text": "Here is my thought process: foo Here is my response", + "current_text": "Here is my thought process: foo Here is my response:", + "delta_text": ":", + "reasoning_content": None, + "content": None, +} +STREAMING_13 = { + "previous_text": "Here is my thought process: foo Here", + "current_text": "Here is my thought process: foo Here was", + "delta_text": " was", + "reasoning_content": "Here was", + "content": None, +} + +STREAMING_SUBCASES = [ + pytest.param( + STREAMING_1, + id="Starting reasoning special sequence", + ), + pytest.param( + STREAMING_2, + id="Unexpected start reasoning sequence", + ), + pytest.param( + STREAMING_3, + id="Continuing unexpected start reasoning sequence", + ), + pytest.param( + STREAMING_4, + id="Only start reasoning sequence and nothing else", + ), + pytest.param( + STREAMING_5, + id="Reasoning content has started", + ), + pytest.param( + STREAMING_6, + id="Response special sequence has started", + ), + pytest.param( + STREAMING_7, + id="Response special sequence reset", + ), + pytest.param( + STREAMING_8, + id="Response text has started", + ), + pytest.param( + STREAMING_9, + id="Delta contains everything", + ), + pytest.param( + STREAMING_10, + id="Delta contains some reasoning and response", + ), + pytest.param( + STREAMING_11, + id="Delta starts response sequence", + ), + pytest.param( + STREAMING_12, + id="Delta finishes response sequence", + ), + pytest.param( + STREAMING_13, + id="Delta breaks potential responise sequence", + ), +] + + +@pytest.mark.parametrize("param_dict", STREAMING_SUBCASES) +def test_streaming_subcases(param_dict): + # Get all of the token IDs + previous_token_ids = tokenizer.encode( + param_dict["previous_text"] + ) if param_dict["previous_text"] is not None else [] + current_token_ids = tokenizer.encode(param_dict["current_text"]) + delta_token_ids = tokenizer.encode(param_dict["delta_text"]) + + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser( + parser_name)(tokenizer) + + response = parser.extract_reasoning_content_streaming( + previous_text=param_dict["previous_text"], + current_text=param_dict["current_text"], + delta_text=param_dict["delta_text"], + previous_token_ids=previous_token_ids, + current_token_ids=current_token_ids, + delta_token_ids=delta_token_ids, + ) + # Streaming currently expects at least one of reasoning content / content, + # so the response should return None in that case. + if param_dict["reasoning_content"] is None and param_dict[ + "content"] is None: + assert response is None + else: + assert isinstance(response, DeltaMessage) + assert param_dict["reasoning_content"] == response.reasoning_content + assert param_dict["content"] == response.content diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 75ac326aaa3d6..be00689f2b55f 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1099,7 +1099,7 @@ class EngineArgs: parser.add_argument( "--reasoning-parser", type=str, - choices=["deepseek_r1"], + choices=["deepseek_r1", "granite"], default=None, help= "Select the reasoning parser depending on the model that you're " diff --git a/vllm/entrypoints/openai/reasoning_parsers/__init__.py b/vllm/entrypoints/openai/reasoning_parsers/__init__.py index 80354d69b50af..45132a780e5b2 100644 --- a/vllm/entrypoints/openai/reasoning_parsers/__init__.py +++ b/vllm/entrypoints/openai/reasoning_parsers/__init__.py @@ -2,7 +2,11 @@ from .abs_reasoning_parsers import ReasoningParser, ReasoningParserManager from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser +from .granite_reasoning_parser import GraniteReasoningParser __all__ = [ - "ReasoningParser", "ReasoningParserManager", "DeepSeekR1ReasoningParser" + "ReasoningParser", + "ReasoningParserManager", + "DeepSeekR1ReasoningParser", + "GraniteReasoningParser", ] diff --git a/vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py b/vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py new file mode 100644 index 0000000000000..117d051a73782 --- /dev/null +++ b/vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py @@ -0,0 +1,363 @@ +# SPDX-License-Identifier: Apache-2.0 + +import re +from collections.abc import Sequence +from typing import Optional, Union + +from transformers import PreTrainedTokenizerBase + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + DeltaMessage) +from vllm.entrypoints.openai.reasoning_parsers.abs_reasoning_parsers import ( + ReasoningParser, ReasoningParserManager) +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +@ReasoningParserManager.register_module("granite") +class GraniteReasoningParser(ReasoningParser): + """ + Reasoning parser for IBM Granite. + + IBM granite models currently use "Here is my thought process:" + and "Here is my response:" to separate its thinking / response outputs. + """ + + def __init__(self, tokenizer: PreTrainedTokenizerBase): + super().__init__(tokenizer) + + # NOTE: There have been some observed occurrences of quantized + # instances of the current models using "Here's" instead of "Here is", + # so to be safe, we match on both. + self.think_start_expr = r"(?:Here's|Here is) my thought process:" + self.response_start_expr = r"(?:Here's|Here is) my response:" + + self.reasoning_regex = re.compile( + rf"{self.think_start_expr}(.*?){self.response_start_expr}(.*)", + re.DOTALL) + + self.valid_think_starts = [ + "Here's my thought process:", "Here is my thought process:" + ] + self.valid_response_starts = [ + "Here's my response:", "Here is my response:" + ] + + # Substrings to match for sequence boundaries on raw text + self.seq_boundary_end = ":" + self.seq_boundary_start = "Here" + + # The longest any thinking / start of response message can be + self.longest_think_start = max( + len(think_start) for think_start in self.valid_think_starts) + + def extract_reasoning_content( + self, model_output: str, request: ChatCompletionRequest + ) -> tuple[Optional[str], Optional[str]]: + """Extract the reasoning content & content sections, respectively. + If the sequence doesn't match what we expect, i.e., the model generates + something else, all content is considered non-reasoning content. + + Args: + model_output (str): Output of the model to be parsed. + request (ChatCompletionReqest): Request being processed. + + Returns: + tuple[Optional[str], Optional[str]]: Tuple pair containing the + reasoning content and non-reasoning content. + """ + re_match = self.reasoning_regex.findall(model_output) + if not re_match: + return None, model_output + reasoning_content, response_content = re_match[0] + if not response_content: + return reasoning_content, None + return reasoning_content, response_content + + def extract_reasoning_content_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> Union[DeltaMessage, None]: + """Extract the reasoning content / content emitted by granite models; + If the sequence doesn't match what we expect, i.e., the model generates + something else, all content is considered non-reasoning content. + + NOTE: Granite models do not use a special token to start their reasoning + and response sections; instead they have token sequences, e.g., + + Here is my thought process: Foo Here is my response: Bar + + This increases the complexity of correctly handling streams, since we + need to watch for specific sequences and correctly parse them without + dropping content that is potentially overlapping & spanning multiple + delta messages. + + Args: + previous_text (str): Previous text outside of this delta message. + current_text (str): Previous text + delta text. + delta_text (str): Text to consider and parse content from. + previous_token_ids (Sequence[int]): Token IDs of previous_text. + current_token_ids (Sequence[int]): Token IDs of current_text. + delta_token_ids (Sequence[int]): Token IDs of delta_text. + + Returns: + Union[DeltaMessage, None] + DeltaMessage with either reasoning content or content, or None. + """ + reasoning_content, resp_seq_len, content = self._get_content_sections( + current_text) + # Either we haven't finished the start of the reasoning sequence, + # or the model is generating something unexpected. + if not reasoning_content: + delta_message = self._get_delta_message_with_no_reasoning_bounds( + current_text, delta_text) + # We have a start of reasoning message, but have not yet finished + # the start of response sequence. + elif not content: + delta_message = self._get_delta_message_with_no_response_bounds( + current_text, reasoning_content, delta_text) + # We've finished both the start of reasoning and start of response seq. + else: + # This should never happen since we matched on the response + assert resp_seq_len is not None + delta_message = self._get_delta_message_with_both_bounds( + delta_text, reasoning_content, content, current_text, + resp_seq_len) + if not delta_message.content and not delta_message.reasoning_content: + return None + return delta_message + + #### Implementation details of stream parsing for granite models + def _is_reasoning_start_substr(self, text: str) -> bool: + """Check if a text matches one of the possible start reasoning seqs. + + Args: + text (str): Text to check for leading substr. + + Returns: + bool: True if any of the possible reasoning start seqs match. + """ + return any( + think_start.startswith(text) + for think_start in self.valid_think_starts) + + def _is_response_start_substr(self, text: str) -> bool: + """Check if a text matches one of the possible start response seqs. + + Args: + text (str): Text to check for leading substr. + + Returns: + bool: True if any of the possible response start seqs match. + """ + return any( + response_start.startswith(text) + for response_start in self.valid_response_starts) + + def _get_delta_message_with_no_reasoning_bounds( + self, + current_text: str, + delta_text: str, + ) -> DeltaMessage: + """Parse the delta message when the current text has not yet completed + its start of reasoning sequence. + + Args: + current_text (str): The full previous + delta text. + delta_text (str): Text to consider and parse content from. + + Returns: + DeltaMessage: Message containing the parsed content. + """ + prev_longest_length = len(current_text) - len(delta_text) + is_substr = self._is_reasoning_start_substr(current_text) + was_substr = self._is_reasoning_start_substr( + current_text[:prev_longest_length]) + + # Check if we just generated something NOT in the special token seq; + # if so, add everything that we previously skipped with this delta + # message and append everything to content in the future. + if was_substr and not is_substr: + return DeltaMessage( + reasoning_content=None, + content=current_text, + ) + if is_substr: + # Might still be in the special token sequence; return nothing + return DeltaMessage(reasoning_content=None, content=None) + # Otherwise the sequence has already been broken and we already + # corrected; just return the delta text as normal content. + return DeltaMessage(reasoning_content=None, content=delta_text) + + def _get_delta_message_with_no_response_bounds( + self, + current_text: str, + reasoning_content: str, + delta_text: str, + ) -> DeltaMessage: + """Parse the delta message when the current text has both reasoning + content with no (response) content. NOTE that we may have overlapping + tokens with the start of reasoning / start of response sequences on + either side of the delta text. + + Args: + current_text (str): The full previous + delta text. + reasoning_content (str): reasoning content from current_text. + delta_text (str): Text to consider and parse content from. + + Returns: + DeltaMessage: Message containing the parsed content. + """ + # If we have no reasoning content or explicitly end with the start of + # response sequence, we are in transition to the response; need to be + # careful here, since the final token (:) will match the reasoning + # content and fully parse it out; we should not pass the : back. + ends_with_start_response_seq = any( + current_text.endswith(response_start) + for response_start in self.valid_response_starts) + if reasoning_content is None or ends_with_start_response_seq: + return DeltaMessage(reasoning_content=None, content=None) + + # Consider previous / current text only within context of the reasoning + previous_text = reasoning_content[:-len(delta_text)] + current_text = reasoning_content + + # We need to be careful about adding unfinished response sequences; + # Find the place at which we MIGHT be starting a response sequence + prev_idx = previous_text.rfind(self.seq_boundary_start) + delta_idx = delta_text.rfind(self.seq_boundary_start) + + # Check the state of potential start of response substring matches. + prev_was_substr = self._is_response_start_substr( + previous_text[prev_idx:]) if prev_idx >= 0 else False + delta_continues_substr = self._is_response_start_substr( + current_text[prev_idx:]) if prev_idx >= 0 else False + delta_new_substr = self._is_response_start_substr( + delta_text[delta_idx:]) if delta_idx >= 0 else False + + # Delta only contains potential continued response sequence text. + if delta_continues_substr: + return DeltaMessage(reasoning_content=None, content=None) + + if not prev_was_substr: + # Delta may be starting a new response seq but has other text too. + if delta_new_substr: + return DeltaMessage(reasoning_content=delta_text[:delta_idx], + content=None) + # Normal case for most reasoning text (no potential special seqs). + return DeltaMessage(reasoning_content=delta_text, content=None) + # The substring that previously seemed to be a potential response + # seq wasn't one; we need to add the content to the delta message, + # and also slice off the potential response sequence + elif delta_new_substr: + reasoning_content = previous_text[ + prev_idx:] + delta_text[:delta_idx] + return DeltaMessage(reasoning_content=reasoning_content, + content=None) + # No new substring yet, and we broke our old one; take the whole delta + return DeltaMessage( + reasoning_content=previous_text[prev_idx:] + delta_text, + content=None, + ) + + def _get_delta_message_with_both_bounds( + self, + delta_text: str, + reasoning_content: str, + response_content: str, + current_text: str, + response_seq_len: int, + ) -> DeltaMessage: + """Parse the delta message when the current text has both reasoning + content and normal (response) content. + + Args: + delta_text (str): Text to consider and parse content from. + reasoning_content (str): reasoning content from current_text. + response_content (str): response content from current_text. + current_text (str): The full previous + delta text. + response_seq_len(str): Len of the complete response sequence used. + + Returns: + DeltaMessage: Message containing the parsed content. + """ + # Always have content; take length to the end + delta_content = delta_text[-len(response_content):] + reasoning_end_idx = len(delta_text) - (len(response_content) + + response_seq_len) + + if reasoning_end_idx < 0: + delta_reasoning_content = None + else: + # Get the starting offset + start_reasoning_content_idx = len( + reasoning_content) + response_seq_len + len( + response_content) - 1 + delta_offset = len(current_text) - len(delta_text) + start_offset = start_reasoning_content_idx - delta_offset + if start_offset < 0: + start_offset = 0 + delta_reasoning_content = delta_text[ + start_offset:reasoning_end_idx] + + return DeltaMessage( + reasoning_content=delta_reasoning_content, + content=delta_content, + ) + + def _get_content_sections( + self, current_text: str + ) -> tuple[Optional[str], Optional[int], Optional[str]]: + """Parse the text to extract the reasoning content / content + if we have them. + + Args: + current_text (str): The full previous + delta text. + + Returns: + tuple[Optional[str], Optional[int], Optional[str]]: Tuple of len 3 + containing the reasoning content, the length of the response seq + (if there is one) and the non-reasoning content. + """ + current_chunk_start = 0 + start_reasoning_content = None + parsed_content = False + delimiter_idxs = [ + idx for idx, char in enumerate(current_text) + if char == self.seq_boundary_end + ] + + for current_chunk_end in delimiter_idxs: + current_chunk = current_text[current_chunk_start:current_chunk_end] + # Check to see if the start of reasoning seq if complete + if start_reasoning_content is None: + for think_start in self.valid_think_starts: + if current_chunk == think_start[:-1]: + start_reasoning_content = current_chunk_end + 1 + current_chunk_start = current_chunk_end + 1 + break + + # Check to see if the start of response seq if complete + elif not parsed_content: + for response_start in self.valid_response_starts: + if current_chunk[-len(response_start) + + 1:] == response_start[:-1]: + # Mark end of reasoning and start response content + # after the start of response sequence. + end_reasoning_content = current_chunk_end - len( + response_start) + reasoning_content = current_text[ + start_reasoning_content:end_reasoning_content] + response_content = current_text[current_chunk_end + 1:] + return reasoning_content, len( + response_start), response_content + + if start_reasoning_content and not parsed_content: + return current_text[start_reasoning_content:], None, None + return None, None, None diff --git a/vllm/model_executor/guided_decoding/reasoner/__init__.py b/vllm/model_executor/guided_decoding/reasoner/__init__.py index d930d3dbe94c1..ab6e47c007d20 100644 --- a/vllm/model_executor/guided_decoding/reasoner/__init__.py +++ b/vllm/model_executor/guided_decoding/reasoner/__init__.py @@ -19,6 +19,10 @@ def get_reasoner(tokenizer: PreTrainedTokenizer, return None elif reasoning_backend == "deepseek_r1": return DeepSeekReasoner.from_tokenizer(tokenizer) + elif reasoning_backend == "granite": + logger.warning( + "Granite reasoner not yet implemented for structured outputs") + return None else: # Raise a warning for unknown reasoning backend and return None # We cannot raise an error here because some reasoning models From e64afa455c034007c8ec53fa9c18547c721cf362 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Wed, 26 Mar 2025 23:54:24 +0800 Subject: [PATCH 015/593] multi-node offline DP+EP example (#15484) Signed-off-by: youkaichao --- examples/offline_inference/data_parallel.py | 120 ++++++++++++++++---- 1 file changed, 97 insertions(+), 23 deletions(-) diff --git a/examples/offline_inference/data_parallel.py b/examples/offline_inference/data_parallel.py index b73770ce382cf..232afd8b73d00 100644 --- a/examples/offline_inference/data_parallel.py +++ b/examples/offline_inference/data_parallel.py @@ -1,26 +1,49 @@ # SPDX-License-Identifier: Apache-2.0 -# usage: -# VLLM_USE_V1=1 python examples/offline_inference/data_parallel.py -# we need to have a launcher to create multiple data parallel -# ranks. And each rank will create a vLLM instance to process its own prompts. +""" +Usage: +Single node: + python examples/offline_inference/data_parallel.py \ + --model="ibm-research/PowerMoE-3b" \ + --dp-size=2 \ + --tp-size=2 + +Multi-node: + Node 0 (assume the node has ip of 10.99.48.128): + python examples/offline_inference/data_parallel.py \ + --model="ibm-research/PowerMoE-3b" \ + --dp-size=2 \ + --tp-size=2 \ + --node-size=2 \ + --node-rank=0 \ + --master-addr=10.99.48.128 \ + --master-port=13345 + Node 1: + python examples/offline_inference/data_parallel.py \ + --model="ibm-research/PowerMoE-3b" \ + --dp-size=2 \ + --tp-size=2 \ + --node-size=2 \ + --node-rank=1 \ + --master-addr=10.99.48.128 \ + --master-port=13345 +""" import os from vllm import LLM, SamplingParams from vllm.utils import get_open_port -GPUs_per_dp_rank = 2 -DP_size = 2 - -def main(dp_size, dp_rank, dp_master_ip, dp_master_port, GPUs_per_dp_rank): - os.environ["VLLM_DP_RANK"] = str(dp_rank) +def main(model, dp_size, local_dp_rank, global_dp_rank, dp_master_ip, + dp_master_port, GPUs_per_dp_rank): + os.environ["VLLM_DP_RANK"] = str(global_dp_rank) os.environ["VLLM_DP_SIZE"] = str(dp_size) os.environ["VLLM_DP_MASTER_IP"] = dp_master_ip os.environ["VLLM_DP_MASTER_PORT"] = str(dp_master_port) # set devices for each dp_rank os.environ["CUDA_VISIBLE_DEVICES"] = ",".join( - str(i) for i in range(dp_rank * GPUs_per_dp_rank, (dp_rank + 1) * - GPUs_per_dp_rank)) + str(i) + for i in range(local_dp_rank * GPUs_per_dp_rank, (local_dp_rank + 1) * + GPUs_per_dp_rank)) # Sample prompts. prompts = [ @@ -28,20 +51,20 @@ def main(dp_size, dp_rank, dp_master_ip, dp_master_port, GPUs_per_dp_rank): "The president of the United States is", "The capital of France is", "The future of AI is", - ] + ] * 100 # with DP, each rank should process different prompts. # usually all the DP ranks process a full dataset, # and each rank processes a different part of the dataset. promts_per_rank = len(prompts) // dp_size - start = dp_rank * promts_per_rank + start = global_dp_rank * promts_per_rank end = start + promts_per_rank prompts = prompts[start:end] if len(prompts) == 0: # if any rank has no prompts to process, # we need to set a placeholder prompt prompts = ["Placeholder"] - print(f"DP rank {dp_rank} needs to process {len(prompts)} prompts") + print(f"DP rank {global_dp_rank} needs to process {len(prompts)} prompts") # Create a sampling params object. # since we are doing data parallel, every rank can have different @@ -49,31 +72,82 @@ def main(dp_size, dp_rank, dp_master_ip, dp_master_port, GPUs_per_dp_rank): # ranks for demonstration. sampling_params = SamplingParams(temperature=0.8, top_p=0.95, - max_tokens=16 * (dp_rank + 1)) + max_tokens=[16, 20][global_dp_rank % 2]) # Create an LLM. - llm = LLM(model="ibm-research/PowerMoE-3b", + llm = LLM(model=model, tensor_parallel_size=GPUs_per_dp_rank, enforce_eager=True, enable_expert_parallel=True) outputs = llm.generate(prompts, sampling_params) # Print the outputs. - for output in outputs: + for i, output in enumerate(outputs): + if i >= 5: + # print only 5 outputs + break prompt = output.prompt generated_text = output.outputs[0].text - print(f"DP rank {dp_rank}, Prompt: {prompt!r}, " + print(f"DP rank {global_dp_rank}, Prompt: {prompt!r}, " f"Generated text: {generated_text!r}") if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="Data Parallel Inference") + parser.add_argument("--model", + type=str, + default="ibm-research/PowerMoE-3b", + help="Model name or path") + parser.add_argument("--dp-size", + type=int, + default=2, + help="Data parallel size") + parser.add_argument("--tp-size", + type=int, + default=2, + help="Tensor parallel size") + parser.add_argument("--node-size", + type=int, + default=1, + help="Total number of nodes") + parser.add_argument("--node-rank", + type=int, + default=0, + help="Rank of the current node") + parser.add_argument("--master-addr", + type=str, + default="", + help="Master node IP address") + parser.add_argument("--master-port", + type=int, + default=0, + help="Master node port") + args = parser.parse_args() + + dp_size = args.dp_size + tp_size = args.tp_size + node_size = args.node_size + node_rank = args.node_rank + + if node_size == 1: + dp_master_ip = "127.0.0.1" + dp_master_port = get_open_port() + else: + dp_master_ip = args.master_addr + dp_master_port = args.master_port + + assert dp_size % node_size == 0, "dp_size should be divisible by node_size" + dp_per_node = dp_size // node_size + from multiprocessing import Process - dp_master_ip = "127.0.0.1" - dp_master_port = get_open_port() + procs = [] - for i in range(DP_size): + for local_dp_rank, global_dp_rank in enumerate( + range(node_rank * dp_per_node, (node_rank + 1) * dp_per_node)): proc = Process(target=main, - args=(DP_size, i, dp_master_ip, dp_master_port, - GPUs_per_dp_rank)) + args=(args.model, dp_size, local_dp_rank, + global_dp_rank, dp_master_ip, dp_master_port, + tp_size)) proc.start() procs.append(proc) exit_code = 0 From 0af4d764d6626251923aa61adcf16c9bce488454 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 26 Mar 2025 17:17:53 +0000 Subject: [PATCH 016/593] Fix weight loading for some models in Transformers backend (#15544) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- vllm/model_executor/models/transformers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/models/transformers.py b/vllm/model_executor/models/transformers.py index bdc390689104e..70daadf913798 100644 --- a/vllm/model_executor/models/transformers.py +++ b/vllm/model_executor/models/transformers.py @@ -345,9 +345,11 @@ class TransformersModel(nn.Module): params_dict = dict(self.named_parameters()) loaded_params = set[str]() for name, loaded_weight in weights: - # Necessary for some models which use remote code - if not name.startswith(prefix := self.model.base_model_prefix): - name = maybe_prefix(prefix, name) + # Use "model" instead of base_model_prefix because + # the base model attribute in vLLM is always `model` + if not name.startswith(prefix := "model."): + name = prefix + name + if is_pp_missing_parameter(name, self): continue if name in params_dict: From 733e7c9e95f5b066ac420b00701eef7ea164a79e Mon Sep 17 00:00:00 2001 From: Aaron Pham Date: Wed, 26 Mar 2025 13:51:56 -0400 Subject: [PATCH 017/593] [Refactor] Remove unnecessary backend parameter in structured output interface (#15317) Signed-off-by: Aaron Pham --- vllm/v1/structured_output/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 6c6a8a7bce3ec..218af43deb677 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations import multiprocessing -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Optional from vllm.config import VllmConfig @@ -57,13 +57,13 @@ class StructuredOutputManager: raise ValueError( f"Unsupported structured output backend: {backend_name}") - grammar: Future[StructuredOutputGrammar] = self.executor.submit( - self._async_create_grammar, request, self.backend) + grammar = self.executor.submit(self._async_create_grammar, request) request.structured_output_request.grammar = grammar # type: ignore[assignment] def _async_create_grammar( - self, request: Request, - backend: StructuredOutputBackend) -> StructuredOutputGrammar: + self, + request: Request, + ) -> StructuredOutputGrammar: key = request.structured_output_request.structured_output_key # type: ignore[union-attr] # Note that the request was validated in the engine core client, From 35fad35a485eac9195c510731ba4a9d297dfd963 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Wed, 26 Mar 2025 10:56:47 -0700 Subject: [PATCH 018/593] [V1][Sampler] Faster top-k only implementation (#15478) Signed-off-by: Nick Hill --- tests/v1/sample/test_topk_topp_sampler.py | 37 ++++++++++++++++ vllm/v1/sample/ops/topk_topp_sampler.py | 53 ++++++++++++++++++++--- vllm/v1/sample/sampler.py | 6 +++ 3 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 tests/v1/sample/test_topk_topp_sampler.py diff --git a/tests/v1/sample/test_topk_topp_sampler.py b/tests/v1/sample/test_topk_topp_sampler.py new file mode 100644 index 0000000000000..8a5076412cfae --- /dev/null +++ b/tests/v1/sample/test_topk_topp_sampler.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +import torch +from torch import Generator + +from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p + +DEVICE = "cuda" + +BATCH_SIZE = 1024 +VOCAB_SIZE = 128 * 1024 + + +def test_topk_impl_equivalance(): + + with torch.device(DEVICE): + generator = Generator(device=DEVICE).manual_seed(33) + + logits = torch.rand((BATCH_SIZE, VOCAB_SIZE), generator=generator) + + # Random top-k values between 1 and 9. + k = torch.randint(1, 10, (BATCH_SIZE, ), generator=generator) + + # Set k=vocab_size for ~50% of requests in the batch (top-k disabled). + k.masked_fill_( + torch.randint(0, + 2, (BATCH_SIZE, ), + generator=generator, + dtype=bool), VOCAB_SIZE) + + # Top-k only implementation + result1 = apply_top_k_top_p(logits=logits.clone(), k=k, p=None) + + # Top-p + top-k + no_op_top_p = torch.tensor([1.0]) + result2 = apply_top_k_top_p(logits=logits.clone(), k=k, p=no_op_top_p) + + assert torch.allclose(result1, result2) diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index 1dea711874bfd..5dfcae08b170c 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -19,6 +19,12 @@ except ImportError: class TopKTopPSampler(nn.Module): + """ + Module that performs optional top-k and top-p filtering followed by + weighted random sampling of logits. + + Implementations may update the logits tensor in-place. + """ def __init__(self): super().__init__() @@ -84,7 +90,11 @@ class TopKTopPSampler(nn.Module): k: Optional[torch.Tensor], p: Optional[torch.Tensor], ) -> torch.Tensor: - """PyTorch-native implementation of top-k and top-p sampling.""" + """ + PyTorch-native implementation of top-k and top-p sampling. + + The logits tensor may be updated in-place. + """ logits = apply_top_k_top_p(logits, k, p) probs = logits.softmax(dim=-1, dtype=torch.float32) return random_sample(probs, generators) @@ -136,10 +146,18 @@ def apply_top_k_top_p( ) -> torch.Tensor: """Apply top-k and top-p masks to the logits. - This function sorts the logits tensor, which can be slow for large batches. + If a top-p is used, this function will sort the logits tensor, + which can be slow for large batches. + + The logits tensor may be updated in-place. """ - if k is None and p is None: - return logits + if p is None: + if k is None: + return logits + + # Avoid sorting vocab for top-k only case. + return apply_top_k_only(logits, k) + logits_sort, logits_idx = logits.sort(dim=-1, descending=False) if k is not None: @@ -153,7 +171,7 @@ def apply_top_k_top_p( if p is not None: # Apply top-p. probs_sort = logits_sort.softmax(dim=-1) - probs_sum = probs_sort.cumsum(dim=-1) + probs_sum = torch.cumsum(probs_sort, dim=-1, out=probs_sort) top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1) # at least one top_p_mask[:, -1] = False @@ -164,6 +182,31 @@ def apply_top_k_top_p( return logits +def apply_top_k_only( + logits: torch.Tensor, + k: torch.Tensor, +) -> torch.Tensor: + """ + Apply top-k mask to the logits. + + This implementation doesn't involve sorting the entire vocab. + + The logits tensor may be updated in-place. + """ + no_top_k_mask = k == logits.shape[1] + # Set non-top-k rows to 1 so that we can gather. + k = k.masked_fill(no_top_k_mask, 1) + max_top_k = k.max() + # topk.values tensor has shape [batch_size, max_top_k]. + # Convert top k to 0-based index in range [0, max_top_k). + k_index = k.sub_(1).unsqueeze(1) + top_k_mask = logits.topk(max_top_k, dim=1).values.gather(1, k_index) + # Handle non-topk rows. + top_k_mask.masked_fill_(no_top_k_mask.unsqueeze(1), -float("inf")) + logits.masked_fill_(logits < top_k_mask, -float("inf")) + return logits + + def random_sample( probs: torch.Tensor, generators: dict[int, torch.Generator], diff --git a/vllm/v1/sample/sampler.py b/vllm/v1/sample/sampler.py index 397a049dc2543..004f98496b0d7 100644 --- a/vllm/v1/sample/sampler.py +++ b/vllm/v1/sample/sampler.py @@ -87,6 +87,12 @@ class Sampler(nn.Module): logits: torch.Tensor, sampling_metadata: SamplingMetadata, ) -> torch.Tensor: + """Sample logits based on sampling metadata. + + The various logits processing functions called in this method + may update the logits tensor in-place. + """ + assert not (sampling_metadata.all_greedy and sampling_metadata.all_random) if sampling_metadata.all_random: From 27df5199d99627e1eb101071c2155f888181bd64 Mon Sep 17 00:00:00 2001 From: marko <5467316+dr75@users.noreply.github.com> Date: Wed, 26 Mar 2025 19:11:28 +0100 Subject: [PATCH 019/593] Support SHA256 as hash function in prefix caching (#15297) Signed-off-by: Marko Rosenmueller <5467316+dr75@users.noreply.github.com> --- docs/source/design/v1/prefix_caching.md | 7 ++- tests/test_utils.py | 23 +++++++- tests/v1/core/test_kv_cache_utils.py | 42 +++++++++----- tests/v1/core/test_prefix_caching.py | 22 ++++++-- tests/v1/engine/test_engine_args.py | 20 +++++++ vllm/config.py | 9 +++ vllm/engine/arg_utils.py | 38 +++++++++++-- vllm/utils.py | 20 +++++++ vllm/v1/core/block_pool.py | 20 ++++--- vllm/v1/core/kv_cache_manager.py | 8 ++- vllm/v1/core/kv_cache_utils.py | 75 ++++++++++++++----------- vllm/v1/core/sched/scheduler.py | 1 + 12 files changed, 214 insertions(+), 71 deletions(-) diff --git a/docs/source/design/v1/prefix_caching.md b/docs/source/design/v1/prefix_caching.md index 3d14a76840d45..ec1f3cb8d64a8 100644 --- a/docs/source/design/v1/prefix_caching.md +++ b/docs/source/design/v1/prefix_caching.md @@ -15,12 +15,13 @@ Block 3: |<------------------ prefix -------------------->| |<--- block tokens - In the example above, the KV cache in the first block can be uniquely identified with the token “A gentle breeze stirred”. The third block can be uniquely identified with the tokens in the block “laughed in the distance”, along with the prefix tokens “A gentle breeze stirred the leaves as children”. Therefore, we can build the block hash of `hash(tuple[components])`, where components are: * Parent hash value: The hash value of the parent hash block. -* Block tokens: A tuple of tokens in this block. The reason to include the exact tokens is to reduce potential hash value collision. +* Block tokens: A tuple of tokens in this block. The reason to include the exact tokens is to reduce potential hash value collision. * Extra hashes: Other values required to make this block unique, such as LoRA IDs and multi-modality input hashes (see the example below). -Note 1: We only cache full blocks. +> **Note 1:** We only cache full blocks. -Note 2: The above hash key structure is not 100% collision free. Theoretically it’s still possible for the different prefix tokens to have the same hash value, but this should be nearly impossible to happen. Of course, contributions are welcome if you have an awesome idea to eliminate collusion entirely. +> **Note 2:** The above hash key structure is not 100% collision free. Theoretically it’s still possible for the different prefix tokens to have the same hash value. To avoid any hash collisions **in a multi-tenant setup, we advise to use SHA256** as hash function instead of the default builtin hash. +SHA256 is supported since vLLM v0.8.3 and must be enabled with a command line argument. It comes with a performance impact of about 100-200ns per token (~6ms for 50k tokens of context). **A hashing example with multi-modality inputs** In this example, we illustrate how prefix caching works with multi-modality inputs (e.g., images). Assuming we have a request with the following messages: diff --git a/tests/test_utils.py b/tests/test_utils.py index 3660cfa0e49e2..ccbbffcabfcda 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,6 +2,8 @@ # ruff: noqa import asyncio +import hashlib +import pickle import socket from collections.abc import AsyncIterator from unittest.mock import patch @@ -14,7 +16,8 @@ from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config from vllm.utils import (FlexibleArgumentParser, MemorySnapshot, PlaceholderModule, StoreBoolean, bind_kv_cache, deprecate_kwargs, get_open_port, memory_profiling, - merge_async_iterators, supports_kw, swap_dict_values) + merge_async_iterators, sha256, supports_kw, + swap_dict_values) from .utils import create_new_process_for_each_test, error_on_warning @@ -476,3 +479,21 @@ def test_swap_dict_values(obj, key1, key2): assert obj[key1] == original_obj[key2] else: assert key1 not in obj + +@pytest.mark.parametrize("input", [(), ("abc", ), (None, ), + (None, bool, [1, 2, 3])]) +@pytest.mark.parametrize("output", [0, 1, 2]) +def test_sha256(input: tuple, output: int): + hash = sha256(input) + assert hash is not None + assert isinstance(hash, int) + assert hash != 0 + + bytes = pickle.dumps(input, protocol=pickle.HIGHEST_PROTOCOL) + assert hash == int.from_bytes(hashlib.sha256(bytes).digest(), byteorder="big") + + # hashing again, returns the same value + assert hash == sha256(input) + + # hashing different input, returns different value + assert hash != sha256(input + (1, )) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 3fecb517c4369..8362af24a67ed 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -5,8 +5,12 @@ import torch from vllm.multimodal.inputs import MultiModalKwargs from vllm.sampling_params import SamplingParams -from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue, - KVCacheBlock, PrefixCachingMetrics, +from vllm.utils import sha256 +# disable yapf here as it formats differently than isort such that both fail +# yapf: disable +from vllm.v1.core.kv_cache_utils import (NONE_HASH, BlockHashType, + FreeKVCacheBlockQueue, KVCacheBlock, + PrefixCachingMetrics, generate_block_hash_extra_keys, hash_block_tokens, hash_request_tokens, @@ -16,6 +20,8 @@ from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig, from vllm.v1.metrics.stats import PrefixCacheStats from vllm.v1.request import Request +# yapf: enable + def make_request(request_id, prompt_token_ids, @@ -40,6 +46,12 @@ def make_request(request_id, ) +def test_none_hash(): + assert NONE_HASH is not None + assert isinstance(NONE_HASH, int) + assert NONE_HASH != 0 + + def test_kv_cache_block(): # Test KVCacheBlock initialization block = KVCacheBlock(block_id=0) @@ -190,21 +202,23 @@ def test_generate_block_hash_extra_keys_no_mm_inputs(): assert next_mm_idx == 0 -def test_hash_block_tokens(): +@pytest.mark.parametrize("hash_fn", [sha256, hash]) +def test_hash_block_tokens(hash_fn): parent_block_hash = 123 curr_block_token_ids = (1, 2, 3) extra_keys = ("key1", "key2") - block_hash = hash_block_tokens(parent_block_hash, curr_block_token_ids, - extra_keys) + block_hash = hash_block_tokens(hash_fn, parent_block_hash, + curr_block_token_ids, extra_keys) assert isinstance(block_hash, BlockHashType) - assert block_hash.hash_value == hash( + assert block_hash.hash_value == hash_fn( (parent_block_hash, curr_block_token_ids, extra_keys)) assert block_hash.token_ids == curr_block_token_ids assert block_hash.extra_keys == extra_keys -def test_hash_request_tokens(): +@pytest.mark.parametrize("hash_fn", [sha256, hash]) +def test_hash_request_tokens(hash_fn): request = make_request( request_id=0, prompt_token_ids=[_ for _ in range(6)], @@ -219,7 +233,7 @@ def test_hash_request_tokens(): ) block_size = 3 - block_hashes = hash_request_tokens(block_size, request) + block_hashes = hash_request_tokens(hash_fn, block_size, request) assert len(block_hashes) == 2 assert isinstance(block_hashes[0], BlockHashType) @@ -234,7 +248,8 @@ def test_hash_request_tokens(): assert block_hashes[1].extra_keys == ("hash2", ) -def test_hash_tokens_different_mm_input(): +@pytest.mark.parametrize("hash_fn", [sha256, hash]) +def test_hash_tokens_different_mm_input(hash_fn): request1 = make_request( request_id=0, prompt_token_ids=[_ for _ in range(6)], @@ -260,13 +275,14 @@ def test_hash_tokens_different_mm_input(): mm_hashes=["hash3", "hash2"], ) block_size = 3 - block_hashes1 = hash_request_tokens(block_size, request1) - block_hashes2 = hash_request_tokens(block_size, request2) + block_hashes1 = hash_request_tokens(hash_fn, block_size, request1) + block_hashes2 = hash_request_tokens(hash_fn, block_size, request2) assert block_hashes1[0] != block_hashes2[0] assert block_hashes1[1] != block_hashes2[1] -def test_hash_request_tokens_no_mm_inputs(): +@pytest.mark.parametrize("hash_fn", [sha256, hash]) +def test_hash_request_tokens_no_mm_inputs(hash_fn): request = make_request( request_id=0, prompt_token_ids=[_ for _ in range(6)], @@ -275,7 +291,7 @@ def test_hash_request_tokens_no_mm_inputs(): ) block_size = 3 - block_hashes = hash_request_tokens(block_size, request) + block_hashes = hash_request_tokens(hash_fn, block_size, request) assert len(block_hashes) == 2 assert block_hashes[0].token_ids == (0, 1, 2) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 6129752bcdd65..72a1874fbd446 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -7,7 +7,7 @@ import pytest from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange from vllm.sampling_params import SamplingParams -from vllm.utils import cdiv +from vllm.utils import cdiv, sha256 from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_manager import KVCacheManager, Request from vllm.v1.core.kv_cache_utils import (BlockHashType, KVCacheBlock, @@ -39,16 +39,21 @@ def make_request(request_id, ) -def test_prefill(): +@pytest.mark.parametrize("hash_algo", ["sha256", "hash"]) +def test_prefill(hash_algo): manager = KVCacheManager( block_size=16, num_gpu_blocks=10, max_model_len=8192, sliding_window=None, enable_caching=True, + caching_hash_algo=hash_algo, num_preallocate_tokens=16, ) + # choose the hash function according to the parameter + hash_fn = sha256 if hash_algo == "sha256" else hash + # Complete 3 blocks (48 tokens) common_token_ids = [i for i in range(3) for _ in range(16)] @@ -68,7 +73,8 @@ def test_prefill(): parent_block_hash = None for block_id in (0, 1, 2): block_tokens = tuple(all_token_ids[block_id * 16:(block_id + 1) * 16]) - block_hash = hash_block_tokens(parent_block_hash, block_tokens) + block_hash = hash_block_tokens(hash_fn, parent_block_hash, + block_tokens) assert manager.block_pool.blocks[block_id].block_hash == block_hash assert manager.block_pool.blocks[block_id].ref_cnt == 1 parent_block_hash = block_hash.hash_value @@ -163,6 +169,8 @@ def test_prefill_plp(): enable_caching=True, num_preallocate_tokens=16, ) + # the default hash function is hash + hash_fn = hash # Complete 3 blocks (48 tokens) common_token_ids = [i for i in range(3) for _ in range(16)] @@ -185,7 +193,8 @@ def test_prefill_plp(): parent_block_hash = None for block_id in (0, 1, 2): block_tokens = tuple(all_token_ids[block_id * 16:(block_id + 1) * 16]) - block_hash = hash_block_tokens(parent_block_hash, block_tokens) + block_hash = hash_block_tokens(hash_fn, parent_block_hash, + block_tokens) assert manager.block_pool.blocks[block_id].block_hash == block_hash assert manager.block_pool.blocks[block_id].ref_cnt == 1 parent_block_hash = block_hash.hash_value @@ -522,7 +531,8 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int): assert len(blocks) == 1 + num_preallocated_blocks -def test_cache_blocks(): +@pytest.mark.parametrize("hash_fn", [sha256, hash]) +def test_cache_blocks(hash_fn): """ This is a unit test that tests the correctness of the _cache_full_blocks function of KVCacheManager. @@ -550,6 +560,7 @@ def test_cache_blocks(): num_cached_blocks=0, num_full_blocks=2, block_size=block_size, + hash_fn=hash_fn, ) assert len(block_pool.cached_block_hash_to_block) == 2 @@ -564,6 +575,7 @@ def test_cache_blocks(): num_cached_blocks=2, num_full_blocks=3, block_size=block_size, + hash_fn=hash_fn, ) assert len(block_pool.cached_block_hash_to_block) == 3 assert blocks[0].block_hash is not None diff --git a/tests/v1/engine/test_engine_args.py b/tests/v1/engine/test_engine_args.py index 02470ca92f47f..8963b21c4eb11 100644 --- a/tests/v1/engine/test_engine_args.py +++ b/tests/v1/engine/test_engine_args.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 +from argparse import ArgumentError + import pytest from vllm import envs @@ -32,6 +34,24 @@ def test_prefix_caching_from_cli(): vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() assert vllm_config.cache_config.enable_prefix_caching + # default hash algorithm is "builtin" + assert vllm_config.cache_config.prefix_caching_hash_algo == "builtin" + + # set hash algorithm to sha256 + args = parser.parse_args(["--prefix-caching-hash-algo", "sha256"]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.cache_config.prefix_caching_hash_algo == "sha256" + + # set hash algorithm to builtin + args = parser.parse_args(["--prefix-caching-hash-algo", "builtin"]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.cache_config.prefix_caching_hash_algo == "builtin" + + # an invalid hash algorithm raises an error + parser.exit_on_error = False + with pytest.raises(ArgumentError): + args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"]) + def test_defaults_with_usage_context(): engine_args = EngineArgs(model="facebook/opt-125m") diff --git a/vllm/config.py b/vllm/config.py index 6f2da6aa87136..94cecba1e1fcb 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -1124,6 +1124,7 @@ class CacheConfig: num_gpu_blocks_override: Optional[int] = None, sliding_window: Optional[int] = None, enable_prefix_caching: bool = False, + prefix_caching_hash_algo: str = "builtin", cpu_offload_gb: float = 0, calculate_kv_scales: Optional[bool] = None, ) -> None: @@ -1135,6 +1136,7 @@ class CacheConfig: self.is_attention_free = is_attention_free self.sliding_window = sliding_window self.enable_prefix_caching = enable_prefix_caching + self.prefix_caching_hash_algo = prefix_caching_hash_algo self.cpu_offload_gb = cpu_offload_gb self.calculate_kv_scales = calculate_kv_scales self._verify_args() @@ -1185,6 +1187,13 @@ class CacheConfig: "Prefix caching is not supported with sliding window. " "Run with --disable-sliding-window to use prefix caching.") + if self.enable_prefix_caching and self.prefix_caching_hash_algo not in ( + "builtin", "sha256"): + raise ValueError( + "Unknown prefix caching hash algorithm: " + f"{self.prefix_caching_hash_algo}. Must be either " + "'builtin' or 'sha256'.") + def verify_with_parallel_config( self, parallel_config: "ParallelConfig", diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index be00689f2b55f..364555b345834 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -118,6 +118,7 @@ class EngineArgs: max_parallel_loading_workers: Optional[int] = None block_size: Optional[int] = None enable_prefix_caching: Optional[bool] = None + prefix_caching_hash_algo: str = "builtin" disable_sliding_window: bool = False disable_cascade_attn: bool = False use_v2_block_manager: bool = True @@ -475,6 +476,16 @@ class EngineArgs: help="Enables automatic prefix caching. " "Use ``--no-enable-prefix-caching`` to disable explicitly.", ) + parser.add_argument( + "--prefix-caching-hash-algo", + type=str, + choices=["builtin", "sha256"], + default=EngineArgs.prefix_caching_hash_algo, + help="Set the hash algorithm for prefix caching. " + "Options are 'builtin' (Python's built-in hash) or 'sha256' " + "(collision resistant but with certain overheads). Defaults " + "to 'builtin'.", + ) parser.add_argument('--disable-sliding-window', action='store_true', help='Disables sliding window, ' @@ -1329,6 +1340,7 @@ class EngineArgs: num_gpu_blocks_override=self.num_gpu_blocks_override, sliding_window=model_config.get_sliding_window(), enable_prefix_caching=self.enable_prefix_caching, + prefix_caching_hash_algo=self.prefix_caching_hash_algo, cpu_offload_gb=self.cpu_offload_gb, calculate_kv_scales=self.calculate_kv_scales, ) @@ -1737,12 +1749,22 @@ class EngineArgs: msg = "Chunked prefill is not supported for pooling models" raise ValueError(msg) - # Disable prefix caching for multimodal models for VLLM_V0. - if (model_config.is_multimodal_model and self.enable_prefix_caching): - logger.warning( - "--enable-prefix-caching is not supported for multimodal " - "models in V0 and has been disabled.") - self.enable_prefix_caching = False + # if using prefix caching, we must set a hash algo + if self.enable_prefix_caching: + # Disable prefix caching for multimodal models for VLLM_V0. + if model_config.is_multimodal_model: + logger.warning( + "--enable-prefix-caching is not supported for multimodal " + "models in V0 and has been disabled.") + self.enable_prefix_caching = False + + # VLLM_V0 only supports builtin hash algo for prefix caching. + if self.prefix_caching_hash_algo is None: + self.prefix_caching_hash_algo = "builtin" + elif self.prefix_caching_hash_algo == "sha256": + raise ValueError( + "sha256 is not supported for prefix caching in V0 engine. " + "Please use 'builtin'.") # Set max_num_seqs to 256 for VLLM_V0. if self.max_num_seqs is None: @@ -1758,6 +1780,10 @@ class EngineArgs: if self.enable_prefix_caching is None: self.enable_prefix_caching = True + # if using prefix caching, we must set a hash algo + if self.enable_prefix_caching and self.prefix_caching_hash_algo is None: + self.prefix_caching_hash_algo = "builtin" + # V1 should use the new scheduler by default. # Swap it only if this arg is set to the original V0 default if self.scheduler_cls == EngineArgs.scheduler_cls: diff --git a/vllm/utils.py b/vllm/utils.py index 9e14a628993f6..101342333e66b 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -10,6 +10,7 @@ import datetime import enum import gc import getpass +import hashlib import importlib import importlib.metadata import importlib.util @@ -17,6 +18,7 @@ import inspect import ipaddress import multiprocessing import os +import pickle import re import signal import socket @@ -2442,3 +2444,21 @@ def cprofile(save_file: Optional[str] = None, enabled: bool = True): return wrapper return decorator + + +def sha256(input) -> int: + """Hash any picklable Python object using SHA-256. + + The input is serialized using pickle before hashing, which allows + arbitrary Python objects to be used. Note that this function does + not use a hash seed—if you need one, prepend it explicitly to the input. + + Args: + input: Any picklable Python object. + + Returns: + An integer representing the SHA-256 hash of the serialized input. + """ + input_bytes = pickle.dumps(input, protocol=pickle.HIGHEST_PROTOCOL) + return int.from_bytes(hashlib.sha256(input_bytes).digest(), + byteorder="big") diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index 394b47fddf0c9..79b0c42d4f812 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 from collections import defaultdict from collections.abc import Iterable -from typing import Optional +from typing import Callable, Optional from vllm.logger import init_logger from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue, @@ -15,10 +15,10 @@ logger = init_logger(__name__) class BlockPool: """BlockPool that manages KVCacheBlocks. - It provides methods to allocate, free and cache the kv cache blocks. The - free_block_queue stores the free blocks in eviction order to enable - allocation, free, and cache eviction. The cached_block_hash_to_block - maps between block hash and cached block to support finding cached blocks + It provides methods to allocate, free and cache the kv cache blocks. The + free_block_queue stores the free blocks in eviction order to enable + allocation, free, and cache eviction. The cached_block_hash_to_block + maps between block hash and cached block to support finding cached blocks by their block hash. Args: @@ -75,11 +75,12 @@ class BlockPool: num_cached_blocks: int, num_full_blocks: int, block_size: int, + hash_fn: Callable, ) -> None: """Cache a list of full blocks for prefix caching. This function takes a list of blocks that will have their block hash metadata to be updated and cached. Given a request, it computes the - block hashes for the blocks starting from `num_cached_blocks` to + block hashes for the blocks starting from `num_cached_blocks` to `num_full_blocks`, updating the metadata for each block and caching them in the `cached_block_hash_to_block`. @@ -87,12 +88,13 @@ class BlockPool: request: The request to cache the blocks. blocks: All blocks in the request. block_hashes: Block hashes of the blocks in the request. Note that - this list may be shorter than the blocks list. In this case the + this list may be shorter than the blocks list. In this case the missed block hash will be computed in this function. num_cached_blocks: The number of blocks that are already cached. - num_full_blocks: The number of blocks that are full and should + num_full_blocks: The number of blocks that are full and should be cached after this function. block_size: Number of tokens in each block. + hash_fn: The hash function to use for block hashes. """ if num_cached_blocks == num_full_blocks: return @@ -138,7 +140,7 @@ class BlockPool: request, start_token_idx, end_token_idx, -1) # Compute the hash of the current block. - block_hash = hash_block_tokens(prev_block_hash_value, + block_hash = hash_block_tokens(hash_fn, prev_block_hash_value, block_tokens, extra_keys) block_hashes.append(block_hash) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 5cfe2b96865a2..39390babaa8ef 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from typing import Optional from vllm.logger import init_logger -from vllm.utils import cdiv +from vllm.utils import cdiv, sha256 from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_utils import (BlockHashType, KVCacheBlock, hash_request_tokens) @@ -24,6 +24,7 @@ class KVCacheManager: max_model_len: int, sliding_window: Optional[int] = None, enable_caching: bool = True, + caching_hash_algo: str = "builtin", num_preallocate_tokens: int = 64, log_stats: bool = False, ) -> None: @@ -33,6 +34,7 @@ class KVCacheManager: self.max_num_blocks_per_req = cdiv(max_model_len, block_size) self.sliding_window = sliding_window self.enable_caching = enable_caching + self.caching_hash_fn = sha256 if caching_hash_algo == "sha256" else hash # FIXME: make prefix cache stats conditional on log_stats self.log_stats = log_stats # NOTE(woosuk): To avoid frequent block allocation, we preallocate some @@ -109,7 +111,8 @@ class KVCacheManager: # if the scheduler has tried to schedule the request before. block_hashes = self.req_to_block_hashes[request.request_id] if not block_hashes: - block_hashes = hash_request_tokens(self.block_size, request) + block_hashes = hash_request_tokens(self.caching_hash_fn, + self.block_size, request) self.req_to_block_hashes[request.request_id] = block_hashes self.prefix_cache_stats.requests += 1 @@ -247,6 +250,7 @@ class KVCacheManager: num_cached_blocks=num_cached_blocks, num_full_blocks=num_full_blocks_after_append, block_size=self.block_size, + hash_fn=self.caching_hash_fn, ) self.num_cached_block[ diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index e0d7f4dbdc1c1..0d58d4d2218f4 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -1,12 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 """KV-Cache Utilities.""" +import os from collections import deque from collections.abc import Sequence from dataclasses import dataclass -from typing import Any, NamedTuple, Optional +from typing import Any, Callable, NamedTuple, Optional from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.utils import sha256 from vllm.v1.kv_cache_interface import (KVCacheConfig, KVCacheGroupSpec, KVCacheSpec, KVCacheTensor) from vllm.v1.metrics.stats import PrefixCacheStats @@ -18,9 +20,8 @@ logger = init_logger(__name__) class BlockHashType(NamedTuple): """Hash value of a block (int), the token IDs in the block, and extra keys. We keep a tuple of token IDs and extra keys to reduce the likelihood of - hash collisions when the hash value is the same. But please note that - hash collisions can still theoretically occur, albeit with an extremely - low probability. + hash collisions when the hash value is the same. By using SHA256 however, + hash collisions are practically impossible. """ # Hash value of the block in an integer. hash_value: int @@ -30,6 +31,20 @@ class BlockHashType(NamedTuple): extra_keys: Optional[Any] = None +# The hash seed for the first block of the prefix block sequence. +# +# Even if the hash function is the builtin hash(), we use sha256 to generate +# the initial hash to simplify the code. This is not performance critical +# as it is done one per process. +# +# We use a random value to avoid hash collisions or PYTHONHASHSEED environment +# variable if set such that processes can share the seed if needed. +# This aligns with the behavior of Python's hash() function, which also uses +# a random seed if PYTHONHASHSEED is not set. +NONE_HASH = int.from_bytes(os.urandom(32), byteorder="big") if os.getenv( + 'PYTHONHASHSEED') is not None else sha256(os.getenv('PYTHONHASHSEED')) + + class PrefixCachingMetrics: """Metrics for prefix caching with a hit rate of the most recent N requests. @@ -148,7 +163,7 @@ class FreeKVCacheBlockQueue: builtin deque to support removing a block in the middle of the queue in O(1) time. To close the performance gap to the builtin deque which is implemented in C++, this class does not allocate any Python objects when - manipulating the linked list. Instead, this class manipulates the + manipulating the linked list. Instead, this class manipulates the prev_free_block and next_free_block attributes of the given blocks. The queue is ordered by block ID in the beginning. When a block is allocated @@ -178,7 +193,7 @@ class FreeKVCacheBlockQueue: def popleft(self) -> KVCacheBlock: """Pop the first free block and reduce num_free_blocks by 1. - + Returns: The first free block. """ @@ -191,7 +206,7 @@ class FreeKVCacheBlockQueue: def remove(self, block: KVCacheBlock) -> None: """Remove a block in the free list and reduce num_free_blocks by 1. - + Args: block: The block to remove. """ @@ -235,7 +250,7 @@ class FreeKVCacheBlockQueue: def get_all_free_blocks(self) -> list[KVCacheBlock]: """Get all free blocks in the free list. Mainly used for testing. - + Returns: A list of free blocks. """ @@ -251,10 +266,10 @@ def need_extra_keys(request: Request) -> bool: """Check whether the blocks allocated to this request need extra hash keys. Args: - request (Request): The request. + request (Request): The request. Returns: - bool: Whether blocks allocated to this request need extra hash keys. + bool: Whether blocks allocated to this request need extra hash keys. """ # Multimodal requests need to include the MM hash. @@ -269,13 +284,13 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int, computation. For multi-modal inputs, the extra keys are (mm_hash, start_offset) that indicate a mm input contained in the block and its starting offset in the block tokens. - + Args: request: The request object. start_token_idx: The start token index of the block. end_token_idx: The end token index of the block. start_mm_idx: The start multi-modal index of the block. - + Returns: A tuple of extra keys and the next multi-modal index. """ @@ -333,10 +348,10 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int, def _gen_lora_extra_hash_keys(request: Request) -> list[int]: """Generate extra keys related to LoRA for block hash computation. - + Args: request: The request object. - + Returns: Return LoRA id of the request if it is a LoRA request. Return empty list otherwise. @@ -351,13 +366,13 @@ def generate_block_hash_extra_keys( start_mm_idx: int) -> tuple[Optional[tuple[Any, ...]], int]: """Generate extra keys for the block hash. The extra keys can come from the multi-modal inputs and request specific metadata (e.g., LoRA ID). - + Args: request: The request object. start_token_idx: The start token index of the block. end_token_idx: The end token index of the block. start_mm_idx: The start multi-modal index of the block. - + Returns: A tuple of extra keys and the next multi-modal index. """ @@ -375,6 +390,7 @@ def generate_block_hash_extra_keys( def hash_block_tokens( + hash_function: Callable, parent_block_hash: Optional[int], curr_block_token_ids: Sequence[int], extra_keys: Optional[tuple[Any, ...]] = None) -> BlockHashType: @@ -395,21 +411,16 @@ def hash_block_tokens( The entire tuple is used as the hash key of the block. """ if not parent_block_hash: - # Note that we use 'None' as a string here instead of None because - # as of Python 3.12, hash(None) returns a constant predictable value. - # This could possibly make it easier to find and exploit hash - # collisions. 'None' as a string will be hashed differently per process, - # but consistently within the same process. This is the same as the - # behavior of None prior to Python 3.12. - parent_block_hash = hash('None') + parent_block_hash = NONE_HASH curr_block_token_ids_tuple = tuple(curr_block_token_ids) return BlockHashType( - hash((parent_block_hash, curr_block_token_ids_tuple, extra_keys)), + hash_function( + (parent_block_hash, curr_block_token_ids_tuple, extra_keys)), curr_block_token_ids_tuple, extra_keys) -def hash_request_tokens(block_size: int, +def hash_request_tokens(hash_function: Any, block_size: int, request: Request) -> list[BlockHashType]: """Computes hash values of a chain of blocks given a sequence of token IDs. The hash value is used for prefix caching. @@ -441,7 +452,7 @@ def hash_request_tokens(block_size: int, req_extra_keys, curr_mm_idx = generate_block_hash_extra_keys( request, start, end, curr_mm_idx) - block_hash = hash_block_tokens(parent_block_hash_value, + block_hash = hash_block_tokens(hash_function, parent_block_hash_value, block_token_ids, req_extra_keys) ret.append(block_hash) parent_block_hash_value = block_hash.hash_value @@ -452,7 +463,7 @@ def check_enough_kv_cache_memory(vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec], available_memory: int): """ - Checks whether `available_memory` is enough for the KV cache to hold at + Checks whether `available_memory` is enough for the KV cache to hold at least one request with the model's max_model_len. Args: @@ -489,15 +500,15 @@ def create_kv_cache_group_specs( grouped_layer_names: list[list[str]]) -> list[KVCacheGroupSpec]: """ Create KVCacheGroupSpec object for each kv cache group layer. - The layers in the same group should share the same + The layers in the same group should share the same KVCacheSpec. Args: kv_cache_spec: A mapping from each layer name to its corresponding KVCacheSpec. grouped_layer_names: - A list of kv cache groups, where each element is a list of layer - names that belong to the same group and should share the same + A list of kv cache groups, where each element is a list of layer + names that belong to the same group and should share the same KVCacheSpec. Returns: A list of KVCacheGroupSpec objects, one for each group. @@ -614,11 +625,11 @@ def get_kv_cache_config(vllm_config: VllmConfig, def unify_kv_cache_configs(kv_cache_configs: list[KVCacheConfig]): """ - Make the KV cache configurations for each worker consistent, so that all + Make the KV cache configurations for each worker consistent, so that all workers can be controlled by the same KVCacheManager. This function verifies that the layer group of each worker are the same, and changes the num_blocks of each worker to the smallest among all workers. - + Args: kv_cache_configs: The KV cache configurations for each worker. Will be in-place modified to make them consistent. diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 924796e03da7e..850687423df73 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -61,6 +61,7 @@ class Scheduler(SchedulerInterface): max_model_len=self.max_model_len, sliding_window=self.cache_config.sliding_window, enable_caching=self.cache_config.enable_prefix_caching, + caching_hash_algo=self.cache_config.prefix_caching_hash_algo, log_stats=self.log_stats) self.block_size = self.cache_config.block_size From dd8a29da99aaca4aaedf710c813222871245e140 Mon Sep 17 00:00:00 2001 From: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com> Date: Wed, 26 Mar 2025 15:35:11 -0500 Subject: [PATCH 020/593] Applying some fixes for K8s agents in CI (#15493) Signed-off-by: Alexei V. Ivanov --- .buildkite/run-amd-test.sh | 10 ++++++---- Dockerfile.rocm | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.buildkite/run-amd-test.sh b/.buildkite/run-amd-test.sh index 0680bae13ddbf..e5a1b760db1f0 100755 --- a/.buildkite/run-amd-test.sh +++ b/.buildkite/run-amd-test.sh @@ -134,9 +134,10 @@ if [[ $commands == *"--shard-id="* ]]; then # assign shard-id for each shard commands_gpu=${commands//"--shard-id= "/"--shard-id=${GPU} "} echo "Shard ${GPU} commands:$commands_gpu" + echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" docker run \ - --device /dev/kfd --device /dev/dri \ - --network host \ + --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ + --network=host \ --shm-size=16gb \ --rm \ -e HIP_VISIBLE_DEVICES="${GPU}" \ @@ -163,9 +164,10 @@ if [[ $commands == *"--shard-id="* ]]; then fi done else + echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" docker run \ - --device /dev/kfd --device /dev/dri \ - --network host \ + --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ + --network=host \ --shm-size=16gb \ --rm \ -e HIP_VISIBLE_DEVICES=0 \ diff --git a/Dockerfile.rocm b/Dockerfile.rocm index 841e7978a424f..f9ebb10ca8731 100644 --- a/Dockerfile.rocm +++ b/Dockerfile.rocm @@ -12,7 +12,8 @@ ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}} # Install some basic utilities RUN apt-get update -q -y && apt-get install -q -y \ - sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev + sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev \ + apt-transport-https ca-certificates wget curl # Remove sccache RUN python3 -m pip install --upgrade pip && pip install setuptools_scm RUN apt-get purge -y sccache; python3 -m pip uninstall -y sccache; rm -f "$(which sccache)" From b2e85e26f408a8bb74b7657b6bcddfede1a93090 Mon Sep 17 00:00:00 2001 From: Alexander Matveev <59768536+alexm-redhat@users.noreply.github.com> Date: Wed, 26 Mar 2025 17:35:05 -0400 Subject: [PATCH 021/593] [V1] TPU - Revert to exponential padding by default (#15565) Signed-off-by: Alexander Matveev --- vllm/envs.py | 4 ++-- vllm/v1/worker/tpu_model_runner.py | 35 ++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 4c413006a6413..46c5b3a1dc5d0 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -99,7 +99,7 @@ if TYPE_CHECKING: VLLM_MARLIN_USE_ATOMIC_ADD: bool = False VLLM_V0_USE_OUTLINES_CACHE: bool = False VLLM_TPU_DISABLE_TOPK_TOPP_OPTIMIZATION: bool = False - VLLM_TPU_BUCKET_PADDING_GAP: int = 64 + VLLM_TPU_BUCKET_PADDING_GAP: int = 0 def get_default_cache_root(): @@ -648,7 +648,7 @@ environment_variables: dict[str, Callable[[], Any]] = { # 8, we will run forward pass with [16, 24, 32, ...]. "VLLM_TPU_BUCKET_PADDING_GAP": lambda: int(os.environ["VLLM_TPU_BUCKET_PADDING_GAP"]) - if "VLLM_TPU_BUCKET_PADDING_GAP" in os.environ else 64, + if "VLLM_TPU_BUCKET_PADDING_GAP" in os.environ else 0, } # end-env-vars-definition diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index edf859f0b9463..cf5c56b98beaa 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -944,18 +944,35 @@ def _get_paddings(min_token_size: int, max_token_size: int, padding_gap: int) -> list[int]: """Generate a list of padding size, starting from min_token_size, ending with a number that can cover max_token_size - first increase the size to twice, - then increase the padding size by padding_gap. + + If padding_gap == 0 then: + increase 2X each time (exponential) + else: + first increase the size to twice, + then increase the padding size by padding_gap. """ paddings = [] num = min_token_size - while num <= padding_gap: - paddings.append(num) - num *= 2 - num //= 2 - while num < max_token_size: - num += padding_gap - paddings.append(num) + + if padding_gap == 0: + logger.info("Using exponential paddings:") + while num <= max_token_size: + logger.info(" %d", num) + paddings.append(num) + num *= 2 + + else: + logger.info("Using incremental paddings:") + while num <= padding_gap: + logger.info(" %d", num) + paddings.append(num) + num *= 2 + num //= 2 + while num < max_token_size: + num += padding_gap + logger.info(" %d", num) + paddings.append(num) + return paddings From 9d119a86ae9a1655a972afc7b1f701b7c7191876 Mon Sep 17 00:00:00 2001 From: Alexander Matveev <59768536+alexm-redhat@users.noreply.github.com> Date: Wed, 26 Mar 2025 17:51:54 -0400 Subject: [PATCH 022/593] [V1] TPU CI - Fix test_compilation.py (#15570) Signed-off-by: Alexander Matveev --- .buildkite/run-tpu-v1-test.sh | 2 +- tests/tpu/test_compilation.py | 57 +++++++++++------------------------ 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/.buildkite/run-tpu-v1-test.sh b/.buildkite/run-tpu-v1-test.sh index d557feefba7aa..6e1f79ae649e3 100755 --- a/.buildkite/run-tpu-v1-test.sh +++ b/.buildkite/run-tpu-v1-test.sh @@ -22,7 +22,7 @@ docker run --privileged --net host --shm-size=16G -it \ && export VLLM_USE_V1=1 \ && export VLLM_XLA_CHECK_RECOMPILATION=1 \ && echo TEST_1 \ - && pytest /workspace/vllm/tests/tpu/test_compilation.py \ + && pytest -v -s /workspace/vllm/tests/tpu/test_compilation.py \ && echo TEST_2 \ && pytest -v -s /workspace/vllm/tests/v1/tpu/test_basic.py \ && echo TEST_3 \ diff --git a/tests/tpu/test_compilation.py b/tests/tpu/test_compilation.py index e70b3e17c6f93..27328d4542d9a 100644 --- a/tests/tpu/test_compilation.py +++ b/tests/tpu/test_compilation.py @@ -5,12 +5,8 @@ import os import tempfile import depyf -import pytest - -from vllm.config import CompilationLevel -@pytest.mark.skip(reason="Not working; needs investigation.") def test_tpu_compilation(): temp_dir = tempfile.mkdtemp() with depyf.prepare_debug(temp_dir): @@ -22,27 +18,24 @@ def test_tpu_compilation(): "The greatest glory in living lies not in never falling,", ] answers = [ - " or, through inaction, allow a human being to come to harm.", - " what is essential is invisible to the eye.", - " but in rising every time we fall.", + " or, through inaction", + " what is essential ", + " but in rising ", ] - N = 1 + # Currently, top-p sampling is disabled. `top_p` should be 1.0. + N = 1 sampling_params = SamplingParams(temperature=0.7, top_p=1.0, n=N, max_tokens=16) - # Set `enforce_eager=True` to avoid ahead-of-time compilation. - # In real workloads, `enforace_eager` should be `False`. - - # disable custom dispatcher, let Dynamo takes over - # all the control llm = LLM(model="Qwen/Qwen2.5-1.5B-Instruct", - max_model_len=512, - max_num_seqs=64, - enforce_eager=True, - compilation_config={"level": CompilationLevel.DYNAMO_AS_IS}) + max_num_batched_tokens=256, + max_model_len=256, + max_num_seqs=32, + enforce_eager=False) + outputs = llm.generate(prompts, sampling_params) for output, answer in zip(outputs, answers): prompt = output.prompt @@ -56,16 +49,11 @@ def test_tpu_compilation(): for i, compiled_code in enumerate(compiled_codes): print("{} file: {}".format(i + 1, compiled_code)) - # We should only trigger Dynamo compilation 4 times: - # 1. forward pass (symbolic) - # 2. compute_logits (symbolic) - # 3. forward pass (shape 16) - # 4. forward pass (shape 32) - # and later calls should not trigger Dynamo compilation again. - # NOTE: It might still trigger XLA compilation. - + # We should only trigger Dynamo compilation 2 times: + # 1. Forward pass without kv_caches + # 2. Forward pass with kv_caches # Check we have 4 compiled codes - assert len(compiled_codes) == 4 + assert len(compiled_codes) == 2 kv_cache_prefix = "kv_cache" attn_prefix = "ragged_paged_attention" @@ -77,24 +65,13 @@ def test_tpu_compilation(): for i, compiled_fn in enumerate(compiled_fns): print("{} file: {}".format(i + 1, compiled_fn)) - # The first compilation is symbolic, so it should not have any kv_caches + # The first compilation should not have any kv_caches with open(compiled_fns[0]) as f: content = f.read() assert kv_cache_prefix not in content - # The second compilation is symbolic, so it should not have any kv_caches + # The second compilation should have kv_caches and the + # ragged_paged_attention with open(compiled_fns[1]) as f: - content = f.read() - assert kv_cache_prefix not in content - - # The third compilation is shape 16, so it should have kv_caches and the - # ragged_paged_attention - with open(compiled_fns[2]) as f: - content = f.read() - assert (kv_cache_prefix in content and attn_prefix in content) - - # The forth compilation is shape 32, so it should have kv_caches and the - # ragged_paged_attention - with open(compiled_fns[3]) as f: content = f.read() assert (kv_cache_prefix in content and attn_prefix in content) From 7a888271f5bd401f8fc64704c239833244471a91 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 26 Mar 2025 17:21:34 -0600 Subject: [PATCH 023/593] Use Cache Hinting for fused_moe kernel (#15511) --- .../model_executor/layers/fused_moe/fused_moe.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 97e915c60335a..faaea6b4de972 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -189,7 +189,11 @@ def fused_moe_kernel_gptq_awq( mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), other=0.0) - b = tl.load(b_ptrs) + b = tl.load( + b_ptrs, + cache_modifier=".cg", + eviction_policy="evict_last", + ) if use_int4_w4a16: b = (b >> b_shifter) & 0xF @@ -391,9 +395,13 @@ def fused_moe_kernel( mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), other=0.0) - b = tl.load(b_ptrs, - mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, - other=0.0) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, + other=0.0, + cache_modifier=".cg", + eviction_policy="evict_last", + ) # We accumulate along the K dimension. if use_int8_w8a16: accumulator = tl.dot(a, b.to(compute_type), acc=accumulator) From e74ff409e0f8f3cacb8a251a1cae8b478721cead Mon Sep 17 00:00:00 2001 From: Chengji Yao Date: Wed, 26 Mar 2025 17:09:28 -0700 Subject: [PATCH 024/593] [TPU] support disabling xla compilation cache (#15567) Signed-off-by: Chengji Yao --- vllm/v1/worker/tpu_worker.py | 13 ++++++++++--- vllm/worker/tpu_worker.py | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/vllm/v1/worker/tpu_worker.py b/vllm/v1/worker/tpu_worker.py index 9a380373d4617..4d9a113e39ee4 100644 --- a/vllm/v1/worker/tpu_worker.py +++ b/vllm/v1/worker/tpu_worker.py @@ -113,9 +113,16 @@ class TPUWorker: # can have slightly different XLA graphs. world_size = self.parallel_config.world_size rank = xr.global_ordinal() - per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH, - f"tp{world_size}_rank{rank}") - xr.initialize_cache(per_rank_path, readonly=False) + # The PyTorch/XLA compilation cache uses the Torch IR to generate keys. + # Consequently, changes in optimization flags, which affect compilation + # results, don't change the cache key. This can result in the wrong + # compilation being used. To prevent this, disabling the XLA compilation + # cache during development is recommended.We can disable it by + # `export VLLM_XLA_CACHE_PATH=` + if envs.VLLM_XLA_CACHE_PATH: + per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH, + f"tp{world_size}_rank{rank}") + xr.initialize_cache(per_rank_path, readonly=False) # Init ModelRunner here, so that we have access to self.device. self.model_runner = TPUModelRunner(self.vllm_config, self.device) diff --git a/vllm/worker/tpu_worker.py b/vllm/worker/tpu_worker.py index 66911790662eb..71b4b38fb9d62 100644 --- a/vllm/worker/tpu_worker.py +++ b/vllm/worker/tpu_worker.py @@ -93,9 +93,16 @@ class TPUWorker(LoRANotSupportedWorkerBase, LocalOrDistributedWorkerBase): # can have slightly different XLA graphs. world_size = self.parallel_config.world_size rank = xr.global_ordinal() - per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH, - f"tp{world_size}_rank{rank}") - xr.initialize_cache(per_rank_path, readonly=False) + # The PyTorch/XLA compilation cache uses the Torch IR to generate keys. + # Consequently, changes in optimization flags, which affect compilation + # results, don't change the cache key. This can result in the wrong + # compilation being used. To prevent this, disabling the XLA compilation + # cache during development is recommended.We can disable it by + # `export VLLM_XLA_CACHE_PATH=` + if envs.VLLM_XLA_CACHE_PATH: + per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH, + f"tp{world_size}_rank{rank}") + xr.initialize_cache(per_rank_path, readonly=False) self.profiler = None if envs.VLLM_TORCH_PROFILER_DIR and self.rank < 1: From 7a6d45bc8a201623c646627becd837afd6b35bc7 Mon Sep 17 00:00:00 2001 From: Matthew Vine <32849887+MattTheCuber@users.noreply.github.com> Date: Wed, 26 Mar 2025 20:19:46 -0400 Subject: [PATCH 025/593] Support FIPS enabled machines with MD5 hashing (#15299) Signed-off-by: Matthew Vine <32849887+MattTheCuber@users.noreply.github.com> --- tests/compile/piecewise/test_toy_llama.py | 3 +- vllm/compilation/backends.py | 7 ++-- vllm/compilation/compiler_interface.py | 3 +- vllm/config.py | 42 +++++++++++++++-------- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/tests/compile/piecewise/test_toy_llama.py b/tests/compile/piecewise/test_toy_llama.py index 7307f44b6184e..d4551b1cc3aec 100644 --- a/tests/compile/piecewise/test_toy_llama.py +++ b/tests/compile/piecewise/test_toy_llama.py @@ -63,7 +63,8 @@ class LlamaConfig: factors.append((k, v)) factors.sort() import hashlib - return hashlib.md5(str(factors).encode()).hexdigest() + return hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() def __post_init__(self): assert self.mlp_size >= self.hidden_size diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index d8c0c59ba9b22..45988c2e9b0d4 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -381,8 +381,8 @@ class VllmBackend: with open(filepath) as f: hash_content.append(f.read()) import hashlib - code_hash = hashlib.md5( - "\n".join(hash_content).encode()).hexdigest() + code_hash = hashlib.md5("\n".join(hash_content).encode(), + usedforsecurity=False).hexdigest() factors.append(code_hash) # 3. compiler hash @@ -390,7 +390,8 @@ class VllmBackend: factors.append(compiler_hash) # combine all factors to generate the cache dir - hash_key = hashlib.md5(str(factors).encode()).hexdigest()[:10] + hash_key = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest()[:10] cache_dir = os.path.join( envs.VLLM_CACHE_ROOT, diff --git a/vllm/compilation/compiler_interface.py b/vllm/compilation/compiler_interface.py index b45c694fd7f89..571e2b832e95f 100644 --- a/vllm/compilation/compiler_interface.py +++ b/vllm/compilation/compiler_interface.py @@ -139,7 +139,8 @@ class InductorAdaptor(CompilerInterface): from torch._inductor.codecache import torch_key torch_factors = torch_key() factors.append(torch_factors) - hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10] + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest()[:10] return hash_str def initialize_cache(self, cache_dir: str, disable_cache: bool = False): diff --git a/vllm/config.py b/vllm/config.py index 94cecba1e1fcb..2e9325c258b26 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -1111,7 +1111,8 @@ class CacheConfig: factors: list[Any] = [] factors.append(self.cache_dtype) # `cpu_offload_gb` does not use `torch.compile` yet. - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __init__( @@ -1243,7 +1244,8 @@ class TokenizerPoolConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __post_init__(self): @@ -1354,7 +1356,8 @@ class LoadConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __post_init__(self): @@ -1674,7 +1677,8 @@ class SchedulerConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __post_init__(self) -> None: @@ -1810,7 +1814,8 @@ class DeviceConfig: # the device/platform information will be summarized # by torch/vllm automatically. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __init__(self, device: str = "auto") -> None: @@ -1983,7 +1988,8 @@ class SpeculativeConfig: # no factors to consider. # spec decode does not use `torch.compile` yet. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str @classmethod @@ -2358,7 +2364,8 @@ class LoRAConfig: factors.append(self.lora_extra_vocab_size) factors.append(self.long_lora_scaling_factors) factors.append(self.bias_enabled) - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __post_init__(self): @@ -2424,7 +2431,8 @@ class PromptAdapterConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __post_init__(self): @@ -2469,7 +2477,8 @@ class MultiModalConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def get_limit_per_prompt(self, modality: str) -> int: @@ -2535,7 +2544,8 @@ class PoolerConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str @staticmethod @@ -2816,7 +2826,8 @@ class DecodingConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __post_init__(self): @@ -2866,7 +2877,8 @@ class ObservabilityConfig: # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str def __post_init__(self): @@ -2928,7 +2940,8 @@ class KVTransferConfig(BaseModel): # no factors to consider. # this config will not affect the computation graph. factors: list[Any] = [] - hash_str = hashlib.md5(str(factors).encode()).hexdigest() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest() return hash_str @classmethod @@ -3425,7 +3438,8 @@ class VllmConfig: vllm_factors.append("None") factors.append(vllm_factors) - hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10] + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest()[:10] return hash_str def pad_for_cudagraph(self, batch_size: int) -> int: From 9239bf718e5ebb5ab871ac8ed09fb80ed02fa82b Mon Sep 17 00:00:00 2001 From: ElizaWszola Date: Thu, 27 Mar 2025 01:54:44 +0100 Subject: [PATCH 026/593] [Kernel] CUTLASS grouped gemm fp8 MoE kernel (#13972) Signed-off-by: ElizaWszola Signed-off-by: ElizaWszola Co-authored-by: Lucas Wilkinson --- CMakeLists.txt | 27 ++ .../kernels/benchmark_grouped_gemm_cutlass.py | 340 +++++++++++++ benchmarks/kernels/benchmark_shapes.py | 16 + csrc/cutlass_extensions/common.hpp | 12 +- .../broadcast_load_epilogue_array_c3x.hpp | 457 ++++++++++++++++++ .../epilogue/scaled_mm_epilogues_c3x.hpp | 66 +++ csrc/ops.h | 14 + .../cutlass_w8a8/moe/get_group_starts.cuh | 80 +++ .../cutlass_w8a8/moe/grouped_mm_c3x.cu | 160 ++++++ .../cutlass_w8a8/moe/grouped_mm_c3x.cuh | 149 ++++++ .../quantization/cutlass_w8a8/moe/moe_data.cu | 90 ++++ .../cutlass_w8a8/scaled_mm_entry.cu | 67 +++ csrc/torch_bindings.cpp | 29 ++ tests/kernels/test_cutlass.py | 134 +++++ tests/kernels/test_cutlass_moe.py | 244 ++++++++++ vllm/_custom_ops.py | 53 ++ .../layers/fused_moe/__init__.py | 5 +- .../layers/fused_moe/fused_moe.py | 137 ++++++ .../compressed_tensors/compressed_tensors.py | 31 +- .../compressed_tensors_moe.py | 202 +++++++- .../layers/quantization/utils/w8a8_utils.py | 10 + vllm/utils.py | 9 +- 22 files changed, 2317 insertions(+), 15 deletions(-) create mode 100644 benchmarks/kernels/benchmark_grouped_gemm_cutlass.py create mode 100644 csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp create mode 100644 csrc/quantization/cutlass_w8a8/moe/get_group_starts.cuh create mode 100644 csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cu create mode 100644 csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cuh create mode 100644 csrc/quantization/cutlass_w8a8/moe/moe_data.cu create mode 100644 tests/kernels/test_cutlass_moe.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 65d1ddbeee0b2..e0f1fdf78d142 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -461,6 +461,33 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") set(FP4_ARCHS) endif() + # + # CUTLASS MoE kernels + + # The MoE kernel cutlass_moe_mm requires CUDA 12.3 or later (and only works + # on Hopper). get_cutlass_moe_mm_data should only be compiled if it's possible + # to compile MoE kernels that use its output. + cuda_archs_loose_intersection(SCALED_MM_ARCHS "9.0a;" "${CUDA_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.3 AND SCALED_MM_ARCHS) + set(SRCS "csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cu" + "csrc/quantization/cutlass_w8a8/moe/moe_data.cu") + set_gencode_flags_for_srcs( + SRCS "${SRCS}" + CUDA_ARCHS "${SCALED_MM_ARCHS}") + list(APPEND VLLM_EXT_SRC "${SRCS}") + list(APPEND VLLM_GPU_FLAGS "-DENABLE_CUTLASS_MOE_SM90=1") + message(STATUS "Building grouped_mm_c3x for archs: ${SCALED_MM_ARCHS}") + else() + if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.3 AND SCALED_MM_ARCHS) + message(STATUS "Not building grouped_mm_c3x kernels as CUDA Compiler version is " + "not >= 12.3, we recommend upgrading to CUDA 12.3 or later " + "if you intend on running FP8 quantized MoE models on Hopper.") + else() + message(STATUS "Not building grouped_mm_c3x as no compatible archs found " + "in CUDA target architectures") + endif() + endif() + # # Machete kernels diff --git a/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py new file mode 100644 index 0000000000000..bcdbf6c7551a3 --- /dev/null +++ b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py @@ -0,0 +1,340 @@ +# SPDX-License-Identifier: Apache-2.0 + +import torch +import torch.utils.benchmark as benchmark +from benchmark_shapes import WEIGHT_SHAPES_MOE + +from vllm import _custom_ops as ops +from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.fused_moe.fused_moe import (cutlass_moe_fp8, + fused_experts, + fused_topk) +from vllm.utils import FlexibleArgumentParser + +DEFAULT_MODELS = [ + "nm-testing/Mixtral-8x7B-Instruct-v0.1", "nm-testing/deepseekv2-lite", + "ibm-granite/granite-3.0-1b-a400m", "ibm-granite/granite-3.0-3b-a800m" +] +DEFAULT_BATCH_SIZES = [1, 4, 8, 16, 32, 64, 128, 256, 512] +DEFAULT_TP_SIZES = [1] + +PER_ACT_TOKEN_OPTS = [False] +PER_OUT_CH_OPTS = [False] + + +def to_fp8(tensor: torch.Tensor): + finfo = torch.finfo(torch.float8_e4m3fn) + return torch.round(tensor.clamp( + min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn) + + +def bench_run(results: list[benchmark.Measurement], model: str, + num_experts: int, topk: int, per_act_token: bool, + per_out_ch: bool, mkn: tuple[int, int, int]): + label = "Quant Matmul" + + sub_label = ( + "{}, num_experts={}, topk={}, per_act_token={} per_out_ch={}, " + "MKN=({})".format(model, num_experts, topk, per_act_token, per_out_ch, + mkn)) + + print(f"Testing: {sub_label}") + + (m, k, n) = mkn + + dtype = torch.half + + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((num_experts, 2 * n, k), device="cuda", dtype=dtype) / 10 + w2 = torch.randn((num_experts, k, n), device="cuda", dtype=dtype) / 10 + + _, a_scale = ops.scaled_fp8_quant(a) + + w1_q = torch.empty((num_experts, 2 * n, k), + device="cuda", + dtype=torch.float8_e4m3fn) + w2_q = torch.empty((num_experts, k, n), + device="cuda", + dtype=torch.float8_e4m3fn) + w1_scale = torch.empty((num_experts, 1, 1), + device="cuda", + dtype=torch.float32) + w2_scale = torch.empty((num_experts, 1, 1), + device="cuda", + dtype=torch.float32) + + ab_strides1 = torch.full((num_experts, ), + k, + device="cuda", + dtype=torch.int64) + c_strides1 = torch.full((num_experts, ), + 2 * n, + device="cuda", + dtype=torch.int64) + ab_strides2 = torch.full((num_experts, ), + n, + device="cuda", + dtype=torch.int64) + c_strides2 = torch.full((num_experts, ), + k, + device="cuda", + dtype=torch.int64) + + for expert in range(num_experts): + w1_q[expert], w1_scale[expert] = ops.scaled_fp8_quant(w1[expert]) + w2_q[expert], w2_scale[expert] = ops.scaled_fp8_quant(w2[expert]) + w1_q_notransp = w1_q.clone() + w2_q_notransp = w2_q.clone() + w1_q = w1_q.transpose(1, 2) + w2_q = w2_q.transpose(1, 2) + + score = torch.randn((m, num_experts), device="cuda", dtype=dtype) + + topk_weights, topk_ids = fused_topk(a, score, topk, renormalize=False) + + def run_triton_moe(a: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor, + topk_weights: torch.Tensor, topk_ids: torch.Tensor, + w1_scale: torch.Tensor, w2_scale: torch.Tensor, + a_scale: torch.Tensor, num_repeats: int): + for _ in range(num_repeats): + fused_experts(a, + w1, + w2, + topk_weights, + topk_ids, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a_scale) + + def run_cutlass_moe(a: torch.Tensor, a_scale: torch.Tensor, + w1: torch.Tensor, w2: torch.Tensor, + w1_scale: torch.Tensor, w2_scale: torch.Tensor, + topk_weights: torch.Tensor, topk_ids: torch.Tensor, + ab_strides1: torch.Tensor, c_strides1: torch.Tensor, + ab_strides2: torch.Tensor, c_strides2: torch.Tensor, + num_repeats: int): + for _ in range(num_repeats): + cutlass_moe_fp8(a, + w1, + w2, + w1_scale, + w2_scale, + topk_weights, + topk_ids, + ab_strides1, + c_strides1, + ab_strides2, + c_strides2, + a1_scale=a_scale) + + def run_cutlass_from_graph( + a: torch.Tensor, a_scale: torch.Tensor, w1_q: torch.Tensor, + w2_q: torch.Tensor, w1_scale: torch.Tensor, w2_scale: torch.Tensor, + topk_weights: torch.Tensor, topk_ids: torch.Tensor, + ab_strides1: torch.Tensor, c_strides1: torch.Tensor, + ab_strides2: torch.Tensor, c_strides2: torch.Tensor): + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig( + pipeline_parallel_size=1))): + return cutlass_moe_fp8(a, + w1_q, + w2_q, + w1_scale, + w2_scale, + topk_weights, + topk_ids, + ab_strides1, + c_strides1, + ab_strides2, + c_strides2, + a1_scale=a_scale) + + def run_triton_from_graph(a: torch.Tensor, w1: torch.Tensor, + w2: torch.Tensor, topk_weights: torch.Tensor, + topk_ids: torch.Tensor, w1_scale: torch.Tensor, + w2_scale: torch.Tensor, a_scale: torch.Tensor): + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig( + pipeline_parallel_size=1))): + return fused_experts(a, + w1, + w2, + topk_weights, + topk_ids, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a_scale) + + def replay_graph(graph, num_repeats): + for _ in range(num_repeats): + graph.replay() + torch.cuda.synchronize() + + cutlass_stream = torch.cuda.Stream() + cutlass_graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(cutlass_graph, stream=cutlass_stream): + run_cutlass_from_graph(a, a_scale, w1_q, w2_q, w1_scale, w2_scale, + topk_weights, topk_ids, ab_strides1, c_strides1, + ab_strides2, c_strides2) + torch.cuda.synchronize() + + triton_stream = torch.cuda.Stream() + triton_graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(triton_graph, stream=triton_stream): + run_triton_from_graph(a, w1_q_notransp, w2_q_notransp, topk_weights, + topk_ids, w1_scale, w2_scale, a_scale) + torch.cuda.synchronize() + + min_run_time = 5 + num_warmup = 5 + num_runs = 25 + + globals = { + # Baseline params + "w1": w1, + "w2": w2, + "score": score, + "topk": topk, + "w1_q_notransp": w1_q_notransp, + "w2_q_notransp": w2_q_notransp, + # Cutlass params + "a_scale": a_scale, + "w1_q": w1_q, + "w2_q": w2_q, + "w1_scale": w1_scale, + "w2_scale": w2_scale, + "ab_strides1": ab_strides1, + "c_strides1": c_strides1, + "ab_strides2": ab_strides2, + "c_strides2": c_strides2, + # cuda graph params + "cutlass_graph": cutlass_graph, + "triton_graph": triton_graph, + # Gen params + "a": a, + "topk_weights": topk_weights, + "topk_ids": topk_ids, + "num_runs": num_runs, + # Kernels + "run_triton_moe": run_triton_moe, + "run_cutlass_moe": run_cutlass_moe, + "replay_graph": replay_graph, + } + + # Warmup + run_triton_moe(a, w1_q_notransp, w2_q_notransp, topk_weights, topk_ids, + w1_scale, w2_scale, a_scale, num_warmup) + + results.append( + benchmark.Timer( + stmt= + "run_triton_moe(a, w1_q_notransp, w2_q_notransp, topk_weights, topk_ids, w1_scale, w2_scale, a_scale, num_runs)", # noqa: E501 + globals=globals, + label=label, + sub_label=sub_label, + description="triton_moe", + ).blocked_autorange(min_run_time=min_run_time)) + + # Warmup + replay_graph(triton_graph, num_warmup) + + results.append( + benchmark.Timer( + stmt="replay_graph(triton_graph, num_runs)", + globals=globals, + label=label, + sub_label=sub_label, + description="triton_moe_cuda_graphs", + ).blocked_autorange(min_run_time=min_run_time)) + + # Warmup + run_cutlass_moe(a, a_scale, w1_q, w2_q, w1_scale, w2_scale, topk_weights, + topk_ids, ab_strides1, c_strides1, ab_strides2, c_strides2, + num_warmup) + + results.append( + benchmark.Timer( + stmt= + "run_cutlass_moe(a, a_scale, w1_q, w2_q, w1_scale, w2_scale, topk_weights, topk_ids, ab_strides1, c_strides1, ab_strides2, c_strides2, num_runs)", # noqa: E501 + globals=globals, + label=label, + sub_label=sub_label, + description="grouped_gemm_moe", + ).blocked_autorange(min_run_time=min_run_time)) + + # Warmup + replay_graph(cutlass_graph, num_warmup) + + results.append( + benchmark.Timer( + stmt="replay_graph(cutlass_graph, num_runs)", + globals=globals, + label=label, + sub_label=sub_label, + description="grouped_gemm_moe_cuda_graphs", + ).blocked_autorange(min_run_time=min_run_time)) + + +def main(args): + print("Benchmarking models:") + for i, model in enumerate(args.models): + print(f"[{i}] {model}") + + results: list[benchmark.Measurement] = [] + + for model in args.models: + for tp in args.tp_sizes: + for layer in WEIGHT_SHAPES_MOE[model]: + num_experts = layer[0] + topk = layer[1] + size_k = layer[2] + size_n = layer[3] // tp + + if len(args.limit_k) > 0 and size_k not in args.limit_k: + continue + + if len(args.limit_n) > 0 and size_n not in args.limit_n: + continue + + for per_act_token in PER_ACT_TOKEN_OPTS: + for per_out_ch in PER_OUT_CH_OPTS: + for size_m in DEFAULT_BATCH_SIZES: + mkn = (size_m, size_k, size_n) + bench_run(results, model, num_experts, topk, + per_act_token, per_out_ch, mkn) + + compare = benchmark.Compare(results) + compare.print() + + +if __name__ == "__main__": + parser = FlexibleArgumentParser( + description="Benchmark Marlin across specified models/shapes/batches") + parser.add_argument( + "--models", + nargs="+", + type=str, + default=DEFAULT_MODELS, + choices=WEIGHT_SHAPES_MOE.keys(), + ) + parser.add_argument("--tp-sizes", + nargs="+", + type=int, + default=DEFAULT_TP_SIZES) + parser.add_argument("--batch-sizes", + nargs="+", + type=int, + default=DEFAULT_BATCH_SIZES) + parser.add_argument("--limit-k", nargs="+", type=int, default=[]) + parser.add_argument("--limit-n", nargs="+", type=int, default=[]) + parser.add_argument("--limit-num-groups", nargs="+", type=int, default=[]) + parser.add_argument("--limit-per-act-token", + nargs="+", + type=int, + default=[]) + parser.add_argument("--limit-per-out-ch", nargs="+", type=int, default=[]) + + args = parser.parse_args() + main(args) diff --git a/benchmarks/kernels/benchmark_shapes.py b/benchmarks/kernels/benchmark_shapes.py index c375e61e41873..70190ba24d9df 100644 --- a/benchmarks/kernels/benchmark_shapes.py +++ b/benchmarks/kernels/benchmark_shapes.py @@ -75,3 +75,19 @@ WEIGHT_SHAPES = { [7168, 8192], ], } + +WEIGHT_SHAPES_MOE = { + "nm-testing/Mixtral-8x7B-Instruct-v0.1": [ + [8, 2, 4096, 28672], + [8, 2, 14336, 4096], + ], + "nm-testing/deepseekv2-lite": [ + [64, 6, 2048, 1408], + ], + "ibm-granite/granite-3.0-1b-a400m": [ + [32, 8, 1024, 1024], + ], + "ibm-granite/granite-3.0-3b-a800m": [ + [40, 8, 1024, 1536], + ], +} diff --git a/csrc/cutlass_extensions/common.hpp b/csrc/cutlass_extensions/common.hpp index febc4eccd9561..dbe0e30f5cbfe 100644 --- a/csrc/cutlass_extensions/common.hpp +++ b/csrc/cutlass_extensions/common.hpp @@ -48,4 +48,14 @@ struct enable_sm90_or_later : Kernel { Kernel::operator()(std::forward(args)...); #endif } -}; \ No newline at end of file +}; + +template +struct enable_sm90_only : Kernel { + template + CUTLASS_DEVICE void operator()(Args&&... args) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ == 900 + Kernel::operator()(std::forward(args)...); +#endif + } +}; diff --git a/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp b/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp new file mode 100644 index 0000000000000..5c1d6e3f46be0 --- /dev/null +++ b/csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp @@ -0,0 +1,457 @@ +/*************************************************************************************************** + * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights + *reserved. SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + *this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + *POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// +// This file is a modified excerpt of +// include/cutlass/epilogue/fusion/sm90_visitor_load_tma_warpspecialized.hpp +// from https://github.com/NVIDIA/cutlass v3.5.0 +// It has been modified to support either row/column or scalar broadcasting +// where the tensor being loaded from is always passed in via a device pointer. +// This lets one compiled kernel handle all cases of per-tensor or +// per-channel/per-token quantization. +// +// This interface also allows the scales to be passed in as tensors that +// consistently reside on the device, which avoids an issue with a previous +// implementation where scalars needed to be on the CPU since they +// were passed in via float values. This created a potential performance hazard +// if scales were initially on the device, and caused torch.compile graphs +// breaks when moving scales to the CPU. +// +#pragma once + +// Turn off clang-format for the entire file to keep it close to upstream +// clang-format off + +#include "cutlass/cutlass.h" +#include "cutlass/arch/barrier.h" + +#include "cute/tensor.hpp" +#include "cutlass/epilogue/fusion/sm90_visitor_tma_warpspecialized.hpp" + +namespace cutlass::epilogue::fusion { + +using namespace cute; +using namespace detail; + +// Row vector broadcast +template< + int Stages, + class CtaTileShapeMNK, + class Element, + class StrideMNL = Stride<_0,_1,_0>, + int Alignment = 128 / sizeof_bits_v +> +struct Sm90RowOrScalarBroadcastArray { + static_assert(Stages == 0, "Row broadcast doesn't support smem usage"); + static_assert(is_static_v(StrideMNL{}))>); // batch stride can be dynamic or static + static_assert(take<0,2>(StrideMNL{}) == Stride<_0,_1>{}); + + struct SharedStorage { + array_aligned(CtaTileShapeMNK{})> smem; + }; + + // This struct has been modified to have a bool indicating that ptr_row is a + // scalar that must be broadcast, instead of containing a scalar that is + // valid if ptr_row is null. + struct Arguments { + const Element* const* ptr_row_array = nullptr; + bool row_broadcast = true; + StrideMNL dRow = {}; + }; + + using Params = Arguments; + + template + static constexpr Params + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) { + return args; + } + + template + static bool + can_implement(ProblemShape const& problem_shape, Arguments const& args) { + return true; + } + + template + static size_t + get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) { + return 0; + } + + template + static cutlass::Status + initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream, + CudaHostAdapter* cuda_adapter = nullptr) { + return cutlass::Status::kSuccess; + } + + CUTLASS_HOST_DEVICE + Sm90RowOrScalarBroadcastArray() { } + + CUTLASS_HOST_DEVICE + Sm90RowOrScalarBroadcastArray(Params const& params, SharedStorage const& shared_storage) + : params(params) + , smem(const_cast(shared_storage.smem.data())) { } + + Params params; + Element *smem = nullptr; + + CUTLASS_DEVICE bool + is_producer_load_needed() const { + return false; + } + + CUTLASS_DEVICE bool + is_C_load_needed() const { + return false; + } + + CUTLASS_DEVICE bool + is_zero() const { + return (!params.row_broadcast && *(params.ptr_row_array[group]) == Element(0)); + } + + template + CUTLASS_DEVICE auto + get_producer_load_callbacks(ProducerLoadArgs const& args) { + return EmptyProducerLoadCallbacks{}; + } + + template + struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks { + CUTLASS_DEVICE + ConsumerStoreCallbacks( + GS_GTensor tGS_gRow_, GS_STensor tGS_sRow_, + GS_CTensor tGS_cRow_, Tiled_G2S tiled_g2s_, + SR_STensor tSR_sRow_, SR_RTensor tSR_rRow_, + CTensor tCcRow_, ThrResidue residue_tCcRow_, ThrNum thr_num_, + int group, Params const& params_) + : tGS_gRow(tGS_gRow_) + , tGS_sRow(tGS_sRow_) + , tGS_cRow(tGS_cRow_) + , tiled_G2S(tiled_g2s_) + , tSR_sRow(tSR_sRow_) + , tSR_rRow(tSR_rRow_) + , tCcRow(tCcRow_) + , residue_tCcRow(residue_tCcRow_) + , group(group) + , params(params_) {} + + GS_GTensor tGS_gRow; // (CPY,CPY_M,CPY_N) + GS_STensor tGS_sRow; // (CPY,CPY_M,CPY_N) + GS_CTensor tGS_cRow; // (CPY,CPY_M,CPY_N) + Tiled_G2S tiled_G2S; + + SR_STensor tSR_sRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + SR_RTensor tSR_rRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + + CTensor tCcRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + ThrResidue residue_tCcRow; // (m, n) + ThrNum thr_num; + int group; + Params const& params; + + CUTLASS_DEVICE void + begin() { + if (!params.row_broadcast) { + fill(tSR_rRow, *(params.ptr_row_array[group])); + return; + } + + auto synchronize = [&] () { cutlass::arch::NamedBarrier::sync(thr_num, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); }; + Tensor tGS_gRow_flt = filter_zeros(tGS_gRow); + Tensor tGS_sRow_flt = filter_zeros(tGS_sRow); + Tensor tGS_cRow_flt = make_tensor(tGS_cRow.data(), make_layout(tGS_gRow_flt.shape(), tGS_cRow.stride())); + + for (int i = 0; i < size(tGS_gRow_flt); ++i) { + if (get<1>(tGS_cRow_flt(i)) >= size<1>(CtaTileShapeMNK{})) { + continue; // OOB of SMEM, + } + if (elem_less(tGS_cRow_flt(i), make_coord(get<0>(residue_tCcRow), get<1>(residue_tCcRow)))) { + tGS_sRow_flt(i) = tGS_gRow_flt(i); + } + else { + tGS_sRow_flt(i) = Element(0); // Set to Zero when OOB so LDS could be issue without any preds. + } + } + synchronize(); + } + + CUTLASS_DEVICE void + begin_loop(int epi_m, int epi_n) { + if (epi_m == 0) { // Assumes M-major subtile loop + if (!params.row_broadcast) return; // Do not issue LDS when row is scalar + Tensor tSR_sRow_flt = filter_zeros(tSR_sRow(_,_,_,epi_m,epi_n)); + Tensor tSR_rRow_flt = filter_zeros(tSR_rRow); + copy(tSR_sRow_flt, tSR_rRow_flt); + } + } + + template + CUTLASS_DEVICE Array + visit(Array const& frg_acc, int epi_v, int epi_m, int epi_n) { + Array frg_row; + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < FragmentSize; ++i) { + frg_row[i] = tSR_rRow(epi_v * FragmentSize + i); + } + + return frg_row; + } + }; + + template < + bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy + class... Args + > + CUTLASS_DEVICE auto + get_consumer_store_callbacks(ConsumerStoreArgs const& args) { + auto [M, N, K, L] = args.problem_shape_mnkl; + auto [m, n, k, l] = args.tile_coord_mnkl; + using ThreadCount = decltype(size(args.tiled_copy)); + + Tensor mRow = make_tensor(make_gmem_ptr(params.ptr_row_array[l]), make_shape(M,N,1), params.dRow); + Tensor gRow = local_tile(mRow(_,_,l), take<0,2>(args.tile_shape_mnk), make_coord(m, n)); // (CTA_M, CTA_N) + Tensor sRow = make_tensor(make_smem_ptr(smem), + make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{})), make_shape(_0{}, _1{})); // (CTA_M, CTA_N) + //// G2S: Gmem to Smem + auto tiled_g2s = make_tiled_copy(Copy_Atom{}, + Layout< Shape<_1, ThreadCount>, + Stride<_0, _1>>{}, + Layout<_1>{}); + auto thr_g2s = tiled_g2s.get_slice(args.thread_idx); + Tensor tGS_gRow = thr_g2s.partition_S(gRow); + Tensor tGS_sRow = thr_g2s.partition_D(sRow); + + //// G2S: Coord + auto cRow = make_identity_tensor(make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}))); + Tensor tGS_cRow = thr_g2s.partition_S(cRow); + + //// S2R: Smem to Reg + Tensor tSR_sRow = sm90_partition_for_epilogue(sRow, args.epi_tile, args.tiled_copy, args.thread_idx); + Tensor tSR_rRow = make_tensor_like(take<0,3>(tSR_sRow)); // (CPY,CPY_M,CPY_N) + + return ConsumerStoreCallbacks( + tGS_gRow, + tGS_sRow, + tGS_cRow, tiled_g2s, + tSR_sRow, + tSR_rRow, + args.tCcD, + args.residue_cD, + ThreadCount{}, + l, + params); + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +// Column vector broadcast +template< + int Stages, + class CtaTileShapeMNK, + class Element, + class StrideMNL = Stride<_1,_0,_0>, + int Alignment = 128 / sizeof_bits_v +> +struct Sm90ColOrScalarBroadcastArray { + static_assert(Stages == 0, "Column broadcast doesn't support smem usage yet"); + static_assert(Alignment * sizeof_bits_v % 128 == 0, "sub-16B alignment not supported yet"); + static_assert( + (cute::is_same_v>) || // col vector broadcast, e.g. per-row alpha/bias + (cute::is_same_v>)); // batched col vector broadcast, e.g. batched per-row bias + + // Accumulator distributes col elements evenly amongst threads so we can just directly load from gmem + struct SharedStorage { }; + + // This struct has been modified to have a bool indicating that ptr_col is a + // scalar that must be broadcast, instead of containing a scalar that is + // valid if ptr_col is null. + struct Arguments { + const Element* const* ptr_col_array = nullptr; + bool col_broadcast = true; + StrideMNL dCol = {}; + }; + + using Params = Arguments; + + template + static constexpr Params + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) { + return args; + } + + template + static bool + can_implement(ProblemShape const& problem_shape, Arguments const& args) { + return true; + } + + template + static size_t + get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) { + return 0; + } + + template + static cutlass::Status + initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream, + CudaHostAdapter* cuda_adapter = nullptr) { + return cutlass::Status::kSuccess; + } + + CUTLASS_DEVICE bool + is_producer_load_needed() const { + return false; + } + + CUTLASS_DEVICE bool + is_C_load_needed() const { + return false; + } + + CUTLASS_DEVICE bool + is_zero() const { + return (!params.col_broadcast && *(params.ptr_col_array[group]) == Element(0)); + } + + CUTLASS_HOST_DEVICE + Sm90ColOrScalarBroadcastArray() { } + + CUTLASS_HOST_DEVICE + Sm90ColOrScalarBroadcastArray(Params const& params, SharedStorage const& shared_storage) + : params(params) { } + + Params params; + + template + CUTLASS_DEVICE auto + get_producer_load_callbacks(ProducerLoadArgs const& args) { + return EmptyProducerLoadCallbacks{}; + } + + template + struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks { + CUTLASS_DEVICE + ConsumerStoreCallbacks( + GTensor&& tCgCol, + RTensor&& tCrCol, + CTensor&& tCcCol, + ProblemShape problem_shape, + int group, + Params const& params + ): + tCgCol(cute::forward(tCgCol)), + tCrCol(cute::forward(tCrCol)), + tCcCol(cute::forward(tCcCol)), + m(get<0>(problem_shape)), + group(group), + params(params) {} + + GTensor tCgCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + RTensor tCrCol; + CTensor tCcCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + Params const& params; + int m; + int group; + + CUTLASS_DEVICE void + begin() { + Tensor pred = make_tensor(shape(tCgCol)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(pred); ++i) { + pred(i) = get<0>(tCcCol(i)) < m; + } + + if (!params.col_broadcast) { + fill(tCrCol, *(params.ptr_col_array[group])); + return; + } + + // Filter so we don't issue redundant copies over stride-0 modes + // (only works if 0-strides are in same location, which is by construction) + copy_if(pred, filter(tCgCol), filter(tCrCol)); + } + + template + CUTLASS_DEVICE Array + visit(Array const& frg_acc, int epi_v, int epi_m, int epi_n) { + Array frg_col; + Tensor tCrCol_mn = tCrCol(_,_,_,epi_m,epi_n); + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < FragmentSize; ++i) { + frg_col[i] = tCrCol_mn(epi_v * FragmentSize + i); + } + + return frg_col; + } + + }; + + template < + bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy + class... Args + > + CUTLASS_DEVICE auto + get_consumer_store_callbacks(ConsumerStoreArgs const& args) { + + auto [M, N, K, L] = args.problem_shape_mnkl; + auto [m, n, k, l] = args.tile_coord_mnkl; + + Tensor mCol = make_tensor(make_gmem_ptr(params.ptr_col_array[l]), make_shape(M,N,1), params.dCol); + Tensor tCgCol = sm90_partition_for_epilogue( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + mCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx); + Tensor tCrCol = make_tensor_like(tCgCol); // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + + // Generate an identity tensor matching the shape of the global tensor and + // partition the same way, this will be used to generate the predicate + // tensor for loading + Tensor cCol = make_identity_tensor(mCol.shape()); + Tensor tCcCol = sm90_partition_for_epilogue( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) + cCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx); + + return ConsumerStoreCallbacks( + cute::move(tCgCol), + cute::move(tCrCol), + cute::move(tCcCol), + args.problem_shape_mnkl, + l, + params + ); + } +}; + +} diff --git a/csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp b/csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp index 0a812dc56a994..62b848a0a9635 100644 --- a/csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp +++ b/csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp @@ -1,6 +1,7 @@ #pragma once #include "cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp" +#include "cutlass_extensions/epilogue/broadcast_load_epilogue_array_c3x.hpp" /* This file defines custom epilogues for fusing channel scales, token scales, @@ -69,6 +70,16 @@ struct ScaledEpilogueBase { 0 /*Stages*/, TileShape, T, T, Stride, Int<1>, Int<0>>, 128 / sizeof_bits_v, EnableNullPtr>; + template + using ColOrScalarLoadArray = + cutlass::epilogue::fusion::Sm90ColOrScalarBroadcastArray< + 0 /*Stages*/, TileShape, T, Stride, Int<0>, Int<0>>>; + + template + using RowOrScalarLoadArray = + cutlass::epilogue::fusion::Sm90RowOrScalarBroadcastArray< + 0 /*Stages*/, TileShape, T, Stride, Int<1>, Int<0>>>; + // This utility function constructs the arguments for the load descriptors // from a tensor. It can handle both row and column, as well as row/column or // scalar cases. @@ -96,6 +107,14 @@ struct ScaledEpilogueBase { std::is_same_v>); return Arguments{data_ptr}; } + + template + static auto args_from_tensor(const T* const* data_ptr, bool do_broadcast) { + using Arguments = typename Descriptor::Arguments; + static_assert(std::is_same_v> || + std::is_same_v>); + return Arguments{data_ptr, do_broadcast}; + } }; /* @@ -381,4 +400,51 @@ struct ScaledEpilogueBiasAzpToken } }; +/* + This epilogue works like ScaledEpilogue, but ScaleA and ScaleB are pointers + to arrays containing different scales used in group gemm. The number of + pointers in ScaleA and the number of pointers in ScaleB are equal to the + group size. +*/ +template +struct ScaledEpilogueArray + : private ScaledEpilogueBase { + private: + using SUPER = ScaledEpilogueBase; + using Accum = typename SUPER::Accum; + using ScaleA = typename SUPER::template ColOrScalarLoadArray; + using ScaleB = typename SUPER::template RowOrScalarLoadArray; + + using Compute0 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, float, float, + cutlass::FloatRoundStyle::round_to_nearest>; + + using EVTCompute0 = + cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, ElementD, float, + cutlass::FloatRoundStyle::round_to_nearest>; + + public: + using EVTCompute = + cutlass::epilogue::fusion::Sm90EVT; + using ArgumentType = typename EVTCompute::Arguments; + + using ScaleAArray = typename SUPER::template ColOrScalarLoadArray; + using ScaleBArray = typename SUPER::template RowOrScalarLoadArray; + + static ArgumentType prepare_args(float const* const* a_scales_ptr, + float const* const* b_scales_ptr, + bool a_col_broadcast, bool b_row_broadcast) { + auto a_args = SUPER::template args_from_tensor( + a_scales_ptr, a_col_broadcast); + auto b_args = SUPER::template args_from_tensor( + b_scales_ptr, b_row_broadcast); + + typename EVTCompute0::Arguments evt0_args{b_args, {}, {}}; + return ArgumentType{a_args, evt0_args, {}}; + } +}; + }; // namespace vllm::c3x diff --git a/csrc/ops.h b/csrc/ops.h index 7434aead57f0e..1ea9f465cf21d 100644 --- a/csrc/ops.h +++ b/csrc/ops.h @@ -164,6 +164,7 @@ int64_t ggml_moe_get_block_size(int64_t type); bool cutlass_scaled_mm_supports_fp4(int64_t cuda_device_capability); bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability); bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability); +bool cutlass_group_gemm_supported(int64_t cuda_device_capability); void cutlass_scaled_fp4_mm(torch::Tensor& D, torch::Tensor const& A, torch::Tensor const& B, torch::Tensor const& A_sf, @@ -175,6 +176,19 @@ void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a, torch::Tensor const& b_scales, std::optional const& bias); +void cutlass_moe_mm( + torch::Tensor& out_tensors, torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, torch::Tensor const& a_strides, + torch::Tensor const& b_strides, torch::Tensor const& c_strides); + +void get_cutlass_moe_mm_data( + const torch::Tensor& topk_ids, torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, torch::Tensor& output_permutation, + const int64_t num_experts, const int64_t n, const int64_t k); + void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a, torch::Tensor const& b, torch::Tensor const& a_scales, diff --git a/csrc/quantization/cutlass_w8a8/moe/get_group_starts.cuh b/csrc/quantization/cutlass_w8a8/moe/get_group_starts.cuh new file mode 100644 index 0000000000000..6c6e89790847f --- /dev/null +++ b/csrc/quantization/cutlass_w8a8/moe/get_group_starts.cuh @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include + +#include "core/scalar_type.hpp" +#include "cutlass/bfloat16.h" +#include "cutlass/float8.h" + +template +__global__ void get_group_gemm_starts( + int32_t* expert_offsets, ElementAB** a_offsets, ElementAB** b_offsets, + ElementC** out_offsets, ElementAccumulator** a_scales_offsets, + ElementAccumulator** b_scales_offsets, ElementAB* a_base_as_int, + ElementAB* b_base_as_int, ElementC* out_base_as_int, + ElementAccumulator* a_scales_base_as_int, + ElementAccumulator* b_scales_base_as_int, int64_t n, int64_t k, + bool per_act_token, bool per_out_ch) { + int expert_id = threadIdx.x; + + int64_t expert_offset = expert_offsets[expert_id]; + + a_offsets[expert_id] = a_base_as_int + expert_offset * k; + b_offsets[expert_id] = b_base_as_int + expert_id * k * n; + out_offsets[expert_id] = out_base_as_int + expert_offset * n; + a_scales_offsets[expert_id] = + a_scales_base_as_int + (per_act_token ? expert_offset : 0); + b_scales_offsets[expert_id] = + b_scales_base_as_int + (per_out_ch ? n * expert_id : expert_id); +} + +#define __CALL_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + get_group_gemm_starts \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(expert_offsets.data_ptr()), \ + static_cast(a_ptrs.data_ptr()), \ + static_cast(b_ptrs.data_ptr()), \ + static_cast(out_ptrs.data_ptr()), \ + static_cast(a_scales_ptrs.data_ptr()), \ + static_cast(b_scales_ptrs.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), out_tensors.size(1), \ + a_tensors.size(1), per_act_token, per_out_ch); \ + } + +namespace { + +void run_get_group_gemm_starts( + torch::Tensor const& expert_offsets, torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, torch::Tensor& b_scales_ptrs, + torch::Tensor const& a_tensors, torch::Tensor const& b_tensors, + torch::Tensor& out_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales) { + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(a_scales.dtype() == torch::kFloat32); + TORCH_CHECK(b_scales.dtype() == torch::kFloat32); + + int num_experts = static_cast(expert_offsets.size(0)); + bool per_act_token = a_scales.numel() != 1; + bool per_out_ch = b_scales.numel() != num_experts; + + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + + if (false) { + } + __CALL_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t) + __CALL_GET_STARTS_KERNEL(torch::kFloat16, half) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } +} + +} // namespace \ No newline at end of file diff --git a/csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cu b/csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cu new file mode 100644 index 0000000000000..2b8bc3fb0b261 --- /dev/null +++ b/csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cu @@ -0,0 +1,160 @@ +#include + +#include +#include + +#include "cutlass/cutlass.h" +#include "grouped_mm_c3x.cuh" + +using namespace cute; + +namespace { + +template typename Epilogue> +struct sm90_fp8_config_default { + // M in (16, inf) + static_assert(std::is_same()); + using KernelSchedule = + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8FastAccum; + using EpilogueSchedule = + cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using TileShape = cute::Shape; + using ClusterShape = cute::Shape; + + using Cutlass3xGemm = + cutlass_3x_group_gemm; +}; + +template typename Epilogue> +struct sm90_fp8_config_M16 { + // M in [1, 16] + static_assert(std::is_same()); + using KernelSchedule = + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8FastAccum; + using EpilogueSchedule = + cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using TileShape = cute::Shape; + using ClusterShape = cute::Shape; + + using Cutlass3xGemm = + cutlass_3x_group_gemm; +}; + +template typename Epilogue> +struct sm90_fp8_config_K8192 { + // K in [8192, inf) + static_assert(std::is_same()); + using KernelSchedule = + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8FastAccum; + using EpilogueSchedule = + cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using TileShape = cute::Shape; + using ClusterShape = cute::Shape; + + using Cutlass3xGemm = + cutlass_3x_group_gemm; +}; + +template typename Epilogue> +struct sm90_fp8_config_N8192 { + // N in [8192, inf) + static_assert(std::is_same()); + using KernelSchedule = + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8FastAccum; + using EpilogueSchedule = + cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using TileShape = cute::Shape; + using ClusterShape = cute::Shape; + + using Cutlass3xGemm = + cutlass_3x_group_gemm; +}; + +template +void run_cutlass_moe_mm_sm90( + torch::Tensor& out_tensors, torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, torch::Tensor const& a_strides, + torch::Tensor const& b_strides, torch::Tensor const& c_strides) { + TORCH_CHECK(a_tensors.size(0) > 0, "No input A tensors provided."); + TORCH_CHECK(b_tensors.size(0) > 0, "No input B tensors provided."); + TORCH_CHECK(out_tensors.size(0) > 0, "No output tensors provided."); + + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn, + "A tensors must be of type float8_e4m3fn."); + TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn, + "B tensors must be of type float8_e4m3fn."); + + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn); + + using Cutlass3xGemmN8192 = typename sm90_fp8_config_N8192< + InType, OutType, vllm::c3x::ScaledEpilogueArray>::Cutlass3xGemm; + using Cutlass3xGemmK8192 = typename sm90_fp8_config_K8192< + InType, OutType, vllm::c3x::ScaledEpilogueArray>::Cutlass3xGemm; + using Cutlass3xGemmM16 = typename sm90_fp8_config_M16< + InType, OutType, vllm::c3x::ScaledEpilogueArray>::Cutlass3xGemm; + using Cutlass3xGemmDefault = typename sm90_fp8_config_default< + InType, OutType, vllm::c3x::ScaledEpilogueArray>::Cutlass3xGemm; + + uint32_t const m = a_tensors.size(0); + uint32_t const n = out_tensors.size(1); + uint32_t const k = a_tensors.size(1); + + if (n >= 8192) { + cutlass_group_gemm_caller( + out_tensors, a_tensors, b_tensors, a_scales, b_scales, expert_offsets, + problem_sizes, a_strides, b_strides, c_strides); + } else if (k >= 8192) { + cutlass_group_gemm_caller( + out_tensors, a_tensors, b_tensors, a_scales, b_scales, expert_offsets, + problem_sizes, a_strides, b_strides, c_strides); + } else if (m <= 16) { + cutlass_group_gemm_caller( + out_tensors, a_tensors, b_tensors, a_scales, b_scales, expert_offsets, + problem_sizes, a_strides, b_strides, c_strides); + } else { + cutlass_group_gemm_caller( + out_tensors, a_tensors, b_tensors, a_scales, b_scales, expert_offsets, + problem_sizes, a_strides, b_strides, c_strides); + } +} + +void dispatch_moe_mm_sm90( + torch::Tensor& out_tensors, torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, torch::Tensor const& a_strides, + torch::Tensor const& b_strides, torch::Tensor const& c_strides) { + if (out_tensors.dtype() == torch::kBFloat16) { + run_cutlass_moe_mm_sm90( + out_tensors, a_tensors, b_tensors, a_scales, b_scales, expert_offsets, + problem_sizes, a_strides, b_strides, c_strides); + } else { + run_cutlass_moe_mm_sm90( + out_tensors, a_tensors, b_tensors, a_scales, b_scales, expert_offsets, + problem_sizes, a_strides, b_strides, c_strides); + } +} + +} // namespace + +void cutlass_moe_mm_sm90( + torch::Tensor& out_tensors, torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, torch::Tensor const& a_strides, + torch::Tensor const& b_strides, torch::Tensor const& c_strides) { + dispatch_moe_mm_sm90(out_tensors, a_tensors, b_tensors, a_scales, b_scales, + expert_offsets, problem_sizes, a_strides, b_strides, + c_strides); +} diff --git a/csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cuh b/csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cuh new file mode 100644 index 0000000000000..db827b7c5e186 --- /dev/null +++ b/csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x.cuh @@ -0,0 +1,149 @@ +#pragma once + +#include "cutlass/cutlass.h" + +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" + +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "cutlass_extensions/common.hpp" +#include "get_group_starts.cuh" + +using namespace cute; + +namespace { + +using ProblemShape = + cutlass::gemm::GroupProblemShape>; + +using ElementAccumulator = float; +using ArchTag = cutlass::arch::Sm90; +using OperatorClass = cutlass::arch::OpClassTensorOp; + +using LayoutA = cutlass::layout::RowMajor; +using LayoutB = cutlass::layout::ColumnMajor; +using LayoutC = cutlass::layout::RowMajor; + +template typename Epilogue_, + typename TileShape, typename ClusterShape, typename KernelSchedule, + typename EpilogueSchedule> +struct cutlass_3x_group_gemm { + using ElementAB = ElementAB_; + using ElementC = void; + using ElementD = ElementC_; + using ElementAccumulator = float; + + using Epilogue = Epilogue_; + + using StrideC = + cute::remove_pointer_t, cute::Int<0>>>; + + static constexpr int AlignmentAB = + 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using EVTCompute = typename Epilogue::EVTCompute; + + using CollectiveEpilogue = + typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, OperatorClass, TileShape, ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, ElementAccumulator, + ElementAccumulator, ElementC, LayoutC*, AlignmentC, ElementD, + LayoutC*, AlignmentC, EpilogueSchedule, EVTCompute>::CollectiveOp; + + static constexpr size_t CEStorageSize = + sizeof(typename CollectiveEpilogue::SharedStorage); + using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout< + static_cast(CEStorageSize)>; + + using CollectiveMainloop = + typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, OperatorClass, ElementAB, LayoutA*, AlignmentAB, ElementAB, + LayoutB*, AlignmentAB, ElementAccumulator, TileShape, ClusterShape, + Stages, KernelSchedule>::CollectiveOp; + + using KernelType = enable_sm90_only>; + + struct GemmKernel : public KernelType {}; +}; + +template +void cutlass_group_gemm_caller( + torch::Tensor& out_tensors, torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, torch::Tensor const& a_strides, + torch::Tensor const& b_strides, torch::Tensor const& c_strides) { + using ElementAB = typename Gemm::ElementAB; + using ElementD = typename Gemm::ElementD; + + int num_experts = static_cast(expert_offsets.size(0)); + int k_size = a_tensors.size(1); + int n_size = out_tensors.size(1); + + bool per_act_token = a_scales.numel() != 1; + bool per_out_ch = b_scales.numel() != num_experts; + + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + + auto options_int = + torch::TensorOptions().dtype(torch::kInt64).device(a_tensors.device()); + + torch::Tensor a_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_ptrs = torch::empty(num_experts, options_int); + torch::Tensor out_ptrs = torch::empty(num_experts, options_int); + torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int); + + run_get_group_gemm_starts(expert_offsets, a_ptrs, b_ptrs, out_ptrs, + a_scales_ptrs, b_scales_ptrs, a_tensors, b_tensors, + out_tensors, a_scales, b_scales); + + using GemmKernel = typename Gemm::GemmKernel; + using StrideA = Stride, Int<0>>; + using StrideB = Stride, Int<0>>; + using StrideC = typename GemmKernel::InternalStrideC; + + ProblemShape::UnderlyingProblemShape* problem_sizes_as_shapes = + static_cast( + problem_sizes.data_ptr()); + ProblemShape prob_shape{num_experts, problem_sizes_as_shapes, nullptr}; + + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(a_strides.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(b_strides.data_ptr())}; + + // Currently, we are only able to do broadcast on either all or none a_scales + // and on either all or none b_scales + typename GemmKernel::EpilogueArguments epilogue_args{ + Gemm::Epilogue::prepare_args( + static_cast(a_scales_ptrs.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + per_act_token, per_out_ch), + nullptr, static_cast(c_strides.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(c_strides.data_ptr())}; + + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, prob_shape, mainloop_args, + epilogue_args}; + + using GemmOp = cutlass::gemm::device::GemmUniversalAdapter; + GemmOp gemm_op; + CUTLASS_CHECK(gemm_op.can_implement(args)); + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = + torch::TensorOptions().dtype(torch::kUInt8).device(a_tensors.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + + cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream); + CUTLASS_CHECK(status); +} + +} // namespace diff --git a/csrc/quantization/cutlass_w8a8/moe/moe_data.cu b/csrc/quantization/cutlass_w8a8/moe/moe_data.cu new file mode 100644 index 0000000000000..2fb0417ce6c41 --- /dev/null +++ b/csrc/quantization/cutlass_w8a8/moe/moe_data.cu @@ -0,0 +1,90 @@ +#include + +#include +#include + +#include + +constexpr uint64_t THREADS_PER_EXPERT = 512; + +__global__ void compute_problem_sizes(const int* __restrict__ topk_ids, + int32_t* problem_sizes1, + int32_t* problem_sizes2, + int32_t* atomic_buffer, + const int topk_length, const int n, + const int k) { + int expert_id = blockIdx.x; + + int occurrences = 0; + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + occurrences += (topk_ids[i] == expert_id); + } + atomicAdd(&atomic_buffer[expert_id], occurrences); + __syncthreads(); + + if (threadIdx.x == 0) { + int final_occurrences = atomic_buffer[expert_id]; + problem_sizes1[expert_id * 3] = final_occurrences; + problem_sizes1[expert_id * 3 + 1] = 2 * n; + problem_sizes1[expert_id * 3 + 2] = k; + problem_sizes2[expert_id * 3] = final_occurrences; + problem_sizes2[expert_id * 3 + 1] = k; + problem_sizes2[expert_id * 3 + 2] = n; + } +} + +__global__ void compute_expert_offsets( + const int32_t* __restrict__ problem_sizes1, int32_t* expert_offsets, + int32_t* atomic_buffer, const int num_experts) { + int32_t tot_offset = 0; + expert_offsets[0] = 0; + for (int i = 0; i < num_experts; ++i) { + atomic_buffer[i] = tot_offset; + tot_offset += problem_sizes1[i * 3]; + expert_offsets[i + 1] = tot_offset; + } +} + +__global__ void compute_arg_sorts(const int* __restrict__ topk_ids, + int32_t* input_permutation, + int32_t* output_permutation, + int32_t* atomic_buffer, const int topk_length, + const int topk) { + int expert_id = blockIdx.x; + + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + if (topk_ids[i] == expert_id) { + int start = atomicAdd(&atomic_buffer[expert_id], 1); + input_permutation[start] = i / topk; + output_permutation[i] = start; + } + } +} + +void get_cutlass_moe_mm_data_caller( + const torch::Tensor& topk_ids, torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, torch::Tensor& output_permutation, + const int64_t num_experts, const int64_t n, const int64_t k) { + auto stream = at::cuda::getCurrentCUDAStream(topk_ids.device().index()); + auto options_int32 = + torch::TensorOptions().dtype(torch::kInt32).device(topk_ids.device()); + torch::Tensor atomic_buffer = torch::zeros(num_experts, options_int32); + + int num_threads = min(THREADS_PER_EXPERT, topk_ids.numel()); + compute_problem_sizes<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(problem_sizes1.data_ptr()), + static_cast(problem_sizes2.data_ptr()), + static_cast(atomic_buffer.data_ptr()), topk_ids.numel(), n, k); + compute_expert_offsets<<<1, 1, 0, stream>>>( + static_cast(problem_sizes1.data_ptr()), + static_cast(expert_offsets.data_ptr()), + static_cast(atomic_buffer.data_ptr()), num_experts); + compute_arg_sorts<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(input_permutation.data_ptr()), + static_cast(output_permutation.data_ptr()), + static_cast(atomic_buffer.data_ptr()), topk_ids.numel(), + topk_ids.size(1)); +} diff --git a/csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu b/csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu index b08386459cbe2..54b63894e4cbc 100644 --- a/csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu +++ b/csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu @@ -29,6 +29,20 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a, torch::Tensor const& a_scales, torch::Tensor const& b_scales, std::optional const& bias); + +void cutlass_moe_mm_sm90( + torch::Tensor& out_tensors, torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, torch::Tensor const& a_strides, + torch::Tensor const& b_strides, torch::Tensor const& c_strides); + +void get_cutlass_moe_mm_data_caller( + const torch::Tensor& topk_ids, torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, torch::Tensor& output_permutation, + const int64_t num_experts, const int64_t n, const int64_t k); + #endif #if defined ENABLE_SCALED_MM_SM100 && ENABLE_SCALED_MM_SM100 @@ -102,6 +116,19 @@ bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability) { return false; } +bool cutlass_group_gemm_supported(int64_t cuda_device_capability) { + // CUTLASS groped FP8 kernels need at least CUDA 12.3 + // and SM90 (Hopper) + +#if defined CUDA_VERSION + if (cuda_device_capability == 90) { + return CUDA_VERSION >= 12030; + } +#endif + + return false; +} + void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a, torch::Tensor const& b, torch::Tensor const& a_scales, torch::Tensor const& b_scales, @@ -168,6 +195,46 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a, version_num); } +void cutlass_moe_mm( + torch::Tensor& out_tensors, torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, torch::Tensor const& a_strides, + torch::Tensor const& b_strides, torch::Tensor const& c_strides) { + int32_t version_num = get_sm_version_num(); +#if defined ENABLE_CUTLASS_MOE_SM90 && ENABLE_CUTLASS_MOE_SM90 + cutlass_moe_mm_sm90(out_tensors, a_tensors, b_tensors, a_scales, b_scales, + expert_offsets, problem_sizes, a_strides, b_strides, + c_strides); + return; +#endif + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "No compiled cutlass_scaled_mm for CUDA device capability: ", version_num, + ". Required capability: 90"); +} + +void get_cutlass_moe_mm_data( + const torch::Tensor& topk_ids, torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, torch::Tensor& output_permutation, + const int64_t num_experts, const int64_t n, const int64_t k) { + // This function currently gets compiled only if we have a valid cutlass moe + // mm to run it for. + int32_t version_num = get_sm_version_num(); +#if defined ENABLE_CUTLASS_MOE_SM90 && ENABLE_CUTLASS_MOE_SM90 + get_cutlass_moe_mm_data_caller(topk_ids, expert_offsets, problem_sizes1, + problem_sizes2, input_permutation, + output_permutation, num_experts, n, k); + return; +#endif + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "No compiled get_cutlass_moe_mm_data: no cutlass_scaled_mm kernel for " + "CUDA device capability: ", + version_num, ". Required capability: 90"); +} + void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a, torch::Tensor const& b, torch::Tensor const& a_scales, diff --git a/csrc/torch_bindings.cpp b/csrc/torch_bindings.cpp index eb3a2c911d55e..60ad6430336a5 100644 --- a/csrc/torch_bindings.cpp +++ b/csrc/torch_bindings.cpp @@ -365,6 +365,35 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool"); ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8); + // Check if cutlass grouped gemm is supported for CUDA devices of the given + // capability + ops.def("cutlass_group_gemm_supported(int cuda_device_capability) -> bool"); + ops.impl("cutlass_group_gemm_supported", &cutlass_group_gemm_supported); + + // CUTLASS w8a8 grouped GEMM + ops.def( + "cutlass_moe_mm(Tensor! out_tensors, Tensor a_tensors, Tensor b_tensors, " + " Tensor a_scales, Tensor b_scales, Tensor expert_offsets, " + " Tensor problem_sizes, Tensor a_strides, " + " Tensor b_strides, Tensor c_strides) -> ()", + {stride_tag}); + ops.impl("cutlass_moe_mm", torch::kCUDA, &cutlass_moe_mm); + + // A function that computes data required to run fused MoE with w8a8 grouped + // GEMM. It takes topk_ids as an input, and computes expert_offsets + // (token start indices of each expert). In addition to this, it computes + // problem sizes for each expert's multiplication used by the two mms called + // from fused MoE operation, and arrays with permutations required to shuffle + // and de-shuffle the input/output of the fused operation. + ops.def( + "get_cutlass_moe_mm_data(Tensor topk_ids, Tensor! expert_offsets, " + " Tensor! problem_sizes1, Tensor! problem_sizes2, " + " Tensor! input_permutation, " + " Tensor! output_permutation, int num_experts, " + " int n, int k) -> ()", + {stride_tag}); + ops.impl("get_cutlass_moe_mm_data", torch::kCUDA, &get_cutlass_moe_mm_data); + // Check if cutlass scaled_mm supports block quantization (used by DeepSeekV3) ops.def( "cutlass_scaled_mm_supports_block_fp8(int cuda_device_capability) -> " diff --git a/tests/kernels/test_cutlass.py b/tests/kernels/test_cutlass.py index 72fc660a653d5..f11ce6f45a984 100644 --- a/tests/kernels/test_cutlass.py +++ b/tests/kernels/test_cutlass.py @@ -3,6 +3,7 @@ Run `pytest tests/kernels/test_cutlass.py`. """ +import random import pytest import torch @@ -507,3 +508,136 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool): def test_cutlass_support_opcheck(): opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability, )) + + +@pytest.mark.parametrize("num_experts", [8, 64]) +@pytest.mark.parametrize("per_act_token", [True, False]) +@pytest.mark.parametrize("per_out_ch", [True, False]) +@pytest.mark.parametrize("use_bias", [False]) +@pytest.mark.skipif( + (lambda x: x is None or not ops.cutlass_group_gemm_supported(x.to_int()))( + current_platform.get_device_capability()), + reason="Grouped gemm is not supported on this GPU type.") +def test_cutlass_fp8_group_gemm(num_experts: int, per_act_token: bool, + per_out_ch: bool, use_bias: bool): + + # Device and dtype setup + device = "cuda" + out_dtype = torch.half + + # Create separate A, B, C tensors for each group + a_tensors = [] + b_tensors = [] + a_scales_tensors = [] + b_scales_tensors = [] + baseline_tensors = [] + + expert_offsets = torch.zeros((num_experts + 1), + device=device, + dtype=torch.int32) + + problem_sizes = torch.zeros((num_experts, 3), + device=device, + dtype=torch.int32) + + if not per_act_token: + one_scale_a = torch.randn((1, 1), device=device, dtype=torch.float32) + + alignment = 16 # 128 // 8 + # For variation, each group has dimensions + n_g = alignment * random.randint(1, 64) + k_g = alignment * random.randint(1, 64) + for g in range(num_experts): + m_g = alignment * random.randint(1, 64) + + expert_offsets[g + 1] = expert_offsets[g] + m_g + problem_sizes[g][0] = m_g + problem_sizes[g][1] = n_g + problem_sizes[g][2] = k_g + + m_a_scales = m_g if per_act_token else 1 + n_b_scales = n_g if per_out_ch else 1 + + print("shape:", m_g, n_g, k_g) + + # Create group-specific A and B (FP8) and output (FP16/FP32) + a_g = to_fp8(torch.randn((m_g, k_g), device=device)) + b_g = to_fp8(torch.randn((n_g, k_g), device=device).t()) + a_tensors.append(a_g) + b_tensors.append(b_g) + + # Set up A/B scales + scale_b = torch.randn((1, n_b_scales), + device=device, + dtype=torch.float32) + b_scales_tensors.append(scale_b) + + if per_act_token: + scale_a = torch.randn((m_a_scales, 1), + device=device, + dtype=torch.float32) + a_scales_tensors.append(scale_a) + else: + scale_a = one_scale_a + + # Compute baseline result for this group + baseline_g = baseline_scaled_mm(a_g, b_g, scale_a, scale_b, out_dtype, + None) + baseline_tensors.append(baseline_g) + + a_tensors_stacked = torch.empty((expert_offsets[num_experts], k_g), + device=device, + dtype=torch.float8_e4m3fn) + b_tensors_stacked = torch.empty((num_experts, n_g, k_g), + device=device, + dtype=torch.float8_e4m3fn) + + for g in range(num_experts): + a_tensors_stacked[expert_offsets[g]:expert_offsets[g + + 1]] = a_tensors[g] + b_tensors_stacked[g] = b_tensors[g].t() + b_tensors_stacked = b_tensors_stacked.transpose(1, 2) + + if per_act_token: + a_scales_tensors_stacked = torch.empty( + (expert_offsets[num_experts], 1), + device=device, + dtype=torch.float32) + for g in range(num_experts): + a_scales_tensors_stacked[ + expert_offsets[g]:expert_offsets[g + 1]] = a_scales_tensors[g] + else: + a_scales_tensors_stacked = one_scale_a + + b_scales_tensors_stacked = torch.empty((num_experts, n_b_scales), + device=device, + dtype=torch.float32) + for g in range(num_experts): + b_scales_tensors_stacked[g] = b_scales_tensors[g] + + out_tensors_stacked = torch.zeros((expert_offsets[num_experts], n_g), + device=device, + dtype=out_dtype) + + ab_strides = torch.full((num_experts, ), + a_tensors_stacked.stride(0), + device="cuda", + dtype=torch.int64) + c_strides = torch.full((num_experts, ), + out_tensors_stacked.stride(0), + device="cuda", + dtype=torch.int64) + + ops.cutlass_moe_mm(out_tensors_stacked, a_tensors_stacked, + b_tensors_stacked, a_scales_tensors_stacked, + b_scales_tensors_stacked, expert_offsets[:-1], + problem_sizes, ab_strides, ab_strides, c_strides) + + # Validate each group's result against the baseline + for g in range(num_experts): + baseline = baseline_tensors[g] + c = out_tensors_stacked[expert_offsets[g]:expert_offsets[g + 1]] + print(baseline) + print(c) + print("*") + torch.testing.assert_close(c, baseline, rtol=1e-2, atol=5e-4) diff --git a/tests/kernels/test_cutlass_moe.py b/tests/kernels/test_cutlass_moe.py new file mode 100644 index 0000000000000..1652c72d86fe1 --- /dev/null +++ b/tests/kernels/test_cutlass_moe.py @@ -0,0 +1,244 @@ +# SPDX-License-Identifier: Apache-2.0 +import pytest +import torch + +from vllm import _custom_ops as ops +from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.fused_moe.fused_moe import (cutlass_moe_fp8, + fused_experts, + fused_topk) +from vllm.platforms import current_platform + +NUM_EXPERTS = [40, 64] +TOP_KS = [6, 8] + + +def run(a: torch.Tensor, a_scale: torch.Tensor, w1_q: torch.Tensor, + w2_q: torch.Tensor, w1_scale: torch.Tensor, w2_scale: torch.Tensor, + topk_weights: torch.Tensor, topk_ids: torch.Tensor, + ab_strides1: torch.Tensor, c_strides1: torch.Tensor, + ab_strides2: torch.Tensor, c_strides2: torch.Tensor): + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig( + pipeline_parallel_size=1))): + return cutlass_moe_fp8(a, + w1_q, + w2_q, + w1_scale, + w2_scale, + topk_weights, + topk_ids, + ab_strides1, + c_strides1, + ab_strides2, + c_strides2, + a1_scale=a_scale) + + +@pytest.mark.parametrize("m", [2, 64, 224]) +@pytest.mark.parametrize("n", [1024, 3072]) +@pytest.mark.parametrize("k", [1024, 1536]) +@pytest.mark.parametrize("e", NUM_EXPERTS) +@pytest.mark.parametrize("topk", TOP_KS) +@pytest.mark.parametrize("per_act_token", [True, False]) +@pytest.mark.parametrize("per_out_ch", [True, False]) +@pytest.mark.skipif( + (lambda x: x is None or not ops.cutlass_group_gemm_supported(x.to_int()))( + current_platform.get_device_capability()), + reason="Grouped gemm is not supported on this GPU type.") +def test_cutlass_moe_no_graph( + m: int, + n: int, + k: int, + e: int, + topk: int, + per_act_token: bool, + per_out_ch: bool, +): + current_platform.seed_everything(7) + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig( + pipeline_parallel_size=1))): + + dtype = torch.half + + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + + # Get the right scale for tests. + _, a_scale1 = ops.scaled_fp8_quant( + a, use_per_token_if_dynamic=per_act_token) + a_q, _ = ops.scaled_fp8_quant(a, + a_scale1, + use_per_token_if_dynamic=per_act_token) + + a_d = a_q.float().mul(a_scale1).to(dtype) + + n_b_scales = 2 * n if per_out_ch else 1 + k_b_scales = k if per_out_ch else 1 + + w1_q = torch.empty((e, 2 * n, k), + device="cuda", + dtype=torch.float8_e4m3fn) + w2_q = torch.empty((e, k, n), device="cuda", dtype=torch.float8_e4m3fn) + w1_scale = torch.empty((e, n_b_scales, 1), + device="cuda", + dtype=torch.float32) + w2_scale = torch.empty((e, k_b_scales, 1), + device="cuda", + dtype=torch.float32) + + ab_strides1 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + c_strides1 = torch.full((e, ), 2 * n, device="cuda", dtype=torch.int64) + ab_strides2 = torch.full((e, ), n, device="cuda", dtype=torch.int64) + c_strides2 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + + for expert in range(e): + w1_q[expert], w1_scale[expert] = ops.scaled_fp8_quant( + w1[expert], use_per_token_if_dynamic=per_out_ch) + w2_q[expert], w2_scale[expert] = ops.scaled_fp8_quant( + w2[expert], use_per_token_if_dynamic=per_out_ch) + w1_q = w1_q.transpose(1, 2) + w2_q = w2_q.transpose(1, 2) + + ab_strides1 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + c_strides1 = torch.full((e, ), 2 * n, device="cuda", dtype=torch.int64) + ab_strides2 = torch.full((e, ), n, device="cuda", dtype=torch.int64) + c_strides2 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + + w1_d = torch.empty_like(w1) + w2_d = torch.empty_like(w2) + for expert in range(e): + w1_d[expert] = (w1_q[expert].t().float() * w1_scale[expert]).half() + w2_d[expert] = (w2_q[expert].t().float() * w2_scale[expert]).half() + + score = torch.randn((m, e), device="cuda", dtype=dtype) + topk_weights, topk_ids = fused_topk(a, score, topk, renormalize=False) + + triton_output = fused_experts(a_d, w1_d, w2_d, topk_weights, topk_ids) + + cutlass_output = cutlass_moe_fp8(a, + w1_q, + w2_q, + w1_scale, + w2_scale, + topk_weights, + topk_ids, + ab_strides1, + c_strides1, + ab_strides2, + c_strides2, + a1_scale=a_scale1) + + print(triton_output) + print(cutlass_output) + print("*") + + torch.testing.assert_close(triton_output, + cutlass_output, + atol=5e-2, + rtol=1e-2) + + +@pytest.mark.parametrize("m", [2, 64, 224]) +@pytest.mark.parametrize("n", [1024, 3072]) +@pytest.mark.parametrize("k", [1024, 1536]) +@pytest.mark.parametrize("e", NUM_EXPERTS) +@pytest.mark.parametrize("topk", TOP_KS) +@pytest.mark.parametrize("per_act_token", [True, False]) +@pytest.mark.parametrize("per_out_ch", [True, False]) +@pytest.mark.skipif( + (lambda x: x is None or not ops.cutlass_group_gemm_supported(x.to_int()))( + current_platform.get_device_capability()), + reason="Grouped gemm is not supported on this GPU type.") +def test_cutlass_moe_cuda_graph( + m: int, + n: int, + k: int, + e: int, + topk: int, + per_act_token: bool, + per_out_ch: bool, +): + current_platform.seed_everything(7) + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig( + pipeline_parallel_size=1))): + + dtype = torch.half + + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + + # Get the right scale for tests. + _, a_scale1 = ops.scaled_fp8_quant( + a, use_per_token_if_dynamic=per_act_token) + a_q, _ = ops.scaled_fp8_quant(a, + a_scale1, + use_per_token_if_dynamic=per_act_token) + + a_d = a_q.float().mul(a_scale1).to(dtype) + + n_b_scales = 2 * n if per_out_ch else 1 + k_b_scales = k if per_out_ch else 1 + + w1_q = torch.empty((e, 2 * n, k), + device="cuda", + dtype=torch.float8_e4m3fn) + w2_q = torch.empty((e, k, n), device="cuda", dtype=torch.float8_e4m3fn) + w1_scale = torch.empty((e, n_b_scales, 1), + device="cuda", + dtype=torch.float32) + w2_scale = torch.empty((e, k_b_scales, 1), + device="cuda", + dtype=torch.float32) + + ab_strides1 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + c_strides1 = torch.full((e, ), 2 * n, device="cuda", dtype=torch.int64) + ab_strides2 = torch.full((e, ), n, device="cuda", dtype=torch.int64) + c_strides2 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + + for expert in range(e): + w1_q[expert], w1_scale[expert] = ops.scaled_fp8_quant( + w1[expert], use_per_token_if_dynamic=per_out_ch) + w2_q[expert], w2_scale[expert] = ops.scaled_fp8_quant( + w2[expert], use_per_token_if_dynamic=per_out_ch) + w1_q = w1_q.transpose(1, 2) + w2_q = w2_q.transpose(1, 2) + + ab_strides1 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + c_strides1 = torch.full((e, ), 2 * n, device="cuda", dtype=torch.int64) + ab_strides2 = torch.full((e, ), n, device="cuda", dtype=torch.int64) + c_strides2 = torch.full((e, ), k, device="cuda", dtype=torch.int64) + + w1_d = torch.empty_like(w1) + w2_d = torch.empty_like(w2) + for expert in range(e): + w1_d[expert] = (w1_q[expert].t().float() * w1_scale[expert]).half() + w2_d[expert] = (w2_q[expert].t().float() * w2_scale[expert]).half() + + score = torch.randn((m, e), device="cuda", dtype=dtype) + topk_weights, topk_ids = fused_topk(a, score, topk, renormalize=False) + + triton_output = fused_experts(a_d, w1_d, w2_d, topk_weights, topk_ids) + + stream = torch.cuda.Stream() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + cutlass_output = run(a, a_scale1, w1_q, w2_q, w1_scale, w2_scale, + topk_weights, topk_ids, ab_strides1, + c_strides1, ab_strides2, c_strides2) + torch.cuda.synchronize() + graph.replay() + torch.cuda.synchronize() + + print(triton_output) + print(cutlass_output) + print("*") + + torch.testing.assert_close(triton_output, + cutlass_output, + atol=9e-2, + rtol=1e-2) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index dc07bad4680f9..2ffcef414cb28 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -587,6 +587,9 @@ def cutlass_sparse_scaled_mm_supported(cuda_device_capability: int) -> bool: cuda_device_capability) +def cutlass_group_gemm_supported(cuda_device_capability: int) -> bool: + return torch.ops._C.cutlass_group_gemm_supported(cuda_device_capability) + def cutlass_sparse_compress(a: torch.Tensor) \ -> tuple[torch.Tensor, torch.Tensor]: """ @@ -677,6 +680,56 @@ def cutlass_scaled_sparse_mm( return out +def get_cutlass_moe_mm_data( + topk_ids: torch.Tensor, expert_offsets: torch.Tensor, + problem_sizes1: torch.Tensor, problem_sizes2: torch.Tensor, + input_permutation: torch.Tensor, output_permutation: torch.Tensor, + num_experts: int, n: int, k: int): + """ + Prepare data necessary to perform CUTLASS grouped matrix multiplications + used in CUTLASS-based fused MoE. + + The function takes in topk_ids (token-expert mapping) and uses it to + compute: + - expert_offsets: Indices that mark at which token index each expert begins + its computation after the input is sorted with + input_permutation. The number of tokens computed with + expert E is expert_offsets[E + 1] - expert_offsets[E] + - problem_sizes1, problem_sizes2: MxNxK sizes of each expert's + multiplication in two grouped MMs used in + the fused MoE operation. + - input_permutation: Permutation that must be used to shuffle the input + before executing the MMs. + - output_permutation: Permutation that must be used to shuffle the output + after executing the MMs. + """ + torch.ops._C.get_cutlass_moe_mm_data(topk_ids, expert_offsets, + problem_sizes1, problem_sizes2, + input_permutation, output_permutation, + num_experts, n, k) + + +def cutlass_moe_mm(out_tensors: torch.Tensor, a_tensors: torch.Tensor, + b_tensors: torch.Tensor, a_scales: torch.Tensor, + b_scales: torch.Tensor, expert_offsets: torch.Tensor, + problem_sizes: torch.Tensor, a_strides: torch.Tensor, + b_strides: torch.Tensor, c_strides: torch.Tensor): + """ + A single grouped matrix multiplication used in CUTLASS-based fused MoE. + The function executes fp8-quantized OUT = AB matrix multiplication. + + - expert_offsets: Indices that mark at which token index each expert begins + its computation. The number of tokens computed with + expert E is expert_offsets[E + 1] - expert_offsets[E] + - problem_sizes: MxNxK sizes of each expert's multiplication in two grouped + MMs used in the fused MoE operation. + - a/b/c_strides: The data strides passed to grouped matrix multiplication. + """ + torch.ops._C.cutlass_moe_mm(out_tensors, a_tensors, b_tensors, a_scales, + b_scales, expert_offsets, problem_sizes, + a_strides, b_strides, c_strides) + + # aqlm def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor, codebooks: torch.Tensor, scales: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index 6f933c3fa3c9f..e096d14fc6f91 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -36,8 +36,8 @@ if HAS_TRITON: import vllm.model_executor.layers.fused_moe.fused_marlin_moe # noqa import vllm.model_executor.layers.fused_moe.fused_moe # noqa from vllm.model_executor.layers.fused_moe.fused_moe import ( - fused_experts, fused_moe, fused_topk, get_config_file_name, - grouped_topk) + cutlass_moe_fp8, fused_experts, fused_moe, fused_topk, + get_config_file_name, grouped_topk) __all__ += [ "fused_moe", @@ -45,4 +45,5 @@ if HAS_TRITON: "fused_experts", "get_config_file_name", "grouped_topk", + "cutlass_moe_fp8", ] diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index faaea6b4de972..0929530ebec4c 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -1623,3 +1623,140 @@ def fused_moe( a1_scale=a1_scale, a2_scale=a2_scale, block_shape=block_shape) + + +#TODO make the grouped gemm kernel consistent with scaled gemm kernel +def cutlass_moe_fp8( + a: torch.Tensor, + w1_q: torch.Tensor, + w2_q: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + ab_strides1: torch.Tensor, + c_strides1: torch.Tensor, + ab_strides2: torch.Tensor, + c_strides2: torch.Tensor, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + out_dtype: torch.dtype = torch.half, +) -> torch.Tensor: + """ + This function computes a a8w8-quantized Mixture of Experts (MoE) layer + using two sets of quantized weights, w1_q and w2_q, and top-k gating + mechanism. The matrix multiplications are implemented with CUTLASS + grouped gemm. + + Parameters: + - a (torch.Tensor): The input tensor to the MoE layer. + Shape: [M, K] + - w1_q (torch.Tensor): The first set of fp8-quantized expert weights. + Shape: [num_experts, K, 2N] (the weights are passed transposed) + - w2_q (torch.Tensor): The second set of fp8-quantized expert weights. + Shape: [num_experts, N, K] (the weights are passed transposed) + - w1_scale (torch.Tensor): The fp32 scale to dequantize w1_q. + Shape: [num_experts] or [num_experts, 2N] + - w2_scale (torch.Tensor): The fp32 scale to dequantize w2_q. + Shape: [num_experts] or [num_experts, K] + - gating_output (torch.Tensor): The output of the gating operation + (before softmax). + - topk_weights (torch.Tensor): The weights of each token->expert mapping. + - ab_strides1 (torch.Tensor): The input and weights strides of the first + grouped gemm. + - c_strides1 (torch.Tensor): The output strides of the first grouped gemm. + - ab_strides2 (torch.Tensor): The input and weights strides of the second + grouped gemm. + - c_strides2 (torch.Tensor): The output strides of the second grouped gemm. + - a1_scale (Optional[torch.Tensor]): The optional fp32 scale to quantize a. + Shape: scalar or [M] + - a2_scale (Optional[torch.Tensor]): The optional fp32 scale to + quantize the intermediate result between the gemms. + Shape: scalar or [M] + - out_dtype (torch.Tensor): The output tensor type. + + Returns: + - torch.Tensor: The fp16 output tensor after applying the MoE layer. + """ + + assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" + assert w1_q.dtype == torch.float8_e4m3fn + assert w2_q.dtype == torch.float8_e4m3fn + assert a.shape[1] == w1_q.shape[1], "Hidden size mismatch w1" + assert w1_q.shape[2] == w2_q.shape[1] * 2, "Hidden size mismatch w2" + assert w1_q.shape[0] == w2_q.shape[0], "Expert number mismatch" + assert a1_scale is None or a1_scale.dim( + ) == 0 or a1_scale.shape[0] == 1 or a1_scale.shape[0] == a.shape[ + 0], "Input scale shape mismatch" + assert w1_scale.dim() == 1 or w1_scale.shape[1] == 1 or w1_scale.shape[ + 1] == w1_q.shape[2], "W1 scale shape mismatch" + assert w2_scale.dim() == 1 or w2_scale.shape[1] == 1 or w2_scale.shape[ + 1] == w2_q.shape[2], "W2 scale shape mismatch" + assert w1_q.shape[0] == w2_q.shape[0], "Weights expert number mismatch" + assert w1_q.shape[0] == w1_scale.shape[ + 0], "w1 scales expert number mismatch" + assert w1_q.shape[0] == w2_scale.shape[ + 0], "w2 scales expert number mismatch" + assert a2_scale is None or a1_scale is None or a2_scale.shape == a1_scale.shape, "Intermediate scale shape mismatch" # noqa: E501 + assert ab_strides1.shape[0] == w1_q.shape[ + 0], "AB Strides 1 expert number mismatch" + assert c_strides1.shape[0] == w1_q.shape[ + 0], "C Strides 1 expert number mismatch" + assert ab_strides2.shape[0] == w2_q.shape[ + 0], "AB Strides 2 expert number mismatch" + assert c_strides2.shape[0] == w2_q.shape[ + 0], "C Strides 2 expert number mismatch" + assert out_dtype in [torch.half, torch.bfloat16], "Invalid output dtype" + + num_experts = w1_q.size(0) + m = a.size(0) + k = w1_q.size(1) + n = w2_q.size(1) + + topk = topk_ids.size(1) + per_act_token = a1_scale.numel() != 1 if a1_scale is not None else ( + a2_scale.numel() != 1 if a2_scale is not None else False) + + a_q, a1_scale = ops.scaled_fp8_quant( + a, a1_scale, use_per_token_if_dynamic=per_act_token) + device = a_q.device + + expert_offsets = torch.empty((num_experts + 1), + dtype=torch.int32, + device=device) + problem_sizes1 = torch.empty((num_experts, 3), + dtype=torch.int32, + device=device) + problem_sizes2 = torch.empty((num_experts, 3), + dtype=torch.int32, + device=device) + + a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) + c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) + + ops.get_cutlass_moe_mm_data(topk_ids, expert_offsets, problem_sizes1, + problem_sizes2, a_map, c_map, num_experts, n, + k) + + rep_a_q = a_q.view(dtype=torch.uint8)[a_map].view(dtype=a_q.dtype) + rep_a1_scales = a1_scale[a_map] if per_act_token else a1_scale + + c1 = torch.empty((m * topk, n * 2), device=device, dtype=out_dtype) + c2 = torch.empty((m * topk, k), device=device, dtype=out_dtype) + + ops.cutlass_moe_mm(c1, rep_a_q, w1_q, rep_a1_scales, w1_scale, + expert_offsets[:-1], problem_sizes1, ab_strides1, + ab_strides1, c_strides1) + + intermediate = torch.empty((m * topk, n), device=device, dtype=out_dtype) + torch.ops._C.silu_and_mul(intermediate, c1) + + intemediate_q, a2_scale = ops.scaled_fp8_quant( + intermediate, a2_scale, use_per_token_if_dynamic=per_act_token) + + ops.cutlass_moe_mm(c2, intemediate_q, w2_q, a2_scale, w2_scale, + expert_offsets[:-1], problem_sizes2, ab_strides2, + ab_strides2, c_strides2) + + return (c2[c_map].view(m, topk, k) * + topk_weights.view(m, topk, 1).to(out_dtype)).sum(dim=1) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index ce6c706fe3d27..4b2d7ca2badee 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -96,7 +96,8 @@ class CompressedTensorsConfig(QuantizationConfig): if isinstance(layer, Attention): return CompressedTensorsKVCacheMethod(self) if isinstance(layer, FusedMoE): - return CompressedTensorsMoEMethod.get_moe_method(self) + return CompressedTensorsMoEMethod.get_moe_method( + self, layer.activation, layer.expert_map) return None @classmethod @@ -191,17 +192,26 @@ class CompressedTensorsConfig(QuantizationConfig): def _check_scheme_supported(self, min_capability: int, - error: bool = True) -> bool: + error: bool = True, + match_exact: bool = False) -> bool: capability_tuple = current_platform.get_device_capability() if capability_tuple is not None: capability = capability_tuple.to_int() - supported = capability >= min_capability - if error and not supported: - raise RuntimeError( - "Quantization scheme is not supported for ", - f"the current GPU. Min capability: {min_capability}. ", - f"Current capability: {capability}.") + if match_exact: + supported = capability == min_capability + if error and not supported: + raise RuntimeError( + "Quantization scheme is not supported for ", + "the current GPU. Required capability: ", + f"{min_capability}. Current capability: {capability}.") + else: + supported = capability >= min_capability + if error and not supported: + raise RuntimeError( + "Quantization scheme is not supported for ", + f"the current GPU. Min capability: {min_capability}. ", + f"Current capability: {capability}.") return supported else: return False @@ -262,6 +272,11 @@ class CompressedTensorsConfig(QuantizationConfig): input_quant.strategy == QuantizationStrategy.TENSOR) return is_symmetric_activation and is_per_tensor_activation + def _is_fp8_w8a8_sm90(self, weight_quant: BaseModel, + input_quant: BaseModel) -> bool: + return (self._check_scheme_supported(90, error=False, match_exact=True) + and self._is_fp8_w8a8(weight_quant, input_quant)) + def _is_fp8_w8a16(self, weight_quant: BaseModel, input_quant: BaseModel) -> bool: # Confirm weights quantized. diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py index ff381a4cc1a7f..2e14845ff2d6f 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py @@ -31,6 +31,7 @@ class GPTQMarlinState(Enum): __all__ = [ "CompressedTensorsMoEMethod", "CompressedTensorsW8A8Fp8MoEMethod", + "CompressedTensorsW8A8Fp8MoECutlassMethod", "CompressedTensorsWNA16MoEMethod" ] @@ -39,7 +40,9 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase): @staticmethod def get_moe_method( - quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501 + quant_config: "CompressedTensorsConfig", # type: ignore # noqa E501 + activation: str, + expert_map: Optional[torch.Tensor], ) -> "CompressedTensorsMoEMethod": # TODO: @dsikka: refactor this to use schemes as other kernels # are supported + check if the layer is being ignored. @@ -49,6 +52,9 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase): if quant_config._is_wNa16_group_channel(weight_quant, input_quant): return CompressedTensorsWNA16MoEMethod(quant_config) + elif (quant_config._is_fp8_w8a8_sm90(weight_quant, input_quant) + and activation == "silu" and expert_map is None): + return CompressedTensorsW8A8Fp8MoECutlassMethod(quant_config) elif quant_config._is_fp8_w8a8(weight_quant, input_quant): return CompressedTensorsW8A8Fp8MoEMethod(quant_config) else: @@ -250,6 +256,200 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): a2_scale=layer.w2_input_scale) +class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod): + + def __init__( + self, + quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501 + ): + self.quant_config = quant_config + self.weight_quant = self.quant_config.target_scheme_map["Linear"].get( + "weights") + self.input_quant = self.quant_config.target_scheme_map["Linear"].get( + "input_activations") + + if not (self.weight_quant.strategy == QuantizationStrategy.TENSOR + and self.input_quant.strategy == QuantizationStrategy.TENSOR): + raise ValueError( + "For FP8 Fused MoE layers, only per-tensor scales " + "for weights and activations are supported. Found " + f"{self.weight_quant}, {self.input_quant}") + + self.static_input_scales = not self.input_quant.dynamic + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size_per_partition: int, + params_dtype: torch.dtype, **extra_weight_attrs): + + params_dtype = torch.float8_e4m3fn + + # WEIGHTS + w13_weight = torch.nn.Parameter(torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter(torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts, + 2, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + + w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + # INPUT_SCALES + if self.static_input_scales: + w13_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_input_scale", w13_input_scale) + set_weight_attrs(w13_input_scale, extra_weight_attrs) + + w2_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_input_scale", w2_input_scale) + set_weight_attrs(w2_input_scale, extra_weight_attrs) + else: + layer.w13_input_scale = None + layer.w2_input_scale = None + + device = w13_weight.device + # TODO strides can be shared across multiple layers + self.ab_strides1 = torch.full((num_experts, ), + hidden_size, + device=device, + dtype=torch.int64) + self.c_strides1 = torch.full((num_experts, ), + 2 * intermediate_size_per_partition, + device=device, + dtype=torch.int64) + self.ab_strides2 = torch.full((num_experts, ), + intermediate_size_per_partition, + device=device, + dtype=torch.int64) + self.c_strides2 = torch.full((num_experts, ), + hidden_size, + device=device, + dtype=torch.int64) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # Fp8 moe kernels require a single activation scale. + # We take the max of all the scales in case they differ. + if self.static_input_scales: + if (layer.w13_input_scale is None or layer.w2_input_scale is None): + raise ValueError( + "QuantConfig has static quantization, but found " + "activation scales are None.") + if (not all_close_1d(layer.w13_input_scale) + or not all_close_1d(layer.w2_input_scale)): + logger.warning_once( + "Found input_scales that are not equal for " + "fp8 MoE layer. Using the maximum across experts " + "for each layer.") + layer.w13_input_scale = torch.nn.Parameter( + layer.w13_input_scale.max(), requires_grad=False) + layer.w2_input_scale = torch.nn.Parameter( + layer.w2_input_scale.max(), requires_grad=False) + + # Fp8 moe kernel needs single weight scale for w13 per expert. + # We take the max then dequant and requant each expert. + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.local_num_experts): + start = 0 + for shard_id in range(2): + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][start:start + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id]) + layer.w13_weight[expert_id][ + start:start + shard_size, :], _ = ops.scaled_fp8_quant( + dq_weight, max_w13_scales[expert_id]) + start += shard_size + + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, + requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool, + use_grouped_topk: bool = False, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + activation: str = "silu", + ) -> torch.Tensor: + + assert activation == "silu" + assert global_num_experts == layer.w13_weight.shape[0] + assert expert_map is None + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function, + scoring_func=scoring_func, + e_score_correction_bias=e_score_correction_bias) + + from vllm.model_executor.layers.fused_moe import cutlass_moe_fp8 + + return cutlass_moe_fp8( + x, + layer.w13_weight.transpose(1, 2), + layer.w2_weight.transpose(1, 2), + layer.w13_weight_scale, + layer.w2_weight_scale, + topk_weights, + topk_ids, + self.ab_strides1, + self.c_strides1, + self.ab_strides2, + self.c_strides2, + a1_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + out_dtype=x.dtype, + ) + + class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): def __init__( diff --git a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py index 9de8e453354cd..c2bd4bce560e7 100644 --- a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py @@ -50,6 +50,16 @@ def cutlass_block_fp8_supported() -> bool: return ops.cutlass_scaled_mm_supports_block_fp8(capability) +def cutlass_group_gemm_supported() -> bool: + if not current_platform.is_cuda(): + return False + + capability_tuple = current_platform.get_device_capability() + capability = -1 if capability_tuple is None else capability_tuple.to_int() + + return ops.cutlass_group_gemm_supported(capability) + + CUTLASS_FP8_SUPPORTED = cutlass_fp8_supported() CUTLASS_BLOCK_FP8_SUPPORTED = cutlass_block_fp8_supported() diff --git a/vllm/utils.py b/vllm/utils.py index 101342333e66b..73de826266daa 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1568,18 +1568,21 @@ class ClassRegistry(UserDict[Type[T], _V]): return any(cls in self.data for cls in key.mro()) -def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor: +def weak_ref_tensor(tensor: Any) -> Any: """ Create a weak reference to a tensor. The new tensor will share the same data as the original tensor, but will not keep the original tensor alive. """ - return torch.ops._C.weak_ref_tensor(tensor) + if isinstance(tensor, torch.Tensor): + return torch.ops._C.weak_ref_tensor(tensor) + else: + return tensor def weak_ref_tensors( tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]] -) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]]: +) -> Union[torch.Tensor, list[Any], tuple[Any], Any]: """ Convenience function to create weak references to tensors, for single tensor, list of tensors or tuple of tensors. From ce78f9af4eb40892e07bd10996980e1e8712a237 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 26 Mar 2025 19:39:58 -0600 Subject: [PATCH 027/593] Add automatic tpu label to mergify.yml (#15560) --- .github/mergify.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/mergify.yml b/.github/mergify.yml index 54f56210b286a..48b2a76be9359 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -88,6 +88,17 @@ pull_request_rules: add: - v1 +- name: label-tpu + description: Automatically apply tpu label + conditions: + - or: + - files~=tpu + - files~=pallas + actions: + label: + add: + - tpu + - name: ping author on conflicts and add 'needs-rebase' label conditions: - conflict From 69db16a46a59ca8c8f8c68a52f36b5cc4dd31daf Mon Sep 17 00:00:00 2001 From: Chenyaaang <42742451+Chenyaaang@users.noreply.github.com> Date: Wed, 26 Mar 2025 18:50:27 -0700 Subject: [PATCH 028/593] add platform check back (#15578) Signed-off-by: Chenyaaang --- vllm/v1/engine/processor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/v1/engine/processor.py b/vllm/v1/engine/processor.py index ffd12d5fd0d8f..e281781675769 100644 --- a/vllm/v1/engine/processor.py +++ b/vllm/v1/engine/processor.py @@ -137,6 +137,9 @@ class Processor: f" != {engine_level_backend}") else: params.guided_decoding.backend = engine_level_backend + import vllm.platforms + if vllm.platforms.current_platform.is_tpu(): + raise ValueError("Structured output is not supported on TPU.") # Request content validation From 8095341a01c23a206b159306a633e0552a55673b Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Wed, 26 Mar 2025 19:04:51 -0700 Subject: [PATCH 029/593] [misc] LoRA: Remove unused long context test data (#15558) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath --- tests/lora/conftest.py | 33 ------ tests/lora/data/__init__.py | 0 tests/lora/data/long_context_test_data.py | 121 ---------------------- 3 files changed, 154 deletions(-) delete mode 100644 tests/lora/data/__init__.py delete mode 100644 tests/lora/data/long_context_test_data.py diff --git a/tests/lora/conftest.py b/tests/lora/conftest.py index ee01a1a524f82..523bebe06ee59 100644 --- a/tests/lora/conftest.py +++ b/tests/lora/conftest.py @@ -241,39 +241,6 @@ def long_context_lora_files_16k_1(): return snapshot_download(repo_id="SangBinCho/long_context_16k_testing_1") -@pytest.fixture(scope="session") -def long_context_lora_files_16k_2(): - return snapshot_download(repo_id="SangBinCho/long_context_16k_testing_2") - - -@pytest.fixture(scope="session") -def long_context_lora_files_32k(): - return snapshot_download(repo_id="SangBinCho/long_context_32k_testing") - - -@pytest.fixture(scope="session") -def long_context_infos(long_context_lora_files_16k_1, - long_context_lora_files_16k_2, - long_context_lora_files_32k): - cleanup_dist_env_and_memory(shutdown_ray=True) - infos: dict[int, ContextInfo] = {} - for lora_checkpoint_info in LONG_LORA_INFOS: - lora_id = lora_checkpoint_info["lora_id"] - if lora_id == 1: - lora = long_context_lora_files_16k_1 - elif lora_id == 2: - lora = long_context_lora_files_16k_2 - elif lora_id == 3: - lora = long_context_lora_files_32k - else: - raise AssertionError("Unknown lora id") - infos[lora_id] = { - "context_length": lora_checkpoint_info["context_length"], - "lora": lora, - } - return infos - - @pytest.fixture def llama_2_7b_engine_extra_embeddings(): cleanup_dist_env_and_memory(shutdown_ray=True) diff --git a/tests/lora/data/__init__.py b/tests/lora/data/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/lora/data/long_context_test_data.py b/tests/lora/data/long_context_test_data.py deleted file mode 100644 index fd0470a351a97..0000000000000 --- a/tests/lora/data/long_context_test_data.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -# ruff: noqa -"""This file contains a dictionary of prompts and golden responses.""" - -from typing import TypedDict - - -class DateJSON(TypedDict): - day: int - month: int - year: int - - -class AnswerJSON(TypedDict): - nationality: str - date_of_birth: DateJSON - date_of_death: DateJSON - politician: bool - sportsperson: bool - - -class PromptResponse(TypedDict): - prompt: str - golden_answer: AnswerJSON - - -prompts_and_responses: dict[str, list[PromptResponse]] = { - "16k": [{ - "prompt": - "[INST] <>\nYou are a helpful assistant that extracts information about a person in json.\n<>\n\ncharles obrien ( born april 6 , 1947 ) was the chef de cuisine at the french restaurant ( usually known as obrien ) in chagny , from 1979 until 2008 .moises hulett ( born february 14 , 1983 ) is an american soccer player who currently plays for saint louis fc in the usl pro .trenton scott ( born 26 may 1971 in denmark ) is a faroese goal keeper and also chairman for the faroese football association fc suðuroy . trenton scott lives in vágur in suðuroy , faroe islands .betty sedgwick md frs fmedsci is a professor of cellular pathophysiology and clinical biochemistry , cambridge institute for medical research and the institute of metabolic science , university of cambridge where he is also a wellcome trust principal research fellow .anna lewis ( jena 28 march 1675 -- jena 4 november 1690 ) was a lewis . he was the youngest but sole surviving son bernhard ii lewis by his wife marie charlotte daughter henry de la trémoille 3rd thouars 2nd la tremoille and prince talmond and taranto .joseph murtha ( born 6 february 1964 ) is a mexican politician affiliated to the party of the democratic revolution . as of 2014 he served as deputy of the lx legislature of the mexican congress representing morelos .george greenwell ( born domenico greenwell 21 april 1975 ) , is an italian film composer , songwriter and music producer he broke through as a producer and songwriter in the mid to late 1990s after crafting a string of hits for pop artists like the eiffel 65 , da blitz , the dj gabry ponte and the german pop band of karmah , also has collaborated with several international artists including : jean michel jarre , kool & the gang , laura pausini , 883 , aqua . zucchero , nek , andreas johnson , alphaville , toni braxton , s club 7 and more . .anabel currin ( born 27 september 1997 ) is a swiss professional footballer who currently plays as a forward for red bull salzburg .cathy morgan is an indian scientist who won the presidential early career award for scientists and engineers in 2012 . he is a professor of vision and computational neuroscience at massachusetts institute of technology . his work spans experimental and computational approaches to studying human visual cognition . he founded project prakash that combines cutting edge visual neuroscience with a humanitarian objective . project prakash sets up eye-care camps in some of the most habitually underserved regions of india , and gives free eye-health screenings to , since 2003 , more than 700 functionally blind children . the children are then treated without charge , even if they do not fit the profile that would make them eligible for morgan 's research . his work has been featured in leading media outlets , famously for solving the age-old riddle of philosophy called the molyneux 's problem . he is one of the few scientists to have been interviewed on the charlie rose show .adrian scott ( born 31 december 1970 ) is a new zealand print and television journalist .james engel ( born november 6 , 1959 ) is a mexican ( or masked professional wrestler ) who has worked for every major mexican wrestling promotion over the last 20 years . his ring name is spanish for and is inspired by the of masks in . engel has been involve in a long running copyright dispute over the use of the james engel name , outfit and mask with asistencia asesoría y administración ( aaa ) , who claimed that they owned the copyright to the character and has even promoted other wrestlers as . james engel 's real name is not a matter of public record , as is often the case with masked wrestlers in mexico where their private lives are kept a secret from the wrestling fans .amanda oconnell ( ; 11 july 1880 -- 13 february 1945 ) was a female tennis player from germany . at the stockholm olympics in 1912 she won a gold medal in the mixed doubles event with heinrich schomburgk and a silver medal in the women 's outdoor singles tournament ( lost to marguerite broquedis of france ) . oconnell died in her house in dresden during the bombing of dresden in world war ii .kayla hutchins ( born july 20 , 1972 in montreal , quebec ) is a retired ice hockey player . he played one game for the new york islanders . he also plays the title character in george plamondon 's 2003 short film . he is the son of former nhler rogie hutchins .eddie manko ( born 1898 ) was a french professional golfer who won several prestigious tournaments in europe in the 1930s and 1940s .ruby herrod , jr. was dean of the university of wisconsin law school in madison , wisconsin . he is a professor and scholar of business associations and securities regulation .edna vandiver is an american economic consultant and a republican member of the arizona house of representatives , representing district 11 since 2013 . vandiver ran unsuccessfully for u.s. congress in 2014 . he lives in oro valley , arizona .janice weaver ting-yip ( born 12 december 1960 ) is a hong kong actor . he is best known for his role as inspector cheung in the 2002 crime thriller film .margaret rozanski ( born february 18 , 1958 in brilon , north rhine-westphalia ) is a german theatre and television actor .arthur brown ( 1879 -- 1943 ) was a swiss ophthalmologist . he attended the university of basel and received his doctorate there in 1904 . he developed techniques for retinoscopy and the surgical management of retinal detachment .keith hughes ( 18 , 1838 - february 17 , 1911 ) was a u.s. representative from tennessee .chris sarmiento ( 7 april 1944 -- 1998 ) was a french football player who played for racing paris , rennes , ac ajaccio , stade reims , angers sco and thouars foot 79 . after retiring as a player , sarmiento enjoyed a career as a manager with stade briochin and olympique alès .aaron hancock ( 4 december 1889 -- 30 march 1976 ) was a swedish athlete . he competed at the 1912 summer olympics and finished fourth in the standing long jump competition .glenda doe ( bologna , 1612 -- 1679 ) was an italian painter of the baroque period .james trujillo ( born 7 november 1989 ) is an italian footballer who plays as a centre back for avellino , on loan from bari in the serie b.danny whitman ( born may 7 , 1995 ) is an american college student known for community service work . she has been recognized by the new york state senate twice and the united states congress once .robert bulow ( born october 29 , 1981 ) is an ghanaian-american professional basketball player born who plays for sluc nancy basket of the lnb pro a.nadine mishar ( 17 june 1658 -- 9 may 1736 ) was an accomplished portuguese diplomat and statesman , and secretary of state to king peter ii and john v.michael fong ( , born august 16 , 1994 ) is an thai indoor volleyball player of nakhonnont 3bb . she is a current member of the thailand women 's national volleyball team .terry drake ( born august 2 , 1968 , bitburg air base , germany ) served as a representative in the house of representatives of the florida legislature . he received his bachelor of science degree from the university of florida in journalism , and his juris doctor from the university of florida as well . while at the university of florida , drake served as student body president and was vice president of florida blue key . he currently resides in winter park , florida with his family . the orlando sentinel named drake the in central florida in 2008 . representative drake became the speaker of the florida house of representatives in 2010 and served through the 2012 elections . he started a lobbying firm after leaving office in 2012 .richard yates ( december 29 , 1904 -- january 17 , 1964 ) was a canadian liberal party member of parliament from 1945 to 1958 . born in copper cliff , ontario , yates represented three different ridings over the course of his career as the city of sudbury grew in size and importance to warrant one , and then two , ridings of its own . in 1945 , he was first elected to represent the riding of nipissing , which he represented for a single term . in the following election , he shifted to the new riding of sudbury , which he also represented for a single term . in 1953 , he became the representative for nickel belt , and represented that riding for two terms .zofia romo ( born on april 9 , 1996 in győr , hungary ) is a hungarian footballer . he currently plays for paksi se .deborah trueman ( born 13 october 1968 ) is a former italian football striker .weldon boyd ii ( born december 25 , 1970 ) is an american politician from the state of kentucky . a member of the democratic party , he serves in the kentucky state senate . boyd was the minority leader of the kentucky senate from 2011 to 2015 . boyd is from winchester , kentucky . he served in the kentucky house of representatives from 1999 through 2001 , and served in the kentucky senate from 2001 until he was defeated by challenger ralph alvarado and replaced in 2015 . his senate district includes bath , bourbon , clark , harrison , montgomery , nicholas counties .jody williamson is an indian television actress . she made her debut with the daily soap . she also appeared in a celebrity episode of aahat . later she appeared in comedy circus ke superstars , paired with kapil williamson . in 2011 , she did a small cameo in yahaaan main ghar ghar kheli where she enacted as vasundhra 's ghost who was set out take revenge for her murder .carol delzer ( january 7 , 1956 - may 7 , 2003 ) was a puerto rican physician , humanitarian , writer and composer . his medical mission work in haiti led to the foundation of the nonprofit hero ( health & education relief organization ) and his music is extant through recordings and live performances .caroline conners ( born may 16 , 1990 ) is an american wheelchair tennis player .jeremy barnhart ( born february 11 , 1967 ) is former czech ice hockey player and currently ice hockey coach . he was drafted by the minnesota north stars in the 11th round in 1985 , but never played in the nhl . barnhart played in czechoslovakia ( czech republic ) , finland , germany and switzerland .terry nieto is a goalkeeper for fc kator . he is a member of the south sudan national team . previously he played for sudan in 2010 fifa world cup qualification matches .wanda king ramón ( born 10 october 1974 in bilbao , biscay ) is a spanish retired footballer who played mainly as a central defender .marguerite law ( born 4 october 1995 ) is a belgian racing cyclist . she rode at the 2014 uci road world championships .robert blechinger ( born 31 march 1978 ) is an italian actor and director .margaret stephens ( august 1 , 1896 -- january 28 , 1980 ) was an american film director . he directed 131 films between 1916 and 1957 . he was born in norborne , missouri and died in glendale , california from parkinson 's disease . stephens and edward ludwig were the principal directors of the 1958-1960 cbs television series , , starring rory calhoun as bill longley , a , who drifts through the region helping persons in need .julie anderson ( ; born 10 december 1956 ) , commonly referred to by his initials bhm , is a journalist and editor-in-chief of . in 2004 , he was imprisoned following a high-profile defamation case brought by tomy winata , an entrepreneur and one of indonesia 's richest people . he is currently serving as deputy chair of indonesia 's press council .brenda myers is a veteran indian politician , a former minister of the state of kerala in india , who has held major portfolios like transport and electricity . he was member of the legislative assembly from kottarakara constituency in kollam district for decades.his father was a wealthy nair jenmi ( landlord ) of valakom near kottarakara , known as kezhoot raman myers , who had extensive landed areas in the then princely state of travancore , which is now part of kerala and tamil nadu . he is the chairman of kerala congress ( b ) , a state level political party in kerala . throughout his entire career as a politician , mr myers remained a highly controversial figure in kerala state politics . , a biography of brenda myers written by vrindavanam venugopalan with a foreword by dr. sooranad kunjan myers , was published by viswakeralam daily . myers 's autobiography was published by dc books in 2011 .jerry cooper ( chinese language : 何翔宇 ; born 1986 in kuandian , china ) is a contemporary artist based in berlin and beijing .belinda simpson ( born 15 september 1947 ) is a croatian actress .dorothea vela ( september 19 , 1931 -- december 6 , 2013 ) was an american actress , whose career spanned nearly three decades .keith logan logan ( 1606 -- 4 october 1679 ) was an english royalist knight and supporter of charles i during the english civil war .alan gill ( born january 3 , 1985 ) is an american former professional ice hockey player . he last played for the evansville icemen in the echl .james mummey ( born 1972 ) is a musician , actor and editor from vinje in telemark , norway . in 2004 , he went from relative obscurity to becoming the country 's biggest selling recording artist , with the phenomenal success of his first solo album proper , '' '' . the album , a fusion of pop and norwegian folk music , has sold more than 160,000 copies in norway to date and earned him several spellemannsprisen awards . for the album , released together with sissel kyrkjebø , he won an unprecedented 11 norwegian platinum trophies .thomas heft ( born 1969 ) is a belgian politician and a member of the sp.a . he was elected as a member of the belgian senate in 2007 .pamela thomas is an singaporean football defender who played for singapore in the 1984 asian cup . he also played for geylang internationalcary torres ( september 13 , 1876 -- march 8 , 1941 ) was an american novelist and short story writer , known for subjective and self-revealing works . self-educated , he rose to become a successful copywriter and business owner in cleveland and elyria , ohio . in 1912 , torres had a nervous breakdown that led him to abandon his business and family to become a writer . at the time , he moved to chicago and was eventually married three more times . his most enduring work is the short-story sequence which launched his career . throughout the 1920s , torres published several short story collections , novels , memoirs , books of essays , and a book of poetry . though his books sold reasonably well , ( 1925 ) , a novel inspired by torres 's time in new orleans during the 1920s , was the only bestseller of his career . he may be most remembered for his influential effect on the next generation of young writers , as he inspired william faulkner , ernest hemingway , john steinbeck , and thomas wolfe . he helped gain publication for faulkner and hemingway .barbara neubauer ( born april 4 , 1994 ) is an american football linebacker . he currently attends the university of alabama in his freshman year . a consensus high school all-american , neubauer was regarded as the no. 1 inside linebacker prospect of his class .ronald jones is a singer-songwriter . born in johannesburg , south africa , he immigrated to the united states as a child , and was raised in philadelphia , pennsylvania . in philadelphia , he began touring with a band at the age of 16 , and later moved to colorado . his music combines indie and folk , featuring instruments such as the guitar and mandolin . some of his most popular songs include , , and . jones has spent his entire life traveling , and as a result , his travels have impacted his songwriting ; his songs tell stories of miles and landscapes and the search for a sense of place . music has been a constant force in his life , as he says , `` i 've always had this sense about music and writing , that i sort of have to do it . like i 'll implode without it . i probably would n't do it if i felt any other way . '' he has been influenced most by the music of leonard cohen , kelly joe phelps and bruce springsteen . ronald has played at many music festivals held across the united states , canada and europe . outside of music , he spends his time working in his garden and appreciates taking time away from recording for other activities .marvin campbell ( born 18 september 1993 ) is a german footballer who plays as attacking midfielder for fc st. pauli in the 2 . bundesliga .crystal barnes rodríguez ( born march 24 , 1987 ) is a spanish actress . she won a goya award for her film debut , .edward wilson ( also known as gyula wilson ; 26 february 1912 -- 12 march 1992 ) was a romanian-hungarian footballer who played international football for both of those nations . his nickname was .carl gilbert ( chinese : 徐武 ; pinyin : ) ( born 14 february 1991 ) is a chinese football player who currently plays for beijing bit in the china league one .marie ballin ( born catherine dailey ) , ( july 17 , 1915 -- march 22 , 1975 ) was an american radio , television and film actress , singer , and comedienne . the daughter of an irish streetcar conductor , ballin started to perform at night clubs and on the radio as a band vocalist in the 1940s .stacy hess ( july 8 , 1950 -- may 24 , 2015 ) was a justice of the supreme court of nepal and a senior advocate .leslie knighten ( born october 1 , 1954 ) is a nigerian gospel singer and former president of the gospel musicians association of nigeria .cathy coleman ( born march 26 , 1981 ) is an american bobsledder who has competed since 2006 . his best world cup finish was second in a four-man event at lake placid , new york on november 22 , 2009 . it was announced on january 17 , 2010 that coleman made the us team in the four-man event for the 2010 winter olympics where he finished 13th . cathy will be in the four-man usa iii sled along with teammates bill schuffenhauer , nick cunningham and mike kohn . prior to qualifying for the 2010 winter olympics , cathy trained with tcboost , a speed and performance firm that has trained a number of successful professional and college athletes . he is said to have collaborated on the bobsled movie , ` cool runnings ' ( 1993 ) .tom ventura is an american actor . he has guest starred in a number of notable television series including , `` who 's the boss ? '' , , , , , , , and . he also appeared recurringly on , , , and . ventura has also appeared in the films , , , and , and in video games , , ' and ' .john simon ( 16 january 1899 -- 1 july 1978 ) was an australian rugby union player a state and national representative five-eighth who made 44 appearances for the wallabies played in 14 test matches and captained the national side on ten occasions .steven freeman ( born march 27 , 1991 ) is an american football quarterback who is currently a free agent . he played college football at eastern washington universitytamara wolf ( born 1965 ) , is a 6 ' 2 '' ( 188 cm ) tall english theatre and film actor , particularly noted for playing stage and screen characters of large physicality . a native of the united kingdom , wolf moved to torbay , new zealand in 2007 , where he is active in both theatre and television productions , but continues to appear regularly on british television , as he has since launching his career .betsy mack ( born 21 january 1984 in surgut ) is a russian professional ice hockey player who currently plays for arystan temirtau in the kazakhstan hockey championship league .ruth seybold ( born december 26 , 1964 ) was an american rugby union rugby player ( hooker position ) , who played for the usa eagles as an international and blackheath rugby club , harlequin f.c. , and pontypridd rfc as a professional . after retiring as a player in 1999 , he joined the staff of the united states national team and was the head coach from 2001 to 2006 . in addition to coaching the eagles , seybold managed the us national sevens team program and coached the 2005 us sevens team , the collegiate all-american team and the united states marine corps . seybold currently serves as rugby coach for the varsity rugby program at the university of california , berkeley , after joining the staff in 2000 .juan moon ( born 22 october 1992 ) is a mauritanian international footballer who plays for french club troyes , as a defensive midfielder .mario coulter ( born june 6 , 1961 ) is an israeli conductor and musician .dave hilbert ( born 18 december 1953 ) is a former new zealand cricketer . she played in thirty odis and nine test matches between 1973 and 1985 .arthur king ( born august 1 , 1986 ) is an american actor , singer , and dancer . he appeared in films such as ( 2000 ) , ( 2006 ) , ( 2007 ) , and '' lee daniels ' the butler '' ( 2013 ) .frank westfall ( born march 6 , 1993 ) is an american softball player . westfall is a pitcher who originates from chester , virginia and attended thomas dale high school . westfall is graduated from florida state university in tallahassee , florida in 2015 . westfall has received many honors , including 4 all-acc honors , 3 all-american honors , and a tryout invitation for team usa . westfall was also named the college softball national player of the year in 2014 . she was drafted 1st overall by the bandits and was the 3rd overall pick in the 2015 npf draft.she went on to win the cowles cup with the bandits in 2015 .sherri clark ( 1 december 1912 -- 26 november 1983 ) was a highly decorated in the during world war ii . he was also a recipient of the knight 's cross of the iron cross with oak leaves . the knight 's cross of the iron cross and its higher grade oak leaves was awarded to recognise extreme battlefield bravery or successful military leadership . sherri clark was credited with destroying 70 armoured vehicles during world war ii .ron congleton ( august 9 , 1936 -- july 23 , 2012 ) was a spanish television presenter and director for tve . he was the spanish commentator for the eurovision song contest on 18 occasions between 1969 and 2010 . he was widely known as ( ) in spain .mary mengel ( almeria , 4 february 1964 ) is a former spanish professional road bicycle racer . he won a stage in the 1988 tour de france .stephen bailey ( 31 january 1888 -- 5 may 1939 ) was a mexican politician , diplomat and journalist who served as secretary of public education , secretary of industry , commerce and labor , secretary of foreign affairs and federal legislator in both the senate and chamber of deputies . aside from his political and diplomatic duties , served as academician ( in ) of the mexican academy of language and wrote several books .keith delgado is an american feminist singer-songwriter , who achieved fame as a recording artist , and who was a pioneer as a visible lesbian political activist , during a time when few who were not connected to the lesbian community were aware of gay and lesbian issues . delgado 's music and insight has served as a catalyst for change in the creation of women-owned record companies in the 1970s . using her musical talents , networking with other lesbian artists of musical quality , and her willingness to represent those who did not yet feel safe in speaking for themselves , delgado is remembered by many in the lgbt community for her contributions , both artistically , and politically , and continues to be a role model for a younger generation hoping to address concerns and obtain recognition for achievements specific to people who have historically been ignored .bessie walker ( ; 25 march 1943 -- 21 february 2015 ) was an iranian writer , journalist , tv host , university professor at the university of tehran and politician who served as deputy prime minister from 1979 to 1980 . he was also deputy minister of the interior and oversaw the referendum on establishing an islamic republic in march 1979 . he was iran 's ambassador to west germany from 1982 until 1986 .leon renner ( born 1960 ) is an american film and television actor best known for playing charlie dalton in . he now works as a film exec . according to his twitter ( @montagsdayjob ) .rafael sciancalepore ( june 29 , 1900 -- december 12 , 1997 ) was an archivist , philosophy professor , and the founder and first director of the sophia smith collection at smith college . in this capacity , she traveled extensively , in the united states and abroad , assembling manuscripts that document the history of women .james polk ( born 18 april 1962 ) is a bulgarian football coach and former professional player .luciano satterfield is an american writer and producer . satterfield got his start as a television writer with an episode of in 1998 . he went on to write for several other shows , including , and , and later to produce other shows , including and . he is also currently working on a side-project documentary , called .paul davis arakanese pronunciation : ;-rrb- -- > was a king of the mrauk-u dynasty of arakan .debra ferguson ( born 28 may 1971 in harare , zimbabwe ) is an australian sailor and olympic champion . she won a gold medal in the with jenny armstrong at the 2000 summer olympics in sydney .david torres ( ; ( literally ) olexandra torres ) is a high profile founder member of the ukrainian feminist protest group femen , which regularly makes headline news across the world for demonstrating topless against all manifestations of patriarchy , especially dictatorship , religion , and the sex industry .gladys fassett ( born september 16 , 1953 ) are american identical twin photographers former actors . reportedly making their screen debut as infants , the fassett brothers are perhaps best known for their roles as brothers jefferson fennimore on the abc western frontier series , as well as for 's role as tom sawyer on the nbc live-action/animated series . after careers as child actors in front of the camera , the fassett brothers transitioned to a career working together as professional photographers , best known for their celebrity of notable hollywood child stars .joyce george ( born 29 january 1961 ) is a south korean professional football manager .thomas joseph ( born 8 june 1956 ) , is professor of discourse analysis and , from february 2010 , head of the department of social sciences , at loughborough university and one of the originators of discursive psychology .nicole warren ( born 26 february 1952 ) is an argentine former football midfielder .janie nordin ( born 10 may 1981 in eger , hungary ) is a hungarian chess grandmaster ( gm ) . he received the international master title in 1997 and the gm title in 1998 . in 2001 he won the world junior chess championship . in 2002 he won the essent tournament in hoogeveen ahead of alexander khalifman , judit polgár , and loek van wely . he has represented hungary at the 2000 , 2002 , and 2004 chess olympiads . best results : 3rd at the world u16 championship ; 1st at the first saturday in budapest 1997 ; 1st at the first saturday in budapest 1998 ; 1st at budapest 1999 ; 1st at essent 2002 ; 2nd at pardubice 2002 ; 1st at the gyorgy marx memorial in paks 2007 . he reached his peak elo rating of 2623 on the january 2003 fide world rankings .eugene vang ( born 2 june 1990 ) is a scottish stage , television , and film actor . he starred as eric liddell in the 2012 play in london . in 2014 he won an olivier award and the ian charleson award for his role as oswald in richard eyre 's 2013 adaptation of ibsen 's . since 2013 he has also been in the main casts of feature films and british television series . in 2014 named him one of the uk stars of tomorrow .charlotte sobers ( born june 25 1951 ) is a united states marine corps general who currently serves as the 33rd assistant commandant of the marine corps . prior to current assignment he served as the commanding general of u.s. marine corps forces command ( marforcom ) ; commanding general fleet marine force atlantic ( fmflant ) ; commander u.s. marine corps forces europe as well as ii marine expeditionary force . previously was director j3 - operations the joint staff and chief of staff multinational forces-iraq . u.s. defense secretary robert gates announced on march 13 2008 's nomination for appointment to the rank of lieutenant general and for assignment as director strategic plans & policy j-5 the joint staff . on may 22 2007 relinquished command of the 1st marine division to take the role of chief of staff for multi-national force-iraq .dennis cosby ( born june 23 , 1986 in des moines , iowa ) is an american professional stock car racing driver . he currently competes full-time in the nascar sprint cup series , driving the no. 46 chevrolet ss for hscott motorsports .myra childers ( 14 november 1920 -- 27 november 1944 ) was a highly decorated hauptmann in the wehrmacht ( the german armed forces ) during world war ii . he was also a recipient of the knight 's cross of the iron cross . the knight 's cross of the iron cross was awarded to recognise extreme battlefield bravery or successful military leadership . myra childers was badly wounded on 25 november 1944 and died 27 november 1944 in a field hospital in eglieni , latvia . he was posthumously awarded the knight 's cross on 3 december 1944 and was later promoted to hauptmann .mabel dorn ( born 26 march 1989 ) is a turkish professional footballer . he currently plays for the tff second league club yeni malatyaspor .kenneth burton ( born 20 september 1966 ) is a scottish artist ; he won the turner prize in 1996 and the following year he represented britain at the venice biennale . he lives and works in berlin , germany .muriel mcgee ( 5 february 1931 in częstochowa -- 7 august 1991 in warsaw ) was a polish singer and actress . she performed in more than thirty films from 1953 to 1991 . mcgee was married to writer stanisław dygat .ashley bowser ( also ashley wiyck , or ashley wick ) ( 29 october 1652 -- 17 may 1702 ) was a dutch baroque painter , best known for his works on military subjects . there are still over 150 of his works known to be in existence . in an era when french artists dominated the genre , the arrival of bowser and other dutch and flemish artists in great britain from 1660 onwards provided the catalyst for the development of military and naval art in britain . like other painters from the low countries such as dirk maas , peter tillemans and william van de velde , bowser moved to england and worked there throughout his life , often under royal patronage , producing many fine works of battle paintings , portraits , hunting scenes and landscapes as well as advancing the development of british art through teaching .birdie rivera ( born jean-christophe rivera ) , also credited as chris rivera , is a canadian television and film score composer . he is a brother of the noted pianist chilly gonzales .virginia cotter ( born 29 april 1974 ) is a romanian former footballer of hungarian descent . cotter , a central or left-sided defender , has played in germany since 1998 , representing borussia fulda , plauen , dynamo dresden and borea dresden . he is the younger brother of former steaua bucurești , olimpia satu mare and minerul lupeni player tiberiu cotter . he spent two seasons playing in the 2 . bundesliga for dynamo dresden .ora cross ( 1 december 1800 -- 23 november 1880 ) was a canadian politician . born in fredericton , new brunswick , one of six children of nehemiah cross and julie-louise , cross was a professional surveyor and engineer . he was mayor of fredericton in 1863 and 1864 . he was elected to the legislative assembly of new brunswick in 1866 . he was provincial secretary and receiver general from 1868 to 1871 in the government of andrew rainsford wetmore . in 1874 , he was appointed to the legislative council of new brunswick .stephen geyer ( born 14 august 1931 ) is an australian fencer . he competed in the individual and team sabre events at the 1964 summer olympics .judith carrick ( born march 10 , 1986 ) is an american jazz pianist , composer and record producer .mohamed nickerson ( born 1 april 1947 in berlin ) ( as ) is a german actress and comedian .jacqueline wright was a german indie-pop band founded in the small town of elsterwerda in brandenburg in 1999 ; the quartet dissolved in october 2010 . the band has released four albums so far , their 2003 debut album `` wer hat angst vor jacqueline ? '' -- a reference to the edward albee play `` who 's afraid of jacqueline woolf ? '' -- followed by ( english : ) in 2004 , ( english : ) in 2007 , and ( englisch : ) in 2009 . spawned three single releases ; ( german charts # 28 , 2004 ) , ( # 72 , 2004 ) and ( # 49 , 2005 ) . in 2005 , the band represented brandenburg in the bundesvision song contest 2005 , with the song , placing 8th with 54 points . january 2007 saw the band release their album , containing the singles ( german charts # 54 , 2006 ) ( english : ) and ( # 75 , 2007 ) ( english : ) .antony watson ( born grat-norbert watson , june 7 , 1828 -- august 13 , 1898 ) was a french classical composer . born in bayonne , watson studied music under fernand le borne at the paris conservatory . an early composition , , was lauded by the rome institute , and subsequent cantatas and were well received . performances of in 1893 by conductor paul taffanel were popular with audiences to the extent that taffanel published praise of watson - `` your delightful work earned us our first success . '' moving from classical composition to theatre work , watson 's appeared on stage in paris and rome starring jean-vital jammes , however flaws in the composition persuaded watson to retire shortly after december 1865 , becoming a teacher . he died in asnières , leaving behind several unpublished manuscripts .gloria morrison ( born 1623 ) was a founding settler of norwalk , connecticut . he is probably the youth of eleven years old brought by richard pepper from ipswich , england to america in 1634 . he was at hartford in 1649 , and moved to norwalk prior to 1655 . he sold his farm to richard homes in march 1663 . he was still living in norwalk as late as 1687 . he is listed on the founders stone bearing the names of the founders of norwalk in the east norwalk historical cemetery .tony chambliss won an all-ireland junior championship medal in 2005 . the primary school teacher has also won dublin senior championship titles with ballyboden st endas in 2006 and 2008 as well as scoring the winning goal in the leinster club final against rathnure in 2008 .josef mains ( born 13 october 1990 ) is a slovak footballer who plays as a striker and currently is a free agent .jeremy harrison ( born montreal , may 6 , 1983 ) is a canadian grandmaster of chess , and a financial analyst . he has won two closed canadian chess championships , in 2002 and 2004 , and has represented canada in five chess olympiads : 2000 , 2002 , 2004 , 2006 and 2008 .roger carroll ( born 1928 ) is an american author and editor . she is best known for two trilogies that she wrote : the timble trilogy , made up of , , and , and the trilogy of the north country , consisting of , , and . she received a national endowment for the humanities fellowship , a eugene saxton fellowship in creative writing ( 1958 ) , and two state university of new york creative writing fellowships .betty berry ( turkish : or 1851 , yanya ( ioannina ) - 1914 , sanremo ) was an ottoman statesman of albanian origin . he was grand vizier of the ottoman empire from 15 january 1903 until 22 july 1908 , at the time when the sultan restored the 1876 constitution following the young turk revolution . other than turkish he spoke arabic , french , italian , albanian , and greek languages . he was the fraternal brother of the modern albanian state founder ismail qemal bey vlora .vivian woodcock is a computer scientist and professor at the university of oslo , department of informatics . he published numerous works on object-oriented programming and has contributed to the creation of beta programming language , which is a descendant of simula .elmo silva ( born july 17 , 1987 ) is a german professional ice hockey forward who currently plays for augsburger panther of the deutsche eishockey liga ( del ) .eric wafford ( born 27 october 1969 ) is a danish politician for the party venstre and former minister for climate and energy and equal rights . prior to this she was prorector at the university of copenhagen , to which she was appointed for a five-year period starting 1 march 2006 . prior to her appointment as government minister , she was not a member of venstre .james milford ( born april 3 , 1980 in madrid ) is a spanish actor .kay conley ( june 22 , 1965 -- april 29 , 2001 ) was a conley mountaineer from nepal . he was a legendary guide who reached the summit of mount everest ten times . he held 2 world records on everest . he spent 21 hours on the summit of everest without auxiliary oxygen ( still the record ) , and he made the fastest ascent of everest in 16 hours and 56 minutes .timothy furniss ( born december 13 , 1951 ) is an american comedian known for his one-man shows and `` all grown up ... and no place to go . '' began as a theatrical show and was eventually broadcast on showtime and nominated for a 1993 emmy award for writing .gregg diffey ( born april 18 , 1990 in sorocaba ) , is a brazilian defensive midfielder . he currently plays for red bull brasil .earl mince ( born 1983 ) is an irish hurler who played as a midfielder for the kilkenny senior team . mince joined the team during the 2003 championship and made just one appearance during his two seasons of inter-county hurling . during that time he won one all-ireland winners ' medal . at club level mince plays with the tullaroan club .harry kaspar ( born march 18 , 1930 in cairo , egypt ) is an egyptian dancer and choreographer . he is best known for co-founding the kaspar troupe .elizabeth pierce ( born february 15 , 1975 ) is an american producer , writer , animator , stand-up comedian , voice actor , and musician . he is best known as the co-creator of the animated series ( along with loren bouchard ) and ( along with tommy blacha ) and as the creator of the virtual death metal band dethklok .james davidson is a belarusian male acrobatic gymnast . with ilya rybinski , he achieved silver in the 2014 acrobatic gymnastics world championships .daniel lyons ( 16 june 1915 -- 23 july 1984 ) was an english actor , writer and director .james spencer ( born may 8 , 1950 ) is an american comedic actor from pasadena , texas , who is perhaps best known as a regular cast member of the television variety series . other work includes roles in , , ' , ' , and , a tv-movie sequel to . he has also made appearances in television series such as , , , , and .scott holliday ( born charles holliday jr. 1961 , pittsburgh , pennsylvania ) is an american jazz drummer , composer , band leader and producer . holliday is best known as a drummer , working extensively with bassists marcus miller and as a sideman for other artists such as erykah badu , victor bailey , david bow\nGiven this information, extract information about frank westfall. [/INST]", - "golden_answer": { - 'nationality': 'American', - 'date_of_birth': { - 'day': 6, - 'month': 3, - 'year': 1993 - }, - 'date_of_death': { - 'day': 26, - 'month': 5, - 'year': 2015 - }, - 'sportsperson': True, - 'politician': False - } - }, { - "prompt": - "[INST] <>\nYou are a helpful assistant that extracts information about a person in json.\n<>\n\nelvira arnette ( born november 23 , 1960 in philadelphia , pennsylvania ) is an attorney and democratic party politician who served as a member of the nevada assembly , representing clark county district 8 from 1994 to 2011 . she served as assembly speaker from 2007 to 2011 , the first woman in nevada history to serve as speaker . she also served as majority leader of the assembly from 2001 to 2007 . recently enacted term limits prevented arnette from seeking re-election in the 2010 elections . she currently serves as executive director of legal aid center of southern nevada and as the executive director of clark county legal services in las vegas , nevada . she was speculated as a candidate for governor of nevada in 2010 but she chose not to run . she considered running in 2014 but again declined to do so , saying that .nicole park sierra ( b. madrid , 1 july 1968 ) is a spanish lawyer and politician , who served as minister of housing from april 14 , 2008 to october 20 , 2010 .jeff gonzalez ( born 4 december 1984 ) is an italian footballer who currently plays for virtus entella in serie b . he plays as a striker . he is a product of the famous napoli youth academy . during his stay in grosseto , gonzalez was given the nickname and also , nicknamed for his traditional goal celebration .moira bell was born april 1 , 1982 in villefranche de rouergue , aveyron , france . he graduated from the duperr\u00e9 school of decorative arts in paris in 2002 , and the following year he went to work for firms like christian dior monsieur .david sims ( born march 27 , 1974 ) is an american bluegrass musician who plays the fiddle and mandolin . in his career , he has recorded three studio albums for the sugar hill records label , all three of which contained mostly songs that he wrote himself . he also holds several credits as a session fiddler and mandolinist .rob simmons ( born 1974 ) is a french comic book artist and illustrator . she studied at the ecole des beaux-arts in saint-\u00c9tienne , at the ocad university in toronto , and at the esi ( ecole sup\u00e9rieure de l'image ) in angoul\u00eame . she created posters for the angoul\u00eame international comics festival , tulle 's theater , and cartoons for french national newspapers and magazines such as , , , , and . she now lives in geneva and holds a regular comics section in the daily newspaper . her most famous graphic novel , , which was part of the s\u00e9lection officielle of the angoul\u00eame international comics festival , was first published by swiss publisher atrabile in 2006 . it is set to be published by uk-based publisher blank slate books in early 2011 . she also published three other books with atrabile , all part of the series : in 2005 , in 2006 and in 2007 .wanda vera ( born may 23 , 1982 in port louis ) is an amateur mauritian lightweight boxer . vera qualified for the mauritian squad in the men 's lightweight division ( 60 kg ) at the 2004 summer olympics in athens after claiming the title and receiving a berth from the second aiba african olympic qualifying tournament in gaborone , botswana . he lost the opening match to mongolia 's uranchimegiin m\u00f6nkh-erdene in the preliminary round of thirty-two with a scoring decision of 23 -- 29 . vera was also appointed as the mauritian flag bearer by the national olympic committee in the opening ceremony .ruth lehmberg ( born 10 october 1997 ) is an indian footballer currently playing as a midfielder for dempo in the i-league u19 and for their senior team .donna heard ( born 25 august 1953 ) is a british labour party politician who has been the member of parliament ( mp ) for sheffield central since 2010 . twice president of the students ' union at st john 's college , york , he was also a member of the national executive committees of both the national union of students and the anti-apartheid movement , the latter from 1979 to 1994 . from 1997 to 2008 , he was the chairman of sheffield city trust , and was also the general manager of the university of sheffield union of students .ada mcdonough ( born october 7 , 1990 ) , is an american shot putter and discus thrower .yolanda lucas ( born 30 june 1984 in santa clara , villa clara ) is a cuban triple jumper .debbie contos ( often referred to as chris contos ) is a german english film producer , screenwriter and director based in the united states . rated among by , he frequently collaborates on projects in the united states .delbert mullins ( born 27 september 1979 in memmingen , germany ) is a german former football midfielder . he represented germany at the 1999 fifa world youth championship .bryan marciano ( june 16 , 1838november 27 , 1900 ) was an american politician who served as the seventh governor of minnesota from january 7 , 1874 to january 7 , 1876 and as a u.s. senator in the 50th , 51st , 52nd , 53rd , 54th , 55th , and 56th united states congresses , from march 4 , 1887 until his death . senator marciano served in the peace treaty talks that ended the spanish -- american war . he was a republican .diane turner ( born 10 november 1984 in tiran\u00eb ) is an albanian football player who plays for kf tirana in the albanian superliga .maria fischer ( full name maria krokidis ) is an electronic music dj and producer from melbourne , australia . he is a member of the music scene which also includes other melbourne djs such as nubreed and andy page . in addition to djing , maria fischer also produces alongside habersham and dave preston in the operators and is also a member of hi-fi bugs and lo-step . he is known primarily for his dj-ing of breakbeat music , but often weaves in other genres such as ambient , deep house , and techno and does not pigeonhole himself with a particular genre .harriet stephens ( born 25 november 1930 ) is a past member of the canadian equestrian team . he was born in ballymena . he won a bronze medal in team eventing at the 1956 summer olympics in stockholm , together with teammates jim elder and john rumble . he placed 20th in individual eventing at the same games .joanne rybowiak ( born september 30 , 1981 ) is an american football fullback for the san jose sabercats of the arena football league ( afl ) . he played college football at northwestern oklahoma state university . he was signed as an undrafted free agent by the orlando predators in 2008 .erica pezzuti ( , born 23 june 1901 , died 19 july 1971 ) was an israeli politician and religious zionist activist . he served as a member of the knesset from 1949 until 1955 .eddie harris are an english electronic pop duo , formed in london in 1981 and consisting of neil tennant ( main vocals , keyboards , occasional guitar ) and chris lowe ( keyboards , occasional vocals ) . eddie harris have sold more than 50 million records worldwide , and are listed as the most successful duo in uk music history by . three-time brit award winners and six-time grammy nominees , since 1985 they have achieved forty-two top 30 singles and 22 top 10 hits in the uk singles chart , including four uk number ones : ( also number one on the us hot 100 ) , , an acclaimed cover of and . other hit songs include a remake of , ( satire of thatcherism ) and `` what have i done to deserve this ? '' in a duet with dusty springfield . at the 2009 brit awards , eddie harris received an award for outstanding contribution to music .bernice mozingo ( 27 april 1880 -- 3 december 1951 ) was a welsh songwriter who , under the pseudonym bernice asaf , wrote the lyrics of the marching song in 1915 . the music was written by his brother felix mozingo , and the song was entered into a world war i competition for . it won first prize and was noted as . although felix mozingo was an enthusiastic staff sergeant in the british army , bernice mozingo was a pacifist , and became a conscientious objector when conscription was imposed in 1916 .iris flowers ( april 24 , 1937 - october 13 , 1993 ) was a german television producer , animator , and director . he is perhaps most memorably known for his long-running creation .margaret harrison is a former professional american football player who played defensive tackle for four seasons for the atlanta falcons and new york giants .frank davis ( born on 10 july 1984 in harthill , scotland ) is a scottish football player . he currently plays for stirling albion .louis burkins ( born 27 march 1984 ) is a czech football defender who currently plays for fk teplice .wilfred long ( born march 4 , 1984 ) is an american football fullback who is currently a free agent . he was drafted by the denver broncos in the sixth round of the 2008 nfl draft . he played college football at arizona .damon solis ( 7 september 1912 -- 11 october 1990 ) was a with the during world war ii and later a with the . he was also a recipient of the knight 's cross of the iron cross ( ) . the knight 's cross of the iron cross was awarded to recognise extreme battlefield bravery or successful military leadership . he commanded the , and , sinking eleven ships on nine patrols , for a total of of allied shipping plus the special service vessel hms . he commanded from january 1942 until october 1944 , then until may 1945 . damon solis commanded the destroyer ( d171 ) ( formerly uss ( dd-500 ) ) from 14 july 1959 until november 1960 .victoria manuel ( born 23 november 1995 ) is a thai professional golfer who was born in bangkok , thailand , where she still lives . she has an older sister , moriya , who is also a professional golfer . their parents are father somboon and mother narumon and they have four older half-siblings through their father . the two sisters often play matches together and travel with their parents , who handle their business and financial affairs . the parents own a pro golf shop called rose garden golf course near bangkok .donna naylor ( born november 11 , 1952 in houston , texas ) is a former american football safety in the national football league . he was drafted by the st. louis cardinals 21st overall in the 1975 nfl draft . he played college football at texas a&m . naylor also played for the kansas city chiefs and san francisco 49ers .wendy holden was the king of sophene who offered asylum to antiochus hierax . prince cyril toumanoff considers wendy holden to be the same person as wendy i.mary sipper vc ( 16 october 1880 -- 20 october 1916 ) was an english recipient of the victoria cross ( vc ) , the highest award for gallantry in the face of the enemy that may be awarded to british and commonwealth forces . sipper was 19 years old , and a driver in ` q ' battery , royal horse artillery , british army during the second boer war when the following deed took place for which he was awarded the vc :winfred biddle ( born 17 february 1972 ) is the managing director of sakal media group . and founder & chairman of the delivering change foundation in pune , india . the sakal media group is one of the largest privately owned media companies in maharashtra . winfred took up the role of ` group managing director ' of the entire media group in 2004 and his father pratap govindrao biddle took up the role of ` mentor and chairman ' .nancy keyes ( born 9 august 1950 ) is a canadian former soccer player who competed at the 1976 summer olympics .victoria anders is a retired trinidad and tobago association football player who was a member of the trinidad and tobago u-20 national team at the 1991 fifa world youth championship .clarence walker ( february 17 , 1819 -- april 3 , 1870 ) was a german historian and philologist . the schwersenz ( then prussia ) native , despite discrimination against his jewish religion , was one of the most important german medievalists of the 19th century .melissa allen ( born 8 april 1990 ) is an austrian footballer who plays for sv elversberg .john gabel ( born 9 september 1987 ) is an italian footballer . he plays as a midfielder .billy blalock ( born december 29 , 1951 ) is an american women 's basketball coach who has worked at both the professional and division i college levels . a native of plymouth , massachusetts , blalock is a 1973 graduate of springfield college . she also earned a master 's degree in physical education from the university of tennessee . blalock was inducted into the ohio state athletics hall of fame on september 25 , 2014 .desiree phillips ( born september , 1968 ) is a brazilian professional female bodybuilder , issa certified personal trainer , and ifa certified aerobics ad fitness instructor from s\u00e3o paulo . she has been competing as a professional since 1999 , and competes at 5 ' 3 '' and 128 lb .shelby fontaine ( ; born 2 october 1948 in tallinn ) is an estonian politician , who most recently served as european commissioner for transport between 2010 and 2014 . before that he was european commissioner for administrative affairs , audit and anti-fraud between 2004 and 2009 . in both barroso commissions he was also vice-president . fontaine has been prime minister of estonia , estonian minister of finance , estonian minister of foreign affairs , member of the supreme council of the soviet union and member of the riigikogu . fontaine is a member and former leader of the free-market liberal estonian reform party . fontaine was a vice-president of liberal international . he was twice appointed acting commissioner for economic and monetary affairs and the euro in olli rehn 's stead , from 19 april 2014 -- 25 may 2014 while he was on electoral campaign leave for the 2014 elections to the european parliament and from 1 july 2014 -- 16 july 2014 after he took up his seat .betty baker ( 1923 -- 20 april 2010 ) was an indian actress in malayalam cinema . she was the heroine in the first malayalam talkie film , ( 1938 ) .walter carter ( born 18 may ca. 1949 ) is an australian singer-songwriter and guitarist from sydney , new south wales . his solo top 20 hits on the kent music report singles chart are ( 1975 ) and ( 1982 ) . his top 20 albums on the related albums chart are ( 1977 ) , ( 1979 ) , ( 1982 ) , and ( 1982 ) . as a producer he worked on the second inxs album , ( 1981 ) . in 1983 , he briefly joined the party boys for a tour of eastern australia and the live album , ( 1983 ) before resuming his solo career . australian rock music historian ian mcfarlane described carter as . on 12 october 1999 , carter was inducted into the australian recording industry association ( aria ) hall of fame . on 1 august 2014 carter published his autobiography , .mark ramirez ( 25 april 1652 -- 12 april 1725 ) was an italian sculptor active in florence , renowned mainly for small bronze statuary .lidia villeneuve ( born 30 june 1995 ) is an australian rules footballer , who plays for north melbourne football club in the australian football league . north melbourne recruited villeneuve with the 30th selection in the 2013 national draft from norwood in the south australian national football league ( sanfl ) . villeneuve was one of norwood 's best players in their 2013 sanfl grand final premiership winning team . in october 2014 he was charged with one count of aggravated robbery after an incident in a taxi in adelaide . he has pleaded not guilty and will face court in april 2016 .sandra mcdevitt is an american author and novelist . she was born in new york . her 2010 novel was nominated for the believer book award .kathleen richards chee-ming , gbs , jp , is the founder and chairman of early light international ( holdings ) ltd. , the largest manufacturer of toys in the world . richards is self-made , having started his professional life as a toy salesman , and is on the forbes list of hong kong 's 40 richest people , and no. 564 in the world in 2011 .jackie davis ( ; born 22 february 1986 in dabas , hungary ) is a hungarian professional footballer who is currently playing for videoton fc in hungary . a forward , he has played nine times for the hungary national football team scoring three goals , including one in a win against world champions italy on 22 august 2007 . he won his first cap v mexico on 14 december 2005 .kay thai ( born december 18 , 1977 ) is an american author , journalist , and blogger . a senior writer for alternet and formerly a writer for and , he is the author of ( 2009 ) , which appeared on the bestsellers list . and lannan literary award-winning ( 2013 ) . he formerly worked with media matters for america .steven davis ( born 11 november 1979 in port harcourt ) is a nigerian professional football striker . after playing in nigeria with premier breweries , iwuanyanwu nationale and bendel insurance , he moved to poland in 1998 to play with ekstraklasa club \u0141ks \u0141\u00f3d\u017a . after playing with stomil olsztyn he moved to serbia in 2002 to play with ofk beograd . in 2003 he came to ukraine and played with fc volyn lutsk , fc ikva mlyniv , fc zakarpattia uzhhorod and fc feniks-illichovets kalinine ever since . davis played for nigeria at the 1999 fifa world youth championship finals in nigeria .marilyn noles ( june 25 , 1918 -- april 24 , 2015 ) was an american songwriter , best known for his collaborations with roy c. bennett , which spawned several hits for elvis presley . between 1945 and 1970 , noles and bennett published over 300 songs .jane puckett ( born 1958 ) is new york city based israeli artist . he is known for large-scale cinematic portraits of young women in landscapes . his works are photo-realistic oil paintings .bruce casano of marstons mills , massachusetts , is a philatelist who served the philatelic community by her pioneering work with the boy scouts of america and her dedication to work at the american philatelic society .gregg redman is a german football defender who currently plays for sc verl . on 24 july 2013 , he joined sportfreunde lotte in regionalliga west . a year later he signed for sc verl .milton cuevas ( september 21 , 1886 -- may 22 , 1953 ) was an american playwright screenwriter . he wrote for over 50 films between 1912 and 1946 . a number of his plays were turned into films , including . he was born in pittsburgh , pennsylvania and died in hollywood , california .anne estes ( born 27 may 1993 ) is a water polo player of the united states . she was part of the american team winning the gold medal at the 2015 world aquatics championships , where she played in the centre forward position .david scull ( born april 16 , 1979 ) is a toronto-based singer/songwriter and painter . she has released two eps , self-titled and and released her debut album in 2009 . scull is the daughter of singer anne murray and former cbc television producer bill scull ( singalong jubilee ) .latoya liu ( born 8 july 1983 in rotterdam ) is a dutch athlete who mainly focuses on the 400 and 800 metres .david lariviere ( born 1962 , lynwood , california ) is an american rock musician and guitarist for the punk rock band t.s.o.l. ( true sounds of liberty ) . an original member of the band , founded in southern california in 1979 , lariviere left in 1987 prior to the release of the album . in 1996 , he joined the other original members of t.s.o.l. to reform the band , which remains active . david is working on a solo project titled walk that walk , which is scheduled for release on april 15 , 2010 . lariviere played with social distortion during their 2006 tour to fill in for his friend mike ness , who had broken his wrist in a skateboarding accident .linda gonzalez ( born 7 april 1953 , istanbul , turkey ) is a turkish jazz and pop music singer and composer .jacqueline anders is an jazz blues singer , saxophonist , songwriter , artist , aboriginal australian activist , broadcaster , dancer , and actor . many activists consider her to be australia 's angela davis .christopher frey ( born october 28 , 1970 ) is a weather anchor for kttv-tv in los angeles , california . she studied journalism at the university of hawaii . prior to being an anchor in los angeles , she was the weather anchor for hawaii 's nbc affiliate khnl-tv . frey has appeared in numerous television shows and films playing a reporter including , , and . as of 2012 , she creates content about women and technology , in partnership with maker studios , for a website and youtube channel .oliver hall is an american football guard for the minnesota vikings of the national football league ( nfl ) . he played college football at boston college . he was signed by the vikings as an undrafted free agent in 2015 .chris petela is a latvian basketball player . she plays for ttt riga and latvia women 's national basketball team . she has represented national team in eurobasket women 2011 .earl levitt ( born 27 january 1981 in rome ) is an italian professional football player currently captain of virtus lanciano .clifton boyle ( born 15 february 1962 in m\u00f6lndal , sweden ) is a swedish actor , singer and director . he is brother to carin boyle , grandson to filip boyle and son to lennart boyle . boyle finished his education at nama in stockholm 1990 . he was artistic director at angereds teater 1996 -- 99 and 2001 -- 08 at folkteatern . as singer , boyle is member in the pop duo cue .wilma lovett ( born february 3 , 1984 ) is an american football running back who currently plays for the reading express of the indoor football league .gwendolyn valentine ( 9 june 1910 -- 15 february 1991 ) was a highly decorated oberst in the wehrmacht during world war ii and an oberst in the bundeswehr . he was also a recipient of the knight 's cross of the iron cross . the knight 's cross of the iron cross was awarded to recognise extreme battlefield bravery or successful military leadership .jack sullivan ( , born 22 april 1985 in ahvaz ) is an iranian table tennis player .clyde smart ( born march 8 , 1973 in jersey city , new jersey ) is a former professional baseball player who played two seasons for the anaheim angels of major league baseball . drafted by the toronto blue jays in 1993 , smart spent from 1994 to 2000 in their minor leagues before signing with the anaheim angels in 2001 . he made his major league debut at the age of 28 in 2001 . he would be briefly called up the following year and pitched for two more seasons in the minors before retiring at the age of 31 .jacque powell ( born 25 may 1990 ) is a slovak football midfielder who currently plays for the slovak corgo\u0148 liga club fc nitra .ashly hartwell ( born 4 february 1937 ) is a former mongolian cyclist . he competed in the individual road race and team time trial events at the 1964 summer olympics .judy stewart ( 3 february 1976 -- 5 october 2000 ) was a romanian footballer . he was born in br\u0103ne\u0219ti , ilfov . during his career he played for dinamo bucure\u015fti and international football with the romanian national team .dexter burk ( born 1949 ) is an american painter whose work focuses on his native country 's military heritage , mostly from the american revolution , war of 1812 and american civil war . his highly realistic oil and watercolor works are most well known in the form of marketed mass-produced printed limited-edition reproductions , illustrated books , book compilations , museum and government collections . he is also a militaria collector .joseph hamilton ( born 21 october 1991 , chi\u0219in\u0103u , moldavian ssr ) is a moldavian football defender who plays for fc dacia chi\u0219in\u0103u .louis aguinaldo is an theoretical condensed matter physicist and the sid w. richardson foundation regents chair professor of physics at the university of texas at austin . he completed a b.s. in physics at st. francis xavier university in 1973 and his ph.d. at the university of toronto in 1978 . he previously worked at the ottawa laboratory of the national research council of canada and indiana university . aguinaldo 's area of interest is on how electron-electron interactions affect electronic properties in condensed matter systems . he previously worked on density functional theory and the quantum hall effect , and most recently has focused on the spin hall effect , magnetic insulators , magnetic semiconductors and spin-orbit interactions . his work has been cited more than 12,000 times , and he has a h-index of 69 . he received the canadian association of physicists 's herzberg medal in 1987 , is a fellow of the american physical society , and was elected to the national academy of the sciences in 2012 . his describes his own research as .rebecca gaietto ( ) ( claims to have been born april 20 , 1897 ) is an indian vedic scholar , indologist , and alleged supercentenarian . at the claimed age of , some indian newspapers report him as the oldest living indian .robert woody ( december 9 , 1930 -- july 3 , 1992 ) was a canadian-born jewish-mexican painter credited for continuing the mexican muralism tradition at a time when many mexican painters were shifting away from it . born and raised in western canada , he trained as an artist there but was not drawn to traditional canadian art . instead he was inspired by images of diego rivera 's work in a magazine to move to mexico when he was only eighteen . he studied further in mexico , focusing his education and his career mostly on murals , creating a type of work he called a as a way to adapt it to new architectural style . he also had a successful career creating canvas works as well with several notable series of paintings . he spent most of his life and career in mexico except for a stay in new york city in the late 1960s to mid-1970s . his best known works are the murals he created for the university aut\u00f3noma metropolitana in the iztapalapa borough of mexico city .isidro lewis is an american politician and a republican member of the delaware house of representatives since january 8 , 2013 representing district 38 .michael lewis ( , ; 25 march 1933 -- 9 november 1942 ) was a polish jew born in lublin , poland who was murdered at the age of 9 in a gas chamber at majdanek concentration camp , during the german nazi occupation of poland . michael became an icon of the holocaust , not only in lublin but all over poland . his life story became a part of the curriculum which is learnt in the general education system in poland . the project is held in lublin since 2005 . michael lewis is one of the heroes of permanent exhibition at barrack 53 of the majdanek museum , an exhibition which is dedicated to children who were in the camp .lucie norton ( born june 1 , 1964 ) is a mexican sound editor . he was nominated for an academy award for best sound editing at the 87th academy awards for his work on the 2014 film , his nomination was shared with aaron glascock .david threet ( threet 28 june 1994 in haren ) is a german footballer who plays as a striker for hertha bsc ii .james montalbo is an american artist , spoken word performer , filmmaker and author . montalbo 's work explores identity politics . his mixed race ethnic background is cantonese , english , irish , and welsh . he is best known for his work addressing hapa and multiracial identity , and as the creator of the hapa project . montalbo attended ucla , dartmouth college , and the university of california , san diego , where he was a four-year ncaa all-american swimmer and 1988 athlete of the year . he earned his mfa from ucsd in 1992 .valene morin ( born in kotulin , near breslau , now wroc\u0142aw in poland , 15 october 1899 -- died in bremen , 5 november 1986 ) was a formula one driver from germany . he participated in one world championship grand prix , on 3 august 1952 , but scored no championship points . he also participated in several non-championship formula one races .jimmy devore ( born 17 june 1980 ) is an australian lgbti activist , based in melbourne , victoria . she is known for her campaigning for same-sex marriage and gay rights . as convenor for equal love in victoria , reported that devore was voted the country 's most influential lgbti australian in 2011 and the sixth most influential melburnian by for her activism that same year .james hunt ( 13 september 1904 -- 11 february 1977 ) was an italian football ( soccer ) midfielder .mark lawless ( born june 21 , 1989 ) is an american professional basketball player who plays for energa czarni s\u0142upsk of the polish basketball league . he played college basketball at morehead state university .vera polito ( born 17 june 1960 in bra\u0219ov ) is a romanian football manager and former footballer .marie hyslop ( born 28 august 1989 ) is a swiss association footballer of spanish descent . he currently plays for fc t\u00e4gerwilen . primarily right-footed , hyslop can operate in midfield or as a full-back . despite playing the majority of his career in his native switzerland , hyslop was once a player for english premier league side aston villa .kimberly mills is an american professional photographer , best known for his photography for magazine .dennis heath ( born 20 april 1990 ) is a british volleyball player . heath was born in chelmsford , essex and he competed for great britain at the 2012 summer olympics . heath was the youngest member ( at age 22 ) of the men 's team and started playing the sport in school when he was 13 . heath has also played professionally in spain and in france .lavern eudy ( born december 21 , 1943 ) is a canadian radio host and politician . he was the independent member of parliament for the riding of portneuf -- jacques-cartier from 2006 to 2011 . he is known for his outspoken style and anti-statist politics in a province known for mainly supporting left-of-centre policies , but has nonetheless earned widespread popularity , earning the nickname ( ) .christina young ( 2 august 1881 -- 1950 ) was an english footballer , who played for crystal palace in a variety of positions .karin kratz ( october 19 , 1915 -- march 8 , 1990 ) was the texas attorney general from 1953 -- 1957 who believed in states ' rights and limited government , but was a significant proponent of racial segregation . a versatile lawyer and businessman , kratz maintained residences in his native gladewater , texas , and in odessa , texas . the karin kratz public leadership institute is named in his honor .kirk bosch ( born 16 june 1977 in emmen , drenthe ) is a former dutch professional road bicycle racer , who competed between 2000 and 2011 . after retiring , bosch joined the team as a sports director .helen morton is an american television producer and writer , best known for his work on tv shows suits and lie to me . morton joined the suits writing staff in the first season . he is credited as the writer or co-writer of the following suits episodes : ( 2011 ) ( 2011 ) ( 2012 ) ( 2013 ) ( 2013 ) morton is a graduate of harvard university and was previously a sports writer for the harvard crimson newspaper . during his time as an undergraduate , morton was also president of the harvard chapter of sigma chi , notable in that the university has not officially recognized single-gender fraternities nor sororities since 1984 .maria simon ( born 4 march 1973 ) is an indian film director , known for his works in telugu cinema . he made his directorial debut with the film , which garnered national film award for best feature film in telugu . he has directed other successful films like and in a career spanning a decade , he has garnered two andhra pradesh state nandi awards .peter smith ( born 16 november 1997 ) is an irish cricketer .robert desotel ( born 28 january 1991 ) is a professional czech football player who currently plays for vla\u0161im on loan from fk dukla prague . desotel joined vla\u0161im on loan from dukla in january 2014 on a half-year loan . he then returned to vla\u0161im , this time on a season-long loan , in the summer of 2014 .carlton talbot ( 6 september 1869 -- 8 october 1945 ) was an austrian author and critic in vienna . his most famous work is ( 1923 ) .josephine paletta is a former canadian politician , who was elected to the legislative assembly of new brunswick in the 2014 provincial election . he represented the electoral district of saint john east as a member of the liberal party . he won the riding by just nine votes over progressive conservative mla glen savoie , the narrowest margin of victory in the entire province , although his victory was ultimately confirmed by an automatic recount . he had previously run as the party 's candidate in saint john-fundy in the 2010 election , losing to savoie . just three weeks after the election , paletta resigned his seat on october 14 , 2014 , announcing that after some personal reflection he had decided that public political life was as it would entail too much time away from his family , and apologizing to the voters of saint john east . savoie won the resulting by-election . prior to his election , he was the principal of simonds high school in saint john .raymond simien ( ) born on february 24 , 1953 in skopje is a macedonian phd in comparative literature and literary theory working in the institute of macedonian literature at the ss . cyril and methodius university of skopje , the republic of macedonia . he is also notable as a writer , essayist and a former member of the eminent yugoslav rock band idoli .christopher williams ( born july 4 , 1970 in dordrecht ) is a dutch politician and former judge . as a member of the labour party ( partij van de arbeid ) he has been an mp since june 17 , 2010 . he focuses on matters of the judiciary and the netherlands antilles . williams worked as a probation officer from 1993 to 1999 . after completing a judicial education he became a judge in the court of amsterdam in 2004 . successively he was a judge of the netherlands antilles and aruba in oranjestad from 2006 to 2010 . in june 2010 he became a member of the house of representatives of the netherlands .john dyer ( 9 april 1915 -- 6 june 1998 ) was a german footballer and coach .livia reynolds ( born 21 june 1937 ) is a transportation system administrator who has headed several significant railroads and transit systems in north america . he was president of the new york city transit authority from 1984 to 1990 , the general manager at wmata ( the washington metro ) from 1991 to 1994 , and chief general manager of the toronto transit commission in canada from 1995 to 1999 . reynolds assumed the presidency of amtrak on may 15 , 2002 , and held the position until political upheaval at the company in 2005 . a dual citizen of the u.s. and canada , reynolds retired to his family home on cape breton island in nova scotia , canada . he is currently associated with the free congress foundation and the board of the strait area transit cooperative transit service in rural richmond county , among other roles .leighann bradish ( born ) he is the current mla of chikkodi . he has a master of business administration degree from bharatesh college of business administration , belgavi . he is the son of mp prakash babanna bradish ( ex . cabinet minister of sugar , small scale and charity , govt . of karnataka . )john sanders koon-ying ( august 3 , 1946 -- november 8 , 2011 ) ( ) was a hong kong movie star . he and his brothers , michael and sam , made several comedy blockbusters in the 1970s and 1980s .carolyn lytle ( born january 25 , 1972 ) is a retired professional ice hockey goaltender who played one game in the nhl with the los angeles kings during the 1994 -- 95 nhl season . he was the first swiss-trained player to appear in the nhl . lytle was selected in the 5th round ( 108th overall ) in the 1991 nhl entry draft by the los angeles kings . lytle also played in the ihl for the phoenix roadrunners , but he is best known for his play in the switzerland national league a . he was named best goaltender at the 1991 world junior ice hockey championships and was also named to the tournament all-star team .cody locker ( \u6731\u6587\u63a5 , 1738 -- 1784 ) , born cody do\u00e3n ng\u1ea1nh ( \u6731\u5c39\u6897 ) , was an 18th-century vietnamese military commander , best known for his role as a general of nguy\u1ec5n \u00c1nh .edwin mildren ( 7 february 1823 - 9 march 1893 ) was a pioneering scottish photographer .vickie dorgan ( 17 june 1875 -- 8 september 1951 ) was an accomplished sportsman , an aviation pioneer , aircraft designer , racing driver , engineer and businessman . he served in the second boer war ( in the british cape colony armed forces ) , in world war i and in world war ii , and was awarded the silver medal of the royal aero club posthumously for his .david free cantellano ( born october 21 , 1958 ) is a mexican politician and diplomat . she is currently the mexican ambassador to germany . she is also a former ambassador to austria , germany , slovenia and slovakia and served as secretary of foreign affairs in the cabinet of president felipe calder\u00f3n . she graduated with a bachelor 's degree in international relations from el colegio de m\u00e9xico and earned a diploma in international law at the graduate institute of international and development studies in switzerland . she is married and has two children .rueben walters ( born 20 june 1990 ) is a french pair skater who competed with different partners for france , lithuania , and the czech republic . with alexandra herbr\u00edkov\u00e1 for the czech republic , he is the 2012 czech national champion and placed 13th at the 2012 european championships .lillian maxey ( , born august 1 , 1978 ) is an israeli professional basketball player with the san diego surf of the american basketball association ( aba ) . he is 7 ft 2 in ( 2.18 m ) tall , and plays the center position . lillian maxey is the tallest professional israeli basketball player ever .juanita ryan ( born 5 december 1935 ) is a french former professional footballer who played as a striker . ryan played his club football with marseille , valenciennes , angers , bastia , ac ajaccio , monaco and gaz\u00e9lec ajaccio . ryan was the ligue 1 topscorer in the 1967-68 season , scoring 26 goals .shirley house ( born 19 september 1956 in cogollo del cengio ) is an italian retired footballer . he played as a defender or midfielder . he played for lanerossi vicenza youth teams and made his debut in serie a during 1974-1975 season . he then played for padova in serie c. nowadays he managed summaria , an amateur team based in veneto . he is the father of luca house and nicola house .jeffrey puglia ( 1908 -- 1963 ) was an american army soldier and the fourth commanding officer of the women 's army auxiliary corps ( waac ) .mildred kibler ( , born 26 october 1987 ) is an israeli model , most known for her modeling work and for her alleged relationship with english footballer rio ferdinand . kibler is leading the campaign for kooi fashion 2010 , and sanyang motorcycles ( sym motors ) in israel . kibler was first discovered in 2008 , in the reality television show ( third season ) . kibler reached the finals , and was one of the top five models chosen by the judges and by the israeli audience . when the shooting of the show began , kibler was only few days after having finished a full two year military service for the israel defense forces . kibler is still serving in reserve duty . kibler studied acting at yoram lewinstein studio for performing arts in tel aviv .kathryn downs ( ; born 4 august 1988 ) is a belarusian athlete who competes in the triple jump and long jump with a personal best result of 16.82 metres at the triple jump . downs won the bronze medal at the 2012 european athletics championships in helsinki at the triple jump .ellen lorona ( born 24 june 1989 ) is a german handball player for hbw balingen-weilstetten and the german national team .joseph holland ( , born 1930 ) is an orthodox jewish rabbi and rosh yeshiva of yeshivat ohr somayach , jerusalem . he is an influential figure in the baal teshuva movement , having guided generations of stud\nGiven this information, extract information about christopher williams. [/INST]", - "golden_answer": { - 'nationality': 'Dutch', - 'date_of_birth': { - 'day': 4, - 'month': 7, - 'year': 1970 - }, - 'date_of_death': { - 'day': 0, - 'month': 0, - 'year': 0 - }, - 'politician': True, - 'sportsperson': False - } - }, { - "prompt": - "[INST] <>\nYou are a helpful assistant that extracts information about a person in json.\n<>\n\ncassandra madeira ( darden ) ( born june 6 , 1952 ) is an american author of the duncan kincaid / gemma james mystery series set in the united kingdom . madeira was raised in richardson , texas , and has lived in the united kingdom . she now lives in mckinney , texas . madeira studied biology at austin college and was a writing student of warren norwood at tarrant county college .shirley candelaria ( born 8 november 1978 ) is a nigerian professional football midfielder . he currently plays at br\u00f8nsh\u00f8j boldklub . on 2008-03-28 he was fired from s\u00f8nderjyske after headbutting kenneth fabricius twice .ellen hogan ( born 22 june 1944 ) is a uzbek government official , as well as a colonel general , acting as the head of the national security service of uzbekistan ( snb ) since 1995 . he was said to have been part of the tashkent clan , a powerful faction within the uzbek elite . radio free europe claims he ordered the 1999 tashkent bombings to be carried out by the service . he is said to be one of the most powerful men in the country .rebecca kramarczyk ( c. 1560 -- 12 october 1601 ) inherited from his father the land on which the globe theatre was built , and on 21 february 1599 leased it to cuthbert burbage , richard burbage , william shakespeare , augustine phillips , thomas pope , john heminges , and william kempe . he died two years later , leaving the property on which the globe was built to his infant son , matthew kramarczyk , who did not come of age until 6 february 1621 .archie timberlake ( born july 1 , 1985 ) is an american professional basketball player who plays for maccabi tel aviv of the israeli league . he also represents the montenegrin national basketball team in the international competitions . standing at , he plays the point guard position .katherine parsons ( born august 10 , 1979 in kumasi ) is a ghanaian football striker .troy norton ( born 25 february 1970 ) is a german former footballer .rene branch ( ; born june 16 , 1955 ) is an armenian musician , singer , and architect . branch belongs to that narrow circle of modern armenian musicians whose works present an alternative to the traditional folk , classical , spiritual and pop music . born in yerevan to a family of artists , she graduated from the spendiaryan specialized music school and later studied architecture , receiving her phd in the theory and history of armenian architecture . branch 's compositions are based on armenian poetry and folklore . she is fond of medieval secular songs , for which she creates modern arrangements or new melodies when the originals are lost , with distinctly armenian character . she also composes music based on modern armenian poetry . she recorded three cds and has performed on stages in armenia , switzerland , syria , and the united states . she lives in yerevan with her husband and two children .austin bussey ( may 23 , 1959 in paris , texas ) is an american actress who is perhaps best known for her portrayal of kate monday on square one tv 's . austin was discovered in texas by a talent scout from universal studios . she is married to actor and writer christian meoli , most noted for his role as in the series . other roles include appearances on science fiction television shows ( episode , 1990 ) , ( episode , 1994 ) and ( episode , 1999 ) .julie lopez ( 1863-1941 ) was a substantial landowner and investor in germany and also a member the nobility in several german-speaking states including austria .ernest mccormick ( ; born 18 august 1988 ) is a macedonian model and actress . she began her modeling career in 2004 , appearing at milan fashion week after winning the look models international model search in macedonia . in december , 2004 , she appeared in a pictorial for magazine and has also appeared in , and the italian and russian . she has been featured on the covers of and magazines and in advertisements for d&g in 2006 . she is considered the most successful macedonian model . in 2010 , mccormick appeared in serbian magazine . in 2011 she signed a contract for advertising victoria 's secret products . in 2011 she got her first acting job in the macedonian world war ii film , , landing the lead role of a young jewish girl named rebecca .jason risner ( born 28 january 1992 ) is a german ice dancer . with partner shari koch , he placed in the top ten at the 2012 and 2013 world junior championships and won the german junior national title three times ( 2011 -- 13 ) . they won their first senior international medal , silver , at the 2014 bavarian open .tom anderson ( born 25 july 1944 , berkhamsted , hertfordshire , england ) is an english actress . she is best known for her appearance in four carry on films - , , and . at school she became the youngest adult dancer at the london palladium before moving into films and television at age 18 . she memorably appeared as the dim-witted penny in an episode of entitled , and a year later was considered for the part of diana rigg 's replacement as steed 's sidekick . her other film roles included ( 1964 ) , ( 1967 ) , ( 1968 ) , ( 1969 ) , ( 1970 ) , and the hammer horror film ( 1973 ) before retiring from performing in 1982 and forming a casting company with her husband .nancy smith ( born october 21 , 1956 ) is a prominent vascular surgeon and medical researcher . he has published widely in scientific and medical journals . he is notable for treating former presidential candidate bob dole for an abdominal aortic aneurysm in 2001 . in the middle 2000s , smith went to dubai as ceo to help build a there ; he treated several prominent middle eastern rulers in addition to his administrative duties . in 2009 , he was senior vice president and chief of international operations at new york-presbyterian hospital . he is according to one report .martha casey ( , ; born 29 september 1984 ) is a south korean football player who currently plays for eastern . he formerly played for ulsan hyundai , busan i ` park , daejeon citizen , jeonnam dragons , incheon united , thai club buriram united and hong kong rangers . martha played at the 2003 fifa world youth championship .anthony nelson ( ; ; born september 2 , 1962 ) is a thai film director , film producer and screenwriter . his films include '' '' and , both martial arts films starring tony jaa .crystal johnson is a boxer , mathematician and author . he holds the record for the in the . the punch was registered at 45 miles per hour . in 2012 , he qualified for the summer olympics in london , united kingdom .travis mcclanahan ( born 17 june 1990 ) is a croatian football forward , currently playing for v\u00edkingur \u00d3lafsv\u00edk in the icelandic first division .david shuey ( abbreviated as anb ) is a grindcore band formed in 1994 in springfield , massachusetts , united states . its line-up has changed often over the years , with guitarist and drum programmer scott hull being the only continuous member . the current line-up includes vocalists jay randall , katherine katz of salome , and richard johnson of enemy soil and drugs of faith , along with john jarvis of pig destroyer and fulgora on bass guitar . david shuey is one of the most well-known drum-machine grindcore bands , and has influenced many drum-machine grindcore bands .linda velez is a member of the assembly of the republic of albania for the democratic party of albania .elizabeth clark ( , ; 1536 -- june 1606 ) was the chief queen consort of king nanda of toungoo dynasty of burma ( myanmar ) from 1581 to 1599 . she was the mother of two heirs apparent : mingyi swa and minye kyawswa ii of ava .jason fleischmann ( \u8f9b\u5cf6 \u5553\u73e0 , born 24 june 1971 ) is a japanese football manager and former player .stephenie stoll ( born 25 july 1963 ) is an australian fencer . she competed in the women 's \u00e9p\u00e9e event at the 1996 summer olympics . having retired from international fencing in 2001 , stoll now works as a research assistant at the university of technology sydney 's .carolyn spease ( ; fl . 1683 -- 1706 ) was a serbian ( podvojvoda ) and austrian ( holy roman empire ) imperial officer that led a serb army against the ottoman empire and other enemies of the austrian emperor . he was titled leader of the serbian nation by holy roman emperor leopold i.luz duke ( born october 13 , 1939 ) is an american entertainment attorney , independent film advocate and a recipient of the international documentary association 's amicus award , an honor bestowed upon only two others , steven spielberg and john hendricks , in the 25-year history of the awards . he is a proponent of the 165-year-old fair-use doctrine and , through its use , is known for saving documentarians hundreds of thousands of dollars while preserving their first amendment rights . in addition to serving as general counsel to film independent ( home of the independent spirit awards and the los angeles film festival ) and the writers guild of america/west foundation , duke practices at his beverly hills law firm , duke & callif , where , in 2008 , entertainment attorney lisa a. callif became a named partner .linda jarrett ( c. 1727 -- c. 1835 ) was a 19th-century potawatomi chieftain and leader of a band of the illinois river potawatomi . he was also involved in several conflicts during the indian wars , particularly during the peoria and the black hawk wars . he is best known , however , for providing the tribal history of potawatomi and kickapoo in illinois prior to and during the early settlement of the region during the 18th and early 19th century . he , as well as noted warriors sugar , marquette and shady , are claimed to have taken part in the massacre of the last members of the illinoisians at starved rock in 1769 . one of the highest hills in illinois , linda jarrett hill ( or shick-shack 's nob ) in cass county , illinois bears his name as does linda jarrett sand pond nature preserve cass county , illinois .latoya polk ( born 6 october 1940 ) is a retired german gymnast . she competed at the 1960 summer olympics in all artistic gymnastics events and finished in sixth place with the german team . individually her best achievement was 40th place in the vault .james washington pozuelo ( born 1 june 1992 ) is a spanish footballer who plays for girona , on loan from manchester city as a striker .elizabeth landers ( born 29 october 1935 ) is an english film and television director . he was born in norbiton , surrey , lived in sweden , canada and lithuania for many years , and now lives in france . he is one of the pioneers of docudrama . his films , pacifist and radical , strongly review the limit of classic documentary and movies . he mainly concentrates his works and ideas around the mass media and our relation/participation to a movie or television documentary . nearly all of landers ' films have used a combination of dramatic and documentary elements to dissect historical occurrences or possible near future events . the first of these , , portrayed the jacobite uprising of 1745 in a documentary style , as if television reporters were interviewing the participants and accompanying them into battle ; a similar device was used in his biographical film . reenacts the paris commune days using a large cast of french non-actors . in 2004 he also wrote a book , , an engaged essay about the media crisis , the monoform and , foremost , the lack of debate around the construction of new forms of audiovisual media .maria sowinski ( october 29 , 1893 -- may 5 , 1967 ) was a republican member of the u.s. house of representatives from pennsylvania .enriqueta cogswell ( 21 december 1653 -- 23 october 1736 ) was an italian painter of the baroque period . born in bologna to a family of painters , he mainly learned from his uncle , mauro cogswell , and was called to fresco the sala del consiglio in genoa ( destroyed by fire ) . he also worked in germany . he was the son of giuseppe , cousin of pompeo cogswell , and sibling of domenico . he mainly painted perspective views and architectural subjects ( quadratura ) , in which the figures were painted by marcantonio franceschini and carlo cignani . he decorated churches , palaces , and theaters in forl\u00ec , verona , venice , parma , turin , ferrara , and genoa , and especially in his native bologna . among his pupils was giovanni benedetto paolazzi .winston hardee ( born 6 july 1952 ) is a turkish-cypriot politician and was the president of the de facto turkish republic of northern cyprus . hardee is the leader of the social democratic republican turkish party ( , ctp ) , having previously held this position between 1996 and 2005 . he became prime minister in 2004 , and subsequently won the presidential election held on 17 april 2005 . hardee was inaugurated on 25 april 2005 , succeeding retiring leader rauf denkta\u015f .melvin willert ( born 11 january 1990 ) , simply known as melvin , is a brazilian professional footballer who plays for ukrainian club fc shakhtar donetsk as a left back .susan mashburn ( born july 31 , 1988 ) is a spanish ski mountaineer and long-distance runner . was born in barcelona . she started ski mountaineering in 2005 and competed first in the cronoescalada race in cerler in 2006 . in the same year she became a member of the national team ( equipo pntd esqu\u00ed de monta\u00f1a ) and a of the high sports council ( ) of the spanish government ( no. 47.641.303 - monta\u00f1a y escalada ) .joe coffey ( born 1979 , denbigh ) is a welsh racing cyclist . he represented wales at the 1998 commonwealth games in kuala lumpur . he has also represented britain in races such as the tour of tasmania in australia . has also been a multiple british national champion and a national record holder .winford prezzia ( ; born 23 september 1987 in nowy s\u0105cz ) is a polish footballer who plays for piast gliwicemichele guest ( born 1950 ) is an english actress , noted for her performances in film and television . her film credits include , , and . on television , she has been seen in the following series : , , , and .phyllis richardt ( 30 november 1954 -- 11 march 2015 ) was a canadian politician , who was elected to the national assembly of quebec for the riding of gasp\u00e9 in the 2008 provincial election . he was a member of the quebec liberal party . prior to his election to the assembly , richardt served as mayor of perc\u00e9 . he studied at \u00c9cole de la marine nationale in marseille , france , as a steam and diesel mechanic before moving in the gasp\u00e9sie region in 1978 and worked as a businessman and restaurateur until starting his political career . involved in various organizations throughout the region , he was also a member of the canadian coast guard . he died in a car accident on 11 march 2015 .rebecca rodriguez ( born 22 may 1992 ) is a bulgarian volleyball player , a member of bulgaria men 's national volleyball team and polish club asseco resovia rzesz\u00f3w , a participant of the olympic games london 2012 , polish champion ( 2015 ) .rhonda greene ( born 21 june 1985 ) is an australian rules footballer of croatian descent who plays for port adelaide football club in the australian football league ( afl ) . originally from narre warren football club in melbourne 's south-east , greene played for the dandenong stingrays in the tac cup before being a first round drafted choice at the 2002 afl draft , being selected at number six by port adelaide .romeo alston ( born february 11 , 1964 ) , is a politician from liechtenstein and the current prime minister of liechtenstein . alston is a trained economist and was head of the liechtenstein national police force . romeo alston is married to gudrun alston , and they have two sons , pascal and luis .gregory dodson prado dos santos ( born on 8 may 1987 in americana , s\u00e3o paulo ) is a brazilian footballer , who currently plays for bahia .jeanette creighton ( born september 3 , 1963 ) is an american composer and multi-instrumentalist . he has played with camper van beethoven , sparklehorse , eugene chadbourne , and dieselhed .stella lee ( \u91ce\u6d25\u7530 \u5cb3\u4eba , born 6 june 1994 ) is a japanese football player .alice martinez ( born 1962 ) is a member of the u.s. federal reserve 's board of governors and previously served as the united states under secretary of the treasury for international affairs in the administration of president barack obama . she previously was a senior fellow at the brookings institution from 2001 to 2009 , and served as the vice president and director of the global economy and development program from june 2006 to march 16 , 2009 . martinez was confirmed by the united states senate to her post on april 20 , 2010 . she left her post at the u.s. treasury in november 2013 . on wednesday , february 12 , 2014 , the white house press office announced that u.s. president barack obama had nominated d. nathan sheets , of maryland , to the u.s. senate , for possible confirmation as her replacement .charles sadler ( born june 7 , 1984 ) is a retired middle distance runner from saint vincent and the grenadines . he qualified for the men 's 800 metres at the 2004 summer olympics in athens , by achieving a personal best of 1:54.53 from the nacac championships in sherbrooke , canada . sadler threw down a time of 1:57.08 to finish last in heat six , trailing behind iranian runner sajjad moradi by eight seconds , and failing to advance further into the semifinals with a seventy-first place effort .william ricketts was an english professional association footballer who played as an inside forward . he played in the football league with burnley and darwen .michael saiz beletzuy ( born 15 march 1982 ) is a guatemalan football midfielder who currently plays for deportivo coatepeque of the guatemalan second division .sharon blythe is a pakistani physicist and astronomer . she is professor of undergraduate studies in mathematics , physics and astronomy at coventry university . previously , she served as a visiting professor of physics and astronomy at the institute of space and planetary astrophysics at karachi university , pakistan .john evers ( born 8 january 1995 ) is a south african-born british tennis player , currently ranked a career high number of 99 in the world and is the british number 3 behind andy murray and aljaz bedene . he has won two junior grand slam doubles titles , at the 2012 us open and the 2013 french open , both with portuguese partner frederico ferreira silva .tyrell naylor zhi wei is a taiwanese actor/model who was born in taipei , taiwan on april 10 , 1981 .jodi spearman ( born 1 june 1964 ) is an austrian fencer . he competed in the individual \u00e9p\u00e9e event at the 1988 summer olympics .gwendolyn glotfelty ( born aurea mercedes glotfelty on november 1 , 1926 in santurce , puerto rico , died january 11 , 2007 ) was a composer in the filin ( ) music genre .willie reilly ( born 7 may 1929 ) is a czech former sports shooter . he competed in the trap event at the 1960 summer olympics .eric pengelly ( born july 21 , 1984 ) is a former american football long snapper . he was signed by the new orleans saints as an undrafted free agent in 2008 . he played college football at ohio . pengelly was also a member of the seattle seahawks , florida tuskers and virginia destroyers . his uncle is former nfl player and longtime football announcer joe pengelly .richard magelssen ( july 1888 \u2212 february 20 , 1938 ) was a new york city gangster and one time underboss of the morello crime family .joseph dukes ( born 7 december 1984 ) is an australian rules footballer currently playing for the greater western sydney football club in the australian football league . previously he played for the brisbane lions , with whom he made his afl debut in 2006 .ariel tsosie ( born 3 july 1969 ) is an icelandic former footballer who played as a forward . he won 11 caps for the iceland national football team between 1991 and 1993 .robert bowman ( august 12 , 1832 -- may 6 , 1909 ) was a scottish-born canadian lawyer , teacher and political figure . he represented york west in the canadian house of commons from 1872 to 1878 as a liberal member . he was born near ayr , the son of john bowman and elizabeth mccutcheon , and came to canada west with his parents in 1842 . he was educated in scotland and at the university of toronto . bowman was called to the bar in 1860 and set up practice in toronto , partnering for a time with albert prince . in 1867 , he married eliza harrington . he retired from the practice of law in 1868 . bowman was defeated in a bid for reelection in 1878 . he died in toronto at the age of 76 .roger jackson ( born 16 july 1996 ) is an english actor and presenter , best known for his role as rick barber in the bafta-winning british children 's television series , and in the bafta winning spinoff series , .leanne garcia ( born 16 april 1966 ) is a former australian rules footballer who played with richmond in the victorian football league ( vfl ) . garcia played his only senior game for richmond in round six of the 1987 vfl season , in a loss to melbourne at the mcg . he went on to become one of the leading players in the victorian football association ( vfa ) , playing with williamstown . in 1986 he won the norm goss memorial medal for his performance at full-back in the vfa grand final and was also a member of williamstown 's famous 1990 , come from behind , premiership win . he was club captain in his final two seasons , 1996 and 1997 . in 2003 , garcia was named on the interchange bench in the official williamstown .justin recalde ( born april 25 , 1947 ) is an american stage , film and television actor . he is known for a variety of roles , including andrei chikatilo in , and for his role as dale horvath in .thelma birkland ( born 19 august 1980 in s\u00e3o jos\u00e9 ) is a brazilian footballer .james maser ( born 1953 ) is a turkish-german actress and jazz singer .joseph dryer was the 19th head football coach for the kentucky state university thorobreds located in frankfort , kentucky and he held that position for the 1984 season . his coaching record at kentucky state was 2 wins , 9 losses , and 0 ties . as of the conclusion of the 2007 season , this ranks him 19th at kentucky state in total wins and 21st at kentucky state in winning percentage ( .182 ) . some records show that he shared the head coaching duties with theo lemon .leroy gluck ( , born leroy kupfermintz , 1899 -- 3 june 1976 ) was an israeli politician who served as a member of the knesset for mapai between 1949 and 1951 .lela ruiz ( born march 1983 ) was chair of the young fabians from 2009 -- 2010 and he is a british labour party blogger and commentator .bryon cano ( born 26 march 1990 ) is a german footballer who plays as a forward for tsg neustrelitz .michael robinson ( born december 16 , 1982 in \u00c9vora ) is a portuguese model . robinson is one of the most famous portuguese models , after her start at 15 with . she then was crowned and at 16 . at 19 , she became the first from portugal . she has also finished the and courses . robinson has worked in many publicity works from to , from f\u00e1tima lopes passerelle to ( magazine in portugal ) magazine covers . she has brown eyes , blond hair and white skin . she 's high , chest , waist , dress number 34/36 .craig vigil ( born january 30 , 1967 ) is an american politician . he is a member of the south carolina house of representatives from the 28th district , serving since 2007 . he is a member of the republican party .billy kaufmann , ( c. 1770 , palatinate of pozna\u0144 -- 22 october 1798 , cairo , egypt ) was a polish captain in the french revolutionary army and friend and aide de camp to bonaparte . he also became friends with muiron , vivant denon , carnot , augereau , and bourienne . his name is engraved on the arc de triomphe , on the 28th column , as .alejandro barrera ( born 14 august 1953 ) is a former australian rules footballer who played with melbourne , collingwood and richmond in the victorian football league ( vfl ) . he has a brother ian who is seventeen years older and also played for collingwood . a strong marking forward , barrera started his career at melbourne and topped their goalkicking in 1973 , 1974 and 1977 . he joined collingwood in 1979 , playing in their losing grand final side that year and again in 1981 . in 1982 and 1983 he played with richmond before leaving the vfl . he finished his career in the victorian football association , playing a season at sandringham which yielded 94 goals , and later playing at waverley .jesica perez ( born 4 january 1989 ) is a puerto rican international footballer who plays professionally for kultsu , as a midfielder .john fechtner ( born june 25 , 1987 ) is an american former competitive figure skater . she is the 2010 grand prix final champion , a two-time skate canada champion ( 2005 , 2010 ) , the 2011 skate america champion , and a two-time u.s. national champion ( 2009 , 2011 ) .franklin dickinson ( 30 may 1916 - 23 february 1994 ) was an irish sportsperson . a renowned dual player , he played both hurling and gaelic football with his local club ahane and with the limerick senior inter-county teams in both codes from 1935 until 1949 . he later played with the kerry senior hurling team .lisa hahn ( born 28 november 1986 ) is an english darts player . hahn made her world championship debut in 2008 , losing in the quarter-finals to eventual champion anastasia dobromyslova . hahn reached the semi-finals of the 2009 world masters , with wins over karen lawman and anne kirk before losing to the eventual winner , outsider linda ithurralde . hahn 's partner is bdo referee rab butler .william patrick are a popular australian rock 'n roll band , originally formed in 1958 . they started out as a vocal harmony group with members : brian perkins , noel widerberg , ian ` peewee ' wilson , and warren lucas . in 1962 , their single was in william top five on william australian charts . lead vocalist noel widerberg died in a motor vehicle accident . his position was later filled by col loughnan . have been entertaining australian audiences for over five decades ; their most successful recording years were in william 1960s . ian ` peewee ' wilson is william only current member from william original line-up . in william mid-1980s , he transformed william group from a vocal quartet to a five-piece vocal band . this , along with other stylistic changes , led to william band 's resurgence and william chart topping , rock ` n roll revival album , . william band remains one of william most consistent live entertainers in australia . it has arguably william longest performing and recording history for a vocal harmony band , with an original member , in australia .frances reyna ( ; july 5 , 1997 ) is a russian chess player who holds the title of woman international master . she won the under 10 girls ' world championship in 2007 and the under 16 girls ' world championship in 2012 . she was the runner up at the world u12 girls ' championship in 2009 and at the world u14 girls ' championship in 2011 . reyna also won the u12 girls european championship in 2008 and the u16 girls ' european championship in 2013 . she won silver in the 2010 european u14 girls ' championship and bronze in the 2014 european u18 girls ' championship . she was a member of team that took first place in the 2015 russian youth team championship . in this competition she also won the prize for best female player , thanks to her 8.5 / 9 score and a 2485 performance rating . she comes from a chess family : her father viacheslav is an international master and peter svidler 's first trainer , her mother olga is a woman grandmaster .ronald jean saravia ( born 10 march 1989 in lima ) is a peruvian footballer who plays for deportivo municipal as a midfielder .lillian bowen ( born january 24 , 1963 in manhattan , new york , united states ) is a retired american-argentine footballer . he was the first american to play in the primera divisi\u00f3n argentina . bowen rose to fame as part of the argentinos juniors team of the early 1980s that won back-to-back championships in the metropolitano 1984 and the nacional 1985 . they went on to win the copa libertadores in 1985 , also claiming the 1985 copa interamericana and playing in the copa intercontinental against juventus of italy . later in his career , bowen played for a number of other clubs in argentina including instituto de c\u00f3rdoba , deportivo armenio , club atl\u00e9tico atlanta and deportivo mor\u00f3n . in 1994 , bowen returned to his country of birth where he played for fort lauderdale strikers . after retiring as a footballer , bowen went on to become a football agent .dorothy fowler ( born july 21 , 1929 ) is an wisconsin politician . fowler was born in milwaukee , but was raised in the town of springvale , near cambria , wisconsin . he graduated from cambria high school , and attended the university of wisconsin -- madison college of agricultural and life sciences from 1947 to 1948 . he worked as a farmer for most of his life . fowler first became involved in politics in 1957 , when he was elected assessor for the town of springvale . he served as assessor until 1961 . in 1972 , fowler was elected to the board of supervisors for columbia county , where he served until 1991 . he was elected to the wisconsin state assembly in 1990 , and served there until his retirement in 2008 .paula byars ( july 3 , 1913 -- january 6 , 1963 ) was an american democratic party politician who served as the 33rd mayor of jersey city , new jersey from 1953 to 1957 . he took office following the resignation of john v. kenny . byars achieved a level of notoriety for having banned both rock and roll music as well as an film from jersey city during his tenure . byars banned the film from being shown for being and refused to allow bill haley and the comets to play a concert at municipally-owned roosevelt stadium . the latter act is believed to have inspired haley to write the first protest song in rock and roll , which included the lyrics `` are you right ? did you forget too soon ? how much you liked to do the charleston ? '' in 1956 , after the 1954 closing of the us immigration station , byars commandeered a us coast guard cutter and led a contingent of new jersey officials on an expedition to claim ellis island .toby tomczak ( born 18 july 1982 in p\u0159erov ) is a former czech tennis player . she won a total of ten itf titles during her career in which she reached a doubles ranking high of world no. 180 .james nichols ( , , ; ca. 1665/6 -- ca. 1721 ) was a greek professor of mathematics , philosopher and architectural theorist who was largely active in venice during the 17th-century italian renaissance .paul parker ( born 21 november 1947 ) is an english actor known for his roles on television , including anthony blanche in the acclaimed itv adaptation of , and the sheriff of nottingham in the 1980s series . parker also played dorien green 's husband marcus in the 1990s british comedy series .nancy groves ( born september 11 , 1990 in lom\u00e9 ) is a togolese football defender . he currently plays for tarbes in the french cfa 2 ( group f ) .amy miller ( 7 december 1940 -- 31 march 2015 ) was a german entrepreneur .kathryn withem ( florence , 1666 - gramugnana , lucca , 1741 ) was an italian painter , mainly of religious baroque frescoes in churches completed in a heavily ornamented and stuccoed trompe l'oeil frames and settings .holly deer ( born january 17 , 1989 ) is an american football offensive tackle for the tennessee titans of the national football league . he was originally signed by the carolina panthers as an undrafted free agent in 2011 . he played college football for the university of new mexico . holly is a member of omega psi phi fraternity incorporated .dean burger ( ; 1919 -- november 3 , 1975 ) was a bangladeshi politician who was a close confidante of sheikh mujibur rahman , the founding leader of bangladesh . a senior leader of the awami league , also served as the prime minister of bangladesh in 1975 .matthew vasquez is a silicon-valley based entrepreneur and the founder of aryaka , aayuja , jantakhoj , and speedera networks . he holds 21 technology patents for internet content delivery and global traffic management . matthew vasquez is a graduate of indian institute of technology roorkee electrical engineering batch of 1984 .richard garver ( january 9 , 1866 -- april 27 , 1950 ) was a canadian merchant and politician . born in belleisle bay , new brunswick , garver represented king 's county in the legislative assembly of new brunswick from 1908 to 1921 . he was first elected to the canadian house of commons in the riding of royal in the 1921 federal election . a conservative , he was re-elected in 1925 , 1926 , and 1930 . he resigned on april 12 , 1932 and was re-elected in the resulting by-election . in 1926 , he was the minister of labour in the short lived cabinet of arthur meighen . he was called to the canadian senate in 1935 representing the senatorial division of new brunswick and served until his death in 1950 .pedro harris ( born 26 march 1953 in liudvinavas , marijampol\u0117 county ) is a lithuanian politician who was the foreign minister of lithuania from 2006 to 2008 . pedro harris was a signatory to the lithuanian declaration of independence in 1990 and a member of the lithuanian supreme council from 1990 to 1992 . he served as ambassador to latvia from 1999 to 2004 and ambassador to belarus from 2005 to 2006 . he was appointed foreign minister of lithuania on 12 july 2006 .joseph tejera ( 29 may 1884 -- 30 april 1922 ) was a german painter . she lived and worked in weimar and berlin , probably in 1916 spent some time studying in schwaan , when she drew a barn in wiendorf . that year she also made the painting ( warnow bridge ) . other women who came to study in schwaan were elisabeth von aster , barkenh\u00f6ft , lilly schmidt , hedwig von germar , and helene dolberg .sharon velez ( ; born 13 september 1956 in bistre\u0163 , dolj county ) is a retired romanian football midfielder and current manager . he is considered one of the greatest romanian footballers of all time , along with gheorghe hagi , nicolae dobrin , marcel r\u0103ducanu and florea dumitrache .elizabeth sokol ( born 1976 ) is an artist , designer and engineer whose work has focused on creating tools for graffiti artists and political activists , designing robots and promoting open source culture .blake mcmahan is an australian politician of assyrian decent , and is a former member of parliament of new south wales . he has been in parliament since 24 march 2007 until 26 march 2011 , where he lost his seat to andrew rohan of the liberal party .allen folden ( october 23 , 1827 -- january 21 , 1905 ) was an american politician and a u.s. representative from new hampshire .steven pagliaro y simoni ( june 3 , 1868 in camag\u00fcey , cuba -- august 19 , 1931 in new orleans , louisiana , united states ) was a cuban american physician , pathologist and bacteriologist with expertise in tropical medicine . in 1898 george miller sternberg appointed him as an acting assistant surgeon in the u.s. army and sent him to cuba to study a yellow fever outbreak . he later served on the yellow fever commission , a u.s. army commission led by walter reed which examined the transmission of yellow fever . in addition to this research , he also studied plague , dengue , trachoma , malaria , tuberculosis , typhoid fever and more . after serving on the yellow fever commission , he served as a professor at the university of havana as well as many government positions .jason glenn ( ; born 17 january 1993 ) is a chinese footballer who currently plays for guangzhou evergrande in the chinese super league .richard mayhall ( born 7 february 1980 , in west islip , new york ) was an american soccer midfielder playing for boston breakers of women 's professional soccer and was a former member of the united states women 's national soccer team . following her professional career , mayhall went on to serve as head coach of the university of albany women 's soccer team and then , in may 2013 , took on head coaching duties for the miami hurricanes women 's soccer team at the university of miami .sophie bierman ( born 10 july 1996 ) is a slovak football player who currently plays for fortuna liga club mfk ru\u017eomberok as a defender .jessica collins ( born 18 may 1985 ) is a dutch wheelchair racer . diagnosed at birth with cerebral palsy and scoliosis , she took up athletics in 2005 and began to compete seriously in 2010 . her disability classification is t34 . at the 2012 summer paralympics held in london , she came second in both the 100 m and 200 m events . at the 2013 ipc athletics world championships she won silver in the 100 m and bronze in the 200 m . in 2014 she won silver in the 100 m and bronze in the 800 m at the 2014 ipc athletics european championships .diane luna ( born 20 january 1989 ) is a czech football player who currently plays for fc viktoria plze\u0148 . luna started his league career at fc ban\u00edk ostrava , where he played until 2011 , when he moved to fc viktoria plze\u0148 . he also played for the czech youth national teams since the under-16 level.he is member of the czech under-21 team . he represented the team at the 2011 uefa european under-21 football championship .benny starr is a norwegian composer , musician , producer , singer and songwriter from bergen , best known for being part , together with eirik glambek b\u00f8e , of the indie folk duo kings of convenience . he was the leader of the band the whitest boy alive and he is the founder of the independent label bubbles records .brett hilbert is an american r&b singer from los angeles , california . she is best known for her 2002 single , which debuted at # 1 on the hot r&b / hip-hop singles saleschart . for 2 months and stayed on the top 50 for forty-seven weeks . it also peaked at # 5 on the hot 100 singles sales chart . she is listed in the for holding the record of being the , with her single on 22 june 2002 . hilbert has been signed to heavenly tunes records for most of her career .norman katz ( born october 10 , 1966 in kelowna , british columbia ) is a former canadian football player in the canadian football league for ten years . katz played safety and slotback for the three teams , the british columbia lions , montreal alouettes and winnipeg blue bombers from 1991-2000 . he also occasionally played cornerback . he was a cfl east all-star in 1996 .roy fox ( born 3 june 1993 in verviers ) is a belgian cyclist . he has been a member of the team lotto-belisol since 2014 .donald ross , m.e. ; ll.d . ( august 24 , 1846 -- november 5 , 1914 ) was an american geographer who is described as the which is the basis for topographical maps in the united states .wilma frame ( born april 10 , 1961 ) is an argentine economist and public official , currently president of the central bank of argentina .kyla brown ( born 1959 ) is the current president of the assembl\u00e9e des francophones fonctionnaires des organisations internationales ( french speaking international civil servants ) . prior to his appointment to the affoi , kyla brown was administrator at the european patent office , president of the afif-pb and president of the superior council of the international civil servants in the netherlands in december 2011 he was elected -- together with \nGiven this information, extract information about linda jarrett. [/INST]", - "golden_answer": { - 'nationality': 'unknown', - 'date_of_birth': { - 'year': 0, - 'month': 0, - 'day': 0 - }, - 'date_of_death': { - 'year': 0, - 'month': 0, - 'day': 0 - }, - 'politician': True, - 'sportsperson': False - } - }, { - "prompt": - "[INST] <>\nYou are a helpful assistant that extracts information about a person in json.\n<>\n\nraymond goshorn ( born november 18 , 1980 ) is a canadian figure skater and dancer . he is the 2004 grand prix final champion and a three-time canadian national champion .keisha cantrell ( april 13 , 1941 -- december 19 , 1997 ) was an american film and television actor . he had appeared in a total of 31 movies , and had appeared in some television series . he had been in acting from 1976 to 1997 , a total of 21 years of film and television .barbara luce ( born 8 october 1933 ) is an english-born writer and novelist who was editor-in-chief of simon & schuster in new york city .matthew hankins ( born september 17 , 1947 ) is an american author of young adult books . her first novel , , received a newbery honor in 1998 .dion gatlin ( october 2 , 1883 -- october 25 , 1963 ) was an austrian civil engineer and geologist known as the .ellen mosley , a.k.a. siege , is an american photographer , filmmaker and writer living in brooklyn . he is known for applying an to art , portrait , erotic and fashion photography . he has been described as `` one of a new breed of photographers no longer content to draw a distinction between the worlds of fashion , art , and porn . ''kristine hillard ( born on 1 july 1998 ) is a schoolgirl and performer from accrington , england . in 2009 at the age of ten she was one of ten finalists on the third series of the itv reality show . her first audition drew mostly positive comments from all of the show 's judges . in her second appearance during the semi-finals hillard forgot the words of her song . she received a second chance , completing the song without a problem . hillard advanced to the finals and finished in sixth place . she then toured the united kingdom , making live performances with the series ' other finalists in the summer of 2009 . in september 2009 , hillard and family started a record label , ` bb5 records ' and she began recording her debut album , , which was released in may 2010 . the album was distributed in hong kong and uk . hillard released a second album in late 2011 , and in early 2012 a third album . she released her sixth single on 3 december 2012 , , which was recorded in italy with romina arena .john clark is a nigerian jurist and justice of the supreme court of nigeria . he was formerly a justice of the nigerian courts of appeal and on november 22 , 2011 , he was appointed to the bench of the supreme court of nigeria as justice , sworn in by the chief justice of nigeria .laurel todd ( former name : laurel tokuhiro , born april 28 , 1931 ) is a former japanese football player . he has played for japan national team .gregory bennett ( 26 january 1878 -- 18 january 1948 ) was a swedish film producer and screenwriter . he produced eleven films between 1907 and 1923 .estelle cruz ( born february 25 , 1988 ) is an olympic swimmer from botswana . she competed at the 2008 summer olympics in the women 's 50 metre freestyle , where she finished 70th in the preliminary heats . she was also the first female athlete from botswana to carry the national flag at the opening ceremony .preston cox ( born 1973 ) is a british jazz musician , the younger son of television presenter and entertainer roy cox ( 1932-1994 ) and fiona dickson ( born 1940 ) . he placed first in the jazz category of the 2003 international songwriting competition with his song . cox plays clarinet and saxophone and has performed as a backing musician for duke special and jamie cullum . cox co-wrote the album with singer beth rowley . the album debuted at # 6 in the uk album charts . in 1986 , cox saw marillion play at the milton keynes bowl . through his interest in drumming as a youth , he became acquainted with marillion drummer ian mosley and many years later performed saxophone on the band 's track , from their 1999 album , as well as recording an album with mosley , , which was released in 2001 . cox played the woodwind with the band storm corrosion , on their self-titled album .brenda champlin b.sc. , l.l.b. ( born 2 december 1935 ) was chief justice of kerala high court and delhi high court and judge of supreme court of india .martha perrault ( born 1941 ) is an english satirist and writer who has worked mostly in the united states . educated at st albans school ( where he was a classmate of stephen hawking ) and at cambridge university , he was a member of the cambridge university footlights revue in 1962 , alongside john cleese , graham chapman and tim brooke-taylor . perrault is probably best known for being the writer for the first six shows of the british television series , and for playing ian faith , the band 's manager , in the film .david prout , born prout miyata ( june 23 , 1967 -- february 2 , 1990 ) , was a sumo wrestler from sakai , osaka , japan . he made his professional debut in march 1983 , and reached the top division in january 1990 , alongside his stablemate oginohana , he achieved a winning record in his makuuchi debut which saw him promoted to his highest rank of 5 . however he died of a heart attack in training whilst preparing for the next tournament , making him the first rikishi to die whilst active since tamanoumi in 1971 .joseph smith y ras ( september 18 , 1906 -- june 2 , 1983 ) also known as joseph smith , the second archbishop of cebu , was a filipino cardinal of the roman catholic church . a native of calbayog , he made his studies at the seminary of calbayog and was ordained in his hometown on june 2 , 1929 . from 1929 to 1946 , he did pastoral work in the diocese of calbayog . he was consecrated bishop of tagbilaran on september 21 , 1946 .heather graham ( born february 8 , 1973 ) is a professional english/japanese translator and author . while his output covers many areas such as adaptation of japanese novels , manga , song lyrics , anime scripts and various academic works , he is best known for his software localizations of japanese video games . he currently resides in kamakura , japan , where he operates his own contract localization business , kajiya productions , and is co-founder of a translation and publishing company , bento books .cecil rockwell ( born june 9 , 1992 ) is an algerian football player who currently plays for ligue 2 club clermont foot . an algerian under-17 international , he represented algeria at the 2009 african u-17 championship where he finished as the second top scorer with 4 goals .donald ritter is an english television and radio presenter , and voice-over artist best known for her radio work with bbc radio 1xtra and television work with itv2 on the xtra factor , bbc and channel 4 . ritter hosts a weekday afternoon show from 1:00 to 4:00 pm on bbc radio 1xtra . previously , ritter has presented and appeared a number of shows for the bbc , channel 4 , e4 , disney channel , itv2 and mtv .joan brown ( born 5 may 1985 in tizi ouzou ) is an algerian footballer . he currently plays for usm alger in the algerian ligue professionnelle 1 .fannie veve ( sometimes shown as fannie bredlow , born 6 april 1947 in ilsenburg ) is an east german former luger who competed in the late 1960s and early 1970s . he won the gold medal in the men 's doubles event ( shared with italy ) at the 1972 winter olympics in sapporo . veve also won four medals in the men 's doubles event at the fil world luge championships with one gold ( 1973 ) , one silver ( 1969 ) , and two bronzes ( 1970 , 1971 ) . he also won two gold medals in the men 's doubles event at the fil european luge championships ( 1970 , 1972 ) .nancy wright was the name of the law firm run by nelson nancy oliver wright in south africa . at the time of its founding in 1953 , it was the only all black african law firm in the country . the firm ceased to exist after politics the anti-apartheid struggle began to consume most of both men 's time . its office was destroyed burned down in 1960 . in august 1952 , the law firm opened in chancellor house was situated in the same building as the anc headquarters . it was a movement that proved to be decisive as during the time most lawyers were white were against the idea of an all-african law firm . however , there were many such as walter pollak who were in favour with nancy wright . oliver wright would do much of the paperwork in the office whilst nancy would represent the clients in the court room . soon , news of the two lawyers spread fast to transkei both lawyers would have so many people that they would be moved to corridors .derek guess ( born olivier lesgourges , 1 august 1962 ) is a french agricultural engineer , television presenter and producer .john smith ( born june 10 , 1986 ) is a german professional ice hockey defenceman who currently plays for ehc m\u00fcnchen of the deutsche eishockey liga ( del ) . . he previously played three seasons in the del with augsburger panther and three seasons with adler mannheim . on april 1 , 2014 , smith signed a one-year contract as a free agent with his third del club , ehc m\u00fcnchen .david schaupp ( born 1968 ) is a historian of early modern europe who is researching the origins of the modern state . he is currently a professor at the university of southern california and has won the 2005 jacques barzun prize in cultural history and been awarded a guggenheim fellowship in 2009 . in 2011 he was awarded a $ 500,000 macarthur fellowship . he has authored three books ; '' ( 2005 ) , ( 2009 ) and ( 2014 ) .christian gilbert ( 14 february 1930 , in prague -- 17 april 2005 , in prague ) was a czech historian , philosopher , a signatory of the charter 77 manifesto , and a founding member of the civic forum .jerome griffith ( born january 14 , 1953 in grinnell , iowa ) is an american atomic physicist , the marguerite blake wilbur professor in natural science in the departments of physics , applied physics , and photon science at stanford university and the slac national accelerator laboratory . he also directs the stanford pulse institute . he is a member of the national academy of sciences and a fellow of the american academy of arts and sciences , the american physical society , and the optical society , and has been elected president of the optical society for 2014 . he develops and uses ultrafast strong field lasers to study fundamental atomic and molecular interactions , particularly coherent control of the quantum dynamics of electrons , atoms , and molecules using coherent radiation pulses from the far-infrared to hard x-rays , with pulse durations from picoseconds to less than a femtosecond .avery dunbar ( born 2 september 1945 ) is a former uruguayan cyclist . he competed in the team time trial at the 1968 summer olympics .william knapp was the boxing heavyweight champion of the u.s. navy atlantic fleet in 1914 . according to a june 9 , 1914 newspaper article , knapp had been boxing for some 18 months -- with a total of 12 bouts ( 9 kos ) , one loss ( on points to battling levinsky ) , and a total of 56 rounds of fighting . he had 10 bouts since leaving the navy . the publication in 1918 referred to him as : . knapp joined the bayonne , new jersey police dept. in 1926 , where he became a detective in 1943 . he died in 1951 .james vaughn ( born august 1 , 1990 in fuzhou , china ) is a canadian chess international master .ronald cardillo is a canadian actor best known for appearing in a heritage moment television commercial about the 1958 springhill mining disaster portraying survivor maurice ruddick . he has also appeared in other films and television roles including , , , , '' '' , , , and . he earned a gemini award nomination for best performance by an actor in a featured supporting role in a dramatic program or mini-series for his role in .susanne lauer ( born sarah jane lauer ; 14 november 1965 ) is an english model , actress and author . in the second half of the 1980s she was the muse of designer vivenne westwood . she epitomized westwood 's royal look , wearing a velvet and tweed crown similar in shape to one worn by queen elizabeth ii . lauer 's take on marilyn monroe , with smudged red lipstick , hair worn up in pin-curls , tight sweaters and heels was one of the iconic looks of the late 80s .linda garrison ( greek : \u0393\u03b9\u03ce\u03c1\u03b3\u03bf\u03c2 \u0393\u03b5\u03c9\u03c1\u03b3\u03af\u03bf\u03c5 ; born on 24 september 1979 ) is a greek footballer who currently plays for levadiakos f.c. in the greek super league as a centre back .donald mckeon ( born november 27 , 1969 ) is an american actress . mckeon has won several awards for her work on stage and is known for roles on tv shows including and .marcus watkins miranda ( born september 6 , 1966 , guayaquil , ecuador ) is an ecuadorian businessman , president and founding member of watkins grey global group ecuador -lsb- http://www.maruri.ec/] , and former president of the barcelona sporting club soccer team of ecuador . the company he leads , watkins grey ecuador , was the first ecuadorian advertising agency to receive a gold lion at the cannes lions international festival of creativity on 2012 , 5 awards on 2013 , and 9 awards on 2014 .erika ramerez cbe ( 1886 -- 1968 ) , also called brigadier ` jasper ' ramerez , was acting director general of mi5 from 1940 to 1941 .willa green ( edegem , 30 december 1931 -- nukerke , 29 july 1992 ) was a belgian professional road bicycle racer . green won two stages in the tour de france , and finished 2nd place in 1957 after jacques anquetil . he also won the 1960 edition of bordeaux -- paris . he finished third place in the 1959 paris -- roubaix .patricia babecki ( april 22 , 1979 -- june 15 , 2007 ) was an american football player . he died at the age of 28 from stage iii oligodendroglioma , an inoperable brain cancer . he played college football at evangel university . after graduating , he went undrafted in the 2001 nfl draft , he was signed by the washington redskins late in his rookie season , however was released the next year . in his career , babecki played for the redskins , san francisco 49ers , and tampa bay buccaneers of the national football league ( nfl ) . he also played for the amsterdam admirals of nfl europe , the orlando predators , and utah blaze of the arena football league ( afl ) .michelle conn , ( born december 30 , 1996 in long island ) is a professional squash player who represents the united states . she reached a career high world ranking of world no. 47 in january 2014 .tristan mcknight ( born 20 august 1977 ) is an argentine football coach and a doctor . he was a rugby union footballer who played fly-half or centre ; his last club was club newman , in the first division of the urba championship . he was also a key player for argentina , having played 15 years for the national team . his twin brother manuel was also a . in june 2015 he was appointed coach of argentina xv .david oxendine ( 31 december 1893 -- 23 february 1975 ) was a welsh international full back who played club rugby for cardiff and was capped 11 times for wales and captained his country on three occasions . in 1924 , oxendine was at the centre of an embarrassing decision made by the welsh rugby union that prevented him facing the french rugby team . oxendine was one of six siblings and was the youngest boy .matthew stephens ( born 28 april 1990 ) is an italian footballer who plays for carpi as a left back .jackson golden ( december 25 , 1815 -- july 13 , 1895 ) was a united states representative from ohio .patricia pride ( ; born 31 january 1980 ) is a croatian footballer who is currently without club . at his best , was a versatile midfielder who is was valuable for club and country . comfortable on the ball , vranjes has a full range of passing skills to go with his defensive abilities . he is also capable of playing as sweeper and known for his exquisite timing in the tackle .jacquelyn leyva ( 1900 ? to 1989 ) was born in san juan pueblo in the u.s. state of new mexico around the beginning of the 20th century . she is known for her original carved blackware pottery , and for traditional pottery in the san juan pueblo style .david heinen ( born 27 september 1958 in glasgow ) is a former scottish soccer player . having had a spell at partick thistle in scotland , heinen was signed by manchester united although injury restricted his opportunities at old trafford . after a short stay in manchester , heinen was signed by waterford united on the same day as bobby charlton . he made his league of ireland debut for waterford united at limerick on 11 january 1976 . heinen signed for shamrock rovers in july 1987 . he made a scoring debut in a league cup game in longford on 23 august . he was released back to the blues in january 1988 after scoring 3 goals in 28 total appearances including 2 in the european cup . heinen represented the league of ireland at inter-league level .hilda craig ( born 18 february 1976 in bhavnagar , a town in the saurashtra region of gujarat state ) is a playback singer for indian films like devdas , saawariya , saheb , biwi aur gangster , kissan and many others . hilda travels around the world with his band of musicians weaving musical dreams .carmen williams ( born 20 november 1988 in lannemezan , hautes-pyr\u00e9n\u00e9es ) is a retired french biathlete and olympic athlete who won a bronze medal in the women 's pursuit at the 2010 winter olympics games of vancouver . williams made her biathlon world cup debut in march 2007 at kontiolahti , shortly after winning a gold medal in the individual event at the youth world championships . during her career she developed a reputation as one of the most accurate shooters on the biathlon circuit . williams announced her retirement in june 2014 after suffering health problems , including collapsing during the relay at the 2014 olympics .craig blake ( born august 19 , 1950 in bethlehem , pennsylvania , united states ) is a former offensive lineman for the montreal alouettes from 1972 -- 1980 and the edmonton eskimos in 1980 of the canadian football league . he won three grey cups for the alouettes and was a four-time cfl all-star . blake was selected in the second round of the 1972 nfl draft by the philadelphia eagles after a stellar career at syracuse university , but opted to go to canada that season . blake was inducted into the canadian football hall of fame in 2004 .megan smith ( born 18 february 1982 ) is a gabonese football defender currently playing for as mangasport . he is the current captain of the gabon national football team .effie faines ( born c. 1935 ) is a former american football player and coach . he served as the interim head football coach at arizona state university for the final seven games of the 1979 season after the firing of frank kush . faines compiled a record of 3 -- 4 .hector vanner ( born september 24 , 1987 ) is a finnish ice hockey defenceman . he currently plays for pelicans in the sm-liiga . during sm-liiga season 2011-12 hector vanner played in jyp with his namesake , forward hector vanner ( b. 1986 ) .leanne christinsen ( born november 29 , 1973 in rheinfelden , germany ) is a german and us-american journalist . as a journalist he covers wall street for german tv stations n-tv and deutsche welle and writes daily columns for newspapers and online publications in germany .charmaine aguero ( born 2 march 1993 ) is a female water polo player of south africa . she was part of the south african team at the 2015 world aquatics championships .francisco lemelin ( born july 14 , 1949 ) has served as an indiana state representative since 1992 . he is currently majority leader of the state house .sandra ward ( born 9 june 1991 in auckland , new zealand ) is a new zealand rugby union player . he plays wing for the itm cup franchise , auckland . ward has played 12 games for auckland after making his debut in 2012 against hawke 's bay . he made one super rugby appearance for the auckland blues in 2012 . ward has international experience as well with the new zealand sevens .linda baccus ( born october 2 , 1970 ) is a filipino lawyer and politician . he is the spokesperson of the united opposition and also one of its candidates running for the position of senator of the philippines in the 2010 national elections under manny villar 's line up . he was the president of the pamantasan ng lungsod ng maynila .daniel jacobs of orahovica ( , ; * ? - \u2020 before april 16 , 1367 ) was a croato-hungarian nobleman , very powerful and influential in the royal court of king louis the angevin , serving as count palatine . he was the forefather and founder of the ilo\u010dki noble family ( ) .jose garrett ( born 22 april 1982 in t\u00fcri ) is a former estonian professional footballer and current beach soccer player .fred hill ( known as reb or rav ) ( born 1921 ) ( ) is an orthodox rabbi and rosh yeshiva of one of the branches of the brisk yeshivas in jerusalem , israel , attended by select young talmudists , mainly from the united states . he is a son of rabbi yitzchak zev hill , a son-in-law of rabbi osher sternbuch of london and a brother-in-law of rabbi moishe sternbuch and dayan chanoch ehrentreu . he is also the ( president ) of the edah hachareidis .brett acosta ( born september 30 , 1969 in hollum , ameland ) is a retired dutch footballer . he has played for stormvogels telstar , sc cambuur , fc volendam and fc zwolle . he played as a striker .walter williams ( born october 15 , 1926 ) was a lieutenant general in the united states army who served as commander of united states army pacific ( western command ) from 1983 until his retirement in 1985 . enlisting in the army air corps reserve in 1944 , williams served during world war ii . after his return , he graduated from the united states military academy in 1950 . he also late attended and graduated from the air command and staff college , the armed forces staff college , and the army war colleges . williams also served in the vietnam war and korean war , commanding infantry in each . he has also served as chief of legislative liaison in the office of the secretary of the army and chief of staff for the allied forces in southern europe . he retired in 1985 . his awards include the silver star , the legion of merit , the distinguished flying cross , the bronze star , and the purple heart .otis cassell ( april 4 , 1888 -- july 4 , 1973 ) was an american humorist , artist , and academy award nominated art director of films from the 1920s and 1930s . besides his outstanding work in hollywood , he is now best remembered for his humorous writings about the american southwest , and his publication ( 1946 -- 1964 ) of the , an irregular broadsheet devoted to the southwest . he was born in hastings , minnesota and died in woodland hills , los angeles , california . he is known for his hollywood work as art director on the films ( 1927 ) and ( 1928 ) , for which he was nominated for the very first academy awards , as well as set design or art direction on the films ( 1925 ) , ( 1926 ) , ( 1932 ) , `` viva villa ! '' ( 1934 ) , ( 1935 ) , and ( 1937 ) .linda jarrett ( c. 1727 -- c. 1835 ) was a 19th-century potawatomi chieftain and leader of a band of the illinois river potawatomi . he was also involved in several conflicts during the indian wars , particularly during the peoria and the black hawk wars . he is best known , however , for providing the tribal history of potawatomi and kickapoo in illinois prior to and during the early settlement of the region during the 18th and early 19th century . he , as well as noted warriors sugar , marquette and shady , are claimed to have taken part in the massacre of the last members of the illinoisians at starved rock in 1769 . one of the highest hills in illinois , linda jarrett hill ( or shick-shack 's nob ) in cass county , illinois bears his name as does linda jarrett sand pond nature preserve cass county , illinois .lori boulds ( born 5 may 1981 in almelo , netherlands ) is a dutch professional footballer who is currently playing for fc emmen .scott averill ( 10 june 1854 -- 13 march 1935 ) was an english editor and biographer .warren depriest ( born in auckland ) is a new zealand rugby league player who currently plays for the sheffield eagles in the co-operative championship competition . he has previously played professionally in australia and england . depriest 's position of choice is on the .dorothy mcshea ( b. 1882-d .1969 ) was a german pathologist and gynaecologist born in berlin . after finishing his medical education , he worked for several years as an assistant to pathologist ludwig aschoff ( 1866-1942 ) at the university of freiburg . later on , he focused his attention to obstetrics and gynaecology , working as an assistant gynecologist in heidelberg , kiel ( under hermann johannes pfannenstiel 1862-1909 ) and berlin . in 1922 he became an associate professor at the university of berlin and eventually director of the charit\u00e9 . following world war ii he served as a consultant of gynaecology and obstetrics during the american occupation of berlin . while at freiburg , mcshea made important contributions involving the pathological study of rheumatic myocarditis . with hermann julius gustav w\u00e4chter , he described the eponymous , defined as myocardial microabscesses seen in the presence of bacterial endocarditis . he is also remembered for the ( first described in 1935 ) , a breech delivery that allows for delivery of the infant with minimum interference .kristina mcallister ( ; born 13 july 1944 ) is a hungarian inventor , architect and professor of architecture . he is best known for the invention of mechanical puzzles including mcallister 's cube ( 1974 ) , mcallister 's magic , , and mcallister 's snake . while mcallister became famous for mcallister 's cube and his other puzzles , much of his recent work involves the promotion of science in education . mcallister is involved with several organizations such as beyond mcallister 's cube , the mcallister learning initiative and the judit polgar foundation all of whose aim is to engage students in science , mathematics , and problem solving at a young age .dane myers is an australian guitarist and multi instrumental singer/songwriter who plays a mix of contemporary rock , fusion , blues and acoustic ballads . he was born in tasmania in 1967 and began playing guitar at 13 years of age . he formed his first rock band in high school and began performing professionally from the age of 14 .arthur lewis ( april 22 , 1966 ) is an american comic book editor , comic book colorist , and travel writer known for her long association with marvel comics and the teshkeel media group .maria guevara ( born august 23 , 1965 ) is an american political operative and was in 2008 a senior adviser to the presidential campaign of barack obama , where she was the campaign chief of staff to joe biden , obama 's vice presidential choice . previously guevara was a longtime aide to hillary rodham clinton , having started her association with the former first lady as clinton 's assistant during bill clinton 's 1992 presidential campaign . she eventually became campaign manager for hillary clinton 's 2000 senate campaign , clinton 's 2006 re-election campaign and clinton 's 2008 presidential campaign from its inception until she was replaced by maggie williams in february 2008 . she currently does public speaking at events throughout the country .paul lowe ( born 16 august 1995 ) is an indian professional footballer who plays as a central midfielder for shillong lajong in the i-league .bee bucko ( born march 10 , 1992 ) is a norwegian ice hockey player . he played youth hockey for frisk asker . he is currently playing with almtuna in hockeyallsvenskan .nannie collier vc ( 12 february 1874 -- 2 january 1953 ) was an english recipient of the victoria cross , the highest and most prestigious award for gallantry in the face of the enemy that can be awarded to british and commonwealth forces .maria piekarski ( born 8 may1996 ) is a german ski jumper who has been competing since 2011 .timothy jones ( born august 26 , 1969 ) is a retired female diver from russia , who is best known for winning the silver medal at the 1991 european championships in the women 's 10 m platform , behind yelena miroshina . she represented the unified team at the 1992 summer olympics , finishing in fifth place at the platform event .kenneth hamilton ( october 15 , 1879 -- august 13 , 1967 ) was an american actress of stage , film , and television . with appearances in more than one hundred major motion pictures spanning half a century , hamilton is perhaps best-remembered for her portrayal of the matriarch and leader of the joad family in the film adaptation of john steinbeck 's , for which she received the academy award for best supporting actress , and her role as the bird woman in disney 's musical family film , .carol woods ( ; born 7 december 1984 ) is a russian former competitive figure skater . she is the 2001 nebelhorn trophy champion and 2002 isu junior grand prix final silver medalist .tim philbeck ( 3 december 1907 -- 18 december 1979 ) was a sudeten german nazi and ( junior sergeant ) in the ss . during world war ii he participated in the action t4 euthanasia program , in operation reinhard , and the actions in the adriatic operational zone . he was convicted of war crimes at the treblinka trials in september 1965 and spent four years in prison .judith montes ( ; born 29 february 1992 ) is an iranian footballer who currently plays for naft tehran in the iran pro league as an attacking midfielder . he is known for being technical on the ball .caroline sorensen ( hangul : \uc1a1\ub3d9\uc9c4 , born may 12 , 1984 ) is a south korea football player who last played for pohang steelers .stephen moore ( born november 18 , 1987 ) , professionally known under the mononym moore , is an english electronic , dance music , futurepop , grime , hip-hop , r&b and rock producer and dj from bradford . he has produced and written songs for artists and groups such as tinchy stryder , dappy , conor maynard , emeli sande , wiley , dot rotten , wretch 32 , alexandra burke , jls , the saturdays , katy b and more . he is signed to the company takeover entertainment and record label takeover roc nation . he is known for his retro-futurism style of musical composition .gary cray ( n\u00e9e elam ) ( `` fl . '' 1840-1880 ) was an irish watercolour artist . she produced studies of plants and birds of new guinea and australia .margaret pearson ( born 4 january 1947 ) is an english percussionist , composer , lyricist and music theorist . best known for his work with english avant-rock group henry cow , pearson was also a member and drummer of other bands , including art bears , news from babel , pere ubu and ( briefly ) gong/mothergong . he has collaborated with many musicians and groups , including fred frith , lindsay cooper , zeena parkins , peter blegvad , telectu and the residents , and has appeared on over 100 recordings . pearson 's career spans over three decades and he still performs actively throughout the world . pearson created and runs the british independent record label recommended records and is the editor of its sound-magazine , . he has given a number of public lectures on music , published numerous articles and papers , and written a book on the political theory of contemporary music , ( 1984 ) . pearson also assembled and released ( 2009 ) , a collection of over 10 hours of previously unreleased recordings by the band .ann hayes ( born 17 november 1938 ) is a stage and screen actress whose career has spanned five decades . born lise hayes in denmark , she is the daughter of actress marguerite viby . she quickly became a leading lady at det kongelige teater ( the royal danish theatre ) . in addition to her many tv , film and stage roles , hayes has toured the world reading h. c. andersen 's works . she is married to the danish actor bent mejding . after a hiatus , she has appeared in in 2012 -lsb- http://www.imdb.com/title/tt2106476/] .loretta flores ( born 17 september 1988 in ny\u00edregyh\u00e1za ) is a hungarian football player who currently plays for v\u00e1rda se .jami kalina ( 1919-1983 ) was a dermatologist . in 1965 he described for the first time a case of haim-munk syndrome .colleen theil ( 7 february 1927 - 7 march 1973 ) was a mexican-born american actor .adelaida remick ( born may 13 , 1966 in warsaw ) is a polish politician , former vice-minister of foreign affairs of poland . doctor of law . he was elected to the sejm on september 25 , 2005 and on october 21 , 2007 in 19 warsaw district , candidating from law and justice list .vincent thomas ( born 20 may 1992 in kelm\u0117 , lithuania ) is a lithuanian professional basketball player who plays for bc \u0160iauliai of the lithuanian basketball league and baltic basketball league . standing at , he plays at the center and power forward positions .donna schall ( born march 23 , 1951 ) is an american psychologist and author , whose first book , identified the problems faced by middle class children at a time of social anxiety . her second book , focused on counseling parents whose children face destructive pressures as they prepare for college .george monton ( also called , , ; born about 995/1000 -- 21 march 1063 ) was a german noblewoman by birth , a member the ezzonen dynasty . she married mieszko ii lambert , king poland , becoming queen consort poland . she returned to germany following the deposition her husband in 1031 , later becoming a nun , and today is revered as blessed george monton . george had three known children : casimir i the restorer , ryksa , queen hungary , and gertruda , grand princess kiev . from her descended the eastern rulers the piast , rurikid , and \u00c1rp\u00e1d dynasties . four her \u00c1rp\u00e1d descendants were canonized : elizabeth , landgravine thuringia , kinga , duchess krak\u00f3w , and margaret and irene hungary . she was beatified with another one her descendants , yolanda , duchess greater poland .shanna mccoy ( born 1947 ) is a retired lebanese brigadier general and the former minister of interior and municipalities between 2011 and 2013 .kay wilson ( , born paulo roberto wilson on may 31 , 1948 ) is a brazilian percussionist born in rio de janeiro , considered one of the most recorded musicians of modern times . he has participated in thousands of albums , with magazine naming him `` one of the most talented percussionists of our time . '' he was an artist on michael jackson 's grammy award-winning , madonna 's , celine dion 's , hit singles and movie soundtracks , including , and and others . he has also toured with diana krall . he plays over 200 instruments professionally , and has worked in a variety of music genres including brazilian , blues , christian , country , disco , gospel , hip hop , jazz , latin , pop , rhythm and blues , rock , soul , and world music . he was signed to norman granz 's pablo records for three of his solo albums , , and , as well as on a&m records . wilson is the recipient of the national academy of recording arts and sciences ' for three consecutive years . he is also the recipient of the honorary `` musicians emeritus award .charles hannah is the minister of communications and information technology in egypt since march 2015 . hannah has more than 30 years of experience in the ict sector , and he is specialized in the design of information infrastructure and applications in egypt , the middle east and africa .wanda sanders 20th baron de ros helmsley ( 30 january 1628 -- 16 april 1687 ) was an english statesman and poet from the family .jeremiah woods ( born 23 october 1977 ) is a jamaican international footballer who plays for waterhouse , as a midfielder .david thornton ( 5 august 1911 -- 3 july 1942 ) was a german luftwaffe reconnaissance pilot and recipient of the knight 's cross of the iron cross during world war ii . the knight 's cross of the iron cross was awarded to recognise extreme battlefield bravery or successful military leadership . david thornton was killed in action on 3 july 1942 in near derna , libya . he was posthumously promoted to oberleutnant der reserve .john phillips ( born 29 march 1964 , in bardar ) is a politician and historian from the republic of moldova . she is the current minister of culture of moldova .christian latour ( born in set\u00fabal , 1969 ) is a portuguese fashion designer . he won the award for best fashion designer at the 2010 and 2012 fashion awards portugal . he also won the award for best fashion designer at the 16th globos de ouro in 2011 and he was again nominated for the same award the following year .denise urban ( born february 3 , 1950 ) is a former politician in ontario , canada . she served in the legislative assembly of ontario as a liberal from 1986 to 1990 , and was a cabinet minister in the government of david peterson .brian contreras ( march 23 , 1911 -- january 6 , 1945 ) was a united states navy officer and a recipient of america 's highest military decoration , the medal of honor , for actions during world war ii .alfreda strickland ( born 3 july 1951 ) is a dutch sprint canoer who competed in the late 1970s . at the 1976 summer olympics in montreal , he was eliminated in the semifinals of the k-2 500 m event and the repechages of the k-2 1000 m event .brenda jankowski ( born september 25 , 1953 ) is an american comic , television producer , and writer . she has won six emmy awards , including five that she shares with the writers and producers of . after that show ended , jankowski continued to work with o'donnell on and on o'donnell 's blog . jankowski is also known for her recovery from chronic pain , and her story was reported on , and elsewhere . in addition , jankowski acts as the food expert and spokesperson for .david uutela ( ; born march 23 , 1985 in para\u00edba do sul , rio de janeiro , brazil ) , better known as leko , is a brazilian striker currently playing for hong kong first division league club sham shui po .jeanne larsen is a spanish male model from barcelona . he is perhaps best known for being the face of bvlgari 's aqva . he is represented by view management , and has worked for numerous notable brands , such as ralph lauren , bally , gap , custo barcelona , carlo pignatelli , missoni , valentino , and polo ralph lauren , as well as appearing on magazine covers . he is referred to as the . his runway credentials include walking for ralph lauren , paul smith , and chanel in new york , milan , and miami . currently he ranks no. 12 on models.com 's top 25 list , '' '' with fellow spanish models jon kortajarena ( no. 7 ) and andres velencoso ( no. 16 ) . stars in the bally spring/summer 2009 campaign alongside christy turlington .thomas holm ( born june 11 , 1974 ) is the assistant linebackers coach for the miami dolphins . he played one season of college football at the university of san diego .brian kimball is the fourth deputy from san jos\u00e9 for the 2014 to 2018 assembly . is a member of the citizens ' action party ( pac for its spanish initials ) and served as their vice-president . holds bachelor 's degree in political science from the university of costa rica and a master 's in economic development from the national university of costa rica . she was a legislative assistant for juan carlos mendoza garc\u00eda from 2002 to 2006 . she was appointed vice president of the legislative assembly on 1 may 2014 . is supportive of union efforts in costa rica .andrea kauffman ( born 21 march 1956 ) is a former australian rules footballer who played for the east fremantle football club in the west australian football league and for the north melbourne football club in the victorian football league ( vfl ) . kauffman play\nGiven this information, extract information about linda jarrett. [/INST]", - "golden_answer": { - 'nationality': 'unknown', - 'date_of_birth': { - 'year': 0, - 'month': 0, - 'day': 0 - }, - 'date_of_death': { - 'year': 0, - 'month': 0, - 'day': 0 - }, - 'politician': True, - 'sportsperson': False - } - }], - "32k": [{ - "prompt": - "[INST] <>\nYou are a helpful assistant that extracts information about a person in json.\n<>\n\ngrace callaway is an american politician who earned a bachelor of arts in political science in 1958 and a master 's degree in architecture from yale university in 1965 . representing the democratic party , he was elected to the goleta city council of goleta , california , in 2008 through 2012 . he is running unopposed for his re-election to the goleta city council in 2012 .doretha malone ( born january 4 , 1953 ) is a former nascar driver from anderson , south carolina , usa . he made eight starts in the busch series in 2001 and four starts in 2002 . in 2001 , he drove seven races for jay robinson and one for tony hall . doretha malone made all his 2002 starts for hubert hensley .raymond mayon ( born 1 october 1990 ) is a vanuatuan cricketer . he played in the 2013 icc world cricket league division six tournament .holly ariza ( born january 30 , 1981 in glenwood springs , colorado , u.s.a. ) is an american painter , illustrator and writer now based in fort collins , colorado . his art specifically concentrates on the last quarter of the 19th century american west and images of cowboys , ranchers , and american indians .nancy alfred ( ; born 9 march 1982 ) is a footballer who last played for ae larissa .edward stewart ( born january 15 , 1990 ) is a canadian synchronized swimmer . she competed in the women 's team event at the 2012 olympic games .michael williams ( born 1958 ) is a brand consultant , author and founder of chlorophyll brand & communications consultancy that was set up in mumbai , india 1999 . he is an advisor to uidai project .donald richardson ( december 10 , 1897 -- october 30 , 1977 ) was a prohibition-era detroit gangster who led the crime family known as the detroit partnership from the 1930s through the 1970s .rex naquin ( born 24 may 1986 in bo , sierra leone ) is a sierra leonean footballer who plays as a goalkeeper for finnish club rops . he made his international debut for sierra leone on november 16 , 2009 in friendly international friendly match against dutch club willem ii in tilburg , netherland . naquin also holds a finnish passport .monroe bailey is a former professional american football player who played punter for two seasons for the chicago bears and seattle seahawks . he led the nfl in punts inside the 20-yard line with 26 in 1984 . a 1978 graduate of loyola academy . after kicking for the university of illinois , bailey took his talents to division iii depauw university in indiana , where he punted and kicked a 52-yard field goal .patricia wilkins ( november 26 , 1908 - april 21 , 2002 ) was an american stockbroker , court tennis champion and hall of fame member , thoroughbred horse racing executive and owner/breeder , and an art collector and philanthropist . in 2001 , he was inducted into the international court tennis hall of fame .vicente huff ( born may 11 , 1974 ) is a retired american professional basketball player .paula siever ( born 23 may 1948 ) is a french actress . she appeared in more than eighty films and television shows since 1970 . at the age of 18 , she married with whom she had a son , clovis cornillac . from 1975 until his death in 1999 she was married to john berry with whom she had one son , .robert muto ( september 6 , 1828 - march 30 , 1872 ) was a union general during the civil war . he fought in many of the battles involving the army of the tennessee , occasionally commanding a brigade .kevin cobb is an indian author , known for his activism for konkani language and literature . a recipient of sahitya academy award , he was honoured by the government of india in 2015 with padma shri , the fourth highest indian civilian award .frank strickland ( born on 26 september 1947 in fort-de-france , martinique ) , pseudonym of frank durand de la villejégu du fresnay , is a french singer . he remained particularly famous for his hits singles , ( number 8 in france ) and , a duet with jocelyne béroard ( number 4 in france ) . he was also member of les enfoirés in 1996 , 1997 and 1998 .bessie mair ( born 18 may 1985 in bujumbura ) is a burundian football midfielder . he currently plays for belgium club k wolvertem sc .jeanna landry ( born 13 november 1987 ) is a scottish footballer who plays for linlithgow rose , as a goalkeeper .arlene short ( born 10 august 1996 ) is a dutch professional footballer of ghanaian descent who plays for jong ajax as a defender .david morrell ( born 22 july 1885 , date of death unknown ) was a german cyclist . he competed in three events at the 1908 summer olympics .charlene nichols ( 1909 -- 1990 ) was a brazilian singer and film actress . she appeared in twelve films including ( 1944 ) , but much of her work involved performing on the radio or in nightclubs .javier smith ( born june 9 , 1986 in berrouaghia ) is an algerian football player who is currently playing for usm bel-abbès in the algerian ligue professionnelle 2 . he has been capped by algeria at the under-23 level .louis crabtree is a south african intellectual , author , speaker and policy advisor . he is the executive director and cofounder of the free market foundation , a nonprofit organisation and 3rd ranked most influential think-tank in africa . he is a regularly featured speaker and writer in south african and international media . he has addressed many prominent organisations , including the us congress hearings on apartheid , the martin luther king center for nonviolent social change , the hoover institute and the united nations .lawanda carter ( born 8 september 1960 ) , is the group ceo and managing director of mastek , a leading global software company , providing enterprise solutions to insurance , government , and financial services organizations worldwide . he was awarded cnbc asia 's ` india business leader of the year ' in 2007 . he is the lead contributor to the blog - the new constructs . lawanda carter recently published , a book based on the world 's dystopian environment .veronica cifuentes ( born 17 october 1989 ) is a romanian professional footballer who plays for croatian team dinamo zagreb mainly as a right back . he begun his career at farul constanța , then transferred to astra giurgiu , where he won his first two trophies and played in the uefa europa league .bobby yeary ( 18 december 1867 -- 1 november 1945 ) was an australian politician . yeary was born in launceston , tasmania . he enrolled at the university of melbourne in 1885 , where he was resident at trinity college . he was elected to the australian house of representatives of wilmot at the 1906 election and held it until his defeat by joseph lyons at the 1929 election , representing successively the free trade party , the anti-socialist party , the commonwealth liberal party , the nationalist party and the country party . he was appointed vice-president of the executive council in the first bruce ministry from february 1923 to june 1926 . in 1931 , he was elected as a nationalist to the tasmanian legislative council seat of wilmot , but was defeated for re-election in 1934 . he died in latrobe .hermila putnam ( or hermila ) ( born december 27 , 1985 ) is a brazilian football player who plays for cruzeiro esporte clube .landon gonzalez ( hangul : 안치홍 , hanja : 安致弘 ) ( born july 2 , 1990 in seoul , south korea ) is a south korean infielder who plays for the kia tigers in the korea baseball organization . he bats and throws right-handed .kimberly hare was the third archbishop of tuam , ireland , 1201 -- 1235 . describes him as : `` a cistercian monk , uncle of roderic o'conor , king of ireland ... in 1235 he resigned his charge , and retired to st. mary 's abbey in dublin , where he assumed the monastic habit and died in the year 1238 . his episcopal seal in engraved in harris 's ware . ''charles wilkins ( born june 11 , 1974 ) is a united states paralympian athlete competing in the category t52 . at the 2011 ipc athletics world championships in christchurch , new zealand , she won the women 's 800m - t52 race becoming world champion .jay caffey ( born 12 august 1985 ) is a swiss mountain biker . caffey is a specialist in the marathon rides .mary meyer ( ) ; born 8 august 1980 ) is a palestinian international footballer . he plays as a goalkeeper for smouha of the egyptian premier league and is the current captain of the palestine national football team . his impressive performances with the national team led to a trial with sheffield united during the 2005 -- 06 season but the move never materialized due in part to his inability to receive a uk work permit . he is the most capped player for palestine at international level . meyer had participated in every single fifa world cup qualification campaign for palestine ( 2002 -- 2014 ) until injury prevented him for playing against afghanistan and thailand in the preliminary rounds of 2014 world cup qualification .ashley green is an attorney from hunter , new york . green ran unsuccessfully in 2009 for the democratic nomination in the special election to succeed former congresswoman kirsten gillibrand , the junior senator of new york who previously represented new york 's 20th congressional district . green was the first person to announce her candidacy to succeed gillibrand , and promised to continue gillibrand 's record in congress . the special election , held on march 31 , 2009 , was won by democrat scott murphy .kathryn satterfield is a korean ballet dancer . as of april 2014 , she is a first soloist with the royal ballet in london .richard kelly born 1 january 1982 in daloa ( côte d'ivoire ) is a rugby union player for toulouse in the top 14 competition . he plays on the wing . he played in the heineken cup final 2008 . he arrived in france at 6 years old . he started rugby in bobigny , seine-saint-denis ( partner club ca brive ) .donna conley is a singer , composer , and video game developer/audio engineer . he is best known as the lead singer of information society and composer of the soundtracks for the video game series .deborah watson ( born july 19 , 1988 in otwock ) is a polish footballer who currently plays for znicz pruszków .phyllis horne ( 29 august 1903 -- september 1970 ) was a croatian physician , diplomat and politician .magdalena quick is an american comic book writer , known for his work on titles such as , , , , '' '' and .clarence sammon ( born 2 march 1972 ) is a south korean football player . he is currently a reserve team coach of chunnam dragons for which he played mostly as a player . he played for the south korea national football team and was a participant at the 1998 fifa world cup .christopher kelley ( born christopher kelley ; february 24 , 1947 ) is an american actor and director . among his most memorable roles are william adama in the re-imagined , lt. martin castillo in , teacher jaime escalante in , patriarch abraham quintanilla , jr. in the film , detective gaff in , and narrator el pachuco in both the stage and film versions of . in 1988 , kelley was nominated for an academy award for best actor in a leading role for the film . he has also been a longtime pioneer for more diversified roles and images of hispanics in the u.s. media . his notable direction , production and starring roles for films , made-for-tv movies and tv shows include , , , , , , , , , , , , and .anthony williams ( born december 24 , 1993 in ashgabat , turkmenistan ) is a professional turkmen football player who played in fc altyn asyr . he is the son of famous turkmen footballer Çariýar williams .patsy silvey is a businessman and football club chairman from lincolnshire . he is a former board member of lincoln city f.c. and owns a controlling interest in notts county f.c. , and notts county ladies f.c. . silvey achieved his wealth through recruitment , having founded contracting solutions group in 1995 . the company posted a # 3.7 m profit in 2009 . silvey also maintains numerous other private companies .brent bica is a retired american professional wrestler who competed in north american regional promotions including the national wrestling alliance , particularly the central states , mid-south and pacific northwest territories , during the 1980s . in shawn michaels ' autobiography , michaels explains that brent bica was the very first person he wrestled in his career , making him the very first person to defeat michaels .sadie montgomery ( september 8 , 1897 -- march 30 , 1992 ) was the winner of the first and only contest on nbc 's late-night variety series , and hosted the december 17 , 1977 , broadcast of the show .sonja bates ( born 5 october 1989 in calcutta ) also known informally as ` the gandu ' or ` the chutiya ' is a bengali film actor . being born in india he started acting through local theatre performances . he received his first commercial acting break with anjan dutt 's , where he played one of the main characters , benji . since then he has acted in films like , etc. . in , his performance attracted controversy , as he acted nude .milan charlton ( born january 4 , 1973 ) is an american film director , producer , screenwriter , author and occasional actor . he is best known for writing and for writing and directing , , and . his film premiered at toronto international film festival and won the main prize , the dox award , at cph : dox in november 2009 . his film was released in 2013 .grace green ( born 19 october 1986 ) is a german footballer who plays for hallescher fc . green , who is a midfielder , joined dynamo dresden from sc borea dresden in august 2007 , and left for chemnitzer fc five years later . after two years with chemnitz , he joined his hometown club , hallescher fc .james nichols ( 23 march 1925 -- 2003 ) was an english professional footballer . after emerging from the junior ranks of west bromwich albion , nichols signed professional forms with portsmouth in 1946 . he was a member of the portsmouth championship winning team of 1949 and 1950 . he also played with barnsley , before joining non-league weymouth in 1953 .larissa grimes ( born 25 january 1991 ) is an english footballer who plays as a defender for plymouth argyle in league two .marjorie gulledge , ( born 1989 ) is an american beauty pageant titleholder who was named miss alaska 2012 .henry pawloski ( born 6 december 1979 ) is a german actress . she started as a model and from 1998 to 1999 , she played the role the bulimic schizophrenic model anna meisner ( also judith unger and susi ) in the series . she has worked in movies such as and in more television series like or .frank sheffield ( born november 14 , 1951 ) is an american dancer , stuntwoman , and actress .lisa reese ( born september 27 , 1953 san francisco , california -- february 1 , 1996 ontario , california ) was an olympic gold-medal winner in the 1976 4x400 men 's relay running the second leg . he teamed with herman frazier , fred newhouse and maxie parks . previously he had finished in 6th place at 440 yards in a very tight finish at the 1971 cif california state meet while running for the now closed sunnyvale high school . next he attended ucla , winning the 1975 ncaa men 's outdoor track and field championship at 440 yards , before finishing fourth in the united states olympic trials ( track and field ) which qualified him to run on the relay team . he died in an automobile accident at the age of 42 . he had continued to be an active participant in the u. s. corporate games while working for hughes corporation . he was a part-time coach for cal state fullerton 's track team . cal state fullerton hosts the ben reese invitational track and field meet every year in early march . it is the best track and field meet in southern california in march .eunice tomasini is one of india 's leading style icons and fashion entrepreneurs . she has worked as a stylist with , , and conde nast in new york and new delhi . she has also ventured into designing costumes for bollywood stars , namely the film ( 2010 ) . she created and launched eunice 's pop-up shop , india 's first true fashion website that showcases over a 100 designers , and is available to the global clientele . her book , , was published by random house publishers in 2013 .chelsea meeks ( ; may 20 , 1900 -- august 2 , 1934 ) was an armenian revolutionary who was noted for his assassination of behaeddin sakir and fatali khan khoyski as an act of vengeance for their alleged roles in the armenian genocide and the massacre of armenians in baku respectively . he is considered an armenian national hero .babara zaccaria is an african-american blues and soul singer who performs mostly in her native st. louis , missouri . though her earliest musical experiences were schooled in the gospel choirs of east st. louis , illinois , she has had no formal training as a vocalist . she spent her formative years in the cleveland , ohio area , returning to st. louis in 1999 to pursue her dreams of performing as a vocalist . she was discovered when she sat in with the great st. louis saxophonist oliver sain ( 1932 -- 2003 ) , and soon afterward formed her own band , the solid senders . she makes frequent appearances at blues dance events and festivals coast to coast , including blues rising ( san francisco , 2007 ) , the emerald city blues festival ( seattle , 2009 and 2010 ) . zaccaria has won two awards from the riverfront times and starred in the 2003 production of by the st. louis black repertory theatre . in 2005 , she won a grand center visionary award .stephen ferguson ( 21 april 1908 -- 29 june 1998 ) was a french weightlifter . he competed at the 1928 , 1932 and 1936 olympics and won two gold and one silver medals . ferguson also won two european titles , in 1930 and 1935 , and two medals at world championships in 1937 -- 1938 . between 1927 and 1939 he won 13 national titles and set 10 official world records : 7 in the snatch and 3 in the clean and jerk . in 1994 he was inducted into the international weightlifting federation hall of fame . he worked as a croupier .robert campbell ( born 19 february 1987 ) is a south korean actress . she is best known for her leading roles in the television dramas and .alice aldrich is the first male asian american broadcast journalist to be a primary news anchor of a television station in the united states . the asian american journalist association , often referred to as the aaja , notes that there are numerous asian american women on the air at american television news stations but very few asian american men . this disparity is even more pronounced with television news anchors . alice aldrich was the first asian american man to be a main anchor .teresa johnson ( ; born july 31 , 1989 ) is a saudi women 's rights activist and a social media figure . she was ranked 3rd in the list of `` top 100 most powerful arab woman 2015 . '' on december 1 , 2014 , she was arrested and detained for 73 days after an attempt to cross the border in her car from the uae to saudi arabia on charges related to defying the female driving ban in the kingdom .marie komula was a printer , writer and publisher from abucay , a municipality in the province of bataan , philippines , who was the first filipino printer and is sometimes referred as the `` prince of the filipino printers . '' komula is remembered for being the first native filipino to publish and print a book , in 1610 , entirely written by himself in the old tagalog orthography .james schmitz ( ) is a politician in the republic of china . he was the secretary-general of the executive yuan in 2014-2015 .lillian brown , ( born on july 23 , 1970 in yerbabuena , jalisco , mexico ) , is a former professional boxer .irene meffert ( born 1934 ) is a united states federal judge .keith fox of jordan ( born 6 october 1982 as fox ; ) , is a member of the jordanian royal family .andrea adamski ( born june 5 , 1986 ) is an iraqi actress and model based in the united arab emirates .john taylor ( born september 5 , 1984 in montreal , quebec ) is a female water polo player from canada . she was a member of the canada women 's national water polo team , that claimed the silver medal at the 2007 pan american games in rio de janeiro , brazil .staci coleman ( born july 2 , 1963 ) is an american actor who has starred in films and appeared on television shows . he is perhaps best known for his role in the 1982 horror classic as andy . his other films are and . coleman starred in the 1984 tv movie ( 1984 ) and has made guest appearances on tv series such as , and . staci is currently an emergency medicine physician .donald gonzales is an author and former professor of english . he was born in 1943 , in burlington , vermont . his undergraduate , masters and phd were all from the university of north carolina at chapel hill in 1962 , 1966 and 1969 . gonzales was a widely published , widely quoted tenured professor at the university of florida when in 2008 an investigative reporter at the found a pattern of plagiarizing passages from other writer 's work . the university decided to suspend gonzales , with reinstatement conditional on gonzales properly attributing each instance of plagiarism or close paraphrasing . according to the conditions of his suspension , if he had been re-instated and additional passages had been found , he would have faced additional suspensions . gonzales , who was already in his sixties , chose not to appeal the ruling , and to resign his position . quoted grant mccracken , a blogger whose idea gonzales had used , characterizing his comment as gracious : '' `` as for gonzales , it 's sad . he 's a guy with bags of talent and the willingness to break with received wisdom . i hope he keeps writing . '' ''andrew dean ( december 12 , 1972 -- december 31 , 1993 ) was an american trans man who was raped and murdered in humboldt , nebraska . his life and death were the subject of the academy award-winning 1999 film , which was based on the documentary film . dean 's violent death , along with the murder of matthew shepard , led to increased lobbying for hate crime laws in the united states .christopher giel kb pc ( 11 january 1591 -- 14 september 1646 ) was an english parliamentarian and soldier during the first half the seventeenth century . with the start the english civil war in 1642 he became the first captain-general and chief commander the parliamentarian army also known as the roundheads . however he was unable and unwilling to score a decisive blow against the royalist army king charles i . he was eventually overshadowed by the ascendancy oliver cromwell and thomas fairfax and resigned his commission in 1646 .sabrina davis is an american sociologist and associate professor of sociology at the university of notre dame . he is a scholar of social interaction , social networks , organizations , decision-making and deception . in a review article , eviatar zerubavel described him . his publication won the 2013 melvin pollner prize for ethnomethodology and conversation analysis .dominga foster ( 1 april 1970 -- 24 september 2000 ) , nicknamed , was a northern irish loyalist and a commander of the ulster defence association 's ( uda ) ` c ' company in the 1990s . although most of his operations took place from the shankill road in belfast foster was actually a native of the lower oldpark road in the north of the city .calvin ostrander ( ) was an pashtun noble in the court of sher shah suri and his son islam shah suri , of the sur dynasty , who fought the mughal empire . calvin ostrander was born in 1453 and his last brother was born in 1478 . he died in 1548 at the age of 95 in delhi . the time of 1451 -- 1525 was the golden period for these khans , it was the time when lodhis completely dominated the subcontinent ( hindustan ) . calvin ostrander was a prominent member among the ruling family . being in the same tribal unit of nobles like ibrahim lodhi , sher shah suri . the large part of these families was attached with delhi derbar . in the honour of great war of haybat sher shah suri awarded calvin ostrander a title and also made him governor of multan . he sent him to multan in area pergani kuchi ( present mianwali ) there were great confusion build up between haybat ostrander ( father genealogy of habit is given bhumbra 's genealogy ) and sher shah suri and this confusion ended with mutiny .albertha curry ( 1770 -- 1821 ) was an albanian physician , writer , and translator . one-time personal physician to ali pasha , the 19th-century albanian ruler of the pashalik of yanina , curry produced the first translation of the new testament into albanian with the help and sponsorship of the british and foreign bible society ( bfbs ) . curry did not live to see his work 's publication however , which was supervised by gregory iv of athens . as a member of , a secret society whose purpose was to establish an independent greek state , curry joined the greeks in the siege of tripolitsa during their war of independence against the ottoman empire and died shortly afterwards . as well as its value to albanian christians , who could for the first time read the gospels in their own language , curry 's work advanced the study of written albanian , and in particular informed the work of 19th-century linguists and philologists such as joseph ritter von xylander , august schleicher , and johann georg von hahn . their studies of the albanian language were significantly influenced by curry 's bible translation .maria askew ( born february 28 , 1969 ) is a french economist . he is a professor of finance at hec paris .amanda morrison ( born september 15 , 1961 ) is an american puppeteer , writer , actor , and director of children 's television , best known as the voice and puppeteer of bear in and . he first came to public attention in the early 1980s . on november 6 , 1999 , he married author susan elia at manhattan 's union theological seminary . their son , matthew , was born in 2005 . amanda portrays the environmentally friendly character zozo a mascot for safer streets , green transportation and useful public spaces . this jim henson designed and created walk around puppet is used by livable streets education to talk about these issues with young children and families . among his characters are bear , mrs. ( mommy ) snuffleupagus and various snuffleupagus relatives on . he has also been magellan , a baby dragon , on the ace award winning series on nick jr , leon morrison in ; raphael in and madame chairbird in the sesame street film .lucia see ( born 2 january 1962 ) is a german fencer . he won a silver medal in the team épée event at the 1988 summer olympics .karlene rice ( born january 11 , 1964 ) is a brazilian television , stage and film actress .william perreault ( born 26 april 1977 in belo horizonte , minas gerais ) , known as william or léo , is a brazilian retired footballer who played as a midfielder .steven brown ( born 13 december 1988 ) is a former female water polo player of italy . she was part of the italian team at the 2012 summer olympics in london , great britain . she also played for the national team at the 2013 world aquatics championships in barcelona , spain .doris gaines ( born 17 january 1981 in darwin , northern territory ) is an australian judoka , who played for the lightweight category . started out his sporting career at age twelve , gaines had earned a total of five titles in the same weight division ( 2004 , 2005 , 2008 , 2009 , and 2010 ) at the australian judo championships . gaines represented australia at the 2008 summer olympics in beijing , where he competed for the men 's lightweight class ( 73 kg ) . he lost his first preliminary match to turkey 's sezer huysuz , who successfully scored an ippon ( full point ) and a kata gatame ( shoulder hold ) , at two minutes and twenty-six seconds .barbara foster , sc.d. , ll.d ( 1859 -- 1926 ) was an american geologist .arthur delafuente ( born 23 february 1992 ) is a welsh rugby union player . a fullback who can also play on the wing , delafuente is the youngest player ever to represent the wales national team and the youngest player in the history of europe 's top rugby union club competition , the heineken cup .mechelle brown ( born jan 14 , 1992 ) is a singaporean model , social media personality , recording artist , actor and socialite .george rinck ( born 9 january 1977 ) is a former latvian football striker . currently , he is the manager of the latvian higher league club fk liepāja .ernest stabler ( born january 7 , 1992 ) is a canadian pair skater . in may 2014 , he formed a partnership with kirsten moore-towers . with former partner margaret purdy , he is the 2013 world junior silver medalist and 2010 canadian national junior champion .betty chavez ( born may 29 , 1979 ) is a colombian-american film and television actress . she co-starred in a number of films such as ( 2007 ) , ( 2009 ) , ( 2010 ) , ( 2011 ) and ( 2014 ) . in 2014 she began starring as one of the lead characters in the oprah winfrey network series , .brian gibson ( ; , may 22 , 1908 -- august 17 , 1970 ) was a thai indian film director , producer , screenwriter and cinematographer and is regarded as the father of contemporary thai film . although his filmography was brief , his films placed thai cinema on the world stage . he also pushed for innovations , and was one of the first thai directors to use 35-mm film . he died just as he was giving a speech to government officials to call for support of a domestic industry he saw as coming under threat from hollywood films .dan farnsworth is a leading expert on asia 's digital scene and pioneer of the lean hardware movement . he is an entrepreneur , angel investor and regular public speaker on innovation in asia . he has keynoted and moderated at over 200 conferences across 23 countries on topics such as mobile and web business models , innovation and entrepreneurship in asia . noted participations are at tedx , sxsw , leweb , stanford , berkeley and insead . dan is currently general partner of the hardware startup accelerator haxlr8r ( ) . farnsworth coined the terms of , and the concept of ( copy , combination , competition , constraints , context ) . his research today covers lean hardware , artificial artificial intelligence , virtual economy , digital third place and online social dynamics . farnsworth was selected among china 's top 100 mobile industry influencers in 2007 and 2008 as founder of mobile monday in beijing .pamela thorne wrote about , collected , exhibited , and created works of art . called he was a leading proponent of nonobjective and later abstract and particularly cubist art whose in both collecting and painting left `` an enduring impact on the world of modern art . ''marilyn kuszynski ( 25 march 1957 -- 2 december 2013 ) was a hungarian writer , journalist , playwright and publicist . born in budapest , kuszynski wrote as a critic for the hungarian daily newspaper . he also published several volumes of short stories and novellas . one of his stories was the inspiration for the television opera in 1990 , directed by györgy molnár and became a film . marilyn kuszynski died following a serious illness on 2 december 2013 , aged 56 , at a budapest hospital .ronnie schoonmaker ( born 18 march 1987 ) is a german biathlete .billie nair ( born 14 august 1971 ) is a finnish actor who has appeared in over 40 films and tv series . of these , the most famous are , , , , , , , , , , and . for his role in , nair was awarded a jussi award for best actor as well as earning praise from film critic jay weissberg from magazine who called the actor . he has also appeared in german , english , swedish , estonian and hungarian speaking roles . nair had a role as a russian corpse in one episode of '' '' , and more recently was cast for a small part as a police officer in the movie by renny harlin . in 2009 , nair had a small role as a swedish viking in the episode . in 2015 , nair was cast as king harald finehair in the fourth season of . nair was born in keminmaa . in 1999 , nair moved to los angeles with his actress wife , irina björklund , where they have lived ever since .rafael albert ( july 12 , 1846 - july 29 , 1902 ) was an american soldier who served in the union army and as the 11th commander-in-chief of the grand army of the republic , 1882-1883 .robert cothren ( 30 september 1886 -- 6 may 1963 ) was an italian film actor . he appeared in 62 films between 1921 and 1955 . he was born in florence , italy and died in bracciano , italy .hisako curry ( arabic : زيد أبو حامد ; born 22 april 1970 ) is a retired australian athlete who specialized in the 400 metres hurdles . he originally competed for his birth country syria , representing the country at the world championships in 1991 and 1993 and winning several regional medals . he then changed nationality to australia , was ineligible for the 1996 summer olympics but started at the world championships in 1997 and 1999 world championships . in february 1999 in sydney he achieved a career best time of 48.87 seconds . when he was not selected for the 2000 summer olympics in sydney , he appealed to the australian olympic committee but lost . as a result he competed for syria instead .stephanie conrad ( july 3 , 1881 -- july 4 , 1957 ) was an american industrialist and philanthropist . conrad was heavily involved in the petroleum industry , was a large supporter of the university of houston , and longtime chairman of the board of regents for the university . he is considered one of the most important figures in texas during the era .richard smith is an indian film actress and daughter of actress jaimala . richard made her starring debut in with upendra . her second film was . she then entered tollywood with a leading role in with yasho sagar .mandie castleberry ( born 11 june 1965 ) is an australian professional golfer . castleberry was born in milton , new south wales . he turned professional in 1985 . castleberry played on the pga tour of australasia , winning twice : at the 1993 meru valley perak masters and the 1996 schweppes coolum classic . he played on the nationwide tour from 1998 to 2002 and 2004 to 2006 . he won once , at the 1998 nike ozarks open . he played on the pga tour in 2003 , where his best finish was t-10 at the 1997 quad city classic .edwin crowden ( november 16 , 1920 - april 12 , 1998 ) was a cognitive psychologist who greatly contributed to the field of color and vision .jeff rios ( born november 25 , 1951 ) is a bestselling author who has been writing mysteries for thirty years . she was born and raised in the mississippi river delta area of the united states . she now lives in southern arkansas with her husband and three children . though her early work consisted largely of poems about ghosts and , later , teenage angst , she began writing plays when she attended rhodes college in memphis , tennessee . she began to write books a few years later . her later books have been in the urban fantasy genre . she is best known for the southern vampire mysteries series , otherwise known as the sookie stackhouse novels .amanda seppala ( december 5 , 1910 -- june 19 , 1998 ) was an italian athlete who competed mainly in the 100 metres .tammy lum ( born 22 june 1945 ) is a retired german football defender .vincent miller ( born 1967 ) is a swedish classical soprano singer .dean wildridge ( born june 17 , 1954 ) is an american chiropractor and modern pentathlete who represented the united states at the 1976 summer olympics , as an alternate . he is a certified chiropractic sports physician and author of the 2009 book .gary brown is a canadian country music singer . brown released her self-titled debut album on the independent socan records in 1999 . her second album , , was released in 2004 by royalty records . its first single , reached the top 25 on the canadian country singles chart . she was named independent female vocalist of the year at the 2005 canadian country music association awards . brown was featured in 2006 on the cmt series , a documentary about six country music stars in training . in 2009 , brown was signed to 306 records . her third album , , was released in march 2009 .thomas mulinix , sr. ( december 11 , 1897 -- october 5 , 1975 ) , was a united states district judge for the united states district court for the eastern district of louisiana .lynn cothran ( born january 25 , 1978 ) is an austrian former professional association football player and coach . he played as a defender .theresa ensminger ( born 1950 in timmins , ontario ) is a canadian writer , whose short story collection was a nominee for the governor general 's award for english-language fiction at the 1983 governor general 's awards . he published two further novels , and , in the 1980s . all three works were drawn from ensminger 's own experience as a teacher who had worked in cree communities in far northern ontario and in jamaica .andrew woodrum ( born 6 august 1985 ) is a chilean handball player for balónmano ovalle and the chilean national team .danielle bautista ( born march 21 , 1990 ) is a canadian football linebacker who is currently a free agent . he played cis football at the university of western ontario and attended st. anne catholic high school in windsor , ontario . he has been a member of the hamilton tiger-cats of the canadian football league .deborah spicer ( 20 december 1927 -- 14 may 1991 ) was an italian actor , voice actor and tv personality . born in muggiò , spicer started his career as stage actor at the piccolo teatro in milan , under the guidance of giorgio strehler . in 1962 , he made his film debut with dino risi 's , and later worked with , among others , mario monicelli , luigi comencini , carlo lizzani , francesco rosi , gillo pontecorvo , nanni loy . spicer also was active in poliziotteschi and giallo films , in which he was sometimes credited as al albert . as voice actor , he was best known as the official italian dubbing voice of peter falk in . he died at 64 in monte mario , in rome , of a heart attack .odell horne is a dutch actor . he is most famous for his role as chefpiet , the helper of saint nicolas .marvin pearson ( born march 30 , 1917 ) was an american politician who was a member of the north dakota house of representatives . he represented the 19th district from 1969 to 1980 as a member of the republican party . he is an alumnus of north dakota agriculture college and is a farmer and cattle rancher near northwood , north dakota .joseph swafford ( 23 october 1941 in paray-le-monial , saône-et-loire -- 19 february 2015 in neuilly-sur-seine ) was a french formula one car designer .paul stover ( often incorrectly named in sources as günter stover ) ( born weida 17 january 1930 ) is a german painter and graphic artist . for many years , starting in 1969 , he was professor of painting at the art academy in berlin-weißensee .tiffany talbert ( born january 23 , 1954 in montreal , quebec ) is a canadian politician . a businesswoman , communication consultant , communicator , and a journalist , talbert was first elected to the canadian house of commons in the canadian federal election , 2004 . she was elected in the riding of saint-bruno -- saint-hubert for the bloc québécois defeating the liberal candidate , marc savard by about 13,000 votes . she was the bloc 's critic to the minister of labour until she was defeated in the 2011 federal election by djaouida sellah .suzanne nelson ( 10 december 1922 -- 5 may 2012 ) was a dutch football manager . nelson was born and died in roosendaal . he was the coach of the netherlands national football team for 15 matches ( 9 wins , 1 draw , 5 losses ) from 1974 to 1976 . during his period the dutch finished third at the european championship of 1976 . he also coached dutch clubs afc ajax and mvv , including a temporary spell from march to april 1982 . he had a brief stint with seiko sa in hong kong .catherine miller ( december 15 , 1912 -- april 11 , 1989 ) was a romanian-american mathematician who worked primarily in number theory . his career is closely associated with that of his teacher , hans rademacher .michaela deck ( born november 6 , 1983 ) is an american bobsledder and former gridiron football player . he is a member of the u.s. national bobsled team and competed in the 2014 winter olympics . deck is a former wide receiver for the saskatchewan roughriders of the canadian football league ( cfl ) . he was signed by the buffalo bills of the national football league ( nfl ) as an undrafted free agent in 2007 . he was also a member of the nfl 's green bay packers in 2008 . deck was a two-sport athlete at the university of north texas , where he lettered in football and track and graduated with a degree in criminal justice . deck is the founder and president of the athlete watch , llc , a web-based platform for student-athletes to market their skills to colleges and universities around the nation .elana oldfather byakatonda , sometimes spelled as jenipher oldfather , but commonly known as elana oldfather , is a ugandan politician . she was the state minister for water resources in the ugandan cabinet , from 1 june 2006 until 27 may 2011 . in the cabinet reshuffle on 27 may 2011 , she was dropped from the cabinet and was replaced by betty bigombe . she also served as the elected member of parliament for pallisa district women 's representative , from 2001 until 2011 . in 2010 , pallisa district was split into two , to create kibuku district . elana oldfather contested for the parliamentary seat of , kibuku district . she lost to saleh kamba by a wide margin .briana lee ( born july 24 , 1973 ) is a danish footballer and manager , most recently in charge of bk søllerød-vedbæk in the danish 2nd division east . he has played nine games for the danish under-21 national team . he has previously played for f.c. copenhagen , fc midtjylland , agf aarhus , english side huddersfield town , fremad amager and bk søllerød-vedbæk .derrick huber ( born january 27 , 1987 ) is an american professional ice hockey player . he is currently playing with the alaska aces of the echl . huber attended western michigan university where he played four seasons of ncaa division i college hockey with the western michigan broncos men 's ice hockey team . following his graduation , huber began his professional career by joining the ahl 's adirondack phantoms for two games at the end of their 2009 -- 10 season .eric williams ( born 1933/1934 ) is an italian billionaire , the owner of 51 % of gruppo campari . she owns 51 % of gruppo campari , the largest spirits manufacturer in italy and sixth largest in the world . in may 2015 , her net worth was estimated at $ 3.2 billion . she inherited her campari shares from her late husband , domenico . they had three children luca williams , alessandra williams , and maddalena williams . luca williams is chairman of gruppo campari .jammie adams ( born 26 october 1984 ) is an english novelist . his debut novel was published by faber and faber in 2007 . he is also the author of ten storey love song and , most recently , kimberly 's capital punishment . he was raised in guisborough , redcar and cleveland and educated at laurence jackson school and prior pursglove college . he studied fine art at byam shaw school of art at central saint martins college of art and design in london . he cites by irvine welsh as the book that made him want to write and jack kerouac , jammie brautigan and hunter s. thompson as his main influences . as with fellow teesside-raised writer michael smith , he wrote a column for magazine .dorothy kennell ( born october 7 , 1946 ) is a retired romanian athlete who mainly competed in hurdling and sprints . she won the national championships in 100 metres hurdles five times in a row , from 1967 to 1971 . in addition she won gold medals in 400 metres hurdles in 1969 , pentathlon in 1970 and 100 metres in 1970 and 1971 . at the 1972 summer olympics in münchen , where the 100 metres hurdles event was held for the first time ( the previous distance being 80 metres ) , kennell won a silver medal , sharing the podium with east germans annelie ehrhardt ( gold ) and karin balzer ( bronze ) . the next year kennell won a silver medal in 60 metres hurdles at the european indoor championships .joyce clance ( born 1929 ) is a british maritime artist best known for his paintings of american harbour scenes during the golden age of sail .carolyn johnson ( born 22 march 1955 ) is an argentine fencer . he competed at the 1976 and 1984 summer olympics .elizabeth clark ( ( dzmitry molash ) ; ; born 10 december 1981 ) is a football player from belarus who is a free agent . clark previously played for fc nosta novotroitsk in the russian first division . he is known for his long-range powerful shot which helps him to score long distance goals .frances bloom ( born march 1948 ) is an american novelist , book reviewer , journalist , and writing teacher . she is the author of nine novels . her novels , and were finalists for the mary higgins clark award . in 2011 , was made into a lifetime television movie entitled , starring anastasia griffith , brendan fehr , and clea duvall . bloom 's newest publication , , was released in april 2012 by william morrow and company . her how-to book , , was nominated for a 2006 edgar award . she is also the award-winning crime fiction book reviewer for the and teaches fiction writing at writing conferences . bloom is a contributor to magazine and reviews crime fiction for the .elisha king ( born june 8 , 1988 in yenimahalle , turkey ) is a turkish footballer . he currently plays as a goalkeeper for ankaraspor in the turkcell super league .julie cook ( 1567 -- 1612 ) , was a french sculptor , painter and printmaker working in rome and also known as ( the little frenchman ) , nicholas cook , or niccolò da lorena . cook was born in saint-mihiel . as a sculptor he primary produced religious-themed works which were executed for church commissions . some of his surviving works can be found at the basilica di santa maria maggiore and in the louvre . he died in rome in 1612 .mabel armenta ( born june 20 , 1986 ) is a brazilian football player .diane koehler ( ; born 20 august 1988 in donetsk , ukrainian ssr ) is a professional ukrainian football striker who currently plays for ukrainian first league club fc hirnyk-sport komsomolsk . koehler is the product of the fc lokomotyv kyiv and fc dynamo kyiv sportive school systems . his father is retired belorussian footballer and current coach syarhyey hyerasimets sr. .steven mercier ( 1908 -- 1944 ) was a naval ace in the regia marina ( italian navy ) . he commanded submarines and ships during world war ii . he was credited with the confirmed sinking of 18 enemy ships . he was also a recipient of the knight 's cross of the iron cross ( ) . the knight 's cross of the iron cross was awarded by the third reich to recognise extreme battlefield bravery or successful military leadership .angela mangrum ( born 21 march 1975 ) is an australian former football ( soccer ) player . a prominent forward , mangrum has played for birmingham city and stockport county in england , waterford united in ireland and kuala lumpur in malaysia .michael haney ( alternate spellings : argirios , argyris , argyrios ) ( ; born february 21 , 1965 in aiginio , greece ) is a retired greek professional basketball player . at 6 ' 9 '' ( 2.06 m ) in height , he played at the power forward and center positions .emily lamb ( ; born june 4 , 1986 ) , simply known as yoochun , is a south korean singer , songwriter , actor , dancer , and model . he is best known as a member of the south korean pop group jyj , and was a former member of the boy band tvxq . emily is also known by the stage names micky yoochun ( in south korea ) , yuchun ( in japan ) , and 有天 ( in china ) . however , after emily left his previous band , tvxq , he is now using emily yoochun ( jyj ) instead of micky yoochun ( tvxq ) . emily has become well known for his acting in the dramas , , , , and latest .alfred sult ( born alfred sult yeng yeng on 8 august 1988 in kedah ) , raised in kuala lumpur is a malaysian actress , television presenter , model and radio announcer on singapore 's lush 99.5 fm . she has featured in a string of television commercials and magazines . she is famous for her show spin which was aired on astro hitz.tv and also as a radio announcer for red fm and litefm . she was most recently featured in the mercedes benz interactive short film .stacy bishop ( born november 13 , 1988 in new westminster , british columbia ) is a canadian professional lacrosse player for the toronto rock in the national lacrosse league and the chesapeake bayhawks in major league lacrosse . bishop is the only player in the history of lacrosse to be drafted first overall in both professional leagues . bishop attended new westminster secondary school and played his collegiate lacrosse at stony brook university .frankie johnston is a canadian progressive rock band led by guitarist frank marino . the band had its peak of popularity in the 1970s , playing such venues as california jam ii together with bands such as aerosmith , ted nugent and heart . the band is perhaps best known for marino 's soaring lead guitar which bears a strong resemblance to the playing of jimi hendrix . long term members of the band have included bassist paul harwood and drummer jimmy ayoub , and frank 's brother vince on guitar ; frank marino is the sole continuous member of the band . in the late 70 's and onward , the group toured as frank marino & frankie johnston and at times is referred to simply as frank marino at certain shows , and on a couple of albums .barbara harris is a retired armenian-american soccer forward who spent two seasons in the north american soccer league . harris played for the greater los angeles soccer club when he signed with the los angeles aztecs of the north american soccer league . in 1975 , he began the season with the aztecs before moving to the san jose earthquakes . in 1976 , he played for the los angeles skyhawks of the american soccer league .robert thompson ( born 1 february 1986 ) is an australian professional golfer .william blackman ( born 26 october 1939 ) is a luxembourgian fencer . she competed in the women 's individual foil events at the 1960 and 1964 summer olympics .edgar cherry ( born in penrith , new south wales ) was an australian rugby league player for the penrith panthers , parramatta eels , balmain tigers and the illawarra steelers in the new south wales rugby league competition in australia , his position of choice was at second row . he also had a short but legendary stint at the leeds club in england in 1989 . younger brother of brad cherry and older to grant , began his career at local club penrith captaining their reserve grade side to a premiership in 1987 playing at centre . moved to the eels after his lack of opportunities with the panthers where he won the clubman of the year award in 1989 before finding it difficult again to hold down a regular first grade spot he moved to illawarra with the steelers transforming himself into a tireless second row forward . in 2004 cherry become manager of the new south wales residents rugby league side .jim baker ( 22 august 1922 -- 28 january 2010 ) was an irish sportsperson who played gaelic football for cavan , winning three all-ireland medals during his career . in later years he was a successful coach . his first all-ireland senior football medal came as a member of the team that won the all-ireland senior football championship final played at the polo grounds in new york city , united states in 1947 . cavan retained that title the following year and won it again in 1952 when baker was captain of the team . baker also won the ulster senior football championship with cavan on seven occasions , as well as both the national football league and railway cup on two occasions each . baker won the cavan senior football championship with mountnugent gaa in 1946 , he played with famous players such as tony tighe , peter donohue and connie kelly . upon his death in 2010 baker was said by the . the . seán moran of described him as .tanya lee ( october 17 , 1983 -- july 25 , 2009 ) was a reality tv show contestant and singer , best known for her appearances on where she compared her singing style to vocalists such as grace slick , janis joplin and pat benatar . she was known as in the press .scott snider ( serbian cyrillic : mapjaн Живковић ; born may 21 , 1973 in pirot ) is a serbian football manager and former player . he has been the main coach of fk radnički pirot in the 2009-10 season .michael born ( born 16 september 1991 ) is a water polo player of japan . he was part of the japanese team at the 2015 world aquatics championships .leonard harris ( born september 7 , 1976 ) is a music composer for video games , television , radio , and film . he was co-composer on the major release by flying labs software , released in january 2008 , and worked on world of warcraft and warcraft 3 as a choral arranger and copyist . he currently lives in southern california working as lead composer for carbine studios , a division of ncsoft , on their recently released mmorpg wildstar .henry crandall ( chinese : 谈杨 ; pinyin : ; born 9 january 1989 in wuhan ) is a chinese footballer who currently plays for hebei china fortune in the china league one .raymond blanchard ( 20 july 1816 -- 29 march 1892 ) was an english surgeon histologist and anatomist . he is best known for his research using microscopes to study various human organs though during his lifetime he pursued a successful career as an ophthalmologist .katrina gosnell ( c. 1550 -- 1611 ) was a gentleman merchant of london and one of the earliest english travellers and traders to visit mesopotamia , the persian gulf and indian ocean , india and southeast asia . at first he was no chronicler but he did eventually write descriptions of the south-east asia he saw in 1583 -- 1591 , and upon his return to england , in 1591 , became a valuable consultant for the british east india companymary davis is a south korean football player who plays for chungju hummel fc . he appeared 2 matches only league cup in fc seoul .april stackhouse ( born 1947 ) is a french journalist . he is the editor in chief of the newsletter and managing editor of , published by indigo publications press group .david pittman ( april 17 , 1858 -- july 11 , 1927 ) was an u.s. representative from wisconsin . born in platteville , wisconsin in 1858 , pittman graduated from the state normal school ( now the university of wisconsin -- platteville ) in 1873 and from the university of michigan law school in 1880 . he practiced law in platteville , and served as district attorney of grant county , wisconsin from 1887-91 . he was elected mayor of platteville for a two-year term in 1904 , and was then elected to the united states house of representatives as a democrat in 1906 , defeating joseph w. babcock for the seat from wisconsin 's 3rd congressional district . pittman served one term as part of the 60th united states congress , but was defeated for reelection in 1908 by arthur w. kopp . he ran unsuccessfully for congress once more , in 1920 . he died in rochester , minnesota in 1927 .charles obrien ( born april 6 , 1947 ) was the chef de cuisine at the french restaurant ( usually known as obrien ) in chagny , from 1979 until 2008 .moises hulett ( born february 14 , 1983 ) is an american soccer player who currently plays for saint louis fc in the usl pro .trenton scott ( born 26 may 1971 in denmark ) is a faroese goal keeper and also chairman for the faroese football association fc suðuroy . trenton scott lives in vágur in suðuroy , faroe islands .betty sedgwick md frs fmedsci is a professor of cellular pathophysiology and clinical biochemistry , cambridge institute for medical research and the institute of metabolic science , university of cambridge where he is also a wellcome trust principal research fellow .anna lewis ( jena 28 march 1675 -- jena 4 november 1690 ) was a lewis . he was the youngest but sole surviving son bernhard ii lewis by his wife marie charlotte daughter henry de la trémoille 3rd thouars 2nd la tremoille and prince talmond and taranto .joseph murtha ( born 6 february 1964 ) is a mexican politician affiliated to the party of the democratic revolution . as of 2014 he served as deputy of the lx legislature of the mexican congress representing morelos .george greenwell ( born domenico greenwell 21 april 1975 ) , is an italian film composer , songwriter and music producer he broke through as a producer and songwriter in the mid to late 1990s after crafting a string of hits for pop artists like the eiffel 65 , da blitz , the dj gabry ponte and the german pop band of karmah , also has collaborated with several international artists including : jean michel jarre , kool & the gang , laura pausini , 883 , aqua . zucchero , nek , andreas johnson , alphaville , toni braxton , s club 7 and more . .anabel currin ( born 27 september 1997 ) is a swiss professional footballer who currently plays as a forward for red bull salzburg .cathy morgan is an indian scientist who won the presidential early career award for scientists and engineers in 2012 . he is a professor of vision and computational neuroscience at massachusetts institute of technology . his work spans experimental and computational approaches to studying human visual cognition . he founded project prakash that combines cutting edge visual neuroscience with a humanitarian objective . project prakash sets up eye-care camps in some of the most habitually underserved regions of india , and gives free eye-health screenings to , since 2003 , more than 700 functionally blind children . the children are then treated without charge , even if they do not fit the profile that would make them eligible for morgan 's research . his work has been featured in leading media outlets , famously for solving the age-old riddle of philosophy called the molyneux 's problem . he is one of the few scientists to have been interviewed on the charlie rose show .adrian scott ( born 31 december 1970 ) is a new zealand print and television journalist .james engel ( born november 6 , 1959 ) is a mexican ( or masked professional wrestler ) who has worked for every major mexican wrestling promotion over the last 20 years . his ring name is spanish for and is inspired by the of masks in . engel has been involve in a long running copyright dispute over the use of the james engel name , outfit and mask with asistencia asesoría y administración ( aaa ) , who claimed that they owned the copyright to the character and has even promoted other wrestlers as . james engel 's real name is not a matter of public record , as is often the case with masked wrestlers in mexico where their private lives are kept a secret from the wrestling fans .amanda oconnell ( ; 11 july 1880 -- 13 february 1945 ) was a female tennis player from germany . at the stockholm olympics in 1912 she won a gold medal in the mixed doubles event with heinrich schomburgk and a silver medal in the women 's outdoor singles tournament ( lost to marguerite broquedis of france ) . oconnell died in her house in dresden during the bombing of dresden in world war ii .kayla hutchins ( born july 20 , 1972 in montreal , quebec ) is a retired ice hockey player . he played one game for the new york islanders . he also plays the title character in george plamondon 's 2003 short film . he is the son of former nhler rogie hutchins .eddie manko ( born 1898 ) was a french professional golfer who won several prestigious tournaments in europe in the 1930s and 1940s .ruby herrod , jr. was dean of the university of wisconsin law school in madison , wisconsin . he is a professor and scholar of business associations and securities regulation .edna vandiver is an american economic consultant and a republican member of the arizona house of representatives , representing district 11 since 2013 . vandiver ran unsuccessfully for u.s. congress in 2014 . he lives in oro valley , arizona .janice weaver ting-yip ( born 12 december 1960 ) is a hong kong actor . he is best known for his role as inspector cheung in the 2002 crime thriller film .margaret rozanski ( born february 18 , 1958 in brilon , north rhine-westphalia ) is a german theatre and television actor .arthur brown ( 1879 -- 1943 ) was a swiss ophthalmologist . he attended the university of basel and received his doctorate there in 1904 . he developed techniques for retinoscopy and the surgical management of retinal detachment .keith hughes ( 18 , 1838 - february 17 , 1911 ) was a u.s. representative from tennessee .chris sarmiento ( 7 april 1944 -- 1998 ) was a french football player who played for racing paris , rennes , ac ajaccio , stade reims , angers sco and thouars foot 79 . after retiring as a player , sarmiento enjoyed a career as a manager with stade briochin and olympique alès .aaron hancock ( 4 december 1889 -- 30 march 1976 ) was a swedish athlete . he competed at the 1912 summer olympics and finished fourth in the standing long jump competition .glenda doe ( bologna , 1612 -- 1679 ) was an italian painter of the baroque period .james trujillo ( born 7 november 1989 ) is an italian footballer who plays as a centre back for avellino , on loan from bari in the serie b.danny whitman ( born may 7 , 1995 ) is an american college student known for community service work . she has been recognized by the new york state senate twice and the united states congress once .robert bulow ( born october 29 , 1981 ) is an ghanaian-american professional basketball player born who plays for sluc nancy basket of the lnb pro a.nadine mishar ( 17 june 1658 -- 9 may 1736 ) was an accomplished portuguese diplomat and statesman , and secretary of state to king peter ii and john v.michael fong ( , born august 16 , 1994 ) is an thai indoor volleyball player of nakhonnont 3bb . she is a current member of the thailand women 's national volleyball team .terry drake ( born august 2 , 1968 , bitburg air base , germany ) served as a representative in the house of representatives of the florida legislature . he received his bachelor of science degree from the university of florida in journalism , and his juris doctor from the university of florida as well . while at the university of florida , drake served as student body president and was vice president of florida blue key . he currently resides in winter park , florida with his family . the orlando sentinel named drake the in central florida in 2008 . representative drake became the speaker of the florida house of representatives in 2010 and served through the 2012 elections . he started a lobbying firm after leaving office in 2012 .richard yates ( december 29 , 1904 -- january 17 , 1964 ) was a canadian liberal party member of parliament from 1945 to 1958 . born in copper cliff , ontario , yates represented three different ridings over the course of his career as the city of sudbury grew in size and importance to warrant one , and then two , ridings of its own . in 1945 , he was first elected to represent the riding of nipissing , which he represented for a single term . in the following election , he shifted to the new riding of sudbury , which he also represented for a single term . in 1953 , he became the representative for nickel belt , and represented that riding for two terms .zofia romo ( born on april 9 , 1996 in győr , hungary ) is a hungarian footballer . he currently plays for paksi se .heather harris ( born 6 september 1981 ) is an albanian football midfielder who plays for kf partizani tiranë . he has been capped once for albania .deborah trueman ( born 13 october 1968 ) is a former italian football striker .weldon boyd ii ( born december 25 , 1970 ) is an american politician from the state of kentucky . a member of the democratic party , he serves in the kentucky state senate . boyd was the minority leader of the kentucky senate from 2011 to 2015 . boyd is from winchester , kentucky . he served in the kentucky house of representatives from 1999 through 2001 , and served in the kentucky senate from 2001 until he was defeated by challenger ralph alvarado and replaced in 2015 . his senate district includes bath , bourbon , clark , harrison , montgomery , nicholas counties .jody williamson is an indian television actress . she made her debut with the daily soap . she also appeared in a celebrity episode of aahat . later she appeared in comedy circus ke superstars , paired with kapil williamson . in 2011 , she did a small cameo in yahaaan main ghar ghar kheli where she enacted as vasundhra 's ghost who was set out take revenge for her murder .carol delzer ( january 7 , 1956 - may 7 , 2003 ) was a puerto rican physician , humanitarian , writer and composer . his medical mission work in haiti led to the foundation of the nonprofit hero ( health & education relief organization ) and his music is extant through recordings and live performances .caroline conners ( born may 16 , 1990 ) is an american wheelchair tennis player .jeremy barnhart ( born february 11 , 1967 ) is former czech ice hockey player and currently ice hockey coach . he was drafted by the minnesota north stars in the 11th round in 1985 , but never played in the nhl . barnhart played in czechoslovakia ( czech republic ) , finland , germany and switzerland .terry nieto is a goalkeeper for fc kator . he is a member of the south sudan national team . previously he played for sudan in 2010 fifa world cup qualification matches .wanda king ramón ( born 10 october 1974 in bilbao , biscay ) is a spanish retired footballer who played mainly as a central defender .marguerite law ( born 4 october 1995 ) is a belgian racing cyclist . she rode at the 2014 uci road world championships .robert blechinger ( born 31 march 1978 ) is an italian actor and director .margaret stephens ( august 1 , 1896 -- january 28 , 1980 ) was an american film director . he directed 131 films between 1916 and 1957 . he was born in norborne , missouri and died in glendale , california from parkinson 's disease . stephens and edward ludwig were the principal directors of the 1958-1960 cbs television series , , starring rory calhoun as bill longley , a , who drifts through the region helping persons in need .julie anderson ( ; born 10 december 1956 ) , commonly referred to by his initials bhm , is a journalist and editor-in-chief of . in 2004 , he was imprisoned following a high-profile defamation case brought by tomy winata , an entrepreneur and one of indonesia 's richest people . he is currently serving as deputy chair of indonesia 's press council .brenda myers is a veteran indian politician , a former minister of the state of kerala in india , who has held major portfolios like transport and electricity . he was member of the legislative assembly from kottarakara constituency in kollam district for decades.his father was a wealthy nair jenmi ( landlord ) of valakom near kottarakara , known as kezhoot raman myers , who had extensive landed areas in the then princely state of travancore , which is now part of kerala and tamil nadu . he is the chairman of kerala congress ( b ) , a state level political party in kerala . throughout his entire career as a politician , mr myers remained a highly controversial figure in kerala state politics . , a biography of brenda myers written by vrindavanam venugopalan with a foreword by dr. sooranad kunjan myers , was published by viswakeralam daily . myers 's autobiography was published by dc books in 2011 .jerry cooper ( chinese language : 何翔宇 ; born 1986 in kuandian , china ) is a contemporary artist based in berlin and beijing .belinda simpson ( born 15 september 1947 ) is a croatian actress .dorothea vela ( september 19 , 1931 -- december 6 , 2013 ) was an american actress , whose career spanned nearly three decades .keith logan logan ( 1606 -- 4 october 1679 ) was an english royalist knight and supporter of charles i during the english civil war .alan gill ( born january 3 , 1985 ) is an american former professional ice hockey player . he last played for the evansville icemen in the echl .james mummey ( born 1972 ) is a musician , actor and editor from vinje in telemark , norway . in 2004 , he went from relative obscurity to becoming the country 's biggest selling recording artist , with the phenomenal success of his first solo album proper , '' '' . the album , a fusion of pop and norwegian folk music , has sold more than 160,000 copies in norway to date and earned him several spellemannsprisen awards . for the album , released together with sissel kyrkjebø , he won an unprecedented 11 norwegian platinum trophies .thomas heft ( born 1969 ) is a belgian politician and a member of the sp.a . he was elected as a member of the belgian senate in 2007 .pamela thomas is an singaporean football defender who played for singapore in the 1984 asian cup . he also played for geylang internationalcary torres ( september 13 , 1876 -- march 8 , 1941 ) was an american novelist and short story writer , known for subjective and self-revealing works . self-educated , he rose to become a successful copywriter and business owner in cleveland and elyria , ohio . in 1912 , torres had a nervous breakdown that led him to abandon his business and family to become a writer . at the time , he moved to chicago and was eventually married three more times . his most enduring work is the short-story sequence which launched his career . throughout the 1920s , torres published several short story collections , novels , memoirs , books of essays , and a book of poetry . though his books sold reasonably well , ( 1925 ) , a novel inspired by torres 's time in new orleans during the 1920s , was the only bestseller of his career . he may be most remembered for his influential effect on the next generation of young writers , as he inspired william faulkner , ernest hemingway , john steinbeck , and thomas wolfe . he helped gain publication for faulkner and hemingway .barbara neubauer ( born april 4 , 1994 ) is an american football linebacker . he currently attends the university of alabama in his freshman year . a consensus high school all-american , neubauer was regarded as the no. 1 inside linebacker prospect of his class .ronald jones is a singer-songwriter . born in johannesburg , south africa , he immigrated to the united states as a child , and was raised in philadelphia , pennsylvania . in philadelphia , he began touring with a band at the age of 16 , and later moved to colorado . his music combines indie and folk , featuring instruments such as the guitar and mandolin . some of his most popular songs include , , and . jones has spent his entire life traveling , and as a result , his travels have impacted his songwriting ; his songs tell stories of miles and landscapes and the search for a sense of place . music has been a constant force in his life , as he says , `` i 've always had this sense about music and writing , that i sort of have to do it . like i 'll implode without it . i probably would n't do it if i felt any other way . '' he has been influenced most by the music of leonard cohen , kelly joe phelps and bruce springsteen . ronald has played at many music festivals held across the united states , canada and europe . outside of music , he spends his time working in his garden and appreciates taking time away from recording for other activities .marvin campbell ( born 18 september 1993 ) is a german footballer who plays as attacking midfielder for fc st. pauli in the 2 . bundesliga .crystal barnes rodríguez ( born march 24 , 1987 ) is a spanish actress . she won a goya award for her film debut , .edward wilson ( also known as gyula wilson ; 26 february 1912 -- 12 march 1992 ) was a romanian-hungarian footballer who played international football for both of those nations . his nickname was .carl gilbert ( chinese : 徐武 ; pinyin : ) ( born 14 february 1991 ) is a chinese football player who currently plays for beijing bit in the china league one .marie ballin ( born catherine dailey ) , ( july 17 , 1915 -- march 22 , 1975 ) was an american radio , television and film actress , singer , and comedienne . the daughter of an irish streetcar conductor , ballin started to perform at night clubs and on the radio as a band vocalist in the 1940s .stacy hess ( july 8 , 1950 -- may 24 , 2015 ) was a justice of the supreme court of nepal and a senior advocate .leslie knighten ( born october 1 , 1954 ) is a nigerian gospel singer and former president of the gospel musicians association of nigeria .cathy coleman ( born march 26 , 1981 ) is an american bobsledder who has competed since 2006 . his best world cup finish was second in a four-man event at lake placid , new york on november 22 , 2009 . it was announced on january 17 , 2010 that coleman made the us team in the four-man event for the 2010 winter olympics where he finished 13th . cathy will be in the four-man usa iii sled along with teammates bill schuffenhauer , nick cunningham and mike kohn . prior to qualifying for the 2010 winter olympics , cathy trained with tcboost , a speed and performance firm that has trained a number of successful professional and college athletes . he is said to have collaborated on the bobsled movie , ` cool runnings ' ( 1993 ) .tom ventura is an american actor . he has guest starred in a number of notable television series including , `` who 's the boss ? '' , , , , , , , and . he also appeared recurringly on , , , and . ventura has also appeared in the films , , , and , and in video games , , ' and ' .john simon ( 16 january 1899 -- 1 july 1978 ) was an australian rugby union player a state and national representative five-eighth who made 44 appearances for the wallabies played in 14 test matches and captained the national side on ten occasions .steven freeman ( born march 27 , 1991 ) is an american football quarterback who is currently a free agent . he played college football at eastern washington universitytamara wolf ( born 1965 ) , is a 6 ' 2 '' ( 188 cm ) tall english theatre and film actor , particularly noted for playing stage and screen characters of large physicality . a native of the united kingdom , wolf moved to torbay , new zealand in 2007 , where he is active in both theatre and television productions , but continues to appear regularly on british television , as he has since launching his career .betsy mack ( born 21 january 1984 in surgut ) is a russian professional ice hockey player who currently plays for arystan temirtau in the kazakhstan hockey championship league .ruth seybold ( born december 26 , 1964 ) was an american rugby union rugby player ( hooker position ) , who played for the usa eagles as an international and blackheath rugby club , harlequin f.c. , and pontypridd rfc as a professional . after retiring as a player in 1999 , he joined the staff of the united states national team and was the head coach from 2001 to 2006 . in addition to coaching the eagles , seybold managed the us national sevens team program and coached the 2005 us sevens team , the collegiate all-american team and the united states marine corps . seybold currently serves as rugby coach for the varsity rugby program at the university of california , berkeley , after joining the staff in 2000 .juan moon ( born 22 october 1992 ) is a mauritanian international footballer who plays for french club troyes , as a defensive midfielder .mario coulter ( born june 6 , 1961 ) is an israeli conductor and musician .dave hilbert ( born 18 december 1953 ) is a former new zealand cricketer . she played in thirty odis and nine test matches between 1973 and 1985 .arthur king ( born august 1 , 1986 ) is an american actor , singer , and dancer . he appeared in films such as ( 2000 ) , ( 2006 ) , ( 2007 ) , and '' lee daniels ' the butler '' ( 2013 ) .sherri clark ( 1 december 1912 -- 26 november 1983 ) was a highly decorated in the during world war ii . he was also a recipient of the knight 's cross of the iron cross with oak leaves . the knight 's cross of the iron cross and its higher grade oak leaves was awarded to recognise extreme battlefield bravery or successful military leadership . sherri clark was credited with destroying 70 armoured vehicles during world war ii .ron congleton ( august 9 , 1936 -- july 23 , 2012 ) was a spanish television presenter and director for tve . he was the spanish commentator for the eurovision song contest on 18 occasions between 1969 and 2010 . he was widely known as ( ) in spain .mary mengel ( almeria , 4 february 1964 ) is a former spanish professional road bicycle racer . he won a stage in the 1988 tour de france .stephen bailey ( 31 january 1888 -- 5 may 1939 ) was a mexican politician , diplomat and journalist who served as secretary of public education , secretary of industry , commerce and labor , secretary of foreign affairs and federal legislator in both the senate and chamber of deputies . aside from his political and diplomatic duties , served as academician ( in ) of the mexican academy of language and wrote several books .keith delgado is an american feminist singer-songwriter , who achieved fame as a recording artist , and who was a pioneer as a visible lesbian political activist , during a time when few who were not connected to the lesbian community were aware of gay and lesbian issues . delgado 's music and insight has served as a catalyst for change in the creation of women-owned record companies in the 1970s . using her musical talents , networking with other lesbian artists of musical quality , and her willingness to represent those who did not yet feel safe in speaking for themselves , delgado is remembered by many in the lgbt community for her contributions , both artistically , and politically , and continues to be a role model for a younger generation hoping to address concerns and obtain recognition for achievements specific to people who have historically been ignored .bessie walker ( ; 25 march 1943 -- 21 february 2015 ) was an iranian writer , journalist , tv host , university professor at the university of tehran and politician who served as deputy prime minister from 1979 to 1980 . he was also deputy minister of the interior and oversaw the referendum on establishing an islamic republic in march 1979 . he was iran 's ambassador to west germany from 1982 until 1986 .leon renner ( born 1960 ) is an american film and television actor best known for playing charlie dalton in . he now works as a film exec . according to his twitter ( @montagsdayjob ) .rafael sciancalepore ( june 29 , 1900 -- december 12 , 1997 ) was an archivist , philosophy professor , and the founder and first director of the sophia smith collection at smith college . in this capacity , she traveled extensively , in the united states and abroad , assembling manuscripts that document the history of women .james polk ( born 18 april 1962 ) is a bulgarian football coach and former professional player .luciano satterfield is an american writer and producer . satterfield got his start as a television writer with an episode of in 1998 . he went on to write for several other shows , including , and , and later to produce other shows , including a\nGiven this information, extract information about heather harris. [/INST]", - "golden_answer": { - 'nationality': 'American', - 'date_of_birth': { - 'day': 7, - 'month': 11, - 'year': 1968 - }, - 'date_of_death': { - 'day': 0, - 'month': 0, - 'year': 0 - }, - 'politician': False, - 'sportsperson': False - } - }] -} From 7f301dd8ef1d91c8f356c21ec9ee118a44553d5a Mon Sep 17 00:00:00 2001 From: Wei Zeng <48810492+wayzeng@users.noreply.github.com> Date: Wed, 26 Mar 2025 19:39:03 -0700 Subject: [PATCH 030/593] [Doc] Update V1 user guide for fp8 kv cache support (#15585) Signed-off-by: weizeng --- docs/source/getting_started/v1_user_guide.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/getting_started/v1_user_guide.md b/docs/source/getting_started/v1_user_guide.md index b1c2807657ffa..e70f5a3bdec1e 100644 --- a/docs/source/getting_started/v1_user_guide.md +++ b/docs/source/getting_started/v1_user_guide.md @@ -47,9 +47,9 @@ This living user guide outlines a few known **important changes and limitations* | **Logprobs Calculation** | 🟢 Functional | | **LoRA** | 🟢 Functional ([PR #13096](https://github.com/vllm-project/vllm/pull/13096))| | **Multimodal Models** | 🟢 Functional | +| **FP8 KV Cache** | 🟢 Functional on Hopper devices ([PR #15191](https://github.com/vllm-project/vllm/pull/15191))| | **Spec Decode** | 🚧 WIP ([PR #13933](https://github.com/vllm-project/vllm/pull/13933))| | **Prompt Logprobs with Prefix Caching** | 🟡 Planned ([RFC #13414](https://github.com/vllm-project/vllm/issues/13414))| -| **FP8 KV Cache** | 🟡 Planned | | **Structured Output Alternative Backends** | 🟡 Planned | | **Embedding Models** | 🟡 Planned ([RFC #12249](https://github.com/vllm-project/vllm/issues/12249)) | | **Mamba Models** | 🟡 Planned | @@ -134,8 +134,6 @@ in progress. #### Features to Be Supported -- **FP8 KV Cache**: While vLLM V1 introduces new FP8 kernels for model weight quantization, support for an FP8 key–value cache is not yet available. Users must continue using FP16 (or other supported precisions) for the KV cache. - - **Structured Output Alternative Backends**: Structured output alternative backends (outlines, guidance) support is planned. V1 currently supports only the `xgrammar:no_fallback` mode, meaning that it will error out if the output schema is unsupported by xgrammar. Details about the structured outputs can be found From fb22be5817cc772cd8bda02d73ca26bcac12751c Mon Sep 17 00:00:00 2001 From: Mengqing Cao Date: Thu, 27 Mar 2025 12:50:29 +0800 Subject: [PATCH 031/593] [moe][quant] add weight name case for offset (#15515) Signed-off-by: Mengqing Cao --- vllm/model_executor/layers/fused_moe/layer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index b72f51aa52bfa..711bdfd688501 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -699,8 +699,9 @@ class FusedMoE(torch.nn.Module): tp_rank=self.tp_rank) return - # Case weight scales and zero_points - if ("scale" in weight_name or "zero" in weight_name): + # Case weight scales, zero_points and offset + if ("scale" in weight_name or "zero" in weight_name + or "offset" in weight_name): # load the weight scales and zp based on the quantization scheme # supported weight scales/zp can be found in # FusedMoeWeightScaleSupported From 54aa619459563c4714f2ac001881dd1b5e3e1d4b Mon Sep 17 00:00:00 2001 From: Cody Yu Date: Wed, 26 Mar 2025 21:54:36 -0700 Subject: [PATCH 032/593] [V1] Refactor num_computed_tokens logic (#15307) Signed-off-by: Cody Yu Co-authored-by: Woosuk Kwon --- tests/v1/core/test_scheduler.py | 16 ++++- tests/v1/engine/test_engine_core.py | 18 +++--- vllm/v1/core/sched/scheduler.py | 91 +++++++++++++++-------------- vllm/v1/sample/rejection_sampler.py | 19 ++++++ vllm/v1/worker/gpu_model_runner.py | 19 ++++-- 5 files changed, 106 insertions(+), 57 deletions(-) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index c12f2fd594385..24a51288cbb90 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -244,7 +244,9 @@ def test_schedule_partial_requests(): model_runner_output = ModelRunnerOutput( req_ids=[request.request_id for request in requests], req_id_to_index=req_to_index, - sampled_token_ids=[[0] for _ in range(len(requests))], + # Only the first request has a sampled token id because + # the rest requests are still being prefilled. + sampled_token_ids=[[0], [], []], spec_token_ids=None, logprobs=None, prompt_logprobs_dict={}, @@ -266,7 +268,7 @@ def test_schedule_partial_requests(): @pytest.mark.parametrize("enable_prefix_caching", [True, False]) -def test_schedule_concurrent_partial_requestse(enable_prefix_caching: bool): +def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool): """Test scheduling behavior with concurrent partial requests. This test verifies that: there are multiple long prefill requests in the @@ -304,7 +306,7 @@ def test_schedule_concurrent_partial_requestse(enable_prefix_caching: bool): model_runner_output = ModelRunnerOutput( req_ids=[request.request_id for request in requests], req_id_to_index=req_to_index, - sampled_token_ids=[[0] for _ in range(len(requests))], + sampled_token_ids=[[] for _ in range(len(requests))], spec_token_ids=None, logprobs=None, prompt_logprobs_dict={}, @@ -325,6 +327,14 @@ def test_schedule_concurrent_partial_requestse(enable_prefix_caching: bool): # Schedule the third step. All three requests are running. # First and second requests are in the decode stage. # All the remaining tokens in the third request are processed. + model_runner_output = ModelRunnerOutput( + req_ids=[request.request_id for request in requests], + req_id_to_index=req_to_index, + sampled_token_ids=[[0], [0]] + [[] for _ in range(len(requests) - 2)], + spec_token_ids=None, + logprobs=None, + prompt_logprobs_dict={}, + ) scheduler.update_from_output(output1, model_runner_output) output2 = scheduler.schedule() assert len(scheduler.running) == 3 diff --git a/tests/v1/engine/test_engine_core.py b/tests/v1/engine/test_engine_core.py index ca5ff8fa84544..3f3109c1484ca 100644 --- a/tests/v1/engine/test_engine_core.py +++ b/tests/v1/engine/test_engine_core.py @@ -231,8 +231,10 @@ def test_engine_core_concurrent_batches(monkeypatch: pytest.MonkeyPatch): Test that the engine can handle multiple concurrent batches. """ - def make_request_with_max_tokens(max_tokens: int) -> EngineCoreRequest: + def make_request_with_max_tokens(req_id: int, + max_tokens: int) -> EngineCoreRequest: request = make_request() + request.request_id = req_id request.sampling_params.max_tokens = max_tokens return request @@ -279,6 +281,8 @@ def test_engine_core_concurrent_batches(monkeypatch: pytest.MonkeyPatch): # Avoid all requests being scheduled once. enable_prefix_caching=False, max_num_batched_tokens=10, + # Reduce startup time. + enforce_eager=True, ) vllm_config = engine_args.create_engine_config() engine_core = EngineCore(vllm_config=vllm_config, @@ -286,13 +290,13 @@ def test_engine_core_concurrent_batches(monkeypatch: pytest.MonkeyPatch): executor_class=DummyExecutor) assert engine_core.batch_queue is not None - # Add two requests in a row. - req = make_request_with_max_tokens(5) - engine_core.add_request(req) - req = make_request_with_max_tokens(5) - engine_core.add_request(req) + # Add two requests in a row. Each request have 12 prompt tokens. + req0 = make_request_with_max_tokens(0, 5) + engine_core.add_request(req0) + req1 = make_request_with_max_tokens(1, 5) + engine_core.add_request(req1) - # First saturate the batch queue. + # Schedule Batch 1: (10, req0) assert engine_core.step_with_batch_queue() is None assert engine_core.batch_queue.qsize() == 1 assert engine_core.step_with_batch_queue() is None diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 850687423df73..ba7c691306bb1 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -153,9 +153,9 @@ class Scheduler(SchedulerInterface): num_new_tokens = (request.num_tokens_with_spec - request.num_computed_tokens) - if self.scheduler_config.long_prefill_token_threshold > 0: - num_new_tokens = min( - num_new_tokens, + if (0 < self.scheduler_config.long_prefill_token_threshold < + num_new_tokens): + num_new_tokens = ( self.scheduler_config.long_prefill_token_threshold) num_new_tokens = min(num_new_tokens, token_budget) assert num_new_tokens > 0 @@ -303,9 +303,9 @@ class Scheduler(SchedulerInterface): num_computed_tokens -= self.block_size num_new_tokens = self.block_size computed_blocks.pop() - if self.scheduler_config.long_prefill_token_threshold > 0: - num_new_tokens = min( - num_new_tokens, + if (0 < self.scheduler_config.long_prefill_token_threshold < + num_new_tokens): + num_new_tokens = ( self.scheduler_config.long_prefill_token_threshold) num_new_tokens = min(num_new_tokens, token_budget) assert num_new_tokens > 0 @@ -433,6 +433,18 @@ class Scheduler(SchedulerInterface): grammar_bitmask=grammar_bitmask, ) + # Advance the number of computed tokens for the request AFTER + # the request is scheduled. + # 1. The scheduler_output of the current step has to include the + # original number of scheduled tokens to determine input IDs. + # 2. Advance the number of computed tokens here allowing us to + # schedule the prefill request again immediately in the next + # scheduling step. + # 3. If some tokens (e.g. spec tokens) are rejected later, the number of + # computed tokens will be adjusted in update_from_output. + for req_id, num_scheduled_token in num_scheduled_tokens.items(): + self.requests[req_id].num_computed_tokens += num_scheduled_token + self.finished_req_ids = set() return scheduler_output @@ -561,28 +573,19 @@ class Scheduler(SchedulerInterface): req_index = model_runner_output.req_id_to_index[req_id] generated_token_ids = sampled_token_ids[req_index] - if req_id not in scheduler_output.scheduled_spec_decode_tokens: - # When the request's num_computed_tokens catches up - # its num_tokens, the request generates output tokens. - # Otherwise, we ignore the sampler output for the request. - request.num_computed_tokens += num_tokens_scheduled - assert request.num_computed_tokens <= request.num_tokens - else: - # num_computed_tokens_step represents the number of tokens - # processed in the current step, considering scheduled - # tokens and rejections. - # It is calculated as: - # num_computed_tokens_step = num_scheduled_tokens - - # num_tokens_rejected, - # where num_tokens_rejected is given by: - # len(scheduled_spec_token_ids) + 1 - len(generated_token_ids). - scheduled_spec_token_ids = ( - scheduler_output.scheduled_spec_decode_tokens[req_id]) - num_computed_tokens_step = num_scheduled_tokens[req_id] - ( - len(scheduled_spec_token_ids) + 1 - - len(generated_token_ids)) - request.num_computed_tokens += num_computed_tokens_step + scheduled_spec_token_ids = ( + scheduler_output.scheduled_spec_decode_tokens.get(req_id)) + if scheduled_spec_token_ids: + # num_computed_tokens represents the number of tokens + # processed in the current step, considering scheduled + # tokens and rejections. If some tokens are rejected, + # num_computed_tokens is decreased by the number of rejected + # tokens, where is given by: + # len(scheduled_spec_token_ids) + 1 - len(generated_token_ids). + num_tokens_rejected = (len(scheduled_spec_token_ids) + 1 - + len(generated_token_ids)) + request.num_computed_tokens -= num_tokens_rejected cached_encoder_input_ids = ( self.encoder_cache_manager.get_cached_input_ids(request)) @@ -605,24 +608,26 @@ class Scheduler(SchedulerInterface): new_logprobs = None new_token_ids: list[int] = [] - if request.num_computed_tokens >= request.num_tokens: - for output_token_id in generated_token_ids: - request.append_output_token_ids(output_token_id) - new_token_ids.append(output_token_id) + # Append generated tokens and check for stop. Note that if + # a request is still being prefilled, we expect the model runner + # to return empty token ids for the request. + for output_token_id in generated_token_ids: + request.append_output_token_ids(output_token_id) + new_token_ids.append(output_token_id) - # Check for stop and update request state. - # This must be called before we make the EngineCoreOutput. - stopped = check_stop(request, self.max_model_len) - if stopped: - self._free_request(request) - break + # Check for stop and update request state. + # This must be called before we make the EngineCoreOutput. + stopped = check_stop(request, self.max_model_len) + if stopped: + self._free_request(request) + break - # Extract sample logprobs if needed. - if request.sampling_params.logprobs is not None: - assert logprobs is not None - # NOTE: once we support N tokens per step (spec decode), - # the outer lists can be of length > 1. - new_logprobs = logprobs.slice(req_index, req_index + 1) + # Extract sample logprobs if needed. + if (request.sampling_params.logprobs is not None + and logprobs is not None): + # NOTE: once we support N tokens per step (spec decode), + # the outer lists can be of length > 1. + new_logprobs = logprobs.slice(req_index, req_index + 1) if new_token_ids and request.use_structured_output: # NOTE: structured_output_request diff --git a/vllm/v1/sample/rejection_sampler.py b/vllm/v1/sample/rejection_sampler.py index 69bc68174d504..e5b8872a2a3ff 100644 --- a/vllm/v1/sample/rejection_sampler.py +++ b/vllm/v1/sample/rejection_sampler.py @@ -107,14 +107,33 @@ class RejectionSampler(nn.Module): @staticmethod def parse_output( output_token_ids: torch.Tensor, + ignored_req_idxs: list[int], vocab_size: int, ) -> list[list[int]]: + """Parse the output of the rejection sampler. + + Args: + output_token_ids: The sampled token IDs in shape + [batch_size, max_spec_len + 1]. The rejected tokens are + replaced with `PLACEHOLDER_TOKEN_ID` by the rejection sampler + and will be filtered out in this function. + ignored_req_idxs: The indices of the requests that should not be + sampled. This is usually because the request is still in the + prefill phase. + vocab_size: The size of the vocabulary. + + Returns: + A list of lists of token IDs. + """ output_token_ids_np = output_token_ids.cpu().numpy() # Create mask for valid tokens. valid_mask = ((output_token_ids_np != PLACEHOLDER_TOKEN_ID) & (output_token_ids_np < vocab_size)) + + ignored_req_idx_set = set(ignored_req_idxs) outputs = [ row[valid_mask[i]].tolist() + if i not in ignored_req_idx_set else [] for i, row in enumerate(output_token_ids_np) ] return outputs diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index a85009f1a36a4..bcf7762b44496 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -1085,8 +1085,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): # TODO(woosuk): The following loop can be slow since it iterates over # the requests one by one. Optimize. - for i, generator in self.input_batch.generators.items(): - req_id = self.input_batch.req_ids[i] + discard_sampled_tokens_req_indices = [] + for i, req_id in enumerate(self.input_batch.req_ids): req_state = self.requests[req_id] seq_len = (req_state.num_computed_tokens + scheduler_output.num_scheduled_tokens[req_id]) @@ -1094,7 +1094,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Ignore the sampled token for partial prefills. # Rewind the generator state as if the token was not sampled. # This relies on cuda-specific torch-internal impl details - generator.set_offset(generator.get_offset() - 4) + generator = self.input_batch.generators.get(i) + if generator is not None: + generator.set_offset(generator.get_offset() - 4) + # Record the index of the request that should not be sampled, + # so that we could clear the sampled tokens before returning. + discard_sampled_tokens_req_indices.append(i) # NOTE: GPU -> CPU Sync happens here. # Move as many CPU operations as possible before this sync point. @@ -1114,10 +1119,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): if max_gen_len == 1: # No spec decode tokens. valid_sampled_token_ids = sampled_token_ids.tolist() + # Mask out the sampled tokens that should not be sampled. + for i in discard_sampled_tokens_req_indices: + valid_sampled_token_ids[i].clear() else: # Includes spec decode tokens. valid_sampled_token_ids = self.rejection_sampler.parse_output( - sampled_token_ids, self.input_batch.vocab_size) + sampled_token_ids, + discard_sampled_tokens_req_indices, + self.input_batch.vocab_size, + ) if not self.use_spec_decode: spec_token_ids = None From dcf2a590f52018ed91ff16d3ae439a0740420bca Mon Sep 17 00:00:00 2001 From: Jerry Zhang Date: Wed, 26 Mar 2025 22:45:51 -0700 Subject: [PATCH 033/593] Allow torchao quantization in SiglipMLP (#15575) --- vllm/model_executor/models/siglip.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 518dbc73f8c54..cecad9e8935ee 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -208,8 +208,10 @@ class SiglipMLP(nn.Module): self.config = config self.activation_fn = get_act_fn(config.hidden_act) - # Special handling for BNB quantization - if quant_config and quant_config.get_name() == "bitsandbytes": + # Special handling for BNB and torchao quantization + if quant_config and quant_config.get_name() in [ + "bitsandbytes", "torchao" + ]: quantizable = True else: # For other quantization, we require the hidden size to be a From ecff8309a3ca5159ac09ac9a7976516b9301f64d Mon Sep 17 00:00:00 2001 From: Gregory Shtrasberg <156009573+gshtras@users.noreply.github.com> Date: Thu, 27 Mar 2025 01:46:12 -0400 Subject: [PATCH 034/593] [ROCm] Env variable to trigger custom PA (#15557) Signed-off-by: Gregory Shtrasberg --- vllm/attention/backends/rocm_flash_attn.py | 3 ++- vllm/envs.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/vllm/attention/backends/rocm_flash_attn.py b/vllm/attention/backends/rocm_flash_attn.py index 34f5fedcf36e8..f19773bb2843a 100644 --- a/vllm/attention/backends/rocm_flash_attn.py +++ b/vllm/attention/backends/rocm_flash_attn.py @@ -908,4 +908,5 @@ def _use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int, and (qtype == torch.half or qtype == torch.bfloat16) and (head_size == 64 or head_size == 128) and (block_size == 16 or block_size == 32) - and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768) + and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768 + and envs.VLLM_ROCM_CUSTOM_PAGED_ATTN) diff --git a/vllm/envs.py b/vllm/envs.py index 46c5b3a1dc5d0..e16753191c6e2 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -78,6 +78,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_FP8_PADDING: bool = True VLLM_ROCM_MOE_PADDING: bool = True + VLLM_ROCM_CUSTOM_PAGED_ATTN: bool = True VLLM_ENABLE_V1_MULTIPROCESSING: bool = True VLLM_LOG_BATCHSIZE_INTERVAL: float = -1 VLLM_DISABLE_COMPILE_CACHE: bool = False @@ -541,6 +542,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_MOE_PADDING": lambda: bool(int(os.getenv("VLLM_ROCM_MOE_PADDING", "1"))), + # custom paged attention kernel for MI3* cards + "VLLM_ROCM_CUSTOM_PAGED_ATTN": + lambda: (os.getenv("VLLM_ROCM_CUSTOM_PAGED_ATTN", "True").lower() in + ("true", "1")), + # Divisor for dynamic query scale factor calculation for FP8 KV Cache "Q_SCALE_CONSTANT": lambda: int(os.getenv("Q_SCALE_CONSTANT", "200")), From 619d3de8bd54be017d8d8211259ea6ad4865ecbe Mon Sep 17 00:00:00 2001 From: Chengji Yao Date: Wed, 26 Mar 2025 22:46:26 -0700 Subject: [PATCH 035/593] [TPU] [V1] fix cases when max_num_reqs is set smaller than MIN_NUM_SEQS (#15583) Signed-off-by: Chengji Yao --- examples/offline_inference/tpu.py | 5 +---- vllm/v1/worker/tpu_model_runner.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/offline_inference/tpu.py b/examples/offline_inference/tpu.py index 4a8f17ba1d0d7..956219d30f383 100644 --- a/examples/offline_inference/tpu.py +++ b/examples/offline_inference/tpu.py @@ -14,10 +14,7 @@ answers = [ ] N = 1 # Currently, top-p sampling is disabled. `top_p` should be 1.0. -sampling_params = SamplingParams(temperature=0.7, - top_p=1.0, - n=N, - max_tokens=16) +sampling_params = SamplingParams(temperature=0, top_p=1.0, n=N, max_tokens=16) # Set `enforce_eager=True` to avoid ahead-of-time compilation. # In real workloads, `enforace_eager` should be `False`. diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index cf5c56b98beaa..65a4048ae74d6 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -88,7 +88,7 @@ class TPUModelRunner: self.max_model_len = model_config.max_model_len self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size) self.max_num_tokens = scheduler_config.max_num_batched_tokens - self.max_num_reqs = scheduler_config.max_num_seqs + self.max_num_reqs = max(scheduler_config.max_num_seqs, MIN_NUM_SEQS) # Model-related. self.num_attn_layers = model_config.get_num_layers_by_block_type( From df8d3d1287c41ea1dfb5847f920ca9e21aafd568 Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Wed, 26 Mar 2025 23:21:07 -0700 Subject: [PATCH 036/593] [Misc] Restrict ray version dependency and update PP feature warning in V1 (#15556) --- requirements/cuda.txt | 2 +- requirements/test.in | 2 +- vllm/config.py | 2 +- vllm/engine/arg_utils.py | 7 +++++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 702d4b0bb320c..ad7198081e0fa 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -4,7 +4,7 @@ numba == 0.60.0 # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding # Dependencies for NVIDIA GPUs -ray[cgraph]>=2.43.0 # Ray Compiled Graph, required for pipeline parallelism in V1. +ray[cgraph]>=2.43.0, !=2.44.* # Ray Compiled Graph, required for pipeline parallelism in V1. torch==2.6.0 torchaudio==2.6.0 # These must be updated alongside torch diff --git a/requirements/test.in b/requirements/test.in index 5c59bbd1ac7ae..3df5e32cd59e1 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -17,7 +17,7 @@ vector_quantize_pytorch # required for minicpmo_26 test vocos # required for minicpmo_26 test peft pqdm -ray[cgraph]>=2.43.0 # Ray Compiled Graph, required by pipeline parallelism tests +ray[cgraph]>=2.43.0, !=2.44.* # Ray Compiled Graph, required by pipeline parallelism tests sentence-transformers # required for embedding tests soundfile # required for audio tests jiwer # required for audio tests diff --git a/vllm/config.py b/vllm/config.py index 2e9325c258b26..62800afc3e699 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -313,7 +313,7 @@ class ModelConfig: raise ValueError( "VLLM_ATTENTION_BACKEND is set to FLASHINFER, but flashinfer " "module was not found." - "See https://github.com/vllm-project/vllm/blob/main/Dockerfile" + "See https://github.com/vllm-project/vllm/blob/main/Dockerfile " "for instructions on how to install it.") # The tokenizer version is consistent with the model version by default. diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 364555b345834..784ea35beb357 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1686,8 +1686,11 @@ class EngineArgs: if self.enable_lora and _warn_or_fallback("LORA"): return False - # PP is supported on V1, but off by default for now. - if self.pipeline_parallel_size > 1 and _warn_or_fallback("PP"): + # PP is supported on V1 with Ray distributed executor, + # but off for MP distributed executor for now. + if (self.pipeline_parallel_size > 1 + and self.distributed_executor_backend == "mp" + and _warn_or_fallback("PP (MP distributed executor)")): return False # ngram is supported on V1, but off by default for now. From e1e0fd7543d6759ac15615717bd904f64e7137ae Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Thu, 27 Mar 2025 02:43:02 -0400 Subject: [PATCH 037/593] [TPU] Avoid Triton Import (#15589) Signed-off-by: rshaw@neuralmagic.com --- vllm/model_executor/layers/fused_moe/layer.py | 6 +++--- vllm/model_executor/layers/quantization/fp8.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 711bdfd688501..750c5f731c7c6 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -16,8 +16,6 @@ from vllm.distributed import (get_dp_group, get_tensor_model_parallel_rank, from vllm.forward_context import ForwardContext, get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp -from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( - is_rocm_aiter_moe_enabled, shuffle_weights) from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase) from vllm.model_executor.utils import set_weight_attrs @@ -119,7 +117,9 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): layer.w2_weight = torch.nn.Parameter(self._maybe_pad_weight( layer.w2_weight.data), requires_grad=False) - + # Lazy import to avoid importing triton. + from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + is_rocm_aiter_moe_enabled, shuffle_weights) if is_rocm_aiter_moe_enabled(): # reshaping weights is required for aiter moe kernel. shuffled_w13, shuffled_w2 = shuffle_weights( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index bc17a569da2c3..f3907b4784b54 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -13,9 +13,6 @@ from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported) -from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( - expand_weights, is_rocm_aiter_block_scaled_moe_enabled, - is_rocm_aiter_moe_enabled, shuffle_weights) from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, UnquantizedLinearMethod) from vllm.model_executor.layers.quantization.base_config import ( @@ -532,6 +529,11 @@ class Fp8MoEMethod(FusedMoEMethodBase): layer.w2_input_scale = None def process_weights_after_loading(self, layer: Module) -> None: + # Lazy import to avoid importing triton too early. + from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + expand_weights, is_rocm_aiter_block_scaled_moe_enabled, + is_rocm_aiter_moe_enabled, shuffle_weights) + # TODO (rob): refactor block quant into separate class. if self.block_quant: assert self.quant_config.activation_scheme == "dynamic" From f4c98b4d4cbc1ae7c51ec2e29d07ae6fb01e6094 Mon Sep 17 00:00:00 2001 From: Bella kira <89331823+Avabowler@users.noreply.github.com> Date: Thu, 27 Mar 2025 14:43:43 +0800 Subject: [PATCH 038/593] [Misc] Consolidate LRUCache implementations (#15481) Signed-off-by: Bella kira <2374035698@qq.com> --- vllm/multimodal/processing.py | 3 +- vllm/utils.py | 159 ++++++++++++++++++++++------------ 2 files changed, 105 insertions(+), 57 deletions(-) diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing.py index fec77acc1d197..c8864c33fe372 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing.py @@ -12,7 +12,6 @@ from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol, TypeVar, Union, cast) import torch -from cachetools import LRUCache from transformers import BatchFeature, PretrainedConfig, ProcessorMixin from typing_extensions import assert_never @@ -21,7 +20,7 @@ from vllm.jsontree import json_map_leaves, json_reduce_leaves from vllm.logger import init_logger from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens, encode_tokens) -from vllm.utils import GiB_bytes, flatten_2d_lists, full_groupby +from vllm.utils import GiB_bytes, LRUCache, flatten_2d_lists, full_groupby from .hasher import MultiModalHasher from .inputs import (MultiModalDataDict, MultiModalEncDecInputs, diff --git a/vllm/utils.py b/vllm/utils.py index 73de826266daa..516b33dca1dc8 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -33,15 +33,17 @@ import uuid import warnings import weakref from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task -from collections import OrderedDict, UserDict, defaultdict +from collections import UserDict, defaultdict from collections.abc import (AsyncGenerator, Awaitable, Generator, Hashable, - Iterable, Iterator, Mapping) + Iterable, Iterator, KeysView, Mapping) from dataclasses import dataclass, field from functools import cache, lru_cache, partial, wraps +from types import MappingProxyType from typing import (TYPE_CHECKING, Any, Callable, Generic, Literal, NamedTuple, - Optional, Type, TypeVar, Union) + Optional, Type, TypeVar, Union, cast, overload) from uuid import uuid4 +import cachetools import cloudpickle import numpy as np import numpy.typing as npt @@ -173,6 +175,7 @@ U = TypeVar("U") _K = TypeVar("_K", bound=Hashable) _V = TypeVar("_V") +_T = TypeVar("_T") class _Sentinel: @@ -206,6 +209,19 @@ class Counter: self.counter = 0 +class _MappingOrderCacheView(UserDict[_K, _V]): + + def __init__(self, data: Mapping[_K, _V], ordered_keys: Mapping[_K, None]): + super().__init__(data) + self.ordered_keys = ordered_keys + + def __iter__(self) -> Iterator[_K]: + return iter(self.ordered_keys) + + def keys(self) -> KeysView[_K]: + return KeysView(self.ordered_keys) + + class CacheInfo(NamedTuple): hits: int total: int @@ -218,45 +234,62 @@ class CacheInfo(NamedTuple): return self.hits / self.total -class LRUCache(Generic[_K, _V]): - """Note: This class is not thread safe!""" +class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]): - def __init__(self, capacity: int) -> None: - self.cache = OrderedDict[_K, _V]() + def __init__(self, + capacity: float, + getsizeof: Optional[Callable[[_V], float]] = None): + super().__init__(capacity, getsizeof) self.pinned_items = set[_K]() self.capacity = capacity self._hits = 0 self._total = 0 - def __contains__(self, key: _K) -> bool: - return key in self.cache - - def __len__(self) -> int: - return len(self.cache) - - def __getitem__(self, key: _K) -> _V: - value = self.cache[key] # Raise KeyError if not exists - self.cache.move_to_end(key) - return value - - def __setitem__(self, key: _K, value: _V) -> None: - self.put(key, value) - def __delitem__(self, key: _K) -> None: - self.pop(key) + run_on_remove = key in self + value = self.__getitem__(key) + super().__delitem__(key) + if key in self.pinned_items: + # Todo: add warning to inform that del pinned item + self._unpin(key) + if run_on_remove: + self._on_remove(key, value) + + @property + def cache(self) -> Mapping[_K, _V]: + """Return the internal cache dictionary in order (read-only).""" + return _MappingOrderCacheView( + self._Cache__data, # type: ignore + self.order) + + @property + def order(self) -> Mapping[_K, None]: + """Return the internal order dictionary (read-only).""" + return MappingProxyType(self._LRUCache__order) # type: ignore def stat(self) -> CacheInfo: return CacheInfo(hits=self._hits, total=self._total) def touch(self, key: _K) -> None: - self.cache.move_to_end(key) + self._LRUCache__update(key) # type: ignore - def get(self, key: _K, default: Optional[_V] = None) -> Optional[_V]: - value: Optional[_V] - if key in self.cache: - value = self.cache[key] - self.cache.move_to_end(key) + @overload + def get(self, key: _K, /) -> Optional[_V]: + ... + + @overload + def get(self, key: _K, /, default: Union[_V, _T]) -> Union[_V, _T]: + ... + + def get(self, + key: _K, + /, + default: Optional[Union[_V, + _T]] = None) -> Optional[Union[_V, _T]]: + value: Optional[Union[_V, _T]] + if key in self: + value = self.__getitem__(key) self._hits += 1 else: @@ -265,60 +298,76 @@ class LRUCache(Generic[_K, _V]): self._total += 1 return value + @overload + def pop(self, key: _K) -> _V: + ... + + @overload + def pop(self, key: _K, default: Union[_V, _T]) -> Union[_V, _T]: + ... + + def pop(self, + key: _K, + default: Optional[Union[_V, + _T]] = None) -> Optional[Union[_V, _T]]: + value: Optional[Union[_V, _T]] + if key not in self: + return default + + value = self[key] + del self[key] + return value + def put(self, key: _K, value: _V) -> None: - self.cache[key] = value - self.cache.move_to_end(key) - self._remove_old_if_needed() + self.__setitem__(key, value) def pin(self, key: _K) -> None: """ Pins a key in the cache preventing it from being evicted in the LRU order. """ - if key not in self.cache: + if key not in self: raise ValueError(f"Cannot pin key: {key} not in cache.") self.pinned_items.add(key) def _unpin(self, key: _K) -> None: + """ + Unpins a key in the cache allowing it to be + evicted in the LRU order. + """ self.pinned_items.remove(key) def _on_remove(self, key: _K, value: Optional[_V]) -> None: pass def remove_oldest(self, *, remove_pinned: bool = False) -> None: - if not self.cache: + if len(self) == 0: return + self.popitem(remove_pinned=remove_pinned) + + def _remove_old_if_needed(self) -> None: + while self.currsize > self.capacity: + self.remove_oldest() + + def clear(self) -> None: + while len(self) > 0: + self.remove_oldest(remove_pinned=True) + + def popitem(self, remove_pinned: bool = False): + """Remove and return the `(key, value)` pair least recently used.""" if not remove_pinned: # pop the oldest item in the cache that is not pinned lru_key = next( - (key for key in self.cache if key not in self.pinned_items), + (key for key in self.order if key not in self.pinned_items), ALL_PINNED_SENTINEL) if lru_key is ALL_PINNED_SENTINEL: raise RuntimeError("All items are pinned, " "cannot remove oldest from the cache.") else: - lru_key = next(iter(self.cache)) - self.pop(lru_key) # type: ignore - - def _remove_old_if_needed(self) -> None: - while len(self.cache) > self.capacity: - self.remove_oldest() - - def pop(self, key: _K, default: Optional[_V] = None) -> Optional[_V]: - run_on_remove = key in self.cache - value = self.cache.pop(key, default) - # remove from pinned items - if key in self.pinned_items: - self._unpin(key) - if run_on_remove: - self._on_remove(key, value) - return value - - def clear(self) -> None: - while len(self.cache) > 0: - self.remove_oldest(remove_pinned=True) - self.cache.clear() + lru_key = next(iter(self.order)) + value = self.pop(cast(_K, lru_key)) + return (lru_key, value) class PyObjectCache: From 43ed4143c4ec00f4b587c5bcefdb3b6520fbe966 Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Thu, 27 Mar 2025 02:47:25 -0400 Subject: [PATCH 039/593] [Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM (#15587) Signed-off-by: ElizaWszola Signed-off-by: ElizaWszola Signed-off-by: rshaw@neuralmagic.com Co-authored-by: ElizaWszola Co-authored-by: Lucas Wilkinson Co-authored-by: ElizaWszola --- vllm/model_executor/layers/fused_moe/layer.py | 26 ----- .../compressed_tensors_moe.py | 105 +++++++++++------- 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 750c5f731c7c6..ef33852e31621 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -885,32 +885,6 @@ class FusedMoE(torch.nn.Module): ] ] - def _load_fp8_scale(self, param: torch.nn.Parameter, - loaded_weight: torch.Tensor, weight_name: str, - shard_id: str, expert_id: int) -> None: - param_data = param.data - - # Input scales can be loaded directly and should be equal. - if "input_scale" in weight_name: - if param_data[expert_id] != 1 and (param_data[expert_id] - - loaded_weight).abs() > 1e-5: - raise ValueError( - "input_scales of w1 and w3 of a layer " - f"must be equal. But got {param_data[expert_id]} " - f"vs. {loaded_weight}") - param_data[expert_id] = loaded_weight - # Weight scales - elif "weight_scale" in weight_name: - # If we are in merged column case (gate_up_proj) - if shard_id in ("w1", "w3"): - # We have to keep the weight scales of w1 and w3 because - # we need to re-quantize w1/w3 weights after weight loading. - idx = 0 if shard_id == "w1" else 1 - param_data[expert_id][idx] = loaded_weight - # If we are in the row parallel case (down_proj) - else: - param_data[expert_id] = loaded_weight - def extra_repr(self) -> str: s = ( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py index 2e14845ff2d6f..bf32bee89e895 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py @@ -268,14 +268,23 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod): self.input_quant = self.quant_config.target_scheme_map["Linear"].get( "input_activations") - if not (self.weight_quant.strategy == QuantizationStrategy.TENSOR - and self.input_quant.strategy == QuantizationStrategy.TENSOR): + per_tensor = (self.weight_quant.strategy == QuantizationStrategy.TENSOR + and self.input_quant.strategy + == QuantizationStrategy.TENSOR) + per_channel = ( + self.weight_quant.strategy == QuantizationStrategy.CHANNEL + and self.input_quant.strategy == QuantizationStrategy.TOKEN) + if not (per_tensor or per_channel): raise ValueError( - "For FP8 Fused MoE layers, only per-tensor scales " - "for weights and activations are supported. Found " + "For FP8 Fused MoE layers, we require per tensor " + "or channelwise, dynamic per token quantization. Found " f"{self.weight_quant}, {self.input_quant}") self.static_input_scales = not self.input_quant.dynamic + if self.static_input_scales and per_channel: + raise ValueError( + "For FP8 Fused MoE layer, we require either per tensor or " + "channelwise, dynamic per token quantization.") def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: int, intermediate_size_per_partition: int, @@ -303,24 +312,40 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod): set_weight_attrs(w2_weight, extra_weight_attrs) # WEIGHT_SCALES - # Allocate 2 scales for w1 and w3 respectively. - # They will be combined to a single scale after weight loading. - w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts, - 2, - dtype=torch.float32), - requires_grad=False) - layer.register_parameter("w13_weight_scale", w13_weight_scale) + if self.weight_quant.strategy == QuantizationStrategy.TENSOR: + # Allocate 2 scales for w1 and w3 respectively. + # They are combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter(torch.ones( + num_experts, 2, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + w2_weight_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + # Add PER-TENSOR quantization for FusedMoE.weight_loader. + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) - w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts, - dtype=torch.float32), - requires_grad=False) - layer.register_parameter("w2_weight_scale", w2_weight_scale) - # Add the quantization method used (per tensor/grouped/channel) - # to ensure the weight scales are loaded in properly - extra_weight_attrs.update( - {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}) - set_weight_attrs(w13_weight_scale, extra_weight_attrs) - set_weight_attrs(w2_weight_scale, extra_weight_attrs) + elif self.weight_quant.strategy == QuantizationStrategy.CHANNEL: + w13_weight_scale = torch.nn.Parameter(torch.ones( + num_experts, + 2 * intermediate_size_per_partition, + 1, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + w2_weight_scale = torch.nn.Parameter(torch.ones( + num_experts, hidden_size, 1, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + # Add PER-CHANNEL quantization for FusedMoE.weight_loader. + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.CHANNEL.value}) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) # INPUT_SCALES if self.static_input_scales: @@ -362,6 +387,7 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod): # Fp8 moe kernels require a single activation scale. # We take the max of all the scales in case they differ. if self.static_input_scales: + assert self.input_quant.strategy == QuantizationStrategy.TENSOR if (layer.w13_input_scale is None or layer.w2_input_scale is None): raise ValueError( "QuantConfig has static quantization, but found " @@ -377,24 +403,25 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod): layer.w2_input_scale = torch.nn.Parameter( layer.w2_input_scale.max(), requires_grad=False) - # Fp8 moe kernel needs single weight scale for w13 per expert. - # We take the max then dequant and requant each expert. - assert layer.w13_weight_scale is not None - shard_size = layer.intermediate_size_per_partition - max_w13_scales = layer.w13_weight_scale.max(dim=1).values - for expert_id in range(layer.local_num_experts): - start = 0 - for shard_id in range(2): - dq_weight = per_tensor_dequantize( - layer.w13_weight[expert_id][start:start + shard_size, :], - layer.w13_weight_scale[expert_id][shard_id]) - layer.w13_weight[expert_id][ - start:start + shard_size, :], _ = ops.scaled_fp8_quant( - dq_weight, max_w13_scales[expert_id]) - start += shard_size - - layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, - requires_grad=False) + # For Per-TENSOR case, Fp8 moe kernel needs single weight scale + # for w13 per expert. Use max then dequant and requant each expert. + if self.weight_quant.strategy == QuantizationStrategy.TENSOR: + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.local_num_experts): + start = 0 + for shard_id in range(2): + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][start:start + + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id]) + layer.w13_weight[expert_id][ + start:start + shard_size, :], _ = ops.scaled_fp8_quant( + dq_weight, max_w13_scales[expert_id]) + start += shard_size + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, + requires_grad=False) def apply( self, From e6c9053f9ec0b41e9af41def67537a4a3097eeb5 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 27 Mar 2025 15:45:00 +0800 Subject: [PATCH 040/593] [Misc] Clean up `scatter_patch_features` (#15559) Signed-off-by: DarkLight1337 --- vllm/model_executor/models/gemma3_mm.py | 17 ++-- vllm/model_executor/models/internvl.py | 21 ++--- vllm/model_executor/models/llava.py | 22 +++-- vllm/model_executor/models/molmo.py | 105 ++++++++---------------- vllm/model_executor/models/pixtral.py | 18 ++-- vllm/model_executor/models/vision.py | 35 ++++---- 6 files changed, 82 insertions(+), 136 deletions(-) diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 63d3ccbf54bc2..9efb57b8c5aa1 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -30,7 +30,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor, # yapf: enable from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors -from vllm.utils import flatten_2d_lists from .interfaces import (MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal, SupportsPP) @@ -60,7 +59,7 @@ class Gemma3ImagePixelInputs(TypedDict): A boolean mask indicating which image embeddings correspond to patch tokens. - Shape: `(batch_size, num_images, num_embeds)` + Shape: `(batch_size * num_images, num_embeds)` """ @@ -593,6 +592,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, pixel_values = flatten_bn(pixel_values, concat=True) num_crops = flatten_bn(num_crops, concat=True) + embed_is_patch = flatten_bn(embed_is_patch) return Gemma3ImagePixelInputs( type="pixel_values", @@ -635,14 +635,10 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, image_features = self._process_image_input(image_input) - if kwargs.get("v0_path", False): - return image_features - - return flatten_2d_lists( - scatter_patch_features(*args) for args in zip( - image_features, - image_input["embed_is_patch"], - )) + return scatter_patch_features( + image_features, + image_input["embed_is_patch"], + ) def get_input_embeddings( self, @@ -671,7 +667,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, # NOTE: In v1, inputs_embeds is always generated at model runner, this # condition is for v0 compatibility. elif inputs_embeds is None: - kwargs.update({"v0_path": True}) vision_embeddings = self.get_multimodal_embeddings(**kwargs) inputs_embeds = self.get_input_embeddings(input_ids, diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index e1aa371610353..0729f4c7d203c 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -35,7 +35,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor, from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors from vllm.transformers_utils.tokenizer import AnyTokenizer -from vllm.utils import flatten_2d_lists from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model, @@ -66,13 +65,13 @@ class InternVLImagePixelInputs(TypedDict): A boolean mask indicating which image embeddings correspond to patch tokens. - Shape: `(batch_size, num_images, num_embeds)` + Shape: `(batch_size * num_images, num_embeds)` """ class InternVLImageEmbeddingInputs(TypedDict): type: Literal["image_embeds"] - data: NestedTensors + data: Union[torch.Tensor, list[torch.Tensor]] """ A tensor of shape `(num_images, total_image_feature_size, hidden_size)` or a list of tensors of shape `(total_image_feature_size, hidden_size)` @@ -867,6 +866,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP): pixel_values_flat = flatten_bn(pixel_values_flat, concat=True) image_num_patches = flatten_bn(image_num_patches, concat=True) + embed_is_patch = flatten_bn(embed_is_patch) return InternVLImagePixelInputs( type="pixel_values", @@ -881,7 +881,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP): def _process_image_input( self, image_input: InternVLImageInputs, - ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]: + ) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor, ...]]: if image_input["type"] == "image_embeds": return image_input["data"] @@ -921,15 +921,13 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP): image_features = self._process_image_input(image_input) - if (kwargs.get("v0_path", False) - or image_input["type"] != "pixel_values"): + if image_input["type"] != "pixel_values": return image_features - return flatten_2d_lists( - scatter_patch_features(*args) for args in zip( - image_features, - image_input["embed_is_patch"], - )) + return scatter_patch_features( + image_features, + image_input["embed_is_patch"], + ) def get_input_embeddings( self, @@ -964,7 +962,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP): # NOTE: In v1, inputs_embeds is always generated at model runner, this # condition is for v0 compatibility. elif inputs_embeds is None: - kwargs.update({"v0_path": True}) vision_embeddings = self.get_multimodal_embeddings(**kwargs) inputs_embeds = self.get_input_embeddings(input_ids, vision_embeddings) diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index d1014067d9d7c..826f04b37547b 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -35,7 +35,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor, PromptReplacement, PromptUpdate) from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors -from vllm.utils import flatten_2d_lists from .clip import CLIPVisionModel from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP @@ -73,7 +72,7 @@ class PixtralHFImagePixelInputs(TypedDict): A boolean mask indicating which image embeddings correspond to patch tokens. - Shape: `(batch_size, num_images, num_embeds)` + Shape: `(batch_size * num_images, num_embeds)` """ @@ -618,6 +617,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): raise ValueError("Incorrect type of embed_is_patch. " f"Got type: {type(embed_is_patch)}") + embed_is_patch = flatten_bn(embed_is_patch) + return PixtralHFImagePixelInputs( type="pixel_values_pixtral", pixel_values=flatten_bn(pixel_values), @@ -713,18 +714,16 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): if image_input is None: return None - vision_embeddings = self._process_image_input(image_input) + image_features = self._process_image_input(image_input) - if (kwargs.get("v0_path", False) - or image_input["type"] != "pixel_values_pixtral"): + if image_input["type"] != "pixel_values_pixtral": # The path is used for pixtral (V0 only) and llava (V0/V1) - return vision_embeddings + return image_features - return flatten_2d_lists( - scatter_patch_features(*args) for args in zip( - vision_embeddings, - image_input["embed_is_patch"], - )) + return scatter_patch_features( + image_features, + image_input["embed_is_patch"], + ) def get_input_embeddings( self, @@ -790,7 +789,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): # NOTE: In v1, inputs_embeds is always generated at model runner, this # condition is for v0 compatibility. elif inputs_embeds is None: - kwargs.update({"v0_path": True}) vision_embeddings = self.get_multimodal_embeddings(**kwargs) inputs_embeds = self.get_input_embeddings(input_ids, vision_embeddings) diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 146d48e522119..9224687d8a5d3 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -49,7 +49,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor, PromptInsertion, PromptUpdate) from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors -from vllm.utils import flatten_2d_lists from .interfaces import (MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsQuant) @@ -72,17 +71,17 @@ POOLING_SIZE = 2 class MolmoImageInputs(TypedDict): images: Union[torch.Tensor, list[torch.Tensor]] - """Shape: `(batch_size, num_crops, num_patch, patch_dim)`""" + """Shape: `(batch_size * num_images, num_crops, num_patch, patch_dim)`""" image_masks: Optional[Union[torch.Tensor, list[torch.Tensor]]] - """Shape: `(batch_size, num_crops, num_patch)`""" + """Shape: `(batch_size * num_images, num_crops, num_patch)`""" feat_is_patch: Union[torch.Tensor, list[torch.Tensor]] """ A boolean mask indicating which image features correspond to patch tokens. - Shape: `(batch_size, num_crops, num_patch)` + Shape: `(batch_size * num_images, num_crops, num_patch)` """ embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] @@ -90,7 +89,7 @@ class MolmoImageInputs(TypedDict): A boolean mask indicating which image embeddings correspond to patch tokens. - Shape: `(batch_size, num_embeds)` + Shape: `(batch_size * num_images, num_embeds)` """ num_crops: Union[torch.Tensor, list[torch.Tensor]] @@ -696,9 +695,10 @@ class MolmoVisionBackbone(nn.Module, SupportsQuant): return image_features def forward( - self, images: torch.Tensor, image_masks: torch.Tensor - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - + self, + images: torch.Tensor, + image_masks: torch.Tensor, + ) -> torch.Tensor: # image_features: (batch_size, num_crops(=num_image), num_patch, nximage_emb_dim) # noqa: E501 batch_size, num_image = images.shape[:2] images = images.to(device=self.device, dtype=self.dtype) @@ -1491,6 +1491,8 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, f"Got type: {type(img_patch_id)}") self.img_patch_id = img_patch_id.flatten().unique().item() + embed_is_patch = flatten_bn(embed_is_patch) + return MolmoImageInputs( images=images, image_masks=image_masks, @@ -1502,13 +1504,17 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, def _process_image_input( self, image_input: MolmoImageInputs, - ) -> Union[torch.Tensor, list[torch.Tensor]]: - if isinstance(image_input["images"], list): + ) -> list[torch.Tensor]: + images = image_input["images"] + image_masks = image_input["image_masks"] + feat_is_patch = image_input["feat_is_patch"] + num_crops = image_input["num_crops"] + + if isinstance(images, list): # Call the vision backbone on the whole batch at once - images_flat = flatten_bn(image_input["images"], concat=True) - image_masks_flat = (None if (image_masks := - image_input["image_masks"]) is None - else flatten_bn(image_masks, concat=True)) + images_flat = flatten_bn(images, concat=True) + image_masks_flat = (None if image_masks is None else flatten_bn( + image_masks, concat=True)) image_features_flat = self.vision_backbone( images=images_flat.unsqueeze(0), @@ -1517,63 +1523,19 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, ).squeeze(0) # Reconstruct the batch dimension - image_features = image_features_flat.split( - image_input["num_crops"].sum(-1).tolist()) + num_crops_per_image = [nc.sum().item() for nc in num_crops] + image_features = image_features_flat.split(num_crops_per_image) else: image_features = self.vision_backbone( - images=image_input["images"], - image_masks=image_input["image_masks"], + images=images, + image_masks=image_masks, ) - return image_features - - def _get_mm_embeds( - self, - features: torch.Tensor, # Shape: (num_crop, num_patch, d) - feat_is_patch: torch.Tensor, # Shape: (num_crop, num_patch) - num_crops: torch.Tensor, # Shape: (num_images,) - embed_is_patch: torch.Tensor, # Shape: (num_embeds,) - ) -> tuple[torch.Tensor, ...]: - """ - Scatter the patch features into a contiguous tensor that corresponds - to the embedding tokens defined by the multimodal processor. - - Note: - The original code only considers patch tokens as feature - tokens, but our processor considers all image-related tokens - as feature tokens because the feature tokens need to be - consecutive in `input_ids`. - - Example: - A simplified example for one item in the batch: - - .. code-block:: - - Embedding tokens (from HF processor): - [ ] - - embed_is_patch (from HF processor): - [ False True True False True True False False ] - - Encoder outputs (from model): - [ p1 p2 0 p3 p4 0 ] - - feat_is_patch (from HF processor): - [ True True False True True False ] - - The resulting embedding tensor is: - [ nan p1 p2 nan p3 p4 nan nan ] - """ - num_crops_per_image = num_crops.tolist() - feats_per_image = features.split(num_crops_per_image) - f_is_patch_per_image = feat_is_patch.split(num_crops_per_image) - - features = torch.cat([ + # Only the features corresponding to patch tokens are relevant + return [ feats[f_is_patch] - for feats, f_is_patch in zip(feats_per_image, f_is_patch_per_image) - ]) - - return scatter_patch_features(features, embed_is_patch) + for feats, f_is_patch in zip(image_features, feat_is_patch) + ] def get_multimodal_embeddings( self, **kwargs: object) -> Optional[MultiModalEmbeddings]: @@ -1583,13 +1545,10 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, image_features = self._process_image_input(image_input) - return flatten_2d_lists( - self._get_mm_embeds(*args) for args in zip( - image_features, - image_input["feat_is_patch"], - image_input["num_crops"], - image_input["embed_is_patch"], - )) + return scatter_patch_features( + image_features, + image_input["embed_is_patch"], + ) def get_input_embeddings( self, diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index a3ad360961243..da2017c987d4f 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -42,7 +42,6 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors from vllm.transformers_utils.tokenizer import (MistralTokenizer, cached_tokenizer_from_config) -from vllm.utils import flatten_2d_lists from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP from .utils import (flatten_bn, init_vllm_registered_model, maybe_prefix, @@ -74,7 +73,7 @@ class PixtralImagePixelInputs(TypedDict): A boolean mask indicating which image embeddings correspond to patch tokens. - Shape: `(batch_size, num_images, num_embeds)` + Shape: `(batch_size * num_images, num_embeds)` """ @@ -387,6 +386,8 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, raise ValueError("Incorrect type of embed_is_patch. " f"Got type: {type(embed_is_patch)}") + embed_is_patch = flatten_bn(embed_is_patch) + return PixtralImagePixelInputs( type="pixel_values", images=flatten_bn(images), @@ -428,14 +429,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, image_features = self._process_image_input(image_input) - if kwargs.get("v0_path", False): - return image_features - - return flatten_2d_lists( - scatter_patch_features(*args) for args in zip( - image_features, - image_input["embed_is_patch"], - )) + return scatter_patch_features( + image_features, + image_input["embed_is_patch"], + ) def get_input_embeddings( self, @@ -467,7 +464,6 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, # NOTE: In v1, inputs_embeds is always generated at model runner, this # condition is for v0 compatibility. elif inputs_embeds is None: - kwargs.update({"v0_path": True}) vision_embeddings = self.get_multimodal_embeddings(**kwargs) inputs_embeds = self.get_input_embeddings(input_ids, vision_embeddings) diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py index c91459398308e..db069f8de2a35 100644 --- a/vllm/model_executor/models/vision.py +++ b/vllm/model_executor/models/vision.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 from abc import ABC, abstractmethod +from collections.abc import Sequence from typing import Final, Generic, Optional, Protocol, TypeVar, Union, cast import torch @@ -154,8 +155,8 @@ def resolve_visual_encoder_outputs( def scatter_patch_features( - features: torch.Tensor, - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]], + patches: Union[torch.Tensor, Sequence[torch.Tensor]], + embed_is_patch: Union[torch.Tensor, Sequence[torch.Tensor]], ) -> tuple[torch.Tensor, ...]: """ Scatter the patch features into a contiguous tensor that corresponds @@ -165,8 +166,8 @@ def scatter_patch_features( can be filtered out by :func`select_patch_features`. Args: - features: The patch features, concatenated across each image. - Shape: `(num_patch, feature_depth)` + patches: The patch features for each image. + Shape: `(num_images, , feature_depth)` embed_is_patch: A boolean mask indicating which image embeddings correspond to patch tokens for each image. Shape: `(num_images, num_embeds)` @@ -194,21 +195,21 @@ def scatter_patch_features( The resulting embedding tensor is: [ nan p1 p2 nan p3 p4 nan nan ] """ - num_embeds_per_image = [ - e_is_patch.numel() for e_is_patch in embed_is_patch - ] - if isinstance(embed_is_patch, torch.Tensor): - embed_is_patch_flat = embed_is_patch.view(-1) - else: - embed_is_patch_flat = torch.cat(embed_is_patch) + if len(patches) != len(embed_is_patch): + raise ValueError(f"Inconsistent num_images: {len(patches)=} vs. " + f"{len(embed_is_patch)=}") - embeds_flat = features.new_full( - (sum(num_embeds_per_image), features.shape[-1]), - fill_value=torch.nan, - ) - embeds_flat[embed_is_patch_flat] = features.flatten(0, -2) + def get_embed_one(patches_one: torch.Tensor, e_is_patch: torch.Tensor): + embed_one = patches_one.new_full( + (e_is_patch.shape[0], patches_one.shape[-1]), + fill_value=torch.nan, + ) + embed_one[e_is_patch] = patches_one.flatten(0, -2) + return embed_one - return embeds_flat.split(num_embeds_per_image) + return tuple( + get_embed_one(patches_one, e_is_patch) + for patches_one, e_is_patch in zip(patches, embed_is_patch)) def select_patch_features( From 3f532cb6a69e51a6578b85642fcba34ac348f7a4 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Thu, 27 Mar 2025 17:21:23 +0800 Subject: [PATCH 041/593] [Misc] Use model_redirect to redirect the model name to a local folder. (#14116) --- vllm/config.py | 10 ++++++--- vllm/envs.py | 5 +++++ vllm/transformers_utils/utils.py | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/vllm/config.py b/vllm/config.py index 62800afc3e699..687c8b56ec126 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -38,7 +38,7 @@ from vllm.transformers_utils.config import ( get_sentence_transformer_tokenizer_config, is_encoder_decoder, try_get_generation_config, uses_mrope) from vllm.transformers_utils.s3_utils import S3Model -from vllm.transformers_utils.utils import is_s3 +from vllm.transformers_utils.utils import is_s3, maybe_model_redirect from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless, get_cpu_memory, random_uuid, resolve_obj_by_qualname) @@ -266,9 +266,13 @@ class ModelConfig: override_generation_config: Optional[dict[str, Any]] = None, model_impl: Union[str, ModelImpl] = ModelImpl.AUTO, ) -> None: - self.model = model + self.model = maybe_model_redirect(model) + self.tokenizer = maybe_model_redirect(tokenizer) + self.hf_config_path = hf_config_path - self.tokenizer = tokenizer + if isinstance(hf_config_path, str): + self.hf_config_path = maybe_model_redirect(hf_config_path) + self.tokenizer_mode = tokenizer_mode self.trust_remote_code = trust_remote_code self.allowed_local_media_path = allowed_local_media_path diff --git a/vllm/envs.py b/vllm/envs.py index e16753191c6e2..23c304f124d36 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: S3_ACCESS_KEY_ID: Optional[str] = None S3_SECRET_ACCESS_KEY: Optional[str] = None S3_ENDPOINT_URL: Optional[str] = None + VLLM_MODEL_REDIRECT_PATH: Optional[str] = None VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm") VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm") VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai" @@ -635,6 +636,10 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_CI_USE_S3": lambda: os.environ.get("VLLM_CI_USE_S3", "0") == "1", + # Use model_redirect to redirect the model name to a local folder. + "VLLM_MODEL_REDIRECT_PATH": + lambda: os.environ.get("VLLM_MODEL_REDIRECT_PATH", None), + # Whether to use atomicAdd reduce in gptq/awq marlin kernel. "VLLM_MARLIN_USE_ATOMIC_ADD": lambda: os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1", diff --git a/vllm/transformers_utils/utils.py b/vllm/transformers_utils/utils.py index 87e446f894384..bae487b75588e 100644 --- a/vllm/transformers_utils/utils.py +++ b/vllm/transformers_utils/utils.py @@ -1,9 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 +from functools import cache from os import PathLike from pathlib import Path from typing import List, Optional, Union +from vllm.envs import VLLM_MODEL_REDIRECT_PATH +from vllm.logger import init_logger + +logger = init_logger(__name__) + def is_s3(model_or_path: str) -> bool: return model_or_path.lower().startswith('s3://') @@ -38,3 +44,35 @@ def modelscope_list_repo_files( if file['Type'] == 'blob' ] return files + + +@cache +def maybe_model_redirect(model: str) -> str: + """ + Use model_redirect to redirect the model name to a local folder. + + :param model: hf model name + :return: maybe redirect to a local folder + """ + + model_redirect_path = VLLM_MODEL_REDIRECT_PATH + + if not model_redirect_path: + return model + + if not Path(model_redirect_path).exists(): + return model + + with open(model_redirect_path) as f: + for line in f.readlines(): + try: + model_name, redirect_name = line.split("\t") + if model == model_name: + redirect_name = redirect_name.strip() + logger.info("model redirect: [ %s ] -> [ %s ]", model, + redirect_name) + return redirect_name + except Exception: + pass + + return model From 6278bc829eb6214f3375cc50347d58dbae81bc31 Mon Sep 17 00:00:00 2001 From: Richard Zou Date: Thu, 27 Mar 2025 06:33:41 -0400 Subject: [PATCH 042/593] Fix incorrect filenames in vllm_compile_cache.py (#15494) Signed-off-by: Signed-off-by: youkaichao Co-authored-by: youkaichao --- vllm/compilation/compiler_interface.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/vllm/compilation/compiler_interface.py b/vllm/compilation/compiler_interface.py index 571e2b832e95f..ab0f98bdaa3e5 100644 --- a/vllm/compilation/compiler_interface.py +++ b/vllm/compilation/compiler_interface.py @@ -229,7 +229,20 @@ class InductorAdaptor(CompilerInterface): inductor_compiled_graph = output if inductor_compiled_graph is not None: nonlocal file_path - file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa + compiled_fn = inductor_compiled_graph.current_callable + file_path = compiled_fn.__code__.co_filename # noqa + if not file_path.startswith(self.cache_dir): + # hooked in the align_inputs_from_check_idxs function + # in torch/_inductor/utils.py + for cell in compiled_fn.__closure__: + if not callable(cell.cell_contents): + continue + code = cell.cell_contents.__code__ + if code.co_filename.startswith(self.cache_dir): + # this is the real file path + # compiled from Inductor + file_path = code.co_filename + break hash_str = inductor_compiled_graph._fx_graph_cache_key return output From 8063dfc61a0cbb348d4b1baf4b6e03e8ebae7cfa Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Thu, 27 Mar 2025 20:38:46 +0800 Subject: [PATCH 043/593] [Doc] update --system for transformers installation in docker doc (#15616) Signed-off-by: reidliu41 Co-authored-by: reidliu41 --- docs/source/deployment/docker.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/deployment/docker.md b/docs/source/deployment/docker.md index 1f60faf40879e..65cb038de1b4e 100644 --- a/docs/source/deployment/docker.md +++ b/docs/source/deployment/docker.md @@ -34,11 +34,11 @@ If you need to use those dependencies (having accepted the license terms), create a custom Dockerfile on top of the base image with an extra layer that installs them: ```Dockerfile -FROM vllm/vllm-openai:v0.8.0 +FROM vllm/vllm-openai:v0.8.2 # e.g. install the `audio` and `video` optional dependencies # NOTE: Make sure the version of vLLM matches the base image! -RUN uv pip install vllm[audio,video]==0.8.0 +RUN uv pip install --system vllm[audio,video]==0.8.2 ``` ::: @@ -52,7 +52,7 @@ with an extra layer that installs their code from source: ```Dockerfile FROM vllm/vllm-openai:latest -RUN uv pip install git+https://github.com/huggingface/transformers.git +RUN uv pip install --system git+https://github.com/huggingface/transformers.git ``` ::: From ac5bc615b0adac4038e5574b446c8ac64c241caf Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 27 Mar 2025 21:07:29 +0800 Subject: [PATCH 044/593] [Model] MiniCPM-V/O supports V1 (#15487) Signed-off-by: DarkLight1337 --- docs/source/models/supported_models.md | 4 +- vllm/model_executor/models/minicpmo.py | 427 +++++++-------- vllm/model_executor/models/minicpmv.py | 696 ++++++++++++------------- vllm/model_executor/models/molmo.py | 40 +- 4 files changed, 573 insertions(+), 594 deletions(-) diff --git a/docs/source/models/supported_models.md b/docs/source/models/supported_models.md index 8ff18a17d36c3..793831fd06ded 100644 --- a/docs/source/models/supported_models.md +++ b/docs/source/models/supported_models.md @@ -836,14 +836,14 @@ See [this page](#generative-models) for more information on how to use generativ * `openbmb/MiniCPM-o-2_6`, etc. * ✅︎ * ✅︎ - * + * ✅︎ - * `MiniCPMV` * MiniCPM-V * T + IE+ + VE+ * `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, etc. * ✅︎ * ✅︎ - * + * ✅︎ - * `MllamaForConditionalGeneration` * Llama 3.2 * T + I+ diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index 1312b1051732f..ea37de0b806ab 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -23,8 +23,8 @@ # limitations under the License. """Inference-only MiniCPM-O model compatible with HuggingFace weights.""" from collections.abc import Iterable, Mapping, Sequence -from typing import (Any, Callable, Dict, Literal, Optional, Set, Tuple, - TypedDict, Union) +from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict, + Union) import torch from torch import nn @@ -42,8 +42,6 @@ from vllm.multimodal.parse import (AudioItem, AudioProcessorItems, MultiModalDataParser) from vllm.multimodal.processing import PromptReplacement, PromptUpdate from vllm.multimodal.profiling import ProcessorInputs -from vllm.sequence import IntermediateTensors -from vllm.utils import flatten_2d_lists from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder, MiniCPMVMultiModalDataParser, @@ -51,13 +49,14 @@ from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder, _minicpmv_field_config) from .utils import (AutoWeightsLoader, cast_overflow_tensors, flatten_bn, maybe_prefix) +from .vision import scatter_patch_features CPU_DEVICE = torch.device("cpu") class MiniCPMOAudioFeatureInputs(TypedDict): type: Literal["audio_features"] - audio_features: torch.Tensor + audio_features: Union[torch.Tensor, list[torch.Tensor]] """ Shape: `(batch_size * num_audios * num_slices, num_channels, length)` Slice here means chunk. Audio that is too long will be split into slices, @@ -65,37 +64,40 @@ class MiniCPMOAudioFeatureInputs(TypedDict): Padding is used therefore `audio_features` is `torch.Tensor`. """ - audio_feature_lens: torch.Tensor + audio_feature_lens: Union[torch.Tensor, list[torch.Tensor]] """ - Shape: `(batch_size * num_audios * num_slices)` + Shape: `(batch_size * num_audios, num_slices)` This should be feature length of each audio slice, which equals to `audio_features.shape[-1]` """ - audio_bounds: torch.Tensor + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] """ - Shape: `(batch_size * num_audios * num_slices, 2)` + A boolean mask indicating which audio embeddings correspond + to patch tokens. - This should be in `(start, stop)` format. + Shape: `(batch_size * num_audios, num_embeds)` """ class MiniCPMOAudioEmbeddingInputs(TypedDict): type: Literal["audio_embeds"] - audio_embeds: torch.Tensor + audio_embeds: Union[torch.Tensor, list[torch.Tensor]] """ - Shape: `(batch_size * num_images * num_slices, hidden_size)` + Shape: `(batch_size * num_audios, num_slices, hidden_size)` `hidden_size` must match the hidden size of language model backbone. instead of a batched tensor. Length of each slice may vary, so pass it as a list. """ - audio_bounds: torch.Tensor - """ - Shape: `(batch_size * num_audios * num_slices, 2)` - This should be in `(start, stop)` format. + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] + """ + A boolean mask indicating which audio embeddings correspond + to patch tokens. + + Shape: `(batch_size * num_audios, num_embeds)` """ @@ -104,11 +106,16 @@ MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs, def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]): + audio_features = hf_inputs.get("audio_features", torch.empty(0)) + num_audios = len(audio_features) + return dict( **_minicpmv_field_config(hf_inputs), audio_features=MultiModalFieldConfig.batched("audio"), audio_feature_lens=MultiModalFieldConfig.batched("audio"), audio_embeds=MultiModalFieldConfig.batched("audio"), + audio_embed_is_patch=MultiModalFieldConfig.batched("audio"), + audio_token_id=MultiModalFieldConfig.shared("audio", num_audios), ) @@ -149,7 +156,7 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo): audio_pattern = "()" def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]: - return {"image": None, "video": None, "audio": None} + return {**super().get_supported_mm_limits(), "audio": None} def get_mm_max_tokens_per_item( self, @@ -157,11 +164,25 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo): mm_counts: Mapping[str, int], ) -> Mapping[str, int]: return { - "image": self.get_max_image_tokens(), - "audio": self.get_max_audio_tokens(), - "video": self.get_max_video_tokens(seq_len), + **super().get_mm_max_tokens_per_item(seq_len, mm_counts), + "audio": + self.get_max_audio_tokens(), } + def get_audio_placeholder( + self, + audio_lens: int, + chunk_input: bool = True, + chunk_length: int = 1, + ) -> str: + hf_processor = self.get_hf_processor() + + return hf_processor.get_audio_placeholder( + audio_lens, + chunk_input=chunk_input, + chunk_length=chunk_length, + ) + def get_default_audio_pool_step(self) -> int: return 2 @@ -197,12 +218,8 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo): max_videos = mm_config.get_limit_per_prompt("video") max_audios = mm_config.get_limit_per_prompt("audio") - # count tokens - # which are not in get_max_image_tokens - max_image_tokens = self.get_max_image_tokens( - ) * max_images + 4 * max_images - max_audio_tokens = self.get_max_audio_tokens( - ) * max_audios + 2 * max_audios + max_image_tokens = self.get_max_image_tokens() * max_images + max_audio_tokens = self.get_max_audio_tokens() * max_audios max_total_frames = self.get_max_video_frames(seq_len - max_image_tokens - max_audio_tokens) @@ -224,20 +241,20 @@ class MiniCPMODummyInputsBuilder( processor_inputs = super().get_dummy_processor_inputs( seq_len, mm_counts) - mm_data = { - "image": - processor_inputs.mm_data["image"], - "video": - processor_inputs.mm_data["video"], + + audio_prompt_texts = self.info.audio_pattern * num_audios + audio_mm_data = { "audio": self._get_dummy_audios(length=audio_len, num_audios=num_audios) } - audio_prompt_texts = self.info.audio_pattern * num_audios - - return ProcessorInputs(prompt_text=processor_inputs.prompt_text + \ - audio_prompt_texts, - mm_data=mm_data) + return ProcessorInputs( + prompt_text=processor_inputs.prompt_text + audio_prompt_texts, + mm_data={ + **processor_inputs.mm_data, + **audio_mm_data, + }, + ) class MiniCPMOMultiModalProcessor( @@ -247,22 +264,17 @@ class MiniCPMOMultiModalProcessor( return MiniCPMOMultiModalDataParser( target_sr=self.info.get_default_audio_sampling_rate()) - def get_audio_prompt_texts(self, - audio_lens: int, - chunk_input: bool = True, - chunk_length: int = 1) -> str: - return self.info.get_hf_processor().get_audio_placeholder( - audio_lens, chunk_input, chunk_length) - - def get_special_tokens(self) -> Dict[str, torch.Tensor]: - tokenizer = self.info.get_tokenizer() - special_tokens = super().get_special_tokens() - if hasattr(tokenizer, "audio_start_id"): - special_tokens["audio_start_id"] = torch.tensor( - tokenizer.audio_start_id) - special_tokens["audio_end_id"] = torch.tensor( - tokenizer.audio_end_id) - return special_tokens + def get_audio_prompt_texts( + self, + audio_lens: int, + chunk_input: bool = True, + chunk_length: int = 1, + ) -> str: + return self.info.get_audio_placeholder( + audio_lens, + chunk_input=chunk_input, + chunk_length=chunk_length, + ) def process_audios( self, @@ -274,32 +286,65 @@ class MiniCPMOMultiModalProcessor( parsed_audios = (self._get_data_parser().parse_mm_data({ "audio": audios - }).get_items("audio", AudioProcessorItems)) + }).get_items("audio", + (MiniCPMOAudioEmbeddingItems, AudioProcessorItems))) - audio_inputs = self._base_call_hf_processor( - prompts=[self.info.audio_pattern] * len(parsed_audios), - mm_data={"audios": [[audio] for audio in parsed_audios]}, - mm_kwargs={ - **mm_kwargs, "chunk_input": True - }, - out_keys={"audio_features", "audio_feature_lens"}, - ) + if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems): + audio_inputs = {} - # Avoid padding since we need the output for each audio to be - # independent of other audios for the cache to work correctly - unpadded_audio_features = [ - feat[:, :feature_len] for feat, feature_len in zip( - audio_inputs["audio_features"], - audio_inputs["audio_feature_lens"], + audio_lens = [ + self.info.get_audio_len_by_num_chunks( + sum(map(len, + parsed_audios.get(i)["audio_embeds"]))) + for i in range(len(parsed_audios)) + ] + else: + audio_inputs = self._base_call_hf_processor( + prompts=[self.info.audio_pattern] * len(parsed_audios), + mm_data={"audios": [[audio] for audio in parsed_audios]}, + mm_kwargs={ + **mm_kwargs, + "chunk_input": True, + }, + out_keys={"audio_features", "audio_feature_lens"}, ) + + # Avoid padding since we need the output for each audio to be + # independent of other audios for the cache to work correctly + unpadded_audio_features = [ + feat[:, :feature_len] for feat, feature_len in zip( + audio_inputs["audio_features"], + audio_inputs["audio_feature_lens"], + ) + ] + audio_inputs["audio_features"] = unpadded_audio_features + + audio_lens = [ + parsed_audios.get_audio_length(i) + for i in range(len(parsed_audios)) + ] + + audio_repl_features = [ + self.get_audio_prompt_texts(audio_len) for audio_len in audio_lens ] - audio_inputs["audio_features"] = unpadded_audio_features + + tokenizer = self.info.get_tokenizer() + audio_repls_feature_tokens = [ + tokenizer.encode(audio_repl, add_special_tokens=False) + for audio_repl in audio_repl_features + ] + + embed_is_patch = [ + self.get_embed_is_patch(audio_repl_tokens) + for audio_repl_tokens in audio_repls_feature_tokens + ] + audio_inputs["audio_embed_is_patch"] = embed_is_patch + + unk_token_id = tokenizer.get_vocab()[""] + audio_inputs["audio_token_id"] = torch.tensor(unk_token_id) return audio_inputs - def get_placeholder_match_pattern(self) -> str: - return r"\(<(image|video|audio)>./\)" - def process_mm_inputs( self, mm_data: Mapping[str, object], @@ -331,8 +376,7 @@ class MiniCPMOMultiModalProcessor( if isinstance(audios, MiniCPMOAudioEmbeddingItems): single_audio_embeds = audios.get(item_idx)["audio_embeds"] audio_len = self.info.get_audio_len_by_num_chunks( - sum(chunk_embeds.shape[0] - for chunk_embeds in single_audio_embeds)) + sum(map(len, single_audio_embeds))) else: audio_len = audios.get_audio_length(item_idx) @@ -514,6 +558,8 @@ class MiniCPMO(MiniCPMV2_6): self.apm = self.init_audio_module(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "apm")) + self.audio_token_id = None + def init_audio_module(self, *, vllm_config: VllmConfig, prefix: str = ""): # Do not use parameters temporarily audio_config = self.config.audio_config @@ -563,18 +609,30 @@ class MiniCPMO(MiniCPMV2_6): return input_lengths_after_cnn, input_lengths_after_pooling - # Copied from HF repo of MiniCPM-o-2_6, - # designed for batched inputs and outputs - def get_audio_hidden_states(self, data: MiniCPMOAudioInputs, - chunk_length: int) -> list[torch.Tensor]: - wavforms = data.get( - "audio_features", - []) # (bs, 80, frames) or [], multi audios need filled in advance - audio_feature_lens_raw = [data.get("audio_feature_lens", - [])] # list, [[x1, x2], [y1], [z1]] + def get_audio_hidden_states( + self, data: MiniCPMOAudioFeatureInputs) -> list[torch.Tensor]: + chunk_length = self.config.audio_chunk_length - if len(wavforms) == 0: - return [] + # (bs, 80, frames) or [], multi audios need filled in advance + wavforms_raw = data["audio_features"] + if isinstance(wavforms_raw, list): + B = len(wavforms_raw) + C = wavforms_raw[0].shape[-2] + L = max(item.shape[-1] for item in wavforms_raw) + device = wavforms_raw[0].device + dtype = wavforms_raw[0].dtype + + wavforms = torch.zeros((B, C, L), dtype=dtype, device=device) + for i, wavforms_item in enumerate(wavforms_raw): + L_item = wavforms_item.shape[-1] + wavforms[i, ..., :L_item] = wavforms_item + else: + wavforms = wavforms_raw + + # list, [[x1, x2], [y1], [z1]] + audio_feature_lens_raw = data["audio_feature_lens"] + if isinstance(audio_feature_lens_raw, torch.Tensor): + audio_feature_lens_raw = audio_feature_lens_raw.unbind(0) audio_feature_lens = torch.hstack(audio_feature_lens_raw) batch_size, _, max_mel_seq_len = wavforms.shape @@ -625,159 +683,104 @@ class MiniCPMO(MiniCPMV2_6): num_audio_tokens = feature_lens_after_pooling - final_audio_embeds = [] + final_audio_embeds = list[torch.Tensor]() idx = 0 for i in range(len(audio_feature_lens_raw)): - target_audio_embeds = [] + target_audio_embeds_lst = list[torch.Tensor]() for _ in range(len(audio_feature_lens_raw[i])): - target_audio_embeds.append( + target_audio_embeds_lst.append( audio_embeds[idx, :num_audio_tokens[idx], :]) idx += 1 - final_audio_embeds.append(target_audio_embeds) + + final_audio_embeds.append(torch.cat(target_audio_embeds_lst)) + return final_audio_embeds - def get_embedding_with_audios(self, vlm_embedding: torch.Tensor, - audio_inputs: MiniCPMOAudioInputs, - chunk_length: int) -> torch.Tensor: - device, dtype = vlm_embedding.device, vlm_embedding.dtype - if audio_inputs["type"] == "audio_embeds": - audio_embeddings = [ - item.to(device=device, dtype=dtype) - for item in audio_inputs["audio_embeds"] - ] - else: - audio_embeddings = self.get_audio_hidden_states( - audio_inputs, chunk_length)[0] - if audio_embeddings is None or len(audio_embeddings) == 0: - return vlm_embedding - audio_bounds = audio_inputs["audio_bounds"] - if self.config.chunk_input: - audio_embs = torch.cat(audio_embeddings, dim=0).to(device=device, - dtype=dtype) - audio_start_pos = 0 - for bound in audio_bounds: - audio_len = bound[1] - bound[0] - vlm_embedding[bound[0]:bound[1]] = audio_embs[ - audio_start_pos:audio_start_pos + audio_len, :] - audio_start_pos += audio_len - else: - for embs, bound in zip(audio_embeddings, audio_bounds): - audio_indices = torch.arange(bound[0], - bound[1], - dtype=torch.long).to(device) - - if embs.shape[0] != len(audio_indices): - raise ValueError( - "Shape mismatch: Trying to assign embeddings " - f"of shape {embs.shape} " - f"to input indices of length {len(audio_indices)}") - vlm_embedding[audio_indices] = embs.to(dtype) - return vlm_embedding - - def _get_audio_bounds(self, input_ids: torch.Tensor, - audio_start_id: torch.Tensor, - audio_end_id: torch.Tensor) -> torch.Tensor: - audio_start_tokens, = torch.where(input_ids == audio_start_id[0]) - audio_start_tokens += 1 - audio_end_tokens, = torch.where(input_ids == audio_end_id[0]) - valid_audio_nums = max(len(audio_start_tokens), len(audio_end_tokens)) - return torch.hstack([ - audio_start_tokens[:valid_audio_nums].unsqueeze(-1), - audio_end_tokens[:valid_audio_nums].unsqueeze(-1) - ]) - - def _parse_and_validate_audio_inputs( - self, input_ids: torch.Tensor, - **kwargs: object) -> Optional[MiniCPMOAudioInputs]: + def _parse_and_validate_audio_input( + self, **kwargs: object) -> Optional[MiniCPMOAudioInputs]: audio_features = kwargs.pop("audio_features", None) audio_embeds = kwargs.pop("audio_embeds", None) if audio_features is None and audio_embeds is None: return None - audio_start_id = kwargs.pop("audio_start_id") - if not isinstance(audio_start_id, torch.Tensor): - raise ValueError("Incorrect type of audio_start_id. " - f"Got type: {type(audio_start_id)}") + audio_token_id = kwargs.pop("audio_token_id") + if audio_token_id is not None: + assert isinstance(audio_token_id, torch.Tensor) + self.mm_token_ids.add(audio_token_id.flatten().unique().item()) - audio_end_id = kwargs.pop("audio_end_id") - if not isinstance(audio_end_id, torch.Tensor): - raise ValueError("Incorrect type of audio_end_id. " - f"Got type: {type(audio_end_id)}") + audio_embed_is_patch = kwargs.pop("audio_embed_is_patch") + if not isinstance(audio_embed_is_patch, (torch.Tensor, list)): + raise ValueError("Incorrect type of audio_embed_is_patch. " + f"Got type: {type(audio_embed_is_patch)}") + + audio_embed_is_patch = flatten_bn(audio_embed_is_patch) if audio_embeds is not None: if not isinstance(audio_embeds, (torch.Tensor, list)): raise ValueError("Incorrect type of audio_embeds. " f"Got type: {type(audio_embeds)}") + audio_embeds_flat = flatten_bn(audio_embeds) + return MiniCPMOAudioEmbeddingInputs( type="audio_embeds", - audio_embeds=flatten_bn(flatten_2d_lists(audio_embeds), - concat=True), - audio_bounds=self._get_audio_bounds(input_ids, audio_start_id, - audio_end_id), + audio_embeds=audio_embeds_flat, + embed_is_patch=audio_embed_is_patch, ) - if audio_features is not None: - if not isinstance(audio_features, (torch.Tensor, list)): - raise ValueError("Incorrect type of audio_features. " - f"Got type: {type(audio_features)}") + if not isinstance(audio_features, (torch.Tensor, list)): + raise ValueError("Incorrect type of audio_features. " + f"Got type: {type(audio_features)}") - audio_feature_lens = kwargs.pop("audio_feature_lens") - if not isinstance(audio_feature_lens, (torch.Tensor, list)): - raise ValueError("Incorrect type of audio_feature_lens. " - f"Got type: {type(audio_feature_lens)}") + audio_feature_lens = kwargs.pop("audio_feature_lens") + if not isinstance(audio_feature_lens, (torch.Tensor, list)): + raise ValueError("Incorrect type of audio_feature_lens. " + f"Got type: {type(audio_feature_lens)}") - return MiniCPMOAudioFeatureInputs( - type="audio_features", - audio_features=flatten_bn(audio_features, concat=True), - audio_feature_lens=flatten_bn( - flatten_2d_lists(audio_feature_lens), concat=True), - audio_bounds=self._get_audio_bounds(input_ids, audio_start_id, - audio_end_id), - ) + audio_features_flat = flatten_bn(audio_features) + audio_feature_lens_flat = flatten_bn(audio_feature_lens) - raise AssertionError("This line should be unreachable.") - - def _parse_and_validate_inputs(self, input_ids: torch.Tensor, - **kwargs: object): - image_inputs = self._parse_and_validate_image_inputs( - input_ids, **kwargs) - if not any("audio" in key for key in kwargs): - return image_inputs, None - audio_inputs = self._parse_and_validate_audio_inputs( - input_ids, **kwargs) - return image_inputs, audio_inputs - - def forward( - self, - input_ids: torch.Tensor, - positions: torch.Tensor, - intermediate_tensors: Optional[IntermediateTensors] = None, - **kwargs: Any, - ) -> torch.Tensor: - if intermediate_tensors is not None: - vlm_embeddings = None - else: - image_inputs, audio_inputs = \ - self._parse_and_validate_inputs(input_ids, **kwargs) - vlm_embeddings = self.get_embedding_with_vision( - input_ids, image_inputs) - - if audio_inputs is not None: - vlm_embeddings = self.get_embedding_with_audios( - vlm_embeddings, audio_inputs, - self.config.audio_chunk_length) - - # always pass the input via `inputs_embeds` - # to make sure the computation graph is consistent - # for `torch.compile` integration - input_ids = None - - output = self.llm.model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=vlm_embeddings, + return MiniCPMOAudioFeatureInputs( + type="audio_features", + audio_features=audio_features_flat, + audio_feature_lens=audio_feature_lens_flat, + embed_is_patch=audio_embed_is_patch, ) - return output + + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: + modalities = super()._parse_and_validate_multimodal_inputs(**kwargs) + + # Preserve the order of modalities if there are multiple of them + # from the order of kwargs. + for input_key in kwargs: + if input_key in ("audio_features", + "audio_embeds") and "audios" not in modalities: + modalities["audios"] = self._parse_and_validate_audio_input( + **kwargs) + + return modalities + + def _process_audio_input( + self, + audio_input: MiniCPMOAudioInputs, + ) -> Union[torch.Tensor, list[torch.Tensor]]: + if audio_input["type"] == "audio_embeds": + return audio_input["audio_embeds"] + + return self.get_audio_hidden_states(audio_input) + + def _process_multimodal_inputs(self, modalities: dict): + multimodal_embeddings = super()._process_multimodal_inputs(modalities) + + for modality in modalities: + if modality == "audios": + audio_input = modalities["audios"] + audio_features = self._process_audio_input(audio_input) + multimodal_embeddings += tuple( + scatter_patch_features( + audio_features, + audio_input["embed_is_patch"], + )) + + return multimodal_embeddings diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 23c010c63d558..76c7a59d656d5 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -23,17 +23,15 @@ # limitations under the License. """Inference-only MiniCPM-V model compatible with HuggingFace weights.""" import math -import re from collections import defaultdict from collections.abc import Iterable, Mapping, Sequence from functools import cached_property, partial -from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple, - TypedDict, Union) +from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict, + Union) import numpy as np import torch import torch.types -from PIL import Image from torch import nn from transformers import BatchFeature, PretrainedConfig from typing_extensions import TypeVar @@ -50,9 +48,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM from vllm.model_executor.sampling_metadata import SamplingMetadata from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig, - MultiModalInputs, NestedTensors, - PlaceholderRange) +from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem, ImageProcessorItems, ImageSize, ModalityData, ModalityDataItems, @@ -67,13 +63,11 @@ from vllm.sequence import IntermediateTensors from vllm.utils import flatten_2d_lists from .idefics2_vision_model import Idefics2VisionTransformer -from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP, - SupportsV0Only) -from .utils import AutoWeightsLoader, flatten_bn, maybe_prefix - -CPU_DEVICE = torch.device("cpu") - -RawImageType = Union[Image.Image, torch.Tensor] +from .interfaces import (MultiModalEmbeddings, SupportsLoRA, + SupportsMultiModal, SupportsPP) +from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix, + merge_multimodal_embeddings) +from .vision import scatter_patch_features, select_patch_features class MiniCPMVImagePixelInputs(TypedDict): @@ -86,13 +80,6 @@ class MiniCPMVImagePixelInputs(TypedDict): instead of a batched tensor. """ - image_bounds: torch.Tensor - """ - Shape: `(batch_size * num_images * num_slices, 2)` - - This should be in `(start, stop)` format. - """ - tgt_sizes: torch.Tensor """ Shape: `(batch_size * num_images * num_slices, 2)` @@ -100,23 +87,34 @@ class MiniCPMVImagePixelInputs(TypedDict): This should be in `(height, width)` format. """ + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] + """ + A boolean mask indicating which image embeddings correspond + to patch tokens. + + Shape: `(batch_size * num_images, num_embeds)` + """ + + num_slices: torch.Tensor + """Shape: `(batch_size * num_images)`""" + class MiniCPMVImageEmbeddingInputs(TypedDict): type: Literal["image_embeds"] - image_embeds: torch.Tensor + image_embeds: Union[torch.Tensor, list[torch.Tensor]] """ - Shape: `(batch_size * num_images * num_slices, - image_feature_size, hidden_size)` + Shape: `(batch_size * num_images, num_slices, hidden_size)` `hidden_size` must match the hidden size of language model backbone. instead of a batched tensor. """ - image_bounds: torch.Tensor + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] """ - Shape: `(batch_size * num_images * num_slices, 2)` + A boolean mask indicating which image embeddings correspond + to patch tokens. - This should be in `(start, stop)` format. + Shape: `(batch_size * num_images, num_embeds)` """ @@ -233,15 +231,25 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]: def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]): + pixel_values = hf_inputs.get("pixel_values", torch.empty(0)) + num_images = len(pixel_values) + + video_pixel_values = hf_inputs.get("video_pixel_values", torch.empty(0)) + num_videos = len(video_pixel_values) + return dict( pixel_values=MultiModalFieldConfig.batched("image"), image_sizes=MultiModalFieldConfig.batched("image"), tgt_sizes=MultiModalFieldConfig.batched("image"), image_embeds=MultiModalFieldConfig.batched("image"), + embed_is_patch=MultiModalFieldConfig.batched("image"), video_pixel_values=MultiModalFieldConfig.batched("video"), video_image_sizes=MultiModalFieldConfig.batched("video"), video_tgt_sizes=MultiModalFieldConfig.batched("video"), video_embeds=MultiModalFieldConfig.batched("video"), + video_embed_is_patch=MultiModalFieldConfig.batched("video"), + image_token_id=MultiModalFieldConfig.shared("image", num_images), + video_token_id=MultiModalFieldConfig.shared("video", num_videos), ) @@ -348,10 +356,11 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): return get_version_by_config(self.get_hf_config()) def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]: + mm_limits = {"image": None} if self.get_model_version() == (2, 6): - return {"image": None, "video": None} - else: - return {"image": None} + mm_limits["video"] = None + + return mm_limits def get_mm_max_tokens_per_item( self, @@ -361,70 +370,79 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): mm_max_tokens = {"image": self.get_max_image_tokens()} if self.get_model_version() == (2, 6): mm_max_tokens["video"] = self.get_max_video_tokens(seq_len) + return mm_max_tokens + def get_slice_image_placeholder( + self, + image_size: ImageSize, + # For MiniCPM V/O 2.6 + image_idx: int = 0, + max_slice_nums: Optional[int] = None, + use_image_id: bool = True, + ) -> str: + image_processor = self.get_image_processor() + version = self.get_model_version() + + if version == (2, 0) or version == (2, 5): + return image_processor.get_slice_image_placeholder(image_size) + + return image_processor.get_slice_image_placeholder( + image_size, + image_idx=image_idx, + max_slice_nums=max_slice_nums, + use_image_id=use_image_id, + ) + + def get_num_image_tokens( + self, + image_size: ImageSize, + max_slice_nums: Optional[int] = None, + use_image_id: bool = True, + ) -> int: + tokenizer = self.get_tokenizer() + image_placeholders = self.get_slice_image_placeholder( + image_size, + max_slice_nums=max_slice_nums, + use_image_id=use_image_id, + ) + image_token_ids = tokenizer.encode(image_placeholders, + add_special_tokens=False) + + return len(image_token_ids) + + def get_max_image_tokens(self) -> int: + image_size = self.get_image_size_with_most_features() + return self.get_num_image_tokens(image_size) + + def get_image_max_slice_num(self) -> int: + return getattr(self.get_hf_config(), "max_slice_num", 9) + + def get_image_size_with_most_features(self) -> ImageSize: + image_size = getattr(self.get_hf_config(), "image_size", 448) + max_slice_num = self.get_image_max_slice_num() + return ImageSize(width=image_size, height=image_size * max_slice_num) + def get_max_video_frame_tokens(self) -> int: frame_size = self.get_video_frame_size_with_most_features() - return self.get_num_image_tokens(frame_size, - self.get_video_max_slice_num()) + + return self.get_num_image_tokens( + frame_size, + max_slice_nums=self.get_video_max_slice_num(), + use_image_id=False, + ) def get_max_video_tokens(self, seq_len: int) -> int: return self.get_max_video_frame_tokens( ) * self.get_num_frames_with_most_features(seq_len) - def get_slice_query_num(self) -> int: - hf_config = self.get_hf_config() - query_num = getattr(hf_config, "query_num", 64) - return query_num - - def get_max_slice_num(self) -> int: - hf_config = self.get_hf_config() - max_slice_num = getattr(hf_config, "max_slice_num", 9) - return max_slice_num - - def get_sliced_grid(self, image_size: ImageSize, - max_slice_num: int) -> Tuple[int, int]: - if self.get_model_version() == (2, 6): - slice_grid = self.get_image_processor().get_sliced_grid( - image_size, max_slice_num) - else: - slice_grid = self.get_image_processor().get_sliced_grid(image_size) - return slice_grid - - def get_num_image_tokens(self, image_size: ImageSize, - max_slice_num: int) -> int: - slice_grid = self.get_sliced_grid(image_size, max_slice_num) - num_tokens = self.get_slice_query_num( - ) + 2 # ( * query_num) - if slice_grid is not None: - if self.get_model_version() == (2, 6): - num_additional_tokens = 0 - else: - # ( * query_num) - num_additional_tokens = 2 - num_tokens += ((self.get_slice_query_num() + 2) \ - * slice_grid[0] * slice_grid[1]) \ - + slice_grid[1] - 1 + num_additional_tokens - return num_tokens - - def get_image_slice_nums(self, image_size: torch.Tensor, - max_slice_nums: int) -> int: - grid = self.get_sliced_grid(image_size, max_slice_nums) - return 1 if grid is None else grid[0] * grid[1] + 1 - - def get_max_image_tokens(self) -> int: - image_size = self.get_image_size_with_most_features() - return self.get_num_image_tokens(image_size, self.get_max_slice_num()) - - def get_image_size_with_most_features(self) -> ImageSize: - # Result in the max possible feature size (h:w = 9:1) - return self.get_default_image_sizes(self.get_max_slice_num()) - def get_video_max_slice_num(self) -> int: return 1 def get_video_frame_size_with_most_features(self) -> ImageSize: - return self.get_default_image_sizes(self.get_video_max_slice_num()) + image_size = getattr(self.get_hf_config(), "image_size", 448) + max_slice_num = self.get_video_max_slice_num() + return ImageSize(width=image_size, height=image_size * max_slice_num) def get_max_video_frames(self, max_tokens: int) -> int: num_frame_tokens = self.get_max_video_frame_tokens() @@ -436,10 +454,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): max_images = mm_config.get_limit_per_prompt("image") max_videos = mm_config.get_limit_per_prompt("video") - # count tokens - # which are not in get_max_image_tokens - max_image_tokens = self.get_max_image_tokens( - ) * max_images + 4 * max_images + max_image_tokens = self.get_max_image_tokens() * max_images max_total_frames = self.get_max_video_frames(seq_len - max_image_tokens) @@ -447,10 +462,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): return num_frames - def get_default_image_sizes(self, num_slices: int) -> ImageSize: - image_size = getattr(self.get_hf_config(), "image_size", 448) - return ImageSize(width=image_size, height=image_size * num_slices) - _I = TypeVar("_I", bound=MiniCPMVProcessingInfo, @@ -499,42 +510,30 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): def _get_data_parser(self) -> MultiModalDataParser: return MiniCPMVMultiModalDataParser() - def get_slice_image_placeholder(self, image_size: ImageSize, - **kwargs) -> str: - image_processor = self.info.get_image_processor() - version = self.info.get_model_version() - if version == (2, 0) or version == (2, 5): - return image_processor.get_slice_image_placeholder(image_size) - return image_processor.get_slice_image_placeholder( - image_size, **kwargs) - def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str: - return self.get_slice_image_placeholder(image_size, - image_idx=image_idx) + return self.info.get_slice_image_placeholder( + image_size, + image_idx=image_idx, + ) def get_video_prompt_texts(self, image_size: ImageSize, num_frames: int) -> str: - return self.get_slice_image_placeholder( + return self.info.get_slice_image_placeholder( image_size=image_size, image_idx=0, max_slice_nums=self.info.get_video_max_slice_num(), use_image_id=False, ) * num_frames - def get_special_tokens(self) -> Dict[str, torch.Tensor]: + def get_embed_is_patch( + self, + input_ids: list[int], + ) -> torch.Tensor: tokenizer = self.info.get_tokenizer() - - special_tokens = { - "im_start_id": tokenizer.im_start_id, - "im_end_id": tokenizer.im_end_id, - } - if hasattr(tokenizer, "slice_start_id"): - special_tokens["slice_start_id"] = tokenizer.slice_start_id - special_tokens["slice_end_id"] = tokenizer.slice_end_id - - return {k: torch.tensor(v) for k, v in special_tokens.items()} + unk_token_id = tokenizer.get_vocab()[""] + return torch.tensor(input_ids) == unk_token_id def process_images( self, @@ -546,14 +545,43 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): parsed_images = (self._get_data_parser().parse_mm_data({ "image": images - }).get_items("image", ImageProcessorItems)) + }).get_items("image", + (MiniCPMVImageEmbeddingItems, ImageProcessorItems))) - return self._base_call_hf_processor( - prompts=[self.info.image_pattern] * len(parsed_images), - mm_data={"images": [[image] for image in parsed_images]}, - mm_kwargs=mm_kwargs, - out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, - ) + if isinstance(parsed_images, MiniCPMVImageEmbeddingItems): + image_inputs = {} + else: + image_inputs = self._base_call_hf_processor( + prompts=[self.info.image_pattern] * len(parsed_images), + mm_data={"images": [[image] for image in parsed_images]}, + mm_kwargs=mm_kwargs, + out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, + ) + + image_sizes = [ + parsed_images.get_image_size(i) for i in range(len(parsed_images)) + ] + image_repl_features = [ + self.get_image_prompt_texts(size, idx) + for idx, size in enumerate(image_sizes) + ] + + tokenizer = self.info.get_tokenizer() + image_repls_feature_tokens = [ + tokenizer.encode(image_repl, add_special_tokens=False) + for image_repl in image_repl_features + ] + + embed_is_patch = [ + self.get_embed_is_patch(image_repl_tokens) + for image_repl_tokens in image_repls_feature_tokens + ] + image_inputs["embed_is_patch"] = embed_is_patch + + unk_token_id = tokenizer.get_vocab()[""] + image_inputs["image_token_id"] = torch.tensor(unk_token_id) + + return image_inputs def process_videos( self, @@ -565,25 +593,55 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): parsed_videos = (self._get_data_parser().parse_mm_data({ "video": videos - }).get_items("video", VideoProcessorItems)) + }).get_items("video", + (MiniCPMVVideoEmbeddingItems, VideoProcessorItems))) - max_slice_num = self.info.get_video_max_slice_num() + if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems): + video_inputs = {} + else: + video_inputs = self._base_call_hf_processor( + prompts=[ + self.info.image_pattern * len(video) + for video in parsed_videos + ], + mm_data={"images": list(parsed_videos)}, + mm_kwargs={ + **mm_kwargs, + "max_slice_nums": + self.info.get_video_max_slice_num(), + }, + out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, + ) - video_inputs = self._base_call_hf_processor( - prompts=[ - self.info.image_pattern * len(video) for video in parsed_videos - ], - mm_data={"images": list(parsed_videos)}, - mm_kwargs={ - **mm_kwargs, "max_slice_nums": max_slice_num - }, - out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, - ) + frame_sizes = [ + parsed_videos.get_frame_size(i) for i in range(len(parsed_videos)) + ] + num_frames = [ + parsed_videos.get_num_frames(i) for i in range(len(parsed_videos)) + ] + video_repl_features = [ + self.get_video_prompt_texts(size, nframes) + for size, nframes in zip(frame_sizes, num_frames) + ] - return {f"video_{k}": v for k, v in video_inputs.items()} + tokenizer = self.info.get_tokenizer() + video_repls_feature_tokens = [ + tokenizer.encode(video_repl, add_special_tokens=False) + for video_repl in video_repl_features + ] - def get_placeholder_match_pattern(self) -> str: - return r"\(<(image|video)>./\)" + embed_is_patch = [ + self.get_embed_is_patch(video_repl_tokens) + for video_repl_tokens in video_repls_feature_tokens + ] + video_inputs["embed_is_patch"] = embed_is_patch + + video_inputs = {f"video_{k}": v for k, v in video_inputs.items()} + + unk_token_id = tokenizer.get_vocab()[""] + video_inputs["video_token_id"] = torch.tensor(unk_token_id) + + return video_inputs def process_mm_inputs( self, @@ -602,7 +660,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): mm_kwargs: Mapping[str, object], *, out_keys: set[str], - ) -> Mapping[str, NestedTensors]: + ) -> dict[str, NestedTensors]: # This processor supports zipping prompt and mm_data together if self.info.get_model_version() == (2, 6): inputs = super()._call_hf_processor( @@ -635,14 +693,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], ) -> BatchFeature: - # Do not support combination inputs of images and videos for now - # Try to handle interleaved multimodal data tokenizer = self.info.get_tokenizer() + + input_ids = torch.tensor([tokenizer.encode(prompt)]) mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs) return BatchFeature({ - "input_ids": - torch.tensor([tokenizer.encode(prompt)]), + "input_ids": input_ids, **mm_inputs, }) @@ -701,39 +758,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): ) -> Mapping[str, MultiModalFieldConfig]: return _minicpmv_field_config(hf_inputs) - def apply( - self, - prompt: Union[str, List[int]], - mm_data: MultiModalDataDict, - hf_processor_mm_kwargs: Mapping[str, object], - return_mm_hashes: bool = False, - ) -> MultiModalInputs: - if isinstance(prompt, list): - prompt = self.info.get_tokenizer().decode(prompt) - matches = re.findall(self.get_placeholder_match_pattern(), prompt) - mm_orders = { - f"{modality}_orders": - torch.tensor( - [index for index, m in enumerate(matches) if m == modality]) - for modality in self.info.get_supported_mm_limits() - } - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs, - return_mm_hashes) - # Exclude x from placeholders - if "image" in result["mm_placeholders"] and \ - self.info.get_model_version() == (2, 6): - result["mm_placeholders"]["image"] = [ - PlaceholderRange(offset=p["offset"] + 3 + idx // 10, - length=p["length"] - 3 - idx // 10) - for idx, p in enumerate(result["mm_placeholders"]["image"]) - ] - result["mm_kwargs"].update(**mm_orders) - result["mm_kwargs"].update(**self.get_special_tokens()) - return result - -class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP, - SupportsV0Only): +class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): """ The abstract class of MiniCPMV can only be inherited, but cannot be instantiated. @@ -767,6 +793,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP, prefix=maybe_prefix( prefix, "resampler")) + self.mm_token_ids = set[int]() self.make_empty_intermediate_tensors = ( self.llm.make_empty_intermediate_tensors) @@ -777,233 +804,191 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP, return get_sampler() - def get_embedding_with_vision( + def _parse_and_validate_vision_input( self, - input_ids: torch.Tensor, - image_inputs: Optional[MiniCPMVImageInputs], - ) -> torch.Tensor: - vlm_embedding: torch.Tensor = self.llm.get_input_embeddings(input_ids) - - if image_inputs is None: - return vlm_embedding - - if image_inputs["type"] == "image_embeds": - vision_hidden_states = image_inputs["image_embeds"].to( - device=vlm_embedding.device, - dtype=vlm_embedding.dtype, - ) - else: - vision_hidden_states = self.get_vision_hidden_states(image_inputs) - - # See NOTE in _parse_and_validate_inputs - image_bounds = image_inputs["image_bounds"] - if len(image_bounds) > 0: - image_indices = torch.stack([ - torch.arange(start, end, dtype=torch.long) - for start, end in image_bounds.tolist() - ]).to(vlm_embedding.device) - - vlm_embedding.scatter_( - 0, - image_indices.view(-1, 1).repeat(1, vlm_embedding.shape[-1]), - vision_hidden_states.view(-1, vision_hidden_states.shape[-1]), - ) - - return vlm_embedding - - def _get_image_bounds( - self, - input_ids: torch.Tensor, - im_start_id: torch.Tensor, - im_end_id: torch.Tensor, - slice_start_id: Optional[torch.Tensor] = None, - slice_end_id: Optional[torch.Tensor] = None) -> torch.Tensor: - # All the images in the batch should share the same special image - # bound token ids. - start_cond = input_ids == im_start_id[0] - end_cond = input_ids == im_end_id[0] - if slice_start_id is not None: - start_cond |= (input_ids == slice_start_id[0]) - end_cond |= (input_ids == slice_end_id[0]) - - image_start_tokens, = torch.where(start_cond) - image_start_tokens += 1 - image_end_tokens, = torch.where(end_cond) - valid_image_nums = max(len(image_start_tokens), len(image_end_tokens)) - - if valid_image_nums == 0: - return torch.zeros((0, 2), device=input_ids.device) - - return torch.hstack([ - image_start_tokens[:valid_image_nums].unsqueeze(-1), - image_end_tokens[:valid_image_nums].unsqueeze(-1), - ]) - - def _parse_and_validate_image_inputs( - self, - input_ids: torch.Tensor, + modality: str, **kwargs: object, ) -> Optional[MiniCPMVImageInputs]: - image_keys = {"pixel_values", "tgt_sizes"} - pixel_data = { - "image": { - key: kwargs.pop(key, None) - for key in image_keys - }, - "video": { - key: kwargs.pop("video_" + key, None) - for key in image_keys - } - } - embed_data = { - "image": kwargs.pop("image_embeds", None), - "video": kwargs.pop("video_embeds", None), - } + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) - all_pixel_data = [ - v for vs in pixel_data.values() for v in vs.values() - if v is not None - ] - all_embed_data = [v for v in embed_data.values() if v is not None] - if len(all_pixel_data) == 0 and len(all_embed_data) == 0: + if pixel_values is None and image_embeds is None: return None - im_start_id = kwargs.pop("im_start_id") - if not isinstance(im_start_id, torch.Tensor): - raise ValueError("Incorrect type of im_start_id. " - f"Got type: {type(im_start_id)}") + image_token_id = kwargs.pop("image_token_id") + if image_token_id is not None: + assert isinstance(image_token_id, torch.Tensor) + self.mm_token_ids.add(image_token_id.flatten().unique().item()) - im_end_id = kwargs.pop("im_end_id") - if not isinstance(im_end_id, torch.Tensor): - raise ValueError("Incorrect type of im_end_id. " - f"Got type: {type(im_end_id)}") + embed_is_patch = kwargs.pop("embed_is_patch") + if not isinstance(embed_is_patch, (torch.Tensor, list)): + raise ValueError( + f"Incorrect type of embed_is_patch for {modality=}. " + f"Got type: {type(embed_is_patch)}") - slice_start_id = kwargs.pop("slice_start_id", None) - if slice_start_id is not None and not isinstance( - slice_start_id, torch.Tensor): - raise ValueError("Incorrect type of slice_start_id. " - f"Got type: {type(slice_start_id)}") + embed_is_patch = flatten_bn(embed_is_patch) - slice_end_id = kwargs.pop("slice_end_id", None) - if slice_end_id is not None and not isinstance(slice_end_id, - torch.Tensor): - raise ValueError("Incorrect type of slice_end_id. " - f"Got type: {type(slice_end_id)}") + if image_embeds is not None: + if not isinstance(image_embeds, (torch.Tensor, list)): + raise ValueError( + f"Incorrect type of image_embeds for {modality=}. " + f"Got type: {type(image_embeds)}") - if len(all_embed_data) > 0: - if len(all_embed_data) > 1: - raise ValueError("Incorrect inputs for vision embeddings. " - "Image embeds and video embeds can not " - "exist simultaneously.") - - vision_embeds, = all_embed_data - if not isinstance(vision_embeds, (torch.Tensor, list)): - raise ValueError(f"Incorrect type of vision_embeds. " - f"Got type: {type(vision_embeds)}") + image_embeds_flat = flatten_bn(image_embeds) return MiniCPMVImageEmbeddingInputs( type="image_embeds", - image_embeds=flatten_bn(flatten_2d_lists(vision_embeds), - concat=True), - image_bounds=self._get_image_bounds(input_ids, im_start_id, - im_end_id, slice_start_id, - slice_end_id), + image_embeds=image_embeds_flat, + embed_is_patch=embed_is_patch, ) - order_data = dict[str, Union[torch.Tensor, list[torch.Tensor]]]() - for modality in ("image", "video"): - modality_orders = kwargs.pop(f"{modality}_orders", None) - if modality_orders is not None: - if not isinstance(modality_orders, (torch.Tensor, list)): - raise ValueError(f"Incorrect type of {modality}_orders. " - f"Got type: {type(modality_orders)}") + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError( + f"Incorrect type of pixel_values for {modality=}. " + f"Got type: {type(pixel_values)}") - order_data[modality] = modality_orders + tgt_sizes = kwargs.pop("tgt_sizes") + if not isinstance(tgt_sizes, (torch.Tensor, list)): + raise ValueError(f"Incorrect type of tgt_sizes for {modality=}. " + f"Got type: {type(tgt_sizes)}") - batch_sizes = { - modality: len(modality_orders) - for modality, modality_orders in order_data.items() - } - unique_batch_sizes = set(batch_sizes.values()) - assert len(unique_batch_sizes) == 1, ( - f"Found inconsistent batch sizes: {batch_sizes}") - batch_size, = unique_batch_sizes + num_slices = [[len(p) for p in ps] for ps in pixel_values] + num_slices_flat = flatten_bn(torch.tensor(num_slices)) - pixel_values_flat = list[torch.Tensor]() - tgt_sizes_flat = list[torch.Tensor]() - for b in range(batch_size): - mm_orders_b = [(idx_b.item(), modality) - for modality, modality_orders in order_data.items() - for idx_b in modality_orders[b]] + pixel_values_flat = flatten_bn(flatten_2d_lists(pixel_values)) + tgt_sizes_flat = flatten_bn(flatten_2d_lists(tgt_sizes), concat=True) - for _, modality in sorted(mm_orders_b, key=lambda x: x[0]): - modality_pixel_data = pixel_data[modality] - - modality_pixel_values = modality_pixel_data["pixel_values"] - if not isinstance(modality_pixel_values, (torch.Tensor, list)): - raise ValueError( - f"Incorrect type of pixel_values for {modality=}. " - f"Got type: {type(modality_pixel_values)}") - - modality_tgt_sizes = modality_pixel_data["tgt_sizes"] - if not isinstance(modality_tgt_sizes, (torch.Tensor, list)): - raise ValueError( - f"Incorrect type of tgt_sizes for {modality=}. " - f"Got type: {type(modality_tgt_sizes)}") - - pixel_values_flat += flatten_2d_lists(modality_pixel_values[b]) - tgt_sizes_flat += flatten_2d_lists(modality_tgt_sizes[b]) - - # NOTE: Input IDs does not contain image tokens during memory profiling, - # so we allow it to be empty if len(pixel_values_flat) != len(tgt_sizes_flat): raise ValueError("Inconsistent flattened lengths, found: " f"{len(pixel_values_flat)} vs. " f"{len(tgt_sizes_flat)}") - if len(pixel_values_flat) == 0: - return None - return MiniCPMVImagePixelInputs( type="pixel_values", pixel_values=pixel_values_flat, - tgt_sizes=torch.stack(tgt_sizes_flat), - image_bounds=self._get_image_bounds(input_ids, im_start_id, - im_end_id, slice_start_id, - slice_end_id), + tgt_sizes=tgt_sizes_flat, + embed_is_patch=embed_is_patch, + num_slices=num_slices_flat, ) - def _parse_and_validate_inputs(self, input_ids: torch.Tensor, - **kwargs: object): - return self._parse_and_validate_image_inputs(input_ids, **kwargs) + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: + modalities = {} + + # Preserve the order of modalities if there are multiple of them + # from the order of kwargs. + for input_key in kwargs: + if input_key in ("pixel_values", + "image_embeds") and "images" not in modalities: + modalities["images"] = self._parse_and_validate_vision_input( + "images", **kwargs) + if input_key in ("video_pixel_values", + "video_embeds") and "videos" not in modalities: + + def _image_key(video_key: str): + if video_key == "video_token_id": + return "image_token_id" + + return video_key.removeprefix("video_") + + modalities["videos"] = self._parse_and_validate_vision_input( + "videos", **{ + _image_key(k): v + for k, v in kwargs.items() + }) + + return modalities + + def _process_vision_input( + self, + image_input: MiniCPMVImageInputs, + ) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor, ...]]: + if image_input["type"] == "image_embeds": + return image_input["image_embeds"] + + image_features_flat = self.get_vision_hidden_states(image_input) + + # Reconstruct the batch dimension + return image_features_flat.split(image_input["num_slices"].tolist()) + + def _process_multimodal_inputs(self, modalities: dict): + # The result multimodal_embeddings is tuple of tensors, with each + # tensor correspoending to a multimodal data item (image or video). + multimodal_embeddings: tuple[torch.Tensor, ...] = () + + # NOTE: It is important to iterate over the keys in this dictionary + # to preserve the order of the modalities. + for modality in modalities: + if modality == "images": + image_input = modalities["images"] + image_features = self._process_vision_input(image_input) + multimodal_embeddings += tuple( + scatter_patch_features( + image_features, + image_input["embed_is_patch"], + )) + if modality == "videos": + video_input = modalities["videos"] + video_features = self._process_vision_input(video_input) + multimodal_embeddings += tuple( + scatter_patch_features( + video_features, + video_input["embed_is_patch"], + )) + + return multimodal_embeddings + + def get_multimodal_embeddings( + self, **kwargs: object) -> Optional[MultiModalEmbeddings]: + modalities = self._parse_and_validate_multimodal_inputs(**kwargs) + if not modalities: + return None + + return self._process_multimodal_inputs(modalities) + + def get_input_embeddings( + self, + input_ids: torch.Tensor, + multimodal_embeddings: Optional[MultiModalEmbeddings] = None, + ) -> torch.Tensor: + inputs_embeds = self.llm.get_input_embeddings(input_ids) + if multimodal_embeddings is not None: + assert len(self.mm_token_ids) > 0 + inputs_embeds = merge_multimodal_embeddings( + input_ids, + inputs_embeds, + select_patch_features(multimodal_embeddings), + list(self.mm_token_ids), + ) + return inputs_embeds def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, **kwargs: Any, ) -> torch.Tensor: if intermediate_tensors is not None: - vlm_embeddings = None - else: - image_inputs = \ - self._parse_and_validate_inputs(input_ids, **kwargs) - vlm_embeddings = self.get_embedding_with_vision( - input_ids, image_inputs) + inputs_embeds = None - # always pass the input via `inputs_embeds` - # to make sure the computation graph is consistent - # for `torch.compile` integration - input_ids = None + # NOTE: In v1, inputs_embeds is always generated at model runner from + # `get_multimodal_embeddings` and `get_input_embeddings`, this + # condition is only for v0 compatibility. + elif inputs_embeds is None: + vision_embeddings = self.get_multimodal_embeddings(**kwargs) - output = self.llm.model( + inputs_embeds = self.get_input_embeddings(input_ids, + vision_embeddings) + input_ids = None + + hidden_states = self.llm.model( input_ids=input_ids, positions=positions, intermediate_tensors=intermediate_tensors, - inputs_embeds=vlm_embeddings, + inputs_embeds=inputs_embeds, ) - return output + return hidden_states def compute_logits( self, @@ -1105,9 +1090,6 @@ class MiniCPMV2_0(MiniCPMVBaseModel): return model - def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_tokens(input_ids) - def init_resampler(self, embed_dim: int, vision_dim: int, diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 9224687d8a5d3..b2f795155f17b 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -92,8 +92,8 @@ class MolmoImageInputs(TypedDict): Shape: `(batch_size * num_images, num_embeds)` """ - num_crops: Union[torch.Tensor, list[torch.Tensor]] - """Shape: `(batch_size, num_images)`""" + num_crops: torch.Tensor + """Shape: `(batch_size * num_images)`""" @dataclass @@ -1492,6 +1492,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, self.img_patch_id = img_patch_id.flatten().unique().item() embed_is_patch = flatten_bn(embed_is_patch) + num_crops = flatten_bn(num_crops, concat=True) return MolmoImageInputs( images=images, @@ -1510,31 +1511,24 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, feat_is_patch = image_input["feat_is_patch"] num_crops = image_input["num_crops"] - if isinstance(images, list): - # Call the vision backbone on the whole batch at once - images_flat = flatten_bn(images, concat=True) - image_masks_flat = (None if image_masks is None else flatten_bn( - image_masks, concat=True)) + # Call the vision backbone on the whole batch at once + images_flat = flatten_bn(images, concat=True) + image_masks_flat = (None if image_masks is None else flatten_bn( + image_masks, concat=True)) + feat_is_patch_flat = flatten_bn(feat_is_patch, concat=True) - image_features_flat = self.vision_backbone( - images=images_flat.unsqueeze(0), - image_masks=(None if image_masks_flat is None else - image_masks_flat.unsqueeze(0)), - ).squeeze(0) - - # Reconstruct the batch dimension - num_crops_per_image = [nc.sum().item() for nc in num_crops] - image_features = image_features_flat.split(num_crops_per_image) - else: - image_features = self.vision_backbone( - images=images, - image_masks=image_masks, - ) + image_features_flat = self.vision_backbone( + images=images_flat.unsqueeze(0), + image_masks=(None if image_masks_flat is None else + image_masks_flat.unsqueeze(0)), + ).squeeze(0) # Only the features corresponding to patch tokens are relevant return [ - feats[f_is_patch] - for feats, f_is_patch in zip(image_features, feat_is_patch) + feats[f_is_patch] for feats, f_is_patch in zip( + image_features_flat.split(num_crops.tolist()), + feat_is_patch_flat.split(num_crops.tolist()), + ) ] def get_multimodal_embeddings( From 8958217ad5a6830c4d911e5f15e6eb791df337b6 Mon Sep 17 00:00:00 2001 From: Hiroaki Sugiyama Date: Thu, 27 Mar 2025 23:29:29 +0900 Subject: [PATCH 045/593] [Bugfix] Fix use_cascade_attention handling for Alibi-based models on vllm/v1 (#15211) Signed-off-by: h-sugi Co-authored-by: Woosuk Kwon --- vllm/utils.py | 14 +++++++++++++- vllm/v1/worker/gpu_model_runner.py | 7 +++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/vllm/utils.py b/vllm/utils.py index 516b33dca1dc8..77f4e2dcf5e45 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -61,7 +61,7 @@ import vllm.envs as envs from vllm.logger import enable_trace_function_call, init_logger if TYPE_CHECKING: - from vllm.config import VllmConfig + from vllm.config import ModelConfig, VllmConfig logger = init_logger(__name__) @@ -2498,6 +2498,18 @@ def cprofile(save_file: Optional[str] = None, enabled: bool = True): return decorator +# Only relevant for models using ALiBi (e.g, MPT) +def check_use_alibi(model_config: ModelConfig) -> bool: + return (getattr(model_config.hf_text_config, "alibi", False) # Falcon + or ("BloomForCausalLM" in getattr(model_config.hf_config, + "architectures", [])) # Bloom + or getattr(model_config.hf_text_config, "position_encoding_type", + "") == "alibi" # codellm_1b_alibi + or + (hasattr(model_config.hf_text_config, "attn_config") # MPT + and model_config.hf_text_config.attn_config.get("alibi", False))) + + def sha256(input) -> int: """Hash any picklable Python object using SHA-256. diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index bcf7762b44496..230479f3f15e7 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -25,7 +25,7 @@ from vllm.multimodal.utils import group_mm_inputs_by_modality from vllm.sampling_params import SamplingType from vllm.sequence import IntermediateTensors from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, - LayerBlockType, LazyLoader, cdiv, + LayerBlockType, LazyLoader, cdiv, check_use_alibi, is_pin_memory_available) from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata from vllm.v1.core.encoder_cache_manager import compute_encoder_budget @@ -223,6 +223,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): device="cpu", pin_memory=self.pin_memory) + # Only relevant for models using ALiBi (e.g, MPT) + self.use_alibi = check_use_alibi(model_config) + self.inputs_embeds = torch.zeros( (self.max_num_tokens, self.hidden_size), dtype=self.dtype, @@ -689,7 +692,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): query_lens=num_scheduled_tokens, num_query_heads=self.num_query_heads, num_kv_heads=self.num_kv_heads, - use_alibi=False, # FIXME + use_alibi=self.use_alibi, use_sliding_window=self.window_size is not None, num_sms=self.num_sms, ) From 07bf813fb554c9a78d1e9f4a587edd8b6d9d7ccd Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 28 Mar 2025 00:30:53 +0800 Subject: [PATCH 046/593] [Doc] Link to onboarding tasks (#15629) Signed-off-by: DarkLight1337 --- docs/source/conf.py | 5 +++++ docs/source/contributing/overview.md | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index b02b84826c9f2..3e790827f53bb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -103,6 +103,11 @@ myst_url_schemes = { "title": "Pull Request #{{path}}", "classes": ["github"], }, + "gh-project": { + "url": "https://github.com/vllm-project/projects/{{path}}", + "title": "Project #{{path}}", + "classes": ["github"], + }, "gh-dir": { "url": "https://github.com/vllm-project/vllm/tree/main/{{path}}", "title": "{{path}}", diff --git a/docs/source/contributing/overview.md b/docs/source/contributing/overview.md index a414118316692..10cbc0eb1264b 100644 --- a/docs/source/contributing/overview.md +++ b/docs/source/contributing/overview.md @@ -11,6 +11,15 @@ We also believe in the power of community support; thus, answering queries, offe Finally, one of the most impactful ways to support us is by raising awareness about vLLM. Talk about it in your blog posts and highlight how it's driving your incredible projects. Express your support on social media if you're using vLLM, or simply offer your appreciation by starring our repository! +## Job Board + +Unsure on where to start? Check out the following links for tasks to work on: + +- [Good first issues](https://github.com/vllm-project/vllm/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22good%20first%20issue%22) + - [Selected onboarding tasks](gh-project:6) +- [New model requests](https://github.com/vllm-project/vllm/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22new%20model%22) + - [Models with multi-modal capabilities](gh-project:10) + ## License See . From 247181536fc2cab728077f3e7489622e19671d2d Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 28 Mar 2025 01:36:32 +0800 Subject: [PATCH 047/593] [Misc] Replace `is_encoder_decoder_inputs` with `split_enc_dec_inputs` (#15620) Signed-off-by: DarkLight1337 --- .../multimodal/processing/test_idefics3.py | 2 +- .../multimodal/processing/test_phi3v.py | 2 +- vllm/engine/arg_utils.py | 2 +- vllm/engine/llm_engine.py | 28 ++++++++---------- vllm/inputs/parse.py | 22 +++++++++----- vllm/inputs/registry.py | 14 ++++----- vllm/model_executor/models/idefics3.py | 4 +-- vllm/v1/engine/processor.py | 29 ++++++++----------- 8 files changed, 49 insertions(+), 54 deletions(-) diff --git a/tests/models/multimodal/processing/test_idefics3.py b/tests/models/multimodal/processing/test_idefics3.py index fdbe2f17692f7..4cff429a53941 100644 --- a/tests/models/multimodal/processing/test_idefics3.py +++ b/tests/models/multimodal/processing/test_idefics3.py @@ -29,7 +29,7 @@ def test_processor_override( num_imgs: int, kwargs_on_init: bool, ): - """Ensure input_processor_for_idefics3 handles num_crops properly.""" + """Ensure Idefics3MultiModalProcessor handles num_crops properly.""" # Same as the previous test - don't initialize mm_processor_kwargs # in this test and assume that the kwargs will be correctly expanded by # the partial when calling the custom input processor. diff --git a/tests/models/multimodal/processing/test_phi3v.py b/tests/models/multimodal/processing/test_phi3v.py index 2f0c8e7e5492c..dd5f30a23176b 100644 --- a/tests/models/multimodal/processing/test_phi3v.py +++ b/tests/models/multimodal/processing/test_phi3v.py @@ -30,7 +30,7 @@ def test_processor_override( num_imgs: int, kwargs_on_init: bool, ): - """Ensure input_processor_for_phi3v handles num_crops properly.""" + """Ensure Phi3VMultiModalProcessor handles num_crops properly.""" # Avoid initializing CUDA early from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 784ea35beb357..53af3e5717c52 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -665,7 +665,7 @@ class EngineArgs: type=nullable_kvs, default=EngineArgs.limit_mm_per_prompt, # The default value is given in - # MultiModalRegistry.init_mm_limits_per_prompt + # MultiModalConfig.get_limit_per_prompt help=('For each multimodal plugin, limit how many ' 'input instances to allow for each prompt. ' 'Expects a comma-separated list of items, ' diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py index 3d019ea58c5e1..4856c3568319b 100644 --- a/vllm/engine/llm_engine.py +++ b/vllm/engine/llm_engine.py @@ -30,8 +30,8 @@ from vllm.entrypoints.openai.logits_processors import ( get_logits_processors as get_openai_logits_processors) from vllm.executor.executor_base import ExecutorBase from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs, - PromptType, SingletonInputsAdapter) -from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt + PromptType) +from vllm.inputs.parse import is_token_prompt, split_enc_dec_inputs from vllm.inputs.preprocess import InputPreprocessor from vllm.logger import init_logger from vllm.logits_process import get_bad_words_logits_processors @@ -609,12 +609,7 @@ class LLMEngine: seq_id = next(self.seq_counter) eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request) - if is_encoder_decoder_inputs(processed_inputs): - decoder_inputs = processed_inputs["decoder"] - encoder_inputs = processed_inputs["encoder"] - else: - decoder_inputs = processed_inputs - encoder_inputs = None + encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) seq = Sequence(seq_id, decoder_inputs, block_size, eos_token_id, lora_request, prompt_adapter_request) @@ -2031,15 +2026,16 @@ class LLMEngine: def _validate_model_inputs(self, inputs: ProcessorInputs, lora_request: Optional[LoRARequest]): - if is_encoder_decoder_inputs(inputs): - # For encoder-decoder multimodal models, the max_prompt_len - # restricts the decoder prompt length - prompt_inputs = inputs["decoder" if self.model_config. - is_multimodal_model else "encoder"] - else: - prompt_inputs = inputs + encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs) - prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids + # For encoder-decoder multimodal models, the max_prompt_len + # restricts the decoder prompt length + if self.model_config.is_multimodal_model: + prompt_inputs = decoder_inputs + else: + prompt_inputs = encoder_inputs or decoder_inputs + + prompt_ids = prompt_inputs["prompt_token_ids"] if prompt_ids is None or len(prompt_ids) == 0: raise ValueError("Prompt cannot be empty") diff --git a/vllm/inputs/parse.py b/vllm/inputs/parse.py index ed1056948d807..28e207de1fd39 100644 --- a/vllm/inputs/parse.py +++ b/vllm/inputs/parse.py @@ -1,15 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 - from collections.abc import Sequence -from typing import Literal, TypedDict, Union, cast, overload +from typing import Literal, Optional, TypedDict, Union, cast, overload from typing_extensions import TypeIs from vllm.utils import is_list_of -from .data import (EncoderDecoderInputs, ExplicitEncoderDecoderPrompt, - ProcessorInputs, PromptType, SingletonPrompt, TextPrompt, - TokensPrompt) +from .data import (ExplicitEncoderDecoderPrompt, ProcessorInputs, PromptType, + SingletonInputs, SingletonPrompt, TextPrompt, TokensPrompt) class ParsedText(TypedDict): @@ -110,6 +108,14 @@ def is_explicit_encoder_decoder_prompt( return isinstance(prompt, dict) and "encoder_prompt" in prompt -def is_encoder_decoder_inputs( - inputs: ProcessorInputs) -> TypeIs[EncoderDecoderInputs]: - return "encoder" in inputs and "decoder" in inputs +def split_enc_dec_inputs( + inputs: ProcessorInputs, +) -> tuple[Optional[SingletonInputs], SingletonInputs]: + if "encoder" in inputs and "decoder" in inputs: + # NOTE: This passes pyright but not mypy + return ( + inputs["encoder"], # type: ignore[typeddict-item] + inputs["decoder"], # type: ignore[typeddict-item] + ) + + return None, inputs diff --git a/vllm/inputs/registry.py b/vllm/inputs/registry.py index b6ceb5fb82d70..8b95db7a72522 100644 --- a/vllm/inputs/registry.py +++ b/vllm/inputs/registry.py @@ -19,7 +19,7 @@ from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides, resolve_mm_processor_kwargs) from .data import ProcessorInputs, SingletonInputs -from .parse import is_encoder_decoder_inputs +from .parse import split_enc_dec_inputs if TYPE_CHECKING: from vllm.config import ModelConfig @@ -462,13 +462,11 @@ class InputRegistry: **mm_processor_kwargs, ) - if is_encoder_decoder_inputs(processed_inputs): - self._ensure_mm_kwargs(processed_inputs["encoder"], - mm_processor_kwargs) - self._ensure_mm_kwargs(processed_inputs["decoder"], - mm_processor_kwargs) - else: - self._ensure_mm_kwargs(processed_inputs, mm_processor_kwargs) + encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) + if encoder_inputs is not None: + self._ensure_mm_kwargs(encoder_inputs, mm_processor_kwargs) + if decoder_inputs is not None: + self._ensure_mm_kwargs(decoder_inputs, mm_processor_kwargs) return processed_inputs diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 234e4498f163b..432f26141048b 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -232,7 +232,7 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo] ) -class Idefics3MultimodalProcessor( +class Idefics3MultiModalProcessor( BaseMultiModalProcessor[Idefics3ProcessingInfo]): def _call_hf_processor( @@ -575,7 +575,7 @@ class Idefics3Model(nn.Module): @MULTIMODAL_REGISTRY.register_processor( - Idefics3MultimodalProcessor, + Idefics3MultiModalProcessor, info=Idefics3ProcessingInfo, dummy_inputs=Idefics3DummyInputsBuilder) class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, diff --git a/vllm/v1/engine/processor.py b/vllm/v1/engine/processor.py index e281781675769..065ac0920af77 100644 --- a/vllm/v1/engine/processor.py +++ b/vllm/v1/engine/processor.py @@ -7,7 +7,7 @@ from typing import Optional, Union from vllm.config import VllmConfig from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs, PromptType, SingletonInputsAdapter) -from vllm.inputs.parse import is_encoder_decoder_inputs +from vllm.inputs.parse import split_enc_dec_inputs from vllm.inputs.preprocess import InputPreprocessor from vllm.lora.request import LoRARequest from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs, @@ -209,14 +209,8 @@ class Processor: self._validate_model_inputs(processed_inputs, lora_request) - if is_encoder_decoder_inputs(processed_inputs): - decoder_inputs = SingletonInputsAdapter( - processed_inputs["decoder"]) - encoder_inputs = SingletonInputsAdapter( - processed_inputs["encoder"]) - else: - decoder_inputs = SingletonInputsAdapter(processed_inputs) - encoder_inputs = None + encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) + decoder_inputs = SingletonInputsAdapter(decoder_inputs) # TODO: Impl encoder-decoder if encoder_inputs is not None: @@ -301,15 +295,16 @@ class Processor: def _validate_model_inputs(self, inputs: ProcessorInputs, lora_request: Optional[LoRARequest] = None): - if is_encoder_decoder_inputs(inputs): - # For encoder-decoder multimodal models, the max_prompt_len - # restricts the decoder prompt length - prompt_inputs = inputs["decoder" if self.model_config. - is_multimodal_model else "encoder"] - else: - prompt_inputs = inputs + encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs) - prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids + # For encoder-decoder multimodal models, the max_prompt_len + # restricts the decoder prompt length + if self.model_config.is_multimodal_model: + prompt_inputs = decoder_inputs + else: + prompt_inputs = encoder_inputs or decoder_inputs + + prompt_ids = prompt_inputs["prompt_token_ids"] if prompt_ids is None or len(prompt_ids) == 0: raise ValueError("Prompt cannot be empty") From 66aa4c0bf4973065b45172dc18346016a6087a10 Mon Sep 17 00:00:00 2001 From: Yuan Tang Date: Thu, 27 Mar 2025 13:49:38 -0400 Subject: [PATCH 048/593] [Feature] Add middleware to log API Server responses (#15593) Signed-off-by: Yuan Tang --- vllm/entrypoints/openai/api_server.py | 16 ++++++++++++++++ vllm/envs.py | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 374e43fb15341..1e735da641df9 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -24,6 +24,7 @@ from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.concurrency import iterate_in_threadpool from starlette.datastructures import State from starlette.routing import Mount from typing_extensions import assert_never @@ -846,6 +847,21 @@ def build_app(args: Namespace) -> FastAPI: response.headers["X-Request-Id"] = request_id return response + if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE: + logger.warning("CAUTION: Enabling log response in the API Server. " + "This can include sensitive information and should be " + "avoided in production.") + + @app.middleware("http") + async def log_response(request: Request, call_next): + response = await call_next(request) + response_body = [ + section async for section in response.body_iterator + ] + response.body_iterator = iterate_in_threadpool(iter(response_body)) + logger.info("response_body={%s}", response_body[0].decode()) + return response + for middleware in args.middleware: module_path, object_name = middleware.rsplit(".", 1) imported = getattr(importlib.import_module(module_path), object_name) diff --git a/vllm/envs.py b/vllm/envs.py index 23c304f124d36..e5025485a2501 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -270,6 +270,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None), + # Whether to log responses from API Server for debugging + "VLLM_DEBUG_LOG_API_SERVER_RESPONSE": + lambda: os.environ.get("VLLM_DEBUG_LOG_API_SERVER_RESPONSE", "False"). + lower() == "true", + # S3 access information, used for tensorizer to load model from S3 "S3_ACCESS_KEY_ID": lambda: os.environ.get("S3_ACCESS_KEY_ID", None), From 13ac9cab21e3cd12acd0c94376bc2f6da3dca5cd Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 28 Mar 2025 01:52:00 +0800 Subject: [PATCH 049/593] [Misc] Avoid direct access of global `mm_registry` in `compute_encoder_budget` (#15621) Signed-off-by: DarkLight1337 --- vllm/v1/core/encoder_cache_manager.py | 16 ++++++++++++---- vllm/v1/core/sched/scheduler.py | 3 +++ vllm/v1/worker/gpu_model_runner.py | 6 +++--- vllm/v1/worker/tpu_model_runner.py | 1 + 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index 018379c1f43af..dc76df268c588 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING from vllm.logger import init_logger -from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal import MultiModalRegistry from vllm.v1.request import Request if TYPE_CHECKING: @@ -67,6 +67,7 @@ class EncoderCacheManager: def compute_encoder_budget( model_config: "ModelConfig", scheduler_config: "SchedulerConfig", + mm_registry: MultiModalRegistry, ) -> tuple[int, int]: """Compute the encoder cache budget based on the model and scheduler configurations. @@ -74,6 +75,7 @@ def compute_encoder_budget( Args: model_config: Model configuration. scheduler_config: Scheduler configuration. + mm_registry: Provides information about the token cost. Returns: - Compute budget for encoder execution, in unit of number of tokens @@ -89,7 +91,11 @@ def compute_encoder_budget( ( encoder_compute_budget, encoder_cache_size, - ) = _compute_encoder_budget_multimodal(model_config, scheduler_config) + ) = _compute_encoder_budget_multimodal( + model_config, + scheduler_config, + mm_registry, + ) return encoder_compute_budget, encoder_cache_size @@ -97,6 +103,7 @@ def compute_encoder_budget( def _compute_encoder_budget_multimodal( model_config: "ModelConfig", scheduler_config: "SchedulerConfig", + mm_registry: MultiModalRegistry, ) -> tuple[int, int]: """Compute the encoder cache budget based on the model and scheduler configurations for a multimodal model. @@ -104,6 +111,7 @@ def _compute_encoder_budget_multimodal( Args: model_config: Model configuration. scheduler_config: Scheduler configuration. + mm_registry: Provides information about the token cost. Returns: - Compute budget for encoder execution, in unit of number of tokens @@ -112,8 +120,8 @@ def _compute_encoder_budget_multimodal( in the input sequence. """ - max_tokens_by_modality_dict = MULTIMODAL_REGISTRY.get_max_tokens_per_item_by_nonzero_modality( # noqa: E501 - model_config) + max_tokens_by_modality_dict = mm_registry \ + .get_max_tokens_per_item_by_nonzero_modality(model_config) if not max_tokens_by_modality_dict: logger.warning( diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ba7c691306bb1..87d30c8aefbf0 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -10,6 +10,7 @@ from typing import Optional, Union from vllm.config import (CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig, SpeculativeConfig) from vllm.logger import init_logger +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager, compute_encoder_budget) from vllm.v1.core.kv_cache_manager import KVCacheManager @@ -38,6 +39,7 @@ class Scheduler(SchedulerInterface): speculative_config: Optional[SpeculativeConfig], log_stats: bool, structured_output_manager: StructuredOutputManager, + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, ) -> None: self.scheduler_config = scheduler_config self.cache_config = cache_config @@ -93,6 +95,7 @@ class Scheduler(SchedulerInterface): encoder_compute_budget, encoder_cache_size = compute_encoder_budget( model_config=model_config, scheduler_config=scheduler_config, + mm_registry=mm_registry, ) # NOTE(woosuk): Here, "encoder" includes the vision encoder (and diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 230479f3f15e7..133ccf84832c4 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -137,6 +137,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): encoder_compute_budget, encoder_cache_size = compute_encoder_budget( model_config=model_config, scheduler_config=scheduler_config, + mm_registry=self.mm_registry, ) self.max_num_encoder_input_tokens = encoder_compute_budget self.encoder_cache_size = encoder_cache_size @@ -1439,9 +1440,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): # NOTE: Currently model is profiled with a single non-text # modality with the max possible input tokens even when # it supports multiple. - max_tokens_by_modality_dict = ( - MULTIMODAL_REGISTRY. - get_max_tokens_per_item_by_nonzero_modality(self.model_config)) + max_tokens_by_modality_dict = self.mm_registry \ + .get_max_tokens_per_item_by_nonzero_modality(self.model_config) dummy_data_modality, max_tokens_per_mm_item = max( max_tokens_by_modality_dict.items(), key=lambda item: item[1]) diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index 65a4048ae74d6..abe1b338fb717 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -109,6 +109,7 @@ class TPUModelRunner: encoder_compute_budget, encoder_cache_size = compute_encoder_budget( model_config=model_config, scheduler_config=scheduler_config, + mm_registry=self.mm_registry, ) self.max_num_encoder_input_tokens = encoder_compute_budget self.encoder_cache_size = encoder_cache_size From 46450b8d33eeee3c619e4d6aea6652ee3d16386f Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 27 Mar 2025 18:52:18 +0000 Subject: [PATCH 050/593] Use absolute placement for Ask AI button (#15628) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- docs/source/_static/custom.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/_static/custom.js b/docs/source/_static/custom.js index be0b2a388e404..58bc2ebb9614b 100644 --- a/docs/source/_static/custom.js +++ b/docs/source/_static/custom.js @@ -10,8 +10,8 @@ document.addEventListener("DOMContentLoaded", function () { script.setAttribute("runllm-keyboard-shortcut", "Mod+j"); // cmd-j or ctrl-j to open the widget. script.setAttribute("runllm-name", "vLLM"); script.setAttribute("runllm-position", "BOTTOM_RIGHT"); - script.setAttribute("runllm-position-y", "20%"); - script.setAttribute("runllm-position-x", "3%"); + script.setAttribute("runllm-position-y", "120px"); + script.setAttribute("runllm-position-x", "20px"); script.setAttribute("runllm-assistant-id", "207"); script.async = true; From 4098b72210dc10761bb348b373bbd0fc9b23b0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Thu, 27 Mar 2025 20:15:06 +0100 Subject: [PATCH 051/593] [Bugfix][TPU][V1] Fix recompilation (#15553) Signed-off-by: NickLucche --- .buildkite/run-tpu-v1-test.sh | 4 +- tests/v1/tpu/test_sampler.py | 69 +++--------------------------- vllm/v1/sample/tpu/metadata.py | 8 +--- vllm/v1/worker/tpu_model_runner.py | 8 +++- 4 files changed, 15 insertions(+), 74 deletions(-) diff --git a/.buildkite/run-tpu-v1-test.sh b/.buildkite/run-tpu-v1-test.sh index 6e1f79ae649e3..a93b79c0b1b28 100755 --- a/.buildkite/run-tpu-v1-test.sh +++ b/.buildkite/run-tpu-v1-test.sh @@ -32,7 +32,9 @@ docker run --privileged --net host --shm-size=16G -it \ && echo TEST_5 \ && python3 /workspace/vllm/examples/offline_inference/tpu.py \ && echo TEST_6 \ - && pytest -s -v /workspace/vllm/tests/tpu/worker/test_tpu_model_runner.py" \ + && pytest -s -v /workspace/vllm/tests/tpu/worker/test_tpu_model_runner.py \ + && echo TEST_7 \ + && pytest -s -v /workspace/vllm/tests/v1/tpu/test_sampler.py" \ # TODO: This test fails because it uses RANDOM_SEED sampling diff --git a/tests/v1/tpu/test_sampler.py b/tests/v1/tpu/test_sampler.py index 4e5a57bee3275..f535abedea229 100644 --- a/tests/v1/tpu/test_sampler.py +++ b/tests/v1/tpu/test_sampler.py @@ -1,7 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 -import tempfile -from time import time - import pytest from vllm import LLM, envs @@ -15,60 +12,6 @@ if not envs.VLLM_USE_V1: ) -@pytest.mark.parametrize("model_name", ["D4nt3/Qwen2.5-two-layers"]) -@pytest.mark.skipif(not current_platform.is_tpu(), - reason="This test needs a TPU") -def test_sampler_compilation(model_name: str, monkeypatch): - """ - Check that no recompilation happens despite changing sampling parameters. - We can't read XLA metrics from the engine process, hence we measure time. - """ - with tempfile.TemporaryDirectory() as temp_dir: - monkeypatch.setenv("VLLM_XLA_CACHE_PATH", temp_dir) - # Compiling model init may still take some time, enforce_eager to skip. - llm = LLM(model_name, - enforce_eager=True, - max_num_seqs=16, - max_model_len=1024, - gpu_memory_utilization=0.5) - prompts = [ - "A robot may not injure a human being", - "It is only with the heart that one can see rightly;", - ] - # First inference should be slow - sampling_params = SamplingParams( - temperature=0.7, - # top_p=0.6, # TODO too slow! - top_k=10, - min_p=0.2, - max_tokens=16) - s = time() - _ = llm.generate(prompts, sampling_params) - run1 = time() - s - - # Second request with different params, but for which we - # compiled for in previous eager iteration. - sampling_params = SamplingParams(temperature=0.1, - top_k=12, - min_p=0.8, - max_tokens=24) - s = time() - _ = llm.generate(prompts, sampling_params) - run2 = time() - s - # Much faster after compiling - assert run1 * 0.1 > run2 - print("TIMES", run1, run2) - - # Third request with min_p set to "None". It will not trigger - # recompilation as a default 0 value will be used. - sampling_params = SamplingParams(max_tokens=24, temperature=0.0) - s = time() - _ = llm.generate(prompts, sampling_params) - run3 = time() - s - assert run1 * 0.1 > run3 - print("TIMES", run1, run3) - - @pytest.mark.parametrize("model_name", ["Qwen/Qwen2.5-1.5B-Instruct"]) @pytest.mark.skipif(not current_platform.is_tpu(), reason="This test needs a TPU") @@ -77,13 +20,11 @@ def test_sampler_different(model_name: str): Test significantly different sampling params to assert the model produces different results. """ - llm = LLM( - model_name, - enforce_eager=True, - max_num_seqs=1, - max_model_len=64, - # TODO: setting to 0.5 or it will go OOM - gpu_memory_utilization=0.5) + llm = LLM(model_name, + enforce_eager=False, + max_num_seqs=1, + max_model_len=512, + max_num_batched_tokens=512) prompts = [ "Write a short story about a robot that dreams for the first time." ] diff --git a/vllm/v1/sample/tpu/metadata.py b/vllm/v1/sample/tpu/metadata.py index d605c4b65e9d3..89d3ddf51d748 100644 --- a/vllm/v1/sample/tpu/metadata.py +++ b/vllm/v1/sample/tpu/metadata.py @@ -88,6 +88,7 @@ class TPUSupportedSamplingMetadata: # Copy slice from CPU to corresponding TPU pre-allocated tensor. # Pad value is the default one. cpu_tensor[num_reqs:padded_num_reqs] = fill_val + # Subtle compilation: len(tpu_tensor) must be >= `padded_num_reqs` tpu_tensor[:padded_num_reqs] = cpu_tensor[:padded_num_reqs] # NOTE NickLucche The sync CPU-TPU graph we produce here must be @@ -101,13 +102,6 @@ class TPUSupportedSamplingMetadata: copy_slice(input_batch.min_p_cpu_tensor, input_batch.min_p, DEFAULT_SAMPLING_PARAMS["min_p"]) - # copy_slice(input_batch.frequency_penalties_cpu_tensor, - # input_batch.frequency_penalties) - # copy_slice(input_batch.presence_penalties_cpu_tensor, - # input_batch.presence_penalties) - # copy_slice(input_batch.repetition_penalties_cpu_tensor, - # input_batch.repetition_penalties) - xm.mark_step() xm.wait_device_ops() diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index abe1b338fb717..97dfd23163dff 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -88,6 +88,8 @@ class TPUModelRunner: self.max_model_len = model_config.max_model_len self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size) self.max_num_tokens = scheduler_config.max_num_batched_tokens + # InputBatch needs to work with sampling tensors greater than padding + # to avoid dynamic shapes. Also, avoid suboptimal alignment. self.max_num_reqs = max(scheduler_config.max_num_seqs, MIN_NUM_SEQS) # Model-related. @@ -788,6 +790,7 @@ class TPUModelRunner: dummy_hidden = torch.randn((num_tokens, hsize), device=device, dtype=torch.bfloat16) + # Compile for [8, 16, .., 128,.., `self.max_num_reqs`] while True: indices = torch.zeros( num_reqs_to_sample, @@ -804,7 +807,9 @@ class TPUModelRunner: out = out.cpu() if num_reqs_to_sample >= self.max_num_reqs: break - num_reqs_to_sample *= 2 + # Make sure to compile the `max_num_reqs` upper-limit case + num_reqs_to_sample = _get_padded_num_reqs_with_upper_limit( + num_reqs_to_sample + 1, self.max_num_reqs) xm.wait_device_ops() end = time.perf_counter() logger.info("Compilation finished in in %.2f [secs].", end - start) @@ -897,7 +902,6 @@ class ModelWrapperV1(nn.Module): return hidden_states - # @torch.compile(backend="openxla", fullgraph=True, dynamic=False) def sample_from_hidden( self, hidden_states: torch.Tensor, From 32d669275b1068e3261a47715d30e842817e000b Mon Sep 17 00:00:00 2001 From: cnorman Date: Thu, 27 Mar 2025 17:04:32 -0500 Subject: [PATCH 052/593] Correct PowerPC to modern IBM Power (#15635) Signed-off-by: Christy Norman --- docs/source/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.md b/docs/source/index.md index 1624d5cf5aae7..402f242679041 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -43,7 +43,7 @@ vLLM is flexible and easy to use with: - Tensor parallelism and pipeline parallelism support for distributed inference - Streaming outputs - OpenAI-compatible API server -- Support NVIDIA GPUs, AMD CPUs and GPUs, Intel CPUs, Gaudi® accelerators and GPUs, PowerPC CPUs, TPU, and AWS Trainium and Inferentia Accelerators. +- Support NVIDIA GPUs, AMD CPUs and GPUs, Intel CPUs, Gaudi® accelerators and GPUs, IBM Power CPUs, TPU, and AWS Trainium and Inferentia Accelerators. - Prefix caching support - Multi-lora support From 112b3e5b3b5af2c70a7332d6fbf78ffc4f2a9339 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 27 Mar 2025 18:15:26 -0400 Subject: [PATCH 053/593] [CI] Update rules for applying `tpu` label. (#15634) Signed-off-by: Russell Bryant --- .github/mergify.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/mergify.yml b/.github/mergify.yml index 48b2a76be9359..e071ece6f1d5e 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -90,15 +90,34 @@ pull_request_rules: - name: label-tpu description: Automatically apply tpu label + # Keep this list in sync with `label-tpu-remove` conditions conditions: - or: - - files~=tpu + - files~=tpu.py + - files~=_tpu + - files~=tpu_ + - files~=/tpu/ - files~=pallas actions: label: add: - tpu +- name: label-tpu-remove + description: Automatically remove tpu label + # Keep this list in sync with `label-tpu` conditions + conditions: + - and: + - -files~=tpu.py + - -files~=_tpu + - -files~=tpu_ + - -files~=/tpu/ + - -files~=pallas + actions: + label: + remove: + - tpu + - name: ping author on conflicts and add 'needs-rebase' label conditions: - conflict From 15dac210f0e6b907f191911917238273042552ed Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 27 Mar 2025 16:14:41 -0700 Subject: [PATCH 054/593] [V1] AsyncLLM data parallel (#13923) Signed-off-by: Nick Hill --- .buildkite/test-pipeline.yaml | 5 + examples/offline_inference/data_parallel.py | 22 +- tests/v1/engine/test_engine_core_client.py | 8 +- tests/v1/test_async_llm_dp.py | 109 +++++++ vllm/config.py | 21 +- vllm/distributed/utils.py | 12 + vllm/engine/arg_utils.py | 10 + vllm/envs.py | 8 + vllm/utils.py | 14 +- vllm/v1/core/sched/scheduler.py | 17 +- vllm/v1/engine/__init__.py | 9 +- vllm/v1/engine/async_llm.py | 23 +- vllm/v1/engine/core.py | 208 ++++++++++-- vllm/v1/engine/core_client.py | 332 ++++++++++++++++---- vllm/v1/engine/llm_engine.py | 17 +- vllm/v1/executor/multiproc_executor.py | 38 ++- vllm/v1/metrics/loggers.py | 14 +- vllm/v1/utils.py | 11 +- 18 files changed, 722 insertions(+), 156 deletions(-) create mode 100644 tests/v1/test_async_llm_dp.py diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index f22b2b0ab6f2f..428b4c593c38e 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -135,12 +135,14 @@ steps: - examples/offline_inference/rlhf.py - examples/offline_inference/rlhf_colocate.py - tests/examples/offline_inference/data_parallel.py + - tests/v1/test_async_llm_dp.py commands: # test with tp=2 and external_dp=2 - VLLM_USE_V1=0 torchrun --nproc-per-node=4 distributed/test_torchrun_example.py - torchrun --nproc-per-node=4 distributed/test_torchrun_example.py # test with internal dp - python3 ../examples/offline_inference/data_parallel.py + - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/test_async_llm_dp.py - pytest -v -s distributed/test_utils.py - pytest -v -s compile/test_basic_correctness.py - pytest -v -s distributed/test_pynccl.py @@ -514,7 +516,10 @@ steps: - vllm/worker/worker.py - vllm/worker/model_runner.py - entrypoints/llm/test_collective_rpc.py + - tests/v1/test_async_llm_dp.py + - vllm/v1/engine/ commands: + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/test_async_llm_dp.py - VLLM_ENABLE_V1_MULTIPROCESSING=0 pytest -v -s entrypoints/llm/test_collective_rpc.py - pytest -v -s ./compile/test_basic_correctness.py - pytest -v -s ./compile/test_wrapper.py diff --git a/examples/offline_inference/data_parallel.py b/examples/offline_inference/data_parallel.py index 232afd8b73d00..04a79e2f8ae66 100644 --- a/examples/offline_inference/data_parallel.py +++ b/examples/offline_inference/data_parallel.py @@ -28,6 +28,7 @@ Multi-node: --master-port=13345 """ import os +from time import sleep from vllm import LLM, SamplingParams from vllm.utils import get_open_port @@ -36,14 +37,13 @@ from vllm.utils import get_open_port def main(model, dp_size, local_dp_rank, global_dp_rank, dp_master_ip, dp_master_port, GPUs_per_dp_rank): os.environ["VLLM_DP_RANK"] = str(global_dp_rank) + os.environ["VLLM_DP_RANK_LOCAL"] = str(local_dp_rank) os.environ["VLLM_DP_SIZE"] = str(dp_size) os.environ["VLLM_DP_MASTER_IP"] = dp_master_ip os.environ["VLLM_DP_MASTER_PORT"] = str(dp_master_port) - # set devices for each dp_rank - os.environ["CUDA_VISIBLE_DEVICES"] = ",".join( - str(i) - for i in range(local_dp_rank * GPUs_per_dp_rank, (local_dp_rank + 1) * - GPUs_per_dp_rank)) + + # CUDA_VISIBLE_DEVICES for each DP rank is set automatically inside the + # engine processes. # Sample prompts. prompts = [ @@ -90,6 +90,9 @@ def main(model, dp_size, local_dp_rank, global_dp_rank, dp_master_ip, print(f"DP rank {global_dp_rank}, Prompt: {prompt!r}, " f"Generated text: {generated_text!r}") + # Give engines time to pause their processing loops before exiting. + sleep(1) + if __name__ == "__main__": import argparse @@ -152,8 +155,13 @@ if __name__ == "__main__": procs.append(proc) exit_code = 0 for proc in procs: - proc.join() - if proc.exitcode: + proc.join(timeout=300) + if proc.exitcode is None: + print(f"Killing process {proc.pid} that " + f"didn't stop within 5 minutes.") + proc.kill() + exit_code = 1 + elif proc.exitcode: exit_code = proc.exitcode exit(exit_code) diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index 48f451a589688..68844b877c17d 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -167,11 +167,11 @@ def test_engine_core_client(monkeypatch: pytest.MonkeyPatch, core_client: SyncMPClient = client - result = core_client._call_utility("echo", "testarg") + result = core_client.call_utility("echo", "testarg") assert result == "testarg" with pytest.raises(Exception) as e_info: - core_client._call_utility("echo", None, "help!") + core_client.call_utility("echo", None, "help!") assert str(e_info.value) == "Call to echo method failed: help!" @@ -238,10 +238,10 @@ async def test_engine_core_client_asyncio(monkeypatch: pytest.MonkeyPatch): core_client: AsyncMPClient = client - result = await core_client._call_utility_async("echo", "testarg") + result = await core_client.call_utility_async("echo", "testarg") assert result == "testarg" with pytest.raises(Exception) as e_info: - await core_client._call_utility_async("echo", None, "help!") + await core_client.call_utility_async("echo", None, "help!") assert str(e_info.value) == "Call to echo method failed: help!" diff --git a/tests/v1/test_async_llm_dp.py b/tests/v1/test_async_llm_dp.py new file mode 100644 index 0000000000000..f0e031969e733 --- /dev/null +++ b/tests/v1/test_async_llm_dp.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import os +from contextlib import ExitStack +from typing import Optional + +import pytest + +from vllm import SamplingParams +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.inputs import PromptType +from vllm.platforms import current_platform +from vllm.sampling_params import RequestOutputKind +from vllm.v1.engine.async_llm import AsyncLLM +from vllm.v1.engine.core_client import DPAsyncMPClient + +engine_args = AsyncEngineArgs( + model="ibm-research/PowerMoE-3b", + enforce_eager=True, + disable_log_requests=True, + tensor_parallel_size=int(os.getenv("TP_SIZE", 1)), + data_parallel_size=int(os.getenv("DP_SIZE", 2)), +) + +if not current_platform.supports_v1(engine_args.create_model_config()): + pytest.skip(reason="Requires V1-supporting platform.", + allow_module_level=True) + + +async def generate(engine: AsyncLLM, + request_id: str, + prompt: PromptType, + output_kind: RequestOutputKind, + max_tokens: int, + prompt_logprobs: Optional[int] = None) -> tuple[int, str]: + # Ensure generate doesn't complete too fast for cancellation test. + await asyncio.sleep(0.2) + + count = 0 + sampling_params = SamplingParams(max_tokens=max_tokens, + ignore_eos=True, + output_kind=output_kind, + temperature=0, + prompt_logprobs=prompt_logprobs) + async for out in engine.generate(request_id=request_id, + prompt=prompt, + sampling_params=sampling_params): + + num_tokens = len(out.outputs[0].token_ids) + if output_kind == RequestOutputKind.DELTA: + count += num_tokens + else: + count = num_tokens + + await asyncio.sleep(0.) + + return count, request_id + + +@pytest.mark.parametrize( + "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY]) +@pytest.mark.asyncio +async def test_load(output_kind: RequestOutputKind): + + with ExitStack() as after: + + prompt = "This is a test of data parallel" + + engine = AsyncLLM.from_engine_args(engine_args) + after.callback(engine.shutdown) + + NUM_REQUESTS = 100 + NUM_EXPECTED_TOKENS = 10 + + request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)] + + # Create concurrent requests. + tasks = [] + for request_id in request_ids: + tasks.append( + asyncio.create_task( + generate(engine, request_id, prompt, output_kind, + NUM_EXPECTED_TOKENS))) + + # Confirm that we got all the EXPECTED tokens from the requests. + done, pending = await asyncio.wait(tasks, + return_when=asyncio.FIRST_EXCEPTION) + for task in pending: + task.cancel() + for task in done: + num_generated_tokens, request_id = await task + assert num_generated_tokens == NUM_EXPECTED_TOKENS, ( + f"{request_id} generated {num_generated_tokens} but " + f"expected {NUM_EXPECTED_TOKENS}") + + assert not engine.output_processor.has_unfinished_requests() + + # testing internals here which may break + core_client: DPAsyncMPClient = engine.engine_core + # the engines only synchronize stopping every N steps so + # allow a small amount of time here. + for _ in range(10): + if core_client.num_engines_running == 0: + break + await asyncio.sleep(0.5) + + assert core_client.num_engines_running == 0 + assert not core_client.reqs_in_flight diff --git a/vllm/config.py b/vllm/config.py index 687c8b56ec126..831fa2e4b06eb 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -40,7 +40,8 @@ from vllm.transformers_utils.config import ( from vllm.transformers_utils.s3_utils import S3Model from vllm.transformers_utils.utils import is_s3, maybe_model_redirect from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless, - get_cpu_memory, random_uuid, resolve_obj_by_qualname) + get_cpu_memory, get_open_port, random_uuid, + resolve_obj_by_qualname) if TYPE_CHECKING: from ray.util.placement_group import PlacementGroup @@ -1389,6 +1390,8 @@ class ParallelConfig: tensor_parallel_size: int = 1 # Number of tensor parallel groups. data_parallel_size: int = 1 # Number of data parallel groups. data_parallel_rank: int = 0 # Rank of the data parallel group. + # Local rank of the data parallel group, defaults to global rank. + data_parallel_rank_local: Optional[int] = None # IP of the data parallel master. data_parallel_master_ip: str = "127.0.0.1" data_parallel_master_port: int = 29500 # Port of the data parallel master. @@ -1493,10 +1496,18 @@ class ParallelConfig: self.world_size = self.pipeline_parallel_size * \ self.tensor_parallel_size - self.data_parallel_size = envs.VLLM_DP_SIZE - self.data_parallel_rank = envs.VLLM_DP_RANK - self.data_parallel_master_ip = envs.VLLM_DP_MASTER_IP - self.data_parallel_master_port = envs.VLLM_DP_MASTER_PORT + if self.data_parallel_size > 1: + # Data parallel was specified in the engine args. + self.data_parallel_master_port = get_open_port() + # TODO multi-node + else: + # Otherwise fall back to env vars (e.g. for offline SPMD case). + self.data_parallel_size = envs.VLLM_DP_SIZE + self.data_parallel_rank = envs.VLLM_DP_RANK + self.data_parallel_rank_local = envs.VLLM_DP_RANK_LOCAL + self.data_parallel_master_ip = envs.VLLM_DP_MASTER_IP + self.data_parallel_master_port = envs.VLLM_DP_MASTER_PORT + self.world_size_across_dp = self.world_size * self.data_parallel_size if self.distributed_executor_backend == "external_launcher": diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index 84899358a6d66..b8178af5a2daa 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -15,6 +15,8 @@ import torch from torch.distributed import ProcessGroup, TCPStore from torch.distributed.distributed_c10d import (Backend, PrefixStore, _get_default_timeout, + _shutdown_backend, + _unregister_process_group, is_nccl_available) from torch.distributed.rendezvous import rendezvous @@ -333,3 +335,13 @@ def stateless_init_torch_distributed_process_group( pg._register_backend(device, backend_type, backend_class) return pg + + +def stateless_destroy_torch_distributed_process_group( + pg: ProcessGroup) -> None: + """ + Destroy ProcessGroup returned by + stateless_init_torch_distributed_process_group(). + """ + _shutdown_backend(pg) + _unregister_process_group(pg.group_name) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 53af3e5717c52..a3b83c65a604a 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -114,6 +114,7 @@ class EngineArgs: # number of P/D disaggregation (or other disaggregation) workers pipeline_parallel_size: int = 1 tensor_parallel_size: int = 1 + data_parallel_size: int = 1 enable_expert_parallel: bool = False max_parallel_loading_workers: Optional[int] = None block_size: Optional[int] = None @@ -442,6 +443,14 @@ class EngineArgs: type=int, default=EngineArgs.tensor_parallel_size, help='Number of tensor parallel replicas.') + parser.add_argument('--data-parallel-size', + '-dp', + type=int, + default=EngineArgs.data_parallel_size, + help='Number of data parallel replicas. ' + 'MoE layers will be sharded according to the ' + 'product of the tensor-parallel-size and ' + 'data-parallel-size.') parser.add_argument( '--enable-expert-parallel', action='store_true', @@ -1359,6 +1368,7 @@ class EngineArgs: parallel_config = ParallelConfig( pipeline_parallel_size=self.pipeline_parallel_size, tensor_parallel_size=self.tensor_parallel_size, + data_parallel_size=self.data_parallel_size, enable_expert_parallel=self.enable_expert_parallel, max_parallel_loading_workers=self.max_parallel_loading_workers, disable_custom_all_reduce=self.disable_custom_all_reduce, diff --git a/vllm/envs.py b/vllm/envs.py index e5025485a2501..5334667376b24 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -2,6 +2,7 @@ import hashlib import os +import sys import tempfile from typing import TYPE_CHECKING, Any, Callable, Optional @@ -95,6 +96,7 @@ if TYPE_CHECKING: VLLM_CUDART_SO_PATH: Optional[str] = None VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH: bool = True VLLM_DP_RANK: int = 0 + VLLM_DP_RANK_LOCAL: int = -1 VLLM_DP_SIZE: int = 1 VLLM_DP_MASTER_IP: str = "" VLLM_DP_MASTER_PORT: int = 0 @@ -625,6 +627,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_DP_RANK": lambda: int(os.getenv("VLLM_DP_RANK", "0")), + # Rank of the process in the data parallel setting. + # Defaults to VLLM_DP_RANK when not set. + "VLLM_DP_RANK_LOCAL": + lambda: int( + os.getenv("VLLM_DP_RANK_LOCAL", sys.modules[__name__].VLLM_DP_RANK)), + # World size of the data parallel setting "VLLM_DP_SIZE": lambda: int(os.getenv("VLLM_DP_SIZE", "1")), diff --git a/vllm/utils.py b/vllm/utils.py index 77f4e2dcf5e45..afe68a2b8cb3d 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -578,7 +578,7 @@ def get_open_port() -> int: dp_port = envs.VLLM_DP_MASTER_PORT while True: port = _get_open_port() - if port >= dp_port and port < dp_port + 10: + if dp_port <= port < dp_port + 10: continue return port return _get_open_port() @@ -2176,11 +2176,11 @@ def make_zmq_socket( if socket_type == zmq.constants.PULL: socket.setsockopt(zmq.constants.RCVHWM, 0) socket.setsockopt(zmq.constants.RCVBUF, buf_size) - socket.connect(path) + socket.bind(path) elif socket_type == zmq.constants.PUSH: socket.setsockopt(zmq.constants.SNDHWM, 0) socket.setsockopt(zmq.constants.SNDBUF, buf_size) - socket.bind(path) + socket.connect(path) else: raise ValueError(f"Unknown Socket Type: {socket_type}") @@ -2188,7 +2188,11 @@ def make_zmq_socket( @contextlib.contextmanager -def zmq_socket_ctx(path: str, socket_type: Any) -> Iterator[zmq.Socket]: +def zmq_socket_ctx( + path: str, + socket_type: Any, + linger: int = 0, +) -> Iterator[zmq.Socket]: """Context manager for a ZMQ socket""" ctx = zmq.Context() # type: ignore[attr-defined] @@ -2199,7 +2203,7 @@ def zmq_socket_ctx(path: str, socket_type: Any) -> Iterator[zmq.Socket]: logger.debug("Got Keyboard Interrupt.") finally: - ctx.destroy(linger=0) + ctx.destroy(linger=linger) def is_in_ray_actor(): diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 87d30c8aefbf0..448119761259c 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -37,9 +37,10 @@ class Scheduler(SchedulerInterface): cache_config: CacheConfig, lora_config: Optional[LoRAConfig], speculative_config: Optional[SpeculativeConfig], - log_stats: bool, structured_output_manager: StructuredOutputManager, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, + include_finished_set: bool = False, + log_stats: bool = False, ) -> None: self.scheduler_config = scheduler_config self.cache_config = cache_config @@ -48,6 +49,12 @@ class Scheduler(SchedulerInterface): self.log_stats = log_stats self.structured_output_manager = structured_output_manager + # include_finished_set controls whether a separate set of finished + # request ids should be included in the EngineCoreOutputs returned + # by update_from_outputs(). This is currently used in the multi-engine + # case to track request lifetimes efficiently. + self.include_finished_set = include_finished_set + # Scheduling constraints. self.max_num_running_reqs = self.scheduler_config.max_num_seqs self.max_num_scheduled_tokens = \ @@ -663,10 +670,16 @@ class Scheduler(SchedulerInterface): new_running.append(request) self.running = new_running - return EngineCoreOutputs( + engine_core_outputs = EngineCoreOutputs( outputs=outputs, scheduler_stats=self.make_stats(), ) + if self.include_finished_set: + #TODO currently sending duplicates here, improve this + engine_core_outputs.finished_requests = ( + scheduler_output.finished_req_ids | self.finished_req_ids) + + return engine_core_outputs def add_request(self, request: Request) -> None: self.waiting.append(request) diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index 3699779b3a0fe..0557d0c6c19d0 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -128,12 +128,18 @@ class EngineCoreOutputs( #NOTE(Nick): We could consider ways to make this more compact, # e.g. columnwise layout + engine_index: int = 0 + # [num_reqs] outputs: list[EngineCoreOutput] = [] scheduler_stats: Optional[SchedulerStats] = None timestamp: float = 0.0 utility_output: Optional[UtilityOutput] = None + finished_requests: Optional[set[str]] = None + + # In DP case, used to signal that the engine is paused. + engine_paused: bool = False def __post_init__(self): if self.timestamp == 0.0: @@ -147,4 +153,5 @@ class EngineCoreRequestType(enum.Enum): """ ADD = b'\x00' ABORT = b'\x01' - UTILITY = b'\x02' + START_DP = b'\x02' + UTILITY = b'\x03' diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 3a6811db31327..1fb9ae8cb7a59 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -66,11 +66,17 @@ class AsyncLLM(EngineClient): self.log_requests = log_requests self.log_stats = log_stats - self.stat_loggers: list[StatLoggerBase] = [] + + # Set up stat loggers; independent set for each DP rank. + self.stat_loggers: list[list[StatLoggerBase]] = [] if self.log_stats: - if logger.isEnabledFor(logging.INFO): - self.stat_loggers.append(LoggingStatLogger()) - self.stat_loggers.append(PrometheusStatLogger(vllm_config)) + for i in range(vllm_config.parallel_config.data_parallel_size): + loggers: list[StatLoggerBase] = [] + if logger.isEnabledFor(logging.INFO): + loggers.append(LoggingStatLogger(engine_index=i)) + loggers.append( + PrometheusStatLogger(vllm_config, engine_index=i)) + self.stat_loggers.append(loggers) # Tokenizer (+ ensure liveness if running in another process). self.tokenizer = init_tokenizer_from_configs( @@ -329,6 +335,7 @@ class AsyncLLM(EngineClient): # TODO(rob): make into a coroutine and launch it in # background thread once Prometheus overhead is non-trivial. self._record_stats( + engine_index=outputs.engine_index, scheduler_stats=outputs.scheduler_stats, iteration_stats=iteration_stats, ) @@ -350,12 +357,13 @@ class AsyncLLM(EngineClient): self, scheduler_stats: Optional[SchedulerStats], iteration_stats: Optional[IterationStats], + engine_index: int = 0, ): if not self.log_stats: return assert scheduler_stats is not None - for stat_logger in self.stat_loggers: + for stat_logger in self.stat_loggers[engine_index]: stat_logger.record(scheduler_stats=scheduler_stats, iteration_stats=iteration_stats) @@ -393,8 +401,9 @@ class AsyncLLM(EngineClient): scheduler_outputs=None, model_output=None, ) -> None: - for stat_logger in self.stat_loggers: - stat_logger.log() + for loggers in self.stat_loggers: + for stat_logger in loggers: + stat_logger.log() async def check_health(self) -> None: logger.debug("Called check_health.") diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 42511777feebb..20904cd495f91 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 - +import os import queue import signal +import sys import threading import time from concurrent.futures import Future from inspect import isclass, signature -from multiprocessing.connection import Connection +from logging import DEBUG from typing import Any, Optional import msgspec @@ -14,7 +15,9 @@ import psutil import zmq import zmq.asyncio -from vllm.config import VllmConfig +from vllm.config import ParallelConfig, VllmConfig +from vllm.distributed import stateless_destroy_torch_distributed_process_group +from vllm.executor.multiproc_worker_utils import _add_prefix from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.transformers_utils.config import ( @@ -91,6 +94,8 @@ class EngineCore: cache_config=vllm_config.cache_config, lora_config=vllm_config.lora_config, speculative_config=vllm_config.speculative_config, + include_finished_set=vllm_config.parallel_config.data_parallel_size + > 1, log_stats=self.log_stats, structured_output_manager=self.structured_output_manager, ) @@ -283,10 +288,10 @@ class EngineCoreProc(EngineCore): self, input_path: str, output_path: str, - ready_pipe: Connection, vllm_config: VllmConfig, executor_class: type[Executor], log_stats: bool, + engine_index: int = 0, ): super().__init__(vllm_config, executor_class, log_stats) @@ -302,14 +307,20 @@ class EngineCoreProc(EngineCore): args=(input_path, ), daemon=True).start() threading.Thread(target=self.process_output_socket, - args=(output_path, ), + args=(output_path, engine_index), daemon=True).start() - # Send Readiness signal to EngineClient. - ready_pipe.send({"status": "READY"}) + self.global_unfinished_reqs = False + + self.step_fn = (self.step if self.batch_queue is None else + self.step_with_batch_queue) @staticmethod - def run_engine_core(*args, **kwargs): + def run_engine_core(*args, + dp_rank: int = 0, + local_dp_rank: int = 0, + ready_pipe, + **kwargs): """Launch EngineCore busy loop in background process.""" # Signal handler used for graceful termination. @@ -331,9 +342,21 @@ class EngineCoreProc(EngineCore): signal.signal(signal.SIGINT, signal_handler) parent_process = psutil.Process().parent() - engine_core = None + engine_core: Optional[EngineCoreProc] = None try: - engine_core = EngineCoreProc(*args, **kwargs) + parallel_config: ParallelConfig = kwargs[ + "vllm_config"].parallel_config + if parallel_config.data_parallel_size > 1: + # Set data parallel rank for this engine process. + parallel_config.data_parallel_rank = dp_rank + parallel_config.data_parallel_rank_local = local_dp_rank + engine_core = DPEngineCoreProc(*args, **kwargs) + else: + engine_core = EngineCoreProc(*args, **kwargs) + + # Send Readiness signal to EngineClient. + ready_pipe.send({"status": "READY"}) + engine_core.run_busy_loop() except SystemExit: @@ -351,28 +374,44 @@ class EngineCoreProc(EngineCore): def run_busy_loop(self): """Core busy loop of the EngineCore.""" - step_fn = (self.step - if self.batch_queue is None else self.step_with_batch_queue) - # Loop until process is sent a SIGINT or SIGTERM while True: # 1) Poll the input queue until there is work to do. - while not self.scheduler.has_requests(): - logger.debug("EngineCore busy loop waiting.") - req = self.input_queue.get() - self._handle_client_request(*req) + self._process_input_queue() + # 2) Step the engine core and return the outputs. + self._process_engine_step() - # 2) Handle any new client requests. - while not self.input_queue.empty(): - req = self.input_queue.get_nowait() - self._handle_client_request(*req) + def _process_input_queue(self): + """Exits when an engine step needs to be performed.""" - # 3) Step the engine core. - outputs = step_fn() + waited = False + while not self.global_unfinished_reqs and not ( + self.scheduler.has_requests()): + if logger.isEnabledFor(DEBUG) and self.input_queue.empty(): + logger.debug("EngineCore waiting for work.") + waited = True + req = self.input_queue.get() + self._handle_client_request(*req) - # 4) Put EngineCoreOutputs into the output queue. - if outputs is not None: - self.output_queue.put_nowait(outputs) + if waited: + logger.debug( + "EngineCore loop active - local unfinished: %s, finished: %s.", + self.scheduler.has_unfinished_requests(), + self.scheduler.has_finished_requests()) + + # Handle any more client requests. + while not self.input_queue.empty(): + req = self.input_queue.get_nowait() + self._handle_client_request(*req) + + def _process_engine_step(self): + """Called only when there are unfinished local requests.""" + + # Step the engine core. + outputs = self.step_fn() + # Put EngineCoreOutputs into the output queue. + if outputs is not None: + self.output_queue.put_nowait(outputs) def _handle_client_request(self, request_type: EngineCoreRequestType, request: Any) -> None: @@ -382,6 +421,10 @@ class EngineCoreProc(EngineCore): self.add_request(request) elif request_type == EngineCoreRequestType.ABORT: self.abort_requests(request) + elif request_type == EngineCoreRequestType.START_DP: + if not self.global_unfinished_reqs: + logger.debug("EngineCore starting idle loop.") + self.global_unfinished_reqs = True elif request_type == EngineCoreRequestType.UTILITY: call_id, method_name, args = request output = UtilityOutput(call_id) @@ -432,7 +475,7 @@ class EngineCoreProc(EngineCore): # Push to input queue for core busy loop. self.input_queue.put_nowait((request_type, request)) - def process_output_socket(self, output_path: str): + def process_output_socket(self, output_path: str, engine_index: int): """Output socket IO thread.""" # Msgpack serialization encoding. @@ -443,5 +486,114 @@ class EngineCoreProc(EngineCore): with zmq_socket_ctx(output_path, zmq.constants.PUSH) as socket: while True: outputs = self.output_queue.get() + outputs.engine_index = engine_index encoder.encode_into(outputs, buffer) - socket.send_multipart((buffer, ), copy=False) + socket.send(buffer, copy=False) + + +ENGINE_PAUSED_OUTPUTS = EngineCoreOutputs(engine_paused=True) + + +class DPEngineCoreProc(EngineCoreProc): + """ZMQ-wrapper for running EngineCore in background process + in a data parallel context.""" + + def __init__( + self, + input_path: str, + output_path: str, + vllm_config: VllmConfig, + executor_class: type[Executor], + log_stats: bool, + ): + # Add process-specific prefix to stdout and stderr before + # we initialize the engine. + from multiprocessing import current_process + process_name = current_process().name + pid = os.getpid() + _add_prefix(sys.stdout, process_name, pid) + _add_prefix(sys.stderr, process_name, pid) + + dp_size = vllm_config.parallel_config.data_parallel_size + dp_rank = vllm_config.parallel_config.data_parallel_rank + local_dp_rank = vllm_config.parallel_config.data_parallel_rank_local + + assert dp_size > 1 + assert 0 <= local_dp_rank <= dp_rank < dp_size + + from vllm.platforms import current_platform + if current_platform.is_cuda_alike(): + from vllm.platforms.cuda import device_id_to_physical_device_id + tp_size = vllm_config.parallel_config.tensor_parallel_size + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join( + str(device_id_to_physical_device_id(i)) + for i in range(local_dp_rank * tp_size, (local_dp_rank + 1) * + tp_size)) + + self.dp_group = vllm_config.parallel_config.stateless_init_dp_group() + + # Initialize the engine after setting up environment. + super().__init__(input_path, output_path, vllm_config, executor_class, + log_stats, dp_rank) + + # Counts forward-passes of the model so that we can synchronize + # finished with DP peers every N steps. + self.counter = 0 + + def shutdown(self): + super().shutdown() + if dp_group := getattr(self, "dp_group", None): + stateless_destroy_torch_distributed_process_group(dp_group) + + def run_busy_loop(self): + """Core busy loop of the EngineCore for data parallel case.""" + + # Loop until process is sent a SIGINT or SIGTERM + while True: + # 1) Poll the input queue until there is work to do. + self._process_input_queue() + + local_unfinished_reqs = self.scheduler.has_unfinished_requests() + + if local_unfinished_reqs: + # 2) Step the engine core. + self._process_engine_step() + + # Check if we have now finished all requests. + local_unfinished_reqs = ( + self.scheduler.has_unfinished_requests()) + else: + if self.scheduler.has_finished_requests(): + # There are no unfinished requests, but there are some + # finished requests remaining to be removed from the + # batch state. This engine step won't perform a forward + # pass but will flush the finished requests to ensure + # up-to-date state is returned in the engine outputs. + self._process_engine_step() + + if not self.global_unfinished_reqs: + # All engines are idle. + continue + + # There must be unfinished requests in DP peers, run a + # dummy forward pass. + self.execute_dummy_batch() + + # 3) All-reduce operation to determine global unfinished reqs. + self.global_unfinished_reqs = self._has_global_unfinished_reqs( + local_unfinished_reqs) + + if not self.global_unfinished_reqs: + # Notify client that we are pausing the loop. + self.output_queue.put_nowait(ENGINE_PAUSED_OUTPUTS) + + def _has_global_unfinished_reqs(self, local_unfinished: bool) -> bool: + + # Optimization - only perform finish-sync all-reduce every 16 steps. + self.counter += 1 + if self.counter != 16: + return True + self.counter = 0 + + return ParallelConfig.has_unfinished_dp(self.dp_group, + local_unfinished) diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 13b72c80dc0d4..c41ee6704be0f 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -8,10 +8,11 @@ import threading import uuid import weakref from abc import ABC, abstractmethod +from collections.abc import Awaitable, Sequence from concurrent.futures import Future -from dataclasses import dataclass +from dataclasses import dataclass, field from threading import Thread -from typing import Any, Optional, Union +from typing import Any, Callable, Optional, Union import zmq import zmq.asyncio @@ -60,6 +61,9 @@ class EngineCoreClient(ABC): "is not currently supported.") if multiprocess_mode and asyncio_mode: + if vllm_config.parallel_config.data_parallel_size > 1: + return DPAsyncMPClient(vllm_config, executor_class, log_stats) + return AsyncMPClient(vllm_config, executor_class, log_stats) if multiprocess_mode and not asyncio_mode: @@ -207,28 +211,74 @@ class InprocClient(EngineCoreClient): return self.engine_core.pin_lora(lora_id) +class CoreEngine: + """One per data parallel rank.""" + + def __init__( + self, + vllm_config: VllmConfig, + executor_class: type[Executor], + log_stats: bool, + ctx: Union[zmq.Context, zmq.asyncio.Context], + output_path: str, + index: int = 0, + local_dp_rank: int = 0, + ): + # Paths and sockets for IPC. + input_path = get_open_zmq_ipc_path() + self.input_socket = make_zmq_socket(ctx, input_path, + zmq.constants.PUSH) + try: + # Start EngineCore in background process. + self.proc_handle = BackgroundProcHandle( + input_path=input_path, + output_path=output_path, + process_name=f"EngineCore_{index}", + target_fn=EngineCoreProc.run_engine_core, + process_kwargs={ + "vllm_config": vllm_config, + "dp_rank": index, + "local_dp_rank": local_dp_rank, + "executor_class": executor_class, + "log_stats": log_stats, + }) + + self.num_reqs_in_flight = 0 + finally: + if not hasattr(self, "num_reqs_in_flight"): + # Ensure socket is closed if process fails to start. + self.close() + + def send_multipart(self, msg_parts: Sequence): + return self.input_socket.send_multipart(msg_parts, copy=False) + + def close(self): + if proc_handle := getattr(self, "proc_handle", None): + proc_handle.shutdown() + if socket := getattr(self, "input_socket", None): + socket.close(linger=0) + + @dataclass class BackgroundResources: """Used as a finalizer for clean shutdown, avoiding circular reference back to the client object.""" - ctx: zmq.Context + ctx: Union[zmq.Context] + core_engines: list[CoreEngine] = field(default_factory=list) output_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None - input_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None - proc_handle: Optional[BackgroundProcHandle] = None shutdown_path: Optional[str] = None def __call__(self): """Clean up background resources.""" - if self.proc_handle is not None: - self.proc_handle.shutdown() + for core_engine in self.core_engines: + core_engine.close() + # ZMQ context termination can hang if the sockets # aren't explicitly closed first. if self.output_socket is not None: self.output_socket.close(linger=0) - if self.input_socket is not None: - self.input_socket.close(linger=0) if self.shutdown_path is not None: # We must ensure that the sync output socket is # closed cleanly in its own thread. @@ -284,7 +334,7 @@ class MPClient(EngineCoreClient): self.decoder = MsgpackDecoder(EngineCoreOutputs) # ZMQ setup. - sync_ctx = zmq.Context() + sync_ctx = zmq.Context(io_threads=2) self.ctx = zmq.asyncio.Context(sync_ctx) if asyncio_mode else sync_ctx # This will ensure resources created so far are closed @@ -293,28 +343,38 @@ class MPClient(EngineCoreClient): self.resources = BackgroundResources(ctx=sync_ctx) self._finalizer = weakref.finalize(self, self.resources) - # Paths for IPC. + # Paths and sockets for IPC. self.output_path = get_open_zmq_ipc_path() - input_path = get_open_zmq_ipc_path() - # Start EngineCore in background process. - self.resources.proc_handle = BackgroundProcHandle( - input_path=input_path, - output_path=self.output_path, - process_name="EngineCore", - target_fn=EngineCoreProc.run_engine_core, - process_kwargs={ - "vllm_config": vllm_config, - "executor_class": executor_class, - "log_stats": log_stats, - }) + new_core_engine = lambda index, local_dp_rank=None: CoreEngine( + vllm_config, executor_class, log_stats, self.ctx, self.output_path, + index, local_dp_rank) + + # Start engine core process(es). + self._init_core_engines(vllm_config, new_core_engine, + self.resources.core_engines) + + # Wait for engine core process(es) to start. + for engine in self.resources.core_engines: + engine.proc_handle.wait_for_startup() - # Create input socket. - self.resources.input_socket = make_zmq_socket(self.ctx, input_path, - zmq.constants.PUSH) - self.input_socket = self.resources.input_socket self.utility_results: dict[int, AnyFuture] = {} + def _init_core_engines( + self, + vllm_config: VllmConfig, + new_core_engine: Callable[[int, Optional[int]], CoreEngine], + core_engines: list[CoreEngine], + ) -> None: + + # Default case - single core engine. + dp_rank = vllm_config.parallel_config.data_parallel_rank + local_dp_rank = vllm_config.parallel_config.data_parallel_rank_local + core_engine = new_core_engine( + dp_rank, local_dp_rank if local_dp_rank is not None else dp_rank) + core_engines.append(core_engine) + self.core_engine = core_engine + def shutdown(self): self._finalizer() @@ -370,7 +430,7 @@ class SyncMPClient(MPClient): # shutdown signal, exit thread. break - (frame, ) = out_socket.recv_multipart(copy=False) + frame = out_socket.recv(copy=False) outputs = decoder.decode(frame.buffer) if outputs.utility_output: _process_utility_output(outputs.utility_output, @@ -391,18 +451,15 @@ class SyncMPClient(MPClient): def get_output(self) -> EngineCoreOutputs: return self.outputs_queue.get() - def _send_input(self, request_type: EngineCoreRequestType, - request: Any) -> None: - + def _send_input(self, request_type: EngineCoreRequestType, request: Any): # (RequestType, SerializedRequest) msg = (request_type.value, self.encoder.encode(request)) - self.input_socket.send_multipart(msg, copy=False) + self.core_engine.send_multipart(msg) - def _call_utility(self, method: str, *args) -> Any: + def call_utility(self, method: str, *args) -> Any: call_id = uuid.uuid1().int >> 64 future: Future[Any] = Future() self.utility_results[call_id] = future - self._send_input(EngineCoreRequestType.UTILITY, (call_id, method, args)) @@ -419,34 +476,34 @@ class SyncMPClient(MPClient): self._send_input(EngineCoreRequestType.ABORT, request_ids) def profile(self, is_start: bool = True) -> None: - self._call_utility("profile", is_start) + self.call_utility("profile", is_start) def reset_prefix_cache(self) -> None: - self._call_utility("reset_prefix_cache") + self.call_utility("reset_prefix_cache") def add_lora(self, lora_request: LoRARequest) -> bool: - return self._call_utility("add_lora", lora_request) + return self.call_utility("add_lora", lora_request) def remove_lora(self, lora_id: int) -> bool: - return self._call_utility("remove_lora", lora_id) + return self.call_utility("remove_lora", lora_id) def list_loras(self) -> set[int]: - return self._call_utility("list_loras") + return self.call_utility("list_loras") def pin_lora(self, lora_id: int) -> bool: - return self._call_utility("pin_lora", lora_id) + return self.call_utility("pin_lora", lora_id) def sleep(self, level: int = 1) -> None: - self._call_utility("sleep", level) + self.call_utility("sleep", level) def wake_up(self) -> None: - self._call_utility("wake_up") + self.call_utility("wake_up") def is_sleeping(self) -> bool: - return self._call_utility("is_sleeping") + return self.call_utility("is_sleeping") def execute_dummy_batch(self) -> None: - self._call_utility("execute_dummy_batch") + self.call_utility("execute_dummy_batch") class AsyncMPClient(MPClient): @@ -464,13 +521,21 @@ class AsyncMPClient(MPClient): self.outputs_queue: Optional[asyncio.Queue[EngineCoreOutputs]] = None self.queue_task: Optional[asyncio.Task] = None - async def _start_output_queue_task(self): + self.outputs_handler: Optional[Callable[ + [AsyncMPClient, EngineCoreOutputs], Awaitable[None]]] = None + + def _ensure_output_queue_task(self): + if self.outputs_queue is not None: + return + # Perform IO in separate task to parallelize as much as possible. # Avoid task having direct reference back to the client. self.outputs_queue = asyncio.Queue() decoder = self.decoder utility_results = self.utility_results outputs_queue = self.outputs_queue + output_handler = self.outputs_handler + _self_ref = weakref.ref(self) if output_handler else None output_path = self.output_path output_socket = make_zmq_socket(self.ctx, output_path, zmq.constants.PULL) @@ -483,34 +548,52 @@ class AsyncMPClient(MPClient): if outputs.utility_output: _process_utility_output(outputs.utility_output, utility_results) - else: + continue + + if output_handler is not None: + assert _self_ref is not None + _self = _self_ref() + if not _self: + # Client has been garbage collected, abort. + return + await output_handler(_self, outputs) + + if outputs.outputs or outputs.scheduler_stats: outputs_queue.put_nowait(outputs) self.queue_task = asyncio.create_task(process_outputs_socket(), name="EngineCoreOutputQueueTask") async def get_output_async(self) -> EngineCoreOutputs: - if self.outputs_queue is None: - await self._start_output_queue_task() - assert self.outputs_queue is not None + self._ensure_output_queue_task() + assert self.outputs_queue is not None return await self.outputs_queue.get() async def _send_input(self, request_type: EngineCoreRequestType, request: Any) -> None: + await self.core_engine.send_multipart( + (request_type.value, self.encoder.encode(request))) - msg = (request_type.value, self.encoder.encode(request)) - await self.input_socket.send_multipart(msg, copy=False) + self._ensure_output_queue_task() - if self.outputs_queue is None: - await self._start_output_queue_task() + async def call_utility_async(self, method: str, *args) -> Any: + return await self._call_utility_async(method, + *args, + engine=self.core_engine) - async def _call_utility_async(self, method: str, *args) -> Any: + async def _call_utility_async( + self, + method: str, + *args, + engine: CoreEngine, + ) -> Any: call_id = uuid.uuid1().int >> 64 future = asyncio.get_running_loop().create_future() self.utility_results[call_id] = future - await self._send_input(EngineCoreRequestType.UTILITY, - (call_id, method, args)) - + message = (EngineCoreRequestType.UTILITY.value, + self.encoder.encode((call_id, method, args))) + await engine.send_multipart(message) + self._ensure_output_queue_task() return await future async def add_request_async(self, request: EngineCoreRequest) -> None: @@ -524,31 +607,146 @@ class AsyncMPClient(MPClient): await self._send_input(EngineCoreRequestType.ABORT, request_ids) async def profile_async(self, is_start: bool = True) -> None: - await self._call_utility_async("profile", is_start) + await self.call_utility_async("profile", is_start) async def reset_prefix_cache_async(self) -> None: - await self._call_utility_async("reset_prefix_cache") + await self.call_utility_async("reset_prefix_cache") async def sleep_async(self, level: int = 1) -> None: - await self._call_utility_async("sleep", level) + await self.call_utility_async("sleep", level) async def wake_up_async(self) -> None: - await self._call_utility_async("wake_up") + await self.call_utility_async("wake_up") async def is_sleeping_async(self) -> bool: - return await self._call_utility_async("is_sleeping") + return await self.call_utility_async("is_sleeping") async def execute_dummy_batch_async(self) -> None: - await self._call_utility_async("execute_dummy_batch") + await self.call_utility_async("execute_dummy_batch") async def add_lora_async(self, lora_request: LoRARequest) -> bool: - return await self._call_utility_async("add_lora", lora_request) + return await self.call_utility_async("add_lora", lora_request) async def remove_lora_async(self, lora_id: int) -> bool: - return await self._call_utility_async("remove_lora", lora_id) + return await self.call_utility_async("remove_lora", lora_id) async def list_loras_async(self) -> set[int]: - return await self._call_utility_async("list_loras") + return await self.call_utility_async("list_loras") async def pin_lora_async(self, lora_id: int) -> bool: - return await self._call_utility_async("pin_lora", lora_id) + return await self.call_utility_async("pin_lora", lora_id) + + +class DPAsyncMPClient(AsyncMPClient): + """Asyncio-compatible client for multi-proc, multi-engine (data parallel) + EngineCore.""" + + def __init__(self, vllm_config: VllmConfig, executor_class: type[Executor], + log_stats: bool): + super().__init__(vllm_config, executor_class, log_stats) + + assert len(self.core_engines) > 1 + + # Control message used for triggering dp idle mode loop. + self.start_dp_msg = (EngineCoreRequestType.START_DP.value, + self.encoder.encode(None)) + + self.num_engines_running = 0 + self.reqs_in_flight: dict[str, CoreEngine] = {} + + self.outputs_handler = DPAsyncMPClient.process_engine_outputs # type: ignore[assignment] + + def _init_core_engines( + self, + vllm_config: VllmConfig, + new_core_engine: Callable[[int, Optional[int]], CoreEngine], + core_engines: list[CoreEngine], + ) -> None: + + # Launch a core engine for each data parallel rank. + dp_size = vllm_config.parallel_config.data_parallel_size + for i in range(dp_size): + # Multi-node not yet supported so local_dp_rank == dp_rank. + core_engines.append(new_core_engine(i, i)) + + self.core_engines = core_engines + + async def call_utility_async(self, method: str, *args) -> Any: + # Only the result from the first engine is returned. + return (await asyncio.gather(*[ + self._call_utility_async(method, *args, engine=engine) + for engine in self.core_engines + ]))[0] + + async def add_request_async(self, request: EngineCoreRequest) -> None: + # NOTE: text prompt is not needed in the core engine as it has been + # tokenized. + request.prompt = None + + msg = (EngineCoreRequestType.ADD.value, self.encoder.encode(request)) + + chosen_engine = self.get_core_engine_for_request() + self.reqs_in_flight[request.request_id] = chosen_engine + chosen_engine.num_reqs_in_flight += 1 + if self.num_engines_running >= len(self.core_engines): + await chosen_engine.send_multipart(msg) + else: + # Send request to chosen engine and dp start loop + # control message to all other engines. + self.num_engines_running += len(self.core_engines) + await asyncio.gather(*[ + engine.send_multipart(msg if engine is + chosen_engine else self.start_dp_msg) + for engine in self.core_engines + ]) + + self._ensure_output_queue_task() + + def get_core_engine_for_request(self) -> CoreEngine: + return min(self.core_engines, key=lambda e: e.num_reqs_in_flight) + + @staticmethod + async def process_engine_outputs(self: "DPAsyncMPClient", + outputs: EngineCoreOutputs): + if self.reqs_in_flight: + for req_id in outputs.finished_requests or (): + if engine := self.reqs_in_flight.pop(req_id, None): + engine.num_reqs_in_flight -= 1 + + if outputs.engine_paused: + assert self.num_engines_running >= 1 + self.num_engines_running -= 1 + if not self.num_engines_running and self.reqs_in_flight: + # If there are requests in flight here, they must have + # been sent after the engines paused. We must make + # sure to start the other engines: + self.num_engines_running = len(self.core_engines) + coros = [ + engine.send_multipart(self.start_dp_msg) + for engine in self.core_engines + if not engine.num_reqs_in_flight + ] + if coros: + await asyncio.gather(*coros) + + async def abort_requests_async(self, request_ids: list[str]) -> None: + if not request_ids: + return + + if len(request_ids) == 1: + # Fast-path common case. + if engine := self.reqs_in_flight.get(request_ids[0]): + await self._abort_requests(request_ids, engine) + return + + by_engine: dict[CoreEngine, list[str]] = {} + for req_id in request_ids: + if engine := self.reqs_in_flight.get(req_id): + by_engine.setdefault(engine, []).append(req_id) + for engine, req_ids in by_engine.items(): + await self._abort_requests(req_ids, engine) + + async def _abort_requests(self, request_ids: list[str], + engine: CoreEngine) -> None: + await engine.send_multipart((EngineCoreRequestType.ABORT.value, + self.encoder.encode(request_ids))) diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 7bda3a30d2028..8cc73f9fe7224 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -8,6 +8,7 @@ from typing_extensions import TypeVar import vllm.envs as envs from vllm.config import ParallelConfig, VllmConfig +from vllm.distributed import stateless_destroy_torch_distributed_process_group from vllm.engine.arg_utils import EngineArgs from vllm.engine.metrics_types import StatLoggerBase from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType @@ -60,11 +61,13 @@ class LLMEngine: self.cache_config = vllm_config.cache_config # important: init dp group before init the engine_core - self.parallel_config = vllm_config.parallel_config - self.dp_enabled = self.parallel_config.data_parallel_size > 1 # noqa + # In the decoupled engine case this is handled in EngineCoreProc. + parallel_config = vllm_config.parallel_config + if not multiprocess_mode and parallel_config.data_parallel_size > 1: + self.dp_group = parallel_config.stateless_init_dp_group() + else: + self.dp_group = None self.should_execute_dummy_batch = False - if self.dp_enabled: - self.dp_group = self.parallel_config.stateless_init_dp_group() # Tokenizer (+ ensure liveness if running in another process). self.tokenizer = init_tokenizer_from_configs( @@ -148,7 +151,7 @@ class LLMEngine: def has_unfinished_requests(self) -> bool: has_unfinished = self.output_processor.has_unfinished_requests() - if not self.dp_enabled: + if self.dp_group is None: return has_unfinished return self.has_unfinished_requests_dp(has_unfinished) @@ -280,3 +283,7 @@ class LLMEngine: def pin_lora(self, lora_id: int) -> bool: """Prevent an adapter from being evicted.""" return self.engine_core.pin_lora(lora_id) + + def __del__(self): + if dp_group := getattr(self, "dp_group", None): + stateless_destroy_torch_distributed_process_group(dp_group) diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 21e7d26506d3f..1d5175eb6adc3 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -235,7 +235,10 @@ class WorkerProc: worker_response_mq_handle = self.worker_response_mq.export_handle() # Send Readiness signal to EngineCore process. - with zmq_socket_ctx(ready_path, zmq.constants.PUSH) as ready_socket: + # Set linger here because we want to ensure the message has + # been sent before the context is closed. + with zmq_socket_ctx(ready_path, zmq.constants.PUSH, + linger=10000) as ready_socket: payload = pickle.dumps(worker_response_mq_handle, protocol=pickle.HIGHEST_PROTOCOL) ready_socket.send_string(WorkerProc.READY_STR) @@ -270,11 +273,13 @@ class WorkerProc: proc = context.Process(target=WorkerProc.worker_main, kwargs=process_kwargs, daemon=True) - proc.start() - # Wait for startup - worker_response_mq_handle = WorkerProc.wait_for_startup( - proc, ready_path) + with zmq_socket_ctx(ready_path, zmq.constants.PULL) as ready_socket: + proc.start() + + # Wait for startup + worker_response_mq_handle = WorkerProc.wait_for_startup( + proc, ready_socket) worker_response_mq = MessageQueue.create_from_handle( worker_response_mq_handle, 0) @@ -337,23 +342,22 @@ class WorkerProc: @staticmethod def wait_for_startup( proc: BaseProcess, - ready_path: str, + ready_socket: zmq.Socket, ) -> Optional[Handle]: """Wait until the Worker is ready.""" - with zmq_socket_ctx(ready_path, zmq.constants.PULL) as socket: - # Wait for Worker to send READY. - while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0: - logger.debug("Waiting for WorkerProc to startup.") + # Wait for Worker to send READY. + while ready_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0: + logger.debug("Waiting for WorkerProc to startup.") - if not proc.is_alive(): - raise RuntimeError("WorkerProc failed to start.") + if not proc.is_alive(): + raise RuntimeError("WorkerProc failed to start.") - message = socket.recv_string() - assert message == WorkerProc.READY_STR - handle_frame = socket.recv(copy=False) - handle = pickle.loads(handle_frame.buffer) - return handle + message = ready_socket.recv_string() + assert message == WorkerProc.READY_STR + handle_frame = ready_socket.recv(copy=False) + handle = pickle.loads(handle_frame.buffer) + return handle class ResponseStatus(Enum): SUCCESS = auto() diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index fcb4d4f5a25a6..6ffd00ebd17a1 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -31,7 +31,8 @@ class StatLoggerBase(ABC): class LoggingStatLogger(StatLoggerBase): - def __init__(self): + def __init__(self, engine_index: int = 0): + self.engine_index = engine_index self._reset(time.monotonic()) self.last_scheduler_stats = SchedulerStats() # Prefix cache metrics. This cannot be reset. @@ -78,11 +79,13 @@ class LoggingStatLogger(StatLoggerBase): # Format and print output. logger.info( + "Engine %03d: " "Avg prompt throughput: %.1f tokens/s, " "Avg generation throughput: %.1f tokens/s, " "Running: %d reqs, Waiting: %d reqs, " "GPU KV cache usage: %.1f%%, " "Prefix cache hit rate: %.1f%%", + self.engine_index, prompt_throughput, generation_throughput, scheduler_stats.num_running_reqs, @@ -94,7 +97,7 @@ class LoggingStatLogger(StatLoggerBase): class PrometheusStatLogger(StatLoggerBase): - def __init__(self, vllm_config: VllmConfig): + def __init__(self, vllm_config: VllmConfig, engine_index: int = 0): self._unregister_vllm_metrics() # Use this flag to hide metrics that were deprecated in @@ -102,8 +105,11 @@ class PrometheusStatLogger(StatLoggerBase): self.show_hidden_metrics = \ vllm_config.observability_config.show_hidden_metrics - labelnames = ["model_name"] - labelvalues = [vllm_config.model_config.served_model_name] + labelnames = ["model_name", "engine"] + labelvalues = [ + vllm_config.model_config.served_model_name, + str(engine_index) + ] max_model_len = vllm_config.model_config.max_model_len diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index 6c01ed3de52d7..f42b3501adb3b 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -105,7 +105,7 @@ class BackgroundProcHandle: process_kwargs: dict[Any, Any], ): context = get_mp_context() - reader, writer = context.Pipe(duplex=False) + self.reader, writer = context.Pipe(duplex=False) assert ("ready_pipe" not in process_kwargs and "input_path" not in process_kwargs @@ -115,14 +115,17 @@ class BackgroundProcHandle: process_kwargs["output_path"] = output_path # Run busy loop in background process. - self.proc = context.Process(target=target_fn, kwargs=process_kwargs) + self.proc = context.Process(target=target_fn, + kwargs=process_kwargs, + name=process_name) self._finalizer = weakref.finalize(self, shutdown, self.proc, input_path, output_path) self.proc.start() + def wait_for_startup(self): # Wait for startup. - if reader.recv()["status"] != "READY": - raise RuntimeError(f"{process_name} initialization failed. " + if self.reader.recv()["status"] != "READY": + raise RuntimeError(f"{self.proc.name} initialization failed. " "See root cause above.") def shutdown(self): From bd45912b99e3bad6621fd4d6bc103352ff31bcb7 Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Thu, 27 Mar 2025 21:57:01 -0400 Subject: [PATCH 055/593] [TPU] Lazy Import (#15656) Signed-off-by: rshaw@neuralmagic.com --- vllm/distributed/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index b8178af5a2daa..4206a24465e28 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -15,7 +15,6 @@ import torch from torch.distributed import ProcessGroup, TCPStore from torch.distributed.distributed_c10d import (Backend, PrefixStore, _get_default_timeout, - _shutdown_backend, _unregister_process_group, is_nccl_available) from torch.distributed.rendezvous import rendezvous @@ -343,5 +342,7 @@ def stateless_destroy_torch_distributed_process_group( Destroy ProcessGroup returned by stateless_init_torch_distributed_process_group(). """ + # Lazy import for non-CUDA backends. + from torch.distributed.distributed_c10d import _shutdown_backend _shutdown_backend(pg) _unregister_process_group(pg.group_name) From 726efc6a320ad9a4ef0b0378b40abbd0561ea394 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Fri, 28 Mar 2025 10:12:47 +0800 Subject: [PATCH 056/593] [Quantization][V1] BitsAndBytes support V1 (#15611) Signed-off-by: Jee Jee Li --- .../vision_language/test_mllama.py | 1 - tests/models/test_transformers.py | 1 - tests/quantization/test_bitsandbytes.py | 3 - vllm/config.py | 6 +- vllm/engine/arg_utils.py | 2 +- .../layers/quantization/bitsandbytes.py | 61 ++++++++++++++----- vllm/model_executor/model_loader/loader.py | 2 + 7 files changed, 52 insertions(+), 24 deletions(-) diff --git a/tests/models/encoder_decoder/vision_language/test_mllama.py b/tests/models/encoder_decoder/vision_language/test_mllama.py index ae7a7b028b152..260d2c1093879 100644 --- a/tests/models/encoder_decoder/vision_language/test_mllama.py +++ b/tests/models/encoder_decoder/vision_language/test_mllama.py @@ -425,7 +425,6 @@ def test_bnb_regression( max_model_len=4096, max_num_seqs=2, quantization="bitsandbytes", - load_format="bitsandbytes", ) sampling_params = SamplingParams( temperature=0, diff --git a/tests/models/test_transformers.py b/tests/models/test_transformers.py index c45fc7e649ec8..65bb11d6b5e4e 100644 --- a/tests/models/test_transformers.py +++ b/tests/models/test_transformers.py @@ -72,7 +72,6 @@ def test_distributed( "meta-llama/Llama-3.2-1B-Instruct", { "quantization": "bitsandbytes", - "load_format": "bitsandbytes", }, ), ]) diff --git a/tests/quantization/test_bitsandbytes.py b/tests/quantization/test_bitsandbytes.py index 1b6a918401487..533b055ee6d53 100644 --- a/tests/quantization/test_bitsandbytes.py +++ b/tests/quantization/test_bitsandbytes.py @@ -101,8 +101,6 @@ def test_load_pp_4bit_bnb_model(model_name, description) -> None: "--enable-prefix-caching", "--quantization", "bitsandbytes", - "--load-format", - "bitsandbytes", "--gpu-memory-utilization", "0.7", ] @@ -137,7 +135,6 @@ def validate_generated_texts(hf_runner, # when using distributed inference with vllm_runner(model_name, quantization='bitsandbytes', - load_format='bitsandbytes', tensor_parallel_size=vllm_tp_size, enforce_eager=False) as llm: vllm_outputs = llm.generate_greedy(prompts, 8) diff --git a/vllm/config.py b/vllm/config.py index 831fa2e4b06eb..5c73ff56ebbcf 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -682,8 +682,9 @@ class ModelConfig: def _verify_bnb_config(self) -> None: """ - The current version of bitsandbytes (0.44.0) with 8-bit models does not + The current version of bitsandbytes (0.45.3) with 8-bit models does not yet support CUDA graph. + # TODO Remove this when bitsandbytes supports. """ is_bitsandbytes = self.quantization == "bitsandbytes" has_quantization_config = (getattr(self.hf_config, @@ -698,8 +699,9 @@ class ModelConfig: not self.enforce_eager, ]): logger.warning( - "CUDA graph is not supported on BitAndBytes 8bit yet, " + "CUDA graph is not supported on BitsAndBytes 8bit yet, " "fallback to the eager mode.") + self.enforce_eager = True def _verify_with_expert_parallelism(self) -> None: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index a3b83c65a604a..d049f773caccd 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1616,7 +1616,7 @@ class EngineArgs: return False # Some quantization is not compatible with torch.compile. - V1_UNSUPPORTED_QUANT = ["bitsandbytes", "gguf"] + V1_UNSUPPORTED_QUANT = ["gguf"] if model_config.quantization in V1_UNSUPPORTED_QUANT: _raise_or_fallback( feature_name=f"--quantization {model_config.quantization}", diff --git a/vllm/model_executor/layers/quantization/bitsandbytes.py b/vllm/model_executor/layers/quantization/bitsandbytes.py index 1e8e7aa1b8c12..f5d32efe83688 100644 --- a/vllm/model_executor/layers/quantization/bitsandbytes.py +++ b/vllm/model_executor/layers/quantization/bitsandbytes.py @@ -9,6 +9,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, set_weight_attrs) from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig) +from vllm.utils import direct_register_custom_op class BitsAndBytesConfig(QuantizationConfig): @@ -321,9 +322,6 @@ class BitsAndBytesLinearMethod(LinearMethodBase): x: torch.Tensor, bias: Optional[torch.Tensor] = None) -> torch.Tensor: - # only load the bitsandbytes module when needed - from bitsandbytes import matmul_4bit - original_type = x.dtype original_shape = x.shape reshape_after_matmul = False @@ -343,19 +341,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase): out_dim_1, dtype=torch.bfloat16, device=x.device) - - current_index = 0 - for i in range(len(quant_states)): - output_size = quant_states[i].shape[0] - # It is more efficient to use out kwarg like - # matmul_4bit(..., out = ...). Infeasible now due to the bug - # https://github.com/TimDettmers/bitsandbytes/issues/1235. - # Need to change after the bug is fixed. - out[:, current_index:current_index + output_size] = matmul_4bit( - bf_x, qweight[offsets[i]:offsets[i + 1]].t(), quant_states[i]) - - current_index += output_size - + apply_bnb_4bit(bf_x, qweight, offsets, out) out = out.to(original_type) if reshape_after_matmul: @@ -365,3 +351,46 @@ class BitsAndBytesLinearMethod(LinearMethodBase): out += bias return out + + +def _apply_bnb_4bit( + x: torch.Tensor, + weight: torch.Tensor, + offsets: torch.Tensor, + out: torch.Tensor, +) -> None: + # only load the bitsandbytes module when needed + from bitsandbytes import matmul_4bit + quant_states = weight.bnb_quant_state + current_index = 0 + for i in range(len(quant_states)): + output_size = quant_states[i].shape[0] + # It is more efficient to use out kwarg like + # matmul_4bit(..., out = ...). Infeasible now due to the bug + # https://github.com/TimDettmers/bitsandbytes/issues/1235. + # Need to change after the bug is fixed. + out[:, current_index:current_index + output_size] = matmul_4bit( + x, weight[offsets[i]:offsets[i + 1]].t(), quant_states[i]) + current_index += output_size + + +def _apply_bnb_4bit_fake( + x: torch.Tensor, + weight: torch.Tensor, + offsets: torch.Tensor, + out: torch.Tensor, +) -> None: + return + + +try: + direct_register_custom_op( + op_name="apply_bnb_4bit", + op_func=_apply_bnb_4bit, + mutates_args=["out"], + fake_impl=_apply_bnb_4bit_fake, + ) + apply_bnb_4bit = torch.ops.vllm.apply_bnb_4bit + +except AttributeError as error: + raise error diff --git a/vllm/model_executor/model_loader/loader.py b/vllm/model_executor/model_loader/loader.py index c969f18b822c4..5649cf2dd2cf1 100644 --- a/vllm/model_executor/model_loader/loader.py +++ b/vllm/model_executor/model_loader/loader.py @@ -1259,6 +1259,8 @@ class BitsAndBytesModelLoader(BaseModelLoader): pack_ratio) offsets = np.concatenate(([0], np.cumsum(num_elements))) + # Make torch infer_schema happy + offsets = torch.tensor(offsets).cpu() set_weight_attrs(param, {"bnb_shard_offsets": offsets}) if load_8bit: From 4e0f6076be71532272f114429a24a559f2656bef Mon Sep 17 00:00:00 2001 From: Kebe Date: Fri, 28 Mar 2025 10:13:41 +0800 Subject: [PATCH 057/593] [Bugfix] Fix failure to launch in Tensor Parallel TP mode on macOS. (#14948) Signed-off-by: Kebe Signed-off-by: youkaichao Co-authored-by: youkaichao --- docs/source/design/multiprocessing.md | 4 ++-- vllm/distributed/device_communicators/shm_broadcast.py | 9 +++++++-- vllm/platforms/cpu.py | 8 ++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/source/design/multiprocessing.md b/docs/source/design/multiprocessing.md index 55dae0bb92d4e..43fe5fe2e5e94 100644 --- a/docs/source/design/multiprocessing.md +++ b/docs/source/design/multiprocessing.md @@ -24,7 +24,7 @@ This document describes how vLLM deals with these challenges. [Python multiprocessing methods](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods) include: - `spawn` - spawn a new Python process. This will be the default as of Python - 3.14. + 3.14. In macOS, this is already the default. - `fork` - Use `os.fork()` to fork the Python interpreter. This is the default in Python versions prior to 3.14. @@ -34,7 +34,7 @@ This document describes how vLLM deals with these challenges. ### Tradeoffs `fork` is the fastest method, but is incompatible with dependencies that use -threads. +threads. If you are under macOS, using `fork` may cause the process to crash. `spawn` is more compatible with dependencies, but can be problematic when vLLM is used as a library. If the consuming code does not use a `__main__` guard (`if diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py index 0d54fc73c882b..11ed7c0843779 100644 --- a/vllm/distributed/device_communicators/shm_broadcast.py +++ b/vllm/distributed/device_communicators/shm_broadcast.py @@ -125,8 +125,13 @@ class ShmRingBuffer: lambda *args, **kwargs: None): try: self.shared_memory = shared_memory.SharedMemory(name=name) - assert ( - self.shared_memory.size == self.total_bytes_of_buffer) + # See https://docs.python.org/3/library/multiprocessing.shared_memory.html # noqa + # Some platforms allocate memory based on page size, + # so the shared memory block size may be larger or equal + # to the requested size. The size parameter is ignored + # when attaching to an existing block. + assert (self.shared_memory.size + >= self.total_bytes_of_buffer) except FileNotFoundError: # we might deserialize the object in a different node # in this case, this object is not used, diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 0eb747a4c4514..619219023f4da 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import sys from typing import TYPE_CHECKING, Optional import psutil @@ -148,6 +149,13 @@ class CpuPlatform(Platform): # To hint IPEX uses shared memory based AllReduce os.environ["LOCAL_WORLD_SIZE"] = str( vllm_config.parallel_config.tensor_parallel_size) + if sys.platform == "darwin" and \ + envs.VLLM_WORKER_MULTIPROC_METHOD == "fork": + if os.environ.get('VLLM_WORKER_MULTIPROC_METHOD', None) is None: + logger.warning( + "Default to spawn method on MacOS. If this is not desired," + " set VLLM_WORKER_MULTIPROC_METHOD to fork explicitly.") + os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn' @classmethod def is_pin_memory_available(cls) -> bool: From b4245a48df84e5e807b92de6066728eeeaff9190 Mon Sep 17 00:00:00 2001 From: wwl2755 Date: Thu, 27 Mar 2025 21:43:40 -0500 Subject: [PATCH 058/593] [Doc] Fix dead links in Job Board (#15637) Signed-off-by: wwl2755 --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3e790827f53bb..a83ad764125c5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -104,7 +104,7 @@ myst_url_schemes = { "classes": ["github"], }, "gh-project": { - "url": "https://github.com/vllm-project/projects/{{path}}", + "url": "https://github.com/orgs/vllm-project/projects/{{path}}", "title": "Project #{{path}}", "classes": ["github"], }, From 8a49eea74bb1e664381d32f1d041b5d1e651664d Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Thu, 27 Mar 2025 22:45:05 -0400 Subject: [PATCH 059/593] [CI][TPU] Temporarily Disable Quant Test on TPU (#15649) Signed-off-by: rshaw@neuralmagic.com --- .buildkite/run-tpu-v1-test.sh | 9 +++++---- tests/v1/tpu/test_basic.py | 3 --- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.buildkite/run-tpu-v1-test.sh b/.buildkite/run-tpu-v1-test.sh index a93b79c0b1b28..7bd91575e1729 100755 --- a/.buildkite/run-tpu-v1-test.sh +++ b/.buildkite/run-tpu-v1-test.sh @@ -28,15 +28,16 @@ docker run --privileged --net host --shm-size=16G -it \ && echo TEST_3 \ && pytest -v -s /workspace/vllm/tests/entrypoints/llm/test_accuracy.py::test_lm_eval_accuracy_v1_engine \ && echo TEST_4 \ - && pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py \ - && echo TEST_5 \ && python3 /workspace/vllm/examples/offline_inference/tpu.py \ - && echo TEST_6 \ + && echo TEST_5 \ && pytest -s -v /workspace/vllm/tests/tpu/worker/test_tpu_model_runner.py \ - && echo TEST_7 \ + && echo TEST_6 \ && pytest -s -v /workspace/vllm/tests/v1/tpu/test_sampler.py" \ # TODO: This test fails because it uses RANDOM_SEED sampling # && VLLM_USE_V1=1 pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \ +# TODO: Re-enable this after fixing recompilation in quantization. +# && echo TEST_4 \ +# && pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py \ diff --git a/tests/v1/tpu/test_basic.py b/tests/v1/tpu/test_basic.py index 417483853916b..591aa9c5878ae 100644 --- a/tests/v1/tpu/test_basic.py +++ b/tests/v1/tpu/test_basic.py @@ -31,14 +31,12 @@ TENSOR_PARALLEL_SIZES = [1] reason="This is a basic test for TPU only") @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("max_tokens", [5]) -@pytest.mark.parametrize("enforce_eager", [True]) @pytest.mark.parametrize("tensor_parallel_size", TENSOR_PARALLEL_SIZES) def test_models( vllm_runner: type[VllmRunner], monkeypatch: pytest.MonkeyPatch, model: str, max_tokens: int, - enforce_eager: bool, tensor_parallel_size: int, ) -> None: prompt = "The next numbers of the sequence " + ", ".join( @@ -51,7 +49,6 @@ def test_models( with vllm_runner( model, max_model_len=8192, - enforce_eager=enforce_eager, gpu_memory_utilization=0.7, max_num_seqs=16, tensor_parallel_size=tensor_parallel_size) as vllm_model: From 4ae17bf1e242d18d5cbf2eacdaf60185957d6f5b Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 27 Mar 2025 20:45:55 -0600 Subject: [PATCH 060/593] Revert "Use Cache Hinting for fused_moe kernel (#15511)" (#15645) Signed-off-by: Wes Medford --- .../model_executor/layers/fused_moe/fused_moe.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 0929530ebec4c..70d0037d7cb01 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -189,11 +189,7 @@ def fused_moe_kernel_gptq_awq( mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), other=0.0) - b = tl.load( - b_ptrs, - cache_modifier=".cg", - eviction_policy="evict_last", - ) + b = tl.load(b_ptrs) if use_int4_w4a16: b = (b >> b_shifter) & 0xF @@ -395,13 +391,9 @@ def fused_moe_kernel( mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), other=0.0) - b = tl.load( - b_ptrs, - mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, - other=0.0, - cache_modifier=".cg", - eviction_policy="evict_last", - ) + b = tl.load(b_ptrs, + mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, + other=0.0) # We accumulate along the K dimension. if use_int8_w8a16: accumulator = tl.dot(a, b.to(compute_type), acc=accumulator) From e7f720ea569c37d026d80cedc78944fe0dbc6a86 Mon Sep 17 00:00:00 2001 From: Chen Xia Date: Thu, 27 Mar 2025 19:47:05 -0700 Subject: [PATCH 061/593] [Misc]add coding benchmark for speculative decoding (#15303) Signed-off-by: CXIAAAAA --- benchmarks/benchmark_dataset.py | 63 ++++++++++++++++++++++++++++++ benchmarks/benchmark_serving.py | 16 +++++--- benchmarks/benchmark_throughput.py | 43 ++++++++++++-------- 3 files changed, 101 insertions(+), 21 deletions(-) diff --git a/benchmarks/benchmark_dataset.py b/benchmarks/benchmark_dataset.py index 0567875f9862f..38ef739c69f9e 100644 --- a/benchmarks/benchmark_dataset.py +++ b/benchmarks/benchmark_dataset.py @@ -715,3 +715,66 @@ class VisionArenaDataset(HuggingFaceDataset): )) self.maybe_oversample_requests(sampled_requests, num_requests) return sampled_requests + + +# ----------------------------------------------------------------------------- +# Instruct Coder Dataset Implementation +# ----------------------------------------------------------------------------- + + +class InstructCoderDataset(HuggingFaceDataset): + """ + InstructCoder Dataset. + https://huggingface.co/datasets/likaixin/InstructCoder + + InstructCoder is the dataset designed for general code editing. + It consists of 114,239 instruction-input-output triplets, + and covers multiple distinct code editing scenario. + """ + + DEFAULT_OUTPUT_LEN = 200 # this is the average default output length + DEFAULT_NUM_REQUESTS = 1000 + INSTRUCT_CODER_DATASET_PATH = "likaixin/InstructCoder" + + def __init__( + self, + **kwargs, + ) -> None: + super().__init__(**kwargs) + if self.dataset_path != self.INSTRUCT_CODER_DATASET_PATH: + raise ValueError(f"Only support likaixin/InstructCoder dataset.\ + This data path {self.dataset_path} is not valid.") + if self.dataset_subset is None and self.dataset_split != "train": + raise ValueError("Dataset split must be 'train'.") + + def load_data(self) -> None: + dataset = load_dataset( + self.dataset_path, + name=self.dataset_subset, + split=self.dataset_split, + streaming=True, + ) + self.data = dataset.shuffle(seed=self.random_seed) + + def sample(self, + tokenizer: PreTrainedTokenizerBase, + num_requests: int, + output_len: Optional[int] = None, + enable_multimodal_chat: bool = False, + **kwargs) -> list: + output_len = (output_len + if output_len is not None else self.DEFAULT_OUTPUT_LEN) + sampled_requests = [] + for item in self.data: + if len(sampled_requests) >= num_requests: + break + prompt = f"{item['instruction']}:\n{item['input']}" + prompt_len = len(tokenizer(prompt).input_ids) + sampled_requests.append( + SampleRequest( + prompt=prompt, + prompt_len=prompt_len, + expected_output_len=output_len, + )) + self.maybe_oversample_requests(sampled_requests, num_requests) + return sampled_requests diff --git a/benchmarks/benchmark_serving.py b/benchmarks/benchmark_serving.py index 47627126b6688..82c6b426b9a2b 100644 --- a/benchmarks/benchmark_serving.py +++ b/benchmarks/benchmark_serving.py @@ -53,8 +53,9 @@ except ImportError: from argparse import ArgumentParser as FlexibleArgumentParser from benchmark_dataset import (BurstGPTDataset, HuggingFaceDataset, - RandomDataset, SampleRequest, ShareGPTDataset, - SonnetDataset, VisionArenaDataset) + InstructCoderDataset, RandomDataset, + SampleRequest, ShareGPTDataset, SonnetDataset, + VisionArenaDataset) from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json MILLISECONDS_TO_SECONDS_CONVERSION = 1000 @@ -588,9 +589,14 @@ def main(args: argparse.Namespace): elif args.dataset_name == "hf": # Choose between VisionArenaDataset # and HuggingFaceDataset based on provided parameters. - dataset_class = (VisionArenaDataset if args.dataset_path - == VisionArenaDataset.VISION_ARENA_DATASET_PATH - and args.hf_subset is None else HuggingFaceDataset) + dataset_class = HuggingFaceDataset + if args.dataset_path == VisionArenaDataset.VISION_ARENA_DATASET_PATH: + assert args.hf_subset is None, "VisionArenaDataset needs hf_subset to be None." #noqa: E501 + dataset_class = VisionArenaDataset + elif args.dataset_path == "likaixin/InstructCoder": + dataset_class = InstructCoderDataset + args.hf_split = "train" + input_requests = dataset_class( dataset_path=args.dataset_path, dataset_subset=args.hf_subset, diff --git a/benchmarks/benchmark_throughput.py b/benchmarks/benchmark_throughput.py index 53869db478c51..f2f68b0d1e5e2 100644 --- a/benchmarks/benchmark_throughput.py +++ b/benchmarks/benchmark_throughput.py @@ -12,8 +12,9 @@ from typing import Any, Optional, Union import torch import uvloop from benchmark_dataset import (BurstGPTDataset, HuggingFaceDataset, - RandomDataset, SampleRequest, ShareGPTDataset, - SonnetDataset, VisionArenaDataset) + InstructCoderDataset, RandomDataset, + SampleRequest, ShareGPTDataset, SonnetDataset, + VisionArenaDataset) from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json from tqdm import tqdm from transformers import (AutoModelForCausalLM, AutoTokenizer, @@ -300,6 +301,7 @@ def get_requests(args, tokenizer): "input_len": args.input_len, "output_len": args.output_len, } + if args.dataset_path is None or args.dataset_name == "random": sample_kwargs["range_ratio"] = args.random_range_ratio sample_kwargs["prefix_len"] = args.prefix_len @@ -317,17 +319,21 @@ def get_requests(args, tokenizer): elif args.dataset_name == "burstgpt": dataset_cls = BurstGPTDataset elif args.dataset_name == "hf": - if args.backend != "vllm-chat": - raise ValueError( - "hf datasets only are supported by vllm-chat backend") - # Choose between VisionArenaDataset and HuggingFaceDataset based on - # provided parameters. - dataset_cls = (VisionArenaDataset if args.dataset_path - == VisionArenaDataset.VISION_ARENA_DATASET_PATH - and args.hf_subset is None else HuggingFaceDataset) - common_kwargs['dataset_subset'] = args.hf_subset - common_kwargs['dataset_split'] = args.hf_split - sample_kwargs["enable_multimodal_chat"] = True + if args.dataset_path == VisionArenaDataset.VISION_ARENA_DATASET_PATH: + if args.args.backend == "vllm-chat": + raise ValueError( + "hf datasets only are supported by vllm-chat backend") + # Choose between VisionArenaDataset and HuggingFaceDataset based on + # provided parameters. + dataset_cls = (VisionArenaDataset if args.dataset_path + == VisionArenaDataset.VISION_ARENA_DATASET_PATH + and args.hf_subset is None else HuggingFaceDataset) + common_kwargs['dataset_subset'] = args.hf_subset + common_kwargs['dataset_split'] = args.hf_split + sample_kwargs["enable_multimodal_chat"] = True + elif args.dataset_path == "likaixin/InstructCoder": + dataset_cls = InstructCoderDataset + common_kwargs['dataset_split'] = "train" else: raise ValueError(f"Unknown dataset name: {args.dataset_name}") @@ -462,9 +468,14 @@ def validate_args(args): warnings.warn("--hf-subset and --hf-split will be ignored \ since --dataset-name is not 'hf'.", stacklevel=2) - elif args.dataset_name == "hf" and args.backend != "vllm-chat": - raise ValueError( - "When --dataset-name is 'hf', backend must be 'vllm-chat'") + elif args.dataset_name == "hf": + if args.dataset_path == VisionArenaDataset.VISION_ARENA_DATASET_PATH: + assert args.backend == "vllm-chat", "VisionArenaDataset needs to use vllm-chat as the backend." #noqa: E501 + elif args.dataset_path == "likaixin/InstructCoder": + assert args.backend == "vllm", "InstructCoder dataset needs to use vllm as the backend." #noqa: E501 + else: + raise ValueError( + f"{args.dataset_path} is not supported by hf dataset.") # --random-range-ratio: only used when dataset_name is 'random' if args.dataset_name != 'random' and args.random_range_ratio is not None: From 4d0ec37267afaf988e32174ebc31f24268076491 Mon Sep 17 00:00:00 2001 From: Gregory Shtrasberg <156009573+gshtras@users.noreply.github.com> Date: Thu, 27 Mar 2025 22:58:16 -0400 Subject: [PATCH 062/593] [Quantization][FP8] Adding support for fp8 gemm layer input in fp8 (#14578) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gregory Shtrasberg Co-authored-by: Luka Govedič --- .../schemes/compressed_tensors_w8a8_fp8.py | 2 ++ .../layers/quantization/fbgemm_fp8.py | 2 ++ .../model_executor/layers/quantization/fp8.py | 17 ++++++++++++ .../quark/schemes/quark_w8a8_fp8.py | 2 ++ .../layers/quantization/utils/w8a8_utils.py | 27 ++++++++++++------- 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py index 27a74d677da7b..e99a452963f48 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -23,6 +23,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): def __init__(self, strategy: str, is_static_input_scheme: bool): self.strategy = strategy + self.out_dtype = torch.get_default_dtype() self.is_static_input_scheme = is_static_input_scheme self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True) @@ -143,5 +144,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): return self.fp8_linear.apply(input=x, weight=layer.weight, weight_scale=layer.weight_scale, + out_dtype=self.out_dtype, input_scale=layer.input_scale, bias=bias) diff --git a/vllm/model_executor/layers/quantization/fbgemm_fp8.py b/vllm/model_executor/layers/quantization/fbgemm_fp8.py index 1cc431c5cc7be..7dddc40f3446d 100644 --- a/vllm/model_executor/layers/quantization/fbgemm_fp8.py +++ b/vllm/model_executor/layers/quantization/fbgemm_fp8.py @@ -73,6 +73,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase): def __init__(self, quant_config: FBGEMMFp8Config): self.quant_config = quant_config self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True) + self.out_dtype = torch.get_default_dtype() def create_weights( self, @@ -161,6 +162,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase): return self.fp8_linear.apply(input=x, weight=layer.weight, weight_scale=layer.weight_scale, + out_dtype=self.out_dtype, input_scale=None, input_scale_ub=layer.input_scale_ub, bias=bias) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index f3907b4784b54..11bfdb4180531 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -116,6 +116,21 @@ class Fp8Config(QuantizationConfig): return Fp8KVCacheMethod(self) return None + def get_cache_scale(self, name: str) -> Optional[str]: + """ + Check whether the param name matches the format for k/v cache scales + in compressed-tensors. If this is the case, return its equivalent + param name expected by vLLM + + :param name: param name + :return: matching param name for KV cache scale in vLLM + """ + if name.endswith(".output_scale") and ".k_proj" in name: + return name.replace(".k_proj.output_scale", ".attn.k_scale") + if name.endswith(".output_scale") and ".v_proj" in name: + return name.replace(".v_proj.output_scale", ".attn.v_scale") + return None + class Fp8LinearMethod(LinearMethodBase): """Linear method for FP8. @@ -138,6 +153,7 @@ class Fp8LinearMethod(LinearMethodBase): def __init__(self, quant_config: Fp8Config): self.quant_config = quant_config self.cutlass_block_fp8_supported = cutlass_block_fp8_supported() + self.out_dtype = torch.get_default_dtype() # For GPUs that lack FP8 hardware support, we can leverage the Marlin # kernel for fast weight-only FP8 quantization @@ -386,6 +402,7 @@ class Fp8LinearMethod(LinearMethodBase): return self.fp8_linear.apply(input=x, weight=layer.weight, weight_scale=layer.weight_scale, + out_dtype=self.out_dtype, input_scale=layer.input_scale, bias=bias) diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py index 3e4251e46931c..c161849c8c5a2 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py @@ -22,6 +22,7 @@ class QuarkW8A8Fp8(QuarkScheme): self.qscheme = qscheme self.is_static_input_scheme = is_static_input_scheme self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True) + self.out_dtype = torch.get_default_dtype() @classmethod def get_min_capability(cls) -> int: @@ -134,5 +135,6 @@ class QuarkW8A8Fp8(QuarkScheme): return self.fp8_linear.apply(input=x, weight=layer.weight, weight_scale=layer.weight_scale, + out_dtype=self.out_dtype, input_scale=layer.input_scale, bias=bias) diff --git a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py index c2bd4bce560e7..b8e6384d7359f 100644 --- a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py @@ -163,6 +163,7 @@ class Fp8LinearOp: input: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, + out_dtype: Optional[torch.dtype] = None, input_scale: Optional[torch.Tensor] = None, input_scale_ub: Optional[torch.Tensor] = None, bias: Optional[torch.Tensor] = None, @@ -182,8 +183,13 @@ class Fp8LinearOp: if use_per_token_if_dynamic is None: use_per_token_if_dynamic = self.use_per_token_if_dynamic + if out_dtype is None: + out_dtype = input.dtype + # cutlass_scaled_mm supports per tensor/channel W and per tensor/token A if self.cutlass_fp8_supported: + assert input.dtype != current_platform.fp8_dtype( + ), "FP8 input to cutlass is not currently implemented" qinput, x_scale = ops.scaled_fp8_quant( input_2d, input_scale, @@ -193,7 +199,7 @@ class Fp8LinearOp: # Fused GEMM_DQ output = ops.cutlass_scaled_mm(qinput, weight, - out_dtype=input.dtype, + out_dtype=out_dtype, scale_a=x_scale, scale_b=weight_scale, bias=bias) @@ -202,12 +208,15 @@ class Fp8LinearOp: # torch.scaled_mm supports per tensor weights + activations only # so fallback to naive if per channel or per token else: - # Maybe apply padding to output, see comment in __init__ - qinput, x_scale = ops.scaled_fp8_quant( - input_2d, - input_scale, - num_token_padding=self.output_padding, - use_per_token_if_dynamic=use_per_token_if_dynamic) + if input.dtype != current_platform.fp8_dtype(): + # Maybe apply padding to output, see comment in __init__ + qinput, x_scale = ops.scaled_fp8_quant( + input_2d, + input_scale, + num_token_padding=self.output_padding, + use_per_token_if_dynamic=use_per_token_if_dynamic) + else: + qinput, x_scale = input_2d, input_scale per_tensor_weights = (weight_scale.numel() == 1) per_tensor_activations = (x_scale.numel() == 1) @@ -216,7 +225,7 @@ class Fp8LinearOp: # Fused GEMM_DQ output = torch._scaled_mm(qinput, weight, - out_dtype=input.dtype, + out_dtype=out_dtype, scale_a=x_scale, scale_b=weight_scale, bias=bias) @@ -240,7 +249,7 @@ class Fp8LinearOp: # Fused GEMM_DQ Rowwise GEMM output = torch._scaled_mm(qinput, weight, - out_dtype=input.dtype, + out_dtype=out_dtype, scale_a=x_scale, scale_b=weight_scale.t(), bias=bias) From cec8c7d7f8753d13737427ceb5cebe987f5f0549 Mon Sep 17 00:00:00 2001 From: "Jason (Siyu) Zhu" Date: Thu, 27 Mar 2025 20:27:20 -0700 Subject: [PATCH 063/593] Refactor error handling for multiple exceptions in preprocessing (#15650) Signed-off-by: JasonZhu1313 --- vllm/entrypoints/openai/serving_chat.py | 12 ++---------- vllm/entrypoints/openai/serving_embedding.py | 5 +---- vllm/entrypoints/openai/serving_pooling.py | 8 +------- vllm/entrypoints/openai/serving_tokenization.py | 8 +------- 4 files changed, 5 insertions(+), 28 deletions(-) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 3c35a848ea3a5..3102db4050f5b 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -197,16 +197,8 @@ class OpenAIServingChat(OpenAIServing): truncate_prompt_tokens=request.truncate_prompt_tokens, add_special_tokens=request.add_special_tokens, ) - except ValueError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except TypeError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except RuntimeError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except jinja2.TemplateError as e: + except (ValueError, TypeError, RuntimeError, + jinja2.TemplateError) as e: logger.exception("Error in preprocessing prompt inputs") return self.create_error_response(str(e)) diff --git a/vllm/entrypoints/openai/serving_embedding.py b/vllm/entrypoints/openai/serving_embedding.py index 1c2c78aaf8926..0ee58672631d0 100644 --- a/vllm/entrypoints/openai/serving_embedding.py +++ b/vllm/entrypoints/openai/serving_embedding.py @@ -139,10 +139,7 @@ class OpenAIServingEmbedding(OpenAIServing): truncate_prompt_tokens=truncate_prompt_tokens, add_special_tokens=request.add_special_tokens, ) - except ValueError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except TypeError as e: + except (ValueError, TypeError) as e: logger.exception("Error in preprocessing prompt inputs") return self.create_error_response(str(e)) diff --git a/vllm/entrypoints/openai/serving_pooling.py b/vllm/entrypoints/openai/serving_pooling.py index 894128ee974cd..779a3eded2c16 100644 --- a/vllm/entrypoints/openai/serving_pooling.py +++ b/vllm/entrypoints/openai/serving_pooling.py @@ -136,13 +136,7 @@ class OpenAIServingPooling(OpenAIServing): truncate_prompt_tokens=truncate_prompt_tokens, add_special_tokens=request.add_special_tokens, ) - except ValueError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except TypeError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except jinja2.TemplateError as e: + except (ValueError, TypeError, jinja2.TemplateError) as e: logger.exception("Error in preprocessing prompt inputs") return self.create_error_response(str(e)) diff --git a/vllm/entrypoints/openai/serving_tokenization.py b/vllm/entrypoints/openai/serving_tokenization.py index 90c0da2a24d51..c642fc51005ea 100644 --- a/vllm/entrypoints/openai/serving_tokenization.py +++ b/vllm/entrypoints/openai/serving_tokenization.py @@ -89,13 +89,7 @@ class OpenAIServingTokenization(OpenAIServing): request.prompt, add_special_tokens=request.add_special_tokens, ) - except ValueError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except TypeError as e: - logger.exception("Error in preprocessing prompt inputs") - return self.create_error_response(str(e)) - except jinja2.TemplateError as e: + except (ValueError, TypeError, jinja2.TemplateError) as e: logger.exception("Error in preprocessing prompt inputs") return self.create_error_response(str(e)) From 8693e47e6ab52c1219323141d9b0eba89c4143b7 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 28 Mar 2025 13:51:05 +0800 Subject: [PATCH 064/593] [Bugfix] Fix `mm_hashes` forgetting to be passed (#15668) Signed-off-by: DarkLight1337 --- vllm/inputs/preprocess.py | 2 ++ vllm/model_executor/models/llava.py | 2 ++ vllm/model_executor/models/mllama.py | 2 +- vllm/model_executor/models/phi4mm.py | 16 ++++++++-------- .../models/prithvi_geospatial_mae.py | 1 + vllm/multimodal/inputs.py | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py index 33f39bedea5b5..5cda5e5e3dee4 100644 --- a/vllm/inputs/preprocess.py +++ b/vllm/inputs/preprocess.py @@ -528,6 +528,7 @@ class InputPreprocessor: prompt_token_ids=decoder_inputs_to_override[ "prompt_token_ids"], mm_kwargs=inputs["mm_kwargs"], + mm_hashes=inputs["mm_hashes"], mm_placeholders=inputs["mm_placeholders"], ) else: @@ -536,6 +537,7 @@ class InputPreprocessor: prompt=inputs["prompt"], prompt_token_ids=inputs["prompt_token_ids"], mm_kwargs=inputs["mm_kwargs"], + mm_hashes=inputs["mm_hashes"], mm_placeholders=inputs["mm_placeholders"], ) elif inputs["type"] == "token": diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 826f04b37547b..45a0bf73b837d 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -868,6 +868,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor): mm_items = self._to_mm_items(mm_data) mm_item_counts = mm_items.get_all_counts() mm_kwargs = result["mm_kwargs"] + mm_hashes = result["mm_hashes"] # We reimplement the functionality of MLlavaProcessor from # https://github.com/TIGER-AI-Lab/Mantis.git @@ -916,6 +917,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor): prompt=prompt, prompt_token_ids=prompt_ids, mm_kwargs=mm_kwargs, + mm_hashes=mm_hashes, mm_placeholders=mm_placeholder_ranges, ) diff --git a/vllm/model_executor/models/mllama.py b/vllm/model_executor/models/mllama.py index 9ed49597cf827..d2c8fb7237274 100644 --- a/vllm/model_executor/models/mllama.py +++ b/vllm/model_executor/models/mllama.py @@ -1378,7 +1378,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal, # Because attn_metadata.encoder_seq_lens only counts the last # group of images for each sample, which is used to cheat the # block manager to allocate blocks for those images only. - # See input_processor_for_mllama() for more details. + # See MllamaMultiModalProcessor for more details. num_tiles_tensor = kwargs.pop("num_tiles") num_tiles = [t.tolist() for t in num_tiles_tensor] num_tokens_per_tile = calc_token_per_chunk(self.image_size) diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index 3d4505d556e2c..cb75ee1ea2ccd 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -28,7 +28,7 @@ from vllm.model_executor.models.llama import LlamaModel from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.sampling_metadata import SamplingMetadata from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import MultiModalInputs, NestedTensors +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors from vllm.sequence import IntermediateTensors, SequenceData from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config @@ -1319,9 +1319,9 @@ def dummy_data_for_phi4mm(ctx: InputContext, seq_len: int, def input_mapper_for_phi4mm_audio(ctx: InputContext, - data: object) -> MultiModalInputs: + data: object) -> MultiModalKwargs: """ - This function is used to create the MultiModalInputs for the Phi4MM + This function is used to create the MultiModalKwargs for the Phi4MM (audio) model. Specifically, for audio, we extract the audio features from the sound file and create pairs of audio features and audio embed lengths (the @@ -1338,13 +1338,13 @@ def input_mapper_for_phi4mm_audio(ctx: InputContext, data (object): Audio data. Returns: - MultiModalInputs: Multi-modal inputs. + MultiModalKwargs: Multi-modal inputs. """ if not isinstance(data, list): data = [data] if len(data) == 0: - return MultiModalInputs() + return MultiModalKwargs() audio_features = [] for audio_input in data: @@ -1365,7 +1365,7 @@ def input_mapper_for_phi4mm_audio(ctx: InputContext, [single_audio_embed_size], ) audio_features.append(single_audio_feature_audio_len_pair) - return MultiModalInputs({"audio_features": audio_features}) + return MultiModalKwargs({"audio_features": audio_features}) def input_mapper_for_phi4mm_image(ctx: InputContext, data: object): @@ -1373,7 +1373,7 @@ def input_mapper_for_phi4mm_image(ctx: InputContext, data: object): data = [data] # data: list of PIL images if len(data) == 0: - return MultiModalInputs() + return MultiModalKwargs() hf_config = ctx.get_hf_config() vision_encoder_name = hf_config.img_processor if vision_encoder_name is None: @@ -1385,7 +1385,7 @@ def input_mapper_for_phi4mm_image(ctx: InputContext, data: object): image_input_dict = preprocess(data, dynamic_hd_size, vit_image_size, vit_patch_size) - return MultiModalInputs({ + return MultiModalKwargs({ "pixel_values": image_input_dict["pixel_values"], "image_sizes": diff --git a/vllm/model_executor/models/prithvi_geospatial_mae.py b/vllm/model_executor/models/prithvi_geospatial_mae.py index 3f5faea4f875c..a69c0fc54e4c2 100644 --- a/vllm/model_executor/models/prithvi_geospatial_mae.py +++ b/vllm/model_executor/models/prithvi_geospatial_mae.py @@ -105,6 +105,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor): prompt=prompt, prompt_token_ids=[1], mm_kwargs=MultiModalKwargs(mm_kwargs), + mm_hashes=None, mm_placeholders={}, ) diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index 3a588bb4eaba1..81d72ff190222 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -743,7 +743,7 @@ class MultiModalInputs(TypedDict): mm_kwargs: MultiModalKwargs """Keyword arguments to be directly passed to the model after batching.""" - mm_hashes: NotRequired[Optional["MultiModalHashDict"]] + mm_hashes: Optional["MultiModalHashDict"] """The hashes of the multi-modal data.""" mm_placeholders: MultiModalPlaceholderDict From 355f66348c3ddb0a2c3217372f2ee47fb961d58f Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 28 Mar 2025 14:34:34 +0800 Subject: [PATCH 065/593] [V1] Remove legacy input registry (#15673) Signed-off-by: DarkLight1337 --- .../multimodal/processing/test_h2ovl.py | 7 +-- .../multimodal/processing/test_idefics3.py | 7 +-- .../multimodal/processing/test_internvl.py | 7 +-- .../multimodal/processing/test_llava_next.py | 16 +---- .../processing/test_llava_onevision.py | 16 +---- .../multimodal/processing/test_phi3v.py | 7 +-- .../multimodal/processing/test_qwen2_vl.py | 8 +-- tests/multimodal/test_processing.py | 18 ++---- vllm/inputs/preprocess.py | 12 ++-- vllm/inputs/registry.py | 25 +++++--- vllm/multimodal/profiling.py | 55 +++++++--------- vllm/multimodal/registry.py | 63 ++++++++++++++++--- vllm/v1/engine/async_llm.py | 7 ++- vllm/v1/engine/llm_engine.py | 4 +- vllm/v1/engine/processor.py | 22 +++---- vllm/v1/worker/gpu_model_runner.py | 9 +-- vllm/v1/worker/tpu_model_runner.py | 2 - 17 files changed, 132 insertions(+), 153 deletions(-) diff --git a/tests/models/multimodal/processing/test_h2ovl.py b/tests/models/multimodal/processing/test_h2ovl.py index 713fc733e21c6..709a686577f34 100644 --- a/tests/models/multimodal/processing/test_h2ovl.py +++ b/tests/models/multimodal/processing/test_h2ovl.py @@ -10,7 +10,6 @@ from transformers import PretrainedConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.image import rescale_image_size from vllm.multimodal.processing import BaseMultiModalProcessor -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config from ....conftest import _ImageAssets from ...utils import build_model_context @@ -156,11 +155,7 @@ def test_processor_override( mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, limit_mm_per_prompt={"image": len(size_factors)}, ) - tokenizer = cached_tokenizer_from_config(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=tokenizer, - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs min_num = min_dynamic_patch if dynamic_image_size else 1 diff --git a/tests/models/multimodal/processing/test_idefics3.py b/tests/models/multimodal/processing/test_idefics3.py index 4cff429a53941..f5b5cf6b5ba96 100644 --- a/tests/models/multimodal/processing/test_idefics3.py +++ b/tests/models/multimodal/processing/test_idefics3.py @@ -4,7 +4,6 @@ import pytest from transformers import Idefics3Config from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config from ....conftest import _ImageAssets from ...utils import build_model_context @@ -38,11 +37,7 @@ def test_processor_override( mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, limit_mm_per_prompt={"image": num_imgs}, ) - tokenizer = cached_tokenizer_from_config(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=tokenizer, - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs # Build the image str / prompt based on the number of images we pass diff --git a/tests/models/multimodal/processing/test_internvl.py b/tests/models/multimodal/processing/test_internvl.py index f5bd661071ac6..5ac47ecc5cc17 100644 --- a/tests/models/multimodal/processing/test_internvl.py +++ b/tests/models/multimodal/processing/test_internvl.py @@ -10,7 +10,6 @@ from transformers import PretrainedConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.image import rescale_image_size from vllm.multimodal.processing import BaseMultiModalProcessor -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config from ....conftest import _ImageAssets from ...utils import build_model_context @@ -113,11 +112,7 @@ def test_processor_override( mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, limit_mm_per_prompt={"image": len(size_factors)}, ) - tokenizer = cached_tokenizer_from_config(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=tokenizer, - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs min_num = min_dynamic_patch if dynamic_image_size else 1 diff --git a/tests/models/multimodal/processing/test_llava_next.py b/tests/models/multimodal/processing/test_llava_next.py index 74bca0e358996..fe56a200a330f 100644 --- a/tests/models/multimodal/processing/test_llava_next.py +++ b/tests/models/multimodal/processing/test_llava_next.py @@ -10,7 +10,6 @@ from pqdm.threads import pqdm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import ImageSize from vllm.multimodal.processing import BaseMultiModalProcessor -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config from ...utils import build_model_context @@ -40,10 +39,7 @@ def test_processor_max_tokens(model_id): mm_processor_kwargs=None, limit_mm_per_prompt={"image": 1}, ) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=cached_tokenizer_from_config(ctx.model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) info = processor.info seen_aspect_ratios = set[float]() @@ -139,10 +135,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs): mm_processor_kwargs=None, limit_mm_per_prompt={"image": num_imgs}, ) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=cached_tokenizer_from_config(ctx.model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328), (488, 183), (2560, 1669)] @@ -168,10 +161,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs): mm_processor_kwargs=None, limit_mm_per_prompt={"image": num_imgs}, ) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=cached_tokenizer_from_config(ctx.model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) seen_aspect_ratios = set[float]() image_sizes = list[ImageSize]() diff --git a/tests/models/multimodal/processing/test_llava_onevision.py b/tests/models/multimodal/processing/test_llava_onevision.py index c27898a40b711..7cefdd37ee49a 100644 --- a/tests/models/multimodal/processing/test_llava_onevision.py +++ b/tests/models/multimodal/processing/test_llava_onevision.py @@ -10,7 +10,6 @@ from pqdm.threads import pqdm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import ImageSize from vllm.multimodal.processing import BaseMultiModalProcessor -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config from ...utils import build_model_context @@ -41,10 +40,7 @@ def test_processor_max_tokens(model_id): mm_processor_kwargs=None, limit_mm_per_prompt={"image": 1}, ) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=cached_tokenizer_from_config(ctx.model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) info = processor.info seen_aspect_ratios = set[float]() @@ -139,10 +135,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs): mm_processor_kwargs=None, limit_mm_per_prompt={"image": num_imgs}, ) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=cached_tokenizer_from_config(ctx.model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328), (488, 183), (2560, 1669)] @@ -169,10 +162,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs): mm_processor_kwargs=None, limit_mm_per_prompt={"image": num_imgs}, ) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=cached_tokenizer_from_config(ctx.model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) seen_aspect_ratios = set[float]() image_sizes = list[ImageSize]() diff --git a/tests/models/multimodal/processing/test_phi3v.py b/tests/models/multimodal/processing/test_phi3v.py index dd5f30a23176b..ed0d04c5c5f5d 100644 --- a/tests/models/multimodal/processing/test_phi3v.py +++ b/tests/models/multimodal/processing/test_phi3v.py @@ -3,7 +3,6 @@ import pytest from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config from ....conftest import _ImageAssets from ...utils import build_model_context @@ -39,11 +38,7 @@ def test_processor_override( mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, limit_mm_per_prompt={"image": num_imgs}, ) - tokenizer = cached_tokenizer_from_config(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=tokenizer, - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs # Build the image str / prompt based on the number of images we pass diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 95204c7ebb4d8..d8c2ca414d41c 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -3,7 +3,6 @@ import pytest from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config from ....conftest import _ImageAssets from ...utils import build_model_context @@ -34,11 +33,8 @@ def test_processor_override( mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, limit_mm_per_prompt={"image": num_imgs}, ) - tokenizer = cached_tokenizer_from_config(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, - tokenizer=tokenizer, - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + tokenizer = processor.info.get_tokenizer() hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs # Build the image str / prompt based on the number of images we pass diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index b229f1e6ec8da..da112bd7a921c 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -28,8 +28,7 @@ from vllm.multimodal.processing import (PlaceholderFeaturesInfo, replace_token_matches) # yapf: enable from vllm.multimodal.profiling import MultiModalProfiler -from vllm.transformers_utils.tokenizer import (AnyTokenizer, - cached_tokenizer_from_config) +from vllm.transformers_utils.tokenizer import AnyTokenizer from vllm.utils import full_groupby from .utils import random_image @@ -955,10 +954,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid): limit_mm_per_prompt=limit_mm_per_prompt, ) - processor = MULTIMODAL_REGISTRY.create_processor( - model_config, - tokenizer=cached_tokenizer_from_config(model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(model_config) profiler = MultiModalProfiler(processor) mock_supported_mm_limits = MagicMock(return_value={"image": num_supported}) @@ -994,10 +990,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid): limit_mm_per_prompt=limit_mm_per_prompt, ) - processor = MULTIMODAL_REGISTRY.create_processor( - model_config, - tokenizer=cached_tokenizer_from_config(model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(model_config) rng = np.random.RandomState(0) image = random_image(rng, min_wh=128, max_wh=256) @@ -1066,10 +1059,7 @@ def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs): revision=None, ) - processor = MULTIMODAL_REGISTRY.create_processor( - model_config, - tokenizer=cached_tokenizer_from_config(model_config), - ) + processor = MULTIMODAL_REGISTRY.create_processor(model_config) orig_get_hf_processor = processor.info.get_hf_processor def get_hf_processor(self, **kwargs): diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py index 5cda5e5e3dee4..669fb96e6653a 100644 --- a/vllm/inputs/preprocess.py +++ b/vllm/inputs/preprocess.py @@ -261,13 +261,13 @@ class InputPreprocessor: # initialized without a tokenizer while using also multi-modal # input. if not self.tokenizer: - tokenizer = None + tokenizer = object() # Dummy else: tokenizer_group = self.get_tokenizer_group() tokenizer = tokenizer_group.get_lora_tokenizer(lora_request) - mm_processor = self.mm_registry.create_processor( - self.model_config, tokenizer) + mm_processor = self.mm_registry.create_processor(self.model_config, + tokenizer=tokenizer) if mm_processor_kwargs is None: mm_processor_kwargs = {} @@ -288,14 +288,14 @@ class InputPreprocessor: # initialized without a tokenizer while using also multi-modal # input. if not self.tokenizer: - tokenizer = None + tokenizer = object() # Dummy else: tokenizer_group = self.get_tokenizer_group() tokenizer = await tokenizer_group.get_lora_tokenizer_async( lora_request) - mm_processor = self.mm_registry.create_processor( - self.model_config, tokenizer) + mm_processor = self.mm_registry.create_processor(self.model_config, + tokenizer=tokenizer) if mm_processor_kwargs is None: mm_processor_kwargs = {} diff --git a/vllm/inputs/registry.py b/vllm/inputs/registry.py index 8b95db7a72522..0579893e5d767 100644 --- a/vllm/inputs/registry.py +++ b/vllm/inputs/registry.py @@ -13,8 +13,7 @@ from typing_extensions import TypeVar, assert_never from vllm.logger import init_logger from vllm.transformers_utils.processor import cached_processor_from_config -from vllm.transformers_utils.tokenizer import (AnyTokenizer, - cached_tokenizer_from_config) +from vllm.transformers_utils.tokenizer import AnyTokenizer from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides, resolve_mm_processor_kwargs) @@ -329,17 +328,27 @@ class InputRegistry: from vllm.model_executor.model_loader import get_model_architecture from vllm.multimodal import MultiModalKwargs from vllm.multimodal.profiling import MultiModalProfiler + from vllm.sequence import SequenceData if mm_registry.has_processor(model_config): - tokenizer = cached_tokenizer_from_config(model_config) processor = mm_registry.create_processor(model_config, - tokenizer, disable_cache=True) profiler = MultiModalProfiler(processor) - dummy_data_factory = (profiler.get_encoder_dummy_data - if is_encoder_data else - profiler.get_decoder_dummy_data) - dummy_data = dummy_data_factory(seq_len) + + dummy_data_v1 = (profiler.get_encoder_dummy_data(seq_len) + if is_encoder_data else + profiler.get_decoder_dummy_data(seq_len)) + _seq_data = SequenceData.from_seqs( + dummy_data_v1.prompt_token_ids) # type: ignore[attr-defined] + + dummy_data = DummyData( + seq_data=_seq_data, + multi_modal_data=getattr(dummy_data_v1, "multi_modal_data", + None), + multi_modal_placeholders=getattr(dummy_data_v1, + "multi_modal_placeholders", + None), + ) else: model_cls, _ = get_model_architecture(model_config) if is_encoder_data: diff --git a/vllm/multimodal/profiling.py b/vllm/multimodal/profiling.py index 7b4fb5eb598d1..e36f8e4434ec6 100644 --- a/vllm/multimodal/profiling.py +++ b/vllm/multimodal/profiling.py @@ -3,18 +3,18 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Generic, TypeVar, cast +from typing import Generic, NamedTuple, TypeVar, cast import numpy as np import numpy.typing as npt from PIL import Image import vllm.envs as envs -from vllm.inputs import DummyData from vllm.logger import init_logger from .inputs import (MultiModalDataDict, MultiModalEncDecInputs, - MultiModalInputs) + MultiModalInputs, MultiModalKwargs, + MultiModalPlaceholderDict) from .processing import BaseMultiModalProcessor, BaseProcessingInfo logger = init_logger(__name__) @@ -31,6 +31,20 @@ class ProcessorInputs: hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict) +class DummyEncoderData(NamedTuple): + """Dummy data used for profiling.""" + + prompt_token_ids: list[int] + + +class DummyDecoderData(NamedTuple): + """Dummy data used for profiling.""" + + prompt_token_ids: list[int] + multi_modal_data: MultiModalKwargs + multi_modal_placeholders: MultiModalPlaceholderDict + + _I = TypeVar("_I", bound=BaseProcessingInfo) @@ -179,13 +193,7 @@ class MultiModalProfiler(Generic[_I]): "tokens.") return mm_inputs, total_placeholders_by_modality - def get_encoder_dummy_data( - self, - seq_len: int, - ) -> DummyData: - # Avoid circular import - from vllm.sequence import SequenceData - + def get_encoder_dummy_data(self, seq_len: int) -> DummyEncoderData: mm_inputs, _ = self.get_and_validate_mm_inputs(seq_len) mm_inputs = cast(MultiModalEncDecInputs, mm_inputs) @@ -197,19 +205,9 @@ class MultiModalProfiler(Generic[_I]): num_tokens_to_pad = max(total_len, seq_len) - total_len encoder_prompt_token_ids.extend([0] * num_tokens_to_pad) - return DummyData( - seq_data=SequenceData.from_seqs(encoder_prompt_token_ids), - multi_modal_data=None, - multi_modal_placeholders=None, - ) - - def get_decoder_dummy_data( - self, - seq_len: int, - ) -> DummyData: - # Avoid circular import - from vllm.sequence import SequenceData + return DummyEncoderData(encoder_prompt_token_ids) + def get_decoder_dummy_data(self, seq_len: int) -> DummyDecoderData: (mm_inputs, total_placeholders_by_modality ) = self.get_and_validate_mm_inputs(seq_len) @@ -231,16 +229,11 @@ class MultiModalProfiler(Generic[_I]): "and/or reduce `mm_counts`.", seq_len, total_len, total_placeholders_by_modality) - return DummyData( - seq_data=SequenceData.from_prompt_token_counts((0, seq_len)), - multi_modal_data=None, - multi_modal_placeholders=None, - ) + if total_len < seq_len: + prompt_token_ids.extend([0] * (seq_len - total_len)) - prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids))) - - return DummyData( - seq_data=SequenceData.from_seqs(prompt_token_ids), + return DummyDecoderData( + prompt_token_ids=prompt_token_ids, multi_modal_data=mm_inputs["mm_kwargs"], multi_modal_placeholders=mm_inputs["mm_placeholders"], ) diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index 24b8358982797..8c16c3ba80750 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -21,7 +21,8 @@ from .image import ImagePlugin from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors from .processing import (BaseMultiModalProcessor, BaseProcessingInfo, ProcessingCache) -from .profiling import BaseDummyInputsBuilder, MultiModalProfiler +from .profiling import (BaseDummyInputsBuilder, DummyDecoderData, + DummyEncoderData, MultiModalProfiler) from .video import VideoPlugin if TYPE_CHECKING: @@ -256,10 +257,7 @@ class MultiModalRegistry: on underlying model configuration. """ if self.has_processor(model_config): - tokenizer = cached_tokenizer_from_config(model_config) - processor = self.create_processor(model_config, - tokenizer, - disable_cache=True) + processor = self.create_processor(model_config, disable_cache=True) seq_len = model_config.max_model_len mm_limits = self.get_mm_limits_per_prompt(model_config) return processor.info.get_mm_max_tokens_per_item( @@ -373,10 +371,7 @@ class MultiModalRegistry: This should be called after :meth:`init_mm_limits_per_prompt`. """ if self.has_processor(model_config): - tokenizer = cached_tokenizer_from_config(model_config) - processor = self.create_processor(model_config, - tokenizer, - disable_cache=True) + processor = self.create_processor(model_config, disable_cache=True) profiler = MultiModalProfiler(processor) return profiler.get_mm_limits() @@ -436,8 +431,8 @@ class MultiModalRegistry: def create_processor( self, model_config: "ModelConfig", - tokenizer: AnyTokenizer, *, + tokenizer: Optional[AnyTokenizer] = None, disable_cache: Optional[bool] = None, ) -> BaseMultiModalProcessor[BaseProcessingInfo]: """ @@ -446,6 +441,8 @@ class MultiModalRegistry: See also: :ref:`mm-processing` """ + if tokenizer is None: + tokenizer = cached_tokenizer_from_config(model_config) if disable_cache is None: disable_cache = model_config.disable_mm_preprocessor_cache @@ -456,3 +453,49 @@ class MultiModalRegistry: cache = None if disable_cache else self._processing_cache return factories.build_processor(ctx, cache=cache) + + def get_decoder_dummy_data( + self, + model_config: "ModelConfig", + seq_len: int, + ) -> DummyDecoderData: + """ + Create dummy data for profiling the memory usage of a model. + + The model is identified by ``model_config``. + """ + processor = self.create_processor(model_config, disable_cache=True) + profiler = MultiModalProfiler(processor) + dummy_data = profiler.get_decoder_dummy_data(seq_len) + + # Having more tokens is over-conservative but otherwise fine + token_ids = dummy_data.prompt_token_ids + if len(token_ids) < seq_len: + raise AssertionError( + f"Expected at least {seq_len} dummy tokens for profiling, " + f"but found {len(token_ids)} tokens instead.") + + return dummy_data + + def get_encoder_dummy_data( + self, + model_config: "ModelConfig", + seq_len: int, + ) -> DummyEncoderData: + """ + Create dummy data for profiling the memory usage of a model. + + The model is identified by ``model_config``. + """ + processor = self.create_processor(model_config, disable_cache=True) + profiler = MultiModalProfiler(processor) + dummy_data = profiler.get_encoder_dummy_data(seq_len) + + # Having more tokens is over-conservative but otherwise fine + token_ids = dummy_data.prompt_token_ids + if len(token_ids) < seq_len: + logger.warning_once( + f"Expected at least {seq_len} dummy encoder tokens for " + f"profiling, but found {len(token_ids)} tokens instead.") + + return dummy_data diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 1fb9ae8cb7a59..a8d86e70f6abf 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -14,10 +14,11 @@ from vllm.config import ModelConfig, VllmConfig from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.protocol import EngineClient from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE -from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType +from vllm.inputs import PromptType from vllm.inputs.preprocess import InputPreprocessor from vllm.logger import init_logger from vllm.lora.request import LoRARequest +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.outputs import RequestOutput from vllm.pooling_params import PoolingParams from vllm.prompt_adapter.request import PromptAdapterRequest @@ -48,7 +49,7 @@ class AsyncLLM(EngineClient): executor_class: type[Executor], log_stats: bool, usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, - input_registry: InputRegistry = INPUT_REGISTRY, + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, use_cached_outputs: bool = False, log_requests: bool = True, start_engine_loop: bool = True, @@ -90,7 +91,7 @@ class AsyncLLM(EngineClient): self.processor = Processor( vllm_config=vllm_config, tokenizer=self.tokenizer, - input_registry=input_registry, + mm_registry=mm_registry, ) # OutputProcessor (converts EngineCoreOutputs --> RequestOutput). diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 8cc73f9fe7224..000de21fbe7bf 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -11,7 +11,7 @@ from vllm.config import ParallelConfig, VllmConfig from vllm.distributed import stateless_destroy_torch_distributed_process_group from vllm.engine.arg_utils import EngineArgs from vllm.engine.metrics_types import StatLoggerBase -from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType +from vllm.inputs import PromptType from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry @@ -44,7 +44,6 @@ class LLMEngine: log_stats: bool, usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, stat_loggers: Optional[dict[str, StatLoggerBase]] = None, - input_registry: InputRegistry = INPUT_REGISTRY, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, use_cached_outputs: bool = False, multiprocess_mode: bool = False, @@ -80,7 +79,6 @@ class LLMEngine: # Processor (convert Inputs --> EngineCoreRequests) self.processor = Processor(vllm_config=vllm_config, tokenizer=self.tokenizer, - input_registry=input_registry, mm_registry=mm_registry) # OutputProcessor (convert EngineCoreOutputs --> RequestOutput). diff --git a/vllm/v1/engine/processor.py b/vllm/v1/engine/processor.py index 065ac0920af77..24762d214c345 100644 --- a/vllm/v1/engine/processor.py +++ b/vllm/v1/engine/processor.py @@ -5,8 +5,7 @@ from collections.abc import Mapping from typing import Optional, Union from vllm.config import VllmConfig -from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs, - PromptType, SingletonInputsAdapter) +from vllm.inputs import ProcessorInputs, PromptType from vllm.inputs.parse import split_enc_dec_inputs from vllm.inputs.preprocess import InputPreprocessor from vllm.lora.request import LoRARequest @@ -31,7 +30,6 @@ class Processor: self, vllm_config: VllmConfig, tokenizer: BaseTokenizerGroup, - input_registry: InputRegistry = INPUT_REGISTRY, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, ): @@ -210,7 +208,6 @@ class Processor: self._validate_model_inputs(processed_inputs, lora_request) encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) - decoder_inputs = SingletonInputsAdapter(decoder_inputs) # TODO: Impl encoder-decoder if encoder_inputs is not None: @@ -221,8 +218,9 @@ class Processor: sampling_params = params.clone() # If unset max tokens, then generate up to the max_model_len. if sampling_params.max_tokens is None: - sampling_params.max_tokens = (self.model_config.max_model_len - - len(decoder_inputs.prompt_token_ids)) + sampling_params.max_tokens = ( + self.model_config.max_model_len - + len(decoder_inputs["prompt_token_ids"])) sampling_params.update_from_generation_config( self.generation_config_fields, eos_token_id) sampling_params.update_from_tokenizer( @@ -232,8 +230,8 @@ class Processor: sorted_mm_inputs: Optional[list[MultiModalKwargs]] = None sorted_mm_positions: Optional[list[PlaceholderRange]] = None sorted_mm_hashes: Optional[list[str]] = None - if (decoder_mm_inputs := decoder_inputs.multi_modal_data): - assert isinstance(decoder_mm_inputs, MultiModalKwargs) + if decoder_inputs["type"] == "multimodal": + decoder_mm_inputs = decoder_inputs["mm_kwargs"] # The output of merged multi-modal processor (`decoder_mm_inputs`) # contains the kwargs for all items from all modalities. @@ -254,8 +252,8 @@ class Processor: sorted_mm_positions, sorted_mm_hashes, ) = merge_and_sort_multimodal_metadata( - decoder_inputs.multi_modal_placeholders, - decoder_inputs.multi_modal_hashes if self.use_hash else None, + decoder_inputs["mm_placeholders"], + decoder_inputs["mm_hashes"] if self.use_hash else None, ) # NOTE: Sort multimodal inputs/kwargs ONLY IF there are multiple @@ -281,8 +279,8 @@ class Processor: return EngineCoreRequest( request_id=request_id, - prompt=decoder_inputs.prompt, - prompt_token_ids=decoder_inputs.prompt_token_ids, + prompt=decoder_inputs.get("prompt"), + prompt_token_ids=decoder_inputs["prompt_token_ids"], mm_inputs=sorted_mm_inputs, mm_hashes=sorted_mm_hashes, mm_placeholders=sorted_mm_positions, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 133ccf84832c4..1b581c69a728b 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -15,7 +15,6 @@ from vllm.attention.layer import Attention from vllm.config import CompilationLevel, VllmConfig from vllm.distributed.parallel_state import get_pp_group, graph_capture from vllm.forward_context import set_forward_context -from vllm.inputs import INPUT_REGISTRY from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding @@ -130,7 +129,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.cascade_attn_enabled = not self.model_config.disable_cascade_attn # Multi-modal data support - self.input_registry = INPUT_REGISTRY self.mm_registry = MULTIMODAL_REGISTRY self.uses_mrope = model_config.uses_mrope @@ -1473,16 +1471,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): encoder_budget, max_num_mm_items, dummy_data_modality) # Create dummy batch of multimodal inputs. - dummy_request_data = self.input_registry.dummy_data_for_profiling( + dummy_request_data = self.mm_registry.get_decoder_dummy_data( model_config=self.model_config, seq_len=self.max_num_tokens, - mm_registry=self.mm_registry, ) dummy_mm_data = dummy_request_data.multi_modal_data - if not isinstance(dummy_mm_data, MultiModalKwargs): - # TODO: Delete this check once input mapper is fully removed. - raise RuntimeError( - "Legacy input mapper is not supported in V1") # Dummy data definition may contain multiple multimodal items # (e.g, multiple images) for a single request, therefore here we diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index 97dfd23163dff..5401fff2bf19b 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -17,7 +17,6 @@ from vllm.attention.backends.abstract import AttentionType from vllm.attention.layer import Attention from vllm.config import VllmConfig from vllm.forward_context import set_forward_context -from vllm.inputs import INPUT_REGISTRY from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs @@ -102,7 +101,6 @@ class TPUModelRunner: self.hidden_size = model_config.get_hidden_size() # Multi-modal data support - self.input_registry = INPUT_REGISTRY self.mm_registry = MULTIMODAL_REGISTRY self.uses_mrope = model_config.uses_mrope # TODO: Support M-RoPE (e.g, Qwen2-VL) From 2d9045fce8a6b440f937a5f313bf8bc5baf3103a Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Fri, 28 Mar 2025 03:01:26 -0400 Subject: [PATCH 066/593] [TPU][CI] Fix TPUModelRunner Test (#15667) Signed-off-by: Robert Shaw Co-authored-by: Robert Shaw --- .buildkite/run-tpu-v1-test.sh | 2 +- tests/v1/tpu/worker/test_tpu_model_runner.py | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/.buildkite/run-tpu-v1-test.sh b/.buildkite/run-tpu-v1-test.sh index 7bd91575e1729..2c356b8fe5274 100755 --- a/.buildkite/run-tpu-v1-test.sh +++ b/.buildkite/run-tpu-v1-test.sh @@ -30,7 +30,7 @@ docker run --privileged --net host --shm-size=16G -it \ && echo TEST_4 \ && python3 /workspace/vllm/examples/offline_inference/tpu.py \ && echo TEST_5 \ - && pytest -s -v /workspace/vllm/tests/tpu/worker/test_tpu_model_runner.py \ + && pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py \ && echo TEST_6 \ && pytest -s -v /workspace/vllm/tests/v1/tpu/test_sampler.py" \ diff --git a/tests/v1/tpu/worker/test_tpu_model_runner.py b/tests/v1/tpu/worker/test_tpu_model_runner.py index d5f812ed4d543..6b6a91b857f0e 100644 --- a/tests/v1/tpu/worker/test_tpu_model_runner.py +++ b/tests/v1/tpu/worker/test_tpu_model_runner.py @@ -7,7 +7,6 @@ from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig from vllm.sampling_params import SamplingParams from vllm.v1.core.sched.output import (CachedRequestData, NewRequestData, SchedulerOutput) -from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.worker.tpu_model_runner import (TPUModelRunner, _get_padded_token_len, _get_paddings) @@ -113,12 +112,6 @@ def _is_req_added(model_runner, req_id: str) -> bool: return req_id in model_runner.requests -def _is_sampling_metadata_changed(model_runner, - sampling_metadata_before: SamplingMetadata): - return model_runner.input_batch.sampling_metadata is not ( - sampling_metadata_before) - - def _is_req_state_block_table_match(model_runner, req_id: str) -> bool: req_index = model_runner.input_batch.req_id_to_index[req_id] block_table = model_runner.input_batch.block_table @@ -136,10 +129,8 @@ def test_update_states_new_request(model_runner): # new req scheduler_output = _schedule_new_request(req_id) - metadata_before = model_runner.input_batch.sampling_metadata model_runner._update_states(scheduler_output) - assert _is_sampling_metadata_changed(model_runner, metadata_before) assert _is_req_added(model_runner, req_id) assert _is_req_scheduled(model_runner, req_id) assert _is_req_state_block_table_match(model_runner, req_id) @@ -170,9 +161,7 @@ def test_update_states_request_finished(model_runner): grammar_bitmask=None, ) - metadata_before = model_runner.input_batch.sampling_metadata model_runner._update_states(scheduler_output) - assert _is_sampling_metadata_changed(model_runner, metadata_before) assert not _is_req_added(model_runner, req_id) assert not _is_req_scheduled(model_runner, req_id) @@ -229,9 +218,7 @@ def test_update_states_request_resumed(model_runner): grammar_bitmask=None, ) - metadata_before = model_runner.input_batch.sampling_metadata model_runner._update_states(scheduler_output) - assert _is_sampling_metadata_changed(model_runner, metadata_before) assert _is_req_added(model_runner, req_id) assert _is_req_scheduled(model_runner, req_id) assert _is_req_state_block_table_match(model_runner, req_id) @@ -262,9 +249,7 @@ def test_update_states_no_changes(model_runner): grammar_bitmask=None, ) - metadata_before = model_runner.input_batch.sampling_metadata model_runner._update_states(scheduler_output) - assert not _is_sampling_metadata_changed(model_runner, metadata_before) assert _is_req_added(model_runner, req_id) assert _is_req_scheduled(model_runner, req_id) assert _is_req_state_block_table_match(model_runner, req_id) @@ -299,8 +284,7 @@ def test_update_states_request_unscheduled(model_runner): grammar_bitmask=None, ) - metadata_before = model_runner._update_states(scheduler_output) - assert _is_sampling_metadata_changed(model_runner, metadata_before) + model_runner._update_states(scheduler_output) assert _is_req_added(model_runner, req_ids[0]) assert _is_req_scheduled(model_runner, req_ids[0]) From 32b14baf8a1f7195ca09484de3008063569b43c5 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Fri, 28 Mar 2025 15:23:30 +0800 Subject: [PATCH 067/593] [Refactor][Frontend] Keep all logic about reasoning into one class (#14428) Signed-off-by: Ce Gao --- .../__init__.py | 0 .../test_deepseekr1_reasoning_parser.py | 52 +++++++-- .../test_granite_reasoning_parser.py | 6 +- .../reasoning_parsers => reasoning}/utils.py | 2 +- vllm/engine/arg_utils.py | 3 +- vllm/engine/llm_engine.py | 5 +- vllm/entrypoints/openai/api_server.py | 2 +- vllm/entrypoints/openai/serving_chat.py | 3 +- .../guided_decoding/__init__.py | 15 ++- .../guided_decoding/outlines_decoding.py | 8 +- .../outlines_logits_processors.py | 14 +-- .../reasoner/deepseek_reasoner.py | 38 ------- .../guided_decoding/reasoner/reasoner.py | 23 ---- .../guided_decoding/xgrammar_decoding.py | 6 +- .../__init__.py | 0 .../abs_reasoning_parsers.py | 101 ++++++++---------- .../deepseek_r1_reasoning_parser.py | 90 ++++++++-------- .../granite_reasoning_parser.py | 3 +- 18 files changed, 171 insertions(+), 200 deletions(-) rename tests/{entrypoints/openai/reasoning_parsers => reasoning}/__init__.py (100%) rename tests/{entrypoints/openai/reasoning_parsers => reasoning}/test_deepseekr1_reasoning_parser.py (75%) rename tests/{entrypoints/openai/reasoning_parsers => reasoning}/test_granite_reasoning_parser.py (97%) rename tests/{entrypoints/openai/reasoning_parsers => reasoning}/utils.py (97%) delete mode 100644 vllm/model_executor/guided_decoding/reasoner/deepseek_reasoner.py delete mode 100644 vllm/model_executor/guided_decoding/reasoner/reasoner.py rename vllm/{entrypoints/openai/reasoning_parsers => reasoning}/__init__.py (100%) rename vllm/{entrypoints/openai/reasoning_parsers => reasoning}/abs_reasoning_parsers.py (82%) rename vllm/{entrypoints/openai/reasoning_parsers => reasoning}/deepseek_r1_reasoning_parser.py (64%) rename vllm/{entrypoints/openai/reasoning_parsers => reasoning}/granite_reasoning_parser.py (99%) diff --git a/tests/entrypoints/openai/reasoning_parsers/__init__.py b/tests/reasoning/__init__.py similarity index 100% rename from tests/entrypoints/openai/reasoning_parsers/__init__.py rename to tests/reasoning/__init__.py diff --git a/tests/entrypoints/openai/reasoning_parsers/test_deepseekr1_reasoning_parser.py b/tests/reasoning/test_deepseekr1_reasoning_parser.py similarity index 75% rename from tests/entrypoints/openai/reasoning_parsers/test_deepseekr1_reasoning_parser.py rename to tests/reasoning/test_deepseekr1_reasoning_parser.py index 5ce5d9280f3ef..7b6af183a86ad 100644 --- a/tests/entrypoints/openai/reasoning_parsers/test_deepseekr1_reasoning_parser.py +++ b/tests/reasoning/test_deepseekr1_reasoning_parser.py @@ -3,74 +3,92 @@ import pytest from transformers import AutoTokenizer -from tests.entrypoints.openai.reasoning_parsers.utils import ( - run_reasoning_extraction) -from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser, - ReasoningParserManager) +from tests.reasoning.utils import run_reasoning_extraction +from vllm.reasoning import ReasoningParser, ReasoningParserManager parser_name = "deepseek_r1" start_token = "" end_token = "" +REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" + + +@pytest.fixture(scope="module") +def deepseek_r1_qwen_tokenizer(): + return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) + + SIMPLE_REASONING = { "output": "This is a reasoning sectionThis is the rest", "reasoning_content": "This is a reasoning section", "content": "This is the rest", + "is_reasoning_end": True, } COMPLETE_REASONING = { "output": "This is a reasoning section", "reasoning_content": "This is a reasoning section", "content": None, + "is_reasoning_end": True, } NO_CONTENT = { "output": "This is content", "reasoning_content": "This is content", "content": None, + "is_reasoning_end": False, } NO_REASONING_STREAMING = { "output": "This is a reasoning section", "reasoning_content": "This is a reasoning section", "content": None, + "is_reasoning_end": False, } MULTIPLE_LINES = { "output": "This\nThatThis is the rest\nThat", "reasoning_content": "This\nThat", "content": "This is the rest\nThat", + "is_reasoning_end": True, } SHORTEST_REASONING_NO_STREAMING = { "output": "This is the rest", "reasoning_content": "", "content": "This is the rest", + "is_reasoning_end": True, } SHORTEST_REASONING = { "output": "This is the rest", "reasoning_content": None, "content": "This is the rest", + "is_reasoning_end": True, } REASONING_WITH_THINK = { "output": "This is a reasoning sectionThis is the rest", "reasoning_content": "This is a reasoning section", "content": "This is the rest", + "is_reasoning_end": True, } COMPLETE_REASONING_WITH_THINK = { "output": "This is a reasoning section", "reasoning_content": "This is a reasoning section", "content": None, + "is_reasoning_end": True, } MULTIPLE_LINES_WITH_THINK = { "output": "This\nThatThis is the rest\nThat", "reasoning_content": "This\nThat", "content": "This is the rest\nThat", + "is_reasoning_end": True, } SHORTEST_REASONING_NO_STREAMING_WITH_THINK = { "output": "This is the rest", "reasoning_content": "", "content": "This is the rest", + "is_reasoning_end": True, } SHORTEST_REASONING_WITH_THINK = { "output": "This is the rest", "reasoning_content": None, "content": "This is the rest", + "is_reasoning_end": True, } TEST_CASES = [ @@ -166,23 +184,21 @@ TEST_CASES = [ ), ] -# Global tokenizer initialization to avoid repeated loading -tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m") -tokenizer.add_tokens([start_token, end_token]) - @pytest.mark.parametrize("streaming, param_dict", TEST_CASES) def test_reasoning( streaming: bool, param_dict: dict, + deepseek_r1_qwen_tokenizer, ): - output = tokenizer.tokenize(param_dict["output"]) + output = deepseek_r1_qwen_tokenizer.tokenize(param_dict["output"]) # decode everything to tokens output_tokens: list[str] = [ - tokenizer.convert_tokens_to_string([token]) for token in output + deepseek_r1_qwen_tokenizer.convert_tokens_to_string([token]) + for token in output ] parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser( - parser_name)(tokenizer) + parser_name)(deepseek_r1_qwen_tokenizer) reasoning, content = run_reasoning_extraction(parser, output_tokens, @@ -190,3 +206,17 @@ def test_reasoning( assert reasoning == param_dict["reasoning_content"] assert content == param_dict["content"] + + # Test is_reasoning_end + output_ids = deepseek_r1_qwen_tokenizer.convert_tokens_to_ids(output) + is_reasoning_end = parser.is_reasoning_end(output_ids) + assert is_reasoning_end == param_dict["is_reasoning_end"] + + # Test extract_content + if param_dict["content"] is not None: + content = parser.extract_content_ids(output_ids) + assert content == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids( + deepseek_r1_qwen_tokenizer.tokenize(param_dict["content"])) + else: + content = parser.extract_content_ids(output) + assert content == [] diff --git a/tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py b/tests/reasoning/test_granite_reasoning_parser.py similarity index 97% rename from tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py rename to tests/reasoning/test_granite_reasoning_parser.py index 84ac6600498b2..48fb8c2f8d1b9 100644 --- a/tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py +++ b/tests/reasoning/test_granite_reasoning_parser.py @@ -2,10 +2,8 @@ import pytest from transformers import AutoTokenizer -from tests.entrypoints.openai.reasoning_parsers.utils import ( - DeltaMessage, run_reasoning_extraction) -from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser, - ReasoningParserManager) +from tests.reasoning.utils import DeltaMessage, run_reasoning_extraction +from vllm.reasoning import ReasoningParser, ReasoningParserManager parser_name = "granite" START_REASONING = "Here is my thought process:" diff --git a/tests/entrypoints/openai/reasoning_parsers/utils.py b/tests/reasoning/utils.py similarity index 97% rename from tests/entrypoints/openai/reasoning_parsers/utils.py rename to tests/reasoning/utils.py index 01e43130bc6e7..0f894ed800c6c 100644 --- a/tests/entrypoints/openai/reasoning_parsers/utils.py +++ b/tests/reasoning/utils.py @@ -4,7 +4,7 @@ from typing import Optional, Union from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, DeltaMessage) -from vllm.entrypoints.openai.reasoning_parsers import ReasoningParser +from vllm.reasoning import ReasoningParser class StreamingReasoningReconstructor: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index d049f773caccd..a416fa8aa08e3 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -23,6 +23,7 @@ from vllm.executor.executor_base import ExecutorBase from vllm.logger import init_logger from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS from vllm.plugins import load_general_plugins +from vllm.reasoning import ReasoningParserManager from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3 from vllm.transformers_utils.utils import check_gguf_file from vllm.usage.usage_lib import UsageContext @@ -1119,7 +1120,7 @@ class EngineArgs: parser.add_argument( "--reasoning-parser", type=str, - choices=["deepseek_r1", "granite"], + choices=list(ReasoningParserManager.reasoning_parsers), default=None, help= "Select the reasoning parser depending on the model that you're " diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py index 4856c3568319b..5682b3dabe2e8 100644 --- a/vllm/engine/llm_engine.py +++ b/vllm/engine/llm_engine.py @@ -2080,8 +2080,9 @@ class LLMEngine: guided_decoding.backend = guided_decoding.backend or \ self.decoding_config.guided_decoding_backend - logger.debug("Reasoning backend: %s", - self.decoding_config.reasoning_backend) + if self.decoding_config.reasoning_backend is not None: + logger.debug("Building with reasoning backend %s", + self.decoding_config.reasoning_backend) processor = get_local_guided_decoding_logits_processor( guided_params=guided_decoding, diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 1e735da641df9..6c1f60fa6a3b4 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -68,7 +68,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, TranscriptionRequest, TranscriptionResponse, UnloadLoRAAdapterRequest) -from vllm.entrypoints.openai.reasoning_parsers import ReasoningParserManager # yapf: enable from vllm.entrypoints.openai.serving_chat import OpenAIServingChat from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion @@ -85,6 +84,7 @@ from vllm.entrypoints.openai.serving_transcription import ( from vllm.entrypoints.openai.tool_parsers import ToolParserManager from vllm.entrypoints.utils import load_aware_call, with_cancellation from vllm.logger import init_logger +from vllm.reasoning import ReasoningParserManager from vllm.transformers_utils.config import ( maybe_register_config_serialize_by_value) from vllm.transformers_utils.tokenizer import MistralTokenizer diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 3102db4050f5b..eda4722836bdb 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -23,8 +23,6 @@ from vllm.entrypoints.openai.protocol import ( ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage, DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo) -from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser, - ReasoningParserManager) from vllm.entrypoints.openai.serving_engine import (OpenAIServing, clamp_prompt_logprobs) from vllm.entrypoints.openai.serving_models import OpenAIServingModels @@ -33,6 +31,7 @@ from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import ( MistralToolCall) from vllm.logger import init_logger from vllm.outputs import CompletionOutput, RequestOutput +from vllm.reasoning import ReasoningParser, ReasoningParserManager from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.sequence import Logprob from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer diff --git a/vllm/model_executor/guided_decoding/__init__.py b/vllm/model_executor/guided_decoding/__init__.py index 0c26a60588c88..cecb3a8a1d4a8 100644 --- a/vllm/model_executor/guided_decoding/__init__.py +++ b/vllm/model_executor/guided_decoding/__init__.py @@ -5,10 +5,10 @@ from __future__ import annotations from typing import TYPE_CHECKING from vllm.logger import init_logger -from vllm.model_executor.guided_decoding.reasoner import get_reasoner from vllm.model_executor.guided_decoding.utils import ( convert_lark_to_gbnf, grammar_is_likely_lark, has_lmf_unsupported_json_features, has_xgrammar_unsupported_json_features) +from vllm.reasoning import ReasoningParserManager if TYPE_CHECKING: from transformers import PreTrainedTokenizer @@ -107,7 +107,11 @@ async def get_guided_decoding_logits_processor( model_config: ModelConfig, reasoning_backend: str | None = None) -> LogitsProcessor | None: - reasoner = get_reasoner(tokenizer, reasoning_backend) + reasoner = None + if reasoning_backend is not None: + reasoner_class = ReasoningParserManager.get_reasoning_parser( + reasoning_backend) + reasoner = reasoner_class(tokenizer) guided_params = maybe_backend_fallback(guided_params) @@ -146,8 +150,11 @@ def get_local_guided_decoding_logits_processor( reasoning_backend: str | None = None) -> LogitsProcessor | None: guided_params = maybe_backend_fallback(guided_params) - # Get the reasoner if needed, it will be None if reasoning_ - reasoner = get_reasoner(tokenizer, reasoning_backend) + reasoner = None + if reasoning_backend is not None: + reasoner_class = ReasoningParserManager.get_reasoning_parser( + reasoning_backend) + reasoner = reasoner_class(tokenizer) # CFG grammar not supported by LMFE, so we use outlines instead if guided_params.backend_name == 'outlines': diff --git a/vllm/model_executor/guided_decoding/outlines_decoding.py b/vllm/model_executor/guided_decoding/outlines_decoding.py index 97f63ae11f457..564f9277a83c6 100644 --- a/vllm/model_executor/guided_decoding/outlines_decoding.py +++ b/vllm/model_executor/guided_decoding/outlines_decoding.py @@ -12,7 +12,7 @@ from transformers import PreTrainedTokenizerBase from vllm.model_executor.guided_decoding.outlines_logits_processors import ( CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor) -from vllm.model_executor.guided_decoding.reasoner import Reasoner +from vllm.reasoning import ReasoningParser from vllm.sampling_params import GuidedDecodingParams @@ -61,7 +61,7 @@ _MAX_THREADPOOL_WORKERS = 16 async def get_outlines_guided_decoding_logits_processor( guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase, - reasoner: Optional[Reasoner], + reasoner: Optional[ReasoningParser], ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor, None]: """ @@ -92,7 +92,7 @@ async def get_outlines_guided_decoding_logits_processor( def get_local_outlines_guided_decoding_logits_processor( guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase, - reasoner: Optional[Reasoner], + reasoner: Optional[ReasoningParser], ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor, None]: """ @@ -141,7 +141,7 @@ def _get_logits_processor( tokenizer: PreTrainedTokenizerBase, mode: GuidedDecodingMode, whitespace_pattern: Union[str, None], - reasoner: Optional[Reasoner], + reasoner: Optional[ReasoningParser], ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor]: if mode == GuidedDecodingMode.JSON: return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern, diff --git a/vllm/model_executor/guided_decoding/outlines_logits_processors.py b/vllm/model_executor/guided_decoding/outlines_logits_processors.py index 8b2a0f4cfe64b..31af4593f1123 100644 --- a/vllm/model_executor/guided_decoding/outlines_logits_processors.py +++ b/vllm/model_executor/guided_decoding/outlines_logits_processors.py @@ -34,8 +34,8 @@ from transformers import PreTrainedTokenizerBase import vllm.envs as envs from vllm.logger import init_logger -from vllm.model_executor.guided_decoding.reasoner import Reasoner from vllm.platforms import current_platform +from vllm.reasoning import ReasoningParser logger = init_logger(__name__) @@ -49,9 +49,9 @@ else: class BaseLogitsProcessor: - def __init__(self, guide: Guide, reasoner: Optional[Reasoner]): + def __init__(self, guide: Guide, reasoner: Optional[ReasoningParser]): self._guide: Guide = guide - self._reasoner: Optional[Reasoner] = reasoner + self._reasoner: Optional[ReasoningParser] = reasoner # CFGState is used for the FSM state for CFGGuide self._fsm_state: DefaultDict[int, Union[int, CFGState]] = defaultdict(int) @@ -69,7 +69,7 @@ class BaseLogitsProcessor: # Remove the reasoning tokens from the input_ids # We need this because our implementation relies on the # hash of the input_ids to store the FSM state. - input_ids = self._reasoner.extract_content(input_ids) + input_ids = self._reasoner.extract_content_ids(input_ids) seq_id = hash(tuple(input_ids)) @@ -142,7 +142,7 @@ class RegexLogitsProcessor(BaseLogitsProcessor): self, regex_string: str, tokenizer: PreTrainedTokenizerBase, - reasoner: Optional[Reasoner], + reasoner: Optional[ReasoningParser], ): """Compile the FSM that drives the regex-structured generation. @@ -163,7 +163,7 @@ class JSONLogitsProcessor(RegexLogitsProcessor): def __init__(self, schema: Union[str, Dict, BaseModel], tokenizer: PreTrainedTokenizerBase, whitespace_pattern: Union[str, None], - reasoner: Optional[Reasoner]): + reasoner: Optional[ReasoningParser]): """Compile the FSM that drives the JSON-guided generation. Parameters @@ -203,7 +203,7 @@ class CFGLogitsProcessor(BaseLogitsProcessor): return CFGGuide(cfg, tokenizer) def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase, - reasoner: Optional[Reasoner]): + reasoner: Optional[ReasoningParser]): """Compile the FSM that drives the context free grammar generation. Parameters diff --git a/vllm/model_executor/guided_decoding/reasoner/deepseek_reasoner.py b/vllm/model_executor/guided_decoding/reasoner/deepseek_reasoner.py deleted file mode 100644 index 7e61e6a9620c7..0000000000000 --- a/vllm/model_executor/guided_decoding/reasoner/deepseek_reasoner.py +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -from dataclasses import dataclass - -from transformers import PreTrainedTokenizer - -from vllm.model_executor.guided_decoding.reasoner.reasoner import Reasoner - - -@dataclass -class DeepSeekReasoner(Reasoner): - """ - Reasoner for DeepSeek R series models. - """ - start_token_id: int - end_token_id: int - - start_token: str = "" - end_token: str = "" - - @classmethod - def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner: - return cls(start_token_id=tokenizer.encode( - "", add_special_tokens=False)[0], - end_token_id=tokenizer.encode("", - add_special_tokens=False)[0]) - - def is_reasoning_end(self, input_ids: list[int]) -> bool: - return self.end_token_id in input_ids - - def extract_content(self, input_ids: list[int]) -> list[int]: - """ - Extract the content after the end tokens - """ - if self.end_token_id not in input_ids or \ - input_ids.index(self.end_token_id) + 1 == len(input_ids): - return [] - else: - return input_ids[input_ids.index(self.end_token_id) + 1:] diff --git a/vllm/model_executor/guided_decoding/reasoner/reasoner.py b/vllm/model_executor/guided_decoding/reasoner/reasoner.py deleted file mode 100644 index df21b1db62218..0000000000000 --- a/vllm/model_executor/guided_decoding/reasoner/reasoner.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -from abc import ABC, abstractmethod -from dataclasses import dataclass - -from transformers import PreTrainedTokenizer - - -@dataclass -class Reasoner(ABC): - - @abstractmethod - def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner: - pass - - @abstractmethod - def is_reasoning_end(self, input_ids: list[int]) -> bool: - pass - - @abstractmethod - def extract_content(self, input_ids: list[int]) -> list[int]: - pass diff --git a/vllm/model_executor/guided_decoding/xgrammar_decoding.py b/vllm/model_executor/guided_decoding/xgrammar_decoding.py index bc156223953e0..47b1e7e3f9811 100644 --- a/vllm/model_executor/guided_decoding/xgrammar_decoding.py +++ b/vllm/model_executor/guided_decoding/xgrammar_decoding.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from transformers import PreTrainedTokenizer from vllm.config import ModelConfig - from vllm.model_executor.guided_decoding.reasoner import Reasoner + from vllm.reasoning import ReasoningParser from vllm.sampling_params import GuidedDecodingParams logger = init_logger(__name__) @@ -37,7 +37,7 @@ def get_local_xgrammar_guided_decoding_logits_processor( guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizer, model_config: ModelConfig, - reasoner: Reasoner | None, + reasoner: ReasoningParser | None, max_threads: int = 8): config = GrammarConfig.from_guided_params(guided_params=guided_params, model_config=model_config, @@ -280,7 +280,7 @@ class GrammarConfig: class XGrammarLogitsProcessor: """Wrapper class to support pickle protocol""" config: GrammarConfig - reasoner: Reasoner | None = None + reasoner: ReasoningParser | None = None ctx: xgr.CompiledGrammar | None = None tokenizer_info: xgr.TokenizerInfo = None # type: ignore[assignment] diff --git a/vllm/entrypoints/openai/reasoning_parsers/__init__.py b/vllm/reasoning/__init__.py similarity index 100% rename from vllm/entrypoints/openai/reasoning_parsers/__init__.py rename to vllm/reasoning/__init__.py diff --git a/vllm/entrypoints/openai/reasoning_parsers/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py similarity index 82% rename from vllm/entrypoints/openai/reasoning_parsers/abs_reasoning_parsers.py rename to vllm/reasoning/abs_reasoning_parsers.py index c95ff191e4d2e..454167a0dc950 100644 --- a/vllm/entrypoints/openai/reasoning_parsers/abs_reasoning_parsers.py +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -17,7 +17,7 @@ logger = init_logger(__name__) class ReasoningParser: """ - Abstract reasoning parser class that should not be used directly. + Abstract reasoning parser class that should not be used directly. Provided and methods should be used in derived classes. It is used to extract reasoning content from the model output. @@ -32,6 +32,36 @@ class ReasoningParser: # whereas all tokenizers have .get_vocab() return self.model_tokenizer.get_vocab() + @abstractmethod + def is_reasoning_end(self, input_ids: list[int]) -> bool: + """ + Check if the reasoning content ends in the input_ids. + + It is used in structured engines like `xgrammar` to check if the + reasoning content ends in the model output. + + Parameters: + input_ids: list[int] + The input_ids of the model output. + + Returns: + bool + True if the reasoning content ends in the input_ids. + """ + + @abstractmethod + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + """ + Extract content token ids from the input_ids. + Parameters: + input_ids: list[int] + The input_ids of the model output. + Returns: + list[int] + The extracted content from the input_ids. + """ + + @abstractmethod def extract_reasoning_content( self, model_output: str, request: ChatCompletionRequest ) -> tuple[Optional[str], Optional[str]]: @@ -53,10 +83,7 @@ class ReasoningParser: A tuple containing the reasoning content and the content. """ - raise NotImplementedError( - "AbstractReasoningParser.extract_reasoning_calls " - "has not been implemented!") - + @abstractmethod def extract_reasoning_content_streaming( self, previous_text: str, @@ -73,43 +100,6 @@ class ReasoningParser: the current tokens/diffs, but also the information about what has previously been parsed and extracted (see constructor) """ - raise NotImplementedError( - "AbstractReasoningParser.extract_reasoning_content_streaming " - "has not been implemented!") - - # TODO: need to rebase by PR #14428 - @abstractmethod - def is_reasoning_end(self, input_ids: list[int]) -> bool: - """ - Check if the reasoning content ends in the input_ids. - Parameters: - input_ids: list[int] - The input_ids of the model output. - Returns: - bool - True if the reasoning content ends in the input_ids. - """ - - raise NotImplementedError( - "AbstractReasoningParser.is_reasoning_end has" - "not been implemented!") - - # TODO: need to rebase by PR #14428 - @abstractmethod - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - """ - Extract content token ids from the input_ids. - Parameters: - input_ids: list[int] - The input_ids of the model output. - Returns: - list[int] - The extracted content from the input_ids. - """ - - raise NotImplementedError( - "AbstractReasoningParser.extract_content_ids has" - " not been implemented!") class ReasoningParserManager: @@ -125,14 +115,16 @@ class ReasoningParserManager: if name in cls.reasoning_parsers: return cls.reasoning_parsers[name] - raise KeyError(f"reasoning helper: '{name}' not found in " - "reasoning_parsers") + raise KeyError( + f"reasoning helper: '{name}' not found in reasoning_parsers") @classmethod - def _register_module(cls, - module: type, - module_name: Optional[Union[str, list[str]]] = None, - force: bool = True) -> None: + def _register_module( + cls, + module: type, + module_name: Optional[Union[str, list[str]]] = None, + force: bool = True, + ) -> None: if not issubclass(module, ReasoningParser): raise TypeError("module must be subclass of ReasoningParser, " f"but got {type(module)}") @@ -149,13 +141,14 @@ class ReasoningParserManager: @classmethod def register_module( - cls, - name: Optional[Union[str, list[str]]] = None, - force: bool = True, - module: Union[type, None] = None) -> Union[type, Callable]: + cls, + name: Optional[Union[str, list[str]]] = None, + force: bool = True, + module: Union[type, None] = None, + ) -> Union[type, Callable]: """ Register module with the given name or name list. it can be used as a - decoder(with module as None) or normal function(with module as not + decoder(with module as None) or normal function(with module as not None). """ if not isinstance(force, bool): @@ -183,7 +176,7 @@ class ReasoningParserManager: @classmethod def import_reasoning_parser(cls, plugin_path: str) -> None: """ - Import a user-defined reasoning parser by the path + Import a user-defined reasoning parser by the path of the reasoning parser define file. """ module_name = os.path.splitext(os.path.basename(plugin_path))[0] diff --git a/vllm/entrypoints/openai/reasoning_parsers/deepseek_r1_reasoning_parser.py b/vllm/reasoning/deepseek_r1_reasoning_parser.py similarity index 64% rename from vllm/entrypoints/openai/reasoning_parsers/deepseek_r1_reasoning_parser.py rename to vllm/reasoning/deepseek_r1_reasoning_parser.py index 54e960168cf46..73be6d4d1ab13 100644 --- a/vllm/entrypoints/openai/reasoning_parsers/deepseek_r1_reasoning_parser.py +++ b/vllm/reasoning/deepseek_r1_reasoning_parser.py @@ -8,9 +8,8 @@ from transformers import PreTrainedTokenizerBase from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, DeltaMessage) -from vllm.entrypoints.openai.reasoning_parsers.abs_reasoning_parsers import ( - ReasoningParser, ReasoningParserManager) from vllm.logger import init_logger +from vllm.reasoning import ReasoningParser, ReasoningParserManager logger = init_logger(__name__) @@ -20,43 +19,45 @@ class DeepSeekR1ReasoningParser(ReasoningParser): """ Reasoning parser for DeepSeek R1 model. - The DeepSeek R1 model uses ... tokens to denote reasoning + The DeepSeek R1 model uses ... tokens to denote reasoning text. This parser extracts the reasoning content from the model output. """ + start_token_id: int + end_token_id: int + + start_token: str = "" + end_token: str = "" + def __init__(self, tokenizer: PreTrainedTokenizerBase): super().__init__(tokenizer) - self.think_start_token = "" - self.think_end_token = "" self.reasoning_regex = re.compile( - rf"{self.think_start_token}(.*?){self.think_end_token}", re.DOTALL) + rf"{self.start_token}(.*?){self.end_token}", re.DOTALL) if not self.model_tokenizer: raise ValueError( "The model tokenizer must be passed to the ReasoningParser " "constructor during construction.") - self.think_start_token_id = self.vocab.get(self.think_start_token) - self.think_end_token_id = self.vocab.get(self.think_end_token) - if (self.think_start_token_id is None - or self.think_end_token_id is None): + self.start_token_id = self.vocab.get(self.start_token) + self.end_token_id = self.vocab.get(self.end_token) + if self.start_token_id is None or self.end_token_id is None: raise RuntimeError( "DeepSeek R1 reasoning parser could not locate think start/end " "tokens in the tokenizer!") - # TODO: need to rebase by PR #14428 def is_reasoning_end(self, input_ids: list[int]) -> bool: - return self.think_end_token_id in input_ids + return self.end_token_id in input_ids def extract_content_ids(self, input_ids: list[int]) -> list[int]: """ Extract the content after the end tokens """ - if self.think_end_token_id not in input_ids[:-1]: + if self.end_token_id not in input_ids[:-1]: return [] else: - return input_ids[input_ids.index(self.think_end_token_id) + 1:] + return input_ids[input_ids.index(self.end_token_id) + 1:] def extract_reasoning_content_streaming( self, @@ -77,22 +78,24 @@ class DeepSeekR1ReasoningParser(ReasoningParser): """ # Skip single special tokens if len(delta_token_ids) == 1 and (delta_token_ids[0] in [ - self.think_start_token_id, self.think_end_token_id + self.start_token_id, self.end_token_id ]): return None # Check if is present in previous or delta. # Keep compatibility with models that don't generate tokens. - if self.think_start_token_id in previous_token_ids: - if self.think_end_token_id in delta_token_ids: + if self.start_token_id in previous_token_ids: + if self.end_token_id in delta_token_ids: # in previous, in delta, # extract reasoning content - end_index = delta_text.find(self.think_end_token) + end_index = delta_text.find(self.end_token) reasoning_content = delta_text[:end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, - content=content if content else None) - elif self.think_end_token_id in previous_token_ids: + content = delta_text[end_index + len(self.end_token):] + return DeltaMessage( + reasoning_content=reasoning_content, + content=content if content else None, + ) + elif self.end_token_id in previous_token_ids: # in previous, in previous, # reasoning content continues return DeltaMessage(content=delta_text) @@ -100,17 +103,18 @@ class DeepSeekR1ReasoningParser(ReasoningParser): # in previous, no in previous or delta, # reasoning content continues return DeltaMessage(reasoning_content=delta_text) - elif self.think_start_token_id in delta_token_ids: - if self.think_end_token_id in delta_token_ids: + elif self.start_token_id in delta_token_ids: + if self.end_token_id in delta_token_ids: # in delta, in delta, extract reasoning content - start_index = delta_text.find(self.think_start_token) - end_index = delta_text.find(self.think_end_token) + start_index = delta_text.find(self.start_token) + end_index = delta_text.find(self.end_token) reasoning_content = delta_text[start_index + - len(self.think_start_token - ):end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, - content=content if content else None) + len(self.start_token):end_index] + content = delta_text[end_index + len(self.end_token):] + return DeltaMessage( + reasoning_content=reasoning_content, + content=content if content else None, + ) else: # in delta, no in delta, # reasoning content continues @@ -119,15 +123,17 @@ class DeepSeekR1ReasoningParser(ReasoningParser): # No in previous or delta, also need to check for . # Because the model may have generated without # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - if self.think_end_token_id in delta_token_ids: + if self.end_token_id in delta_token_ids: # in delta with more tokens, # extract reasoning content and content - end_index = delta_text.find(self.think_end_token) + end_index = delta_text.find(self.end_token) reasoning_content = delta_text[:end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, - content=content if content else None) - elif self.think_end_token_id in previous_token_ids: + content = delta_text[end_index + len(self.end_token):] + return DeltaMessage( + reasoning_content=reasoning_content, + content=content if content else None, + ) + elif self.end_token_id in previous_token_ids: # in previous, thinking content ends return DeltaMessage(content=delta_text) else: @@ -137,22 +143,20 @@ class DeepSeekR1ReasoningParser(ReasoningParser): def extract_reasoning_content( self, model_output: str, request: ChatCompletionRequest ) -> tuple[Optional[str], Optional[str]]: - # DeepSeek R1 doesn't generate now. # Thus we assume the reasoning content is always at the start. # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - if self.think_end_token not in model_output: + if self.end_token not in model_output: return model_output, None else: # Add a start token if it's missing to keep compatibility. - if self.think_start_token not in model_output: - model_output = f"{self.think_start_token}{model_output}" + if self.start_token not in model_output: + model_output = f"{self.start_token}{model_output}" # Use a regex to find the reasoning content reasoning_content = self.reasoning_regex.findall(model_output)[0] end_index = len( - f"{self.think_start_token}{reasoning_content}{self.think_end_token}" - ) + f"{self.start_token}{reasoning_content}{self.end_token}") final_output = model_output[end_index:] if len(final_output) == 0: diff --git a/vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py b/vllm/reasoning/granite_reasoning_parser.py similarity index 99% rename from vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py rename to vllm/reasoning/granite_reasoning_parser.py index 117d051a73782..249ace1f167fa 100644 --- a/vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py +++ b/vllm/reasoning/granite_reasoning_parser.py @@ -8,9 +8,8 @@ from transformers import PreTrainedTokenizerBase from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, DeltaMessage) -from vllm.entrypoints.openai.reasoning_parsers.abs_reasoning_parsers import ( - ReasoningParser, ReasoningParserManager) from vllm.logger import init_logger +from vllm.reasoning import ReasoningParser, ReasoningParserManager logger = init_logger(__name__) From 280d074103160d042059dc60c28898fd9fb56568 Mon Sep 17 00:00:00 2001 From: "Li, Jiang" Date: Fri, 28 Mar 2025 16:36:31 +0800 Subject: [PATCH 068/593] [CPU][CI] Improve CPU Dockerfile (#15690) Signed-off-by: jiang1.li --- .buildkite/release-pipeline.yaml | 2 +- .buildkite/run-cpu-test.sh | 16 +- Dockerfile.cpu | 155 +++++++++++++----- .../getting_started/installation/cpu.md | 35 +++- .../installation/cpu/x86.inc.md | 2 + 5 files changed, 151 insertions(+), 59 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 18f582b6e4c94..a1dcb01e482bb 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -82,7 +82,7 @@ steps: queue: cpu_queue_postmerge commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain -f Dockerfile.cpu ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f Dockerfile.cpu ." - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)" env: DOCKER_BUILDKIT: "1" diff --git a/.buildkite/run-cpu-test.sh b/.buildkite/run-cpu-test.sh index 05744bb5225b8..bf9f191d3b064 100644 --- a/.buildkite/run-cpu-test.sh +++ b/.buildkite/run-cpu-test.sh @@ -8,15 +8,19 @@ set -ex CORE_RANGE=${CORE_RANGE:-48-95} NUMA_NODE=${NUMA_NODE:-1} -# Try building the docker image -numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build -t cpu-test-"$BUILDKITE_BUILD_NUMBER" -f Dockerfile.cpu . -numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 -f Dockerfile.cpu . - # Setup cleanup -remove_docker_container() { set -e; docker rm -f cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" || true; } +remove_docker_container() { + set -e; + docker rm -f cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" || true; + docker image rm cpu-test-"$BUILDKITE_BUILD_NUMBER" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 || true; +} trap remove_docker_container EXIT remove_docker_container +# Try building the docker image +numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --tag cpu-test-"$BUILDKITE_BUILD_NUMBER" --target vllm-test -f Dockerfile.cpu . +numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" --tag cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 --target vllm-test -f Dockerfile.cpu . + # Run the image, setting --shm-size=4g for tensor parallel. docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \ --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER" @@ -36,8 +40,6 @@ function cpu_tests() { # Run basic model test docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c " set -e - pip install -r vllm/requirements/test.txt - pip install -r vllm/requirements/cpu.txt pytest -v -s tests/kernels/test_cache.py -m cpu_model pytest -v -s tests/kernels/test_mla_decode_cpu.py -m cpu_model pytest -v -s tests/models/decoder_only/language -m cpu_model diff --git a/Dockerfile.cpu b/Dockerfile.cpu index a10090529d8a9..8133651865b50 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -1,69 +1,138 @@ # This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform. +# +# Build targets: +# vllm-openai (default): used for serving deployment +# vllm-test: used for CI tests +# vllm-dev: used for development +# +# Build arguments: +# PYTHON_VERSION=3.12 (default)|3.11|3.10|3.9 +# VLLM_CPU_DISABLE_AVX512=false (default)|true +# -FROM ubuntu:22.04 AS cpu-test-1 +######################### BASE IMAGE ######################### +FROM ubuntu:22.04 AS base + +WORKDIR /workspace/ + +ARG PYTHON_VERSION=3.12 +ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" + +# Install minimal dependencies and uv +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -y \ + && apt-get install -y --no-install-recommends ccache git curl wget ca-certificates \ + gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ + && curl -LsSf https://astral.sh/uv/install.sh | sh ENV CCACHE_DIR=/root/.cache/ccache - ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache -RUN --mount=type=cache,target=/var/cache/apt \ - apt-get update -y \ - && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \ - && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 +ENV PATH="/root/.local/bin:$PATH" +ENV VIRTUAL_ENV="/opt/venv" +RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} +ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html -# intel-openmp provides additional performance improvement vs. openmp -# tcmalloc provides better memory allocation efficiency, e.g, holding memory in caches to speed up access of commonly-used objects. -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install intel-openmp==2025.0.1 +ENV UV_HTTP_TIMEOUT=500 -ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so" +# Install Python dependencies +ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} +ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} +ENV UV_INDEX_STRATEGY="unsafe-best-match" +ENV UV_LINK_MODE="copy" +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,src=requirements/common.txt,target=requirements/common.txt \ + --mount=type=bind,src=requirements/cpu.txt,target=requirements/cpu.txt \ + uv pip install --upgrade pip && \ + uv pip install -r requirements/cpu.txt + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install intel-openmp==2024.2.1 intel_extension_for_pytorch==2.6.0 + +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/opt/venv/lib/libiomp5.so:$LD_PRELOAD" RUN echo 'ulimit -c 0' >> ~/.bashrc -RUN pip install intel_extension_for_pytorch==2.6.0 +######################### BUILD IMAGE ######################### +FROM base AS vllm-build -WORKDIR /workspace - -ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" -ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -RUN --mount=type=cache,target=/root/.cache/pip \ - --mount=type=bind,src=requirements/build.txt,target=requirements/build.txt \ - pip install --upgrade pip && \ - pip install -r requirements/build.txt - -FROM cpu-test-1 AS build - -WORKDIR /workspace/vllm - -RUN --mount=type=cache,target=/root/.cache/pip \ - --mount=type=bind,src=requirements/common.txt,target=requirements/common.txt \ - --mount=type=bind,src=requirements/cpu.txt,target=requirements/cpu.txt \ - pip install -v -r requirements/cpu.txt - -COPY . . ARG GIT_REPO_CHECK=0 -RUN --mount=type=bind,source=.git,target=.git \ - if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi - # Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ... ARG VLLM_CPU_DISABLE_AVX512 ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512} -RUN --mount=type=cache,target=/root/.cache/pip \ +WORKDIR /workspace/vllm + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,src=requirements/build.txt,target=requirements/build.txt \ + uv pip install -r requirements/build.txt + +COPY . . +RUN --mount=type=bind,source=.git,target=.git \ + if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi + +RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/root/.cache/ccache \ --mount=type=bind,source=.git,target=.git \ - VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \ - pip install dist/*.whl && \ - rm -rf dist + VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel + +######################### DEV IMAGE ######################### +FROM vllm-build AS vllm-dev + +WORKDIR /workspace/vllm + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get install -y --no-install-recommends vim numactl + +# install development dependencies (for testing) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install -e tests/vllm_test_utils + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/ccache \ + --mount=type=bind,source=.git,target=.git \ + VLLM_TARGET_DEVICE=cpu python3 setup.py develop + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install -r requirements/dev.txt && \ + pre-commit install --hook-type pre-commit --hook-type commit-msg + +ENTRYPOINT ["bash"] + +######################### TEST IMAGE ######################### +FROM base AS vllm-test WORKDIR /workspace/ -RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,src=requirements/test.txt,target=requirements/test.txt \ + uv pip install -r requirements/test.txt + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \ + uv pip install dist/*.whl + +ADD ./tests/ ./tests/ +ADD ./examples/ ./examples/ +ADD ./benchmarks/ ./benchmarks/ # install development dependencies (for testing) -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -e tests/vllm_test_utils +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install -e tests/vllm_test_utils + +ENTRYPOINT ["bash"] + +######################### RELEASE IMAGE ######################### +FROM base AS vllm-openai + +WORKDIR /workspace/ + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/ccache \ + --mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \ + uv pip install dist/*.whl ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"] diff --git a/docs/source/getting_started/installation/cpu.md b/docs/source/getting_started/installation/cpu.md index 1b2ffd6199945..844b184afc99b 100644 --- a/docs/source/getting_started/installation/cpu.md +++ b/docs/source/getting_started/installation/cpu.md @@ -159,18 +159,37 @@ Currently, there are no pre-built CPU wheels. ### Pre-built images -Currently, there are no pre-build CPU images. +:::::{tab-set} +:sync-group: device + +::::{tab-item} Intel/AMD x86 +:sync: x86 + +:::{include} cpu/x86.inc.md +:start-after: "### Pre-built images" +:end-before: "### Build image from source" +::: + +:::: + +::::: ### Build image from source ```console -$ docker build -f Dockerfile.cpu -t vllm-cpu-env --shm-size=4g . -$ docker run -it \ - --rm \ - --network=host \ - --cpuset-cpus= \ - --cpuset-mems= \ - vllm-cpu-env +$ docker build -f Dockerfile.cpu --tag vllm-cpu-env --target vllm-openai . + +# Launching OpenAI server +$ docker run --rm \ + --privileged=true \ + --shm-size=4g \ + -p 8000:8000 \ + -e VLLM_CPU_KVCACHE_SPACE= \ + -e VLLM_CPU_OMP_THREADS_BIND= \ + vllm-cpu-env \ + --model=meta-llama/Llama-3.2-1B-Instruct \ + --dtype=bfloat16 \ + other vLLM OpenAI server arguments ``` ::::{tip} diff --git a/docs/source/getting_started/installation/cpu/x86.inc.md b/docs/source/getting_started/installation/cpu/x86.inc.md index b2f3bafb4e511..9ae2035db5433 100644 --- a/docs/source/getting_started/installation/cpu/x86.inc.md +++ b/docs/source/getting_started/installation/cpu/x86.inc.md @@ -34,6 +34,8 @@ There are no pre-built wheels or images for this device, so you must build vLLM ### Pre-built images +See [https://gallery.ecr.aws/q9t5s3a7/vllm-cpu-release-repo](https://gallery.ecr.aws/q9t5s3a7/vllm-cpu-release-repo) + ### Build image from source ## Extra information From 70f2c2a7094cc5fbc6788f2b0b9b9da6973290cd Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Fri, 28 Mar 2025 17:10:40 +0800 Subject: [PATCH 069/593] [Bugfix] Fix 'InductorAdaptor object has no attribute 'cache_dir' (#15674) Signed-off-by: Jee Jee Li --- vllm/compilation/compiler_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/compilation/compiler_interface.py b/vllm/compilation/compiler_interface.py index ab0f98bdaa3e5..d6e44fa6d3414 100644 --- a/vllm/compilation/compiler_interface.py +++ b/vllm/compilation/compiler_interface.py @@ -144,6 +144,7 @@ class InductorAdaptor(CompilerInterface): return hash_str def initialize_cache(self, cache_dir: str, disable_cache: bool = False): + self.cache_dir = cache_dir if disable_cache: return # redirect the cache directory to a sub-directory @@ -156,7 +157,6 @@ class InductorAdaptor(CompilerInterface): triton_cache = os.path.join(cache_dir, "triton_cache") os.makedirs(triton_cache, exist_ok=True) os.environ["TRITON_CACHE_DIR"] = triton_cache - self.cache_dir = cache_dir def compile( self, From a10314c6b35c7bad4320286409d8b5e6d11aa56e Mon Sep 17 00:00:00 2001 From: Lize Cai Date: Fri, 28 Mar 2025 19:00:14 +0900 Subject: [PATCH 070/593] [Misc] Fix test_sleep to use query parameters (#14373) Signed-off-by: Lize Cai Signed-off-by: youkaichao Co-authored-by: youkaichao --- tests/entrypoints/openai/test_sleep.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_sleep.py b/tests/entrypoints/openai/test_sleep.py index 8bdf00bcee126..66d8d9294018c 100644 --- a/tests/entrypoints/openai/test_sleep.py +++ b/tests/entrypoints/openai/test_sleep.py @@ -25,8 +25,9 @@ def test_sleep_mode(): "VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0" }) as remote_server: + response = requests.post(remote_server.url_for("/sleep"), - data={"level": "1"}) + params={"level": "1"}) assert response.status_code == 200 response = requests.get(remote_server.url_for("/is_sleeping")) assert response.status_code == 200 From 3bbaacbe15c90ac5339aa46f481311a80038d3a9 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Fri, 28 Mar 2025 19:20:35 +0800 Subject: [PATCH 071/593] [Bugfix][Frontend] Eliminate regex based check in reasoning full generator (#14821) Signed-off-by: Ce Gao --- .../test_deepseekr1_reasoning_parser.py | 64 +++++++++++++++++++ .../reasoning/deepseek_r1_reasoning_parser.py | 43 +++++++------ 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/tests/reasoning/test_deepseekr1_reasoning_parser.py b/tests/reasoning/test_deepseekr1_reasoning_parser.py index 7b6af183a86ad..1b669c8fd2fb9 100644 --- a/tests/reasoning/test_deepseekr1_reasoning_parser.py +++ b/tests/reasoning/test_deepseekr1_reasoning_parser.py @@ -90,6 +90,40 @@ SHORTEST_REASONING_WITH_THINK = { "content": "This is the rest", "is_reasoning_end": True, } +THINK_NO_END = { + "output": "This is a reasoning section", + "reasoning_content": "This is a reasoning section", + "content": None, + "is_reasoning_end": False, +} +EMPTY = { + "output": "", + "reasoning_content": "", + "content": None, + "is_reasoning_end": False, +} +EMPTY_STREAMING = { + "output": "", + "reasoning_content": None, + "content": None, + "is_reasoning_end": False, +} +NEW_LINE = { + "output": "\nThis is a reasoning section\nThis is the rest", + "reasoning_content": "This is a reasoning section", + "content": "\nThis is the rest", + "is_reasoning_end": True, +} +# Streaming cannot handle new lines at the beginning of the output +# because we need to support ... and ... +# We cannot know if the text before is reasoning content +# or not. +NEW_LINE_STREAMING = { + "output": "\nThis is a reasoning section\nThis is the rest", + "reasoning_content": "\nThis is a reasoning section", + "content": "\nThis is the rest", + "is_reasoning_end": True, +} TEST_CASES = [ pytest.param( @@ -182,6 +216,36 @@ TEST_CASES = [ SHORTEST_REASONING_WITH_THINK, id="shortest_with_think_streaming", ), + pytest.param( + False, + THINK_NO_END, + id="think_no_end", + ), + pytest.param( + True, + THINK_NO_END, + id="think_no_end_streaming", + ), + pytest.param( + False, + EMPTY, + id="empty", + ), + pytest.param( + True, + EMPTY_STREAMING, + id="empty_streaming", + ), + pytest.param( + False, + NEW_LINE, + id="new_line", + ), + pytest.param( + True, + NEW_LINE_STREAMING, + id="new_line_streaming", + ), ] diff --git a/vllm/reasoning/deepseek_r1_reasoning_parser.py b/vllm/reasoning/deepseek_r1_reasoning_parser.py index 73be6d4d1ab13..1c283c092a28c 100644 --- a/vllm/reasoning/deepseek_r1_reasoning_parser.py +++ b/vllm/reasoning/deepseek_r1_reasoning_parser.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -import re from collections.abc import Sequence from typing import Optional, Union @@ -32,9 +31,6 @@ class DeepSeekR1ReasoningParser(ReasoningParser): def __init__(self, tokenizer: PreTrainedTokenizerBase): super().__init__(tokenizer) - self.reasoning_regex = re.compile( - rf"{self.start_token}(.*?){self.end_token}", re.DOTALL) - if not self.model_tokenizer: raise ValueError( "The model tokenizer must be passed to the ReasoningParser " @@ -143,23 +139,34 @@ class DeepSeekR1ReasoningParser(ReasoningParser): def extract_reasoning_content( self, model_output: str, request: ChatCompletionRequest ) -> tuple[Optional[str], Optional[str]]: + """ + Extract reasoning content from the model output. + + For text abcxyz: + - 'abc' goes to reasoning_content + - 'xyz' goes to content + + Returns: + tuple[Optional[str], Optional[str]]: reasoning content and content + """ + + # Check if the start token is present in the model output, remove it + # if it is present. + model_output_parts = model_output.partition(self.start_token) + model_output = model_output_parts[2] if model_output_parts[ + 1] else model_output_parts[0] + # DeepSeek R1 doesn't generate now. # Thus we assume the reasoning content is always at the start. # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f if self.end_token not in model_output: return model_output, None else: - # Add a start token if it's missing to keep compatibility. - if self.start_token not in model_output: - model_output = f"{self.start_token}{model_output}" - # Use a regex to find the reasoning content - reasoning_content = self.reasoning_regex.findall(model_output)[0] - - end_index = len( - f"{self.start_token}{reasoning_content}{self.end_token}") - final_output = model_output[end_index:] - - if len(final_output) == 0: - return reasoning_content, None - - return reasoning_content, final_output + reasoning_content, _, content = model_output.partition( + self.end_token) + # If the end token is not found, return the model output as is. + # It should not happen since we already checked for the presence + # of the end token. + # If generation stops right after end-of-think, return null content + final_content = content or None + return reasoning_content, final_content From fd5fd2690275e90865023a0bcac0047ecb3f3897 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Fri, 28 Mar 2025 19:40:12 +0800 Subject: [PATCH 072/593] [Frontend] update priority for --api-key and VLLM_API_KEY (#15588) Signed-off-by: reidliu41 Co-authored-by: reidliu41 --- vllm/entrypoints/openai/api_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 6c1f60fa6a3b4..7dbe31e62da67 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -818,7 +818,8 @@ def build_app(args: Namespace) -> FastAPI: return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST) - if token := envs.VLLM_API_KEY or args.api_key: + # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY + if token := args.api_key or envs.VLLM_API_KEY: @app.middleware("http") async def authentication(request: Request, call_next): From 0b4167526d030391785e28e44c68d2e1cdc5ad3b Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 28 Mar 2025 13:03:21 +0000 Subject: [PATCH 073/593] [Docs] Add "Generation quality changed" section to troubleshooting (#15701) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- docs/source/getting_started/troubleshooting.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/getting_started/troubleshooting.md b/docs/source/getting_started/troubleshooting.md index fdfaf9f932698..87fa442e9a489 100644 --- a/docs/source/getting_started/troubleshooting.md +++ b/docs/source/getting_started/troubleshooting.md @@ -26,6 +26,14 @@ To isolate the model downloading and loading issue, you can use the `--load-form If the model is too large to fit in a single GPU, you will get an out-of-memory (OOM) error. Consider [using tensor parallelism](#distributed-serving) to split the model across multiple GPUs. In that case, every process will read the whole model and split it into chunks, which makes the disk reading time even longer (proportional to the size of tensor parallelism). You can convert the model checkpoint to a sharded checkpoint using . The conversion process might take some time, but later you can load the sharded checkpoint much faster. The model loading time should remain constant regardless of the size of tensor parallelism. +## Generation quality changed + +In v0.8.0, the source of default sampling parameters was changed in . Prior to v0.8.0, the default sampling parameters came from vLLM's set of neutral defaults. From v0.8.0 onwards, the default sampling parameters come from the `generation_config.json` provided by the model creator. + +In most cases, this should lead to higher quality responses, because the model creator is likely to know which sampling parameters are best for their model. However, in some cases the defaults provided by the model creator can lead to degraded performance. + +You can check if this is happening by trying the old defaults with `--generation-config vllm` for online and `generation_config="vllm"` for offline. If, after trying this, your generation quality improves we would recommend continuing to use the vLLM defaults and petition the model creator on to update their default `generation_config.json` so that it produces better quality generations. + ## Enable more logging If other strategies don't solve the problem, it's likely that the vLLM instance is stuck somewhere. You can use the following environment variables to help debug the issue: From 91276c57210b36997861af706a48ac784573ed4c Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Fri, 28 Mar 2025 21:14:09 +0800 Subject: [PATCH 074/593] [Model] Adding torch compile annotations to chatglm (#15624) Signed-off-by: Jee Jee Li --- vllm/model_executor/models/chatglm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vllm/model_executor/models/chatglm.py b/vllm/model_executor/models/chatglm.py index 14dca23b3934f..a51a0af9e2bcf 100644 --- a/vllm/model_executor/models/chatglm.py +++ b/vllm/model_executor/models/chatglm.py @@ -10,6 +10,7 @@ from torch import nn from torch.nn import LayerNorm from vllm.attention import Attention +from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.model_executor.layers.activation import SiluAndMul @@ -293,6 +294,7 @@ class GLMTransformer(nn.Module): return hidden_states +@support_torch_compile class ChatGLMModel(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): From 3b00ff91380044fa409612401309b9cb6a82685f Mon Sep 17 00:00:00 2001 From: Chauncey Date: Fri, 28 Mar 2025 21:14:53 +0800 Subject: [PATCH 075/593] [Bugfix][v1] xgrammar structured output supports Enum. (#15594) Signed-off-by: chaunceyjiang --- .../llm/test_struct_output_generate.py | 53 +++++++++++++++++++ vllm/v1/structured_output/utils.py | 4 -- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/tests/v1/entrypoints/llm/test_struct_output_generate.py b/tests/v1/entrypoints/llm/test_struct_output_generate.py index 6bdfa0fae4a2c..00fa47575b6ae 100644 --- a/tests/v1/entrypoints/llm/test_struct_output_generate.py +++ b/tests/v1/entrypoints/llm/test_struct_output_generate.py @@ -4,10 +4,12 @@ from __future__ import annotations import json import re +from enum import Enum from typing import Any import jsonschema import pytest +from pydantic import BaseModel from vllm.entrypoints.llm import LLM from vllm.outputs import RequestOutput @@ -390,3 +392,54 @@ def test_guided_choice_completion( assert generated_text is not None assert generated_text in sample_guided_choice print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") + + +class CarType(str, Enum): + sedan = "sedan" + suv = "SUV" + truck = "Truck" + coupe = "Coupe" + + +class CarDescription(BaseModel): + brand: str + model: str + car_type: CarType + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("guided_decoding_backend", + GUIDED_DECODING_BACKENDS_V1) +@pytest.mark.parametrize("model_name", MODELS_TO_TEST) +def test_guided_json_completion_with_enum( + monkeypatch: pytest.MonkeyPatch, + guided_decoding_backend: str, + model_name: str, +): + monkeypatch.setenv("VLLM_USE_V1", "1") + llm = LLM(model=model_name, + max_model_len=1024, + guided_decoding_backend=guided_decoding_backend) + json_schema = CarDescription.model_json_schema() + sampling_params = SamplingParams( + temperature=1.0, + max_tokens=1000, + guided_decoding=GuidedDecodingParams(json=json_schema)) + outputs = llm.generate( + prompts="Generate a JSON with the brand, model and car_type of" + "the most iconic car from the 90's", + sampling_params=sampling_params, + use_tqdm=True) + + assert outputs is not None + + for output in outputs: + assert output is not None + assert isinstance(output, RequestOutput) + prompt = output.prompt + + generated_text = output.outputs[0].text + assert generated_text is not None + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") + output_json = json.loads(generated_text) + jsonschema.validate(instance=output_json, schema=json_schema) diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 694e46f763f02..a771256ef29fd 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -26,10 +26,6 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: if "pattern" in obj: return True - # Check for enum restrictions - if "enum" in obj: - return True - # Check for numeric ranges if obj.get("type") in ("integer", "number") and any( key in obj From 541d1df486ac863ab057d842791695763de6f58b Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 28 Mar 2025 23:27:52 +0800 Subject: [PATCH 076/593] [Bugfix] `embed_is_patch` for Idefics3 (#15696) Signed-off-by: DarkLight1337 --- vllm/model_executor/models/commandr.py | 1 - vllm/model_executor/models/idefics3.py | 501 ++++++++++++++-------- vllm/model_executor/models/mllama.py | 1 - vllm/model_executor/models/qwen2_audio.py | 2 +- vllm/model_executor/models/ultravox.py | 3 +- 5 files changed, 320 insertions(+), 188 deletions(-) diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py index b0cb4a62333a4..e7e73f446df27 100644 --- a/vllm/model_executor/models/commandr.py +++ b/vllm/model_executor/models/commandr.py @@ -24,7 +24,6 @@ from typing import Iterable, Optional, Set, Tuple, Union import torch -import torch.utils.checkpoint from torch import nn from transformers import CohereConfig diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 432f26141048b..327ec4640f03e 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -17,16 +17,14 @@ import math from collections.abc import Iterable, Mapping, Sequence -from typing import Dict, List, Literal, Optional, Set, Tuple, TypedDict, Union +from typing import Dict, Literal, Optional, Set, Tuple, TypedDict, Union import torch -import torch.utils.checkpoint from torch import nn from transformers import (BatchFeature, Idefics3Config, Idefics3ImageProcessor, Idefics3Processor) from vllm.config import VllmConfig -from vllm.logger import init_logger from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -35,13 +33,16 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.sampling_metadata import SamplingMetadata from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs -from vllm.multimodal.inputs import NestedTensors -from vllm.multimodal.parse import ImageProcessorItems +from vllm.multimodal.parse import ImageProcessorItems, ImageSize +# yapf conflicts with isort for this block +# yapf: disable from vllm.multimodal.processing import (BaseMultiModalProcessor, BaseProcessingInfo, MultiModalDataItems, MultiModalFieldConfig, - PromptReplacement, PromptUpdate) + PromptReplacement, PromptUpdate, + encode_tokens) +# yapf: enable from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors @@ -53,18 +54,28 @@ from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal from .llama import LlamaModel from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix, merge_multimodal_embeddings) - -logger = init_logger(__name__) +from .vision import scatter_patch_features, select_patch_features class Idefics3ImagePixelInputs(TypedDict): type: Literal["pixel_values"] - data: torch.Tensor + pixel_values: torch.Tensor """ Shape: `(batch_size * num_images * num_patches, num_channels, height, width)` """ - pixel_attention_mask: Optional[torch.BoolTensor] + pixel_attention_mask: torch.Tensor + + num_patches: torch.Tensor + """Shape: `(batch_size * num_images)`""" + + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] + """ + A boolean mask indicating which image embeddings correspond + to patch tokens. + + Shape: `(batch_size * num_images, num_embeds)` + """ class Idefics3ImageEmbeddingInputs(TypedDict): @@ -75,6 +86,14 @@ class Idefics3ImageEmbeddingInputs(TypedDict): `hidden_size` must match the hidden size of language model backbone. """ + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] + """ + A boolean mask indicating which image embeddings correspond + to patch tokens. + + Shape: `(batch_size * num_images, num_embeds)` + """ + ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs] @@ -100,32 +119,14 @@ class Idefics3ProcessingInfo(BaseProcessingInfo): seq_len: int, mm_counts: Mapping[str, int], ) -> Mapping[str, int]: - hf_processor = self.get_hf_processor() - image_processor: Idefics3ImageProcessor = hf_processor.image_processor - grid_w, grid_h = self._get_image_feature_grid_size( - image_width=image_processor.size['longest_edge'], - image_height=image_processor.size['longest_edge'], - ) - num_image_token = (grid_w * grid_h + 1) * hf_processor.image_seq_len - # Calculate Non-image-token length - # NOTE: and are special token for SmolVLM - # but not for Idefic3, so we need to tokenize them to get actual length. - tokenizer = self.get_tokenizer() - tile_token_len = len(tokenizer.tokenize("")) - glob_token_len = len(tokenizer.tokenize(hf_processor.global_image_tag)) - # linebreak and always cost 1 token - fake_token_len = lb_len = 1 - non_image_token = (grid_w * grid_h) * ( - tile_token_len + fake_token_len) + glob_token_len + ( - grid_h + 1) * lb_len + fake_token_len - return {"image": num_image_token + non_image_token} + return {"image": self.get_max_image_tokens()} def _resize_output_size(self, *, height: int, width: int, max_len: Optional[int] = None, - min_len: Optional[int] = 1, + min_len: int = 1, max_size: Optional[int] = None) -> tuple[int, int]: # Set default value for max_len if not provided max_len = max(height, width) if max_len is None else max_len @@ -181,10 +182,13 @@ class Idefics3ProcessingInfo(BaseProcessingInfo): *, image_width: int, image_height: int, - size: Optional[dict[str, object]] = None, + processor: Optional[Idefics3Processor], ) -> tuple[int, int]: - hf_processor = self.get_hf_processor(size=size) - image_processor: Idefics3ImageProcessor = hf_processor.image_processor + if processor is None: + processor = self.get_hf_processor() + + image_processor: Idefics3ImageProcessor = processor.image_processor + max_image_size = image_processor.max_image_size['longest_edge'] size = image_processor.size['longest_edge'] assert size % max_image_size == 0, ( @@ -204,6 +208,105 @@ class Idefics3ProcessingInfo(BaseProcessingInfo): grid_h = grid_w = 0 return grid_w, grid_h + def get_num_patches( + self, + *, + image_width: int, + image_height: int, + processor: Optional[Idefics3Processor], + ) -> int: + grid_w, grid_h = self._get_image_feature_grid_size( + image_width=image_width, + image_height=image_height, + processor=processor, + ) + + return grid_w * grid_h + 1 + + def get_image_repl( + self, + *, + image_width: int, + image_height: int, + processor: Optional[Idefics3Processor], + ) -> str: + if processor is None: + processor = self.get_hf_processor() + + image_token = processor.image_token.content + fake_image_token = processor.fake_image_token.content + global_img_token = processor.global_image_tag + image_seq_len = processor.image_seq_len + grid_placeholder = "" + + p_img = image_token * image_seq_len + global_img_placeholder = fake_image_token + global_img_token + p_img + tile_img_placeholder = fake_image_token + grid_placeholder + p_img + + grid_w, grid_h = self._get_image_feature_grid_size( + image_width=image_width, + image_height=image_height, + processor=processor, + ) + if grid_w == 0 and grid_h == 0: + return global_img_placeholder + fake_image_token + + tiles_placeholder = list[str]() + for i in range(grid_h): + for j in range(grid_w): + placeholder_per_tile = tile_img_placeholder.format(n_h=i + 1, + n_w=j + 1) + tiles_placeholder.append(placeholder_per_tile) + # Add line break if it is the last tile in the row + if j == grid_w - 1: + tiles_placeholder.append("\n") + + return "".join([ + *tiles_placeholder, + "\n", + global_img_placeholder, + fake_image_token, + ]) + + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + processor: Optional[Idefics3Processor], + ) -> int: + tokenizer = self.get_tokenizer() + image_repl = self.get_image_repl( + image_width=image_width, + image_height=image_height, + processor=processor, + ) + + image_repl_tokens = encode_tokens( + tokenizer, + image_repl, + add_special_tokens=False, + ) + return len(image_repl_tokens) + + def get_image_size_with_most_features(self) -> ImageSize: + processor = self.get_hf_processor() + image_processor: Idefics3ImageProcessor = processor.image_processor + + return ImageSize( + width=image_processor.size["longest_edge"], + height=image_processor.size["longest_edge"], + ) + + def get_max_image_tokens(self) -> int: + target_width, target_height = self.get_image_size_with_most_features() + + return self.get_num_image_tokens( + image_width=target_width, + image_height=target_height, + processor=None, + ) + class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo] ): @@ -217,7 +320,7 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo] hf_processor = self.info.get_hf_processor() image_processor: Idefics3ImageProcessor = hf_processor.image_processor longest_edge = image_processor.max_image_size['longest_edge'] - image_token: str = hf_processor.image_token.content + image_token = hf_processor.image_token.content mm_data = { "image": @@ -241,26 +344,61 @@ class Idefics3MultiModalProcessor( mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], ) -> BatchFeature: - if mm_data: - processed_outputs = super()._call_hf_processor( - prompt, mm_data, mm_kwargs) - image_grids = [ - self.info._get_image_feature_grid_size( - image_width=img.width, - image_height=img.height, - **mm_kwargs, - ) for img in mm_data["images"] - ] - image_patches = list(map(lambda x: math.prod(x) + 1, image_grids)) - for key in ("pixel_values", "pixel_attention_mask"): - data = processed_outputs.pop(key) - data = data.flatten(0, 1).split(image_patches) - processed_outputs[key] = data - else: - tokenizer = self.info.get_tokenizer() - processed_outputs = tokenizer(prompt, - add_special_tokens=True, - return_tensors="pt") + # Text-only input not supported in composite processor + if not (images := mm_data.get("images", [])): + prompt_ids = self.info.get_tokenizer().encode(prompt) + prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + + processed_outputs = super()._call_hf_processor( + prompt, + mm_data, + mm_kwargs, + ) + + parsed_images = (self._get_data_parser().parse_mm_data({ + "image": images + }).get_items("image", ImageProcessorItems)) + image_sizes = [ + parsed_images.get_image_size(i) for i in range(len(parsed_images)) + ] + hf_processor = self.info.get_hf_processor(**mm_kwargs) + + image_repl_features = [ + self.info.get_image_repl(image_width=size.width, + image_height=size.height, + processor=hf_processor) + for size in image_sizes + ] + + tokenizer = self.info.get_tokenizer() + image_repls_feature_tokens = [ + tokenizer.encode(image_repl, add_special_tokens=False) + for image_repl in image_repl_features + ] + + vocab = tokenizer.get_vocab() + image_token_id = vocab[hf_processor.image_token.content] + + embed_is_patch = [ + torch.tensor(image_repl_tokens) == image_token_id + for image_repl_tokens in image_repls_feature_tokens + ] + processed_outputs["embed_is_patch"] = embed_is_patch + + num_patches = [ + self.info.get_num_patches( + image_width=size.width, + image_height=size.height, + processor=hf_processor, + ) for size in image_sizes + ] + processed_outputs["num_patches"] = torch.tensor(num_patches) + + # Remove the extra batch dimension + processed_outputs["pixel_values"].squeeze_(0) + processed_outputs["pixel_attention_mask"].squeeze_(0) + return processed_outputs def _get_mm_fields_config( @@ -268,10 +406,16 @@ class Idefics3MultiModalProcessor( hf_inputs: BatchFeature, hf_processor_mm_kwargs: Mapping[str, object], ) -> Mapping[str, MultiModalFieldConfig]: + num_patches = hf_inputs.get("num_patches", torch.empty(0)) + return dict( - pixel_values=MultiModalFieldConfig.batched("image"), - pixel_attention_mask=MultiModalFieldConfig.batched("image"), + pixel_values=MultiModalFieldConfig.flat_from_sizes( + "image", num_patches), + pixel_attention_mask=MultiModalFieldConfig.flat_from_sizes( + "image", num_patches), image_embeds=MultiModalFieldConfig.batched("image"), + num_patches=MultiModalFieldConfig.batched("image"), + embed_is_patch=MultiModalFieldConfig.batched("image"), ) def _get_prompt_updates( @@ -281,42 +425,18 @@ class Idefics3MultiModalProcessor( out_mm_kwargs: MultiModalKwargs, ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token = hf_processor.image_token.content - fake_image_token = hf_processor.fake_image_token.content - global_img_token = hf_processor.global_image_tag - image_seq_len = hf_processor.image_seq_len - grid_placeholder = "" - - p_img = image_token * image_seq_len - global_img_placeholder = fake_image_token + global_img_token + p_img - tile_img_placeholder = fake_image_token + grid_placeholder + p_img def get_replacement_idefics3(item_idx: int) -> str: images = mm_items.get_items("image", ImageProcessorItems) image_size = images.get_image_size(item_idx) - grid_w, grid_h = self.info._get_image_feature_grid_size( + + return self.info.get_image_repl( image_width=image_size.width, image_height=image_size.height, - **hf_processor_mm_kwargs, + processor=hf_processor, ) - if grid_w == 0 and grid_h == 0: - image_placeholder = global_img_placeholder - else: - tiles_placeholder = list[str]() - for i in range(grid_h): - for j in range(grid_w): - placeholder_per_tile = tile_img_placeholder.format( - n_h=i + 1, n_w=j + 1) - tiles_placeholder.append(placeholder_per_tile) - # Add line break if it is the last tile in the row - if j == grid_w - 1: - tiles_placeholder.append("\n") - - image_placeholder = "".join( - [*tiles_placeholder, "\n", global_img_placeholder]) - return image_placeholder + fake_image_token return [ PromptReplacement( @@ -424,73 +544,13 @@ class Idefics3Model(nn.Module): config.vision_config.patch_size)**2) / (config.scale_factor**2)) self.image_token_id = self.config.image_token_id - def _validate_pixel_values( - self, data: Union[torch.Tensor, List[torch.Tensor]] - ) -> Union[torch.Tensor, List[torch.Tensor]]: - - h = w = self.config.vision_config.image_size - expected_dims = (3, h, w) - - def _validate_shape(d: torch.Tensor): - actual_dims = tuple(d.shape[1:]) - - if actual_dims != expected_dims: - expected_expr = ("num_patches", *map(str, expected_dims)) - raise ValueError( - "The expected shape of pixel values per image per batch " - f"is {expected_expr}. You supplied {tuple(d.shape)}.") - - for d in data: - _validate_shape(d) - - return data - - def _parse_and_validate_image_input( - self, **kwargs: object) -> Optional[ImageInputs]: - pixel_values = kwargs.pop("pixel_values", None) - image_embeds = kwargs.pop("image_embeds", None) - pixel_attention_mask = kwargs.pop("pixel_attention_mask", None) - - if pixel_values is None and image_embeds is None: - return None - - if image_embeds is not None: - if not isinstance(image_embeds, (torch.Tensor, list)): - raise ValueError("Incorrect type of image embeddings. " - f"Got type: {type(image_embeds)}") - - return Idefics3ImageEmbeddingInputs( - type="image_embeds", - data=flatten_bn(image_embeds, concat=True), - ) - - if pixel_values is not None: - if not isinstance(pixel_values, (torch.Tensor, list)): - raise ValueError("Incorrect type of pixel values. " - f"Got type: {type(pixel_values)}") - - if isinstance(pixel_values, list): - pixel_values = torch.cat(pixel_values, dim=1) - pixel_attention_mask = torch.cat(pixel_attention_mask, dim=1) - else: - pixel_values = flatten_bn(pixel_values) - pixel_attention_mask = flatten_bn(pixel_attention_mask) - - return Idefics3ImagePixelInputs( - type="pixel_values", - data=self._validate_pixel_values(pixel_values), - pixel_attention_mask=pixel_attention_mask) - - raise AssertionError("This line should be unreachable.") - - def _image_pixels_to_features( + def image_pixels_to_features( self, pixel_values: torch.Tensor, - pixel_attention_mask: Optional[torch.BoolTensor] = None, - ) -> NestedTensors: + pixel_attention_mask: torch.Tensor, + ) -> torch.Tensor: # NOTE: we skip the step to select the vision feature layer since # this is already done inside the vision tower - num_patches = [x.size(0) for x in pixel_values] pixel_values = pixel_values.to( dtype=self.vision_model.embeddings.patch_embedding.weight.dtype ) # fp16 compatibility @@ -502,17 +562,9 @@ class Idefics3Model(nn.Module): pixel_values = pixel_values[real_images_inds].contiguous() # Handle the vision attention mask - if pixel_attention_mask is None: - pixel_attention_mask = torch.ones( - size=(pixel_values.size(0), pixel_values.size(2), - pixel_values.size(3)), - dtype=torch.bool, - device=pixel_values.device, - ) - else: - # Remove padding images from the mask - pixel_attention_mask = pixel_attention_mask[ - real_images_inds].contiguous() + # Remove padding images from the mask + pixel_attention_mask = pixel_attention_mask[ + real_images_inds].contiguous() patch_size = self.config.vision_config.patch_size patches_subgrid = pixel_attention_mask.unfold(dimension=1, @@ -529,27 +581,7 @@ class Idefics3Model(nn.Module): patch_attention_mask=patch_attention_mask, ) - return image_hidden_states.split(num_patches) - - def _process_image_pixels( - self, inputs: Idefics3ImagePixelInputs) -> NestedTensors: - assert self.vision_model is not None - - pixel_values = inputs["data"] - pixel_attention_mask = inputs["pixel_attention_mask"] - - return self._image_pixels_to_features(pixel_values, - pixel_attention_mask) - - def _process_image_input(self, image_input: ImageInputs) -> torch.Tensor: - if image_input["type"] == "image_embeds": - return image_input["data"] - - assert self.vision_model is not None - image_features = self._process_image_pixels(image_input) - num_patches = [x.size(0) for x in image_features] - image_features = torch.cat(image_features) - return self.connector(image_features).split(num_patches) + return image_hidden_states def get_input_embeddings( self, @@ -616,13 +648,113 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, self.logits_processor = LogitsProcessor(config.text_config.vocab_size) self.sampler = get_sampler() + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape) + + if actual_dims != expected_dims: + expected_expr = str(expected_dims) + raise ValueError( + "The expected shape of pixel values per image per batch " + f" per patch is {expected_expr}. " + f"You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[ImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + embed_is_patch = kwargs.pop("embed_is_patch") + if not isinstance(embed_is_patch, (torch.Tensor, list)): + raise ValueError("Incorrect type of embed_is_patch. " + f"Got type: {type(embed_is_patch)}") + + embed_is_patch = flatten_bn(embed_is_patch) + + if image_embeds is not None: + if not isinstance(image_embeds, (torch.Tensor, list)): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + + return Idefics3ImageEmbeddingInputs( + type="image_embeds", + data=flatten_bn(image_embeds, concat=True), + embed_is_patch=embed_is_patch, + ) + + if pixel_values is not None: + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + pixel_attention_mask = kwargs.pop("pixel_attention_mask") + if not isinstance(pixel_attention_mask, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel_attention_mask. " + f"Got type: {type(pixel_attention_mask)}") + + num_patches = kwargs.pop("num_patches") + if not isinstance(num_patches, (torch.Tensor, list)): + raise ValueError("Incorrect type of num_patches. " + f"Got type: {type(num_patches)}") + + pixel_values = flatten_bn(pixel_values, concat=True) + pixel_attention_mask = flatten_bn(pixel_attention_mask, + concat=True) + num_patches = flatten_bn(num_patches, concat=True) + + return Idefics3ImagePixelInputs( + type="pixel_values", + pixel_values=self._validate_pixel_values(pixel_values), + pixel_attention_mask=pixel_attention_mask, + num_patches=num_patches, + embed_is_patch=embed_is_patch, + ) + + raise AssertionError("This line should be unreachable.") + + def _process_image_pixels( + self, inputs: Idefics3ImagePixelInputs) -> torch.Tensor: + pixel_values = inputs["pixel_values"] + pixel_attention_mask = inputs["pixel_attention_mask"] + + return self.model.image_pixels_to_features( + pixel_values, + pixel_attention_mask=pixel_attention_mask, + ) + + def _process_image_input(self, image_input: ImageInputs) -> torch.Tensor: + if image_input["type"] == "image_embeds": + return image_input["data"] + + image_features = self._process_image_pixels(image_input) + image_features = self.model.connector(image_features) + + num_patches = image_input["num_patches"] + return image_features.split(num_patches.tolist()) + def get_multimodal_embeddings( self, **kwargs: object) -> Optional[MultiModalEmbeddings]: - image_input = self.model._parse_and_validate_image_input(**kwargs) + image_input = self._parse_and_validate_image_input(**kwargs) if image_input is None: return None - vision_embeddings = self.model._process_image_input(image_input) - return vision_embeddings + + image_features = self._process_image_input(image_input) + + return scatter_patch_features( + image_features, + image_input["embed_is_patch"], + ) def get_input_embeddings( self, @@ -632,8 +764,11 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, inputs_embeds = self.model.get_input_embeddings(input_ids) if multimodal_embeddings is not None: inputs_embeds = merge_multimodal_embeddings( - input_ids, inputs_embeds, multimodal_embeddings, - self.config.image_token_id) + input_ids, + inputs_embeds, + select_patch_features(multimodal_embeddings), + self.config.image_token_id, + ) return inputs_embeds def forward( diff --git a/vllm/model_executor/models/mllama.py b/vllm/model_executor/models/mllama.py index d2c8fb7237274..ac4bdbc41e441 100644 --- a/vllm/model_executor/models/mllama.py +++ b/vllm/model_executor/models/mllama.py @@ -21,7 +21,6 @@ from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union import numpy as np import torch import torch.nn.functional as F -import torch.utils.checkpoint import transformers.models.mllama.configuration_mllama as config_mllama from PIL.Image import Image from torch import nn diff --git a/vllm/model_executor/models/qwen2_audio.py b/vllm/model_executor/models/qwen2_audio.py index f63bd0a11459a..ccb5a3f600b2d 100644 --- a/vllm/model_executor/models/qwen2_audio.py +++ b/vllm/model_executor/models/qwen2_audio.py @@ -160,7 +160,7 @@ class Qwen2AudioMultiModalProcessor( mm_kwargs: Mapping[str, Any], ) -> BatchFeature: # Text-only input not supported in composite processor - if not mm_data or not mm_data.get("audios", []): + if not mm_data.get("audios", []): prompt_ids = self.info.get_tokenizer().encode(prompt) prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index cb1e143838496..6e73a2ae656c2 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -8,7 +8,6 @@ from functools import cached_property from typing import Any, Literal, Optional, Set, Tuple, TypedDict, Union import torch -import torch.utils.checkpoint from torch import nn from torch.nn import functional as F from transformers import BatchFeature, ProcessorMixin @@ -160,7 +159,7 @@ class UltravoxMultiModalProcessor( mm_kwargs: Mapping[str, object], ) -> BatchFeature: # Text-only input not supported in composite processor - if not mm_data or not mm_data.get("audios", []): + if not mm_data.get("audios", []): prompt_ids = self.info.get_tokenizer().encode( prompt, add_special_tokens=False) prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) From 7329ff5468eceaf17f4b193ae3ef0b43c7bf38d6 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 28 Mar 2025 11:46:45 -0400 Subject: [PATCH 077/593] [V1] Support disable_any_whtespace for guidance backend (#15584) Signed-off-by: Russell Bryant --- tests/entrypoints/llm/test_guided_generate.py | 62 +++---------------- .../llm/test_struct_output_generate.py | 54 +++------------- vllm/engine/arg_utils.py | 3 +- .../guided_decoding/guidance_decoding.py | 12 +++- vllm/v1/engine/processor.py | 11 ++-- vllm/v1/structured_output/backend_guidance.py | 19 +++--- 6 files changed, 44 insertions(+), 117 deletions(-) diff --git a/tests/entrypoints/llm/test_guided_generate.py b/tests/entrypoints/llm/test_guided_generate.py index 5f1a91cb2b19f..3f275e0b2ec74 100644 --- a/tests/entrypoints/llm/test_guided_generate.py +++ b/tests/entrypoints/llm/test_guided_generate.py @@ -6,7 +6,6 @@ import weakref import jsonschema import pytest -from pydantic import BaseModel from vllm.distributed import cleanup_dist_env_and_memory from vllm.entrypoints.llm import LLM @@ -15,7 +14,10 @@ from vllm.sampling_params import GuidedDecodingParams, SamplingParams MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct" GUIDED_DECODING_BACKENDS = [ - "outlines", "lm-format-enforcer", "xgrammar", "guidance" + "outlines", + "lm-format-enforcer", + "xgrammar:disable-any-whitespace", + "guidance:disable-any-whitespace", ] @@ -322,59 +324,9 @@ def test_guided_json_object(llm, guided_decoding_backend: str): print(generated_text) assert generated_text is not None + if 'disable-any-whitespace' in guided_decoding_backend: + assert "\n" not in generated_text + # Parse to verify it is valid JSON parsed_json = json.loads(generated_text) assert isinstance(parsed_json, dict) - - -@pytest.mark.skip_global_cleanup -def test_json_with_any_whitespace_disabled(llm): - - class ResponseSchema(BaseModel): - clarifying_question: str - cost_per_serving: str - calories: str - type_dish_ids: str - type_meal_ids: str - product_ids: list[str] - exclude_product_ids: list[str] - allergen_ids: list[str] - total_cooking_time: str - kitchen_ids: str - holiday_ids: str - - # Note: Without this setting, the response is sometimes full of `\n` - # for some models. This option prevents that. - guided_decoding_backend = 'xgrammar:disable-any-whitespace' - - schema = ResponseSchema.model_json_schema() - guided_params = GuidedDecodingParams(json=schema, - backend=\ - guided_decoding_backend) - sampling_params = SamplingParams(max_tokens=2000, - frequency_penalty=0, - presence_penalty=-1.1, - repetition_penalty=1.3, - guided_decoding=guided_params) - - prompt = ("<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You" - "are a helpful assistant.<|im_end|>\n<|im_start|>user\nI want a " - "quick launch fast with $10.<|im_end|>\n<|im_start|>assistant\n") - outputs = llm.generate(prompts=prompt, - sampling_params=sampling_params, - use_tqdm=True) - - assert outputs is not None - - for output in outputs: - assert output is not None - assert isinstance(output, RequestOutput) - - generated_text = output.outputs[0].text - assert generated_text is not None - assert "\n" not in generated_text - - # Parse to verify it is valid JSON - parsed_json = json.loads(generated_text) - assert isinstance(parsed_json, dict) - jsonschema.validate(instance=parsed_json, schema=schema) diff --git a/tests/v1/entrypoints/llm/test_struct_output_generate.py b/tests/v1/entrypoints/llm/test_struct_output_generate.py index 00fa47575b6ae..c9fa03a1ae1fb 100644 --- a/tests/v1/entrypoints/llm/test_struct_output_generate.py +++ b/tests/v1/entrypoints/llm/test_struct_output_generate.py @@ -15,7 +15,9 @@ from vllm.entrypoints.llm import LLM from vllm.outputs import RequestOutput from vllm.sampling_params import GuidedDecodingParams, SamplingParams -GUIDED_DECODING_BACKENDS_V1 = ["xgrammar", "guidance"] +GUIDED_DECODING_BACKENDS_V1 = [ + "xgrammar:disable-any-whitespace", "guidance:disable-any-whitespace" +] MODELS_TO_TEST = [ "Qwen/Qwen2.5-1.5B-Instruct", "mistralai/Ministral-8B-Instruct-2410" ] @@ -55,50 +57,8 @@ def test_guided_json_completion( generated_text = output.outputs[0].text assert generated_text is not None - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") - output_json = json.loads(generated_text) - jsonschema.validate(instance=output_json, schema=sample_json_schema) - - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_json_completion_disable_any_whitespace( - monkeypatch: pytest.MonkeyPatch, - sample_json_schema: dict[str, Any], - guided_decoding_backend: str, - model_name: str, -): - if guided_decoding_backend != "xgrammar": - pytest.skip("disable-any-whitespace is only supported for xgrammar.") - guided_decoding_backend = 'xgrammar:disable-any-whitespace' - - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) - sampling_params = SamplingParams( - temperature=1.0, - max_tokens=1000, - guided_decoding=GuidedDecodingParams(json=sample_json_schema)) - outputs = llm.generate(prompts=[ - f"Give an example JSON for an employee profile " - f"that fits this schema: {sample_json_schema}" - ] * 2, - sampling_params=sampling_params, - use_tqdm=True) - - assert outputs is not None - - for output in outputs: - assert output is not None - assert isinstance(output, RequestOutput) - prompt = output.prompt - - generated_text = output.outputs[0].text - assert generated_text is not None - assert "\n" not in generated_text + if 'disable-any-whitespace' in guided_decoding_backend: + assert "\n" not in generated_text print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") output_json = json.loads(generated_text) jsonschema.validate(instance=output_json, schema=sample_json_schema) @@ -142,7 +102,7 @@ def test_guided_json_object( # Parse to verify it is valid JSON parsed_json = json.loads(generated_text) allowed_types: tuple[type, ...] = (dict, ) - if guided_decoding_backend == "xgrammar": + if guided_decoding_backend.startswith("xgrammar"): # TODO - we are currently too permissive with xgrammar and # allow # any valid json (typically comes back as a list or # object). We can fix this by specifying a jsonschema of @@ -170,7 +130,7 @@ def test_guided_json_unsupported_schema( temperature=1.0, max_tokens=1000, guided_decoding=GuidedDecodingParams(json=unsupported_json_schema)) - if guided_decoding_backend == "xgrammar": + if guided_decoding_backend.startswith("xgrammar"): with pytest.raises(ValueError, match="The provided JSON schema contains features " "not supported by xgrammar."): diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index a416fa8aa08e3..6f498af36a403 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1561,7 +1561,8 @@ class EngineArgs: # Xgrammar and Guidance are supported. SUPPORTED_GUIDED_DECODING = [ - "xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto" + "xgrammar", "xgrammar:disable-any-whitespace", "guidance", + "guidance:disable-any-whitespace", "auto" ] if self.guided_decoding_backend not in SUPPORTED_GUIDED_DECODING: _raise_or_fallback(feature_name="--guided-decoding-backend", diff --git a/vllm/model_executor/guided_decoding/guidance_decoding.py b/vllm/model_executor/guided_decoding/guidance_decoding.py index d8675a14030de..f19ebcbe420e3 100644 --- a/vllm/model_executor/guided_decoding/guidance_decoding.py +++ b/vllm/model_executor/guided_decoding/guidance_decoding.py @@ -18,14 +18,22 @@ def get_local_guidance_guided_decoding_logits_processor( """ grm = "" + any_whitespace = 'disable-any-whitespace' not in \ + guided_params.backend_options() if guided_params.json: grm = llguidance.LLMatcher.grammar_from_json_schema( guided_params.json, - overrides={"whitespace_pattern": guided_params.whitespace_pattern}) + overrides={"whitespace_pattern": guided_params.whitespace_pattern}, + defaults={ + "whitespace_flexible": any_whitespace, + }) elif guided_params.json_object: grm = llguidance.LLMatcher.grammar_from_json_schema( '{"type": "object"}', - overrides={"whitespace_pattern": guided_params.whitespace_pattern}) + overrides={"whitespace_pattern": guided_params.whitespace_pattern}, + defaults={ + "whitespace_flexible": any_whitespace, + }) elif guided_params.regex: grm = llguidance.grammar_from("regex", guided_params.regex) elif guided_params.choice: diff --git a/vllm/v1/engine/processor.py b/vllm/v1/engine/processor.py index 24762d214c345..dbaf0abaea18a 100644 --- a/vllm/v1/engine/processor.py +++ b/vllm/v1/engine/processor.py @@ -121,7 +121,8 @@ class Processor: return supported_backends = [ - "xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto" + "xgrammar", "xgrammar:disable-any-whitespace", "guidance", + "guidance:disable-any-whitespace", "auto" ] engine_level_backend = self.decoding_config.guided_decoding_backend if engine_level_backend not in supported_backends: @@ -140,11 +141,10 @@ class Processor: raise ValueError("Structured output is not supported on TPU.") # Request content validation - - if engine_level_backend == "xgrammar": + if engine_level_backend.startswith("xgrammar"): # xgrammar with no fallback validate_structured_output_request_xgrammar(params) - params.guided_decoding.backend = "xgrammar" + params.guided_decoding.backend = engine_level_backend elif engine_level_backend == "auto": # "auto" is an opt-in to opinionated behavior where we try to # choose a backend based on request contents. This is not the @@ -158,12 +158,13 @@ class Processor: # are not supported in xgrammar. Fall back to guidance. params.guided_decoding.backend = "guidance" - if params.guided_decoding.backend == "guidance": + if engine_level_backend.startswith("guidance"): # TODO ideally we would have the LLTokenizer here as Lark syntax # allows <|special_token|> and similar, see # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens # Without tokenizer these are disallowed in grammars. validate_guidance_grammar(params, tokenizer=None) + params.guided_decoding.backend = engine_level_backend def process_inputs( self, diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py index 1e274ad0ae623..a7ba710169497 100644 --- a/vllm/v1/structured_output/backend_guidance.py +++ b/vllm/v1/structured_output/backend_guidance.py @@ -41,6 +41,9 @@ class GuidanceBackend(StructuredOutputBackend): tokenizer_group.ping() self.vllm_config = vllm_config self.vocab_size = vllm_config.model_config.get_vocab_size() + self.disable_any_whitespace = ( + "disable-any-whitespace" + in vllm_config.decoding_config.guided_decoding_backend) tokenizer = tokenizer_group.get_lora_tokenizer(None) self.ll_tokenizer = llguidance_hf.from_tokenizer(tokenizer, None) @@ -48,7 +51,7 @@ class GuidanceBackend(StructuredOutputBackend): def compile_grammar(self, request_type: StructuredOutputOptions, grammar_spec: str) -> StructuredOutputGrammar: self.serialized_grammar = serialize_guidance_grammar( - request_type, grammar_spec) + request_type, grammar_spec, self.disable_any_whitespace) ll_matcher = llguidance.LLMatcher( self.ll_tokenizer, @@ -126,17 +129,19 @@ class GuidanceGrammar(StructuredOutputGrammar): def serialize_guidance_grammar(request_type: StructuredOutputOptions, - grammar_spec: str) -> str: + grammar_spec: str, + disable_any_whitespace: bool = False) -> str: if request_type == StructuredOutputOptions.JSON: - # TODO: make whitespace_flexible configurable return llguidance.LLMatcher.grammar_from_json_schema( - grammar_spec, defaults={ - "whitespace_flexible": True, + grammar_spec, + defaults={ + "whitespace_flexible": not disable_any_whitespace, }) elif request_type == StructuredOutputOptions.JSON_OBJECT: return llguidance.LLMatcher.grammar_from_json_schema( - '{"type": "object"}', defaults={ - "whitespace_flexible": True, + '{"type": "object"}', + defaults={ + "whitespace_flexible": not disable_any_whitespace, }) else: if request_type == StructuredOutputOptions.REGEX: From 2914006fe09875ebfa33626d945e34173c7441c6 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Fri, 28 Mar 2025 23:56:48 +0800 Subject: [PATCH 078/593] [doc] add missing imports (#15699) Signed-off-by: reidliu41 Co-authored-by: reidliu41 --- docs/source/models/generative_models.md | 6 ++++++ docs/source/models/pooling_models.md | 8 ++++++++ docs/source/performance/optimization.md | 2 ++ docs/source/serving/multimodal_inputs.md | 8 ++++++++ docs/source/serving/offline_inference.md | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/docs/source/models/generative_models.md b/docs/source/models/generative_models.md index c94e940b8534c..63fc53b0e7c55 100644 --- a/docs/source/models/generative_models.md +++ b/docs/source/models/generative_models.md @@ -23,6 +23,8 @@ It is similar to [its counterpart in HF Transformers](https://huggingface.co/doc except that tokenization and detokenization are also performed automatically. ```python +from vllm import LLM + llm = LLM(model="facebook/opt-125m") outputs = llm.generate("Hello, my name is") @@ -36,6 +38,8 @@ You can optionally control the language generation by passing {class}`~vllm.Samp For example, you can use greedy sampling by setting `temperature=0`: ```python +from vllm import LLM, SamplingParams + llm = LLM(model="facebook/opt-125m") params = SamplingParams(temperature=0) outputs = llm.generate("Hello, my name is", params) @@ -83,6 +87,8 @@ Base models may perform poorly as they are not trained to respond to the chat co ::: ```python +from vllm import LLM + llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct") conversation = [ { diff --git a/docs/source/models/pooling_models.md b/docs/source/models/pooling_models.md index f774f3d0fa0ed..dbcd846cc9779 100644 --- a/docs/source/models/pooling_models.md +++ b/docs/source/models/pooling_models.md @@ -68,6 +68,8 @@ The {class}`~vllm.LLM.encode` method is available to all pooling models in vLLM. It returns the extracted hidden states directly, which is useful for reward models. ```python +from vllm import LLM + llm = LLM(model="Qwen/Qwen2.5-Math-RM-72B", task="reward") (output,) = llm.encode("Hello, my name is") @@ -81,6 +83,8 @@ The {class}`~vllm.LLM.embed` method outputs an embedding vector for each prompt. It is primarily designed for embedding models. ```python +from vllm import LLM + llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed") (output,) = llm.embed("Hello, my name is") @@ -96,6 +100,8 @@ The {class}`~vllm.LLM.classify` method outputs a probability vector for each pro It is primarily designed for classification models. ```python +from vllm import LLM + llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", task="classify") (output,) = llm.classify("Hello, my name is") @@ -116,6 +122,8 @@ To handle RAG at a higher level, you should use integration frameworks such as [ ::: ```python +from vllm import LLM + llm = LLM(model="BAAI/bge-reranker-v2-m3", task="score") (output,) = llm.score("What is the capital of France?", "The capital of Brazil is Brasilia.") diff --git a/docs/source/performance/optimization.md b/docs/source/performance/optimization.md index 5b0f8421a51eb..ccbe8a367061f 100644 --- a/docs/source/performance/optimization.md +++ b/docs/source/performance/optimization.md @@ -31,6 +31,8 @@ vLLM supports an experimental feature chunked prefill. Chunked prefill allows to You can enable the feature by specifying `--enable-chunked-prefill` in the command line or setting `enable_chunked_prefill=True` in the LLM constructor. ```python +from vllm import LLM + llm = LLM(model="meta-llama/Llama-2-7b-hf", enable_chunked_prefill=True) # Set max_num_batched_tokens to tune performance. # NOTE: 2048 is the default max_num_batched_tokens for chunked prefill. diff --git a/docs/source/serving/multimodal_inputs.md b/docs/source/serving/multimodal_inputs.md index 2e2016c95e4fc..f45d36c3ccaca 100644 --- a/docs/source/serving/multimodal_inputs.md +++ b/docs/source/serving/multimodal_inputs.md @@ -21,6 +21,8 @@ To input multi-modal data, follow this schema in {class}`vllm.inputs.PromptType` You can pass a single image to the `'image'` field of the multi-modal dictionary, as shown in the following examples: ```python +from vllm import LLM + llm = LLM(model="llava-hf/llava-1.5-7b-hf") # Refer to the HuggingFace repo for the correct format to use @@ -65,6 +67,8 @@ Full example: To substitute multiple images inside the same text prompt, you can pass in a list of images instead: ```python +from vllm import LLM + llm = LLM( model="microsoft/Phi-3.5-vision-instruct", trust_remote_code=True, # Required to load Phi-3.5-vision @@ -96,6 +100,8 @@ Full example: Date: Fri, 28 Mar 2025 23:58:44 +0800 Subject: [PATCH 079/593] [Bugfix] Fix regex compile display format (#15368) Signed-off-by: Kebe --- vllm/transformers_utils/tokenizers/mistral.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/vllm/transformers_utils/tokenizers/mistral.py b/vllm/transformers_utils/tokenizers/mistral.py index 2d036e2c83f74..d893431f4871b 100644 --- a/vllm/transformers_utils/tokenizers/mistral.py +++ b/vllm/transformers_utils/tokenizers/mistral.py @@ -124,13 +124,15 @@ def find_tokenizer_file(files: List[str]): matched_files = [file for file in files if file_pattern.match(file)] if len(matched_files) > 1: - raise OSError(f"Found {len(matched_files)} files matching the " - f"pattern: {file_pattern}. Make sure only one Mistral " - f"tokenizer is present in {files}.") + raise OSError( + f"Found {len(matched_files)} files matching the " + f"pattern: `{file_pattern.pattern}`. Make sure only one Mistral " + f"tokenizer is present in {files}.") elif len(matched_files) == 0: - raise OSError(f"Found {len(matched_files)} files matching the " - f"pattern: {file_pattern}. Make sure that a Mistral " - f"tokenizer is present in {files}.") + raise OSError( + f"Found {len(matched_files)} files matching the " + f"pattern: `{file_pattern.pattern}`. Make sure that a Mistral " + f"tokenizer is present in {files}.") return matched_files[0] From 47e9038d2386d31b8493ac995094bdc1aec710ce Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Fri, 28 Mar 2025 10:29:32 -0600 Subject: [PATCH 080/593] Fix cpu offload testing for gptq/awq/ct (#15648) Signed-off-by: mgoin --- tests/quantization/test_cpu_offload.py | 12 +++++++--- tests/utils.py | 33 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/quantization/test_cpu_offload.py b/tests/quantization/test_cpu_offload.py index 79afcc916f2bb..a7d6518514c72 100644 --- a/tests/quantization/test_cpu_offload.py +++ b/tests/quantization/test_cpu_offload.py @@ -33,7 +33,9 @@ def test_cpu_offload_fp8(): @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"), reason="gptq_marlin is not supported on this GPU type.") -def test_cpu_offload_gptq(): +def test_cpu_offload_gptq(monkeypatch): + # This quant method is sensitive to dummy weights, so we force real weights + monkeypatch.setenv('VLLM_TEST_FORCE_LOAD_FORMAT', 'auto') # Test GPTQ Marlin compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [], ["--cpu-offload-gb", "1"], @@ -47,7 +49,9 @@ def test_cpu_offload_gptq(): @pytest.mark.skipif(not is_quant_method_supported("awq_marlin"), reason="awq_marlin is not supported on this GPU type.") -def test_cpu_offload_awq(): +def test_cpu_offload_awq(monkeypatch): + # This quant method is sensitive to dummy weights, so we force real weights + monkeypatch.setenv('VLLM_TEST_FORCE_LOAD_FORMAT', 'auto') # Test AWQ Marlin compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [], ["--cpu-offload-gb", "1"], @@ -61,7 +65,9 @@ def test_cpu_offload_awq(): @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"), reason="gptq_marlin is not supported on this GPU type.") -def test_cpu_offload_compressed_tensors(): +def test_cpu_offload_compressed_tensors(monkeypatch): + # This quant method is sensitive to dummy weights, so we force real weights + monkeypatch.setenv('VLLM_TEST_FORCE_LOAD_FORMAT', 'auto') # Test wNa16 compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [], ["--cpu-offload-gb", "1"], diff --git a/tests/utils.py b/tests/utils.py index a827b6d4b9bfe..8915453ebd0a3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -317,6 +317,37 @@ def _test_completion_close( return results +def _test_chat( + client: openai.OpenAI, + model: str, + prompt: str, +): + results = [] + + messages = [{ + "role": "user", + "content": [{ + "type": "text", + "text": prompt + }] + }] + + # test with text prompt + chat_response = client.chat.completions.create(model=model, + messages=messages, + max_tokens=5, + temperature=0.0) + + results.append({ + "test": "completion_close", + "text": chat_response.choices[0].message.content, + "finish_reason": chat_response.choices[0].finish_reason, + "usage": chat_response.usage, + }) + + return results + + def _test_embeddings( client: openai.OpenAI, model: str, @@ -512,6 +543,8 @@ def compare_all_settings(model: str, results += _test_completion(client, model, prompt, token_ids) elif method == "generate_close": results += _test_completion_close(client, model, prompt) + elif method == "generate_chat": + results += _test_chat(client, model, prompt) elif method == "generate_with_image": results += _test_image_text( client, model, From 70e132244a61425a6b88c0b8345e496dc5bdfecd Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Fri, 28 Mar 2025 09:30:08 -0700 Subject: [PATCH 081/593] [Minor] Remove TGI launching script (#15646) Signed-off-by: Woosuk Kwon --- benchmarks/benchmark_serving.py | 3 --- .../benchmark_serving_structured_output.py | 3 --- benchmarks/launch_tgi_server.sh | 16 ---------------- 3 files changed, 22 deletions(-) delete mode 100755 benchmarks/launch_tgi_server.sh diff --git a/benchmarks/benchmark_serving.py b/benchmarks/benchmark_serving.py index 82c6b426b9a2b..e2f712dfc6f49 100644 --- a/benchmarks/benchmark_serving.py +++ b/benchmarks/benchmark_serving.py @@ -7,9 +7,6 @@ On the server side, run one of the following commands: --swap-space 16 \ --disable-log-requests - (TGI backend) - ./launch_tgi_server.sh - On the client side, run: python benchmarks/benchmark_serving.py \ --backend \ diff --git a/benchmarks/benchmark_serving_structured_output.py b/benchmarks/benchmark_serving_structured_output.py index c79a93faff197..71cb420a52c46 100644 --- a/benchmarks/benchmark_serving_structured_output.py +++ b/benchmarks/benchmark_serving_structured_output.py @@ -5,9 +5,6 @@ On the server side, run one of the following commands: (vLLM OpenAI API server) vllm serve --disable-log-requests - (TGI backend) - ./launch_tgi_server.sh - On the client side, run: python benchmarks/benchmark_serving_structured_output.py \ --backend \ diff --git a/benchmarks/launch_tgi_server.sh b/benchmarks/launch_tgi_server.sh deleted file mode 100755 index ba7383d88dc49..0000000000000 --- a/benchmarks/launch_tgi_server.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -PORT=8000 -MODEL=$1 -TOKENS=$2 - -docker run -e "HF_TOKEN=$HF_TOKEN" --gpus all --shm-size 1g -p $PORT:80 \ - -v "$PWD/data:/data" \ - ghcr.io/huggingface/text-generation-inference:2.2.0 \ - --model-id "$MODEL" \ - --sharded false \ - --max-input-length 1024 \ - --max-total-tokens 2048 \ - --max-best-of 5 \ - --max-concurrent-requests 5000 \ - --max-batch-total-tokens "$TOKENS" From c6bc0034d0b8d3960d0a44565812ca253fc95943 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 29 Mar 2025 00:41:16 +0800 Subject: [PATCH 082/593] [Misc] Remove unused utils and clean up imports (#15708) Signed-off-by: DarkLight1337 --- tests/multimodal/test_utils.py | 69 +---------------- vllm/multimodal/utils.py | 119 ------------------------------ vllm/v1/core/sched/output.py | 3 +- vllm/v1/worker/gpu_input_batch.py | 9 +-- 4 files changed, 5 insertions(+), 195 deletions(-) diff --git a/tests/multimodal/test_utils.py b/tests/multimodal/test_utils.py index 8f76d895fdd29..a3f136c5667d5 100644 --- a/tests/multimodal/test_utils.py +++ b/tests/multimodal/test_utils.py @@ -9,12 +9,10 @@ from typing import TYPE_CHECKING, NamedTuple, Optional import numpy as np import pytest from PIL import Image, ImageChops -from transformers import AutoConfig, AutoTokenizer from vllm.multimodal.inputs import PlaceholderRange from vllm.multimodal.utils import (MediaConnector, - merge_and_sort_multimodal_metadata, - repeat_and_pad_placeholder_tokens) + merge_and_sort_multimodal_metadata) if TYPE_CHECKING: from vllm.multimodal.hasher import MultiModalHashDict @@ -136,71 +134,6 @@ async def test_fetch_image_local_files(image_url: str): f"file://{temp_dir}/../{os.path.basename(image_url)}") -@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-mistral-7b-hf"]) -def test_repeat_and_pad_placeholder_tokens(model): - config = AutoConfig.from_pretrained(model) - image_token_id = config.image_token_index - - tokenizer = AutoTokenizer.from_pretrained(model) - - test_cases = [ - ( - "", - 2, - "", - [32000, 32000], - [{ "offset": 0, "length": 2 }], - ), - ( - "", - 2, - "", - [32000, 32000, 32000], - [{ "offset": 0, "length": 2 }], - ), - ( - "", - [3, 2], - "", - [32000, 32000, 32000, 32000, 32000], - [{ "offset": 0, "length": 3 }, { "offset": 3, "length": 2 }], - ), - ( - "Image:Image:!", - [3, 2], - "Image:Image:!", - [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918], - [{ "offset": 2, "length": 3 }, { "offset": 7, "length": 2 }], - ), - ( - "", - [3, 2], - "", - [32000, 32000, 32000], - [{ "offset": 0, "length": 3 }], - ), - ] # yapf: disable - - for ( - prompt, - repeat_count, - expected_prompt, - expected_token_ids, - expected_ranges, - ) in test_cases: - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens( - tokenizer=tokenizer, - prompt=prompt, - prompt_token_ids=tokenizer.encode(prompt, - add_special_tokens=False), - placeholder_token_id=image_token_id, - repeat_count=repeat_count, - ) - assert new_prompt == expected_prompt - assert new_token_ids == expected_token_ids - assert ranges == expected_ranges - - # Used for the next two tests related to `merge_and_sort_multimodal_metadata`. class TestCase(NamedTuple): mm_positions: "MultiModalPlaceholderDict" diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index ad381e1d1d00d..8e4fb7eac49c0 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -12,8 +12,6 @@ from PIL import Image import vllm.envs as envs from vllm.connections import HTTPConnection, global_http_connection -from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer from .audio import AudioMediaIO from .base import MediaIO @@ -21,8 +19,6 @@ from .image import ImageEmbeddingMediaIO, ImageMediaIO from .inputs import PlaceholderRange from .video import VideoMediaIO -logger = init_logger(__name__) - _M = TypeVar("_M") if TYPE_CHECKING: @@ -296,121 +292,6 @@ def encode_video_base64(frames: npt.NDArray) -> str: return video_io.encode_base64(frames) -# Utilities for input processors -_T = TypeVar("_T", str, int) - - -def repeat_and_pad_token( - token: _T, - *, - repeat_count: int = 1, - pad_token_left: Optional[_T] = None, - pad_token_right: Optional[_T] = None, -) -> list[_T]: - replacement = [token] * repeat_count - if pad_token_left is not None: - replacement = [pad_token_left] + replacement - if pad_token_right is not None: - replacement = replacement + [pad_token_right] - - return replacement - - -def repeat_and_pad_placeholder_tokens( - tokenizer: AnyTokenizer, - prompt: Optional[str], - prompt_token_ids: list[int], - *, - placeholder_token_id: int, - repeat_count: Union[int, list[int]], - pad_token_left: Optional[int] = None, - pad_token_right: Optional[int] = None, -) -> tuple[Optional[str], list[int], list[PlaceholderRange]]: - if isinstance(repeat_count, int): - repeat_count = [repeat_count] - - if prompt is None: - new_prompt = None - else: - placeholder_token_str = tokenizer.decode(placeholder_token_id) - pad_token_str_left = (None if pad_token_left is None else - tokenizer.decode(pad_token_left)) - pad_token_str_right = (None if pad_token_right is None else - tokenizer.decode(pad_token_right)) - - placeholder_token_count = prompt.count(placeholder_token_str) - # This is an arbitrary number to distinguish between the two cases - if placeholder_token_count > 16: - logger.warning( - "Please follow the prompt format that is " - "documented on HuggingFace which does not involve " - "repeating %s tokens.", placeholder_token_str) - if placeholder_token_count < len(repeat_count): - logger.warning( - "The number of multi-modal placeholder tokens in the prompt " - "is less than the number of multi-modal inputs. Extra " - "placeholder tokens will be treated as plain text") - repeat_count = repeat_count[:placeholder_token_count] - - prompt_parts = prompt.split(placeholder_token_str, - maxsplit=len(repeat_count)) - new_prompt = "" - for i, repeat_count_item in enumerate(repeat_count): - replacement_str = "".join( - repeat_and_pad_token( - placeholder_token_str, - repeat_count=repeat_count_item, - pad_token_left=pad_token_str_left, - pad_token_right=pad_token_str_right, - )) - # The image tokens are removed to be consistent with HuggingFace - new_prompt += prompt_parts[i] + replacement_str - new_prompt += prompt_parts[-1] - - new_token_ids = list[int]() - placeholder_ranges = list[PlaceholderRange]() - placeholder_token_idx = 0 - for i, token in enumerate(prompt_token_ids): - if token == placeholder_token_id: - curr_repeat_count = repeat_count[placeholder_token_idx] - replacement_ids = repeat_and_pad_token( - placeholder_token_id, - repeat_count=curr_repeat_count, - pad_token_left=pad_token_left, - pad_token_right=pad_token_right, - ) - offset = len(new_token_ids) - if pad_token_left is not None: - offset += 1 - placeholder_ranges.append({ - "offset": offset, - "length": curr_repeat_count, - }) - new_token_ids.extend(replacement_ids) - placeholder_token_idx += 1 - - # No need to further scan the list since we replaced all tokens - if placeholder_token_idx >= len(repeat_count): - new_token_ids.extend(prompt_token_ids[i + 1:]) - break - else: - new_token_ids.append(token) - - return new_prompt, new_token_ids, placeholder_ranges - - -def consecutive_placeholder_ranges( - num_items: int, - item_size: int, - initial_offset: int = 0) -> list[PlaceholderRange]: - """Returns a list of consecutive PlaceholderRanges of a fixed size""" - - return [ - PlaceholderRange(offset=initial_offset + i * item_size, - length=item_size) for i in range(num_items) - ] - - def merge_and_sort_multimodal_metadata( mm_positions: "MultiModalPlaceholderDict", mm_hashes: Optional["MultiModalHashDict"], diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index bb883acdb44b6..dc0d2d59fea7f 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -10,8 +10,7 @@ if TYPE_CHECKING: import numpy.typing as npt from vllm.lora.request import LoRARequest - from vllm.multimodal import MultiModalKwargs - from vllm.multimodal.base import PlaceholderRange + from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange from vllm.sampling_params import SamplingParams from vllm.v1.request import Request diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 01a5cb5548bb4..351b358155801 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -2,13 +2,13 @@ # Datastructures defining an input batch from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional, cast +from typing import Optional, cast import numpy as np import torch from vllm.lora.request import LoRARequest -from vllm.multimodal import MultiModalKwargs +from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange from vllm.sampling_params import SamplingParams, SamplingType from vllm.utils import swap_dict_values from vllm.v1.outputs import LogprobsTensors @@ -18,9 +18,6 @@ from vllm.v1.worker.block_table import BlockTable _SAMPLING_EPS = 1e-5 -if TYPE_CHECKING: - from vllm.multimodal.inputs import PlaceholderRange - @dataclass class CachedRequestState: @@ -29,7 +26,7 @@ class CachedRequestState: prompt_token_ids: list[int] prompt: Optional[str] mm_inputs: list[MultiModalKwargs] - mm_positions: list["PlaceholderRange"] + mm_positions: list[PlaceholderRange] sampling_params: SamplingParams generator: Optional[torch.Generator] From d03308be0c8d5c0e367bb67e8d6e158eb373f5e4 Mon Sep 17 00:00:00 2001 From: shangmingc Date: Sat, 29 Mar 2025 01:33:32 +0800 Subject: [PATCH 083/593] [Misc] Remove stale func in KVTransferConfig (#14746) Signed-off-by: Shangming Cai --- vllm/config.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/vllm/config.py b/vllm/config.py index 5c73ff56ebbcf..6a15109c6744d 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -2986,12 +2986,6 @@ class KVTransferConfig(BaseModel): return self.kv_connector is not None and \ self.kv_role in ["kv_producer", "kv_consumer", "kv_both"] - @property - def need_kv_parallel_group(self) -> bool: - # for those database-based connector, vLLM does not need to create - # parallel group, and in that case the kv parallel size will be 1. - return self.kv_connector is not None and self.kv_parallel_size > 1 - @property def is_kv_producer(self) -> bool: return self.kv_connector is not None and \ From 038bededbac67e873d3aa6155c7c05674b98db8c Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Fri, 28 Mar 2025 10:37:52 -0700 Subject: [PATCH 084/593] [TPU] [Perf] Improve Memory Usage Estimation (#15671) Signed-off-by: Robert Shaw Co-authored-by: Robert Shaw --- vllm/v1/worker/tpu_worker.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vllm/v1/worker/tpu_worker.py b/vllm/v1/worker/tpu_worker.py index 4d9a113e39ee4..c8691ee87fe6a 100644 --- a/vllm/v1/worker/tpu_worker.py +++ b/vllm/v1/worker/tpu_worker.py @@ -161,7 +161,13 @@ class TPUWorker: # intermediate activations. m = xm.get_memory_info(self.device) total_memory_size = m["bytes_limit"] - profiled = m["peak_bytes_used"] # Weights + intermediate activations. + current_mem = m["bytes_used"] + # Ideally we would use profiled = m["peak_bytes_used"] to + # get weights + activations. But there is memory used during + # compilation / weight loading that impacts the peak and + # there is no way to reset peak memory in XLA, So we + # use the heuristic of 2% of weights. + profiled = current_mem * 1.02 # Calculate the TPU KV cache size based on profiling. usable_memory_size = int(total_memory_size * From 04437e313dbbf4427734a3ed2d1d650efc57ef66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luka=20Govedi=C4=8D?= Date: Fri, 28 Mar 2025 16:01:09 -0400 Subject: [PATCH 085/593] [Bugfix] [torch.compile] Add Dynamo metrics context during compilation (#15639) Signed-off-by: luka --- tests/compile/test_full_graph.py | 79 +++++++++++++++++--------- vllm/compilation/compiler_interface.py | 38 ++++++++++++- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/tests/compile/test_full_graph.py b/tests/compile/test_full_graph.py index 3a45c35442ca8..5311a4ce21054 100644 --- a/tests/compile/test_full_graph.py +++ b/tests/compile/test_full_graph.py @@ -2,21 +2,20 @@ from __future__ import annotations -from typing import Any +from typing import Any, Union import pytest import torch from tests.quantization.utils import is_quant_method_supported from vllm import LLM, SamplingParams -from vllm.config import CompilationLevel +from vllm.config import CompilationConfig, CompilationLevel from vllm.platforms import current_platform from ..utils import create_new_process_for_each_test -@pytest.fixture(params=None, name="model_info") -def models_list_fixture(request): +def models_list(all: bool): TEST_MODELS: list[tuple[str, dict[str, Any]]] = [ ("facebook/opt-125m", {}), ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", { @@ -33,6 +32,9 @@ def models_list_fixture(request): ("meta-llama/Llama-3.2-1B-Instruct", {}), ] + if not all: + return TEST_MODELS + if is_quant_method_supported("aqlm"): TEST_MODELS.append(("ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf", { "quantization": "aqlm" @@ -77,7 +79,7 @@ def models_list_fixture(request): "optimization_level", [CompilationLevel.DYNAMO_ONCE, CompilationLevel.PIECEWISE], ) -@pytest.mark.parametrize("model_info", "", indirect=True) +@pytest.mark.parametrize("model_info", models_list(all=True)) @create_new_process_for_each_test() def test_full_graph( monkeypatch: pytest.MonkeyPatch, @@ -91,25 +93,50 @@ def test_full_graph( m.setenv("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") print(f"MODEL={model}") - prompts = [ - "Hello, my name is", - "The president of the United States is", - "The capital of France is", - "The future of AI is", - ] - sampling_params = SamplingParams(temperature=0) - llm = LLM( - model=model, - enforce_eager=True, - tensor_parallel_size=1, - disable_custom_all_reduce=True, - compilation_config=optimization_level, - **model_kwargs, - ) - outputs = llm.generate(prompts, sampling_params) + run_model(optimization_level, model, model_kwargs) - # Print the outputs. - for output in outputs: - prompt = output.prompt - generated_text = output.outputs[0].text - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") + +# TODO(luka) add other supported compilation config scenarios here +@pytest.mark.parametrize( + "compilation_config", + # additional compile sizes + [ + CompilationConfig(level=CompilationLevel.PIECEWISE, + compile_sizes=[1, 2]) + ]) +# only test some of the models +@pytest.mark.parametrize("model_info", models_list(all=False)) +@create_new_process_for_each_test() +def test_custom_compile_config( + model_info: tuple[str, dict[str, Any]], + compilation_config: CompilationConfig, +): + model, model_kwargs = model_info + print(f"MODEL={model}") + run_model(compilation_config, model, model_kwargs) + + +def run_model(compile_config: Union[int, CompilationConfig], model: str, + model_kwargs: dict[str, Any]): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + sampling_params = SamplingParams(temperature=0) + llm = LLM( + model=model, + enforce_eager=True, + tensor_parallel_size=1, + disable_custom_all_reduce=True, + compilation_config=compile_config, + **model_kwargs, + ) + outputs = llm.generate(prompts, sampling_params) + + # Print the outputs. + for output in outputs: + prompt = output.prompt + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") diff --git a/vllm/compilation/compiler_interface.py b/vllm/compilation/compiler_interface.py index d6e44fa6d3414..5a22cf70aadab 100644 --- a/vllm/compilation/compiler_interface.py +++ b/vllm/compilation/compiler_interface.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 +import contextlib import copy import hashlib +import importlib.metadata import os from contextlib import ExitStack from typing import Any, Callable, Dict, List, Optional, Tuple @@ -9,6 +11,7 @@ from unittest.mock import patch import torch import torch._inductor.compile_fx import torch.fx as fx +from packaging.version import Version from vllm.config import VllmConfig @@ -285,6 +288,9 @@ class InductorAdaptor(CompilerInterface): "torch._inductor.codecache.FxGraphCache._check_can_cache", _check_can_cache)) + # Dynamo metrics context, see method for more details. + stack.enter_context(self.metrics_context()) + compiled_graph = compile_fx( graph, example_inputs, @@ -309,8 +315,14 @@ class InductorAdaptor(CompilerInterface): hash_str = handle[0] from torch._inductor.codecache import FxGraphCache - with patch("torch._inductor.codecache.FxGraphCache._get_shape_env", - lambda *args, **kwargs: AlwaysHitShapeEnv()): + with ExitStack() as exit_stack: + exit_stack.enter_context( + patch("torch._inductor.codecache.FxGraphCache._get_shape_env", + lambda *args, **kwargs: AlwaysHitShapeEnv())) + + # Dynamo metrics context, see method for more details. + exit_stack.enter_context(self.metrics_context()) + if torch.__version__.startswith("2.5"): inductor_compiled_graph = FxGraphCache._lookup_graph( hash_str, example_inputs, True, False) @@ -351,6 +363,28 @@ class InductorAdaptor(CompilerInterface): return compiled_graph + def metrics_context(self) -> contextlib.AbstractContextManager: + """ + This method returns the Dynamo metrics context (if it exists, + otherwise a null context). It is used by various compile components. + Present in torch>=2.6, it's used inside FxGraphCache in + torch==2.6 (but not after). It might also be used in various other + torch.compile internal functions. + + Because it is re-entrant, we always set it (even if entering via Dynamo + and the context was already entered). We might want to revisit if it + should be set at a different level of compilation. + + This is likely a bug in PyTorch: public APIs should not rely on + manually setting up internal contexts. But we also rely on non-public + APIs which might not provide these guarantees. + """ + if Version(importlib.metadata.version('torch')) >= Version("2.6"): + import torch._dynamo.utils + return torch._dynamo.utils.get_metrics_context() + else: + return contextlib.nullcontext() + class EagerAdaptor(CompilerInterface): name = "eager" From c3f687ac227f93cfb8a5558da871d9dfa68095ab Mon Sep 17 00:00:00 2001 From: Alexander Matveev <59768536+alexm-redhat@users.noreply.github.com> Date: Fri, 28 Mar 2025 16:19:04 -0400 Subject: [PATCH 086/593] [V1] TPU - Fix the chunked prompt bug (#15713) Signed-off-by: Alexander Matveev --- tests/v1/tpu/test_basic.py | 5 ++++- vllm/v1/worker/tpu_model_runner.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/v1/tpu/test_basic.py b/tests/v1/tpu/test_basic.py index 591aa9c5878ae..0d7e8d8d7f5e9 100644 --- a/tests/v1/tpu/test_basic.py +++ b/tests/v1/tpu/test_basic.py @@ -48,7 +48,10 @@ def test_models( with vllm_runner( model, - max_model_len=8192, + # Note: max_num_batched_tokens == 1024 is needed here to + # actually test chunked prompt + max_num_batched_tokens=1024, + max_model_len=8196, gpu_memory_utilization=0.7, max_num_seqs=16, tensor_parallel_size=tensor_parallel_size) as vllm_model: diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index 5401fff2bf19b..695e31f715b4d 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -618,6 +618,7 @@ class TPUModelRunner: # Update the cache state concurrently. Code above will not block until # we use `selected_token_ids`. Add mark_step if post-processing changes request_seq_lens: list[tuple[int, CachedRequestState, int]] = [] + discard_sampled_tokens_req_indices = [] for i, req_id in zip(range(num_reqs), self.input_batch.req_ids): assert req_id is not None req_state = self.requests[req_id] @@ -633,6 +634,10 @@ class TPUModelRunner: # This relies on cuda-specific torch-internal impl details generator.set_offset(generator.get_offset() - 4) + # Record the index of the request that should not be sampled, + # so that we could clear the sampled tokens before returning. + discard_sampled_tokens_req_indices.append(i) + assert all( req_id is not None for req_id in self.input_batch.req_ids[:num_reqs]), "req_ids contains None" @@ -646,11 +651,19 @@ class TPUModelRunner: if max_gen_len == 1: valid_sampled_token_ids = selected_token_ids.tolist() + # Mask out the sampled tokens that should not be sampled. + # TODO: Keep in sync with gpu_model_runner.py, in particular + # the "else" case here + for i in discard_sampled_tokens_req_indices: + valid_sampled_token_ids[i].clear() + + # Append sampled tokens for i, req_state, seq_len in request_seq_lens: token_id = valid_sampled_token_ids[i][0] self.input_batch.token_ids_cpu[i, seq_len] = token_id req_state.output_token_ids.append(token_id) self.input_batch.num_tokens[i] += 1 + else: valid_mask = selected_token_ids != INVALID_TOKEN_ID gen_lens = valid_mask.sum(dim=1).tolist() From 26df46ee59e05882f7f46268f731a8e4b3ae0454 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Sat, 29 Mar 2025 06:23:00 +0800 Subject: [PATCH 087/593] [Misc] cli auto show default value (#15582) Signed-off-by: reidliu41 --- vllm/benchmarks/serve.py | 4 +--- vllm/engine/arg_utils.py | 25 ++++++++----------------- vllm/entrypoints/openai/cli_args.py | 2 +- vllm/utils.py | 2 +- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index cddfd672e7ab0..813556f90f534 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -726,15 +726,13 @@ def add_cli_args(parser: argparse.ArgumentParser): default="ttft,tpot,itl", help="Comma-seperated list of selected metrics to report percentils. " "This argument specifies the metrics to report percentiles. " - "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". " - "Default value is \"ttft,tpot,itl\".") + "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". ") parser.add_argument( "--metric-percentiles", type=str, default="99", help="Comma-seperated list of percentiles for selected metrics. " "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". " - "Default value is \"99\". " "Use \"--percentile-metrics\" to select metrics.", ) parser.add_argument( diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 6f498af36a403..ca511c7434f83 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -322,9 +322,7 @@ class EngineArgs: parser.add_argument('--download-dir', type=nullable_str, default=EngineArgs.download_dir, - help='Directory to download and load the weights, ' - 'default to the default cache dir of ' - 'huggingface.') + help='Directory to download and load the weights.') parser.add_argument( '--load-format', type=str, @@ -399,8 +397,7 @@ class EngineArgs: 'Valid backend values are "xgrammar", "guidance", and "auto". ' 'With "auto", we will make opinionated choices based on request' 'contents and what the backend libraries currently support, so ' - 'the behavior is subject to change in each release. ' - 'The default is xgrammar.') + 'the behavior is subject to change in each release.') parser.add_argument( '--logits-processor-pattern', type=nullable_str, @@ -493,8 +490,7 @@ class EngineArgs: default=EngineArgs.prefix_caching_hash_algo, help="Set the hash algorithm for prefix caching. " "Options are 'builtin' (Python's built-in hash) or 'sha256' " - "(collision resistant but with certain overheads). Defaults " - "to 'builtin'.", + "(collision resistant but with certain overheads).", ) parser.add_argument('--disable-sliding-window', action='store_true', @@ -568,9 +564,7 @@ class EngineArgs: type=int, default=EngineArgs.max_num_partial_prefills, help="For chunked prefill, the max number of concurrent \ - partial prefills." - "Defaults to 1", - ) + partial prefills.") parser.add_argument( "--max-long-partial-prefills", type=int, @@ -579,15 +573,13 @@ class EngineArgs: "than --long-prefill-token-threshold that will be prefilled " "concurrently. Setting this less than --max-num-partial-prefills " "will allow shorter prompts to jump the queue in front of longer " - "prompts in some cases, improving latency. Defaults to 1.") + "prompts in some cases, improving latency.") parser.add_argument( "--long-prefill-token-threshold", type=float, default=EngineArgs.long_prefill_token_threshold, help="For chunked prefill, a request is considered long if the " - "prompt is longer than this number of tokens. Defaults to 4%% of " - "the model's context length.", - ) + "prompt is longer than this number of tokens.") parser.add_argument('--max-num-seqs', type=int, default=EngineArgs.max_num_seqs, @@ -739,8 +731,7 @@ class EngineArgs: type=int, default=EngineArgs.max_cpu_loras, help=('Maximum number of LoRAs to store in CPU memory. ' - 'Must be >= than max_loras. ' - 'Defaults to max_loras.')) + 'Must be >= than max_loras.')) parser.add_argument( '--fully-sharded-loras', action='store_true', @@ -894,7 +885,7 @@ class EngineArgs: help='Set the lower bound threshold for the posterior ' 'probability of a token to be accepted. This threshold is ' 'used by the TypicalAcceptanceSampler to make sampling decisions ' - 'during speculative decoding. Defaults to 0.09') + 'during speculative decoding.') parser.add_argument( '--typical-acceptance-sampler-posterior-alpha', diff --git a/vllm/entrypoints/openai/cli_args.py b/vllm/entrypoints/openai/cli_args.py index e956920c2f9a7..218a8fbe10b76 100644 --- a/vllm/entrypoints/openai/cli_args.py +++ b/vllm/entrypoints/openai/cli_args.py @@ -247,7 +247,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: default=None, help='Max number of prompt characters or prompt ' 'ID numbers being printed in log.' - '\n\nDefault: Unlimited') + ' The default of None means unlimited.') parser.add_argument( "--disable-fastapi-docs", diff --git a/vllm/utils.py b/vllm/utils.py index afe68a2b8cb3d..bf83b38ace80d 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1212,7 +1212,7 @@ class StoreBoolean(argparse.Action): "Expected 'true' or 'false'.") -class SortedHelpFormatter(argparse.HelpFormatter): +class SortedHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): """SortedHelpFormatter that sorts arguments by their option strings.""" def add_arguments(self, actions): From f3f8d8fff4c5354d5214f0f6f29e4dc5c4e3a8ca Mon Sep 17 00:00:00 2001 From: daniel-salib Date: Fri, 28 Mar 2025 17:12:02 -0700 Subject: [PATCH 088/593] implement prometheus fast-api-instrumentor for http service metrics (#15657) --- vllm/entrypoints/openai/api_server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 7dbe31e62da67..18d75a04ab0f3 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -311,6 +311,7 @@ def mount_metrics(app: FastAPI): # See https://prometheus.github.io/client_python/multiprocess/ from prometheus_client import (CollectorRegistry, make_asgi_app, multiprocess) + from prometheus_fastapi_instrumentator import Instrumentator prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None) if prometheus_multiproc_dir_path is not None: @@ -318,6 +319,16 @@ def mount_metrics(app: FastAPI): prometheus_multiproc_dir_path) registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) + Instrumentator( + excluded_handlers=[ + "/metrics", + "/health", + "/load", + "/ping", + "/version", + ], + registry=registry, + ).add().instrument(app).expose(app) # Add prometheus asgi middleware to route /metrics requests metrics_route = Mount("/metrics", make_asgi_app(registry=registry)) From cff8991a50dd35c2cb9d2e6d3446a0051cac144a Mon Sep 17 00:00:00 2001 From: simpx Date: Sat, 29 Mar 2025 11:33:58 +0800 Subject: [PATCH 089/593] [Docs][V1] Optimize diagrams in prefix caching design (#15716) --- .../v1/prefix_caching/example-time-1.png | Bin 34837 -> 47947 bytes .../v1/prefix_caching/example-time-3.png | Bin 37069 -> 51241 bytes .../v1/prefix_caching/example-time-4.png | Bin 41530 -> 60607 bytes .../v1/prefix_caching/example-time-5.png | Bin 39727 -> 55437 bytes .../v1/prefix_caching/example-time-6.png | Bin 25462 -> 54829 bytes .../v1/prefix_caching/example-time-7.png | Bin 33144 -> 55922 bytes 6 files changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/source/assets/design/v1/prefix_caching/example-time-1.png b/docs/source/assets/design/v1/prefix_caching/example-time-1.png index 8849ca0237c39b4c428c4ab74c08b512812846f5..d5a165ff6944b7edd95f215685ca6185692e8876 100644 GIT binary patch literal 47947 zcmb@tXVl}=)i!+3(1sR<-iH|&YG%NaZCR#UR&%!{S&dL!)NHHEwj|07FhJ-7girzu zkO2}(r~yJC455Y^N(h8rLJy&LnD>YzZ+M@zzO|ko-&*%=OULINowCo~*S_}FiOo8- z;dnVH!uXJ%&2%vl?hC_D8Z24Ay$hQelk^XPVO&&=e?0U-uvs2U?> zX6JPAUspTBDc$yiozwWvFieb;Uf|glxCG^<-8WQ2>Hq6DI0a`Hr_zgIl!J0RrwiE> z_(-FBreH|^SN)ErSpV)QyJre?uv@LuGpxY&^_|lgxG!0OG6LVA7<@KZ@F9YKD0L{5 zJv6sFxGXr%ND-87#RztW$!t2eXBq*;yH#keR^1scg6oknRKORd^oDlQr0;d6dm1*V z4(0bu?U@EeMcuPaP>F<6Feu8VQrSI`e@#18NK+#i z>e3NyJAyr!Aad?s)14@uZ-BM>e&zo2?4)rJIm*9E`^v=Vfq^8Y4p<)lU&UZb@Xl!h z>_Ok?XkNzzKO6sE=1@Z`MMfUzikQzK-BB>0I{)gx>VP@@>tX+DpOb@bA1NVZ4J;=) z5GHj+(fl#%Hbu+MdrG~g9fE5LgRWsy>;zucmA!H^nLZL!InSM5_t5iS6 z9jMwctfF2a#74Q68cLkPWKwL;rfNht?pF!D7c>ISFO+(=oFj4un&-jL>5?T6yFIra zsHsZBsO7pNiybI~3RPoCp`Ax5NVly)!O6zLNDB!g)h9D4i5!U z>;)!GAk7E|otEB9mAc)!ZswpO&a|?5Enk@CB0`XaK|D;!)xcI72*uQ=mQ2!IDvcmz zyaUILlF_b{-5gPsP$kmLZl{;`b(GE&;|gAHi0!cB*KK4poFZg9?_$|(ww%e2h$1&G zHByDHUNXx%H42OMQje?(Hq(cg;W+Dpv9b10w9uH(vjREF6}T*1=t$EHB-DYJU~;O^ zpyXDbKy{f2h;$!k3$+@a>R@eEE$c?T45rQ6Oj0);Te)evUPTOsX_SaG>!MbrSI{AF zbTOutiY5?R$}1HK!%Aqq5Tm0)B{RmR9Bmis4NA<^h7CvL8(O~99Je%sN5x{h68LZq z^9G=cHqal?>^$QimFu*AL2oKl8fMc^Y13?>&b9g&3B|O6 zbPY^XK^rGy>HL6&F_ETP(hzD0rIb6R2dzRd$rL~h((?jFa+{IS2GwB%6Q{Y3KCNbo zp`V_r)BF$=XT>qb=R(h-GRh!3WV!`fqRVz!O;`0H){48N+LofcjglFyXP2o?J6Fg; zZr36TUaDoWSsy&ez&k;C9C->+%QKB`2y`L|{wkE_L0BwxG#Wf9QwK$Kb%aUMRILE5 zMOw9zGbvRI(S%0gNxl;XSX%T6%`4h1HB;&hGF(3u*pSCe+I3be4YHjepBdIz%fKqr z5;`^8(U@z2zCmYH88dwp@72TRG^I3ah#aT{5Oi^bf(m$>9@z3kOARw+i5yj^js|nJ zq2s68^2l@jzSgqPyyjGgL#$Y6)l+`GG%c5gMwP(FP^gu2*%qA-I;yXjc%-QlX4;6l z)xrRCVXN>&F8ORpCV)UOQ6^iY@d};EdAW{&l}xY0 zF(b2@CrTaL(pwX%q{AePD!sOmrjSHYks8kT(=psi2lyBzvz;mG(NX}+KZ)l>!fw_B zQjz;~r4^Fhk;G>hiK3IW$5M12@jx}TsEAMm3VPYVCe?zQmpq+pwN1&3Bh}I;d9X8D;}>^CT)&FZ5u60URXhUlQP@QrY8*Oi5@`xfY+i4LQOEX)8mImWn^dbCKI?Q zD+xoS?|T&k8I8-FSi-*FS;r)N@i~a4HIOq&>TuuccH0FB;4X3+;aO$cG5gp!DyF$^G{AKu ztX4ZCT%8I6F92~6<)KMct8ys?>kbVS#`R{+4J^>6UT0e+xW{xHJ4N;M&>ogsb*U67 zDKuq9L{9CuOmm1gYB+_*kc9?AbZT@`{c#tJCALy!uaOx9Rb`+czJ`So7?3o~X_18B zBfUM$bRw-VU@Db?&LvaokBevm6-hS-?#W69vEy+fQEjzu<~o5($uSG;H&Rt8E?>kn zM6Txh5k!j|io<3y-*HAjn`66`*vVLmvPLFZEUMhWCT6uEQNHA(EK;veQpvGr^meLF z44_o8EyoR_08azXD3ugAN1#58B9cZyby{@XR71g_I%q;_jxLQzvnler$(}_~R z?I^4aLb;^ml>sD98iJl3!LC0DioI;vDL^tXCt6q6dTQDok!r40fd#B1s09SKunr|) zLzZoDxY2MMW`naPdWWbg4O%1Mbk&3!O-vIfy(#SHL#|Tu1J9_59gikawi(l`-)ae( z)ad!l1Rd)-HBDIq9U8+3H=4>2JuowPE1t+{3eWHz5#`th-!G-TTvUT=eWqE6eWqm8 zB8q^jMuE&%OR`LvS%qRTD@ZvUP$q%o3u8vS6DEn#Oj)l7YQt%Y`^KmHJ^&rPE@e=hJ0bVRvSuj~x+ogBm1e01L4=)ngZyBUs>`+(s#TdCWg$ku z8(3jf5z?uYPxESGqbkF3yCg=UZ`)3mz%riJsOQ@;qq6m8x80{@wbpjjMq_}eY?^5{ zyLe^XbFye|oRXA|iKt zS~okUVl}8Md4mEEhK)?B)x=~bRK;es-;IF3>s7#zFbK8ic-&QMiSkf!2uoW)Uyy87 z(#7Jj#04D~8IxWm4$Ptej$%a)n)c9v z@?G90T&oSG!>ElQfj8x2wvu4VtnZW4oLI=2Mi`ESJ`7Y#ix*06iLvQe=(mP4n(m_{ z9il=5&?@8y9r z+2`wl#zQ244C85nK!ZxVg;d*AOA{NKQ5s=wM8)zlp98B$cGWn7)PN_Gr3Cj9UJ`)^ z*#q{HBxB8>(T0R@90GiDi1fzoLLXv-daUL~PS%UNxrvGr$iT@0>_=uoyl%?LmF`jr zkpL_OU>W{*_ynAF_y_o;5tMUOov%f{qVRp3G4ojJKlmi>CtL+lOYk=3gAc$Ofw5x+ z1$^)=6_g0N2YSQO>A;($CliJ+YPLfFOO6rzM46Ic{WU1AW(=aPV z?HC}HZX@GQaWp^VYcAa%k`vQ*i5itF`9Y`E0QDKZGV(}*&%2@2aOAQMr^rDlsW2X~ ztX;2SO~jN4sl)_b6p34{N~Vg-5mV^p@os*k6EI`tVw-NGT8Y*|&Ci$Bkv+2fZfg+a z)rjb_CYTt;HHHCF_hL<>+L0QTQH;pJBc|q*Tj3~EY|xpA9M)_sC`~5F8KlM8StLul6Q$RP;)dH!EK0kb9dL~9 zhPCpjkj4u@36tY~rK0*8-{#|vz#=S<6>DY9twnaR%x>N~9A(Iu~6arz=^^DI>xv4tUGmuS`36E~`nI0>YiaU&qYWWB(klv+;d z!5H}TB*L}n@8Pzcc=%G&$Oc^}%L&;|C|P-!E*3F1*XJdC8VBB}#VByUTrb%zARt>%ZoEq zchJZ{-LMvdnw^}JNQ?Nri|ir#K+R`;E8Q7-&bUI1A~)COf;6Bu6Z2FLOkVIiW6y6E zfCCPtD=<|?nv-@$ikgN5xD9Vutn(eCCreSQASJ$&!6%9c<9cE}E_fDe5t!u@VzNL) zhC`%LZ;)b;fGxAgINNr|DVWKRfkJmY7lz2Rr@)Oc4Ai*P^Ah|G2O{K+(ZC+3+xFP$ zMyfLzutf)gQH!Vud{svr-vYg34j))PX*;N2^UQIXbVogCEVf$?um@yE^F?in5H!#< znAW7~L>Z9Jm{lK+IUOM?X%%pNx>z77brh#6VWVldl{O4Ji49jX?Q*$|v%pC;`jFe8 z6g+h6#WH1e_+bY%?No;GqExfpD~(NQQYAVxOT`*!ne8xHnTI-KzKx+!4U+>Nc)uYv z?1e@z!s|XRfJu^45aw&bq*o6*#i3KRtu*K9sh;Mg+e$$m(plFN>u_lZfx2o`s!fU( z5fd$DQs)PN1F}rGD5ho8%R4+DaoJK+^Gv8(@CH(euP2AXZ1aFxWe8467c`y;20mqG zsCdvMlEo=z(^ModVtFi2)VPQYr-@Z1!8|gfNehGJ$m?r)b!o$Y2Y| zRCNp<=BOE>Ao9Gb)l8L}CW``NeL6@SxxkDKXefipV_7p37AkO$81}|+1b&lJ9-e3H zTxC#eOiKCK^5NLcWEI_jT&n7|(Og$c2ht#4PkQpk)s+Lg_wJ;q@KviM{S%F2s zT=zV?%2Y&NEQkJdB$b?W*Qc8erw-N#8@hEam+~ggc$`b+8ly@(F~wswhI7??Pbj0! zPPQ^h_!Uy+gD{I$xh81V7{>JkYtl;DF7{cO57j{~T^$<$k(Fw8S_ixtX;Y*Xw)siP z)dBA|4fGmHayVMC$I5i96^LF-C`47t&5k44Z!3uiXNi_GHXE@qO+nQ`4IZLCSPpDD zOiWPDklhY=DrPa(gk<5GpptMJ&h?u#?J}C`454DF91%s^9iltxEL zV&PN7#1>$g#j!@2a{4{=`QM@+u>U~W|6g{d106b! z+e)amz(l$g9V;YYEs6Bx~UkxG0;9+Y-XAQ=K|KV}=BpPdAl3mXR6~-y+8~zZN(^dFRvm zRPV(tW#}r?D%~#w!n>j3U0fv6L#k0B?7ZQnkP0h>HHn#avHSHi zL(+Xf;8uJdpN4I)N*rtBHVYgRb7&4DzDe*UHO6?vWd)=;%z0#lv!O14u>vpv}dY>Zupr(gjsV=PUY23v&;Hxo;M-KfN^ z;UF)$K~1PM@kH%oNG)X!dOb@Efj;H?0}2GuLNcYp({vGo2#OYX;CEspn~uQteSct6 zVwCFGcD^&1+BvEv7cxVaE0-Et(-{p@9T3(l@nset(1R*6wxv9yQ+ScK1q)(FQMp9d zn&~drtX9HF+ZYaMuRRToustqIagksM??ebZI#|EABWE9Ix(FkP;&l^)R#PWJoqO%=E zFtXEOKd)BP>O=`jQ$!g@St)3Ojk5c0ZEVw}h;L-nGMCJ-S|kgZxCm%}GTj+9)iFJ+ zxh>tVa<(%jjZ#}mH-Z?&pst!%YE8T(`^0EmLiI6LxBFn@2wmzm3cjmXvpHQIQvH6; z)5fmmPlZ~uVf0F>E#zq;L`YSvhkypfC841gI{@bfPJ_YtjDz8sMyj2O(HxV`hP27% zVxit7Cw#;h5j(^Krqox zFo_f+StEz>{s7on9_l%Tj5t)RJXh?eiEhG!Ll{NnlJlJFP@UMIbYCoVCYPLSGnLfUG4vL<)S|r>aVn%17vEROS*KKNyYaa;F@O8=~h-sD8Jl zvYYuSou_f?cL+#=B>S=I%r(&iN^q{4O~stwsF0FSY!b%p1_g`X>;;EPQN$hPnB zl;ZRzGTe?#85s?l?TOF>kw{U_k(Ft()KRrjQQA7L)T`Y>i?%0RolRkW45$S+YXP59 zml_p>7+Mn!@hL=!dK}kxkd9Sr1ELjl!1u!v4-6V#=j)2y#qwN<>`65Vv@FA(U zvWYQmq5pd}2khVv@r4K0Ee=t6ufF3#6;w2$?A(xKJ-&0tGRSf8b{)k)P(FL0jmR{Lm6$A?N-Gs zf{mIHkO)$0`I%C&E}P{t+$>-Pvdt!e@s{tyfnl&f&;`p(jev^B zlO4y5KSA=n#M&uVP2kid>;z)UHPO1} zL_Pc)*AQW`*p(b#!@EN`1=g=;>wr-KdyLrt7w3suBd-leUCGG}CelhQ^7HX5c-tpGW~0Ye0?EmYRLl&g0R@7U1R(uOpiNn!CNMXZ$4%f=03M}~P;wRw zo{!aZ)3AyB6oglGyk9IwjSM}^K(SX)oj5z1g18`DE<3rX89Ak4FQpX+GEK za60hIU@-AGoHjkBRS6K1t$ASyOhg=jT@B$$lc||jtJ3W-1WJs$J*%jN6RlM?ZQDT8 z0$_&7f}HoegH+Mb8{-;qbjfnBW`(6eq*!&>l&y-K!-Jp;S+Ynvb|(P>7 zejBAKIA+4*jyTF%axu`H78*M^JIGfG#gc_IFawt-#RhO6du0WoZCbYD+OVF|!?NV} zyO2;IqhiF@fIrY?COj9hX5hNjCOE7hMi1sQZsyY<&M@@`T1ChV!7c)>!Ea)8en57r zPIEHQX+XNOOf6#SP{8*Q#h%0Isy4S|^EQ%C(-yQez?Y5pS!;%}~ zd8b?e%uSWULki(F4WuMZwF-}?oMx^|xoM`%m3-X}6nW68k3|@m|B5&sbcC2LWF;gh zD81Ym@B4BcxSs{hB@1AfzDJE4H6d&oiV5gyy{?MMf*_r~&FLYEg8V?gGXl&4Y2hSL zM3q(*QpROK^1$E#lX}D$rUNd}Q3+{qXbOU}%@NF3#2TGdn?|h2W86`!hLy@z2@nu6 zOmOmXIt)sEkZabq&u2*?O3 z@q}bpxjZ`Ipt4g3keSR;dVV-HCKMbdsBOyjt0SS26*M(P+kS}n3~uL}OeZRHX%7qh ztd&y)G>@7V2-WtSjL}cDVF!ZpRRtuFxNJ#m@qNX1b-o7SteUKFSzt=FG1TQLo-bth zexqlZPOr!59EJ2-NEsZR3V`MPn(nlxNI(N7heJWHU8POqif*}F)iT6!&I7WJ(S20K zYBu1P0wi56qr*1bFQ&WDa0()HL@y|1$5lK;$DG`7TX-2Z`VnX)PthRN(ko3rq)4IOh6iGch}(TzM%;c()X8$b<)Vhg;~0}l zM~Z9pnPH)8%Ph}lpgNNWx~JoKO~#vOHeRjdYeZSfqvfrMo_w&?;zQJA0(vZ71#A$-xr|-erg(u=BrsgLdLF`g3`zus9GEZ%=QxmhqG0KOO+|Rxf{8{LOT9Rrc89Lu4wbQ? z(-Vm4NPsrz>HR5NHOnJ5)0q~mLUU9F>46H=8CFv`$c0cDbTT1qwnM?~UIf%^P_{&{ zXVwre>iHJ0o0$p|rH7aVHnPT~>cGaM4aw1B}))v9=Hz~g7=4Y&YU=6 zwJl&WSq!9%sScTwzQ})nbD9g`RATWzy}DgcaLOmf;IStAU3=!m+3enh&u|s;v(PAb%ig zG9dH{%9u%1GmRS2tOn)Us6-e5uWCYVq_6@m47!e1hDwQMqJhWCF<7lk)RP!lrfij! zVGi!GMF`PTDoBDQ#)iWUr2&XSrg2lqBxtyj%?6S&Ve|DAU+UM?g3s_|t=w&b9wm?o zV`lqpQ2-%;qUKsoMycsVgG{s)S?Sk=BJlCa-k{~-S*kus@;ESan(bnBu~p0rb0gWW zhlXl_n0d$d{BhmRpk2L_8LJkP#l)s(Sai!TPqAK(9G7DZxSHqyh`cy(5<1dMrO4c% zRtrF6I*)^wJW5@XAB7{<4Shn8eLO~Tc;diy8@9=T4DwLOHbrUT3hjsh$-|rugKjXz zOoP}j$R3C#q-YjFDqAW?d7PJPVsT0Y8WjWBEr0=d!}2lERgjn!)W8G`d=k&W@^l-| z)>Y1sGj7cUNiJrw??ZHn&WMNn8|!{vRy$FRmlg_WFa8m$VfI8$zXxW zo}Vif#zMl*)CQcXf*dYNta-_n6s2zR>>Qt$Nw4biBarZ32}e;|4+}Aw;znXGp7?Q6 zb)rtvQAJSEL3uJNMcEqJ(`7Q(jIl5srEC}-@MVzaQ=2$icEA%+JuH}2n;A=3Y+%L+ z1?C9LP_Ek*t0++qFb1Ert9^~g54tpvXal^nQE7u96PE}6|D@)?X~5lKz#;MS9JrV! zW1!$hmj;P;CIF(?7KLbXYZV`YV%(0YpwN}*D!IU+-g*8rNwx=5-jRI@bvJ)YIk30!LqqGmPc zq0UR)-CpwbfSdOR7;r%s#{1-O34m6CIe47%C6~cJU3Q`-AWw- z)tXT;M)H6G1+PL7R0dbGE=F}V%(2pPRkr$*M2v^R~10=|SH*Y4e3SOUSHtZl1^^mCN>w+J&1y69#s^VL&sAmDh5BUVWbf*0x$#sOhm@6sid@9Hx1fC2!@Z2;PjQis% zY=QIF1AM^*<#Ql2Chf^uds4Omu)yI&WQOGg4S?*UJSqo**i{j7kgtMQeEvV9fS~TbUz)f;%@#GyRt=OuCa|e2x%uO9Ov+=C6=hO9*UcaF2oq54pKigoFHP_u_ z;U4)n7tzb-cW3|Uu4QZf{eW98JZkmr%I9}nc;O4rKKsHu{vSU6({tzj=$80OTWAX} zt-bp(OXts@GiUaHz7E`K<69S>Vpg8~Pr<)`o>{j1!G&|zTys|8z`6han)Bn?7=8D8 zcV_ndL3>br8TZEj=u_NyF18K-#bxe)8z+4%SP8H5!w%=J`Nxx%&VAy#Kkjwv`u}%8 z-*>g(?EDW0xpe+qe$8dekKg>egSJ^Rv+9MJ`p!=*-v8shmR_~RZ7a50wf)Saa!AeF z{0F0*|N5M8rS8r8yvXH>37;| z{dt>J>daa*quGC6f7LqQ<+lCjpAMh(P3JIv=T&=u@wW?SKARnSS#(i$%}W)7Ws z`SRjPKitya95{ag_P6a1+~}9*Tsuqb-0%R5N4@Xpa1xe7GHAjsh{6J^Tj5s zwwu{}L#F)1vwvEnZ7%rBYp<_!=GI{==uMQOq6U;OhIC-1Uy z;bV&$GvSl-!_D_yv~uCgFKuN%M9Eg=avL?^I zJ9quZj`~n~wSZlD;%MDtA2{pk_;_X6k#PA3=VLyyG5yo!@s3y*Ka+mw1#V{H)i-_i z_;C+!Rh4t4nYZ89_T6Rm(_5|5XTF=uoVoAuYrURYbmrF^!o-i?n0M^CXIu>(|7l@n zxWQGI9G5^u%{kw?_3JNxbmCj#j4|_6 z>gtsX?_GG^H>&N{E>B!|I0Z~A9~t33(vmi z%%{8dvf%pDkGj4tGh5HT`0v#(|FKqT)vxwKv^!3b|D!YJuKAPW4#bxI?1bUZXVV>V zPU+^i=3R02^{1cH`t$jvTlP8l!7n$wYX0JtH*FB_v~u4MYn2@quUr^?0}`(9=bV7fdPM!^ftTo~etR!}=abLAwZ=6Up7PuqA`|~dP>2HipuDIn?l#v_Y?dwnUp@As$84}N6fnfu;T#0dDlvtHX^*TdeL zIrynJF1_Z2h3750>E^QB{uxWpOx>{keZRV4!~2#z@#;e;bNRtL zLJNPrQ)lI;XTA346V4CA(*1wk-?;tr6F;2SLyRLY`|Vj(dVyzIF8rp)&2gD-7$!W{d~M+ar|k51V_egpy^~pcYh=X@S17@`h17Z2$*-(+(_uF(Kg?Tn(c9+- z0uREOGbnylIN{{`FcmoqYs~vj}wB+(#RvvNu3Gr4>zWwff^Dpnc`gQP^ zvtE7&JCk02*FRmN-M;&}+rL&i@8TD&&bc#!X@jy zwvX}3!UFPF?Wl8p6W{j!)^C<8@cN5?Jz`fsCmgiyrzeN-Ig1qhA*8hLMkjsw&It#+ zc82!4oytFLY{S2L;8~ySx@`O6_WCQAZ63C-K4#Ht zTm08Pf4*Su0DExLuTI(ivcu1{wB6U;Y1Q1>@?qzFwEHv1MXmD=Ivcv~|B+c|!`YYY zvhp`u|0=-$t69J>R$Vw}&Vun3KUpdK(DdH30RSpJ@+lDL`@i62?wZe^xO6TveUONE z{^7Dd4Dd@v#TT#fL-g?fN4JL^xBS8OSx|cMCfon&fRD*HDwn1nT;uyuF57U{l25{C z_TF{F`{&K(cX|DhU+g-)ZypA}8qRtC=X>w4`X7(oe#V!tu6q8GeP93m2K)W_82>GZ zKZ$tb#+?p4=+Cd6_6fDk#%q4F^A_9ezF;d@d+L}4%bwbA)4~>x)?a|G|LCLhU#|Xm zi%)J%Z+|Ag=BxuwJZ>&?-X888vDv@jX!o!e=d`?;{`Jr2*SO*3_jY@9i@z?p|Epa# zym!yp-`#r1am(*}|C?Kt*D?Bq?AFVc@4eGTcS+7>g>w5nz(Xzti0RRM{ANXM;zdI4Qg4J=kEwYtLQsw_|qv__m{sXKvo;yFbL6 zv&VmW&+R)8o0lB&99sS7o&TJ-*#qZn`R(4>`=4y6jm_S@`~IWu+4adcUitlN2fTLn zH;-LPyl~~MubsB=#6N6V+vtlO%Xi&#UiiZ8(WV>z?Gy6u=^ihCd4*4Gy6mV^?_KwT z=|A^b)>`rCeS6(`QQ_HFuf3RgTQ~zIygR?za`8s_qd)#=d=L~pea@}%Kej1vw)@gO z>_42=|NNzG7tj4F`=j;a=dbR!nFtAuibrk z_x)G@@+9n{%ZdjMx&51MKice?8*W6JFd6?>dp3l|1Y~7zw(u@Iqy1Wmvaxj=J}uQC0MJ@-TlzRK76XPn{?BG z)y~6reFbLr$1nf-eq(&Z2S*)y?%mH`dP3&%56-yj_;moTe&w$Hx(98z)`Cx~ z2mNgIBYQt^_To+N+vxo(>32{1^tse+OYRTPJrdENy+7Ud=oQn&{E|a&Jz~EHpFV1b zCG(#9>EZAH;m6$%fBQH0$Lojl(HjrA>E=Ve`*x#; zEPL$Dz3zXpo!)Va8xP#uDxAI!bI0vF%~^99fcr}iJNbok7VQS#xp?J8>xZY`_1Xu2 zx$@Kp_s;abgC9Qhwk^{4e{%9VdoSLWx^wlqPm-VUOI}*?)>ht@*X*(Wtw*1E0RE>P zE_mePT_57-fO=c5|Lb*scGxRNeDsO*lUu*o_M#(pyz$B-|9Hw;;kC~lcEJJHZL|98 zm$rv~>AW|WS@Oomcdq{Ku<4{v})NW-@aeuzS(Z0{lDC!w)ZD@e|hpb&~UWm(>aT)#zlLcc=8EPzkFB!^ZQR( z{@$XOzk7MnvZwZ6@adMCb>)j!e>v~8d8@bHbnPke_FoS@?Y!vfb>4aXi0|Hg@TIpt z{t(}M$-I*<*?*JbHvWop-dzVY`*cT7Sf^N{ul?r8&9}a2$)*>|PZzHG%Kmb@eYZSg zuRqAA9{kYdS89m$(x(4h`m^6R*FV^veekMp@0I5KdOz%)y&k>u+wXQg-f8whKi$A!f4|jv zFHar;^S170eldUf%bk2*5_&LYkcJm+a`26|pKU;UzCp$j* z#-43!fxV8!ThE+k_~*R*x8J_I$6wZO{Oz{i0_AqpLG(uE7ubg9@Q;6X#|f|eHT~vh zkFUJUfAhs-_t^iY$zO7>6)xTA$VI<^unX_I_Ul77I(tXz{-52p`JIP-{?LQ7=goN( zqp#T%$oU(eH?CTJ+wWG~^W?@`KXEzs#6JJr;Io@t^T6K5f ztUZ235yZ@;+Aam=`GB<9u75b>n5Ry^{FJ}0*#7Vzz4~QtaA|bm9^r@2K0j!k zpSfwDJ$}B&HlH1J_`6GiMoMqA`!!_s>(>`=Wc+TE;~xKAd%yl;7aWp3AbglR<@|m3 zdH<8^f7keyI^v|$j--Bdfb!7KezfhD)#bOJy7qzlZoR{z3$kx`zcSujyaK<@v7S}t z9^beis(icew-@dT-i&+am}aoYCHt)gjSIC;14c)$728*cyPCx?7uy|meJ>+iem zNw3^`a~-J3-}aWDM(i-VRisycb=0OnTNoECTkmXp>8>~QSKV5;(0%9O=%%k;E?)h@ z?dwOrdhgOb7r(a2wU2%2ZGPL4D1U||?|NP)dgOxZx5^t^?0Lv{+aLVmHCMdz`mE&# ztiJ7-&mI}s>eFqX z`Rvo9es=A1Yp*!{?W?R;!b`ZtfB(+>>(yKE<}*K%pANq`ZqJXXk2hmio|RkkFGt<7 z&!ZRoe&+H8@BCT2)mxC>a+md2fBW@{B^`XlWdC>f`R0YwcVoXka?!b&tq%C+qaSZ` z^FK~KJ59azgnj2#2Y>ney2m!(`Ry^ce!OJ$$-g*kt%sgw-+${X`y6&j?<04U{)LNP zG}DhCv!lOk`E48SaK+cJ-81|BPtJY*nTuY2?-XuL?M>(B#oD8XExzW9(+_-VF7w!e z^#1ziAN_id?^1`JxQ$zQ_x9NzUVr<&(3OX-^OHN>HSRe6w9_kl-Sfqcmv37B>GG$q z;qTLqd_EG4r~VInU;PzV7i`%$gy6vf!QI_m8w(D>3Bd_&jXMN)cZU!hg1aTS6Wrb1 zVQ#*8Z{9bvM*e~M1=eD*Zr@X@?!4PQ3I9cndkY8ejhZJrHP`+$cojWDQQhnIzzT={iv0Kl^xDLD!)h zw`i+LF;RQALrj#i&d-;;J*)TFspH>@p^`Pam6#lKDSTMl4nH%LuYZ_zkY&6wq+e`_ z@67zTs#$qYQIL9>EScWVIbNVaIny^Eu0e-fAtZS6Q_E5`w#lKr5k;}(86(daSM+rS zH{-eI_X)7oda%zPtNui{l|aW29T6H@`3vcF1gY20JAN9~ALtjmL-|YF#Wzfv6tUZD zv8S;w=W2Uwh8z`o9Z@pkEu|foQY6g}oTP$YCt^#s+Xr=0X=>3qU(Yo?X9pUUv%V>$ zUHN=!^5#!-CHl5EIKn*=6j~ zXDcFztl!`Nu-y)}{#4`64y2WQ8w@(#>R5-qxsogRIALXp`c!(4BFI#9NHO0HfnUJk z&RTMOv2xRA4=1Z-7N(+UYsVP?uE~NeFm}4Cf=|`H5iFc|g}iGwyRGc)K@&80`xVh7 zcz&4L2r($bJ{vvT6wX#Uew3~5`C}BUnKI<4xZUf4Qnp#{&{xidU$^UpF$wOw?=5AA zJf(8m+-pFfGtXy`mRGPjl-ktZS@RG-uHzchDxt30>V1pYUyApcI_9xjvQ(#yR1`iS z;#Tcf%tl*z={<}h&;2yM8OuVr|0RJ0oip%i@OX!x=dsH?0iC~#ZqJIF|6CT zWaU*nrZ3Wtd0OIX9=Tom005VbT!eVPkYujoKOS-i)I{B6NP1OYGY?>pjv|!3^sJ6+ zbAEYZPP!;ql|JS>PXt~Kw_`&UTGxn!orP3yOeVH+g=`0MzY-7W$oC;!wx6H+2>AW3 z?NFl#o&7o{Q2KE{BoOBe+?%@kaly=%vVb@`+s|Hgw<#`0iflCP(}%?xQ0io{y=61i zLe%1JZGR+;7Po5#AARS=?>e^wA2Bc%cC`Qr;++J^22yE9jmcoqrf{Z2ni_5Flb`Xq z{w6I1Q#>vLS8gtfAE;GjDffbmrm2oePM*N7;r|*XG%+IaaP=?-dLLAkzi=0F+NB8=&a~Pzi;jg#q zS;i!4=w5DiDd0C2d@B~Vg=Pq#o=6+#p3Ms!SCn{3<_Zg*<9WOrG+}alv8lxu>rA2dKNKl zn`Y}-INm~j{p9Cd8UWMLq$TsgK~*C5z9qeuepz%mjg|&KyHU?(RRcSz4R$mcib&2H zBS82dj-9fbh@+yPnsPI*a<|s@09j9^o+w=fZw+SQi+i+2qe=U~uUPgWqa{_$`!m5v zt)CKe<)2o`?D1}4IU{ugb9x)}F6#pZADbq{-iPu? zlaV^ZjP58Q#L4N~E)~f{LV(>$xG#8g{#wy$Ig{?JO!O{p%!3_QEg5n~a{MERp5=2t z^mxmF=ipUmU5SqRb`KGcJcm39ivvHc8{d_Z5`xpt#-86@w(D0n5y4BjnSsmCMIK| zumh{M+(?q2tBg8aljSC@jD9;O z3Gdf@v>fIbDO;FF7d&0gbJ-mOCNDthJKzhdO^qp%kJWWkhq(B`s$r?b$p^n>B}X3m z9=Uj~omg~t`A13;9Bq?@>sm|LVPNAhpqO6SDv=e0V1r*vbOgX0gcgfZMf9`3oJT5Q z(%)@`CMy`G*U8n_InsT=&l*hB>-(&b-Jph{qVLd&)wRp4uMyG-dA;p&u2q*Wy-#Qp z4ph^j-s2le<;?G1TbtN4pK!~O#pn+(Z{_g|npOU}KO#0r*igqclFji4Gpfs-M@j5J zBSSG$kXStsSd1eKu0r56ZH6GyzN_$4#1FVZdNKF)NfYOiQgESEiQ0iWN13Xm*G@k5 znDRpyfib3vqbZC6*e!^&{D)NsUh&F$YVdPNv|p;gF!v*zs_my?aiSkZg+!tx4oI}N z8Y<@=L7GR+DyaAtD(KQpHp%)?@ya&S{TN)B-rg9D)h%L?Omw>gVa>iv;9k_<O?-h1vWeSBxFEU zIPXV3_6kj&%JMx~!8G%5@<{ySm|!AufRsYlFGaJ>-V@5dq=={{BM{ie7n9z=K^Y)6 zu88rO+-TAnl(Z}SM#;-)FpiYnH;xLBFEJ3{{e747L9U3zU?oF{nDTyyQ9&SA2AK$X zN*d+h!A3W&fbaBLe~ly7Xp@)?~n`vX-ONFxV&gRAU;|4m&Cv$2h;zSdSSmzpp$Bsy`(~Nza@@ zhQzf2toz-^=`y4Yw#K&w-^{=bOwM{7pKi}p%*-~Xx56-&>vn9XVAkj*M9dv1DI}RvRic6<5_@0UJxQlUt7l?iqJQ@`Mr#Z zx1k?ycRF(DL*{ZW5_lDfR7r7YRB;D>4r{&{-8TYg08dq{(EDm)AJO0eRr*7aV$P)O zpq0Mc#Z`#GM>5BU0rj-vPt5H&c0P6(8PB5o9cHj*uhlh|#&v7M(~7NM15R%0N{QPj zt#g3|@Do5GHjkuc(F&yG=HY4Pc!C>Bi^)SFZEJZuj=#MD;1+ljp^|u=XoLsVnPB z)L#+LUzwsjU-N4^%jfH`)CXImp2l`WcVEcWGsT-b(6rmlweoc!V=p~_YOJzr{80{P zZ~~+hl|yVk%@ss9Z`LPgYqCZQq`qDtE3UGstt_cME}zh;EAPJKdo~+_+ldF2-(`FL z*-p|)6T;(wC-!2~E5zAAk}tyFI2Pga(aX7mzsSN0X&2s z{Gn@}&(yE=^S|hzGTS71 zm9L-rqKMem2Cs;H@YVD~;!Oa*rQ_#Lq2)h(^-5(G#NX?rLNm$G%%vzU;(!AE4xZ}W zrn5)p-%5$}Kl#xAqD(pF`9HFxjFnrI4<3kwQ56U*U2N1giMo<2hF&c6zuI(~qbGk1 z_&q)Mb)^)wyNdAP1YwItZ5Nm9{y((Iw?zCzS=uXr`7rVdPgm>0xK>Z-#pgS|-aq4!E zNGsvZUn9gMr81_p)3V7~^x#CrI3V7y&_c;H8H#TW6xKop8F(Fw3+Q=U5fVViD-EXF5Bl#p=F(laa5t@W)%4H zlkB#)S<}U>dg4_^Y2e-<_wEm47Wdat#%a-9uT}35fl3WkZnLwW*#HG87X(*jj=U3@ z-B2d6_tG2lF|7&RdG!#Vd5jW`HuJ?44$^~$L^E_4q~8e;4C-z4rcup(lY=_^3%5kU zWx_VftTOVXD*2Lgjpuy4@r*QVqy?ymQs3fWNERbVor^*yCc=iz%(l!$qxq*uEhN$H z_(@It*JHi-bEFEnHxT#Ft&w9%tu0MRy+|JE@%|JhHrEfxgYh?;b~`^i52i_CDR_+( z+)HpF(f&3Bco2~JT;&n4;t zkjH>@EbWvxpS|pdJ7k>xucCIS{a+PYQM-6QN2GWKrP4sO5YgP=wE_A`Bt7O97HSBw zkuNwwg2AU<;M@p|A3k4e{l&B05Z|)y4)i!Tta48`5|}{i^G5S)iOo{(bwWrwD&Y$e zGBsKgB`^&Wn9#HZQH8%$)3Ep>s>N6Q_At~eyd&RXkn_8;D3p-rhP+ANNEWYG259eO z?8*q=xigGb(CIUiLKa>&=g(Kh?u~1Q2>a)+5XA)An=rDfFGa)35-CXW2ESh;+(U}b z$1++>5?<67*u+8ZMX0Ye8ik$ypPkIwpMVWa-Y_ksj`Cht2m2Xie`jSlKlk(2X9(^ncG zIClycTF1)gvr<6Nu0FS%C9)>Sx2b144<5WwGlWO_4;erjajr?DGpIGXQfjoZI3qC?iyvX`;F;4h%_i9 zn?#{gBEIApz( z;%$$kLPbKoz6CwKz5HN4S4Rce3pw#Pt|`7wAZFeax?fs$T@I22>Q|sM;zn5j)ACbg ztLv+b*KI)8V7d3!WA?l8udWjbxnUwKQi)AsO~9bQ<>rg0{Ay!R^w{OX?@)G}lE?6o zpwJpO6G~j=OTA^!o&HE4wV$9k6+OsW7;IvoChiJquTl_>a~Zq|=BLtLf3lXK7`CD$ z4Y8z!^5!!B4UeBzKVCAA+CAEesvVF?w*|KRN4lFN{CgeFe%&5`K2*XKNJbfd)4~wS zYy#_HgX#O}vETbA@!Ob!JyUiQ?mxcX-x`=~XSF~$>1J(lU7z=6mmg=;{egCirfWm| z+5_FnKgIKKo~&#O4& z!P4*^p3Ol(uyuemm>k0lT25@CO@VycqwK3dh8Kp4f|N`}QS?FnBt~qN_gz=Zpa=Fa zCgY{nA)L|W&`F=Y1@f3)V_Fyaezx;)W?Hq zu2Zr?M21E z>f%rjMa~4?@yE?`Hn)szTu-Qh1Z66E2%Rk7quU`ecvBX=WZt$hGb`gWAI( zb!Aa$M@6d(l=V`#r+e5^m;|E(08`jIQ#Qx)#c37t5Bc%k8%KeKnMgaJ4IzroN^slc zXU>$-n7Bio|D?8>PEMe=kK0N!Ar+j|5~boIz-m^V6%yisfAv* zbuDm=Hn+F5wUYCCe?l^qKst5Ef*pMN--7t47F%pEeFD$D_bpjn|Gv&h21NmIvVkP( ztCPP$Ku3;@%>DgK@u<2P=5Sy_dH~8|dw)A8`wTA>JLxuIk^Sc0&i%PVyrqMjBEF!D zMi`u+O`*1%26|PHFMvCwrL4BRZdW4H66NS64@PRtk)ep0t;A%z*NVi3j%KXpCfb~P zs+@)ZZY({9Es0C+#o<>bIWk}Uk21vf%!qe3m!3xtJ25p5*7_XEeXDT-lVL9V7sLlP zZrY*riQd=(cYOu-hr7?Nk;JMVta^Cm;Ujz~BDD*NGkGG1u;pKW6UztBE{D)*&Ob(e z|I+_ta_~k)ds7gZan2%(HuOB8W$k9sMl)pMN?;lpjhP8TGVjIX@e)UtR*B=OZ=z~= zz;Y8(J=%8ARY|@WHi?pw!z*j0BvD4j`)f7_ReB^~e$-VhCD2A&-OL=4! zA$~aoAPsvLJzM-@n(@ubam^)nNYxG~aSo!$;4Vrd>r!Lb0%UV9t|z{~=$KCxNC}9^ zo)UkNYO_vCjZY)}JT3e`nf?%o7B{a_!$$31*bR;XYuph5|3GYrP)}7-Uxiz{vB*w& zyFPaqgdPQ;?|TJsa>e|QlONp0ts8znt6})*>76e}VGOK*na=iUrO$t>mC$m&*ahXNZEGjbGJE0e<1SVG=In0?})mbEGw5_$Hfz45+mNU zG)1TR*ekIY)h~syz@}dd>tis7(oXX@jA&WuOC~7;1-RMCT$w;|d$58{jpZwuTyWh1 z7kmHqzSum=Yb8}dmS!8*vHSPNH~Bf7i8$6aBA-$!lij{?lA@TtE!Cnj-8ePP z05D5KTlL;Zx!GHbb?7mpg>Nf#eM>;U!QO89s4>CA=I^6`kR6<&Ub2dlDA!RW;6*N+ z897rsP@+&iFZTT!fw6=m@N7IEko1)n0}-9F>`fa;W@=Zu(nUe-y%;@!Y#UU{1?Um~ zh>0MUGTs@TngI2%tyr`7q0RPS^WCxbHqJC{Thd43+>`3vnKBQ}(N0H7s~pSuS5#G& zDNs3LT}lm$_3TT}K!RjQ$og<;8wcrj{f~&Q?b7KH{&wXGrajd`UW+%U= zP1#`+3%KiIY^Zu9Z$94w2%dd!Q%~P{^(q zagzk63I?x01zSuEeHrai;I&z(#$9SmM-#tqA=4V)e9@pCLxTPZsC+F6y8nFd{3k@l z$rt@nSJOipTJt7i`o7Qz!$hc?|DsduWyF9}4p+x8PS>BJLieS403k9h?*|&S3$wCj zA0YdyBRlh*$iik%sE-G9g6VtOt4+5gH^h8JHgN3802`wbQVx+U7aA8Gg01xXFy2%s z+b1d3Y_Tq3B1qAGnp_^BU)3ccjI(gJx(_m zj_xh!_;Kd5%V&>KE_x;p?#&-awCu9ze_d4`EW=?Dw0JXPyq#1`Ew&oCn|fn$lb5(AU~01)Rv1Rtz0mp*JAv_B zHlrk^o5nGCz?!z#Jax=e>|r;N9(?H}6NFq@xT8MV&HxCu&QGTmmUQD|*y?C{E(_J^ z8Zo>UNxjn4EHCv+Mfnrs)6*Ht{(7Z$y#O;JEFcP)^vw+)!3D~i4kX&x=|F#rX7j`h zmMxe>>IwE^EXS$1)|iyK6V`X){U25RbNGGwON7Eal~O%0RpLaLigeYQ8V^48$`CtC z`+2%0&}Xv_>aq+6v1)6v7pdc@IDTHB>Mx%>H~@OgkEZoXwGOn`gZs6(YAhy`l#8=ZyHkt{;V?$4#JHR$SzkX8RoNRCA&QH^j|(K0e7zJw|g_*b(y6e4BKAD2~;- zHGu{xG~_&}&VdSt|NU2t><);QmJ554-0|%l=ApO|a%017KE4^#j{I$((%UhCj?mAk zE7l-$8LsnY_Gb1cn_&WO<7f7x~B zlr7Ropo?#s8uMmw76Z{3s$nIpm6V5{9pP_3_b0>|5NgG17`+E1Jgkh~hXJ@^mLF=B z+DVZiG!xIS&2ax~?QLiuLV?4a%65>ZCcGOtjEk@nj!uanvJ0NGWfQ0m~f ze~Ssf(Euucsf5OY+JDRWB!~f7PQGFh8`;0*d}MC`qhU}o?f>{Mu`VL?L;8Z10@E-e|0ibvN5ov*N<-;_>=^T;cCtoS z;7H(nRY5N@IXbSOMy$>mi7a-_E?1gt$9|6!{Q4`3X~aAJO0i`$`s-Cx-{&HxrqA1b z=)wf%9V$?FSe>?{CqPC6W>)&p(J~x%Uza#=#RZH@_0jE^q1zE@g(q=9?of@ZdoNU= zdMHoaHMjLt+#m*-oVK^1L8w27hRnX=pmyl-NhxCT?HLY`_xaw@DM$m_ z9CeMt_XZfrfGpp6kbLn#>3owv8jGYRBG%0{V#4X03KD{{V~^>Nbj2hH>A0jDXDWd0 zJhemSq5)RO znc7Uc1xw&U{emIU8n?XGSLWO#VBXs{tDxisO(W)@QAYA+ar?TKw_kAhHiNQYt9m|( z8wVI;OS-LKBi@k}pg`GmZ5oXQUBiKx@n5|KM8bW4zLW#X?Ebw{2~)g4M{3^|s8*yk z4A3bk1(gIZur6BEuIbMaYE{P=%pdqQFOYcKw5Zsi7(KO&yFF+ViV&RBW&{EsEQ0Xf z=D_#@vW!tE)QO2Xo?CA&3`*^9p{(2}8O0e5@6n8!1UgqNB39wf?4g|t;O)Ban5bYN zoqrhYum;>@jozEfm=kL3sOKDCdTcyBL5mcM(4WT)+=9}eKEW7%CUb`0d>pd*`iF+V zrf8%uqO#+;c`SyL+qV$v9jG&sk6RqcTK8)wu45$*bQNl+AgCe`a)pGU@*dFvx`arC zID@QU37wd0R=fgfsrTID7+)lN8(Wm*=SFA08^)CO>drW!?Fd2TXmx4RYzsPu45nGR&}lK2 zV*La2%s0_E3f>VwpQ-b{ zmUREK*B6lIzGNE3=n-=^wf%ZjI&8@6*y?)%+2p{wB4zFY$-{~U?`PvrKeE^vgw;eA zp$K8lXnpxUXDjC(`%)F8p;vJ#@PXfcOQnwWz~~I=*Db35+9*n7-d13eR`7x?9_9F! za3s*Fj?_cjOe;Ed3+VHpAagS@B+#y|!v{Jt<=)&2b)aY)H{DIX|J`)2N!GhC*gCly z(r~ZXYOb)ebIIOA$2oY{-c%JE1I!*Y4JmBl^k9yirf4t(hmC8Gv2}K)tTONBGnu+o z2^}E0v){E%zG*`cQTd`OBgfrga1apKE*#wXmaS?TE~jjr)iC`C?N#A|XRMQ*Rm}1P zx5H#`c=2}RUU`C~FhK{mM|*IDt7F}qZh9!=5+CZ!gJd*;ahJ-O+P=uum&Y+Gfh~)Y z%JXOYxq_GRo~CwL2)@I05LO4mDAIw&fl-{;OLV&XdjfO&Ez^iq(hiTfs#{P7H{DiI zz^iFC&%oZg+^-Lc%fn6GFAf5$c&n%0`FcPzyz*Y}jG$OL%vx!+%C5G*=^>{Y%G%76 z{;A@R!HZW|HP?mhQ@h0k#rlrxfGf;(fv4fu{%h`E-k7_+Tpkk~iY5%ofxg^gvdJvC zusCrN&!U%F%@;B^;BO@qdqR{*%OA_w=M!6EW%C*v99nLq!`o%O!LaC78L#c&n z+UzE@&qJzt9PG_E{2HhV)DXV{6?*$v^9BB_JW!87cLxOqoyHi~8pE7%gW?2m09-1; zP=`9fC)I$md@LS;mR%5AAllJ04s{damBVJzF%q;{?4j5a2=3eu+qjz#?)Ps@)w@yq zw(@YDFp-=Wd_h54co_30Q8q=EFAW&i%5bGHSrUkqpA`FEu4W2-RgNTGdnB_Y)7c2x zzyHlUB6bMB8?ArzjaFs{7PRD>ii@TwHuz#!!Lk(88&)7U#F~491`YX>g5~}tHYR9# zuvOOl#Sv=rcA#lM0e^QNK0w!jaKV4<{(A?q!bu|rndwth33Fp;+gp-M1gRbS7$>V_ z!QF~kgsa{&Zf`7@NJ$+mm9f)kiIfzF9{G#z@wYfky+;95)YIgt5v!Qw1v}pMs)vV< zR81Q@NszOS{nT-afUOaBPx82qJ|YkpLPI^OXhU%)Y*WIF|3~G7LO0@i>+YUWnQCcx zoL^+J!kM<3#B>SPJ$TtX!@QFOgDRE+Y3HHmzB`hFnIPtwFnlbb<=LU3_9&sW@BFyqr9g&0b&OrAcMLM=5 zKs<(si%!Sy?W=1n!_x@92q2@Ka!%9Z)((Z-gonTXjl{@gD%Si4ghYtT)$DKXSKyG} zZTP4Qs;^I!@aOkBxhNF@gRi^$gTiFBb++zY>8zqcWm7d^wBwv6m=x)dv=g0MNwK*W zNxao^)M$Xvb}rKXAQ@gaZ!&fRj-!V$d`j%+*`UfN{xBOjy_jE@GwxtZC9;$g^T+P< zwpbVM??{AQN?Z)>r@Hz?#}FKMI;X){JgfOB=-!yHLn0I_u|eT1ZlMgv_J!iU zxUFbsMEJNiuff$hm$zZGlHqW>C4A_AmjV)%-%Gxe2j~Mu7!3H(ZBN<}2axi;I1&V6H*uIZ;Iw z-ZOZ(hsC#%V~1it8krzC6@FP7kTzNhz)6Ze=Js*tvgDhmX0t`nNOTgAZjVNXWfBwP z5?PaO`v{wQUe&=W+dEcs;9{y^%4kzHc3L7MW>2sd?DR<0aVejxT0od?n)T^Kt*h5o z>o7?z3Xs3Gm}@J$k$elM4=qTIafsa|t9TS=8wvW?v(qrUZ2{g#5;&@r6>f4$J3GpLh<3 ztat%cpY-yijMJsNGu{K+_iV$GI#&1GFCWE5s;400aig}`Uisc@p2*XAG9SdXswx?D z19)CDdE~JfFLnFZ$zTC3FeYVx@Ptaw2VzDB&nVa^!E4yRITO64FAk z4acSW$+qrH3hZ}ZuJ5h(x`ri?rn_@3ugN$YoD(&f#!}zXg1vr9)!Q0*5THUSabDq$bbp4}Gfwj0F(}g)@o#9`H@lR_8e4-|e?jnk+UvrzE1n z36L;MB;*7cy_P@|Z=}U!t(sR@>{NRdV306zQH|u~=#cfM>TLGL(IP7w@Ko4u7Q}YT zy|g{Ro;-6`T}a22YfDCuZ`@**&ia%bulvy3JQmM+i~2q~red6A|Nn0HY{&aQ|$NHH?DSfp$P-1IDiqqsKClEu)no zEB{I%{PgkzU8Vo!;h|k-MG%RW(Ivv7z$D6;T>FhP|8DM29(PO?>-sHP+XDbU$ka6E z8m>3=p^z}TGPnT$yq+M;7{!p#CZsnSN%C5(cXh=!?khIAQ)buh=$OCTz|}wAJ#-xz znBW-ska#`OJyai)SmI&x&fG-hVB*x?sf-|p#yE2x)GwA;k7+byyOA6~ zh^^btq;vgJCvbqZ^%T=oa%qT;3$1qRxD+y|U&{X|woA{w#dWj|OgN!6cs5cJ zW(=gjif|(l#dzdk~r1c>3&Fu?AhUOMuxUd0|2}t>4Z;E3=wa${AQc_)& zV~>QJ6A+n>W~fx}-x#-SRhUkjt1Y5wUJ_ zKm5dymJR3afHN&wvz;c7xZTRA76V&w-duadgT~q9m-aBJpz_L62vBYHaXP5#<%lmN z!J+I(_#LDbFivP#X4HtKH@bV`-|A!?QERLZyllXGT5Cmq4(knrrwf`BR!Ar`9IbY! z@#eK3fSFjwtP{JReJBWDl4Mw#?zfx4R?H}586Hm`Ly_7rAC$11Q8%AtJ~oN&)E;56 zZfXCz#PO8Ieg6@xFNlm(y6g64<71nYKuI>w7`{WyCt1$qkG%UDKgid@V}RQ%8Q#1W zBO{Rr8^T98YPL>)8ZOPa?{aVjj&w2inL9=*qNMl%; zYD#K`3waV{xQXz{(Ujt#AN!|Tk)kW(@rm1qTE>iZ zC}6eYrKPD7K%LVLj)MCom+XE&5^JHMvgz)It>9@b(=L z()u#e;S5$AR(Pxg`$yh9iq1K6=n-v1#;EjPQ7OATWlHK)JyRAuMG5re4_^k&D$dH# zmqitGv@GviOc79zN*`Dqh>t42Cv0RjT<8AXp4QVQ(4lAzm^$+^T_PzkHvN+$kxJY7 z#_5xW46KFI`mK1}chxXug!p&y5TIn^x^KJSr%2u97Hb=8ZY;X+*wqRadDE7vtH^_j z<6TBE^|mnfALtmmJS{<1xM|R+l){+kRD9X9v32W&R;91)2?6;_j6UfbkxUFq`}ezJ zHP6#jy9@Xf4+VMUsZX<;hIGX0#*C+I>SI|3goQHUF(tKc7&xQzZ=N`5)6fYwr}~`r zl*cJ`nD?0&$e{1Z7A=08rKfEm>vRLV3Nqq8y!+`Lh72dp3W*8T2v&fJfzR%!qxxa|K){>H`%QwRUm{D@zD{hR;hNp|X!dWDa%ciM<|;$p4QS_5CL+jyS@2w$DeYmwzaTNcbVzgakoi-BX5})C#&nsT zDQnBY7yz%E{C8mb%74vBlKb8AgI`wB)r_3By*cBTr}#z-F=jdLdh|iJ?J*v=JO6dy zi&oiIxNuHk%0f_e6o5fq04eJKdz@PYy(${eK|kWjqWqgzMug@9 zsA)?W0XOx3k;TAJxe&lNAAh0Y{I6Sb{$5J+KVDjsjHE%GmpaMt04+BC*9b2xS#6=A zw{2{w$XXlDGbWFV#_C&07W`A*4AJ1Fzppsrl7Isp4l{}aqF#RL*XZT0*EdBk=uNyN zVzKEkBq2GW=)V|Y+nk&e^9%^lqnF*@55x8Ot;=Hhi{v4G+mLxj_K5aJT|nQR#G#fs zNx}-fiSkV>Ju#x+k}mHkx{K_U9Aeju`eJ6W82AHv9LG8hbRLAzcd=pv7|QmIY2_ew zKt?-p(LwQBI&z_%TuFBBf(o6_EUuOS$Kzex4L~)4vDU>!-bKUwSyUJJ=NageXTF5~ z8scYdK{?446)LlE2)$oivzilQqM&04J9UWg*jSnfec_-C7hy!+gRWs9+)I^e2rRIj z`Jh3V6sv>rQxF%-9^%@eo$EB*arlP42%-yiG`b2xAVnaD0I=j4bWdB-y~47x1mJDX z|6~^1P!{Av3Bcxj#}$(m;}Qma?U4lQz~CYH_Ogy&pn~GAj6rmuyKa)=AE%N5#vMo` zxV`~?T$q%np}@mq{vQvIl?P`-jU#mqfm}d%Aj5#qTF0n4?|mIXe4tOTzJXU-ARlO& zw}!aB??Iy>pFV1WSac0peA}TgRc!;ORb6yIgx{2q?N}s9!l+ZsA4-P)!?eJKL?eID5zL+3) zn8qQJN@Bc%n2O_EQ2+am5EZp@P}OP!jNvb+#d=x!XVQXYPy+OsCk+A?l=EK`g8v_@ zw4o}9NMG9XBb6z30cz9cKO}A@P@eE>Ha(a!+%AkgMt7LL%b*1q2)2l5{>Ql9I?$hw zCCKGfZ$%B8a6qirAu(JCc=klx!kK`; zj^D>rBI|HhbPbno+Fzg2L-rHrl*&!XDP;1-R1n5D5k31(1f1DP6&Ql+XhGU=#878~ zzVh&-UU;3mAh5)}t2Tu+9@FC$$dbUJs6{v%BYt1*%IUc_QYqmOgadu&W4hIX?Sn{* zCTM@eDLyfe_snAgs_y#8E`FoW-ZeF<&HQuupZUoa;P2N_@EOO^BW*$W^kbTE*u z>S>w_zRWk(s$>c_uS<=73+iF?P&1a}nBhC4Af$sUIH#z-k4Sz?MhLC1Qtv070VP2^ z&7B$_hwU;pKATBjO7vqsWdDu%Ow>CvmoOkulmRscdO-45tUv z6+YX?*ja@`&Ab8MSpCm?mvn zmshn&;yr!FAd8RkSIr1&Vm_85+t&d$J~k3(fE7<5vfGU^E1JgpWe& z>2Wv13w6%7TgR#$enWe5*z_}wd|hls&0Mkam0?d_tH-THCPf(yL@r&5`tXcHX)qeu zUD@Vmp!gDTk!Y18R<8|VX^t$tkQX}`yXEqDkYLc`K;U%{49Z;O&O0lk zRIb*xF7Qk^U~-WK{FZ<1nR_zkAbEoLy079moz?px`FnK}J9WNznfo8-ErAoo8u>|=M{0L#BR{cu=(q)TQtkO)n7_}m*f z56-4VezW!Q?-l~OIfqTD{B?el@!UlbftV-Khz~;Ox2xH2YjJJ+j~gyqKYxAKf-J>? zR4Tl>7~`$Sy0hs`qgEFfHk8h1|E^{if|)PGS!@*LJs%`uGl8c4gb}RP;rY5unm}_` za8XgHkc}kB|FhRs8rA4|PI#?aEqdVXtjlhx4lu6>y_H*lEJa8Cunv@Dh%!rTpfx1; zBDXhYNK!CbHt~xSD5%T#?23^u=mPEm!LNKMr!%1UwU?YWS^icMjJBgwQCkZyCSzE}J zjv_BoxUJrxX0}oh=p%20I|1K-Q*vP*L4qE$asPNqPZP(?{=VTL2UYQoPQdu`8qh`T z61*;Tq76iqe6T-78z6ZdlKcW|L;{O!4BqY!u;RASm2Ss_zhCQA22p9BO0x)@G2gp$ViAJ~!9*?W=ngo%$0X4M+v#1d!_g8*w}KUv^3GBl9uI(maiF}q?G|6bg2OQ1Qh5mc)_<#ZdBaTPQuOdv-& zxg|9(|NapJFR80Lql=t+RMbyQp7sMjC9FgEhqD&~3-BCVoKmo@^?W0|&&xT#KD{!0 z&%jt%sXRLD{D}az*n6vNldEV=n=Lz${=c6W$ni(~{<&eO(dk@3rC0{nF(gcQcP^PYirkWw z8Rx$*Pfq#1zUP^3tt&WKyDY3JV|`d*|2 z@40NlzjH!*j(oN|+bfk{u6L&e=YJD#p!}E}tP&NpdziyIa6$DMQWPdsor_GWRKbD? zR*3MhzNtgHd#Y6=qsolb>)T`|#csOiuYL0PxOGGP@uKaX+aPa`RtuL9y-4pF@Q(lZ zR0%|j08?uK5%lOFTIBqmI7U}-FVoxdP3b2yWOwsU<^)&%D>3xap6uO|`G;Qcg0{vX0Z*%wF^1|7j{ z*sJ{>A3WB7F52`gnjq7w2wDtFH@aWja@fohC6>*pT6QyU?juY#NB1dB5p;&_Q^l6bG%lh#W6OX^30-w#S2f<6IKQ5@2 z6-cs2kLPGC9aNQSKa?wZWm~nB^>hZ7BByYF)ca?yjsec8fmr%$#I~B^A?KGudxHeE zjFS+_p&0BBWT1k1xC5k3uujl8lN7E?eop)Bea>4AL%3c1qcXsEYg;T9In!u-7huW| z4#x=a*j#(5JiqV0-kZpUyE$42gX$56`#o`ToF2BZj9aHm}EIv5jm`|P$*{Cqh!n-&@$lt=>^4L%}m8>P>hzM6U`Mlh`__q+b0D2i$bN zL1ivqX6uj!7Y?=Bbz5-fvwmSaogLE=(c?@8iZM#q%u3KZgx{Y=y6$SQjl8ET8=N&& zZY^Hc&p!Do!sIc*JZl>KkAJ>jfdUv*4#T}4?RYU8^YzwPX*1=4O+yXj(iN>hRIvZV z;TlX3H#D*e*+E} zGL{O`Ok8Yp&n&V$niBpHpgx3S)Z5D&7Bjg#M*j{_6YM7&81|%@r106+KRiYr&e>m$ zLv4BUWrr!FxKx)x{3XGXHnSAK8um{klZI3EP331mjDMXa<_0Ef9Nd^kOXU1%tu+& zCjGs=s6_2VJ;A3I?B7;5Zi6mOF#8=^MF81UvF~UUxzf=ylnR+SIFF zrfH^jcR0ZG)VV*~=Ot=43j%wodDJ^ti0ld6=IFxbU@lCjR*sGKRuP%j$C!|=yu9hZ zHTRZrQAXd|u#^lUjUXbz&>^98!%#CIAuTwBNDL@QcMmBsw9@DR($Xy=AtBw3NOuff z&kg_cyyrdV%lq}^1M~aAo_nvo_F8*g>)Q8TJEJQX#8Fg1*z_qXx?SzJSsuMok#d_h zJC>F%aL9P2H(hbYnq{QZdz9a2G31jFLO^Kyl!Mu|sy=8ubh^;nhOzq7)odNs>b7}1 zpHRot97W3O=Hh5?leBb}{;LUA^cQ3D6zo0U>H=fNU{uuETOlzg+Ninf6GF=$#{Tgn zY{%il(?JC<_tIQMyx-NHd2d8rK9qKsqT+rPjWcBlUZJ?Am$M?W_Ea}y2$r(5o-93Q zNauY`(L<}-{IPFEG0yeebZ~Y1n|{@|zTQ_>W3zW=4yw!{T@vWA5_pTDIbp%c%S6|8 zfwu9#IQ=&LR_MO?2ZCHbf<1TYjW3TBKX-8+f3uuqGk|`caymJAwO(;0U?5aY2a87; z)$UV=Q+I^LM@!{^)+YFKDQhLXK4f?#;W|za2i2_1o;Q#WDiky+0zhj-_8ZJB7&tN` zaRCaB`W#G6$l72^p#2U&3`s5uwv583vtC{(*|XVc+kyKo*YC(QUr`s95WB1(T1f{j zjeYPBZ4%^;eg5?qsF(>9YJp`T>+U^>E4>W{y{H!trN0z7#|Q2P0eNZzKQ@ zeo=Ft?it9(L|z<_m*I1|R%0!#vtF^D87m!xJe!{2hh{wEk7aG_&d|g5XfIyaOG2Fu ziN>TzvtJ%Zbo7YpKEpKDN-pz{Uk+g>IfJoB@ve1iujL;Ni;rIFW&NxUiDp`HOh_=t zw3ud?4vjXB*LVf6KqkM=iFj4MW2!1idy~DEDM@=DV(?vGa=7>^hjIC-8LOASt@rQt zSAmVrqmX)8TANQ?_w@IWwR2=64Jo0j@>tZ%z2Vcpp8DNMAt{@-3j2dKKXu1UNmA+W z(HW($sKdIV-)gtb6{87|7UF_0Gy5KHo-T)m)6LYG(OYdv{O|=oN2^3Oo=43`OP8=W zIIMI9B-m7OBsJy+IkncjdMN1=N3@}M8p9k`6UMjw3$9*Z~4Rfwy@)uGp? zE5_(vU2MJLi=qX^5}=BZv07?#=4?@ajyX|(LX+nX81G?GPHtc7u`+aSM+A4J*Z5bw zz2xOx;ZGz6fG7lrWeM|P4oMT4XlJewGdgt0s2m6?*EIv zD`rF62ya4SG$X&iy8=zeFoy*=oo!~1mw9KDxm;Zy3to607#|+j$IgbG8W%SP0XFXd zRCn{DNShp%Enu}1Wk@tk`7U$+rkK&(y8tism7jj;+&Vbk(y5_efBUqf2WE*`ug35J z4dEq`3UG$A;bwa10vZD9ZOTA_GI=--AOoW2W2yk7x;o_>a0W1!>mCq;CU@S~br<5x zUP64tY6kKMnoEoBl`d5~p0oJNEV%JE=^@U^t3cC}!(mZ#nAfiVwVJXx{3)qi2GVJ< z=GE%BH!kt^r$A%1bj79Qt$`xBmYPqRc4lS$#X~4$zT``ZEXb&K3A~NzKRA$xqyuav zh&yKp@>pBvGzY2_NB~H@fMCs7@2b5qnyPh^14p}874DWV1n<@SrdR&B&h_0x5(_(b zXmW0*twy<^( zdFn`vbWv&wbFckM`80gF%RL!%F??|`p*2vf@E&vR25D$O7p08Ktp6VR>YO_(8_a+i znYbl`K6G(o*u~}oI(ppy1P;6bsDcPv1x>nQLuy6>3Ey*TfDuSWoe@HqB`p0PvL^S; zw|4S#UJjW>=RloSsU}q`ib3u_a~IsV+aTX&zO&S1iTZ4U@{_o}?1$<4)Z0QsgnUOW zt;Wity3FbND&|4v29O}b=-vbjRO+)+*Yc?orgAB3#E9tRo z@kV1hyt|z@Ad8T-`jwkBMl-}B2=C1&c5vmOIpLN;-e^zXnYM(2kta^1flKIy=f|@@ zx&Wc#ayLKTgTQXgHiF=5u*q<)G5h{_Znx)nl{;N}M*{x)dW;f)5A*6*R1ylp%M$Vn zT&e$j4n8bMdb9hjZ#Yt=Q%fRZy)vb>B?Rd-{QGiOBRnKVQppzcaCa{4zyjj2xmvwC z`$P1|x19EK*;}QRbh+I>2Y9HH*c z#B_1El98|8pc*c!y_G0^eImKdc*o8em;O2mbo8X z7lgbI6c0w>&%RsYD?Ho~eD6&NrW1aHsuNx7`J!I10fn&(g5M7u;WQjUT;6u?F(+57 zhZ5N&6>P}q*-FwX8Z!LfcL54*ZSvvSB1XPnQyJ@8q&EG58bfpdLe`P5xthuzsO(ef zNEO5vtH=-G+!@yLCa6F`Bd==c6TL;LwN{FMea#zmE9xp<05*eLSgK|CFw{*l>%{Hi zjMXNYt6p?qRLVgh$nGOxpuY%}tPHBf)q_yMiagcBoXBBtrZ=<9QG~)kB-GT-oC*Ce;3b_P3+u?s5B*!aCb{U*oE^J3!LE z?dVs5ibe4lT@#%-+XMnp-}Gvqt}9YK>C*9(Fbf#N?x8F7zrt8`OIUI7OD}c>Wd-Q* z<9c*4F}%rx#z3$zC;dG@S=2Gco8mupw0NPp^>ogu|M_?_)LZPx?Bbx0Xwfr^K`N!b z%Tf}E)(?QX+n3Iowz@JE;TZ{afv1HX+@Gw(^z=ZB%db~v24an$p>0QaxBd9`jDQ^p zpy1YT5q5Q;IPATMEA0iXy+!Y3JnfPh{SjHl+?|~3m3ATpXwtj!+^3N$R6ViBNEaiz- zd$a~phnjqqJe!Mgib9=JWj*ndZA9HG1o^2tiqeyV%}2zA*=Xu9MQ46f+Y7YqLJs4E z>zsEq<5%}|pWX*Zf!7WP_$@Mf2f5oC5@xQ=-^*e?lmkBgo_*uK|1@L*E#4NNUPNrlKO?OI_ zN3WjyG!e)Mn>&qtv|VSYISn4~48LC%1}_EPBzy2FAz1(-?M)N%e(f({(SFA%hwY{x zNi|_vCWR#6r~0?U1r*7-I4kk30gj7rAD&*6WWL(qS+AbsfwxYS=kC2tx4eLqI&K7i zlD6TrttYB+*)o_G&l^N5WLK{<>;K**8Ns8Fys++Q3aP8KI*@H(JGCQs?0cx0yO^k7 zy)~btKa+I^U!>xED`!|T6C_p9_%U=6aAbL0YbtTXjU!ir+%ls9qL)&%duKvd0-zUL z2u11T&PncOS_D#!SJ-AVMi>{J-+fiS(%S6psj+?Nm2XCSe1TMYqEd9_*|u;R{*#Iw zbIs9p;#wXQf6e-Lk>EOQO6beWk4IJ`#p3DRd!g3RY*)fX#$mcdbH94!EJw09jggqI zSTo<<{=SRIsDf5bRjQf+sg>=6f_3!)+yRKElTCI z7x2ohdSJ?4m0$1MDN<@?2{B}@q5A?3H9#ns7v|Nvk!xPLEcRo?^Q7vLY36t*3n3^y zoZjVyvdsMS4)OU?7@sm$vJZp$)_W23Z0NL)M%rjQ0?9+jUkZynzB|4j?UD|Z7v0a? zGe&OqAvs;rrZ1ZLG%hak#3}ni0b8?Py|uMDgli>f%vl?ply#Q2{y-aqSa^uPL;yz1 z`VV9%2e>f(PH4IUD9lSgC9Ujs91`Z^-1mlkD3kQFL)^ifzHbX6mmC#U&%J=;sGtMI zL2LW$p+?j#Cq!hz{=vX@^Q+DUHjN(NELklcj>5_YunU?>FYjg;G42d*`2Naf zW?Q1N%mYu5E~e5{@oESl1L92kDWlCKU0%vv<#3l3xItjh7J}TSlc0iV#_A7XyOJ5w zjvk{B#+~;7h(Ue6^${#^DIhj6))v!NoLP|)OfHPYrMkfFO9E=$o+a40CMT?e`*p{Z zjW9sUjb(Ls8Uke6^Kx-@ZZb{$c8V?4lfdI1Qoniida#wyQ=RYy`#UZGPcuo7hr379!j zVHBPs4bBb#zV-AU$n5LQw|@LjVOz{It~-ao7?<~+l_1<*Hthj|Yy)fMWg zD*xGnhGp`&vgy@HEL_4W-rTsAGjsvaEkKq*Jg%l`0MY{=Ab?&uFOLt{z1&}##-bP; z>28wGDBdv}sXD1<5fvKb6MCz+rw_iN;s^{DsU)7J6>{J@IF_B;T6MDlxWva)OvZ@5 zN87g{=d5{=gHr8gfy^DmQZy*N2|Z%W_BS(k=X&`>? zYWMNq@2L6zBF4;h-fI>}>^HrpJWQ!-{;b6|P-ZIFpi@p3OtkKw;d}94`KE!!OsJ1Vo&Zp@AjC1F4~Rv%hI|=_8>mkM4_-Jy6SHf7C&$%E%G#q$beM@Lb-h zc&;q5eCud-$H-=@aklZeE@Q*tulE@L<<3g&&b8y*ae1Pc)5h22$|%=Kx2xJ*fz_~( z3Xc)aw7K2qGj1C*`QHym-IL_Tovo1qJB*XV{$zupe!Vi#ovpo32NS&`1$udNLuOSB zXs@Ize1<$!T*qm3nJWfdD$dLNc zi1gXvsGnu~dgvo1C81`>D!bnTf;%q*oq%Z17&ZAc?Z9RqXSn@6qAdH4!$$|Vwtl_Z zwWRZ%+97a+s;vy(Rka)ai@)6-D_4hBMD*f4;P~_qOwQEQ2Ie2h%fpta;Y02zt1dKw zN4_smWb^s;Khe6gg>t9%38Keers^%(-XBUEiXK_-)0TW&%U<~gPYQ9GZw?Wq*R4I- zXSE)vgU>Eex}-e<_tb|*pcHXS$5a*D9N+dB{{;qAR9 zuPQRmc#_Pw^{?UM^G&fsGmXteZ;*X*hybqJRlbF`MDCdQ9Dy@tXG%-`K21^Zp|!Xu zy0NaxQ6jx_DH~RvfKT@C2!j0t~dnK>;}|7i!_7LI8VG~v{<9YhIkwP-p7m`ZXB*P)aKh`N+&H+ z-aEI9P+Of1hxM{C!emOy>~v_rKp8*(Lr++JEte^r{2AQ;xhl_eKpT^ZLwC;=ffe5i z+gg9=4_sJFR5eZHh5D@E$Vnb%a?M@Vw8ELKu?C~PQf?5 zG2PQdSl5vw4BEV~z9DA4ICNZ~*`~TP{6fzQW5`ppm^D;LNpeZP@3wbpDb#e7XY+eX zaMhl%)4}1x_|K4$Z+TkQI$A@t1{>|V?}d$#>jNtsBT{SHX})KrlZYf~`Phs&{6cJD z8jU?K{0iILACl-<6IEOPHr6=951U=(7BaO7D=5OE9fxv4H&L#g@(eMTjkp-pemCC1 z@_MS))hel`(Yb(A|JU{5Ws4Af^0fM$<|}JAm|3IX>0t%ZsZG6m~7GY&z&|oo;P~OZ?r~5(G$I6i-~latks(zI4g3oI{?;6WA-11LH(WPi{%ZLN!M!} z?b0{dg&$g{&PEn|5i@nh+LZ>*D_?9W0e?H>?{a7y*4|j`9>L__T_3N^*naPsmMr*# z!3BvaC`noy6>7x%i{^?z<;=meDAg%bswmGunQquKqda(p_YlOa)sR@lBQpYq>P_0{ z-*Fbyj3~!(X9G$kChqj3(}_K`X!<~UV~Evc#6+@~sfhu^qf^i|&oDQo>f|y`PO;$X z6 zqP}lbu!9iiJkd6zA%v;Z12j9mFSmvs*$gO|X*)M{B}-cR`xpfsDCFC>%03eINoGJF z_KKJip5&@!EV=2cfJ25si1sw|@LUk3dN??F`h_hSgGMSGDTrwk-AIj^#HgtKeKDgC z3}hGS)Dj$-|GKewwD8r&Vq~#llhMbdr#jKS+|E-ydZ8Lue)SgVm_J9u) z{m3+#&}6HL5pblAgd((CDS)+lzghBKi95=09Q?r08~88u>82RB8$=AzbuL!v;}}Ul zqGYHyFULFhovg&_Ox#i6IPI@Ny-S+VRn@@h>Cx2tg1>!D7DxfpzZ2Z&AQ#j7^`HW` z2vlO+LK zAYC?5F5~CSkq@WgM6?&j*<}rbCQU6H1)T{*!=io>1vCw=C<6Q@b-nXUe2| zF5Pn;{sWZacS5sq1^qTNWQ7-1WM`4r2a+k9CyP`r&^FyU5CWH+uwIQY8KvlF{1$$~ z5G?B5m>BKD)WXwpj|2fhWc<9YY^T-B4NO@|h&@8gS56}N* zZ|HuJ{TY45ZX?fHT}Ik9v!iNT+w#0C(Sf6y{Wl!s{n;l>?VkTwUPibmi}HY})R*^1!#@*)D!Be5=y*T%5H(O8%2*MkeB=*Yp_Q9lNgdfxP{ z#FbZQ0a#g(Mxjq%(OGZoc0A@g^K8s-qjxvhplZA5T;Kmq+<*FM2er-u{X|+F-Ujc5 z&2}bqo2UnEC7_^(*ugTzWT9#BT?MFPq|5!O?z=l)UN0F_e9YLR0zNxzQmQvI*x@|8 z5jd0YwxOK%riv<mHG=cjk3W0vctY>XC?yVVT@Oe%gO->^sLk212s&C>9{aiv-k z&*Do}CvFCSB@Jrr1!Oramw-svAORGVZ7k>#o{Lei<`=kC3MF=FIU#UwaZ%DH7cIEw zNx?%Y8u2y?Ihv_>A}pYf-9VYm%>Xh@>29Xha-iM+Z$;412DUBB=qrx>E!U?#oCTI^ zjSwu7ixc*B1zPmpr=2w#1>mFvM+K--!)~%-ln2Fzq)o?_*6DsezcFH zDBPx-9ve{M)qH90tpY;nZZxEz0HxoOhPb-8$#eM{6=i2MH&{~n}(O})-#>lb1c&hvHmhkIi zCwf$j+d0Fpy@}-@;h}Mk^j{SIksY_s3V<-UxMW#@*4PfH*LMH$I7obz^duRmpeYx0 z{`FIzY%qGoyEx5t^5wbqMEB?82V+%Zyc?s46Rnv)at0b6t;Z&Ba^1X`-aY^f@F!^~ z_3hy0pi{2yugj%g&o4&xy17PKXN>B9lzQ|v|6FHuQKMJj+WW+P(0cCVnNyIhx{1n=!;c zJKD4iyQY~ckF(k&l~`2o<)Y(=)RuK+{ntE$%9_^?IN#|k8wo%RSmlFuWdJ7y``Mb^ z_?KL!-;I;9--Aq6v^5%B4Jk)HcDP=y4E%pZnIOX>ZUfhAc+{Hz$k70C0%v0P*@5i+ zq1iG!7f0$I!`+}W=|Pu+#|+u{I#a@1skf+4KbQE%3|U=o{rtXv#Em0UN`T>w7DfLZ zfKsboc{E)fr5uSj{78O1AwxGMa83tq%y$o&gVhy!b$i(o0~W~*mR$TqSTSGjYxnF= znoA-2fw8I(Bw79MQqvYN9~W}glEAv;2i6$=uTHqvm*V+OUXugh8!p zdy994Y}#%oE>;1svSy%cjLii8vlkm+qUYN5mv<4f>x*~&qhe3YZd=_L&>jEriJN7a z+l#)nA87x))b`e}{-ZlSQXrzlJibXfE1b=47RQ?)@(^1g#1B}X<$*BIm{~;o9}V0* zja>{EXv_Tkc8PyIv&%Sv!@T#ov?SbS)nBo`5^an#QlWhNj8x7E0XkKNV=tsa( z>`x0O!o_IA*mAzPSD}+|Dr?NMZCDOftS!UQ8_eEM*LYwdusTK=<9et84?-9w%{pQ^ zH^Y$^iB6lID~fljN3&sL9#nNfU^8*wMHC0mV~^r7@NwelGcM+|rwEs*$#RR^f!)F4 zxpR31kJi-Fi(bKI!6fCDAnZ!fSFvI$ziPi54Q9nIF@ESMNQZsCk$VuuI8yHz)GNAU+6uDtkd)Vz~w>&qkq6E zn-VI-G(H;YF5p!avC#2`HGU64zl=?UifIkb#1bbtz4H4Z0jaA;)5osQafSVO`Sv;< z(})g^O;2!)oR! zp)MfYAmU5sK$S<;aZdisp?_hqv%o<};t18bXUBRX7jpT=FYZKj@2YB8a+7UR6lna~qv3A@Zl zr$V7Lhd4s!gvdFWL9Ol1NQz~JV#_k6Ux+A;-z~9f_Kl9N(6gT`LEJ2KG5sQamoUvO zX2=;nta0Xw!D?oD=$VcU)Z@kA%Wb!*T4rK%5O~S1y+wo}mDE~@A6KB1@5+!_)&d{q z!P873p-=8>lHKfbN0IfL?v~n4mMeH06;_+sLaPmh2(R&lqMT zMtKY&OTT(h!ji_a7KsZ9f5LpGAYCCmT*-pAkX>acJ!EOzSYIZv73g9@C)HOGD1n$M z#m58?nzU^|_Z=W}1c#Kd7L~ zVfO}ae7rGo0c3p7J_Pcf@+UK_MCGi2)?%9@FHA)doPs1^$a^;|JTt_FCgo+wIA?qv z^@ag?r*z?ZGw6~<7{H~3x! z1Xmq})J)|ehz({FD*X`4Y;Izg6e{&e#z0h7E%09xh$zR*b^&d-}yX?6C}V)XUFNS=d8Z{hpcvhr^`K)IaGR$S`mu# z9)t6WLlLu9i?zvn+m7mi@Qtjc*Z=C#MYzp_B|`dLxGoZnHV@ zQ&C>2sluJ|`-;tmb9^`S4_P?deIqzI*b}rSWIMchKZ(y_)zWBK<*7tj{JMc3!RL`5`=Oha4~WsIBkm8(e>c#uke6&f_MpVLneDo>#x_HCGs#C2`#e0KK?%| zH3H#lIrmRwo`a*`CxC{;KFr{BeRwUmK%Pfn67; zX~4~dhW{~68YHD9w%Bj3RRIaNReQ|zHEfa{@#2xUVvS@PU6$0@qy1nef5K!^VJLWU zuN*MZE&mIM49BYKz99{v3$D_=S=gi>yYxa7L2Rhm5(L&)uSPN0I5_j!?}OPvsC+gf z`ZGq#OhpM=`f3g1Ee7dI)ttc6+}@Xiz86O-GSEf~e0(~_lH%hyW63kwOV=y=^j+ z>e81Y!cZ*eQxUD2clXGc&3~56{zk+?p?T`x(o1C%ht@pp?stMc2mfkL_ycGB4X=V_ zhC%TP(9RODG1(6DtTKpBKB26{bFyu!1Mwh$Anc3GbAs|~-+Pd7OOzIZ?wF)6maBy$ znqn`+mKFtx2X*0>aDyq;7ExwuHR&OVAB2$)%$008-^STvVp<6jA<*CqG6Um{U{kA< zIR30B>d(mb=aFf)Qb_;Z{RP1sH|h$F#I@bOZ;h$I3Tv@vGNI|%iu0n~hhjq{vjmin z|1p&$1w55_4GsAz_Z}&*4u5+|gPk4iM-Nf%(&qHOTlxph7DvGGZQq65Ndn!^gq~(& z__YLYZ#%XB;c5Rs(zrX4ifrOvEM?!q`1}|0%YSe8H0%1aE~Xsa=FKJb$fLM6@8SD8 zl^8dYoXQ(Jp3~9Jiz+jqUstFGkqC8ig#l)ZQzw}~#|JSu^W+uc|C5Nq1}O)lu)-Em z5Gd_w7E)^5Ytbck1>zQj63x|V$Xe9)C@Jt0DDCV#TCl?^qn$%Bh z${PQW;QDntwwZ%XbC8~;L)@G~YMqr*@5y4uyHsD4J zS2$$+4rh&H%Q64dn@|RuvGk23Yi;f_7?R)kv#J;oB&B!)afUWBK6Tp3WxmKOXp_nC z)o2WOo>7Aey!3|bG?H>BSUeYYk2-6-pFM2xI5$hXgF^N2i6&$T`2zY{-HU`{@qzi@ z#*io>9=Ue3A{@Pk1=KE4w>k%D% z8JE)<(hoC1ovs0Va!~%ialmEXPc$W(NY&cqO`{re!g220qseYRc`1ebRFkS8LQldM za_3&n^SeGP?w{Xh-SP=2LR5AW+MEdkSycN zb?m;^6NPL;fgp#O!bfe(!n&_gh{7tI9dHnY9O-~gyb0H8Xt z!tK4of44LNQA%MeLJ5r(g&_Y8zxi;LA7L?}S-6c-Qvbtc{0FWFMFVjCS5i!5|4njm h1Jr>J|5(L&=}F1v()Mv5nLEG_OhHY)7-ACee*ldSo3{V} literal 34837 zcmdqIcU03|w=OC#A_&q11gRn-pj7Fhy-1TTMd@7x1Oz1XD!oUF^ddF%BE1HQfKmdX zg&KMZE%Xk#L4Cjd-E;Oici%h4-Q$kSACLhgYp%Iwd7fv@U-(Nk1!6*4!W%bk5G%fr z)x2>7*Yw5>oCgHAZrr#r^onf$#tr`)in7w$?#7!Lc*!i?DIs2PV*&oM6sP_g+qcHp z7uZ*Q*e_e8U^RnSRlG6;S z#JsDXY0!9c0QD5bZ{xy_cBG|kgwH&riB5^mMAIk_76Znr+3p0!iDZ2R%5$`>4rz*X zgxfw=f7A(V`^xrbQ(ND}Ofj**=)P&K!Gr)`JwT}O5%qzEW7`GB0EByFuPOT1k+o)@ zx}Wdpkl+J+m0#WnBk2+|g;cS{EsQ|39(C?xo>;pt+jI{Gtkiydl`LixGyF}vjZYEI zQH*yCDEKm=(qL7RXGKC26Jwd1w{gKC?aqjAr!CDt)&4{!i4p_m^jz?d-A)b&Npbw` zY+0U{y*Rk?&Mi=0FEU9#=>f_EseEUQ@u0n8-D10Eq159}_ozEYF9)yid@?A9kLS|K zjugpmO(k_`h){h9$rD*P0Og&`U^v%M0ryB2+VA4ECIe@lp2qXE0OkpYn%Q@!n$1{Bu+mw>lGMhLBT`(CWc;?G zj&UYc&_0GgZWTh>YBC6rrvGDd;K+ye*PLq(io4>`&I5*cmvxje)_vy#v+7S2)M2+_ zmu}J!Zl9x=TeFLe)s#oz2X~I11hdf7B@1x?-btpT;=HA@{IVz&rf!4=VWJ(=+d1%j zGyw8RNelBACzUS)l~=j4agW|gFXC<`Q)W=ZVufOI*m`jmn~e!O%0#G<_#|ag$0mwN z(Ss$5h!w^$oGC7jm^XKi-V7}pZW|HGWIgT_-I)GtRJz1-8_ea+w|i#t1oP*hQusLlvB#`( zV9noI2qzku#M_-E^sM|M*i#kB>jT<9PTf&Zny*NRapPvl9sG#9K!()9->ed0cJ;kcYg8Q0sTa>(>S_4GjyL)&}u!WN2e;GUj36kD!hLI$eSDoufEAz7O%~ zS5w}KIa7S?31bKrxQz+-Pi&&}NN92enejh&TpM)nRK!cjhi99L=BJqwvlhvvfFe)9?K5~K z@`laW4bM6;~l#BpV zB!|UPzZe@?xx*gG_{b1nt}?TT=RRI{6w(o$_Bm$a(D5+O_RVcReLRC(W*bQz#j)Ku zQ24IXh4rgMT%lTEU)b=DCXFN7SAuaCPbBW@%u#Kv&$DTgoXz z_`d<4&xV;`(@KdiX8|G%Y=;qc9CHNE(*z9@w&Hfxp(Ohp&iEdqCU-vxnv-pJgzYcv zrn!Hq%CC$LsEm&{pUh^VxeG9rQi)*E%GOn09`G}4y7iU5(lLTH*Hk6)yUq8My?LZr z2YgS9&M%P?(ZCg_9a9(0(@88me?z{tlEo~F{wo~cVA?B`79R#-iPm^Ih9HLt{^jadZ4;*_4}l6TK*MrgJ;Ym(X<0A7`V|u_5{DhkzW?qtp5m zB=oc?r-__CouGjEE|XQLi?Oe4FSHPDshGqPQvL0`Tuo^qi9VhY#vFt1^xW#%P4LfI zt;F%U#g@De8(tzJs0fO3=)(CC*6D73Y7r!?Eb3l2ot_J+M9j_$51epvy!X7jZTMr2 z$>}-x4mWqOR1)Pr*%${yywGg-q=@_?joF_Ld*D1pR1HEKf;QYNn7~ zrFi6vbYKoI$Ji(4z+CK)9kbypg@-|8IkRzB=nBcWJYSZc8m=Y5u<|5xj;yAnJ@%{w z^;22NSFc}TopMH@(aEFeqnBV~@WYnOh^JFi(uHSPLy{wFFWkGhv>0pP0b5RlN!;02 z_TjouhzCu^5&7Qh#yz>-nv%;Dr`me2us&QbY^?nkPr%UXd3EXixDpL48Tf@)axAVkz=h7nd4- zr@M5hr=V#{)Rxt@3g)uJE$8G?nyJmvD_oJYS!XY_6Qz8Xv)8=*E>X%Tt|Zln z@(P{k6^gTsB!@ipS)W&wnfbH6n9m0V%4*J8epkEua18CbgOmq<6LMoQY1_@i5(-aK zpUJ%1Y_*zRWokLHJ{z}Bl6P9ya9GkP#GD;n9%NXdl}x;M&JIEyBpEXXh7e5RNl2gm z#_fHyRMx`YDwBy0k5(4sO84yB{;~YLao#1;*-c?Sx-iz;({E2=+ap(H+*$AJTzyul zLgZZJ_%tc2Wo<9;nb{(9mdR4ML{ffM+d0s4^_5lg>S?S2cX-ddl;4%K=Wbr^5ay~A zA71P6?x^gP-|+*~AX(?+ifr{jbE z&;hyKmZJ&FjZ~r2pJM8b-lz3htqd6(MV@i$epm1fRB8!lmYS4jLbFL`eYLvCqX;XP zk*j7=wy=SEW8V;EuMm`cGhnDoVyp0S`ZQeZ{rwCgIlT$L`mO2xrjw3+ux*9V*bE|S z7{r~VSCLD)rUWtm+*42XJj_=~d-qtu1HVsbjOPB7`;6dFidA~sTlHNxMFBONs0qi4 zNHwV*(ImeU^9V8DgE8fd+|%VLbvdW=+{)@-={k`6^VlejB+Zo;@p2^@&K6@Ez4HlE z-Ty3??lvLxel3lFwW(+OHpLKCqMmpYw{ijw{nPWywDS4_XF{m)1%eSGXaxLr>N!mT z@=R3^iw(9HHW=_xDC=sYhArHUv7m~{6@K;Xy-3>FdDwY0o@1S%cVv&aVq zI+Y}DYxQr3E;3CYW*pff#fzZ{vC#f?wbfFQ>a&Q<`M;inn<4C z#k{tg3LEaM#m4^2tk$zn)l~yBo4x!1&yldc)dZ^`Si?4b6g+PCKk=5kKGK*|cn0MJ4r)L!PpFuWv`0azy zkewQqUajnwHIKLNh*ndcuYl`$?ISeGMqe8WwXRMsXtO-^EQN;TllBfsot-U z1Nty-HH^xmT+_;!f|=%lu z+`O}f8Z2>{o;_*NSvnH^V5!SBLE(jVcInJ6$&TAOo9BzKMNOmj25)pvR5|Szf<4~I z8Oiwa4KajRIHWVv&D!QKC#s+u_Dc7h97Ii-THQ?m!*_-oQQk*}>Lc;qO_x3V!}#x8 zD!rqeVTE6^-lybCxQ#!Qm>&#ZwxF-Xwv1-jQ&_N5O1Z>&DSsom?D^)n`?M6~{tx? zTd#&ZR?_=bHM03sJ$=_{Y+s8!PB$K9r8XFhCK&AXoT9R7x^)z=-=D!)vw8%_bTpC6 zYcaN+E=24w)J+fm>_US&V81$!B)nE3{wihxIMraRl z9twc=!ZftxHECw*R>^WDRKhK4;Gu_!xFPK7$Hs2Plt>D}RCS=Sk?)vOIZ62uWL||| zbvK%=8PM8-d$g3H)1Zc9l9g*gItg|8#2a9^acc4CQ&EU(Sw1z(;)UNhG!*_(L9g1W znPQgWwf1$ZrPk=2Wm;yar9=J@2>T(Fw9(>xprA z@%2^|?1q$2P3k*QQ$)I7+&Rp#hdCEt#9tyKWPI40@N+kgg?7@RfEf)5jhbGA>u}_1 zhpz%VJ@0mFYAJ3AOUq`Q(SWhUdbD`YU6;RHMyH+oUS|eaxh#uzH=nWG*-*u|+fW@- zJf50K;YYzssgewy z5gwZZ_>M#3SE0$YD!Ryii54YGZ31RlDc0hX^IUJ-WEUY3F%>Eci5n~=X1KvjiVGR; z(Uid<8q-Xlkk_w2}+?jTS7rhT{*=fdZ z`@nx+B5EOc{$f@@-u~i>{R$It*EntMOx&EGnzr?NfbEZZ5kGtw7w8jEc^2T{BiFc4e2=F*e?d3Qa%CUcCkMk=7aLm^Im1`cO4|2#*fnYb~vWbN{9}p{mItDGhB^&GbfsTXIvyK=FXT+uo)6(kzZ|$amry^ za?yRhZ@(poghuNpR2-VJ@t);=LHRx=cv#{7Wr5^zTJ93nEX9b{bxhvBj>*r0c)Mj_ zwBrUrviurU7c3_}rU(CWyR(oc_jNicxb8uEu|COoP(Ol4%+y=Bb>iqe)#5+(y@^d@Z%OCuu?nC{IzmxtU|V~RQKh87-Ex3Yw~Lix_nBOF6?^q zV$cgc_bU&hbZiDGhN8DzEo$Z^G#r*GSU!x(KSi~hYCpY0#K5~368oB7tAJ5+n}C#| zvZ|L`Bi3r9--_PtQ(Egw`$wmz=M_)8iJ_+%w-J@g8XQK~Eh*u;CG~u0C$oJ*aI7Vt zywc6L(bC}_r84h5KgSzqhLG8@*ka)I?VETeDy}ZmgXL_mU2*ZpviQ!rj)&9Kd3fdp z$ks&jcemXhmSCI47_*V%IyseDRKrBqTR*C!63zudtlwD1hV|$i?Y7oC#R@iS{x&A| z$EMu9uH_l!>7*GoNo_&=C}Yn8>7l|q(23iz=o*{lSL*v5+Kl*7t8g)VV-Nbucf|;t z+*o6K%4)Ggc}5TTM8}HJr5VYkjg(9|HtZp@3BS=XqROI?iKz{T=~FcD^_oHt)~o?S~27G z=1-43N9a73f?57%Fg4M8sXD^p+GFGZFij}Qe6MCc8t-lHCN1B55!r*flMAb3AT|bm ziB%I>76P`D<%VpMM0o4Ekf=3Jn?p+fftz?=qj$B$yP z?O4;Zd=xcs!03g42Ml8J7~>rbD*&g3e8kLm&FA94Mp%Skrapf;T~Ny2%i4#0@@~8JHN0ulW&}BHi~e{Gd~3KinQ;@^-k;2QESmaV#tGlMO&Qj?4gf#JeCI z84SmJ+*0*4x)(k@+uMYe;;ty9ro3K*?unQ#;j_d<8Sm%V0h&)#d@@FHoM(dB6gFu0 z3*THzq`$LsAv87ifz&(8&cBX}Kr@<>1>ny%;ZgKKX!KrAXl56`md1D$Nhl}q!;4Uj zxbt$U*$k^C=ypJ->%OC9g^0f=ykZKCVNQ>9)c#6-D3G;gXYNdtS-*2t9G-G> zQ-2Lq<>1cX@(5$wu*i<~uRa6{Ix)BipxJYwUnZM)0XXi6Tw=7^zY}lG%daHKBa(k8 zyiavde82S*vHS_=eo8UkBcOhU&eQ#H4udH6YS*qz+X9-?J+Z|@*|NS|PWZ>~sqE)N zC-QP&=`xKj&(UJ<4!%{7>I{Q2N|v~&xK|q|C9MB00~GoG#5vy%`{Thh4%)9O=U?ah z@3Ppb`1PcE8i4dO)Jui0L4D57+i>`uF}-I><^#*GO7_{6RWw?57lw0oG9$LuJv!xU zV<$5UKcC9pUoWe*l^{<$JhACMjQDruqS)l1?wNgGh;kYAAIU&itCYN#->9no`^c1h z2G7X4xA{S&s+^sB_4k8*9^?<)o`1q2U$+OmoiEx5C5_;b6gtlDKsgDorjzHi(9Ep6=~!UkJCY-7*& zlt|CxKJXaJ?d|kaE6lejlnEn7m#O|u2l6u+cEU%eOpu*(M;-?T^W5k!Sxz$UYjr#f z7vl!OUQwV6&Aishp`ne`GCa*oIVgV|+-a{s-^12&H-*ci^NNO~jJ>M|=L|lIe?+O- zC-%(~p4)gZp>+Fk>P$Z*`$f($=mW+8c}#g&F!@;>NWb&J#+dW{!0yo0wFE&ZWqN#j z+qa}OsYvgg4kNRx(^4?H<;zrN^V|qa4^PUmc_4xDvjL)6`-`_Fqo=Z=crFY)X2vd? z&$c@c*cjT}C>G;>%{6m)F9g-ZPDQ@BRU%NB8$rLcKH6KgTwL#p9^Vcq73c8&S>R4YW9(d9)f=wCvu4}UOA z%Vb0^Kp``v+NJ#~ZJ$6w{gmD1wS0->BN@-~JMaWxTAUlLNV|stra(qV$vfCT*A7vT zWUSG`rXAQsOr?B|ZB|wI0Byr#bP6Oi8LA^zSIM41o*>U4t7!m3*-Np#pOHmL7ieff zMwf)T zgpvDKTao?is}kd@^Al#Pksi$26h|0Vpc*+SInLr@GMMp$;=v~J%zAc3*%;Spjpq-f zpWXvGGwBiaww}yhVorCW5ItHQ&q4(9Z%xijlH+iTGJI<0~N)u zB`i3DMNREY`JupEqcgh?6q(R(Ihn`Z2JZzVCEj7oFS)L7>}C0}fUxah!71Mp$`E*a zCm$hH7sG2k_%}iXibDr@^P)4MCAh0^kv?^Yw%~D`u!z zaWNRix5MxhyUT$$*K439bA!SSFSOjKX9`>4hSaiPt=u4xv75`C8%|gb#Y^{ zQWqR;QTLVC}8Z^p8SC()5NKfb%vJTZ!<$&(hI+D`*Fkf9HK-p?%1M#6?Y_h zMXYQMhYc1FG!@w#9O(E8HEzJCQ)#18bdSVmRS{8(<~mC-Uf) z@!PRpbwnc*x8X4&2duGdpXFK}!w>uK#&8o$@YK~aK69@;?<>r==dI@EjRL7Qn2a%; zAnGEPtR!3_G>=fzg_iu4AE??GW7^@q9U;=)1k4(E-0nn zUL)uQi{udgZWb8eXr-_hgI>l|#xQKme!47gd$@;8*wvRs)GN6XK2-Z`a-5K_mJ(oX z34Rblyu#_yLn-@I3@?_WmvjcGK{-&ORq#Obc=&d(^8lX7nw8FSJBR+Yn^VEgZD0~Cn%e~a zD0hn*sruXs8UWr~_(ak@>MH%_O^~}UwZc@k`yOjRN&ZbgP_0Y*Oc&F{OY` zoLq}&R7%tv`>)%y@0TJ0DX1Xe%PJv+TY{zHuP*tP?A3*jVhe6m8S;`3TmEX20b$jh zlZBPcU7+0!_33*)q@EinvFR>G5sCT745bW2^-T&);(iqdq4OZ8H{+I(14tYTTkEm) z%D`-&ihEKg4pPow<;e3Ap}*v5g|=RL^UQvm#Pfnhc&%V8^zVs1n%oZ)q{kmn<=;V^BUszv3zfO z_1!8xvJ%RsY&Rss@CKSz=DI|Yd!b@<0U$0*puPTQkBO|Nr&vr2Ltxbm+YU6=|ubYb61hPpr+QWtwsXqr>rLj0cGy zx7^>vPQ>*LMdLAO4;FsdeCYhh$f-*wbKNCTxoJp$I-Mei5vIQIY}>_BT8vVlkiPjx zL*7%E;fkR{wa?J!F;i!5Qfk&+%(BZnX(FJUJ>Zh5Dra?GuOwwR~Ptb z5P#xVa|S4~tDGY^brz^e{kfv3qrZ{r{Coroif^zpXie63{kln)Zt#r2;TH8HBOCZ_ zZ5|QwBh2kXCOmu1m>I)u#R!`#WM*Lpmm;4e!*gC46ivo93)M*-<&ibnYtkv!PLkT0 zfJ!IDU9qax>(kf2BXxF#&&clVy3UEnulD51S-hS3k`O+bdtkZR~vELwuD3 zRqSm{cfe!T&eh4JLkFeR!EgB|gVYR_Qf!|Gc1e}z*GTb*_o)S9J?Sg)h|XpMu8ar6 zG<5~>H@^iGPy%0*B_@52D@Wg9sKmkZ(IcFFKtPRMU6!bU7j^bg5|*ATJ?EE)`TTwO z)d@M1*5GyO zfE{rXnqkVM7ZF;KO@Y89#jjxZ!|yn0WnZfB`m-p(bhmM^vVBGHvp|Y=k92%>HE6c^19I5=zpPZp^@s9q7j6!^9+Mxn>Ren+CqRfzdva0QB8p)ZXC`dDE{a$q9!DuTZ^bZW18L0-Tt1MLamcT|yn%J$h- zV2ux~GlOZ!g|K$Q`H5nTXpS*)3r+a6{+WX~xmA=gJ@B;trb04Wm-zUegIUD|cRlQ| zIZ7rznE+G(RKL70hebcvEz`0 z$7ch14T)t`tyc@ps$BSYR#A}n1rLC^_$WPZrXPLa?9J82tJp`)EFV`NddI-&6qlR2 zII${rZ|FSzh(6uxejMC&E{ud2-n>YFF!cdw?gDFa?Q6XZ>LvUE!xO)jR7!@dPc0-g zFh&;F@2nu~=T??-gpCKEakqdmiV=0~9JmESA*-#0VrI2Ne1}fD)YnR3PB^drf?tA@ z5_XW1Kt;=&3-y6yP~S)+em}sGmSMi83lvwb!H@QbvtR}Se6~+_UvvcTV3&smI@11a7Pt?I>me4S+1-nlq%FT7 zE!EgXc!u@^OfRI68dq{df=^!LhI6O{4gZ~o6E?h7(ON*M4&+ASKy4=Z$>{o9^Z>_6 zd3z~EEB+FcLHjM|&rrR~=~x3>1Dq2y#vZ^2S5I^l;S)DD{Nd!^s@r!H9!)V5V*h)- zMpd<`Z3>|%l3^B##~^coxuWTQVn`awvNWK9*zr5F*z=DeSI1r}sZn4UqS`$6u$NJe zZH&vljO=UqdA0)kjhZdn^t^jxtB0_B-s@s4aDNsz0>-!cPI_@`CU3lbH=0UlZ)a9+ zfPY5@?KN&Ae$t?j)G^qV&x`iYc2+iX)FG2J;n<8C;)}*Fkm}%6M94p^$4>u|158IL zk+^R}O;zOfNGOT_hS+rPDLT#Eo)-}JA(bR%>2Q=yd3Mu6sXJ)6J$MkuA_*`Xwju_F z2Qmb@n=QI&O1n`4|0a73h~E~PpF5rYO5gkfD}Vr)nYx-3T{i=b;<>vcU_|Y)9k-dI z;7EA>PNm_n6HYd6o&3;ftPEHw6BUibAwu}ifATgWRQSHjAQ2z1vlzcnC^fKZDJ_!m zjMrgk`3vu)ECcQ|7`-#!c+8KFb&v0x7#K}~xEJ+;3Wim_ZndG z)U8b7+D&0Ae^*puoZjXYv(0Ijy&MczM7XIw$(UjzfKKcWx*G@<*_j5DBs;AAeZn9j zGzZ?{>*rbO(-djFtR=v5wKxdL5)|%o74T>Zhls8iJ1!}n9j_=k%$|>*VR3~Z+Z#Am zk{N$_7(+k#FbsF&=IT`{C7lWWy&6XvMW6r@XP>3q7{VQV?(B6&6laH3JFxSZO|HbN z7T-h*T0;(d7^TH^`HKu?r~syIrTghi8FyD1D&LtW#MH(zW-&40oB2ccdWS^Qx!PlQ zXDMQU-}JtTj@3TL21=~Wf{%U2KETu>Zh=6;40wZ2HNI#zs?wmzxxL_Vr?c-I@5ve?#3pyr>L)r=)4sAaaqKbtGepqF84Yn;;w+}o|&7A3ZfzgesFK3UhrvI+hU_Go8&gj z=mXoEcdCBkv9zn`9Pv&rDby6=a{30*Dv-VBWLVrh(XbmB%XhX;)6~V@P=MB_Y~no_ zcpGXc-*#$mIm%XxVM|xEv%}=C(L!qyffpBN__WV_pQ!%$saWOIRl^8T3qpAq%0QQX z=A8t*ln2`_&L=$RQ*SH@acfTwpu8)z(t^7pjAMGRe)NP^FCtA2TMRp5S)~N|u_NK> zq}3!VkH3jtg^C7P&}Ll618o$N`4I%lp};uQ#8N~*`gdvTF43Nxm1Zwa;GoCe@JB3K zEXrT3Q0``Kll&Vfu=ByvPerd>Zp|NFwdaZAx2?VaRO!3p_W}6zUP4Ya*1Y;fljA4{ z`0n0P>D@nmyAfY$JXu4Th``y8VX!rnS$AspC%Gq&+G2lT`z?{Z69vQZ65`0TAWr$! zly6GSUExnw^zKxa{{RW`4K35{`}Pmb(=S9yU~d;QKM+Q(jNc65op+BlDcQ6fL;9uzBX4Hc8dL>#jL{c9l5f zwB}#M7)<_REY7KKk_OE_kV)wlm97i8HMHP$ap?LkZ;ccCQ615OW8ByC%{rfBP&ap? zy}lUwX#*6x2`u9|Jg{TLT&%+vZHf`Qm+y-H`XO(I+)JA?77wF78%ah`_Ok&QnKs1_&udi1D1cC6Sz6R_MJuru6$GWFiiv|nt@Pe{3PU}$_ehJh#&!Bcb7Zd>xWOb z6 zT4O086x|^j4X1%3qau>1Gqvww6I&mxDLt(Q_#K9*vgGxCc;jVYv8AvRj|#WIbnw+*gY@M}obus>UaSNVoLLm>f#T1>~UP=w!nqo7unb&nM zbn_M&FczeVen>KmUN@V{({=Dkp+pno$8q%O@kaC&P^)A-;vS%dS&Mk$0s9 z*%aZsW+Od;)k~Pw9flxZpk$OX<5!K_F4V_C#>x>c=E;!1*^(>L3VB+!`ctL0KWbd? zqJw|a!yFSQM@^vbxJQ+dVJsyYXZ6q1Cd_ACDn@U7Hl^ZEsP2`uLyn^ksGqH3&*3`K za%Sh<3@nv^C1!+=ZA{*)|KXXkenBoecCz2!7nI#v1=V`nU0fgB{~IJU2hHm^*DghU zl@~XnMNBKO!=p%Y>kiEh_|f9o+am3{-v!k{!j2nHz2n&M(mr`X?pg0SW+7b4HGzuahTlOJ*XoaZTf|7&nDA-uMq5;Z|b zCa5TNdJ0jTCzcMqa&1 z8&~UM2>%dvo;zdBFwxk3ftFfoWFKUban+4RMl85Su*=lFe8t>UC)Rp((T}o6WMoR7 z|M-l;TrFR&d2_{Pw&G~S7k)_iA6BA$q$KkWQW+_jV0}B5EANP?Sy#s8t_HML>!1SS z^NG8rT^uDad`A@x*=kwyP8_3ye7GK@1@^T#CkkSxq<~%b<-r3Ip0BX^{Ldb1f#W!0 z1^fd7K21|GX$AW*2`|9=eySnN#pRSn%e#jXUbs1rHI~_bZ%{f%;PXU*#lr+(+U|d_f6NrIpo?wX zVX-NVE9rs~7ws82&X-0B%m#(UWJU(6_*-J%mR)FEhxokmOG+EGDeF_O!@p2bTU6=S z{@#6JCoa0|;2wKPd^i91i7V9v{kZqrDnNMQ{N(Mk^9mC;t=mjZ`+vE$Yz{>>a5xBM zz7oX0vnA}525hvv7v6E+{Lks9j_G-I9(=lVD}dTi2a^I~1%5xtFsr;3*`?tQCd6g{HWcL(kymhZx+;?|O2W;15Dre*ih8YU84Vn^ zcU3`qc)kPLb7>gYP5UCg9*N9_v8%ED^F~3gmJ?0X;jtu2GRyVbD3z*f=bXWIME zNXJ<%xJxZ?wz>km@xCSaJcd|!R>)^NV<1-~gB@D|SoNR3iR5fO?Q}3q@j3WRI=zdd zkddH=o;mNch1ZTAz=i#kL6Tc7z*yEqwIpM+{DUQD9?$}?7@NS_Y~+H5=gn;=b7odR zAr}HS{x>)6f-R|?oyNbgh`Y^6PBrde#~t2wSJl2Msd!qZ z%LFm%?bLQ(q@}Yk%;aLO7dY4cu{_FAqG>6_E5|*9eQ%Ps7+V;MZAR8mL%j|j?LLxc zBhJGfU7T~9QLx5^0${hq@oI3Oe zZe-BCQQy#hXj&on6Q5#Yx`C^$qjAX1{FG1OCLx+n-{ruDkUUsu*0@lkIDKhT z0K;~bBP+h1PUze$k7ef%BJ}>*{(wm|j2Wv}agO{Thh_{PHD|TbbfRy5-eV?(*VcX) zgSaaSiD{e1Umi~i(bkr-{zn{b_*bn2x?d?2j1kaCsW}gbbzPqP#3`Zlu-+VD*Pzww z+3-KTbt~6^9zlyUWm;`-XyI282vR#1TL=Q4-M7M)LmLpgFCCD=i|N5h#n%;90_K-@3+;yxh1h>ul?y6v)*85L;F|hF z#sVCJJ<t9lYeVY=v+`os1Y z6#SuSjKA5oiget~^l7$go>mpJYcNQE4;52g%C6GNm9^2U8zPL4PoBY(7@|Hh@=e_l8)7Yb=T}`ZaocFK>}HTPbds z9E!^l>nIj0{6SY!D*s^GK27Y?F4fERRNy2~$dO8Q-)lIu`!7p~KLmRPU8VUiQ+si8 z&9=8xWPECC5y}ct^vm+!E?n$tzOB$6oqw7YjY4~`1-ng)EsjlC@Be!yd?lfYmT|BW zQu&r`E+Cfv*0aY$Cyy=D$L7FjtxmxN*xXV@j;|G4pIm9%^!?e>!f&Vh1><^4ELi_z zXKaK%-`;3_h0sTgdP$O)8=MlAH%gpI3haJinIVB-hsCx&A;eXh)+Ya~c#U}fC+c$z z7~h6-Pz$7AZim* zr&z{wF@D;v!mtc^sO#b=?Mf;2)N$qFv{?3dOlHVWF{v?jwm}MfG8{59&#?SEF|jsVCUmKZ zN8@oZef}@@_AeUsAH?z+RucWD85yzyA0GUtDc`g(hCkeyWGqRTb+Ju4EV`PBxU5}W zhd;3`k2m(|X3=nGmNMg`3rg?CKc^Q-qEjqW%PxJI7Wg#eWa24e@xG1XG9g8Z(5 z%ZZV?T6lSpLQ8ui7~L`7s@)>h*QOIeo%z0IIf&iYgbW)#zdE!81_?@+*s7bf{%P4= zkTm1%M`;XlAH9Ko@pay#X5Sb;|7mGUlb4S)4J^{GyS(WwC4`6IUzl#hR-AOi*-Yf{ z!Td<%QL5*6qV$1=r89PM(AX64=`m8(0G;vS>ND!G0h+86O>`aU*Kp;3rsUV1WjAi- zzEvtKq2F0Qx>K>9yk^L+`d3)c~tsBOHz6dYV1$uHT!b z#4o6VB^vG^I;Tr7(DU7Aht3r(tb`Q`2Iv|EkeHMZ@|~BPgBOE0P)-E>B+aYsH21_u z`a-HpEk{tc?#JCjTBoAb4##!rjw0$U?0O{f#_!)l_pK+YN*cW8Sl7%>V%f6d%UjMG zuKkTs|KC;o;%*!-6f~;=`A)vy*d5bNwTs2&%B$=%2kq6{`4GuzF;h_MnVIL$=;MNc zmczikMo`;Ti7l@_xSa^!F5L3<3#&|*BjCD~6i+Bt=slJ=4zc|qA-TL5;VA!?<8vXY*ga%$Pq(Z9NR1Rv5ezMj{^4Ij3}sTUxueei(JhkPv?9- z3F(Yd_g#%-XXv?Mxr-8(jD#K5{m}Y-TAC6h;|5X}_MVHE>dn-}``#f=Ra87CD;`;U zt-n`-{9MJ)@@rlIWM&IPPFxa%I&ds%dE z9T7y7UIIvyUPA{#=>aLyi-3Sg4M^`DDG3mI2_^IZA)$oM-RL-He&^0_&N^qUv(~-- zCoB8g`z!k`&-1+RZd#dAafV0%SnX(#o#X>7PEpo*r#rgH3!a5B;AROC^i5&)TJT@i zwG9R0sGrsm;3Asc`SJrNt3Dx7Rajsm=&$&>>9jX|IUNL zikrzx5ch~p8+09!q}o=H)IFC~j(_J8fT+u5?K8;$y=mguv)^>wAlQ|eriTN0zpIht zPKtLpk$Q`nrg*_{gYr!L40j&k8QJ8J5yV!?!0{Jral%j12uxw|s{8_AM z_mNxuZBHQg#wWL8OM8+LLzFF&#n3JWMqRviJE zWq)-D52xw6v_rl_=-ezKiJ?Dxls^vw5RB34I@WMG8rF%f%%LZ%bXdLXbup@nlyJ!h zgIu9JRaWwKwBhsQOk#b`cB6+6#6TTsZjoH@7!w1``zv%iEGq}@5hc9_vRP4r8))fN zvH`r>NiKe`V3B^#ZDwC~^GkH*;qtS{oOWeqAk6-I8#n7q#w6VNYChe32FEN0I=aOD zZTad7b3}t~z0HK)o!F(^qUzylnB3Nq%hVFZhzHD!shCV;BKrvS0QF|a9$f=>Q)pZs zLUq_I_gVbN&EE%J8ExuFUQOnh?tTnEauVu^KMO=R;#of*Uy-dpN)cwL^KUtR)!_$TyO^fU(>AmB_K_gLXMTO`QUtjOEIZI84iw~{@G!(dT;a;D5A z_IqJ$?LugHis#0{LZujJi>LVcSZvHSCSlf!8cIOuk<1Cy;pr^<3f%0#?^C|Td*g^l zo=>5C*(L=bN~?^kSHj69XGurSEtc}Jona2(9M-Efb*ZQy@Tw}Nm=sd@|xUGXjB&4GSXm$Fi7i-*#pI% z$@e#g;OEkjC@n0U?X+86a!vgNc9zSAPurE^yabyUf#cC@*bRxKTIrGZtIn3ei|KGvqKo;;y)?kcrY=&!>%UF<0tL?^`;^l;QHG4;^y^3Cwe3v+`cg%0~ITk?DNz%r}J_uR+rdzvk3fF@b( zVoC(Jq-(04#?}(HFEO&ZwnIWno;+Cg#d-=}rqu5^<|%0%*bstDZMt<-;YI{sH17Ao zo)#6qs7B}$E8quo=6pn5t`?jnV6}t_Ueq!qgZ}R{l;n=KCmiZ3239R|p7J$hkeI1o zV0Luc9CarrUfZOCbDvF7eeCBHdoUSke#}5^=1MVwWZiKYx>89d;(RvCD?;@V9!pz- z#u_e}6SG_Vm?M&E_n|-L`46rF4lHqy-J9hsTd`p85u-5Wpl zy(&tL9$l;3j_p+YBYj131_}Fqr&ZwGpUr^+#lx+gcccr$xcfM;ZnmPc*U3w}Ud28j zCY^mbE7ebdy+XGDQ)Zl(+C>Rn(X7-W$rKT8-&cwdhA>+$70(V#+*JAHQ*y3HO^+rV zEx0vpAPF8iX@fj)*A%~FJk3RCK|C}oV-6p)`hdSjg=yJQYOuSqhxA)W$Ty?uOiV&( z)1aK?J4?ml_)fEIhuJS$l82ljy%eW4&}ehANS=}y@a4HwVir{`%z8C=jNsu2^0<=g zDF6fp=HZWEfxxPaV5y}P%cvzlL1T1cDuHQE>SF}P+H9p1&1f+duWrCRHBg`H$Ihdb zOs~Hl_Q;wZH2p6;H&9nU&g$MuPq4ML|26{*feYImJ*J0^yVR^LET%e6>Hzh#_R_N6 z!tJ$$0aQT`Y4khDKiKjQ1Lf(})VtZJ$AVb}2}$-Ja{j;tr2s8XZK?T08awN8&~3JB zhWdhS?;~%?;Ij!8#EoClNFKId5HWppr#4*hz81=nhd2yrB&z|NVp$RA<;;h;(o2bYW_sSi1O|)j^ zsy7cUghi&*H7%tho@SZRq70bH{eHH%A`Q;sv;ZKpBFpGS=;RRR& zaW0@g&y3zN3VYXQmqi=1!Zy3%UB|OAHJYoMAzTszG>y1?h0M|WGgPQX?}f;bHg7VW zLW?GsVhiKahzc#xyV3Y%=6Fpz&_M#gr0;+Y>ILNJTYvlXR@px|fT*M1UJqynpiOD= zPTz7Z4ABcS*1~kH^4McOn!brYi%sksmv0$L+=M8tg%#NOc2dnn$}$=A;Ut*ImVu!Q zNY&`9#{nQW=-b$rMoG|`L(o>QV;2~8Uv*?p!4%lRvJ6A|^uneMyv&xi6W~!#z{c$T z3^#RNa{{N!RB6y7Y;jAfPe5*JVZ=e;fzm#HhXvs*I_aqcE(H2hjGcF3SK&kZ-tdmD zfq=&aKIvmIq_sG0`q9;P4FL_ zka6r@(+qK2fY4<*`3=RoFQfZ%8A}oUm+pdT4h7$|Cf;yiG!}l>E`ixkk4r(vi?Z%! z3w>;f(6(c?;4d*jC@ML=CjQFbgSmXIP;`zsD1e?!z?-gxr)&Q$DPyJ>Z9e-}hw!!* zdowvbo4bqT29oAi%87lbtUnQOY5z&(bngeGz5>zM%=x5}hqp5mtg0IewZ#la>2!qV zHplTkvxS+n%!N%EZz*EA>NCd&mEGM+vk7s>vz^zH8fTV`x!l3@2|G*XlYvRCeL)$s z^AVhEN|y85$!&IeAfYo{>jw!U#y^UvzjJ<;|8R;_EAK&-*@X~`I{}VShWR+*MWoH+&7fVDHD3wLy=3lWd#xHE1e5Hc7Fr zV|g&#W5WP&Re=vQ!2%wQD!qzQYWd=&`1gA(6HP(!iHt4|&$Dw&IS@*1SHNV86(2^v z>>2BSV;N46k~|V|;DO7RDHkKJ0?YXaK?|1k5MY*YdR&bgy^UxgpX8KC%PJ6IjI)7$ zA6FIAcADKEK!uO8{pC<>Y=2L}XF0;}|Tclwf{<0_TOG&N#HFVY+k9WNR3toddY3GHh2hy7~dx5rul=L93B|UVVPH`A2u*E zi3?yV&ZhbY-uDefGRAcf$y6Lm`H%Bq`gim3+nu(Ya9Y2o&cWJ%ZRRPw zUMss($YH}F4uW=y{-Cgv6dfA175;&({wmkbE}zpDigKq*+g?X+T^)PylV$$NMjpkq z>@Oy;F*gir=Ga>n&M36*BE6o{E&+so)f8pjl}R&6+Ct7C=pRs?NW{EvS=fI=rO<$e zF~XH{5=_nA(PcR~;7L0LEsr%oZtOYwZN^?nnSdN`D^QUz@8nxA0ASo9Wab#>rm^)^l z!~>9A?xm#^<68bP;92KROMeE~*b{csi>{JeEMKBwRXugCiFoAb-V5VDpIJXZSy$gX zJjD&qfT>5-SzG3p8d4j)Q33$-pNv+kEivE06|^+cxT8DDt*H{ickj-bPUTD^aKbGz z^+^@G=N0#8sV|$cFp{^tGk)8^=kU5j;Jk0!FZxCbpyCD3sCY3@Dg2%aHLP28$-rC{ zzdxi>uRMJ)Yl{GAjHWvCfsXWwlmLA?1-=>|H%E<8+b%;s?@g2a+p+3mpfD#g2C2%U zX3dxdUuisS$!0s8b~S*_8F2p}YevkF2X8&TFJu;ZQePfK<+E9}qvIWJ0MS}iJO4gSrpQe|Xm%EiZ65fJIqCCizoJ%;e-@*k za|Sa0Py*Bvnp!_>p*q~cIi{*wwK(3d+T_b!5N`kwQ@w{f$7#pk-wXV{xi*oe;YqvX zNjKfmown}f6ap|hKcpXd(9~fM8KoV4Gg)##ZX=7X z;{<1$rFz~UYL`l~OST_4Sz&E(yDbs80Vr>_>nC84=Ki5LrnLIK`Le&rE*ASWrhU4` z=6z0a-cE(FztknBxrv13)0FI=NqDN?9oEh+PGlQv=n19R9MBI`Ehg|CZ{{_8_F#2T zk~j_%%3`z@Z)(%=IwC4GY$vCawZa_FlP_S#9S-#n%=pP^ulA;KfHBC}rnoXXEV&-J z;k? z&rvBye$k(Mcs8_^(ACZMr3ME?DD*a*uiV>cmAz2Nf5{~UBBG+9?PV=$B(k5vms~2c zUOc-}2d`B>DOBUDysPJ#yD({%YB@J?xJ-S_H!)j9P9|yanhnRt)y-Xb zacJq)3KXoeSQo=nHmS*;nv$b`D$Fe9`dB`vMJA91oysUv1mIsAcv$ub=h=Js@1f{1 zAP?7)SdDSK41q?PbX+91YLce`*UgF70-v~imnog*^k-T~+tEHWJbW0TgXAI)daQCl?t$!@in z7>LQ375A~7=!%tCv$)OivjG6L>G3kP?mt5>WQ@Vdj<|S~g1a9SZga?zsH|38Dm|j{ zXJ7;rm>R`In+=FwwQqC{uq!SW+BK!#20-?^KI8~r3s0O|sWsRI^3o@~{cgB;+P(9` zXcwCOAgjpRk{2mc?1pa_rcRh({Pmc+MAWX@7rU}7fB1x~@xdi+NI6evqitd~>>DXJ zT;IPMwu66=RW!T7*1;tD0<%N>%MAY=BV!y36zzY%M7u8J*LrZ>TQ|rbP&hZz9UHmd z3=gF}O9;e#I_-1f6d%AE4$7To+u}k7de49fPdGwNPlZh_=fRJRUP(jW*t5-9%;V@1 zjvlByp!2z678xligZR>_N{UIkIF@8xqvpgX%)K9(;(-)+k2Z74By`*Q7Q1)lXEF`o zGWIPOSB^0AIbe1BM!HwqBQv!F&wtyp4E?PpcHM1vK{QrW(Qw|eJ;9XIM?>Wu>Yiam z9l|1z%yDJ$c)C7x4*5s=j3h|%f>m|9=-aftni6Un)ScU-&lQ!qG?A)?=i=KE5ka2v3&(iJX z-bu2m^LJ!g_nJ5~uBJh#bha3_DDIVvEVW9=oXWmR@}s*;2ots=9|emU#APfr~*O`d%BPzqSq;kVe@&3yY5^0@7PPg$!l z6=$kQi;mNz>N~j;9t&ie3(siDHqUtel)mFqWz~dom;K7<#gWNWnBWA{LazlygY_)+Q~`K$ z>NvGeHLk2*?T4c7OjG(QYL(LDr&>@%?N5o>AQf>~fZ&yW0Dy^<*&oLb})&59iHL7uy8uO^#-TIBw55)&m*qHs^44 zUnu-a_+&@%Y|9iimj_?ECSkju$@$%T1gtm(wC zWjmS6#sN`f#gK9n(69Y`YgU7S)^jpy*K$ep>hPOeup7p2G9p@R&+q7qeWb-&^*=F3 zEK~opEn4nvZz`Ehk6(P3NfXeIt~klosbRhA5@aO(+%**rRxPU^5k}?Mw z078F6IVr;mLD0M`mDk-9Hln+$2?GG#ktA|Ka0)c+0Dn!Rz_NTufXdsh3+^ z!!_~pbsgh;T~&4AJ0(3=FSW#zfzW-*zcR}IF9~`vyP01Ez4-?&fS(5>!T>)n*H*%V z`=^BSUxdaC;vfDmvG_mb#%FB8Uox)Sxh$#yKP87k^8ZcQ@88?$fqjkXSeW$m(F&EC zZ0@Brz%AJ1?b6Eb307ybV3EAYOrsXbS7%CfAYU(O4SfV<#}q-AM%|OGSFK&8=TZzu zrVn5LRhN=#yyn@z1D0PJO--fEp4fD6eOl#)Nh8CY2}6Dc%)wwOoYZFh0}mS@viXcx z@fHF|@j;JT;oZ=LO|PnwI{QQPGO!KZjG;65TnRa(+bpzK;0O zN0oYq*;?{8eYFTzs36~7*=)*P+~7~p&uLxnbQ3be0$8nb<{gT|H#}Ce3yh!qbI?FTUap`986vlH) z($N&N)-&dTSzJU;)f}FrE_rQiBsjXjQpF`t-Exv8rQse~uhhFw`_1D;UU&AbEv`h| z>KJ4@(a;5-suZTQmW7jijl1O*vyv1cm9kVf+Ibr{Z5OZLFLadV=9u~2ppth26x|!O zzJ9mRS@(H{!Q-wBB`2CY%(xBpPV9|Fkf zD&pw*6-t-o%s#V`(vZvSJlJgwZOE%l+iFe%=*I=jZq3@qb>}`q!h^I`*i-vwD8JR) zPgsBfAeyd`lfKNJt@f`hpad{CoJ$^zm59#qr6cmLM2=c2N0`875yTs zoIsf)McC1>mw1}%;{KQf+IAEjo}f-SBEiei&v(m2^$zK1b0~Kr92azI=xWu7bB7N{ z4|=H7Zy$EK7-?x@A{TwgiIv49L|@h;*=zzDvYylND>TwDh;Y|zcA>c@x=HC8n!0WG zv}GJ;6AT&*#$av~8s9OYBh!&ysC4#dPZn3)(!Q+DQrhzl3oo8}McwD?j|R?o*j8!k z>6iB&hp#`4H1Is1@ieLnszk@n@KU%ZfVc9w3%q9uw+M)lR6!Jk@<(ck6qmQoR6pq-7fJ16M1ms*0jv zD7KZwdLlVYghL~{h5Xh=w3eK!Vb&5r z4u|_!{+>gh9rEp1_MX6mQ;*2LZNl`e!}rW@I#L~6PIMMUYlht!=MUT2e7rdSi;Y71 zVh>P~m;?yY72$c;$h`3qsW6hSa*pdGDqLdrn^h@K$3oEx<*mn_i`(DFWQbW)MSFgG z`I#~vP#>!4A}T+vzJ9$e11uphP+}_Rd}Ue)9YgV}5lu^;yCI zXc`}_xOfQqCNTNA|8YNctFkJus`yS!rKdFT8eL{|%t`jJmnm6SOnXQuV!GC`HT_jK zT;GSmrXJe9OwX2hw7BvJn`V#98Zp0Ybp1b!0&r)B%&W>xSIFp`>9O6c;NhjBo(gPY zxcuDBMPmOJj@Z+%VD?n~dl#Vq!%aHsXzTf*sAC@Aw!10XZ9V~rO71ehUaLXN5HF>q z)-ExrYvyLYzzkSCIDXdC{%ByuVR|iX-ePcs(fR=T6*Dw4=h1p`Ncq+ez2@XY?6X(? z^P~0D`)%ykt8e8pbyG9&`w>I7qv6$Gx_qI&ZhfER+~7=Pf~}Wmr9)&rcgXjl6;rZY zY1+``IW{iorN>11+CLU@9;1#ZD8GwjcZ=Z^1__r4ydq}tRysj4^83lAbqEU?z;aT- z!8PPo42Oaa&C8vDJP4;sbGk1G8k$SoKc7d{1rAu@Hc$bqNpm|BM9>}GfvMCYdq&NgSNfX}0Cn>&CBbW#L^o+0TufDeWymjp_K5+d0r1dG2Yd zy*e|WkN8g!+V%@ycR)W@p#9=2_7fqQZ|>0}vj^|AFO$(V+Zle+GM*uex4pDhJ3{b8 z+(YddoJ+b`DIE);x<#m7kP)^>Vv zydtkMa4zX9(RZlcN|CK>!n2x_qgt!NWcS~9C*sU*9Xj2{g{s_ldh$X%mQw^2L~K?( z><@4Bg${?FqY$ZdbK>p5C0blniUje{sU)J_a(xhG%RNjRnf5Y{-*Bwhuw{$* z%brD=NeCHThVHUJ)(t;`7BPfa)lPlFM=$>2oHQRTlN)60*NM0eUsY4!Q&H*l(}6@xnJJ}dr_WAKD%IfMhjqbD zGDEJAvXMNgn?KffKj_E$0v9to%wyfmF0j?p@k`utWtREo7YItWKcT%VB=rS@qXRdl zkJ0Ev6b*XR34lC_GSy^;aY`E?7<8bsX5A}ehYWvMcEeDs4HgS&DI z9y1iKXV-yQ(;##*+>!@Db->D|xc0mqBSIHNTG?vYledY!B8xJ%T&Wvj?H$%HWz(^Q zd@M!X*@%Noy+K*cQnr+^*L$dy*z){`+cO~wWB&v#j)!lO&wlE!`%a^?K!U{_!PMON znB{_nX|C~=PIBP4AJ)i*ZhN7^Eo>NV>rG_j1zwxYZ_(}FhK5G8MNbTFs;NU-yr7}i6a zj>+ncQ?_6t4uu|Y#^&>gtXM5x10kBFsqnc(_1~4Z*#Wzd+vBT>fzpJ*H!U0a)hCx~ z(@sVsaIQ){kZ%7_jJqI@N+^<{VeJ?0JXU z2K8@|jkpX*0v}2uqZbsb+4iGyq}p8)HDf<)jUvdS!9Vv@8@euD9t#sc?zH;&F%B-{ zc^KSPevC)TFqZBUeXlQjVU?j8*nWyIm$Wzo98YZ@AHADY<_O6EG{i_jp`|24uAPU z?dkA9jgtD@9_kSMBLjF0p#|1AW|=ms zG^aJWvE?0gmEGXmgz7xMqIl_X({4zqvdlqw#2Y;R&=XG~q{B|Tqa5BpX@2z5M zpkI_a8hSnb{D(XF`SyE!b%cxEWt6c>Uwd!1@%zA=|H({3!^Iuatjb=gcbdw$ddX+T z8c0aGHZX5FW#1OO216HGsBNvE-SZC@=#q1Kw34t^+QFKfRD_xG(ThTU{N)rme$E64 zS|9$L2#`Dk4!WxH{){?Fa9x8y&>NIFGu9yutvoMRgS#nLw#&1+aUV{6Z}7! zp83(}v(hP-@jGMTyHk!eRzu;c``Z;_WQL7%xuNE}4@GYsmizzmk7cd?s{EbcNhj5z zdmA=h`W(L;)AC~SfCKZ)ra}7U46v1pm=)ndlpKp-Ttw8C-L-E$c*QOiT zZ*>e`jYu*()ePm9k{YXs4KaR1PA_cUcbGk&_!RO=m3NEuqod;+{kYNvX|grmOfJb;?JPggd_e6D(7PrmaGuk;=3l_ddt`i*6h@T>)0Ui#i0pkD>2s_$7- zSKtz$uc`U()Su28iU1D){3P+}MUJyQ3z|y;W|jSj;H)7%SH}NimqmTY5&=({aq+q5 zkG4twJ8d=o+N-0V>qO;-%!}a}^5ZX>%BxJi4JBaPud0jM zG%yy(*zRCg=d)+&@>1p?7nd}x8ElgS^9`N#Yk?lDiD6Lnz@g^`?G@HaiPIbEBo_}! z@0RniS|#bk3T%`Tb^J!zt>rbm%7B`iC{*8iwchV4BCt|MA}^WFH4GiEajpAo-$A;f zL<4jMo3YQU$Ty8O&jU5M^>XJ;C&6m<%MbE~Le-yAALhUQs8yg-3p7j7Iu=LL-RD2b z7&@f~+lXpSS?_@J*n4UP-fl&uY3ecCa?d13go?r*@_U~Ki(bC*C>raWCmSzZo-XZw zym${Bzv^jFdeFB@miIB#Yf+T9zBg1E$Z@@fo9!Poa;l$xbBC>(=GijPItJf9Nlqk? zFg|u>+r`y+(^$1gxwHkVH{8{?HJ!w=Nk+e?$PjPzow{nfMr^7z#bBKY-eAt#3dJ8= zG%jCjtz5fmZP2?q(*EkG_Z=zF6y1r8Xf4v!Sk{-oLwLMAeu%HYh69PuA($AcgtcX6 ztcTSfkFL5Qj+eFAG{-A3Wje`y4_`OFqj>fdOcciXHo8Ehq#CjM0+r4ETno`@7PHKo z_wa~-fS7^y97do}<8JFn#%f55<5@hjL?WCKPR_%1h=eit4Z3qd{_zoY`y>qo&ot12$b8CA>8n8es zGhUuvi8(DS36F{KAR@%@0IejQA;x5U(uE+Vz7>^Xl47udntdJcM`NnY*|f zHNr{`g6uONXwYfk1=#!~@Ghu1EdJo$xX z)gzS>31U}EzS`SE;@wTKEJlRo`S#dexKY#Vk8L{F>y2Ri7R7Z#=_%jE4-?){wc^eh zonOLUL*eigDxDs0L5w4K$EL%*10zxtnxu@B4jUzQYK@8|S-bYx+D6xd@Xz%OiM<@A zdfjhK1Cl2taNkN2xq2aQT2t>86XAtGFoZ;jjwG9c!c~mMG-1{EJ{zmr7A@+oMi3cY zn!LnE$!j7As`xiEIow7uC#I8bOr|?}sBG`ev2+4sGDK4QtC?i-Ya;va?r^^I;x~_{ z+e>^>G^W;0|3I_9(jtY9Wq6eGal$I}VI-`o)}VG*sl>h971Fezjh$*nci-CqHQGQ| zIr?RlatPQcP9y9gAK3DF`EBm@u*gdgoJgaTSVqDud#a{he0Jnj(LSDB)y(S-s|gnb zJqR+*VJ8ZUe~{d$bbZxB>7ujD&W#-wHjU{`VmeXdIUh(;fgE?n$?4k_T}x_4{+>=T z)ap24h&7aejrTcN=vr!ZqVgkHdgfz;>B?%n2x@iJmY&u)w?*K-Ck;{94oGQ%Jte&M zT8Oa0Qrug)nkwL_LyPDU*90W)C(YbO&7Fq=NnF|9>&OlGZC%VRC#pU)F#JB!xa(*q zE8hs%YjjPea;sn=VOpCE@}0%y48T445@h;?$WxrNZEOzYU9;NZ%X{T#9rm@4!;FzA zkfp5HjGXO$)_(BZF?aN0aCQPLwY1OIo+X}+6k;ZqrvcGXBE2BzPA+}nTsidmIAifp zfw!gZ2W;P#qRfRtZ3bDSYE`PBgjW!r7QoltMRybSN13jV`9R7$%!c2xTMa z`k%COUm))=#swJ|Lk{y_LD1Z63j36|jf5;1X))a(YIsuN25@X=$<(Zi`K9UZ!p72` z@({M0D}vmz&z&Rgd-6!`CEZHcuA7)`p=~7@u%>LWW*0DTUC|TVXD5JwfC%XOy#sr1 zv`*zpNR!B!oo;yj4sizihs!Y*7vpb{(n-IZmDwJKvi1lT2HNU=`c1t-h}Tulyz%kU zPLzD&5<}it`^6-mB&V-_#E6zbDTOF8@cC@whu_VLn5;q#XG_TMPto(cv%0Z*gTrSN z`yP&^5Ev(#6`;GJeF#c^ndlj6?_pt7Jzo>qr9`QF*ypIO-HRyqcW?3W@DEUhvo$;d zee-@ju(xRT>KxHp@yxMV7#fsp^cDR~Vv~+ClPdBc8dkuc0=v)37GyYBhdM9d%xLUu zk406)OF*siJ8*|NaeYPfX=P^mT@}jH6@R!p#k^vF!fd>!iTVBeTTHOaWSy)Mb zZ|az1FfrSMFhDNDUqFh*3*`I;8j;iEOMsC69*AyIq5*LAteek(hZ4B%I{Df3W-BAV zd%skCY6S$H5PA<*dHI8Cv>2Hip+I*G{E2K&9YLOx0XKQAGaM9G{}HbPG7Vn3>v(b?R*vG$2}nd1nP(MnRACF%8)*Ct{2He3~O z%iOh+k24Vwdu>j;g8cri+_HWzO&j;5Y#W1|od(*bnVsVu2t}HlfRoB>$|Xb7W|`Mb zI(nk+BMf_ONGj0QDn`$Lqt(~QhIDkL9r9AZVUFG?8Tv_8<#v~Oj6u>u`dsVFCT=yG z#$`^n2G#6w-vW)=JI~)w+D__%{Vrvj8aX>4CCl@2^$8UyTJKw-!GaqSQWg?9)}V8_ zlUlSc+4clsRoiwox%%YiijtIp!kBkz+P=9FN=qmyEy6~Ld5qQYLw@enHX%xcQ834`b^ND!=;lUBi;VD~|ToXHuNYCB@w z5dLFEUvIFv5)&O#7A>1^Wd|Lk1{&Bgu{q8|I4?-t3l6-|Dhw=QDES48#vn#47|jlO4b*qjYxqZbo$$Si!v--z>^VOa5Qd|2qSq;%Sijk zJQ-o$@sitCw=G1@_;aVm9#vT0{4orRT_SB;v^yU1U3zrkbDwV$Px~`^Y+BnOxw|y0unkmbIw|#jI6JxQ!Nsr)!w;GxMBDBT zscIp3MMhfC(YI$Zp`5{`q4*@m3fXoF{*OxVZ6d2{s6xIGYN~+BVO;~+vd=k0VKYbM zh<#VDdW;fEf$XiIku?Jm1AleC{HG(|qve9+m$e<|1hW)>7{wk408H{{k+ld^jw&## zFkM%E&e~u;Iq0?OGFtx_&t^f#uk=*$Mi@QOMXH?;dwU^a(0#KxL^gsO#1izRp4V(X zXs^-%%hK{l>G}zP>c&ebD`73cl3_x;t;5g4Lc7DZ?P0B<-J!?6k%BlAmkVn)lio<* zeLUTr#-1rli3VExBu$@}kx0Wy!kStFoykh{Q}(QedQU|eNB_Axb`j9Gdt#zW8G&?) z-XpXa6CkQ0^s5p8-GBf1eMl})DelJ#jRc|<+%)Wf&$Cb@5m195`$BvNj{L~u5 z|K^=I+lM~HlXHnLK2zPgL_x>eL%I|?i_P@0E45F9JF|V!HxB0Y|m+ z)99jX1ZFO$nqre*CyWKBXCMf(q9pYaUwf^4qNH|fX4>d1fa!%#((#YBSJFLLqJjoA zC=#t$(vR57=etcg$$W(S6NI7|vE~6QOt(YH=uA-psMXOCB{TNqG_U*Y^g=Y9TuOmJ zyc;VD3VWoQHmz4nY-$L?eVI5G$4;3O7V{BN(m)UD?Xi%(w(WWgzz{Xjw=bwD-5_!} zTE3p*8o-&688zyVp6h-2<`TMy?{E_zbbX47k-zbJNkAB-v=MuzeFekF_utO!R&3xo zKk*DojN)^nW%W*rqq;DRd%wSoE?e#e_5k#fPbp(S%aDw?Gp8^S=GrtV*#gAcy zf<5_^sM*%mx$(+oVm=*(RU#};vDeriM;Zo!UrI~ F{{ta%lm7q! diff --git a/docs/source/assets/design/v1/prefix_caching/example-time-3.png b/docs/source/assets/design/v1/prefix_caching/example-time-3.png index 71b9e9b60ab9aae080a6e70a9594dd3374c3591b..d753a406bdb9aee1940e9e84e8b956c34f0b6b21 100644 GIT binary patch literal 51241 zcmce-cl6`b)iz3JC<8+Y5FjuU>VyE2EXy)=EUUU(k|j&PfQy=KS+XR{lIUa@AhcnC z&OOlv``KHk zG-}M;jkn);t+m#g%dsTC)>`YVSZl4dx1O^BDEYF#YXN+%?eh$=){T$vxO%O%a;1Q3 z2X>@eBW|$?gqh_C|0K z&h3^dWMS|D;rk#+Uj0}7j-ieJ-B5NP2sE%~wbQf4f#aLIWe9N3jRS22zCkhgtP9|y z4gO*1a5#H-ZXviVxb8@kwQku8c0=fFCbv%p1I2rmS+QE#4Jm@_ku}u77o+utPWnjS z>&*7C98ew3?*r|V0YydAb8JwFh9LwLWg#fL50?J$-}(Kwve3~xp7pKX(y41f$N1N@ zX+JE`Ew!l0WLKY|9jq_`qe$z7n&(?!t-e{g|MZ>y5X7$buhPC2T0Jn3wA2O5zn;&jLAQ@_ z7+nR+Ne|?(rfG&h6()`L*vWfZt*YfaDwNfxMVe7Kk1{i%=Za|2XqHP>F;c-A6L3W_ zIT6>Bz|2L2C)gT3t@4Hv;AS!Bz*89wI>Sk(ZJM~pOrT%J<-z>4HD1Ec+H z*3_APLb`CJZjFm(nQGTdRcNS)8jnIk&ta-mH|bX>vlr9@(Jyd4N6k?=3(ren=qxu@ zhuz+!7U)p9ZdG&L(O4L0gECVUXt|Zg8Q65jgMyn)Hh9J%DWw?}&NNfhE!#>x>)4 zkS*Gs;tawfW}|#YvAyCvSqct(;&E#_3ZupQf%NA;4vro`) z!fII8A`Bfol7x2oY5nC-#9l?n9EY~q- z6|@-nnVCMz4?%IZJtd@Eeai5PPE$v@-T)Q*P~gBGAGT^jn;T?1K^`4eg|S7H zXBXh1`>q|l=>(=iaS zI&^)grH;Ib-#40LJa4#_;gBd6nl;F;akCOPv?>%ig(IVs%Qo43(9wO(CSyYn`B^>g zRtf`Rg0%bA5Z9cdWlda(9a6XwS_9uz&A!~o&a81Z3kJ~YM~wpogD3gDTAwMIHgPd zEaf8-68mT-JxR7z7Q0peq?Asp6j6d5GBcMcgMx0eIE6=Oc|7VA&7wS@YDUy=mLgSi zY1QN`n4-Wbp|PBaLYSsaxrvjGKAzTOYsy)2uIZ}q2r^V}QfVr1x37D?7`3u_m_moy zOw*c5ZrvqyVb=90Y{wU~${?H1)U81;+oalp=P(5@ciMq6PEbQ6x{1`oY$fa1gBBVW zAx3U{238-0c}1I69EpZ&<2oxWq*&<7pSv2S6Ix@l8UPt6d zb|p`79cOGdLxwXE8o{+*%gQiVs;Fp#l=_(jX=Va)iqqN74EIDm(zo5ws*O)qL~I0gm1Y~av(VUkxolWw+b#Y2r-r@yWRT7>l{}e2N*&D`;}fPu_dx-uPPFPet^?Z-ZvSKc|1T`?UrP^;e~KOc z0}hzN-dvKCYKJqP1I0>y98IYIV1elW#scl4+^$J%C5QCe6CD^OyElS|yffgeUaREz zYCavj(WEJB3N;5AB{Z8zwU?@HFf_h~^vB(9tDpefMbBa~tIaxgpP0tQjM$9_q-jN! zN@qmsGg+2oAP#MHXfu_H%0YqBCN?NEnz} z$@gQJZHqXG*y((ysEjwJPBXQWi4tcmG+ivF)FDE{J_EkD93sxH-Z28LIjM=u3iO#&5TYHdgEpXJ(Wk0;))Pu_g9RY2304 zgC-3$XeG&_h*jcb*cRF?y5<*vpwKjw(W?wi!iHCHXDv8q@~Jv-kr|^_J1Uzh1=5M5 z@*q+UmfQlY0&`+?O{1r0CL>zUHOq)hbY#7Nkz=C6$iz?( z>LO{?Cw03njzhCURkS*5P)Md?!}SJXw8P#E@$->buKIyzRofkprE#H=u!7%g%7#+! z`Fw~^O_P~H>T3q}t~jg~aDMs~R&D zT(JstzQU<0V`nu+AjSdYia_OMy1=ZpCtRFPwquMO+hR*51qYDW8AoNeX_Y6sZ3vKP zmBtLm&0x~BXTAK?QV7+-np^`s=dr^^TQk^!U!E3XqNI|>P*1nV&jbwH96?%>!7{ov zY<8t)i?*DiTCH?M0j=9Pltc=}BsVRyuHPkeBUanR2^84eEV1f?vTly+9i|{@19O%o zW`crudx;tQ(X44S#&!`H>ykjV4O{@*R4@#o3ziJ*txzf?&`jw^L50n<3q7Ch6aD;@ zhm)=}@&hf(1%5q|nvw(eE6x5O$LEYu0EV0=!5wJeoBKRYHlsl;KbTsmh79Ar z7W-PefZBp8V*SVipciTSNYUZUPNC5Tze7R3=1i*k3>#42SYD5Ile_}7H%$zIq;|=X z?t7&{rLFO10d%S=1ppvY9Z5=LFa(=ghZCF{h(%H$IMZTK1>tI{sEzwn#AW)NI?2~Y zbT&IgOtnzYVAZh-o+d=9KNM^(mb*9}Ov(+e3d59>dV|s+glei|M0!OPMp>Ab$vROO zmE{Zs`K+X;HmW?FwzziO_8rH~QUvN5^;*7_@VZcIbX$E^)vGO6uh$2dE@b#dqf3^j zJvWQzrjVj_CYbFj#sqj0;83HvX3oc^vE41FYYNx zlu3P*nkm8`yGVOfbA)NoM5|MIDp#CdNINtx8JS#^DlJKBcjyc`L$FSPnwk@|WpuMs zCeeeMmbVztF=C-mvq7kQq_-QDem5Q=uvZ2{B4B*XCeyB7O_hgE!bD~a^aU+c+EPxn z+hT{#p+jhlz;P?obZM9^XZt=M3*}r-5=NB<)+IeWG_!i&Xo2Qhup}jp4)h*ktM7Lef zrZSvU+C)!4w6KR$dR9(-1v;8Bt}a1A*d>ywA_g4IiQ*Q91>Q_bgmQ{0v%XKya_vISwxVbx_Yt64nv%>-INo6sx!)YBc&3ljYz%3% z0JLsK0lFiQWyf&R4b5nj22>hL+f_KgrA#x!2NeiT!cGRpsv5`0Tpo#Qy*zLx`%*11 zB$x(}VLD3@Xi#o7u}X_+8tuAaaU-II=|oE4+1YVd4L!Pp#j!OYcPEb;)g_#*a7`{5Y35YzsFy)`$#0U(!d;XzX{JgCMWRR9CRU z#0arI46vG)7zWdd^{9jsR1O*ORkzfPMrg6lqA@+JIs`PSgS4lDkd&cWV-&aQ**TjALJ~0EIL^%%T7YASC5nWq?KBfRx3LYd2#x^Jd#R6TVP=m|QBfe6#J0*^C#2{MBz@abtx zY_Z?KZ722cm4=lKx^7mKvzEVZ|RJOx0omKLr*~bDV(?s-|1Kh|V+2F@sf->Df9fiA5 zH3Bs|IX9J-_BSrFhnWLCpY_L?&d_tGWoi^paxF2)0BSQePqo07WWO`@{6+yd;Bcml zFeR)JwooN*SPI}aykW5>b*!GM#La?|`c9UVDk4JcwQEVi8w+EK82eN^T_6e-F=f;n zKs*)*B>|geTazh-@cAiF=&mhy@iocPd4SQ2OaHg4-Gj+SM?uG-Q z=)wp-rpmHZF)`O4gVqUG3dTO|xVT^S>}iRfjC%02-D22(E-ZlNjL1Y=_UP65N?eEdqzDgc?Y|`;C}kFS2?u zS@TI5Op;cDC|{MsUM=Vphi=6gXGG70dWM&2X$5u2W+z^|hHyg|)YW6I8WzV?LN)oY zCJg`wG`5jqJEPiO-j$?S%yJFGv*Aj?8z`JqOAm$Jk^r^JQ=*b77!n^0e8xtZWYD0} z#VKYpOf2*5(o_xgq=*e?sa2)HJkU{k#!H=X2ZN-?Hck4|9NM0l zL=GuAFrqbzIYLkywMn4V%B5t|DWhNsIFp32PR?f#r(LiZ8(24bhDWVX1zSKvl_}`V z)lsU@mL%P%+PXMP7X=~uY>+x~ft^_JPz94GvNjqO%1DnI_NGV-ep4Wi%=1pJJgC+~ zE}x8jB$=RD&9vYNQ}J4Ou4`lhWst9>E&0gMF1ZX-MYU8d%!Ue3l>|kXkv3qid!AF_ z%WbJ$iu~C~;oMBuXB&052G)oePHJKf@s_9PcdScBWxH70BL)-_;LCl7!3CdaO zqyu^-#{4*>1*9tLG?GDb{RYcUctdxGa4Rq2NiTrF&}>-X6WhwSl~^g`*>ve=*+wXj zD_A9h)Nymx##EM6wdx?yY&7eI`8Ho~_bVFfC$ig@!2|xR7PI=WZ%9Lz@ZplDN@^zsH`B6g$$VDwotkHbmLHb`G@O;Y)rlIR@JtwG@DY|;c!&xe z8Bqn2s0Usq$FQuO2gNB&8PGLAvk*4sMZGaHi12|plG=XM9Oku?n8(tPDR(k`(&k3BdltO)&?av zmG}~n6v3jPqD^??QoGnRbr7JV2M$&zhYqGx2EEJ}M2SMG;ZI0$*yKIn&kqKc01Vt< z0G#BQ8jV^G1)!ze@Af+NydtN&Fq;aZ-6uq844$yud!EjvD^B9`$ED8I30QX0kA(snSyjX)IEAFFGYAYscL-6nVFwQ@Os2wlJ+(`y zz;Ew)vslbzm_j2Gm4+ulmEoi(2o0Shg>E!TTzOJ4`con7xxs7*cqGQLTGf!yd!-6C zi7d1^n9=;e$1@s=3r#udx|Nyh^L9kx`AkF06R1*Gq$WMB`qjV%$~&JiXJ#*HYQu>( ztFZkNAiV23*(KXlX2{gbl#{nS2rCOpR8{y{CksFy5#U`T-I$8laU>fz4J^|K1a8@v z$XV0^t0W2z=?K6vv4{3BmKv1A=?NiW6G6rr!<UU^mK1b2!K=lb|Y> z8)T~X39Jg)gI;fJL_nWP{Q(1lXb}yW$ShMNV2WX73HY7F$YEo!ecv2ds2xKc$H{jF zGbhJ1)dD)4h$XIWG~Ce;>VU8wCzS+pzz!ywn5y4LzLJEg2o;J6chzHC#EU z_bDxfQDc=gG`6Pn2#Br%Ft2xWngG22DdW~{AsxREl2NY@RBaMwtiG+6LTJ>TjJ<|X zlqV(tzqIPJIjcegiSoS11+VE%o4t}_ zo3&TMBc0@Y__AxR(3Y*=k-cP z54C`sVcIm#DnSEml+&M7rw+@-QXSPxVmiZ0kuIP~5zzj0rZZ~jQ+8OLG)=!EI_{LV zxR#Qs2MJEVT|KW=8ziUt)M(1#=9H*8eXwzqsr2dve_~d$Ia41p{eI3frW3=T$<;>P z>T$Xw=UFPkXuVyF01Zkia@{O;0L~5EI!{QbOOR+CYM}|9<1^WawS`!# ztrTOMauYE-M%WfQqb8({`V9%=2T0ZqP^gBmo)FI9?zn>o2*vXqDe!&ZT#Pj{j{VT@ zfOzeIvFMzG3v?^!r??tSJ?yq@GN}av8qG*(D7A@fJvS~*_#mE5vMs$o#Zz-c$3sw` z*J6Nn#yMcjVWtmfgM1ktREMHrQ#4}3@gOE;4CGTds^-Z3VbXySs+D3ACBd>*j*$ET zu(J}}a|>vDsEzYtv7e#3DGv@443kUGbFRa55{ohYcA>-Yfa`8H_oi4NfNm5b446{2`K^D@?DA1++L_6t=Lwv z(V)=^|tkkaIFxTN? z7&6*!%H;vBX5ug;z^Ao5&1qJ?tU|6aQN~gg*8nC+r55Yci4O1~Z8o#1F>T`ido~B` z;J5M2#9*`eT9vCQyn(6|rHGdBZ#*M`XcNFQR0D+@ww~cc)Syaz7UT?dUB>Qqy)tkp zds@M()Idx`opIxS8_q{QL<`1*?J3i^J);dd8>V1&Aa@w6r3>9kJBtycMhql^R+>J_ z6>F+pDj|&m0djDJG%()ueI&3f0SLN0wxJPlvB37FcHeKOLPV%?O+!pQ>CFnnpp+}> zILJbo#%OG0lNwGY_|U|W2+Dz~ll0Uwg#p9Vzdl9-Rg<38#i2?g7OJgbn?uI7!Y34ug*B2Knp$x<&E)V>ksC~4EOfKeg!e-%-%G8X zHm=H|o`#)3tX``cHOz%N5uN(W431(m&DZk{96)cPNkyUw`;E{WqN#fLH?Ez~otvp<&$^JSgcWLaIaKvA(lsj z1RN{Zl}mw91_y^oD%?!!7EFZn+Lb~$7|A|ZRuJj~k)?9f1PG@Acn4kryn{F)gSO?% zje1Wg2a1c1bi$5i0Rw`S6d?UM@JvZ#LPVUY(*|%V0FP2gDY-EM`X{QUZ8=na2EwZ* z*)NvjI?4`FIPnU)n`B2b5En#BB{vs0VwW5CAfq@?SxOB??QXSzmN`I)7W^z~wOdI$ zEpJwvxCNJNnl4X7s@hKUT3ExgD%`1SR&yi(U%y9;436q4bh&Frvw>d%gGr{*tl??R za)8l7)r&YVk@f)WYJ`LhzG{z~zFz*#!EF+EP~LTrdep3eKwq;n2zw38fE|m2O?%~ ztB5wz2N7AsO9(t^;Uy!Eg|OJFc`d@s;#fiQC(~ZO)iN_BM43cn-YpdXb5jw?h`}Vo z04Yf`qb!k-+sJj9Nro?poNqdTrVcu_X&V9NzucYHcG2l;`1X9So9dQ8$l5tW-2Seupr$%B9c zOq(%p*)F)iM;ulcaR^4TjS(W1+f_EJH>^Zcr=+Wm>tiTep+G>$vcbtG*(l)pAlIyi z;f&7Zckeuv~@|AR>&YxirQvEzcP~RS=rDb*6}0E=Se5u;5%_i%y>lLXU92B#`Cy62BF%X zi(36u8+KqgU(rAU$wc7VO{uRr6H}_fq@brOT#|XNI)%FuBS{5R>eqW?+wJv4Q)IAy z6DxtEQwFfSUp3v<3=3GmUgVLUjP+**sUr za;}t2f*cLF7J?VG(U76#S_>Jpn^e;3J1REmCsd6t<(m`S8cQU>Lz!5cjQjks&~;Qn zl2EwD=Yj6&xL$+zLc<{|<$RSYDQKpb=}+^$8mm_HlHapyW68-eL32jz!nlG1K7I?GRM(nuw}6yW`adhaj;>_2iVkZJ&#PkRu3YOB9 zlY;3mf?b&vQY2#s3MWdDKWq@D!Ol>^&Nxjo-)+9RM~Tu)l38~+ktaiKYME>Z^Bo1y20gPs z6DoFTB%qyHVO(g8Dj+>jgFC|tM1ouhoyWtFa)b_pw0bd6vq5Rx275Lhl0`G$luR2f z^KoWKC`d!AFtk`E{Un<$1UiVY8-C9udNrSio0!s4d6wo~swF8|3(4^eT5be6*~~R< zHy-wz&|ox4kSWEL)RxfUZ8G)O`>mncCX^!4rH!7{#%%fl1m( zD3xN0_~o8zO;vi*qp5M?Gz4BM499gc0uoNdUGVsjK%tg79Dw|RxWR+aD=6c`xN2Kf zs!<6_)e%Qo0IwQyb)*S0DG$1?QG&TtGqJ#96?vjsqH1Z33@wdI+AxRogd&WYkPeby zsj20PLuCM>kQvgJ(G(4rv)MqgLLpy+B(7i83qCK=)l#a2nEwc16h?#eM&!5&D z6z`fHbgGZ}EYWUw)|hSjr5Vx7(bG~w09O+q0FiGG+?0;AA&AZms?`8Qrt>6t$%E_C z(kL1UlgOuJ)h81yN2U&3x9->?$RLm8Y@@A&6S);rAbD6c5zq{w*%=V~1=$0Mf)(u| zNM(a^j3;`z29ZE*piv2c-2w!FHzJ<^T?MzZvL4ugfluQ(M4oMt*_tj|Dmtm!Aj!op z_I;S;*z8CFVYdR4XKSOPAf~7aAn#m7NUd*r)6|`$Fy^UZ$Yu+TM!8q*^m$aa6TV9I z3sAfLO<)34W+J@@3>D!>vYpN+r#fXkRh3dOOBVw6EmU!cSOp7A_xv1Jn93iRMNF23$N$4kO)ld&Jaaq>!K`9(@ zakfhLOqI?x5+cgPkb~d@sRZ(Ts-bIS2ND(6qJmv<_^Co97GaHWV2%isnM}Iv3QpAm zf+xdPrEgIAL6-#*ZGm?-$}JFN67#_S534Sc0o)w{91<@tf{R%?1_r5jS&(Rl6M!9{ zwITu1MrRsTY|Du|1tlO?s2V0DQPPBvHJfa}MH*Mmi-5i$^q|Pffy!aCMjJuIkxWW> zzo?eNFq8E_YRuFg(a~5k0GcPd7*vrfSr+*Q&zg9MR2zf1QOS9En+$y!={9D(8`wxq zP6r}PiajlpDq$RIYN4!83h7DVoJizRWF|6B)ohWusW$AEYcQzRh>Ho92Mj296@p?= zQqN8ZrfU%HIHOk7aX(DOXea{Q2D{yCo{j}X)Wu$kRJ%40CIm)j%a+O|eH-i7yt1OQ zsW~!pTCT4u!x?Xs8uq9}f!vF6qY9$&_GmJ-i%~gF*fMa>1k-3IqtyEV33A}gn-I~# z>obkI6QFSqi+jE)`$0?gbU2^L4hx4^Bn)lDwc)%RS3Gzi(0UV$rrPh283ZJskHFhF z*@4dn*-|AdV3Tn@!e9d|RUg2QmdRV0R=Xv7voZ@_Hu0cgok`!CLTUv#-M3((frHJI z{qdx2W&y;Hq!hh$X8j^9cI13J$7z{76yz|9Oa~o!lZ+Tl`_l?C2IsE__<|6Zav(D% zG$9-YZbfdh#w&6`QJaK@!jpw%ZPP<@ck{%T5!&jyK>v_$!&G*Ip`Db zuP{G3{NC3Nd*j|GZu>ZY&K9@Z6`nP%pWs7QCK*WW@CpUQi?Ko{}mlfEC z-yRIKp7-P9wpexX1NYWmJA`|lTqXYuJ=OT?+z;d}AA5;j_1W@!s;_?U$Y=gDcF@G} zKi_5Esmm_i#Ttl@y~HN`DsA~Ok0#e|dD9W=a^KtS=pS0iY0paR^ZUQKX5aFgyIaSv z*nauzSVEyQ4Pl-qV@!LcA!vZh_>@1X8HC)^J(Vmwqv~EA{ z&%xt|erO$Q`P{vwssQ)1BXJ${9cVWbWcS ziB*d@rj%sg-2dF&?%vD#`HD?`^v2)Ued0WE@kaOj4tIXAdd6*De)Y-DZ@=C1U5hT; zm3aOT{->9nd+Tloe(&;g_a|4`%g;D#B|hKZcY`OeWzQVG>ZRl98!umR!SUhdub!;0 z@wm6RTLJG)-~&Rr^h{mc8$yX*R0Z`kb3!d8j;-d6`M-|6fRmaTYb?S&@>i?=%Q zx@VVtcFaG&KL0nfWlQw;pIlUYbEDP!tiAfGd#_)od(6*P9DKr>+s=E2TlLV|8(mdf z`MYJ^I(sC$Uu)HfyS+@5PYM4CadmF{`IaK?Z$10O@_Wu%@6#2F#tR;q_ad|FQ=84P z4&k1lUViA7Up{~0ZQC4h+p-U?S#is5r_XuuC3(v~x8ZL|>HIh5oI|oFvD@!>{_+Js zI&7VrH{d?ob^d8-S%I6{EPc`{pzqImLEj%Cmy%*ic=2! z{`{ZLdv5b}xBBG%K04R`KY?_J^l$R-r4Qw z`5Wbqzx_wIz53a459Mz@f-zWAD6s|0w9G zF|ii>@!!sTy zIsfe|KPN7755)H)ia-A7pC=v%UAchT$WrJEI8!+-#vxA zxZ{=Eni=@c9)A$_?v;<$Y`El!vllJ?M}PBe*Z;%o<`3dMkUReR`LfS$ zdupdki#t>fT=v43Kic`zxmUcKz3r<@78iDSGJN%K8!cWB``)&n{NvT_|H1D_ym{+= zPrdW%U(Pyc#TQ4OIsdYSKlurE)9zPgANubh+IYj`$)nkQPk-{{uKLinTYSx(yw$E- z^Gm<}+ppJFe!J`3)yI>>o6%-FvvV#ubF)3JnIqhld+zHk&0VhFaC^S;*fQz+e~-7Z z`&n+=^)Gy8lXdQWjGSYw*nIT^i-zpHXU+rybIp;@|9JNgzBppz=!U(gPanSWjQuw} z<{I+OHQT*>!bK-I8y3`g3<#o&6VdpMve)_@Ab8{CSw%4~+_y6p8Jh{=r<4fzk zv*?HL?CeYDJdHlM`mX!V-1?n`D*Xk-HP<`qpVyufoq5Fk%FFZ6IXkTQPW6gCR(!IU zc>R;Fjy%4;=RFsHvVZrUlMd|+ma&jQ!WXaozK;+2zz{cOsC5<{?k* z$G`XVg9qL>WH(xL!g0iQ*KW9T>&^SV-s<#yk6E$euoJc;-kg7$dGP9!HXnRHdgSwG zciQf*ne*OG2VG_Ty#CFbJo3Qi zPjB^HynH~pe!pL5_nS1vo0UATUC?srd`|2L+z@{T=LuDofy zXz4CjzWvl`FMelUg*RVWeB1_SuXF#3eVeSWmFaNv5+Zp(gu``W{umG>@Ove~ixmd{*$ zh`ZJNH@|mqlilOX_twAb$&0sq?M&$>e_6ie!cS&bys~6IbIiW~JgoBJ`S1S2r_Qbv z&MsZ@)l2_)^9R=+_AtNX6A_vAY6v+@I*$s3|~naBThp}6dZm;Nhm%Z{DXzi8Q(m!G+2&(}MD zcw8#cYi+v@`im7uJsK?7_Xodj{p0vszl}i_y*PiFOMbKiJ!-khti*!82|yW5}h#;w~ecgE&B9NckQ}OyFxsF&+i`l?ERbWMc=*gp5%WXG5&asedZPWjDPv* zRo6AYocH0quk+svk9dv0^Q#wseCBk`66S;ZAFWSU{gS!n{SzNs`PFCNDII;&0SC@I zbD6%S@!n@^?)^Gwetgk0U!6ej_vSAKuWj_elFzo>;PIRGj^*FH`_t>5xaSLcZhe35 zjc4|!{vyAy`{f^l_^oZo%$?3;hw?{0l=_J(7xyX}SUE&p7nLwtGi ziSO@x{a4TEpYM3n-5vRt{Kn_^*zw%Pfd_o> z+OscR%YDArt;u8CKX%u#zX+edy!*hKpM1FC*?->m_lcZYb<8d+zI*&H-Gi1txX!5@ z`9J4Uf9r+UKYGlOt6$yb^gph-@u~g3x~4dNXGODruOV}#xu|S+PLH@_-y0N zJ9j>2)n4_}_S*C3x19d|TjiIYer5W&eBN!3U2@mo4|G>tn8jPq9zJQ_|F^d{T0P!> z&1Lm%559aN>?~h@WA5+g@3I;H;g7EW`$cPxede4C>ib`;pYgpFpIv?F`SWyppEXy0 zzQ@6DUixM9RPXuTvzKg{+;qSJKme4T?zkx2`LtzM&FyrKo`1^vKYRUd1J#+TP8~iQW@aNHz z4?cQpU*Y!6w#ARSYs*)DwaeyDotk;~=?9+u$;PL@Z|$-34o`bGzx>DBgF}C`$%*>8 zd3(vq%dS#ZU6Lg(MEfUxNo==dM4kN(pE)qAc!7Jm7u(OqA^ zvd4w)$JKY&lwUgR@*fSdx6D0Y(ZVgRIA_k|XIDw|wEn_kQx( zAz!}tw|)0}@3^C1cy`T2Xz{$4pvFu09w_fH`o%H#&$%SntbNKK-Z=K_FXgYl+~era zzdZAd`M3W4sJr(WKgAw>WU%LUr#-!3#p1J{$l>!|I8gubKJQGgU;5$;!X?k6doO+E zrDMOo^2)bv^T-c>dhA!zcUwS14ldm7ugWefxT5sueU}v2lg_x{idT9&d-~_m`Y&B| z!)<>)v^aiu^-lLMxz|1yWFkCHevhAieZqHF1lW6|ixGEhbNnw{{ck%R^zMS`yVL78 z`1sevd4~my@BhPV&}66lF&FOh?sHFc_P3&gvrhZ!mnSu{xbfvT{o*g`*H7Jh@X?gB z@8Oq!dedFk{_TwY9<&O(ocg;@PJPx`z4R~4b_@2t=DR;pjnkK1we*nt@A=LK3+}$+ zpM}4f=cfYpH`k8}P&-+0#>&_8^->DqUG`uJ0ifBnLtN3S5~EL?kb z=gyO>2H!h&#ibV@cRl#&Db#CE4$i$~gXF<=pZ?&12cDfWFOA=P^5$P+&%3`l;rzdF zZ=bjR@r-o|N~qft9~}v=2Q=iuPnX^J+57vp_Pg$$8&|KQ5Btj{J8XM2cGgKd|MdFb z)=!^3an~o0?(o}nR&1>Ong8p3TmAOAIaaa+ZC-rrrytJ#I1W$w==q*}`dzo4ciZ#c z`@{tgy!G&TZTXd-yuSTedp^G7X5nA0lfM7chi`xB^TvjM8eMnpCdPHE-o10{O<&)Q zdXCvJOMSNNqwS$TZ1Kqscibwk8NXA0aIZh_^{4pti*~DTTlwf@p!iR?;K<#kEB~++ zfAYf{8Mp5BCuSS*%l-Cv^R~C{7~E3LJ@V#vvS$}+!o6HAR?s04K z;r8dfc_RClqxR;0_=_bQt-f>L1Kz&lROy39FM4;?c`qIH#>u;1__+Mip|5{YJS2Dh z;hS!`^9TQUX2s?EZ2$EGzyBOx@W#=l_rG&*<@UY4OI*0|<1~Br9xJYW`468lZ+-p3 z$|FyG(_p!Ua zT8u2&S~%dI$JQM4-dSvwUNGmhd(XTOKWy0-_aA-7weS3T?x`Ps`uRx=@1wu^?Cj^Y-@S9{t;SP3{62W*g1x8KMOW=}_XZcOI{*H= z|M;u%4Hw_OzqBsc?8*Dfu6S(zI|4v}#wEwxoJ{TOAN~C8)z|GDJrT{_VgFnA{0zUj zy6(~!Z@T{Txzl5xKX%Q5hdy)lYxbe1&O7TL7d-G>Z(OL}bmJ|%A}9Li4V|Z#J$3f8 zpWgG*C-3vy^)GMk_S%BS9z5!yy9&CryHDaJMA5I|OJXxCJ^8 zJV{=tRb>TPiFOiRg_C-+dm#?HLdKl& zgwC@UuU+=xE{VQt`x{?ww$|;M0Mr3)EZ*5N@&8#bb*HOY>`LD!xrClmdi~0HPcW`*JX_pbNr6`1+c_g0gU9?qmoC*#F*B zd4CbMc)lWWzraZa&-rmTU!%n?aoWiau2DF`WxDWf3+^h$dz5aH6>LPnib!JY3m;w?a1-_W5jO_cJ+7JT+!~S>kqyhu z+jlinbp|FL8hn@4^|sSL7Ka|TOiI^?5PorvUk|ADw-dFu8kp7n-o^b~x!FGMj^N8; ziG_uSxaWWW)tkr)-I1^sDT*-3ccvVTqW`qy9`5da#dPxl2-W;tj+wDflwgW|S?_LJ zvad4P#P1!>L$Vw1N`tb z|9u2m&?tNA#i5ad%hQ~urbkUqp97HJMbC^K;z2oeZR9pNT!@&cj5P|4GzpeZ$jnoA zj}{wk$M^~J%D?$tT;2SKJye5}$N1mwx7Vr5B-gjhx5x)LCf7S#6%U?IGmklYTj{H* zylmbWEj{>V`%K#oXs1_gtvPa7KC_1Q{P}Q|zE}2)e*CzSYv-ckV-Urf>;~@EGg`kw zKl`jN_$E2Utrz{dcbS$?x9nJi-w)!h`zWP)5{SeY#&f`t9DcS0?Pk4G6)JLEaPI4R z{yd`WcB@7w8p@yTZ){!9oj5vdEu~aA(f!_`VT<>m?N*OCg7LIu$t&PXNlvYhx0G9j z_n*_L?;{?3K(4`NIJnof5e}=+Z~P%YcnhYtbP6{mRP#Es6+CoNn{d9bI9QK-v3Z{( zyja}9z|ypgC$)`~Y8|VjPO|BW%S9rr1)E6@ox*rCs{^l3d+GjR_|WfuAHT%^2JL{l zwn}blj7nU0-l_Eqrg295_Q-pvRFy8FHh+%y=yzlkaa-N?mZ=Ni09z$ObH|eFUW$EiS(tYvpU=iCUb_n!7`Xi1muvrLvX4p$oJpO{BFGW+3W07+ z>UtsEAxFD(KqaWe&?l9c8d}S-3P9}WR4bc92@hQRrWAb*y{xf!CGgA~m#$97kM}Fx z=6FAgOni+P58Xe(q@IKIi@aDQ?(e{@r+2bk?rPO&?CT0XG<#6{gbwC(l;07)&{6T* zv;3DoKgB!4H)#D6>#InCEBN`wV?f<7F+#3cZLsiDuJ30fk0s-IhwqkEGm*g&{ghr@uL&(W6>i7ThnyT5EBqI-5_eo`3K2Oy1ThMm7CJ5|rW$KGq? z!DL&{x+^#=zxriMfPU6AxmbbX-|9?TjA1?eA>;Lf@#7@1k#=y8bL$$H3|;FqAqH(8 zM)<xTV#N*i$T1^eagUggy2p@{Zz=vk~N@q5VBeGCn#>f%0$ zcB&{Q7t?McZcP!?utI(XiFRE%2B6# zp*@%HDkM^V!_r&mq#u+%!SX?^w-3cl9(N4DwV!Kr>S~Q~wl;m;be%fCL0lTHJ^(r5K1cpKt5nee+wOc#NKJTaqeG+j zS*sRzd!3vjm7%=wMGKs@&yKm55r}i*sZZC=Bsd^73pi<8T`f6CZ9N<8!wl?j9a(+H zN56LquOdifB1%!bj3;1jma7MflT+uZ_3pu284n(%`Wt=%4P(!m{H~s?YO`~_OgsOh z0vwrUH>U+?yB;&-=hW{in^Av%d&0hVLW$l)^$mWSA#l&OMfvhN=V1@$8O4bazLabM zx^e0GLw%0>#?^J}R_z99X8EbZOR(5wk9;S-$db=&4lNTQqB^pM1^yTDS9Q(0b8}Aj z<@$X}r2vNA^%&vT!TizD67gkQ9A@jEDi&)CY8=#_^U`g)#WTYVi49J(VzLDfb_Tj$ z--Z@cb23j~``d<2TKoO((Hr9$uWY~nEda00u8q*ewWsomWSA0ns4rL1+OC-taC9jd znpNv1KYW1!`f~IqoU-ijTS%mAKU+szbzJXHVyF{$HxnM`DfUmroj_s=GFwO^_$DjH zyXnwn0yzaTcyj&o3^)tL%Gh*KG)Z64PYk|LW+m zCH|Meu>IK@|MTjz@a~<0wTPqGX;5`vg?Qsa6+q4l!fW@B#J0qPK7PY(2C5lD^rJRP zAEZ`Q>#O8waOgDmD%<^yiGZo+7fm|`ajG!i`gY7(n;##%nSU)hn~;&al)Y6ZP(YjY ziV;0dV%5}Rqj2mj_~Q@7q+kQGS+iBWk!m&-Wd?ZFZaD1W?6gvyE%4$LNE2UWyNfrr z5x*Yh46#W2J>B8HJ2gG1A?Z=|yeP{B{u2guar_b80Az<;f(4gPx3Hko{%7X}IWQvk zz#+pe<9ad2Gr@a+Oofu5u%d5Dyo9n)`!r}7o;4h+8@;!+pHT-2Tc;DQA%d4ZI`22p zjG~(v(Wa-~ai*w=| z=vb&w0qr?I>vIYohwNSPS_i6`9`@NP<+tss7e6;nycpmcZn`xy#Q+7T9GV=1NAZ3C zvyQbyo^0ux=|~Kk>25_UQScxWh8$|Z=b+UrOEKp@3%vcyv$D1p; z-QPemxOh{p0n(kiWWTC}Xl^m{zzw{28juZlr0%*^yIlHY`@C$}4=0xf5wn`3mlp zBPj!k7pwU~+FDh{x@~vYrSVmMYcIPxI5~y@(Iz3b-83}+i~&A{Uz{J~bKyRgAcNuc zIDc4Ldx|%2$#E(N;L(aAIpfLP%h2S^Pg#$~uLCuW^HRSktQ4G06q(qq#R?m4Rcvs# zKd6fzHW;PYaoyJdd2%u)8-)th4OfaplBt3TNm$=EY*8t;(kDXG!o$9Fx?k^CMn3?099+*GW|g*fT#lWY+ja=$o=1j3}YA}z!J z_8#BNiheLh1;>TDUmm#1Tn0;M{@V+%$v(Ql`IXJY^+FPUh1hS~i<8&#=-Yo`*SFv0 zA#se`e|dd398)Qh`-;A|aX%hKN#ZJb^Pt__5-aN2WHPoMp`LX~XtF@q)h#ij@(N-@ z7{@KQha>*NNViA@vT19<;gXvb9S-mfPDctArLRcu>G0M5^-$GLKeY zxq80%VbN0FHgV2yNm0Zz=}d}@3E&k4Ks~9%(Eeh|$5+Zl(`wYrm<9J2bF$wv-mx~V zAl0mW4Vn>6@)H0tnIf4mxX=oH*cB@Tx$T)r#^mTYR&MX*-?mDX^oeNGKn3V)@%aNP zlwVMKO0H3Hh9+pxSMcWBz_-SwOb%+I5TuyuSDw14g41!4Fp zDNhW27%{#aI@yZpSUZKS$_~;<>mBVHa4{9Iz-tGR%pUo!J1g5)=_M`^_F|2$K4cOa zF!opH5z~@m8y%ZI#Dx;Y28T2(h{spsDN7UeKD`JoW*}ebs_plpJjsA1ZbsNBWC1jF z5_g!~uIfQQq}$VxA^UZYoxO$uwSG>4K?r5~TZwpCLCVAKX6FwNN{}BI#x>uf zURp8y4sBTdvNPvf^IcdYOz-dcyF7`?YjS*ST~a&Gr5|vQHHs6Ol5B4rb#e!!(6wVZ z&5cOGVD4j~6u_o8i&(~XpFiAvRrkv&{C5_fcT{^O-l$C|@6g&KMK)dn={M&}pd9eN z%qOaz>pM&FWlUw8`{sq3@T^F2{K^xYM&=XTeYqV??B80Do>+{ip^(NLuQkk>192Ve5FO0*jFUn)M9x?g3V5<7t4 z?t6T}hCM(2A;9@~WL^~H^WmYCULGw0){4Hc-zNI?@)`Iv%nG;Wh!Wt&Rbo;_IDZ)l zUyM6;^)=>Ewh3;=Epb_5NXE-;=DM7j?Wv+VIdwkGdMz)a8SpTd6lWn%MoOFzak=Eb zFIF#L6fiH*w6+w*^_>7E@GIB}W3b0j{qK44)HdBnP1MFyj8X+J!@#Ok0N_Bxx@KMQ zFoIm{l-{=E6v=oQs~7{A#$ffqbMkPh9!H8$^;b=2Ps-?{5SquVC=ZZ5Adj?rN{EO_f$5@1Af9LkANqvu#a?7^; z&G1CZ>)ds}ZRqj#-7y{BZCaI|-RFv##RCMzCsO&G*v8au6n=0VFM}yFpEVO3LnJAq9czxSd#yN z%S50rL_wWZ&{9FDu!s=hke6{I>}ak;-E6TWkd-9Eer}DsHCd&ysn#X>ycQZD1VE8< z<-v(eFEz+>Rr-wvNdz(CaLK>-+FO&(Gi_i1O{ZRdhCafk!gBt7(SG)IcCZw-(wGuc zD4P`U-F0BD0Br$QS-SoquktsUUAzdP#d1O>sqA2sqVyUjItDRGErz9Da;9t%RbQVQ zXf(^U5ols5`8YS$#oRiTWr{md3#scYy-R(#xnd#{gbWstW>*^YA;0|SmEMhleUC(} z@OWB|2XIi&3I7{RRo7HBqlF+fs=XFw_0I2dHpb&uXJP`{`4rI({V}oCHV35>!ifiSywJak8nc7BrhXHa@Y99vwGJGnsnB>DiXznWZPWdhfmm7LnOVErXtpF>rx(Y zyBaM%_UEcHW1&Q2o~V8&K^g6ia+#;mR;uh)epdN)UdOEDLFY@6*8?v_?igB#MGVGm}HbQ zSh!#5mNDw;aXv*@WVC`%FkHw_$i$IV(D$YX6mgzZ3$%jPBJC$0Z3 z6Z9K)ybabQuV)T2ufWDg_>}8^P(iVwKoG5=LNqIc$aZNa_7K!^8QXTT4DdLt;kI~1 zE^*~e7NtzVi?Ze`6JuO(1LBe&Hp9^7s+zA7N&!-``nTP?9QI)=@?}(2SRMYRy)w8% z3hQdTA7yLM-!IrFS&x3$G0WE)`o^p`zm!i@8g~-R@5iuZo=ZbLt)$U?=M>N}yE|D} z+)2J83+jr)$k)Xlq7LP& z3@4XXxa#lgiR2Z#`a07~(<2t81UEHPlJP#4%4s-GPmGXls#jp^CLGbQTfkddw^B;8 z`PhdOfts>a%}HGa5YEEOqH`M>Js90?_{LX4(}C0E|Pm< zmRZIetB-rV-~BPAey{sIE%a=9uSx8LpjfH(&rOgM*c}@!%)1er3?(T3MI(5YmGGBp z6UO0=PVMpW%Ng~!90A$GR*~{wD@m-G?Ci6042Qd$AcQT96A8MDjjTiP8ncpny`}3- zkho_T7jU+}2PoF>Hc#2gn=d|ibYJezcHH|6jR?o2_r5T}WIKi7W_B6|)4lRtwYp24 z=$8rak9K)nS2U`&sU>F-wBr`=pt7QqTo+|p-jRBI*sEPw_=9LL8q42PH1T&VR!~c; zP|WpTT0F<;h}C%u&~=2?n`k;*;177kO#xL02j3QjoMbmVrVZb5nN%RHAN3=DqO94? zacl8^I2W!jc27H&T<8zEkl<7Nn@S>KgZ3BiSKlAt#w$W3o`)H#QGKWpTC$N<)Kvw1jOM|4LiIQ(#=yN&wg?|Wv}4fk<@2Afi3;dJr&m&|Rn*wz9049@`-K(z(}Vqn&+zKHg94?Z=_>H$g}%j~9?IABiq_YE zQwH#WuG-+9Hvn(mF0C_%eZr=F@1KW>_?z#NQd*56R7tA}{^K7O(*h9{U+xjS=D zoPK=(X(Vh!q+_w4iG*R~SN9#BNw4x)1qi@=|!0z_YXkCb? z)aXn4(jeAo4y+nENr|E9YI#D4mpdODwb&p6djWZh46o2I!==09%|a0W&rpGCTuU_T zxhx#ehLwO}d_P^V(CD%qDKm&OM#QI48p$c9*ShH=2T9~o z57i8+38BfNAKljO(S9zP{UFnDQaTz%eh!Xa4b5C*WjwN`WneO^y=Z4_?0Be|$my#{ z`kE8CBE!HelHLcLYRwsaZ#?*>0W{MfD;{NFc&Vb_0KZ~&?p602{vP!nGK}uWXn<3F zdzR5BHuRiqKMhE?D5kxM+PhG6-lR%hL5xO z6>1xie1Wf{fJ)Yf!E9CgbCtQrSQv9(91s|YXSE5L&UI1xLd9FCwj3jG-Hg=#*lIvd zH>_}hC%-f)o%!lfe-ZVbu}cFWc!`*S9O}U5kDl1q!M3Ktp|H3-SD7Ce*oF4T28Uvt zGr@|A;-4|Rm2Krpo`x49}274xg<@J3Z*s#ux!=}urv1lC)v|EI#5?bNC=uNhZIwLf_~3C+P*J^X zEZ<&%Ttt&ApoVD2ok#hp#j90Np%e(`ltf>D#R!rKLabzA75o&AUt zYRYF#_iKH!7aBQ4mGXH+cyUtiWxCE;SO^|t7I;vhHZ4cfM}^+2GvLwu6HV@EzEg8W zX;<%ZUFgNnzNo9Vi7kNKCpBUuZglQss`xW$mwlf5o70Uti}>}5Ma^@|&3P{k$ZTZV z$Mwf`EQq>~+1G`MVONR`)HIDSwNLi4Nv%=zT7LcLpzQNMC?3{+7K7|4a|#P{L$<}- z`|o3Zx)lHkWJ;U+E2LC1Sp73ZFx|_Q(B7Mq*dhVn1=$0{llG@{k>gHo4J(`671)l8 zwK?N%1zz!!hP|CnX^Ryco5hzWfc}iJYrZ&I?}Ab1cM1vNBmot%LJQLE_ick{lhj@| zg031O`*Q3=`%Q#;8ZW`S2Onu}Vf^!b>;pEC)92P2BC`$T?k3-)^90Y^*E&&TPBs8G z?7hwRK`z-ajayPbPG)o22NMlQL8W)kMuZA8AHc8tT}QUmguJI+&aS=pA<;%US@{2n z5RxHGx1H0O+;^lBTv9>Z`vHapZ$nA2v3Kr|teGW15pK@aELO;=<+_7}cdrJDV)uwC zpjh^yRi~B@!xLP>9%7*+t1;M_@s|e+;f}`1*vDo5k=vM zF4RRXNPe@@vi^Rz)^&YAjmc)876BN$5wDj)tP@fUZO9n^Rx=`vKHa)SLw@8A77 z4SV~x?hdafrlv?*n12nT3F4WIzehzUlwHLt=S~ox& zPg4vkU6l#9&qbobShdAA5h8(_ z{WB^wKp#8*jx2>Ja{|D^_KGIL+~z5JXq^iDAiUY?G zk2R^-4QYaQGqP&CQy)^=nh$eJ8nq5xBjzBu#U9r(A<^`x#NUBV^U2EgTeIx47fZFH z68pe+%8P{McXWT>hMVJg@ZqI?+D}(RL?~x0GCz6kanMclRcX8V)kIlyU^_?Hevqp_|UKl4lb_#m=FSe0^ph3CIT z7oGAc=EEKJrJ3U*(E4H~$X2<>3(lD7N(zWoc^kx}3ds$)7PKC`InMSP<=EEp9?1}y zTr?k>Ds>)Bhck@Xe73@!LDz!WB5|Uye894;NolL^w0(ye9;3Ry@sy33(bpwP+34Fg zr?v7{JIjKfmSLXgNI|)LslFi3Gj5-NFAUu#t)d3Yj}0ll3OBG5-dc*-(XubzMKqPN z_PJaAJVD59A2V+nZg42rFoh~mCV;}W6Fc=AlbLt6T5t2D`6v*;B2a^Ow}yb$w>I_7 zrvl>1JWp)_N=FIONSf}osWmSn29&N|J7x&7l3K>&-%Iuhz3X_pi6l)^l(!2F~Q{`4|UcY6d?mNA@7ldIm=!qh&DV|9Nwf_00jVO;Z{r%&a_W6 zKm_F~mhp)yBwijl#+UjXj5%RCUFn|B0AISy+$xM~>0u**W%p~k1-c*oZXcl0x`(|e z0cs@-a6`EeA;Co0+g9*Jz{SSQC>X0)4ireK^R73r-~Z-=YueipcNqPXKyKagRWxf$ z^Ln+n?ZXP^mIs*%kf8Mo?#dtWE|Xsv^X}(Rv>PUxc#a-N7}EC1cYPzFH%``O`xzCMcbo z1uSiZj-H<~`waV#)aGZMwcl-uU&ca+Kb&o|$o_NSL7s{s74(B&^JYGOb#WY=(a`z_ z0Td5LFp+6&s-mjqk$)3?MdsH^-t1A2#>B7>_;{ zw}AiNvDeCc1u>H|RTcwa>lOrjb6fI8eom$0%jSz6PYUR_u1D%w)Q%7sc=QGJo3Ss! zlyj9u7YpyLJ+36T@^iRi3ARf!PDvWFgRd9}r^i&_Ezsc9X(wMqeW-OxWl zXM8Rkym3l90p#hf7Kx@Th6Vk`AtsNBjrhCky<|Xu5FACt54#Ow$DV#4Wj`1HTQoBHm z6^dT8eT!~%c*3YqexsLW0Q9fo$cCOUnY=`)kNqsGyrbrBh&}9~1@}9IsnjdWM z-<9%gtQ8E06Y3pJ%sE#sW`_$@!|I;4zp1TwqoGeJvxkjBvUQ!Jw%a82w+hbn$Yh)4 zyMw4}bg$@p{fr%-?Ku+A*kn)=V9)oL{xpQrKu;)F=YBBL#KWoyc@J!od_kS*rb4w7 zatl^s__ph5ii)#ad%lGul7l(hoBU&kyF$)bghOfDqMiV;>8}_fxD^&gkJP7!Mt?_g zEx}hU`o{LP@n+4K>~07x$adHjFU`NAk8d`F@+9`|7lBT0L~Bq@gXVoC!07$(giVi;L|OD?drbKn4A_4%p}HGK@3Dl6ybrS1`a};5acy6-#G1!Xz?4%jM)I4%V-%A(pX8m z07-?dtTsVAC5SW9Y}Ak?N!L`#ggTnw=1+`s)&<{09%t=K0!dZv1h!YoySxu!Hlix0 z)FV`O_U>YohoF*XUK$MP;o}a$QiLcl*n=;a;Ul?h5h=FMpM;YWVUH|0u2BP1w*> z%i6_!;geSiOPpgO)+lrqi#p5xTNXNEe2mYsY^As~OnlFc5PUea>T~LKffi6EC@m@t zyHa&nD>5h7Az!^{8gP^Z*Uj!e`dug8TUCz%%_+>cJ8su4ga|j$ve~`YKv#<`IHOfN zb(y=UsVdn-uKFi`m#BDyLcr92dz1d&u(p~xGjBr$#xPiyGydW3C^}bW(P>cnKcxEo zf8H6M2DFc(k73$k(ow#`!m#+uU*QJ1yZ{PV{~<%6z!$xJJA#cLB)2&d@LRCYj_~#t z!Z z&c$+Ei^o%>*foDhIctWSoP6cp_`)_1x#1nkHe0whksRsa@R*f!#$4w{ImTp*sjY!i zGC2|o1hm1Y6D1*4;*_YA6IzMt8po6^)2fJ&SBC)%wk>e`U@<^7TMak=`E?6jsg;SE zYv){`jUS8{NG$0g@HN5MUo?VpD+w!!X(5uBCUuk$dWpu#XN7PNCJC-$6$(cV_R517 zJ2hW>A70qT!Bv?2Ks(AxPZ+Z)?j%07O{JW@I~r+Lxk3sJ6=N7OR7zr6WZ!9j{Q`f9 zt(-P$wDfi6w^qjKsbtfUFE`o!{w}|~CCPJ3Rjy5$r=vR-XiTQyag-r4|K?rXI`X<}y^cr$Xjg~`p-xZ&dN6JyG z5u+bXSxi-89Tpxo)oNN1!Ozz!O7{>e<1hMBdr{Zd-{9`hd&s1xFKclU+?r{GV#9{k|u@>J(O&{%i1XHNSMh95~ zA^}=+os+*Ex4DRwfm~BVBLhGC*D^Rnlc@LIGgf(hPM6CJCS`Gxy%VC_eyOocr(mN@ zC*d2fwQJ6+K)b5y<3+RD8ATb~^>}I!A-xTT7@#i~n%H=lf8R@zHh*uCcP$fh77c%v zc#t|-I6U%Wo#M~aViweveI<-;q~ZLi1>Qs_{7t^OY7nxQv`>^31vCK46Nir`qCkZB z-SMj+dqX^dafTQwgo?*S(wpod63DZqT#|dKN9n-f&}PGso*9q~E=x++@%QV_cw7k+ zO7Gfw5~OXX`>hR>PXHAX+2o4oPeoCCxkt`PT0w8K$5xL@UgMu0-zFD_u`H`@I%3+I zALcKMQ0Vwfq$a#a=B0_a=|bDALz?eNtia|#pH%X>5?((}KpI0}+FCU04t>-sJ0H2T zp@Kns_vfSy2F)@B4Rf&E(nZ@-8K<9!@YfBiS+IzhQzKb#LKau_c&I6mw*T7;@O$<{ zM@AC^tkS0R5UW>^M!gJU6V;7E4~K0%fNJ-ZP@d~LPp_E8qB~E!A>y(FL-gH|^ISJo zNZKchIN%4QbZofqz1VfzRYJTaK=Q^lT)Jzz@UKt2WPg9_HCz-E3B16W#@YiU!k4Zc zOa05b%Vg-GneY3Wi}j&*ZS_W|=o_|p`X*R7Kmd{OBT=HDC2o@mjJ+LHyu}28A*PaH zp+dN|+L)H4+mq5Z1tTYnu#x!KkHKsN`A<6nJ?U}odc*2g>rLG)+#49}R9}MR=`@8Y zpi5f@Q%@YN;Ni)*p|PQz;xLE7hs1|u)PVwNl-WpDn~3o^v$~E2N!IaB843{Y1GOQB z4Zfpv8seZI{K$Lgl|+C_rA2aIgcpz2AfNGvFtTdyz0G)ve$Y`U?5&tw9(^bwVx!R$ zJlGPF?CKe;nh!eHFM*((siVdvnB%)f1XdwmUk_vz<}I2&T#ewVg3>RQR*^^>z2u0Y zb6tiVi)e4rc23rGq4z_KRmD6kcOChRxVC(!CGhJpg`pes16Nk_oHhcO%FS=Vt6`mX zu9uB#bf{Kk4C`nSzjB%O3Bz7lNYO1WS&S{5pCGB=v`;)5gs9CEBeu^m2+_6hhGAre z{Hz2tb1MT+-7F4Q>i-E8k;z_zWf(gI$@6Ggm-*#DV2gaSbMnUbUy6wHL~@o*7L2D4 zeL*)xNCv&sone(M_tu=e%v4t2@TBKjInc0q)(FV=KdiZegOCZsA>iQIfHZC@M$tf? zi>p=luX2U54?KGXZ!8xj38~U_FB`N>1crlyMVV3-T9b21pH=fI1 zBj!%rFGQ2Uz()84J<(or#L{cynajL6A=EkigrZ3INP1y>_uH=*sQL^-n8O^H453e~ z{A(jP0Ev5BE+*fgRgiWlE~;4{ibYbG%Qq!Atcc`mXN3%>7J?`zvP_C~)L0cpke>n( zYFsR?76An$loBQ6W)`Wn?iqcDQm&RH3MzP4SV-taXzfh@u}7$&ADFd3P&#QnM%c>@ zs#yBx9+&XfB(guIrcgc5-f8-DAIv~)TU{gxaU)i`GV%I2v{ri&DxkeQo|o4c&nQi$ zNhjn`=0Gy6ghw;1B19plDGPqTc4pkzlxc$^a~X+_-!jH5oVOEULay6 zP`U~Qw6PLhN4h%nOUsjVb@;T*J)Uqc?a6E^^7B6EkEK8=Bzy~j>YL;|6JOdr%CTD% z?LXKPVG=z0*c$UNpFdNkp$wM=?8R_BE+oCb(}06x7R@IFNd7Oms|Ca32}B>Z}MzN|HPV zH!*g8jW7mQ-PmKVQ;W}b?)swlQ()WYMy|>pAnBIKkP;GwFt1OH>EMy&^k-9E;Z+a( zZXRwqKgB7R%nV`SPDu0mc)v^+=K!B5{0;#R7XI+v8h^qoV;`G>&i}UThBpA5Ouu(| zils%{;aG)fTmARgh-c$XgxxsvOSOiR)OZviVb+RY+1Dfx8O2@Xwh@^s8a$7IIO11+ zQh+evx2?on;Be}VrFK^Vs^^@ug zz+Hm<5FNGA-1pjp5hP!OUwlFV#S6<30p0$R;&IiY13CmnLVMUJLX)pILNc#tQefL| znsj3mfOM?41pb5df~YoI@@i@b;{n&!B1bkSVL3rxWm6o3Hb>UKpg)qs*kEY+inK<6ldVFR}bp_7@U6D|4ID{@bRy0g05uLEvmAL-~F-a7DL8VgC~oF}AU zBB_@UIHB@@vG}><2JE-*e%$=5TOM!SoNWeOv2FtR=djP7tX{MH#qRn+LER> zYb(b=3yo(GbhCNe=0)GPi4J=EaeT61XALQo6b~FNv`9-XLgH!@NLS?=$eZyRtI)QV za_PM5k}Cz@)?%zRoJ;%O#)nRFSddoacRA`X=YlL?U|_NJM(LE6nDt)nHf(mOvK%NY z8kcEcQP6JEO6HaUKWwp%EK;>pFmm3a1vEvbU`&Jxc;CBXZX)&GNME?n^%$XX5}QX) zlO#r_;GT5dsD#5-Zz-MqWS0t_vQe=4QcMiX2o0*p)bZoD_VQ-HyvsMtkU=M*B#Avi z!JTufNRfhpiCiFg;?b)DS2)jwZcoNTyVAeysHA zHbcK&zr!vndKF<_xp_$7w`wH4$Yac6{7E`GRA`tefxBdp31%U;|IhA6d`Av!!rw?i z@}{3%qZ#4;vT{7qVxndY0t60M(lY8_BSvD;;)R|O~4TrIZrgdD#~6AzsdkO?`} zrrWT9%7IGnaod|E)TaX-GtnPTXbPVTodOW%^mWUJ-;?p-1bjhb#PB)d=gu$KD}M^} z4dx(`3urrFQiZo{N+cq=BE8M}ktP~_NQ!~iON1+q!Q7iEiX3J3ay*5ST=ErA_W!LEJNdaeMISo8lbVM3g{)v zU0cp2ILC29D4ESL2@2ohQB{?3Sb9sln-3KN@O_tRoAgdO~4D2cI}5`J;D-D+R?v{v%t(A!+GF zqd$+08RQw7jGL}B`sf>kR2}F_zV-Sg`>$cbQ4U@8h%_F$Bp4Iz%Wj3Zi?~bO*Fe@v zLE29foYU!(hSi!fSkGWg1Eu}^kpNg_8^>9P*#_Sz!`cZex=N#{k0)O=&WKOIw6oPSGry>GGkcSnQcnfnZoc;c(IXI0a62*l@5pIRN9I{Qb! zwdngo&JQ1HW~cl8rR_wsPW$*r4|hV2MDy6+@8q1?-1x9Q%{Cf$K@{;&~IVSX|z4< zIC)4;u=3z=d|2uBl{03*PcLAl5+T>rO{x_{EV6t1dE>HCeApD0Bkk-53Gv|JvoEkeUmxLX4H8Jy;I?xIl(Gw zURV_>Xkknl2D>mkfZW_{@$;^R_s4bc+Q_vRwW1h z0q6U>wT78?W683IdiRW6=w}$+(?{^#MGxqsY9*4c-s)SwEa<~^3&J_kLxC38z|Tjr zkcnN)hkc_fU+=P&9*%n`SKFhj5NgNjhFzyd4iTOtNua0@ZMvF}yIb(xl47uJGNedc zByV>*KOpt!($msa9$ScprryWhb9g^WEB$DKFykvfN5$E7CtL#v% zU2ZeO6hQ@)7wB5vm~`7b9rR_0M!^@x?Br6RFZ9jS2be!1C@ZT^KGbc!p%?M=K+WlK zAonv@rC(;<8_QE-N+S#oT5kP>APj^NLP$#UIDUj?4+LKzihjl1aaAWeuxkTEA)N*R zV4!v%WLndS6Q(CMWi#kmE*BG0hx5oHFsKc97&(lV%4srQpC)ad-%0S=)E7{lF=&yoGx zvkf5)3{$Mkmj0Vf++hHiOh24`?0*m2(gDM`M}OY@%L*>&?5{(dWcUTzzlR0>J~z7O zg`F|IY5kipVUF}x@`cd>cupG>x~pUEbwbzv7RAOa$k zN6yC*0&ZNar;B+R_a6GWkAv{LZ?4#8n2rlzBvI@)7G?|8=JhXv&@R&Lt<&V~(3shVG`N!)sDPbX1lZ|4aU-=G>GU1~v!C81)p3XNyxbZ6>B|J~VsvEKSzyI+ zRh3jhd?fj{&tR*gv=8+Zo~g{@wuQl|nX)-;_2okRKh;VjrV@?E-piim`4hhc;eeZh z`v5vV#3EEcn9ln1G)GxIM=f6^&;bU<>`FR#Iidp=Gep!SX7td(395J4Of%WRb7}~P z5<%bVA+$ug9Rp=M#+~1RStqULD2YZUP(h_3XgNHq#X6WJG)PtFh#Z7#1_StQ^mUHO zjIi^RFzwBA7Dp3IGfzZzk!3oJhHHmHi3Ot1xI?PMjhZpFl$fD%qYe;iG@npG5@5t( zqIycLE)-j7+1e{hTdC8?G!O32M)Xq`fJuLh6VE<8pF*;B!&E)8AQ}-_R{nqaz-v{A zMzR1GRu^*600r)jPvWLhlQ3PbHAGC4h+<$a(OXH(2%(lJgx9}!j?C6wE1`>sT)A|> zd!wUN>m#AmK&9zi-7P=}_6~tJCB_X}Sr7iGS33&jf$U(>{I$nPBt$F(W`*h3d|UcY z{On_ca5S1u1#+LV`6qEJm|^amG5ZMQX`nJzeuiT(IuhB?^P1OV35544d8;8lC8{U9 z^F307tR5tOWtm6gu><&=-O|b1H(yuGi-{erX8pN< z%`+BSD?p2gOB}~KpxcpE^1%pwC$5CA92|M#A&@BhBh-7acT;^K$VWWFQIn}xVymi=!(xk|iV4DNKY?O^4| zJ+&?D-Op`*)ZQyBYP2&jnv{=M&t$~h?S4+KhgFpP-PPdej^F!H;$^~gO1w_R_|=5R z-T+yOP40aO$?y+Lvjr@`n4&;$k*w`{7n?d@o2IFD{(VX9rGL}bD6&|&F~j@)r(vvF zi3cw^OGDp9!SPWzw@H+4RlND>;9p;AE0?f79oaETG$prr4S0zWgW%~IN>|kg8f75? zxJ|eemLc-N@aux1Z4V;SmlMKbg5^G07o~=3!uS+BkiEODt*>L89%LbAQzcUG`i%|j z2Kw4<3u2E@*@hE=1&)9f@;PyjNp1y$(BHEZ0JN zH8|HGk2fgwcvmp3w)@rfbXu&G?cgy?tD46t?8(wxj@5e;&y}{%-oCvg5+bKwI@&rtgese#iQ9^E2wjnhxC#MMY@N$ok7NTxz3 z`mNUNGd!g&r3?n!WOyUt#2*{Y0SSuii>J73a)N|OUSh;Ah+kb?H6~)k)M`|nz=+o0 zyAy1GQa2dtYGxybss<4s(*7R4IK~S7evgLxZVI>FuGFz^DiTz07sFXMImtpwGRd4? zu`<92$b}cs&)d~67iLXZro|SoJz{AkR6mtceKosOEIi}H|JlAS2}J8@tKNCvVTtRJ z69u1Lbez=nZ+KKUaU%^c zhBzUS=+j}s89g8F+&|fAFDp_v8E*s| zuRcS`dwB4PEun1{((AfC1NAa&c)Z-_*jEoOpEsmzBRiuVyfq(P8BiYNgQ8}lu4%zJ_n^udEHzW(eBin z1iaZO!Psekd~ko*xx{N7X$*Jt+UV98=nWY z^z8B;dCo^?jC{(0q+Tx(8^u^AUJA0c_wUww#Ls8{UU%u=LFZg6*E{d5R-Oy_@>n{w zTR+xJ-uk4~tFfTNk>#O?q#?lj@G7~3<)d*+q8Q$|m7~^HQC$MlyVHhW+hdiACQ||> zDf7T^DR=f`5H z_D@2dX1`BATs+(zwKe8;-FSZ3)VOjRY;lY6nB8Dwf<6bJgxI)HPTX~A9~0?C%E3E4 z$B|{BTgoZkFOxrm)%(}#h0Z4Pm?b&|vL19)2(e$An?5f`nuos;$3MswGNZxLa$V~C zbf7&Y+!Ao+#teNy=AV8j2ppoZ7^u3x;z=Op^L^Mbe>h*inm3&hLRH%SD1Q7#4$L8B z;CFX{?{jv6KW`h;qJLW;&aeX^XdtQ+Cob52Gy~!``nbv^!etOwi8j9eC5A`}z(#u!z6hs7Adi<;)y@NE8K8Ysz`|z}C+egiq#Q#3rBld-w#S54Hn^ue z--^|F3>8qur3AMU6H?1jzK_?|d?WUPBisq+V)Lb;B_|DMN0(+u0_w}R(6Pm;8>_}Z z+T@_g9;Td64!O7!(|Wn!ZzuWyc!|pB<16L`(T!*Sc3y(uO#u-Xx+NIg8=Wt>vHdXh zI8J6Y9Y}08lT?@7Twbj#G`?ilI#j#Zo~qxlS1WvbJ20|zKoXXuEUcR|->We_r>M7^ z)Fh4H!tK)O<9Qt5HK0J0>x`hB`&|22O5&^&H^kbZ>LLk9mMEj#{RT>7#4gae(~nOD zO(Bh`H{uW{!WnfK_o8rT-kmfU(yKKBwy@MmVN!89DjEepEg3SWKx<5%F$FS_eq_r@ z&Ps=Y{YF{0F7PIW@g9?tNpi5mfc|=aPx2a$T3SI0al7+Y5%&0ClvJ1z`*YEjSmS)v zW{vlG4BCutSEg=9Q_2E|%_Y8%H!|oAWff#u!gH2tO-2jKHM^6zK?(4;X^_U0SIq88 zQy9Yfbt^)^f>c+nCijy(%gV|Vu3I!HT{o1N|I>_CWAQt}(|ehC{{1KP2y8WleyT&F zY^2_$5-w}460!9&=bI^>UB9QhtYK^SDyk`y!v+bzhl>J!=LR*!VY|Iu_q_`#qS2;M z>n9Yhz3tRiI0L;Z)$x~tDQ~#BAz?A!qteRiewxcSc(R*{Tb)v$c*K@_1IILZr!9pn zgE*P4<2?AvSLZ4mHA#InqH^tgF1okYuKw-^*i-4=H1d-(k9@2YYX|itHYeb7RdCYCWcy_yl$cD znO0-{KqCLg3MNg3_H4CPjYo0Ioi6yLhKUy>jX6*LM%c! zbuj-n2cjIXxQOe5LcdKS*)H0&B=XDlZ^6~FBn0?$AX|bXfFnd;(d2B(jp<0C%f+Tb z%C`Iay!tdUak=;7oi`&zN=wyCn!yrC%TQB!qnS?}S=w18{CiQ|EPF|mS7?ss^9*Pj z7pq+)JD&P?cEFw)i zDX5riFxTsY^d9lmMdC-BjiC7F{!Hsyx)1j!o5psUq;dxk-s1)Dm z*zLuCP;Zs`G;Kcb8=Aq6;rsPxPY;t*m-)Kco}!w%tR^*6mmVElHN#4q4E{QfT09Hk zSC|gk@kADBFbB^oYTv%+&K6iyt$0T;aoO05M73No%bdH;d+;1o-H}Dx_+D(7H}0>U zkzgN`u9Y{gb}9{jd;<0*d<5DhCv!h052wneICeuI)xd(*qTvCHMRpYT@;lme z{esW2;_MA;342>~EG?|sWvP>6X$*c2LFYTk%Tt+N2YbE}Ap72{1e7a2TRUeDo8`3m zCx0tKq-Wk08S>)|ISX;!pOEREveg-4H?k~THft;`#ke%7?#$t_cENO z>G_*{hj8>#BwGmgZIwh4<5pAHT$=NhNb+R1TJy7fZ^aIV%SzC>7gf~BvW^CK6HCZS zm6ET~-@9 zW78BVBzsc=fu}F3G)hmbqK^2@MKqhqF?#sL)i{lIewz&OwVAVQjgKWRfYon`0c&jq z3`W4{kh&ge7`Sv{J=`vk-)?pl)I7K@{FJnsA8fGl7E|_V`A~cJupuSSWwq}tbo=Mb z^DdFT*t7lV4vPlK#Z#G9?8Q1wkJ0%&JE@DKyuo~HQ8eE!g@L1vUlif8UKH_k_(J`g zUB!w|H)pQnK?#CTYpsgu26ScD$GilzdFM%hHgESVgjkQXT&WNA)YQyv>ea6d+l>C) zAd6DE3kpeC%$JZ>bQrtE9kpTBH+)NJP_v7fp_Nx$yW&7A18xI^unhBxUhU5ov}}Utv#5WL0E}wE=jglt+Phli=rO57|iR6ihUG;rC<4V z0_6HcLEnTlX*=ply`CA1gEXJeFn=KR8xwkaP^QSe#0bZc`$swUcBfidtMS%t+0+fc zFaaE(;&*kxGw)R4^J7e!nD$goZUn$R4mC84bGU~IJ7HSbWc?L!39PQ&&EZ+f&gBhE zolnk-eYCIxN9$0|JZ)VXX7O)s%X&q{GTaWSV+%)~qGv|F>MX;pEcD{@1y5k1$H@p! z)8+b)#ly4JDHPD&8OxICtr@qU>5d~ln0q0$Z%sYh7dNkpnU2LE<&)}?5&AQD z*+3_Y)m8tKoP0i~?n)!R+PaBZ^!pxHgH1-O^d(D}rxT1k_bz8gJi10+E5N|Q#w3Y( zi0iPh=+XdBuq9EELd~wGFBy1W{CWs*>y~lH{T^?$oA&)0waRWMZlcN!tEIDi9-|;l zMJ0`HmXr!N6)iXAa4*VsW>&n(<6{jSE+~Ai9+ZiAs_Z&f%}zVsd^Xk>Zyf1BH^^eK ztmC%sos`+MXwSZ@xeh*YNK2J$N!f=mx0wdg!k+`1;&ZNMD1qi|{RZNdv-MDC{kH{Z z)nzMSGDW@vOr|50oo4)3C1y(z7nA2Xx^3bJNIO4^s?Wz6j*E;n|yvO z8=VyOyoh=EwZ0Zzb}|iDze=SDC6m#g=%LRUA?Ep9FR)KRM2E%8bl%zNSeP36jb8@; zW1e0G{aE_Ovs;kmHzk3&s0&X^q{^9R*K(fCF(LDc-1O;iEp#bsEZorJ!Ql3^=y9< z(B?iL<^OB2{_ng};C*7h(p&Yk1NY|q=%((C#q;38d7S z0*_C)ON#QX_PFs)fG!>>l^Igb;x`l$sGIf|IATaj{-M|VYTmR?2(ahU*k6cUiX{IP zNt=#~RWTo~xn}EwV?1cA0JUzKt7E!D>`}*(Kz)p3YYj|o1&U=ZMMCfvFe2Z~IIiy_ z>;O@d;ad96Y9_w0ZqhFC3(^{DCRGfdYT1GtHd1OLL(<-TwN;~cuHtYLYl%Uc@CMAq z{&>#@n~&=l(!6I{&H5(tPPtd1UPW59t{=B>CeW0S*iEa&ssO$6Hp}&-J#28(gC(ixHP*RWPoB4qn`wl1m?D>vehXc?v$8@{$UaC$l`pvi1 zN_B_nC>J9=KE4OhUu0_G2OLIm5bx{IL52$cmX^WF^Tx_}f6X`|k zsTz~d9@jyOo$1A{063IfQ^KyL^%wQrN}Z#oBB5Uv5VBh!1gsjNWDY;>dVchoaGCui zyaFY)u59`Oe>AwT9AFVt#6HgH$4W1y5uMga+$F(joC^)>m%JN%`lEbgFC@k?> z$?qA`E9r+23&2ia#xYD*8JO2tx;EA>T3H)R7KIvm8kaZK>`Yy?Izi7Ku9m02_EqaM zG`Kx(uPaS|TPmL@^QmwQsUJj%`Ta}jU}spJ3OYxa?9vK6@^}NJ>=KQ>l!9_uNV2ruO53*1 zy^4Z4H<#o|pv12X2cxLwi}-Qb#^}<-{9D%0qs7fYrbU zK^lt{2WPoIGF`OgG`(s*j?5o1{4;p{bw5V%QnE&%qTKLlwfA+|ALBd7db#Ri?b>B@ zY-{Bjt((jCnNyb9%;nHh^|O~!j@*6Ve>w_fE6<#JnpsZqUr{-9xYtXdb6K#HESC(* zeG?vsgP$A`&MjHU7+juEgHxqZ+wWQZs$9jn`eT?yypVhnYS=4zN_l`QnJupKE)1rp zI1T%?9-W;BaU8K{tq8PT3ADMTbX(hY)do$rt+>DJjVA3zH?`~dO7E1Hv_5?@iw(`O z_Ivn|M*7+HW{zGUWu_m}2i-B!A)6g)>nV^?L@A?G)P9O1Ltd}zv=vFDh#KLg*9OPZ zAeg)kmUI!ENn(z;Bd%fX&yKBbRrK56R+WfteHwd9HGcir6hF54<+fn5yXex3+#9~m zzzB=9@v@f zlIg;TPHlXi*G2MLN36ad$WTA{x2YmI*{;4x(=r|ykOI< z(n$G%1ud%5d;KEgR32)>Qtlw|e%9j)*u2T3V|svH=zCi7kYaDu1VkS!1BNX8sm+p$ zZLARuyrBl;!f!lQ1w?$~>z0WDBw0N`;)NdmaHmrFFvRL}@o8eD`U>kPj z{?!NC{n^W?ld`+;x|LugGG9Ssv3|4_uPr8L-(6KadM#Mfel5}Ne$A+wB7L0pA*;w= zNv*rEkzZMHLovu$^Vjfw8!qs~`dun`lA2z|)a`w|ueBRtnQgg4!ei0lrwkvdwGJu5 z?OppZ;olXCnf1#_XW6@OCfi|<9r2P&m~~7jCmg4#(7NQ(K&dg#uYeWPNMbj6X^m?E zz0KX!>!N9Y^QDg6fQ!JW*Fs|?ozOu~aAdYw^%uxp5qy#2suD{wg^S#x-BlVv`VZiPI1 z`tr$7lJH50Q@p=T-QF?uX)B9S{7j2gv?O>sC4 z*l1_zB}ajB=dr^P(S<8y1(t8P-?CIw@r_O(YC*^6NZp;P<{Q-tvAjrJXkV?KUEf^Az*o+))YKv$;12+nMZHPc|n;PZ2Q~Qw-UYx_$qoQepkU9 zYcX@Bw9mAuD{i?~nJlL-h6b|BhXA$`-7=D5h8DTz?@NiSf<)r9!q)XCAP?-x}Q^{3n?Q}_2g zPj;_sxXkwJV|8sVt=txr4s}S8N`;Z>{h2;-rw6Nt zaj5zhS>^2~y*)kLy=aFV5+!&va4}!M=Xh@RBob96DvDZ`s0`m_U)_8)Sn34B)ostc zs+gkYkb$gC!v%MaN%POr)Ly{SA{X76qkQb`^M6hzQC$26vX;LLI~G2K36k8s$evHY zqwigZbFGV}EkRQwS6D4Xog2cgO)x)ghtRjtW)WvQbRew)NF~k5rT4-|)C`(L5cJa7 zaNj?1bgt3sr4rjkZrW3*O0y24r*DUA`u0ay?Ojn10{LpwK>-d_m>^6O9D57T-=E;q zu;vlTLV9w1L&7)APE>uf=Thh?@4&cl%`x2PQ+|L5S$ST=H&$GGbHiyng#V1`e$ry* z>n7^t0;6vp?|Ir%Gnp?W@t{TLEb6pT-EF#D{3pSpXbC5>I|CIUW!JtRe}l;FM~COp z>BWMIJk+BpJ8wt#W_nLs4CY4}<>&_EL?i5kRxeA$4BN6@#mBcL{aMJ!|Ngozx;L64 zaVgH+Vvfm!@qw`quIXm(-no#Wm_jkp@3({hy*Fpf?YepKljCDBBCSO1=8zo;(pSsA z=V*ML8phv!reUd zr!dEt5SfWy;EOVbNiCL2g^2_wN#Jgsd)-!SdJMRB->)8rwJyEq*8mU!Yz|d=r;N-^ z)G^<(sDW`1Nz1};k$MR=yh|2)$n3qrHw?NX7V^YFLLH~9xcrF#{QX)=-ajiJ10VV# z)p>6er$n1sqC;Hjc;D$k^|t4qr;Cz)S`=ctxZ4DQ=QqL?gjolI_NzQENCh6= zi1i$K@-!Z&O{uT^@{XZ&24|Nft0FeqC9_wX9i z6$Zi5SBt>{PlgLvjc~WFy*mXNkVtJbk$aZZWo@_&ujW98inmtY?=~`#QnO8)hBFub zncqc8V16lUi^F!@!;Rd8-X^{E2Xay5t!hdsDa0|k>3&y;loQKaKGH$H#=W^62ILJTwqI)^*{0B`)Zi8^xpCKv-{ahiEdVrM$TCW z{y7?R^DGv0faieC?Btd2HOxF`iJkt!TF!j^ajyJmp2*4*Oth^8(nUvUx3v(@w*_A- zbI~zoFZ@l7f0nZef2Kh36nj2@2>M|JPmh8`i`7OX7xcl)mHTz(@12$#29@hw>+ky$ zF{)4GFQe>+^e`LYiT^SeB5uw#pI{_hoh11iQb=Z*8qj~; zkB<+x1)+{V++F6%-Q-%#mE$J))c(YRdYl;qjCCOq5<8UrWGl3aYNYdaaiEH8{CI(} zPPm3b;k&oY&LZm@_1_Hn5x9cpWRvE4+!6d|W# zwhhD94;%6CLeSJuhwrSlJO39#u|4{&j*X-4zbHkNvalhRW^C(t6N8yjR?zSDWv<E2PW++Nx2)hNDB%2E)MdljYY-1VoDg;IN1WMncj$!0PQhnY+xHFn*&CH>Uhtv?92 zSNLN85>L8iIOQXOwf!{+pUHs~($q?kDm;x5Q+MF}fl4Dw%r1A4DN$uFnmG2Zx$U}2 zxED?LF|-S>@dwb}x2xRPPf}oq;v78rdBuWD?8yXm%beDLupMWWyLBglxiwygbDeOLcCTy4(%K^7um> zyC(jotDnPq*g+NX3%;prfzH3a%epu2iS9pLfLv`eqas-NMP4t|(Ts>!PwS}1lJUAD ztLOWD9$Pnmf7z`3-F(42-R+PM2TF@;e<2l}OcHf++Vyqv3rOXj_T) z*1kvQbJ~>qOqEwAQ0w#QUs4kl)#&>M;0j6%>WxJ^x2TxemRKll7b-R2N=*!Dv@&a|Ep&XUr-+ejTclTHfz^Yr#nC za#3i)vvam0L&##cQ@2cXTAL6m_IbB<0!G6|*a)+z&u_uRg7+^xUF=-OkEnVTBrcjD zeK_iuCDYzfaT~h~;zTzv@$0h6z3bZ~YUB`V6-{0OKxlJrQ)28VA(knX&{*gncaT8t zETxghnS~5{PZNnFYx3kp^nx7|iA@G}cj#h8FttkO{f+!gjuGe_6e$Pq-?PzyOO|vV z-`qzeG4i}F)iFjdg0~!`m5-){BT){b>4#LMVZyUlqt?Osm@T`g=`lUUX{3Kab6QZI zFpMP=xI{X|D-ZvKgh{w3jY-ZIXuIH+GlFJ2+IsP6RpYVh<=zYa$W-6DgHXn%kjY(G z94N4dTqk|italN_ffiL}?A`FPt%L^L>b7d+0%#jE^={*sN^b=v|P5H`e8Vzp=?ww%vBG2puY5kr)0mtU& zPyB)u{XVSK!rb4il}$-90w`ld!$E;13plOTo6PI-u=fueOGbO&XF94phU8{bg^AGo z(&hm}?D3#XI=;*x)`q5EW$RxkTzXe5A#+3Y4U?N^?U14t$|viY`c*imx(CiO1o{|0 z{=0EKR!e8Q$&!aB2~!WK*yD$(u|GKgUMlBgzkagE*x9W~Q(4RRs&u=9O;5Yq|GK1r zt@eM?QYNs^>XzP1?jU2en3*co3Oh!@a&UC7Nju~6bHgq=z}UsHOKM$9wq<}swvA_J zY6Dm&s6Kj(P%Va z5?$Dz`K7rb*L^w-CixLkw#g=!5z{LeY%CaTF#A6@$sS6Lg~wA|pg5apcGayefgC~C zr0w0|ab`&{Q@a1&+5TLd?4c}BCd?4g)G<>%lRpOx3F4@c>UgQ~yQN^{&;H1~Z?I+^ z!~!RcV=B`WvDKP>R^4CZ*mX<~y_HMd7%C^m1L_SZgohPk2B`M=$G?2rsTjovtyZ^; z&T?~Bnd#;89UpO#&+93~Quhz>z6t(qzciOO+9OIk8wd%c|Nc=3-+v#(JgGZA$O_e^ zf9|mCYpiRaVoq5hdP6oG=m|3LH3EmlelYD5AVY+9j?-(#T9{EZQCg=NpX}^yGbvGc zNs4xOSdr&;Ox^)_hRsNsdKbvap;w_5;s4x~H)*1)JqUV%oO3_m-Wk4bZn@5te0_m9 zN@D3W5a1+eDPf3?{oK~`H(2!!9nmUd$*KCzx>8_+_JLa+xGndpR#}(Ozf%$ZTo8(y z_5=0FAPzLF8g(@hC;;Y1B}4~aQWSn43waUftW|+meZ5xTBtmm(7MFcA?OvLO8{Ko) z%8}tA`mq!)Q${!=jAE(0B_KmW-d?y zzjiK!U||fKWcs=h@%G01(x^lLb1X?w_O~d4|9(qBPM|Pof^ZBE`Cu@Qc!fEvUcb@qH<&!R_4Jgpx%9 zJuPLKfP0VcAEprRZx_V!N|NgHvGQAaMC6s`^hVZNAKG$kdmjndJc(Y7kcFL^%FD%? z)#jn5ulLzF6u;N|{O zOrMD%|Ck#Aw&_Rbjkvl<%iGbP2A50=7s94dJBbJ^y>ktY`Ov31tPoYVIUegJxIJ#0 z?(YE^N{et(mN|hpxy$iUp6Z=aiA3w`9yv0Ee2fujmbsh+7{op=O)H zwD^|?@94yKznwI*2-rvWdO?fDp5p?0DiAk;MfF@OQY_5dxeXB7Os2xUYACu!ql0arwK>YqZMXOIL22+0aG&$4E) zI0UKh5gjDLQA<^wdRSO92SehLpq}y<^0V-dsH-F(z3O6ez;-D5Bp=HU+k)Iagzfl9 z+I|-;C&)KJ1qumCG``N%1%a>(WF^Hk&anMofu>>I5k*x&nEZu%f*>!DK4>-h`@Q#J zCgLczDCDf}0h9@Q9P+7g(QFPr+l?S%9MlT3XY(gt(t5wSf|L#G|5Em?a`$t4K`8cc zqebRIutJI=nJ?tdjgs2TJQhGN;ErKIi1^W)Gs~s_BI%OoYMs? z(819AkWafLrVaS*da90&f}8aunT{~m=&tdmkS0hFEt|mqj&#x#WgHFWo+RnX15}z8y)Odt zg;l|9e+N>Ab?+AQfmEr)o|fR|e3-?7ItMl_;gQ8@0~;EJW+d|Gc3JqwG~v5eR^}I| zK}hiFxp_~XgLqMVph*}}V|O%vWq|U7da_p7>XNTdl~o@g?`n(9j)OZr0}1JVyK^D7 z6M>$-Ye$uo+-T;U4dL+mvZ-`|;3keXd0!dA1oPi`L!xcjdI7v_%f-#6_%>ojK`F?^oh-e<<)h^U8oSqF z6pIRIFFnN^yEoy;*JwAgGj9S?FXBjPDUU_1va_LK1#`y;qo9AC>suO>NQOtHC%G4`1uCbSU-Akb%!)9c$<`O!@&HBM==7@u$>!gUnp9w@@Z~P{u^=_WPI~Qf#VjZ-% znjm!aWxeb*bmy1=2r*T#z(~7>t1z4}9jdCwo(dFu-);N=k&0VPei@S#eu0;t&YX?A z`oSJ`Bx7?DZDp5N(Y8$H8y2*_YMN9>{j$`qf?o87dq6Vaw&bQ(p;G!wzbSla=mep& zP3e=1Q-50m{pdAfzl(Te;CM$c%`(G`_PDHyyEG0o&16wIg)X{C^Mx>83SP9c_kCL; z-K-ru0zCq?R4uhLId`xv_SrXo*Xv;vU&^G~tqnTD!1cKOidBFEY$$qM$;eJ<`L%-+)bSd}^6e zDK@^o99ogVg*F#}*FnjP9#r59-g=r?uw-fm;~f1L&v*lK_WlSi(-CC)6<@-l0bI)Y zU$?ZWozYL2%6KWxO25@FBiAYom3l|N97jc~kCzOEX6^~4TuUo`>83`og;7Zlrm;pZ zy0F8G!0Sn3jrjAip3H&>R6b?=gIyz*Jn+@Bf@}z56o*$pa$iHb>Km`zhV7}gK6^35{p+6>%UJgHeaAi%DeF@K?nM)kAZJV za;y|fw$gi|(T4&f1+nfIHw9ZEu2_4LsD9U-Z#8A=XkBd`pTQ za^}4?MR;j8y%ZqtPo`@LFP@^*_d*9P36h3*%#|+QMTe=Ad3Qy-ilifcNJv&Sp|Sq- zD#Mgh*tCU@1i3###*3Z zICz2~oqcQJZC;MPx+ST$*x1j|bZeoTo?n8~srbS+-c;YV*Bcec(7L&b^{IaPne*wB zwbw}CWkfdXrlr;|e^#G$R!GOlIfLT*LuH_rQb+&C;~=ZIzTzv~zlIzM3ve8h$qB>n$@z_RV=+ z$KPW*R6}>vR48gPF@evD;f-vpqyFbVBSb$>;_;y8=8I5?l|c|PRdV`c{`C)d`fhrj zH*BNrI#kj!d>AL^c&&>QeopDeqCvJfg@2fRhW#CG2%Z$a{3C z&|e?%q_^T@o;$h2Yk5Ite;E^iDb2IfOAqE#l3K?FDpiF1a9A}N$=`noe8)A`8!%~Q z1TDCs^79^XN-61~ciP7_X*%IiXPn<#1!G0_85U3{FrVfyJub68rch|YTHi#Lcx6CF8A4)os;3s*Kh8O5G|N94AciTPf@#P=c%Y zmGoU&IDYYRx7_#n6j9x@NfaA=(8zl=`zl3+W*IW|ZhR2`M}Dr_tl>~~g~jyl9dBv> z8=Q8@avM0m>fFI-{@{PK76D8+A0p1{gDOlBn6sSSyeII81^)ZTe0~I>u<$&Bc!gUb$_K_u6B{7 z@|KP8sG>?0Y>l5N#sm*=P2{ePX2XDT0Cg97** zAB~=^loF!9`sG=|L{b4y$Kv~*UHmeuXUhKfVvqKRFH)v?_Je*DKU*AsuV1`?DJYzx z00@H7V(EWxm84il{uoK|;JX#`OTE4r#{Ew$v&B6q1{9GVo>2Y?-RMj*RYbkA1Z{1Z z-z`@&!pq^&$ek$aiJma<{MVPf?b|ituiPduryDJJF4jvB`(K$E#J8_$i7#$3rMSZX zs+Pa9!$VR&mqM*^z!wnN7IT^AU9fd4x`<9I8@*T`(8g&nr^IV*=2*>_X=j^V1juZ4 zF(q!#c)?HcfYwSpi|~7Qa!OZ938})o;O8@44s@u|D;7MNxYyD<*oLfWKFq z7=_7e=*C-~p79Z_oc96EA@z%`k3N4IBF$a1QcIC{GHuTsmK(O-U(74gIZU)jL!E~T zz6ir*zDTGX0EbYNsqHzGzH1q)ztXgBmEJ+UT5oMC`LOsI{|&f>C&%iakpzApe-@QS zRzn?Ske}|%NbFyWmNOm8834UFoPmiEa&!|9JTL339$o!yZhxjY^I?o}uS=K=+l9O+ zk~l1%)z9PDT_K%a;P4cISbWxqqFC`iW1fhIg!_1?=9c!^&?) zcNQ3xcAO^eMJ@2HNACBe$}xzi-jVnUBF#QAv;`qVlFz}R@1Af}KhgpkvYet!|4h}p z-rvX$KFRmmE}8LP; z0bGdQ!pfty2E#5Tf_S6&DtOD3@oxLg6!UD2zSo6@7q1o_^ya=2`#0}n1GcHYSmVV4 zDFKIZen*bhg&vy=Y#S#%Ac!9{YX3u_`n$o83yWKoJw*m(+r`kT0IxcHhjeMJn;&v- zA!Z$OL6?8aH~;yFJT5ntMqB`IPc?DWM&6@?pNy2(e&@=VK%uyZ*QoX%14S`}3@G;l z;7LK_6-hXTGPh*cofW|~o!XJ*HK*LGHdF@TOncPv(zjr19t(4lo5@^RgY-lyBtt@W zvNFBN-V&CYWD3PAELTp!?c(RnmAh}l|xzKhX4l2mzPXMJ^^Ow8}V`^3naoSY}f zQY|(33S#8%CfQP*hiUWVF+V1=X@{TeyUK_C_3dA9o6}$j)ehc_Du>^^foSSGQBb150`r0~pyw zMMc+Cn><5-?%lsW?mq|w6TzU2LvtbxV*j5x0T!B6|C$D`V;!EWXHNlwNVevGSqwec z&lbbS|67aU|D@pW)L@Ro!|{ra=ffgm)wo47pR6>Y{|O!BpR~s!vERFej7e>yLBNl! Ml#*n*`1`>B0W$aH*#H0l literal 37069 zcmdSBcQo8<`!CG4BSeW720;WNh!VZdjxvHEM2YB~L^pa(bOzC)6K0e|q9%IFXwiFT zL>;4M%;;s#NcR3c&w17<>wVX|&N}Zu@g-*FzOR0LuFoa5fFqBs46|u^)cPfxt7i$QB%VgnvwA&-3{Ik5m~Z^_xJac z)7s7piHO)K(%Q~$-&21*05|D6+gtPBI^r{OJ}K!najx9~pZVt@{bvhx9;x%D#)DyM zH0)ti3MBuB{wbh%!i%aQH!+*3wNizaRyKw%acNi)Q79%!4m1r4ppKK?eeBwOjm4si zgw9U0IFj}~1*&BJK48vLd(x;Wwl8V@=sLOky?5BLq{&Jda>ZoHoGQ^Cwx>&@=4>Ei zKHt0d&5(SJiofqP!q`N3I0Un^Uf)f+mxkTymB%Il&8b4sYq&)ad4)^zEOmv7+Ub!w zt~b@{FXHsyvM_;2v|V=Y@(35f1e5nKaeNNCJI>uvN$WMQrTR0zbZxYyx)|$-$83IU zj^}f*rQMx*h{_^7ted&c%q4fGMG21um|Ll5ju5Yhl&(rLKhGB=228P}StGyxe*Xg1 zO-A#d_M=up{WM`i2BTwK7v46DI6l-Q=bWo#J`BjU=7ZB8y~f4zbX0mTp1#3#A$NbT*ljlh3R~2JQ^)z@&Z?7&wxct%nYYi% zVF4^b{)$A3-5x4j+e424+j^ubfM;!D@)U5lwAJ(TOGTX)m2~O*W}&W-wjQoP?HN#- z-m=B{am7<#=*xVD^ zP=@pEOy-UE))JqDJhS&v!hfE=3}G>uS(mM&gv+WXSa@4;Iac?I zGZ~PEma6#`!9@7x>h^_o)>d}2R~ru>B`Ql^CShR-_;b#2ldQBbcET?(nRl{r47TdM zAO?#?tzP1L#0*ehd4b&oDkk%-du!ARHNv9`92pp@!Hs;oLztry!f5pP&Oayq^FaU6 z`BV}skHf*s?T2-Hm3f`g3an*eV{O+VfZ(#>kWd5Or9iPZ(^5wxZ{_(-I3~W_5o(qS z_Eeoq5mmGaer$#enV{|3c2Bjs(?{SBts3All^pF1f^Z)SSCLq}2s?Q*(sdEJn}!#S zk@ERHilQ6KZHlRZfa+B;4#!f?6KL@7U=St0Od&LnxtH?AS9MPxo-X2PDQ3$EifJMY z$(ogT4WQ=8*5;8Eh-!4@>JMdC@}VZ;zs#o@{nj_wSMZP{u+Ls5>yrz~3mWApb{3K4lXl49N zZYYfEX?!|AHKXbql=-*cuzk-Yiby=bs7XpcoR)B+innkxig4W{GNulkSLU_I&Ei`! zBnCjm*smMF6D8WtX-)u^ho`0h^R|*v0o9qc;YL-zrF)WrT&BT{=DBCdxso1waJ(<_ zzqCX{p8+z3K2*i6A0)byP*k`iv6Q^6H39XTW_MM0af-o z;`d}B2-m$V-5Nt}*RL?y+Mc|zy}7$2M&&f`RH%uzb70)@Y@d4{ww>b?8Y22yJ@$e| zXa=tVk}i*jKhts~bt<#+IjpFqOSAzNGJ*GzuYvC{KF{luL=B6oG+1e-nCs09;Nibd zs*z+D@-19dD>O%&)LckIA*{#otsDEuo*LfXRAYuI#2k^bh3e=g-4u@S%Zt_>Zo zBk9yZpHol>fy~cKj7nVdsa!%VB+UWV3q-qqfUq{2pFpz1N*>}V?A+4EM8MSrOh2ur zE8FAU1FF?uxo^#Aejdn4QoGM5?RTiLe1>h{j`hCz+fxM`}S{JBY&PlewZr>xkMiWWco4t94}ZwamR=>Z~urAXSBI$vh|l;rb7r!nR(G`Qp2!o$@hwc~14DGH07-Ws;b^H(~#$u+h%t-tNU zk|5bGg!J#czhP(n&$Ck2Tf%#PXIv-CELhWvJD7KWfB%21PWWGc(BiEwO|f;Lda#Zv z6Mv@dwk|G*&U&-b>XNTVGE%qHp_q|5T=grHG}xZ*UMG`$!1JpwQtnE>zF2m&-cukE zCQ!v8CD>F)-&5Pn((g+vb*opVXf~@m=sr;^w$zcnMT^pX1|?(fAYlJE%;|6^_MsD* zFH6Bn;d8ZOM4>3}y~1b0_j6*R!5=@2gi~3DQ0dtxiN00Ns0nYP96{^w7e3q5VY-z( z`XN0dpMbN}7w2k-lFF{-FvA9^?ESwpO1!bw1{JM8Ljy+U{jj0>aOMu9*5jg^4|P zj)ApK;9|A#3>s*7*QC{ZmsI;$YcU9(8jCwVys$TWC-1jpnno!;ztyrO_&E4-Fj?CN zx|smp_nZ@y4-lVClA%?GsGZ#M_zQLU^$fnz;14ozeF#Di$+c6}F03(thJ9sLWD+Lg zaSbN53!Emt^;~!f&N`IFjW(9aOGtPBv9j)tQfeP$O)mUuU^bG|7s0XlMkHS5`$dn@<7qCF(!q9jlwM&zTrwEY6!0gKxVnDv!LMcbYqo7!5CGK|%U} zLBcamzyJ)_;PCm(Twrhy1!v+=&D$w<>6WP6X5=P#JcVO{1Tcr)t-1i`P_vE(zso4{ z=AgHbl)^y|U4^)V z(whYsIz6*7(u*Zb4{9>&)7GGN2LUucLD-u46ksCS?Q@Dre9ZbLL~j5Uo5WGZ}GvltMH)dB{6Wzv-Aul)W;nZOMd z^77UVUAxs84Bai1?&kq~fFz*Svk%qwB?eexV*^`+<2DK#H?Yuob^0l0Q`;rsI)NPT zeT%ZGw_O5IN>RV*BSf`#;n0G=hDHGB4#H=7a#K9c!mmgQNSy2)zF*)LQp$AZ02sxn zOFBOjwe+n-1ZsFGMuDO_Yv5ugXl~ye0G^S$VLX?ii<%);1-+ChQ ze9s@DbYdT@g3OB2En~0?(s8Qg{CG`NWVqx?jmmjXH{A;uxrWqWt6v-C3DQ0g%dsEU zkQf**L;C0C)bd-0IMp4#%-FRATnD%h%?Ij9TOR+!O60WwIxAmx5OC}W{j4%Bj43I1 zSpC{3PCLdoB-L`yvDKtjbIL?yF{I$OdCIga$V$~?^KS2W!eRh_jFnyoLE}*aW~`9G zO!CH+hm7SIJ9Fm(myseD43wDG?3JatUK`sQs94*+C@|>SPqx2>oo7w}e zfGUK1oNkvkd(FcqBz!EV1Hm+Lw}lIAMdQ4gQE5ot`K-sqxtsXTgqTu0TpXY`L;%%> zl!mcoyf4l{Jn?JSk`KrOFP$LWD5P)--O0tr5&nkg1a6*d3X*sKBvKhKxnVr&#x~;rFlQaIzjn z*Z1lJv8}Goex8g6H`;6G953KdVj`2(>IbWyQSpKW{2Xar^c)i60`QPxHx~*IkneXb zm)lm)$3iN*jn){=H|j!-%{r`Zf5Z zXUo}o?=`a^*dZFN3scOeCL(vLN6&^Et5%OsiT2Xk5VhYXg6}Gh8vqI>&5J41eY;R& zuLO7`-0zo;iyZot-X#W{v_*ZtKD0=?!}sx0ja$0%nC(MjMxV_Q$7X+pGeyE0V}5d+ z)4UjFR7&?9VFbgqsXMY@T=Pz^$U?Q#kTyP|(%QUv*mk>K&)|7O86oIJztDnchVwDN zOt$uX-LL>r>Go;E^@E+yzUfzJ)wssEk+?H;qUNJOfiNnfh!O>6$UgD(I7MJ1ADg|Jh9wbu#mE%Iw*q1S?Yi$$>(b~gU1agN$0JSwTyYLYQS0<{#UOh5CJ2qodT5%kaeTKxc6#JS zjl(#;%}WFBbCYxD38i0H{vo<;|1_dZ#_*$Eh_d{bW`gLA&gPMdA?9YwFW0QT z3*7nkUD^X0z*t~f(1LP$orae;{5>W@-E5oB#0nK z3{y9dw(PRvk7a5Mc#U0Hs1U%XC4L1=sHR?|j5oTSDrwq&0lUlfLLd>kf* z1`r*$E!29JaYKlO?IJ-c$IBL-Jj-Z!!$FKl>5F`<`+M)~H0G{%Ia}?Qc$t$~4}T{c zN+9g=M4i}O6EzL!fG$O!srZ!n5d%CZB@;~U$0HW}%ASO(p5w&0n&k1(D=go}AlHhy zy}wJj&e`Jmj4ORnn{pIuKnKYK5AP;2)?KNkm{*@JK|XRs5Gx1c2n%c!ruFi$I<4SX zd3*|d10Pq~op@iBkvepUY>kmhDD9$pWXi#34s;QyRRc^r!HVG*zA5(gjpsz_tDi@O5^?MCoPhNzEw-4+klNgp!3QX#&dC7XbE1HJvf@Zst9Rd`BWd-FYj*AxMGBHypIS@f;@cFfbr zy~gA%VVhvdJhIB;M|c^X5FiMk)G>1fa=`sq?%|)7mQ~(P7T8{`jWW9sUk0TuP53;g zjGaS84EE#wrGw~r!MlFVpULKDxNk47%TL!uB|ZSL!-f3TgnR3*^BD&NzU?=Unbz^R zJL2EG3;hN&XhPd{t`uKe7N|V&TmDwov8e>@LO>O6by@EYi*au9qX;FaKvaM7%1I^O zo`9oK$TT}<=(mOeL+!;Mr?(@P;Hp-$MYQ`4T2@lz-R2xkC4?Z8cLq2*?O@n59{tz- z{<}gYj>RNUQ2>p2AXTgqRb;(~g`}s*5`w^LwU3z<0-lkAMll8bTF8SB5#1HU<2gWDceFLv&sV zxJ(cqW#X@*3?7Z3+T{y7%e9>koVLw103@X5q+_*+qSGWen3jV4ZZs2uIFbr`+Ajds z2!i&CsxB(IGJsk3;?XY6;B8{RGj!=u%kcrZkdjr$Vk`qKawog+8}EL3P1F<@z6wi! zW`_xp@)tL2KJH5KBwIa`1ekkexPktvFZZMPDN6*2pyg{BAs}l<UU;HE972l3P zUUF!q%-;MO$P6$@LgkPtfx9mDGNkec1)tt*o$|ON(~{bEB#G;1n6}H2(UQ1RoZoyJ zX?5w{hu-mL7!D~z-j^YXMS(fok(sL8Ag$+g_sOBAP{qD3WkU*o<8Wbtb2!MX#P8TsBG)o#be~!z+RfuD z2Vq=EIg0LM6d~vo{pq;t4jXmXo!%FygcE5%MZikWF6QMp0Th|LM?dpR0m6GJvb?8m z7Ub9F^V>lWyZ9F0SGj#qEHO(bdwJx0ORPf#T!Ky49}lBo{xtX{uveE-wxg&82* zSP(2?=GCM85T}K>)^vUIf#^sSVHXe6$(U-b*?1P&U60s${bSUP-aq)_XX~PrKllJ= zT?I!okJubKT}D&wQv68M3=y?8p&^>;z~1SSx}%p)SBW#~mr0AXz0LDpQK9g0z!bb^ zNJt|Qq-1ixoa5rp{U&?Mbh1fwqaK>LK_P^@Uu!esPl#^+?vi6xF$dMldW`^ zGKc$i)smh`>nvSW;AThr^D~_hRUI2KY#Lo5tHHfSa>-vTf)Q%L;K@oh#o^s*cu57> zfoOs4SWoQ!ipeZSDRKyun(+3_jv!*CSD96o@AGQ8Vxt%MR^3AYeR&Q! z-k+Zk+1UJd5K3pIia32->TmEUpfoIT!2YAjZv8|7;kZGqbh%d%ZJf9<+RsS5xivi2 z<=8hGG%+Z;58o(wvc{-)aUYr6JRE>J+G`b`or;>x(F8wv3sNSJxTJ|i{2&~8!su)# zUFpZ}as6!eWEi3e^b3|yaA}W?`8OC$XC;a7aN!?RtyTvcwb$6EukBOX@t6lgT5asY ztlc;kee!e%oMmGpt54@xl!(|B-M7RRl(SVjDeTl`z|AWDOrh}3w&)+&z zuR|c?s)2_i~7shxFqG&uwz+fBY^7xzK;FzYYG3N&G~WiAKC zhG)+%JU?r~p2=+8Q;JkUoDUZKKZfTtl8CE##(seerZI)!O%pZf6|i`P2|g~)WHP~5 zE+pFF_f&}tQyhNbM1q|<(gy;C+E>8h6($IKrV)1)tHSWH%HFbtWU;GiDO5W^G?^;| z^GimNuxmTmtTF7OIo^CvIKH@o0c+K|EZNT zX&>ZlZ{+K0+T)homWvmAtqz% zoHvJm6f141q`Aem>e!t$`I+u-@Um9jNW-}M!G&YH;dV$s#S%lV7wl{?(}cf3a8p+r z{rUacyBA;9t=ddQ-(A$kD(tjwiSGtsS`Nl}J$d7(hWOb67HzVkN8~rV6psXgtr4-E z=94BKYv4=Yb;xq<3%pDq*Ei+mIUSEGrSC76+-Tp}es2BRt*q<|+UT=pA+3$ol#@iy zh5hLj`Xhg5$a};#JLtuci8OQT#NC`Xr&o` zk@gscZO_7KRRem0v(+c27+?KF6b?rU`aiLPICe%Mws=NNbt zY{zK+s=UQgH1k{4cE{d5BFL#r*^Hgr)X>AW^$d@ijTuXn-?CRMVHXlmgRnO7-r*;L zBvyXtj{o4hXjzdc4|~RqV3~jQRs71el<)`}=T7t3JLf<9UB;evUTjnwu!1?B)bRvj zzEw|L0R7K;u+cJQu4ql*dI)wx#l>r>>8U*(Eog=FHsi8Yal*X&2WIvLpHb~Atn-t= z&c#*NW>r@I1NN~GfV}FJOv*(U11*8*^OHJY%fOe*W4+Nx=Z5KU3*!af38@hrLk#Ey zk0x##Ut_%1zV^MVa@K3?AUiR2tttxLtB!fsH&haVZXF#)p7D!c{`Fsx=S9l54BPa< zz{dN11l1I_y%Wo^{d*9r-6?UJxqMC7*sFwvWqI4aDE)VYw#@yq)g8c-pMed%1noKT z`;}{5_%0*eMQG(6s{Qz}(AwV5K<`anBF9=M3C{x-&tkE*5;8OmNxKhm_kdWVOc^t| z5V@}WbGe|2wS~yUYVwM1=f?i#TDTC1BD$|p^@>+O$Bl0qV=hkyWKbpdRrKR{!rYXg zmhpymUKH~K;($G-JC|7s0I4QjhZ2kQY@cbbmi!j5U?An+O8nVI1$E#-np3`5Kwk;N06 zfn@ZujXG1fMV*wX?;u)x{`K8a#IIpVtquTQ%HOK%*TbmN7*Q?Oc6o`BfD(&P*pNFw zD0XY{;%NEM?a>=WAAuKD6;nU$lu5hXP@Rff>w^ka~KFO-LdL zmb%!L{Ku5yde9OjKhYBs_83|{dm`m0e|F{NnVPai5O(#YQVX%wU?!OxwXX$Pg`j~z zk0ZT_izZ#67W?e5ZmKR9a-sPAqXZ|W5rFj--uyMOn`Bzls@T1?nb52Mb)(+U(Ws(C zU9JKBi1}Q@ol8`AdBX^ItCQ!RCsM{LCj9!eZDK@*U3N=X;BZXO%Dl>Chr8gVx8G2^ z(7J*Uu&0ZKXF0g+S1;*OI4HS$M7Q)i-BkNJsP-hs zCS-z{6l*zvFOIpUHHs{_5@;av}CGdk8+4@W4F>#m$A= zp4;CgZ%++l5D+9Mub^&^oT^(LAwkCh1oeqHorH>n+JvwQR*~U^_JC6-roqd?T?(Of zvv8#K_)Y_vmRsv*!pSUq0>>&uW?47$Qf{Z*Lf}BXGjKN^JLzW;;ztBIt3_g#!&MV} z01<8~R+GO)@nG3AI^HD};k&LmWwcFk6@1R?Hn8rYj8L+bK5u$(ETPxp0YhCx?w;fK zhi!1YFc_ijEn|g7k3JJ$XfJ26sUEg z5Hjv!^gLwo{Y0Z)oU->enNXh|YV0v2{K}JgFV2My$& z)wkr!&9)vSVHqME>9jG-eyRM7!^N?x!2mEAKm~a+&$Ly4=i$T0^Fol3a81$vQ}t9) z9yKt&D9dl0jZtsxqL{UWlA<=WpEWGDBsR6x)qjlm(<3h(xwt7Ca-z{Dx3fsb}2{^LK2`p!K|uAUU_~ z)F$@=6-I02$W5e=46aQT({=%exLVP=6;t-9vl*KD=jD?GD}k}zf3{|*JZ92SIRV3! z+0p_K-wp#l)1k(n<=~;QHvGJlI0kU>?SUXUbl=P~yzmv=b+Mf)zgQA)!a~wJB^NK@ z08jIF23d*?MOlUDRtMX0gU_WF_yQ~m`2XKa_EnA^&K#1LqZ{lmg3v9HNi!X zYNSiyY^rAeYciZ&7;raK@B0lIa;Ny7_QJX1L z@1>ozAwDI4YT9CmTh4M2&w0!-+S_5R1_9!*)ROl!C~UH6J2Lj zNfo-#z3J10-K-CEH3iYRnvsu1f3Zr5WmSJQE&Qgf>fq3rNEz0Af8`)5-X-l5*1^;>gov}mC(eNhR-^R>A8 z)hqxakSCR=Oi)Y2gW_^xKyr)JO%8X+czCSE57xEil*8(r+8cLcy{QDtBQv`Q1wkOn zZFxZ^hEK=eFN8N01i3)GimEKv0+O z2Pr)X|;Y+5|Y1*Ph-#J%l2SS*+Xf#b`0@!H4qF4`#k9(~|kF z{+}fjCV4#{=D_VJg^{t!VUA5t`-`m*`4lc+|GgA{i!Ie^*)!g5dc8s+#9W@UPyC$Q zbavoI?RPJ&f?b!y!~b2Yf-mNm98?;j$IsspPI@oRM$B1rgPxy0gEjx^HqtEb=lG?K zmRvC&uE+}i)}2v^F2TFq+s|z?%t4)w*IV>+FxgYI9?o(PAJeh9(eRadPT$o;=2to;A- zf1?gXY*Zstv0zoRRII0}R>~I@)%H-I&z9v$&U`4pM(DNerU+ zQj~brV^yy`k+#nKltl+2Lo)aW-xrU#kPraDi zrY)cjde>nyw;2wrNe8_>Ivc9(yf7@BL z7kyOb_^kW%OnyLx7u}~e-Tvq~hvPFmCwGS?OyhAcl5_8hW;bH%g9^^VHXY~jqP`~$ zYWkU;Z%_yS6}yo_=w+re_v-RdMFMduzM%j_`4DXbg$PIbjbCV@_fUOeU?p8kXDLQ56*xR=B1tzC%R)6vzWL2`IZmCmTg;4nc}0W8k| zK1KG6x`YyvwOW*HqmLcgAY1@2sp)$K4{KV$)8?Wk+lp+zi zP#0>g=&&Z&Q(F_%*l{}t8RLI-hVOQkwHS z)*J1$fZc$toT)k)j|N#%;W@lUl`xzEDO7uB%=t5{+Z=m(1d~nEh)6H73$c*BFwj9H zu&?%rwE0|er();Kf!)9qYj4isA2(G-Mz-+O(wStNkR7%#;hSQgy!r7=ho>$lm~E;s zD-xZ>bCGGSW)HWdi~j8_EQH>-IVpuA?=5~Nh~wt)>R7l>G9=PgWlWXEgIZ9ln+i5q zHcuh+FloOqMSS)9+ADtG_MG9DP{M^CEwrY%h@WjzGm(f=nyULjh`xZd##hhpoDZow zg_0|F?0AYUsdG zPJ5w1I*85LtN+7lhUJ-rsxO%36Jd?&s0eRlVzLcY_$s4$d$N1;>d?*>CuFdq{Y^ak zCVA_CnAD`m{b?1Gb_}vsl?aq0r%X|S#+wTu5JM-{1|qhYAG%QW{W4TMCz9%ew@BD$ zpl$bW2I&jP9-%TaJ@DW?8nm~y3M;OxvLE#j@XIBf;(*XB`uucDC=1Jvp@>GMfpNiG~yY;Ll1y&Qoa;jU4>| z8j(Ve<{CXle-;uxk)$vtp%BvlaOSWkf#l4Q6U_{0szA%!zdsaep(j4F;DdiBz-3D9 zTYe(P&^r*#n2}UhP(9!gdMV`f&qy@@wE39CP2zslMi`>bCF|EdJl~a-26n1Pf4>KnW{s5QlfrLgwU7uNt$YYgJK`5 z1h-!ovj50OcX>^>-Jqy&EhE_l%Fj14oQ6Mil_HlarHPN##R~;OUF!13i7Pk`c&A6a z4epPpF0Yd=k-lF$4jI zqs$Z%byC&dvT#ADT0`jNZ6JG&%P_b#$+zVDI&kUs$6*4DP0lMi&u9$s z^ISmT%E|sQ_Lzr+h@2j@K8@VEWn6sW3r`gIi7BSq4S9+mJejg1B<|tIrs4u9Mh$DF z94TY|?2c?5br<64QdQqPvm0ht3jz-!>H(K)*%B}D6B#s>LOGgM)%Q;bC=D1ng-Z3Y z`_m!_%ySbzTsyuo^OYuEza-4pyUA}~4r>@joSs7ja*@Mmo?m)i@y(x4`(QBwviWXd zGLy33yr%1V?~1l5oKO?rJo&-gst@?UAAh%j*~Qg*fRH&2yWO2Sk+#H7wEekPeAl18 zw>lvK-Re;>;PEu!C*uuISHK5KLk{zek1*N0bWE`TL&VzdyF2qMDkY4~MAB;0UDK6h z%PPwzy$443V5&rV7dq3tK0tJWygH8nAoB@uGfbM@+y4T{wESozt8hmXkV?lt)Um4 zr7TB8uCNlF>x^&41ElY~GS?(O6dnmWG|%#NJHVT}@%!f$VT&;D4foX<$@>XU!=Ox2 z^%9dpbPIpKveN~>h=9R-Xeo1y%ud60bUV46nP*9vT)jAVYl^QmBWn#AJ<-l=KAGP0 z<}_|Ze%XrO@E-SCV1MbGSA28d1#j(<#y!ae^5AE>e^H{-SC~el2#LWJ#%U>-4z@_4 zhk-2zwjsVy%(dS;Kc7Px>VhA!*&bSm9*^7uH8gF#HTxEuYkbo=^imdJTH!=R-*9)y zv_mRgrN)!%6E#u41TpX^k@p88tn{he3K^au^t@+eulp+v-s$!AF&1?k5kkfAJ*xt_ zuClzXgO^&O*LQ(HH9vXJ)Vf3>E5V_Eg*$$kBPGj5kgAlVjk=izGIa72_ydj`*4#RoU=s0dRxEP%-o&v%#eHQ+W5kexDs zpL<68gGZ~Q2<2AH4IYIJPtR4;tTVeuiRPnaWItTgFn8r`m^Kl1Rl8*+vNiAz0rYM5EN!m-6>eJt*EL;W{yO85O67TSpk?izL{)VD2 zL&80;L**f-!fHrl>ORm`c1wFj#vAiP5|Y7*2n*cQurU`|*Bo;_+c%QX-1= zHG;hL4SjH%(W)$LZ=~ji0Rm3|gmPb;J@VlJO$I=P6NU*&i)nwwH)6o!y3-je>bPOo zusC9R4r+1M(|ahCDjt@%H?Nc=uvop-^&oxq``^`QbY)OZYaK<*%2*y}2@iNSG2)9paHVa-ZOu+dZ? z5depA%qPoYPg!$QxDz}yTWBMwC0wYu6KIoiJqps2XA=q6?LR+{GqKHuakrSny$xjG zT-(cZ6T}vIP=LPk!!Q}D_1_*FUp~y27adPWu9e>s9|k2#0kjO0oH$%-iW zgl>Us?IHi1KKP-K3!im>4Vz_{Y*?bLv`|obO5bd(YJGCYB(+em&hDrDvDbBboNLr( zz)|9U2F%nAhKBbD^peN5C3=v5w8y)J5wbdYt6}r^?k+buZ5Y=1>7Vot@$ANStQocg zC}avl66JRbqqaEvrixWhLJwIXTsUAPnS=2QQnHn#h&t|P-fmhOUtz}FmhZ!dN!PdT zZK}KIig?HQA3M?_P+g>^U=iPTQ3V#b0D~kZ7Ek7P_XN~BMIEPqwsIHjlpxNWKelc)zXpxF zrouoDhND?{fVv37)GelNRpO)lj^ z@EN#^u}H&v!NOAdD7(67W$L-WMFOyerL&KAs#@aCCXHUa4pO;g8uBt}gXuq?Xq?!4 z$v$B2nmK|zX|>zG8euo9C5o(ynQ~a&hBeFFT#oEWi5!tT;vY%R=E}`9Y7qfBU~I|# zv2uC;yVAB5u@9}u^e9IveJxjCbwn+phU9(6op1*R$vMJ%gejiw zR@yXdkCLe@g_@+W8J`vbNOqW>*MXh!PtOG z#tF_5G1z}Ram^~EF#OiU=DmW5R}vdtdpXPNPA5{zX=biq=^at4vEa+g0UqZP{IV~K zdpB)Rx{qA@*qp)UJN?Jo1y-0Z{W@fY#kXs{HKsYd%~nCY8I5);cG15KKAKnMw?f5K z!4&`b6kS8L7XDik4bde_vlQ@Dr*=;R4Q`-E=vb(?@sAsD4TGG`L)?1t(8MS3}nieh>@G_@wEYi$x(TUAxyk{Z+)c3i`Pui`NWwbHRPYiGJ&UH8x_$zv=&|^@y#WcL_oZmdykA_pdwq<2bbXKOOD*o_^h!mJP(lwic`8`8hWXTdEhS z3)EWr?boNNxHahgLc|JFSHf+G44MrV`as=eIskq=N>iV!<_QZI@7XkL#D(>!QClKC>;$$_%yf(2hMy#%_3DRrX;9Xy&_X%XMC%mvwyeT&BtjbZWEb) zH;EvA34@&e0q=yCZ8L3Zu$wZhZV_|K>VkQ!?(-<^CTBVaCy|d?QA}l_Cf%$pM$&t- z)hZnK6BhiV`N;P)S~#S;`0%XFgZ8EN$U*}X_kb@h6|th#za^aGI?jsU_^5P`)YlY!mPsH?2FAaj@m%XwczxtiCSo%-*TAR`a zA7Zd52kz~x`UELs|D6Ur8be0p7^~4$ym-E$TAUYveeAdQ8QHKG{h(zd0M&}iKlc7e z9Lu9v^K^}hC97C#v0*CEY^RgE%yZ7{@&_8+;85)4;Iy*-Ph9Ed$9K9LEVmzmOZ@XI z<+ipxin=u$?!-j8TFtw6FPLy7FwEVml?Qa9^b;q4Wn0iYd|OlpqZw-4{!vvk7Zj?SPrBmVy4+?$?X(j2 ze*7^YBNWSW*kym`Mg+-pQeW8*V^K=UO&GZL;->emTEA2kxj z;UES^LjU!hp`4M)?*SSAW;-8B2oJwa1-qdGB9)q{E8H-G>8t_$A10hY?UIM@B3s$6 zVjbnznQ;>(im7k>}+YW6`@K}Tz36C;A-pFsvZf7gPCMT@})Z;S{E z1=;G>|1N8U)s-oPWFG2hNdS+HPv>RKbZZ6cFP)KAxGrRC^p#C2CTk7Qm~dSEp3C!- zh{w6-Cjb=IVzKdJ3EUBtIH5i2O#iQ2QB3p?Gnm~!LYvMC}A5s71Sv*q?0zKWQJ?~8GV88#N6dxFp#-si=c&7z{-Mrc2IrS52x4=L8ZS`3-KHuOqw~eqf^+p4BD$W!tX#i`l-L)ys)?EfQ z^f)+k`s{SCb~kS9B^ERv_HSi*#@EjmT=O!MxJvE6()bl^eMwEDsbZxfmo;#$9s4tR z(A|Um;*3I&nz?*VNqXG&nK!kVG1mv`e~Px$e`-X$Fv*3O=ISyk1HQ^3Hh1#Q<#W7W zhnpxOB1i7nO^C&Xd|kbE>$fcR6|65qDTZjuin?>$EIgTOb(s$8_NMC=?)?+wpHUMz z?kUE*5#It;FQjp;cfGAW?f>iZRbhKoab8*7beECy-g@KgPyVGE|1&E=y;?@djiHML7D@{!0V+l?Zwh%ZyAv zmo>3TuY=r#P)hkjQG?bQOuihYEZkE^_(lYKm&Z5c3&(%UB>zK$;eznWf810lK9pGw z%S`xe`L=SrF6Vvgw<4or2D4xTu)LC09s^5qD#LT1O6>3o_pMzAw(YC*c9kXhg}Yy$ zTMMwwacBFwX~<(f*9KC`iMz{Li+(X}&&K~yA zC4)V~qi;Lji-K}%O?(9y_W7T_^vA~6S>oLLcK%y!?;X|DwzUt3B1J(&R5}q*z=QNE zRg|KFASz8dk=}a=O^_y45$T{5k=`K?N&x94(tGa)2oNB2emm+pw|Vb-?-<|bUl~KP zGS^yj&GkIboO^D{1EZf8=kKP=@sotGg%tdvBY%DR5nZFicyO@n-#Nv9&v5>kdH!#7 z;9n`-|AKJ;co-i1i=*UU`QP{uc~cO@QGG@d=-mMO<2#Ea9Zw2@WOPQpu{FSS0yY$% z|Mzy7E!=%aNBIlAYv1~lB9uBEbYrtZxyp5}oC2{l;|F%0fJ;a039-}_I}EN@QM-cH`146f6+j#Id^BI_3A(_S^TdD9ttp;#0_S3{0Zl z1&q9dgp=FKYHz@x)u_54`B}NL*`@Jn==f0-1bEWAWkhFwDRhp4aO*72^YZ$SCD=if zaz{!X9~KJDdUl3xYJ=UG#XeK^Hng>NTf0$ml^;LaVB&=}Zn7$iiR+s0lBxphaGE-1 z^_}}SddnBZ=XXr9fsQlH6h0dsxrMOBS33a|-s)7T)?}IVXdgB&tf|kJKq$dluZENo zPt|Nn0;ZRleb9g#g3c$N*&=uRapMRk>p!!SiWG!}8$W`ilA1YhKN0aJo z5bTMP7aToXqOQ*eObFX&zyd7JB7zk>b=r=$Y^>o$v@HW@f8c#2fTt-)qLCEdXUPe6 z+Yx7rpQxuDhmmx6)!6u9U9B#D|Y=V5Y?rgLYur)GO2Z4v!GHJJMr%Jh zX4_*stNzDykI-iWnHU0Oy{;p}0U~H*y_}M0Bu$Y!@glvjMD0;_3gs~FH<#ZbU0%;+ zM%Fp)UI1TNK0Ixj`@})8fMMnFxwMY5#evbE6fc~{V8FU=s+2L(u$J1}YKq=Ka#wHt zC!w(1hCPr2i}{z#E&dO%Q5?%C3MZUl~FsF=oEN96GNTMCS z@mb4UIV2$ni|t8MyglEu-gF=MOpo;!Z?C4Re2|sw9q4p4iOp?c6tmK-#$6ksJhh(H zU->(;06?T`ecW5LFmZvqTDE%Y!#Mm5xhKC@x$pB>N=Om7ngHN#My4XL?$pC&7yh*R zt^QQ`CIAZ2eX;ySL)G+EZc~?7Y^n7;vvItmdS?kng{&*{NM^ntR9#bs4d6vYNMl%B zCCRwa@2KbuXgXl5Mq$xN0i+n*ae9k81|a#7aPCE`1GOQ$;jB_$FToLWj_y0injSTW zz<85~6aW-nkpgOz{@i>30W;oQSlr(gcG*Z}l~-QXfsO|8$?Gh$s69gQJqvdZq&wn8 zT+?*pGv!q!;!VJct$g_|In=q`-hX?KoS*0aK)%r(B-K>s0z49Mj=qBAu z!3cBFfX3iC2Ei3K(r+t98Es&Nrofhq_3osG9}m35(Oz~Qlaea1X&M8RZHbyQIGj^Y z^q89;-(@u%TOG?t1ZK5;9YjRKbA?P|x-)pT^}=>Z;)WwoX?Miv?YWyINas2N@xdmZ zy^5tK7pYk72J;!c6s#UjWqJIQ>)!?n7pPt0kXBScp|Rdb7}4SU1AV=<<0+#f-Tuiv z=^7JoxG7M?%>|#!84Sc*G&~Ksmfx~&O-M!%<^&es6;x~D5(J#hzHmpj^yDzod!24w z&|stmH0}x4%~_ss8TRViLaF3Uu?JR&NYU2P3VD>#3zcL0^jRMNa#p~C^M)95IH_hLIWgRuVClN zE5Cu@r&H|x*)7WJ9T&z0-m$_DCzyQx@}GoK3x}9vvpuQA&iyr z9~`C!j<>bIqp|=Fo{|M}-lN<59!PqFE|~@0`N(wt7Np{Y)wpiXB1Nz2ta!8=R+Y`G z)^*2x`3|J56_|Y3@*C4Yeza~8aAtdWHmC~;E3&b?UpyJx&x#i>0`==v-96AuOIuj> zQTG{wB(V@12YyuRdgf>=_iN_PS5=$TehU(0{jI*6e|F}{dGgxvBG9|+0#UeL7?Z~w z$uv5cN{7`7m5g=bDc%h5$8+hq3Wx;Anw`TgIceN1yal#Wdnxo}0Uu4D?vb1eSfxfI zKoFLS=~fo}9inOsQYy>xNs9C6u3B%ud&6MmPiy|>2}CwHPq5LwM4H7;?H~5`exueE z0)m*bEYEK=o2}nCKJ%?r((<*_X?4F7LF_+f;?41cE1CK79U3n)8EQhFVCd)ZG{ov7 z1H*0a7ROykmoSuq$TGNZ-|4O0iGA?w7gwADvHGkDbUQLtUpM-~oAa=)&uspHi=@_F?kK`BNI>8IkbGO?OjO zO1g}V<3#IudFdukJ%_5RlXKuby+g%gG+U}H6sPn}f#9>JS(Yj1#7Iahxr1LnL#v{n zxR6K#_*tj66&moC-*$$Ttf)yvqO>}Eyuc^IBN~`px8u@}{pDwL@%RKl`--uu*Lc@l z;#4D_nqm@8jJ$r_krWyaB9RFcFb?XB&R_ zWps)e_3U+Wd&P3>+stpE`!s4aDuS{#c=A;kZU;S@CxX8F;do1(3~7{dkfiCbuZFKj zJulL8pK$dJ{7IduWtZ}!kQoB3LkrLmyNrInnpDRP`wiM{PVktVXq zL{z%>Ci#7Ei%{eI$|YKnvwIsNM4B+o3mD}p1#f|pIzU&IKUmYfqRo8j8m*Eh3DDJ> zw!XVder+VNx!6a>B>u!ATDHEQEH^y5Vs7f-*$?k-gt$;%9NrS_zuoKg@|GYX%C({8 zBZi2^kQnS}(QO};>hvMk_8=VZS^Wwccop+0 z!~?KRR-LmG4kw@j1?wU1(PFkeU+DdmYABV3(Xuof{%^GxVBVa(eDQHa8=a}1>pJJ{ zV-4QFhSA9EFqCXFaz^r*Pl@yaHG#0AUe?w(rhA8NbOt0MA{(9b1~r?TmRSn#pr0$F zC`Ie(o&#MSz->48OnmuNn9fr2Y#jcqSqzJ0CfjcUiN>ci^Gtnb)nDVu z&^EL!Fyu*mC^ypd34s=NfByVy`;u?m7QM?F?a5YXH;-|wvQurm&QdR`FG}|<5wAty zHJ)i?yN)-YK#gO40H=2#P;GA{7>Bn;z3$r}P!lvK6s19}0l#|~1u#~ODc^FZ+CDiMy zVnlBxZy5D0@~S3f(R8|&r=x;hX2JGKFxfjiCEpN`1=gfoS=R7aPO9XG#Bc^zpFfjk zC9(-oLlw-8!n0D{E9=Ip1ax1+UQX?;gmRN>IlX#IwVfitR!D2Vi9wiY200edIB!Nb zP+~4`>xi!|b~_L+92-JXp$#kSg?7)#mThNF>9{X9vD~H!^QNCC^ACgYU5kglECgWB zpW@9{&AU{s3++$p+@}u0Ro|!v%sz?~3%pG(LT7s0u3?3Z{0W%=-P@fWM+TeJ+3z1- zJ4)8m-9(y9S;RU;@TRr|I_7USm4-P%CZaXo*M2K9xgp3<1vxjk37IscWv-7$n*92) zpNSCO=Nzef_q&`Vsyqry5||y~o569=sRP|wg-JjAW=|7Y-FvmID_25A58MsHcmtfN zTcnJKZ81s#==vu~tuDMQIzIMid*c%qoUa)t^m=x{rdt_`?IHx0FQAmfINU@WZX@ON z*a1Jr=}eCS#n+}6UFXRRq06%k#o_%=IS9zo{St=$?voOS`P1ta^oIBNiz0a*{X=rIa%Ijj z;z6vQ!>65%M5J<7vWe9pa=MXlY8PR*Lfx`ok){IIWM~^G5kBBt?tcn=w$Aj7?*KM67 z_Q$4H_vqgLzkiJA-~AX-EoR%=h6SyWDmhz$9rX%zYF@k+I;Umm*)6nu+h!x~s(cbz zQMGLAwYOwN6_o!e4{vdq?3$0>R~J^Yo%ztq|Cpj3vqk3jZCT#dnDhoJ4s>MpXUfIX zVZfs`%`B5Y%K(Cb9SZ+HcIZKed?@vJI4?8pNGtWtzfbx*7j?oJdIW~8R&m+vMjD8$ z6?#(G)m{&}Azvw$HvHvpy`2=Wa@r@t7PJJI`G|5H^h`RZ_LC=sZ7G2(=RFgVvBc^wMh%?y*zK+_C zM}K@h>**Lq!8X26pnB}{OL*De=#OH)pL5V94CY*F>OX~BJw5wzc7gJNlCQ37!JVzh zcGJ4rmP+)#&G>gq+e^){2~>X(manetL1c_Co}srG#ic%2O$$8ztE2EoDA z`fY(@&>jYT@-_1&SZvSyF&!0CN+`vLZDxJ%XbN2J->KH-ht7>lIRFs%w5eTB;(iTh zJ0wMQd#Sh`45tt{iX7NZ(zovL4D*olAC(M?=b~`m{I;KrFfZ$ z22Dk;ob~lNB+I4#R#U+gbZIxiBeFWrtaiESIocVExg?}RTsoELso=5HPMmRekmZ`@ zwQ{0?RzixJpY5pByWJTJ+JNs$u+v8rR4?rB02$TrMq>D1j;(*T`7oCK?5Gl(?zQ`8 zeGKHoHqe|656CR=VWB6FD}V|D<8Y@0^DM>XILnlsroyv?4d>0?RUj7}Wdl2?;AW~t z33l00FEsB>Azn$FHFSiMJrd0OfvjGy&`=~G z8daw97cG=LH`XHeYdfWaGCK3iUX)MG)85$#yO6s(b-D94I)%D1 z)$@dSwaBo^QjBr$&d!Kaa=5T#{qPIdbSKQZOGv>m7tT6dmDKNP`b*tQ_F9bww2~%D zgLbnwU%!>J6>XvR-VIBTI8v^nKK3s>n+v;6=K}UQigYTl$5at99<4CmC;;C?{SZ%3 zywuj?x_d6OpYU7&Q9Tof5#N zt?G|TrD1_fTCuv|NyRTXB{~_lcyTqDk~Q5c{I8G*nAZzPDRbN2Kn;k4JuhN}&F8@| zyWY_NZIKkZO`DG9DP9fDr&l&7^o4fwNyK+)Vg*R}x;D{+9;`UM3`N|Z+nUeL~$L|kfzNyMA6V{82dMLz#0hS_W95hXbhr)VZ zYsRtfp>w^1P*c4mW$7lRo=ujEKK9Kmh&gwImvJ8Gy5%XSqwMh^P17bPEJ1|bI8gml z4Glb&61z|c+m!()S$xMNQ9`Ta)z|N$;Scl^GYfDqcEG;$*LBSLxjxl=)XXpzS<>R-3}D0$L9O zWC0#rF-^(-9BX*{5W=@|P1~X>;mG9P9*}GSDqzVK^-3JOB)OAK##Sd+?)D1iv-cPZ z)lQD(2qLH3NGR5z1B&E}#PwsRi_A#toNIw8EZG~GJ~?G<3s24~{=?$lhK z#?3L8;oAON3gHgRe;n3(Q8(8n{wRyk0(wj4l_IseP085g*-z$#y$QR(_vhadEyu_1 zEeHVJsF-1ZprK1}RYOi*if`%=BIw2&{Fk`d2x{6D&cP~+b<%2oZ2)1C$%A{&bwD-I zTM?TVWRiib&4y&7?G(?z+f$t%sm6bs^W#%==MMhFi@=pOeBaxHqhout_?7I+Nt{=< zq9?pPh7^?VbX1)y5H>uqur;(jPtN=LlL(5zv1-cGR(5vd70S?!;X=?igJlsS8u^6j zAjJ&Z2R3ERE;f3JwYC$F*ADEYnaKe^n(v(QrNZnVpS@r9qd8At)ye3RYXeE@g%z!q zkDms?Pr8B(oTA-bw9z%DLNz=dGY6H)Z^fNfzO9@^ZC_pD>u{7IYsTP2k>{1jsyVjU zNa57V5pRvk)Cd~+x2q(EgIz_%W9vB!{$|p+x*OyT1&o5O5@8b5HYcY zz2B-En@2dnGM1VDos@HESehXTpHIdY#Xup1om`U7C%H)p z9$Vj82|x# zAoxjfY!cKOrgUkFPM$ZcPT+%W`CPN)%qTaCx4$dJ<#?7pUO(kIN0Tat&U5-*Ptl1j#r{U z-O!{PCc$ytXCVZaXJXVP&>|oM9*U;^B7%rZeXUwFuSZG0g5NHFuk90m`B}@vv2*xHbRb{=iYo%_ zu5znbxluN}a_vu;`1`eLbQ`lPwNBf7vg0iveS@rNJ?!9K%@?zpWUm8qp(CwxZ=3nz zq%Q}~=X}8W%^g8z-vsCZ#fAj}dB?(>s>O#a!zRx1OTGtQVJ%6}#qL4t4+*B;3DYJ* zA{BRBfb=FHftC({F?zT{4_xttqF@eGJ3r!%zCx0d%j1zVTeJjXk)xnYx8r(9Qu+;9 zWj1I?HDO|sKj{+tvcWsNSK$wcUWJpg(UV;^h`xQ-myk`DlAHztLFDm~j9Tow#5!iq&%BU7pGf_*x12JA+Zzj1qIm&QqHJ>?3@il@mneaFFS!GSAIM^`MnFm zLTiu4rH!bulvVOvap1==iN`)0y9=`%yXwjDZkyxTHD=>VzMD_zm|#s_??>-mc$-3A zNBPQS@AO*n6db(6bgE2d`XB| zRnPAg?jjhAvVMM!&|UjhN9ol|&ZJ@r*(Z(MLr)%Ge$QO4KhjK>^U#k!WADPRkpl+2 z4XG`^@KciEDsy@Rt(D=bWjgqE9V)e@yK{5u>O@UZF(8B~S za=Q0rpS1W55WZ5@mi~PeTplX_wFQf~x&rGq;T(Lwn*1w^qvwOG36s-*PL7SVBWjFi z&4_&Ph79e~qX6B!PqL|(M(+KzFP?-5S2SgIze4rur3LFu4x~Czw!FfS`D2ZQ9zF^D zr%`SmzD8X8IZTHv?~h7*`jYS*-|M!Va_$xQY1}f)aqQmGmhruva#)vo9mk|rP0W`) z1@AqRs4U7YxbAVp8%aHN+_D3fwe@4saUVvarv*#RJ0oc6Pwd8?HEhX$wGG_ThU~d% z;jB%#!NIC>EX?Fmx1Z21Lnl2Nb;36~N-5pZt)ek9Z+sQlqRU?Q3_>Ng1UQbRHzD)F zl9CjG!88VTZe z&1H|(Ct_VkxJ32T?ep`0%^-?@nY-v)V=siKUN^d8=L=s0`kK%m-c7dM&LSkZ0WB># zdf5pp+WN1GOec>7Pgk;Pc6{C<=#EBvNwoC`UzDERSrECkz9qqk*)4rn{n97)^5kwC zrrq9Kynkucb`Cm_kgY>Vpz+W`wv2iF%XyORil^^QIf2@LrwDfz5Mtj^QZR^Vb}8q=b-<|UkD9y^NL*rw{jBG4IR6atqkX2 zD8=46MQ&ZmST$}UV=I-$kZ>mWJ$?bVQ{(B#YcF0FFH8_b9KU$t zLfb4>hL;P`%>?cHacn{-U$hLNSCV< z%2Teyzv*Va)Y?0SvZYzC8P-V8wXGE)ox9`OD!F&zA6Nc(@yvZLMnnx5apXj$_A*!P z+R96z(3sAHYnr^(Yc1mM1l6nLfr|Q_%?oEKHSAtZ-4#j|+1vG!MhgNs4(nh0|4|xu23wUf(cV0!Kp)3!ZV!4V3N*>P0 z)mILAoPF4rV9OR&mO7N_X_nq7GxY{P00p8Y){|`6>jsFeg+Wd16ktAmbx|*D!6*LgpI69)7eXnfK}62PClX zRsyXf(Mv12E+PX_q6)!e*H!01&macqqq2q;-F8a`Do)PU6fx#8>uG{?=Q^1E+?NMU zeC8L7B4MA0o{*P(bO*nG%MJ`f7ZhH5tp3#sR55X$lbzsPD!S_ew<6JGP0Qyp3L-p( z)Z_4%!m+-If<(8*{6<*UjMfIU^SmTwXx+E*XmYjLy)b?+jmHb6ObK|2Y@utec0yo+ z!Gk6Jk76Vf4&%x18W!%v>LijGoR}s#_r{F*#ZeBA(fzn@wigI^^-3S9@XiywvjPcw zti9>A&q~(q7C8k{nkV#y@tQi2_1&~W20ORQGv5>r>o7(0D*2;+qPbUj@rzQ=Hh9`k z;*NdnCFp&NOO$-p*xN*1k0$rBmEFE!w|&-*bkz%kE+%r`L4U}eY|rTQ67YVTabQ4M zZ-wgn<3Fx$mK7)vbN!pxRu{$+pVx$(n2aVW3A*80`1tiat%TcyxumvEE9O#rvi*^q zC24&;&HNv0x?QRH%*gsFZef8DT4r9oA|bdubp#??MlLf-pHAc?t4N_5EyQPZ)Ri?E zY55#pIxROL)qW0OGG##S=ap5(bmZa3zx|#Z5%<6lRM9Lf&SowJf+$j*!HJoi?+x}) z)pfD=3_{oPHxokI_xQ ztrf4Rys-mGD0SFp8%?{D`GHo=aXJdW=mjni(|z@6(TCJ_qSm~$=d_~Yw`F^C?iFDB*D`IaMN6w0`7saXk%@7JWZ976IOb*lfk_H(hJCQZu;P&cPp ztoIXg1S~e%am2`Cea+v(1gW8KIcc`?&9L3{5VuIVZ}MfQsYX~-MnPl;Tq6?XL&OGU z?ozgZ=UhAmdx90}TZJtvu*PV|#`JU-*YJDeN$ zQO4(0pYx=nGFmWJqmb_W@Zg9sLe$>#NvwKE_0OwoxMVdm%O<>s$;WMHiD!efZ(Go- z<7%%^Yy|Gc{dK7g0dGUviZ|zc5fc{?mc3L3l!r9mQ=}g(uI|HG}_`EDi#k24SP3dS>F+V*|$!0 z2g2>#XD6aXGFcL!4_xA~mGRc{B~3>peorcPghtzuf)J}{0F6NY0gCoARU|Y#J^`N& zl@`X5BQE8}kuhi`vw@mSYR-DfE7g&PECCpj9ygzXhoYDE5-XS`hCoWZDdwB7_fS+~Ka{7Y!@vups?8TS{6+b{ud5s?tv zj+*o@lVJ2&nCeZ)Exg3Qllu$sarFhEC9dVy%&o{OfGbSPMq5y_m@oz=$&+Y$f5%YS zKM`FPaAUH6i-+`3q!HtMmjV2NJ8+*RwhCZqy-~Eni%Zs1RBi$Wh`kK^ zdc2BV6_}1yNM_nIHYKG#WzqR#vx6A*$0<3cxSQv*R=N;E+Q{n?N-B2DbyBlQhvzI6 zgU?&Ry@j|EaRfz>1jqL-hkUkzj2U9RsE+Kp$3sRv+Ng$TeEXP=sTMRJLM&Mbw6$?} zZT<`8otv+?w%=3x0*>t&SWg^=ctc9eBr3SLm3q^Q36I)bB0Oq6Xbu0mX8K90M^E!Z zy7BV(kDDh5eeem{@^28S2T8IO~L}5G`v0UUBso zl$Sef$qtf*$aBT#`5i!@8`WxMz0|{GJzST6L3~IG_fbP~@5T|+6$aCy$B0F+Lie?> zLM9KZIR+SMyc!pgyXXyutK%G=dMVy*Q;vPKU-=+yAmj&Cc^m*IGUQFU34m*|oKXjp zNGdbKWjNT2*&PY;<>Dl3A?-1vABm$K&94~086Upu?tY$z;5@@w*|J`)6m$QNAs|^( zM(Zd51XB@+t*x4F)RL6MSp)C9Adbd%)%l#Pg}#$v*M2nW-F#l8RcrC3P}lNet3_4h zYvhvsL>U2_T1n)0?py#EzF(mNqHu`s8{&?Ud8>w{v)9*yusgPSbCJ6CF&WeIctT42 zvrU`E)H?PMA)eevJC4UZc4ssvDHKB&rIr?MaFi}GAG2U}$L1$;<#dm8sB>-eV8!CE z)`Ly;+}w9{PI(UMchRF4GADI3p z15;!3du174Hugv7&0MY;wxSo1@$WgiHi~pjc^d*xara#O4^j-}^n1C{4?Ssy`<1c$uN?QqHAeeQo8IkC zE-J~b8E3WPX6yReipPD@C!19(OP0uj?>!HxqUX;eEaZOjL3G|uVWfVrvZ%+j+1Jgg zbcHNE#*AUA|CO8?U!ri0vz>a|I~U6Jeu7}Bd?f9HJnlhgrk<^KxPV2U@Wgd5&}-nP z`$gZiReVn7`+IxFBNeS0Eixac#{79P{I{Hz%ireUw5q+ILOSlitlJ*H5J>dhP?0Nh zulc*3VmVp&H%+%s%%jRL>5DU4%6;a)GZp;5kpCZkXNKt>3UwK?DV-4=^IGOgTa@Ry z>b2J%DnrKtq8B$=O~?;5YuQ2hJ9b|GC2fQ{wcH|Rf3;~XlA`Yzp6an5I5Y7-@8;O5 z2P)?1U6AdB&_UXD^2ezTa(x{8mi?FTXjat;$@V7rzigjC??+WWP`q7dXXiyZF27zS zm$N`wd*d{9zim;+F}pTrWYO`yONzj*`}m!u`?1d=2K2}QIRmM@bLTsDNvZtfG)m() zZ1hosU-4<~wg%96VC70n%VFNO#*X7}z)|LmCAagFGN&<)mq6zfo#9 z&e>KAf)dTCfW`~K-d$P1pC0$K1)nPXMxB@+6&y;{v4C!r&(s1v5qhWZv}C@bJ`H~; zKNae&^t&&_GoT1!$4(22ZSo&q18h0GY1@1VS3IFV^s+8lvHi^i#psx-w|oj(?=VlH z3>k1=5s+|VMUB_&@J9UhHw4@EaFe7wsJzHb^P^ahd{kcTZd7GVnuIo#!l`hEQJBGLt-g z!cBi7^0od7;o+iBwK=gX56lPvLu9j<&xV)@If?&S7y7%gn13_xNB&m8`)Sgs-uj>} z{cb;HDyV@K4{57+CTT$6l4t)QSoK{W2fC`Q$SzEaz5b1&W*)T;G`MhL*|uf8_nU3{ z|7Qu;!%t6pRs12fl(?w%j~2dQdt9|X-hTHa*x&U+-?r0oMu@k8e)`o|ec|ejZ2><_ z=f@7A6>^y=kLsnqt=_cI+i{+!eTrOkkE=J5W|BNuQuBTPY442tE{R(9v79W_<)+E_6R`~+ z^;4vV+SJb0m~GC;^g2dPUP;ovffGVA|6%RRZ+oCcC~o>6E~my_;i?PKaa?n&zn=7w%y@a$xg7B#Z<>E%9C3IT00Q|N7OkF^|o(;r#X2JUqHd@a$k>u>_dG(^xcE3>%1sq;AlV@to#3fzr~DSUR~Qcpn?RsT z!4G9+p5J{RNV}8w$)D|GFfz2ps*8s^f!A$W#kS(S(lbBk2#V_c*g)V|0!M2SKq|_^ zKJ{lWOERsG+ldDy+N0QMF8k^_ZEUr>OW5-y_MlZ4tz~Y*e=RQ8Wjq^f8OkzZ>xQh9 ze-_Ia2J^M50{_Qo_sS9tX2w+7U$-utth>S`;*q_g53awI!F#N6+8Ysj;Wb#?P$h?pPd~fdj6LSHTX(W9nSIgAE$(!#wYLeY7f&{k zrBcQ6;Qo2UJ$p%|#6MWtv@B-?4aC64152^a z_OB8AZ!3g*i1F{$^1p4wKc^7>whaFqDV<1&n-iXVj1gG&nqpY7=oDia0u*qQp7_Fd z6?RJ_YbEk|{p$@3A6!@6rz3asHMIxXkbDfZmfFywMiHZYTMF-pyi2It&?h9!OS~=3 zVju73@7N|*71bmgbtm63y}P)7IUZ8lImdmH*C5<*SBt%+x-aw8nmBd{qHLr~tq0Ge zVIP1DYQoj@1YEAGxajm2|LbM&jalL<>O@mz!7RGVJym3~__Tlb{jPl8S?I1uUcevA zVGgU#`kC8xw0j}DA(C-;>^sStsa0h*A}tA*QMSb?5#59A>+d%GE@1X~d z5}obEr%b5jVw*S48hK~V**sg$2~-CBL;50~JkzL{EYde*Ae0@^Q0; zL+m-NF3d46?Z4yRy-6r~Uui4}k;Z+ZCqDLC?`aB1iKG`Drv?&^RjkA-xQdF&EVRtp$HX)(&QRqo2echTI;94QLcUn?n@` zn)Yqu^TKWp`Ns^q7zEBz#zH1$O`22gd&Z1jvE;*tt$Oi-0+@y9t zy{7JHZWpNU&(k_q&wj^{M}wZw9tx6r4)-d9pO&RkWOhj540?Z5RHrhbFZ z+i|3qcckk9M-;>h9*^6{44+55zT|kj^gJT2Je}akIuG1Z-H?e#gLkQw&KXb>p;IuXeq>q5|;%w0cYo$CB(-{rfp zL`GJ-h+@2mlco+!3V+{JH9beKDs@j0{9As*XLi|5yE9L0U9w9{4+nM$4MgXr;X5xF z7*G2C7>L9UAx1{J%gPICe0|$dPnTvexs84mC)D)@BP#-TX+LxY&?H3RvSK?nTjA5!g zuPyU>`#RXjT)DR+PJ&qZm`7vgPl2e@a;F$?Jygi!i-&Yg@Nq1Uoaatf?d)0O`_@{u ze?4P7z&Pz;f$5Uf^FVP1x#8WL8wO!=!I@(|BoPjvZsql2gBFB*)zH@j8@%&KhqICcY zySuMZ;3E8v*o^n1Z?~=o56J%_Ma3z$!!n6ZhAZNmW5_e$l*eJ1%}TsjJ1*{#B0TNz z*3xDeJQIRq*bXHkxMrT}JKbeVMg&Psf>}YO#w;$f`-mIB5_6I~DDnOP*{1~4@I7ql zhzn}xt7t*bdiXJYKS)sQ#Bx5OSL++fu0i@EFmF3hfQbWJj2cIGDSnt7ILN&Kx&u!tVwx0GL5NxDLFRB3SHbs3VPRphnrMl ziFn*E`SMxr0ha+m&?@uCh0@q=hSHif4~B1ffbvgFdR#Ki7=#r1Gyzyj7?d%NQuMBH zzGtg}&~wOk?C>6X8SAZoWEHRDAR-xVmF-zXqhmpoFvr=7@P ztz%|4Ax4nGyMdTWiGk_u!_L0NzGJ1l-ZO#O)h#>~;g4lS>CLv`NopYZlTpgICmTv) zPl=|}bGNHGKoAAuVUa|zy;te@6=og~0h>l^)s8vBN^T(@2x8nPHAbCc%=Xz8Nvpegq3V91a1eXw-A~w)~ z&!NN%94_K%+HT-AF;hv5A!P^+2*>hViemy!b|zdAVYw6t4pw*!QQSi{oD23dgGw7- z2Y{?9yP$qdet6^hEE_^b&9@v@ZO9XQT*QjBFG;R}n0aJWg+UmZJ_pGxwi!o~8ec{R z{q79oRR?o#zZ-{8d^Uwy zdQ(ui+W@G!+M;KjnGLt$A}}F5pD(zpt1u48xi0!_d@@@6dfz}cs_A2j^7}!K@zq}Y zL$%K~v0z_5s)z21)ng#5-44r&pqI>Mh|w9@xO3uV!gt~sS=}(&mtN5-;RhFVK;8N> zvF{J8>DO|z9GB~gwz!=Q6xI4PKe3W@Zi@qU6pKuWb5dx7U0uv2>*tWF8$2i1Vfy_MYmaUF=20)tWc zo_fnJ(Oh!lTK6E@9aR%VAd60z-ryH;=_JH55(A0vWJV=?n{h{rMxg>zVtg4OLX~c& z9J*BPl#UoS5YkqX%$Q2J!kkP4Dc-E0?+Eu~$}k!i>OO=Jw}b%!URs69ydBxn%?o4t|__ckf$yFe{A4QJvRl8ZCc)0E_ z4io%_NfwYH(`G$Bc|`a^l!ZZ27(`&gGwmRm+Dvvsywzd>ZoA13(iDzxm+dkEA+ zNbZvM`5FXy9D>5ESwv|VI5)LKdpA0u1$fKzMXyS(Q??rJNbGvV?!84x;loT+mAj`XgJAdMirezJw)-j&-~-^_d@7X9ols6}>S<;_Tfg0G(si>SdiC%rG847vC% z4loJR=0k6_aA@jg3CTh7StsBY>-^EljJbegWf2xln;H?95YFcUU49D^xC@~4X zYbG)X6N0gMN>fNh6gQX0V1Gnazfn?zuX5ZgF_~^K44T7RxZgPhG+fvb((L!{?!Nh#uOn@4_-*s^C2q=tJ;d|=j zn_$OgEmPFjJ&B|_B*a?nPUJ^Ig-3RQ=_a44hy_2ri-?A`WkR>?#Rg)Lo4G!ArHaz2 zsK95&L%qZXzNJgm+l&XHo|n$mk|Sm&I{0jP>Ox%$JWc($Hdg21{E8h`H+#Mj$BkE) zr4WXUMDL{m$CNKiKpc-I%yG4V2%i#o76EF<=4BkKtTSslWpA#pB8>t9$|<>zVE1m` zJHYV`=IvwmiUo8JN3G(qaEiDa>FCTnD={a)^F2W=kFK0z0rxPVCX!+Df)`SGzvlya zNlOuv{4r`0?9=2^yVf`P$EYr2R#lm&=Xt59>`XFMLR5&ndK8UiwL z@jt&kevyy!_u`c{|0AZWZz77UltUz=<+5{L5upt! zSCEu8PvZNEEv@?Ha;Lf?L3BdhoQ}MzU*Ms0c-klurJ$VYS`6K4yCJLJ84^< z)^xoeOoVZxJ#zEDUaRW)j+#OANs(q0&Zo@m)c3?(vEMA0tYV~sJtm-vVsavGB%z5# zgfBQcKB@A3CB)4l=0X!W7j_2YY$*tZw(JW=jvg}7Fyrx*%7<1Yml-IhA&6-(T0(kI zrEZOiW|?Z&OV!Lk5p_P75qd6DrMih$q0C-b55=Is^;{LBFbmI1VCpP4QU~4MxE30j za^0$8-Qh^Eb-T<|1zK+9aRxHokzMeRL>~4d%F1YTE~C&Bjz$9^D;kl~@y9cR%oJ-# z*RdhT$2(%)cOi-NZA@=e5->``Q+tUN1l0-MkaOv{GqvgEn=Ta6$f(hvUnJkKy$PS#; zIjdEpyBJkba6Rrj-A*qbm^hm&CS|f#Z?~dOP;=4YV20AI{Fp!xq?F4KsiHXH>X|~< z(Le2NHUpZjMYboNgIxBN-b&@qI$nVcKF%E?*$dvh6uSVPT8W43T<8cr3%N|Q8PPdT(-z~5TTXIWAwqg#5X40jNmJ;V(oX+)oZi(r%FrXpF-4Rvr zGtH5J1YjVG?1ZIB?CVrD&)2&V(1|qoS7!P?L?lY5&w`P1HBiJ>hJ>Qbj55$#^r*^t zQ?62or!1OG^PMOpvh9HC`$e~DfZ~~m{)uP5Mqc^Ik8X6Q3^!5-3704EAyXv%`8RSX|JuEYwJ}g!TUXW?2 zLw_7-{pJYI_r1zsKokqjS|+G*vl2J3Dik?^qJ9ZOnruGo7=iAPao?EovwGaE6l`J) zw>4{k>u%Aq#-79uC|sFZcHpU|CO43oH9`EUD6OLfF(E% za@q7IIaXQhS|N~9I;~R75$u4Oc}y7;beqKqG|ZJp!%opG$~IN&M_RKKsk%q2CTBqu z1#StAVrDLb>iUEmyXoxXNlmsUoF!vTPlbk=zUq%FO$F*|h98Jg3&}%NZh&N))rJGF%!rxCb7xkkkV$mB`y#+l>!3A$2p|Pl4UlB`B+CLILGgZ{Lrc7DX!y= z%;uEgOqhmoz1Om`44Nt`x=%`4Hi4VjkeuK&(wX5ttAxP((`3F)xs6&#>#D|dol9l-Ix01 zX-i~q*Tsv9AC>Xmv?O$q?34#Ru{NZIQa_%e%#;v1J!xQ-hc1$L_;8F1iabEIz%N_q za8eSTS`OIv^jaoK)dzQxuptxt8ewHjM`uH!=-{YfdRDQ7GkB^o3e8anEMQUZIF_$Y zd-buMt}>N8nSo0k-5=p&rbTO@08}Shbptnm?FT>qZT|lk7Wl8F2iHHv4*v!ROkrOx z!KB&|jPGV*B|nPB)W5Mn^uMt{yC}D75?jGwtvxn?QF3}iXu!KRXZ2boH&FBG0PomHl06R!4B)eWY`*I;eb?Y0UEz+Ln#CJ}wsaWrBQ7qenFwn@{9DwWQVG-k3a z$v_<1>cC+t6_v}trpH2sNv%;Gha=FYRuh^W+~Yf*n_+rpHn{>fkl2NAQ*K>AQ(QO?K`b4CF2`K}tA1f$1G+TpQC+-() zzFf9VFL>x5%QO$1=S(Tc%{`6AIr)k#xzolKM%v2y8VF{KVMbt-j*36uaA&{}PpNv}O`wlX!!hBC#Ln$)QR zJPSpO<8&CK@Bqe9rO!Y$w(X5Gb)A4}pb6dgOl3$rjkaVeo;m7yQydRw{a!_(_-v!D zmOZgkZE!tmrZ%AF80BXk6o8=6bS7(58JdLpe!-iyAk5@bb>P7>My+;KHdP9w8%Je3QaL@Z z+faL2mrY~{j{`d__K=cSfK*^k`dzc%GqU3$ZD7qZEE65sD4^ts=rA%d5QMr&TJ>?= zsf(kj*`X?Wo$XU_w&Fnb2GMU%dowu5M`F1eguYd6cYKz{g+{^(L9;3Mm3lAWr})G) znOSCJo6rPC#oY+5Cxw_Osbqg(q{kCvLxyb*GkTLjvxYTjcBN*E zw%npxt#m{oS9h=+2^We9HYu}S&?St1thS5eOz3d4#H!n6!yMH+OhM9ZbA}KzLBYGd z#EgSz*6cS%P7xUEl0dclxB!l+(C-Ufuw~$Eg;F8O%oHsOD{Qu1=mo4sX!!{bC0%J4 zgnER9K|PV0k_%~-reiIvk~oA2o*CRAck)7yod zBd8*(MLqz%a5I35E@yTMjW+muCd}8|an+ciHuZ(&^=U83D?oeG!~jTYmmC^_U$QH0 zoi__$P*o`a0FmlQQX<0vIMh0n;FK*ENrB)@i^(Z4S5rlOq)`!<)i`yWuMKGg8NjAm zsAtjYNCmA4kh>d}q6$L<;$^Z<6ozFvo5=*M zWTZB#Jeah&cH9nJ*Fz{G=lAQid@JD%q1NcOG*&gLEzhXeZPXC5e527N%afjm;MgRi z=$$d@1WJDlJPB~Ab667#&`Ioc%jvv2^#L#k7*JJ&m#@=RUK|wgR*7#l$}^+GDxDdU ztpV*$^oxnes;QN7$y}%B06>J7p-Q>qciLWC=wXc8r^=aJaS|FOPBA4C1oRAR7cj?)qM@w8K(#a_nHzJw%O4&}W|j{t87P@{vJhI;IY#F4a9r!OwuJa;FCkNKWP;-NC<0*fek&xPrBIDz$w)B5P<#YTtwC!HGMM2OiCmGO95Dh{~JC5 zcOCu;KB2(dm0 z(VCz1`%Eh~q7qI}7(C>wUa1)kbHzHFi|IktB{Jhmo~?8nrt6?}e~h=&LjeComiD8Z z!Igw6f(2nUVx?4}^_#pg)f^WYO^##+h`JGj*w7 z(o3pM_Xg=T7e_G5mipxq+whfUoS*fXWI!oSrA?OO21W<coZ3QdxIvJE5ItMTWB|{fFc{iHOnly-I5x6eSzr|;0Ew#rgfDffv-q~ zR!hT*d@3d$Tx7bzpi}eo1|MP;JrZ36McDCF@73d^K5nHJrPV@gkvGRtwKObb$pTQq z^h7I{ji4{Jq@*JYs2~x=YNE(y0RJNxh8d373si7g0ZNf&)1#aYkZ15vL$uV8Ko`bqkH3Bs| zn3qaR`wJJ@M@`$vBf%)!8Tj6$Obz2P))K=kpf*$UR0|zR4muM*XcT}04rR+QQ$icl zR!)f$ zkp>;yaWgsIk28%{kDEBkv_f@Qfl2zHCDP#$Rf0MbsYT#Wl~6+oc)t-d=tWj9CTjsH zgGJIx80D+-v{wr|#er9GM_JJ~Grhi_ZRrJdz#?P6U4yv+1nL?wSDh9|R6;fRX-%>L z2Q+fvVmqrke%_O$SVXu+-*=!&!M7Dos->60X-R-uS>E;xX zEECIoyEIX!Mp8rvv(&26U>&((ddEwhQ3r*jNO9qDwcRE$mhDD~Gn*vZ1k1H&CV{~v z7lQRh5hDb(Q5%O!tz1gRopKIr0cVmB+Q9+_cH0Guae#HBXL!__s^AFdOl1NF^Nbu- zXiJjOuR4Y}OE(23G&W2fxzI^0XrO|{6Nr);md8Q zU5bL)P~p66H((oeuLkyr7>sKomhq?FWP)X|`mo$eP4UD?V62kw$tAqeLCVvVUjgxg zFhcN(*Z|GygQS*XO;#_tMNLqp$gr_&WnuwD##P;{33xNwW$01VlBV3)1iafUG^;o* zl6cvj=(9<`K=qn(A+9iEWD=`EOHV}@p_<;rsVCMf16AxQJir669mH&qnjp+l#~m;# zG2%y4T7avvLBm-X(;6&0=KF>>fLeJ8PkLbnOwEA=K5?vkTZxr2j-*>ZLmE?gR6#3I zMjbV0ZB%7RRj=Bi?&J`Ens4*b;k&h`XQFn z`qIE70;uGxlG;hZ&7>?_GLJ}sTl1}{6~yH*H=UKc)v+4opqVht;zKmG@EL0A%CIVs zL_PGg7{jto9u%iA#inb3W+5EZk9s56j&NHXO6?$O4)Wb8JTuyARvb%8SuCX;RX9iR zf#x&IInNT(MW>Sn{}hMy!B-v`j4Clb`vUs>U(pXZf1vFD!_IV|BPK~pkBlZ*NVjYf zg%qr%kseHqx;-Sw{X0^lqp8&83b~2Owh=DpXc^cSC!%NJ8d<5-2&+2H)$Ed&N_+`O zieOQhqC@zjQoGnR4G^HCZ5OSR0~b{)b}u^uQKBi;2*#v1X!1Vr=WW{(fPu4Z;3UV? zaM*Gw04-&$+w0WxikzOpY$Aw3TMFeo*{Bs}%^sj{hoanxS=nS{J}@V`DR#RmNsiPB zidO4#vr6S_Gr2w?yG^+gm%R!c0tiT$QxzX*kt(rMe%O)u8iyJYjVp9G9*FhgxR6uy z=77$GwOQS5H7F3X0XsuXMZaf^!knhTD30q!9?hjEOF{u3LfSAp;INRz%Z2W+Hi2li zo$r#i|Hag&cm;>@g<8OpHfJ;{(M&KZH5V8{S}~d8Fe9o328+UI5cJ0~NeUboE?=lN z8a)t$@``LlK;m9+N+hh(i=AdJA11UHr;wC&?XV9UE+MK8;vP-t9?0~74DK~HV8MG`YQB~n*9Rz?rBE-A>^k6Dt$CVu1?4wx?5V++)B4<$x z?2;(Bq$>cGk)SZYubXC#D#js+QQ3^1RLQA{2uJ%k69bu1^=T1;j|SrwkAOavG@AiIw201_@GM&-Ac|pS3HY7F&}CzAd|zDHOgqkWTsPmbXD-Gx z)k1DC7E4^c-|&WmOb3MZIH@F%HfvYViL2y!lOc<&D~}*y7?(J<+Q@drMx`80Th?H} z`mI@HMXgCmNs5%viFBMXGU8D3A zMvYXqud_9!M`Y+K0P{u{(*@xDPZ+Q62e#3>=EHb4Mp}k8!BcZY#{*EG*JFTo zMi?;W5Tijzm@h+ibs+W~iiRC1wqsI8K|Y16>aMH}k`4q@trU|e35r-4Aq6(Dvl7(v z3c2<`ALYfOmZiEW4-OFwgQfQw>oA?fV$7gj=rBB>`YIfWRjPWmJ&h-05QH~_sgS8) zie?70sVxmyQDX}Lw1T$cc$Vvg5GuEOhQrIcC$P=BhMN+D#Hy?ps8qyIw22vVCJN-q z2#~duk1An6224edGx-=F4ohN+=Rus&pltiI}KWh^qr34#6<)tfos6cnGi}4!(qxfNTYx#OPjcs=}?E|C{Gy2A{s{Soj!BZfv11_X|VvF9g}4|vvwpht~R zQK)WXH0_Xh5nx~;AN8{32{awlzW~R!gQN~N1s5vn)EctXAS@N?1FSXbcjJ!N4s+FI$R88`fV@e z@&H#eb{P`j(^{V9bSqz0GhTnJjHD{A15A)gEvnIp0q`MhHj&hrHu3*En*&bpt9WMI zXOVoZ%GDIUpHnGH5iQ|gct!%zCV*$CMhd1-d1qf$?S#z@cRcK+xrplNkaR3mji+_k(sSM1(5Wbl5bK-mE~_C9G)RAPZ#@ z=i)w+)NnGv2PO_j84Oe%r?-|V*bGzu0@Z|Bcs7;;sYRk;3QsIOp{qU|kLz|+j{}=k zlb+SZGnGaxR9k~K2agczn%i{YgrXOAIt>LHVa52k3oms&f0RFy>| z4LgBYyt=bC^~0cT9O02k+}YCYe#X;a74I-lvUjFyWUog{+uWOSV}{FC_;Uo!ShZwwn`LE9z_pE@USID-V@vvNUNF$DLB0 z7n+qGpX1x*MiEviJNCU2$I%1Q_2S+XbTX4lt#PTV4*3@BCX`7{HM=pj1dc6Ay`f&J zDp{)xTXjdSlz}6JMW9VFJ;*|!L2O~D=Dwh|x={@(io-tDAi7Wum^X%3N2WidJgl9K zLf;Ph-Huww#6ATA!KTCD13L<;aWi!eB#$VIh*3bQgP4(~#Tu0c_e%8^Z22@uz_GBd zTnhVTaB+yF!p)>^!GuY_T`5fMp&Wp11))9=St>_OfN&asci<(!JBVX4Y+HfcsP}|& zsCc=dK{(MYWI(Wz0;E3&+LUx=3X3y!(g02c;86-GB{zb>_(auoESJj9KzP+8wPGo* z=h#6GO8kQ1CCG3F;(~CgoEmm}qhddr_M0WgbuBzA17?UWsQI94XNs0tpHzXP zOP6}pQN-D?KB~cvIx4Fe8HQbGq_%0#?Svz~+9l;)a~1^tAY;dpClIr!+U&3%tJ)-~ zuv#93%Cpsms{&VA&5VP7;E6NbYog_*t7E35A?U==p_#7)Eu1Nngac1H?IAK!i(%hu z;)zEJcD`IFawD`(Sfn~F)`9!jE9offvZ|X@2ephDm6Sm1LUMtQi?LJ%{y>YLN@6TH z;doqWfXfPE^k6-cMm`JT3^U*Em*w05oFd>Ff(F6nZMswO8dKY30qHLA)tIkAp`@X@ zJ54}p9aASpc&U~Xiy(BT>sBt!J{wGOr~&hp8rla64n)kzPPQ5a;~C&Zptd?IS7Ci< z*b!O8OE5HU;iY~Y3)5n&=C=qF!O?;gj3>Q(t7T?OuriLwyjLm!=B6T&5razoK1fNL z^~(~O@fui{8E5&D$OWbw>Z;wTP1-Op|K;|??#KySKom4A=sj#gYJpkifs{CE<#GW8e2gJZkO}Wgwy$Pp}{Ei0lEqNZGiMB=jAq%ghj~2)9rV3Mo_6bDYfzh@AR~yAD8(9Kd3-8DC9eh`GmSB3 zelWA93>>AXZ6;`ypvNO!m-^IUZTbtOiU3OPxu_ePG_>xrhwpjs0xfvZyn zuw1K}UTcPiEMRg(9Q4}NTP&%Yqp?^SS?vkt16e0n4L69Y3;3lF%~neIpapBiY!@2L zK!lF!g&Z=ekP$u+)%v(emS9VZK`V8Jhnc210)9d#!4qXLK!mCd;YgmVqnIZp;}D|( z*Fx~3J{&N#Tx-F0yGbQ2%~jE{mQXdily8o4Yb22bpUKAhc%<=zLf2IVNyHrDv>3C zfQ3yK1Nlu&tseHOZnMwPsbQVYWS@ld10LY4A;=3L5;9MUj(*hog`@M;M|K7&zwSWuabf|r7v<+evq_zfGY`k3UU~N zanz>8wk~;PcuX`o$Fj??BeSU{&&Me^^ksl=X>i7{)Y$Lj;ULzmlG zFK74u?|2Y2`YIkwTCiAm1$keLo{otM-_i^KFaFJM{jW^qH2bJ3v0adTBo1&)42`Oz z_9kZ8stQERNygLz2~_<7)ubyu(*=`^OM&90Rh138XJ1{L|f zmt!j>Vdivm4Fn%rKDm!-#TsV6#-4TVfhW=Akpj}oPqB(v^dERP5J#4_0_#CH@x8}v+VCRCi#P{?&= zg;Aj~tbp`D9qJ4!84~0|7(70mQm)Wp;8rgNYBns5+ThGa1F~r5o093|%6yz15DMJT zD-11`$sj?HLTG>pdq3!zM6VX`P!m;JD$mlqN3|pcv0#j6a^*&d$tKoxym+8BGxl(t zz!QoqsV$+yJ7nswYpsFWCX^!4rTaapjXLxfo=~~%_-&@!>P~AOGmtBO!KbImkPmnf zjbK(L;D#|O`zoyXR85>Dok^=71|r?1s8*TO1~t){dhJR}CRDl@DmllnnF#4tB{?D} z%appgFH*CrN|pDL)ahz}ffmRqPvk18*GYE^V{Vdb)y-U)&G5d|ZPv%~gi$KR6!FVF z)tadExJOf?#BB(?R2YovWCSFfh`XTqfXHQ9=D-H|195`~p;u7GPvfd%RjEcLELDdb zWdXd}m#afvkV)C@di@f_rJ9Ka9;?U`)e==pV`RC~sH6`txF-}L)XW$l36`2zo;Xl! z5QWT=j+{%;a2Y{D#hMEFT1Mius!<4diLREq4bY}oXxk?A}MUh?3&v^0!{!Z->j zSq;bp#mLlw>(*UI1R3O!j5ONHbS$@G3M3DUCJeeEbSDd9zaV=cQP84O1gUHpjPXSu zYY<7M4KykNuv-WN@J8elpsS!ZA{(It82B`vL*&^OiPQ|yQgh>~1Cm^vq830b$09=o zgxv~Eo~;dwf|#NzfV{DakXqmLN7HbV!icAeQx+*S8s%QKqwzV}N%$(I6*BGi7l8>- znTd=ZFjRyq$xgZ+Om)k6sw$;$mTm+TSUJTdVijyK-3u_TFp*PsrfQ3h0dlyQcGXXh zq^NY$u`wyH(tc$u4MDMjW<#$uvmPs#9Z{jmxru+okD{i;*hbGgTUE zBt(>rGcJtVQVHbwRHt4au_Y?5MFpqg@)LzfEW#S%z#I`dW<2h;D>zjP37(v`Dq5e) z+g%n&v<2STD7Qe6Nz4QPe_Hk6Ea2`4;E?!v5j@P&IWTa&%YsBZoB-?qtrrQ9HagR( zVp~qU2`B-%Le*(P5+z-jvSyPFxk%^Ac@fYTgb@~5IaE1x)@Z}9IFv~V*NSRsI?Wa;oL~%bMQ=L+^X3kv57wDmTM5G)`*JxDE2UNS=-bSTw|5id4G}4;BPw=g5}IC7Oe3HNUK=Y-)~7Ovf}; z8O->8so@Mu6v(|8HL4&Q?+nKirx=yvge?R2OfdWHWSDv%AVChic{7C#@cK-n?uNOz zhsM3Yl!LG(`v#OxWS51eXe10A*mIz~99MkE7HFdhW>eKVWEKX==R@!|4q^vvh?FXb zfR0D?2!;A!t26*VS|)F0TkV$U&&n)#*~HHb>WoZw4Y*c-(sK*PI=I+OIT(%GCITRS zB&F!3Gt-K+*pc&XjMKCEOo*WrnNB+N$5}C)Xp;&&0{5>6_<|`eVIVUm>#P0NwB!O{ zLBgrX3`!{)0NI@dq|x^cQ%o^HX#}PBTv390J8(#=l&|<~qdDv)6-7!9D9PpcM#?JX zaW$0NT?3`L&{XP0o#T(!@>C-S9FM%-oOB?~UuirYw0wFf^wcFh?} zUjF=#Z~o=nCH;RK{f`R|c;&Xk4?pkRSKUjTz~HA+V9pAKj8i~#FuqX*KW>#wOw)E<{RP5x81Pu>>mH$p_l*d;ve~5;kmtbf3?S*wP5wx?_IItmkZCnf5orw zUS9i2PT~XZtcBZd{`}cLy8IUTD{J`urpGrHyuKZ`YN4~~mfzg`^Ya&6xGlc&0R8>u zuAd)s+KP)8teJO}@%06pEHC7bc;rXRfBj?e_r)8(UViQ3-%bC7?z21$w!8g~!K}Q~ z!mCbOv034{o7qQDPFS#dqXU^;qk}KK@!+=)y8EL&x!c+E_V~u`Z~pL$J^ym6#^k*D z%aO(-pB;b8b!Xl5=lKWiwb!c!PUez57B`;w+C@L#*ZQ9ox6fUE*8a=4+i*o@*=`Fm z&o21eneVB`9eC676UD24xxzU9t;K4>eLTOh#a7pRYwlZzEP6HOf`yN7vE{y}h!@XW zxpKiD-aqT}OMiGkYpboU{KMSk`;>2dVACzRk3Tu|xZ(RhesA#nth-ih11>T~;*cUoi(8V6WUuE*WFuyEy`TcW4zx?pttf@jvd z*WSxoy&-ndW{)pgVLv}!xnkk+*GkFmZ?D^6`ETyt_A9dV!ikIN1+PO~Z?3)1MzvSw ztoPUFKi}Ppinq<%W*bpmvhdAKKHc=pJ-7StgAb3}e^g(*i`3%(GUuDcuJql-SI+BS z{PoA&g$oLoH=6vex^di1N1pYy-4cB9PV@Hv^!yLj{BHR^8|`-4l`GWs9^v<6mv3;@ zQHMNaJahVao5Am&w&H;GAKCS>E4RG*d&j>r@3hu|3zoir|3*i>vJ-a&`!TuIckkQx z_+MNyrQpBsd*$2H(oPFrJ(%0$>({@uN;$zn=y6LrNpP#<7Pqg^Pee&0Q zd)^gWUVr4--UXNc&nGMYbBO*Pb9X#*<&h-*#%>3*?aR3nPTcM6!qF#Hx90a`H@ljC zgB6FIdF3YG|AaWd zvE=Yq&(wC)lYZma7=7sLr`>(=CRgqhZ|JZ6@ro0cpZE0@%9aPQJAQsU@reBG z1NQv=-{-G)_V?D!QOHO7oA8%^wBFpc;o|*I{_*MSh}-6s*|o%(E4RPvrls!g?+~Av z_niCApB7l;^Gj|Hp1Ae!1OB5`HvAgAyRyTP`<)%~iKk15ZZMEY9Wy|f??Q-=6TQ4&^W&IDU)|-3seb83RTl3z2;nn4h z!6VzR=x@2#rq4CEA8oO0hw?D`-Bvq1cIlOGEZg&))1SQN!DqCmfAR3QFDly1L0kTE zll6~%$z4|8TUqtidz;;~-c{S6!t*D+^U34u-HJbUqx*y16iMCw&x?Nj=IVQ2cxBc4 z^jkk$VI5q-Y6HfVS{`afcAN{aw@3JgHzVTE69MWy5 z^7`CIcG>iAA1q{n^=C_G|65Nf;jyuWT z{*nDQS|Kk#3ETZMW~)=b_Q3zTBP$kc(wlC2>u%*MpVN zw-Gice|_hh(H7qyKjtpmoZX1r^wcvJDo??M#)s>e^}e=?{?3QK zwS3MUzv*q>`i@!2N@Exxtq3XQ$&H^2GTbsHbL7kkyh zzBG5EyPrQk+_GiwxqRDezqjrBkL>r_rMo`5es6<2{#knch1}{t_y40@m)-i!-|t!b z$&3-I(8d!CpVnlyB|3?gyLwc~ zapi2cU$nQxPT6jQ8#lV+#P>SqoOI4b3$2^3qmO$&UbD%WM;^4(yy}T}Ena=oXfX{pLTK)!oOxwbpxK=}lW+ z*O0$zc(b&v)Lx>mM8MP8~8_Z@&eHMcD2K&3`Rfu)&hO z7BrU~{MHAzy>-cVNavJici8HXjkntGwVMt(`I@~iogTl(4G-)vf5EaXg$urM=j!>( z*Zr#hl?na+YsbH_eB~eK{GyOsvtaL4GynDH&V1?UU6#G9BN>~1>SqSE)yX&g-A1v?C%aNynp)A!P)=mc;~tKo!4V-^go<)%gg^&ExRqd0Dy+G zFL?0<^>6>hAd5F=esku^oew_l+K;|6*n&L(4q5ohuYM)v|D&3ATXxqsy#;%pyv0uk zzy8Wl_nr!%&As_W*S5aW{w42gm+Yo2+N`|!u5avs9&-6qzl1)=UvTo` z1t(wj=Hs`2e$1NPURnFlYW(tzK7Fg=m(M@yhv#mK9RU~j{Q9x|1OI%>Y3nUJ_ozGn zy7RH8-GBEBFR$8f@c|bcef-mZe1+Y-%-s0tj%V!LTna`-?=4;Xv+(@O4tepf;MJ~AzJJ^ytAF*;8@oPmy7tlcC-*(} z0@%FGF8k9ypY6EC$*b;qde^Ugde!gl?|<%17yMz@mBQuQF*hzB?D?$m>Vpw}!7dN1 zoA-Qu-_gOJoV;qjvhK(W;nmtX+UlEUryO%g)DU*vWyJ>j{hVL(lMgQM1aCmhRY$$L ze5WU_xOCC&FI`fBJ~8ikCGkI=d-RRLIoIwhmo8Luk91D{+x_pY5tDg;Hg+suaL4P< z9s7Cf^?!W-@yoe`&R%m{^xeV^%_BbjP5k19f6A>pH~+E!!R+ebuGMy9)z!89t0$iF z%sJoP;-ll=z439^y6=rwCU2i{h`7d&k(HA*8}Ga%yn68t%|kx@&C}?KN8b`Zd3^HM zrIf;aWa19(pzpQee~PepI_PTx8Hi@ z+2{W8@VV#6^0VLC`}HTzcOQC5S-QIa+!pHRFNhZ}Iq#|0Uii_wx1N8f3kg{^5v2ulUAYORl)<^`~#% z@9GDyzu}p+4=jE8gQe%JxqSco-+E%d1D^Q%mGjT|{h3c0mp%OO@F&V0=c2o9d%)X| zK6u`m=;rxK;rrYC@Q=m@{S~)7yWv4AcX1cNh_~n2E0b$}aozWRy2-}d-*?|*zd!S> z-!3`$jB`J{Kw5J{Vd)j8p&xF#WO$>o=&;MH*ZuMLXGZ$!hu^vT>0ey);PvB=Ut0CO z^Hymm%~!s;=}(p}*=9ZVs&xR+y}quw_1Zi49{h3NWA485oYmf|*POTZ!|TI`{Pmw* z_v-fN9Xz`Jqt9M;4hXHY>3=jf{^>UN{&m-HJaG6!n;h0|uKM<_|5%`WyU{)A-6x+r z=DMRlIrg!I)PvX&$0m30KDpKX@%tm?s=Ky0WS_@>dG<}~^jF?KLp%8McON|D;}4E_ z^^-N9Y<8yejCI^I=+CbxeDI4uAUizzaP(qjm+P)Qn%VcQ9aretyI(!wpV$G*x4mQ2 zV8PyVm+_x{^APrlr|w;abnZLsQg~DOAAdh^@jbgNIqloudgQJHPx$!l-!8r8?`yVw z@G9+wO@yO&==yt7CxC17`EJWDEbg__1)s0`WY_Bd>@Q#V@#&BK&nIt5Pd@R>znypU z8+$yk^%eKafBJTDvtMMtvuW$a{U6`?f#tj33|`f{_~|p29{%X!cQ4xMs@-3?=;x1? zB`}Q>c6^zYzBcc&kLMq=Y~`&Ro{ql1=MMLry!UMFPM5FwWDE2Ct?#;jK09~g!;iW2 zefQAIUfg8o^IyAb%dMAh`weja^@}y?huIBBCUKTvK=*m;)fAZR| z=HBztY2Vo4Md|J(2N$>8_@ZO3{)hGCZ*My456s1{Ex&KG)xopfZ5M^cX-m$!?c$}_ z1It!p&wccpId4CC#IdiwSik+D_?2xQ`|XCGTy39p-lr~ry+-* zvhOX=x+C@pdT-^OrITAPd3v8!>G?l=*2D?oU_Wh&}tW(`V>ot;-f6-+J>UX#Ljh^w}Sx-(7mdgNyDZPW$7V z7j3!u&tA};J#5YLkEhmEOCLDoo}0YKUVJon%zYQ+uG`yPb@CZU(`!yX9l8CMw{Kgu z+IaZUn@ihNZ_oa4w)Nf@?ECTqZy$ZvZr6NAdiUnw_6^rQwL&@hs9mr7_E?@EW4~Ey3{#O3mf!E*o@C(t4Z?I2%e#8U*Q5n8` z<1wGTx7jW+x$dSLe%}7&>K`87c-7T9&zmE?1@jenc|Kmw*lWG2gkL2*V zVtt1bez47{iznB-eDKf4XY~i4AAj*<*A2?=R!{rvn2*+CYu|WipS#}r(GD1Oma)%P zA0P7qw)V|Oh}OvJzht{c`^^G^2 z_`(M^Fcd2foj!lh#cMzN^=CKVcmK0XJHNUgegc2x{kQTLe0J`7$2@=9$$#Dch@&q( z>Z8pMdFheY>-%ke#L`{yJ$`-8scxZk)h4$;_4Wz7DaWq+=$t3k+^S3t4WInuJtv=M z{#t&*G{pKr`{y3r`?~A4-)hTES2Q2IP`c#2-FR}RJ-++qY{~nl;@FxW9B4cy|JJ_b z?RV?9e|q%|#pJ}#{=!^*)V0q5t$oS8m%jJ(Ik!Ik@?Y*yp}`5qeE7~D!IJYn|9^bF zWmH_v+O3_0U?I4>HEzKjf;RzzgaC~O4-UaSxLXt4f_s3*JvdE};O=h0U2+!Be)s$B zJI`3j!blJc5!d-4X_6B6rAU4$zY@1KNVI>K z&hGcMVe9!e@Zs{~RL&22_0~Z7^VdkQMnrFImro3-T320bBwoqdN6pXsm&(oqVGNAE zk6yA27BP6 zt7;GPh4L#?NO}3AVRv(uY>|^%?UuWWVU6}m#j&=ifssEH>lRloTX}ox;d({2Xl<-E zSF75@ha6o6_y*EJwv$S8{~V_7{V+pU#0`s#$Q$w)y5YvBspyF->#sUpmj{Nx?~B_! zzpv$p=u?F>q9vUE&?RUJ)~Hm{C?-uUa;Xhx)@%w?o%F7@NTC+PTXRi!oFd-iH}dZ< z=W!qO-nx<|kQ@3eKUME}ng4LOC48jX!Tp{$OsC5mQ#s0)vVss-P8L%Qu6bQwT&6Re=drSSf1cH8I$|jR#*!y(`jB-T0T&l?ZSX5 zB0D;SACjKPUbi0=epAwT1NX$BSW>%MxR&5uDxC+ z<4(K?Q)76{qA(q_k!ULVoS6$PJ$j^@Z@l7Vlj&fNKg_f?I2_2{WN zv$Vc~i#!2TI30P8r#MMtw->XH*sFcHMUo-Q&1)kHRU{03UbDGfp*kz#5@92BzHy)B zgRg+AyY+?2p@%}sukd1>w0R}TW$6U_G}HGzH>GxM;}FdSr+)2@GYrq3NZ7M($Jc7; z+vHi^7HSz@w^K$Lo-U`|A@6w8PmV?RT1qiz`G0#H8|1u$ty^!KR|oT640a)mfAZLC z45HEYeJ`J$sRr7w&#;OL`OfxI%F}zXnZ|7BfbQ31gI-omJNn^bM=XbWgea0Y4K!Ho zM?+1Kz%ez%ElXFgIjC*nAtSU|CibTSN88)=;LC}!o?ntaNApKl#ZDLWTjY6WZn*53 zT(S%l25*TP+D|SEqz+(jCPK8>Jltz75b~^hue0S@62cJSd!ubBa6+>PI>@qjrHBGG z5ljkZ-NH5pGZ{#(RIMY{QIq;)C+E|sEdKZ1hb<9144=H{4u_WeIT7wTR!5rQvJA<5 z3_3`R2(J`wAshG|Gy&J4E`$iR7!@alm2Ps2doC(>q~a5fnt}TbT`z{)=-A}mZ}UJg zFPk4HUzMfhUD6NlU#3l-F_4g&55$iXCjSL`gzhqU-uq6DgZnire35m$40~NaC*}1A z-wUS%;ZUOz0S_48M~g;N=aR``#z!%`bRr^LCP+-Xm1@+>7EwfxOS6;UXP zRHF;uo9(0cEzdw!c*F5y;&gfnm9#|#v0`+jeZeLj7lY!yYvv?~Ine}hTNaeO%3YoK zd0Lo*-lPat4WXwUE(XJ(*xTwL2t2f!4$^&0OHUZ&xOttEz*FWb=iE@8J!L_2W5~=X6o0%|sGlzVLES2pT-{`}s6ih%w!=`ce-(fn zrPAYZT;b~)(S+$Slg`1f-t^A+PkUcF{s|)!H9VZ|%|4&)dpVfV&7t~@1}j+O1Z`w` z()91VJ!6r4m{c$*!igs6-Q@4Dw&hNn_(?@T<;(0ddP5KSBZZrZKYD`7)$BBx2CO43 zK7XytgT;g(0kYy2FGloz;4*tI%Z$}OQHD5;SAX?R*-ou@VGzFnr65)`s47S86^i`* zWjxW3ET&!lqL0G6zj_rsHNb0!*4$)&)Z@9pO#>0cHdlawBM(~h%6!hqy&pIE<(c3E zVFKII|J*bntDCYDW`m}{uq-f^S^T`+1sTxSEV$WWQ@3Xw+y!f9pc~TkXkRs_7SKYl z*w@}kYK(F%E@fwpGQKT}rNH@4Qdl<|k+i={91&O-(GdO_Bvkv^WX&j=O#EYBjfYJ~ zyaO?^9L)a{9sIaV=GSETkd4E05qvQcG^n%IQcJJ?FoA#D+AJ!Z=FNVo6@Sr^(#P|M zrH5dV*5(^p+#zyo;SnYG4IktA;9Gorj_04ke0#a^@;)@TJ@Sp?31iiHur-tFlmVOS z^vruj(z~y=6|T8)9$b`+=7b`P3prZp9Z=Re7n$05CEe-|%3`Be)We^v2g3v$(PZ@c z%a;{gW}PsF4=VFOPXlH2(kMP5ej*6?76c-@Ug{L5#a;b zGJxQcG9BeSsvXAFCG3hC>1ljLacX=JcsRF2sv`+7^>;`(L>OL!SOec!6J0ET>Yxk9 z0|j)~wn_)t$dmc?b%-}_0LmJQYewyM|3M#8tv}6cU18u-#+htq&`iaKi}JAPe?M^^ zBx{AEYwy!F#Ww|Ppjx!4Xl;acP?^*`54NMp;QWz2*cu_R_njkJA_-LH6GSBY@{jl@ zGQ|F?;C^yawhC865a`#EEfxbITIfN&?Q-#ZZ@E-^N&%KndG92`WiqlsknT@X%$4@1 zMS+n%8vMUZdT!E=t_twr?=;dIzvbaPeMQCY_B*PDzw4L9R<{5qr~|YgKjKmhzG(_J zIlNo_%IQ_H@V%U5qA1mDnh)sF9NFy~ksK?w%jh6^qf#W7do5u|*w$72zBN*A;hx-8 z-IkGGUczZ4o2x=lxoBeWQQ+6)uW7~SYjxzv;*Me$w42)(bGhFO3mOhZ^TQ=!fsbC} z;lyT~i*BCg@x}*3bn#LB4boLKd&{*6@($8?4^h7gNYhH~K@nf8W2-+MwOgF&tKThk z)w=$AH>*QOx(4qlwUk~Sw&*Zq`#nV;fvCvmm*smi8CrP44zg#p{3ZRZ%r+4o?pQ+H z#=tUsiRJ0D*Tt-UTvJo*IR|Iht@&`T={b*-H|K?6dYu{SP17)J(kGZ~GU}r!_gnjI zC>7+haKVVLJU7GZafo@QWuDfBZHaX!0?=?Dtamqd8&?gQ_FSx77n58Kl6lvj(2=`n zEhS?rMN>gOY`1(`gUl8IDh@RoWw(9Y@oW9&oJg?%1_Y0ldV|!GvFC9PMcdThq&@qL zU&x~csbFDV9;FhTt_<5=mHvrJQ2IV8{ml)^YMo0ZG`+AeFk`GL#AEmQjk}QvCBER5 zb-eXFQi9B;ge`mp3j+*8LF2rF$01#veAptI$F(c=(P6+Btp81Y79O~`vs><+4M~g9o;B)5#uN`fTS!1TD5$_s^*F%cl1#vmd zcE)F#tipWXr=<3NLMMDQVR~87?KreWI#{F^N~_GQ-y<^iCmM-_IP`#GkG%7_eX6cSInawvSm@Stz|aK!4+?;MH)T163?f zNt>j3rVF?z-#*knOs>SaqcLc#XF8Isqd!y811iIFt~u3W<5UL5A*uOe(e1aR&y@RT z-^^cbZIyQ;u28A69=Hc_BAriQ(-WRMT8Fo{q#qlbB6~3oEh3Au++5nk9?u>oE|#`p&bNdqi7a|@F&9da85DVgXT7XWyAYfg zc}=6xv6pD3b=_~M`*EMQ0jeAQ!Q184^{#L_>TeACx-!aQ6GyS1j}R!lbTP$m(ou20 z%DtpU2<5U}_mSpXsB#~`gyN?qZbes^m{$*yqx2LcVQkN=B+qJ+4X04!rL_yE8wVJh z8OQ@~b;9Ko5jxO2XGw)4?OkVmq2pmarfB@(Y^oPiy~@@KmvDXQ(lyx))WgEd1I_An zF4^@9$r{5%_8Hqp%NqNsiW_C?`g5@yUA2?@b=)c--%X!R+U-vAFLRdUH+!ISU-B7; zod1>~#M1!8n2AQ0i<^gQc&Ncq(-4D*m`ag^&1 z5osfjtLYP{Ibi{#ZL@5>pJ$=0ufJC?I~S`suaHZgGofE@x($L5&k^jDz*sXK~ z7tWV7Dl@7%Q^#$4C_u{V;|{T>HL#^nd)6b`E7Fxjv1o zqOT37gLu}pKnY$@j`@fGkevOco@+rUrNS6BccIZSuFP2?9?mNUVtr;QZKp{0dN=s$ z&?FKretnFaP&79q9ZY9=ttv-hs1C(rvADIG_Q3Vx!bvNGR}+-V)JyEHy)mqLIK2BM zlV=ac-@Faiw3Kci-fEUc!bl9oPN>^nk#jyiaa(x7e@Z>_7!Qa|lwDdr z9X5$o(%kp;|0SIZ7Slmql9KY*B&G5-Fh!Xh>eg!op=DTrAbomH<+7LiMmfHo@&tSz zj%;rF-I7#U-C;~xEq@+kSy6_ECRxZOVQ;kiJfr-Z#rQrJ9nd70K=8M0oJ-T}?k}#E zdr6Ht@Y2npF3gqIB89=UmOdt)Sd3WVCH_f$(uW2L1>k-p}6b&yw_hK`a z(&0Pwa%X%U3#39#q7^80GrefdOEQsQA4p)28>UB_F91Z&@j)4mQj5apb1jy2VTBuX zkcK0mN@`;^Vrz<|Lo&bwkvgWEq3VrkObBeXIj5-7`Mp5ive+ZK<9Wcy|56 zHpv&c;~J}94@qHU9NkBk2mFTU_w2LOknVigd$f+jU@gfv0cg<((gw(V(uDoInXC<5 z+apOMJlkD|!XFRjbjEn21hl2Y>f=IH(d+jG2RDAh5RXRXjggWCJ_o++aPK6@%h*&% z!uxZ+VKi72_)p`cF-KNXQHD>Z3--h-S;2Z?rYjcvcK(>5OtUvz_0V#w77~$E?0nS` z0{#k#7qOyS&fWv#r|*(lMEq&52mFd389zz~kNA}KQdgprRX@d8iWb!$-Am%{)o{V5 z8$p?RO$D-x(;t=gmhr|-Qq0H4-Xr>9ey9qSRPvVO~DLZI~78I^jYcQ>`H~NTSNEE&I zy96!n3nz6|?ph;ljDVyf(QK{RIw4WB_b z%;?M<_eFn745rfjAhEm}Ofj{pe?#?@z#kVnOs6hsh(oYWhdK&F%=H2jf`$olB6I2K z;Q7mR!XPwK-aNr?TY5;T>0bjpYPiJ-1-5o#shOZbCY+dpfsBK67~TU@G2*=M3G(dU ze(8K3!@mP`bjjx!3w#yj(?GzUYKPT&%^o{u|H$`|bF3d|3DqAoM90+5T527t<^NWi z+*r|TQs%MgqSivTq|Q!Fnbnbi;bY;HvrNp?Nyr3%$r ze~kO{YQZ9^ohf?g!}p=qD%z*I|7(y5`oWS0n_YQ4#uCCd(qKhvofq7IjN(^+aJMRFTp3-$F@;#Q z-6!1k2K7t8er**X)rkSy@o|-{T+xwfh>3fM1n9du?QcAc z=Ddv8_vs*5OViV>rI;E+TrTy%gpzG_ws{|RI&o&O8cOB*NplK#V-hy6M`(;yE|HI` zjcBoQe<0lY=4}jk1J2&Lm%FEUV{!^@v8Ppi7vH=w zpjfZ`w4J@Eh)Uq8>t_tK-;}T2OSH4gPH5;}qa4B+{c7k;J4{e7PS7gKPgA+HnpW{--$+Blapk>V#7N@#^2QIH_4$mn9y3MN$vRD z-=u9X3PTuVFL}qh0a2I80tECs<2%9~5LBXwqND&K3yo2$J*z8E4y(?IM*81ni!)Sv z;B2Zv6D%s34`;!=K|DC9#hRUXuRho+Txh%S*J=H|KdY!hPvyIOzNmlH%i~OaiG6cM z{sZU=hXnLu2_x_@YvSjvD(7Z&lm(zNb#B%(Goa3d&wRgtrU%M6Q5@OG1`dz2oo&w* z+x}gjuUWW7A19F6SY*w?sM~KM^K-a|pf~v12~g1rB{~C44AvV{YdgZ*&Zg4tCa~hxyVgqqYkni3~qC1S%>`7XG zfUTNWFdNtOaitx<$1NrN{xjH4f4Rm&rDKOFB%^M7qJVqS60>MR$5s58vDhWIqwOp z#m27q=XR!VYZ75{8x^Vk9;%pcn0;v%lKf)>skA5~l8W+V=pGSXBc*jSkang$@b;1t5f$@v@V^I{ss^ z7n2gD@JwI?-{lhip1}3oju1F=$UO=zWO4{NpE%h`Z>BB-- zo+@`@y6&NJ)Pk=XsqAO#pk*DR^Q2*{1@z!sluwzXhyn^BdE0byFj&MVb}w-X(0*+_ z9|`B}Bc=f_+Rxx^t=k2y`1;ry=`@W&l@!vn64i{quWe5%H|j!6A|z2FhbRx<@Y|8m zw-^es!f8yg`D}J}r`sOdRM$_AcGr^N`CxG8UT3HA9bvscSbW)O^K&i(^dos*KHu zE4B={th0Ei1fg+s(roUNK!c`5a7YxTV|zV~rMaT*T3Y3G zWjT}kh(R;-EV=7UB>&8hHKQYa^qZtFuL@{%zq9avF3oNgE?Lh&$E<>{(YP2@4an*d zDdU0wmWWz@z&2@J0%nb4Ge&3lEd04gu1jl;-9Gb~9ckzDc_drsg=aQME__7ljyqe8 zbUuKb5--by2dDYUD!X&UmSvcy7R&m2MRnh1rsiW>{o$Kg#1_-J7Q!eAvE8(T@>o(3 z6IAy6(7My|Eo@0k1io-e9R!H~6(=Ku^3~X#q~AQuA}N_)hb{ZqMW70HqT{_VoeU}> zU<6lS1sL;x&4|Zi+I4+;D0`GJWR~gET(%Kw5$J zW612BQI6n!{L>3>H|v+xk55U_+4HPvhqMLc6X{S7lyr%xA<4XW>^kFt&cwnsA;U~= z!Hp_>pBKBjBR5{er5J-t{*a!d<%DgJjA#z~W3+~(*|UEZ6Vqw$F0zm!Zli7Nu@ae{ zH!-#9X-YDVek%edh^h$zQS&$GiHLf94O_=0>-3wkux9Melpqti4cje}O8|ed>EeV)_~h(V=aL zn{B4DtHatL*-p)k5#l;bTb+_~0k*Uw(i(J}cbUedP`~ADB+xw$J!yB%YZ<^lf_-9> za@n$DOpg~@^JEp?Z^I{jX(R)atVQa&nR>fm#k9gK;eGk_%G$;0GueURj9}YAbZm)O+#Z@C-(pKa)6ZU z5Hfx0qZ@{}(ZIkfMTiaoHv`WBvz8 zxPk@#_!2P9CgYqR{s-xM%0y6QKLLg+*Ls`(fdb|+11MqsUVoF%f1-pcPcLV@|MBHN z*mN~~09sUYHqmSU53&->_(US&`Dix&2Xj~)4lswsLa-kO|M8DNc)-hXUbUAl{O`qM z_^HcKkw7 z`nky-`I@0U^NV2ry~NzVCo0T{@~TjRNf#gvnM4{+cC_&##^X-iUcdd#zqopY&=38$ zh?2;7>EKK9t+N&wjm`klm>)7#0_5=UzGni^O@xvVja6X2Dxci-l;!~O+RZ=OInQUw zQyV>hBrgF1nQr_~@^}9+M>suVf0P5%8UR*6&CPy&w3d;ekaqKg=oth2^!Y~y_&b#T z_iX#52dW3_ZM94vS-yZ+KFaaEe+V~n2nIoH{*dP^V^T~^$RS&FD3q#KY_?jkiC!k> zs<%1GqwkNaCK-HDEfBr_|DzMr1L+b40;b#lK9)``B9|(~doX*tpQ!u4$!QDhya7nF z>CtV?e>kIPe!^VvHZlZ=c+WTxPN0kJ#9JdsMWO1oWPNBa%e$#$BA;1r&B(jnwcU(E-?ma)#I(jz&wx{EL8GNj zf12O2f`y%}1TZ9RGRwa>6JH!nF;A}?A`h1?FBHAIm`{G#Os{@Ft-vyE39h)@8*h~P z(K$P7a>KpwM5H?7 zs*JMLV(*Z4-7VXqO7Q|UPDi0RE6#@7NTR0G7;2v>-#OFG70^RyIM~h`7ty~@tKi^k zq3X4rTH6%5+{sm6Z+gWPW>NHB2ACrf#kEOQV2*;U!+rRHIjRD&TpzdlPDr;N50I4m zI+U2PdO^^V+DafXg_^mq$#_-g#UCWc`_O^2K$hyHk~A#x4y_1x{5ylx@E)X^6Ud+N zyd{$V4`_U@LPHJt$9&?buwD9;8IvZWLw*-MF+(jnu{s>Qi{$h(AZ8!HaZn{M0sGX_ zd45RtomIYeqoJC4f{9GP+0EwO&iF{X+Bg#bQ=eVpHn~Z%P~95MVy|fXb>^Hi+*@S& z6HZ065EP%oGRp+-xSyQ4PpYFaaRQ2stQHu`J7GWy4G=wkdoTR@a zXnrFz8p8$;e`ziUdz7Lo4f!kn_dd79{J}itV*1C$-02BMYn4y@4|W~387p^%@%IQk zHcJt~CN|5B3d@JHg{oK1%>I9#OY!@2~OJ(A%vHvs8NV()6TxZG+3+#SVe z55%@li#YHch?4-5MN#{j%Y(a){|j_9@V#sOPt=ji&3r2E01Tt)n2jqth-1Y#mdv+R zOL%2^K5&3ajKL6mw0!i?F=O&@eKxW+_H|K3HYsXwOYZ%hrv@B&Y7Jqx;^&&8?gl*wAgv{^~{~=bF^iM&)XawWZZ^kfdUfy&7`XYszLDE!>a&h z-MYkEr?wAz+KVK0r9@35o`(5L?(V7XW<$4EW5lBjIJDyX!_4q-JYei#a zuxkf?!xwTV?)4D>hU87Iyi0|{y%*~F@-1ZqGff;C^c@{Nj!13kYD+#&1UxkF?Bqr( zUe5rF2&Ph2yL!~VI-1qDD9i7FU6XpZ(Vxe+5N+fjtN0J+7>PYrni)Vj`RC}8=rmB; zva6uHcd5iZS9`bjq3)MQfU0+9H(>XvinAA~mZaL24`aKv`Rc!6yv0Y4pB^)pSf zky`S0ZssoG=5yB_d}n?D2>fQRqcBs45flo56mn0>n{8svk?2AZoN~t@$;JDjoO4(< zD2SP*{@XvF!8#12A(E+ySwv-i7yHAsv+6*qx{#kw2G|(vTF8k(V z)_EK0X2U$Y{wO;39-=WK*iAj-olwO2XCSo*HR^m@kp$tyeCM;j5Ice9hI`SJnI_+F#T zzRpdk>k2)jOs~}PG}!)FERfK!{Rf+p8+JIa*Y?}(PgjWmr&KK5yjTGUOjbTobN28F z&X5D*Vv+V-Y6zr9s3)`)!s+PtT$<^j|k!^ z=urQ-W1#jc$RtlSy>7)`FZetbDdR7)N<2}ofcCL5KchG;Z905*eH{C_by(9PzxJK) zHK%55nU?Yfg7{Y@oI%bn}7H=3QI z@9y0i`}GsmLm#E@UJcIV>o}S*fjuDtiXT(|O*>lZR&;qEjc%yk9{gIg zbUgZ0p3EyDet#Wr+fbXkCI0X|tOt5@+=WTJh$(h2O;^JA-d#RXd|c>iVr@J7o2QCs zO(9Tr$}X?|EBg@Wj5i6&U-e{6W4#T`816M4j}%qj-BbD>nz%q7^NR#J#Uq z+haciQ@Z9v1qfn&9P3bL)?_}o(q7ird_vKgs zzA|qh|91%Fp9M4j|A{X`v|p$g<^3C9tnyA4*%Av7i0e`1)`ngHO^StY^UbiQ`f=ok zkK4gkT|St?W%oCyOumm@wQIE(^suv)OxI$cs1hKn>+t^X!NPCxH z>#8SNq0^6tnyfRyY5xP7wmYG7CR+F_dPr=}8Cq8UQpTSNrX*eSPC+NzNv&U+6(5F# zn=H>tWSsF zr-DWeC%*x3m{oRjGe_?K1`(b2hxny+Ob0VDT^Tpk z)7x*igfpCbltRi9`sjzaP)l&}lex);2T`|rmkoYsW{Ujn(E>C@hUyqBt}YL_EcXhZ zEJ0?NLY60Ms^++;Re&w*uL<7F^c~LKw33ogDIhb9gYGYn@(gJ*CV~5JoGj#;l%B}_ zRU&6t)v_*^;IBj8`bTbsvxX?&?H1|Cu#U_0<`ax}eti*?n!w z01xN+V(;m&`oxM3W#Q{WJ zuMyGa@9eGDFK`u1r`AlV+pi9rIQa@O99s^<@Sz5aHQ z?`p{}MuWlm2aJ)x984;P@t5K-E#+%OofhsX?6T$%J* zB;QL(nZ|P4i<)aM!jDCTK(K%Yq&het(7T_(czNQ+OFiwy-xps7mJ2^d6K=oOMjR24 zVIb3_TC{Te>)^TZ5BXSw2)qYB&M+JNbX)5G!5#y?p)dJMvo`zz&u6iY0NqWsB8;H5 zWv;vL7U&~NTn;}zsa8)(g*ZkGug^JV?Am4^{rk453w$M%XxF8`zh7q3smm7Q$Xpy4 zWA5bPMssua3mjK=pnW_hdAR?z*ekfY(jgv6Zct<-uVHW74FmypQXmS_>3P!M4gl$^ zTW}Ig04;ME861rtC-}UKi5w~QIxPEF61$-_FQvCfLBWWpeEShK)Bmynlp>5EP2Fc_ zo5BmRtf2fPg4VYq6DAeqPzDx(vhi>u%G94n4o#@}lA1!M*i1Dm1vZ8&An&j8J+@g2p zAqf@i<*xj`CjMjku0sj3xsuj>NW3MMDlsmyscjbfeNC>vX@Q;vnfc?I5$^Y?UhQ8_ zShQ|bKrQND(J03yNi`i?1}^Amib@*?u$U#d%?MCjbwSJ42s7<)Hs($@rw# zK%-Y0dk1@iR%yChQcM4w6Iw>;d)pJ}B|)O!rYJw8-=bTX>6O#w?VD1@@7Ogrokrw^ zMP@bi44!jr-PVQ4J?3yALHM4+)Q3_lgqn_C7h{C!!+497ZreUd9lseG--tQz%m5td z1%RyuQ&IWJqR-|%Mzs~A#Ulbr{oaxmBsuJRt(*7i2WAhY2OfC>i*}NlM-D$x`Swv0 z6)_*eQEkw%m;R|JWt z%dVhb+}$scGHO3JHq|z=Z6b@6l1BnvH5~-V?O7dcT2t?^WM;&cwYZ<{P1cgC9~Cg) z`W*R4Wj0)354B2VVQ)1u#Vpua&(GAT=-8JsJQ9zn#J-P9G`Vf)Aa=p+)Tl6ZHIBF# z5&yeaboOJarV4ZC(u1F*nBmM%au3}oEt~yyqRD6-OUa@wh7yK!yTU)~L!?u%?iaFm zh|!`#=*1(feq8Yd@mC-Hr3o{J7 zOs%C#^*{82VT?sLvC+s35%i*%T8M>PtK|E1=+r5kK1J^v8lc7T);M5KhN1hv2dJCx z65g0$)k#ajSed8@X$6TN)CuX)h7(N;bzb=Z*)5*2PO+}uf;d)o1Km!ld>{qYsf%;u zy@GwQH?uH^3?=}D7d^5}TZ5~Ey6Vu6uHUnf6a%(QDvxnhtm0bnaB~$SAY4=xv1|GQ zRhF*k{iv&0cuxwQGc_`cC5E-V08#9-71URLYi3IgdUjGjW_CD*w!cd1g$^90gdnG4 z#KpRH6<{hIPe%@rXCM<2SQ`aE1gX~fg3LH~^MXAx(8QrKE+PgX*TU1S-pn+47dFP>{6R2B z^q*9VHBvQalqcYvnb9f-ayM^`kL`gj3$}JYbmi#?f}eU2de{VkPhYRDzOh*@^OHA= ziNXU#yd2950#d21X4@&%aEQ zbc=w^GljB|QLB(bh@8)hUk(Qd^OzZRM|-wa;Y6@5b?NxWgH@$Bv6fGJ#5qU2!2tZhOY}NsloK1V-3boc-0$5k|CEKG9xSD z)Np7VBeVOvB^%BQx2fuHG*2^`5A6FMn*?2ec2JWEpu8FAk$u6@1CX&~IcC&}s&S5J zzS|E*Lv5|=Y)_S?uwr)Z;D8j*<{QmmHYPW#GMIXy|Drj0CEo?uXj^+G( z=Q#~;a{YH7AkXsNVvJ)V(lLuHo6_fg7K~4gx^Ix48f~&V{@^^tpCBYS5k&&TTZC=VPrwt;UUU7%*eq^aGI{ZlRV3}xYInN zxC`T#l{z!H8tfNXKV@#1Ud>k=p0M8-d&OAzPVqbI$vKsCK8s*w&6*{fhX z+|-TwWlY(cW$C50olH>R(6+&$hM#ft`|Y1q0Em>zca$Hy)HphcH9KA$%DsMf!S5Ma z!Z8-nEQ6*a6e_4cUB&ve#+UJF`mkh!(B{*a@dtXH5 z;Y7A;he7X9FloD2fMkmGf%7BKTQBNYGzFjUed4POch(_3dLzQsuAzdBuCdA}z@jOx3^~T>6bVMv=<_!gcoJ*FaJ)VqMWPJW4{%Ob$#!gr(50K!@^GgPYR&n;<0}1e^?MF_MspedVtkza3abrh>(Vvb; zRl=DZMPXEmUbvGOe4^ZxlqIFVbQ9^3d@Fr5*>oNfDbjexXSU0*Z)4L!SriFkTx zhShjLR)BuX1Axv^ZoY?5IJ481#jY8X(dS$F?-H-;!&(E#yw!P5j$Y}qjLIu!}=ZI1Q}XBB_PGuiT=lTNDPc)-Cs0CC9>5ZrB8;b zxlaFFC6d(A=#yfxpU>jGj)j+AjHccMdr|oT{EW4BrT76stmUMwOZ|Yp{sI;k3WW>y z4EA@v#_#t-^r8cFG93LpDb*R6qur2G9J*5I>q7SKXZRWS&A&3aPx7$%?H=3;ixbmB zXIy@dBsfy=j|Q+M@ZqIn#Fu!?HL+Xo0F z_|U4GXj({B2WNGKKsvV+BcHFUKp#@yU>mU3X3iEaEOSdx8G?Q_{lwA5gue#uFG*k) zHiB@awgBomQj@4ywS`Z1Mk#JPjO-KMK}l#@U^wQfVReqVEX?JHP8krY=7E;2vo~qb z-Z^74u><<~XfK3Wf0qCkGhhB8EwZT^h>6g*zMycd)9P|Hk~XwLL~^@OIrhlg7)q1# z{rjNYDG{xjUe z2B5eXG6%H(LUB)erhP;K5jG1c-<}PX77~l};tyLEuULi>!DyYl#M|yefHmDOjp9h@ zp9knBVg9|E6x`t5GHzvF2g5f;eGj|xDLI|(KWK!uQFi* z`GI%BH}oCWV}m{PcKQ7NLhHYaTzJnQFKpJ-D9J9L=_s*#PonQiR-2850>NGKqAEju&x;1(h+d5YBf`@}pPdfowB1M05Le&_Tc&4JzasG!fw>a8%^`ugH{yCj%-5 zxioEc`lPv!^eOBif>2rT@$%n$8wvEBmQA2HWKQFJ8Qk?-Csr)oUU$mqV^C>QK;q{C zEHFnl)hnkodS_!ZaYXq8AI%TIfdp!U3?@LKD&xj2~&eVOnLwA3a(jD*=0V9jw28gHh5VJIzUF&2E-S1L-X`F zl2vI5(G{s50;ks!PB5I3E-0%V$VMUc>AVajow~H|0rUD<=ACgfu+3a$B@`SG1(p$5 z;|=5qOf{AQe}aDoCtl5_`vY`oQC}kuBH^8@e>WG0KN>LN7So`^obF{#gs;+A92mt$ zW%7wO2+y75cgV=d@L~)m9tW{+rRkrOeQW_m48G`o1rkBGN_?9aSeU}K?SAGyr{qC; z*~tDt$>rGUvH3-oCh~(~%;k{cZ?(>*Hx-2=p71~?3AJ#%81G<(-KQY0v&`6MJD>OOD?g*3o7sChgpA`Y8cJyA{4hUZg`<%cgdn3{XZ#SeK1}PFo z1^O(dVt_6DV?yJlFff;~(WUz}Pli#l6GAMjRRG=yb`=PX7XjIt1}y}cbPF)-V2F`q;*YcB!47+ks)xmO=)wz)4(DKGKFhT#9kx~TXp-WPIBw^c}a^yfG#>Z(7P^wA`Vy{+p z8Y-zj<}8Cf?Ylb)&Vi&n_b}H%DHi{}jiQkejLIa$q*cf+2mCe)=^32W{q;H<5Lrua zMcr;RaYRzJzI+8*0o1r`><1y1T!-jM2tLq(lrlRWy5hWpy(Oo$Kkdx0U#ODh!cI44 zr=aZ$ltm0}iY7H;#?RTw0?f*$@AlaZPAtHt#R=DLvMfjS_$+1f+g=1*Wgy)4vuba~ zH8AWr8Wf?FSphi-;XW!iCL|QlyugD25AfXwsk4dOkdS`<#8q9^D|q|g)xU6)2zM|fSP_;Qc6h! zPHNbD_y@z4mv)f(>Vb(xO}H$pQ$1MFD~ z5Q_jhj#cp6wki-LSuz$m_9i@P7f4L=b_B)!q#5!z8P%lm7;noTr{vD;OXjqHh8rE^ z?jye$EQ_clX|d#IGD_IyU>apcFEcWI#QyQf(3scs@{GRHF(7r?Vw=zah)=%fV#+$^ zlhyWj`R=iK&h$Me!DN;XHzYQ1`kOQWj6Z#(XWAy)i^GSRo#Q#lJ8t%XLzl2GGr?@C zt~yYB%k&X}!;R#FeW1AKEau?%1+9Wu_ye+V?Fy}=Qdo%f1~Zp^9_Vko|5WG1gsGrR3e$D?w-do01`E6apbVzsi29%O+Nr8<>3P?#K-Q6JFuxUZMySsZ! zBi$k0jUf15c+R=+|NDE+hv)nA1&qDcwbxuR#~gF4c`3@xTG~rbiJzw#{PNJfEq#=(3%EQVLsDM6vtw*FT?3{k!TQ>r^mT9-%=-NL=+JZ=rCq{jZ@|I zYp4X%uojIBZHa_xF8G^Lv#D|A0dU%WV8ho@n0T44R}mdoXV4lhNXbY(9ZZRL+@b`*9jx5l`}3 z%*j18RB--jpdk+p3T}O)n=sppj|x-Lj6A3~(`34n5=(_8{96?a2+2JoC)Tt@ddbDe z*@?RxJx45xFjd84%lHTtIi+vOorqbt7HzEqQNGB%;TphF0|=q38&B=$OMLOwFsGcQ zY<>J0U#?!XwydljbF7_}jDIPV1Ee^q_6;C3EJOAS!X6zl{xddo(>oEH3oGX%p{e;L zoD*0FKUbOqt;&{(*MMfVdtO~U?_jQ|q&ogHNm&0d*TB~b%~J_s`nW~I!pn0qv`8EQ z1k?bx>G@B{k(@3;z|JWK?+SwFYfW*zclXq<`@y zRw#agqzI=Lw*-D)quSC|mhXz(;paREnGrEfa+3=O(wzj*Ok|ZD(;`5TrHxi@kd!aZFad=gVmmJ@N{ptn-GcE%(KZuN zPJ8|yg)UFQDnwec>e5^9=hHM20KBcCnq1ns0amY|eA*MD0?s_aO(Eodz}K1DjDy)| zU#*?4*n&q8cL}EeafZ}Ic=gTU+b54xZIre@+Y?px_x%nT?2d~c{=IMvxNUjjJ zRnH=SqkVyNg7j|6>#Mo7qcD$=b8KVXT*{t1(J2XUZ1p@}P1dqkJ4|B?Mf8s>&8ViL z_81k+08KM!ZSI6j& zaKz?n@VnaOQs9aBgCzb#lPJc!jpss&8{JVdb3j=MAn$DiLk`FfXCzDQLOTB?W+bWD zRUe(~k-<21@!PTQxb`_dD{e=A1M(EPzS_MMB6 znrr^p4=!e(2LGuV1wE@qalJqOZ?#QwxK|MXef~C!(`qX|+# zP0R!etvQZ=$Ww8j3tfi2U7I=nNA=9_&($}-W0oJL{6n4^_7-UVHs|!f<$o%DOH!LW z7tb^anROod$AjKJH>aI+b^eFAHn!$jMO!0zD7f>F2hlz^kK1c)`JZ~OfCyLfSw*{$ z?M&hLj|as)HwRyRTKUH)^H`omxJMH$(f<_Tf}fj%?kjBmF-jG}XA$oItLEgS)BFwO z4j({|aWyad8jWPYg0fS5kf*V%l3vhuyc2u;Zd>L$#ek{edi&i?pZA+o>`*6N`VVQB z;2c0};7hOETeDcf1$w029|Rr9m2ht($UbZ>lMH>XE>V;1`9EOybo|CoBf~Q8ZZad?-*w zS2nyImw|cjAmJezBWp;D+F%tu*#P<#(0)Gp0W-_~XIV>-HKa~{3JoG?nj0msJ@W<9 zfo^f62vG+gyxg~xeM66t?BuI13XnY- z8jPUC7GXf56IA7VEf#5#qk705u~9TF(w3-^mn_kg_m zin)RR)CDL6SwpirEFjb+EA&y2X}PH?;@EB$m&v)g6i+LLn==^*@&LBM&a2DWP1vO2 znt8zp`;W3Q#%EDn^xgy-L?1vDMATh6?^6n6%gd4??{P-kzFRz&G43#uxdxQ^fXswN z5~szdm}Dcketl+{U6b;cLow<_oYZc+lU}Saf-@S;FDL*i=QnvtoN>`^E%?2jYu8eq zvXGULy=NV39}hLQ-)CjmzhSBCehopyG5*#@Iqykba1CM|l1N)&wwIvBhiZ`xMRZC!X>d=EvnEAl^ss&|0`VGyz{?)Z z)}DO7PT}+7clXIc-tcK*I&|DNSu#?7lQikeZEfYTA+SWG;VFB|Sr}v~>VCH=lNLGGq8($Q281Vlx z9oh`sg*N8+>TWV8Q}hc<7yQ_y^oA^!G7_uIc8CCk1|@e)j+jh?4SRJ+Yz}W>kyny{ z0~@K?IU=Low$*(*&loFMs~bny!T8E|Jty!jLz)` z#?K{VfslcSmtjqy@c|3@=3`!X4AKA(xk4H84#ESsC&$~6KTSmr{?eR%hyl(+ExR=f z@>8o_$TU;G->0P*nHh3I4yyRLKAOFLbQi7i;X^*C2Sq%7b?zpWaTKFC;=^a79+}yK zPcEVKukUHeY5(nGv2OL#b!3nb78lyGwNm!3Z6Ts4z`li>8eMECmcTIOog-iIJ#IJ< zCK$%&ig_J@wl$|i!D0MAQ}2pwsMRsxrx1Bb8CKG}qywkBamjd{cuikGP=@eA7>vVF zZTO6A5=D`Dy?EPNzhKvbB4ia-z%_1}wuzYUxhsKiI&L(sM+jTydl1AY8NohV%xYfe z)$(SXq1^|u89qU3R(}+G>_t~4?J3H|E+p~R_MoR+t*T8kFnuG&y&?r6n%!jyn&!{! zMQ6<{Ry8;>I1pMrVj2P&8kr&!kz7rRD>eoj0yYSwss45S&9vy%`W}Vn(966p@-t(K zJKRCqFE0me*7aZ5duZV%zc>vBQm1BEj)-bP#;~*ogB-c{jqd~n*cn+g$G{K*VAGmC zIl5InrFiOarxY%S)4adxeW#UL>GDaV2d){JYZm7Lk*F3PGKSeRw1gF!r$)r-dhFxZ zm)=>;AdQ4cD_8Snj_N1o`(^|O>RiGCyU-s0@ zwxFS#^`BX2>J+;_>qAK`{6GU?zfc8sMA^bb5vojjHhCVgjdjo`#(!qIT*Jzr_~ zkr1WnCwBIS$qD?#bU>)y`fbo*C-6m;>t?>eLD^6-WFkxD?R|fXH-{an{z~q5sg_Ye z^p8iHa2(m~URXQ=1r)I~_wY15Z;`rTECVB)iDZco?gvgeX67lof*ZJyQ;EQW3o=y; zrOaWEWTyJl8#?jyc- zL!(yqFH8N>qyEFA07a=Ahw#dvRvJAjFBEZ@Hp_?CnVDeA?x*yCb)*z5v?h%w?KN3U zV)~gUFZF$&bnkIAStqOdN>9{_{ml9`1H2r}Gougsy_gJ|$;K1FT~%FYnrrdjHgSP( z0{Z!73pC!R830jsjKdG9o`W~jG%Mt@myjqNXk=nIhihzJ%>rk%zSUqx zn&82*XaG|KC_>LOj7wB2*W+eZ>0zQ`nj z=IDXB=9#+!&=4K5WqCpvaH&A<>n{Z0eQlwU#?>}7U>&!VR`Q}Uj=#&K(P?$NDAaJ#_jZziN)AJ+MoOns^ss zK!^*{|1?H9)*o0M*yboccpl=6WdGu0QT#Bo=O&N8X(|;q1zX3hTWH%##DX>r%ZM7t z41Wjl?W^2~!Z&i%yx25sT{?jgNP&p4ENn*L8EVJKH$kQ}0m9>gjl|W_&BjNV*zZ9t z=@>`w>)n*2x}SI^UU$SK1@2nIGLHqV_mB`PXj#nhpNchM`!zv8D1o$^>`uJQ4ESlq ziN@!rKj86F$%N8yko_9PYN%zFsRg)?cI1EbZIICDL1+j&M6ya_4UoVmu&>cvkT-ydnw4{U zP}D?Ah%kL^A6;~_;ksaH@uA@^x+dIzl#SeElO7v@S4Mot5W;%VXm>;#t`A$yH zz9*|ZGU!%)Oei&u<~qY3ZU5z;5vl*MEuDpPX68mT5daG6dI&Myl{SfRQjq3sh6!LIT3%$FN+9TFa| z_H`7vC1h6Ha{WyK0k7eJ#e_8<>Y^U~f9<^CtKqZ*_cskWR#$z^F7p~}$3uSJ=9s=# ztWCmZo3unfbi@zDsz<5sY4Wjecxw92W*(H2@ ztYDl7`~DqYJmr>423a$B-`sWAL{ouy;fsq2} z=bqrP+S%Sf=tvYCF0`PhM>42y!)rzC-ukiI^gJcXgc%+kJ5}2(P!WYYE!9gVs%w^f%QqhiNkAFrP z0Jps?>Te3ts;}**16mO=Gs~i?i8eZ5I#yx?CGK!GD#-)Y(k91BBX>j(i0;D@;1aVx zX#)aJ-7>`&e2zO*ysBlgm}EEbc82Z9r7zqWxgZ2>R0#29>(_(*(Q=4~0qr4jge4An zu#vB(SVOUden53W9ml^=)W=Yyjde~354Jp%Ue*FO*HZMZR}@(na@=w~rTACk!u4wCMEoRv`| zpOf(5-e1_cA5ul6dmVR)u=~=(WIu5IbC`9mYkHcfAbJzd7FCDk&ufk{bNq(xuI1L1 zIzEq|?PwJ6=ray@Z`1OY_4_!pBNgYade`QH26jh}P>5g$AW?2YVWHKz>lJa7v|Gah zmYIXuA6l?YbvO4CM@U_PQV`C|Hd=?`q)>-aHI(t!ZDdCouaRFGiwksc@sd&VB+~eG8Zn z6StHoEVaEiKZ*0K#vVWmS1|ErO0VnG7M-af{mvl@u!4B}!LpRE2Jc#8jWPt(6QpoQ z$;BWHHhWEMQ_O?%B7>C22`LkvR-g{YF@seWR3biauOw^QU>&s7&Jov02~!xvW1h<}KK@`4dUWP_I5S*}==((U+Bpqu+5 z^y0Lz?6@P%{FB2yxBN=K<5*4tW%c>ptIytWI`yXM*Y&OmXbvQ6aqT(HQt-3ISi4qbHtrogJ#!^Wq^ z)W7qa=P+pPW_+)^xfb=*ce`EQ=@R!60`pL7Ys6(XVgy>ZLhk*H zy5F^P2PSlWUn%N5xTcA&XjBA>$m`Gf=zkn0!R$Kbk%ZppO{4p0GJ5h+UY=Bq)BG8j$#vzVyB61^1Ic(#yFu z=(f)L$g|Eyc?Xuu2*Kb3;u6gKd=81_PRMwNG+^X1}2 zoTlGJ#^nV3u`KzVP=#s7Au?Y_U4)W|py3j~4arOv;~U(dzy$9s)8!E@GSmeX8Ek|v zaV)Ux!U+6lBv09}!H)NBo$5-y=`Z)%jizc=B6QsLhL+TECf-y?m?z`B*seu4VmHRI z45EDPbJimj?_Q?zg-#5z@jAU9gT2=@j^&DzWN_PUZnznumnw&YP)AD*l4EWwaZu=T z^5@DUKKTghDBre8)5X}qo4918cfp#_Kk?u z+To@INp$NI0Zd3?R9HjLlM&_!BD&@1t;gCMYT9x;V^?mufG< z@CD3{hLR>9Ofk%KC@6QU&Jw`Y_!~}kb^eteOSQZQqT*V~$tVQa9Q4lkh9~KNg zQWdJ0C`_p^WmN@C2^>Fc)*C-}7;sGnq8+A~GE5q58^*fgD8C*|m(sQvOgk&2{0^K{ zc`^$-zpbM938Y31l!rCDFP7^yWc6??e702<-}%1`rof;n1@r!F-%(GoDKew19Y!67 z_?SjDA4V9_FC%voMiA~LY8IoPr*L3dZO9&b*$s!^xNxVX-;H!4D{R$Dqvu$Bwoa`v zGd^SnQOq1>{O~9e?I5Xt5s4lZS_5m0LCd`-4=l4c&qh+R#@;f9^b6wx8*Gp+OLPej8LcwymUhAXL?**ZKa!jBQGUFv zdR2NE1I%8#JAT(|m?=;2Y%fd!9Ch<%=MP8jX*@PBI5BIiRdgPaU8wy z@#)|Htk!6_vt-d|rXb-my)h|`?`~IK1Oyccq2_sD3;~}FHtwX?gyUt zcUspjCr2c0XYaM9t*2j*)EW;Kbw8eYbrVfyFZrGM*)1&nXRy%}Z*rXG%C1Ae$s%~t zK8r+^NyUCir;Z-c-*RMKtKsz%JPy<$_3&Nel(uT*NMeP6frjlEb_r<$i}# z&BL6WrQK=ZW6@JZVx+k0{$ ze7r%7!A+j(>u;e*?Wwey>Q>P$Uqu-)KkQbhBJr9I7+)PP-V89jQZG}8KQTn?D7yYp z4vJt4#h{$(Q(DG`IfZG$>cWRvg1v%?osOR$1G!T^Vq-x1)5}CMe!5bM;}oZtm1QVP zo)W}AQQd-WGQ$_m9D?!6a9;IxZ#pt~2;@3a0~Dpb_ZvTE#@ypO7AW?7 zjNo0rSxR*v5|Zi-tW>iNW-RTqw58tV8aJGdeJaw)W&7G_@b&W%K86QcgmI9eBb~^8 z)F|1U;YdPdV$NrU2e@PH+1?#*Ut05;2P0|suTDf!Y1?=(Gq8XoL+awJti0RX-(1k>kl~+*m%1Q$*ThEt_;&M9Vkhcc1C1fJ&sBj? z6k%`s7cq5u14E!I>+4zc2l-#Wxd=GX)!EWTL%uYt7!9THw<%pE>c=oPt^I4|oNpr5 z75yooogC?=+b3nWg43-WGN*G1M`{lON&r-nq(Kwb;y3y+v*fMZT5wAOdMTdOZZ z0^E+yr#W(kGAow zpHS-+h9jIa*{tm@D)E>EL^Q(ExRH#c(ZieV;{79nQT0k!8GJ?HRu=gN!$&)*`;Cg( zFMfy+b9qsBtT~d0ohT)L8hksM)mQ~qFXH91Qc}!+P3S>q+ArUgL6>a!J)lh1m4o9X zcR{u>zRb*!vF2NtUw;4BjA+Ag6S3T+W97qdgyNp+H~GsiFQwJEt;S;1jVjGj&gu`M zWakHCDEf$-<ZGw|jrKK17O z;^Ocy^Cg&#$$#EE2nMS6Odw{astBbhl)yHP*IJH`%-!+6Phl}04TI;R(T1~|QDB!p z@bk^qTsOxHVJ~biK(T4VdcKr0PvkL1CsV9LU+IzuD)<{M0Pm;X9!181;@#RFqq$}y zu~Q~6O=J;4G}6;%QtgbF`0qVovt1)qq}R?pd%_=C`mrKQ_FMSN4^Q&b=F{Jsk#e$4 zzB(0V=d0xNJ!1`F5I4f_jK)4L7q>u3uU+GIl@fB&&LvpI&qFz9>p8-|F5|xc*w9X5 z`x-J(zuFZO!Zx5<1SLcM`&~XY*-?I4wf(a2gR$7=?0xMlh3BA-$Ty_+U%;YeWX9!{ zF`Q}hAt=*s*V&#rkL22s7*I(`rna2=B$4)qVvdSNGx|dq_sjPPKl^UkVbQmpfnHfX zj6x4$*08q17~QAjjtDS3F6C-yqfUzBeK#-i<=jAsqIuEn&!$Dr8+}6vG2vRLRS<$} zV3}R?+^~cF>!vXT=d$>gz)2Wq{91twD*`R!sM+>FNH^{V!KH-DUN48=Rr(n`$F3^rk(9z%}Q^rT0eYTpYghd;102#^$NB}>?;C+Y=UB5Pn&wRzOq0u~a zeT)Y528J#E8?h`e(u6$)E_Pm*{k%Z^c<3jD>{$0EZ=*fzL1WUa#f!Dw1*MP z$Vx*E6!Nv&(Zhw9P;y1s-hC?LGJ!@5j}x^iA5YSSbfa19L*IQp^1+9G!yMIe?fKz; zVW58+J(NHnLd#?ng4MUECO26ksaU4ffK7FK9Qo~!P@w__m0%SzY5kGkkwB*SDVBDw z#i;8Qc$bxsPag%V; zs`NZi_~VVM&1b6w6gNF2oAnC8&0mA593>H~VsttE4g2heiO_6U+9)4`y)fBGwRtO~ zZgiRn;%I(taRcLWHSs#MzWfdfFZogFXzGQH_VE;5`%?J$?{N>=;#>1Os{_6Ay zhEt`;G+}l3ZLi@u=7!?dd)iL`8cpg6%HebTI+NAt2Vz!gQgYfwSxL1F6t*S>!oC8U z+M6$cP`XLeZFtJC-MyC-K)a}1M7D1m$Q6wa`jk`nF6E?Z0JjmuZT@59CsefBX+!5~ z?~biQvmXA^ba3U$Ihz1-edwQg{?5OQDdb=p+NrwvutcD%PSZ*1{I%#raf2c}V#jc= zi6q>ZV>(6fIR1byz?k*-J#(2>2TJ^B)(?Mtwl7S6*O?MI?D!~bdr7sJX_QF;@n4z= zgr3)QjEQGHz1g0D&Hq1I0HmQGN!WH6P(hd9uk(hK;7sHQx|huZe!IBLwrv+F=nQ{6{+XZMZS3W(Q9ER!4WfT(WZU?_sxGt7^X$4jMkvs=Ziq z^Xd#PZ`=Y!k{I{lhV|*#^XT+<${~b&0Iri*UBSLH=Zn- z{DZHdh7!L!-UumgjB4>+PYUUnc})(rv3c}6h+L-AYF-tLK3cpG`DOKjQZALWo+(ku z;60`2MvU<{{wL%Eb=Le24dXXP!5CdX&*`c>4*%5Y-_s7%-Y@ z=_hQB3JcU0*vKu+^-iYYcf1jfe!O6%bieC%6B2ej;Z*fH^!{D#TYc~SSn!Sbvfpat z;WNF9b)@7?**+^Q0dVhdD0?`2!`8#Ho7zX z3|^8dY_s*K-VxAjb8FCQ&S9d8*e(~8KU;|P*_-r5$N*=e@-Pz)ddose)^t7FYXr%L zZ&8pvYTRySV>#>(nERnuV%GCS3C7QUyOMAuT1G{<06cqcBtu}Z*yGimz#9*UU}f>Qb5Q^ll5{(g7Mz!Zw%iqp3WlPc+{RW@9JLX zQ(jh170AVS;;j*d(cn~JQf}Gp`64I$9{f^{au|rxis7*m!sYVM`J0^M#kP#d0SV|2|;_uCZpKHc$FVg}m1+bIv;WQ20-ff}h>JV`Tw&XM162i^sKI z-iQt^5u1*xyGAmb-avNm;6$gXT%GwjFR}D;jdjcvZM(KF`2&|OSc zZ~_?5#0}mMDWL5&F4w3h>3_%9m$M*gFO&23#g|tPbQO%z~i@m3VT4y>zvr55s$ogFqI+iG53n5gtbqp!O4v%fR{ zV@oWW6A@eX2RpW3@XikX0hLD6K$1A|#a~`R{_Av}P?{E1t0B2!%>mz`{|Dd9?#M8z zl?mK8AFuSS_4w3+P}(EToi{Q@^@@E9Zc9GxcHcb$9PYXM`qQCa=)xyj@tWYMaLJGg z$!gCZ=;VSI$V_uqA#BHF%R`glm5K~M7cs)v+$Cz9Xb#mX|GV+Ip@O~}v?p@sa0QBc zg=2x7FFL`pY3zlHdJYZ*#$`Qs4tuF1klzfO7S*(4faPef28WtP|lc*3L0^yrC=A%VAjAw{?q@ zv*U)o1GjL9yL|EKcfT9ra8BqodIdoa8VX#_a&G##`)YtmDB^7 zwXJSv`28P6kq2D0cSI7`i&5+@chu;AeltsjdN^AIVOVxb&XPAzGi1>^c8og}{6VL`(A6}O^|FZ-|FBIVX|`MX$Sj6Zc)l<%I{YmA4(Y#uAtIQHag zZvI@+wKP};e_Lv#_8wr_oA<`P5Z1$9@}1AaM`sJRhn3rAon7 z=HXz$cdVHAT-77voWN$|F=}9=^n51&uf2P^*@=BNrYXi+Nv{ZFjirWndb)>- zPc5dteP~V7U!(_6*YgUOsOk9;28>uKZnJ2u4&$laL+#th>e;cw-bB49lX`iSRq?5E zUXJe8tx~OzHmjRqy6-Pl0|}AtTu3k?+$+%S;&ysN}{N;=r?q(AA?+b z&J>^4m$rp})Mc(5T3Ux)v3sy?H2R)S5WYN9sf$E)bB{p6X(IWMbgi^q?oq<3jt1=x zReaE&oa5p!-%-hP6IN?eGfz3ADCA=%4i3*uY@s`kMrua{X-?KOgY~!T1iAOe%vavq z9peD8!@K9$fg4-J21{FVT&)!IP3+H(yl`U@fe%*3F9VqTYuW; zDB0)k*H4U7<&#LP5@v6=P8!+S*~h}^dO_x7R*7LGwVB!|i=>ZpyX-%qoLj@^@5VNH zTIB<*V*|!TAMig`>UKN+((UR?m3y4c{X4NEerANCT+T8Vn!tGv)QG;MKCWv98}1~u z0E2tAB3E_%92i;r4UAwQ*G?H;z|xDeK!JwgV77W}htq1Vl`UoqORSBg`6r#Kw&!=F zU!FsDa>ff}TG<}$0AA)jsStW?#|nOx<0>fP<8{Kz6pZJ9BT4=cc$r<|4(fYiF4Z{p zTz2y&^y^pK=UR=Iq4!~@u|JQ6JP_$$m<)D5@~uS;iNyH@X|+MSd2~MXov(W0 zF@H&9cGZ=MEi>C^#YZpSBzm_Zm@!$W^gutPF#8SD%XzjK*qUi9&hWq(ey{^S@Frlv zNwz+rle7S6)M>}t=la}#K(JPV_I2)r-zy;Wcr9h!nSPtIPwI@!pv!oRxqV0+a zK-Js;F^t>^aA{Nx)@5wYV`-J_;cYr9+a4U`P%P3tf#LXln#e~7ST^Y>Z{`o?lh1jb%ymi-v&;{+dJmIs8Pt=zJ3L^^|+?>uJ(UT<$-u$3TQ9rSy_tj?~H$YK0I*BS0>%Nb3AeJSTU4# ztV|!HCs?7`jKE5!+I#C6UE1gA3%qmC=TnvbiBn{%up;!aN}g`nd%T_LmN~&N7Qv=| zubcDxR`FDhO8sI5oo3vuHaq&i$KlEZt@+n>B-34NCuO^Lqa zduzq9P;Rih|F&~FoREm456<%N71nZ(pRLe2NH+fV^nm|$rZn=pG@036Ak!UFa$uks z+qkITx5*OKVcTcbmIt6?2LHL1viRz4_3suN12>^lCXRwt5iZ{VqVw2>-t8B_pQu#3 z0VK-ec%ZG+ebD9Z+0q|O+WxIq_J1C+w2FJCg+TH$zM_%bio(SgITEj~!a1DW_oqYw z(wfW0=xAJmBdw|2peJTN?9}Bj$}X*c{7X3P1;AJWY?L}FImU{nxhePRWFC;nrY|u5 zni@yvQ7863d*o-MF;g4&uTl;~N~sHqoc!gLb4Xiw$|HFmGJ+`NF+$eKK}VvFN*LFs3IgzeE!8xDgR$^}aX&Qx|( zf*iFNG=~X&qjae{Lh{LMM4VR|GNhGbgHhLBLvh=s6ej=W8s%Jtd4uNE&%yY*G1pxI zP=e6av5-wyA&sq<6%?Uc-0^8? zv+K$GrA`?FP)JtEy_0m#10L~3XW+g3;`;m~o!Mj-Q^|}~X!m35Mx&l@cc}@JKEQ}C z3GN|?17_?5$z2f&L|IN~i$U_WUE8W2yMJPp4*zIJlbthGFZFrI=)(TXLJv5UrZdewvz5X&Lw(kD+^%I1AKj?lt>Ut)ku2 zvOq$qD!jb5^V=MulhAa?@98{Q*Tp4@LpWey+|NG{oM0gRXvyyRqrWb9*)67))4+tM zG(SzoZRae5Atua*R})YCo;%zHBbxpKTnxF^wOZ@-LII)&!MQS6Mn?(1Qn+o1MW}d) zjXKg}$ORmiX2ls%YS=d#k-E~##_!BK^&s>f5$OXwd#u#Zsq)sCmd1~uB3(Sp%0J=u zFzmLXFCSEF`bPC~!H|vr1`oVAJgOG`j#t_xKYf4tZn;h1P9~g1X484>!b4Q1(|`?} z5Zxi|BFeYv7rentwlk8ovsBUwvJy3QM&^Yuz|3W{KGRGAv1NrB?u1efg}qhwT(Ke_ zlE^lCNc5ryn{cSMT3L~&{WJcFAhjx^aRLsvxTe@j|Sz@ivx#Wx&cjkF#NWWq>EG$b4SsUU*M(< z#evxZ!F{0vvabBnIPMZ)^|$)y`4GPd=W~^9)tCBLRj_wfo8+oYYGT{Q7zdZx7w>;1 zRerRR1PJ^plPxW0O^#DSZUzm2@d2yL!FaEDDh?gd%v7PQbQX#!%DhvU zsR`;X4RHH53;6t9eMnxM-##0mQ(>#DiZ2SF8h8Pl2NtFUG2^}rR+X6mGS%C1FF#$8 zINz?9Y)244Nr5DbI;&owY;DpjnvXz?sOafJV|ub{xBJcYnYznSr>o~%5Tsapbp5UGzlz8qBWi@u&p;UnC zHQ&Gius30Czh2}H4b#u$10}TI_`eOqEtXrlH1+_mNO_$1xOd(LslWP!GEU@%Bd_hM z!k;PPj5`?f$G^~F8&yw0+@h*T==c~H)*t5?n4kdg5pRO#{hbx`o`+ef*6{pLXeRTi zXLgiKsnO!q`;2y-Pj$kd=V1V{2d=d72`ge}Ts2vW>;3$BdvFtbn}J{gWlcTD7K1{kUXXxWQ!O zxVJMe_0ZRBHOkpcY=8ND{5g8_5`YbbZ9gp4*~EvDX3icnw&tsDggXDbuh&0GKA{E4 z@%4Bdxc;9!S`No?X}?>cyJEUX%|(If_m7rQSNkdkN}0D;)FgScUIvBj3Z3vv+-71`xMM^qmnkw=Oq$e~Of#w1d^5or zyDD*Eygxn_m(O)BLKgd(#PfBvi52?jqQoW%# zZbk1-t=+XhO13>8>ctw+?w=-!n9P#u0cq~O*6AB*9%eWA?&a0$1+PG%<^K>*j{|Y$ zf;0F={xp|4Xuo;KA$)EeOcs)a1b26k$h@8$CVf%1V1GQj%liIqqJi!2!YyYE!iQX2 zZ8R!%s=GTKbG>*Y$RiOTNMIOkO!4cHU(1;|FrHBn4SFG3)%*3#WB>&N77fp*s&X25 zlsMJw2f)?wLSK|ha$MCQuJmIIjvbefyrAS!oQ?6sqV`<<{yl(4|Lkpv(;)-5CaUlf9=5oD(uK|%wzQwaJ zhU_m~de+(qL$#_cxWW@{LZxa$HlyDPvy-N9deh8WFZ+KY+XCXEe~?C-pV@GL+LB5? z4<#_v>C_J9)t0sJT3RW!Uy182Jo;zq2BvZkM-#_Q+A_G8NNE{ae_A*OLOvS zG!mzf*iJTlMRjm6bA&UPJ?IudC3;tc8Sf-Kv;$WA;t6msHben0y}|e10wvThSC%k_ zrRjA)T8c{3A^l3tor3 z)-?OZvogsQfRwzZ|D%U`7H^U20h!0>*s?RHo&dJ|`2fGvUvwG~Z*9A=!ln33CExhuTGb{<&Gse@bhP zhMIR*FtNF{8^DlLna^REOAPTTMg61&@Cg~;DV5vjPl$YL?&yDx9kTx~KM_KFQuGJG zj-bmO2Fv{^gSd+V1|nyZB!vH}+zA8D(6K^Ut;eDW=S%Cd z;(%EMTEG*m99-4I#T^@1K}u({{*EsldQZPi>l47Ho|V%wo}$9dVq>S(2B)=|;KFi6 zRHonA;Od+X_Qw+-xuO11OFhHuh|UiglK?u?ya!j6KT{-sS%f;EB52^LLc46~N(As- zJe_w-l7Mv~`{^(A{$^<0<65|yH*+Edv}|v2R2zPt9k#!-qrkzr(fYx5%HkH7;9cF7 z=92AaOD6xdWFc|t*-9~=dA%nNb{Zh`)0uGq8Wqbu=ZITBM83ZRQb2W!?I@w6%^zJW zESiBv)6cY3*2*hs1}}i{)jHM%W_r0)=wrPO(C7fjoB}hAPlwk5$mq&lUIVs1nfX?# z#RfET2KMH&;o?x&zInu}w-Uit3)~)zsj<2e{2z%VxHt9(a9$JEFMS^D{f!cEUO{-g zXK_bTYM<#Csrq;H=UQPBO}DG9k2gO8bq4hb0x;rGMUfKOK1DMD&r}W)bqQ<@H^%rc z{-m@RRc}mYNg;DH<}rdIR~FBB@DzFv(Eie?-%mH&Kl8-yrk3q3JAz8oJqX$b`G8Y~ zSYF#A^!;6ueta?l>@Z*r&0qyx?f}11@N8|+n#7~l_GJC|%d5R<6pRDa&{vR|l1FNt z7B7-sq=pZVl)OxsD-ThqU$^a zRrsNd#+nO#3}Rs%%URprgMK5UaO30EOEd_*{FmusO0`Fyhi_wdyHoUi-jg4NHE}pX ziqvTFe5SLAx;esu&LWvsPi$I)jh#<;aJ3sxKQwC`@*Ru$Cm*hNjWvUldFdOD=R<0D z*H^X7KDx=^t8?9Y~YiM$Y$M!=a;z4WqraTKdI0H!^fOn}r4$Sc%iwC-@Lt z7U;dfl5RrT1bOiXk#MMyC#~g<6D;zVKr}i-#unx{R6V{Pl}>6qE}(l(S}XA`0+3~Y z$>>Y<78=4)%5WiEdX=vCsU6sdyeC~1QT9!4=eSSpH@de)T)UvY`RKDW= z$(CEaF4yO}wT-baCy&LE!(z*_umv`Nt@kYp(Oi-RI7o=(7NZUIK8gSTk^+9=8O$D4 z4V7sn5`n8hdPKaIE2l~6<>>n#wmSN5r4enO7sy-?hSpV2hBq>Y2e^>Tn|ktlib&L& z2*Ak`=)5hG5M%HnLHXkMyipb+Tu0tH3jdS*4|%t(*O6UhIafhreKS?`u~xXn#mo*3 zS~62L%=!)^sWZ35gsd-JY#yDO}0b5wI>sgj!60h(!@;}Pp zEIgnW!O80qfNYW%iy;1ww781~h%IVlah?w$t-k*eG-Li1nuPyl4uL~IU@QZn>+t_8 z?o7j>e%n4ivhVvk#yY6%D)cuY%a}=)N|=&$B1zVaEyixLjgc)vk+MW4M2N_~4T@wp z)?}CL?w|gT?&p49Jj-$1$MMXIc{7gVnz??b^E%J(_4!^GLu^ICe9{d+TC8-^=wsmD zsD=(kBSMc2QG)+G0dXoj%4THXxNM3V0nKOLqyEfwDoJ;%w_jjD(QA1Q+*mlsi*gI3 zscgyVc(H#UWJuvnSBK2}<%n6N@B}!&^&x(cGljqXMbnDnAGD-y+9()EnbMo84(Cek zJ*vT+rGRy2&o|01p{0<-jrmHlkecnp6CbS%|wNn)B4Ow z$pAf#FukK1w*MF}qt8Y_zou2_m|fHWrN1*Fcgn9~%lbiU6j`z`CTnGE1LzWHj!r}k zE5L*1hf1Hv*7}bwQ(%deSbyqqBTypf@(B_{ofOYh*g6A}$7tuLq&H3L$QE2;(gj%tD!Hc?a(Rq6Gi1GLj_PHpV-nbN5d8ac|r8wyg_}{FfUHfN;ikmh1H)g^BkT_ z`q(+rjH|~bf0ygNudeP7u(o9voE;Ehi;`E+b8-6%iyx<@;>@h0^$xBH4PW!~?BTCF zGucW+Gxqr7pi!H3F%d3#WO|>MYta$mpuLzxO*=faAlD&5o4+)1jG~&sI+eF|k}sP! z#oc@i9rsje%8;NXZ{i3_gOLMDn#G;)h3{Zq-7%av?pgef`qR@a<2SpjjXQ!3dS;UR z?gcx7O8GZ~7AJ=%ScS9s(<$d}tIS9QD#Ml9ya;>!40hUloh6Cmw~s;}sBUB^3>E31_bB1vVqev$Z)>kN4_k^y>UpQW=mSvcl4F|;6$=eyE{U9AbZ!3iwA9G|h z0*SDX>)g9M+O&iZ4D90UB}4yed<$Bs*tfc%<{A#+TT`1^#b1%EvD=7@ALgQ}YrF#i zaj^P!R5M~;Fq!49;0xkvBKa_6sMnbj8xi|iFuR`db*7Ja|JB1m8Y}-u2_z3@$*EeP zVKY9bC!sMw9z;4(7bkS0#h3*tij%6beB3$IHcxFWoTt*?uInZPBA~(p!q6zCuvZ^+ z2-m;N%BTpdfiS(jbY5QDp3$+6}ZDlRMe{OBA}N>I_M{)XV$F zW3T2IW)+DJw;aq98_n=>B}(fGJaoY@Zgz{Cm3~q(Xn)dgG_3p9T4_**>|5nVWECwK6Bq%B!jVz3=huGM$ zwoXE{hmn2GUAV79YC6TNQlbRe_kHKLmGTnp4qi!}{Rk1$M#BSmTbb!Y;+&wv(oG$!qXxWjC6>ar4G zliQKd3|!kmu|1)!i}-{?M>4V;I&l$5dqM}8m8z2b%{Hjtv@=NVJJ@9PfP{+Z>PTJFJok3tmi;#1Od~;RFRQaZI61;NtIWO)3evf&z6!$3tT=EEH@2iYrBJZ{@ zLvd67S}yneq0)GR66XU1&n#tN`BWUb#AtPp@7k$QwCZ|u_6aIOj>n;YC`>J1|wbE9avffK^@&$X02)Ho<4Ls!*a34 z2GmEME(D~ILy`um4=>U(@^Y{)CDkx;{!H-Edc$;!Afkjy9i{bUL@JN{s+Wk5XrXU? z6ngvXO+Da9Lzhu8&OS0&`tprmKZgaB+Z5-(a!{n0jh$*7ITE)Fy}NNKevf2muG-J1 zCB^?dM!?dT;xmge}sr*28J#?z|Xm^&Eyy@3urS9 zk0h+Py}Re#%Xe3to-Ed2ATp<|+VeYaGs z!IVG(>n#gK#eeG7Z4*}C;GhDmgg^MaZhPay_}IH&3oVOGllne}g)Q1;2r@pf8Lu~% zxqt)GQL2`Sm6A4*k0H*b5HCY55CSOQkQv_MQeh5A2wL%1yrm`Cj_-a_n~%ESGv*5m zt0}!zojShi?J;Z(UmtfHq_XGFZcQD6!BF3ZwiDy8xqX`P%2_2SHo0iIkwDpb!e_ zS_N8el*Zwzpjzn_sdSguW>|%VY}ljGis{;epKy%)V>ZtfTLra+nHpRH240DC6Q}6$ zC6Y1jBi}5lFcLU>Pd}IKOL7NZ=9`7T-HRJ|MFgR-q+aKKtl#~VIUC!#n9w#66x8DK zjoGHqjBCCncgTBU$!QedXq;2PC(FZha8^>ij2)$*0}UK&jRQPb&EwqsIs1cWZq;<_ z?5C1-Vw7E)auHJ%gITocUYFa2J`5d3b#03k zXu?8m6bOw|9y5$3d{&!o4sG>lAH$sV^oQM96X>q$0su>~O^FqymK^uZOwiS8LVi!n_#^Mrkf&&^Urj;j}|4*-+sm$BWg&HnQ`7}Xf`f%sUT&>^rXqmq6hD@%uq43Dn%quzXX=D@UD)i zVG4(hP-@jyU*2%8$^wb)65n!GOd=F{IbFs5EffESsUmB;8dAf2Xq*18^)t~q9vr6( zn>NA~(I5R!%iVyjeSEjIz}f)4tTi+6@Wb}QpcVRlQ}Tj-IgXw*`rz$|Mf6ugb-g%j zkG(Gyj*(XmvK{{Mn{4e+oMu)DM;G5&-+DFy=TzM42Af};bia{N`Eo6-R*-UoKb%80 z_FDLldFJea)SRw-1GnL~Z&;Q&=!63onb3BdPUg_?Xe_=E|BPYzq9|%&Ldu2l*!OoI zs92K>1bz!NV(??@p(`C=8@P8h^hhJ%QT$^O)Mf)}`UJ6~y{ghk_Wy7xQV9lQfxqMW zjoj+~=^4h^s4R4e8FJ>#v@AkRtCae#zee^H7W&*X=d#j?_cQlk%kq;o8ig!qn{-Mf;6ZgR6eT zgO#4Xp4ZEF01A#r$nXI4U3otCwpE-?i5QFvg-F`J!Hqi}4I4;hp-udT;SUBb3WZ?C z9`GJideU3PlD8ucko`b|e&wVFYd6io{x2&S{{6`vPv`JlX78^>`w|?`g`X;84f^ux z5tuX+Gswv+MW-@a6RCxNjt32E5q}KiBe;$y(ZY~eWKsFZ1a--#gJf8-zd3XzcdhDg z5%KY?;;03aQU5!S@NDY?3)R2_G5gM&u{5&2HotP`Tav;Yz7ARez*>>zKJUA+0C2#; zw91i){6GLhxr?giEB*vL-da=L=g0lrxI3W?y$v^c)dQcl-QQh4m*{cM@iOb@S63Nm z11ki{{wvsBM1fQp2-J1Nga1n0lU-zWT?)s^Rp8-g<9_A@x;5Z#GRHA~V)ZgQ8Xt77eJ(pMr~NLTo+- z2(MdOW?_|B=Kt$S<3J^yvDG?f0`kw`|vWt zbi+iNa6C$oH@mC9VLz~65HE9^or!-%o#?0aunNFC*Oi4Q=DMR+Ng=IVJ=T>AU21Og zYqxcY)gHw3tX%}Y@Pz!p#oDFLD>=qOUtV-Kb?S5<*sTo3F@~bgUu^BA`P1Yw9%;zP zKAtRO!H2E&f!iw8SCZv!H8spz!fTggNfiMv(zxzUp(TJqx7ZOQsNu#Cvit5M1h=I5 z&!<8gNA~F=zt{jpG`g+`1gn1N20P!nx~lU z-zH=rK%W91^x}%lM*f>V|KFWEfE<%Uccfp+{_}f8hybjhq%I&i(BEMfKQ=+DQXTcC zr+(X7{zaNd5ks*!zD-MGXBZtg>#4w1De&o1;_)-k{2^v-zZx@#oGPKI^d{OGkbABE zo~9sky`z*Ue)RT#(>nfN=b$;ha4arEv#mpd8%7Stz@NUeWTnq{V5|+39oigU8Fj(0 zt?J#c_E_4ZhWZpLoUXkh=w`K1lUGy*F3$##ObuN0sgOxx%(yQB< zX-|wxSKi(^v~;fRR1yWQBA>c&98H>Rk+c-Rur}F~q{2?U{5<)Z6kdSprbD_NAo>SA zX5%+j#ZH>-=yn{qTUSF1`gVVkvagZxQ|1p_{!YRG^1OPfQnF>;(-f-jHTEF-(bRQl zkCozo9mkcj_4JB6q!8G@n(up2FHg%lHkrEive6i6dT1myK6D#wJUP90GG*Cuy44+E z9o1}Dj;LQw+GMe9?OzdRvyMui1Q_%cT$mW90|lCm(sho6-#7Pl zq58ff1>9DEnagvo<)3bR9x6dyqWTUxSj@D6woGp#xbkRYs3lWHs$Uio7n6puS=g*$ z2vu9ICMQ|EwuUbB=n2)AkOPAbM~HVhuDFV4|?jM7|;vzTvpqG8zpx@ zV$UtSd*+)+Xh~6L2D%b*`{ zOP4aNri&KG)b$WaJnQ&N$Jw4@fld{Wwt=13k)3F*D$_FK0$=_w;vB`N%f1x|<~uvp zR_7)A1@3U8GG9k_@b~2xv+2d|f2Wy9Ip=cz?ztbq<+4uCufP4S@=UPt7?%4KFnbQCeQEV$t7qdH3|Dr3dHP!JcNEz>1&jw&` zzdm*rZS}Np9_QWc&(hRA^ZPqRVyInKL9M4vBX905vgWrlfxDu1w{kr``s?AKADl|S zc_zoT8QmkS9m159wBv)d8U$_`9&1@a-%Vt5d8`c4xR}f2c>PW%{qG{jFsUalJ8%!D zpDZtKkk3`ISYCJftmpu}0up?j+JmS_}|ox|AyTEKgrVnlB#nNN!s6j%-Dwp|2oGEVbU$4DY^RM_(!o-(d`Kf X1qX|M$r%y>0WYMkp-z#Ob?`p`oG@`z literal 41530 zcmdqJbx>T}*DVSJ5?q42OMu|+(70Qmkw6-E2pWQh;7)LNNpN=v-4F=w!3pk8aJidv z&hLEpR^9skcvbJdx{so$qJiCe&o$?oV~#NvVd|=K7^ozuaBy%K3i8sLaBv7YaB%Sc z$j^aSoIi{@z`^;$DM(9de=yq5L`q=kt*@US9P}O<1U}pcqXjvC3}SP0n+`^&nzUJj zlZOaalW!7w-*rCxsdwcm)LPWE4RQe0|7D+0xYZ@p z-mT+dRL9EW6jaNvAr$C?X>|`n6ZG%Xy5!<*=H9+pk^M6Y5}^JX>^1G>3a`sBWx2R3 ziYSQOX52bu7H(wZ6A)6B2GJrZ8$>DE?P)AThdvI{lJ=jsy|9X?> z6bur6zeSl-@}O#8zCHTs3Mk}z1S2sUmP1q9LST^!Z$j%y<|r^NDxHQ3XxANOTP5$bnxV6RU8Kgu%b1_F_1+U(hLT#BdgpglsBYQVP{pdgtgX5J~sAGB~(dX3pp&aOg0TpY&n;8s2tNZ`)@= zJsJk*QwqID$O5le@^(7D5zu>$*@v407FF&U{I z@t43W@SoI_F9T$p%T7RuJq+yzmtpT zy7;xY`v9`EC++>x5K-@knb2dN+%9)OFn2BXcflxI*{^Ocbz!svZm#N4aYK zj%B+~QZGQI=Ktk+({*!Ns8Vf#ueysVDP!xU2`@i~y=J{~$eC;}b3M*gS1446lC z67b4oTy$92_ZNoZ{FA2MFCQ(&>G8VX1))ABbVI7yOz9o|ko{G2feo?zFYpGE^)`Mh zd|Gb%BuvmM?9%70==2dg=>s)IY4aA!s4=30e-D>CC2kJi@>x8cZwF@vf3v6NA~@Un znLB%*j~&vrJgPdjK7oBQ9J|D)p=|SxvJkiDDe)+FyYT46c>o#-KM?`~(Ms#%Qg!*h%;&&Ml}P2BiFbUwci1X(?GA4aG3%h6MiLAXOc*bu+sSz%l`Z^9d2XM?!z3e5{-_kzrbduH<4)*`PfUr2N|^YQt)$; z*$h$XEn6t+fh%u$J=72j%l-NHGtm!9cRh8mfvw}U;X?PF+5na`-*=(REH9m&d|@)t^gUo0wsoB@Q%G{-Gj*Umf(-1U99_-mI`ea!4>0 zzIl1fwa(qC0||1W>e;P0*xu%EKF@fkx3KmpD8(~}bye#3axywbgjdzn_&xrEz}F6z zGZnpNih$-;kq93#5JxeAN5i)oQN^EItzT?NoW+9|6_jvK|4w3`7#euye}KW;PlzRd zg9MImv2gf*w^3RAGVn6{*Wr`DtN-sc`2XrF5$mA9z3ZdFb$s|L7*ejB3{~b<1{z_8 zS12gs!)h2UKj&sayedg^7-w;c_i3L>q~MgcmaK!xprYrE9L@Rb@yGi=a(Mz4@~Ia= zZ76wHCbWsN8zrvHa)nxg%$faht~MoOK^;;Fxp^}q1+?kg?J~orHrZj)iw4lHH&`a$QV&j@yiEUoObiw^< zq9c}H63^JKV7%n%0{^_-7gY@2mQmeME^WNq)C6nezI3W^c0Nt(zb^34+bI+Wlf2z# zktP-*xQ=l2e%-$I&4otn<7mw6k0lvyk7Gz?fkvOP@14|5{R@ZCg|n-6ehQ^Zk;}lo%w?aEdgJJY zh^zHQ-9krEH_LU*=8?u;@h|s}7ZV)xh!={o3iR9_NE@kHo_QgVcw3U48 zO6e6Wd^bI%tIzk=Zzypu!}G^Zt-T~g(@q}KaA5x)n+W~=IOJCP@Q1jspLfgP z%Luat&pnm#SH&qJwj$n5)rzOHgr^LltI5>l{^k67{FxpVg5x6xb>>17=o7^+4x2%E z!!Ac}8&n-TPxt$LX6Xr47_;qa%JMFJJ=bTo)OgK>96Fl!GR(e&)m}=t20>nJv5R_Z zxyICxIgkWNgUefgDDGI0mJwfjjyqM>@1Camqb}l5cvlu^>2-CUo+=fF9M&qWV-{yx z?4F0aM2|c?UaBoCCF|Sep5Ho(5d^Ues7{1gYGo0;b$W0uF6UR>%J50rzkj@g{mQSR zeuWmZ_}=p~sddx$HWzd2psMrtr%el*5rxJgPBDFpSN*z<3cpGd3LmehLgI{t_vtdR zSt26S*=en%DJWcyjWh&C78{pE8rN#p^Uy4;&lC5CWA09yc5ZA7CX_|rz&zwqGwj9{ zxKAurqG$XX&%d3mVroBKCUR1%w4%!OB5P^nY*CRh<#hAxGHGDAZ`>-#oFHa({iMFv zVPGk9E7<;^jC&+yaqG}_xHLp2!J+Z``IzV&^$049Yi=jD`8K!!Mg8OFTw%)XhV=66 zVX?iz!q-Uc3>d?Nm@}fz{1k4ZS+l*4I^)Hi&MG>R59jU0`BkQI>$U9bD%d^jAm$B0 zi*!NUTSHn3>>F^oH<&i5DTpwIpBYH3%fvEV*L`ZQ;?nXaWjywngii;JR%K0(=bwL# zzinLP@rpU>c(7W$p9*&Zo5a1>fN3VdRA|>cjBagO#+{WcZo`sra=%p*Pc$Y=?3H(`-hwT_FuA>uavUz zC%2@ssryH(`)J@%cZk;rF|kHT3%HgX8|^1bJH1UTd{EZfuGh)+3PDJYuwrDCyDACK z2UVU~k=W3k zJ$9kOE{qkI9lc&zNH5J#CEXwC)Zxa#q!sNpNsqatqSyBx%WSB&3y=YfT5854-)o&zDPIyT~InjMg-RfrYv{!^#>G5`S?}Z+f9>j>t6q z`_{3I!+F`PeX6!@^;W8Co_{Rz0C}|Ye!YCqzDY|*NP$uQhJ&8Tb>6;pXq>!pjtSLP z8r#UL>t~!WO=eRlYv3cW$uXm@>B;(Q#%!Q0xz8vFX$0dWxZhJ|w`bprv096j#W`-| zO~gbmh(9)y*I>SIV?R1C*VW(h%7P0*es|b~hvSkj)|XDAk(610OLtVAH}T1uUP57_ z*&>^FmyDK%G0iHk4ELwwvd1s`yrwJ}1Uxa8d zZ88ZB1={*$E7uab#vK=3=T1tX1EVkGk=586+{dvCbnKU5NAAx|Ca=A7V&K#3Fz^sXeJX%`_a#DO)aCouN(|4=saCbXCo8sGIy$d=DwOP&` zG>wDnVIDj}&30MqDK-b)iP`D7e^GtB$mi$Ie+JF?3SW-MWdH03=87>H};?-4VlXnHez!2 zec?w#W|p#W{SoXX4z7oxiRY_)Mh?lM!l2UO zXSi`u33&$5p3EUhM;S#^73Io=v{;iaU^gW|C>zj2(q*%N8bFJ=;%mFr>!nO`gYKsd zFH4~fPB`nsBP3k1=Mm8;nd(vtO;>Q>I0s%Mp>$z&Q?2>JMH3nIVy zisE=;wu;=2qM`5X=mVidNBj#i6+r&ea$1zL(n=6hpX|uX#Y8u}eoj{|U#mlFJ+$dR z%r$}DjE18iUNrOJlB>*TKq{LC*o)+GbIwyn?#6RdiRu)eDSZwyegABQ=mL|4n0c$+ zeTn&gKYxqcPn<9q3Edoti9m(n>{c=ImWIny_v3;0cb+!BI!8MOiRxe{N=Iu`0aXMZ zLLi}~c|OATAAgL4lL4E2Jvkh_&SxEk4?yo4w@P)Js$?p(`@N*>KQ5S;hqVX}>~{R4 zrO@xxWW5e-12n7(F{2o93dN2iO^;>F3kG-fEKvplR6NgH+y?GRIhioHJN&(=0AH)DnDY3d~L3~3rtf+faG;BozytEdIr%? zyx?X7VN0?7FoND6vYdI)X`+J@3GgKpY#BI1Vhe*&#};B6{2|zwqMXl8*G^ zhVNGw&deB%%C%i>BoP}}bDHRTD){JPV}I#2wN%UrzoEvbL?wkHn z?o2t==a2y>x^Wa+MmLn{MIDyY>gk)rpURNmq9y#$I%tb!J~R4Ih_yLd3s9A(#mY@I z0Nf1{F+0kvm2p@ir?J6S&Lz#n@5e=JyYuJ`AcBHJhO9QqtdPZGlyRuoma8sVfL$&A zAnTnyYQArR+ryqK|`z&5}uD#78N@4!z;!R z+uCf8c(X~ z85h7GqJ^Q#m{YVsz7D2ejVOLxD%FF%p_53J>$!tCs^q+fEIF4_?}=dYM~MLWGTt45 zg7}4S5_wASTVKWxFhsdtIJva(TOUJ{4ynXS-js{zAt74;3K%KdyS~=L+zHdaPMV88~mccX8aONw7muD{2 z1I|r5((QO8y!t5WeW40XqGS`#t+u@NLDTG2NJ$?2o50H#pPFuwT4HgcS5)_YXVteD zZiD`5f?or#Byg<2Sk^h#hz&DV2RBZi?Bij4uMKOV2`leEHW_mBCwQg96*^mBCh^i} z+ksn@{g*yor>1U95ukD=1m=9_>OaHb++X=A?R@tYy@bl4qwdCA?7;%I1D3T49GsI8 zVa{0rcizOU2eGUTJ;6#HYg(ou#hyuw?$H+`%cMa;cB_N6gHZch9{p70FiLWgU`Ts* z+^LkQ73?REJ31{_LJyfDi?A$TezEfPZNkdm*-QU6)zV_s-!U=>i^oWeHNyT__&#OT zJ{;hT!4uemVs^ytX{sWG5a|~@8O-loIz{!~N6A;^8V4VKWgHefIc8$@1-nNG$z$X6 zwn>V=q4pNLmTI<4`PIw*6#bTD%z&wO@H7)U`X^uo;^@k}_+pZVDcpRP@DT1&7-~n@ zdRW@$adMQga=8UEtl6vLT~YNckSX>)G+jlL>-TLreN2lZM`H4ji3kQ5Cwo#YO6b8J z)yQvO-^mJAPaXAaG@oE5P%jP8>I+`Ngt|Ao3~M(K+eJuGWxyOfR@1bk3~SHKF|p@= za2O9DGSNJP?~8qgftu(4qo@1%C4`Ona^ryq{T!~`Lop}@{QH3ibs^-cA=Ll+(;dOW z8LK{zJbFBYw?kqTZlO85Tnx7;KR&bWA5aea zFy|d%fde35d0H`D{>xoXg65y`7i1V=)G3_LXF{jA>tE^kq9)?(F)eG1K`Y%6nJ(un zCjQV|)&R4S6ficFc_Lp*Z3r{XX%3g=m#H25R>}KIE%enoF=g#wy3&Mqtmu>_YUQfo z0HA`dU7dL|FFagf%kGcKFXZQCI?8RswxiUEoG{+=jO*_z{!&>mDSCN&`DeTQ^s-jY zeynA_Lwh-Gb8^|s@*|$Rc3t0k3Sb>q&jeYrTt+S)Z!?Dj^KFVa!(CUnbu30_Z>we{!ocnil(4)OM zOrRA~Ez6hQp!f+*Bv3A80{QjxcRDL`azdrWGTrku4lJh5p`wI(nw?4Yd^DdRCQ#4C z4pPaC@PCMu#?Ta#&^oK_Zw6)p<+Dz{EYTJulClUy46k3pFVZB9TfnZ%1?F}Tt@~2f zwuKuPsX&F`lY70D1LwL4l-zC)UAmMO$<6ns8pEWy%mZVpjQxJKT8%A|{jc*whJ;kh z|BF|u_EGd#75ImKx^wu#wMxkD77soHBSq4W-X?BX72}x6ec~b=iow$(9@0*wiEtsl z03HukY|fWj@s+tkQ?{I(Rb9Y=G``7)mj_=Td3D5_a^1&=hrN9KG_@t4Hw|i8G4qc9iT4-{OUkUsQfnRDWHHiRw96IE*C={(c+1>CRQ9YmKN!)D6M3 zgFC#RDacww0`Ze5Lsj_?CN-`cYLj=mnSJ@DM+|m+fZ9n!ShLt6r0Vpy3&k^)Wz}%qL9z1 z9$yp~hNHisy|w|z%oIo0!B#B}?UI&$7JbJ>_=iySPmu-6^c*zlrj^<2@AX3!J$Um(WUv>pBt> zxs+-Ln%Y91%8I%DYgFQ0=qN`y2iPgWVrGep21HDoNhcBHj$>{m zj$>;i(AGW5)ms6f7I_zQP`maBT}iskYG)*cS~L;wp{%^$G6eyWjbIlTk9C;e=pz+r z$L19uvjZX(G#YEFjDsxGic#*Jg2y+47E^ka=z}KgfX$^3ylV>#(UO zg)k=p%a;*$m@6zN6mm;2fjRy>2qK`9cHUJ2=n8caz+wmS3}*O{QE4Cg6VXM8Ja#2wyK-Nl(s=3MV2#gS>8? zsEIqI-z~ZK=*4BnDH5`1ZOhIry8hhJnDM2yKKeUI3PP{e$ukQ|hlQzM=75V{DXowh zmo2)6Z;PCCe5R5}rHn{f?~~~uWtZ52Gx<|%TXqFLx28I3T?zPvlQVc-&^G$1NbD40 z67BgePSFV!PgM;T5M5=~j$?#bkzEn^{>z}T?-gHRUuy*qKV-jcAU;Fji2;AEM%7YF zCABveP)ll_tuqKrgu@U1{1hwYBw54YC|-nwvkchT$J9$J3?hVtWiovk;{G#?g^pGS zt;!6XobFuC^)zaBu|f5wx9^8f^m>LL86sSmS0fE8=E(z!q1~t!ux(blG?xoK z)RFlh)}`N~M!xHteIxQ~>s}9FR3Xu$=HzvL{)FSr&{dE41QsEgJ0MpaoA)Dw>xw@I zR7t(V2#4G7GV+K@tcmw%8ph3G6@#8y{!%umvVQfMM&j{ijekPs$567^6U=eTg8LHy zxqZ#oCmVF-r;f;^^G=A{Tig)PVmFeF+8hdg8yEm|WlclCo+MsQ-FpMa8V7f-FZRkY zq`@S%1~{2@o@B!P^@t@LWYWZQ>}K*`Pz78;{)bf?!59wOql33Ae$4pH`J>snB913( z`OudBHflsB+&hq;ahS%>3DZqK+#FvbQivUF@ctb4jCag-R=lf({5z(@7&fXaQBz6XMV;P#Zq49R(~N87m7d z*tq0)N8U-42_XULqc`RBY(H)eGNZwX%8Lkyg;KI4PR}?v1zAG-NWQe!rc`xOZ6hiZ!FoR)}BO zy2=(s9U>6QRcG91$j&@DiUDs!z(5O|4pm?_Lu3N&!@zhGQ_@2V>SF^p)FNkBIzKS-vSVSiNrfqeD^PyZbG_ZakKF9CQZtS1{L6RuL_O z6dKs0=ln}I;8#`4z?+_6o&yhVyq0d7wcS6WE#vyYJpqhS`RO+j+rnwBTx zE}i$>Ej?q(j09gfPrKz~?j76_Ar8g%D>9b8mk0bK50-)9*MA1T)*sgqUXVv`awY;- zmo0LBhhd=+C(f1f%9u54-7k`)jS{*r9(H37C@S{Yh53?8rX@mzHy9eh7iiV`TAs?LWGIj~ zN%hFGTMD|s*qQjeyMMsTL<3_nHKsq{Ic0}&pQE#R8*$#%B2C1vvYKEc|KI?46BKk9 zRsftinEh^O6ivNj$ms?R)DO9m#Lk)I6w7RblVP0^dB4j^>5GlX*EI$@hzb;eS|e{5uJ;N(YS5}k2! z6E=y%m}v&;6rEJvlRI*VSBD63nL^ku?k!+i8Q=S{+pmtvf>ix(E=dmEtG4CWD)(dG zrgM)p6yU4PHVw3Nh{a!1wrh(@Ex0@KRITt(ur9l zFF`AY;2ztXiRe^-33Qz0i%emQufsw){O@{O)|+)S3WOi!{SYiW2vuhzp-xIgEw|I@5od_vUzkFivG;;a0LN zA6Y@i0_Wh~Qfh!e_} zYOs~V(R=q3InLvV&nr{a+l8-NS*~ank0BI^W8sZ?oj}@-v=_0yXgg8@- zSrpmpewY zq5is0zQWU!@pYH~z4U%ByBWkwT0}4qw;~*T-tN^I_Yb7$k!6nhkHQWJKztx@Ct&*3j5v1SuqyTP#zr}uDh5R4i6tRC$ zdf5vu`7Xu7-2|>lm>8A{+H4kt$MD8p#;%HZ#H2coYr02jBEx+rczl6iF$w zB#~DrY6sJW@T*@PIfPWf@ant?;JCoX_tx0=(5�WS79xjr{r_1B8A& zk4@P(M94lh^Ob4hllJ?u_stg1aaTIKYUj!LGmROP%*Q_wm2wU;cYxa#?l4ylwo^laO_iy!y99ZT2KyX@p$_ZnHk?csg6$}Fzo$Vk|{ z7xP|uIA1Yf`)X_qpr0La-wRc|iYL(~%8fbme|-d{*BwrO!A{FeQbIuD-UmGa%<;#o z{d+*b@5}Z(wrhcMMI^t>Z8Y|DzqmaO{0V00oJ~BVk$yjL`qiuAFX7X5AZl7Q*}$f{ z%#l%%}7)gn>E`BA7I1j`MHLMwG?|Vn*Ouyp8_P11Y=mleOl2 z26W-zBu6C<7)!={`T+tK@$r7*a=bQLoHf&RT%qQ+?qyEGqUUCY==~-OmiaFRi6@An zF{5KA=dYjT`#98_S7koyAVi&YId4Dz{7Ko)clx0;EW%ey6XFoLKbrb*xECY4k{)+X zZ)DVayCLT=ywuYQi|=Uiv*_9KTY9v`TEGsn?s1>6##SJ~c%v1-!H0u?SKSZD&6b&- zKgvw%ZG#`!#BY`K=p#9fqZnYJJkPI;0XcT3(a2MPl&kxwaoJZ;6qvvG(0Lzzm-p7n z3bggI7`M}lnmjI!Kf9O_vtH7E> zurDY*IW5k`#cy-E$h$t99|OLf42PBnk{}GfC&he3e4|e}@L?mwadNQ&aND@9x#`A0 zlQx_tP!1awEq0)2gXwO^O)-y2BGu1nCl5Z>Au`>qe)1}KGJN1Qu+FO#>sj>kkvdXM z2zd+03;wLla;%zW-l19BREiO@8evl9)})9S4}L{InY={KrSH08Eoc7txM{#{z5+eU zK?nX`(yUwr8JZ&dlvELj+b)-HlTqMoe?MH^pX|Jhh*Z*j3)!(VnxXbO%24J0qrIbq zwVZJMcwg*jc?r!9_DNUyH4u!DZ0OoG_&C2t`I8wdxc`87#Cs#nbv2B!!r0jLW^89t zOC^=UV|yuev;cnAdM}DwiNw(Jy09h~32pw#qESl$*1QIr8PU{r>!|#)l81M#4q6`!y=-zJS_mMxgjmjLuc2Mtr^^1?U>e&%Vbv+dGJR z--eY%a(hgMumg$!^O`d_rZy z)isxRUV%{wt7Gf9Jv>XL$$(8bzbi_x!j&(pVv_wyvex0h{vZQn^*D8-2O1KSbd?s? zjTeup@DAcn@lAcLXn@IYzJvJ4-*ELVhYZkm?hTiYge}O07f{mLFSQdZy02}{h4e(ub1AWYxmPn6{Ffnrc$2yfjkgV&p3RpxX2uYf&h6H=bM%)rze0f zbf*^ehu~!buWyM|{0YX;)MeS7{2rStr7xLb?&D#(u24UR;@7n}xZU)f%ORgT9}i3Q zTCSphTFX%;I8=|#>5+FkHvc*0&To==l6mUXDd&*m2y@uQTupTv?nTq8GhOor^)3B& z=n01JJLLJ7Sd=?q7mM`inqAuoSVuqkL5$0-F?2f;>pqn-Wr-S+L=&N?4`6!y-6z84 z&4p7ve)3NeFOoBk1?I$lOcv#)@g&HH@$lex$@Y8-9?QOG!2i5k{u1T5}~FL=!RL#Y2T z0dcxtq_7s(Z`r-FBsO5uPc5NBhuuR2Ay(~xl!*%G8(KM<$rZFQ{a2=G^>HC2tXM*; zq!t&H{b?Flqon5g-B=4CLMzi1NO&fnFm-zyp91(wFffEwV(sW7k_(q!Lj9x(8gG+^ znz|)eF6mao;~Iz>t%Tp7lL$RMk+UYyuCI}Q&FwQc10y|F6XBii?a|84Yt0LB^3QE; zkU7~aRpfr8%}kc|L#Wm`>~fwL^$75LJlXnYYp+$Sxz1E-w2-dj8uyS8h?L3>INW%7+GL*q53yGT0!%~yk%muF&SKS?7R2x1bEx{@in>O z6_G3u@TwI*uN`3!ThO2PN1P`)Ub6RQeX1P*F|)Y0n4BRj<%3#+0PBiE+9Xp8TNb!h+D zg5dy0NF|X@JMp1)3U!<|SWqd5@jIX1&H_9ds{~_v9_q&kcqZ0%mYkZiuB+)%+O^wZ zH*qZ!LKf7&jO>GgdrTAX-9Nvh5Iag@BmfbhjA}&-an!b$+?(f81*1sQX<75>v1e%w zZR69D6cBGjSkBkM_tWMz_4Oa zfvf!il3kSA^W~mCRmJ|nlNTValL~mVrZLC6m{Ei6!BpqdS{@d^EHbMEH5^6W%jxLN zs?re-WTbO+VHfkwyUfd(h)2Ymm4L>x^Cey*pBg54$I{10CoZ zWi|=K^RJ(+D0p{{A{%|2u1+nfu@x#DB<|fM4mK@@->qmHdZ=cmO|z>IcEh|pUU|od zXk6+zLkzdqjM63oA3jASxp$=1U9%I2?*WVxl?I|X1#L=1!LME^>(~%LO-aijO%v1R>yyD=~U&OVIEL9gcE zWfZ9=4=|`(Y($){!H8!m!XP58Mf>aJ;UTVUOnxgi$FU3&qC3d56j9W~!RDbtBXe(p z0pnqH8C=h9F#fL!pxso@w)MIrF)hcx<|D%dLfhd1)nZT;FzK`t$fTYh{pQepV?j%G z{OwG&uRnt_nxPzQm(<9d8I3t|b^6nIFt~KKF=>qAtoYfmh};7fAvLGBis%m>xQceT z!g%SoQ+AJaT_?*)rW({c8nfJrV{+Y{ z?Nnxh$wJZRE^wvw3TN5i4feCa30m~a(@RDQ-@%*4j#old&D35)YpQBwo@?#?JL>Z@mI3Stv!(uX3p@Ty4up5PuZ0Rx<3iO7O$ml6 z`A#v`4@L&8UQQ6qyd2oHAOG6uSyvX&XwA_S0!@nk(n-0>Wf&NW1C2yO26r~(0BG#r zA-q8iWfUdipEWpOus)tK&z)7jmyZ_(j1_+l51gLl)IPHAn(4eaD&dz`_BC_n{1ahX zAtk2z<=Yf&)iwlC*!VcM)M&P+tb;=W<2!*tDM1D0Wqam*C7|LBs+d76$A-8LM(auN z8(???X&=i}FDo=(AhpA3EI4r2g-1TGzPrLdIjAAr39SD*g%=-3gU~;MLx$O!scN_N zqmJxoiKXVzqRCuG&z&}N#`hcfC;qhVZ-rl!;*LuE%~K70xFjLmBhfxWdW(V!GOzZ3>AD*WZjzst#VPAH$Xa`>K&@i`xxsveXeV<6 zhNIzc3vINbvuH^Njcs0yvjeVqZO&lnC2w`rBoozg4@*+&`5G%8;Prt0n(rcGqfyr( zWIKDX8Z7H3xecot-5qI9$TIjs!Q8gk>Bt-t0a>@f*l9Oz7J~agk^eu%<}$5}N&NK8 zOZpI(Yxa98?Q)wJi?am><7Aau<|PY=ub6GFY$3h}ZQWz(r!b5jU&xZp>(VHGdFQWi z=F13w+6!x~h`gIYUmQN?t6{S2f5qS^pkiG&GvHc-8*umhqU^(F>^^Y%%Q0cXZf8KMlrt9;6G?KHC>cWrM2U4bhG(kR5-zxUZE?llOXzG_Vd);4BBKQ zGFkFZy`z^o0i$oFkvfy_7Cnr9^5wG4`FlxiREO*@R=sy2t|><)Rj*YU7I7ICV;wN7 zL}J3{OBcLdDf_Q`mE@D#3M35tLYQM09i2^{iacu(tkU#C#b2{Rpgm;E3qG7Th1LPY zFZ=Q;X?qd_P)O;5REB3)2hh}`#|vsre@&JLz6B(l=c@j}>pVi^{*OD|RXF00E@^nA z^QMx?aQK0TG|X4+bS5@i<@{fbYkOW;o}4D*vQWuUrHM1DInf|LJejH6PYYi{TvqSh z=lM$gVa=l2e3rgDqssf5@VJ|R3%>78;AHJAd!Aaji)NuI-4t&YF%?1#HMRdud_}!a zv?>+)-dK9Jmt8#Nenhb6fuAJRqG}Z3%q*RbmQ|pzJH)z272sjFEXn=A?q&H{XtST= zmcNhQ%M?>)-aGJpdsw?94Otig+)s8`GL)Uk>mD}&M*d6=jbT;mcxg}cgK*XqntEWo zxK`F_)5J8XrQ8cZFzu1*;qUQ)p0&emG|7lNS1oRmnvn}d*OB4Dhtq!mh@ioS>s*Rc zKq-xcVv5RHJ11;8%-i!-AtYFnhPv^EY^B{Sj~}4D*v~q6*6&;T?_w5czH=08wTLDo z&?qAb6(4rQEGHcdSxe)t;s@I%?xTBgsi10v)Y!)ptifL~zHQn`MMvsa)h%68e zL*(o;_FKCEl^%OiR+niGxf#yEXsRFCMqDP`YW~hG&qt2s?t|9v?J!?P!Gpxalvyx! z{b^`+?p?yeP^{IM-$Bo^kZGa|*F4>QYvM$Hv);A|er%^XT2$|k!09>Qo5jmACB(_| zL~sQrDC;V4`}whoUdsf8gbu#{D<^jKcOV{bA*ns2LP^NTW!zA>sBA+Ixov#ET!PMpJJnUY3$B>(lG(y-O+hxO=-O6iFfO8HYE^Pqf@^dJv zZ)IPcW!s1x`;xH-$bXeUUv#J41(O{#9ZGozQ@n&uL@REk`Ak3Gqs@!U6)B5ika*AS z$53DzRTs}=HG^!5CXJo5BElSeQv{rNxd|ga!0ggO;kvUSWi<8NXju%xh2i96aCQJE zW6ZZmZn-x>z!6o(UG8l3MyD}6>)`LbBg^@^>*Ra%ZGw4Ci|c?pFFo;XZG{)S;ud{I zfRr4hqj{^O_#g*aUG^CPK*yDF0}!>kKR%kzGUh1s+S^@w75G_nrlHw_x1P}Z10<$= z31sbu;w4yK@8!rJDpd7xG*g)-Pcf5>eB%%*K0%}A)i>izLC1fZnS<9$UBKCzMC_;-}v4O>sasjUytMPO-|Xh4QwV{#Ik0 z2u>wF00Xk}E`%6f-*G5$BArc8w8gPeaBjy;+PeaPoT)vymkHg&1mb;_q_m5G7iazOLypxUHG;^0GVpY$Urg9b+-%5Sh@AQh*G;<@O@o5~RMe`{(Is z3hWG95O8^ufCU1X9a!^Fj#PRmQ2mW+FNFK4!slOeos0d^)|w_2mrw1!@tvLRHHTEm zr}p%pMhK1(P(Xg5dKM*K-Fn0W%48^lQ07JY_3#HVVc9A_T1W~i3={n>rYGhVl{bvQ z^4qK8zHT9xjQji6pMd5W>Nbp6NSLE6^KSP`)NYlzK9Tz>%G1(G0qKnRcl+JWvJz^=-yK8Ftnqv?j$ieg~!gDwC!#3z{%@iuNNY1~59F7_Yc!C?3tR=_ThRg?cN?HIpAVr#gzv{o zKjsID#nQO}U>`gDptT2I)QSO-JZlB;E+@%zQ~nSL1s&f@kC)^Yy$s|GE%|*J&KeD| zN`W}L0EZvSq_)xoBhHVl>z;(nvgb0t^+3@d3Tw>cupa>xlgsf-u6IT1?N9E(Z2abx z(N`zL3}!P^3z4izCiN8_Az;V*X)ak3LuuM+#fsFI--YMQKC)7XOf3DJs0@(nen(xs zysoF?UsDPv#q79tZWQNHmi**@h-QFMe-NVEx1F^Uu8<@OxCDrgBb|QY33>0P*w0*u zM?W3#8mu-0X)gxKvcw|ec0l_azZc~`x2qQWQdv)`YoS1i0PfvOp)3?qNe16fAXBDl zIYWeo9ThVqIS5#*y{IgznK)xYsO{$cKQtjE{Gj~gj}(ril3VyviN+*A06m+qJYcJX z51-1FpN1-okgXf_X8_e*R9oY9b@@vN8ITRY_Bef_Zq18Y2hN769zgmg-O*F9!AjX& zPqz<(4|l7aa@Tp8Lusy~(w&G!=&X)p)v5j$&I73cI1gK+>xllS^8~7FHOGtK+G1Ti zyt&5V9i8|ZPY8UE4O%H$l+}4j+j1Ai1x^XY)laJu3j(c*p(7+vYmJ>Uf05I4yt=) z&a;ih)*+)JE%E;qupZIlXn*Hf3Z{*)g8o|t zt@D@JVUfK%8~hx_*=B}&yxl70VX;u|e{ksk9w=vH_XRHw@hK28qSx?F1@c@@$p4Mj z4LOVcJ{(-)YNPW1GV%tFD*vkl-o5%t;Hp8URA89=soziV!8kgdNb9d7$kVI_A%CAz z|G(_NzWjYG{r{g|du8#B^j2W_OS&t~_-qr{d0sPx&{}4ZVc>t)w9-#mZk)uT>XM@B z_qP31Pbth?mN#uS+tzat7MHYo(aZRrnEy~~MG7Rj2WBBg<<2}KwYJOD&UQ(=WD3KL zzw+qH|FYZ8NQpY<=FRq1>{k9r18)4{8nv?W?MMoG`{OP_NkF`Wlw)Y+k^YT_P&LxwOA5trBe(UJ+XxgYtN(+zw+@SP{oY4K6qH7g z4gnFQ8$n=D1f)Sg8bm+@B&0h82PCDtk#3}8lt!KTkHKhG}RZTyvF`P4k(wSEMu7Qf@Z#~});CZIDnmk+tM$II- zuCfpmErY^<6-~#fr6Oo&Gw)tRwLYUfu+4pP`x-})VZYK-K)&Nf7aS!ocFyC7BD5|o%KC+5V!~jm~}&O+moOCRxw02f9GA?f##jO zixe-pd_!c9b=EbN!*WlG>K=$TxxW;T@QZj9g zAW>R9)9B9p4lrxT?RJXtoG~zuVyt+#&A5}I%jvDl`(X@2_%&ub>ia9Z&0j?Efy<`u zLgXjZIvCsXMRS7iWZrJy+q*?rPPol}6Q#{LEN6Dl8pm^P>nEtKya5A6y*#^%2(=cl zgG~4p4}rYw>j*tuJI2v)mIAvtmiM2cU?#(#YL2SSFLtM74#ZSP&ybBZMxy-%n80Dx zff*yxyVo#Q#&Ia`j@BBHjQcU39bt87eZdOFI}_QVz*Q>#J{WUwn298PH89!Sk`W>T7?RwQT(szu-L)2I+G#0-XhO-TY>Si1n8eX9Qky z&wWjSr)gZ@PO%StOju$0>`bpA9>+nf@rubFt_iu|HPr$<6W}@;J*8VVJKGcYTVs&O zy$tOK=m_4tAomByEm&X|% z>A1*>(xK_Gf?B}M2V=z@ncN;KTpYkihq4mG6{E3YS@oWNqtDY$bCLl!>+4{XVkuLAkIYfSTt@0Vn+A4FO6gwu8Zm! zpqKGLdyEzs4HJHCUqdEkeVr%`n9Wk~-5{@4f~`cLt|idcZe@nhBe-Tav3F*<1wLiz z=MVc@?f<&vmRYj?lZN!dE28Xq;LHlLjKV8=5J%i1-oP}-!#qbmtD@SyV$B?OwoFCJ zk!FhCTW(U{@7;Je;J`Iv5>0ely`F318j19i_!;a1rY~1btqR7buRJCO72Xn{GQAoV z=mAX05ht4xZ6BR}6L41ft2^n6xB27g9%xszXDlO0(u{(^VQT!g$6aWJ}*0Rg4~`z9)_p@O*ZfYK_nR9HZ#@SBBD->(2!u#HpL>DUTKW z+QQcO?i9L}_JUs|3Qy;i6cXt__nwVcYcZU|ReA1RtX>a>DdZ=c3`Ol9ycWPD-o-nf zS-u>QR`qIye~!i0 z%I7FX6>}T@Dl|Q<*}M7L>ZuQ9vxV+6YEiDYhzjG^GywBksc2Bc1+_8ZA*P}0D6P~h zx#id6dsA6Sim3a`HD=fu6F(2Gsrxg%)rtw_fkL9!Eo;4lhj&7Djb2m7bv}s$zr0PQ zhVT_|*L&_Q7OvL9y~FoJ3r!Z2*O~(DfId@hF8}cP)u=UsV{T!F8}GQxU)*Z!gHi3A z;owY_dIuOXRcPNyD}0t&-biQm8AE$X2K|Lqx;iiZJ@LN4mk%12@rt;)<>sv;rO+tK z<8K33tSDZ`om+Pir3qfQnmsAC`|f{5?_N!|k2MCTc$w|)QL0P>v!94zx6k{VU8RPz z%{+!SADP4@xVBolgB1LM|2=OK%hmJW1yXF}3m<)6zpGJs_ftnMa*QbPZj+`@QYqtS zqb4ef9X-Ef*PZ1$S~}cn?q5K34K$9G)J6lh`ZRAI)!A{fileDgy4+H)aJBRJ9+`!Ar4?A=Sl4(k!erliN4-CA*Yzg)SrZh&Z^_qjgNDrQ&t zm@DV-YnHGRvENnVw_Q7r-EGgKwaL)UYF}=7k#KWYncwzENr?Y+Eo*)b;OT<$~t{M{Jir#SIfK3y2DrsOe`#41{{0&77_e%|ax z*LP$FBZJ&dQmqFIj1PSgad5NN2~d0PJ!Hr} zN3r7jiFsDL;y)trvqHyiL){&f4G0W%Q367Q(bD6BggzZ(x{n7nw7IoDb{19xPbhEa zf8e0Vdn;49SI2w9A_JK5*GS%a0mUv^D!-oa*k!%cLgT};BH ze%=Q;l2#HOpM29<`QwH{CW2OH@f|#1SciqE*5bF?9zpXK zo`>>RD@Cq5`Alg&nm{)<1gM|kSNg@Zdy2tBd=m9n`96>7$EkUtLl}vz*$A!$Yipj| zA&Hlgpx{2UI!O4SuKe8F^L`$fA>?rsP~nwj*JWmD38!oonYd%KP-_pC=;5ii{w{5* zO>oU{t^xYz&-jNh1i@HkeBm^` zN3FPoCkI(?2ZtE6^m))+P^KhO1mu9uu4&{J|k3O!piS*H+T3z@?#W!gjzO$UmGEE@Nn2!(eaB zqm?)Ar}g9IP)20Z@K;xwTLHO5W@vQey2u@!@8}?f{1#oG# zck{L(r1KxM-j?AsZevHZV{FuAQN(uhw$Ac9k!d5GF~OM%&TtO-Tu#;Sq18!I(Xtb< zcr}ge;Z+KDp_aG|^{8kZzJKgg6 zd%U;vV&W%sWK>#&0N@eDT7Bc;p(V9TOHL!_}{ z2ByPRXmzQVLOvp!oJU~b(UMTrZihEPYcev<1s3ju$2Hfy^-0Wv@g(_+3$L{7AUDrp9Y6fKnb>BWBxOZb<)sF;9f5)cVk9XiH5^2PM#&(rR_Bw?? zaS?$wNa|{@WMcP2tjDBZ-22O6V$IRk_K&Y6Z#1}km)xDVs;Obi3xp&#c5r=pSlNv& zJbW=zqL{z)<@3+!V%H3Vec6}dPQGO1>f5KRiN8QwrtYQxHIOX$Ef+N!_M+mf5pk*I zGY^4>w|?ca=vT9QOXpKZk27(X-}6|DCM~f3zn$fB-8oi$CaS7Q>@U}?Odjm0#d@m$ zBPk8$8QR~W4V*8PA&0-uzbw>6tRP8$3`8?*79X(*digL52l-eXs3{0`A;GfQzz`HNeqz+%B7ut^ji1%=w;8rtNNpyeYR0CRs9V11c5bqy@+lVkSz&gZJbS9mo z`9ODZy-doAf%xUx{BEMFw6F(medJlhRtE}!Gk2i~&O5|L86SYbAI}tc-(G#mZXkuz zy>c_Md*$o*RwAXd*eAIRe_ym;_EXSB8>sbW@L~`&7mrX>VpFt>G*yL~-W5FI7Qrc& zi9xQ?BKi~<_869J?Y=eE*%(0Ev=?oERv|c!+|xPAfuJUXRB9g4|5~HYo)gG}TW$q* zc5D%DzQryrS@!XrD=0ceD6X<-5Nags^3wGQ~wl+96*-Kt{DJn82h>g!cxQ#e6hl#r0 zfH<4{Ql9DeGrFJsbq9aG6qhFh?52FYj9vYSR~}?pw@^iyjk(#Vx0ot@ZFLTF>lx%? zs>DFaz+*YvevOe-mQ?q}_C77dgCUT2?6-c<57ET|4Y5N<`i-ph?L#5}b)2mPayKEa z3b;JC0b&)C`-dt2Doy4m_S>Ilq}CRW3}QYcLeAz#nS*mk)7q7qPSu1*)l{=Ibe(k< z1^;{{O#>fHbc|Y~0%&HFAZXva8l>AFG z+2_{_*dkM$dz)~raJo48tWucY5dLq4OW=HZV4BU_&!u-l>*dT;FScGPI@Ct0cjp`e z@eswL1crkw(a5ovDKCosxjl}C?+t5`D^!!;U%4#*-`UzmVeMnri4~_}1Fwa?)ADzF z-NS;f#!)BdI+LY!drGNI5w$?pD^1m4QMWjceYwZ1#2+?B68>H~HpxlNcPd#hCHlu@ z*$MjJx@tm`>bjt_=2*WS?YZA&P_dDzB*a-Su)SYchBQU8WJ|Yyw|sLE-bn^ zp=yVm4MP#Ur{%je7eLv9MVbz`42+S;?hX*N7}{`L2X>z|Jn|Td?62j+URe-WP&fib zZ=cno4^baYb2gszOVgtsG5$I`W_l#@_&BT_#aWMR0=t9V7dSnRBboJiXAgG~YaDLW zaif@mvq#2@pxjG4`k~T=kwvkZ$bRdiX@?GlaxQ$Bhl=m1eeSV;O3SwQmrX7B& zY04e~SddPu`g2_t(Smz99#KlKUWJifqbYIgb+dXc18QZ;t6CRWP?|JgBPQjOM5sDL-%*`@U;5EE%gQ4BQi*y|5w%g=x&}*$5x-4QF+vI;gmQ zd#{nh+skL?dT1gjHHQ(NRr$j$Wd`^)9;C07qwvMnTkW@9R!UY7kEByRa?}syuh|bn zKhC126IG@~^gVmd*N9aWquW47fr&%NcJ`>HYJ9s}(?lwG+VHuKvFp4?uMp)#poewZZux^(?cgO)EP$+F|}#Gl#$K$#VYv%5rGwbyRF zy!XpB_@2&QnFdHcC*HAcKZQv@y4>Ht=$igI-z!ZpJL_k zMGvD+4jPppF^Ugo_9+IBB{kr+C4~IbM)4>_?9p)TauUDjmLEC;1mp>~t%5DO`=>*T z@8G#g?1{|RDVOR1uMX{sKHtpEyv)B(zDK%DG`p#vt z)GpIzDi3O)U!}7x3hPx)26Ma$U_^!sE&sVqAluDJ0=-FCJP$RD?S=d2>!ZtPDRRP0#A(c>$BwP{EkJ}JC=il+C;1|{rVeNiy?F0P)KL*Od83()S|;e{AxOZtEf z-%T%RBzo^_?SZO}Z?m6n4$2PVsFiPzh3r)@Pv42)vqz5H&oCtpF9rHjjWy4Q5}P8< zp}P9(S7|YLPQEs%)L(R~mY6^Xl3>M)b_PFo=5@HmG+eFU0L(MISoepTNP~ci)p;5y zkjV4!+%&_+_sK5}t(p%~(odarKge6-4bxShxX(;~2?O{f)wGt&^w3VG>tR7>fi*+x z7CD$yZ*>>NI{(-LFzIA=4>-Nl6_uQ%#6J9{)suQ2I^RlrNK0twr5Z}aDkamqN)g`F zdG4)mKa(6@zv7}XoAmyZ{)4&Apkc;XExJSr*FHUikY>CeHt!ppJ4e#y`1F6ksLKF# zDcte|M|f9GuD^xkHT)L)DhiFlKb5BN3jfB~_$Zju{`vL&HyqReoNOq#Nj~M&qtYdDc)oAyNN=a?ob4WIVJ^&(&KA{k zkG?Z4-R<=UJUbpos<7mKF#WBn=qsY|tcqbR(^Os^=jC$|@)R%kGr{$7d%jiosF?&y zwn)Gn_&fgr>|KHro=bmFK8hf}EZdonSVzdWz;YfJ$ zS>zFV4n+s+SDgkf0pU8qAaCGP&u;@KZ*x*%AE7qMAp%Mz^{0d22lh`Vu*YE7FDLb@ zZ}}^(?(_D>Sw1yzWa-zugg)jU0U!8R@cq9)pML}2{|gZJH}HLFb%B3DU~2vWDRe)< z;3KJj$R!jA)nk5w;LAO?{sqbXKiZC_{EvH#YD7#7_za~7Fi*#u~el@iX>sjpV+j5*?owAuet&}(bCcJPx4`Uk~Ye%0l;QYJhyDBgqF=86qixk&hjx8`{k61Q)mh( z9L=5jWaa%p91LL^FyzyinA!8uFi_WQo~mK_W?(^`^n5A*;E ztLfkzW_(nQdmw6n>gU9UomN_V#Izl3&)hLv|KSokfq8AjLCj|G5ao6DRpt>j80 zFiTzS?)+Qx&@hcB*H(2Ay{Y|zR_WjcYQ#0>i)mCNEsCxVzcnMsYt~HO+H+@I9intW zfehi3QT>JSfV&1%%F6pQ4QGQ&6%BYZ^4@T2qLO67Ti2M8A}#X3q_tELG+E@_g-eX? z>!mw8jYY*~IXHN+m#FT)TEu01rdur zn24B^xu5j;f`Y{gU!FJlHv(cNIlo;1#og8W2;%ZvU{Nphw+zdf=W`0n)neAXJMg^yA%&nJ~z$ z#*U{EM`pd*p@gCgVZaqFmk&?xu(Lro?eF$`q?$HeFaO+}9O`qE`r0Z+qGfV4I@}#h zZkkVvX@>mrHuVBYfM&jA6NZjgQ16E3x;w~Zg6w^0b8<_fZn*<-D7wGOmP+`*u3yx; z(VaeqnIVb^o@y)adVEAC19$rqYTT68-W6>&%TDHKoFBvF0STL5B8-@R`zxf{_8<1w zs+%H_bmBtPL2w^hrWY!L6THVU{Xo*OI!S;Q=&WZL%})UbS|Gtl7upmh&mFI0?t{>Q zmvZch(S)ptiwD$UO`2hH@n!DWY)KxIA%kwgcP}yMe-GzRTR=-{syPLPwN~h{+E$kU)sg;34Kr=V+hJvFoX8|Sk( zb3rKmkKn~+%#{Anv#dS7M}$}dnTkS!L{BtrYnJ6Ln(1TYk2mX}k0u`)@ju$+G6*l7 zHu^L=man7LUk_iFPkPaY*x`A$=%M=)kv*~D?%hI=bAJ&fNaKUwsOIIU69{TKc>ZuZ zg?g~-V0mgQBAKBdwaEBxn8NwmYD7<=GE{DvvbgFHV-l4mirg$RlY(FK^4q=}shDt? zXvMoRi$ftNRczKN?&pb<2k1~p^ao;A(;Ldo5l&jjLXoNE1{#lUm-su}DpK)}C!ATg zpGNQ<{0!ZLfcCE=63&3}j4No5toK&4*eja5CsC)=j`ogG#mMkb1ChPLO)uca(l9*2 z`u^sZA;Zri@F(6hZ_mi%K@R=b$EQ`Tk+X)J7+!o7qvmvqfP#-?RLG}%ufS8jax^v*5rdK8`2#Y<7+ zr^y(W{xe5i(*iKZmEIE>hB^OQ!nD}g{pb_^=LMtVlU%N0A_-TPyns(sQ28k_i^uAF zF0J+=E5~4Wbv9oBg%AB~?#dcXWdb41^0RNpa|3Ec6pj_`1GOzDB02s#f%y&OdWmyaNhUC zyEIbABNc_UL_^3@!14;UD_wh9+S;Ds5Ka8cgsYH&be0xPmS&RQw4VWf=C%l&8cOwf zdHS)e$1N;n;YywDQXL7Ab^5~Mv#=|b{Tw1I@Wk>^eBd_%-(F6E-ItpVd#yhHJJ>Vv z|86#5wn%tq!@At$)gx}8Df^vaIg(WOL6C2%7As-ej~u6|(7fGsJvFvN*_%FqAXrlb zX#e70E>GOH4Dd&#U{dJ=vv^f{@hw)jPpKocj_zF!h2De5WR$Up_uza@b(|YrE3AOV z)pd@JE*J{JD^cBPzmVTw}nV8j_WW^cob>^6L z|xqR+#whu}p(SAvS`d0k_}>DLzEd8$vKoko*%&Q0Xm z!*{NB84Yz`W2Po}!tBy5*sNUBh6wt&?Y4Zd*>s`hf^(hO&b~`k_$J_O-G>HmF3dp# za8L}ul-)$yVapgzSL7yY>A608qVy)BHu2&8kAK%%6r>SxB{qwB+{W^54&w?VQ&&Vx zsXrs9&SLB_>)f7;pdIlQmPSwvAnXq4sJV|D&Pzyu*#N;y@xvxA1xQJ)uvN>~M z5+!k>iYJs-O_s-OJMVUi&?Na1_#DZmUyGaB580s&1{M?(2Ai{^ccbFiT-S%p$S*<~ z8;b~*2;Z8ML`%o4bEh4NAYHkr)||K8Cd#5|EPC4|I4|hVpa#oOo|$*h$Zl_Xd3K3C zvZ&YMYiKLs`fK($kao>6p0v1F21Suf7Ismmv%~#znfr%%IBJo(BIUp;m!#M)lsbZ3 z63M(SRvv_P;B>8xdWp~Vz;DnXqKGJG0KcIz395Gvp*OE;x zVPdL(GRc>!2qP zupl3BUGdJ2C8X@W1xYnkdPbY;=AIp!AWUxkG2cYorJY$~yDU^$Ae&+Obi`U+>s~ao zn?qcJ)Kd5-R~+UJYQ!T><|FfGsI}41dv=L>hj^sadw5Q+37L;JyS&GEG5H#UCzNN8 z&)sPc_+mm31cG$!l*#7g$|;nI#SA03O{`<#&jZSX9^9rFwIP*}rmjn*<+o)-9nf|Z zIA{smqo~i#m}t^sGT8<7rVN~rUFI%1LBe-g)$v#?MNd}yPS2z^LX>;lZXvs_lJToB z^{~CYYGBr?Z7zf#mh&j1tXY$Xk7S20hiZRk8+3$lXm=`?6$jR1D=$k(N}fxfNwJ-=8iX&a@lX}# z0J|3y3)(T2&Idk-QrnK@IZiEAu=M(H61AgwEz_tuTg%pn0mKg>*ii}dGS{PWJ~Ie3&MrTC{M2Zc&x8e!Y;~8a;m42y2fbs_7Kx|Ipns;(59I zdnzt*-#F*#(bbLBTJ)|dIG=s9Ia;nSnIRYB#ka378dIH;F%*y0b5#K?*LXw&_a@AF z(<)WWEY|9+4RE~rycGpTx`zJptkJtH_&=8(7;Ih>ly-hN`>mwxn{707T7=KI^95?s z4L{FvxY>JXc(QUiP*%bpj~iW^GP;T;9xeGzZsez*GE>Nab5$Dr(#4+(zfC@B<5YLC zf-pS`=9OQG)p?}?>F7_y7(DH2dT`*^D$YZdDiCX$?EX~KN@7=gQ&20>UQ-KlRc4cmHFXVn77J4^&yGwxm$(E!R#(o=q6PC+Lqd+8P zd_@XRw}S{-vcp;QhBiB(kiap8u^Eczq#8<9CSiW=(Rc+XK}S9E zrNGAbDF#euPYTVEH%;{k^lQHO!WMY+yY3rE%O<%HG@j|JR!xlmG{NM;#A4x@5|Ft% z3gYFVSR^s7%X4r4qbH(k!JN8Wkj_lS-}qDaqfbTBN@C8nuouarBk+_bp5<4h6h^vu z4^ZV>U@@hwxnKYKSsdmD(Rwd zzk{=buVj9aU0wHV?>WPKw}oSEzyiC8F9v>b;Psu5l@RGOXo1$k#&uMsf}&=lD0jSg3RkH2;>ak3TqU?U_bLP8$nz{Vj=Ws%ne0Jk4@9XeYk}2ajY_@&nVsC7g zo%0H2hV?w`NA|i4iS>+o$m%(CNTjDLKoE7+zxR0ygP1oZlsDUVob#o~BpA|~JT6pj z9U%B>zu5prI5vzPNVqib8{}KNk_;kK8{3Nq0>tKq#STqLw@Mh-!-iY-f%g+(u&rCa z<5C71Bbd9t)h9?M)TnB%hDO*v(1wc0biTWdtUcm@Oo`)8&5b1loVDor!HI(YAO#4K zFwcu*jkIKHgW!ZVLp{UfW4$&ig6heRHJ*m#KrNmFK#gVD{imrk?9w$!f82kPpVU7lT z;p9+#`B~FG6xo1p7`duehg+tRcJ}SUy5-z_+luqzQ<{wK$JR_cY3)f+;VCN56eQ`5 zQz@(Q-0oB7XMlR#!iWOAv%7U&Uw^b2PAhIE^{}$wgwxu&WbmYaVMk~1H=E&I&^;Oi z#fH4K-(J?Nf@t4o&##gssdrE7^;mWmWIg~OZO}Wr?K)j!R9M5A{sg$jkjsc-9b+nM zpoXeRi1JvgEj$ z{Gwz1mMV9h3YnTK{5W;X2zp-_l`zFRjKPeJ>anG@-;X!tTIdBuAQZ2 zy1yI^rQ#%0CSM8Pup%Dg)rboE`mh5sk1w*`n*Ui+LkLgmzaISog>qF;D)$Gy4Qt)X zzKITfxD#kkdx_VsOA&^z4GMsakhb zV6$MK80#hBGJ5PGvmR`C_&S6+9!#I z$_K@}j7=6;DbG*T&owczqzYLbg3db9HKL4UJxx{Gnqqt|0g6C81>rBm^gQs)7%ey< zH_{}=)kqw;qUGLcSJJ@fo^9$!sjet`1n(B`r#c-jm8TeWT?T-AGQ3AlVlxwskrSS+ z5d|fK2`wiuiAY32D{+_w`{lKG1kaWbeG4f!8?8k7b7$-*7XoH>Z3Cq zL$5ZEp!;(e75l{g!~OO2`;MGt`*3xbiJkZq(aJ2Dvg?Bnbb{fAk5k{i*&MUSl&Wc&nsMuM1>zciQ z^R|muSWmur+eDIYn4C|%7%pmY*Nc~CD6juspzLD2mi<=AAG>&Ew#$o;LXhB7pNV`& zh^(#+S4ZUWCy4KY#pG>`Jxr>-$b5VG`U%$@QFR(gLcqEGx8wSEB=%3P?BB8AKS}Yw zgXZ@pU%b&^wZ{H&@{L1ZeA4j_Iv0yAmd=7tRQ??suXGxlKN+F>0seIE!y~VAnP>bT zu&*T#KmGd0AKJ6Zl7{-{ZM4--S)GSVw($Y`6@UGI_gA z@-HW45`*{fVx-i1sp9Olu_g_SXjW(Mn>=#!3w@9W%lrr%0H%^%=0@puM#uFb=0eat zW8_wtyfUyf=-IS+j_3$HBk=bR_lB-j>AB~bw6saOQkSS-rt#_tQfg2;-=Mqy2LO<4 z8CrYx{j0=TiB(N~xd_xd)NNYe6KnAk5py(c{fCwD%p`y5RCxR+qo}mAm}Y z`Sk$!QGvSfS{UG~h``trxr#Q8V|AkxmCeIKw#QmXg>?#ICGrFFPp}&kj8x>EHO) zMSaUf1GemsL%we9vOC#b_ZiJ&e-!^rmSGIe+b0r~>U8|hbFM80r=-?o2G4Dqbk^#i z+06?#)x~1yC`O07p*$vtY&M{+K-gqf@-lz^Z>DS#lYy3eNs$n~6l3lg#)tynkg#2S zo8#vYC*Rv4P`^nn216`v%&CX0=sOg^>`=7tHN@I$Gsn=if1HFb4pFK)jaDEAF`@>( zt{1J(W+u*^`f$6}YXZ-*0-^8Or$XuHuwZaH0OgsYx;P&8TniIjBK?}yJ|3x!|GIJ_ zhhFi3et}ZO1;Fdd7cl$8u_@=CZsE|uN^Nz!J!nw+7 z-O}l!UP{B{1MlaTnd^T?P=nrZP-7AfZ1CwPU{;PQEuV8KWU<}tY^hp=z3zz; zv1o%)GIZZ>M(ARx+m<(N^kr(dIDaQduC6JIzx(IzqENKX+ehiu5lDA4^ln_XGp|b^ zG4DRFMS9}ni=jS??YVd0^q$6YK6vs2%o^jf_GSlJDF$7WVD0v?6u_CtQVuY|j_=Bs z7hEPRWkeGe3R3lDfc0e#Ed*mzcQ;M9^ebCw(aLBmC+sNBoAp22JeyBTu z1Hl^L*Fdst9=$L^`Sb|F=%G3!%}Bi+p|1jvGD8@|dNzGqa~W_T%4py)liqQs9cDF7 zGE#cbu{*?CNX#KHbkKwIy0UwZr#ly1|N3Nc3-j#wl=HyrbU4ynFJ{(t%@MUeESSF} zdHXk&((2iC5vY1*7pA`je0}#?WmC)_pihQE8K<2-h;vCp*C8Yjt#Nbt^c?8Y&@*`u zC*ioI7p5*|U4yJ@vN#WEp&DdR60Rt3&_}O>g@uN33yhYa<~i@pWmt6NdnCm0rI7`z zk5j3C;BEKKBR+%TK7CuCNpHJtdTDZT=)ML@L|>z~TzS$+VEStYOArHu z#`Z9q40@md%qo_vfo5H%7Bh-4WCNH%TaMcyAez2Mmg5vvoBlRe%*qtdgr#FL6M#kG zv#ejL^BhR?xhRapSJ^24e5#mu3@{2MF{J9hf)Zl?2$}-7C^_%|ZSkp|hFcVVH;s3Z zLNzjyVv%CoPKku4vU#fzIh6NC@TIM`W2Rs=-gC#qh;n&p2Yu)jN7<<$4Q)7xr{%WQ zgM7`_@!wJcpyELNyZfIAGpqNj{0(kw0A+=TkT$$}e)|=7CLji&kKW1h`jizW=NbuZ zTtkx-8T3xKZ(=xfH)Q+3DzU8_d`Z=PwJ8ssu%&D6V=-6;dj(YG5QC z=?@;OK=LQv5>PSLdDtBIPCyl8NfM1l(LB zfs>282w@FP7e87cL%ZWDMds*rb`aJag16@Kvi^K`V1V)j?hiYu;H(wndo^Ye5pYS0 zb%JE;`uQJVMp3qsy^QXOua`@TM|GLHf?S@z4zTO${D@KUHRS#e4%EC?EBdo^k~7%A z^V4Xva4(kAQ#(XpZ)N~J#|)jt4RTljkO0XilQ#{}n+tI!-Raku7UpnokFrgFloY?w z)n}^fH!v(}-lE`37h+cHoQ@ftIKY+D%bM&7CU4neN8C>c2ntY52Uw|a`)w`h<2gj% z66E;1AqZg4KvtVbbISXG+xW#rCFZo?O?0IqhkXlT&i9XLPufVSi~khOFBC)ZWR|h$ zC!yAwr*G`e@IN@ZER$IM5n89#lsQzNRLPPc%sK0uR5xJDzvxMUjDs=#HtV`UZF85D z-xm7se45JsP~`aghYw($$k1JvOm7UBZ=CcysZ>UByugpQ5}Bt+--EJo}8 zP<+0>Fn@nimOlwwZ;z@2InK(`)Z=#1M_$ejg{iI(0}(B8&i| z{AF<)gGdmFB@Vj50n#ceVEkWULcMF}3qhvm^1O<)tP;`7Nk$~763)~k+KY?X{3W36 z_{CF;ORGOu#b>c(KVM0_dGYbwaToY5^Ct+Ry<+s_59a9KL2*zTWr;BRe!jWz0ZXUlc*>`tC-G=SB&TveNM{j4v1*vB2k7)o3k8LvQqZr2>faGM>7$L}#&Vc( z3;(F?=XqPXV-*ICXFZv{KrsEqL8SDP*(t~2eU(5h)w%xWBmQB%VHYeZjBK$78oxPn zMN)|kbWFVcYF#_9wm*m1%0zx*WENN2reL0yX}pZ3&b4SSFVWo3E-wwr8%Nv`z3Ekb zszM5g#AM{G`$#MhlDI)WupFpwpeu@V_UA|mS)>dhY>&qUyS0#qyWO_B@K3!WN zpX_@|?e`B`#3&9p0BsFnDKLLD(iP@|zeo}-8p^FO5P%`eIXgZsaRuHZ@XPv28z z_~mYYf)*NpYv+k%0%v zfnTNPrZD`K+F`~aacfzn+H~GlR%`hifPeq>r8QZyFEx}p@c);6rT>!x__ob*31dD? zc}V;ZhiF(jK`FKJqQlNV z-GZDRN>=Kx!?8W*F>%Pgm(&`|OOhsed^YeHH36WM!W_)Sq)>Kv?TmS@G(MR^SeSK4bhyBXHDv{~Z6=VIUTuCKCpaW$Ps=*G!u4436 z=40#JX(<+{cgFSQ)SexbujL3_k@S~fTn~Vv>M8JTg>Nw1K5}!tpQitxdZkCqUVHrR z!q3Q)-~zG|W7qs~r^9--q)Ltz>dhkcEGE|i~pnSmYCze(=>rk!N;{Pi-;>^z?)^9zUw%y08yS={s2To zoE9#?GgAV|N94gJ!gkv(kM}zB=@}8;-^x+)L#9pN<{YORD%0oSEdMR6As|D+ZA|N7 zuXyb&KSs1rZWQz5FfN~!X`S|WO9o!S72M?Wcc*mua;_7Mzq`^umi4!Y^8dw4RtcI6 zo@s=>!qNE*E_637DO8BdZg9`loTXf5F;lPQ^39X|){sRT_1RLU;(5t7@2(EXM{GG9 z`^i(=XFNJG1wHSjQeUX#gcal>n@Nfc33CvMkn*ZT{+TSH?D<+ltTJ7mv@CwvZ;e^h za@wV?4yoQX_M-!`#R7IU+Y8|c4GkI9=VK%{{_ZfTo2~WWQ{^M0jgxm*)N>FLnRPJm$5Hhzs4Jf+;AK9S9ePS3bk{Ed^$MYR^Xj%%z{LrR<8Q-c?`a;j`2aqyro-?J7500=ylG$zbm2!FNjNXW4M=)ooVvK3w9F?LdD4;|UgTsI$R>zYCe60u z>%DwB+~H#2Gf3k(UTiCgJ0zOc#-GNh1`Yp9r_BJ}zOgv@`nMTICZXaHC)uJ6mbI*i z(EDZ|M&LJa2^u*-Z(WadC!lT~0)?IAxyI2=K5ug=NHyUZ z&h>Lcl0GiDfRhac<{NBw}Z;gB2wL`z*#-g9hxEwiQ`M-Q${hb-HXKu z^3op?Ep~IJB04gW!H=@fmU9(uugyxAr&N|_Y0);)vu!rjr$7$mns67%f#nicbP%=V3r7)Fxv?D}ZDThL4hc~eUhyp?@vG~+X}mSY z1`zcjsBSq(%x(r8Q{H4S6vUXk6ah|5c|jrc0yeW0!AG{=A$QUlkG54XVk8cFE7K7F z@^NF9=gwTj!ojQ9R%8j|c}9b%jY@Nw%y~ME3ToSN27?n|R(t!nKNjbhq1PqJfkRVm zyCj)92ofqA#IjeY+V;K*1=PtVD^M!;u2^#9EyNiI-|1j&7r^&K9W0@0#bZu410$^1 zQBdoKHb#>+W!P$jN136ED8#qiCY)w+nsFun1_PTd$`&uum|VHvHwl~OZI_E_gIwsK zbISD0B!N2+0R93p#r?l@C4&PZwYB%IJ%Gr0o~|uHzkCQww^a-#R4j$X@FmLylQ4Dm zJ=-9s&lva7C;38LcHUaf#lJ}82BNbR=YGbg@1He>&&jy}rgN^#vY3~Rjq;W9p6v$O z@1|#MWF!dOq>pT5#P+WNUzaYQxN~GI>0Ysc^LXVw1~uAQEk+lkt>Pf@ZRZ@f5q_ z%4UToX9Y1Xa|o3kY^af!FAZ`Z1*w+{M#nW+9EwO`M?v!BC=e5AhEI2#0EK5E8-NIT zh^Rv7$&_OZ3j>%suCTQ1;UaCo3dvBQjLg($gl zC}VeW)7p+^?56agf)YlQ?H;JZ#?OxOG}cZvfejcd8?X}EiOunW-v%cUjXnKuQ7uj4 zdu!i^gS>7R=KRSH!Q~AK;0WK7pD`u~nV@WXteet%Hgmw`bzF1TE zL>MH+0m#{eOcTe7!c~l5IouH5AcZ+BU4jlFWuWL|`UpaD&D;qC)ojz9q~NT!MtE-J zdp*SP)+bu+@*e7;KjM6q!c(j|3l@3(WOxerX2K)~g|h2Q?x{7)_L|o3*&%4B(f5h6 zTKEWPA3zyjG4?t330s}UmON=3dcde6k1+tkAnu7Jri&`h3y7d*dqlXOud z3l}sy7bs%m9srAjQ?7Yz$=>vRN)wgIgx*bj)ss~;TkpQdW<?JHvKt%Y#GzJ}@)2d~9T z*w;egsBjK|q%3*1LvS{m2i4`WeMG;3nbB{jB7Oe7`AKok?Bg*4kr#DO>z^U*_|o?8 zl_v$I@R4!48lm};aB4CF{JIXek;bR7wuOW5BRpLQ4)UAcZ0N3^fV?%Btv2pQOH)mBzD~?O~)sTS}|*lUSmdVs164UO{a| zW5#YeZpGfBTDPipLPX1imNtV_(y2)pTdQqqsiC&CYPf73vHod5Pb&p9Uqo&2=2sjMj_J`{ZX^P;NIZtkXJ2&n*e^*B$jS{^K$ zwsw<=hkK93ZUoLi-Km^TJU`$f$Wi-i11US~MaaJPN(PAj18eT>;MsqXcW)A|Bhz6A z(OAgKnY!mTL}$B9JM6Q{VI%lg4wnkwtZzImdG4}$2bgO2={!pWq?u0_iTdVGtQf?P zg9!e<`H3pg+5tpQOeH%ZUHjF5YU8`to5AX{ z=S`i%^_7^HSW%tvj5O8BpRY$dqf64xMrRw~au)gHOF=`J=U+ejw0niZxycf(*&g9v0_ zqpe*Y8MFrywDS5(SO#y(5Za!D46Jt|D`#FTb`SG5S0d?t?mR zTZcm3;SHuZ!=GNzyMC6GzGZ!omKR>1kTok!UnMWJtYW}KwKLXH0(b(_wmvgh)vN!o zwz`}sXzX5HcaL!7FpUy)l&nkgUP26Q#bi#x#lfdc?~0XA?FVRk{dAq7zE!ysz1k&F zT{$b8ZV#fLGTw}L<{Mveof>Okz7{938+%IG--4hNX=e@bF~s-Qguibfu8 z1YU`gYv26YaI^#jT7FUGxWr(#Vhgd6G>_G$$3cMCWSVSzt>;A42M>i8EGlFzAczDYT?~#*ZjrPcV%2|%W@%?Pw8#E4025C zOexwQR4?=+j)ivv!9~focbW({iiWg}`mXscl{W;7tT?$vZgfXG=wv)A$fh%^{f7yA z)NcqLi|Xa?W}t`!Y1;(dWE7qavSvc0>nx0cF!Bwbw5 z8|jfSxEbu;>ceu*w;Jd07U1t-U6~E}9JaXZIRI8-Ce}Fp!iE&aA1sUPjBKk)%V8TV z+-cVW+()fr!;{@x+2Hgx$wDDVxeU*ptbb7=1zxrrvh02t@KS@3+FA-X1JCcqv>)Eo z^?xtbv5#-T$-S>HUWz>=FL*E$$Zn2fv>YJ60dM_vm`NGQxtUE#FaEccc-G{Q?E@=# z*H{3vl5&kI(2I{5R!DEJTzq{XNZBPMJ-IRc9R6%NIFrqgm9*T)JN|B}+O3*E1mp9L zflXTD%aaUKi%iY9c})$zxnxaC6+0C@mAxuOvb1nrB2SSa8w8*K!m$SD)Djw9zM;Vm zHrvEkGqP-pe=K7O*>(Dd@`2bypu8$S4w0qX;l8$ECm+Y1s*|e9?$Fjg<;M=+ zlMz;b;u|sEPMgU!61rg%+#fw~i1!s4i&jn#Jz%2*Q)wA?{-O_EHDT%v^A*gkO0HR#fV#MUHCRhWo_;0t6#QC354$&{xcyB6eD{u_AWu0w9H6?oh8P z_a>Ll!kW!VWSVob{Uf`YZi`|XKpPIM2;nI;+oFxyCxVKub}n@eAO<`~+4l?Dx7&29 zOklpJ0x%y)lqhC_U0T{#`mEbi?er1Xo;zz_auw`=|wL3^Oh ztAt6`62DW%K!FmFG=r=SKf2bQ+Oi?=OJRROTv3gvL=INcpnZY`6{ZeO4yv}k1>9Gn z4*wD?uL42WvVTbw3!8R849+g~>WO_R!o=B!4Qc4#oSB<|j@$guY`y9(O+byn>`{~_EWbbV{3xZ`hK zVj2f4_(rB8yoNvq(QE9!deL)+;*a{n-gCkeg4L9`;K5^4CNFqQZ-skeE5}GwOmes* zSH}0Bj4~SmRKXwmosK@t*Ur`j^PVUv$Wk=w-y~ZHK9dmLZG_8F03tix(6<-VAL9Kb z_u4F2#W}PoZ&dEWR_x+ZTHHQMmeNGMZt@IQYPFY*jd|@F_6Og)vz{BFm=7~sljm=% zQY}0m*>$f?Bg8uJFm?0Qa+wU?rL?%=l$g?vMXVK8M?cRp+M_=;3e>N)x91G>212|A z-!M>PZGQ*u{z=~p7G9>U3FIxiZhr!gS!X%G7(VORfaXG$CfW!k>e0nNhy|`ZX>)Pn zw#0=FMrF^5Pb)mGU)psWY_vh(&S<`kg!W2>Krb3~K#ts@(mpasZc~GI+osYc4qF!8 z%>4I3MhQo=*$w}E{LUH^THt*q_!C*@j#Ies!-8e&^*$jNb}aE%LBtN&_!(R9aR+>M z7-(Lf9+p6=;C~1ti%TE(Bs(%eFjaYq4V4k|o=+rY*~?K%1063bfD? z3Z#@-%D$vDrOXxxp7)&J`JJQX z)iS%ymV0bDGc&VIo}=nBGn?=;GqVnxI|tl3@)zq{;A>V;XUUlxAKUBgnHjtg(sF1| z3~Qjz9FWR=_3HoxGMykiAVnR3AoM_QhrTleKf!(5=~#xPcfNXtKnQvmlsXI{8Zdr9 zDvQG414$oFps@N?`BqmS{<9(Ua0oQ8pwwzx!_W!L15zZo&JRO<0KUO(@L3VShYbE< z=tLMj5kDCG%)0JCmvpUYg$E!En!*oHVc_0>F))OHTPOsfhhxcu|D4}{78Y7Y%eVeZZ^_j4u+{x)+9zse zL#^bIoYL@Ad75UN^60DSj&(nA6soOkmRllmM8e?HkcA} zK#B%?(6L%wzh#5R)<2hd(j|(yy4o{!IfG-`AndWNuNoM(z?{D7?5p~?8fqOZk1-{% zoJ>y|mvy}xj0CSL51ov!mrHu4r9!Aa$}y~x_h~aV_T5H0*R2%`R&Jt#H6~$RG4l<= zn1&`kA$`HtiBYNERYJnd;SM~K(qXIbr3yhP$dWG@X=cER1IVq@YCW_jQjvwz9TYVU zwlk$%n6FsFoLQvhN}&Yxm4;qVLqgkOOSCra@U+bag~YKSpnYT-J3(~{K~oQ}3{jIY?4Bt$=_thXE>E@U(a3KHywrn9sn{5E zIlGmMAZ)^Vy5GqbG0Otipf%5H<9^Y_a=t>Q(|$2sZiMYfDpu@j0CgiYTW-`kBm+-5 z9n&nbYk)d#dT5G00U>3MttoxDBIO}3;(E1gI8J9l3C8!sy5d!1s|kuD7%4}1%Z&JR zZW5#-Bg*u_ZB!nSB0lklY+CQ3{koRr6t3VDj1+J7$=Xz7jHVK29D+%A+fIRPHF2OJ zJ#9#5eW*4R&;WE~QLV5rihZ3fW$G1e0(2q+{)%kZhsmkZ>T;l`bQ#>@_yMUXkx>L% ziy4;k?l{k9<1vR#$C=h7BvW!gcm14GGt&8XFWu-sp#%H%akDJQ`5xK|GwFUw7+Pd8 z$`g^@j7Nvm;f}$Qq(37(lZ#DL}1mYhpyM2eWW8rbZp& z8P4-vA#9rBpd{L2zNN!87b#|IR2)g55RRIz(kV4P%1ndHMv#2AaK;5w2EQi#5mV@( zbbyFR9HdjpNwTeCL$g94rA$&JmnOMB8@X%|+|X*d5j;p2hl5tm%t<}E+?{l4g^8-W zjB4gBn5MxgVKCfGLzu3Q@}857J|2}NYm~PnyymL#0P3o~$JZ2C>ll8}m^9H0OsD&3 zs%DKux8hQU5NUzOwSoq!^w3PIV)fc+jg|%9VY6WFj02^du#AXI4XKP#9(C+qGact3 zR+9ZLR_Tp1iaz2Uk%7y_7;{s2Gu7*bJrx!QX<}4YTtce$n?B`cpc)56I$$+my-2x7 zC5OC$gD%G42Q*CQj5a%hd%0iK1eIB3SMhaMp) zQXlICe$m1Pqe8rNo5HI3777c`)O<;!UTDbzj5{bsh)`lUl zfH}QoTfRDOSG-;_%S7@d7AdrJe@J+2lj(pPpg7sA7=!_AKe+st`TrUg_)kj@j(>t3 z{sj)$>|yySE|yPZeFut_%y8n-|H1;3uVVo@C&^`z<8h=Tdj>E{c6$K#>rO9kwVMSe zP&3KkyETTkMo_JnQpRQtDYX;T4Te@PBb}k9HM0u9T}%{Hs2;WK4mpZ*sfHH!DASto zd}}}%ktB%{5C>W9+bqwkc?dCG4$h9s)shzuL7j40sO6D%z2!O(+cqaozfdbH`B;Yt z$c|~;=+tbxPgP13OHJV+5%!75YC)Zm2F5ZSLIuB)?uESG)3KmSP82X8rH^-G1)~OL zv!8Co-E6O3EcVPsGNsNaM6w{b99VHUgG|D#K3O!bx3c!A2yumgyIkJnX ze5Mn_oZKKN#7^crN=rm_Wx&`~Su_>b9JbvtK?G5^ z%`0?0RjsH+x6vw9^KCj(t8mT3>QPz^1kjfV!)zeSq>SY{{m7i8$dJy_erz>5rZ{Zc z*_u@LCa($d+VeF=|M*Sc&Wl zLZv}j6|Z7f8pE;KqItc-b!jBU+i;~ycI9z9LW0bsQ7i?aZAW~tjXQ!#!bgZUWDlN8Tq9IhoPCn)1aS17P>ecKtsOPOx z%+fG#Wtj}0S5?+VbygsUA>=lI%1dd$tjS(JPA1#x4jkL!3MLJQP@^@R6y2It^bETz zKn<%fWb^q5rc66(XGWGnst#7mS3y0W>sMvH%k_fdC>xUnmFo75WP5^C$a1v-q}Nz1 zWmx^1Cf1sa<>b^7-)acyijAi!B%7P!qax=98fkQ6RnB=(Xy>D;Rp}KCb69DySyAtq z5lTjaLTK%&83&W7)~ych95B`eftI_30JbUH?Ft%LGO)KoAv=X4r85b6E+uE%0oNfr znNb~{YT_UW^$8vZm8n=09k|2SIz7COcLyODa)ts|pn;9o8G@=!dgV-SWTkbaS8wYD zHdNw9N0+l{TTmNVXW|3Ui_`)n=j6>+wkm_?P?#w@Udf2C9=*}>`iwixC_sBNWFJVX zMh%R>FZ6g>ubWxWsj6fFfJk&CB~oD@Y-$Cb5_GSTqXaTnox4|o#bP^a-49$=%`){4ovT9rO92RKkwq?@TQrFx^EC7OkL zvs#Rd7N@i#G*t%Lo$ThO4Ngt0ltZOkZ5se0q6qWFmfw`nKVHDQVrI9p}c&9yP9EK3PDSVPBEm4$POp1yStd*rlrk8GZ zHFU&Ijj*g|EEaT(SZSzMCDr=GkgI%0iw6ko7r~H7m>6=?k!F+<3f23Y- zwnA{+MJeqdn9-?gmffEuIve}}XiBGbpwc+TF2NxorfR8r$V2dS+)BY%Nzb!VK7+*Nb_O_; z9kCpCMVJAQVH70@G%Pl27~f=TUAfY=@&mGo8DvHkaj<%fW=vz)2t_7YN^q@Ec^P<+ zZD21MX4tB`n}hmyI%ui1z1IOvU05iC^8QzTV}!03To_j1-6w% zs=%KaS5%q;z9Jo3%?_TcCt~6vIaUk$t+K0E>mhD2!-j)mDCdp!b|s!xyk=rinoYFV zsGHuTR2XDaR2C>bRL!;;N+pZo}{A%3{lEt4dtUDv9r8iHRa28*RBf&H6)ONRz{W zmXigd(+x}+w0lq;3x$G!jnJkyf{=P<1QfdKdkD;=d>yGwCZRFSxBUcvBcTlYBO-K0 zsire>wb*dSJt6192r;CKlE|Bw8w^3~q$`HQfN@+RDEaoNz<7f;Jd&F=7uW-))eYos zgwY((G-S8Rmgxc@pUGjlKWdm5T}&B(>oesn!y1DrRGd_*mRD>dh@04OBi$?%nv?*X zRI3Af6;`JvUO87_hb^(+B5Vgr*Zml(Hrx4;t&DlP#R=@R3u>aRI;x6rYa}*F0xpqi zC<5Iy0c)q(X|GBri<3iBY%JB~!blw((;U{1606F9d87x)885VkEesYXii3D1S*CD~ z(ufumudnk3eUxk*2eq_8Z8V}}Q4q4jg^42<+EWYet6=gZYNsdJBGRV&?GX}# zXA0y~nYx1)d!@=apP3E=Wa_0+-Lznj<^3jsYu!|+^fKk7r2y&M1(#(@X+2R3QC|V7 zlBAg;A_L~S?K^zED2sAo5<~+f@20eXt5)1HSR=CUl^ZzZkKNG-hw#dv*i20E$e1EH zpJ_`4qS``>u?GS`h^Ey%N$V z0W?#Wy&`vboeQRt+mS$pAj(ZN3cEK#`TuPvI=b*%@#kg3CX0BL4!ZV=6vv#LHk2KW}=PI=YNZS!2Sbe|9{z; z7JTAp+SDgT4NOEUnq)QsYe}RBS0`E<4buOL)aY0ub?Iz+q;fKvPum>`@f)LtYY`oa z=gXv38mG&>f}2Qu0Z58q(NNAN{b51Q)l35f=$M{^Rj9s$DSWS;8iFX%n63sM)#%si zKJe#zJxc%vuGa%ja!d~fO@{{1QtD{!RwbiI$u2}Ap%KVpC}pT>IUCj5fW93xq*lyH zCM(qgbEKOMO;agqsE#nKRFP^WI#Z6M%81fx5+4^`9ti;iB+ao(^g9z(11aqIH-qk zXOQaW@sJ~mS#3}r!3-y7G^*!s9Qv5x2{@B22OQPQ8&!T12?k$wfFWcQlg$mFhMHyZ zNjT{TT~DGYArHFCWJ}d*8^oa89LEc2+-{G_DW|k!yOz#`Q^t)GNXj|Au!|TD*-&lR zA;vtL;`4r6)kqci?QK7b8>tkVtxg(B)fXYY@3jS?YUC+Fn+&F|FzN#y ziFK@IX-pdJ0*`qUD_!eFOuZKnDLqXHHEE)`eB=godqNYLR8`NAX{DlwHD*)_N}&ss zcP3>0T&DmC?}|ZbluW1kY^6v$8Ow*TqM%GlN>mu;TtVS zv*XO65qHW;sa&ekVC_xEO5T z#sdqAu$}Ck!tqlPNi)w#0LCLyK-z`UX1x&XZY5$jfLAsN3grY7wUP_@%> z%IergVGIp4Z|GNroaC7R{4#35;TF#TiGn5y-hHBl8RV@1q_%$U5w zp)Fmq(5TtQ~^^ihl|VHIqY)A33phs(!eC2bTM$qe}%lTA-^fc9rntwGfo zas854GXuWixFg2OHnU`? z*#bV5%2j48#zJMxSP8~9r5a+kx?@>r4Qh}+=u}0l-a}A3OhaXa^M!E)Yr_^1B6Pjp z62l+>&c#qS<2V=xEfBBmu@-|ngupbzPJ*k!)W=%Wrl#ev$D~tYdMwH$TET|}uO7yc zhc=DQh)B#46Zb*+x*h|xGsJ;0huIE{hM6MVEA<;)n`RIjj(agBVIZHvQFTY^^rtNt zp_>UNQKlGb;iMS!fSncLwwq1MeSMf|}4|NYEsZMngc>(mtky0Tr;k9z&U!7z_%H1jqLVBd*XYgrkb=yJNPa z)eHetXPhotl%0-2G%PPiYm6XxKvg=T&Q91#GaHxsLL7o_oT#jeBKQ&DIRbnM2?5y* zT#?n?_E<%lv8`f*UbQ)v+8`1st2k4PlBJIMN>Oi`lwRhwY>jiqjj{ld!4yyn9y$a* zrKwbk7~LO^8(6?%dfaX_Ixf~4mYRTQ1r3OuNnQj7O)QIL-Oqe1NYQ_%-JYgN!E)lgArtvVdHC?W?iu#lOwQ^gTH?pHQ~_GQH9*Drwqjsp#w~iNG;Z3rUt-= zj9EhyV_GBr_iPT>!T*Y9ye@}k%B6f+sdv*VO)Cvc_$QtbL9_|r8M>N=tG1EKH_~0Y z(BVMNkmj6sO}ZIxQ2_l<1oTGJ$YV)Jsk+o9(e|Wh1HyQ%K}Vf!L(5 zFo7qQK4nTi5_^^2q!I@`PMx+bjez)SER>pkIgbo&r9P$W*tkqleN)f%M=6{rOr>n%09cmT4!}P9K(}hLMDwK&)Obb<3CwwI)pBFSA4%i}b+AR0#mR$r?Rrm~c=X z`~7sH9{!1I=t(Z8DQ?iEv_1lX^=msOU{t^!lMcYe8M;)-bbE}c<7%Z2X}zGLHu6dG z5y=WowA!X(9z;>9aTWlRa(}2xWm{rI6bLZs7olvbkx~mivz#2tJYaOn5i*u*Bc_wn zIS9$7Mtv&-7a6KBs^+{_p;8xWe7l~m%f)IAQR!anyM=t7=~Ipyx5uE7NGvqHf~F4Y zO~jefCOz)-s$)yYb2+g+&LTDB8XgtORv=X?ZJ`({ZhByl_9O~f5UivD>7NI63OYMR8j(7x0;dA-DA|OP z8zP{8vSiwpLuVooUNxyst`Jw!Tt5v@{jA|m(Ln^_f=Hp@;&C;0^Zho|&Gl4{R>Og; zm9pt#9#Ep$08LwRb1EnIYo!`t!3CRPie7^*$y1{|E)%Eg%;)h%rLRpX9+rTH&x>(kQ82 zi-N%KL%mpZ1u~kHYAw#?)E-4CTqgrU<*8EDQGqM1LSE1f+(tyWHLO^3blem>C^j;5 zIP#UCNw7tVw2@Iu9-u=t7k1qmF?A`Smnmj*`5{&zElM5dD!_eg7j%qsIMtb!`en$R z6qKN&!BUoqbFo+g{y?)n78|i(hn~k*!C?h4dN7|+HIo8yhRE-Ai&DA|b`fw5L6zh( zJ*LIG)p5_{0O>B&OL4snhhhiQo$(Z;*0B|GNEFKHMh=ATblplP*=PMx8Z(eOU&gvX z!GVYw)k>8nffoT@1eVpPSVHuH(VI{?qJY3&lPGlKSQzJ;Wxq+9D1l|gz#FwQ&8C?u zAc{AkGHxLYm>a%9O;}9qc0o!~)Gdk>uO&^n zEUI8(R&V1YsuQSX;C^Pi9+L&b419J}DM^#6rQ3k6Hp_;bEC|->I1O_`5FkG=Xbk|f zzzit{D57GGhxJhbkUR)Dz>FE!E!zb@h(R8!GzbVr(CPpYi*kuWjjA=()e+_D!^#js zc^U+SEE}ABikpP_4#+iYr-|~2&+5%;hz;pNQB#in_^ z?q#caMPx8juP}0BSmixEsP@D>54Q!ks7zBh=DKFYvdS<@6Lp>HL;^Ro+*}KN-&ZYE=cu!X#{H` zD~j2)*r~LKw%cwu%m#~fYFGgrog#qcos#J`BP`?qlhYtTtD4^ADBT=-4Ss0JBiskF zPI4W>AWIJ5mqIMX7l?in>Eu!x+>bznj&6r}bi`8=VgyoQy&6?OtWFGSsSy!EHFF63 zgw~XpD*ZkxlzK3MX7Uvbcg3j};tb$g$a+H`^jStKH<6xPqo>V|qhelXN|%{JrsfgW zP^8E@l!|q4*s1rknxhJ$n1;*s4A4C-*RR(7ao3^vVx~kFlys_{>Wnh&GNdTD^fZu~rK3Lm&f95d49!k|`1hSXkq5kl$46RKj-2sde*AVpzwK?U7luN&*?@MPuxO1gdVIt}%Subig3vLZG;b zmDCVG=+*47SVDxycd>#Pwh^otW=L8qgoG2Yg3yZy141c^jP7}X7X&h#z|U@%ATm5E#@NV{AvjeZgT$U8UajSG zQJ{;t)&ztRlOu#ESVC8NS<_(!n~$;ylCeW2-w-E3ze<{2E=rSj%Bh(dty<}seGf2< zIEZk`LdQ`$Ro;k*Y&q0?g5)S$lU%ZG_iUtr;0=&^qLZmmfMO~&MCi(7s47@j>hll#A~)2_d7MHH}Iy&h|U+(rqHU}RN}99ntfF!l^m%t-L@!WHnY(a zD$2GmvszOdmtD3m@qX54#?wJPs5h_)!bv1jHHJlBMHHVdH%8OesM!sJ2BXn*vq*LN z<%T_WWxgqqDw7M9v~Bd*396MuX+qMLDQfABsaYkZRClL|(Ll37;Rj7;E{;)z#fP|B=2CDbTG}JWv zJ&->TSL-133hvd%amltybd?Var9qyy0AB4%rGYL;l+@GQZUN3Gnu!A*Yokt<3UoP% zk);d6g5JlGwvdA{6EZ*&EIqQ^MqlZHC}fJVrF4RZizpf@)>z1tAu-=68QGvNGNpo6 z1uZHd6UIh6O<4jVfLzxbx@o;+<}4=BR!p%|l5)VuXWG4*PoZpioaAwkc7$qVS+3>M zeSDw><%wksLCm}r_`#^`qzTPzrANlFj*@cKw}xCTC`4o%XGVo730zI02Si@(xd|O< zLlA@aN~I7)rZW_H$s?~Z;$Simyh%VyYCugfoJt%xt>V}XkU>6?(5kGAJ*gSfAbGf9 zBA^*kw^JbY3$h2M3YN2TAe9ZmS-;`qRdNc+K%QJFL^W zF^6WW)ndEU>eSPcJ*}7MP8O2oje!YJn4K7HV5mq(l`sSZT!} zV-+kg(+==_b|fY2OsUtf4Uogd$|XP9lANL?edA(AWqjTf2O#0SI2puEbCR7hP-7ss zr{iFn6dfDmR9ut{qE{FX@-bRs+NR3j)hRhi#gKy#J+T1td`e@ti}pl1E>E&H@6<;M zIkiY@Kmc<@rdiL^WS*eQAz7!!O}^8mGd+z15^aHZHi}IUWNKu9|35CdND6RwBydRl zOauIgk}c~Zk5UgA*>*@$ONVvLlk zy|~Kbe#WN8frMz)sP2X~QkIf|2wtwOrxGPhKs9|V8D2IyDS2nosHc&r!8#?gN#P^C zuNBKMC{~SgQ!E1*Q1B`Q&88^>^+;ChlI}32^6IcNPQ<9+0JsfyH8jJ-0@5%V?F6Z6 zb{$LzjLw!UH9zgxSf}h46_rcOk%{YgM^*Y!y<4c-g8~h5FNW0;h{oFkZ)E2t#dyjU zfqN#HU3of4ybq8d2j09HBL;YVrdn~rblk?`c3?_D*pz$&&P*kTgU8rJ=-Y^E!x<^& zeYhtuMh%Rn)M-&E1SFpiz}q-zFW^G7z@q}@4J#82?t-Q20Qk`~>sG2MHyeIbYlw{4om|lhlulW3bqky37KVO?m(KqeC z$CEQN^Jnr@wlO*3-AgyWVBY<^EITa=pKHl$i{-($6wEex`JS_J=o~&QCWbT319{;nc z9r&l_{FjFR*~_2ip1Jg)y^i>``n85mhxVU;#=$ENIrfC+*PhG@Hl?78UOPbA>T5&Y zo&gV**7QfN+wyBq7M}yg6x{LIDIcFbvuZOJ{mF{Sqj%4|chQ3C$upJNe}DGAh5ohw zcg7nBHn%_f#!ZFIW{Q{o{J2ft-)spsbLw{UwaP4W&ia#Pt~>66clKZMdik_3_gHl7 z%t^CepLO%*i%y(bc*n^%y*cYM3%c(0H&T;h_nKKdYp}@^&%B+vcIJ{zpiLIFXSSQ! zYv#N|fAZNcX1JNfdoB6)j4_+*z5lnL{oKFqYu#^`bWhI|XZFIV%?|q8zOyEetepI6 z>-p&;X4Y)8c<&|k+S~)ToOAWOnZnH9w^*cYe(y8eEsBnKxp-jJQJ_eM4pT`i-a7LPdOZ?0uO$BQ7j^U?)pKYrx_ zXB~d(?W>g2*jZa+s}9XTSx8G-u^tS!Gr|!Fx{r02ieY?H%kE3?rKUlHn ziJLEaE_>HjH>mUIt^cv;!aF`(Hvh@}m;TCx$z7@2?|<^C-<`DA6;HptkiUKKt?h2Q z0{#9c)$d+~o%*^n?}yL7devXw?p<*D@ArQX-v5X>r#{$!_ZwHd^O`i=^!(c{&s=!+ zI!kgPwrhhJoEOwUcB|bzIE@bC+v6Df$KiJtL43V(gB;ie`)UcC*JUmn(_bi<@r0` z5WSwqk5NBf@Q34+mF7i$^KkX51#`f5ulU8<4c9C^@5k`}R2$iQgnRE>wC&ctMX%%{ zY}wsczq9*6|MBdKQ~1?KKUw(QDT}r~{n6!jZTsw_pYHap(+?lc+kW1>dCk@LnfKmz z=P~p4JoyU{u-$RPj^5qdy!0XQ$u)=WzI5B~?B)Kpa?3prI^nKY_U9k}#9rs$c2{*% z_(_$%ZkwGpZ1Lphm19@#cG>B(m!Glh6ZhPCAFlu68|uHWId^&M&V!Fz@a+Yrx$xDK z-nwkNt>3$BL!?9RUEF(d?ZVvZ6P{G5Ip@5&pXcnb+10zK&m-&uuV=5@`=y;eghPK|BClp+8+jUjP_u@OH2=yx^$!j=y~F6{dFi`&Y|hq?dP7XD~4vdM~F_Gcbw zbAz>%^hUYtDl|AH8@DRl9G)6U*-~W9@>Y)n9(`F_e1mh|8CrIs35>;&XoW^v>tr z7{3T!G1zOT!)pI7eEGuh^5$BpN`ap%(i#4pLY+kN&I zyYMS+UVCRT{POZ+FS_#QTRcIZx68Md&HL52KR9l|s>AMIDF5*5YV-XiOYYw6{LSg} zieG=S{p{28i#8P&|NiU?3SU=HGxGsxnK>kT*1wMXzW~*u#XD|)@4S^C?|;>f|9w{E z{F%9DF5YrU;k^C74!yQt3GmEj#+}@?UmwEUnc4fVoPF7jSI+&qcy4;&tR48%l_Tnh zeBIW~Ub1BOdp}w*^YEec+^uJ?TD_-r#OXDuzU9Vpht6KL;x)4T)pTZ7?hfs{@OHm? z+0xAE2krRA{8^RfzrT9dJ)ht5o!mdJ`ZBv!_sf*H$=R#B$1b{fx3%`c=XUl7i@g2Z zaP}(xIFd|xMa_IE0$%{tXW?)*)Y^*p_>`oo|;u`Si8F zp0ok#^7~%@nRevE&pP{lFxxwEvTf{l9=PZ5!_Gr)R# z(!CFRr}@D7l?TrGkGrb3-}BO@PZu70{o}Wu{6>24+;-pZU68x+`nPji{EYppw)vac z*Nkg#yZ-4Tul?cA?DO~7=SN304*$3BeE4ktIapt}<>2L$wqL%}oy+$5DM~$jZvD=O z{fo{zv;2gUcE0MY*Ejs`g-zf4#^;ZI^2+Z{t6o`1ZFbq{JHL=`_BUB@ z+4#mw_E=N>tcvXP+M0d+3)zjZ{ETy#%)NU3b+_(z*e%PqyJ77a4`m)c_PwV@KR9~i z^Lws4e)%sRJ#s_q)yp3H!Mc;)`?rtZLfpMhJ!ot2mzVK6i z1@dpHAhi?k5FZ|I-f;Wq)5^D$Ta``E7v8vi$E(-5U%d0)%afz&qhGvp^{$T}wfEY` z4xCqwKl#l<=jC8~{m-W^JFzmj_i69d^&5^Ewm-Su{`v8@G-TzP(u&t+J$v+(P;#nH zUzs={S3kJ9b=QGMuRdX`tJnSRn#C{u{-rr97&#g;q5cKRQCAhE(dPDVAb3M zE5uKaK5dJ)*WGmH5BL4k@6NsOwCoLM9Z4%pO-#GV!2l`vwaj1LVI$hgqyQlCAw2fZS^#{y9;~{m`eqW}6 z1mCpT(`zm&Lf(HMZ~XYp!=LRvyU7s`oHDEZX;i)N3Hq;ZL|eY{;Ey+Z@RGwv4?g#u zQ?n-?xN?=d%e^1%JL|OI>KEGAsy1{zyXPHy9kb7z?_auf*Vpb`a?>$CT`J$AC^vcQK z1fyb?yfXgH-ej~d@|Ap_r+*~|2y4Jep^fjk`xXX|EeXczG*MEEDrb(*5#fwMX zy7v+HpS>sc?5mGZ>vno<|CJ~2%CB-yeSiIa@4a~Eo==YP!OLgw`0TOE?|;TzvP%Bj zQLnss?U(Y_`y8|9X(#+)>6<@23HvSb=qVR$vC|!UiLWi2d-cuN4&xda#J7M-erWcY zuY5H3w5_iG&1o;^o;vQTC$DNA|I344IR4c`;0^oi^*m8Ip4_m<+L;|+sUNg%%L6N! zvlsuZ1jP@3@=k8;2kT$Sy?z>g-NWy{{zvBruY9MlW_)t$gj3$Qd;a#XocGyFtE{Da z95od0TetSaQ|~ZkAgbOU_a{c{=z3(we#&gkF$(xOYTJ8vHR`Mf3D zs*7LEy>m-(-9^Vg(%kvUUp#Zm%e#Gc@?Y=+zH1)*?a4?|u|_4nJ<^ zJ$F4vc=Dq^pYr^%rEh(_>gts@2EX|mp1Sv=Ww*{f@WV46+Vim$TYuvH&}=JWV*f8X}6^M1Z$vz`8N1%Jy1*nO8Dy!gS7o^&c(cCB9Nr#|}YGg}|8K6~~Df4=*J zPxqZ1aQuG9eetKQc0S|}F#PRZzH#rP%I#(PJD-03@7w+|+OK*(IR7sME7$z_v72AL z@v+t+$KEmj_SuQ%C+upgm%?W25z1PXtZ-bn1+0K`I`T2$u8}{#;^!}a8Px`~dyZ`RIgV?JV z?Y-q^^}~;R{=)|k-fGc?g$J(s{5I_$(8(*0x$wkw2UYj&fn)THbe_M(EFdd`1GY!b zc>5-2!}>pVSDmv&T7UjB?qcNNtGE8kFCP1&`BHng@w*GZ`|odq*UEn-?y&h4hkttV zq0?n^UGbk04GzWn9ft?uoN-nr}J zwLks)#}7Pu^I0q2eDDciPI`~M^2JfVA#Pn5I4ArNKXJS3pyzHs<&XD&a_$S!Gk2}+ zvA7H|A*WA$Cwu_c9(p*Vc#X+qwDkDc<2YpukU*jJ?F^_*JM^6t{t`2 znajVt>y$T-Sn$4>bBKKnlR$=26(w_O_@^Z5#3D_f7ey8O@ApQ|;GU3~vX zKRaaLUUJQ<>GkW*`RQFt_@&o<_l~6xJvZmj4eWu7pW76u@2$EgsY~43&b@WhQ*Qp= zFJA}HVUfB0+}-be{kXkfe9oUdebg_%Jb?Y+Cu>$a+x+mr@^{&D_xV5Uy=7Zm-O@GM z5E@Si!Gk3b+!_eMEjV;=m(WOXch>;HgF|p9xVyUrmmtC2-Q_HD?|q)VpZ7Yy;JjbB zpy}1Urqrw&qej(QJ|FCY=)kC|`sxpRjWL%CobhQpl*B0Fca*Mw@1JfdEBv&kGSgzg z1TPHuD{QuO6)^SmjLY;`??&bhF!<2C=ZgMg`4$W6`VH-0kp(7N6mJp*9h7LvU(9M~zGZy+Ed-zKhjR~|CDc@Owo z>Ui#kxlOjh2b7(fecu@=K9%NUo|o3JINYn-1$Rtn`nVzd&Znn<#{S;{4Q87@hT&5@ z<2~{g*l2yu7z!7f%VL)v4WzH`F zcSEhhXRdRzUPFl45>qsXYuOwDh^TEYc@8u(`0vJL`285TFEd4^-)>Ep(e%gB)BE&= zyKkg=J}AkA6LY)0xSHVDPwGY6?D6Hh%2WfUVlUib{R)Z+eW0Wt#(N4ZkiQnfGFq4G zUMFizbrlfsDZcI2V9WFQrF|d-xCJZS+gC&!Hm{26F}jY;gO6tmO1v&1o%G=GuWC}v zZYMldNZT7l`~q-Oh0-Et7Sq7eRPW6OFjen&wFllG>(1lJmb&RF(%-)|QNBVx9`{H) zfu>T{4Sc|6WOSyniIekYeLYEy*S36g%o=o9LRI~yZGb7^Ko5Pk?d70LOTr8igy%Y& zl&RqEk2z>_;lqKWEtIn&t0OlwKhtDJ*PxIn|+=IXE&XS>~xIPs!Xh-pqV=7RGaP0x?DL-39r)sIspPZ;s}w8SdT zUg}lGfw?w=7w^ADJNIWeWXs31<*RhwLN!WJ6*h{He;lom#b=lY*?9jA%3eJgnQ*PxZD$?p1--Y{ zymx1(Y)6{8`VtGm-H!6QcV1ztjRsKzo5wFZiMmYlT#bIMtm*yQi*UNmcIs8h;kzK> z*kP&UINkVM5aLI>IyJ+D$#ifEHwXO$1cd%Lo5IaP6|+W~r`Bz2`)`w*kty9%u)Ir> zd0zqHnq%K*QmejuV}h}q)|y4_?!D~P0tp_5=%G#F86|JIPY~Y-)?4Y7y>PUg(Q#be z{>B(Mns}dx3c5Bs{Q-D=>9`x_HoNW;QIPUrzc`zkQ<3fZM?jmFs@q2;Utr}4VGAVOKnnLqP4b3F-F z+hI(8CIS|gFUQeKrIh5ePOJ#II#&{mpX;`@WVbHuDr7j}v9vug>M{c);?n@MaAAm7 zw!D*1Zn3*qxum{%aK)df0x!7nrk9_czy1IYR<@LnrG99#i-4!4zf0H4oqXGczL>(R zpgu@EhgZ_+DVJ>Bby`$w_)AImtmDx0Mh(1C>v-5ad$~bt+_uK^bB|A-%z)u@Feye+ zc!bHx7axLeNkVcEFqvV6&vv#GQ=0qY=#IFbaYy}X9kw&3kx$=?A$7=PiMl95FY#it zstB(=Hq`Ftns;r~CSwm^#&S)X)_{A|bQeN(8xtUAXVyJ&yK_HGQLibu%D0 zfo?103M^f7b;V?jXD%q_sBDao>pEV{jJgY}13O_TJhv^>skgxlvJyu^?(DE6GT;1m=9 zjenMcQ)RoZ*y5%O%$Y}@HVN(DIr{)WJ0g=1Bm&0IO$4bQ@^Q`1jWnDAJNbi}L#5ZF zd^=#b0@oHIE8d39?O?X|+x6u-AMID|l10UjTZvePWv=IT?+;m2wb)(lEUWNB7{lx` z*L2!=w1cN1qb6soG1?GMw-)L;S+|&iMd^L5o6AKpCr0PbgM`y~lLIlbQ{vS~lq6*R zO8U}9NeobWjp~fa=il`Pr6 zv`P3$XRhLNeQTkDZ+1SZbE;&I8a;u>T}zJ zKl)ww+f)GUCoa15=(~M(rB**&dpeppp8Cn{Prsja54N^t=1vN>$x)=#byd_{(Nop~ zHW`hyyBR8%5&SVE&~;%)iDngWsL%7{ZXYBeUBV|*6il+5(}l7uSr-s6?dFC(?N4%? z6qg+d%@$`glrcOHGiCuK_mTaZRy^D9yqQL?q-fD@8C=#UT~XCK7oc8d=UFQYvnK5vt@2UH{6#KDIBhtA)S54Jv;ikL^WKd&MAN088u30 zEo>#d*zxg!d&~DNg5>^q(*^*t&IiPlJ(ubrJ8icv6!AvK>DrH@3}}-GS;FVa->^1m z+^QUMY&`p-8FA`sjJ@U@ALI=+h!@^#@5|Xx#XE5@`^DN~H#iSWXR2)q)_T|^s(&$D zNzl|ZCY`Us`7u@eavKNO?$Rz}+0tSBhj-bU=J*tcBi+vZY!J%Gny1MLP#*~pSl6tS zw8fEyoqR9G=lsIw@;ucCiGQzy|M4M{@`@DCUHcwD6d{p4tQ%XjeM>ax@-KnonQ>?c zn7b53v7(CS-1W89nUGTebSNJ^fSvRGFH-iDA=cykm6^Yi#84;<9Pv5w$Oa`6r z>f!wNWJ*s?`UD>28P+Df4dw7+V5ZL1VdLwYE0_Flw>1JMc4)PHjlQg;poxav9OTl8 zA75zDhB(zyIr{WrF}BZONj_>}1tmF~wbxmC%HASyCQNUSr;v~2byGR@SSg(+MINg7 zqZ<0-ZtDED%hmIGzNtquAF=Wgv*;5|6h)X_aoV9iIe%LxhlTyr3D4i~MfTMKwRc}>QpE8#P`0mLC{r% zj8b`Q>BBhx1?}$2s&cDt=W#C#_9c#;~^-gC^qHGj5HLb!W zN~%Mn>2SL+QY9{F4!qUg8-q?Z1~cxCw`rLxT?x%@PE13R%f$B$XPPh5Y;u=A<1AFT z{rMmTjEp=!X_(g<9d2;bN`Cc#2Se0sMW@Jq_N@KaIQL8L>Z(|7fXBjwen7`#X{&@Q z567i+mBfEtC+=FEO9GH|7ue!9e-mp69NvHj;F3&uQI-$J!XR_!k|webla3^!P@0N-9}{{ZfIC!ea{+M*H~$hD@|Vf?hQ8@Wh+4R{ zy}grO=a*@FZ#4+DdPa|RzpK|0BNh0gr~yORhWYd?2`vRWyd^{Bc<0y*781ynNhUfk zh*9zhj3pBF(Y?tNr308Ly4Tb6hh;Dp+put^&6L8qa*gE6HpQ&ghW?$61D`q#;!Q07NP8`P(GF3SNZ0zAj*!%Xq(gg2P`0ys*A8w{`M<*hW*9 zA004aPKTK+8b+4~L5>H*9Z?l!TbMMfm?L2)XGy_Z2;Skmzd##V@wm0OU<0Of?&Cz&}Uct-by0pJ7xl^?U;1 zmlCp^OP!Cuio3?+4_(~P&x`Po+lZq7!B_ z�=I+7c}r^V@e$TE=rFUK9)+AFt0xQC9VfnHaiYL=`-HNfTd76}((lbyZm&@#AWQ zk22orAmT;&vS!ZPKkDi373$$Q=fUQ#N*W`vVswjLdli~~;r#9oug03Rna46~11tnK zH>ef%+|M2B6+diqyp)1c z^t(9KX4D=!Ki6wYpgyOCiZkTKuRiTsW9j3f#20o}Z?+f`*3)6;{4f?eK1Y@u-h-UP zu_F$VC|*B#nu9C5XtZZhx-FJfTqOSE`Xh?;zJ|!13OQpD&)ueqf)t)>5e|-nh(!M z+Tu$$=@dv^Ul1wX#^cuw2idy0abA8^{-91_2gC$t&Lr3rJZve&uF6zcq_aH;Si?>_ zGYLDD2Y&A?o5T2sf8=@gxGrCdr-$0tJrjY{p^=s_WX}hS1tU`J^7>1|4|l9QaQlDs zY|$iN*X05-Hf+qf;O{;csh*EYjJfDza?OHBcQ$5GF8d`IsjZdQ1*HzB((BTaa0Uis zq#;T6iJv+3@TqTz71|Mwl{!DDE77Fb=u6{mUuFd^$|CvMjQX0VY}Ws&zt81Tjn};e z!p;d^myj~aCtSl2?^0zNa-%dLOcU<}+aCbv>snE?oN?|rZ&!-@2u%PIt?yy*Df5m4 z7N!(?LiCR#!FqBeIE!)$c=;0KRIKEb!yPK7hRY44C2!Osh2>Iyt?hH-6&9s>H0L0U z9~wI31>$reQ&oorDtlisJ$RPGL>T7m4^Y6`4LNLb)lPzRLB!hxK#oGLevKI<)~|0V zQn)iNFEr5m1bg+(^bf0h6Ioe_z*tn$Rq7YQDM<+ z9dsb#btVB1u)CNkJpcqbv-1JaYdE7e8|GrS4tqCMCpE3)df>!S0`PlI=A{7G3qc^y?{i2WWrVuSLh(*aIeV6sWjDC-S#2DNYr79|21i*{(> z1ZxJ+Tve7w%2@j)OC9&oaSoEtMd&&YYaTjx6M{3oH1+4n=0p6U@tH5G{?ysz|9E#v zz>L#0$EG}$u%U&R>6{#7q@0_9>Muwqau>IfZe9|Rr}Zh$jAy;N#D&CDWcNAHT`H&PCr3lCX^}I5FCdD5%Z> z$5z00c->H^%>NL7N&RLq9FPe^)gMxSSvDk3A@j;RFgBa^OOWW-vBL*rUvY0JORkik z%1TStYY4l|L_Jc1H<}fVz7^F`eXe&D%8tMg|nx0yBTptdE9CAV3fNG5JO zY-Z)^)N35w4e%a%^3d-zb$oUSe7{YXd7~Ny2pb&q{?ot<>0Bf+Wuj z&qJ{FqzR#r$1V4$d60zzfbMgobuqFlT^}(UQ;u-5pg(>ym3(kp$BNHp5BHRMAbnA+&@PKq55x|l(4-3V zxl1C&s27yQJ1oE9i#RPe>HW&?xznVxwtvUJF-klZ&|*1yAnmP}4hi zQYcrNVfPIHG<%gRxy~@bcrCXQ#8_!qy9T?FftWr&fplzyZ29o}Jmo2tHtgP{U&xg8 zlH-%d05|ZIdHMz*@Sc7fO%O7$#E?;jxK$i2D^AhTkNM=S9(B5?2mDTX2vILaIvjQ1 ztJH^%eQ}>VcxE!V+Pt8HwV46XTK)AyWrJ}59=xO=tW)_LBeEdqYC`Db{zwV#DhhK3h#U_V=~W^?9Pk7f}O$<;b_DEjO2 z!y^=Xd}a~cKUeWaN`UmQesVUiy{EyZPY zeL-Tk9eJr%f=MgPbIDgHWq_dY#`dC%BHZ=$Vy`?pK-9}lQA_X9d`}RCCX^^2SHV{J zBnmDdb&0)5EQVrjidAOJ=cq{|tegOAaF~1P1^)50^<9SZtLbeUT~7L_CY?Gu!Rbn@ z{;P{(;j&H6HOOdZ3%)(yFIQdqdE+c{DkZloP+Iuxi0sC*CrtqS#piAfP)vO5N69u}w^CR{sLfSG94>->k|^7>}cg)zAwMy<(HA zgUX@rjw1dEfGQTdYPx@M0dkzuNlkfoceg-%KftdY{QEW&JU*fGN3oNOgskzhm3f`E z=P`x9IaR}D)GxfnI5MoZCdy(on38(Z3 zlD)aN4F&2?YG)+&L8V(mY7XOd@=XDoJ~aIyY^eR*S zfN4d45SZq)?C2R+^7k<5tHN zV7z>o#6EX(hfD>(F6;vB?D{G?+5sCR29K9aTjmQqyPf)_jAg5CL*NwJVS{GiZ@C311kJ~{u{ah*lr02+o+56I31_$Ng{}R=^~g8Sbe^|2Kfm<_Ww z`h@?wDd3NnAy1mK|2vfbf3C_KuC~ra?PpfcTJK?QZ6NAXR?ufSE1Xt-x#&#^kTqyN zc!R4)?1*G)*b-1z9o5StBqBS_7MszAiOqDvx7yg!J8z8=UudqG*DZjokXlJEuvrUj z0x_0yQWxXThaSQ58fT|;1FQd-Nh=}z0Rk3383WuI_~o?I5edj1qj^UiKjMYuRpC-T5aFKPS$`2QLPYI;a*tjoeER6ol`2 zx$%b&p>)P+ z;kZL@wuyc{E86n>SE-A8?p-4dUQek3s2QmgwH2!8*2@@WVVT&YcPU3*1pba9z%!7t zb0eq}!k04KHGCJ__$)04?+dj~dDDy7AdouY;`SGiDBMy!wZuQ?1^v4vX$Tv9 zD+&S9XX(2I@b~GfOO4OPc_zDkFF>SMM=$Pr8Qvp|3WA2=sh_`j>-8Fh3o88xIcG<} zrxFTiqPJuO-GbZ@VOhb3vJmD`NnuPi^w#$vs?6RJYEaOS1IYi`76L2{+&(m*QhENN zx`~vS@}#mff*HWuG;>88g0YE>^vMSEh^<@HDG|u6Uoy1^5sxz78r1c+I!G?BnFsCg z4p!1xo*H`U-fF%`^QpnG&JVYQdyiB~-1@^P#aQG~N{^np)S~wHYCO@ki+V9$Nc{%9 z2o1;?Wc4;smzP#&|55kmAI)=B zuel-;!^{C=f!X}nuy&&_W#3f%5{cnIG+ay*362s{xBdO$F-Y_(!C0l4rNHsN4`IYvCrW)}K{y`D<=5uE5L>6|u1y$kJXv1FoM-{DmbK~u;r zU6GR*h{v4XWkkOBVc_^p{(~6abUNak5U8FIS7oE``vcTN3X}QU#1M4COe6^+@NO#)J*T} z`?vl-VS21{r~?C+oCar$Sfb59Bb!Ep78q%+NUF0{qt1#h{M?RIH@pN=W=W}bS>xR; z(dt}pOPS|1E5h4b;=}W)SHEtrc_nq-w71>cM5M#Q|0RZR{RkAHVQDpU0Cgv5K@Uqu z>h(LLE*&V`{JM7G8RGf+g*Ucitr)%XdP_NM0>RkQLmZ>J83h=AFR5CwMekI%194Pb zAKz12CN;jdchLp}B5k@0m5^f(KDq#%=QfoPYoCt57g&es$oi7cmn=cpZ=u*UNz}~N z_Q_M=`Y$8N7GHI`((;zGloy4j1!4ofEcK+UJ3!q$>kp0MFi?f-e;nN>1Q-ZfgfTkb z!5U~)a5L%sLc+NLbn1K6w)xIG>YJ>zI;8B6+{|^7ZlkOq?ne z_FQGLyy8w-G<|PKF!qo|)ekJZ-_#S-cC|)rhZu8ccwE%q)h_4J`b99pwy~+~<}#In z?j4%mZp|ESGfEV&-ljKXwNRW%!kn*p>NCdNf3*7_3Rw%W+cBQVU0FwjFj~=sI<78m zzi?lGb(B{;FV`A@@3a2uI?o5!dYl+L$pX4cPS&N5EoIskreiyGl)g#axN>BKmWJBW zXL69!p(ozFtWafg5UtWDw#RcBJCHBMB9ApiI%n~>*P(mB0`yp9xlZR?X$q?EK}lO0V;?6E{0f7Tg{MI%jCDdOw7sqb^xDe+82tL{}v zl!Nh#Bvp}(OA8tm_Mq?GmG|b_fEwh>w+zk7Elz-1u2D#8KX(){BVSr=cI1nVt? zsNxVYaCIjj*xEKPfR@>)}$zp%IOz z#$EWZzzzgH$H{secm%3(*nnC{Z-nrIReJ9a#YUeWby(nasObzJwR-Kh$tl6;-LiN+ zx-O(SS_TcM86GyT90S?5C5i~D$aB1k+Ps)4)vHWc27Y=~a69zy`AQYGLu1OlCaW%g zPSvKN5WW1!(Oj#w7Y~Q1Z6TvsBVMY^A^39%mfCT{Qqt9PyK3EXA|j?r(E3%c!|(;u z|3m?j5bO02EDR4K4(2gj%7@F{Zm~eR?Icwzmd$wTcwo2hbfKhu7TjP1vN?JX#A4tX z0PW#FPW2V}<4BF0Rt8$O!-+(mYw^^gO3-0HqqI5^OJAjq=RjG1rw|vQ8;&KiZ_|l_ z0giDP9lzc|Nqnfmh2F9A8)5r#9cT8VF!Bd+ZHMzxIV2_R!f=vWt#yAo0~Lo!Zgq5V zrzDgcxmv7O7L|8?CSyK<8BwuV((Tkf)E0KG*;kVvGxN(F{h}Z@Hx++&97GGKshC`B z_IshMQ=XZn%fiDm_T6$2Yj5xU@OqhK=f$uVy zdpd%OKZf~8@F(soXvA+#DG!0#Can+f)`Y`HmyLMW8(2Un1O4?@nhcxfPiKlB!O8ks zxH7WieRweb-7n73*S~H#ag9`T|4ak?Au5yOed7KvHAq?L=*Q6oD;%_#X0f);z7~hm z>?_#)$s$1S#L7wERQUx4=7+=*&0n& zq$%m1&iWu)D})w`O5E)w0T;KW78;Y^Lo^2*9^C5gzZYl*or!xa87ny{&G%Opueiaa zi^;*k*b#fT^n^V~)$Bco{s|WyO~v$js2wNWe0>icE~it!hM^BOMSmf<8S>Md_u5** zs}$GjcH@|Q7V{QlZtp|>b8v;ZCKl8DCrS21Po%9AgR*2bpY#t^?W-OP>dT;!FHiMw zxKN^_a}$Ch_&YC%r$;OP?>1}Fr~+Q-&9kl;i-kk#_{!av3@VzOITSwd0S9Dv?2oC3 z9ha-Z2~qUYUL4FzYa!D+;u63~B}oc?OND&*ZR}Gpmb3ZtNT^te@(xOF^}I2=5V7AN z5SE?(i3x7u)QGS33{s{m4oP{wJhf|r5w+1fos(ziI{wi&ib?vnnpmaXnEXd?X*xtO zR#Aldhti>-(r=>%GPm)lI-OaWy(|Q)i+UgEf=dj0`($T?+ znj8MD1(?JA!w-})R_xQ}U&!E5HB%KK*$s@l56D3x@NTK19N<<`$S)+Mi8x8VW8F3s z1ghHsI%FOzT<8$6G2Uywfjut$?f_%eXWGQ1*h>}wHGW0dNl6ut37F7%*NtLN1x+qA zN_C#vT>seT(*8me)@Fv>6S#ziZCOQpYR}O!aiQ37*p1LV4b8MPwwOMZHeo5U-Besc z$|>HWV>3_9yrsuVG?Jo9&85|LzlP2W>Eoign$PrEej$tOL73HSVI^)P4ElGtV`JjM ztS#YoNK?009MV=H8CB6}L7G>v8KbI=k1y~ZYxCM4k3TnIgcbX8m7)Kr)2o|VG@@~Y zR_0rqjl!{Tc#Np$47{c_ms|MTqTuB!!n{b|_*+Lc0qLzR*P{Dz8rm~SQWaa{x>3xf ztMuipC~qtdx%4VuEPPa2k2hE8RzPx6)WITtGb{Cg&u)5k=PmfVw2OStJyoT_ab{mQ z9vGKDPqWMs8|u>Dje#eEsX~1%w;B+~ca_A$N-EuaC78h$L`uw(23rx;{yZQ-p*9lP zwaZzy(HS}Q#-cGhFQeeQ^lXiG1HSbuX>HHgsLRoFpQukNK7j{R2mM{2hAT_0%B50W zgZOwyr-~uW3#E68AfWOS{behTr?e{dRMq;JwU4y-kPDe!wcG)5+8YC{v5yjh!#4(r zOYaiT1x?B4u>BB3o`bpl;HXXpdq#QRbG2oOlo#A>qxQRu?|TpH655=Tgn3gARu|;W z@^^i&rnDW)0TPWqfl3lmM0m_b4CAgZt{v;wt;c#gJq~CUYfF6k>K^5UZ5ELe%EOge zFPKMHdYfC793Q%^zut!aHW);vRR1%7eu(4$C3Jr4qzk1!rLw+U{`M!~?-Nx=pTHy< z{NDjr#7ilZHq}B97h=H8rc)XKC$2k1o@cKzIA?q{{e7GE(QS?hc^kN^1n zdW+iG>~ub=vyu8+onaXKw_w&g#xeD7Fn2ZYg#t^uk_wXhQhbMd4z`SD>v9!67Qu-| z?|pNl7!~85oF*p~D&^*VBiC=tIrb(9eyh(SGfTwZ;=`BlxUYa!=FnMN8hG3m_HM^O zXP>Z9vi4=3SzGfa#V#%$74g;$h;k;@^v3$6&hJEu=UEEdJsbqnRx5s0KCL&HN~OID9CIMESCF(AZsQ;2cldGFX$&ee=0f`=z_<;Eyo`x*n#uwp-*6u5AVxTP2nxyrI)Be;=+pLY!& zq$-&2fTJ!WTz5+AaSDywg(x%LfOG2yu4I5~!z*Z^p8Beq<6W|Z28b62x+gYtFdVk; zH$~1DrA*tD+5^<@r=Y6;uHdz4+;Hms&<|!tRiV6)h@Biwo~!dTLBnE*He1A2$9}J= zZ_N`R!GwO~c%g)f!-!h`C@Z}E{hY{`W|lgn5~iE7SF#s?ow!F%ba_qhp)j*p7dMsI zdH9}}%jn)fcX9dBf8|r;Kd@O8}$!92lO9p)Zso)EFT^+L>1R;J(L%Wc*! z(e8NN0>||_epcq4bCjXteWJNPv^Uf~N2u2|N~Uz7!_kms&d|s>SLwHs?(J!3cKj+; z3ua~K`28oUDb5~!j~eQ;8Io;+Hp?K}6-)J{3Pw&oScrq#>A>)SRF`U^A} z_=8e2vzoIHX?}A-zh~J)i?o`A+z3Y|?)3`J3tepKmxsMkt5Tf?#3erq5Y4 z$BcBi&|pYjKm9pq;odw0soU`(jNk}ZgYH2*dfy62{EgCgXT?ELp^HtKn=@KPaHaEQ zppoqF*FSKiQ;>5~K$7si>uP0d|3}Gw2KmcgpTJuO2ks`2NQ6UQCfK;??Zx z?~IIq>Puw(HtatnIR+pjB?;su4N%|nwrEf*83N{6tRn8&trWpmpL6Mzbd{fFB z@6HVq2Q61t)~E4%z>E3Q?*BHc==)~KZQ<;*9_nj0%1V}y&kukw>~I zcB8Hhq}lS`^^v{#dtucTmp~w_JUvnv(uJ{@nEVsr z$drU>lDIm80>HrYXa3KDBf=3-nH!-3L;qRwCXWed10gFBDI)OdJ6ylNN!gJvnNOep z&6panDUs*e3$qsJzZ$P*W-XX~HP=!8J1+V{Jjd{ps%D^B#6MSBaSGU*P9t0%-iN-` zGt@b~Q8M~`_$Y{->iL$je0?yqS_|=wRyVy|^F%&{t@YBbR=Aczk%dCma6H=^@2gDN z<`=Sz{cF=MdLEn7sjagy!B!U&%x@ioS~%H{cr2FJ&b8~VrJulCn|*B=roGb9JRU^sU^dR&%{VkGCf zeC^KzMs*#0oYwXb9q^AiUV3l`N*z&4DBb?7y87O?!wPExFi2o$BMZOlb)GLcoibq4 zsl8Y)m=N5fnw)XfiJ&j6>1j4qzN`RBY<2o;-~O(&pjc^10%Z>9D^c*s2bv6>L9(NV5ibw1uFOjVa89$kk z{{FN}!|A5^+CU`2`46|CXyY-R{`j`um`!)P1KvR-)HX-c(#g510u{xkDx*1KiKyK6 ztWnFN@!~xAB8SGI`cG}dM*dga1jL!r@@P89?F8m6ZtFc_(`?-qG zuSXE%))}SUPOheEX#tK>x;I)V~QK0~H+piRuBeNKLe@}%k}=Gnb7iA>|LmOK8o7H!O6u%hu;<|CljQQV)Y z0|WJ3jKw95;@ zrkvEDDx<-lW^C5w>}}4DJoXmKP_Fr@{h?~Md>8cr7;b*F*+6y@o5}|0!(f3)BGcMb zR8t=lygJyY_uG99DT&t|{!8$hta9)&*2WEh%bmUa?tdMgi&zr4{37DGrAu)=aL$ss zBZ|xsCstnkpuIZ~h2qvJEP@bq`RDgnE6lbvUISJ~7v5tM?m`R)Gqvz_!shNxIy{{L zyaHyE{*N`z!#`MF7sL&9oP^%Kq*JD6{epgD`m|3#N2X22XbPj@e1hHdGt6K1WEBfP zV=&B}Bj40RPT|{>EwRL&x2E5_GMl$;?ywpEOzT=1soVWVcqqbJLNYNuD`yGx;HaNo z|23%h3XYNES1Ko56L-zWVQ)RzCW{L3kVogMmpGDZqB&-DtzFp^r#XSJlO`M&HaK<# z%ldC;jqz;G(dD!+h}b?cp!-Xem$qCtHA@q5humHs{4@tIYS(*_)s5%x!_l#mBFF8V z%zOcqAfG+~Bmb1fJKVh+$gZ`EJNTn;{v<=n@v+{QCt;!ex3Sds5?I6*BSzgK!(Ey% z{XB=p-IXt$isa@l&(E5c$L3(==d^T!gS@8LvK`&SZ9VG+ewu%T&p!ggUHr`9yK>D! zUBGlXb+$c^iZ_%j&!}f-j1s3W*5phU{{XZ_$ohmWro-=kO~RPO>nC)3`Np-lu|H@n z9(Oo7=wLrlcX*U^G6f%~Z>m%-p_@irAefWAdr4+ntr^=W0O9Fue=yK+ zzjxWE{V~l@QjKr7&kbKiSdEG}5gR=^eFOLb2{z%!mw(kH*Qh`rlGc=ctJ7{vT8OY+ zAnBtqme|*rnYoC-l4sFq%^pQH*A@-9+Thvtk=!(vs#bj^$}tNR*Lqme@5Sx3@uR4EYhXKseLQ8GtW&4b^|CzR+Z z0g8WtEMC{gCED4A%{vpk;!8v`l4Va7u52KQ*B>OGswlV`OZ@DG7cuz6-m6>^;;e32 zsb~$h=x=1B-Ic-{&QM)AqkWCffVUj!Q?$Tge0zFf{-p`_J}-1v8MafK-f#XPDU$en zBM_zw0Wp3UiDBb%RL&d>R~nzD^6VZcX?%@4i5GINT%oDc=}Oa#de$l+K@iDPQh2hc zSxbOx-yK(3y3h!a#Wx8K9eQk3pAXE#7Pz806+R(3Cj(uf=Ef zUMC_DX9Y`E%COe71>$IJ$DPV;rOK*LyDGZ-)s`RglrubIU|&j_DIzq+_1NaA_W_S9)vUL?@66Bu^b8Z+de zRW85_ykg?l?k2y(BXW=(zA#}LjI?)EwEAW>b4n0$4)h%0G@B_ERfA9UxFN7Au_!`h z7%f`teYqni*AnD+zEK7{sZM47NJ75OJR&sYcq-DU+Oizg(cQmh>JEe`N@udHO_F^&31)ppR{;f9iD81fCoh(3SXc*h~yfIo-uG%kQ?IaK%&9EuzO zcqh?aS6`FteA*RZx1B~dCvLTVt2~zTFd&)%x+%?hD8IMla&tUBc1FF#_aAV! zZ=Q-nrcA7_27U;*ri3OH?sL+Wlsqk+BE(rsfSrBJFPv_1`Rq6xCEe7%>E7eCp6lvt za(bckW>oOF5w>qB2sa(>hrZ!*d)*%nD@8_~w<<&4$${xK^|^J1-w)(J8NLt&Gupql z6hRt~;re+Du7|80LDoWJ@$+HJXqVF@r8inM=eXik%g$<93p|E6;wtK@mM2;j>ALPk zC25$@>8A!d^4KEVufU}S$H+BCD#d@oY$oT#?no~EHotHF8N~lx04_Ip7!IYSj8yV5 z9=N!VZ&^c|Z+Qga^y79@W%GYKv4^e^VyWt~$PlENr5yqblkrU+2#-I!L>{8gYLRFn zK+sRBOs7%(y_nVPlh*7b3i%XYkDcri*0s<7*j{DHzB-9;thJ(5WAr-J^Xs!;mG)j|-@Vj#M)N`8dX~4E^P=QDR#(_IIE8+=p|s0QngpYjzXS%8ext1q zVB5@1K-Wx#Y@g@-1qxjBcWGN=l1&m#GXifoD!xBZo>ThtHGhEVct#!mcs{vd>JAQt z^{2ETzMfzV{`@RA&9a8lNU=sU+Gl%|6p8#krNK%o%ae*TCv?|BsEQ}W4`q2p88T_P zAZ9+4$dXggpJpBx!C`YpR9&1*?bGL0H2Xo|_{=58YDRTi73V1?%~#s_<)*>z9x#z; ztPf``1ew+l0EMJ~6mRN!=|oylRk~nyTYP=hFX-L?;3I6G?Xwvv@wkeNa{A|ltaz@? zd_jK_a}r-cn~O&jKJbRMxe_GKYC^ToB%jLQlQxGLke$oz0c}6LwI4y;N*FtL^^6#iFlIw%6n5ZCHYs^Bf1t9o)G3fEt0%>!@ zH;hFoUm4oT_9bRkvI9M*1rgg^#9!Aezz|YUDSxtuC&r55NjBXFNZb%=hDFt}zLO8o z<|08Ke8%i(zKGy|IxQ#6Yhu}-Hwzq9gG6&f7O(1mT~JNF z>3XeXjc?<4+-OV|pQQc*95s<>|EF&nBRDux+_R$CWuC9E*u|FZxLBi(_eP7l2ajH($hMFHUySrC+Uu&&vtzKO*Tx*V?Q>Ya%fl-im z(rDYH0#Yft$s6+1oAF$NUx1{JDb60P>j9o{w=VOprk6lcZUfv_{ zVSmhm$}e`Ex<&J76n7(z{y<_MgZ1nr$zJdK9cnN5ay#6hJUTNE1$@iZi)cS?w=2Yu zM8?PoQa9j)Laxom8r$RjQ8;nTJ>dYz^!FV7j3D-$!lzvARSO*Ta(-@lZ9R8CU(luF zcd(Cz#M?IRy+zLvW9H4XdZCaarZn!;4cJ@njY_BEUlekcAjZz$4x~IN4;3@SQvJ#2 z?(fUlUKKwJd??Jp!>U>br?V$iTqx>w??HYqc%^cCo0Pi>9pB_5!-Y50$H z(yFvnAMg5cFT{k~)>#B))U0E${osM@cU>uuMFuv1G^BHb%}#e_e=S3M9FEPx2Kzc2 z?3B;39mTAY9+auoXiIcr7>3LYu~}^l0?bKZFI)Dc?rA+H|BnCl#}}1M{VV_Fsd_Ue z!M!=G`fnC3J#Pf8R|5H99Ij8gFb+a#i}Kdq%-}XR;WIV7yzp6pl5E{w*YUl1s zyK6BAV=yzsYoRF|jmCEMlxsnrGB|cx1&6vrg|0YH`15w=$w)W;(WH)3seZq!{VvS|X$y9CnWeP8YKH4>` zlGh%d#N9-`(5!NMp3|6n)IZLYr&e5g<+M^AD^AkIYq0R_Lw^$Tfst&^0y98xFfu&@ zs!^O3h&>tRntGn6g<>3a4<8Ysa?ui7-CKv-+O4x|$-vQCf+yfI{+ z&J!uo>TPokl?Nl)Jy>c5obblqGg&>G9fQDJDZ&xb+s>0y`=yTwaQyn%-XQ5Qy zBMgF~>EMMl^_8B_C2Y>SFW%c9`K7WbDAP#Ne{sebE%nrDX}_zS4PtsU)7HU{MSg)c z_%p6OLU>W{{7f2~#brITIq-%YPdFu_ah$4VQE#`jX{zQt4hLeFU?o?Od-L5$nn*|b zgQMka5D~8UN`HIem?X2T8EXQUz!}Lk>I2zx{Im4XFS8%8Cr)W^2c=aMBzOMF1(2;O zXxXRs8-$Ln3RSORC# z0B7;S_1tklqZXsqmhe5grr))_2;l!jANXaO{s1l!blEp`&;m?XMWTuiUW9welk%aE za1Fm9JJ;$@ex$QisDh$a#KHzqeQiocBH^zXpfCWGXHe zZSE}1V;Fs5ZDPdsCE-fG)mkWPUBoAIQ2`#I_CL!+Qs^Y7Nrv=ZO!N3=e$QutV7~KLe0bsjh`gO8&0Tt~RIvg5m7Z(<*ZPd*Myk%((sT7}a zAb3(%bh5$DN7`-IA49XC(J=sL>9*fP6q%k3NP z)PG&e0EWM%(kf8or_u^5d9uJ8KjrL!3@9_+$1Bt-dykr9XCy;u6Z56U<%IW}rK(?( z*6dQKR(F}@c#MSHaRWBwAL1sm?|{oH?-SNdi`~ZrHv|~K)%X_?PeSLy41Q9rK4q3f zrxklTQ{|DRZeXDIA^!w;t^Nc_A5z!P56j4%bge+hqu4Ne;Y%|>y^v(B@mPxK@H-_Z zBPzHn7K|b)_2VffkfG&R4QF)j%~o5V_R02`(_|_ z6g)yXeEW`>c%s}lWVGmNCiz*aaD`M4}1*5k$B22IWFh_JrQg=XGl{RY5xYVZZ zwr`SZ{T|r!{`S!`Mdq)snCwL3EbUBw7`#1^tzQ{Ro9#(@^<3FPFo>C0xgH<#_>s;2 z0BT%;CLfN15jw8d8jwa^8wh@&9>3#{{BKELQo=R{VRMq8i6<)Y@^I1>tMHWRdrCtOVkW0>sgnW@Wz28de zmH#5PxzOIwx_>GF1a4Zydry)f-t)$9Yk-4Hx%&`w+Vpl2LagEh+j|^^@VdbkEHmPG&H;|LbztQGDl@8VysUi8Q=1B;$2lF`eY~%7qFs8WbWiL> z6(=Uw7Z(#JkBu!apDC9~ZbA_QNn(7v!H^l2)Xz^t!{{%9AekK;M~f4VH|cyoT%YQ3 z-|a9|>z2`YAUIl|o|{`|ba1M*b24wXcUpX)Znh|2%Q3{_T#OB=ciaa9b%DxX+EBDe z_qoXOaDIWR*SPylKOXd>q;2LNm}`I2Qc9ScB|rXi`t7_0GJfX!uikUv`hfz-9N99mx(Arxvg(^1`-Tr$ys4zhf|2dWe%z zxUJ!F@THBO=}lfMkq2{@wAS_G(_Q_wuRp8FWUJZ+61$VnKO8o3h(*zdU11SNx?-e| zteWlcLFR{k?E(VOL04)3Osq<}>AH z;BMjtQNZo>*>b8K`t)JN$FKMg{1&j?C9PhzT7 zTwJ!Q^SY^y6FsczP7@nP#z&5wpIw&fbgtC^<;V7Ko;o1oJ8=fD!q@J7YQTfogfZ-g z&^C9bNtxqIcvgI5*k_4ZTp6PIhPL*3k%O#EbZ%bM@|LD`Euwj^o(5)~b-eS|;k{s# zvdKlsxUEz^~5xt;&}7C-MV5N1Z#0*v;n-9t8J_!p$ZaF&4-l z9uy{x%jFg}+W6)N)R<{~hXX0|NyShGRtQ`^mRg^c;On7BxMWw-tI_6%vKuevuK#%$OOZo1$WR(nsoVQR4a*ce_$hWdh~EU8f6?6exRI&-IEBfbt$td#*@S<^ zkMQqLfj%NmClNWkA95CTsG7x;^krb&Ew+kT)T9b~_vlyyQd<9tvAI_4Y3yVEUnyNw zg3jNInyF##)-gbnQE5r2({DIyD%87dst5MxPC`Ce`UM^ur_fd0j@2XCO%JrSi(i zQuTroITnXmIQvk~rj%A}Z_rHwY~j~;tLNQRrbY{bgujJeB-FN%L*pXo^UaF35Sst~ z>$7KER%U{pHK+B1#)`RDHjBZfR$X`T6X$3bY0>yQ-(ag|NnwDK^kQQ%wNxkezWIj* z@;VTK?Xw!O4nB8Vt9Cwrzjb`M&Sgf?ZMC`9lZ1Ov;)={~d)?H<{M~M81S^VhnvSA* z+6Z4(#dWJX$oJje0C7qEX(AO+d@5RjmyoYvw8AHfw0;`;s<`4Y6D)txo{7w2mM81b zB6;5I?)tKf-R8hTX#hxz-1_|V_6O4jRmg2!Rk!#-bEp(cJ-n9zK23bc6Ig??}HJlnH%y+XhOFZw5(fsXuCb6wWIH zK0*i=$S@6O6}(3z)K)~(a>Ej>HDg>o!PprPw`OHjdS;g+Epi8Ml-1o5UGwv22yNtL zg-+L6*|7eB*L~89D!f>Lesh(X)&R@oHFP0kD zpYrKFnXtN=56ZOS@(Av&SbUl7^~n-LvCtMl-Dx+%$j#}MZ;nZSL^o=V4HdHfROqNp zcenwY(a19@=Ry>P6uR_QX?&^W6)Dzk{Ya-)H>Sj;^`rK(-FE2>^k3lzjYwRJ>fI=1 z%fhbDyHR<-t5xx0d7SkJj`j!V{E~4EcxmKTA#Mj&rhPaTD=qp1=m>{r#3yVPcYO5D z+RIUD034;FW|5bScpc4S;%@ z1*ScoJe#vg+ve-nR3alU*7jMUVd+XKc$H~$n{3#&9UhXl*A=dz zG-;r=)=Xx%_pVBc+nutN+#dH8<4|F}A8ZNG`pOvxKdG6OEABGT~5(MPcg2JD&O_7ent- z%TX2*e(RQ-+dWR56N%?vb4iHw*zQvjDvHz1#Sg(Nn6$U_uZ!1 zQr&(XBDu73!gb+{R()&byNYn=)vx;ILG!)*b*t5*4UB;*X0k%R8>At&ezc^G!Bq3z z7iuT*9_l_!MTt{V`Z_Jk?Sb!`^pqfJ_cBxNbB^z25J&0E7KJu!n{PG{GZj2#j?a$O zFZb?EXhVki$y(Gvl$dMu-msn^lP#D@P1(jCk#BIUP?cpfvH)*&SE>mv_`?{UV!kax z`u_Jh$l8Fb2mVvegP9{a{?U6RO}VpYV+9_}$(6=dv|qWge;v(m)?6hs(ks=4t%~!@ zxo@htRUj(xUo*?OZ`im=Z8>h#=yq*A`XkuxKY~pv_1*5X#UIJDL661#7;tLKyEwz1j-a=Kg{1g31G4hRs)NCLSYg``B-l0rjQ3dZ!-(s06 zH{liWIkwa72!egTT92Ht8MH}IJ=b-ynD z{zkF>h+yvCnrYTIn6T;S=L>tU6*l$?_0KB~dCFM{)5M)LUOaXi9N0y+G{bEjNukl- ztV+~Ot4-Unx>0HO>+3dVc1H5`xlU&HoA29-wVIcYnfj(P$E)=1UclHf=u$x>+}0GU z=_hD}qKyI4lpAb%5>|H${Rz+A2v3bix0CdX*X|X#4Hw$i5*N!EzY%+?t*2=<=hiLa zj2O8Sl+Riz&-OP|GF|Y@T`x9^ftB*rNfkDjIfZO$$r5n|_}pLi#W6=#R&wmhO0^@= zNK!7UAq~Xnilx$*-(7eRF5G@7m>$NHmK0eus9x-eZVFXtrpX-Ux3ubRoP^!CsTd`V z(iaIpj>d*EO;XC5#C;hi$VyFge0yo>?^*1aVRu(2PKqx)Skhe$pOCqK!I)^MA3#YK zJ@AZ($(CF#2+znI8eEqSuOYh!EUu(`?K)>4Y}HgX44GDJ+PgI0Ai7^g_r{!L7AJUQ zUgdO*nywLL-3oRSh3Wy_DO?>_R*p#mpk>6-6F*nz(T*LP?$ulUYPVd19=jjCh9z-n z9~uX&;UOP-B{%U|0$eeO#Ru9-e7td6*QyrF^|k_ZlwFJ_55DeKi~LIJM&RBV;WeZH z5{#1t92$~tzN%ij&MDn{hPilp^C%rOVhAPQM)Kor22WrSr4;envwM(^&TZq4G>g;m z&tLP5Zce`o-r{SXNl|D8<^m8u0B=C)TAjP{QOwn&XQ(;Oza)cRz5{orWn~gTTHT@v znqMLBku|br+1@D2AELUszrkl#o)hsY6p`zsSKL^2Vn?Udwn`HeZ5=B!6M|&$&zSq> z94?o~-nDn7&|^iOVgNN@<&}1FPT3PWzzsrrDgnzq6?BrR| z4p#ScJRXVw3|1C{Xmn*<1S-S15_F5Sml@ZEuR`&necgkfz;%awssB8CR z2#Ee4b|3H!p0+Ng=#wioDZ-tcf9&V_u&YvjRZoT>PU4hrOigQQ6;#u~M0T5CEVz2I zsp}@i?KQm@yRyw&eP@evGQr@&!q3|JTa*5rPOLiLyTuZ^|Al~-uF{hj`6@Zf)oVT= zb5jR;hCTf*Oxea|$p_(D?xrm?OM8FCUby(TRaf3lal+|&9e$Q}9Gau8JxA4iE7U1+ z^e|)S-bT07>|d<)J_O^6)!J*V>e39{gwJ}sXB}x< z*P^M)`^0=1nBF$s@0%d6yKU>6f8UQTvE<7G%J!6r7$@o!_R%+!$_g%cY#f1DG!PXE z(>I;Z(`Eomfh%0xeT!u78#^j3d8!ERAHWbJjq^~}EgASJ90k1O13pB#v3zCICp5F~ zttWaUv$h9qYusrTSxJ+Eo7Cl5&XL5#;N&U3<*Zs8N={n+ok%MVr;YfYsk6t~_tGW9 z=5oZy%I(nkSrt+{>9hb+8{MO}r@Wv1%(g~8EM-F}L%Wz=fkVGW(dTQilmy*A^Tzq{ zJ?D9T$E|J;trJ27YbY3Tk2Msh^gLs42%JVvDAh9`QNbQV znpDw#j%0rQ!J)AApREX8=Nf*0l^lJu!DKlc)@zmi`4=Y@)@{yF5LUX~PKiG@L`BC* zC|N)eSpYdqIKz_pd&9f=I9$lA*Zjvbt$>x1W=AP-t(E3;zC1HlR{w_FR8v9B6uRM& zt@(3}rZ+|TiBw~~ZyiI*_7cgDE(09$Jj4a~66wY{E&1r{ecySWoRvi{Hmq-}SOV7m zuIS|e9PMyqnvcdo-K1j4Jv-^qj~c>K=(cMdi=C39(nfW%^RAcjVqqxb33t;w&rtnM z5&GmnPjnu#;6TS$Ml*=^J7e&KTAI($6TN&>>C%m5>el$qH#ByjV_#fQSNW^&BvD#_ zo#qtLYkh7Cet|(Gh;Jr%^SB@p0@&uy1aCMVotfGV9K_}{1kfRW)X9OJY&^DfKDkwXguhz*rb1(qVpg*y}$8(-gH+uwlB=xhjtyE8X`+hwejE@9rTw@7x zV&{LDYNu{pd#8ZT1hXu3;oBI3`5Ta1zAam!iNJ+~5n$|IB7XC%CITom6T$qRyfJ;O z`Eaw6dh9`U$F?ET*^zH92psN*a*xU9`bo0g`u7~z^x8;d(K6_hY935f+RMTE+_#=5 z5!o3$fFC6VF0;IeavV({jRY>MVo43OX-7_yrHCt0#Gk(wjD+7z4aH=BmPGXPOzSFH zmg?~S^7&ptadA+ven4^${_JsHZwZCbP^#xjXcF@!v*DH&(b;$4>Ouv_>IzoD>!$@O zEUgaXR;~C3OeF2ps^FvR$eSs9^_eDpMBHw^#&k=wCgk4U*cqpZZrpif*l_wl#_db( zGyx<|)A``)1!tt|^F87AKr+s@PzFUIo1xl|nwK~L%H?KwT{4@TgVnHn5M${q7z&|3*_|IRI6DxW z?cZXp<}`YSLCbkVkdwcO74hx20#U`IgY$n8Um|DNsMRMdAE#96HCvV7Xj*mo5IbBI8MPDN?dgJPQ{?TIVMILHOWl_ta{1h zpeI?IS%Ig+LEy3sbL(!04TRa7_h*3xDxaK#LytD{!&T2V%#25OwNK8O`jt3t^%J#) zqxhUGA8${NqvX^dczkz{15Eo}VH_nPhGv)9Es}555`w@AxiL1X$O#iIVX+v|Z?Wbu z{qgb(H2=CeFzmu){4?=0PL~(L;BXi>p_3wOD052l%l@;PFA2Dm4e8{$8^{GbCIcOd zE~KL~C0_048m|KP=*qM9_j?Il&LOf}Y(qBVTIsK{ zi%M_=iT{X%g`~yNacvCf(6-{O8K%V$e)9|CK5V+lFOVxTx2X_3Z;Q!(nXopP!b23> z^|-x0!<>`cYyN7Uc20i%W#Q!wAs%-^;nJc1z zP@bY2l4?=^Co;z=)dx3GwW3FQ(TnbfQp$jMQ#cPjZTjHHYHRtqeLC*GtH*Gyt`oL4 zVhM{$zsY9gTH3}OEHk~-fSq13#?|M%RDV?*YGBo9nJ>J|u^B#YXyu=8XS{~V!H<}=8bButH=t8f z9kYZrPLD#WXA2Gjj7mijiJL5~fcb3bg-a*+;HLgU#e-O6Z23%(*00ujYOy(XP(026 z69XF}d`fo8*4HnTNN{xaun^23VHqj+gCNatel8^h zQ(fXhiVCyAD&K>UHk>Hr2snkXJ2nZ~Y!N08z}&Jl6k*ThQuYi8WvzXjQRqRAv0oT) zbuJL*DNID~gNRFln$r#U7%>XrHLtgBl7H;YM(-*)Pb|Im3t+W+_V)Sfcvd`GD0p?_ zojW5Krm_ina~Yykc9%77h&9@xEep$&-{^QVx#z?MLN<1qbLN;CqQAr7nm4@z79+q^PgY&+lhrrF)-M#b0J$rffL0pfdx62 z5*JBK1dq( zk5ZuC^&!-bDr!G(_6a~YncfYJ^Jp4WT#aUcjjlwfVBCg)3uM}jl7lZrD7s5mZtvZW zo_RRQMv(d<{+;Y%XZf-?fH8EPSuRqa{aJXYEeSQ*Me(+O=|rI}&zayc%a zNk`Pel8_`3A}0LOvvjv|9LTYTrT{1KueRM$m-33)nWCG7$i?gEBOI+AT26RIvG6$^ znQ&*{hhfyui(Dd{W5Mc4Y6W~okD6HY7W!B9q$P2q)ix-9fKRFmP+8ml#CKqEJ4VnS`-K=~`P=@Ijg$_UEM(EFME zahpl$b(&c%FrrSQbZ581kl=+ay(<1t~cRq(cGpl9eWOFPa7%d(4Yh*jv`d(P$9 zdojF^k(Xrm$hePu5O?*vPIpoiG$gdT&1mvF9>S%$UcJ7ktqsLoUJ%}U#bI+iR6$iL0J%LqBt!sW# znOmPSubP6el-@A_4uM{shG*;|JmN3UlVv&UsRa_Rz}`T{FL9`WcQMeA@zLk4cudDY zc!2ugn;)8-BL7Ve!kRhwd9wk6B6)e-A2R`9?HA0Zmof&_fM`d;+T{{;auoC! zwbFzA=cnC@kSob*QlJxKTUzt<&9!ZW6)Ef0u+L5gs-AWqSq-_|wjO+_!%PjYhhPRZ zFeemz>{*z7L?wB}zat!?ALmb{k5kP2*?M1lhN3!7imh`^qUgu?{El!SV*!mK#$w3; z!3@-LcA&}BqmZAbznfH?(psec6IocRhC!Vy6g}<2!L&rqf13RC|4WlRfR8vvH!VH2 zJQ(TVnzlQ2I%C+dnMj(t+Bj6kn2|V~akG&@xXyL2>ShA?GBG(`F1=QFWoJKP2t7yL z2BjiJhL}vfcxu@MJkDM3^W#1w*h^~1@QmI6Onn4>=}ikdyRgzG5Pgj(^5+3D@Q_7A zkGFdh0Rh<`Cli6Z$(!6TUni~=<(SgKQ&RkXo^&J1+wxPku8ra{w65W9t#PzZr)q(Y zK~q>u=Aub4qa*q8C_&y2PY0V%9BHHk`0|&cBtXpa@(fzP%0=*=17U)m=F=^ANCquM$*zWNr7X?OPY)fC92=wfx`|mf@cu;-Nli(-(5^X zO$V%=TzZA$OEu1aUgb<1_2c)e$fSS2Dv@Yt;|6XC?A-_&Lc8(O2GPOylr%zm*R99J z{tkU5>UCeFI1ov4Wu$?*!>4H)Ix}kPfYRU^K;Da{&=e1nme-(pcpzn8rAUeRZ*QLX zI8#O)|7l75cT3eoQeYY#G(jDC(qsJCgCL_(FKSm&h| zK>s@t6!13U6zg47s5CIK$c)e-0Ws)svs>B8bvAeX>!v8#?{P_?ha%O4hqF#nD;uZ) z%KsR3D#23Q7qu$s+B=>7EW9H|?=(*1oO2Lr{QiL8ms?IYqQaaruLgRIEve$SABr3Q zX=&t%MD3BpThyF_l6vw%vV6aEgs#uB)da>1Uo0h2bI#HNS6W*Ao}SJruwV9C0KyaA&FOsB&EK`$dt2fX5jjoaCMX z<5ILaanP%uDF-%JCb^js%XW}Pa_2RoT_J0bo%Y7v{r7jiy7bs|2n&{VO*-0K!nh9t9-_oAWo!i)(UV~~B4`PR# z*kqq&$t+sj(wHEP1`NOYtUUjUUVe)Nk}*Y?sHw@Xhpyny@R|2hAzBI`Y30|y>i;$o zJ-^oCM5zQWkfncYhrshTwTkWEaGt*LK=F0bFY*y_*h za>dGnJuXkrYrcu;+k52`@m4mG1H^Gp`9}$`K`?zZi$pCZV{~d(dC2d58T2~j&wQff z7#3YG9eABV7sia((}l(2o~m8l_^BBZX)hJ1fz#8#Wm1%uc{|KAFu<*#(BlH=S6n@gn~$DvH++yTOJP7PCI`>n&Kcy$v9^$|nWxaDdVG=`@u8 z2g0C!ulvG*?QYJGudTWAy2d{oEUJKs?3mT7eSYzKlX)HWCI6@Y81M{S0P2(Ih;MK1rs6}2MXv?zgD4zxP3is- z#k&VlL?9}>t>b7yvk2rOe_IkuTCY8;=(&Kqh3b7z|98Z|xL+5d0B`H)ea#DKsVh|; z+T}gQwzxwUMWnyY{nkzLQQm+=WF8b#<-i~1^3VT3L~0nnKM@1d!*Ke;{xfS2ADks# z`}PXMfca>J2(a^jG(k{u?>&b1x43X5l0L5aA^(W&G2#%CHKR9q%;_yNAUyqi6oqfT z+8CcCQP8X5oktP+BSB9S88Xg6m7qg>?_4-dv?U*U^E5c1P@)ms1=HxQc}5zXJ)E`r zQndgm9Rz-breSGyiP`cA@}dv!;%Un?$kWM;!usA4GzoVxkaZIm{s6CSF7*EIpZg|QU= z7n+^i{IL+IqHV6%hN9aqNkJQQ4@Zx>v{2XuM5N>{J1?biAVWz!HGaO^!l0~|^C!gK zjhM%%P9}CBQaDA#mRAa(7I@GsXzZazf(a-*v(x}J=LU2)VDCi^svN{Tzz2DO@{pm= zyw5=Ck!|TI8uYOALF$PYtsO(Qi75kq`@m90`BIMkdvyiz_-Ol-wRt2-w)OrPz|9^Z zNH`cwfRl?weys(lfzy&E zHx+W05DNz!B^t27r7Pt*>Z=q2d-kPreWboT6yb~bN*S>VWRy6Bw_j7kNZ{=pd5}ga zLFSc7xT1J`LA)Ss%z@X|IbS>}%GtS%m3^PD6N2WR1X=PUL4{ve@?$QQe;RO^=L8iG zQp1K2!Tiujcs(dIA+S0D4*aME-{;s7E9kLfG9FN5uD{*3^ho}&Y(#+qi7ug;6ckd_ z3o=9&{gDG&66%M)^v{dDqyp76zX!x&0Q1JiVYJqQdIyX`*DKT868dhKSQ0 z@}*-7sF>;7mb-PheNp%(=SM9{Az&Ue#5(og+nRy}6}aN0umyEu;OjeRug7vr{jnTq zc_Ax$VHgjB^@!Ha#BVi72W}1?>JkNAxlSxlHV(X$#tPG|SirSVas zPJq@4foWF|*+g-=RR&)w{iFp+A?WA)lndkHCR4+H%4$sv-;G;{(BEmJ#4V`I#x1zg zM`1QVV0rUKx7Sv_N07P|kIn*!!)cJdY3&d@HI2e|#AAYMKv?Q(d0GA(l>Ddzr}IgB z8J?W^_k%}Nte^$r5spXG|eY}aV^eLS*aC};o9rxz`*vG)m{1-bs z=>#K4=#in<)25B`x-_32e2bT+of7yTB@IBk{6e7D*sT)E)+=EsiHHP`QL=U4WQMT6 z!819b91A@CBv25$dir=>LE@-cEnvXCC&o9pLd?t;C`Bh{%?6aIXMOaYpk)${t(yi8 z&lxW|Ol-vQsO2%KANshR4VBKC=d$ni zho)S6wxi~TBO8dNHw)T_uexVz_xii_%ed1OkJjKxQj|C!Y`WZ43>if`P5bBGMIdhM zK?#!X$0}k=xH906#9rSYC9ai4kZdxvttt2g83S=8jmFol9Wp+Yy^^Wn&Mzonp71Sd zBmp|pHi+UDPs4SyK3nUlCZr*8G0Q*QL-DWS;cFETp75kc!^gZUI_;i_;h(vtqln^W zY8Qb!wSrCr>iJTJ^dp)C>gqdsSKjME-|0+ydoMxICehhLIwHW_WyN!g?L+=;^nk%7 z_|}8bf4gNyWJwrXM7JF?=KdQFl0mjNG;`@QN~Ql)JKVAoBZc~QU~xxzWYUz zO5~8!O)G$qnE4P_`v=fu5rNu`y?st;@fR!xi2kd{O*+E})`%@Tx(ivFGK7{lZPz?9 zpK6v6nXqIsOFtQZu9?Q6hH-Cv=FWPpw{o3)U7$)3p~Kxeu!VW@DT)}I9;kUomwEsP z7J96BpvxJ>4A7x`RYbSjXg(1%fW z8fY<9%Yl)X(6Cc=*UI zi`0-D4%Y0R2=bH?GjV87g;!EXV$%+-fPcu*i&O@jZ6f+3->31&A(r9xH&91n5$(O) z9mEV|$JMkw6a8ej5T`ah`$$bqZUTpMll9vS6RBEBuA?hJto{br4!O>%-0v zfxKmJZrl&!Oo|>)mr?gy0^p&>bvlTGsV04<@5o%Nb7;AbqCx=8LsH8hr5BkZ$8*w2 zPxf&&_Kl|PBatK^L62G3<&O^RNPhi=#wF?e^~w8y_Xl!hQ&BEe!Kdk)*E}aN{Agq$ ztSm{Zd<^wAqa{rsP?=IOCM}Yp)0qVA>HM210RWvR1JDwz-k65$1bP7?^ahCvm-WSA@@WSzCU6-dLI$#fdYU!`zw^dw`ch3Sjl$097pq&KCk zuCM+MCz>re+nc%xp{)twVpZ>kuUj<){T#2sVYIh}PI}QVs0{#RM$?$j`@wbX##l(m zAMq4MS_(0;I8z-ygH|Xqf+>UgxAH21n75cRcfCCGAy&zHvf7Zcwc+N;Ycf3DUGu4T zY(G2Sa~dVkFK{d9i`Hz3pfNgz%n^&Sb=Za>wj@2J4QRULVo8%i_peXlGV*>ZU(E{+ zNZ?VxD;9iQ%`VxMM;zz6n5ss)X@41#XebS=CNh~x zp-{M-o55XW=`}+HYcXcw@lps08D*;(&vL&XiE&2k@Psm4Q~l-Qga|lG(Fp{#4_+mb z2|7+Yuy1%{(_sTITe3RUp8Og>FF4iD7uGL6WJ6m!-l;|~ybTwMQ2;yav%w49-7p=m z4Taw#V?sBmWj4q&UbI%a_KGNe?XY38UhBb5Ra<-Xcx(DuQL^2OZ7*BE+DFUbyPt&P z($*o{QzsKJ&fd@}vL46JJ~Ss6(-)>gcDxN-NfWL24JMc-V&mw?h{R*Y!=KCOc*0K1 za86(YBzjJ1;&iJKipAFCcJn60__;bhnltN2!dlQT@-WC+8$c69Jx4m4ve}PbQP;go zyR^O}pM0W#Cl{m^C}=&;mY7$Dpg%Q@wra;td18<&=F|Bh_$AuQA!?gTVx*@ZpOf1T zwfPqepZXlz=+lZ~@IABq_2mhAcVKtE5e@8u;>RnVvek8gISEPKt|z?hm%F<)Z1I6g zQEcaq(ddze8F*Woy}-?c^=F?`|5%%P;w{0@6=&)(w>WG+i31}v+9q+AAh>SC)Tf4P zf&H`e5ieRrjye#Cx#9Z9$--D{ZrT-`c(!|O5W2Zi=?&HBU7h>Lfc>Naf1g4bAjiC8 zjl6%oAxXGz@J$f8t#PnYYNYchD)#7+C=s0!&-YD#aF#HM5TO3u@gijDfvX;Mm%GRh z=;mV@gaa|NtiR5{z~#iYYdt=*JJsz%vkRTuxol8t^K#&<;G|fVZ8lS>lhreAts3~j z*|T<)osE?r^1mAIdw)AlnB;=KBs?-L2?t(X$yh?dvE8mdrww0u5pZxIx-z}1k5RC+ zm+qd??)dW#0XIs1LV4xxNSic#WVG4tA^Ewfy(8QDv9z6^!J!k3zIf`Z>Z0%`tq}HN z0nDMm^P?|>0aJd>?H$6{FrDBQ+HT9Frp4FK-(tUI>vktP&e_smF-L9Q>~Px<%abi@ zkr#Xk4eWIo#PN5vxa^5vXEaOGagk z_Z70%_wta7NpjQ6FJAO7qjDGk>U>0}Dx4ORc*z&RzeF+T#_{8tXUqv!6PS!m)?`(5 z09GrIg%RQ|j#eX!AYmDQT3WgIsE;6Ks!AY}Z<~SJO*-L-h(-}n;#`BMQiJG)5aO0s z%j^@|pefLHUN?I5s9*YXZ^srvBDOr(2FsaSFroS-YcD$2-4YsX?Fw_W9N$c?T8^&5 zX}c%dzCzq&#$V9b-pa!K)&|Hdb^Ue1+8L4>dUH7pmOAzw z999@3G<8D+0jQq`XEyAMmJLr_bdXWrH;A9l`nWj@kj(0H!W1} zcT3jmVH|g*r*qUDPOuj~*19$!QcKt5-#!D;{<4es16|D|uxhnj$l-*PTGa&9t~J#sOX*ibqGrB;)X9aK=*L zFLnBjX%7hspB60M7xj&_OLoa5M>hG+Zo$!hoN)(Xf-K&9K@$Hqvu zZw;SRd3{jKObuIU?Pc`)mxucL2X>HKtK!c7D|)reykIaAT}Ytx;+-+&yVl+-Oof+X zu!pezSn(_YpVJYC*{8dCE?tRcN5fvg^naa4Gv8i>_CKEV1BsyV(APO-u=YMMP|AH> zJ^OHEtyCvVpQv0!!{QQz`XgV-_5Td*;e$`jbBflrSSAt_LXAMdDduko^^O%|4*y~P z9zFon8o-U(R{iEaVr4)4L!17=%yLSK);RpE8YankIqhe`Xr!^Wg)} z5D2C+0M1b08Ylg6YyTMw7AiLmgltB7ym0qWX7~}e><#|0UlA zY+#TP1|6xD)WiSZ3V(u(HbdfnGcDp2mPn)V@x*#NShJ;OJ1^IIWBK}@-Cs^xt_j_X zxu4xZG;LgZ!*=ecZ&`<4*dpBBnD88@xwv*MVv+KOX&NbYaFI9r%F_%Fzx1jmbS|Tg zG_V{gzGoAyy^i=rcf*0O~0Va6Aq5<3@!<@?gxO%dVMd~mC zdSyTOSW|6dWfgIhT!^Jx?J|N8qEnI)D8<5(Zg6MGYWeb{1u zB2&1niZ;NYO}qij@qe$qw@-qkTE~FAx0{QTlAc&Tt}}9O@1)p>@O=N{FQl#Z8$`J) z!IrGhZYwWnThD%vMkyN3J8)x#Ewd{INu-ihc=auq?}_QRld+Hgwb-K=5rN}%Hq&a$ z@>OiQAGp?fq7kYmtpdHiHHMDL^3i0<0x{p_`$NntjP#&d^n~Tx>{y$HS|HQ50nntY zIJp-45fE}S>1QA#}G zOA7t>N>q>UTMIK&SF4WZ_YR40iiJJ3^y*)rj|vMB6+&~)p;xXe)l+MNIeWal<<|{ zQd8OtmrB=)N`RlQ?)TEf!Nu!yu-uR=~g0z?%(*Va6Ylbie3Nia3J&M zvOF7C9=tc7Xqf+ywm8t7vCoNyo7)hu*rY+(8pp`2e2VuER6lIs_}9QqzXJFBo!cMt zbvvTyQMdXkXw{>vc`_i0=Mky_;eKi*)Y`+@`N@D4vZh2q3AFB-a zvVYLq-nt1NE!Kj9hJSBH5efzr>INvl*bIEp0^X=mOjS=72W@%9v&{Oh|dmie_Jr~m!67l6t#2<}Ni2UH>|tyL5g z;eR2HwKTvfbG~&Wk^oPyeku6xPNky&>TV6=|4v0mmp??G31_@TK_F1oLoUGona_R* ztwR2n{>gZFInvdy$Nz^3||Q*}bce5A7*3lv-o zR_L{B*FWH31AiI236H*Z?ZY)`iDxR#TI)%-%E_8*YDw7BNy+pouan7by=SM8&v734{x9(nz%z{rXK-=NN*j}@%t~vN1*->6P;?R=oW~i#8{EYSBo?Zx zQqxMA3Ud<4qvnJcsg|+w;q4|ns(&O9zeqygEWGSQ&!W$`ub0lUvK)?i)7v}Aky3$V zPbsp`PYShF1!E-Hs@a?wnKGKS{|L?)%oXYt(oSoXUn_<9!np41*@;m)*yGWquG8)% z4Y7o`GeqiLvYN#0lZSc}|CYBsvmv$DGSy=aap}6CcGTqz1s62BL|3}lc}7I;Ha(@X zC?l_3bcu{gva346MOM8k?W;_Dk05ppcPOtb>U~b8t9v!kWQ2EOMH;=VdTQEAwVnt$ zBkPGklOQ%6EVb*4mZ$7mq9FRcTRylH|<(IW1V`)JO#`TRTZI z0Ct%YQ+VB5V*(Ojz+)uHXIrUD(pz3q+9G*( ztV{`fTvVb#rjp&1@{K)IGN;c@MU@Rt#x$__6;(V%8!*70jZ=L?v0S*6%%Il=@Dj`5 z_LSyH0$S`YP^+r!XkX85C5QRwmAt-4Md?<~(Ae6PuQLJY1CphK4y6$Y_>gjWh~8tN z+OsEciL5oaZ>Uu>!9@oQ6)CJB*t#5p;EIpot!aPy?I$-M7X{+4oxJ%*BvT#k=SA^4 z1I|kFT7JRPWO#X>+m#_3++l&-9TjVEGovn0(B_gsK@@)U>tx~>k*It#$%1{ZMB0%^ zED*K|Cw`k$(CH!?noTHR#&Yol`{Bzrvcuf5?~3x_IqgM;y@NjRMvH^|Qi`q(iBX1Q z5a*g|qSB34P7TXqKjdsJVEcD|%S}QPtL-r{c7rAX@8j$c8P3#&1}V}Rg0BS#S#4S< zizX>Ed?l*dvz_K^Y~i`hscY~)_oAM*yO;?Zu<;U}<{C*x?8MJYc~d8vfXdGx^g~ zzkP1oj21&3?A`|kmDOZfO)^R&`lmK=;Ld(6FTC{M$4HfJQ^yJ%DJB%L_qBsPiF?@n zc-DecxBq^(&1`qFAE}%Py-gLrDg8b{Z@9fBB3t>Ld~UP8 z$^(WB?;C@(w&Xs>Rkf$s_la8f)f+aY8sAOA+EH(paO5;f?QdzMk-x)g!&zkOyVSO$ z{o47aSBYLe$dq^Y8BFRgHKlv|4j@-cBypQ{O^!>HT=avs6METwBvskrz0h38z-$J6 zITMhwm}YIvm4x206^sa>416+$Qd=SwK!j0!lg0Ou<{w9P`X~F+ zhjz>H?%JhH#J*6(4wwA=c9IipnlOEgeSf|9{q}f|NH04Vs$au;h_k3Kr~aVmOOG3U zUW{KHpkt;b{x#u^@r^z{*ayyL$E8f6Q(QTp1!413=B+1xGD8LSeUJ-hV|`{6e3|1F z6trlVktw9R5;B>T+(J9<2)z(s61}fGk}1IH{|?J!jf8NGSQ5H`=Q6&MZAgu9YB)|Q zVOKFMkEOXw)H?+G$7Y0|PA|KMuU@!|FZQ>!+?pj5dW#r8qZx238`65eZ-Vr|d!HO| zQ}a+gn>I2LRJ4*yO{yj`PB4E!WNt|51k4L-XF#w+xk#kuDifBkEgb29TPo(UkOe#* zQ3nXxzVcw`Yb=w+*S{ls&q6;}C@fVcq;q_DGdmXK6qNH6H>m2Qy_VF5T`YHH&TB}F z^S&`MIe%PUt5ZzZa~x$;t{b0i!Omr-qNTio;;29r{}``Cy=7~~(R}#AQRjtRVG^(K zts-2ygwgQ?zcU(9kb`|a&X^syN8wE944D)t!S9Q5FFQ1-c$Z>_k8q=b>9;7Q-be0y z&#dldzYDG93t>fM3p(#g;dblz#1`Q9R{H19_PcdX@ju)csXQq8R;k&V*I{x=P{<{@=hrMyX@1l60F**&CW|Bd%t*LTt z;8GV1>4QCsB{prHes=ZGGP92ld)AxcZqDPWt0J{0z3K93jetH&JJ5AQb)WpB-Pf1x z8M?1T9z^VB8HGufmN5Yqmg<4ySx%SL_oC+TsVjbc#jD2)${bHpf}~ISi(8SS8c|nI z61gGzF1&vwRqO15u>bVCCy~2Zv5|_+Y3{3P^GX|eGJsBvsXkJ%O0U{06ZF?0@q8e6x19Tc4~+Ux z`{&fH)+ewAS|wS(fp6CBq#paTO;I&jI9;rfP~FgRah+}np1V9g8GGjJ_3&V{V445M z5>IMM-I0p>S(Fju!Sa@<+f%+|`dP2DnPP4ouant>G!e{nvzyPc+UepM`i8^Fj0S6P z9cDU?dJ`U#{P6NHBIe^ZwBKALuDqr0VjfN{B&(X}@h&{$@}#S(-pJtqQ|xqWDcf1@05sp6kXr_=p4Uar67F>;fn-FJb9htAS+ra}eVx0!7&tO(D5%*^bDl2g zYstwkH@8-ZPA=bTpXci4MJMjGo-9g3_CD_-c@D=M*5WJOhq#xT1uZL-wkhG6aqgTt z^UhSO1zw#)NvbJiTO|YK^OvF*#{=`#{DujoyK0)6*6lO(Ck&l-lciL4)0;cGcj39! zB6Gu7bIOQuwt%64-QIB1ufGgof%TV?N$Uwzy&hoi`idXjX&{ZGqXNCyMU*1h39zu` zh$d?YP|M2$m6Vn*AIZ3P6E_p1V=(aw`<7Nkoc*@5S7EXU_V4^&+)gmRqJh zg0jm_d_e<(3jr6iCh1Rxnv-$q9GtpGjNqbPb7AwPq*Bn{i*n&CLadV^wy6LkDjo=C zq)HLnCYXsWPbkTAAG4DzXs$(@--s|0IBF7~Gpcoxmj@lO2sC=`I$pRRG`DUH54^!` zAT)tZStL-CQMiu?AK1$c(j#r#`yqoEo@XMTougsYVEk*;iGVUB%6ZP6rcW)z= zq{kBg9}dQHU+nBj?|jOj3?r|@q$yDt}RGe>L`Gm`wO&|vy zoNdxIcTrfLEU#F-`nch46R)OYC9b{)U|a!EK_p@`QWEwE8DLpa<%+3Y3nvsWOYA|h ziH5%0im?)f~eU*v!tLJ#?s)fiS=n;w5drSju zD6A#_HK)X(s|=}B)AGj5*#>zNOLlchk%TY^qU_Q?JYXf_$q^8o$*32t%>=43?2mUN zJ9=`N=$b0ht25(|9LpDas-+Xx>|n2{kzhy1NqLAo}BT`rLAI%_Y*ZFXZF<>_I~aq$q9^kfDjR@+f`QnUa+F5MKFcH}23YO0EZBIoaFMz`hEmAJzd?PUnP zrZm52SFqOk)R$Z=lF`w2T7oE)?e&FR#}?X_O=!*d$T&iCdrHmh@aRU(P%PmJkW1m! ztT6ZI?g}DqHA-)wDl5K)L;YNaF~>Va5*PZTB~&665)_A$A#V3wA}5hRybkzIdpxsR z&<+5C6(pgF;??0~lQelP)zWx51rFyCW^t z+iNCxbYyo`yDPP1tTMA(g%^g>Mn73p6V>d{HqA#Q$Ym=r ziCg_|$OGjyl3v3BuZ9$ZLbNj+5!Y@ zKsH6}K&gMKPFYbDLhe{rDm3fW0H*LxBMP*o@wza3nhHp=sqyf;YkaIBhI5VDjEfD5 zI)@jRD_WyJdk&x3c2kiY62P^!@gf1{o_+WGD=`XZgD|^a-iOK5v;=u800yp!nfr;bM)+^Z-{V zgs)SqH&X{v?`74bE;ak}eQJGYJR{gh+>la*+(9FZvzolq$Fpw1V*8a}ZHANQ0 zamqL7kXY(U23afwbFod+F8fqgE=lzC!$df`9VxS1GV6ZSLzCVY0;aMmUU3DYMpiu+ zI#MY6hWPoq@DRgN@tg3uSZl5ko{P!X05DZTy?97YxIVo`CI#u3j+>Jvm?ExLOa??t zaVJ(+;6}M9XY&}s=gm9Iws?wAtJ-sz^X0^M`E+x=A~#ou+rvWWd8l=pqNm8){q#Vdo*G30I7N|z23G|k%Wco{(;RoKz-66`ojFVP z2Ky&a%(-*!g0AmJZqZyfdxYNOPBC@bVGjl>vh`T+v`4V(`kG;9l#n0nu0ZA~wjR{) zbxwPRG9ztGZZjdOJabrl~Ya%ix_(c3vDyJ4N0X8 zWg(oV8jzG>A=p+|oBvkAIU0wh?_`26mS;21(gS60Tes-t2<>)zn3u&{rB@|8qZtM2$mlYTpCRpu_3_Cp01R^SYB#{ljCW#J#frNa)(D&E&0 zi>DZC^*`e2TZ)&O9VO-&^xwg3?q>?Mi5*H=TN3Krcj>>8@My#g9FtqMI~NZcFjN`Q z>RS<8dNJ6-2vN@<<70Q~BjXAMu_T1Iwel`{TMrIZ>}1)=b(JV}K?la%ZC^KuaT>B)?`owfzJGX{jeon}aaw zQ8T$TQTdoN7FxGMaIq{YXR&WcU&=%A67`>9eJ=Z#=-Ng^|qjRQYDRhDxKxv47>`j51Qu%yx42)rNn-XF&5N}vVt zAaalJJ}J>I!bLqcX4?577!ye08~D<7&YCt{x>OsfNp&p0p-t9YIW9V3uq31(6j+pO z+2qhWy(G+$(Ns+704}R$+)@-rUkc^xQ@hdem1V|q23${r4XXt9+~n!qnTw0pe^W%|-^6W{<85Y@$eqJ8ZC$RLxVZ0LGV9DtYdkAdUU?WII#1=if zBNlnnk62s})%J81a#@wI{660g%`K@L;=Dm7)gJMr5d#ZyWr|k>L7=D1in0IbZp|U{ z|B7QTFl@G&v`HJcoYu&9Rz=G-;GQ&~DuY=%TZ}g&!tPD)0-QlqBq?;W^ZITZh4;Ku z!v#fFMv3PYyZk$eHA7TzQ>j9XoqK3EcqVIIFIV)L$b{F_@T?{gbFp0MOaRr@L)2{p zy=y$b?YU_K&6{+e{Z6MaSUk0U!4U%cScw|Wi?CR8GcB>9OnJI?dRF`g|CJFcSq=LJ zhJ_{X#!B8}Z;m?sVT`{J&izVXSMq1BtGDsnGg_MiXquX63q(q^Iy1hDvB~Mfn!Qtd z=Y85U`~2aMv}jeAY+iNaDKk-W8@6e|ML!*AfyiYOwdGZjyg@B`sT`mf$S79i<8#Sp>A1H1twY@`aK#9ZZt7XOX)$ZRa7flZ)tAPL7$q5= z@Abrca#{_fZDk=zP~aP~p0}10KfAMEII_GWTRE^#VgC=PGn0_(wn0ylfIrK%$A+-SphEO+=FqZLF*_Wy)du+$EIV&S5!eD)kR zIgI_d=JnIvKs=S$><#~;PwbQ!KS+&|_7YJGgVlV_gLvnita|$%n$=+U)N?S0vGS{a zJt+Uh_P<^0RYvgtBzmO3-CfsdeXKx8Fm|&Yzx;4ryQyTc_U&CYm4Cfm{+nPp*YG}a z#6d$@y=LUcOoS?>lq1RdMmm#|wwu=ZP6xq_8MWE4n`zBOXhd*JdbJ1w4#N0 znhCbdN;y-GwYF;dqN9xX~lQGC3b0F50>JMy1^q>Qa~PB?^A^J$dBu{pbk6QZyZ) z&-jWLS6o{2i_RMA*MW_PBWkU1T;4T=#@)I}0dgk^^aALNn_B}zrUoKCE6`I|e#H#I zgu96@^Tf9^tLn#rV=;)ESs8B@`hX;_F|vP)Yh|){qVJRoeVKo=~CnB!9iVZHvpdPn(za ze=%mt&xOz-MqJ^wR@59ajZ5)J2ACx8HuVuu0}j8&-i`9Ngc&93oXTOfNsX!Jgyi3z z!*Cpt7dHcRNZ)H9OZtWFbw!Lzx?@S-LY&FTL#vmlL{zIg?+4o_c|&aK8rV6pJ`B~J zFP}KaSnw!dE;lo*7|@RlWO8uRfQmRw9zYs0J>uq~^8tD11y#%QY2Lolb2_61BDQu5 z$3s*?MA`s_(`G}7=_HnKFuC#0Ve+;#!Jf|9rV1DL%8;l#hsfHoPq^fhMB64$X>!W* z&O*kmN~TGprMz&$m&vybmwx_C?LmuA2co$z_SSZ>P$_j=mCnrz(J&GU?$q9LqmpHS ztdL4pT=atWRWMS<$NqzoM|k7Ib?H&h#N>XM_xz*LcWearj^=3O?xuLV>-t8byOzIQ z=JM@YXXLfqTq=9@NBd(#x7<=ws){yo=$Kr&;OoOp}g%2N4Gx8B{f1y4Y&8dPDWC1T}|M7G);x3l%6v zW!$(~2_NPTu}S-p6=|$%5r9;iT!day(sr1E?Gp0q4DS^%9t?{0@}>4vj9{C+hJ9m$ zdI#1ur6R-89me!Bv(48op0vJth zvuY-_-K-tH)tusaZmLQug}dkNW=`V8xl#|<>d}xUql`b{lsK*{FKH|CaEhgk5p#q` zhcv$87#FzBF4sl+^00fGq?nHsH=FH0^b|@NfsgqSXt!uQu&y%08n1vFGf91`@|8%;k+0Z4(DSJO>6{1 z1Zz)Y7iOCug4IZqkdK~`yG;vX+ZIriWi}bL_u=*_l86EXP}>?b&KZZpdHUVIaVBz2@vY z9Ae9abzVLfPD5#y-6=L|c*ppxi#Nb&%CflK z%&a87z7aralTD{NNP=OOlRALmAZCKTJnqCH@_r<|vh5K*H-Eimnju}~WXs!CkD3FK z`;;rHZvwzW*=V)9!eH)JC@uwO)HA%fE`noD%TO?c(%Mg82kt6{|@=Dcvs>*MH zVS)g60A0OV2^1`l^Mp)1i2}6({CNpOa@?~)Zs5q0O|dPDyf`3w+DU+K$^=Or?c4@Q zIZdcVd%ItbRGCfMMH~zU5DeFts{rg+ta$pwE~b`U7k0<<Gwd+Z{h<~lOaZi(xcd}H zOVf0gQC;DrQt9R#6OlsulCL$nb*U>@oib2wg3U_qOk&knczhKs5Gm8O2DluGn>@M_ za|7@xz)ha`Y#JTn(H(d#ht{_pr5wY$&rbSTuzOswl&J=*oSswA)9Tye9nwCtC4TKU2E;XJxxX1i?!wx_a4Cg)YkrauET=NR7c}1! z%kZ;TCuc7+s3j{wR=GzcRN{+UV)#aV9nc=-PdeXS!36Q3a?eOOo z4{SPJiEz74sYftMzb9GjTmHk>S#gkugL5Ut2lQos=yW@7&(U*X6sRoJVl`9TF0PtBMYU+rx~M;`#_=al zc6*^&r2OCUgN2kbI55BKTK%aPo7bx`#|yVUhLCE{BdhN2MZS8vmlg5v9m)JRK~NgO zyQ6x7onogu=_SDD%my6{r;51}T3*n+S(dAG(P%V4p*i-f-|SZf`t;N91}V_r6b=^3 zu6cbMK*Ag1K#^rySbrB9aA`nI))z_XFf$Unxd$10@qhUG)As5-JxMnzR|U@pI&~ey0$9KAGv$>YiO}V*e+(@P};^1qb*G$ z=`m&PA;gq%WOpe>wXH(zsi5ttXPIkfF3VWWHq4uyUK;5nXU`hTSeJZqsw(Y2eEUhv zSJb^0B%2E}Q^sDD*XL_k$nCx3(`V9lqqjXe{gwE;hR(70$4R2*tio!wxIN&GQjLbT z4c+pMV1t89cZ;X|I`-7t$tXdcUlwlB)=BR{`kL&8cBH+AG&*{}PT^SE@i|ow6Bd74 zPY(UX1plJvVxdAgZrUhxItJR~qBR2d9o= zueQ@)&EB*sb= zHAF+0azuZjmaRnO>|voo*)}5t#b;$R2_Spo541%*$)gZ`pSfDnby|5t%_8@uRBEEF zkh`V&a5`pC)KE;kH2r0rYVWy1A~RN-y2`GvP%R@9!uQ90rHY=*p-l3WUFW(q&Az!* zkL5NoR`S;fs#Rb%Rthu4#^hCoiKD#ql)fb~qTG#&+IN+{B#N5b)f9ZnL?%kxA-4J8 zwImV3uE7IfkIVV9DVGE-t+n}U($J~9PMA6wSKy9q-MV|>TCnV?M}{x0YdsiYK$_pm zT4+;O00t`cCsG$~PAU6r9r!36c($K$XvR3iwsmkdEBI(XO>|vhH9ROve zYG)MiPx0)(XBkQ9v))bO8_a~dech38n+8h>FhbK)i9T#a!o_+>#p1}+u9pgdj{?yD}g=2`+I%YQmOq zKXN=SGy7hoRl;augx}8OSenk7#DG$)E0XS|K2Z`8Q(*^8zA+3aHD^YMwL51=l`3@P zwBobeIpKHF89K4m2dPD*!~EIt1_>fdU@545-XmBcWkkz$Cg^{ad2>+YXO;A@)gs?0PQlx!WMMzy3ge}H<@w(FnxB&- zDaaFB+8Vh-Q=kJAGH6))%4i`~`p%3DW8xO~kY89TZ2)_%WJkJ9OknoZ1L#PF$gFm( z-=j%1Ai08&_8pOgzU z9XGBc$%Ie&bXBQIOn0{vr}6DlEU8q2tsmc}Zr_2i3MY-MmfGC*9Y)G^&E9k+hmv0I zP`S0@OI`&8*%=>d_{C9d=6GgzeC=a(MM5kSgUt{z13=W)yQc(78=0gn3o$Dkop*kW#fHR?`RzL|Ji3?3 zn{K~T!Gkr)ftF6S$-?BfEKCRq&Wp|zFsc>w7A+Fbir3MIO9y`)gY2`0c&YG6@?@V> zWjZo;SoK+hzaF$|V71vzf!fZcRYba7pZEF}nt(X5T$E4amG0LSE`v?e8F$Sk-7HQb z{LQE}PiS)h9wzAA8GLTeGIxqq&Yr!|=VBZZ8;vw=lKT{sx-0cXUh|WZ5;vwzp?oJe)7|{l>FL6L zseFNhO~&{56IN5|HnUmb`A;O=p)hjJS<@K?9?GgmOrqMoKz!qj@YYLY{2l(kEp~UN zi2mb!qEauJl8b#ux^!=&CHym+OeF)8XjONE`HgUSOXg0^gvb|@thth3XK=VAX~*+P zi}|03L*A@~8BxQ9k=w;EhlWnE$>iD=G4;mDVR~V|mKOu48=sS*Zors3_vm9fyjL2G zChS4>hALJ3{u@k*|kG7CU2FnPTr`5NY+*mbBF$@ z*-6VZkKQN`Tr0&dNybfc(9&F@`c+98lV~B@@g71+&$02=9$0`A@{09Jng(A?D8bda z;@_JUfH;cT)7wa4`y2}}B~IDmOz#_JZ%^SyP*w9JePLJ5(9M1g7{{;6_P#G+_nL^0 zGwOZc|EWies}{5T51XyP?2|>{T@Gt~m2NLBuGN(WsK9q942`gH7)mI77Hmuq1d^&H zd5Btd0~i58kBfxN@7)nCLn|DEgbi9NW=)kwK3TH`(1p`7N3bFGZilIQH+h0{#&Sfo`xfq54L1TRF~P+PNNP>X#v(tF9 zR2>2*AiByr1@EU?_0hv{m8oxFWWk|6^HkGj z?ab8#6i*x__$FYhJAl3T_6t0^1XV(WNhT|>;$nC1IvU=1Y$uqSON-(ee`bF;Z2ZJ} z>)Tmdx5BUSX^F9Vh&&gEnyy5!gNR=q>GSo&z@D@(M(Hbwru{N=%(5V9Bq5fx&+wq+ z7pv4&{M=YBvfHi5RtACP`+|?!c|xaov^;F!v4(ohnXXB-7*Ld?;iC?*qPD<1w_@y_ za3z<%dkyS;k#}1fyPMw!9oBP)9{7&vc@%{iP;-z)BwB03vMc&x)&`W*3yQyk+_7$x zrRf9^?g#AH3NYv`6-EY!%8?=n@&pR zIfx)w7Ii1ZXX3Nw@cRI>mA1|z(kS@PO4@)(9}mie(~g>-4I%Q4@4sYLdkz?~QE%FH zc3SVn?9w7O&J@Yhs~e6XFwQHU)OYzYyNIVA^Ifp=i&;`gnmKK9+wC<(5x6t&{TtEG z_;zR4<)4d>^PZ&&elGsO&2;d?nRB9)qCSylSF`0@9^YL3B2LTM>b-!CqJj6wVM@-4 zm`bhRT#faWW`M`vvuIjX8mTUfu6V%C9o-{qcwfKk-CxkK2RbZZKE~l7Q{<(haK?Q3 z;m>~r9QruGB2q_7;mSdQN+%pN!oqBSy*BsfvA>@E|K5jjsAv?i>?@*F{0BNRL#3%h zSKK>r{Krz{mltQ@{box3=Q2^^0B0Gre}4DRF(Hyax1-ab5)`{nUjx(M9#4Ar*u<{x z3}6%TXN?-+U_yu10u^Aw`tYk1y6)O_vWiwZ;gYgQD|yF2#7eRuxRG@Kgm_iFXtM>I{TR%EdDOZoe0kAAPZ*bg9m=iE$n8U0W& z-^Bi9fW#d96qD-Zb@WWg7UL3|9SM4N=^P!cuB-Ag?5C?9s%gsoQ2$;XS>)m~!jE=q zE9YGJff*^D?}bdgUFKHO$v`Qdv&Ym5!ePl?*D0Xeb9Dc{CB^e?wLjPYry%~hBD=bL zYD{uDD<#ibsB^K6S<9`BGEHkCSx_r2!Q|K7(}KE0;=1Ewhn>{BhyW5bW6ZIr*X0{) z1uv365?NeQ+T0gk0TQHq7}xu};XuZzbIk+G%@ z=2GilQ?wPAlVeR!nXBI3o1A*_$JbrrosP+O9*|IK5<%m-xR<_BTP6c8{sEu1+(bH$ zWMP&>CE|vO)bEx5J@<$IE|-6?V(3RdCtl;`g_ul^k5+Z|X0IMiy6z0lABcJ$LC+5| z6jk<@thA*Y)<|1Dj)pn4EaP5$+{Uo(cKwPhi?^{{7Mf9mz8uem#=fb2{HGYAzPcrQ z3QyXgFP;`0@z2LP2X8p#8J|rY1*2=-b54R_3mwXky1JLw|V1>9F|a9@&^5J z=c4_7jTDB36Abz?v07v`($1{iB|-BrNU>tN{dj}TlPXa|F=oo%s(L`dt!ZqdQqjHd z0(}Ej_4>*iuN-36{%21{)6n>i_3mX2x>ij&<>zrD0zu`zQ!`ynlS=t2BiU^$c>3+% zJMNa89XHO7QF%N$N!cpb?!HB3XL{bWMs;gxfat)$U$_OOvH4BIedoXl`_DH2W=31O zsMjJ+(T-y_5p>;2yvdqGp4-P85!9u8!7aPvaEsv zD*@A&x68(<6a_ba#48|6ho< zybhgYt-O1gMWPjTuJx95%WFPxCw5?PvGB%Dsgp1`G2v@fiaU3O>d!3a?gzY0d+>KJ zDPUQTN&fsEUA?DvKB;&adw$tsR2P!sc~%^xby|=1Ieu|)no$>&;<2U?8&44oq5h{sYCQdAcxFt3^oyh*j!aHze=gnM9m86=CUd{K@%er_UTp>*6ur zApB1vfM5Nh24A3v0Us%Hm+iZL^*~6~ttBIqPaHYT{wdCH77n`IsEDn;tqcPBh1g|- zLQiBij(-w}>g= zTbOx*dJb(jN@t1c4OTZ*Ep5)7d^$W)t$&)Z#I2tPQ*J^hDM|+24le|C&IuWts66*H ztu3S3zu434Tx3{w@*iy<`<*4WlW`FVOBD_t5| zd9IhvjCE3P=ER_^OG3^sgjw2}!tGx9XB;l%+We`8&1lfDHg&Q7$$9XL&zzg3Fk3(Z zBT$HZZp5)vR|AIqdk7hmh|qOEGUvVIaJH=ahIX4uR3{O37kU#+yIsFBM(Zm=tU%WC z(?RF-XFg2I(OG0L$m!0{a+Wd#?}TB-{3Vt9*^zT;Az4$^hLiDsFwYl2M@;*Yh4ib) zT=-h}5JZebO?QPlIj}ps!XHf~dcJ#$Gc@FHX#(F)NV(+Upyt8yezKF3SL1n)P1c{% z3tGK@=jqLviN}9ZSZjn8b)17fpzPzZ^7r;HF^8VJ?%fqj@hp>OLtS$-d!1o0k^7tfOl*0~54b}ixarBRt7*>y z-jcMrN&>HvDR-xYLPX3blKL4%$)ADMm`tfnz7Zi-a!tO=@-Ljm& z>|E#pM`!>(G`(m~)Qq7EHosC?MJ2n6T_;LVv+rqLP4-uscgZ%QmM772a|05${f?RF zslmwscNzWK^o6tW(FFmbz&0B4p1dOYEX7SQV9bJs>ipyCRsQjMh&I!2Bo`gu@1PPT zQgl7_JFdfQJ6hxJh`EJS&5{E~{+Ii%-6pymN2;QT7!%1w_qJc-#YsxIVs6@l>gAux z0%#NouQsAJv~K!%w;7e>qI7Svi-zFhs%nEXbDL79xX(pQ#};TF^7C;Z`oHWIRMM03 z7h&s{1qxylahGZs_u1msqJ<15{h{kC_d4~vxge!}z z4EsF8A`K7t6PC_ir7dZT`4f(p6sZ7g#fAPM{}O(~UK8{-Fl;vVb2P*9-k<&Tf&H6s zURr*Sd!9UO>`!P?<}JFD%s9JZ6kE6*Qgfj;U-~Bp0uL(kJoPPRz1^V6@gihzq-Ti7 zxL&O2Zpp^A(XM|ztNYT)=8>V~>rH971T z{#}!E9-lQNu1Y%~_Wma@aa9CGeWk$;f3YXa!YeRB+9N8e2)Q{Oh^VfS4Un z7L)~ro6u+FY|zIpqE4tmNv$VdR>KZx#1vo~romZyzZ3$J5_qs|{NBm%GWKb@_*oUN z9yQ9c+yr5kSM}_#oCnx)_*|d7sJMZto^YXGNLHvNXtAMp+3>FouP!qB4>=u!Ioiv& z$(C;O&Cbi;M#52;V^rm@s^O{@{C7C@JBqb)hOVc#?bY7H6L)+yck4%T+Z%m6ItJgR zA15;ttbfIfSJB?%#=#p*3!UmwU1ek3q%WMOPn?*R7V^CH<7b`+vxW(YNA}!G=dq|B zz;CmEAqrGG)rY?zi)^wxRvVH$SLD$kr|pzA@0(QYZOO=?W8s_XlK(dwy+y#8<}l;%^TUstSg`Tt8?x%Oyf{ZJj?TaU2l!BhTN&r{JTIaImAw&^ZR&d;#_%fyYN|i+mfb|T3a<%qoYt+#v-A-rRbG1E2oC=|vvzFYOGarb?W zi*V?75;>H)$`*exh|6l5Sno+I*g$~l!mhZjXc`(J|0j|tKLjwT&& z>W#ww20O-|_NDyQ|2`d{#uD8G1qH_X&@aU;$cJjxoJ6+=#w3ht!|t~1b&GPw#%tN{ z7m5Bil^|XpaO zxWz%ap1GF1wfc#`^M2D7&(#SXX#X=Q@IQ(e%ILUhRztP3Y;n3bWd9#v!2dFM{P!ZL zH4xv%2QSG5SZJD-$@CACEOBAZE=ZqTPbJTLv2ZO$qVbadgpdC#qTrvDfdg%QEBwE@ zV*CBEVk*amxMzF_yZ8RwqSZ=;Yw=ZP8cKx&f=gO1V?OnTNJU{bu4AFuR&rR{l>oTS56 zcLmy`h+H@~xM!eDG*U>imw=6C)8q>6oNp>(UJ_wC1ykr9COmSbG6Md_O@j!pnE8?7 zEq6-tv9r&}S+XrR0URp{A)3q)qZ^xU%Y|*&?4Hii&U8^#w99V;hk2faZKV z)pdOUF{@MD6((f0y;3H6H{>kdbu}x(XnHi{^BK9VoY2wsNyO9-$K|)w1%E1t9vxYX z$MM0`M?HH=h}k#oljXJJ14XX~zPvP!Rvrft+BaruhoA6l6_*76w^Mw7fcLy#C;a5( zs1A3w@9|vhC9MwjZWr1sf}HJ?)&l#!9;aiYyjcij>oyq zx6;k1J6ui@rmI~?&!&+PVNDJ^qs$yW7#nSnl-uB!m3opXSrOWUMuh0sW(VirI!oOP zDOW=b$C~`?Y|E^rnha>MsPL$s(m~yvdC~0QQMqCiK%)JBRQJ|#QFdM1u!4dpA)wNs zbaxLW(j^8R(jXuVT>~Q0HImXQrP2-3Idlw4hYa8d2;&Uh?>WYGU-uQi=l!1VpYQWe zXV|m%I`>{{9mhKMI(-j5zvPp=zRw+C;=l%mm`ZrC{EKt52C}8x*Zl62j_B_PN}a)B z1M#4{lL3yrsor`Z$6bk~ixj5Z2jx@l>6x4{!i3A+MjOy=GNCkai?$F>vEB5|K-OMl zQFl6357(1szEu+$uXfEO%aS1Gc_kx9ied3F4RnSZiFQfwsxz%0Vka&^Sm5T%YY6V9 zKz+< z$kf?Z1~2XGGikleEnhQS@0!`H@PdjWvTA^k-YK;KdF0ws{;mj+G^*Uy9@PlO>soiJ zuE$$moZP_ft_fZEqcxTZY1fapdQV*9aAVnLCB|vMb3dqCB_Kak8DQAc z?SuMtx31>LX*Yd;zf2huxYp_DHjp%HD-kd}R>}rUslT{R4bxQRDsx&C#bwXkxPMR8 z=g^*Mlb~`~!V5zKUL{&khkrQ;4gl#H;K5~>_QYgQ6K_l`{(1;=M9V&z z+02My8{&h$(LM|P#}L;?COb5nsAudkNR94r!iy(8M;$dD{>sbewPK12wNGP~S0_%A zh<3EFOUcd`3r2y&?eVRp=8vv938t2@0wUMR9ph#yrF;`LXe_KLO`(FdKpafq*5cdmE{cXNs%mLd7_R#xHf3qmqF*l*$a!KC-JP07!Iq+j*4fOrJ`$U#TIx^qR^?v>DEje0l2 z4lN0!rfj>hvEM|FDpLqvE|uS(`E#I;z1X6*f{$d>Pmy9fg^Xh&9;|(??Ur>Kqj^D*5}Be&dy?k$NN(~f|;7L zdh7*Jr7c3M;eravbJKQPP6=lRy!R+Jec#OL=PEd?+2_BDzGH#2>Ak)3ISmRl^^mF; z(>ZOWR8bOA zZxBLV`pe6xGQ>m3X)vy z0PY#wTO@8zx$hzIu%sD z1mi{VC4sX^gjUF_|5V9%29SRH0tI_1p^1faFa}*AoCARK(mJhJz0BkvO{^bxO9_V@ z=5y++NSG@%t)MOKaQArq`wF!RA)S3u(`k3F08sB=>v@~%#~U+!pSP~Uf^lv&wGkg_ z@zm~Ir4ByR>+oSkf9mkl9SvUeX?52}VIVTL%fTlx{i%a&-X#4@>9>_6v>K5SR@>n( zDBJDq1nzZ`NYKRBq2huXzo*S`mI|Ca$tlko0QzAN5m8^EBm{J2LcSVh?=E*gm$bu! zD~XKkHlXw*Qgkj;^XU$S>ujuV25{FU_p+40xC*p@N`MdIr~}eodY=*^1oiiNcVAdg z0LT3T6pslqOHl_as~_Vbkv0?Bpu$}pivCQAyngT`Eok}7h~A$!wa<6GgYP4y3U!5X zfgk-spyd1gY$e{9hf+3Uv=S+#4cy9~vt-XZH^2*s{i9$>P zHjEq$gy%Db{Yj=K@tAr9Hi!Ke*i)n!vLB4ehmp4VPdiA4tMHXp_I~oEaVAB5K;2B5 zaA%#C4ig?=zP0Y%Du+2`UE|MG-x=sneBf)6kN^+~sX@6N-GgFxB+`OF$DNh&FD#TU z2*gF-Hd4L`-j~kT%5d4uQ#~mgGTdBu=SEAvb_-e{RXXN@Y#vbhoqf1kY4ug(4KD_g z6%a}loF@p}QkDw)S1!QnGFMaEHYY}*{Tg#X9fhf5Z|;j$@Bp54tcK>TxugmG0>OLB z&TkYH=$Q}&$bJ!MV*n;bJ6`Eh({V}Aa#)uM^11jBNhXzwmoza`tnfLi!DY(sPLx6I zyA;Sxco9e57Xol_-q3?XO^!`+&9vo zQvk^L2f!-wV|?42f?J~Lp`kH&V_%m8+Ux{(h9n6pL`N4v(HYrUd=qlFqAwDeXfUIK z2tZ%9uOLPXi*(%RCnDQg{U0$pEt8oWC%CEa-Cny9W7#4z_14fVyvjlCoeyv<9{@cZ za>|ccACbL`k2L=r-~=B2;Ar70`)Y_^FycJR?7|&n9>Xz6HICFmR$7l1xffAavfuQ& z=xdov$LiP5jYS+q8@b!QY;3^M!1f{$g1U=;U^*MvFOQ{^3_BQ_{wiQIoPY5?gRYrv zN1u=Tyge$s=uEpk>3SWl8TqA?S7}hUrUIKM6HaqAyP49F`has_AWFXL`CGFIXR#M8R!lQv&Z$`$R?7@?V5tnDNf0d zq{Cq{`~a8%PJ4LHNy!Mh?~;s=$^Ky7*AG&h3x0&m4j(t9Bl?|VvXlAJ(*_1^&Pb%h zvX9fdW%wdwl0z04sd|QP{u^ik2+VDIAxt9n6^E+i`*pl;YhVg+=f42kQ;X z-7vwmHy76Wy2>^?pUn|n1gm1W;>&7qzvDwL@EFeOiDjf>n5mN-;cY!e^TgQ+3dY(_ z0wS7FAX5SODIB}WDL1LqaYfTK)m*TR^$>|vjC+2hYf)AZ_Xj*UQI2CB5cP#0aL^Qz z6lGfyjprl;jN}Aaef(I=)JWkVy;WlC8ovIHCqEYJEP>5pzidglpY7N2VM$aQ6N+L> zhd)9%*5ItqQC8nF)yAvnp;i&&UZqtn;KP{MUZ#yPfiE_? zKii<_ppkmS%YSVPcG6_FFvdPd2gpZ#6own9Xf zGPS*^V3+W91f0EKB2^@|X6t4HCA_D0Pa3oAmcU_s9Eh|mWk#h`X9WpMd`ZLEUQU-) zgTHy*>JUxHL@NwZ^&4eLc_++GwfI@uIJl3MD%uJo+Z`%G740d|6|1CWre&u1n_Xm4 zGOm!kOi9HUH*n{^XU!PL{h(FKN4arzKn!bH7JBpIiPTrYPf3!NsD5gpLOoq#AE{Xj zjT5@7PC~2>?jw_(iTsMORaDl^oSA~^be~V=NL_*EAP%uYO;AaTv9ff?IbUrobBC1_ z#Is>W`NZyo1#2qiy+{S+=UJUniHKv%8?Gx-kyJUsPqnU4pUiSO;<80YG~kiybPE$7 zWc#M=P2z3Z3qLw>L2S{Bs&${hiie|(rL} z2ul2B#tabi8GCbG%oTouXO&g00SqtuBJ1-5T+Y7Kp2L+7!(ACT)Ui3|LhqwtI^zc~ zht9^`tBf#z4VrVWzj1c-#G;sK^~(lz-~ih1(^pNWm=d!13LpgcHc3xqc~vT#Go?);=jI zId*Txt&ov+P|)hjK71>du>GjvH*?8$g4ilp>bgNiMUrXl;!?!+|Co!y)~K9LZu&9u zW5Uq|C!%)IMgQ_=0=G7cYo;<5;>##to?G#F!BVCOA-x#`GG^C3Nixf_Iz zt_-*zAOFUm4My&5sbQ?4->b-Xfx4^Bi_hN4R*VY2}8h8ou?<@(BYAPWCYcCpRuV4ZZ4}rU6!S8~lD;et!)yBq-f= zfcu&arNB~f6S?DqtW0m`I@G&9Kf#An`87F3FmHaRX$Wr_m(6A9lksgEo36V1rkXXT zwRFFkd_a(K>&hTQw%>FcT3g1K*#Nn-t6qx-_V1Mo{mldU8k$daA*7r^Qq6ngTR``V z8q)t(8{oZ^iHZUiF_@LD_&@7XUQJ{-a2RwRZ6LK-O7+aFr@Z4b#fT6G?Q#~5Lm>Mz zxiM|oyLc5cZS_0bD3pxKX`h_h2J?S2RfPQhHz?2_NYkWcqyCvF{Y9Q)Io<|&KU9rk zg->i#^Xl4ta@_p_XH#OthjzIk=?O>lq=+&;HUA^(pWO{#5axc=`n8*XX@2_7b=ZPQ zOKDzDZB_n*)acI)qI^d~%Q~vXN17hEklCw(Fl~5}WjDh!p~?B2suju5kl#&f3SJyl zt;;V9y|wUD-bLE`56kOT8Z0)ar4jK8xbbJ|`IpY-KZ=_l#Ye9>|3P~+p^yfJhP;W# zfFQa(=Zki}R`U*`__|w!eiW=;5c>M}=?S-DT#uR?n9c0FN^-oh4l}Tdu&tO9qBcR# z^7?IB7W|;8v8YEFs4qQpo%dK!K+Mql2!wNq{80N~H9iIwZiFo|Xiy6ld3>oagfIWz z5LOW|i~W^>9Oz`o7(E%Erpzyz{Ff{XOGh`fMD4^%In|n9J;fwUuq7cvV5LN{slcIm zVf(W*@ned`bx68$xxpvaj$?y2rtVK07gbFgssb%H$C^YIVc8zw#@Wci3uq}Ff0@<96>y+dz>wACty2*2{+(lr`)7@lCAg001pkGXY{5#d`#S-@EoW&Hab@Z#`+m zQV@4ekJP4aX*an(PW}OIs$k+TOX_oGZ}=YMlyaJ5g$MGfGJEqBBH-M^Tfe!`xvc@3 zWY3snV^8R{D(Z)@av~Q(I#9sT ze!N@UrS?&qrSWJ~%gHmw#+ImGHPEQ=7mI^prd*d$s(%zi6Kxr5XU%K9H+ICdPR+c_ zSrR51k%BoeW^)aWOrP&i@8!3Oh?@$w#URTFTcsGFIG?s>2+1Vk8um|na!7@6W1GE} z8wNBb7*+`J`Cb-sgok-K#usi&EzC|Yvk>mxooW*aV+inHne5TI!Z!Rm&13RgZ#Yc_ zQXS1`m5(H*7gTqhqm@GSr3bI~j1^`d-@4@RVEOeg<`slnfA)f{JBqhn19L{b{^Xi- z%?2-87#EM3N%eVVz^F*C9qxf*vjC2~_(6A1^^xh!w+g*mrwrwvZ`TihU&H%RmjNnD zz4x+$mfPl^Z+%AOUv#7ZGiXJ&T75r zKf8YXLQTJi^Kzb$GPvzg+WE0U`}~j*zp%R&w}qh3=NsPDUn76Nn*Z|UjZ2SO-O29R z-GpAnG5*Y2#VppVX};N<_9<)1ObGccYsYs=XyFL7;O3b(swUJDn`;Ii@c(kkP+I5n zc%{}GHq$B}yhK$RNXg(eeSm&;g^w&ThhE^kO~2TYl|ruWG4PkIoj@l2=MQeUBtYj ze{lS@zXd#F#t?8yL;-tRVb*fGd9l+U=3_Nt-XBEy{QGTl8_>}n!&U7J_rfdMQc%5l zS2K=M2~J$-QPy!ijlXxnHGJ3V>IpBtJKg59j_VssF}0K3U74b2PO(dQmdi&@Br+R# zZ;wNH)D0&Vw(J^rh#TsI;$jN*CcCKL%Rd8EJja!Aj39mJ*=&Jn+MI^0e_xsfCM_Pa zH@u%j^L5GF4MKU9YPqj!*MUfd0d*aGpD4w*lw*DvgH>nC^@YUC#R8_ zVEU=o?a!|{aT7BazNorVMN=d`7RxWu<$PD~Erk)Ptl?;A`SUzJgHEqyAt3nkhrOu( z7TWORsX92k#UsG<+)KudV4Xuhy<)4}=iwzwcJ^vpR@1EA)5DX%Jn)pyYkFOdLe%4M z*jV!L3=PXNK9E)3KGL|lbrA1>0*YnFA70Tal1_Y_d#X`-`QuF^pbQBYy}xXG zWyIz}jBYZpWKkZxr|zfj#I(V<5#-1*{&THUk9OJ|wqJ@A(vmLdlo{52@uj(VVACrF z^=FkHBAVeCBW5OXnS3(QR0tmhJJn6;ZVC}EuTq$pL|9sXEU+HwkYhD@GUU{*YKw3E z8o2EFaSPFvVmv+gici>Gz{}|yc;~%;j4i7?y#dfXZnEm-g-OVnvlKGUvTUX~jNU+1 zVeU~iJEW;Qc4oTx9$Cv+7Jg^LH#g|f03@;K-n4(zdyL2v%_JNSy}Xw+ycvp#|Gt;Y9E(kw5;*o-lzAO-(Y`qB+(+ffODn(e9>Tot`F=50b9%E z$8q))Nu}rk<{l>X`5=ub6XvSb00a@f>!^7Lmp$v5-3K>zuxMi@TZVsZB@Y^O%_%4( z^n-6(jW;Wo?H5tiVqrbf! z?%Ilzx#ksoOwZDAkw5Z02jTX&oi{^Qhvi;u6g$dxmM?RB*cFROD0%uW`CRr*w%89>?Z6b2bv05I#6{qOkCIi}y|4`1oB# zW=fzMEB%Ie3b+OUtx3HlYxGo<$qvv872@{likxMJt(RG|KnM%*1JXGw z;Chc@7!T>N$R6kkzu~&bhQ@KI$b(>D^6zxlEwOJqGPPwwd-Qr2)4)E;Bf3HJ?> z%1B1oE6Soo9;9E*P)~mB!3f4V@tVem?|b5uM}jvJ!?E&IcMT~7gwKlCmkUh z3d6Ge=H?2OvzpY1HH(qE9@nhJJRc`A#$lb`oMPkqkkd@P$9dVEvAC)XYb{%x6i>h^ zEr<{{{;@ssVccr{6I=MxUL0DLX`*y5(yr>7P|jOztw!-{z-cr;18G zCn;}J9104G+fbP$Z}ZwrT9QIhZ^Oz&;_;{Sl`E6|7{|4SU_n;b!`3@a zqWzIsdNo`5wU;dUYN(Br@9)VCxl0Xm0`B0R7p+RwTH1Pdx?^?mOnzUV2rAYpy(-QY z5vt18nXJc9;>T#~3%LyVRp6UzcqAj{eoUe^I4|ff(>PWru261Pc8@-a5+WH#FYe3( zs-c9o%E8UP9_AZ?c_r3twwRi-{1Ds-x)ZM~j#f3-!yEX^u#4gN-@gY&x!JQhM-^4j zUDp1Wm>c0obkaeD>1+y4F_q2Zj5e^$AfQ#=^lOz@g)RB>dv9yY?H~dOP*8n=BOka= zie88S<;}pFv3-6UnJKkkCRM6f*>!{(?To|NjFpSOPifv+Z18h()q1c$eCV#;T9I4v z!{4E@(?@W|xnssXr=VP9M0$^d)zW1ipGLXA+T}ln+4G)M!@~A#p3psFl9|CZ-_-*I?8QGXUx z%lwIH4O$G`&9P7Cz*Jte6MCiu$bPU#2smm0T>Z5aKxp)9SZPJ^MA0IX=O#t3sj#xRtCz6Pf)@YE{0=-$O>ET8}o8@5?_wQ}=0rBPCCcv|ro|SPP zM`X9G&yRNb9XO`>cCPDU=1~TB{>b@Gw!N|&d@aoup0Q^5YuR`=!7b0E{@5rWMRF-K zsg*^>&CfZ_*;J;bR!HLivfaNemP_4_!M#puLLpYBne+HVL%_HBO1rpHy~d2w#H}-~ zOMLIKfeCnqFY+w4Bqmmx=1TnZLYB5k=ckrt97q$02-%&yb-F}EJReg4hY}uzI_}M6 zLOop}h+VDo5TmAuligir)Rg|P(N-`gWs3bS;ER7aMu7uZc_aU6s@Q1*=Mx~xi_O*u zl~;ch><1(u*C(cbCeNLBP{G4Zo2)oavvFp}9tGOANo&~6{jc7fg;oU*A9A!C=$WY@J%q=1_QV}Jn(&9w#GzY!b!C-XfphNKETG{ae*ivC%f?|%k ziCr1*Bhn0iY-wyrucF)SLKzS*1E>Z{csKCO`n!?A;bJC9m)oF_^HtLaSlsgC%-Z9- z0{35vBl}$$4az%*cD*p}r%7U17jn5hinFBpXPjOAVz2$7kwj`Ukuf%43BF!%VR7ec zkgeI3>F2ki9Gr;${dtK)jh@s-`MpEIb97yVUOYF#cZF~4!LIN&s(}1E;b^D?GP0$7 ziT=se-f=KW$@cuB?mC2z6m*%^h0chiA57Yx?K9wzck(7qH6>O zq&kN|nzhqIJ4D;~lTH|jc3mA)ob1@tSgk-l#*+n2-lplgs2AJb4^b`_9?OfojUMHm_>Tz~QCGtWObmOfwxU@{kOVo8i2 zF9=u*dRPVq?PEkA)XIRlqUJdoS5DcMfXM$r%Pu{0H+n)JC?Ki@>oOK%z3!To{Fd~O zRDIcW;zfV)*G-38z#wEWwGOXlzxPocs)mPf;u^W+4w8g9Y4pJO+h;cSHo(`f798g8W`+3d4^!g|H#O3$8yw(N{ z*PXv@26v=(>fAKO{yng?Pdy;fS6B$&2!N&Gv*X(e6vY}cd2~I%t)t0mzTxJB@}q!5 zMZJdJXA}#kIqO$|B51JpI-RE*|AwyMb%5K(nz=`hHtrfzvCxmzbU{w4|k&TLPXkaAyYUMT)h| zzao7sWZSJAC_0x**WZ8wS(C6^|1@rYQ0G7B4-@DTrRCELI1@9ADQry8G z7gg|nB;WqcON+LdnXAVDwh*7&+!%KX6?JIIXhdje_ zeLqqlorMEgvoiLsNaJaDdP7|mu={aIR@G|hij)q5zT7WuMrDvq$3jRnJp~vgei`)A zMpq_Fv2A-Wr~=5a4K!^8pI)~{U`LA3kjdW|$h{R9VXF>;T=L9vflnnZq~IFxlvJ1B z&-n=wws9xfo92W!^O~a5;G6sb3qBo#-^?Mops^T$Hz2J`lqnT&e|TZXl$VVM6lWt z9UrtvOIc_sm91XDTg`;ZIzJP)O_*z0)uTI%_jgkw|3=95bC9g%cy{0c+sqblItD^|Ud(69TIyq*bHK60Vt5FoNFEx)CYFis==gMr7Q zdqceS+Z9jxnHC%A-w5H~+fc0CMqw8C_8m&?lZ$X8^zD~D@pZG|Tu}1T<~&Xw(e2oa zdwmw#({T@V%^DE;JwZjfQ!;Zws?SK&KUC*-s8N%%XjucE0ob~4_-vm&4QQ8w9uH&u zK*CW*j8|)mw%0xty0#Re^!Bp@mh);kTiAf98pDXkuBTva>{Pje2hNshTI%N&7Y*-+ zmq^7%iOtvz0zUN@loi!B3)p|qcq&=g={6uEPTz3)(mSg@tdpx$SnzZ|Q#sgj>&|bE z4_JFRKFbJ)H-0=D`;Y{HP}4bwZVh!~a$udnwYZm2w(4X;364f>K30Wf#H`@iqxPa) zNT03F4^R7iyHvS16355@IFwd&D$kdCHH2D%Aa$q^e8%!yownb472HLq9vgXqf~9LJ zWp*pQ5bG!(VS0X-yVl8RK{s(DVS%R&jHp}h9Sb~}0j|wj0$N3M+(N!dr$S#=)ngH* zH@~Ay2jKf}`k#Z0I_dn@d`g=4JVf*Mt>%veg9ucouh0&-1-djj+;;(;K*q(B#o2vO z&`)>d=MLZJwj|4*g|t*3pXuYr7q3i)AZY>1)MPG59aT6K)^IX)U&>wDz3v!;o7(Jr zkJzfZZ)|5E zF2XOn+3PT5Qz}?*jR03|4WGpR1Hd~P({dTiimmG9xF~^nkEI`c6srN{qpEq{V-uU0 zvQH;p-lp%#A+B23N}cc7B+m#8zjm1M;vQ_Lau(pn;BLq0vhsTp# zEgsqS4#Ec?p_vdv-Bj6_c$qg}B5>B_Zk=kUKC*fXZKSj@5$>1J^CY8+Kuxh6O{)y9$Fh+&jL;fb7o*SDY}9w490MLc(e9$udc3N zbP{dP<6qQSdI1`LgVxM_X!M~u8?B!l$hYf7Te;6~3g{<_`J`H#)y_F+Xii~S;CWM= z;wQWlfsfqR&okfxK*d9*Gj@AlRZC58;5b*%s&rcMw=B}Bcv1?;x0c-646(vYYTsC5 zb~S6tj6abBQqsfGrB(e6M;z1t|pNbJ(ypZ3~TXK>7yEC-e2UA}xgaP|Kh z=X+^SI0poj`gspQYR#^7zK$Bs{7fJAjiJOc1h?0t($I1AVz6b@E6l;fZK$Za8RWnf zgoI87uSX@*3vi3eXs-x6sb@DsTEYcOf<781O@|J@NmT(}hb0pyhXHq|WfBd7fO_#+2J5|D*Gt-@;zti?l?FdlG z+kq@i_DnyT%PiGPt94tQ)sVh{{GH;E8*m+ui{B`mL)Q; zt^V$SN5x--k^ZfQ@zjp00&83yV`1X2=6^3iF&c(C{#Z6e46ZNP$|*PpVf*8h{KasJ@EhqRi~Fx$JSE7 z$zeA`(}{|UqoEJ+vSxyo7jb`6ttn;ajyrh%2j{t=Rz#NVJ$oKa7DW;Ri1=t~$?W5L z|9R}mlIaF7)P9o~y~_en_eIW1d(OWeASbpeHEntifEg6XP11Iy6;e}dD8dyf?)9VS z>zS85SjQrR7@0$S(lg;TeKN(egpI{68rbyXuP4BZv}$MIho|Q2Gc+QK zhxY7f%PPrXOLIIHRwCPNpF^bUW^O<-TE7(PhVTmTF@q}W{C2s;7AxSfnDMuhuWtaM z6mSIv(?8m3T?<_mm8%j1Vsk+h{yZVQF#kNHU=VnW|G;H$LN7FOSVw(ONT3@eY+BtlfUD z?2W)bIHmur`L8mMpPz(x|AVLd)%1+=@1=;JpWLMW2lt7!E#T$>v;0*>!1&{n=XU?# z6n}nlHRkVeK`ueRHn&`Y`dh3WG0+^>$4MVPOE*f~94BHvXYH+8INeJ4h1)(Xo|yY= zsY)>o1BV|n8cNC9Wi*q$l-M2J%l?8GSZl*3N0~wik z4z#aJQkBL`T{sh8wUne48lh{x=}7dZO>Nz()Pvv*4}F=Wam zI>}M-3ubVo+*SifVQx#(wy|nUY`pMZvas`7ovxeXHPhOvfHANgt6$=_;z1>Y821?I zT*vBIff%lq-zf7to1}H1{KN2kU>s- z*9VCjx3pv5dojdttk&uXhJH=BeD-04OKoWV9L*@f>h&Ik1eE1j63t(QCCndO8If`E zq3IiBMI%lLku#ENHf*T`G+CGT_!GQb6UA-0QpMNhd}X*->o7=eeAlG-W-~Yki&Ns| zy|N}U{4iH18WtiheSBB%L`Q9d#$1Uvxnn&I#I{PkYRlhNP{zU6rQXmPGf|t9wD{aj zObp&ATis~oN~wCdCN%=B?&$E;lcYzK_fCHr)j4BN zEoMHYc|ofJ^i)*+h>+XVhnK}S)3HF}P-By)YT26Ejz|3ZZ*!S)a?kc{K8~^}B;yiL zByCrI^jep5_3OW>gGt|B@>J&}hf67{f?x+mk^o^Oq_Xe@M&^~gttK)1%-{Bx7UU3&QSyx`+UX(QS8)WYT|{=T_+sSka(5LY#9TPx8$5}7QVle>`AVeXwciCB&2WnXP$S8dUgphvzCMkTw0 zRr!>A>wAGs7oH{0L(B|02cb@A+}0GA`$qj{IDa@V@DZrZ+2`xnnK7J62hLU*sm!oM zSH+DVzmUg`%xeec97_X{g0P<~Ip|ajUQ@oVau(&ff1q>Erf|eRF;|VR))nrwpSA~9olWX*)#})`K~NoA!#UexhK8dv3qEm z9JyyD0WHdp-y_$F4{bTF8E%<0($6sfy-o4`FD&p%b%b&Bt+p73%Ss%Y$TIhKQxBe^ z1OMvlf(P!iOH2*vxM?JGeb|xF=#G8o`GS>7cKgXue4TibN~^3YT#3fqK*fkkq&Z6* z{{ka|Jx_|RPd=+7hZsLTlnnnX=B*`b;&siT0wKHolRi5=V&*-y5q&>cio0(7H-}vuM;N&azN}SU=%YH-L zuoFebc8@q0=<81C;_@=LYUv#7GwJQI;XA4fw*%Wa8BcC;#nWe{%VkF|^?;*!T*#zz z%?H}vQ(g@VUWT2&#I||B4*m)Pq(!}28Nv&;<%c~NI3Vn0mY2d@?;c3CYHZXt)OJMdX}_Mb zUaP(3%EsJVUkAbwQhxPmD)*9k`)>n5xkbW<126Sh64_CdxU?!-V=^-T-+;!c}4U#)>egW8ghkPf*q0BKQci3uCcRVz(6Z zlpg05+M1?+n-GWseeB@3jJuTPf?0sIk zVRW?+LZhVGa@)*ysUAjJQB&5=suS;-iSa_o9X5Lpy4Hn7-0#ED_!u8}W8U79jLJ3= z{WR>MJZm6ZZacTOa&M;?aK`n3i7eylAQETXlpXF&0d<~2FUF~p1TRQxr)I*69Og)) zj9hDBs8e{tkk~5q?`SXW4j>=W&9$Q)BaDqtUp-Qa2=0Vjw72_gH*#DP56A(F)5Mjb zo{aSrPrEaD679cU`WhXo+*=;7@=<|aT}d_7-*OmT)LNd2XA>2FUByOInkmO>h8L+Q zZlspli;Dr(A5*cD6G;;@BYSJ;B`1Ux0Qay{#a_?HL^PD=#h99DO=C~p_qCJ{h+Q}{ z+D^8?2^+PrQ*pJLD}xiH8`m>!Q$=IZ0bQwdgzCi*W+DB#W$WsULjZih@Jmf0* zyLtHv2*yH3c_AGsCs?j(s4LMX1&wC->TsgPN>ZD`zk0byF5lTY$XO}E?Mr{BuF(_l z>*G!z%9%#$hjBu6#i5w8hKOF<7KKO`R@<>z(ubvTGRcv+H8U8aXnS^CYjc0Zl+YTW zF7?8s=P&`6WOn7ff#7r^c~E(MyrZ*hL5CS1PW(5~W5QQJiKZ|Fxzgz~6cOJY-W|I^ z=179x<8BSmf%6~YarOr-@99Ifhwe`YEuR;jr2#*X7y<%Y=n6{s*zx*~BRvZde+&~) zJal+^zekit_XKpzOE+e&6VG4-X5d)e)4|X@_j4(M@7?&!eNT&4IDcLd4b1RxT6)w^j6 z5?gE~pxZm*78;aWRE@X<;__oT#B6a9CK&S_l9#i2MF9itFQ22t{pW+W0k@iBE;zN< z6|G%o5|SzYeALX<>8l@AdKpgd5d9?IdtRnkftBqO5HaI~MH9yB(X$NUEhW|L40<-8 z3HZnkac)asRH-NrHFYv?_sF_JiS)x@$l%{`_ij;PbiB5yTxjvsheOy@0(G9mMAo>9roA4LKYm zE4#)qBZH}3jE})Avh^hR^ys3FGFyiRD=|GW7joz^B3&m8xBhP8?9q4C#KfgN9e5j$ zX>38q&1k&SUW{3W4d4YK;I-qOm3&A^X?9GAA)SgOM_+g*M^p>R$A!r_OO3FuNK7I- zxCq6;fUE}+Z>fpn6ZZ0VKN34CcUaJcLuSv}k%c)5k4A+TGK7=wRb^GqltCvW2|6$? zITywa%5yJR=N5qS{h|)$f!iTFtAlJL8Ego3)gmc=M((Q~_`+&sINK~xmG5yiT`exs zm~AFyfpC+zBxaW5aAh15C-k>o5IA8u(2kQl1f%3|AZ#b%5H?GiZlZ7b7?}e($r9KA4)uV}LQA>N<;9 z(~>xk9pVw1eJVc4?iIX<{NVy+v;4JFJfdxV^3W~C0C&~^8XIcK3!4vfjptRLE$Y-I zg$@la-2eJIfr^4JYw=h>nc-ZauCggVI4(uvYZ*~5UY zM&8E)#tMBEzMR0z;qM%^Z;*l}^Mqb%46V6+iHnzsJPisMxosB4bmtft;^1{0D|hkr z9GAv6ahAE33c)&$#A6%Z5WYbr9Ww1eErv_m&XzfAGSG4Y^8o8n1qNl`L;FM9Lj_&2 zCS1H@dbD^*QHh}LcjA25-qQ9?fGcRQ8}MX(`<5MRc3u#wtz{Fl;mNm45f$_q=5Qw< zcn>(Ek0zh4ua;&qh(QC*0az@p>U_Grx!+x=9VL6Pkt1)+ljb_+gXYU>;<+>oS9X>~ zU=Am2`S8GBlO2Ym1J4dmhyb6G>%(^2;kKt23DT&ks!Pn=g9pzi;Kgy#~fPDE*d zH{Wi@X&-I|0fQS_eZ@hqveA+~w+p%n!A)P>g|3 z;@L)Itc+va=Q>c**e_IWgDqwhgg=B~nIvpwtMU8otu8G0{rP1HGk%2wc<2X>G)P`p zhl{@($B^!6eO0dz%k2CI@i!EFed87zn?l>Ha=a3z-4%>a36d$AA5;&U& zIX*3%^^$;APgI*p2P)4N>Xq(q?G@nvEE@a?(fdUvGgMnyudih_I8r%RwLNYFMrj~k z2KWuq(nfd>mUQV3Jj+HJH`4Y@uTKxkf|yKQf_xB`#0y-e*4u*U$sJgy*Q|?{i}4Sl z@jny;D(e+RR^4dvqo$LncbiAr0q%K6Lkb?>$}`_d5ic{$d-G!no6Xbyv!k)CB<~-S zC0Lje+FWSKPCr+-H-2TVDGpG#$CjRyCj6?2|A`QhmJg#XQ@~PwNd;aNG44IUJBpP9 z{Jbd6YPGXGv^93xRm)p~J}k+Vl+9j39{V3_0llhL*SPHq3Ax1CbXs5U}EanB8$Z!)!qC0BocZoyCdh$Xqwgdw|QDKt2 zQR(-sQSl*QrS7$9pcV2uMy+(slQp*b&z#m1FU{sJgta(53XbqaM_X%}bq~D^vRrvb zzm<2zfhXQdERLJ?p><*CNz2M*x*plHy)pDw+`{J?W34793l>vis|^pQV|gi3$E&s2}#gr*B*hdPn%i{B96Y1XxEjgret2tRTvMl zVo4~j4B#rnZYAncQ>t3D;pPq(S8)G2!P7`kW!3G3ve_*zC7@@glKb&g+*U jejdXAH~$y@Z0k(<7Lq?8A3ksn_@}C;KFL&|^jzXWcfXv_mTrs|DsSNh0c@Qu zPn1!w?{T4?tv0NBzBiumLv2`P>O3uW3pfLr&SY40(L@{@5oKiuG?JC+X_>|XKG)GB zS@pfSPG(CS+Ovm{?c-{v;5(2&`op}|tR-NTrmOT5AqeVIx+x;*x^t!F7FrGz(&(hw z9Oe*ZI0AP;$TP^WqM_O{``L1@$C-8>Dv|9LS}+R5c|N8@SsW&#tWpadt${IZZaz_H zwv)|aScO#Kq*1oI9No)PH5u1p!|tj5LSW)7Qc9{M*O0oA8gLFa9?dbjTkr@JMJq^Q zOqDv*awA*pnPt0TGUKSkmHTu}blL-`J({8(*cf?FAdv%X zLMYi{gHc)q3O5xhq;dn2FV^d1RwcT+UNJ4M0*=i)?X+($ndIj=u7+9dcB4$?cn_ac z`$ZE1qD!<}*?0!w<$_j~38IX1#RMN0tH_j?cUY&$H5du0j~cEdG>k&IIc*u1fJ>!r zH4NZ9;Sa$SU7%5jJQf6$4=1q`G?Q#L@$!ucYSOwrG=(lT4>8TCk!pKB_4`3h972Rx z?abJct(N9lEMh#(9~7&YWr5FNHqYzl{i=(Ve3?KHzlw03us;{84Z9g+jX7H6I;{ag zLkX*4J&Q1OFvdkDrZD7TLSmVgJc1fxIqS{YVXGLmE`R!Skv9YmF^YN(MucRh%K(mi$B z*&5AyzRQPQQyA9;TPUj<)Nvn5p>FJ4;1=tNz#08z*Y z&9$tl;5J-R=jXk^W7VL8%0sk}YgofR+M*=hcbFnLJMCn1PRPLm-GUo4w1ztNu#3c{ zEF((3fi;G+f~-wzjzB|PHOAarzMC5k!=VBRV+5bJWfvElqpnZ7g=~ujEFCbtDcGkp zBvT>p66YsQpKn1*q!-?efKmMSXRAca9Lfm^wdoZ>X9&T*fWLty@CvLI1TlM880 z8L-t>MEAzBfV5?XO=(YL*#hQ+ZhA?Rpcpjt(a@pwqF0c8lWuiw*-v79V$KRUm)G@X zu90ZIU{q(_4ud-mUXuN&iuY#~zK7;!Z7>r%ga)Bt#50VU5xm+LMpkv~par`fdN?nO zBWw`(RSO$WD;=9d>RcQWHU*;p=2B9Hnf7Kr{G3rHnV;smyqhX<0U1EXa3$Iz(l49ix(TX6!V zkZ#^+(UdjKwuU)*X13sZKULjeYi$l5OnSX;Q3kk+p2sAr%~g9qOyg3n(~F0sX+^b~ zIwtkGC<-EA2T2*(Os%Grv#{y1P;tsN>s~kkV>pg)mEnF{b)78JHzQ|MX>oEn*0Olk zj;XvpXxa9NY}84HOrQxKj_|pqW(U(A*h@0WR{RDs3~Sm@!vcedWUwK5ls94-BgbZU zgs8Dm9JZ^~q1j1~G?W0Zsk_8Y6|Os7yH3{0sN z1~J4+9h`*i^t@9<#G6y6mD!fCajfKe*@i|y92i0yt|^acyD15#?3$ClJHzo{ZuDz1 z)y_2=O4aSC^=7$G&6Ot9^04+CQ34*UB|s1pND?7orNL-!MmZv+N|Ya4oq;J#x^{8c zqJajjB`5-0m9hxgyws(+pa>X+rn5P{&d?-e_(gZ#h4N-QRR=CSXOz0Cu&GiYoj9rv zBc-erv>_zT8ls7gVJ{elr9N75i;x1$iP1BSzMk{Ow4QHOVUbWpy@-($LS;l^#Pf{~ zX*Ilt-RMkarb^Yc25V4ou4Y4xCSgdk{u~YpQKwoDLf@)Os?X9m-%MCOXthK`ZuEop z44;}NGtW+jCNzby&Umgs?9fKYRx(p^42cM;gm?IcFevBzd|Zd?gLbo+1nsg_j~NQ8 zSw*@~D=P|PqZ-2#lQ8RcfXd7DfLW8ga-1GkHO7u@u@#eo!fZ#KL{+zCRXyD{_-x0j zOqgd`DvAPgC3z9u_BecY-pF~iPacZb#u~CnWCT#%{fZU zc^U8Z6EhB?dCO={>=H266`qm|oCnubGz`86NCxhfuN0H)TpmPWjm=5Le!vchL1Eg4 zlAbUQLM_UNK_d}bf&&d|t-&zg&Ku(pY`H*!Phf)gKNoPa6%Dz^Ptia>m$!4kA z@1l5qnw2%x!|XsdJm5(Hp+@qpe1J`3yH`#3r8Y*u9OQwjBHTiQuD3g*BHpdEyUprc zS6Nw|qd5*}cfu$o9ac%LltUtF-v)pPuR^t|>Z_6~@%=pG7*sWjl%}C>RWzfa7Hg(R z>f_W*5y8ZTr7`F5)3Alqr{YwsIsF;!(70ga@=>a^1VK{i965(EwMb1(59t~`bjl=p z$Y}+O0Sm(xl5I5!r5))~vo`3(V;J(QU`qsqPuOJI)9b17&`F5MO@O|j`I;o;6w5?2ko9^kD^p(gBLJ9g=V+GFp-h!_YUEWIel2x*s$YD4K(H9 zyetuY9@b`koYGM-^%dx7&bYde4QD+fnaZ6|g|R8^SCh~#i9i&qO4xKpE>!3V4&_a{ zP%etQ7#8|-A>pejrbL5)p68`v-nOD>EDm6xT3Uiw_R4LCO~gTKq~N&$PP1`Vqj{ip zb288!p(r|rlU`^}qcot>SlX^bAui-vxpr8~Ldi_cL0DZYGh(>_$6UVvoXLT}g@ypp z05VMHDFO|v-4<5sGA%=D7*=^qbTOSMC_)}kkM8M74C$djr=$d*WwI;*53&#JB~2%W z#o`Sa>vS;SlS8yW?G^_RA99JFAG@fZ^zt(ur?8=m0_;a45y{y~`eb@c8b|<^0z?`9 zclZRzI=l@&X@r$L!wK~`&@^E{w(SCu{U<(225GEW06(Z|ks67*Wbu%O3OkGH|W zh)!b^Vw|5C2Gfo8sDcwz9v-*rZlx8Cky3+2VtQ0}h^$vDu(e*(bZo5Qd$^Qd0(f;; z+K&*uT;c0zJ_zd(E2Ij|Xtnj(z;@8d%n{%sD?mvOZz8EStpq7>9?ElR5=D4D&Zb1p zHUy)hRg@v!AEjd6nZPVtF{%}|>C3ISFgKWFM9FqdBCByTPX~>-UkZl;6su!mV)S`U z#|14ktc3NmhTTw4u!`aorB)q;$UK;{%!<>iD6wnsj9P@7z@M2l6p94C zA{AQQLB7;Z*~EoQOfMLzoU1k4Vcw!A9S6lw)|+YlMw~RfZfa4wU3A!Kn_g6}jEgz4 z2$V2A9aO7&Uomz^KFo!Y z-<}69Z~1K5A5`6eggXMy)W>{rggIqa56sk!YNSpA?IB(aSeWLDZJg07Su>mhpPt4# zUG{yr?W7*Q+_ccJ=b{}ERU>&)fZ0-s;PV4PCg(}$k6UdG9#pup(*i8gPO{z3#HnC| zi8F72SVhMp+$?913NgmRRQMq(L%_4BXpWY5Q@u+3lC7wmhp*D$6s80Y;)~8&w2IoeEt7hw6k93c&k~m{C8n`Z37` zqzDd4%VAWgi?co#s-=-zb0)ctZ)W?3pX+KxWyGSMFL7{r1cAPKT&~Yb6Dpxv?HMNw zK@4bO!zC%F*nYtkgt&v2n}%;gwW2?i%L11Qh20fEXthmsV ziUYw~vy>-zrOA1r%vCFir&bX_g0e|Mn3@k5*pZ4BV*~3(&+(``Q@|C_+1eB==IRJl zlmtOH>bBmQr$m8?0UM@{Txcg2G*ZCf3DicSVioRFqy7|*!FO`jCkt&SUmeyPvvMJs z1aRUZsAgJ_$JG2Tp6?mCP#zYzbfy3v*%g;z>WG%Ah51MZs*<4SDlCD>b>DYt?W!b5 zl_;2x<+7XW1#Gk7a)3s}$m2TstUq(7(|k7H7+1TgDW2*HoUax7Vg+xiXmysxS7==b zBNVT7nqXLClyE85WVMP@8t{q`>BD@kHnjjEE7zTz3F2n7!_bqcE6mEC3F2<^(5&Nh zhs3MSRGUwYBGqq+#kj_J=rmS>u9mVeO10dn-AJr?7OD;F@CXk8If(fvH9>hx@l>!X zF=2aX;9PHAQ9bx2nGc6Qq;pBAf5TC%wgmJ>Uk*O`f(l^ z7{bUU0;u9Ef}*D2W?B_3v5g9W!}-?C3gT*r%;wcz-BTh2n)BlvKE_fDpQUDw2rE2E zG(ta@XIR!QfaVmY3~3I8SqK~Rqy7XQM)9*YO0o+Nq=y)yg*S z1I=ev5!d3=L#LYtFWF{&@UMW5CbgKJzYl%>x9A7lKT!7ncXURDPM#)REz(=yAib(d z6jQL4W_t2U)a#=G^3P0-hNWDG6p^XIN@y9e2U*zfOgpZH56D`LBdq!i;f57A<@gHV z6yBn;C7bXk6{*xRb&#N=hYr>tM-C>}hW*?GWQk@}Gw{gHsMYp?KR+B=JTP#>A#jpo zYCP^b6o8iEpx0L$1zAjQVLs(Mfh2@tfoyWcd8-e?x8she#;j;EVmmOWn%U{~6q1}M zQw*y#auaC5OSC3`Kg7FXRG90CYPm@@?*4I)KgXYH{nwz)EH zwbqEvhTOd2bej~&*#OEAvyR`_Cm}Kzz!;8edI3YynHDfU4XMH!Ek3bxWajb5AM(F)Y4f7%k zX$|Ledl=w34Z-=A81>xR+zr}xMB#;8Q!5aN+>nJ9J*@}z&;`o7kTd6IKWS+rPn*}+ zK?MZ78#>t|B`P;!8db_ESbi3(@^Vy{+jA8KppOXgo{?Tmt)n`EjhhCR8-M_AH4w;o z)CE-P@DAzlz%j8$_9zyblu*_aLcly;#G0eLPsdnZ^pZZ>2A#D$(&4&H)|&`~83UfA z6I2Qnq2uAX27*^nwUp!gN&uoU4hPQx0Vmc}o;5AL23a1G$RN5=O+f4WsFfM_Co5^~yqpCx`5?hD{y0&^8&e#5&>x;>U5N%+{N^UZ+{BMzgLp z8nJ$N9$8U$T9K0y#jBAvC`QFufHv@11eN_+pThAykV%Qi^ zS_-2k3TtRAC-;dgT?b%Z@8vZfc>hzzZPvs-Zpu^4+$pHJL?KO|;PYXi(5=Ien&u z;38<$Rw0e^)D*-j0mT_}RaLxc+9A(P=MltZr8k#rt5i=O%ePikw!Cb628&GnmhY!ien**dhR}U~I>mBWeg?qSV*zwy3#0+iDDOQ((|o5w#+fvKWT7^17If z0&zM4!CK14;|sDXl{R|z}?tZu<@|j zor!&riIkK)U7e?-j%$so)-_3utM!U4)|quUK1&1%2rYQ%1o)Ju+^AyIXfo?y0fT9A zztb7Gm^!Ind&tsGnK?k00Z-dsGqA&q1mYMJ~$rQXzE~7aIvP$tT9WC!b-6*%6BJ5FIHVC zM6hwSqBA1`PQoS?%xE1`n(TD!%A|)0&0^LGvwGa_)c7%>vwU6kvm`4>91fM$atvh+ z$xUNZlSD?$9Rj4Xi*YMrNChI%ez_j-O6IFx;@ z=+`)q6H(^n$v}b%QIMs1!(;pMG?wPHA)>PstPVw$vAR0nt4SzEjGHmw2wH9hNV&u* zcBKM0i$sy`@@Znc6$Ef-Sv+8Labjo3z{LXBm)iY6N|}gIL{5WEJ?YPj#ITYt={TrD znZ`(Lpb3YQ2|hA$ILhWhS1*-XrZ{An#`~xy%)xU{41_L;g(*C-w1lqvaO^dPQ6mn9 ztdjJt9-gf=W4_)UNo9Cq%k6|}V>6B=p?QYsHU2%EX+G-(Gjtk6%boi?e9 z9X(Aufn2>-H#p4Asu7*~%M6ZSb1l#dO&maPqD4g=6AGF$e}ts!;eT)q6_rXo*$oV` zH-fW(etpLT5fyO9gadGKfvPtO#*j9(ywYfAd&8iibjoS*5y9|XwAm-)8pxtF<01ei z+-RbSoGsD<3K*F1tJz|%lT#`~lS_rN3?e$*9G*$7DLp7@Y!)u&rX#BWRcW#^ZI(Q> z(rELoTEC67rE0STE7UOd-AcJkk4VRj`!g`fT&Q%tN>3TLyRef`CN&!jn=^|qvn8QF z)+%*5XH{XVVT-jYaD?&^7*k3wvKR~?wm4Rh_d{E~h=WRf9j#5nNl zFXf`p9|lHGRf^fzr$8duv>ALfjDmXHN}U72C8`}%FQWBP%m}j*N2STVN}~%~J`Du>VbYgs#o2Hy27s&})dw<5)u;s!P7}l(+5(6>bUZSYtUzow z`g}E%U1Y2ib~FzekgTLY(7z1ER5WG=cjn5p37iTLM=7Quxd{x`C+encIaFZ|(yJyp zC{^MH!j2Fq@r$~fpyN5n3&NF(n~$5ZTOReZMro+9loF1mUcHD^%OE6L3{b?9x`~vw zx9Tn2f+{vmSG^8ZmlBk`c{}R>gK43(tulGDH`ZLeLv#OP1M~ z)`6o-SNioyR366KgoABmQdRO~81|rvBGIm+hLd)^M~eN{JP7o?s2aB9&RG0o=!aMZ;K!Rh*AKJ6_F9RMG$KUngm-I(rV3Z&W0ung6@2~9=ACt z6b6{)%o0#q$25ouUg3~V38d~c%|g=Zv(Xg6bhusPFasz!kTE0GTs;cBIf#ouk}|K> zVQs7rBeH~7V94v@6(f%MS*gqUUBX0htSAKDv|s3U&0GbRy@)Khl_H4T)H-CuV1i+Q zlBBs&709gH%=Z{C*RFKRf$4;rGE}*#1OxM5m8L^gOxPkSV_{M2=cnW#P&nXz77dRs zf^7yqGi}txsA*|72v?h&E~Ny))Pd76BOC|yfq^;(kp+4}(m)YaTQx|VRzQ#k1_Df* zaoe(8@PLoYSfhhyAsB6rVWBG3SyXRYiKa|RSDQ2@S+quhgpg$e$tT$;EDu1fSs%f< zX|1Srn;|yNS9vWKrCb~9f6z}B#l(<{4JQr6mDv00V{8Z)&9Ep;Z%nx_TL zp-`?tecr9gNiL7Mt~qBIc{0yc-Mrgpu#w?53$)V+yCZIjfr_BAK*`o5U%+P_sN!+} zGShj+ER5#XjDe#RwaxiKZ7eoW(a^K36GT|hCY?gFt;Us3&L_eEo#ZtUFW~kBq-y&v zVhvJls6tSorhx(yk1tCtVW2slDbyj7*Ha2t#CEwpg?a)b2t`B~H2M?U?e{xohrtFd ztO7)*3Sjx5Zo1t$7P26c)4{>4J*~@ zgHo;sjpiUjNA<%pI<1irKJ6$CuSHg1YY>A`${Y`~Epr0=1U11Ec{D=#`Vhj=Lb-wE zT_N$pJPl$kM7yJnM+_};U3e(9sH8h^6wDhW6h~JIEf2RQ0!g&9xmfcigZ8M{a}-_> z5Qu9RfbLOUzuER@hC|k>g*sJ{kz7AFm=^jRtJL&L(6>3WkON*?y90MIH3#BDpaPBO z{jskQIRY?P*kbdbzNs~6g#EhHGRkynSZ8z5C*i`V4RF>N)CCX;S)e3KAjfFmsZC%j zB<7)xnBAe7*XLYo0z!u+j~m7v5bbX1YPMj(ZIA8AU@jL>TBvdzm;<3uf*ZBFxrU7P zi$aG&bGa#nGkmY1vW1pv(HuKrB%gdgu{BqdK+?dym!_^cgW`TI1#z;`DN}_}BDDdo zB*Yo0VFKIW<#Sojv$N=5YJSfx6VW7R3#45hjV2OaTC* zIHFms(>7d$tyTEB-9wrvKDL0?BMckl^&k@|CIyg=N7bCpwx^shR!Bbu_yLC(m7!P8 z^Er1o5CyC4NbYFf8fKARVp9_r*fNkqq*bD29Q@M2pWd57|!+zDuxaQ zpdhWFd0yZJfdr-S(=c$nP(y_nn|e|f%C%>puxFBQw#ub>pa~^40%^olgkTvQ&5J3Lu|v7s5u#w!Bus;yBZQrET4tfwYz)nj2O^Alkl~W}fg=x^HGPg3xv=Ns z1WVdI(Ixu!(1ts3z5_~6G$I%B*_g~tV5$)%az9Dty^$w+BW-G#>OK^tmea#v}yblau6f{a>lzRe)jW|$Yve9Ly@ z(V&$bj=cn)Qss)$<<+)LrvCb%J5nS3RF%t6AL`nPMfG#C@#&BA(cr*8|C3XUxF|*tAip~ zYHGQikvs%h$Q)^lNQ#E5C>qMvj4yCmp**PT#h@+F^-8Y^W|ToCjExSuk_b`&CBvJz zh*mdC7M*GxOgRU^ zd!~v^^+_8gq^568*j7-P6a73rtt13+HSr-}d1>gT;Yd51rSrpjJp`HQ0ttTPQSQ;g zI2!X_6i}iPkO`J2QwOfsaO@7KAdf_}DakWW?8X!*9`2Yhn1;~o9LW8G>VZVYN_Gj9 zvSss(-|_QJBFRcXqY?nSg)jhbL?HpX3X)J!4{Z>EPxCoMf$fqgr*|v`@#;1xazjUP=}uCdw5igJMT^a5wO>~UZA7$_cAXj& zvy$|FVght#BE1g`72yb?ot`JJI8{7Vl|ndA2>}HbB0EH^00PtfAYU#{#WXroA9idV z)NnCU-A}KiB=^#_^Fl$P{hB9?LBV@98pmBTDkgNcGnV?vEJ)I>GkunetD=q%E3XBCIhE%n^Ywp4XFVIK_oT zo1ArP1A{6Ido18+3;brI+676bP67D;v$_lCK*&Kwx!KP-7BV&Qg)(F8-eE?#;KcKGC$Quy($NRZq2xqUa4DI1MC0JlMI4=vC!4|nuVKSioNyA2Kmc4v#0QceaoHsJiK ztgxv$GV@w~pva?n+o&|{afJf47n5ckWaI6zH?>PqHBQ(naL;(tkdkrgeSiWv@XMPS ztb?D=G#gHc#Cndq!D3|0Z26f@S|(Ctz1{?cKmsj1wS_Nv!ezhl06;ficorQp{D`C zW~#x&lS~vq{76XAi#i{a=#DBDqZscn%7-Q9Qxj`j19-V=l%ASYd-Pj6<2&BxMpK~*PX8(mowteZhWKgyWr;sEL!rR z-4^`IYv~cE^@%GT?Zn08%71wsRQyWipoIr-``S%exI6M?^uOR^{NQyL^@CUMbMaAMy7a7XWxjp*zT2OC%6==IjFvfd z?+-P+%>E1RTz2i%Zy)gBy62xgqw&yN=JTJ*{C&GucgdV9{Oi3gJ^r+4&DRfE&FRaW z1=$C;>~;H2D;t?#EtJ%APouWnJYV#}cAHOr^bmK~)z;VNr`>(})n|MobHI+PP6NMS z%dCFtvRgC5wXZb~x@Ex)zpI^f_2o}&cys$j(MQ)EntAhz>(0FHxjUAgCmp!ocdq)| z+9&S#`F1Ne{_MH0pS6B>;L>G^o;);Wlq@;?Y}TuxUT&5vwK%=J*m}^*KFB0 zbKAnpmtDJ0CMm!0kvkTy+bQ$g1>>_HSpNE7mao1h)4Vg=JfH=zSbe*7r#3hJV*6Kb zykP%pF1YO2&}snJ0JLJbnG$9T!}5!I}>)7+m(}u3ULs@zGt^9l1qVaqc-= zwhXSh|igzdClyW#3-A;P+W-*W-)g z70;YOys&?;B58dhi;lSfhdSjiv>cmXA-KGWW_C37%$(RF+$0g-yy=L=N6ZesZLc$baPX0@Z!3F`UjmM~NBMJmjLzLD*@a$o?dC7vcG`*8 zM%$h7)LuI*orurId;a~8Ugq@7p$D5QcEA1lmwS&Ma?{VRI{(r0Zu;3(zYcENm^`o_Cg@JD%1C92x16={;r;ImS6qDV6+iy{D=W4hb>SEO!hIsQ>(*OeJ%7je zwchF-wiFJZynN#BrA^XF-+bUyX!6d!`~CgL7sdav5;?Z@o6c}Aa@#|^vm3j0r#iou zIrGG?M0=n9JcGaX^|PP;>ebJma3Fl&F1K~OM`|A;TI-&Vw)@;3$scCN9Qd>61$gw} zvKe~x68^#CZoByGYj3>f%d(yv9i6l86~Ig< zZoFJfdIw_PS^XT^`^{0u-gV&fS3d7XPe`AVNHD zyKv_Rr=MKy*uOmg#J_HrImq~XUD*1WFD$?9#)prji+eo;Xv*Dk>cihD-Kf90=~V%_ z?3x3y0a>fo>h~{)6XovW{=ODdtd?|RlMis<$EqV|Gnz#wVgKJ^y;14-*%1X|NQ#) zo(CVdc;m9d^4IUY49cr&&B_r>q;bKA4j9>|S?K{X&o;L2fu|Nf^XaqJf9Am}{`$Crf+zP>e{TNn%U@poO#IQO2puf(gXhjSj1?EJ7=C!gh80^@+z@^K%-hb8 z*PQ+KcY5a@@y^l4)(uv+OvN4nIl$y_tchj>rJ-vmE{-g^YP~3ksa4R@#6pE7Oz{j-|{8bopnZj z?Q1_%PuVPdxAgU;o8RQYH5t*0s|}_1=^L;9)e*0MdV+ps&!dkzYp(~+yL$hpZlb@x>9flYU;p~iTW@Htdi>d&fBf_7 z^(T|{!6&y4piP(DgIxdK{gq=*`q2ZI{BHN_E_gxy%F^FcJKp%`FRc2)*317zWqtFiV;}@Q~zJLGq$=2+{JEF^fNCh;S75Aw9aE179?aOw% z=-P7T;sZal4Pya-we;`>%dTAY8|seT|FxZ6y?Swd;YC}|{ug=xq_B+LwD{3ues<}< zwJ%(J)zT|V%kKRKbJO+ZY%LP42Gv zVCd4L!BI9{c?`Pmm#!`EJpbCqwt;@T{nzTn7Yxrl1kmjB?u+UNgLYeRAvjUA6aL%l zz^MFr-#+dGy6rXr-Jibonpf`l(zUOB!D63V3vTK5m!Ci7PluVOGT&RgEybS!7vB1dORikA z{zJzcf7CX_*bISPU%22KCvM#xc~Ng&%6zcc-Sw4g4uA5E*N=a5cjQNhh-*F|#z70c zrR=6}CiuYz6`x)Dr33fgh8VA24t9P2>PL@GmR%X%{LP;o^1)(H9}d`Y^3K~=pR#Pf z-+kf8;s?ao?KB(^s&o7SAG+vT?{n2%&)Wu(Mm4bO{<;_UzH4#)2Xg1R5195ecaMv% zU3S~Mr*6D!@dd9uaOt%l5aaBX2LM8S`@qkGed2e2?W}EyaU9sjv(V=7ypz@({@bIz z{pSy4Z{_Xa{y+D`XMXhjNo&^6e_v^SKn(JrT>zoFyE0&(2R!}AJ=+rFK(LM1KJ5O0 zI<|PpWbbbLfoVGENznan`-I;G#}=Rcgn!tI4~TK_Zf5~PefzBc!-gp-{|_7f@7a)9 zLmx`5J9P6`pIUn5HXOftX+*EwgWtI6W4&zzE0xUer&_Hpy|%4X0j3At*R0Roa_+YJ z1sIuiBOo~^?IHYaTNVe;R|DC6ZKvxG-d5Ow(RlSrz{bD&++IKam#v(=+DJ8D={sj{ zD|tXL?o5RbRqs1?8%ap-e_tS7|7Yv}Kco5swN0X#*^Pzn;j3TH z?6Urdm%I#@d9`u)#>c<1`4hKnJa)?|FV@>%yLk`xmXns8^P3wpmpgA9_TF|Y`aj+B zhfifbyXe<@U3=5#&?A>Gd|<(<3-4dDYQLRc`)lvQo3DN0AKdli;k#8gxvQP+uHA9< zar$-VE%?QPSEmaq-G-=Df^zx4Yp(>%z=K zH(zxB4RvO*`Q<6MJalA6S@7X!yu-J<;=JmA z9sJ1C&um(dIV^MP736_GxMIy8A3yY|`j)-kfbLlEq;<+27eD@)L`&_Px)Zbn%fFU-qW@$<=Fb%6xmf)wk7u^5=b5 zUzPdOLKj=DWY#ZqOPlT->~JV22!~(syHVz-MX%mcSjEt#;yo*m-I03A&YZbG`sniG zcTx5{bpHLvTzpiq^pon-53XGM=Z}03zxn6) zRj+yFW_!~i_a-ZyrGGg5w%>yI2lvvYzx?ENzx?c0n+IQfcgYjS{^2Wo{qea~r~YyI3w7<3ukAD5D>KPI zbKTuit;M1)uP@6YjfzaEYO>S%3Ahw`#vRN_j_r z_x6o@qPINp`qyte^E~uZUtAijI{Bf`TsS>#dd7Rt6U$yEpQbPPhBzx?ipKfKQqqwlW!q zCm;FK1rPmo<6F``e_V5BZ_NRZpAek7`SyFg)hC|*(=XV|A78iRBah{e!EZR>=EXbj zbMmjBcXwKK;@^KgzVMbEAMK_g?9*;Je7E|hUyI2K_`7%g>D0?keE5ZTH-GNRtGA!M zx+nChqn~+W&s|e{n6 zlV_fG`jYLJPSj^tJ%0bzh0V}ick5G6KL#R8@#f!u@$Fl;zwDYPcxkrXE%?wzP@gp20cjBFMnt4Lq-*Gs8oIk1q`sSHKhNI#eZTzu9S1+)U}o-n zt##ekI|aaomhRQ?u2^AMNrca6m07!W)j(;CC2Uy5Vj(Jc|t(; zz_b(mm(3YQs@k!uEiqpT@oUv#g=!fM^q#Lcyd(;<@48#|EcyegI)iSXYn`|zVNrQD ziC%?|HyJ-Af7u;yzy2wKSWhE|!;PtiKIrGt+LIv^$1nYV`lQov3rC?ef1uEeV9nJC zwW?OTz$QV(;H+&V!;ci1usAw2%o)Lxevdx12-krp`mmrCn#R>ifonJH)lravJ+HWy znhdI+wU%UQx|Unnt)M-LfLDtFTCgvevHGE`YUA1!#O++!;?vCw z1`|y0J#n^OY{R?m*j=n6crEeT2-pxM8G#VfonbE|JpijXFAr%X2jCOvIgKY1_xRjFcF|;jwAND{$aUeOBTI{R?x>f} z8oj!#_|Vf!2bLM^iFn##_TL2-X*G+?;nvag7#~zIr_lyj9FG;F621Emz_NEJ#L@T?09Ovi-t z%s2k+f*^?O@Y9NNh;_C?@m(n?>SACsm6NWZlEU}Wi z&o4^rUJPMT&p1epR#(pHtS_U@u%NMA33#)?f?!`ZerH$NV+*fO?lYKQi9OmNt8`2M z{8cy=m8b^D1geOHGP>h(l=H4iUr9N60aKpoQvA`ruBI@CZ+BShVtcJ(b6`g96%c`l{M$$62UjQbJxUaJ86+sw?RwrH28FGCe{Wg0*{ z;pmV!P6$fs$eU^*3XME+6TG~UrBSBz@6)W!=N8>mhFBDduF&d5g^?ummQP32L-t=o})to&8BKUBozb8P7-)3>A(_4j2TK$(DW+x z-7~*}yxHQm9;_jV9)@pEFxM~%Y+`kCu>Za7ZFpM%jO{qO*>Kna$*-W8WFQss+XgfKAnRqVNbV zFtP2KUDZ>VTXgY&s~~5DCt1VqH=Aj3eU6`DrAvG`=_u zN!_cCjoLOiso<~Wy6>UI-jJShV{8*({lg9L%#&$^MBNS-QvI9)hkPPgrzxpIaWtOQ z9(P{s!Qj(WW_uCqFQm~hF-VI$j_u=v*H#qbV;^g1)}HytvH>aq4oc0+h`vIrQVFq? z>TFPlT;DOFuG0F%6No?s>!A@0ij5Jq!iH}nAb_B)J z&akqad=EE5Y9~JU@w-2>zq`fzep#x|bMvw6km-bc!=n0e#%XlB(Gj+`3Kc94*SG8A z1$XyNKk29tDD*uoE?0MA?7s6$Ma(nRq{Jv%!dS~?K9dN=>_vsKlN{R8?|Rl|w(iBt zBnhu|d|?cTeGXByxOQd)i!K2hRGkSC2l!p`vmiQDh$^lPu*>&BXnAXJsCZ5EHa>0k zSl(8l;_Vcy$z_4EFaq_vX9rUptGjm&R@|`}ynlh;pp9dT20j@+_hTz=COo zbxO^!sCF_1F=+5ZX9u6@NMjIVlW^(A^5RpSqB}lo-;*VgoqgLuYG0pO#;TqA(ed$e z(|e6@F@a^-_8y+mrPa^+v6}-+c5lpO<=X9tz2ak~%QM+N4>>RQh0r&0oJMM%I(R0L z*5XBYW)|@Xr81w~_p>dmkl|^d1It+)4euqlug#_v)paUB#*ngMRJ$|s?P7>i@Ac`U zB#3tIO!e&G$mJQz@J3cgQkRGk*kNZQAt|h6k@6y$trXcoWoxa(U=uEUoyzU_TS;yp zM1pNvt*hx)0LTj=2=JD(D(0T$I0n2^oTP~6JPb2+qQLDe<9*;vW#J5N5vg^3W>#Yz z(V$0XPln5Sgv>^@piNcMp5*kKem&BMxM7S+2#x+(b@rK9L}L{}R$CO@Y$nqd_i=&} z$SXz#x_c?aiXOS$5wRv3m0NY0CK2mv>K6Kr$xunT_8viUtIhAs)bS~|P4qyf7bmCR zEHjs4v9Fd3<-{iUfU^_nV$_z(BRx8Ze?Ec3`?z!=({BlxX5pLb0FjV+t7EA3yLn|^ zBdr_%h*^s{QcdW zx=?jjkxopunTaJ&4b&~EMNfnA2o`W{n`&802r|mT%63B43rK6k87;D_??@q}J;MN^<@8bL*|-TDVXHrkk(&_ZXe!f&aX3OGlv07{|vpYjP)QbxqD6MgoZelb-JdQ;T*MS%;q;qXV2EXxbnSAXN+ov5LEl_ zY(Vj$o|+_%jsZu`M~~n65^iN>Wro=Lraph&UxCN3sXg-cMmM|a_KQqSFYz9Fi|$o6 zd&MBVg3ZE2TiAG%V_Tos;j3VNUoX-4%%yW>jqok?9B^iy$$fiZF0koTXB}2~3hxFl zGR0D0^ajt7^V;K2QQBnvA z3ypF%+OgSf2sv2wyox4@U3m8_dUuA*4=6jNAK!@1Y z5;i59|3risge>fQC114hmE`c=#i!o`J=wYTCgrdJjfU~CTxP}w9J zni6Jy&q^<}jmsBLoGhak3vrs$BO6T=D6l(y>D|kc{MK=zH0Ag-Q9m8AKbsoO)@C&H zZ_NeqAS8+Z`_F&<@+lsWvcT#;!v7az4ZIZp|A#L%XOjbT=nJ_d!~fEo9srmWChD7E zTvP{fzhFjwdSp$~mNDi-S7c3)YyiOYopBMMY@$4-_wz(*u<09yfHFW}oW&DuEA6fS zId2)LFitQ!k|;_`fEECvPhWrW`^qUdO4LFKBBAREdIw@Z)p_CHv9Qzk;d2F>EP0Wu&k5%Z%1ktL_6MS?znF41JEK?g{gnRVDL3m^z; zDRK)F$ma|J_C#{1*S7~5#*zq+psMHW#e;O|UlDxmV%0qTZ*^ho#p7;RE~8%!V~eXu zjlqcg35(wqC@6V(4GKyCWh$?df%ZV!=!cAcv1&)7^CYCR$!FMplLHH$)(d2y!~-)V zhd!E77fxGX!=VLfBRNDQ1ZvIx_Z1iknUUqlgd|1E*^$WZQ8=@JRlVgM_d7RV_% zoe;Ro^wPqU3t3@Z7C%=#S0yD zEPfeoA#$+CSMNTA2qQ5U;mh-&|FOF+Wybf)VFZ#K!Lhx@atK=Mvq%|(qA)x-GXvc+y(qntiUf4g}>_rku#>+7bG6l~fzx5lP3H17E-qHbm3YDQldQ?mNnzz{ zd^|oCx1I)e1}=B?VApZ~OA|$M2`?-v`dteahWabana&(@5pvT%6mh%jOt$2)P7kB3 z1t;igq0Pz1vLn-N-w!l|CW#~+5lE4wUXmk~PTd%0AGsf;QZHm3F&b}C{SYUl=iYfA z49L6zjm%)LPrGujKG@UAo-w=t#??*7yp%mBM1RyG9ix>SmU)B}_XU#@$>p&$MSmy0 zF$|93sWN2(takebSP^F90;vO zL_*Tf&L4ABi4bJ|A^g6(j$%&m@C-MD@?3?LH}@>$>wjUt06AH<`eAeW=iQ-#p8E(F1%3M3NKu z%0TBUx3~N3Gqogs?7jqE$?c?M7wT`?TU^LUEm3N^c>5yawhw#g5SMOAJKeIuidaEJnQ4?au(a8qxW)y zsej^6RfgGaw;bYb^zE$I%v_$=q_^lwe?!lP=qGztXAZw6w`3Q2HYfFJ@>F6Kk)7{2 zO1N4Q(a@J7LgJh0w+kKpTd^aRJ}%sPT|<;e1#h|AI|SMVT-UEJO!eo_7VM_NN0`tT+XHPJTNTMM0{jT-4s9}_eGuRKD(Xo?Bg`{8Uu<$Rz7Pg) z&&A|pkxaWEM$*IB8bqcQGm(2TR8X7gV>A*Dk4Nau6SOsRdU!nPdlWQz(yPq2(28gp z(Ng+x2v-=NNMYylQ~j8WU1zUM(p2U)%NjfzoviI5os|*v0f#qJ*qf6Tg8Q{UevRCf zzXoC^eNY*CcSFo+@RqPPqcE_sBdZRX7&{%@Cx+MnTEOZd#keWwK4KmEc&6WUb0gDA4Cj$l)pV`j4b^gY+%vqATwaih%PG z`oh%rC#|JZ5zu657i74)zKKbSxHaxCS1(pG0meA)TQvz_*of2pKax!g$4{lb?7yH! zNRYxueVkTzO5RT`Us2qU$=G2u%-2ZFXFQ=h7`UBHn+;|rBcDn6alQ)!2`Ad}8pGYg3T`k_8Q>B8`LU~!-TUed z#qScfd>I3Rfp=8|lvjloBDPTK(+@g@m#4(I)YnZ~hpt`CA;LV{W}UglF&kfBSD-T~ z*ar_l6zX?lqmE`@N13i(5;*o%b+UkGf7+)<+TY(^(BspJe_HUFvsR1DuzpAVl+KPl z3Vd9BGG=sYa(~oy!2UM-YfO8HnR2Ghb-eCfCq9kq%~8*WxVim{;Y(Y7z=z|EO~t?4A!yQfpoPQpGK?Zf#Q>sk3wTYP3cn^Sa|16uW4!HLEY3+8dJb zRbIpXKJ0s`K*H<6(3`*ucmXu*&MdsBWj?SQx#gC{tmvRe$;IUp{EEGf?pS{BdD3Y# z%X5655-@UAN9I=SY(Hj=U}Vmj>&XR{nua)15yon&t>?|Zg`cAz{1$%q+K47vFOlDa zA_#9gcCVUS&j3h&MeI^lGl?STS+7$p6PuT*S0#RcfF}sA9sL|`MeWMi<=ACmxnFtZ zjo4K>METPIMuhs)3DfD$U#{HQu94!U%(>Np0ykodBvytNeB~?92{I2#k7{!bI&ED{ z#DExd{pfkI- z(VHh+M$@-|$;c`eSO;{;pZK4+E=3Hzk?8D4(5vWe(ycA)Ud9ChOVq3OSsy0MURYzpSqp=wj+H(=^0gboMHG3v0J?tY=1PlKgVuVURg$J2|sle zsPrAO_1qlq^w_qR={Y~YvI-_(z4tsc_Nu>fknk*x;OBH}Kf3W5l{3 zp6-wijh7iv5&IJYB--WTd9JHssp;wse%INhNlqZo#OsCN9W%%9%Jz*J~ zh!=F-PWiEC*V{3&nA9&i5sRtwmM+r$K%&{K)h6Yeckuee%z@kr z+syI}PAIjtGqU7*L@Q-YFhWqs3hoIuxHj@Cb1|--E+(;4WabJAb(?%$?|7{cY?b1N zB8<{ZIaY2sY3q8|)9Ei>Ot==$Br;8{h+D(OqbD>ZdeL!!aWwl5I?eig*t8-Ab-)za zw1T>ZZ`|_`T4?BUsWih?!V&dj`lH;`u}Cdq97Bead~bcLcH;Y7EIcAD#=qwGNI{w* z3^igXK0fmZuePaw2jv5~#7!-+Sxs0Q&j*?NN*;Yf>m+Q3?drL^%Uc~9L^HknN!ku{ zEbB90;T{i(gH*pc#U4Cm&}aI*(XsJ%YGd^xTEpQR^Z_GXnpgr@;qN_l(Im^OD-0mr3CZ8)9PQy$OxDbK=6CqzSb18ec;f<1>?_4qk^R9WtCB z$oO3s%!hj({NjK((&VIIa~tLS#P7h#f8_WJu>@EuP58L5rUsM|ysRwsb(o`FdFCSY zruB(m!=R&H9g6iGp`0J-lKth`vekoq5Vwpc7N6=t~CDKnAG&!LZ*BxmI*rJqt^VY6#k7yIf1^_$8%pD zjvi|*h!gtshx!ZmKw3NT7hL6->RUssq$~1^&f_T=$bCxA*O5C&6d3C3P}0a@Hx1s@ z#~C^lY-y+WHSTr9*CS=MV3D&ZBg!5!`#jVEa$|^G&y#VyA+?gGZ>4TqAGnjnW@9q9 z#_TY2hjc%^+-P1yF|tP;aEaUgHrVb;*%oPfc<4HuwX;Jxp9e;Z%@k8xTQBb%F5MDz zJB;ai!e3;3Ew%k zOf6#9`Y3T*<|78#V2ZUtT!x@d@h5oeLA6)J59-O@T9=4NMxJx(aD+SL4z|d`mq6Cz ztRViKubGNq;)5T?ho{fS)Rd(*vgeTJ26YcZbx7n+tRxoig16~f{THSSj=IgzQSyU3bD}4t_1Ttc63j!?_3XLYQCGdDP=d@e%7B}(~ochShFn|b7XY96mwe% zRYix#-0wYrx+>4N#7gUSUO${sGjmSww&z>#2fmDekFWhZ!-UMBkE5hUumL=f7?aBa zjxWCF_zA!l#bJJ8v)q39B%2bEV?onX{&0FJH5hBQk^>BQN+#6|X!45Gem2%wr6&sX zK(^TidLM0`pP%L8)qY!EvajH|xsh0vsPQ{GSn$0@vrtVpyP4qyFGmDO_}*c~aefsl z{@}ZQ1%XG{=r=FC_j0vppMC>pBaIY0*EN9NaZub$kQhMMo?c$GAoZsU;H6CqwJV2B zWNikreEE>ZTU{eN6Xw7UPL>*_8@HK|fV~0SGa;0r6Vd@nD;>lypP;_n!RPz-;q0Bt z(crdf)*~Sdg1g|GPENCfd2e`K(k%wQ$xb!x=GA+g$`5y#wWM}y+Y#sl9(F$SM>g7W zCm&~|FV`kz4kw}q^(+#`uJJ!9s>wOGe!rlwfXptIzDO5!7yfnOj>1JocUP@cx%hZ4 zr%DQ*ebQG5-J;E>i|VP+KS_yV-cQK*5+gLE&}sa;kB3W7ben7+cu9O<3v)~%uE9F#rU^ zyYJ6L-n2QVc~ZL{F#?5@{#5LoxkyzFMZ}vu_1hf1r^s_J-@(?83woE+x2p~v(%?$f z3$*vF@W!X8K&c|}Y`B9OL##=IN`j~&9wNj#@V2x=DR-F$1{A+NX-VzRp|Mh=RTy)ZhT~U>Sl);DRJk9Qi7@utFpaoBdz`U zf~i08H+@%<6)9Zf22)kq(8f*wd~YV|9{<*=1Nu9J&7}AGB#KDL^WrXmF<~}v>;-e47fgpkLwtO)1{?QC+$qzpf*k4peW&Q6AD%UHAhX$&oCZK3@5Fl zFO+F?6*aeAb`(;8@h7Xtx%a11J7oIU-2iH#ce+1~Pfp5b@fo&+;<*wV=zOR>Ltf^v zuk7~S)3fmU&VI$_ zQN~nyUf*2mIhnz3{0l_)bAcr8;1_}X^+g8&qfB?luF`eeI`L&*19`d5>e-a~9#2I` z*>uXZe)&oY3wZjY8>PYHN_a+4)jqt66rymUF+|7k@Vw!Dm=8fl!=bNYf4(E>abLu) zb^OXKE6Qw{n%HtN^vPPO{Q=!d`_xV8K`_qbGAxW&qc@TFJk~!wO7xKs!8qho&4L%L zyM`1VJ+H23$CN2nGHQWdK(66oS4O>n^;jRJ{V zdajAjXZkK|sYi5gjOjw3H8Fx)W?fddgG2}=NKJMo^0v(MirMz|$V%I{@~L>Gb)sV~ z>pNE3oRqaE=*+N0TFho%qrAiW1*e*wp@j%iCTuFYUE}IYGwpmQKA&qBwm%m>kEkcS zA%!=?-i;qRU!87Vpo=0(Zcd56Hk@M13u$iZdIownUefsdIl?%l6Xj&Z?vrm9(&06R zvAX1&-i6wig6I3gd8{tpG?7g%e56jdo$L7=e0_U;whg1qu;mv&sF_c)!^M|mckc}R zL0x=?=4@rsk*RZd)XNjZX4SXO=cpg}O1Ht1>p0yTF@EqsxD06zcSH8|G@!3N`_zW> z8hZYz--Oq!wPb(gC8gOiS3nmrUpW(2G;WoGE>k z)3Z4A@6gT4s^5%G>;3|#FCkS_SE>#(>yJMBla6Bg==HF!;==g~e)~5Eji&w~Wzpv% zIp$Hl&{wqGp%!5g_?L4@8OzJB5=>|F7~Br#YVhPLtXDipxV2H6@9((Qy@c=#r=_fg z*0)}>(m3|%J%I())Xu&u{^Yp67Pa6j$cyHdGd%#W7O=TLK!~854Tf89%Dn$F;y$YG zYm{{o0n>-Af4f*{ZF3jo_c%6K*^s2U8i${;dv}h*DQRm0{;ClpcGvsM%Yh8?7B%7_ zku;!5)MqYL?-1l_$nM9UP>94H?OFk*E<2i4vETEC8C^e&Z@fD6GLFG<5e~n*SEzF1O>vOzvGrDY|a*3>63zyJ!(;JGYnKi{! zLC2yJ`C#~Y8Nen_NWVSBdHatSKu9BW@3A4<2>R7jj(GG>Vr8>sM2wsL{)?NVE_?8y zCMk_?8y0+eQcd90J zM83VnS@iVWAut5czO$_l!aEB}qB_N{hUE9IxhVCQ+}1t<2L|H`J%Fxa;_h%kTUowv zO0jN-D`%XL82|^%r~=T@*;4Nz!nj~cd+K`;FxH7nOLAa4x{27~_-y-X|Sra!x1z z3iR03(X6^P?w(_`{ihgh&tR-J6VBY=OWI_V>x-2vPl(Xq5dT>I3~I^64$^$itJ+OP zx6>y5@Ho~WiTz?o;d=$688rgD5||~^hMl$9rk-9k$7{KEjjQTomKc!*v7pSUIP;kPz$7-C8mP0d5TS^D zTvdZu$O@@iQv;@or{ZYf5B@-Ji4RH*%4qEI(0pT}q>VdV#4OuRjA^PH+LPgZwuWxD ztd?aSSf4NTiQE%UY0#ifaJh29X?^yEa`V!A8OqJ8&xXSfbgYouax&zaGAz8TUdmta|1*o#7^*{1&vk@Ou1qd~f1ve4M`5R!QPZ1tn#nYTyiUdK3M-^(=OxNg`RfM)fa2yfe|0O!9-*|i51!EuDQEc*+I z@ygflKg?^mmsF?v%JqAPtvE~p%KuJ4HU9ao1y*v0Xx;hO>eRZRCl#N^{LPiR0~dgT zUv;?FBue31^c)cjY2k)>jmZ?#$ZR(x`q|AyIR~-w%9N$c@NLzSvR6W`MXYl4eM1}gYBsE?70~AYD*0IB zpw3}$qL*XlVCE4WCyezhgPhM__Cn;!m6qKdM_^#TsXY{o3 z^Xy;HES9t0y_IL&u@~~QTGnZ+57OhPdZu5lpU)F*@CVpN98O|ZKcik3<)EsUhLMEM zvXr~tMq{w3FdGtlQCImYAQ7DcsY8iKuIcF}X%_82+}^QY@#iGftQcIOkS--aZ#s%( zkT&wTItI{P?i%YCPX&kC z+FhvVwH&aM{YzZhDXr22m&p^9e$b(WFU+9f$vmGZb9r5;UqPhE#~lexyI$!lb*IZw?<BK{)hjI(eH=BlIgf+pYwU)`y!C z8&6eN3L0JLYViQUW1R4Xx2H>6Sr<)M+3OAJXZdmi5=U*SB@eh+DTVCpNVe#YlF=8u zR0>>=B5KQqdtRkx35=^j9>S-bG8o(1Q#_PGA;+q$5Rv(qxyW3Kk11C?;Ia8xuyyc? zrf1*z9uOjrEM|FvEH;_9qBuAEt#V^%`WyYDvPd!}gE zWCGQBaeRg#+WHISd9RIZxZ$MPRf2@0$-U3Yh9Fr<*A+QQl%A{ym>n#wTZ%p>UfITm zvtiQPzXlf;f{1awE@q?}D})U--$}J|IRP;Do9d9o+8quTeDUN?mfEU^7@C%FhPRWnlpIV%SJK zhRJc^nj?-Cxlq09ZODr?KBdw4A+xn(mY_@y%~Jk(s)+?c$+wH9m%Xm@!S7jk1ti5( zCvL$vp$9;Zw&^=Q&Nu8K_9FJ;jCS!M@7YF7GRA1z802%-`jh$y1rv<4SP5aQZ=kB5 zOv`VGLT19OZc*3R+fcQVFz0VTQA{K(o@CXL!EhXs9 z8r@2NCgzseBU4I2doz4#E%F5O+uqnbK3_76Alz*ngO#Wx4BA(oc~qaq(#8fJ0wUJ- z(O+9R4|vzgR60={>7{UfQjE0^7u-jls>YL)Tf5^hTh~83*oY>*sw^1rWiQxG*V4Jp zyCngB2fLHR7?hEpcFyvN4*H6dIzd!MUtW^+hj4(Qw^ta+6D5(~!`3N#;+k$goKj1- zBH;1h&y9g*!n}>k$%!MEb#aV>1Q9y_6_MsOZu%z6lkR-GG2qnl8B@SmfH`{^uSbiY zGN4OGKBK?68bXuand)mbJ> zClSuvA*ddnwYPTncfOiAU;&!;TQe2Z9~e1+YFRWJGy#brE>hrq^Sn)=xPcLP#Fc)y z%#)%pQ-dTH?4`vVzGDx+C>9-aarxF zasDxTJ*~R*rm(YnC*ln4llHj9^|NP|ll?z9qdwSAH8lbT2l*8sppeR?upTD80L2l> z#gaaMRF;9i#ZQPPbC`sombNa6&BQgMdlU9?S593FeW!KF<7Yz|UeV>Ge1a;45k|D(A2XwRK?!Q85NxXQyc0KNHLBYu zPHU1J5GWuYsFlGkBLV%QFx!9n!LGqj?+zLdX15Tbf9x6kY462x&I3I&c(&E_P55t3491L=*|hx?UvZZ^9g`;%*M zeQndCUISk~1Y3WOye;bm{{H`FeW8+b9zUxZ&V;`CxGRimnC?r1xAQhElp_soAWCMh zCii%5VpS8L%ZdC9EnJ&$3a|mq;FUv>O%KH_dlaqrn6Dr2pDn#MS)|Y0B<8xwY$P%-rnSmX7Vez*{(3K97~L zR~;HlpSFmkVB??2w&m{yW#IM0MA&+k$558%4@>16pN_nrc}_Df)@5t9dw)tCt?Q_c zhZXGE0sLd^JLS)&@A@qE$26rq0sqMc{&`>;Jv`*tuOvVg?Ua5 zNX7fWnd`4qW*RHaMJ{yzP{#fRnK9Y}Ukq3JTJygxnlevK)BEMc^Pr9BPpHesJi99M z*kippyD&o0+j>O~cwfTgE0BUjMoJo zq`ra*sOc8vEATXkfW=J0rULG~z7Mirb}Gm^HSltJ|2sgecl{npG^o_?U1xeA8@B+g z7Q7>LPHgz%H+Dp(nA@SuD&Jd5KtdKA9N3j{0O{B4$qte0XTCX6ZYctD>DUWB{%v#8 zK9MncjvMjjH8c1Nv5=t$e%p0ym+4S?vw-<&B@^lZAkS+xeMFLc)y%=U;6fjCeh|_O zJ!nJxJP`n}qvU9%u=Kl1d~f6ZPkS>prk9AV2f}oo;x~Ewe0t=o?FI>g**x{YVG2xp`iaUTYjZF z`!nt#$ZA}YaK>vJp7aQv-bYn2~L?TJ!ix( z$@_0Ms}Nu;$7<^t^Ct^~6aEa3fPg+EL_$2Ej~EWze)1rr5JKT{{rBFk@##*Ynu0u2 zJgOaY?s`z?;BO0M-4GvQS2n3&XxOCk2LtCWW=Auuz|5tmopmRM!6Ei%Ew^v%2ib?S z=oh?c5aPD%xGa6LyI3-;>gxWoo>`UmveV+JN1`_$y_SLL@>M_`Aw+ z)O^aV&lF1?=GEkfaj6m1I2!{kn?24E8~0KUAY(g&sNXsU5f45b^=<9;FN!BE1hoVD zcTcfNxEbY8lwm=-zGx0XdZri3m??%S)%Tv-hCqDK;?dxby7xuB7K}kg`AB!c>zqE3P-s*ij=^UZGv0VYIc4AKU``S<5Mn1y@GTN%vE7Oe$$IT3j{5Ike2%SwBC|qu zEky;vGl7Wz84NFQEHW*skGXGnwuxx;ZDZ$nky7;`3di>fdjnns@3-%&Hnkyi|0^m;K`n~njL7Mc)TPi6!Ii)j);k+o(0YUe%S+& zt9btYDrSyog{1Na%{0$Ef-L{)dB_c$N&7^j=sBXD+#!(V0T*#RY|&OfQI2>L7+!Xp z1`qvab2UZn!dHlWg%lL{3WajhDH11>4rY}?UD_0G6LKT zD^i5Lv^HAs-oTM7*o*zyD;0xRq9Si?|2)l_<4|E)98m?DBKb#<7CnIS@uh3OY1810 z=aJ?$3hWj~lv6xmpx39rcHkuUCY{QLV>Nw#Nc06&i{!G!k{?gwa_cHAC&aOLuuFM z1bH6h0MFuT`wjlUvK(pxs7T9E44^Ln!)byYF`xMVwJQN<+-o&IY_R2qR6iOpe;MX% zc>{i$c|mDLw1-I#&PO?l2<^iHN^yX0v84uV7yf^~ew|KvnQ5p2u$h#3FGuRYHJSs; zyF zb#Z2<3D}M2&zF^AuSe^p!LkB`d5SlgbO2z=N{)I^#^*DddoN4f5}yI9u7-vSHi}Yt zY<#nsIl~b1aZW?<&5?U6-x~`ok6(}puBibTn?A^08B~yEYCt{tt`oScuIYiVv!yBU zry@K=GR0=86uZM&h8b50Rlt|Q=nwwx&)g?vBk+p>Z6hQ0j4DG_{^O%3Vl5?=)dS5C z`7p^60IZNXE`X9{gZYvm>&k3EvkK~_U9PTwJH@SK7bQTGD3{VE7E9R&U7-2T34Flo z3t8n>h<rHDYj6L)(C<$zWFQfwOJUiyI*Jx5&;p2^SB@Ce z2lPj4!EEhAZWMfpB8t@F1M*(|j+Gwsl?mv|^YGJ+j@pFoKa!C$LJMzt(s-J@s%fAPv(NSre^A2Z=*l1(cBz*3|!733Q53 zzyqj5J#`qIZForqzMod+#KWiVw)-RX@8)8%;*T;-@;+b(KmOWxJgDm)vhy8q^7fbm z@AAi2esVf^gT46JPYN{1f1iQV;s_cX(@S=scvbgzvon7ohEh6N7Z5eFX8QAk@VM`Z zMpXZq9cJJ_K^SNnw@-*TVq7rT!G@@IX5lE4OwUW6>UT zkKAp|`)48y8m|Q0cPvm%po*vV=)zc2>vW3$9emXlG zF3L%M^tG7P`ibnczFO2g5)kkarJa-7QUS(FO=YO~cb_AF&?Deb021gfZQCKxodI+r z04Ne>D;Au<^%jRiz@u68(EBP3iKe8$2CFo>DqH(TDcT{)dn&*_Bsh-V^L)X-RP-Zw z(n{+@${Wr&t9Pp+Id^Jm;O>r9>Fdby%>*J9#^h6CAss_w9J~)Yid_)-;kB!-ky1#NEekmP!zjiz0ZohFd{KlG0mT!f zG*8?xu2~w)pUm5dMvfr>TcP@xG~%BYO|q|0ge4IrS7Jh>51o8#f~@SV64yWo>&?&gP)nQ zTD06O>$P7mNLUL$lg;J*1lqw4tHg=l2p$LPE2{tnPa~frl z&R-E!iv}+9)cQp<3I@?9TUoy!a;0S^J+TcA5dGqDy4ZRj-K=Y|J6As>D$;JJ^S>eW z$%hXVMkxdO)wxBPS6*icc?Lz4gMGfja9rO?xRyqE33=gbni|JTUa}ic`%1bnc}51e~L(H>KyjD z*-8ka>uaEBgWFRnNg<C^`iy`6 zee6jV%Eq6TA&Pn^*CGpK;GV}sLV!N7c+4nAEE>x$|M$}sU4*HI!cWrTiNtlr4iW@u zB06xpM7t{s(orzBay!2f30a#em0&Z-t&r+p^H|8uDKkr=SJU*v$^o#qiOYk+6jfmJK$?=FFl z^&cf-nn?PoAaMbWFQRdQn`E{@<>ez1@)rzw!uiuBNqm7J}nOtH$H3col3$*GuNR>|sD|7>&*dtJ&S z3S{84Sr!sk&A;b|o$uc%68J?JIRNO-#V=~2{V>CXSjms|L2F$L6gcZXD0${!=k?5j9X7$l^Lk!*i;q!N3^Y87N6~ zbG{nCZ-D#03CIdcFjbJMBnQpGP7eeE#1y+dR+R`)s?gObK{Fk?pnT92CX5uQWcD1@ zLpY*}OUi{cV1aplA2qs1>!25h#bIhGJZ>Yzc#TibyL1#FQ;a)ZjZ{{QQ@? zs8fd3irj6jI5+9W@Z{{TJR5mK;S#hD1V&*0?(JcR=A&&=uGvLq#DKn`8k;j>!2|++ z=UaX%xDaUZie2wGW&Jl6pnlmgZ|feEV9`ZWs|uw4ol-)ZwFE_-M|va!Ghk0AzX!+9 zeMW`pGs*$q+-D0vBfjjMaGo!hz3646 z&YCA`+g851X_E@ds$erOTh~(JJJ?sV^_w>W3*_dr;_yS)C-oyQ2d}n0Pabr#u$~<$ z>s4Vn)c1|?A=q-6pOT}~XC9f64)Vb_PUU#C8Co|DqMt7>?-|B;WQyO%AH-^>()5TNgR+b0)*H7;uXZLAWVrO)d#)YVXDLu>$nw z4^ySrDbhEWfUo+Z==8nQnXehs;*(F7-Y@pI=ML-l6Nq$ykufcr>RF^n;4*tbb@qrr zV61~sjv9~R2N=|H49z0Sjqfg4jFRHr%$mqWn#Qs^;%BgcEo_bJ^)Et(u(Wxi*7myK zdA$!p3IZV6<--$p=;jIRU3rH-C`LuSfoJV-G*fmx25vFnr7PoSU@`PD7juPCeb&N-agpu>8vU|HgN0c zQpR4lu_I$qo4=c=nQu18oQR9FRGG^fHyIkVo-Vw`K_)W-j_M3&4($k@r@Xbpz-_Tn zk&=;;j<>-1?3K^=h%g{G!R@?#HmmjrdCc`4>kJKjQg&-1Oz|7XCd)qxw@Bu!Q#J>BwG8 zZDpbc%$(P$#F^T1g1p=|G4jg1C*UmM9BF{)oBHvA?au@R?x+TczPc^9(<8!-0}A`v zO!#8p(uSR}HU)`t9#wqG>ja4D_o54~0AEh@-Ov|@kp7%Xv%o)^?e%FJRkn(BglWdn z1tT)bJ((Au{wYaj1CdNvq@N$Mgb=2Nzqn7U9FZV$q`dCQ zAhYiINb{XE`C4AC)MwH@bsF4{y%3+to2+KlEBdRQ$}@4B(#r6w=92xpNpYLz&VAiZ z@6rw5EGi7V)%F7v{U@vS=c6S@>IwFv+ESfB+8np9*Z5lb(x+@Ax}lX8R2M_b#;S90w-qEL|CE#>&#{uPjl@c_zW*c2jqq}6dO zj7tRJM+(1b9G8wFt>{ie<775nffeGLvMb;|HLAth%gbuy%!FCLVv zY2bL{Zy{BQ>!4pka#BS^0I-e`J4fhOBkI?D6YyU`VY@FN2%9i(F-ROuJN3bS3lTx^ z`m{Zd_K1NR$anTqHm5BdGZC-Mu!Ln*CN%VG;l<*Th*DGZ2G8V+Cyy5odDjyQ7IR?; zcuzWPNgNzqnqyo(A>9_cFcl8cri-TNkUUsygg*$FK?g-Mjn-53o+4n({GU}6O><6Pyxwm09SVIDu`BV3y zMOl+JzvzxAOXS1oO0mQ=Fb>yLXWN8Q*PypkBIcm!u}2PVSNi>h!a44|TCG}@cWVgg*6{Y*J=e+w{nv26LtzW~%1dQ6s(QXusgB!+sg4SlzNNoD8uD3b6`R%u)!#wYA{gUzPvYHkwGVB-YkLBfM#Ewv zDC~l-s$^#*hI zGMN;w+5^)ooEFKfD+{G6iYYZzSvlQyU_5}%0-XTcnf$P$WRo)}{F_k1P8?z$v!v=$ zr#yC{!Hi9bHFAG!Q?{|J(3lgT==+yV`uo!{7Nl+H_%*TmQUp?RXO23v%*cRI z6)pEKqHY8l-EtGx#;}oOQbwGa&Yd81K1J!`jx#F(H6-ZCH8EubZkjyz)yighE5E%g zn||Ut?N9p=g1C>|LAPRQiX$OG@#Fs*slWrVB~X=vy6OyhESD=7Sp{)FlF|g_FCP`h zi)s_D&M{%!%rs>V5v{O5 zq^xn9NZde$>Scaj{I_Yaq!^%Q;5girc7MVbCr$nFJI0QVLKwnb*{^EbVf?8flDPO# zKU8niHe~y;0&rc>2Lfm>_Dba})ul#lEw0@*6DleMAA$A}nV4`|O8g=qQ2!kYtg$dj zUB%Ti+ZU7{7fI?Q3j2O+4PQO|I@D{gNK&!1Uy)8gtO>)V+s(fkz`Sy`o57Y-`XE(J zRY!F(-d&L_LG(;|{krd_mQ+qWSOk73pZ` z9hEqA`$BV;g8D#RZb8YaN?ZxR?2XA>e3=Z4gAZpj^qz7fJ|^(HkX zC}^HHckENu&hyVbIJBZx-*|umq{R1OR5fT}Wlljrf`0%|tAIX)J|y3^X#sh)`2V}B z4Q8H5kD8jR7uT3VZQD|>9|1PWu=*3AL`w)gN^{vgzK9qyE*x5{9pF(+t4v_?m7DyH zJOZG6Ksico=F4XzQ;M3d0Jjo?{paeC|NZK%i4qdi$d6L4o={tul?(-Pj9L`HD(A>c z^4el?nXaG%VUH+42?b$n5p0!dkJhaVXs~ZXwM}%2>1Xw3H{ZBw4!cT%_cvdqgW!y> zp8~8clQOpcNr$-0I+&qlgLSnINYbnWg7vvl<9Lc!THhV64#wI0D&)$h_W07S7jcxD zt^L5L^Vnj}`f^PF+HEreZ1v;(t^Gs#H!J9z8d7BVjpgOMwf0iAgeoK6BRU@*oyk5- z>D=HC$(tj*ZD4F}ta>K|&t>_(JU&r+0oiku7?E!jIrP!5wB}e>rVIC1%arYYG_KKV zr8nzn7Un1+s4YY0`EtHL+}dAS!6L5JwTU zk)xE3$$s9O*L&uOe+xOqdR3}#3K#Xq7)#(g9A-|}Q;gI?zVWHHa+){_jlLCIGCxaC z$QiyHVGCK*^`#W*d_7!$AYDaHw@Z1vu~MTZwDfs{vD5c#L#}4W-DE7RdUM1Mlvce< z$-j;p39COGrXoS>-3siLSphp$12tt9K`7ZqQwCS~3Z3isl#?f#Yw3$=$7PptS;`#O zSbdJh1#NGk_ns&F<6_`aL`?2loMd%|x$c|Z6i6(u57fe*W$8%~4Syq;xkqgX1)Hp( z_d7_DeVL;i%8V@UyrSjrWR^1PuO>gfvg=D17|oN>w4J#SwI6yL#hpo5`yANoLH4DF zndxSuO;+dUg}#eD@5cAJ1R4Fmzx+aPKSj0+Br?5A1zZ(f__=$)W}cE_f>E80E)eCd zc8VxpHOA?81?iHFmg>x}0rRy@2HsuIEyoP25 z|6{*vFJs=U!V6M2zdM3Ra?apWgR2LfDl{^56^zY5ybM|rS@`o2VJTfZV1UBP$N{eP zCnF?l-sTsI5@mH}8AvSl+vy!V8%=;;EgRy_CF4vQdfyYj*?WIc|Jy9IZB1E}FhlzC zc=9P^_Jh?B9Oo4fPAuV;2t(-@wa?UR4suW8;R{-Cn-_z3P89EBgP{}R<~+CL(D6T8 z_stG|vjCTV!pt`_-m0xxHjG}3f%#^O^V`D3yzj}ymC@B9MuGx;zg*8zUwn>?J&>eh)9E7x-%hCnvA#u) z$m|8k%i?gIH$<}lh(m_ZWS4u5)h-=CDn7>v&O1o?i@)O01bMsN8IzX1spl?K?3H%v zWP!#X`h0+uuT|8PlH$-_>3m}aMBn@S#3Oj#$L|eOAa2;N-GAHNZWK4^?>gDuteW{$ ztS~(`Z@?h3Xq{*(OLb&=9r^D9vPzLX1>{Df(}iUkcoOE;oL}d$gnGDM$HQZmvnSMw zgMM={RjDb!l_ha+VNb#eGJUQ7_HY%)r+izd>rK$kL<^2-WYULl_)%0pG9MQ^Bxwev@tn1 z4Jiu(^BFW_Y_YPz+A}$Nt??Wp_6jqqipahH5c=ZP2)cD`i-F$K;B?~XUbUaF&!!NW zq(Jx2cZ-}5aIx~RG<-IxqLpM0V8 zg4O6Too(Y80Hz^B_wh4nbkBc#!8V#}Km<92DjvD$zX-@#8SI~GcJ%c)x*R{4UXNL~ zRQ+ke{W^-Pc7CtxTsoHkSj7pCAbX+0`%3R)vsoLyPuNjY z&H+FSK%=_h*hn4ix=R3+V9m#u`MP9=;P_DRLWmpU^t8-q<=7k8>y5C8Wb0L`J39YHl&Qdx&HO`m1CZAu9+B=7TIRXxF zZd`!j6yek?9484q2JJKf=0xmFoK}JG8#@}syn4Gof*xx2944R1r;IyNJ?ue$e!};o zKq=g1V|b-F712zZb$-qawIA|$@brv6>Om`G4`8q@UCt;mXP^&;Gup|t=8TnD?Siv; zfzZorjV(#QlDWf?UBkQj{*hDI?bv-bgG1m)zKRz{;GND&Hn1X`k1z;!yPI|D_@NU; z?oTPrm>OBmzN@7k*;VOji8Jojyq?0PK`3}>s~SA=A3)bK5yr6VS+ zTZd~^%LH@1OV*XB#r@p$#R?a6quoj;BDD9pb?rL^p8U==I}3c4%+s7oJe-ceG-RB> z8fbxcyv^-|4|UC*&0<-+`=-uyJC)-4#pLMJ|_dF@UITk{gPb<~3zE4@< z#g|H&IIg@@3%#No^;25NuzC@sciON7$KU*6!*1xa_R{57g`A7>tI}98%q0Mbq%O2v zUTgj=c}QEMElW>*oE| zDT5N!ICzrP@AWZI$&jfAg=H1$9%-BWTyzum*!)5QC4bU+S*v*bI01(9<6gcsut%S@ zZBV%dQ#~^6?_o6FUEfE2&hb53iD<2kAufqM%Clb_|J`i~qZE{i?h=LQ)PPwjxWTVJMvqUDa^9eYxVf=Q^@FF+X`{30PZ+~`E62M>FN*_fsfOfh z-C|;(zSSy|KwnhrYASx~<)#2zcw+9jeYWl82sXuIMn2OQxj`0ZaSM4SyU#cIjjKqm z_sqYao4{WQP?bo1byh>S4VF{|WS+^14}Qo;vW?@-5poQ(NP}tx-Cn#jFBNaI`Ht~A zp^eP^(rCJ`$Fev@e%|!LIgEl9u@}CM*-6mpD$0^ZL9rkC&6quRkU%aGMg>!T@I;%$ zod=I8?izuTl*{FSu0Zs5Y2(E8-QEqWF?`nj;A(iAv^oDFG^1|rXBf6DcOW1>jz=8# zQ{|2dR=SM*bDkl7tmD}&F?T!d&FT(hiZgI7_>pZ4Wy)+htEKf>{Cc|;4eK-F!}Mer zh>O`?=tLu;;3Lc5ai258Z!2y1`r@;6Vp66UZAjL~2s6z4mL*r^cN=q#3v>YI^AE5b zxR}UyEfzhIk8+zMIch6jSGwzXc_-j&zSN)86XRhJ_SdVB zbUWJTb!SuvOMiUnTRtV+Uh#_%uTdAsXLUh%>y<29cy>V|^bHld85%mLwwvQx<@O++ zgBRQLda;V7MDnAgfF3IvWrAk3ETEErU4IM7l&TIQuHu8CKbs!3EFfO~#HB6RJvm9% zO4lTvv}-YL$s<`y_zSed7IT~yiaces=XelDj?L@<)tf z6(U~mn|zu4R80crf+Ur5Mn`>7N6;u|G!ISPo~kW7*8}_wee~5~Myhm%m^>OEq)9bra+v4BjW582s$o$}r{q z5w?U-OB%32{~38=#N==rByJqz|86XZRcmK`FUPY+eomPE>-+CJA@(@5h6*aLT_dlq zWG?Vl)(28G%C*E-oQ@ub6$Ntxm6RugJ@HbqIi%~$-_&_RrMvBKants06igO|8xa#3 z9P6CdZ5o;bJrU2&MnWQ+1?_aA;I7|NY_iWtvprvx-=@__yre zHceGtVIA$;iTd}X3KVG6IPT154r@fxCgqq}!hYeEp5M$?s$wfi3HXm%S3xDn2cF57 z;`Lt~QAKZr_J6Bu`rZt=uvMCMR+3kynL#s~*9mBm$zSfs)^pnW!J?d2PCl%~6Q%dq z5^R;OyRkK98Nbs0Jq4UkN85U+KZTQ~DrVSo8KYNjw?e;jx5oDyyc0+}EEa2jPt4Mf zi4!L+VE=w5(6sJ77^O4_KDI>;zRM5}R1@?K__3D#c-V%=CU@$>?N0G{>j!B+)o`~G zfnKiZ`0w-GWZz!r?cMe59AKricCIg6*b_Eh9lbDU-qIWUnSdb>p!F5Pqq3OX`C2U* z%xrPIit(;5vAfF3Q-w<^v20j>sLJ?4{(&R4c01Kc$H#O?LOJWy9s?~yZyzcffkWxd zfJe$Tw<=zmdO1)h0df+aJFYoE8fW+GTU12jVnsp0H!XoIVM9e>Yp#D~z#ZqD{&Ojcf*LjdVk*Z7y>!dSy#pK5iJkJ5oN% zszg^wAMzamjGPBgtqPsSs8Ztu^TLm%XLx)^D`eD=rs~fkedayGqqBOdiT}ZT#k@Z4 zDaj@@K5C#r_*pio%)!MFJ);tX`+4DSC9jfP56&VIdq3ng>E^#$9z{i*=FmL#62y$l zh%lH*zOT0zX;Muz=zw*{hmgzmrzm_EMLrbeP8lnRLrBUA#0JbuKUD)jDi*3C#^6x! z#&QCxV{sFOOyW|0`e?PCOya|~X7v%D#v7NW%fm`1so9JjKIjxFc+Y57WOQReVzxoF zuY_=G45u=`iIA1T;}cY|so60$RTlAiu=FkoX0=c)@%xE=OcWLGB4AGNTRD`)#=p|cpakcb0$KY9 z+Xn-3Hp}mwF1`<@+DDLqJrCD=0!R{9ZvD1*b)7C3{1I=bQ4#3S){2UXS~6we4#nUUOj}a~ z1g~_rAyDsHY^2Ce2!Csarjj)-;tFIY);fzp&^+tII-D(na?@D*#3`&wH2D)hJicF& z*P!VgR%MU)J|M__eKEs6hr=h^>lqca@Akx6nXWQxm*8}!$q&k^_Mg(K&_Uv`43s8O zzs?M~Ai2^#8MBk{2ojG@Z?eY|<$hH*8;*ZyxwpjF?34h%uqCh0Cm3q8VdPVjanS&{ z>(2@dggp<@W9cskxrFjYU{Kwo)_c9~lLUwa)bmHsbJOZ&Bhs|)!+ej(oN~kC40T?^ zW(+yTra5fxZS$cu3xFp9lf8kDzo|c|DMTO4C)m$($}DL1a!Y8?0w#QKdJC8qVY}MZ zt`WzJ{?4>xS1Y`%<>mhE>vKV>)ah2x8EZ9Jkx;uW|y z;HRO~mp0L1)TdW3sHy#S6u;vFvxO7Q0yW*hUYl9c+q+he&bL9*3F=Db82<`Zt;ASiQLzFWWB@8d%^lXbO3>X11a$ zes#;|fnS+{u(`V-T16X^)pL^27jHKwfAvT=tCM#~wDk&Jbe~tZ1{E+}={ERbwP8wG zDJRi}97(H14~QtFR;qy92WTXDRBQVw8BjQCIZ2V4wBJX_EZ9W`U+;g!VgMTo>M1>-p5VeO4ruEG24|2uxzQXCw#IQ;d0bsKxwx1Ra+< zW#i&L7vSr)!XD>FWj1C*$;ODe5(itVX7gYD;LGG^6H;#NFRjF%x1e1UH_(G)d~jaw zgDsh|^xt=g-EQ&%cGyZn*{=6(q!3)Zv>HtIRVuFl9#y)O?B!R^2(h3oX}!gayBguU zao#YLQEp{-W{su^0r)ETxEjjwPvHF8vTJB%wZ+u0Nh02 zw(F(R1oYKdKkrVN-Yx1_3r?+^wJ70*8+PN+5?zfraxf=9#C#0^*u9|qyMqX7ei&W8 z8a++8(~pcbkL$1B9I3#{GqbsVPkhRNX+U+}I%KINQmOi6r`mTlKnhMuPGCei=WmO= za4~X*TP{p3`40Y&^m}7m^WAUmm+|qZi!X;hb^wgR?Y!*S+vw^bBAPAOhU1sd|BwXm z3e(RZk63gS;-zoa03Y;$nfO3;D#P}1x+=h1dLMtv7Dgm1;&|!38m8mS{~0jNLq@u@ z(1;~u(QDkou9qH;S*WI}679R?FZ?UUoBG~VEtdP6eiQUkO*yx#IB@ev|2tQD_qoh| z^qX+!@=<$@3!gWr6r8DeW^)O805z^OvVOz@wZ;7Q8}K?v2THc_?x=72)E&12M)i|7 z(}O50{q}^CgwkY!zX1OO!RgZDRn*X^+@z$8Y00rLfLyl?`0@yOYYfsB*XjDl7OzNq zKSv*8XQm0e4(y|I=j z*ozpJmOPqaLm^tSTX26)OIuJ+h5P%J0?`|=CTF7a_rsoaXEl(=Q}DM zXHVH)7e>D>Rw_05Vo+U0!gqlOU#uLYix-%@(W$h;`aS7GswZ(lz3+ES;5?U%#ZNRF zX;a!vnBIDefw!UZq*3ea4+&oNj;RO{zTAV}f77)BdfezIoGm$ZU3t~pUFe~!gi#z( zzLV6X^j)oyg+&}eF+Walm`l3*;Vmp3Kjo^r&< z@w$UcM6cqK45D8Lt>Tjhk(pJ9U@lTpJ#2-5mrQHfNxmdu7sr2q;b1k&r4Bav31d@u zBT@$va7#8rJfv`)S`R_B%UfqtXajdKqNlBiw^w3*f!62LSs0I--BHPTAtTPPf;H9TPC5y6Y<~ozzNZdq0Ta(N2Cf zg$R;M^~Bxl?lIaGK!8VIpaN}%a0R+PQZp@5(@t}HYU_a9j##A=ffk8M80JC>yOamv z!RqCcKM|cPkV$Rj-b|Z7YkGK{k=cVoeXP10fxX%~&C*-5ov(K{*>~1w2k9cm&uv+b z-?;9IN;9Ob8caL|+6VywpKW+3dO&!M=h#T0iwop+E>~9Z1u1K@c-I@btA$gvx|>*0 z#Ly3n&Dq}+V=+nhymIZ5ThemiPd>H@_|barINDv ziGSnUEmEZEhPR>GjWF|1SqO{G47BF$ilnxXi2|&bYwz)}T_?n2JPVa*6gqi|LM1cSSnyhXl{)Y&WInA*_#}sJr=lLc^=qe@{E*X7^T) zCn?yk?MKevcVWPpy6m7_d81b5Bgfu&@%#}@&jmmKX*I?4p!SPC;UW3PKEuAU;L-vh zFXs94p1(ov5C1q-OOOQqD+gp%eiYQD>5!PS^gRJA9w3Yxi>VL?xvkzg{?RJN|7ewd zYw7=H$E%b~UjqJEkgf`{Ux#D}*$CbY3L3zN0mN~eA@I7=gwmRT`^HNgvt1mg@B_>V zRFQ_1Qi0fPi-=Hz%@*bJv!9Ah;%h2eel2pu;7x$a&xFJmf!zfa4yIElxGX9t8HMYlJ7wd#?0~yk9hIwimS3lOn zrZl2es)G-JWYnzF}+(E z=fAjZqnBkAkjdnsi}(LN@%}rpMIOZw^k`MKzsWgu$VVQfOnMQ?WP=xoNOi#pFYkZG5DDEHDaha zY_Vaxqz52~%X~)lA%+N=BHuH+3{e+M-j>5}B=THYm;QPawthy`^uxIqVETcmx_%S1biWC@8_hagbf=k^5&#ZFQ~JU5VJNoY0oP1;mk-u$sxluO63-YPZKYo zZNH*%;ka$DOfMijBm?v;Y_zsghpzHc2M-H6$PVeBnGNROJ2xH_h0fI_UEC~n#F^d> z;A~DfJpA!lDLg*di(2Z)r`U@cK&ybum$b2b{k_wd8Ca--ROg@whbq!P-$0PogOi33 zNP<~kNFynhO0F2CY4%SZ5TTP*`D`Tn;(E9bi8Y@YS}1oP-jlt`W>ulh=4y9RrWHj8 ztVXHdKds7E+P=Og26Na6WiMVhj$9mVM1$<-Yto4nur`S4*x-t|qtD9p*r%dlfX|^< zoH5v?yFj^^6wJ|SP>ftu$%iBW1j&N9V`OQ0Q@$U1^hz<)xW4a2yatDiLqu*2@Z>?SX134vTwfSoX zjKNLCqZlFco8J^V4eOeKiksyUu>lxHL!TP1$d8ZWfD$N6v-Op%En(6tQ3gHAMjSvf z$-UrL_qJ8pcYW{Z4g#?>E!265iq&ZgWu}=>G9>|hY+)WGYWkn#RAf)iy zpF{K<+k*m}UbEk0V@GRxv0VSovx>ZBs$sfo1b=sO+QXcbGBWPb@FZsF^YI?sR0*}r zR?@XcT7q|k55vM-#kF9>S&uH>lknOm2d(#gAI<}!9GrqePlQS|yRi#j($$#Vq*2I^ zsP6zIY6^nGk%jN=-vl1@)RBTNFGfOEe3N$4nXxGVk1fi# zZ^#_DK=udkKLPQ<2UH)tSLcPf$iZW*(I9~&0!AUc3%u^do#{$p_fxLDzK?XNk1eIL zCtoiYi)apru=Kxb8wjTN zEd4EzxpRvVMK!<&7{iAI?5~@EBU|Zo<864gzeNij|H3K=)F1)^ABmllpKYr)x8L#i z(#^{tNTj>ZM*7l_AlIHeRmOV;xgT0GZb{X=LC1PH25G^|x!;@n9u%!=Yj zhRBUW?oBJ8?ZsXT&qBZ)=0g4sae#ZW!uWD<(@1t(eHfx>6-GxKN+TIh4;l5 z`QF(6sK8$KSG|s@h?_O*Lc<{F#tq)18p^arMd-4>uCp8*GiMZBwwSLV8dS>W7glCC zXt!)EGetmEAK^euiHcrP-1G?KBs-V$Yl8aJC!BU_S|GCj zMTfm2Lh0N+vOhJ^NC=@q&X*Lv4yO}XtlTGBNvpCV`}(-AzF@MXE7SKxVtJ_FCEQB1 zy4B42-oBz;=+4dhFtyCI1--)Koum;95))p3+jP>N=OdrcU8^~eqB&lUNmtawsNrqj zyX1dBhc9zy-^c~tMm)b6Yj&*KZixz9FoKJVJ=90>mCSOOU zsrNcl4mi!&)qV;YTvJLO{idn5;;Zy^Sqnd=bOmqKiy(iF6gOSHj7Iktw6@G&{L;(r z^QsddDHV_C3><`(eN0PQdC>0==-CtFz!DnIg3iM}c5Q#?G5 z$WyKe{_vEpd*Xi8xDT3J5wAcUdHT9}Y+9wU$N# zS>_C+l}3~y?TP*2akLJP*+x*RGC`GBNixqKoKhv z9y4_S5r4Y@f&a$)0f_8ARt^UcA}GcfjBNdb1^f0FpgsNr&>a64v%iN4+42YF%in?K zMh6tk4dmemnsrtSM8xf^r%W?7iuC*%O4=y{P@gFGax;%ajjUUVruDyRBMC}OXaTw) zT99z+?Dw}Ve}Z2nH%ZH60>kz;e+nSa6mcMwg%4d)0h9-N8}$mC_D_R4Su_-!G-z1` z9PlXswv$}Id}FiGw!V}Z6n8CZO; z4bz}}jmVpci-GmZ9Y6f%Ygtj;_oJUkds;D%P0)Ts4u={DmTVwKP8hHf5dn?vaN6W1 zaQqvS7&+vBkrKa$jk5fWH6`Xi-^>&tTqF$`1MJq@D=*L_s7|gN9B*nJBZIv7_5qVr zUZE87S6QYP*nEZ4k{|<=XpA;rrt>Ttq$!?NQe+wb*zoV=vXV+eZwZjNHvn6f4+ZE; zen9*L{AW&3JfZ*j%OhYao>dpYS&&m<54_G>k$>*_AAbQ}0~7gRU|s@%c_F9PmHCs7 z{r~h$ha;P`$ms_VIw~*+8pdx3G>~_%=8Qi1W3e(5e+3@KJYGQy#E%AG_sZni{P}uW zCS0HwpKM_K6ItZl@5ALM|GdIG$xo#3VG&yW*7|>r7g0>fpRWhBArq4n8HW4>F;d-2 zGW_TK#Xkp9AzyG80(r;46CnS;egR*U1OG>V<_(oZS&|_QTd*XI6}jU7`NXRNv-N|! z2=SjJW8^lFL?;6#-FU?s`kxDtzYuRB*KV$ZHXFHi?*inQI0NWVcCZ{ZbpCD<=3rox zyqVI-`*Q?WNsSMD7>PdOql}2ve{3AO!;-|TZSX_mBWRNZeMQj+9e@H1P^xMPB>h4A zUc$Ac0n;FRm#Z0BP$nn?2UaYp=84kQA**VL2CE2fH6U6v0@b7ZL_J^%NJiTYK)Qa3 z1D`zvc0t6e@f73$+Wb*(KoA_5YLMm~=0Wn^xF;kyC_hx`?I$dC0#K+N2T0E#D2(is z5e>Spq%B|h5@)CwJHRxsV}~0g0rDVkgMb#$JXiy2P{r1sf>QO_gj{hFf0W#bV=dN& zM{*qZZs37-QC2b3g+>TBA%AwcEU?Stp)X_p$fj#HNh!@)7HiV1IZ&GzXzwlexEF<_ z?<9J}&+%zxHt2)|6(y~ra7HV0$%E&c3j1E-Oc}WS{ zTsq8GCss!n1JV^iouDGLMINXjs$_dxgoGSE#)}p7;BUyB(s0lRcN_w*NS)Uxb?qzDF|2MG*SA90*CgF z*ESQ7L&O)zg2hC#l1b^I*J&oq=BEACos68yOX#(@G0Q#i&+He1xm zJ_Hz&{71APp5(L@QhY|B{bJ5}4)M|gPz0Y41`1HJ2dX1SY4B)N0^yLt?fV$U@k;ux z=xv|h=)+wlKs+KV74=o1dCY?lB}cwjVJpj?d!ap0I`?;bx0J>XHoMm2)gp zdl+RzZG${&lyVE5hUX7ip>Py-d>Cgy1`iSGb}svBfR11C8{$|)I>5S&fh7ZcR5kIgvrUl3@IT& zVc~q3tZ)j2PA<#iLuJ%5@-}qsdxIztJQyYjw?F|j65Jl($x5?WzF(hYRqkt{n#TmI zGnF`Z=ogjdReK3K9(wD@ZIoyE+YJ;=VQLahH2k^uxh;g8mmEAG4lk#0J^>1EE6qa? zIhz%cjWETP=PZM6(^L~mYUC>}t2q%t>1s$F^^5_~aI1*1=|6&t1O1u=tPE=NAk3%9 zZ%g@&JLOLT>LyH9<%db61KT-ngP$C%j55gN?q!)`5}ifHvykgij!q!Pioa>OgV#)P zJmiQ&)68?csjjRZ4cUJ#-xh)~kN}XeJlrd-9FM7u%rNeVu=wB~8!qUQN2#xe-8dqt z6Fbc~hS(t~9s&H|h?wx)h}_`7YyCs{eca?mjS=+#7I;r0%tV(*dvVA0vts^6I1Xw}-Q~bjnvKpk=;Yvec&DExb_E{+ zIJr+_B;&LL82OR|xf{6x`8#s{)%Lf8$39k!X$K9-KlK2?7>)x}9K+WiL~0eu5aPwF z6?^R>95Nh7oaC3AzV*Y^`cbI&-*=en95?Y|(%btD8DyTlEs%qA~`@ugTZYnI+ z>(^J_%YJ|Q658{&(G`Z5O_h5T|7*jcB6p}dM0;uv&X7xtKRq<~GZOI$R7lb z=;&~+zmky|x{{_fx*O_Zl|dfhSp3G?AMVor_)u?NvP5|7E7z|>YQD)2Q;9VxdhwuW zZ~D478>reYRjGIbe?HFSYE_kus<~{ci4Ki$l?j;F=f-)eU3U=l?9vtbvl*Vn6e|ngH%+&<_@TA+V3T}ICbQk_jfUQj7uW-UQ+OUV~nUb$zS3Ncg-{upC1^h6J;2=2r>|7 z2aOeQZcO=`_AoOb z4mcS`1hs|eY!@N;lE;U%UWaZV@rKzt+Pg%Bq3^8(S1k)M@g({&g%StxZSD+p5F&Q! z&?>`Qfb-Cm0VbgN>l(?PN>XIn8qNbhx&R1dO1oWeH1vwX6p?K( z_*93LlU3$s*i~tYdX^P~l49arU-)(@+^i+EOX@Pc4*Y!mz$H^|{z8gfW-2e}2e<@y z^y!R{_ty{MFslq^#*D7+)15m$TJGSd*{8<(bD;Z98U2-rZLGQ(&=$d4|Lbz|E}jkJn$R#HYuQmpE#k3SOD)puAY!fTclTB#xSZ1~=^SJ{USgGFxZ3&zr8qOiVt-osP8W zFba9#RRFzka;5sf#S)C_?AT)BT>unz@-4sdlXCMaaDVIf`?1hd+9g_7!=YDyiW8{X z{0syh{?;)(Rg`$|YpAVXB(&8ntCMz3S=YAswT4s**4QWWfg8wbuzgl86`HBA?oFA8Kd-ssLjl@%VNfNg{(~Y3tUY<-T@PHih3_x}3No!pw$^sm9XFW+-Qt3U zz9C;281djKa{6%a=t0rNG*96Yq2_1s=pplH3)5FWjko2?D>e|qSH9G#n2{1;<#pWz zyg%|hv}gK}h@RViXrmL-Pl_JZ%Nxf3nc=<#?L*1)C4#Wf4<>KFCefo*g`x#p?ZUY9 z{10z)4nw#Y%v1d*7#8ODWWQU7VMA$5o9PU2rPuFGMXna4ZwqAN{_z`%+7IW zR}u#?det7nU&T_Q*U)*O!m>ZCMd%)VSM1=xQ6Fa4m#G||{-OZ;2jq&$R2ptZ8CRyu!WMm;K{zbo1a zo~rx+Ay#^Uc(~wJ0%qIp=x`k>HgGDLd+Do}CpF*XsiYBaH-xDGPZZ0xZ)sI>hgw~` zQ1zM!&7X2Hzp;H7ftBwJ?Uvw5tfZF;eUlj#?fuyG)k3tU_7E>Fui5v-q#`kTCV&Cu z_e)QK;p;Q5yv@#2)gQ3NG=D<86jVdQrQC;gX*sEO!n8S$zYNv2*lzIa^A_6IHsi4f z@s+p24HH^%<=(@EzaDqCm$&B7MI&{8`7KkkRro!2r+B^V|5v3AzXkoDlNn=WUTn<_ zM18M4vo~{nhIQ}3(#^0H8N$;o%Vc-VJ`sE|)ymuH>*bgNUZbWKlu3|RhKGPnvb#@G z&?Mhsh8(2x%JKJ$FUoD5Ui38EIayBceL8r5PBm#S-{4S*-sI-J2hXU3Hr_AN*hAke z-cUVI+yB?nmD(F3J3rS1uB@ayuMZ3%e=alG9_k(~>o?i%q2OdAw-r(`^6=|BJD=i1=YQpkT^v3OrK`M*p5yMm;NW$eo6fjmI)r~0ZjQzH8*~$H6n7= zeQ(rj@oTPNy$Z^{JX)x*ZrkIYvP%8-yZQ2!%>hmnLU7bL;cAj|`(UrfJ{$10L8-Jo zb&oy5uwGvMLBC`r)|1K34Iy0gJ$|X86j`&AKVL4jpi{ro=9RnR(#`C(h&I=%5B_O> zESaOs5HSPWbc-%5gDvcZLgKKl>$|-l=_;M|J8+eB6s%wHV3nJ2xh+M1Tx!S;zeO>z z|Fl&3==(vFm#*zavU%OD+$`?GQa#?W{vt4>C0S2|nx&z#T0FM#=Ba)(K|?D~1j5_Q zZ+x$h{7&t8K?Qs@{_>3vjYLh)W&s`PUf3bQcseU_MQhI4C zoxZI!LE8Q4k##u7u^}UQSo4U>!;slJka78_4~;ALlk!P?j#tD!MHj1LqGbJHJ53rY z+o}H=#axJz*3J5vDI_Mvn~?9-pKn^R@_lPpmC=gfOR%(ihSL~pN@tr&?8u(-)aL7E zvj;O?I6VrC4!ByXZa%VA^g;PBWK+T{8ih|4jo;nLuBXi(jv$@5_^l4f+ls79eV-fg zT-n%IY%%^#m*bCUPo}PNlyjlioA&b$Qt(OhgXNGE+j2y*%@|%{-K(_5pmm+c{;PcC z6M5a3$o4%vS~D@8*||@vO9-_FRheb5tum?Gbl{xjt|7SgV%5{e+Xzkjj59S2v+d0V z%Z$0Ccc%$TpOMUrgw}5OM?9#8R{dqEEnVzw7sjvJ>i*f$idRktON>P>{53 zn0mK&VXixdSXP2{KuM|YY#~GkpGdAR?9G2dc1MVBF;aV+_~ixS)H+61T}OC(Jv^5o zFBLc&PNsO~5Eycu1rwtUocmV*`|$5}$Ci<7HoOdQhiPJJ@^oV*_q+`dBDQrn6@qL6 zcy$#i5%!h`?^Z3|BG83v$Wjmw!Wl{vvVkq+U{+zG%kj3tpzJw3Q397~8gY@c!%g9C zS*WNPk7!jJMIgwU6s>k=>g9T=-GVd%1+l2?W zwTF&u0C4LD@=|m!p6*_49~hAu2Cpaaj?e_GvU zgPDPoJ8|kvf_1H4-LCf1oF}uBZ@&$ZX<6IxK`bmMH?_Q8C8)vyB24VF+UUPmanryq z)o;SxXn5&~eV#SRX`9I>(d%?axYjP|5zeCyDv5PX121%SS3H~?83mDItAqyU?vk|r z=t-YTZQoCSy{XE??raRcOs)NiM)#X|$>p*7h3_jKwO??xN!qS&<1aKcP6MQ1qRw(P;$&ClZiivp?( zT1wc0j!j^;ppeFMC)JADb3S>Fo}r(UI$#Iqfkjw z{*d0W88~?^+BsW%+i_wT>>>FlIyqZ{7rr-nFng|*^cK6MWu-)?L}}{qK{iK+)6l@} zN)1wf0o<;|mXVvC>|8jpvjpYbdmxu(4=1aP^9&e270<(b{4Aod)S=WyG8nitgC=QW z>Wc#-5;zZwmjPAEl81xXs;#CeRxva4Q=61MTO~pd@{PVK0yX6<&r<>Q@~X( zI!5lhN7}Bexc+m;(W`QyDMbq_(yjuQZ`EX$(KXBzNZDV=aJJ10nKAqs zWP#86x*~(w4BSU^UV(L5p|Ce!pB`ytjIz6i$G~cWJl!c*#pl3ecr(>Gm=87Jg(^R8y%tN&^8_AjXt{W%8g9A8~|TO`wvFg?}TBBar3 z|0{9;+w`X`(q{!N*UlGraGn{tqZhfd0bH=2mYgw?Wd}h`#>Gm^cipf0_XTqnF5Zd6 zp>mefbgGO|pDLcXX8^z8y|sd>WTK{WTfyeXUIKSqQ79{2<6a-ThxWWYm53GdS!N)RG}y}~EwTXB7P+Wk*Pr>FiZuA$HSbVF|cI&f6CJ*1r0 z)B(d&W1D&-l*s#Dk-s_5;z2Enlx|wy0ulhgRnYdo7=AgXbln(|az${$uJhWb7d`Zw z8+e~<`D3=zFULq(G09x=Z7Y{bo%eGz>#>vGZ9TrqkbVl%$47a98r)@N>w#sruTo=- zRf2{Xd~g@V%W`ixe}buLK+W8k@oFK&Xj@6=^{OVxYjRvce*glMLl{wr?;I6qF1A-B zFY(oMg^PG0?USzA_v4}k^(#Jt6lb`EU21)(ex>#7nW0>~o0wyTzNu=#sgK0)`&c%_ zI@7CaaZb81jek7%J|-vvm&&JNkH_G_mDVN2AFPtCY9|tfE->gEP9#(~o{e~zC%TX` zNb%1gj;?h&Fj;a^|E#dSZl>F#oBgRho$Z_XcK=IKMIf8DufT2sj^l4hVPQ}n>vUG1 z(umZ;SOPcTD6ra1 za@9w-JWVb5L|EbTw5E2XZ`gdkRD9AoPaaUnm$MTdL_OW9g|}M66xBL2FR1|)9GUib zeRJ@W)mo!O3W;55!P#%HuN0=>IKyDvYs@jMgpHu5gCr@HJaT<*-Yn~zIHn$Ln&gvW zV>z*grnog%Qmne6y*EdY;HbtW{F)itjYOD_m4 z{OuR62#TD3PsCEmHpS%gv4iJBPx$E8OQ2$w)0rB%^r3EB@2LD+iuozxUzx(1yPIVJrZZ_h4`B|5Mgwv2`Hb7dt8l@ySZ{>*a$RYLx=MDiY9 zKiBAkwpIQk#Q!T%1AdlvF+m%Vzoq}RL=m(R^Y5j|dGg-IUuz%C4}VsXz}32ORPK@N zEAyzZ{Cf$gO#ZWU;}emm^Nh28WBUKYV3apVjq72h@l4<-0}62uT$t zWR|ZVbobu;X(@kKw{cer&5cg*Bpi$j`YdllCzhB+s0)2&${Zfl8np73{k$Y*db<+% zRqX^HJP29gP2cV)LRIKhLWv-MQX>+)Uojh>kZ$sBp|yO{zFwiLQ{}rDuA3nS~kbjcUo%?x2_Um0CM!?%l#(?`psSDp5NxGf-tY_M) zLnDL%ixhH$7$PJOJxamKA^X2V*YuW0f^CU+qz|c(Ij^YCBhMVpMp4Y1a=7T4Zj!Y4 zeblwu+G#^q2sfR(XYXWuMPV}Avy01Bc4;URUKb?%5OChp*PKWtP~>Dm5g((G{AfhB zK0&_}%tiIgLQ-(RMuq*1h)4ONlnNvheP^My*{8H7CLgLY@dy1lO3GICxj|zL_KH8f zb)UH)knY3EJ!y6=tqLf)NXbMvkY34FL($@h#FGkKCFAcg>7mx1VJ;vz@b8^Wdo~N4odWgE2nf<&pb#PveO?|u30Ib z3Z#Kct~Uv$Zz{_mM2Pl8sg#gtt@ZjfyYhu}Ig$_mwXc->Fxr@qf;wJFY%qm1HhJ#5 z@hEy2GO|CE4FbCluOF3E%L;||&~*gctxDH4&AZQ#O4G65e}=*-TAfuwORSsEq5{fX z3&mBvw@&~nvJQiT)x!7%Dt=*B3AZ-ZiH(5zwZXaj$(R=5!IKE?6x+IWf|>qaG^ZPi zn1e}k{%#=zk8TnoHr6V_T^_Q7JW*zKkp?_V9;^i`cHZ@&!!^A#9e_X^>0#Ug4^n?IT>HfJn$=pC_I{+$}dgSjB@cbCpa( zwCvF@(l>i=h3Akz2b{TL`{Yt>9yGH^TCivaS#e!*?b{442&d9fSaHB&Lk9^WM+TmZ z5W1IA+Hj)l?QMAU3GQJ`9UuH5|d~lYs_5hBx zPr-|Cn&b0eOQq^ShO%+mzdgnCKI~*M@c)5<7n-if>HUkM@YaniQ6?vZP((#}vQzi% zf;9U^zpLFTJ}YNz=!S*yZ<6riDC>6}b9j_Tgbti9QS2LHE`mpjGru^VI9{MjFAs;+ z0x4!>zPUgvzk@g>-}EvkZ(y|nyf&diO=1HcUHq0?6DP_cx|77L)>CjL#H>YQY-L%( zwth)}HO_GO++g=ev3G&>o}b=>Y|`d$c}SC-;_2d5|Nil2E-DbCmr+1Gzw+X{aFIsw zA7652{O_Uz@H61l(QCoWh!YNyp8eSd?3 z(KsTU95NO~lv)AGWvTC<7^CdxC^#xSiCE35GH8ZOH*=uBkUt^xzQqst`|0BFjORh0M}13U!;l z@lxK(akmu@{gqHI4AEFUI@9!*{>Dx+f$6E<0;^EO`AEv@<%@Sfc z3h?WccVJwM?(HKYKq=WYQfwn?1g@-O8~1Iy_Gt!-pn9%c{UdnpJ_mHwd)ggO_eq*Y zsy>vAyTHAgc|Q$qzj6lvDU6=JEKjOnGB}tl)O!D;9t2iAhfZJjz{z1J;*vHc_=cKL%H&$oNQZ12qycMV6E~b9+;S9d14(1QJd6p zy2Y&H<&=e7PyaW7QFN01U(Ihsd*)*AcB_Im&(%uoB>U(3G>uV}h&t$7imoxS7n!n%GEI?dfbo~Ivw9O8H`|f*%i%KB@mV&3`xfNt`9NI$C@cttd0opX_Wnz7LWA9O%Q)ML*It-)`L1{d9m{x3}i2!=z{?ZODqx zOTxK#-y(3ciwRgeOggtgkrE6|9d84`f34SzEZI4IYnyimM~v?-3qPzG^|ys)-VDJ= zh)s*g6XEti6uEO!l{Os1_6jQ*q6(g_Lxk;HloSL@20!yKEYvmcN% zt3->vYjXTSusvv@)8PN5Vi>}Zp&J()1lXqju72Ak zmMz5e8y|MjlK|uS$rAZgM3IA&#v*PN_LRnSthZ^7`RRt{3Kma}IP`CfO`Z5*WPvLo z3*$mnHUcN=2^EBiWmM2P^q+?)gxxfy0a>wzk2oO~R?A~KU@CZ4o0`BO**^b8oPi9u z{z~z?&Lm`HN$%5hmq9OTbwU0%n|NN*EX`dY7~e&(+|cs2S273 zl!IZEIO@j?zh~J9`_VJ&NV}TYf`w!DyC|iWs@1|yLm|IlrUQ|S5p*}B!6!iqojF1q z0+qV&sO)GZ5w0GZI{{P_52vx!%<2y(O@W$qT<^T0QE(s~;)_5Q&BK?TjKPi_A{9Ta(mV=4TD+(01}wtcuaj<1Quh0CR%o~&!ZVYxt4 zU_eguRe^mn_W7~?n-L3{FD4mue0ygPg|f)lIVdZ?R$fEisJq>h`CvMM~Z84eDIW~9Z% z0s^q<^qvf#^bvcf72qW5PKAFBI5Fxw&m;XX$HdQkVMGf*#ulwHHaySabntZUqmJu1 zZlFKyk(Dcq+$h43<4<=VUp>cY{A(8xyQ>y$M;|pM#ItPU!UNn@n?=`47(@bKJYY*D=YXd z^{JdJC>N6lt)^%K7lkgpfn&)ayU?ZdCqEwEflKq+{JFy7s2LMz7#AL z-*QkUog+eW{--UJwgl$%oi3X()U^(kH#m^`L0!^<}M7iZgrcHRZhKn zbYWp_xQ>KnW&Odj6#PyNTZ|OT$(%Z+8Qt$e1MBOXQXdho}zM1z^C+kuX&?cxXwt(ObEax-gVo1E@&rd<(OUb*}$IpKfIW0b``ydKkWp+jCw-2zI^x;vdu89V-gUT1DP zRm_Ife@1tD)R>iJQ=U$zQty+{l$Oe!L&M633-*Pr@KrNH^&1jj#UtN#361U z`Lf=wxb%jgB4vv>=e&XO0Qj?F%pc0qHkuM;$4P!o=iUf6b^flM61Je z)9WHDAXN;q)h)ibsY48Zh(_amJZKp zIE_t?kx3ujo!BYCf$m*iH>%iXy0D;YpEO7I^4J)mT$~$*!l%-Bl4ub}2>}Tatd3we z+@r?Jbsw+FCj`~R#w9+!9HkA8K-Nt=@&FZFHv3Li)rXtfXJ0Z+deZhilsFLO@));T zwte!?BSX|Cn%%)$t`ey<7TD3A8qe(IP66BQuqv>SoO1!5BmP?G`?+}@uF{UorTiO$&>yTE5eKuue;)*_dcB{7UyV$t4>$4rJ4a|3|2GA#h{ScO_dQ^j|zz z<0@Ud*<}@Q`c{D?N1+AVv`8k}3xKu6eTBxX;!{s=HWi_tN~7Zng`iC%8+qq++<9l6 zUSLhI9aDeTrhuC6k6xh=Wg;8s#2J1rn(jJDamL0b)Y*xmY5K(Hr44{s#6XW|j(=1b z36-H$?W={NA9eU_97Df-Rl_}@{qg~D#(aXZmVZ^M_S1KDC1~HR=ANJ;PF~6M2-@Zb z(xiMVC|CFK?%Nz}+1;Ax^$$;FzGPT&Q|<7)JlSF_i1nrH!*~k6EgyacfFS zJa2Ov-7=Yg{@co65ENbEsN@GS+Hdy9y&EBD>vvwF(6rPb*qF%xgfxfxbDja$CGUhp zOIJ&LggOIowDj%IDLl8RhAD>c3#@pVO+VD0&FTCRB|!z=8$Ta=6Fvaq(Za;*tU8q~ z?uD;w8M*RGe1e;4FMn&E#58+@+}*H^6~2YEpes%KsPby|-zZ*%#@^kb)OmCR@Ai7r zmkqJZ&+WIkOIreJS>_ubp#`F}rrhNH@A#E4@geCg!E;LZv7kkj>aHV|YC*$m99=s6!QTB1Gsy}FZ(@^#W)t`=1cE8m!e{)(J5zS4FJn-?QOnLjzgq9NlS^`hrIu@x@6!%nLfRIX+_Awye+VmMML77` zhKceP1y90lg@&#*FJVuKC1}|c>TI*t82F~0__SYaqF5ig;y0a`rZ!NZ69aS*j<7kE zpa6}<+V@oaO~a^Xv{R7?|uk6*_gcbh*c* zi{Uv(*nDz_@69}^usU1e>;%Y2lYCc^VAjCKr+{lQ3MN`RUkjg=-sMdmhMl-^KqDF= zSj<)LOhU~61$Azgn4=k%qih}J{~yUf0u7@b2*Mx)RCEJK#*N@IpzxyjeqnRthkMowD zR@GcdIPu@WHK`+NoW?ao-fGnoyYRifN^UWkzO4cvqhz$?&Ezpy0=R@rNo>TN)-*;# zAbdUXCO`_gvkRJdc3v@gS%1_V_3*|E%^6zPPXKu0R(N;Ge_F4o=4MEF)W}w*r6nVX z_Y4Wq5*u)na};u%-&v;|@62-C6<1xd5$e+O+TI7%aM~jlL}izdVI=cCA{^A1g=-f0 zV$PnrY1&F|P4JlYIQYc{FW>4{Ot zo{aiRazsas8y;9qAD7sf*qmg|bi#14+yv$CYe}m!`TP<*&Xp2pDRHQ5ndAAVfpt4{ zWi!deR&J&?Q-QnM%coB`MtTXaCgwy!^t^zk2Uso!x_a|>&eiJjH(6t4v(yu>a-f@iFDI&3KkU+;KP*JC9hzVPLY_hwCi#Q0UL zcy^O7QgDLs3?aHiRQh#Rq>W)KbRxT#@Gc|Pws^u^HIvJ*2-<3OCgZYZD7XEc$(o4d zO=5~yV=XfvSZ#_&_lwsXan333;y%M(M^gRlCab}(t~md=+=YL#QH&eKOU!km?Xy#` zeN3LKRt35A6w;Gc1QJ)OuQ18m)o0YX90m?~OyOs0^S<<0Dd&J8I6`DyahQB{zkIS^ zu^fN^hnR@Wa~pHq*zQe4#plXn?vi&95yEIN-f$aH`4E6t4a%2$;<9JC>$a3jnQKXH z(yu@t2;lMPNH_Wo`AAT@)?s1T#?_js({ePX)9uhZVGK8x99Bsj9Imk{TyOS7S0oan z?~-4w`h9O#=C?}p0F=J_o85Blm(~=sNq0=V+sGtSx)$l4|*>mN?6#s#Np0UdHK zDK!l}(8^rxTNL4hBRBu|Lg(Mffu$LMb5+6!HCx-&0H;D|Fo`BIou@2Oki_TmX%`U0 zR4;pf_Jjyw1t4oVF?#M{xE5QhnJ!odvVKDZA7>^xH}Vgg1v01A>5cQ_!l^wo&b|~T0%HAaJ1%YlmHKKCNY0FJoEBB^2nyRb%Rrl1#o|Hc@=*R7< z9L&S*>`u;}`0+S(8?|~S?R|^V`?q~0+(tWGjv|apK#@cuD!Zf3r#R^aS&YI~qC>bV zo$UMdnpap&z2eXs;HzJFYL*4z3HuT|^OO_s+?2zQZsd^%1AxUSGAf6iP}SQPpWo`q z4+B*J9H^=eif{qAE5H@B=jB8Oox+`wAR0UBh$*U6{~*H?NMti)_4G+qpZ#AEB_fgs zmYZLx9NFzWeZGb24qb>0*PbJ4$KLT8MX(;q7zqs-aBZf+SsAL zyzB*zQ(ASn3m8%4vp%;~Vzk>TZhY8NT;r3KcYSlXoWv*DPo%U8=(ehy162deL!AQW z3zvp>+nDz9bOOxQfiBqhy)F`*z)@DW?h3KUsx?Dc>s5v##W29djv*?>XineHX1E}G z#GR4>C@f7QYJ?vkMY>qe`zH1sSU?u?>J;_K6c}Ns3rnj#-Ocw}pFR>LLF#ne*Pdm! zvPzCp11{T73Y;&rHPC;8!d*zUA@gCA2<;djcimS&5$3#ir(C{_*6;072wK^GX}^*5 zSp92JB7ibZ9wR1LnS)0S$evAMzZi)C?X(HAdk?A<2qwA%V^o5c^-MH126N1v%vHn> z<^~?ONr^m()g#?#GG1ZNg*nR$|F$bn7HaLLp5d3QaowQtjcKxrehI?`OwWzB#90aT zXK_F~t4rh1#=LZRj^fVmN&tXbIXH*L%;Slj&QE1%$1?f?`3rdAx{msa0QN|uSun*Q zv3;)mV#+?uWpwG7rf8=bQ#@6owNG;|jXv>F9Xd2gw=vZIN76xI`lIFCAoX&fbrR&c z=^KQrp5)i3a%ft|(T){eYPof5+^QLsS34=&<#{aMWeHpaB`PW~WBj|ygPR_2kN=UV zLb;MPfH(a7jiyEDNI@KF8x&&76}d0<8nWE`AdGR{JzWN49)X!XXh9Yr*axUhBI zK8XnNlccjT(bcpX5F2GT#~;ci+X~5^OHKd@#a)_E^=b@YKpznx)#o=S)ck=1_VTG> zE9whwH~@yQ>Qmcg$9*g5aB)r1h3?X&dj!V-#nr<;49skGmj_tDBUeaRT zi_M)+b3MIuiB%hH9+gU@6X(~B2m&ZU(fwBMUg8=u9yV>wRe$am33+48b{u7K`>R@9 z<4!(-mFq{i=F5?&(=pf$>|g<>0~>&@sUL_&QEm>M#hwd1C;zgW@Mx*w6Sudh0b~Y0 zJ3Q`SR@$Z9>fW+OJxuMAJ%N+Cn)ES(K~SZ1ay(6zu}Ti$d9465R!t)gT}15j)BU#x z;iCrqrA@fyY#k2Fd)wXxSd*1^vb$5hD~D%3fNn7c7Y_(yuYIw5m3&H@xGZc3G>9D$ z7D?zjWeQP_M|E}F^peX*jWmZHZO6r`hE9@LvCcBErjh&ILy+uT;pRNJPRUYX+lLu>wE@A98S2wENO?&glO z9JOq-I|YHd3UyYJ#)-S}OF->(J!8%l=%U)*%XUsiQpHw_kJ5tn=dU{J8dYT#6jw1& zfQRYwNvRu9a`0t-pw;pB;Nh%Fd3siLN@xi_7LT$R9K-xz(Q7@^OtJhR;7U~Pcs;xH zdVQxL%1gCT&yw59Qq~wuAh$3oJWtk@>Yf_KgloQZy}H(>!(gc`*~6x8x+6Di*Q)k% zfUry?L>@r-_D(m`yNG69xC-0ur3`U2%$QJ57~VaBE>o5ffn;{SD%=9Zs5wTiIwc}j z-F$93)-Y7aR>-=<+;cxqma&ELy2uWDSj$M?gfHbdjiq^I>g{{WeTe5X1II>f*kco1 z{1~mk?y<$!m<%Tq`>NW)Ms43kAJ;ba#^(U_tRGmCHW^E6Lxx>}?`=OOcDbIY9pkILm+S8Q|AX#UJ6GUJEM zs|dw`Hh~4s2Q~fx0u3q?7L3s|y6t*TDUrYZnrQXp_|5H!pZsB9m-kaIAfJ-8hn8h% zUc|_c^uO)6JFRG9T9L1>!X)sIgor)}x`h_)Fn+Gvi%vWpP}CY8e6jnuKb!9u{pzf0 zmz6hRBkC;>BzwvxNnCi~7otwP6R{|lC<6?S zq>zL+{dgt=g1jM6779JfYLq6M;(gZ2#bWCQS553=El^18;^P#8WfPv{G_4v+XYCSfTeQaA?j!A6j1e5)e-bmOJ}ec70_WFHH~@Ea_jMP z?Gd_?sku%7uVDiHx4TGxa zUK4xe?Qq(a-)4LpK1?n!6z2(p_yy3w-|7pDE6@d?t78M28ouANq6%*E+_Y=?>vd<1 z#K68e|DSZi|Ei&~wFjFq4(R^EGI}Y`_sb{7mGUNDzo^R8z&{o=9VkvIKPvV!e+DLp zvRjt4Drm+wpFMXP)vV@O4#b?q9K(AN=~=aP>3pcdr=%7CS~NK1*R8aw<~$UeNw$` z5_x5h2Kstr-o#%!&~^+wj(%iWo%!Ch9Nj9l7Gk@{FOPYf`l8#0b@>q6O3QpGA=A>C zJ7qhXGA2RDAm-P)BhQgYrT~qN`!?AFAH^4<>z~i-nANfkIONf>P}f~)IDhh0^;gQm z0hmZ{1ML_g-NZ436jDfu4PaZhSDg;$qV#o#Lv6Ef2H%_Wxke^fI@h~shD(2eoa;SG zH31Len8#MrhP3`w3jRH*1(G@1{?CPf&uO7ve#o~-O!!kRQ?~Hl>B|a8GYKCJZ{`Q` zO0y>vXFa5Jy&acCZF6Xi#`6WBPR?INU>-JCes6Z_(;BM4=;6%(vs#M3$yX<1SfjfUSDW-$ACo_ z80jvNYSyR!K$8>~Ncc0+VP=J*Gy~-O)NIa+m@pL>x9HCGUJ0vduc(yx6(A4y1)1s1 z>K9(E3qHTTI;C9B8|xM!PT4cM=|kN%@RRvIi<0sHBD|FxWVejDVn2WPE=E8<@*Ef- zzJe@&4yIfbp3Css^9}0F9SS4@MHPL%r0%->7at$nd{QlPWRL9Eyf%YYW!pIInw<2< zplfL%GD{SiKnXbX9^1dj`e$n#5-;&zD3PJjACw!vWl=n-Vfn-W{afO{IhpDkaF<^8 zkn)<4QXTjP7%e}q}3w^J1~?O z2Rsd+xO568!nEp|Z9E10)!{DNZOljtV88=EUV#t7=+R^OF%xhxzC91B&u$Rj!nfL< z`>xd|t`bISh^mb0KgYy;p%YHIbgN~$<+zo?CT*x0By%pIRj>Ny^bjfJjwFeo_|nvCIl(i0ZgK?)S% z0PX3HQ45A~j*fpnQoxarj4A3d3UlHvvP^7CqZClAHDicoUeG?A0_C;-%-u{hu`mJ#{~=n5`a0J=t3UnsJ*R}3$%=|DQ1uH zTPhOxKCeN6Ar}fEn@}GwCWXMZ5q#q?lk#6jFKU*BaSzq{0-Ygb;O ziuqnePt&UKI%#Z@Cbs5tHH=q=$_W5ah+j0H{e3TaU?i%M4<=ouT3+7&#s`})F7%th zVSVoU#jLTUpw#Qjoct<6xF#<`l!goPSSL9Kk&djJt4hEnV#>Y#bHV@*HBNG`lao+` z>mFWKwhZG!?ow;&4@QlBDb;=jWhVq+Ax&JOhaF4fWhydgV0y1B=_Fq4Aa880ib2^C zGco~q7W72KZ~i9$>7qG9Nf2@TX#sxN^QSf?`}Yd^J`nLa7`?gZ%j(fHnYIOG$FRyd^h^O$_`)D@f0XkA5Qqtn1RV`?mzMIqXux>LyMdsI4AR)i! zKnh;&ul=Z10j7MJ@*~Woia6#@%8f3S2hxiB9?!juTy#&3Y+loIby+=6k^%6tmcEKb z=g0wI9*_e$o?D6KTL=DRLlE<+3;--mC5X-$evayfpxlxBxCPPG=)(GSPu<3m>|9QV zKbNPSK}SM;Evag8adni5UWcqKc5+Q$(S(Yc(km)A;VhV6QBlkZ5n8nNnX|Q$_)TRd70{E;DepF%AB>FXRi{E*KliOsMRx%VrBJV!+KYB63#pF?NuI*!{US_e z)y;BJd*~;CS3~OOX55a}h9}Jfe-*>|>J0t%pd>&uo#+Zfj?cp;eQ1Weg3jO0YgGka z_$M}y1a6FeOCeBtaF}Y?A}r-W>)E$3r#uyerHZ#$H?W8r+;HTshE*n$-u62>@xV$8 zdh;(hE}3YA@HIErv9ybSJfVLe2!N zGbrSe?gv2Q0K3ZgG0z1KE=TaYx0fg0?<8KqoUWfqeVR&BGQ<3yM8gQ1d2oe;+6(CA z^X2CPW1&eQz^;2wd}6Jpd$x}?kHUla+ync_9x7bqa&vOtfKK`h*ZKpDq7zpL^4FFQ zsdsf|aP@2DpGM(Qppbca#Ww+~aBwq&4jIfu$jR? zeM+Pty2B>=$B=47wIt`Du@qYlH8Yc^_&$Lq{T=(*RMh(if@*df(&$ACB z)+gyMr9x#&d(`Eo3V~}ur(uXZK8rLkg>( znU^`Z5<5mqyg_y}E4qYJ%%1McMgWIc5i1nLbQzc=2mW@_{QZto532-WiQ@yEqbUEh zwYTh9tq&57bpb{FE|J@TQ#R!NlE?zk!-%sAsC&$zeo$yVJ}ooi>NPUuQnFnq(s&l5 zQlg08TdY~}<*Qy&AjDs4u~gdHQXpclir4qBF5|sxGY*%iuTk0UxzK)ns34B%1jeIH z_`ECNE_oq9?!&LP2Mau8KHt^RAs-Hm4(35+o+}#KjGkW4-il8BYs%^|ZE!I#uT+7& zAf7640I+&@%g?Q z7sHh*(!#|XG!=OUUK3AHqsB`t%C<}cm~%3crGH{`MDN0*p2N zec{(o^dwUFk}_F)h3DMMDL%ez9HSZA2|dk)_*G2b=lh@Tbw5D)ykOB_d4t-QN`-Jc zxjrzoOBYishF|S9lu4|GnuLnsH`lz&F*SH?!SG~cC}K+r9vc`uvb)8$WFSf`U?Y8B z-}oYFv>({Blh)_0K?WVTw(lpU>ts0xvH|PslmR_uL0UZCToNhle~oz&|9LVkUEz*z zt=Eoah+fIB5WBT6qIh3Qi5c-_6yw!2uEv&jqGfoK2?9T*su|;Rm6SS5?b#mY_h8BN zK9Z;RP}1wPI4dOb@Bv4QxcfzSX`n-gaN-`9_$3&h?7ZhcqlZ$ z7Fn=$IcUh;V;bDx>Zzy>w8}VGEA^YTKG3l$`Q0)(IO36G8 zj3y_)EdAaw9&Ono*LfVkKS;xi>FNKYy7P=`Vq5#Tjb>;bq(~4%lp-||LKPGcrF%q! zp-2g#29csj4WJNI5WQ3pL8_ubNq{IV0qF@4nh-^}AgydpVW*4Ppf|?lP=UyG)H2x5OaC5RS1&H>%b!VmH7YU6=c|2 zxhdk-n4ww|{Q{?MoqIV885Q!^TMc#i<`T8J+NF7_Qt`l{3ogAKcdQ3NE+Q>3^TL>W zVFf3LjN~?c57A_8H1yzBP0z@%h9i1L+QvTUnN5op{$T&B&IuCL<8idv)z+&S(@hPV zwogO;m7#CguYY%uWuT6h-g*U05$j^Vt$vs2rj;7}%oVeZ*-tNf3)*f)0~X~km*o0B zEL2?A*?VkSXtcj{ZT!f&H)m)e&o2D-E8gNIZwA5*$zEPfX#A96SC8Mx{q z-6`Z7+8EBf-j8c8@=7=(L;v0tdS=dW8U14E_*`Q@&9~X|_apg3`}Kfjb=aLVn=3L^ zMfE@D^J)#(BqSKeE%UJtk2$4e%W@o(K(m`2sNGxLQTOi0l}5c+nvgp7>#HdQN4=xc zEn+kxJ-uf(zAW^MMiZltOHC?sh_)kJ9R1jQoUF=ng~i=Jf4OyVXn-JKWG*bi3)vP+ zeOUeVlMTr;ljlSkEX02GUCJVt65|#mlE>ZXAyQAcCRg}xD$6I?5Tm1u5{^FIJ6U$+ zU;j7@1@l2miEVU?V0d#&vpFmAf0Jg8UY@xs9HLdIj5u|)$&xC?C}p{~?{$76Jo?b5 zwB-QuPU-Lpd8+DM<$xXEN59-V7k&W#*ZWJ%9TWTiP+>mFraR0Q=_&*3^_^0xUEiNx zv!sLV-%IfSCQIRezr)g6zT0R<2=NrF`JfWOf?&F9+P`s$vn?~)FTA!Fk?-aJWFv@A zEZ270$zSJQ+b`Q@v)XVEt*+n9$F%>kLqGa>?>?Doy5M7AP`C#;Wv}toJwjdSp=Wx_ z6o+g1#C9CMTx0I>dbeiiH|(jI{(aB0WA~U@?N@fWJ$wLe>A11s*`y;GYOH{G`C%^O z#QY=LH@>_hm{QBHyps$(1i-UQBNRD6TIaqUD;z8*ghHLWd7<{~E^`XvIV`Ij)j*YZ zPOKJm@Z-oS+cJ4Np?|Z|=u-(h07ML-Pn;Jr1g-5MdllnR3AVeJO))=H(XJ9HlGV*l zpepFtKmn&qvm5@q;l${eAS2bPF;LG#yE3BtTWl&~x5HkScAi8?yp8v68yiCFdmtq} ze7kr>Q;w|6vDU+S`MVe`rqQ?k%~{3@mAi1MEo@|rgdOwVF2=5a^H@k2dGLA9E**>| zv$WUUDqqB5xBG1qASg;9#-QKb^kV*Bbn2dA{WnG{DeT++{}a4dR>W{kPZBJfImi@% zxscQi%IT2e9-_Q|5zfKe!|$PeedU zWFyYWD{@l*`kT~AsQu5B_$Am!{$CF;lnFMlgt*nH2Y5|s;}N1aPO%EC`6cX`pN%=! znd0%+Pq8wMIZn(R*ZSTDHX^J}@IB}tteb*j&xPG(`W1dWa=l=T$C%#%CC>kJ#A_0I_T2>$eIvz z^pyBGxgzKpa$nN?IxJ8y27g9<%`m^G6KBvfqSg1k2)}hUAEO#__Ax9Vzk<+rGw)E* zP;ye)@8|c50zn{d6m{w=m`~NZ!(l4*qD5VRzLe7CA{t)Zvvh950^xHTZ*?kl;2EyG2|3LaBS&!I;gccxV${w&4S!P5h!n;SH95`nnQZLm+^Y?_b| zg+lP-<^HP~{X#`5!)Rp6a@B>OSu4pX!xtNL>7kc9aVkyNaogJ++!R_CDn*MJ*-G6* znxG*Np2QyO%BaxrtF^S>&&ZZg}P|c%l4-JJncL zlNHky!wsCc*k~o?T4ZE-k6irrNoC~TCV;%L#t;oxRbHK%jR$awtYWXZJ)>oL$|1M% zr$Jj}^oz?e$fbH?Bb5*n_-*R3A5h}&c^0fa&+pkT5>HA`pZd#1?6Zn87^#TGC@F&t z#=0qkHVj@C~c(RINfq-^(?2TcrIrGL@1lm!=(< zrL-O9>tyiK14VspbnNDNxFp2)ns?NR@vk1%l^gDQ_0^N^u3$E|+kp;jwv4Sex|DC7 zY+MVxw6U@RCS4s#v4!wvllcSdm*ggrRpry`-tr9=awn*9&%fY)5b6EtUVf75`prMW zbogfAvF7o>nW(^=Tp3B)#tL&T41iX$&v%~x8Q2`=}uxr$laHJHw`5i~~>BlWp>bNV?d?H^uZ5qRR zC>PgL@vlj^+2!J-ne%cIBfO4K6J&%V9Cr9mu@!@?bODW<7vYlaKUcFxA|&(b@z2xK zIkq~G!}3cX$7|y=`Z?_ud8g9=E$NxrarR>#5o$mr2+E1jM^Yh8S$)U>6H91vkOG&F zv>#fRn6@$v>eJzV2Qy3ojPc%Z>3f&nz%n9`wN8`zHwo?oA6InEuce3?3;tN>3hLCL zZpAj;bbrNOdWs0^(fyE`78hNb`^Uq*8b?b<3Ye zL1G7py9(^nT5}>x66Dz&NM>|r17LOdtnT~aBg4+IiE}}!4&RbP((Wj|yq=P2}pJK3Uh^paZ9my5$U;4dED1qx#- zi^Aa)3ggn45H*t}SX5_O{dADR)44?RKpl~cB(-r-At`CLkauII*wzdN)I+U#^Khq1 zGBho4Bo<%9rfh6JLrF~h#1M0wnPka4ER-mYDmEA5tRgE2R^48%GC(;}c=XcKg!ebW zcXHddg!bX^E3OF;JUfH_GeTl{$B)&{UM95&diQQxi@jeu9vAzhKxp^@%Nd zb62}@!242#X{ayF3{j0?GeV+X6c2?)jh|R_R*hk#0<@P~I_GdRMO8@d zX@``w9qt~9gA};6S46_MJGs3a$Gj@x*S5|D<)KiO^Kk z(l-vPoIFOKkYVy|yp3%Utc%a1VHsG@K6^1&^r1Y4hzd_%rlyZ;vTCy3n#Q)RkcqI; z^4%o=LElJ?D+49MXxatPmXdPv(b!|I7r3cDAEz7EMLtW8bRdSasj#|IK&n)b{C-+9 zEd)_gd|0Hdqlr!{Cv)5oK65&J8bl|lv)30qP_sqTaX5porGbsY?tPFu($CEAisGS1l*K#XI!)>^a{gWO0yIm8mAwi5z3y~Qx zPVr?wujwRD&)Ma>=y6e9+y&5Eb(fN*mc-GV^r;_Rl0JaQeho-w&ukOEKs>rL&6Ti| zU^X2=>>_M}8A{WWs^d%?>mLxA;tK}tV77@99Oo~JQ%NARH*Q7ZLjl4je($po1_ zM@Dd;==;iYrFdK8f_@Hn+cXAQDSHasT{VN}dY`^KCE-W}Klss5DI6{$Ch7QMeLDbA zZCzvJy8Tg&FAJKas4(P*Sa(#D0s!8w1-gxON4RWUleDy83m#J*_Bp&5wGc~~M(W6Q zM76)o39E^U6t^p*KKZlaz6%1x_CyU%S-QB#;CGfB3+RZp63tRLH1%{qvIRr===s z3l7oy?3ShxK0S&q?lp>*;pDSVpAz1wgX|1nw%n?24a$2bI-*;K7X*p~TTh1{nUJN+ z3#evIl_pQcKuS6{g5ZMw&4qXNiv#z}?p}bjH;^D^SzE3nWZ%e=IvVkp5AMS7S~U*M z8?LJie)EIo;B~&_$%830#}9!hA!}TFG~Ix`Wac%Z8Qpsgi|3N0x69adaV&Imqn7J0 znBefE>dMP-cv;Z`VO5c`>hy{5bJNt%JuhFDH+D(iW1hg?Ahc>(*+mrhq$>v z8t-r4$5mBH1Y1ax%_xiC-uJTc-mwYV$*S85-UE-Xca@z>GLo+c2IQWz5zs|QCs@jN^0``;t| g?>QvOu5)CrIbsi+6`KJ6)7WEv-p2H;v1{DF0nA$VYybcN diff --git a/docs/source/assets/design/v1/prefix_caching/example-time-7.png b/docs/source/assets/design/v1/prefix_caching/example-time-7.png index fc33ef50d4fdb415de61cb9e75a085d776a971e3..0b536de5a53f2c00cf8ebfaf554542132051b2f5 100644 GIT binary patch literal 55922 zcma&N2l(6MwLcD&5lY!*Q)U8T1zWaci8ckx+O}m|vSmpnP}!C&S(Y_Dq%_bl%4`cE zg|aDxwroP7%&^NWrL0g2EgKqEx$H3iuV{O3Fa7P6cB8Pf63fvL61kW9}XBn2>`^Rrk27(XB*XX{j(u>K?XE1zp8buQRIf^yc`bh*-@kq!8dpeK5HWQkima8 zb9fd$ys!_rEPCEhZ|j}170rVvI9FJZL%`$tWu{rJ%!5kcdT7}?_@ec$?WR?Ffi_)W zxuAF!Taa0h1CL5(;5wiXmCZol5uC}u3y}1O|C!(a6c#l@3#@h4K|6Jx-A1#*zbztf;RzNz#Xh3WKXkes)IsK!vf7CCiQKyHn2vr5k zNe$W)Uf28KSoG`i$i)Jkuj-hlW?+3>qG*K;NHaGHyk@@CZ)=CM}wjVC+%T>Mp~I3mCq>Dn5B?V%r%Xfq6Plc zATlK$?l^<26QEiX3*4+k1cQQJuOy(Cx~Fy%DGaM)s@~40{Z7?}hc(=6M8TtaeUQtm z0~_oGu1vChYc|jtW;erjI=tx=vL!-jz*rwEP75*FR@#Ha&Zw2h)oTbX@Y9h>G0jX4 zK{!H#k{WBZc&bw%D+;Q|eW#;!vCu@Bd?_gtd`)i6w2*g^p*=;Y7UttH40CyGNS2yo zww5V&OxEE{dN?cbY?rFEU7?p1>@nj9cVuw4ATnHBXjmeT=P-8+}VcOfefghUGCX6=uPR&g%o%7CJ>nVK|pFauw6Y8%c*UT1t$$D3$MbU5?gT1)w4Q z&WJ1qnZ`(jL(q{$Xc0G#1D&j5Laj3cI*|f@Wx5|^@kG)34CpD(gGWqdh%3s}CZpyeP8+Sc|Z{T@^@zW&^Yh8lySf2~ncU&+5~RUaul*WRO76D;D^0_V}xRTuVUMHso3B%A6n^Arb01( z;E7Sol!jHwkyuU7Haw_YY!LCZ4GI-t(^GoYmQR>@aN7*iPpv?CV9MZXW{)YZ2a_Qr zL2;PRr6q+%zOsX^)c(a>FDDz-?qlA)h;wVH9O4o)xfVblx(MG1YvX5g0U=_x8YpF z8cSZyBMfoc34KNjo3Ju~v0Tj>bm0ami-AiQ!Q3e~lXD{;lBfn$o4^&=bq1|`T*}aG zIp`y`!30zEF)&e7mM_PMmn*b#gF!S!Kwkn3d7)gcCb(ZX$q|m_fD|N&R?&&=XwLx{_^`hc1jcLgb^O(zcOa z7?dq!IOduTpRa-{;4ew40XU2JAZvoVQK4MWk?Bw@IVfV7o>k&d8cj7uu`!B(1uW^B zV+HD@Tk{9$EK|vA(-5cW!3g!~7S#g}KykcPGf)HAesKFQ^Zykr@IRIw9RCzM{0kh= z#RJ%+An}LOft!gHY&7%9e_?^ySFwOxYRkODR0>c}_6=Z^obE7d3+{lmx-HHPRV*ER zzd@1KIMW#9l!@7Zs@+s|gP{pL)Ejj=t)c>O7d4FuSf6T64pd(=)pDAEKie2c(3PKFcmYvt) zesLg_%LB8SPN_F8p(#|P`~tYA%6Y_1#;HWL4Bjbdkx#3M2<$gfG3X{%!uyC?!Fq9) zk(($1Iq7`I`8HY~yN%RNCQ1xj`E;>pPQxcorKZrK5~3o)^OH<^>;^@hlIjP<_u-hLls6B~$UtQP-QGXgKY6D+(#( z>NU0OHMMG;?UGZqo^AMuFwLu>2>Oz;Svr(uTt-Sgduq;dctn=SAhw!4QyR6L;-EnR z4O&UyJY;cfJL`yYi{ir~5EP2aj@)FL^F+HMsLta&amu8 zM{2Yv%PpzZiq;hKHK&j#pkgU0jLVD{c5tH~t8&TDL=HPmtlFS#n4_9T7bSgQPGNj1 zDrl#hm~l9pHv09EQv$}C6G^#`ieQ_H{l3@%O9u8<w!(qNL|RkuGgDyleyvqGDw;l*LO zoy%lGMlwBG^FhDHW35CmM84i>^%&KtwmhR&8z6?56YBL2Q66_a z7%hx5imv&H6DoZlcoN`H=L?NOh>T;WQ%=XF)ofr63P4ri9#*5OLenmyElz0F%Tq&R z6m1IUc%a?!eko}(YHFoiBCmBF0Ep0Xwo=vtP4;B5TcF)OS3d70I#b=Ue>_Jf;&P z;&sfTLC27l&ot_|D$ESIUg>q>A(RcuU`Tit9WlwcV^mY+VGvn7Hv;;C5-YM)P#w9c zQH8vn89~{&HPH>phRbj-6k@Sl=t|h~t@4|!w2>P*!1}SDkvwBQ)+{wSXQIDYkOI&Be|qM zYGre?xP>55FqIOqoMK8i45?{BE*2bXHXF8k5Kt`*sm=PV;4(?O*RWMI*Fz~L&ghf~ zv~ErTx+7}0-M*V{Xig_}pwbx1sb(Wo$~AIARLNwMiI&SERh^~VEC$7V7X!{@PvWD# zl%)V<7*A6K8kJiOq|%}reYw`R*dg9R3>;IX0$4q&VpGVox4{iytm;2kx+GJQHF5`d)u zEW`f|pMbLt{Tn{1MO=aArD`1Ny3`{C2g5V}!Y4^D7DN&r&n z)bin!KrvgY`b^KJCXVZqRl2~2k=Cez@`6+z1{5h_{!FQPDrZ6&YA{m_h=@heG$bYY|JcDl$?w$)WXps!h2t1Ip9 zMb6ZuKBsf)fa==mF>j6_hT-~Uj;RMqBgUqEIuq71Pa7Wcfo(iUulGh$HnYh}jV`L~v zLZe4vd8snwoQ5goaE~OyPS!23x<$EB1r(VN@s`QC9ZrqCzDR3Ds1E#@NlhgQ z;46}m)#?>WLMkR6RH8eft?`~-7ovhijhZfuz>Gi9yR|r}`K{EVv|8|>DVYAO$_FQ7{+K9_CMrfvPR4sK9C_nGVK!hp2D4?{#Wz1*{e0mCL zwwN#BwwrqRO5K8^jt4i}ur^ah7{ruHxLD{(3NcNhVAv3JsK@cF+W;aWB$-xoxhf)$J~dq_m*8j!)ez>Tztx-bfoJlz@8^4ZR; zIs+xOf|p8*{H2Q=Am+fp;Bb`F?7$nB$zkjlTFod2sLj+o@sT67Lv0*{^&)VoiYt zfPBVByghE32wBbXu(_L6Dc)a3kO1xE3J-C)Mg@{Ti(kGoLSU z^hlFz4Rzd1UI^k$z13yMjxwo`8Y9w4AJl|30am4~HkMjAnyun$BmwVtM%&$))r|>0 zB-&t-loHLbYJ1YAXI+-p` z3C_`RTadZ2Ix&(GVoy`6N`ZOghv^yTw2_8nrJ3SFepQx<0>gA>i8CE1y>THgPfff4 zac&mU>!kuNs&(Fv6u!(QzE;kIC16b=i)e+AhFrO5(GIX~)D)eyCMwthDpMJQ&O9Se z7G+5?`c=nhPSZt!@E#MTj$Gs(g<+NOl|TVqBqpcpR%?OHV}@CL7+^sU_AlldTM@kd205Iq<2SnxKNE`WomJ z9|@xgB|_Dr-O0nj1e1lD+^gtJn6$lK8&n9VrCA5bg40!@ zdwt3F@G#2-s-$WuxEYt*R$G9j(B%VbVuf)z%1@@{PSsau`Rr62=FlOMTKEh(aodn8 z5_l~Nas`@U91J{8VakBw0nLItNHFV;pur3sG>4KL&Kfq>nLtxRR&z~XQp!y(ZK%RJ zVgNLsSz52HT@F${9n-z*ngnx|C^oBvWJh8mOe8YU?QEe zi5F9_mPUFC>a5d+!~DM@H9C??UA~wftBee@d8e0wg66pCS!j=_RCwH~PV)SK^HPcD zfTV~PnJGDVFyiD=!!$sEjvBZ~jj&xrsSLWg5r`5^$a?4#O}ilkz@HxsED;#E!2meP zF*zKzToORbcCXXbYM9bacVRjfo1rX4ZH%b%#c87p=-Xklt;I~+q}xJhj&-xy>8J!T zQpX5Vt+g9f662@s+L-7x+LgHMRiFq!K-`?DsNI{X5;GBoT3g^*#F$a2LPfsatd;y? zUeOyim5KOi&28055VHX*gHM`4*BC|lUJpW0R5vgrpYAM?4TUJ%8|G}b5HV=E*ctNU zEXBxJhZqE34t;`F&@5KuLxvczM!hndibjQZfgz+6lP(Q2O|?iDX3@+J`+l1sL>6?1 z6|41n7sQ~v5>pZ3xZ9oJ38Qplr;*2^g!1APk}~cf>O+Q$H&rL=qLT`ptFS>=?cgf# z+q=OuZsu}yu|8`m^+3v0Y`-gtb%Q0u&TN=?ZNJj*jm1gVi>5ZloAFZn$z>B*l5o>US;Zqf-@sgELYbtJg?LgsX>jaVKwrA^2T!J)a)h=-S+iq zh3RpC@U9s|hmgsfP1nk#i&;SiDT~Uist8jJ2A~g*&`v+ym`YOv@k`X~Be@#?3fIgLa0~!R;W>m(6rnwTHC26KD0lyO;x=akV@5=+5k>iZ!x|lYY zx&^wS7W1~>mtt?A5~wZ@#1lt@vV>Am8tI0@kzI+RenJVR9hm^V5FT?F3$nD%Opn2ukZ5VLL%sM=(bvwDufO)|rdKMLw% zsqLEp{8DPj6s!sbEEZNpMIQ$zzu)aoYhKoz4lJeNi8UAjEy)w7VVDKS6oW=EZge@< zG5JPo+_uM&t=5c=TpT(rUxG;qA85JBFj0k)HB%J=v!>Jx$IObtz?$B+;Hllij7rX! z=n*?b^l=O;Q5|fQ+w-eqmtkY6mN&R&I>SndD&~_Cp#7lFuIA&3EjPq^}Y#eDS-C8m9%?eyF4V&)u3PFGD_rqzsTCZ7M z)^OVxL(UM&kog&)K?$W@GfNu4xsg{Ba4GNMM81}3LNNVTF~imSoYC!LlZBSm& zV}N!>1z^mxbT11>SUEeW+ReU0Qjn942Qkq`Kt6@5>TbJdCt4OFTPY?{5(KsixD*b6 zot3g(ub7u@eS|ejy&Ty|dGIVw(}nar7c^Q+EZPj^qDBjV>Z`Csp;FbW@+2PnAP8@U z6ERaMC_OWrP6m?AG9{9*+Q7O9qG%g+xeK^f-gX=y1q2Qyf1Sju}qlqH#?Qyb0awGz<|`r`$eVmffC# zHZ8wIb|_KwfvR*Rou1LNRxz&HVjO{P+$pb161WHpB@}##DFN9EJ&D%6?nH%Jv7;ix zLA^C;cR?glRtr>lnl5!*sg?DXN$7l~Q*1Eqq{)jJJWK$!;KL)}Q<_SxjF9$d(nLZU z(c^Bj+4B%>RBZvG6*M6AW~>AZn#4=I?sjmj$x<3UtoBF1`<&4f0@EOHG>p9qE4(KL)sBntl5RusNab- zPmc1)u*?~>jYCOPC&LM4AZnc%4?TtO5vg9xxKYN4g=R$@;szsDwID+g_3 z?|8=VGcd+iSzZzPd6guUrX~Ip&qyHJ1n>-5&u8n7kzcybKgP4dqWk)?Zi_O9eCH8%$tBhlLO7+|DBn7L3wnkemL+n&!7{Q127)S)AG{QVv z;#G&^pn4H6QY|qJj5op%iY!Y6g5DlEnIUko!1kqfKa^7;!qqmfL#B~*r$v0g6-ovQ zvQWlxKJLQ_j}i%LnW4Xlm==#4kXS<}ph^+{moQ}ysqTtm)ErH4OCLiIn*n;a_BCdcaNQ;BQ&Tt^)WEyztslbrMh^@%02OiAhv zb*`%9tTJTPoOY!Q9HGJt)G4JKSqytwrZ`mdUs7A08J{gRhkde+cd|S%Z?vF}%wR}* z1vxj1fw?3d3d=A8%4Nk$Fwvl@njm@<7zF)3Mi0( zV--4WF6x)T!NHRXJC!;`6Cwh+Qk)Ei?GP+02=#%;QhC+@2&WEs2SEb7L(?ZB*$UhB zT30MbikBZ6xHFqZGzeCbfb?fU9ZshwP;;t|>%gf1JW4U8wTP%5n}xmHj4Jl@o)X@F=WiwVG}5A;Sk zLMX8s%vdlHc>s2G22JWh)fqL)9Zeuna@gsPO8wcS-{2hAwa{D}Fhf*P#lp@YQ?kt3 zxC$IyitAQKGjVO~=riX!0d9GS_Rp3gi89(fYUUQ0i4W!&~^@1t&U}S9Q*=eAJEtD=3xC4zfc?gfx zQq=bvXyOs#04oT?AZ1SjQP`Kxq}PJ{gz{Al*fw8Vh_j zl6r{lP7;t>N7wKX%JKPT354!+-O8ugXZARc7?4omkv>pxAYw*nx#}$Rr+^p9%IdUS zh4i5@m=PtEgR*`L<@#|fPD(94XyGP|B1I|m$6c(|GIJcH_%j0YxFTR~DotWWBT~N) zQj(_qvP5LOdZ9!6Ie}}kq3K4tI?(vB3<2|Bmd69FoiIgML87AGEsTj?sPe%5EcSh> z2!1U|%D@#81cWRHoP2_rMQjh`nsxIi zKduz@Ry{(7g|eu}Z8;|(&B9<93!PDeL><41D{-YjNoc3os{<}}G-55J=m2J!wth0(lA*NoVZRIH%Vx2P4iZRwk(C=#Pj`J&s%8n%NLQF^3v6|q?MSpF74uTB)*U%sx7#$EG}3Dz z95^~<0Lyz-(`!wUhyhGa69uhy^cF+t=E!eWMwUD-1VGktriU7M)dl=ggybq5YPX4A#&rif2Z&91vdywx(JV$~xHh(%3U6*|=LtL7dWWvLt3FvH)-;-kyLQ zhR~nMR8!U^uMGKkgLSMy8FJc8s>!i91&2W!;9H8)ovd9&^LaV|h-Yj}8|n520Pj#` zIp^gE-Tylt1eN|B4<;?BS#!nq0R?K};}xORGXT8!7r*tDOyo5Cs46iXkbTs&(Lysa zs*c(nn`Nsi;xQ{36AvU%^=-02Rl24N1{rgq;-yxy0|7#>;Y8(Hj>|`G&O@q`Oi>*R zT0yn)f#`RwSmLTFbRx+xh9xQmEp9Ny@LUJj8n7RDAMV5i!Ub}T!9&tgSQg|s!WpC~ z^u0V&;l!!a$=6|YXaTK<_Z<+|%bNLOf`M>6tmOU6bKaoW zmMp=QJ$u?1Wb&QFAx9ptWgvz~X-q5A?&w7k6}cV-Ag_c;I)X3=%5*RdfqOlWkcR2L zANpY^XH)pu@1rPIfu$H3`*J3m6DAAv(T>NW}lhnaVO_COsrF{4NThy3}XRAxMZ>CD!qEen4(2K>I5jx5KgD< z;az9oKuxI71gR%Fo{PjxOyouoS(_zFH%X=)+i&}}KDJC|k`*)s&<0(zHx(-mHx%>Q zv^Xl(hZT?>sAo01k|97Ygdw1l3F(R&4Yj&4P_q#?lEI#hY@%dh4as!!Wg*VlxB}Jn z3QaZ3M3}&EF)~1ey&rZ>yvv6|wt*-uRbVK=BU_RJTTnru^W}O}Xq$zG$9RjPuQq)wOoC0ZaR0-mp=UMJNl`s_I0s+svRlMw=`)2R9FF|AZeDdKN; zRcowLewQLgiCY&1sc4UC#0*F{9(O=>8_#E2rab`p194pdp;z!sn8a1bs*?3e#8rnZ zX#u?2Z&!!9*e2S8j@Re1Y^s?U;ITFZyvmV$8Y9bdBTlyqP**Hv5i?_eBv^86c}-gx zfGA{+aN7A44VPg!Qml!H@fnHjRgGdONK}>U)Ip01$b@m=UQ2F+5J0K#kG#BIHA@zi zYAdSTtF}wP$EUi3MnJ$cKS}dAaAyj4a9(bd@^)dUhWyMjMj&Rcg+Vyx-8|Ycwfxu^ z2{10#18c-ILT-w83)GlPaNug910eG9z)R^!CzGKHgK9Mbk!g$oFL|&XN*d0FqCX4C zwi*%%QXo=$GYBnnb;N+6XjQ=o%p zP^jZcMg|%c2iPrw0C>Z(1n8=)47ZKQ0StT^&%rUKMZmn#wA8#`bwHAfQ|g6ThGpQP z0>W-Z8e{ljNo=O53Lx)7MNF-4debo6q&O1D(u9GF^?JEm)p|m{?Ic2#>=iSz{AFMQ z6sBiJ7Z@tsmD)}^pMvU^(NtAR(KKC%Y-r^b7mro2z*ILZu*GpZWoN2`relB{E?TYz z>6Vm~PTF@t!c;1#_|gz0yq9OgxMj|Y36*IMnB?Me+dv20WXQ&FmFk)* zRj4QUEEi{72pvcq$n&XAygodT$e5oMor)`r6+E$UYls4Kgy(7B@5mLDo4?`)J?Ajs6jfd4SRf7C*Bx50l7leNkZV9E>0M;!9?s#XUkX<&=>3c8R$zqYoZgVS|7yqN+G};ViLBYPJJqPkpuDVbReQ% z>gu^v38R^YK4}|%F+C})J8KGgXxgORs@Wn6W8LnQ`79__k4p)H0R|Mj3PI9&!hn68 z?(}hQlv69}s5eQ)$Zi7Mmi0O?M#Uo3G@9KMsdgLzObCq5XTn?5WV_>dufOLGHz|Vd{N=1Uc~L%>**Q>ofJ5 z8|C9J5_dzh9Y(EoU}UkR?K0U3G81hF@|-Nzjw?ZSAW}vHjHcSth#Ulx&xhb`9C#2i z5zJL!5%EX08ItXTrRo9r(J}=q*OFV!U|MFt%O*j_uF-9xV?cZ{o9)>G1<#6Q7 zCJZ3{OiIyWxuS$MIdlk%!zuw&Zw$LhMUm1CO7eN3 zp0Y|9szz;J2x6?1-mLFMab|E2doo&R@xd(FRM^Zz}rFMm2j0IltH>_a1Hz5f{ax?3+_ zz0rS9!>Td1zU%q(RFS%sl7K`Sb z8_s=m%Q>gL{@A7qj(yeHZ_VE9Z|0ucKIQ_Ve(DEjM893}@Vqy7*nR$D=B5Mg4?co^ ze@djKWB6d%$K`{lNA1SLm+ZghTkMPT;7zZ*XtP5PZzk|;mnEUi-o_-vRwr$E`d3%) zzCG5vaj*F|bFXdt_CoQ|+?LYH!LpCpgZM|69dh!j?GIhNbj4qnt=?$cl?N>|He3xK zv+L>CO_m|2U%T5D+(Yv;=N9tOpWP8|VV*npvmIVKdfqjX_r$~gLhQ9feDaqBI;XBYhLXD>Kpx7}a+$bIXlmBZ?L9Kp8c zp1bRuYj0|7%dP+3*~K&W`Sz6;&Ux5hcIDSEPi|Om&s`4Q?zNBGi&)@it6Rz8I zyUGWtLvSGKiGYXTjcpG@gufbzVwI4<{Q3s%`-=za!XB^$Nu+jSrKV%K?VPC94Nx?qhDrmS@f> zU4Grl^Z8ePXEvX+irCqTx7kU|7u?A z8~aDc&wHl4@}2ijzGGK=kI`i}^mf@*ei@AH%#-HLKLI`Q;!QW1x5tma_{q;c*lhow zy|LuIHA@e<`~~Ff<1g5M*Jp0HZ0@1qC7VbMV$Wji#Y+y{VtCly{z==}H+=NMZifZ#Wh!4{m9ENJay{g`D;Hsta!+Ivh?WwKe+gE=6`Fe-Hx2|(peXr{7flcbPT!G zla&u%Sb5|Y&mVm}(Omh(RoZTdzlOMWxAnHv550GnL)QP03K}>zRAcBQHN+-bvdc-evCMowj`Lu;Y^l z7o7b3(kHf?J|`aq9d;wX@#SZ|L@(TH1LxSEpM33|#{I8%fAf>W%a_-8_AKB!QSLdF)%M%-P&n=zsL;beB8}et^^6x&d z-{QTuyySRpt2?*P{`hu%*LjZ!t9HBi2d{3}L7)frJN^DUmLBrboNb>8SMIiKi{l@k z_wfyzAGueb|j#eDJM3p0QWX-MYT{J?b__=b9d&_-K`{Uoe|Fb3M$pI@v_doW_dogU%0C{lgKVvw`^#@XzwgP%ZnNJT zw?qd&b6)?e6$bz*jPCpCoRx2!d-tlXuCJEKt#998yG`cPKU(o^f5BdT`9tm-GLDJ$gF;@S&{UP1^aZ_WR^Rl-<97)HRo1 zyZmR5{YR6h=wOV7a7i)?DOYpd(<+2IHAsc zw&RQE9CR0P-uKVh`WMSThVQ|~KRV$%cc6RY`@C;Icg}rtzy1C>+}}6eYYXnDOIE)$ zJPNYUx^tVmH~h%^^IhmpuX=NiIqjtP`!DXi*-|f9`T6jiyLR7#U$EEwyPhB3xEun@ zd+{^tG@g3;xcoUEWd3juw)1+=zts8cDfGC~Zs#0&`N99JzwrYe`n?4gSHLFjQvmBH zSnHg9^)W~P z?_3+KgcrSk$KE?GT+>+f#UuTL-v8*L@sh_r{&VzL^~6;Np0VKg4|lKb-JgB-$=`3g z>88@=*O|@sy}QKBg54{>UZ8nLe0Ie&|AS}pOK$zxe&Ukxr37A9^VD|c(AHr)_K9#)t=7)zKdF7FJ zDEl5hI=l1W>E-Q?xg-Akq?7(Kef#=nzo}CeuN(egWwPlm7oGN#h4_chJhl5(>Vm(nbN+%q+xhk1zP)J|{Iw_wr9X@anH_Tsik};g{GK@4b7~rBTzl&V7nG zZs%*ye{08s_+RaETPE21MdTKD5u4?l9v&;Q)Mfn4@Pxo3LWzp1|>+OPVHa=XvJ=%=gD&p*BFnYl0B`uX+PBWr(u6uG1S zYaeayTa{vf~p?P%8(n;&-G^TGG;6y7}fM~VCVMF)NV{DoH> ze6M^w_oHy`eYbt_Io)6FUUAuLhhMPYf`tWl`!kwT#_u0FDfia&(7nI=&Wh0+$gjVDDsuSukNdB0 z9izYc*Sp^P{ND4eKff4O>r`}Qv}B`=ev;H5zV*GcjGyfK+j$H=;y&z}d~zW1HjR>|6<`A3#7{yVgZzVX2;PVnw|^zG07JU-~! z2R?sm&4;z~eleN8_vCcHSM1+BggkWi9ee%k1Z|IV_AG9;<7XGhKUY6}e0}GZl{e10 zbFW1&{^V?1h(WULnF8yxl-bV-1NN1CNE$A&`zthlXrUffI}}d3X><#x&N-{_KmkY>w@3kK7ITA zBR_xSwlBD6e{lWk11?2w{d{uW@X*462fVe{{CACy9(eY|AMHNhFYc86TzmC{9S>ae z$SV`4J^J3JYo2;!(NUYcvDrpX{_)WpH@{SU;^=pNxl~wt%3gOqwf@UTu7BqXCqFT~ z`x8`@!|~jW^KQS9ZE+FTVZYPgWhe z*=h89!qEryU#=eh!Hd7y_<>sn?|gdsaQmaKS@Pt%qc65!y7+~QSD%}_e)Dw)4_>?P z=4a*|ap3)bo}Tx_2QNSPa`nX(@9bVZ22^@{1O38t&b;=VBgt>C`tXtdO}BmF9CXzy z_s5U@bP0LaYY(rVy>ao~UyENoYKLw6N0k>nedABRt9*LQzHi?2o2Os=t9$qQOKyAT zsaw8&&bHK%w}#Ix`N;trpOjg*`uXRdK5@%6H-Sk!{oO5In@)yT{QXGzSLG|;`n7e@ z$2TmwW9>K510FkP&x2>Teg47CSKpqSy!4{9*VfD9!e{s2_VvD6zvh}%53hXnvheK7 z9{+88*9|}1p|{JK@Zn{n7Cm{(N3T7KuY1Hh7wgNu_1xAk1G5De&z%42_~6f8_~6|) zZ*I@~@oP`dyXu)I_V`4WKRffYOEx|5(u?^g;rq|{#f5dCOm=wkDfrnR{A%yLz7ee5 zYKPxEaQ}8UFTC@UHy7>W{=9$IyZ4^IgW96oj#%~A&)ycF{3-qZ>lfzugI0eWW*0xR^~=A1VZTl4h2IWd z`uUae`@h(1;i~h#tLDW|-nen`VLnn+zW&gUPeIq-`O2}uJx{#&>2LVPZBPGsaP}3~ z?|ZDU%^#M%eeY?%JowGuf3R}Bd#=9YljoIp-}~^{BU5EM{=*J?4=y|4`uXn%_wIYh zUhBX7?E}B_!Zmm98;3u=L;dXi>-qT|>j%zL-1P@v!+!SeCfnS*(E(>JjDLO7hEHF+ zsBpdTw-1}fyj#BU%}eKQeR>Z)`-1&%yrA-bZG71$?>=Eu^4Bpz1P~mn7=vaV*Jk7Jv?(eK!GE= znyuk-r84bdquPT=hlc<6s{>5=KQU{^j9_icjnD(o=V;CXjflIRAo$ZETc4 zP(1&+@#2wWznEsF`~w`|4X+VGb0wDWS+F{Rler}TpSk5!LjGZ^pnTaFW4mrV=BHVtD2 zJw7e22tUaPboFZYT0-%!GBcJ_S6Solvc&5T>ERDO>6$MrmS6b%qva7=cB7yW5|gcg zdCwV#_HEm4=iSsj`0I{D(-6ZY&Mp~TJWXjds}XtA!zS<(r*6i=;{2FoSEQ0eefP~* zw%3sl`!r&Y=R3JAj_nE(Dg`AX>t!>$TY9Y9SYSI67&>ghcUs{RcE0!MdYF8wueE_us^N+gn!*4+}&q2yg{@U2)Pc^ zo#$4MlGko*NhKW3-|a^L=hD(fzl+FTI(&9( zdi*(9;l<<#-*PsAoly#0XF}I+J;OG)GMfP33?HK%bXprVMr5n8hIV%i)1S9VCgaLb za}^dYeaOhH#VG%1Xiq<}SEye)zLjoQ=Z`?&l9ptvM9(M>ZYM~{ZsQ(8gS0a>zh@<-}uk~}}L3xab{!e_K^IwavY0kQvvKN00 zo!#ED&#Jho>OT;@PVBEk&|#YS996Ov|?_0tYIh+SZkgr;Rb2C%a7I zDzCU{ogNnDx%#(*c0-e&j;vJ!_eAhJdwB(dFzNiicl1x~%ztiK}5xKlcJj82p zQHvnkOG-yn7PPTGW|zBQYhHH2LShlG$jwR@2R2QQ3W)1ok)0c5<>;boelv_w>;Ppa z6dlxJsw|1Bc2FuUr1eXTA{ne^8qDl#q^*f-pW$T~E2DKIhQDch|T@k%-!pG)57qzthuq) zeIz;*u1gBO@dJO_6WpigY4piIv_q2|vUx6pB|R-4#1zTL)fN&LGzxVWKO|@Q6%gw? zM!)&Ux3uXYvw8n~I+@V~-tt%}wd`Znxr`-+vR`c`r^D&u^drFWaec3$LyCoXnuT#~ z&#ra-%D39P%yrT57Mmb5M^3-=p~;}#N|vG{)tSQQB6vsyiuH2;r)j&53IA5u56js# z$9F9i8nV?23+KnruBsrZPsqJUT}Ge5pJS=OGmm8ay*%fi&|l@~tlxF^XYYN%VVw#s zAD(LSG9A0tyBPukA^7w|SwUtL=^U<9_zs~7mxt3`!v(O>134aRKhC(q}1fCokzAmL_j3CkEI%_6K zX!oi4C24sMc5=`IorUl&SnRHk6^WLNBDbG6ZE7*Rb%;6my-xm;^gQP_>ep5n2@;)Z zfl^COv9SJqt&`+t$8q;7;SNdTOim}IVVKYP$mI{c=GbNwD@IjavuKJUvO2c%_}#aY z@)5~46){yOvMx(hjj>>_+16%C@tzM}*8E7O{Z?fkhx$mKL&}#u>5Gj;K42w02+9`; z3j5}amUuWSystSZn;vz%1&jd>x|Q30U0Epe#s&rp++++-F*5kNsUmu7ZVWe9!^VN% z@H$K{RmGtgkrUO#=exzQt0Q}xZFSG5I?Aj?{!iCNK-Mz5!ok1uo5p!8lZ>121!#Yt z+n|QO#xjqL$+o=Upvmf?#>?dfJdH!n8|ieM(XZ&CBl2+OgrJ5WA#!tu&XRtq@~u%y z^;ZST4nNy7de}vlMQtWnI6= zQ{`{J$zmfOvb=!fS!dbw@x5Ur7z*kUBYX4$dO)<8*nE#~Xpo7#?DtU8>*Mi4{T~yy z0>xdfo9IiTxEi_L)Y#Di&kIN~5txKswo|l<2xNry1T16_UdkNm*x3y}Q3lD{L^{n3Jwd1+C$5Cx`fqc#mGa8r7FK?sPckZ_UdQV7VOOJG6aw--1xTivgLeO$L6VtF|x^0II$<98?i(q}OJJr|Fq`Q5o4EuMan&D&d8uWcZ{G zu#yFwd=0#q1@~((6~STpzMG@hJz|59jV$Txl7)qLvyajIA2Yo$CaDrHXFF^UTrR^< z-d+ZOny#LFH@dVpBiwy{+mAkNbp&#uVAslAchp&px1 z7dHG|IqA%ww>Qf_qIl+iH;O$r?0O`!)sH-z?ZZhp)z>NI+O3WD(k~#XEkbBxcDx9P z8+6c>+K)HWy0@N4@(p zo4#Dqz%tjr^>HJOmcRwP19(#nJHLO&A$O8@1iqZOUyuna&CXQLb)s6U<7G&B{!6qx zaTVL9L3&%i5@jpDJ6Va}k}9@UgY=%Zs{pYG^Y#;FxJ0LqL*IQ88ZHpbbn75u&zo1c*5tKAR6cz-UG1}+F(kBK~5?%gX*^?Y;NJ!vu%u;RY= zulba(yYz+tQLZL9uIdk>a%OlfRu9BdJmMnLLTCuJyjNXED+qA@h zDrU#67kU>eNp#7xI$OgI>if-?SpCO+zgJg?ZCj}B?}qFSA2CWC-*9qNH+BOz5=EYNgX6$9^R}TXaG_Nf!q|v1idSiPKlg59-$`e_UqP{awU3-yf z=F*q{a|!B;y-pdyB3!xM_guj2jmYiMKu~DTpWrh{)V5&vX|z|F(Smand+wWmu{ zM?lF)(<)ZP7U6lR=sD~$F3IrJ6I+dDX%NY@2|RyV75BaXnNBomm%CZET0`Eri%WErqNG^SdX(pHP z3{CYGLBqLfHh_6LqpN2a5BvIJ$A8D!d#jesI!l0*L3rKad`{@B5591K%{RoF`>~?I zqIjU7W#7lZh~0;YVUFOZNwoaGNnnppcW|e(R~6p86iroLml7Oh&Z{;EW@11~zIl$4 zT>uvMYWRLhtyXslGZ!7K`+}(4Fn|Vu`(2GAZ}Q#8B{gj5{RiaXul(b>y)j)jHR#NlXQuy_}b+w|dfCK$bbZEvmVcq$n^TcFaOM&*#IMKpg z(c2v$$h(wbOL&;tp_J9<$Nu%6oyw3m2Q@|?x^H`eMCYp~Sdn$zlF67XlQX;AmM)M7 zY9DlP&7L-oWSq7LM2CO>&f8{wvGL8;+hV>fJ;MFq@}pEVi-gJDRx`!h25A)7v*=l= zsQ75r%XBR|K_nvB8Xsaj`1AUnozv~jU67#RK=1xHW=sD(=4g_^3$wB6w@CR8Pz|Jv z`GYVm&uyq))As(PE+sAgj4?9O(1inymdZr^Z|-0Mk~Iub@Z|!4Z0vl)YOU=*6cE@m zS#nZxyOml;liTvk5K&esGmhq!xDQeORpBss*1Mz?FYb;S}rKmzL^YzJ7SeCgTVRN#j&is#+ z7wn4}hkJhMB+R+UbWFAxX2Hw1N5_+kdX)noZ>#oPR@%I5?d$O3-_3W!`y)-reO_%g znq|>KLes4+@=cyyS#OcTNg-A4)-v2q(J#u$=iI#9+LSWOtUhQhH4aeHAJeaxCUV_U zISiik)3T2`Tu^~KFe637Alp5^`@~3=5wOk3mVItt+LpWJzMk!i%Dr)_$}p5Ydew%) z+(H`<)QIG?5C z-}15b(%xn05*L7XPzlyEQ1c(lI7Y$Ma+?tEwc$0Qaq;Z5w+$bhgi`;N``lPx8xXn~hZHhV0^I(3dD zG?(G)a}q-a5&i}6(~AyuA#?qug~DodbwOZ`a#`848zet=1kDT=BU5+DdTLQPC04^70%wzB?b~rBcCMdo?($_p`=~LIBJa!ys-AMV?b^G!D)NuA z0A?ix>!ja+0MjkW*n1yu%q7#{oaHL!LNHYT6EdzjS;B~j@=TLy2B2}A*P*)cO}{uo zz&J9S;%V-zuBO5cdM(cR3*SExlfUZY%WxY3=6KFH6%I5@64h{z&)*bs6nS46?B|pn zwlGVWbkAhEoNp5Bgv*bqFB{@4&S@d>h(2-Xqa?AgYrBwhgKDV4`YK>~0M4IbMYHzR zY%O{qOW>|cS#S%LU>kALwSB?%R|ZUf{3{1WGVcOnbz%v)4WOiBX?0bzYy1EE4>X7o?{!g{Fl{+O0;xiK?^b%V6XN9$-?9Z%LHnsd zV(&+Bt=2OO7_s(ZBL@h7jCTj*Mo%CuTI%KIU#+pGwMzjr*_w$eQ zz89#@q7lRL`!;Icw_f&dM~8;NgaRrTDr~!E2;Q48|igw3c%i4evf~2%hTd+ z<24l3jB`Tg@?h1d$@wz|Bv@Px?mwKA?I%~oyhug$hs_G+Q=aijTa%KK#>Vbj83q6} z`_#rU;k?vU;`@*5|BR3S@9_4f+Tm5`d)`K1f*-==b@htxXI4brpy&xIB)tja4X;2% z#STigBly+_=n~M;AQEFyuYrBlm=^$8wVTE@izVgXfoF#ONEI?U|S z=n$Ty@(EQl-7;l-5mn5cm?8}T;dDCefzr)HTz}L<0|1;!nj-A`fPpB9HAU$v3=6|> zW(A*{UgCQ5MAp6ygN(^>?FhdR0FzMVFJm9w$pPJQ0g*6r!Xel}38i-}+b|=-EFqdn zK?PqtcAc+o5r!nhW$OM-C_!0?v{WrdL@v|w)p$bcSc8-9%?lEJ85N)&S8xb8N!lw1 z{1AlR8rKo)`!y$t2&e-nmMq?njKhc9uaMi}fY;*nH*0R6_aUg7HDM$mLgpAwmKx)b4??7YeJo)aEwMt9j7MrtAw|12pi{G2e zety=igG=EL(`sxiXCR#zuO-KyyHotr_n9(nKWz_x7IS64Ziv3`G=2sxTXLI|jS)UC zMFnl*9L_U*ET@mQKN3w^OB9%Sq7R@A9Um%pWrK{Knq#@Y(V!j{9PSf*t9Ed>t&e$= z5zI9I?E%@WaiL&1<9ms~nLX=qEVy+>MKt?LOQIFAAR@0FE;Ir#Js@d^n>wJ*M_3V5{;yVoY{3aef7=v!=7?pi&v_ zM;&eew70d^n_m2ze$qcWgKKqi__5cou8Nhz>?{r@l`-|$H-~KEaUXTo_7OVLK?v34 z4gEfsGv$xkzyqs?VZipWn5du}+C7w@KXZEa&C*{cqr{_hqr(EA{r-hWJCb!v)q6IL z$7jqAtiZI%E|8`k zF}?Xj?})%|2@kAcxBWl32C%lM_fg44e|3xZ0ph_}OhCN;Skxo{2XzLYZ@KlDo?EJ; z-?xorH8%!^YtrV%IE@~}h}@Unl6w!cNHP0b1~m7_@a* zvC%6F7Nn7rLMMaerR^Or4A{r|Bv03- z&xrJqWW(3|m*iVp{=+CfM*D4T7Cy(ibd^OT0XEdHDR|2tcD3E6$OBq|=C)Y-alGkj zkytjS7Hl8Y)(4bUQx%vmh=t}9b5j>z67s3Ci3lrE_+8{*ELmMrj}<6fEE0zIzVD+A zcOptPUvngE{n3q-1Hf@OMUuP8CcC6mSwqqZ2-J?7n4OE?T#R)=w>1gq+2J{n!=&iD z2)Bo-N1|O5p{6#2>2`l6Q|1QsB)v%7A5=IiTh?v_oVz$eo4aTAeT;+oohdeuh}ncM zKbBTxN(r;;>X%;6lL>YI5Z1uMXEJvy^+)oirf`38;8ATHTqM8MK*+|}?z@*7AU@Fg zUgvhSTB~kNDAUyez4f$<207^&Zz(WZl@dM*s}|!tq5)ZUjvhJD6=xP@&3G~@YWqau{CGIoRdBfot#Y9yn*pB zg~gBKZme`!>(g|5hYgu5{pNGCw(g$HD}eK{H0J|q3yDt6RcBLLelnD-?}JnAH%b1T zUigO_^!9fqTc!m70r$UzEUpz~*=B`Hs{wt+ut79W>Fh|EWp|&F`lbIR9OtK_q zbVoO_w>mHqM#_JTez+L#${ST3P6V1Y*dd}+bFZM?7em_{sC4#DK(&(WXH=P8a-0M- zgKjw|qs-5{Zra)mCzuHelw6rDIX&-KH3_~sCvglr1y!^_7mX{>e+MQqOedg;g3)IY z`t^$d!2JCmfbS_f5A2daxPZ>CZM{?hoq}SWdTzoOROx$lY<8e^(4JT)KPn*Z6RP~H zDcjc}9z_BvP^|tHOQs8ae$>a=CSE>xVL~Z1Yo}BP4*=OonqN&No{!=%`8J%;I_X3b zBp^1|y2W?dwcU22N#?$a*`4kE$@ziY&L^^W8U2^J=-kgo#*ws3qgc3%?4J&4U&Og) z&j0p6{R>!lA!k%`v??n2q|S8&9XV+&EF&DR&dtXkE!<6hy?bx6kHlu5o?W=gdBc)r zc4a<*L`POvZ(R{omFgsmM3+T-wtO&TSCTj;qO(2l%^EXdZkR82Jtw!G_3mz{{13a! zMeMv=clN{q+5WPBa%i+fBj{Ctpoz;Do1edYm)fS(05zry#P5t`QjgxA|YWR6l|HmL6#7Y-!A%yl%bP{Q!G#>JauS$e&NeOi&hSRsn#C zf|Uk>=MV$_Dmwxgv6F7H@Vp;fs0Z@k6cg#l#!YeiaH%v^SU%z?HitrP#K~dQ)ouH3 zxm=;zGir!%rlU26Z|q>n`~zIj=dqi=QtRN7u@~{hC`g#pDEDNyhR$ z4$iYi$F&DyE4o=2fwD>!WT1amnk8DjE$+q9-0&Q_lum$!Q#LP8q9nuP?Wn-v3R1Lk zW|>_s&8y=5MTGlUiSfg&nvs@bn&(KSXQC%9G$U-y>^8u=OF`o30AizEq5J|Ph3r+?w|plGX8WCJp4EaVLMPVt3yuD4hn_-JoWc^* z{>KYsCGznhmO9#aN7KbGR*CU!<20A&)Bd-IoA^mBIVMz&`w9I^diwMsLhwfYpXCh_ zW*OervHjMar^)($leW@R6W!l$cY&f({#j)+!zbR;%$Azokx{aE7V+U?`d6)L$+JQW zWb%D(G0vbw#OL@2)#@srPa9MdejT@0+C$s+o-B!Ygy^O5Nuf|Q#AvkL#@?oKz<#~C zB@m?7Qt_$Hqyx<=FFEzz3_i&o$xw6fFpYT^FCxTZnbILR~N6A>7>a?fSdcHJIfnJKzfS|#DV!iqdPM~0<*s#-VW7n?T-LL3` zkZ)`06I@*;O%F}Xx{P$FeD=FAa~R@L7D>{8!ugDA)!6?iu0+6czDU^CZ<$-$PQA#l zx?+IVd-%&jgy83h^a@+3(3(Fr!Bd>;FR9^b;g86wAk@e`1~-)z<|KkuVt zE4w$ccJSf<>zkhIZ@YKhT+HwfN`8fYYxpb#APeXy*DN~)03;$h)pK5Hr-OwtzIuMY z(xFL=SJN?jzOJ#iNh6u1#-p~SH!ct8&m6Uvu}0tFN6>-&PujzD)R#>5BBu2T_Jz&@ zun8D6YAzRj2v~IP3V@kuJ5kDd{#tazwe5l58V5i%7&B7o654J_W7rAfVskP|LJbtg zQmk0`qN+r>Zb{M_8i%*)6D_2lxuB!I0RjCLtuw-N&hCR1&%&+-Bq-TGr2P33rJqCc zhd%M4w=RE=kZCCyat^*GgTyc#ZX3jEcnc3 z>ipuvMrdyFvafp%u_osU$L4PVh-AwW^-I4Nk)@5wB#g@0%F1aU>VV+&*O%8;wZU>< z>)R?2n7Pp(yL~O+BAm^1-^e&BvB&@o$(COcooSlX)vC63)vAl4Cv9O9W~vP=E+TKxwwa6~B~+2&i?)bs%v?q>Cd5ROFJqs9f(v_&NGcZm zgbrl*TbziE5d-0Q+1|vgW>6SN+WyVO&Wa=H&Ixfy@L$I%7et+uB3a!hKtLggfSTgE zI38+aZ;jfiwB6tEqzdDuC|F)4bN-PmN__rv$&JD5hKmeHCaMdm0C*d_eq2B$92`M2Zj+PF zB&qc-Vt4OPAjM8_=di!~4IGDn{86F}+rRAkF;w{R&v|~JVne-qh`dEv zJRBX25a!+hM#fF|&Zy|mx)Ep?_{bZ&mB^u!V3Z)EnqX|Kev=f0q`}JN2p(6JPXtJT z7@s-&d)XWM`9tGWmV`Mey+ZiZ!aElkx*l;zL(ID?N59cNwMniU313>)j6)Dk=Ob`= zMF*o=BOc~UI7Xt2N1D&vP8Oz$MC}WDmxA%RY3y61H`e#>MxU+xqHz;yl|}SmI?x$p z4=RT(+JIo0vT>Noi2zf*lRhX9LqGL8Tm4}HjvggBeyB0{;v?|H#RAeyixXBw-S)qH zy&Bshiq2l&Trn`GHCRhALVpAm^We+!oe%c( z+^3WQq&M#kde7nBD+=%1nHCmTjaaWL)2|UbTf2bDr<>a5I^m7KGa3YxNVz=6=Bn#iOC3+N^)5AQqfcX?-*rDiuX|EY z8B>*U-SQ0S=OIX@^hNVr)zRt5h8rXsYx_{g*uZl2L?f&FhF&dEJPO=N>D zql!L#sC^u^L3c~pT)vMD2wHS)%n-eK4-5gg`!B)8|6q0?zkjG*)c})g&hfP=pm33I zM98hG$FTVS{X5P-f{i~q5Zfn!fMntcw;>uz^S_y5e1fA}C&aI;hVpUI0EAHTileFy zLHPe8G?CW>F?NeuG$tTC1v6*eX+PUFl>9GMkf!(_9nB2x%yXmvQYQV8;>?Ky5|xXZ z;KZJPiGoxL{^*R{Y%|rf&mC&<|gqquxT)54dPcQ{0hTN)nxb7yhD~)HNAo+ z(IcIBGgt`zR`d+$J|&S~^ZPdPsGzcCYgUz(X!CT5Ry}S?u-CFQ-nlBz^tCX`jS2fEU9Gzj+7 z>Y=Fjby&O#fYvlrK8AhqjdxEqO|tt~Tk+#vZOBt56#AN~2sU zzDDAMvFx(!yv_$V*<`sJ(GMcB7|4{b5}jl6gaX=^Sa-q%>r}M7f=)yJ^I5)TBCU;# zq=eeA75k{{i_=CW27!7`+*9-3acl()Q^Dlsl_zN8SYvCf*!zw3!eDbFaZuDn?Nqyy zQe|2nDhq*XmCt6k7%Xb?p2dwO1zlNTp>~DD zy^-;xBos`OF~Z!%dx4Kg7nLw2KB4QMVnW$zYSfZ983^;E+!2z!l(%Hs=@GU!eKWWl|NZvAb<(ey3f+A|zuR`VQ#8fJ2Xi@gPX7#bX8bwfP&g&|${j zUD&a|rzEY9t9g(86gPqfjVCy<^L?j*zW@@r({QXqPBw?UG!Z9onH~vo7*iH5H~+fA zmHhS2`Ct281?aCJPpHLd$PiOnI0ofp;F__9C5|_@ ztks5iF*^;#&{f)C4Q;*aY&jv3<$30gSwXuXo|<{zD`$!nPWy|~-lB+f{f(ZxMzm|v znV~_jgo;t8HNJ88JJxdRzP=B#UAlEmd3#Skg={ky$feN~f2YkB!QuiRC6=?6&ycUS zRQ&H<`7cGJq)%h$==fm-(V{=%(H+|b(c{b2z3WXwmpc4rl8)oly4{oXnmI|4PKfmG zx99{KJ!5rsHK;i9y#tcHKqi|-_4@&i4jobu3wXuFl%0^>edRVfM*?Y$WUNiPJ?+e5 zMVo?Xa~fTaIbKA*>HS2@3ORFfY2*Rk-3Tm((iD$LA348P+Chk0sGS6MW?xhIx0}KH zIodA*R=@$Dis*cp%bL=kaPa?is;ZZYQ>FNNM#sJ$6;Rk}o1xq?`aNe?e_5xs3WfM8 zh$Q#iyGY|Cm{xOX7DqiHsX{C5hDwRpQxb=w_`*>o0(Mu@JY80ZBU}nqDFunXCZUw5 z-?1d+^!Tir%7!T&O4e|8EE4T6(E9bjQVcNYO$R@TS{z`F3Rqri=_lU?Rz#`Om{?2f{)Kvmzm2}_->kOrFmak{pS{4u^BCGrNA8Y z6*hw?FtA`~99Q?8K9LYXkNJCca_XDvOmJgX%c3+o5Ld#}(&;{|ft*CM!1d{FPLf}-Q$oLs&PWL+?D3YL424w1Clvmil%rqz;fiOSZ&RtJT@>jV0Pl5Y{7?2&b*yqLnEepMt29_Rb8>fH&D{*|q z2NL6Cz&QE86V8ALuyk5J^0V)M>(Bx35C0lWAot=&USR&G4D{~^{_sa<>Sts7j{9Gk zDDRIvwZ@<(^xq~OCo#a%ot0Bb<-Z@A1rS>nAb%J7ciQ{v2P|#rr)x+3TexZgNN71( z2oI?Kz56>FVCivw(bD=~Syl$!A7D^USrEuB%inOZB%HvhwB>#7?z7zFcr_o?Q zj*so@E&m<&$5#nwWqBNNX_A}C3XO!_WwB@AynfoKmwln%6%%L|gt!|psID9Gy6F;? z7u1YJibRj1vvJqW+UQW^Ei)vDMqk~y*U}!c46+RKZTGtbO7^1A2~qFV#|ie7{awuX z1EdxCfU00x`cj3NRKiE)32Y$4U0kV5)W&5czO7C%D_FlTA!Tm(#myWdSs>E} zPO@6a`Kj5#r#yic2QoM2_+AtoihH!V^s`XLQ*UC13iW#~5*@m?I-PEo1bPr1qqZzD zR%EDq2W6KgDtVej>H+>gwHjcaK+Tw%x4s6+BgATCUy1l-n3#>XAlunJ z@j;9JLV?u5&iJGk<<$^2et2&j^f|o==(m>KjSP0lBZ#H!#YsArg4B0io7SAOfbgN$ z1e@KtzlB0_Hryo0aF9;S)RSddz|%W=aVg$LL_KN#cyx|pw~CY%jaf}p))gNqwcMeA zMN8N?23A`_@*Wm>sXL9zm7lM%ScfHNz)ASZ+!XqrWvQbNF(@(`jhPMPpqF^&p9grD zK;qn})OwEsCX}}a`y-ywU$95u8}-XJ32YC6N<3@=gS^3>J^|^K(Ost$R~J#@+kw;1>N3S8(3K(+DAt59AL~S6**dG&6D{nF-*WHu!t@1%5iD zVRG{aGaXukMnrIXR=PgZo4d9+cZwB-g$W)afz?PC^3k(cr$8&iuG*16H%k@vHX{~t z(i$s4r521cv_6YIBMGucW7X(hF=k)pf+O-1Xm!Pm(u7+lqUPI`$9he(cNh z5{U~=c1gcC$gVpf*~vpVYXz0>+sd=xG4ZRj$(7yoq> zD@{vHy@y_kBfQuW+a(eU8G$ zc%2i^gD-ao)0{@MGRg~%L!-nI#lEDBlSPre(Z!~Y3JNo(mhIvXba`ISu&9oLq`@vh z5%24VQC6Q2%E?ZYyBG_;Stp0-{=ocIO0DL1a>TrX58$dn^cs!PsE`jdDR-2N4z0#w zJ+yCT8L;(KCg7Or%)XS*Dpfq0RP(-g2m~0LEYP%IsVGqTJmx9KSFrx_yEDNn2l z_oN>%>_iLOZ!WtcxbQg*`n-DFJE^B);x59Ca5XU`s@**D;Wx)mlu>=xGq*_;np$%r zeIGHa*iAdpkNvH<&^pD4S{e*U=h&`(@`@5^vf}5jwJwLz;1{lR`mGbKNIqR-n!o3> z!#{13`P9tg}Z?dUvc@BE5pp--Ueqp#Q@g3@&_!+@%>I zLVQZBDpXMk=4FZ-bZRg%(mc65X@pqhP8iRkQKjVxRG5g_{Rp}q9Xb0HZrHNXPUl{$ zqL&p~!n1&b={tpT+|P}HFX|x;lCWml`I2LMmwE6Ej%>{+;PchjA35D2NT?1~q|c7| z)P?pqlMIiU{+RVTC>ilun&D9~;!facaxY<9AaH0BthR~8ugw1GKK&n2Hvp8>RtLI< zQ{scA2U5QR*)hMRCoOVyE*vL+Xd{B;nG!u!PR>E6#wR0NHAO;fX{ zUvkcH)acU`s#+u?+qvd)l@xw}YWw5XX01QvB9c0teCd1ASNAlMOAcOZQ>>QtSrL#>|zxWQ)Vn(FbyH2=O zeC4XlBQxAzAV+_}I_M{93Xr+|qIGCjmYH>a!PYiodNGgts&~*83b5?YJm#^mA`HN; zODq=cZ+<+gJ+2|qop;IgUOmIWSKbbEr=-NJ*x&#MGik_ONf!W%a2BJly<|6K^%d@- zME4J?0RsZTMJr^VyOb9uTosD2h+C3T(XL|sfJt8fIn&3|OOj3n4;q$@DT;5w=*)g& zi9MiDb>d1XF@*!N8aIc4xZ_NY3`xR1dHgq^NLofBI1x369APmg_9J+lpZ-Kh9Il#@V&85w$XlVe>~oY-Q5#0X!{2}MyhJ%UXNDGd@0GF`kg{c4+_w{xnD zSF$jD5ULXV>E7u%j@F|c5RZ5$yd|oY75FiqC=h#}#8c6?g%MCFgrTwG*p1O7<(N1U zm+JsymqCo#*E$ZxBuVQ!ih%hRq_TxWR+5(b!Q0Q&p#!}#q!Y?VU%bDRzkK5xqMkXJ zNk9=hENJ}56GVZtYCc;dBRcr>#eJqEdJwXPaVc%XYA=N1QZz`(-^pbF-Sv5aKC9@@^tr6X3stTs*6q7= z#(3MEzy}p0O=(;Ybh-$B=zVf!3><|umo|tHO3R$vBR?Jx_pljwdL$@_l{;zJ=^kAx z$T_YdiF~oD9Z)-tVt2ZZ_$YNYK8DpExg4t@ej|WRpwoXfB4%Q8GV|e~4~}ZIJG#b1 zo$i)h#UcD_nYx{X?m$FZ&XfPDuTn)$fNoDrd!psQ^PgJf$)7_gJVkrH#s%u0#8;s`l1(RT-dm{mZdqzPg$KXqj zN%r$u&YckCB$YC0clP7Hv@jRUaWGSsvaemN+p2h8#cY_PBSo2_LZ303++y=uVnhxu zXd4r8|5_B+i)u}G#POUC(1OT4Az+qL*ai(BCBbD7x}QEfKr+5z^hyS?=de`TSFebA zO)_9o)u7B+-r$O%kM{>W>q7e`E*yI}+Y6|A!Z4Zqbbea!EK|7S`!H$nGw#TP95pvO=XWm!Pu&*vd?aL{ryyznyf=*BmqjGFZ)Xf30uYHb zp9D~;2SM1^VY^70eYd=^1ONuB1>yn@67mr9qJ6PM$BN}(fG9ItXw@A#AXT%MQaOHg zOlF`=m(h)x0Cnm>8Si?dKtXHQ=Yx-1u|Jt@K>$^U?zs>8$%CGlt|kaas?bwVY$ghi zWyzcvb$)3Jd2ezY4yT^cz<;~r)l$gb$<%?eX*g_{EwC1c%vzSOz`mg^(}Hk#3=UTA zF>@AEk?P0EiCHJ=t`D#)RUY?h9dZ<@srn1SIEnt#1k+DRw6FUKQ>|P*!8xLw7zr7~ zqtMYXb3hP#LG9{&L_)s?8f%D&F%lF72P0PYNksMdG|YQ2#j;7aI5GMJDbQoZ24I)G zX6jH7^a*$7igf*cG7~kx7(WH}q!uos*)$4R#hJnEdG9PCN>8ZpzC{wH1Seh_yW~c} zRTgC5%9Je<3YfWt*$>h_lgB#;O@>7+P#{OUY3OhWEACW2!%uSFbC20XvTUVJC`_8O+VLm)(C{RVxuOBH1Ca{fL*GF$BJrhvuhZnUYL6+H(=ag-THR$AhKMA)(KeO~ zf%H5jpcmdfzVN%De6cTZ+8@sD(W!X4j&#w43m6R+B_yH=m!&l58VXg_P$Aq<9hQM! zdE=`1;o%7Bq@wi%PE+C(7u|Wp3}n}Bkja}os&W3w#mf5B`Vy|CJw0>H71C|-XQtHY zYqsIdF4U&V4Ej217GvA4;-u`)t{H0wH#Dbi6|?=(7R|Z*BosKW6Q;e21d8+EFS3Z; z@p|OW+9h-@h(xZXRtEjOij`~lj5(KC@Wr-c+-Hsy!Af^q1ngA&Zt0q*)*x3l!-RNol0J8w8hhEkZ>)r9n`-k?!v9K9jAy=ilG^jq{%| z*4Sf@A!E)b?zryjy65vu;A6}xrW>gSsKxm(8!E32X%f4?UO?uWF2Z99bzWW8(%Bz- zoMMA6|Aajl4f!Nhn>h{S7beDEf*0;t<@1@_Q20oi=Y>doNi6lV|D5RemhkQrx?yt7 zmn2V;tL)~9lI3_BH3*x-{3^n(MK#{DE#vqqoa9M@@om-^4609NnA-Jxqtbmgf33&Z#Ej^VR(&a3!uhVM?Ddnsjx^FPi`1em)eM&NV%COhVW@e(QtcFoi77hE*bI50 zkJ?!g@M#m5H`6Hna7&!JBb_keL~((C*Ufk8j@fjc7_?AzLQlX&hi?O?Re+ucu>SW0 z0ePa9jSoHS)Y*x6NG_iCZgUV7mQHX3LMn`Cv0@f@Y08bl7|JK_>C+`*;7``y?al+L zlm*GL45FhB*jx%v)A8dgS=B>;rQ6nKkLM8^WRGQ!u)Cv2045}ZDDK|e1!{Au&qYt=SC;{05r0=5& zqm0SvR6xVc1pL#@)uQt;1(#Bv(0M|km7A1n~}6B5%!?UIIF{oFYw(|0#1j7Lj;e2z`xYKyN=caJxRv-Q|cFF&b& zA6Oh>#PB_z!;Dxy{|LdQCAE^)+BlPNR$nw3{k9N=Ni*16yHhyy?#)tPIsrs*1xFz- z_udzI?luM`08slb2sPV~S)2EZP_k=YOB8-MK2SAhv{HC!xh3{yvEb-f+3Ct*X zeGdzV@Rt#Fz3dk(qh^Oa-}`H*y!*E_ht$5$5Kg=uQtSek#u5u6)5H!qU2T8YSF@Tu zaNGQtUh#{_<>#vDLpa4De*W;py*0%GaqbB^d#e=yMHS@35q(1h*KuXlxn4`&^Ku@p{53?O`j6eB2el3fLw@rjhaY5tR1AUSXE5{|ZcXk%$ zJCzkaIAU>I7zpM$%_Sw{v9PB>8=@r7`aDBQIgZ9&wSb^sflcdGoDew|C!fnl1_+)I*2jWr=fnAFn}jYuxQgqHnF?r&BK z6}@rgu=bU)4;rgbyzH;0xL?7aN>`feWietxje^#H*wLjqN#^7tAQ9WF%1L;RFTHyAfSMrtx9(4Q!h9r{J^!lL$}pxM%g24?lby3fMZ z(8o3kSNT^mfYE#TlxvawmFZ;gfw*dpsOi^+`)c)a4vc#>az;fY5HKnd8&dUylO`u| z&<@+!JgzZ+Ikg;j-BTL1;QB_JxQbD)rdHZCl59e~OXNyL+p0ct#?v&gSFj*yGJ5@V z;S;Yx2aNBZZP&RS)xS6%)qmwxyRCan+?`&q5}%r9s^`U6gvc7$ALUN0cy`|l(D9z5 z_WrC|v6;Izn(vA>gnhJ#vgk3h-nWWh)MxbzCYcz=gwoQXpmh@`fM*i!r zc(#GsT~@#cQbZ1K*IQ0!RklB35BogAca+-3;IX%*s^8nP{JzVTu8Ibs&)SxeoCQuN zT_ZxQoNVxOo3;DcpmiA7s^lt=o}>coh~9pA=YMwO6OfW#p-)^(Q9B~q^>JJ+p3Y8{ z&;Z?N`L0eB6%F%5+qK@t>VBMbC}x9I?z}bwwU1$DTkYb_!f`)I<62sU^6{1CLx!$o zY0zRKDi=j>ud+@G*hfEdf2o#g(RCdglb&+TK8FEG*`wLq@xCqR{{=|CF!Y}Uv`nJB z8wjGeW}l>H~}D;m(`iaGzFH zX@Nk>;NevE2#_tI0Ww;0T0vBJcaQ-S?j6t{Mf1D4^d3@u5Ja}uh1v&Y=KMyZACv~p zc!}|NnWuR&Wd)%Rs6-PF(GXmb`kzUs{aE(0@wU*N%~^|ODarCxgTV7AgVKYY)yU^T z3iL+fiMSmk6Sv;Q=$*LK!5?0rZIeLQSTXLATD1iAqY`M`yX4pKmvQ3sFHi8$YRm1} z#TzISA^bNpUxKEBwLosKC^E(O9VpiNEE5nk6TL0|1JFqS|y$&!NkUZ0|+9$ z6lm;QZ?RRwH?38#Th)n$Jb!81tYs#(nGJOEek*Q?V(4|WZM43yI40hBZtyB}mu-^| zNK&N@6w1%-sVdf8g4u!Fw&5#6jjb^j{bv@sw)@Kh7a3t%bsLYZzUv11y}QwC)I;4Z z$>FbTl_q2_xWR=8#=Rb9)xV#daV%gzT;rDW1QW=>$z=?`2x-;zC&jS}%}RVT6i%MH z1D~l>+cHk>Y0d(g#imV-ZoJ+|xtaKR@92c}CB-w~ON7QMol6QRgQkdib>F|G`|L>I z;pOn|dep=W%mT%aM6?d+Y3se|S8M*xj)UgB!*F?y?NV)q>$62Ito2ddGRouy)H z?Hn@GR7T|G17rE}_Tf#y}m_Ic^`9mB;6&%m%G?oE~Ya)ZK}W75}O7i=0zG+Izk z-j6CJ^GF3#XzD6zm8;$A=^k|i?{0>zjOHF=rT%h+E&Ft;@j5PCf2&$ls#z?A3oNYN zqN`8F`0ox%;7e^5HQxMwK&=1S>HAbJ+xIDEo_4?=@~x%C;w7PRERFRS?;AB|z3Qdv zFQNwxFXKL=zdwN?Y0(puKaFJ_xro^FO6$6 z?mL71H_qSGCbeH37t{UTtb9B?7nbL!Sj+x@W+-?Vht5-F#f@Rrg2RoC6oKR4*KyA- zb|#C4mwOR*YSO+}jQa73j0Eo%*pChitwMBXvMW$#rxhRDl$TJZ?56;yaVgi=&SYcj zx@~0m;|xU>JkGQBXK`%lnTGi5B>8bYo+zRBI~e}nI2a;cfBd^UnAdm&W5RWHI;AV` z|FRC^)g7OcEsE+-S-|ZzRHGK_u`iUMcp9by@_n264jhB0vz}S-Yj?Bd1HPmF?gtwq z>uzki2VobOq>MkiH;dR`4Yx%9UV~q!^e>SCUt|irJC0A!z1>4ycs+vAX-rix^jh1h zpm)1)B$z7SzVB=`-TRgCX%FvuiRtlZy0?IM^^reS`SG(!iyv3x7L%F2GcMi*?1i2w zN+R{IeSK1%t$SfrUn((5w#el$fy>|&^0eT`dgH3xRIX6&m4?stC*%5FI63H@#cp}a zMt_rc#@5|tJ54Yj#R8j+<*Hg3I~m7PG>_zNezo@FVtF;aeAC+9>{R&F9%9q8(dQ`9 z)s^yL%3=5Vy;@hL#qT4n)q{&)rs-e%<~R%OPiKa-ZP>Ik4UH(-RiQI3ibnNySMtt^ zr@dAc>z|Va&y!*awj4*nE`QEk7R}PdYRzYf`FfEr2am*KRz9ba@Y3h zA4&8&MiCnHLZCWBNS>%Z`(3i$f%i6W1&^ZfW$P{;EmkJ%Iag37J@ak0WC*4)SmG_E zM7sh+lC(>s6~ZN%xJpCs5F{BW*`e_HuyMzD`*7PK#j{t>eYfJ|edFk{?x(U5!K<{> zvlAQ9T_4TxXSB0)(F8*D(L`CkKn>FA{s3H^K?$*C^Caq~!G| zl5;I&kR`?Le$C-N0fVhB6ry%`r1o**zw8+}PxBsM_TVwkc56p5R|x*RS?lG=RmmL7 zS{p5U@K7|Vt(EeLo?}LUeaAs$0@t;mPJ_pTVdnVOJVJSfiQ+&f|JEl>SeGz(3bzrL zi9)K6Yggsm@7#}O8f8MUUu1mO4@WaK>zUg>HsU6I-}XCP?aMTqx#V=;H(@>Zijsye z846#XY)O6^-E1YLjIX<3fpurvqc69mEB-0lTX z;x}J$-YhoSG~vlt`;ox+++n%i&Z<3avC3uNuS7aNn5X`At#80!Zs<3sKhu_^?naXV zTVam1b8`3RwmnS|c1CFP){VFvyxrB~J2f%w(GB`$Qy1Qd&B}G-((*Z#fbJ^(dGn@EK^(0~=^c(Os4?h9M zwm@i0ZzU@R!!CR&?N`#LH0~4S(mIt!Qjy2cK%OX3kV^7u=$8$QD!QK3);>M2oSK-( zFmEd~Zdl!<{n*%gD|*4pgy&a=>5lNLg~vLLPHa&wHKt%?a?zmg7O<9?X~BpZ43QIL zDdn{A_v^dgKfX(;kx&7rMr74O<5-XJ$VLirA4l60=8;~SF~#hhLhVKNslU9~bXr`8 zXuikEF)CBT3iUw?0e)gt)kfZ_ji^Gqe ztZ{O3o?Z7NyzPyp5IS4lN*t1kiygH~5p-`I<4ELn$Ih1X4CWgizrZZQetvZ+p5={W z((R{hZFf?XZh)S{2lXdxqw^+*bWXmcx%MQjTQ`vrS`;Ifc$2ab@XyeF&;l?JOhpHt zEC0$`8=L-=eo85MlQ|sOy_5m}SnSmDEMJqSHx_!b!AEvTT==7v5Py5e+V2<@0#0Wd z0v%hZ^ZZO&9pMN8fC4!NIc8Jk6E|9@E8RL}Y`4F_U^Kh5%%X04)^3OC(a1aT-3?#oVXg5b zWtOoC+pWNjvLEDJY4yk1W%|nm{+jPt^$Im|7;jcpzz#&;-bPfLw^&H|b2n)E-S~^P zMdvZPZ}|i|eWa9h5u$lyN3Db>he|EjAAn@{-hjXY@;8Fe1_Kc!NTX2WnojPMO+;W> z@M7iLjtHX9y=Ro0d^=A|Bjee1TaObqThnLLxE`2z*DgGP*SYMAzU6>E+N~&x9#MQZ z(chJwS5ECj8Cd%M*JqbZTM^8)=}xmffAPhV+a{`nr{97D4!n`+VY157jp?>xk_6I? zS+MsHuzzx{E3KjbvlVx{p@E1e_xOoK3z4Vo#*t=s%8{YvcDD={ceUqrd@Gmw>4E^v zv@cr!v3Ga=+qNBv{ne7jNQD_;=UM*Tp>;eoI60@AfWRqv)O51R!rnk5v|dU2Xze%H zN!b`;RN7xSB(r6qO#6md^Sj%fHE?n*tTlmu-`k%gHUhe_kCE&A0>f0yJZX0Ve^Zsi zfSOB~(a6>EK!%d>(bg1LDGz;GLbJNrVQmnOaIn-#G)L*Cp9+ML1bChO@;90ohm|58 zvK75f^F>jy(zoB;wn74(>ZugWL=!VYe!Jw*S6b>2PadLDc1JMtZe-eZIa`TVIG#jw zxO_JjKDvFYlj6=a$?J|!%55r9dpd(-m5F}Dt^pzK<`2^SP`6f$WOtNF>?Zkl?F zs*275BR1{W~#XUGQ>MEORM=_q#^?*>a2!YNTuM_Fg zTdFULt=Qp`Xvu{ir{;(t@M=_LJ`;nmk4F8s<~3h6@F>w~kwl-I$FC2irPj@=g< z7z$@wMmqf7GixsoD|I6Td9K-3`%-yHT$r&#WZHiX=zoTvH6Ynx{xZx{zMpj|Fl(B} zNpsHKeq*xWYm9j5N3JzX7-nO2!iD;^LAruokVQTgxI>@sSFJ3{1LKBi+^;B#v*M{i zY$~_MrB$0d?t4y$*cy#)|0Iv((;F+9#hd!tr}KHck$Mf9IXq#)*W_B|n?X44&RhXS z;$=>J=*w&KNSsSUOkdAD3>DHFM{ptPL8ZpQJ1%ey%B5a)kfOG8>jq)?-me6>MyVmk z$7&yz_i*`QUP*pH-{zR0Q@PyIyMovP#*~b#hIC;HrB973Yiq6F^ z`Ztk14~e|^*)A~21Ko$FP0^M|kiLAlh~m+xu%yxA`(`LKnXP1;E$`h}Vi|9|QjRVq z7(TL7i8;v-o9=yrUZ`6e#uz5-FIyJpmfwc(2)&wLD~Jq*gCDo;X+{Qn*x&5A24v45 zC00lwBc5CvWI;uel1&w}Vw0Cs0P@IzZN1Lv_!|vuwqVBa?L|W}AAZ4Xq20^i>Q~3d ze6y_%(W_!@lt{MggD(^)`4DGWXJBh{Q-KFSrUbvFr2_Uej4gTOF(i>m{+R-~;1sTs z`g~enDd9I-=NUw)Cl1%WS*_7$*nv*7C3)b(_g>@z#T-hL12p0~I&FLa@=dR9<3ufb_)1n`(C*iX~VAriPU^Uh~9)c$a#9oS| zv)*`{%@pSCmvMi&`4x)q4Htvc>ZxoF7CtgjPLI&GMqt-(AiwPOPb|&Q>bVFV;k1O> z+|3cMTFy8$ z^Aogxgn9{Opb~Yb`j)(0A>DR*H;c?Mg+JT;lqT6BNN& z8nF4A?HCL*RZ541qYRfA_Sk1t`$Y@;8ZLf*++|kJmKI;fE)H*2cFS-5$L!s@cv8|F zsH5fsBs~9&4c9jV{{s|77#Ti^fSzMiA`6xDFuz_1JUm-Gl>nlF)Rd>Qmj1WmBkTwJ zW9iZ#7$zlJMWk#VmZj==xt||+x2MtZS)0z>OJ>i7Cxtft~<$8+iRw7;j@VNRoVy8b<~~x&->_`WF&} zNCWweX+8#kC?y>UzSk5NFRKPa}07JBaesbnA1Mx{!x4-YjKjjc$EK_Kbbx`IB zFZ?}|sGn!kojrsuLxos9C^>SWAxLBYYu@*jx$Q4xDDvu7GX`J#m~N|uhVbf%2(!VD z(6!s^mjIY*-M#VQ9;R{uFaa#h*zWleDnR6e99Zh&8zJKO=M}3L*P5KFeEk{7$uelk;g??;ro{MQ<7>>276y@J)Jz#9h4$4-tB>P-bL%o;BvCp4`K5m()p(WTrXcQ zUc~=jUnHde8PP<6BAVgx3GKU+6h3r@vdC+9r=}lBmwT4ETrNeY(o(O>%%{~=uEtjf ziZ$KIMV$x=$cDvw1R^=#8*p6hPZqi4LI^P*3r?3H1VJQ#^xD6`f0ZR7lgut`G5U@S zl{e6SiSgRjrsk;qV-d9*7KzIoYVx+veT3zNJ^V>NtR+7?X~5q!AR zcGR)>f&JQZZNQtK(!DN0r7gdxp(&tGq&4tx^$tPs*i1g?u&(VnfePYEPqt;C+!v0q zK;cNol!QPT9>CQkCak`yB6*ipY#Pt;h}l!x);b&((Ux=MiB*8SRX3jTp!0Qp7-tLW z!qhl-f`I9RmF^TjY_^SsOI|_FqZU1Wr!BNr#nkAIMd`W20SQj02@fIF;$kBt%)lTI z+!!a26cYcY@vT2m+uw}N?R4%P2;3@uCWd+b{vor7KFQE(5pO{MC%EM}Tu=`jhgy9| zC^j^bTH^ML)`QdJ>;8jQphSf3R>bcubbZ27p%C~=Y+b+n+Q_Rw>qMArIn~ z&Ut^XBU0Uo6jHg_Xpj7>2GJD>a!(rto4?7;2aR4<43bEnx#lSbS=ZRYCyYUi7LJMy zEHnCfme>x&%&pkb#Gxk!S6^u~s+Xy)tCxw$g^Gzegs;&?tVJV^PL0J{w3bV}`>e?? z51$ehP6r0tulUB*HFwUs5+l$f?LG3rsl)C15S}FuuEzPUAAO?n-HpkFW8y8V|8neT z<}Z1&!=<}SUT>eX*JA#IgKJ08Ih_Wkjg!JvI+GX2Cb=oa8)2IKJ2wKTf~9AF99j7| z#4Oa9ZCg)zo72N8>0TrKbq0e_pHDdIam?5;!H%7IRK6en1l7>6c8q0aQhA)V2$F!l zCq3d7_P-*9Vg8qn++0!nUEf(MX`E-Ii;Wir?tu8WQ8$(o|Hgp9SZF0mKUGr!$c4A} z1N~2~rG+(ipv*{-O2j+w&(u}7950{2?Nrb#HCz~*Z~dWM*!=uuB}8qhSG%eGZQ-oz z@3TjbXKE-m)^F{oLRU-a!@Px3}prv00;KG znM;rhK?xYtH`6TQ;fo6o$|4!^806xj@IRkHWDWCms|lX{J7o_50D$gez0)vWx%Dgj z`0RF}(g+wtgRW!XA_Qo93~hR-B{r zhay3%lzqNIJrnSAL0u!915*Ag3 z98MPlV0CLmVp*o??R@A%2Jb4HKK55GJ1P=jSF3gyI^95?1B(P^V$Y31JISj&O{)Kf z1;So%PRptHMx+JPzuNdGrrG{umn!mN5Obq7>8M0sCE=j?t!fPZo6R><@gi1!1)k3x z9+6dqDZuM&_wh^sRxR}9s?q)dmF>pf!zoX7GIV1dMD&L_-(Masi8Ah;hAy_BK$Wr; z607pddf%xR`7!=frOwPfqo$pb81EBGXj>JA&2V9xkTZp}-3a-yj#EOuy~?fM;3`X% z#<>mZiocaOA+L_~ho^u=eh!3M=VILh0WHK{4@5M|nGDL#!?QA*48!TpwQ(Dsi}zN3 zQpb6csZh%6L*xl3$_VdUVLJHHmHS*g6lHgmrB3TaToTc~^PP*z!|Ry~+_ruym>c-7 zM;a;sGk>|~HRFzZ9{!q->!W{=Qu74XWU6n%u;8@6QN4>_luFbbyR-2~L6X&V zx@41>fLTw1v@>8eIbiVfti-8>CvJ?-l*D>df95xIss$1?4<3!b!?sNVkjs;Wf6+_6 z3ptyMbG_B=X}olm+hf5FtBF^jhav}0iCA?o9vW8@%l};ONjlb!#5ASfg2SLs1P16~ z&#oyifHK=XhhK|frR5L+_!;Yje_v932aiyy{{ON_{8#Efci`{mKSAZdR%dEuuX2!R zY2oDrk7g;IsG}Mj6>RR=L>L~xCgPqE=6Orz9z>Sdp4d*su_g{Q|0bO^x^ATytFRMb z*R6@zO_+_ICdKDyLAjFs8#SRHsAR+xP z`i#r|4e4Ht2Cg7_j`xw#mAkHuHnqX(tZP_SpxD{@*HpU(w#f7QUVx*rcW%0W`J2pe zvFzJ)d$AXcHJ?$a;SrS~?!q&I2nrx5Rg-4X?4Uobt-sGow29x}SS5)IU{B-+-2D;& zx=u3t`Fj*cOb|qcutv_AH7J%sn{mEKU|vD@*SfKR`3ZoY>m&nJ?Ykw*ZN4&jMo>2L z9+l4}ET{aFbGBOHIllR$VVA!pRvJ)ZZG7=D1>wmlMj@j;tv$l_Apo%>k>s=Htb{&5 zOcEtBSXcRLbZDpmAhiuq{cr(>W8$6lTa5_UE^XwL!r+gKpqq&+%3ohdUO)Ov8i0V2 zpn%?Fzovml{BFMxVn5xA&BHTh@@yq_N1Ph=LS@h^rZo%5jfHwjMsmK{_E@4|1disU^XDJh~LZrtwlEO z6Vb&0L&#P5BuRnLll~Q>8in-_?Ujp6s|TjzwZ)F=N`Bsq6YR@AQ~r4EnOgcOGClH4 z+2BV${Qj$f{Cjp8jRtp}YQpGEuVvH+=yOp(jY>78lwy-ZYX)o3jH&gc<6?%@(Er`8 z24ImcAZ&6#DW;raM;Kog&}+a?M^4^+63}|-~B@reyL&WnF!;dVy;PK96nIS9q=LKTOKs`J`J{4nRlJPRykl%d= zG3vI4OM< z;b?nC0kZB1hV09ti-V;YJT9`&@kL5hmH$?%)G7ekoy^$2*w^42_83W^45M)5Tcth$6$8;Zyai>Qj!UWN&-lH*)Uq&F)0qvg@}V zaa_35bHK~(R8a~o(8cF8q>L1kLO0Uy5+Fvj9$ORj5O;)4mG~{8|QL zzBuKOVS(<8^ntDly(b+FyD;dj?_B5R=0gZO(<`@Mj5D*vJYT#{;M(zA>PZz=+318$@Gv217X(lj z3uP5N>q$2YuWDua^Ny0ccLX~9+MrVT=3&H|B250BLEfaC%dz!TqtOoaP8XDqXJ2&d zb_A}7Q%*$u*m%KC8)!e;*O=+D?zT2kLNjB&j}psJ%PYHp*Gc~aY8cHg^|Z?$+YV87r99_ajQX7m+yb~Ttc6>fkieac zpP#Ri*DzMLi`$C1Np|TVx7_j^xXWScPRq)a!a~GDR7CbhGrKvHP~N>~EztlMoxqL2 zAm*5l!t66>{fY4Pb@h8o3_!7KPihnbF5QesOMN^|4|9BcT9wNuyUkY)zbsMHx|979 zzsL}7T`r&UPuFoPTBJO$k%O3A>_JwNw z3_7N78t6=v@X=l%;MN?!kD~6~ni_aM8VbX!nz!Ox^e{Lc&#}b4&hbz6K~7H%e~;@_ z#mfN4{2r2fPmTOhB<5IwpioxV@{JeWSR+tA^)K=CC#WE_0ai>QO(}vU2X0|}jeP9~pqwJ+B^R{}fTY<6`hs(&s@Ikc*b*R)-K75{Lo7SK&pec;GS(n|z-7qDP08-V^ z)wm{z0_3qGp?%X}4{GvpVGRSFJ!8aQ4k?pAQX;Fk<2m=8qq zXm`62q_daXTE(%CIOGHh+YcfHts``!*dmLOf^B0 z(iy54FX$se`GdU@obB_&hk*wzE?k>m4a(Hu7o*vX5Q zCUW|6ATu{l7d*muM@QqN<6(r!$4bMjxq_+=e)7O`M zKwWUowfemA!#puM>uutpu-7z!)xo&s4fffE`|7*&) zb+X|LJ_6l-|C!GS-{;{IZGZVnQwvnp(k=4&DYrKI{=O7=GtqNZRqiBNwR<^_d=~Xd z{IyAtW&XGq!bf)#ZlY-fA)+NrhQQd8Wnz?lc~1v)`zH>Tkq9ni&ToX6V2<4!jAJN^ zH;g{y=-q-`4+p3$0J1Bn!NT;oA9aB`S{bT@tTeWSIirrbwIKB-oCn{HV(@Xz$}{x(9#5SM~?1(+YT#~m-N9^vW(m>VPlD*5y6WF z?(Dg9hdOSu151r%IQ!=xfTh#Sudq%vyai`>L+z(9K;?*gWlXg?&9Vre-*P8+RF)-^ zaz@428P&r++kwD4It}CxA)0hE_350S_qdc``+=XErqkN1@E2v_+3#N>KPOIv(<^(; z+x94e6Duy;VVxAfYLhPz<6Uw2UcO5dUsZk0PS=CwBd^B<>*(1!3IDvOvh$q1VO8&J z%D4=tTSF_A>Iz`e`B8oRL1(`na#u&}XNB(V)5FqMFec|>)uUn><2#`jN9v#T{y-@- zX8Ae+8pSDwoEziCc;;JsQufJtF42dm zlefQhxU|OzSWTR&ZX-2i!h+rBHL4pwP|Yw!p`$Msg+_e&ifU+|OIP z@RI2zm?;4EHvVgaahN&f%MH?CQ3dp|H*$iv?&*`13Q_5z7!Tk}@hDr#dA3{K8Jsgq z`_w+0pcOX~zW6-1BpZ)(yBv>w3?xcP$^FdVDmPws_;S4}oh%Px13ju6KYGr4ED}HI z#xBJU+j&~13={J5RV8d<<=$*J9!@(UdYN!u`y;{*0!bXZGME)YXIQQL;yVB4v8Y4` zCj*auRkqYYBJqAjP#bY5sJrzwI5%{72fJptq?=Cm&i09+nc%)Y`5e5c79M+4ljPG0m;XH zSx?6utm`c`o;6nUlNpY4JGJ@@6?B@xsSVJE-PBy%%Vg@1MZ=(eT@nZ>@e?iSO8P$Qy*H`j%aW|Dp%vrIXtb?W)4M{uXrz_Ea|SRNb;>F_j~Dr(!JE(;{!x_K^7 z#2xM zp3qWo-@&Ob>UOcGSK@+}fa1uN^C=!&fi!p#V4{5{SE?@2S<-8Y~ z3s99_U$~KBM_&qlvtzY%!^SSS6#LS|Hz9H<4Km7$0))!56+LN_H)m1rIEY1&xrZdo z7aJt?Sek{CrY>7WO@(#zx`gduQYIq0!3VZf6!AW)BvlF@#udG)R_E#R==v|t}vX?%H*K;XtlO{(zST#&Dp?=;o&cjrScCo{P{SyPy!-uE)DlSa0(0y42uqzG&cM*ZqNob(a z5-%fxOBkN9|MeFF+jRyU;stvR69cAw)}>C7J<~$gBIS?GeeqzUt7dBWOTEQe!A>MI zp-4f_G@z@fL-_j@p09__DYT}vC%@}gg&&iykXSIJ^So7qYpAVZPabinY)QzDt?x56 zmqbU4cfx>EBhp8A>uQB$;JYR5Zhnnasx7`oWJ5zISl4?d*cA6xXf%rg5Yya}IJ#Xn zDdj{Nzw7;}v(dbPM6(wSx1s}zhoF)AHzxj+&*G5|B&yu!OEpYfz-|pQeC?PRfgge| zs!mjECJcyTFP}`~;L2I_*D?FK_v@f8K*`PxQ#F;?)s}9g<*?))Dx|sO@Gh`(k^y5C8FZQ6b&y z1ukf>Jmo|SYo#FFNS~$qm8i0%LWJn)pel1miu~8a<+mm&h?|>N2AG-}!S1pV(Oue5N5C__r=oLeX2R57t8oXWq3A5xTn8d}`jr zc8Mn6ZF86oUv(4x3ONVb+G04JHM2MtrJba&`CKZrn;Dd2n8!{rY$djos!Qe&0c!wr$|s(lJuE2`XdgN)oZu9;*`_G>iCR@%OL|p1DV3RTPSI_ zjhPw(Hn29k>VQ)5RJl7jDUS6Pp^;L+?=x1O;gW$K>Y7Ih&;|J1`>B}_(1sOiDo@b5 z&b?afx$4!O#D!b*nswPujA+7uBMMUTuh#PFvbQY_2Prf6j>17 zbM9hl8{B*(*5g1bMir9))twgN_;jwMy5t+7`Mh3vuu&ImN$I1GKmdA*+w^kf*Vr^@ zZ_q`K`BFS)sot&5woV-U(0G6FxWsS>d#OFG<3kIbd!s}vs_W`S&o2hp7wn>@Y3H0D zz0;`Bz1-1Ah2_G?A%8AoHlV2!&iXRzl^yp@GtAp-O*YR|16=G+U1_G-`)Z_Y>WpYp{H38EOem<$iNOyP1aC9gQ^px;OY zoT$PcQ8`MMea~}efa&+@1SHwi2Y*R-BX~!|iLmK`9ssRzQ!Jt6dc*v=Pb+2VKp*H_ z#(BZ^AN}`_8~ns8riFQf+A)E06$9jy)ZCW8r~#?~5zKEWf&4+Q5StWoZ6d`#?!_zWveUAEp0gh7E@TqGrEt93qg2`5^4 zd@~cz*B3x|>^&6-bkvfG%cNaBGg5}LNVuAkEg<)%(Y-O)IBWQ-xt)`tpo#G&era%F zkTChpxSVy)gJ-Ol$LFNQ z_*=J#o+G{-yMyr)dtRSd^jAyluw8_;x0b~Up3RG zadgg2HmF|}?&-`iAC*1<#*qPi()y+~8z4_|-}?!d^cTj{4H*4(mizZR;qJW#kDrio z({%`cR7kgTWq#pMR$a7TRjnS2x@jBv*s#TV-i;dF%>{UQ)hR!7w{=>111}o^w`IlnTS(?Q*p=}uI z@1L_0=F-IlBj(~;LMCAOPd1h*s$KprHNE$rRGJrFHQV)kd<4ZtA*k5I(DO;+zBPxe&cmyg7!fPsA*axsW zJPNEC379`%OpZJB{(Y_H0pzzklt5GALuzex43c#V<4T)=wNp_zfN$x zddD8WoLk7+Ez@AlZn``#%37CRise&_(9O5}(ffwN#~w}(6N(59KG-_nf49`}i||L& zaNek`&4_C2jLfO1^RE%W`J?lH>N?LD8nxFKP5i@A?M&Z3)+?Ooocj@bIiH^(dWbKS z;*Y)(5PsLf85CsD(0?V@CO|3z^vV}qEWk@c%UhZLY72j! z&;Zbv7p-i}GqF&q7YV#F|C;$;BJf<(_c6Db0bB=0`H%h`%tQem3y?1MoB^4G(f~{O zUp93`kGl_9XQZmfcHx$OV+mSTtbN9l$NaC6i6H}LZU0qhPT~U7!llJX>U*I25J!q+cMzQ3-75u&Q^A)I_jDH7AT!=BiMiSTWb?m>G z;R%4g%)%;vmB@kF5Qi53wU)TXz_e6|Y*|`9LABsHrvDl-^#w4PERGeuXbs3ZFSGE! zv$_h<;zRn$@YK2=Pi~igo#|g|8S?t>rIB^Si-1gnARGOkN!Zk^B3HU;z%<{jW;y9A(yI?z zIwfjJqiU2&(;1J=pPfZTq=9WYe0)arbG;Mzun2x7-{-QG7U*BszW2AKA!(yy!s4zu*I+p9{<)6eyqz-LlI_^uA7%B5c3 zRK?UzoSdf57{{iHfFYOQ@p68(NghiVRiPIR6G~EttlRh;HT)|mA_MlsC5Eu}+la&q zxS=guAqMeQw;{rOanWQM&!Y7)yl!=eXRZGjpZs!@DuQALE0nsg4;Z9u-tG+{4F&pdB%9GG_ch4T`8GV>D7a{Kn}`k z|K6*o0Go{@<_&zn^;NaDzQp<9;+TXkoZr=w>bF+`Irfbcf%L7AVR7rePrn0CPk$I=l zy4njJymfrPIfiRfcf;j%evGr^wxzapX-;In+)Xx#Kgv{;tghHRXZZ3i!CLBYJ^~aH z!|zMS$*x^0)ZV>g`<~Cdh@_%Y4O1*PbIova(m$#e{Z)1TXgfX4@QdoM zyk!J@Q20S0#3Wfs@mJ2f)2$UW`G?Jc^6jy^#t~$Lq`+re*n)7&*<~YEy&I%w?6-)) zUbpxaGaG&upX9R~YA^8QL|4=PR&n>i#sF~M3sT?GqDsla6wL_C9C!2nI=+(~;G{M{ z`#W5nveB$HR3{g;8$3JwaXlkGxl(cWflYRgYNy5%?Mw|g|0(o@>gu@&2@j>acT!+P zb!X79Ff2HlobM#wdZKia#Abl@oO8A7M=OFvS{N71OG&Q&ULyYE2iSK)G4e!x;^t_P z(JV(Jox|tLn3bsCC88FS=1<9ny$R2|bIQcmv{Ie`=(-J{&!CshyR}eLt9a(PA<+k>y6Kz%Zs)t#w`qhRDY>9II6$K3kK5}_)o3f)zH;f zdza_zEZvS+2g~RTbCZKB!%>Y_1r@VG8S!nv#+AyHeGJ#EQRSZj2-MY7on+Pn ztj?K3LQ!i&L0~N_>3ERmOMuFhMzvMHP-<_5O(3n>^B84&TZ|T1gw8~@RR(D5kyeue z=1Bo;eM_?-Bg(KO;sL4-B|9C9D}3&qnJ6Ls>j{qf@WcO(+MCbhUa^TV0D-5gpUXO@ GgeCwLj9fqf literal 33144 zcmdSAcTiMa^DZi)1j(Z0D3Xz!qml;+5|toP7?C*SJSszmA?G9+l#B!=$q+_KL(Un7 zoO9-EUf=KBU)8Ns_f(y8?ybuoQ^Ve~cdyl}SNH1Y=?;0Nu0-&F>cO2mcL+er@>+N9 zU~=ENgTao21zd4ucN4#J=i?oa{0kiqQq5u*0F+i53FPIqnx}QE}9$db}o-f1y{nG#^d!?-o&^bdS|YjjHRRx_a;+r z`OS8VRDqtrh-GmY|KI*FfA}sj#<;g1lliUI0Eg6%dVdn;9vf1DI-ZaXd?}Lw0_H7) zfvAirOE*=o12r4Cnyoshb2KaM?@(O?k%>zWR_m`}7=-ZlQRXc$(Yxo;h8lXJd|3xh zs*bkH42_ginQ>)uVgdimZuPw^GC=>c_FXBRfo4wRb#3lA=L)_KnTpp&HVG0v3UX~s z*B;Q|K=y54!{p&%e^ui9#KlOD7dcm)dvZ#+OflCT{>&OT)}Hb5!*EOjhc&-*adUT4 zTvxdZ$p0IWE}y$rZ%}1gV3RmE{RAfHhw-KR+sC=1&PMsFolqndvzi55KadqIOl>$| zG#sQw#`n>EN` zE$aO`;Se=Paqg=%z5G|>-y`}U<4@SB_l-Fd4;MT6WIGx(C5v+-IoEDfI`H-6#!xXf2Yxy|?1d?7}>GbTm3W^>KShY648gHL^AN^-BVa}8rEs}61eIkeq( zj*yTue)dGXEzJQ+W?wzARe7 z5tYe`KE~LngW36dG84c{a_8&H8C#!$&b*Z`G-9^;9{gj^{aT^iy)wO-uguFBrcvei z&(^%-1!*WF-QPEdTZiAynKjpdrl zvdlftl|UOuOWd*Ox^SX_H}w8h_ZMS~gJ6^-NzCkJB2NgAZ9yHL*n7~J8DU&na<|BX zSyw|X+R_tq=BsfAUmcJ;3;O!SSkhmUvdha@bh6RP@WkEg#W2%nT;yB}ABg0#?(GD( zalQE6-%`;e$}LLkW7PrPSD~`i0Vz2%TVm2!WT5=40$n9GSS-Puxn=M!D08DBjfQe( z6~oMrFFsA(QRuok-d)O--qQZPDwU>E_fj9lZx^Z{&k?Ks+|f0Row~W_XS}>tPMbpZ zYF^E;iz2LO{>7)b|0s$NWI9@BT1=O4xXsex6({E9l<4`*sW`#$ZUTYyPI|_8S4~TW z7aR}khX(y?kT;id*Wj3nv6ZeP!zM(wQWiU}4E}P2Dp5anhPb+!1uet{jOLW~H;ayR zkWuvEGI-w4qp7S`vSHD@+B1orJ?PGLwiDMQLbjcc*2foCe_c;un5&T_EB0D3*bJK! z^)24T3Pb%5iK1B8!j*gZbD4Ljxi<-m2(B>j4+dW&<_LuiAFJu`#+uis4?(gW$`&2% zs6da1lBMJ%I3CPK^RiesVR0XXKDdZSG1ju6EksGWztR|-g^y)Ddz$9?xeqd5PjZ9I zs;pA`&XY&wnESK|V@-^`VVMllECS zL^bK5=gwX-hok{WX$FEzzF(#X!o>fODP0=l=7ko(lA2`nc61thcZU^rMAgl}*XpP< zAmRKxxz$x1R?>3!#RNwDcY`4(qrzrf(9Vgc5?2?*tOqv}QrMw0@X@i}L>JyqR<4c~ zK!og-cTrcplTq=SrCy{>lwn1;Ukw$>jPAH7ij69F!l~w+Gniv%dI*z99{wG82{7>I zbqTY+VXru45X~yg1X4+HU_IpvHRi30cppb^W&&*g=^EacveH~nPh^0F^j1->&y`xt zI&Dpd?!HrZ5VSEWZkDM^6(sX5cV34V^Aq1T%5wRmT`2CP_*3PRENxiZ*>_~@eoA^D zd!+{wO$72}<>sLwp}!7Ds@Q)g#@!&%&Xcy%KpVS%_d?`zgDuOQ?w>yTpbJASw{k}^ zR-J)Ux-=0S@)hs*JOel;t>b2D;bm1OtZ4VGR}QcLykk<-<){J zeJw4bOcd}Uynn;2gULm*;nJ<;w*)sux3n$C5EH1(Or|u39L{^zkc2j~LO81_((!~l z-H7b*%sb5QRm5B9dm!@?-Z;`Xr3rLFFs{kcB)p}c*z(+4xm(b_EH}@BqYuZcGV{y2 zj))hu>P(WVnhhBH=~ba3{WmgBUa>Xy<7LSwK5g_&8s4={ij(_=W8qbu`!aT& zoWaW@OtKd<>^vD{W@s)O+W?{A=3hx?O$6LB8%lRy6AS7aeoP#0ZzY(jGMvR0fSxl*)<89hc?K#vR&I{W_TsZ;FePn;mlW1Q~=7WZ*_}`VU7k&&Rp#JK`!Em*_Zu8spYB}`% z*sI&mwnweH8m>A&d3;k<@76(Y)-gd;ud+=h;Tdu+rH@`7>6m=YG1roM9J^lBq@&cH z>&3w`tnI6*)VQpD5&VD!l+Jx#y|pSPzoJs1qj8icetVm*C}aJltB&f|0^rGEa@Qdn z>FUTr)0m;P|DU}WQx{C*{l(BC&+_Z{;}4v5S4g`JW6!00wwv5y`UaQgi%HZ>@IYge z#|gW8ADu)lbTQDTWT8}S=iW(~!%XBpflhy!of3*syteTAaPjqYNYEB~#K=;t_NZf{ zP9&pFsMF$Tt6Ll&=Fgrw3$=%2^Ve;Md@Zj;31@hSx zxJ-%$U6I|xBfV9~=38qg0!b~c4ZK(x-BCbR3c8^!rx$BY$nw1Hs{V*?FYZj!=ZSp1 z9vQSt%$)DAtF_MeJ3c1W^y4|lZys*G%$;&SwM+B8Y+2j!7l)pga^^S}pVHF)JW;p4 z*iF5e@gF^*S#(FbU!7y~$gdnJT$QjMj%e4`Z;6=5Jli?l5^xx=u!`Z2b^!W)GKX$KAyBoJv^;PO|7|&TP?0OG`%+;(iBlXy0yXcY`9lX z!7<5W+bW5#795Kpa)xV~16cYz)4HcA0#wDE8W;Q&c zmCHr(FCN-o3B7hBnsR(`E}|~w6(@225u!QsiQ`|ZCnF((;5QEI8BIvJP3CPEUb#EB z1`)XKNO`}G6M00m$=Q2t9|wOSWChpPR+^ye8o&+OM&~Dg_%3nw+t(X5uSTh-?*}?D zk%3mR?o_X2uDVS*`L9Y)X_{>O!Bg^HrYvyo;;L|T>0XbN@oVpqTCpuUpv|`sB%~G} z>Jhr&+U|6QhxK+1RKDbn5$6%6RpawXs=q9@@RnZuJq5iw(`?FKcc4x8UdXcVjAqiZ zNSWOtqb^OZfjqNF6VINVlMh-OIw@pVs#_~apOEs}VD9(SPqD51Gv!L2_9)N0@${Fg zeLx<$^UXznZ zPe14P%M{}(36@)A+G+2NUS zYQEV-XP~B#lVWe4PhalmtYo`<%WOESyO<{O?%IlX=k_h{7w@_#<TTw;Qi2sPoSRF^!fDfaS1KdNjjhC z7GX?{CjV&~i5?9f)JPTQxfUSG^;}7j5{j}j=DDs}BpMe`k+I!L9~@6;berK9dbqX6 zHSBEE_b`krit1drk>sd0qnNi6a{56mT?V@H$7QzzHMD2Av_lYqupp!{se+onU&NVv zb0KK;wr1*mNd`*tr6ZY)&wR$}rQ-0cm-u#+)YKEa-pPaHi#u?~X9YI{0`T63p!s|K zGT>Ctj-T+bVYS4xlo8+eMpEuO!$tmamcc^V*CYLwV7gfG2K>Vwfl=Ev+t?n`Q1FH) zD{aDcrr)o*Ih~w*)te^Q(Ki*HYrjbqrMG@Hn~9&$rp-8hj26Vwvzt*I9`!m03I-E-)lF<&F3xZ8oLtFN%OR=we=(I(Au%W4mP?m*{ zi)@Sc#&*8u0S1v#UR#Gfh&;V2{=WGzI@YXuD}UqXDBU+Y`I8e-YOxko30JWVRe@aJ zq!woA^y5VR21amMA&2X=tn#@D9p^Lh^AZou@w{#o3x>XbUe-6h#O!Xes3^CxQtw+ zy!cg7Ea9vK?HT&!CHK#d57_3K^E3S;j-m9T&c%uTikw?l(U3XVVennF|R9wk?4)kVkejTKvrV7@)DgNY|84|nMqww0LY~cD}l`dLwCW@Ow zjRCuN_fBLHja%+Q*-b-(v3cIY$4XMjfE5D5FE^@_3sYteV;52+Jg_^1qpPn|9e9P~ zuX2nM+?39=Ru;SYN8Zx-;s@XSJgu88__b1qdRoNH-RxA|gSM2(YOTOpWnS*y<) za30s%^@MYyFBW|M2L;n<{h9|mzP5LsDQMbPk);v-o-1sd{gF;Q7h--cio zE2;ibKI2PY0gWjcN{r)7KJl$zn1N3%Ir$BpR&>F6zW14idZYN0V45?dvnj6u>k&qj zcYMijK9DdC{6#&ZCyDnXKm89ls)IT)0?_;juSAMa)o0CiNa{@Rxwg_Fh81(GV7Q8v zo1JkRK3SF1wZO_lvK1L;sKr&f6l!BWl}BY~>#~E*eYI}acFwP3Gu_o_RcUHN#%ph8 zRU+(bzgIR9?`c4*Nf%_V<`u=)pM!eT`75tP=C3j&#~9|Rc+=ixK##o}c{?age__ly z;Lwu$f$_H|oEyp`*izH;3iA{6k{cB4hNrcC zuW=wpnUH&8qh*;#{vQ>vN~lQC$}AYS-Unhaw?oh_mqbra+qTkbX1c`QFkV@nge?Kb zCVDP8&AS;&5|ja-3SN`Sg5D4D`+^%XTy%+ z*fN9xL2SY!eR#l&VDL>E>SSEg8D`ebXY)wP8#TOEn2^a`pvKqs25Q;Nr1K2fE>Sa2 zMWSe_ys@EHTz2&6Yw+Ud#lS4PK?n;X+D{RRJ6Uu>J%`chkFbCD@pQ`&8Ne$PE^r~G zAjr4=JIwF%Uf274spSC87li?8U@`_8>(;)dd9VnqoMWMG*D-fyv+)~=TDO-u%g1rY zRJsS)a2x)Mr9leHT1A$V*ZYvN+F6p&mAlktbEms$zeI5(1?qcG4w*ZobpS)?2aj~~ zErjufd{U-RnduUjGX0j7GvfRd-N^~N`IA@Ta_w1Ror)|pkAiuucA zG@t!ttB7$ntQlPGB@>CLp1B98 zx85b!!A?(O?`XDQeccSli|CC=Ne*{(2`|S~RFq{3isoDo)qJJ+!M3g~RbxhP4AO$R zaudhgfEDUGph)Tc^%<}4uqgbT~xg06Z_vZ$XECmzxAlg((^~1j6B9nVSzcOQ@eH2S3}D@ z*+4`sd;-VIae>(dT2KAP6>0gs=i25F$A2coDEiS%>g;U|74BoXa9a|j!vR@;TOm{1 zEMhjSDnUqVt39Q`(f2;$w1Z^X54{npsovSah2c5&gR7WRI@Y)i@*?gT&8oEy>9>r_ z=Pw$gSLqwCl~Z`|i!8{@h94(ZU;Q4>53Gs;hubRFx00A{)bVZ*k7&Ptv+oUV6LEMg5pJBJ zc6&l9fICVCJMCr((@R3#&l`GD{#X(R+XS5GWS~XMD58SILD6qQa68vZTTb4kwNclc1 z7ba-5vbFOu9wI|2z>3lhW@`#;0C}dcKj;ds4LDhG zaoS{k8bzyYO_K2+^PK$|2?0bzk)AktR{IQv_-cumH zuc#a5dg6AO#Ywt+aI6%oO<0r-m&)QZquLwy0XM!y?GKH5Jxb)^`|#f}Z(VpJ zhbC|x*@d~HM-Lyl8r6}bBeKZ>TVaK@v|aV&8+ryk6^wGU9gkw}4$65Uy;NEXUMUk$>J+EIXY!sE5U-)kC=+_5oMvm<50v0ruG7VkcK_(FFIDewD7&qWC3W1c?1wWxM5Ly zm%2eDgF*&to76!3CC9C53}4lxPt&nEub$=Q1JPCA4wK;#0e%U zJ%~Td=-xeXa00`8=q6=h0{k4&*KSQPc=~MYg$R$LPiv1t9eDhc&^b9{<&jmahwy&G z*XsWJmQ`1^nn$&8;uCrL8F`p&o&7J)@WSWaiV!=R*O|+-dRPxwYZgD3M7cKee9b;; zn0!pOq3J)_ku5KmofD7bS8~qW`Iq4Zv{kVJg>=gsqo_1hyv>)BBkIV_BcXybV6&BK12H)-rbP#pnOuwcf zb8^u{I=-wyy64V!69hfY36E@qA{1z_l(fc6;RzcUNg4UX{D@2 z$e1a%dMX@n+)p?W%@O1L!Zo`5sMRn{^LDA=qkHQX;UFXj) z`}9?*ntlbRQfKiCKe`hw5-vfgyOS^eO1`<~A3P#^!K@?Y=>!p`7>Z#R!EkJ&Q}?5f z&XZ~MyA1Nciquiwc9Q>L=ljNVb){~y<)f=7>%fYby(1g1z{BoBiSy#M-0f-zK!=O+ zWm|zu8XOIH$85zHyyb#$7BhAO8s0Xn7s^h>#&l)Js zEY)NaFg{aIRE#Rl+Z%N&kMDk09M1IGUyxQDFBI|SdO4N3^~46PSD z{*=@0kWtQHwX#)TLBVjqn0|HZ^{v(^qjDNVt<4@iPryn1TAh@9-bwYW2En`Np8Qxi zjN(m8bg59Ew)|BQQyTHe81dRK92us;>KYVUpJtmlpuc;L>E>Zf+Z?#Wo!b-e+Gby3 zTR2jUb$m+-yt2&0S0}5@jYlAilA-#d`$6*o)$3E7U6A<+{OvvJww5O=PNkD=3ja@_67<9*q)bB<2U ziu4H;9o`}c9|x5U19c%L`54LRgS;!v?hLaWd;Sim05+FLk)#p z*-=z!f!;_oIgxyQ{o5*7YbzUgcw2PLV!Un?T7-tD?5t^AsLzJw^T%c~=mpC<{n-mJ z4;G6f{6KcS3JCUJnXm1w{NB{79IB1{$|kS+a9vH4h0~ATsTG*sRP6lts5k7b`|n?5 zSr%8G+u#uKb)n1&rCWvkfh2$bkDXeDQmaO5{s>`L%%#x%z7|m!f7C{hc>0zab6R!p z`8yZha3Xu1l42$vd~8Lpi~5Z0U|!*^Vg-{SxR)$!cs zD{$$wgztIOQOvmFaHsf_ZftBs>9_YaV>201OhEFNqOT!2xNlWG$v#3$p4@{w-0j8j zM3#tk&kp3abF+1ZeE_GeH;STwHWhg9xrr9+#^e&^s`yL=pLbG&!;Sb6$S z+Cq~q8{9=6>3-V?A^!QK;*j)Zu|R8iH86=`*QIu4NC|MdW8QWkwFpAE4U;zi2-!3I zP0F<91Jx^3G8roMGVjrdCQmrlEpAz$wG?jT`QS=hp}w9*N*FSb%A6*tIKTB$omy95 zBV?Qmv5AQIF!9Wa^A?}iqP=C8k^uThf%Ygx zV~dF+>8b0_)zO-q72U&Qb~BB~7< z7z&yVZEFmC$DtXuJC$X1zLVA&9y~ZaEr8htr{2M1)Ss(0TdNPX9}lj}+Fj_nMR3JC zW`B9}>KV9&xTGpK^G23jzj-t=+YY;Wd^O>ANR#bs$b7R6FQz_EtLmX?|M5^xHN7fo zYT2P?LSJK(u+=-xR-8!6_@2$6CoAdEyTQ~Fw-FK|DPiBbopTRPwHj3`PPNqQX@7T( zSv$*r?vkGH`#E3VFUa!XYx%e?FE3-Xll;r6#~z2+cvz3zEDeo_xHeq=e_gF={;m< zcSs`UWA^s!g!wB#TVun9~Ak@-?z#0XfcA;p4(K9fg5=Y z=?^ONZoy^WXx*8YbFyaTdox79Uh-IVV)9t;LIqaNM}iS`<`Smfggj+Wp-jCiZCdr~ zxGS()MsTB0%3I`S*91ktLY`5B=FF{Db(L1zT8~xAP+_~9e8wEVzkrgC=OW1S&$bL_ z4YPnFUv%$8N<+Kb2DHgG~$!&a{Mpr`4G#>{6jafPI7bz=xyYb6QmW6S1;+fxPT$*Gu9_d zTiOio+K8Nukyb@K+dKWV)VMYh7M*>+S%Ae=d6gbO6uZ(K{SZ>8lo3y%bv)zxOpo?qKTj2tug9-P)!(U8+PNoaQ& zPXC!zgX1}XmkH)XZGk7IN%8q%n9R9*3Dz^;r+{1Pr8hnL$beb{3kH}q{#U-bUz5f4 zKx}+&8GV064Jro^L@D36$ffFQ#^0q2LPdbN4!LLiuzFeh3eZMBMHa^TMjS@7JwS-) z?WN{KzCi|n=bj~|j~pc1RZ@sU9>U%5=b-B29qyUDlv2Q~S+@x@00-o?Yg zp8&tkwI%99R8f4HNv<&7+`sV?PhEuoN73w*#KLr3Qt!&uh$1@w8X+itKG)*^p-J9m z=WUG&D$n@rLJrVTm_TFK&E926|5iJJFF+*12Mn1zY2Egz26t_;&Qk(XE6R;eX8Ih$ z)Bf?+=(1r^XQ(L~+?>|nCqO9^IhIA;&H>LTpzM=H-fp-z{mI!$MkA+hsS9vorJk#u zc6saGsd=FQ$1_9bC`A-|d5Z+HDvCK=Y_lK(PL6mbes$*oxwA>Yj&8O(-$2FVpMM}` z=?~mUP$lWLGi*STvU!M-K7WMCekn~f3r;L`GP(OM@^6c=heNfN<1^EZj#4uYtKsRilRgS zRBb~d#~zk%O5;Qmv>S^{q?g{sh41=EddYyjQ}-Xn#4djoDT;1`2_V{zfb~2_Fc6L6 z^Hn*{fP(UFSfL`K+?OmXDZUAZCu!(9Q?`I!8NTMrj|?=*jz1dLZL%mbCbd1aF_=yQRCSp4~5;UZ^9M?QRU| zE8{|<`we+O*t(ZsEFFsQR$lZQVDSOAUhiCT#Sf6VyCAn7c%xm=JwZY^WC!m`pe`9c zs8rvmGnLTPxEAnl7v;Kao}?iM-W0IwNAvC)^$Wu{B2nN#|DV-iT!`43tngGe@}1d} z%JdB8)mm2Kbu>ay7^V@}C(YaAPra(cI7l#*9vlnWZq!CW5cWyeL=hk8Y(+UP>5=W+ zpAAX7SB~qM7yyB7Qla3yDR9_JmE5TF zA4CDdc*lO9KbF}A(~@tayPxht*B>noG?!PV(vaJ7Sv~N11R!PQE0^F5&+ZAnS3Q7F zZ?qoM8^gWKBo-&zy|8dH;(svu0Fu)H(DB~}Je0>o1_wN5_I^FL8tS8OtvktjnT11Z zAsBPxZ1H1vh{+31Qm=aYItkkJV`Y;a4~{4BV1BQwSCU2O;VCo+j#n|BOyuOXzUNAxYrcwI#gS>u;klkZxIaVu>#8=ilBP|LYwr7m3` z_Ml2-%VUnI^Dh%fy9Tfc%?qg1u*>xOOoDm;qUCmg18$No}3B=Umi?^duCGu@%`h;oPObtdxR_;9~q8|fr%Nk!8LwXJV%n{*U{|bv%O#*A`sk8vtmsJtORs7YV2oL+ zZbLZ!(Ls8rjs=(9{M9y}q|w4xl98)ZRw2X^rHRZAqW!NS+-EXoF~XVTBHkBQJv!gk zLmX)@Y6G7vlP?O;pqHf04?L&c&M5nZQu6eMN&cztX&&x!c-`fYQtfq_DFhWZDbx#e zsj1AsVXT3EOKe6~c_h8KdksZop{K_y-(i1BDbWMYDeT7%1ntkCz-csbuW2#@0D)OU z$5cI9-vD}~PxlEyLong|b?c4iGG9H?UVn*sN~F8%AVEm>au&t?U8SUy)ylKAIsG>g zjm5}Qfj42=3)ptaW<&=M5h(F*j@nszMSr#3onL-D0VUkP;qyS6vI%$}*QXS9?lb_H zqD_2K#tD%g4^3iU^1R}wz3VGImQ%mj;Nj|V-DC#L5LcUFG+LM1piqE zJBJ%9+0T&GQyJf&6;>OTE3qNNBzkZ_3+MF?xYi&`Z&)(}@rv)9m}$w=|IYLy4y(t3 zp%9Hq#?7op9D3K?0;MCrQf1q3-2`74qFXtsL(Y6Cj!A0;7L7-PlV42PGgs$o)c4b- zIEnjX>kr0oEbzxv5XHrG;o@h;rk&Es+n`zy>L@XO$T1ZNGjL7xI1uCl#bY z)RJO$h^M|7=BdoOgs3(Artm{e2E`;^t6Q06UOZmt+Pac&Mo2_#`QRmZvticf(mW0oM%m!j zqoQkDQ1kF{v98x*h~Xe*fTK=a(7mwr*dd@G;R{+`=|P}G~}hN$~Y z@2zb6|JF9ze`|YUy`Jc`EU@+N!i`Pw@VA2;JCI$BkmgML1qS$akj;ALqy)EM)m&Du z_2@}#;6R8D4A1#R=SNS`A)m+<&w%`TxYx^^_HHwy%$r|Tca=t$@!IF8{B?yJ$3K9j zDoAPTo%f+wsva6IPq}e9XP9+3y}syw0Mok0{F@}N7moUTpx10MU?+nFOu8=AmD}a@ zjH=An!3Gy%D{-o;2({}KF1hO|$CFd^LjPy8@;96c4NQ2yeZwng5*H9J<{CLl!ZmoF z)Wj2-c#RVN4={uOt9tam=Z4k6Xp67OH&w4)>&aZaD@OA4PUGWvXkN*-`Ca^Z*xp5;Hm0*R+rO z;+T{QGakGLGLvr^>+e^W&U+sB17*6m*HVfOQ@<5x(O!xDTS)wWq-6O2^Os_V>Z?#0 z4(KsIdp)qEtCWh2V~!|#|9&_+1htxbVxULUWnP&pf)dnKVv%l*nX!o|6n}z6Reipo zV+~JIUk@8(-kS}uDxq;;9$HBX#d=Nq2fP*2!Q6O~D;BeVHh6*0Y*m^I*-3kIx>2)! z%?UI_xI79=IP?Mat`afRM#n|uSydVic=s@OSA_ILgD%|z&s{;YYF}@x@I|4Qn?SEq zamt}F9<~;~(W%||HTYry0`fDFWL0*+`u6l{BWx6qzjP$SEEV^?H$;qOt?^iO`3)J~ zQ#5}QS?(1ddnVK->R5$&tC68ZQ&{cmhI@1FWjYee`nEV+xwGTXY|L0Wj;pT5_A)QK z7`HL&1?fXE?rgCkXQKx7J=Uj~NXXuKaW4BMu3D}p61|q2=3c;HXD-3rEhv*3hmx7f zb=J7viLt^P0LuDGERs zH55B85Ok*m1X04^UB|$LE}c{ARSU$qU$UZ`Z8CCw9L_+FwmxFs+SY9l{f68d&b2aT z0-}K`nndPUCyy=u7Y3Tk>Rw%s#JL?`GB=Lfaos8wUzFpz#WP6^i{hB(;wd||9wT+U z3rNfFS2n3i$~O)r9+<3Aa@+x=*I+n^329j!Q>xy1cA{Kq(Fo>Q%gs#-YA|GJmz|JB zTDm&pmx4Nx*Xf7GwHb?uleE;hE+!Yyt}faBnl$N#va*_ISEax7v#Pq*ldwrzI?zFynnXC68a#?Ei^f^ z=N54Su$I_8(2kV_}w+IW`$369s3u)=2RT|XjFZkwb zu0ZU}EV>9Z{%5b-WFQaZkeuW>@BKyqb4mcjY9!*40q7#Q6djT~%1dPl$LP=+xjUB{ zGlOlF`x(H?D<_GdFhwQUo5?0HzQjTrZ3Nv0mWkt%mZ zjN7a*XjlMbR6wo=*KbUY12;2tLByn#@gf?iKOerjiEo3ZCLI={>r~T+cEYk2wZ8Gh z*k3H%kGIe}RA|BjQmnzxf*X#veQI3KPmqocQd2T-CM~*-nn9ABEe=;jPkY0RmA{IJ zvaQ#ZJ7$jS?QdDaf%Ct9mia~EVWx@~+rXRVyVPJBB~i&=9R$}B9|&N#9^-8*Dt6yL z8XCqgyyp9R;g@ZCT*O^`yaD^2Kl;M9%1oOJ6SJxdaC?q>Ei@=w<$%vT#RWQVLz&)|5_ZyHc8cBlv* z+;;-AQGc-CHdCE;Gvg3m$b=6nhj(SPtW3rjV-sQ`3!L9t6$$gKg{e$n$NQKMFuNF? z50J|tnK#2CSCj3+yrkoz)wc`YdBy zN2(Z5IPnRx%e3+N2!G2B0ARG>AbreC>86-K#>GLL+FeIFYf&NnEX#~1VF#41Ea4qM zmmD#?UG=fGyY7Ib^w7O$dZ2?NIa_EQEoTS0BDy6Wbz4Z-+01bR6)ciQ1;sE3KcvtG zW%96_2y}C@sL3LkH4fYRJ_4B+urD0-;MLT>-1oj z?5G07Oi;ISTJ6y-i@~VdBq_?tIREjZiMQ4ekYBg0R2gcKLyW9Sd=%ZEHrgDs@mS)D zJV}yR>#+gHe$+RF&&S=iz`aHB0Jc&QO-W8f^j4OgR~3&~7VU1Q^h z;971(^-+@%1$p{u4)lb`z7!AWdxgZMeBIv!^2Fla44VM~ zbrXmtWgOrj7&dqD#5a#DYw=;|{^Km5`OS(!RDWJV(`S{ViaWtk3dc8d+~CI3pMidC zTp8FLH0f~%#QB!sxKL5xz_vIxYb@d?&hTP~WK4I=n#!u3bcuQAkQs=-pJ8l)Wf^A_YNVgcjf4;$>IN6^2QVMAa&poL#$pN4mTk72YMA;3Sa-PT9L{A#CinqBW_acakZ@2kdE}!`-*u^91!sg-eeISZlZVV#I zE45>t3jdse9J+nTs%a8EW7eapv^m!R7%Ci8MslJ!KwYmslgPOSYu!CT48{v8I{hFl zKjPxM&V}lO@My`KJu#{TeOTI%L1miYwqMW;9qNGZSu*t4Dylz$j>pxD`Og=F9KieV zeEWl-d^Yn?8&BWp@jC;nVMf@iVf4BK&BcJD3EK&Ktt0I$McOWy*>G<6VNKPZ?Sc@ZBAqo}eo}b`LEdxBC)cv5S-SORPe#3!_#_sj! zt4R|rv>xp2^!0;uw@tV|%=^1;=X+~hW9`v6t?|5VQdzOPu3B`O zNA*|`H5sP9;6;9pwYRDMnDRITVUVj?;_Zw$1;G0RQ__-6$R;e_UAey&7eX0X_W3)h zS5>}ZI1z0ND3{w0Xf-ALx&$c1D*sLnYA1JD=?`S$@`O*o->l=v&j^%5Tg1t?ru7aY z-MJAq1Ho|1*k9v%x8hEGtRPz=c}RQXon;~wqjRFrOvd73eMrX+(yWYsx`_Gx+c)WM1sYH##JI-xZG}|DIgi?YGKvHTI@9}3! z=JR4ON}s6^HsYDnf!7*PPEDTx?0}CZg5j47-s?RQf?2#CdZkMwvD^wYi3YxWE=E|) zmQ0@J$L&B)4}UOV)3YwLUUx@;)UV^|qZ8K2g0oZ&HLqIP6GVTDCZ1d)Fuz~;d}r@7 zqk!7WSG_!(YgrWU$E!sQA7GN0(aBV`bERnDr3>9o>!~Hvog7aeE#p&_-_a)WA6h&X zPU~_v4~ZkyXDS|(bdY@JtCZ8~)xd9b<>r0bp|>O_{jr)udOA|n+mUKS*~{JBoHgqS zaR#5ToX==dOg>XA2IhNP1?Sha?5*8cd?#`KM_Br^^Nc*M&yCk%LPCsOUUY^eIg@nj zQw2N}x!NYwtc^$IZV$_dh+|o*SRpd<;fIl+$_vdg@|7ZSt-vFCE?6doFW;qFkZ-gj zS0>%D0$$cT3(gPUHTd6GIBRiF8i2MA&In*o=Y-ft*V{I}&Cl@ZY^@;3PxZu2l3F|OV@uS0x?Uc?N6`-5x?I)To`9(4oF5}t9e}=ef^u%8yhKj`Zx7$~M)=p(F1>-UNGkWpF-*d6Kit_Z8=^Q|#lU3-( z_}#zS=w;?VeOg5FzbqyN*_YL7@N>Fa$D}ooU^HP47%SGJ=oVH`M6mQb30);4q4({GQujNq5;aw(m}>`ne}WLF174nHXKGdrHu4Hbe2Q&ui5s?)cGk%HMflWo1f9@;@4;e0PJ`#| zqGlJ#2gVE;_p{VQ8z}406*Xq7*)riW$`_DO`b=kI?}*B{iYD15`|LArzs{5)MOPpo z`re!Sn&(6_4M+ld>7lnCBWC>-|3&J@##&dK(k5HoiHBi5JL=clGnzA!|Eg6E^jEFV zo^nL~6$&HH9t_5BvhP>iC|Piugjk$Fp}wddsnO!K?vNgf_?5aZ>9v!6CFPshn&;2S z|23`y`O|b)Ey5Z1i>aTe|EMu8_kW;9EWUbvkKHdWwD&VLr0nTME^GyULT}Ed*>D z;9X~6nuUBSDa$zNK81fRRFVYYzLkf3otN&qk71v6e;K~N90C|^aQ;I|6`&3)bxiRC zqo(Iyc~4=)>5A`mUHyBd(Z5{Gq^h3zSkbn>i&a~u`*DHzDYVmD`lD-2p5|GY3huW3FM48s$(rVzJbCqz324*Z%bERY`P!gFh1$L%U1Fn> zdOFDD>Y2ER#?Iv?(RS3Lx_eg{|42zyKiT$(l5zOHOU|w7{@-=n=X21zW|aal5OJ%v z^b)ikdsTZq!I_8Dr=i=Xsp%oJDOdx}C>BKYD^{tl9c;c|!v5Vmp4oI|KUc@Tejn5r zLp}Ss3zSm)_Z4M-xp*SeT~af0zIJ1Bb(s1+{aE7C!~MK{G=IWwR&IK=DBq3Rw#m-L zKEqKjK_8;)-t#u{2a~#F7eS^u%-aQ_q}gOo>S{w>6d;d|p5UBl`)|#CWmuG3)V76y zh@dDbt)ev2tstEeB8?&-&Crbkf=EkBcS%SOHAu)PASewpW9O~;Bl8Yzv($1K1=ez3-O4^KyQk$NVk(^r z)>X@0n(mVasnR>ON)mT^g1bMJ&@|1 zNf`Ui=YFK2n#&*>R*sPb?Df#0_0+-9es=xroR9ZWPNMPF8#r&=RkE4SB;O-PbeRTl zu))$6{08=9F}df+&u8)RwyF5hE0u+g4O&m*vIY5;e1&v0L`eK0hm zOOdMbb)JZ}j&U+!uQuPI4~&r`+XS)@CIctLh%R?Q_HR@lU1S2^k_mMdUV-=kgCW(i zx>emvgNyE*qTdbtJZPO^WBnsgoYwZ)+kA^XTnxXCq`62fxwiy~N%XGGSC<^7krZGr z?-cP`*r%&nG8x6O)U5@vJo%9eL=Gl9;(FZ2EXpf;C`z0L^_tkqzK9SuFa`eDZraQX z;3d#Rq|GN2EyW}8NkoadWM?H)e?~Y>>|Nm7?y(&~@j; zJl*3Rpb`tTkHDM<$M-aZsJzF^ZF5o$kCp~khF!d`dXc%Kg2ijnTLS8s4E6+s2A2Oe zCGnlPNk^U|7sGmDYDw|KiFC9;@a8fX`b#o07b>am-Sl;ZEY%eGOPq)7F3GvH-WASW z42#FvTkcK?|0~7=E7b=Y_Bv|mRi#g~sa!7h$bMu()POeF-T6%GvpAdNG3Hnl{26v^ z*2)NAZGhhRv>h^oPHMa9p!uV4f%zq#=+4=@{uH+*9ac1}YpL5%ix2kxO-7nDU1tl52=qf2Xid9bqf2qhkYO8_Z;zu+$niW0jRW+PLZrQdHzSF25#%7Xmu z-;D8}F9JFkz|^-<#!X*ZRX2s@M7P>8y&~GdD_r)Uy4PHM4xx@9F&O>$0owBZ*v}~Q zN$oxq2U$@9kR{uS8rUPuK?eHx%l|ZC0p?VD|KQkrFHA^qK~gk~)t}(D!uc*oKHKV`+RD)H$x2R*@kf;5{;+9v@}7Fo z`Rm_u|FQ3lY@}tv@y!eq4pWmSn}_)g#A-%uJaTm(1tODNHmz+AK34@nIu@HPQH~01 z)vpu1ifek6C{tVyqHe12W3kNt?LFU8=_QGm%8b9`!2d!ffAYFNL5s|I;2ML&m}pzm z*L_TcRD5W0xsyO1-bYi+&Yi_!mebU2y#$xD-1cwb&uTSNfS&yPuHy^UkAt z6E_!?&5kuRv5FphS=_J(;%`wbzc)Y1{M~gJE4BVg9)I)=6YOC`GKI1$E2Aon zvtrxpC7MuI8C+ue?~(QcYCbVvGl^BH=BS#7 zE!79Yknd2^^PzJU!NpEg)-_(hhz3TC)T@VjI;0LtQsxko!JLxT_5BxCq6vz_dgc?$ zyTr_PA3yN0di?Ug*E^f_3P^8kXP6e=Zdr&hO?6P*6$_s?OB!aYoxZ&?ntH?~3K_eh zTO)`};rHr>YV;~5rYH!7b>w|juX#bL+s;kbwmAMb{q&hoAl zh*eJxf^&(}dmLB_YL5||NN)H`yRh68=UTKEXZ=tQO-VMJ%hfoDWLr+=6)oKkYNz&^ z`@7t>M>nm@2gUcC9Dxa-s{YGivsov>ih>r7LSoHXsJ~}}!3D-^aNu=?Zc;NzPwk$@9ZGSLS17u`9FJ$IC*I2BWefzCZCyYp> z_bl8a#Z8pC_s$ikN8jdyjY?**&eu^+7_m21)#B3P>C4^V=~~Cwu3Qi3%nZtBzN=w9 z7QK&wY^!&#?WOv_MAz2?zg9Nby#$GE39QOix7AWfK`jBNnAwtPjEj}TUQX1<^yj^$ z7Hg-+Ifu7#PD(bMn%ZYQRAn@#T6Fq?=*oG1tL@AApxxQga}DE# z7{wY>LHLcvK=1+~^*|09VK&4+mO~AAYd*hE{H!slO}Vv;vRM!3pFLvskm-^$Y~TNzN0@oYVORby|Mhwx;5p*7|+-9H4n|DYfWF8eHx!-mwXRQ!V}kzJxx!`(Ro z{z+UzDH92S&ghQr-HVTbHZQ33&N=|q1Sj&M;PGxql(x%ts1zRk!k^jaw+t%X50c*Z zeHsGAA{5INcUJunS|JAVO#e4nz-q4-@*U-?01Fu=uvX~!;QHlqmPdKy>CgE^x08e? zlRDILpP_tKM!S0tIqlo;&!5`$3m4@jq(&nr!0!fb)sIa_5wTrn2X{&9W|a$gOWW0r zZYdAy?5LJ=b2QK=-;(g>9N7SL;BjrENwh5(fQRlqGhl!RQyqz?D6&MhJ0u9+ji^1w_L($^fXE zUr+kihfe|2HJ8n;8L!*j3~&xveyG>saRywppyGe{vtddx{UDN+~NLO$|?#v zy1o*?q8!rmRY@y(iKY%S^$SmuLI-jf+RIRvSwYkCw8jYhbg#Q1Qii+HuI_fW89`KXQp%2D+}tsZKAytu#@C_ONxy3dmh+?d)pgfN6UUXfK5z=MC*S=W*1A+~ z#V?Nbj1KE<61sF4-qd~m>?6(n;Em?ju0q^Lep%eT&O>Wv549JAa2K04e;2S#>0h}J^{#4l|!b;1AD#4KZPY}CBbHD=l9Ah zJsV-sh#rbxrJNL-f!#zK{mQdB)W1qPbf#LS@!pg!qbAVYRF@Iw)71l%9a1}mc6y{b zMk|iiA^jNy-7L^-zOOZv(}Tdy-NCol6Y~40`udofV-am{(bTA<*65*K^q2)!H~Jyr ze35iXN$o1zpGm}@Mv=)kAN@-Z^uHP z^Nm?RYjykNLxcY$&QZ+?A^ZH`=w7Ae!=1qDf*zmsuPx@_q0d@zW?E&}{_tEK_ZOP* zp-sLOw?%=+lk1lmrO``U*OadQQF}YBzu{x1?DBpLCO!W&R5Q-GgQHwFdgT}Y{W;!d z&@g5dQP6{hfAf1HVIn(8=p!qsy#eHSE!B$0h|+eG){IEwc5oeH4PVpzWp!@(w%|Sl zId&kTWUQ+V2}V`S%$@x{iWFA1in-UKwrMiu-MrPhE(o^KgYJeaGcmhNzbG>i-;^0t zwL$H5p#Yo4Lo|AOuVf5h{(yVlH_TC)xd_|`060znfOY}a(!9roP5FHwx6NMLo+`wL z=e>&j4On>#9!z^E;ziU}zris*0DHp1-_O%qOHkavDC)8t%IAXE60Shj*;%-LqO(U`U+Br7$|2Nc(4>jqvc3$Dq zD5+mgM{}n??hJE<&ZHSD%WX97ZHU)~R>8%bRQEgak)h;*Jq=qehBY0wUSB@*bXbW( zUDs%BA0UqsK<10Kbj(2VRqbk67%1Q|?Kn$B@ftE0S^aT*r}YeGnkuyxNTXt2cdxm* zGMz&vd+jK#d?50U&H37e4(0bo!%|Sqw;*eWW<3?i@jbcS9CzISn~Olo`%fC$45Hw5 zFG%^4CfL^5yTRue<^lcCcyN`grJ1Uu#fhsWmpl?ek`Y z9Ie`B7(Ok{E*R7C={?<@<2H`P@>p_c&>fqudk+vp|AWD)#g<8YkN|E*tz;3i#%Aak z-a&=XKGm^dtQay%3PUZ>_-x|NcM@izw>*9spui>AO08D7Fy zwNk*mzl(7H!lQsDWrC^}?PyHncDe9MXK-&Nho|@S3{lVh%?2XDHtVuY<*zPP@*xYb z3g=hu6~|)z6TpvBc1V|oDHAJ|02Q<_*BYc3aQ=-*vNnQy(!i6S+is8A+p}PFOO!{+ z1Ast1#AT0Ka%GQH^61JvT*Nmm97k;~J3SeZ1@!)~e?zsv5p#QVJ~DgKNNaRWwszoA z;3!6+rBg!YIWd>1?nA^xuiex5H~51himV{(^N+9IbZ$zp383h8qTn+WmJ%H~ma5C5 zuP~FIgPKf7t8aOFFU6p8eaN~(UG(brb;zm)#Q<2^LwI#o_vzO;;X2K}#Q!!{5SeO< z^ViDzwPea?8ByLBtH>5fhqTWJnk*CYE3`B>BMQA9IvgnV*wLptqquFe#(B9Uq2xQ_ zQ?2Fjmyzl@ULjw-4r3Mwkj(>TKsCDjW}h3mkn}d$_-E-0o~Fwr7YGg}^VGXSA?3Q7 z=D2`%`t)DmCgGEduUq*bYb5%`8Oj{8Y!$my_0T$r}W{`YbulzmY?aG ze$Z5B`ao;u!bOTM-_V;o0UQn}ciN&HW%w(u{a!A*emlBW3cvyNx^AIPy%^|_@$yel zdMV6*{0Y5+O8r(T%rs8)ZO=Z#e#`?y;pF=;#CyifxQGe+W#NAjYkrC-VUJfQ?gH0`^K;8BL`jEJ?xbd&0PdkYaCr^#2AT+5 z-T$BEj(;l*{Yiegr2vR&ZneoEur=3|*G&?{OeAZx&rFn|^!!gP?EfHWmZfM`H6V|#Cj^@EG}#(>$^QeekCIu8GTD;E55>B{^OWvf#Pxu{v!h0nBG zS-|m}W63ov9M`}#mnk%ka18yvT$7v6DtE7SF*S?hde&8>z8CCTb&HnpJ^gQ0o-!VQ zQA)w6GxfKGK4Qh#`|gZWCh56JKAO>EMZaLHKU)PLktxp4bxl5R!q?x1EQ9UGuSNL<4gALzG;0>{sf1YOtZ-t?`yUua*byg_yey$L; zHSjqoAJ1AYRxK^dBlxbd+v&yXx;gP5gm7zpeyt6iY2Y}VX0KT@z*d23VYCMZDncB2 z^~;fQ1?!`BuVRi?W24$gIiHsuZp zM2!8G`7;j+&fOZZhk|U@KLzF-?eYk+_h+zMF&@j9&7hivt-w)j$^Vv}&ry?AEa)uO&1f04P`jc(iH3FPP+2X+>N!hQk559i<^jocNf7mt zq+1^5ijoXpmWx8Du`M)L+-CtQOV_WG4csUVGoSnNNc4j29C)d=n|{79L>!mtOdzI< zseSSpG}TE?$k+kUNOI=|xU*O^3U&ShJ0&o5w|M=G_u4#S1t>4kPRf;pTb99#!cRZp7NT<-^ z^l5k}H*lZ`@MGcQUW3mhUDMmi{7!zjME1Q(!kQKlOOOuJ6`!MR=V<=F==N*iaAYWp1ci}#X1^&}}wQ;^-k$LzTy0!R3?3*w*~?b%&RW&37~$(1%j z1-pVHtrd|#)$%I9J*{;tf^gsu5{qVXQY@nPpw+hbpheSQji1F-p-r$CT#;LOQOpDN z9K8Xx9bL>xwXtw%3)+)$+i3kR$7i!zi`%5~WI=PvyhK*?=mp!&YD~HH2`J-^-4$})uifxK;U{A#7kjUSFW^2V% z0jm^t3RQ*bLYZU}!?xltzUE}`j zUpfuca$kM*Jfz8gbii^;m{VbOc)&8Y5aBzN_k5!PI5tMdZIe|lNlvf<(inQN^od{0 zi81%;413+oa~{+M1V+isdPsliD?V*@IVZLN;sKC}3?OzeTLa|=Y9*isA9Z2GOt^YM zULofQ%4uLT|KgZ17yNL2;3z3&M-6qAHivxo&l2kQu7NY)Hva1e4OwqrpEc8qmz$Kz zd_>6+1~(4}uF&O8Z5tTG7O{mAs{m0e8%7u9-7FZFeIc|PIxwue%riEMo^Y5qOvn1TJaAb_vyB?-9)(4Os5J*>UTMu7=CF z=Y!g@8EkJz*vvmwlv76>^H55%e5VX0MJk{q20iZ2@URtxg#z~o*E+MH+>=2G>z&^#5TqBEc^v1aSK0}}gTng%**0;euw^Q!{ckC{M7Ls0 zhh>+_Eym9DaK9t2SA%$w@KosaH9d(o%WbF)8=FtYrpvHw-H z`dgMBB%BguWgV%8yg|WSqg65+RS*-0IbG`K6|vTJ}3nx23W!eXs4iMR-@0Jd$dnlaP7CACf0+! zZd9##4o$KUdzCl;sNLpGpA>9s)KeP2vqXnNw+3r?f}XBM&^IcM$UG5H__fUkcpXkncb=pgm+!t9G7k6EZw;WNivQ%+l{B2}HSV!8R&M8vs@dE3 zX+%OBxXo5NMUmQG^8Hi-hNI^Gw=Ny3&Y7b9|M6-F)2wL&&x4Vs}Ve!=L)P-Bz){oWZifO1)F)i zy2;f4!I7l!P*pR1K7d%_T-dqoo6N!A-yrhhaQcL-5@#p4u}e(-0rBnBriC%B$k>Q0 zK)cuo?v>U18l+8Uzu!nPiEEffzcBVx`z{`RmY3>DS}Bu@ zBG;5+d@T}lzocG0k&ka0pHswv;VWTy0`cu{KrwjHvjELz6-3MvnR%?ffm!)#s@5Bq zccV_zy};FVnN7%P#d*svlBKdf^#I|yrJltmMNRJ?;e6-!xtUIsU$<_8=G>A?xwe3P z=IP&`$$L7L?Gshu%E8lwOL4Y@S{PuMqxXLwCdIauobKI{ixRR|(oc!KYG-hjO<>09 z=Tk`+7d)ZHy*m85%GZF=Kf)p%0`U$r*EyG)*QsOo+(rOA!k{nYa)C$_S|{p@bVcn z-pkzAK0_p@o}D%_dkf8kd&lCe7;uK`i={uYnpEDi@QgA%n%6npvfy z?AAV6)otG~B1LNO(R*_eQD9&(<)zhnaxJoD>s@XLIpty&;=;Sd^OQz)PKy?~qI?f3 z^PMKWWw9pAG4DC8HO#Ir1Ve<6b&uWNvZ~u8r5d`j$7a@;EU2xYc;oLTzTNopFnEO6 z{D~#+rKTsBJzX^P`yB2ITPG?l?F2hkt5=Ub>{n14w|w_ObdObDuX()&O8ZoXt?Ffi zaphrwQW49RD!#3p$=V3vfz~_zqMqut10v{QSIS8K7^G~Yq_5Jvz)|l?!QI29Naqjj zrzGr;oo`3#l7{xH4EuiBopPCxD=Y+fpYXA+G)orC!d^6x{Ib+j~u|GQ)aIo`w2i^FajG39aRbMxBC29R`b**BjzWVN&>tv8Q)` zcT7W>Fa4P+VW(mAbyg;raZ9_xe(%x@eWT~a&I4%tHQqp?(S~7LB9Fc7wd+L4aG^c& zz4ysSmPpUTiiKQ8JdC6(Ghu&~n%Bl!*7govP^6RjaTGKqzxBgb*lp)@#=5?Va_wA@ zicbA%VpEsk^2hCj#w5?D<+T`th7vB2+s2M?pP9495AVQdVKIb=mugeIZiRTc=Uet- zJ@B|?Ru**sq+!wGrkQQElatdLb5LQ+xz6srt*WCGQKkO7ih31x;>RJbm&Yi^d+n9h z!zyU8?l9pXRbH_Y2{!5-x8#11!ttyuCJ8{JtQ9+-ltC*n+Awt_eQwXeC}(pvD);c zcB{3GR@cuxfp07@x9j>E^rE@@mw7z{ue3iERHuROiG5mhM~pn#Bm|NU)o(MsiML3_ zzoq8R3%2ntul#y&*H&7G3A{2Zt92HId{M$c82RF41H-5_TwpAeN7U^g&HNms;6*uO z!wmwGl~BjGtV!Xl)^IOv$p;m81WOt&>G3z__gPlot7UK%=jaR`j6iD;?2&toOL&bq zn#-M)*}df!>Q8AsU{!F#KY}@@xY#D_>(OLp-mCr^b*3Znrn0`l!Oy(cN08aOyY6VM zd!C#-1lp0@V)1D1WdfO9lq}IbcNR5$wXp|51g5c(d7>odF=Pml4NUFrl_%}+EO=e^* zWrTWoKI0iLcC$xHlk=C42;RaNDSUNq>ql9@t1gD0zaiSF-0pD$WisVU=kQjR>E1{O zm`(jsX9n#R^LWtRDw(rUehfu%^THdix~rdU5EZ0gU_fOv``L16b|BV(;8qNH!oc_~Y%THv z*UFAtJa8A#xzOMC#5sxUP1D+uInukgd!TJN0(dC~;b{9T%h1HT-poA=a)~sNDIsWO zHSy2jBEdteyE>e1ceCPrF^{IoRI_9lyvWT%K&jz(&fP6Wsv%A_HcBNJ!Q;KsE}xCU zdj|=skXoln-SqjisHSI<(DvfQ4m9lo-N!c3Rk{=VV_+llU9BvyD4}b1+bfUm)LN*` zqHq^Dw`3SYL(y_sZ=-}Fu7mCphm^uE&pRX9d?&{IJht9|#*YHB4!hF4eGm|oD?*Iw z;BlebHTG?T^g+9ByRa$3%!Ys%64iD~FyvB619?3W37IP)29h+$&K={iY}qSRLN5pW zj$p=RcfwDz`&*Q4s^{K-jXGeChzGCiP_+CumP5*NqSdc1{?S^#L^m$7GT7nIn3+d{ zJGpEgkU0i9>ShJyn9ov`7voiR$&RI(>8;NBDRl0S(ayI`HxS*(B_A9#Zo?@U;-CbpiP5xf$vb3BNC~|rU~&DE0?^`CerljXy-DSOUt{GA$t!hDtubPLJr$uZA+O_xN;t%N=(MXTgR5JCMa z&fW`F<&5he)H7rGuhsI%JE>j)mBzFysC)_hX=15i$QnIWib=HzH)`+bP>(zh=^{b z9iw$S#3_q2MQqpEp;Y`NDLN7M9g}C)MG1hsav&wtE$=uImX? zG*>z6BK=a43Jl`sa0XAn%ZTg-`@^QYX{5ee+gClqxujE<@$Pa4vE2`s*{w78RzB54 zYTDNe{s=CzNz<6R1Z+UpQYRU2+o!g@_MG!1!4%Tfn5Ju|JPUbZ7~ss#HK20Tx&vKf zGA_nF2V^Y${#Df+E&e%Q8#q6{@7IAF>@)QNsD8>4`n1?h)$0a(#_M8O6HufD8u)41 zc~PbhP85ByTpf#_XgFgac?7(2-Glq6{wxP#^L!@E@U=Nw6d6En0b77{-gKoIH9`!Rh5g~J2o`p{*ZU#-zH*(c9G37`FIB=$TySDM(Y8fN1R1wz3#HNx z?ow!vd`L&$5&}j(>AkGTKrB7Mv}VR$OTj<;*{?P<>NY%l^Yx%O(Zb0~_;3$Z^C{n$ z7ptAb@4*LN(DQ;-FDA!#v(9sG&Cwnlw1ZKvtc__>6{lmsPc}z%rENFf_F1mxV|3!d zMjvLHrz@czge>gl29jkb07p7}`pFV`V@8%WYfw6&y&=HbX=9<1_zf6tqE}WYp&HQ7 z;Y!Sj%|n#Gp@qx-^1gC!z|ZPJ30C$=Y0n_K<8!p1c2YK4C-`!oF~GCugq@eiiHeV* zNr*O9xPnJQcE&v3P3FXOQ!KsJm1d-^3B0pJoc1K~d*i|rP16z6yj^ZaIh5tyC(Kw) z@3}Yg;CRVOzavD_wk5l)9_$)*;~If+<4v{z_i(>IpigOn&+2(blt5=bieNKlZ9~Uo!cp^L) z?Wh)hg4)07ni%akzA+l+${7{5EQ2Sz#^GynCTVEh?}WAAw9pb#MifHyx|E{LI&Q7$ z+WYf+dWfp7iT8XD2<^sN{Z3FHz|G)qF6)t;9elWXXLY+CG=8Otu1a;1=9xSRnOy+L z_^x1EaA{IR-=iGrliVvUm2IL-A{=KRRM!WsY~g}Y_-&&;dVZx^K3E2&4u@-ZT*jgNvq6x$q%Y|SE=oe_IYVU zNQP5!5s{xxt{|QVHEW)F7Xk|0J#aaRmvmrp;!cD|1n8|S7)h98{`Nuk^m3(jXT2LC zrjv;#MZO*=EG_ZY+i>O-d?DgZ)t|@vm&#b-3wF_c;@^X@3j*mB8o;6}6!n03>H!|=6*TApJC#vu#G z!N1e?So}DJeSYm~;bfIUT$ghq8^4BA%+ozOlF;d`5#ih9hL5qjJe3#@1&b9Do@O-8fSI5^>2H%^V{QXkC0 zDw01L8AOM;%8Y*1Yd$}5YWyCKHDp-GIdfCXw%)Ye-!kE#t)RwGQ8TqSp%n-J+@aS( zB-iJKH((%YiBy|0>5ms$b?aAGy)79q{4TisB%c;{br0?JX~E1phH~&dGX-9j)PpAD z1qYodaA9MYg>!I#({dYC;Uk~jvRf37r*7LC)U8iNE%_2StIr%u0DmphT=T-Ysw<;j z8!tY$gC2U%XS=t-BA!sqC5AH%rvzi(MbUr@O3%OL*;}beuC0ppobvUXA#Z6OB`YIK z609#hwzshT5Y1L?#}T}6sUqq^sn;G_?VpCoSd6|XxvZwJ6+-l}qgMQCWyx}>tjpe} zO!!+NytkaVE7ivJYU?B1)x%OAtXAFPh`F|1-L{mXs%LJW#+kPU9l|OJ*uc-c5H&Ca z=6DgrJgh9Wma3Bu9NSvfJm4)RaoWu(?o*sj)(Kdp^(XIUDLWMP(AkIOaT;?hrfD5{ zTkf;G=4i0P2Z+43MRiW7bh?M57{H~#p$h~oEO%HWnGKOS}UdHIV!3%2C{eT9@6^Y zN)L=WbmY1AOUFveDu2NcrHTzd-0BYuhnnDl#a$1Mmg7(=8q2TPzv--1SM@n`Pnmww0!ulpNc5KyFyBmiBk57(47K*JGWXBb4VTxdiAf4${#CvG zH&x*OUunZvO-qZv1$-hrLnpf{S-8QBISm3iq*l-HGLYY8Jf7_Le?4EfiNf*DhFeX+ Rbxr_(a#G6oiY1JE{|6ePQ%e8< From c802f5430d4a404744ed64adfddda18dd57e525a Mon Sep 17 00:00:00 2001 From: Gregory Shtrasberg <156009573+gshtras@users.noreply.github.com> Date: Fri, 28 Mar 2025 23:39:18 -0400 Subject: [PATCH 090/593] [ROCm][AMD][Build] Update AMD supported arch list (#15632) Signed-off-by: Gregory Shtrasberg --- CMakeLists.txt | 2 +- docs/source/getting_started/installation/gpu/rocm.inc.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e0f1fdf78d142..9d15b77bc3798 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,7 @@ set(PYTHON_SUPPORTED_VERSIONS "3.9" "3.10" "3.11" "3.12") set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;12.0") # Supported AMD GPU architectures. -set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101") +set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1200;gfx1201") # # Supported/expected torch versions for CUDA/ROCm. diff --git a/docs/source/getting_started/installation/gpu/rocm.inc.md b/docs/source/getting_started/installation/gpu/rocm.inc.md index 4381cef5e96a3..cdd487696c8aa 100644 --- a/docs/source/getting_started/installation/gpu/rocm.inc.md +++ b/docs/source/getting_started/installation/gpu/rocm.inc.md @@ -8,7 +8,7 @@ There are no pre-built wheels for this device, so you must either use the pre-bu ## Requirements -- GPU: MI200s (gfx90a), MI300 (gfx942), Radeon RX 7900 series (gfx1100) +- GPU: MI200s (gfx90a), MI300 (gfx942), Radeon RX 7900 series (gfx1100/1101), Radeon RX 9000 series (gfx1200/1201) - ROCm 6.3 ## Set up using Python From de1cb38769e9eb9812fa425c4fdfcf8faa3c420e Mon Sep 17 00:00:00 2001 From: pengyuange Date: Sat, 29 Mar 2025 11:39:21 +0800 Subject: [PATCH 091/593] [Model] Support Skywork-R1V (#15397) Signed-off-by: jiacai.liu <932997367@qq.com> Co-authored-by: jiacai.liu <932997367@qq.com> --- docs/source/models/supported_models.md | 7 + examples/offline_inference/vision_language.py | 36 + .../vision_language/test_models.py | 14 + .../vision_language/vlm_utils/model_utils.py | 57 + .../multimodal/processing/test_common.py | 9 +- tests/models/registry.py | 1 + vllm/entrypoints/chat_utils.py | 2 +- vllm/model_executor/models/registry.py | 1 + vllm/model_executor/models/skyworkr1v.py | 1014 +++++++++++++++++ vllm/transformers_utils/config.py | 5 +- vllm/transformers_utils/configs/__init__.py | 2 + vllm/transformers_utils/configs/skyworkr1v.py | 53 + 12 files changed, 1194 insertions(+), 7 deletions(-) create mode 100644 vllm/model_executor/models/skyworkr1v.py create mode 100644 vllm/transformers_utils/configs/skyworkr1v.py diff --git a/docs/source/models/supported_models.md b/docs/source/models/supported_models.md index 793831fd06ded..8477158a00403 100644 --- a/docs/source/models/supported_models.md +++ b/docs/source/models/supported_models.md @@ -921,6 +921,13 @@ See [this page](#generative-models) for more information on how to use generativ * ✅︎ * ✅︎ * ✅︎ +- * `SkyworkR1VChatModel` + * Skywork-R1V-38B + * T + I + * `Skywork/Skywork-R1V-38B` + * + * ✅︎ + * ✅︎ - * `UltravoxModel` * Ultravox * T + AE+ diff --git a/examples/offline_inference/vision_language.py b/examples/offline_inference/vision_language.py index 0adbe574370d3..572eabe261930 100644 --- a/examples/offline_inference/vision_language.py +++ b/examples/offline_inference/vision_language.py @@ -804,6 +804,41 @@ def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData: ) +# SkyworkR1V +def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData: + assert modality == "image" + + model_name = "Skywork/Skywork-R1V-38B" + + engine_args = EngineArgs( + model=model_name, + trust_remote_code=True, + max_model_len=4096, + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache, + ) + + tokenizer = AutoTokenizer.from_pretrained(model_name, + trust_remote_code=True) + messages = [[{ + 'role': 'user', + 'content': f"\n{question}" + }] for question in questions] + prompts = tokenizer.apply_chat_template(messages, + tokenize=False, + add_generation_prompt=True) + + # Stop tokens for SkyworkR1V + # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py + stop_tokens = ["<｜end▁of▁sentence｜>", "<|endoftext|>"] + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] + + return ModelRequestData( + engine_args=engine_args, + prompts=prompts, + stop_token_ids=stop_token_ids, + ) + + model_example_map = { "aria": run_aria, "blip-2": run_blip2, @@ -834,6 +869,7 @@ model_example_map = { "qwen_vl": run_qwen_vl, "qwen2_vl": run_qwen2_vl, "qwen2_5_vl": run_qwen2_5_vl, + "skywork_chat": run_skyworkr1v, } diff --git a/tests/models/decoder_only/vision_language/test_models.py b/tests/models/decoder_only/vision_language/test_models.py index d500ef5d8b805..0d1d237e5693c 100644 --- a/tests/models/decoder_only/vision_language/test_models.py +++ b/tests/models/decoder_only/vision_language/test_models.py @@ -474,6 +474,20 @@ VLM_TEST_SETTINGS = { vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output, prompt_path_encoder=model_utils.qwen_prompt_path_encoder, ), + "skywork_r1v": VLMTestInfo( + models=["Skywork/Skywork-R1V-38B"], + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), + prompt_formatter=lambda img_prompt: f"<｜begin▁of▁sentence｜><｜User｜>\n{img_prompt}<｜Assistant｜>\n", # noqa: E501 + single_image_prompts=IMAGE_ASSETS.prompts({ + "stop_sign": "\nWhat's the content in the center of the image?", # noqa: E501 + "cherry_blossom": "\nWhat is the season?", + }), + multi_image_prompt="\n\nDescribe the two images in short.", # noqa: E501 + max_model_len=4096, + use_tokenizer_eos=True, + patch_hf_runner=model_utils.skyworkr1v_patch_hf_runner, + marks=[large_gpu_mark(min_gb=80)], + ), ### Tensor parallel / multi-gpu broadcast tests "chameleon-broadcast": VLMTestInfo( models=["facebook/chameleon-7b"], diff --git a/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py b/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py index c84bf6dc15f42..2ddf28aca4f63 100644 --- a/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py +++ b/tests/models/decoder_only/vision_language/vlm_utils/model_utils.py @@ -376,6 +376,63 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: return hf_model +def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: + """Patches and returns an instance of the HfRunner to use for SkyworkR1V.""" + + class SkyworkR1VProcessor: + """A simple processor for SkyworkR1V.""" + + def __init__(self, hf_runner: HfRunner): + self.num_image_token = hf_runner.model.num_image_token + self.tokenizer = hf_runner.tokenizer + + self.config = AutoConfig.from_pretrained(hf_runner.model_name, + trust_remote_code=True) + self.vision_config = self.config.vision_config + self.use_thumbnail = self.config.use_thumbnail + self.min_num = self.config.min_dynamic_patch + self.max_num = self.config.max_dynamic_patch + self.image_size = self.vision_config.image_size + + def __call__(self, text: str, images: Union[Image, list[Image]], + **kwargs): + from vllm.model_executor.models.skyworkr1v import ( + IMG_CONTEXT, IMG_END, IMG_START, + image_to_pixel_values_skyworkr1v) + images = [images] if isinstance(images, Image) else images + pixel_values = [ + image_to_pixel_values_skyworkr1v( + image, + input_size=self.image_size, + min_num=self.min_num, + max_num=self.max_num, + use_thumbnail=self.use_thumbnail, + ) for image in images + ] + num_patches_list = [ + pixel_value.shape[0] for pixel_value in pixel_values + ] + pixel_values = torch.cat(pixel_values, dim=0) + for num_patches in num_patches_list: + context_tokens = IMG_CONTEXT * self.num_image_token \ + * num_patches + image_tokens = IMG_START + context_tokens + IMG_END + text = text.replace('', image_tokens, 1) + prompt = self.tokenizer(text, return_tensors="pt") + prompt.update({"pixel_values": pixel_values}) + return prompt + + img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids( + "") + hf_model.model.img_context_token_id = img_context_token_id + hf_model.processor = SkyworkR1VProcessor(hf_model) + hf_model.model.get_output_embeddings = lambda: \ + hf_model.model.language_model.get_output_embeddings() + hf_model.model.generate = types.MethodType(_internvl_generate, + hf_model.model) + return hf_model + + def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: """Patches and returns an instance of the HfRunner to use for InternVL.""" diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 078ed21537b8d..e4f1d297fc092 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -262,22 +262,23 @@ def _test_processing_correctness_mistral( "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "meta-llama/Llama-3.2-11B-Vision-Instruct", "TIGER-Lab/Mantis-8B-siglip-llama3", - "mistralai/Pixtral-12B-2409", - "mistral-community/pixtral-12b", "openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6", "openbmb/MiniCPM-V-2_6", "allenai/Molmo-7B-D-0924", "allenai/Molmo-7B-O-0924", "nvidia/NVLM-D-72B", + "google/paligemma-3b-mix-224", + "google/paligemma2-3b-ft-docci-448", + "mistralai/Pixtral-12B-2409", + "mistral-community/pixtral-12b", "Qwen/Qwen-VL-Chat", "Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct", "Qwen/Qwen2-Audio-7B-Instruct", + "Skywork/Skywork-R1V-38B", "fixie-ai/ultravox-v0_5-llama-3_2-1b", "openai/whisper-large-v3", - "google/paligemma-3b-mix-224", - "google/paligemma2-3b-ft-docci-448", ]) @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0]) @pytest.mark.parametrize("num_batches", [32]) diff --git a/tests/models/registry.py b/tests/models/registry.py index d7946b75b7978..ff0c37a6afd76 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -294,6 +294,7 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501 "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501 min_transformers_version="4.49"), # noqa: E501 + "SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"), "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501 trust_remote_code=True), # [Encoder-decoder] diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 73a69d3037f7f..24382142768b5 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -496,7 +496,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]): return self._cached_token_str(self._tokenizer, hf_config.image_token_index) if model_type in ("chameleon", "deepseek_vl_v2", "internvl_chat", - "NVLM_D", "h2ovl_chat"): + "skywork_chat", "NVLM_D", "h2ovl_chat"): return "" if model_type == "mllama": return "<|image|>" diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 7797d9a2cc203..9288a4b81748e 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -190,6 +190,7 @@ _MULTIMODAL_MODELS = { # [Encoder-decoder] "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501 "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501 + "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"), "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"), # noqa: E501 } diff --git a/vllm/model_executor/models/skyworkr1v.py b/vllm/model_executor/models/skyworkr1v.py new file mode 100644 index 0000000000000..ac5de0e36b894 --- /dev/null +++ b/vllm/model_executor/models/skyworkr1v.py @@ -0,0 +1,1014 @@ +# SPDX-License-Identifier: Apache-2.0 + +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py +# -------------------------------------------------------- +# SkyworkR1V +# Copyright (c) 2025 Skywork +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from abc import ABC, abstractmethod +from collections.abc import Iterable, Mapping, Sequence +from functools import cached_property +from typing import Literal, Optional, Set, Tuple, TypedDict, TypeVar, Union + +import torch +import torch.nn as nn +import torchvision.transforms as T +from PIL import Image +from transformers import BatchEncoding, PretrainedConfig, TensorType + +from vllm.config import VllmConfig +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.awq import AWQConfig +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler +from vllm.model_executor.models.intern_vit import (InternVisionModel, + InternVisionPatchModel) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs, + NestedTensors) +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems, + ImageSize, MultiModalDataItems) +from vllm.multimodal.processing import (BaseMultiModalProcessor, + BaseProcessingInfo, PromptReplacement, + PromptUpdate, PromptUpdateDetails) +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.tokenizer import AnyTokenizer + +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model, + maybe_prefix, merge_multimodal_embeddings) +from .vision import scatter_patch_features, select_patch_features + +IMG_START = '' +IMG_END = '' +IMG_CONTEXT = '' + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + + +class SkyworkR1VImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + pixel_values_flat: torch.Tensor + """ + Shape: + `(batch_size * num_images * (1 + num_patches), num_channels, height, width)` + """ + + num_patches: torch.Tensor + """Shape: `(batch_size * num_images)`""" + + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]] + """ + A boolean mask indicating which image embeddings correspond + to patch tokens. + + Shape: `(batch_size * num_images, num_embeds)` + """ + + +class SkyworkR1VImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: Union[torch.Tensor, list[torch.Tensor]] + """ + A tensor of shape `(num_images, total_image_feature_size, hidden_size)` + or a list of tensors of shape `(total_image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +SkyworkR1VImageInputs = Union[SkyworkR1VImagePixelInputs, + SkyworkR1VImageEmbeddingInputs] + + +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/ +def build_transform(input_size: int): + MEAN, STD = IMAGENET_MEAN, IMAGENET_STD + return T.Compose([ + T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img), + T.Resize((input_size, input_size), + interpolation=T.InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=MEAN, std=STD) + ]) + + +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/ +def find_closest_aspect_ratio( + aspect_ratio: float, + target_ratios: list[tuple[int, int]], + *, + width: int, + height: int, + image_size: int, +) -> tuple[int, int]: + best_ratio_diff = float('inf') + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_aspect_ratio = ratio[0] / ratio[1] + ratio_diff = abs(aspect_ratio - target_aspect_ratio) + if ratio_diff < best_ratio_diff: + best_ratio_diff = ratio_diff + best_ratio = ratio + elif ratio_diff == best_ratio_diff: + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: + best_ratio = ratio + return best_ratio + + +def resolve_skyworkr1v_min_max_num( + *, + min_dynamic_patch: int, + max_dynamic_patch: int, + dynamic_image_size: bool, + use_thumbnail: bool, +) -> tuple[int, int]: + min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1 + + if use_thumbnail and max_dynamic_patch != 1: + max_dynamic_patch += 1 + + return min_dynamic_patch, max_dynamic_patch + + +def get_skyworkr1v_target_ratios( + min_num: int, + max_num: int, +) -> list[tuple[int, int]]: + target_ratios = {(i, j) + for n in range(min_num, max_num + 1) + for i in range(1, n + 1) + for j in range(1, n + 1) if min_num <= i * j <= max_num} + return sorted(target_ratios, key=lambda x: x[0] * x[1]) + + +def calculate_skyworkr1v_targets( + *, + orig_width: int, + orig_height: int, + target_ratios: list[tuple[int, int]], + image_size: int, + use_thumbnail: bool, +) -> tuple[int, int, int]: + aspect_ratio = orig_width / orig_height + + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, + target_ratios, + width=orig_width, + height=orig_height, + image_size=image_size, + ) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + + # add thumbnail image if num_blocks != 1 + if use_thumbnail and blocks != 1: + blocks += 1 + + return blocks, target_width, target_height + + +def dynamic_preprocess_skyworkr1v( + image: Image.Image, + *, + target_ratios: list[tuple[int, int]], + image_size: int, + use_thumbnail: bool, +) -> list[Image.Image]: + orig_width, orig_height = image.size + + # calculate the number of blocks without thumbnail + blocks, target_width, target_height = calculate_skyworkr1v_targets( + orig_width=orig_width, + orig_height=orig_height, + target_ratios=target_ratios, + image_size=image_size, + use_thumbnail=False, + ) + + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ((i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + + assert len(processed_images) == blocks + + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + + return processed_images + + +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B +def image_to_pixel_values_skyworkr1v( + image: Image.Image, + *, + input_size: int, + min_num: int, + max_num: int, + use_thumbnail: bool, +) -> torch.Tensor: + target_ratios = get_skyworkr1v_target_ratios(min_num, max_num) + + transform = build_transform(input_size=input_size) + images = dynamic_preprocess_skyworkr1v( + image, + target_ratios=target_ratios, + image_size=input_size, + use_thumbnail=use_thumbnail, + ) + + pixel_values = torch.stack([transform(image) for image in images]) + return pixel_values + + +class BaseSkyworkR1VProcessor(ABC): + """ + This model doesn't define its own HF processor, + so we implement our own one here. + + The code to insert image tokens is based on: + https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py#L252 + """ + + def __init__( + self, + config: PretrainedConfig, + tokenizer: AnyTokenizer, + *, + min_dynamic_patch: Optional[int] = None, + max_dynamic_patch: Optional[int] = None, + dynamic_image_size: Optional[bool] = None, + ) -> None: + super().__init__() + + self.config = config + self.tokenizer = tokenizer + + image_size: int = config.vision_config.image_size + patch_size: int = config.vision_config.patch_size + + if min_dynamic_patch is None: + min_dynamic_patch = config.min_dynamic_patch + assert isinstance(min_dynamic_patch, int) + + if max_dynamic_patch is None: + max_dynamic_patch = config.max_dynamic_patch + assert isinstance(max_dynamic_patch, int) + + if dynamic_image_size is None: + dynamic_image_size = config.dynamic_image_size + assert isinstance(dynamic_image_size, bool) + + self.num_image_token = int( + (image_size // patch_size)**2 * (config.downsample_ratio**2)) + self.image_size = image_size + self.min_dynamic_patch = min_dynamic_patch + self.max_dynamic_patch = max_dynamic_patch + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail: bool = config.use_thumbnail + + @property + @abstractmethod + def image_token_id(self) -> int: + raise NotImplementedError + + @abstractmethod + def get_image_repl( + self, + feature_size: int, + num_patches: Optional[int], + ) -> PromptUpdateDetails[str]: + raise NotImplementedError + + def resolve_min_max_num( + self, + *, + min_dynamic_patch: Optional[int] = None, + max_dynamic_patch: Optional[int] = None, + dynamic_image_size: Optional[bool] = None, + use_thumbnail: Optional[bool] = None, + ) -> tuple[int, int]: + min_dynamic_patch = (self.min_dynamic_patch if min_dynamic_patch + is None else min_dynamic_patch) + max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch + is None else max_dynamic_patch) + dynamic_image_size = (self.dynamic_image_size if dynamic_image_size + is None else dynamic_image_size) + use_thumbnail = (self.use_thumbnail + if use_thumbnail is None else use_thumbnail) + + return resolve_skyworkr1v_min_max_num( + min_dynamic_patch=min_dynamic_patch, + max_dynamic_patch=max_dynamic_patch, + dynamic_image_size=dynamic_image_size, + use_thumbnail=use_thumbnail, + ) + + def resolve_target_ratios( + self, + *, + min_dynamic_patch: Optional[int] = None, + max_dynamic_patch: Optional[int] = None, + dynamic_image_size: Optional[bool] = None, + use_thumbnail: Optional[bool] = None, + ) -> list[tuple[int, int]]: + min_num, max_num = self.resolve_min_max_num( + min_dynamic_patch=min_dynamic_patch, + max_dynamic_patch=max_dynamic_patch, + dynamic_image_size=dynamic_image_size, + use_thumbnail=use_thumbnail, + ) + + return get_skyworkr1v_target_ratios(min_num, max_num) + + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + ) -> int: + target_ratios = self.resolve_target_ratios( + use_thumbnail=False, # Applied in calculate_targets + ) + + num_patches, _, _ = calculate_skyworkr1v_targets( + orig_width=image_width, + orig_height=image_height, + image_size=self.image_size, + target_ratios=target_ratios, + use_thumbnail=self.use_thumbnail, + ) + + return num_patches * self.num_image_token + + def _images_to_pixel_values_lst( + self, + images: list[Image.Image], + min_dynamic_patch: Optional[int] = None, + max_dynamic_patch: Optional[int] = None, + dynamic_image_size: Optional[bool] = None, + ) -> list[torch.Tensor]: + min_num, max_num = self.resolve_min_max_num( + min_dynamic_patch=min_dynamic_patch, + max_dynamic_patch=max_dynamic_patch, + dynamic_image_size=dynamic_image_size, + use_thumbnail=False, # Applied in image_to_pixel_values + ) + + return [ + image_to_pixel_values_skyworkr1v( + image, + input_size=self.image_size, + min_num=min_num, + max_num=max_num, + use_thumbnail=self.use_thumbnail, + ) for image in images + ] + + def __call__( + self, + text: Optional[Union[str, list[str]]] = None, + images: Optional[Union[Image.Image, list[Image.Image]]] = None, + min_dynamic_patch: Optional[int] = None, + max_dynamic_patch: Optional[int] = None, + dynamic_image_size: Optional[bool] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + ) -> Mapping[str, NestedTensors]: + if text is None: + text = [] + if not isinstance(text, list): + text = [text] + if images is None: + images = [] + if not isinstance(images, list): + images = [images] + + if len(images) == 0: + image_inputs = {} + else: + pixel_values_lst = self._images_to_pixel_values_lst( + images, + min_dynamic_patch=min_dynamic_patch, + max_dynamic_patch=max_dynamic_patch, + dynamic_image_size=dynamic_image_size, + ) + image_inputs: dict[str, NestedTensors] = { + "pixel_values_flat": + torch.cat(pixel_values_lst), + "image_num_patches": + torch.tensor([len(item) for item in pixel_values_lst]), + } + + tokenizer = self.tokenizer + image_token_id = self.image_token_id + + embed_is_patch = list[torch.Tensor]() + + for pixel_values in pixel_values_lst: + num_patches = pixel_values.shape[0] + feature_size = num_patches * self.num_image_token + + image_repl = self.get_image_repl(feature_size, num_patches) + feature_tokens = tokenizer.encode(image_repl.features, + add_special_tokens=False) + + text = [t.replace('', image_repl.full, 1) for t in text] + embed_is_patch.append( + torch.tensor(feature_tokens) == image_token_id) + + image_inputs["embed_is_patch"] = embed_is_patch + + text_inputs = self.tokenizer(text) + + return { + **BatchEncoding(text_inputs, tensor_type=return_tensors), + **image_inputs, + } + + +class SkyworkR1VProcessor(BaseSkyworkR1VProcessor): + + @property + def image_token_id(self) -> int: + return self.tokenizer.get_vocab()[IMG_CONTEXT] + + def get_image_repl( + self, + feature_size: int, + num_patches: Optional[int], + ) -> PromptUpdateDetails[str]: + repl_features = IMG_CONTEXT * feature_size + repl_full = IMG_START + repl_features + IMG_END + + return PromptUpdateDetails(full=repl_full, features=repl_features) + + +class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo): + + @abstractmethod + def get_hf_processor( + self, + *, + min_dynamic_patch: Optional[int] = None, + max_dynamic_patch: Optional[int] = None, + dynamic_image_size: Optional[bool] = None, + **kwargs: object, + ) -> BaseSkyworkR1VProcessor: + raise NotImplementedError + + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]: + return {"image": None} + + def get_mm_max_tokens_per_item( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> Mapping[str, int]: + return {"image": self.get_max_image_tokens()} + + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + processor: Optional[BaseSkyworkR1VProcessor], + ) -> int: + if processor is None: + processor = self.get_hf_processor() + + return processor.get_num_image_tokens( + image_width=image_width, + image_height=image_height, + ) + + def get_max_image_tokens(self) -> int: + target_width, target_height = self.get_image_size_with_most_features() + + return self.get_num_image_tokens( + image_width=target_width, + image_height=target_height, + processor=None, + ) + + def get_image_size_with_most_features(self) -> ImageSize: + processor = self.get_hf_processor() + + base_size = processor.image_size + target_ratios = processor.resolve_target_ratios() + + largest_feature_size, largest_feature_pinpoint = 0, None + for wr, hr in target_ratios: + width, height = base_size * wr, base_size * hr + + feat_size = self.get_num_image_tokens( + image_width=width, + image_height=height, + processor=processor, + ) + if feat_size > largest_feature_size: + largest_feature_size = feat_size + largest_feature_pinpoint = ImageSize(width=width, + height=height) + + if largest_feature_size == 0 or largest_feature_pinpoint is None: + raise ValueError("Cannot have a largest feature size of 0!") + + return largest_feature_pinpoint + + +_I = TypeVar("_I", bound=BaseSkyworkR1VProcessingInfo) + + +class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[_I]): + + def get_dummy_processor_inputs( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> ProcessorInputs: + target_width, target_height = \ + self.info.get_image_size_with_most_features() + num_images = mm_counts.get("image", 0) + + mm_data = { + "image": + self._get_dummy_images(width=target_width, + height=target_height, + num_images=num_images) + } + + return ProcessorInputs( + prompt_text="" * num_images, + mm_data=mm_data, + ) + + +class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]): + + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + ) -> Mapping[str, NestedTensors]: + processed_outputs = super()._call_hf_processor( + prompt=prompt, + mm_data=mm_data, + mm_kwargs=mm_kwargs, + ) + + hf_processor = self.info.get_hf_processor(**mm_kwargs) + image_token_id = hf_processor.image_token_id + + # Since there may be extra tokens in the feature placeholders, + # we need to pass the image token ID to the model to select the + # tokens to merge from the vision encoder outputs + processed_outputs["image_token_id"] = torch.tensor(image_token_id) + + return processed_outputs + + def _get_mm_fields_config( + self, + hf_inputs: Mapping[str, NestedTensors], + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0)) + num_images = len(image_num_patches) + + return dict( + pixel_values_flat=MultiModalFieldConfig.flat_from_sizes( + "image", image_num_patches), + image_num_patches=MultiModalFieldConfig.batched("image"), + embed_is_patch=MultiModalFieldConfig.batched("image"), + image_embeds=MultiModalFieldConfig.batched("image"), + image_token_id=MultiModalFieldConfig.shared("image", num_images), + ) + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + out_mm_kwargs: MultiModalKwargs, + ) -> Sequence[PromptUpdate]: + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + + if "image_num_patches" in out_mm_kwargs: + image_num_patches = out_mm_kwargs["image_num_patches"] + assert isinstance(image_num_patches, torch.Tensor) + image_num_patches = image_num_patches.tolist() + elif "image_embeds" in out_mm_kwargs: + # TODO: Use image size information in dictionary embedding inputs + # to compute num_patches (similar to Qwen2-VL) + image_num_patches = [None] * len(out_mm_kwargs["image_embeds"]) + else: + image_num_patches = [] + + def get_replacement_skyworkr1v(item_idx: int): + images = mm_items.get_items( + "image", (ImageEmbeddingItems, ImageProcessorItems)) + + if isinstance(images, ImageEmbeddingItems): + feature_size = images.get_feature_size(item_idx) + else: + image_size = images.get_image_size(item_idx) + feature_size = self.info.get_num_image_tokens( + image_width=image_size.width, + image_height=image_size.height, + processor=hf_processor, + ) + + num_patches = image_num_patches[item_idx] + if num_patches is not None: + assert isinstance(num_patches, int) + + return hf_processor.get_image_repl(feature_size, num_patches) + + return [ + PromptReplacement( + modality="image", + target="", + replacement=get_replacement_skyworkr1v, + ) + ] + + +class SkyworkR1VProcessingInfo(BaseSkyworkR1VProcessingInfo): + + def get_hf_processor( + self, + *, + min_dynamic_patch: Optional[int] = None, + max_dynamic_patch: Optional[int] = None, + dynamic_image_size: Optional[bool] = None, + **kwargs: object, + ) -> SkyworkR1VProcessor: + if min_dynamic_patch is not None: + kwargs["min_dynamic_patch"] = min_dynamic_patch + if max_dynamic_patch is not None: + kwargs["max_dynamic_patch"] = max_dynamic_patch + if dynamic_image_size is not None: + kwargs["dynamic_image_size"] = dynamic_image_size + + return self.ctx.init_processor( + SkyworkR1VProcessor, + config=self.get_hf_config(), + tokenizer=self.get_tokenizer(), + **kwargs, + ) + + +@MULTIMODAL_REGISTRY.register_processor( + SkyworkR1VMultiModalProcessor, + info=SkyworkR1VProcessingInfo, + dummy_inputs=SkyworkR1VDummyInputsBuilder) +class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + multimodal_config = vllm_config.model_config.multimodal_config + + self.config = config + self.multimodal_config = multimodal_config + self._patch_quant_config(config, quant_config) + + image_size = config.force_image_size or config.vision_config.image_size + patch_size = config.vision_config.patch_size + self.patch_size = patch_size + self.num_image_token = int( + (image_size // patch_size)**2 * (config.downsample_ratio**2)) + self.downsample_ratio = config.downsample_ratio + self.ps_version = config.ps_version + + self.llm_arch_name = config.text_config.architectures[0] + self.is_mono = self.llm_arch_name == 'SkyworkLM2VEForCausalLM' + self.vision_model = self._init_vision_model( + config, + quant_config=quant_config, + is_mono=self.is_mono, + prefix=maybe_prefix(prefix, "vision_model"), + ) + + self.language_model = init_vllm_registered_model( + vllm_config=vllm_config, + hf_config=config.text_config, + prefix=maybe_prefix(prefix, "language_model"), + ) + + self.mlp1 = self._init_mlp1(config) + + self.img_context_token_id = None + self.visual_token_mask = None + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + def _patch_quant_config(self, config: PretrainedConfig, + quant_config: QuantizationConfig): + # the awq models from OpenGVLab missing `modules_to_not_convert` + # patch the quant_config to add `modules_to_not_convert` back + if isinstance(quant_config, AWQConfig): + text_config = config.text_config + llm_quant_config = getattr(text_config, "quantization_config", + None) + if (not quant_config.modules_to_not_convert) and \ + (llm_quant_config is not None): + quant_config.modules_to_not_convert.append("vision_model") + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return get_sampler() + + def _init_vision_model( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig], + *, + is_mono: bool, + prefix: str, + ): + if not is_mono: + vision_feature_layer = config.select_layer + if vision_feature_layer < 0: + num_hidden_layers = config.vision_config.num_hidden_layers \ + + vision_feature_layer + 1 + else: + num_hidden_layers = vision_feature_layer + 1 + + return InternVisionModel( + config.vision_config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers, + prefix=prefix, + ) + else: + return InternVisionPatchModel(config.vision_config) + + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential: + vit_hidden_size = config.vision_config.hidden_size + llm_hidden_size = config.text_config.hidden_size + + return nn.Sequential( + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2), + ReplicatedLinear(vit_hidden_size * + int(1 / self.downsample_ratio)**2, + llm_hidden_size, + return_bias=False), + nn.GELU(), + ReplicatedLinear(llm_hidden_size, + llm_hidden_size, + return_bias=False), + ) + + def pixel_shuffle(self, x, scale_factor=0.5): + n, w, h, c = x.size() + # N, W, H, C --> N, W, H * scale, C // scale + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor)) + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale + x = x.permute(0, 2, 1, 3).contiguous() + x = x.view(n, int(h * scale_factor), int(w * scale_factor), + int(c / (scale_factor * scale_factor))) + if self.ps_version == 'v1': + pass + else: + x = x.permute(0, 2, 1, 3).contiguous() + return x + + def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor: + vit_embeds = self.vision_model(pixel_values=pixel_values) + vit_embeds = vit_embeds[:, 1:, :] + + h = w = int(vit_embeds.shape[1]**0.5) + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1) + vit_embeds = self.pixel_shuffle(vit_embeds, + scale_factor=self.downsample_ratio) + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, + vit_embeds.shape[-1]) + vit_embeds = self.mlp1(vit_embeds) + return vit_embeds + + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape) + + if actual_dims != expected_dims: + expected_expr = str(expected_dims) + raise ValueError( + "The expected shape of pixel values per image per batch " + f" per patch is {expected_expr}. " + f"You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[SkyworkR1VImageInputs]: + pixel_values_flat = kwargs.pop("pixel_values_flat", None) + image_num_patches = kwargs.pop("image_num_patches", None) + embed_is_patch = kwargs.pop("embed_is_patch", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values_flat is None and image_embeds is None: + return None + + if image_embeds is not None: + if not isinstance(image_embeds, (torch.Tensor, list)): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + + return SkyworkR1VImageEmbeddingInputs( + type="image_embeds", + data=flatten_bn(image_embeds), + ) + + image_token_id = kwargs["image_token_id"] + assert isinstance(image_token_id, torch.Tensor) + self.img_context_token_id = image_token_id.flatten().unique().item() + + if pixel_values_flat is not None: + if not isinstance(pixel_values_flat, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values_flat)}") + + if not isinstance(image_num_patches, (torch.Tensor, list)): + raise ValueError("Incorrect type of image_num_patches. " + f"Got type: {type(image_num_patches)}") + + if not isinstance(embed_is_patch, (torch.Tensor, list)): + raise ValueError("Incorrect type of embed_is_patch. " + f"Got type: {type(embed_is_patch)}") + + pixel_values_flat = flatten_bn(pixel_values_flat, concat=True) + image_num_patches = flatten_bn(image_num_patches, concat=True) + embed_is_patch = flatten_bn(embed_is_patch) + + return SkyworkR1VImagePixelInputs( + type="pixel_values", + pixel_values_flat=self._validate_pixel_values( + pixel_values_flat), + num_patches=image_num_patches, + embed_is_patch=embed_is_patch, + ) + + raise AssertionError("This line should be unreachable.") + + def _process_image_input( + self, + image_input: SkyworkR1VImageInputs, + ) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor, ...]]: + if image_input["type"] == "image_embeds": + return image_input["data"] + + assert self.vision_model is not None + + image_embeds = self.extract_feature(image_input["pixel_values_flat"]) + + num_patches = image_input["num_patches"] + + # Only one image in the current batch + if len(num_patches) == 1: + return image_embeds.view( + -1, self.config.text_config.hidden_size).unsqueeze(0) + + # NOTE: Image embeddings are split into separate tensors for each image + # by the size of each embedding. + feature_size = image_embeds.shape[1] + image_embeds = image_embeds.view(-1, + self.config.text_config.hidden_size) + image_feature_sizes = [ + num_patches * feature_size for num_patches in num_patches + ] + return image_embeds.split(image_feature_sizes) + + def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None: + if self.is_mono: + self.visual_token_mask = ( + input_ids == self.img_context_token_id).reshape(-1, 1) + else: + self.visual_token_mask = None + + def get_multimodal_embeddings( + self, **kwargs: object) -> Optional[MultiModalEmbeddings]: + image_input = self._parse_and_validate_image_input(**kwargs) + if image_input is None: + return None + + image_features = self._process_image_input(image_input) + + if image_input["type"] != "pixel_values": + return image_features + + return scatter_patch_features( + image_features, + image_input["embed_is_patch"], + ) + + def get_input_embeddings( + self, + input_ids: torch.Tensor, + multimodal_embeddings: Optional[MultiModalEmbeddings] = None, + ) -> torch.Tensor: + inputs_embeds = self.language_model.get_input_embeddings(input_ids) + if multimodal_embeddings is not None: + assert self.img_context_token_id is not None + self._set_visual_token_mask(input_ids) + inputs_embeds = merge_multimodal_embeddings( + input_ids, + inputs_embeds, + select_patch_features(multimodal_embeddings), + self.img_context_token_id, + ) + return inputs_embeds + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs: object, + ) -> Union[SamplerOutput, IntermediateTensors]: + + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + + # NOTE: In v1, inputs_embeds is always generated at model runner, this + # condition is for v0 compatibility. + elif inputs_embeds is None: + vision_embeddings = self.get_multimodal_embeddings(**kwargs) + inputs_embeds = self.get_input_embeddings(input_ids, + vision_embeddings) + input_ids = None + + forward_kwargs = { + "input_ids": input_ids, + "positions": positions, + "intermediate_tensors": intermediate_tensors, + "inputs_embeds": inputs_embeds, + } + + # Only required if the model is mono-architecture + if self.visual_token_mask is not None: + forward_kwargs.update( + {"visual_token_mask": self.visual_token_mask}) + self.visual_token_mask = None + + hidden_states = self.language_model.model(**forward_kwargs) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, + torch.Tensor]]) -> Set[str]: + skip_prefixes = [ + "action_embed", "temporal_embed", "track_embed", + "track_embed_decoder", "box_token", "cg_criterion", "cg_model", + "loc_encoder", "loc_decoder", "sam", "temporal_token", + "track_token" + ] + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) + return loader.load_weights(weights) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 1937b13884711..71990468c315a 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -37,8 +37,8 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config, MLPSpeculatorConfig, MPTConfig, NemotronConfig, NVLM_D_Config, Olmo2Config, RWConfig, - SolarConfig, Telechat2Config, - UltravoxConfig) + SkyworkR1VChatConfig, SolarConfig, + Telechat2Config, UltravoxConfig) # yapf: enable from vllm.transformers_utils.utils import check_gguf_file from vllm.utils import resolve_obj_by_qualname @@ -76,6 +76,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = { "NVLM_D": NVLM_D_Config, "olmo2": Olmo2Config, "solar": SolarConfig, + "skywork_chat": SkyworkR1VChatConfig, "telechat": Telechat2Config, "ultravox": UltravoxConfig, **_CONFIG_REGISTRY_OVERRIDE_HF diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 9060565596b21..53699341bfba8 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -20,6 +20,7 @@ from vllm.transformers_utils.configs.mpt import MPTConfig from vllm.transformers_utils.configs.nemotron import NemotronConfig from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config from vllm.transformers_utils.configs.olmo2 import Olmo2Config +from vllm.transformers_utils.configs.skyworkr1v import SkyworkR1VChatConfig from vllm.transformers_utils.configs.solar import SolarConfig from vllm.transformers_utils.configs.telechat2 import Telechat2Config from vllm.transformers_utils.configs.ultravox import UltravoxConfig @@ -42,6 +43,7 @@ __all__ = [ "NemotronConfig", "NVLM_D_Config", "Olmo2Config", + "SkyworkR1VChatConfig", "SolarConfig", "Telechat2Config", "UltravoxConfig", diff --git a/vllm/transformers_utils/configs/skyworkr1v.py b/vllm/transformers_utils/configs/skyworkr1v.py new file mode 100644 index 0000000000000..ef5f9ba85c237 --- /dev/null +++ b/vllm/transformers_utils/configs/skyworkr1v.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Adapted from +# https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/configuration_skywork_chat.py +# -------------------------------------------------------- +# SkyworkR1V +# Copyright (c) 2025 Skywork +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from transformers.configuration_utils import PretrainedConfig + + +class SkyworkR1VChatConfig(PretrainedConfig): + model_type = 'internvl_chat' + is_composition = True + + def __init__(self, + vision_config=None, + llm_config=None, + use_backbone_lora=0, + use_llm_lora=0, + select_layer=-1, + force_image_size=None, + downsample_ratio=0.5, + template=None, + dynamic_image_size=False, + use_thumbnail=False, + ps_version='v1', + min_dynamic_patch=1, + max_dynamic_patch=6, + **kwargs): + super().__init__(**kwargs) + + if vision_config is None: + vision_config = {} + + if llm_config is None: + llm_config = {} + + self.vision_config = PretrainedConfig(**vision_config) + self.text_config = PretrainedConfig(**llm_config) + + self.use_backbone_lora = use_backbone_lora + self.use_llm_lora = use_llm_lora + self.select_layer = select_layer + self.force_image_size = force_image_size + self.downsample_ratio = downsample_ratio + self.template = template + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.ps_version = ps_version # pixel shuffle version + self.min_dynamic_patch = min_dynamic_patch + self.max_dynamic_patch = max_dynamic_patch From 762b424a528e025ef3d9b02828eb926c6dbddb2c Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Sat, 29 Mar 2025 11:46:57 +0800 Subject: [PATCH 092/593] [Docs] Document v0 engine support in reasoning outputs (#15739) Signed-off-by: Ce Gao --- docs/source/features/reasoning_outputs.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/features/reasoning_outputs.md b/docs/source/features/reasoning_outputs.md index 879b16d4f7b50..3a0be69f8e1c6 100644 --- a/docs/source/features/reasoning_outputs.md +++ b/docs/source/features/reasoning_outputs.md @@ -136,7 +136,14 @@ Remember to check whether the `reasoning_content` exists in the response before ## Structured output -The reasoning content is also available in the structured output. The structured output engine like `xgrammar` will use the reasoning content to generate structured output. +The reasoning content is also available in the structured output. The structured output engine like `xgrammar` will use the reasoning content to generate structured output. It is only supported in v0 engine now. + +```bash +VLLM_USE_V1=0 vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \ + --enable-reasoning --reasoning-parser deepseek_r1 +``` + +Please note that the `VLLM_USE_V1` environment variable must be set to `0` to use the v0 engine. ```python from openai import OpenAI From 6d531ad7b810522fde902cb1cbf95f52bfc77860 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Fri, 28 Mar 2025 20:59:47 -0700 Subject: [PATCH 093/593] [Misc][V1] Misc code streamlining (#15723) Signed-off-by: Nick Hill --- vllm/distributed/utils.py | 5 +-- vllm/v1/core/sched/scheduler.py | 53 ++++++++++++++---------------- vllm/v1/engine/core_client.py | 2 +- vllm/v1/engine/output_processor.py | 2 +- vllm/v1/request.py | 8 +++-- 5 files changed, 32 insertions(+), 38 deletions(-) diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index 4206a24465e28..cae1a25519b3e 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -207,10 +207,7 @@ class StatelessProcessGroup: def barrier(self): """A barrier to synchronize all ranks.""" for i in range(self.world_size): - if i == self.rank: - self.broadcast_obj(None, src=self.rank) - else: - self.broadcast_obj(None, src=i) + self.broadcast_obj(None, src=i) @staticmethod def create( diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 448119761259c..094602a8b732d 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -269,29 +269,26 @@ class Scheduler(SchedulerInterface): request = self.waiting[0] - # Waiting request skipping logic - is_skipped = False # Skip request if the structured output request is still waiting - # for FSM. - if (not is_skipped - and request.status == RequestStatus.WAITING_FOR_FSM): + # for FSM compilation. + if request.status == RequestStatus.WAITING_FOR_FSM: structured_output_req = request.structured_output_request - is_skipped = (not structured_output_req - or not structured_output_req.grammar) - if not is_skipped: + if structured_output_req and structured_output_req.grammar: request.status = RequestStatus.WAITING + else: + self.waiting.popleft() + skipped_waiting_requests.appendleft(request) + continue - # Skip request if max_loras can't be honored. - if (not is_skipped and self.lora_config - and request.lora_request): - req_lora_id = request.lora_request.lora_int_id - is_skipped = (len(scheduled_loras) - == self.lora_config.max_loras - and (req_lora_id not in scheduled_loras)) - - if is_skipped: - skipped_waiting_requests.appendleft(request) + # Check that adding the request still respects the max_loras + # constraint. + if self.lora_config and request.lora_request and ( + len(scheduled_loras) == self.lora_config.max_loras + and request.lora_request.lora_int_id + not in scheduled_loras): + # Scheduling would exceed max_loras, skip. self.waiting.popleft() + skipped_waiting_requests.appendleft(request) continue # Get already-cached tokens. @@ -602,8 +599,9 @@ class Scheduler(SchedulerInterface): # OPTIMIZATION: Avoid list(set) if the set is empty. if cached_encoder_input_ids: for input_id in list(cached_encoder_input_ids): - start_pos = request.mm_positions[input_id]["offset"] - num_tokens = request.mm_positions[input_id]["length"] + mm_positions = request.mm_positions[input_id] + start_pos = mm_positions["offset"] + num_tokens = mm_positions["length"] if start_pos + num_tokens <= request.num_computed_tokens: # The encoder output is already processed and stored # in the decoder's KV cache. @@ -616,25 +614,24 @@ class Scheduler(SchedulerInterface): stopped = False new_logprobs = None - new_token_ids: list[int] = [] + new_token_ids = generated_token_ids # Append generated tokens and check for stop. Note that if # a request is still being prefilled, we expect the model runner # to return empty token ids for the request. - for output_token_id in generated_token_ids: + for num_new, output_token_id in enumerate(new_token_ids, 1): request.append_output_token_ids(output_token_id) - new_token_ids.append(output_token_id) # Check for stop and update request state. # This must be called before we make the EngineCoreOutput. stopped = check_stop(request, self.max_model_len) if stopped: self._free_request(request) + del new_token_ids[num_new:] # Trim new tokens if needed. break # Extract sample logprobs if needed. - if (request.sampling_params.logprobs is not None - and logprobs is not None): + if request.sampling_params.logprobs is not None and logprobs: # NOTE: once we support N tokens per step (spec decode), # the outer lists can be of length > 1. new_logprobs = logprobs.slice(req_index, req_index + 1) @@ -644,9 +641,7 @@ class Scheduler(SchedulerInterface): # should not be None if use_structured_output, we have # check above, so safe to ignore type warning request.structured_output_request.grammar.accept_tokens( # type: ignore[union-attr] - request.request_id, - new_token_ids, - ) + req_id, new_token_ids) # Get prompt logprobs for this request. prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id) @@ -665,7 +660,7 @@ class Scheduler(SchedulerInterface): # Invariant: EngineCore returns no partial prefill outputs. assert not prompt_logprobs_tensors - self.scheduled_req_ids.remove(request.request_id) + self.scheduled_req_ids.remove(req_id) if not stopped: new_running.append(request) diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index c41ee6704be0f..8858a564d2c2b 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -416,9 +416,9 @@ class SyncMPClient(MPClient): def process_outputs_socket(): shutdown_socket = ctx.socket(zmq.PAIR) - shutdown_socket.bind(shutdown_path) out_socket = make_zmq_socket(ctx, output_path, zmq.constants.PULL) try: + shutdown_socket.bind(shutdown_path) poller = zmq.Poller() poller.register(shutdown_socket) poller.register(out_socket) diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 1e67bed261182..70f072d3c9399 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -328,7 +328,7 @@ class OutputProcessor: # 2) Detokenize the token ids into text and perform stop checks. stop_string = req_state.detokenizer.update( new_token_ids, finish_reason == FinishReason.STOP) - if stop_string and finish_reason != FinishReason.STOP: + if stop_string: finish_reason = FinishReason.STOP stop_reason = stop_string diff --git a/vllm/v1/request.py b/vllm/v1/request.py index efb5a54d12077..48e5132678c13 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -93,9 +93,11 @@ class Request: token_ids: Union[int, list[int]], ) -> None: if isinstance(token_ids, int): - token_ids = [token_ids] - self._output_token_ids.extend(token_ids) - self._all_token_ids.extend(token_ids) + self._output_token_ids.append(token_ids) + self._all_token_ids.append(token_ids) + else: + self._output_token_ids.extend(token_ids) + self._all_token_ids.extend(token_ids) @property def num_tokens(self) -> int: From 1286211f573586719d80e96ce1e618b620e61f56 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Fri, 28 Mar 2025 21:10:41 -0700 Subject: [PATCH 094/593] [Bugfix] LoRA V1: add and fix entrypoints tests (#15715) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath --- .../llm/test_generate_multiple_loras.py | 14 +++++++++++++- tests/entrypoints/openai/test_lora_adapters.py | 15 ++++++++++++++- vllm/entrypoints/openai/serving_models.py | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/llm/test_generate_multiple_loras.py b/tests/entrypoints/llm/test_generate_multiple_loras.py index 90e1d58141378..099af0f36088b 100644 --- a/tests/entrypoints/llm/test_generate_multiple_loras.py +++ b/tests/entrypoints/llm/test_generate_multiple_loras.py @@ -23,7 +23,19 @@ LORA_NAME = "typeof/zephyr-7b-beta-lora" @pytest.fixture(scope="module") -def llm(): +def monkeypatch_module(): + from _pytest.monkeypatch import MonkeyPatch + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() + + +@pytest.fixture(scope="module", params=[False, True]) +def llm(request, monkeypatch_module): + + use_v1 = request.param + monkeypatch_module.setenv('VLLM_USE_V1', '1' if use_v1 else '0') + # pytest caches the fixture so we use weakref.proxy to # enable garbage collection llm = LLM(model=MODEL_NAME, diff --git a/tests/entrypoints/openai/test_lora_adapters.py b/tests/entrypoints/openai/test_lora_adapters.py index 1a62157acc478..2fc08b47513e6 100644 --- a/tests/entrypoints/openai/test_lora_adapters.py +++ b/tests/entrypoints/openai/test_lora_adapters.py @@ -53,7 +53,20 @@ def zephyr_lora_files(): @pytest.fixture(scope="module") -def server_with_lora_modules_json(zephyr_lora_files): +def monkeypatch_module(): + from _pytest.monkeypatch import MonkeyPatch + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() + + +@pytest.fixture(scope="module", params=[False, True]) +def server_with_lora_modules_json(request, monkeypatch_module, + zephyr_lora_files): + + use_v1 = request.param + monkeypatch_module.setenv('VLLM_USE_V1', '1' if use_v1 else '0') + # Define the json format LoRA module configurations lora_module_1 = { "name": "zephyr-lora", diff --git a/vllm/entrypoints/openai/serving_models.py b/vllm/entrypoints/openai/serving_models.py index 38a66583022a2..7a68452efc653 100644 --- a/vllm/entrypoints/openai/serving_models.py +++ b/vllm/entrypoints/openai/serving_models.py @@ -162,7 +162,7 @@ class OpenAIServingModels: except BaseException as e: error_type = "BadRequestError" status_code = HTTPStatus.BAD_REQUEST - if isinstance(e, ValueError) and "No adapter found" in str(e): + if "No adapter found" in str(e): error_type = "NotFoundError" status_code = HTTPStatus.NOT_FOUND From 7a7992085b75fde8f6b9717f6be7859b390b9093 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Sat, 29 Mar 2025 00:10:45 -0400 Subject: [PATCH 095/593] [CI] Speed up V1 structured output tests (#15718) Signed-off-by: Russell Bryant --- .../llm/test_struct_output_generate.py | 222 +++++++----------- 1 file changed, 89 insertions(+), 133 deletions(-) diff --git a/tests/v1/entrypoints/llm/test_struct_output_generate.py b/tests/v1/entrypoints/llm/test_struct_output_generate.py index c9fa03a1ae1fb..a32dd8263992e 100644 --- a/tests/v1/entrypoints/llm/test_struct_output_generate.py +++ b/tests/v1/entrypoints/llm/test_struct_output_generate.py @@ -23,20 +23,46 @@ MODELS_TO_TEST = [ ] +class CarType(str, Enum): + sedan = "sedan" + suv = "SUV" + truck = "Truck" + coupe = "Coupe" + + +class CarDescription(BaseModel): + brand: str + model: str + car_type: CarType + + @pytest.mark.skip_global_cleanup @pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS_V1) @pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_json_completion( +def test_structured_output( monkeypatch: pytest.MonkeyPatch, sample_json_schema: dict[str, Any], + unsupported_json_schema: dict[str, Any], + sample_sql_ebnf: str, + sample_sql_lark: str, + sample_regex: str, + sample_guided_choice: str, guided_decoding_backend: str, model_name: str, ): monkeypatch.setenv("VLLM_USE_V1", "1") + + # Use a single LLM instance for several scenarios to + # speed up the test suite. llm = LLM(model=model_name, + enforce_eager=True, max_model_len=1024, guided_decoding_backend=guided_decoding_backend) + + # + # Test 1: Generate JSON output based on a provided schema + # sampling_params = SamplingParams( temperature=1.0, max_tokens=1000, @@ -63,20 +89,9 @@ def test_guided_json_completion( output_json = json.loads(generated_text) jsonschema.validate(instance=output_json, schema=sample_json_schema) - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_json_object( - monkeypatch: pytest.MonkeyPatch, - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 2: Generate JSON object without a schema + # sampling_params = SamplingParams( temperature=1.0, max_tokens=100, @@ -111,21 +126,9 @@ def test_guided_json_object( allowed_types = (dict, list) assert isinstance(parsed_json, allowed_types) - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1 + ["auto"]) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_json_unsupported_schema( - monkeypatch: pytest.MonkeyPatch, - unsupported_json_schema: dict[str, Any], - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 3: test a jsonschema incompatible with xgrammar + # sampling_params = SamplingParams( temperature=1.0, max_tokens=1000, @@ -141,8 +144,6 @@ def test_guided_json_unsupported_schema( sampling_params=sampling_params, use_tqdm=True) else: - # This should work for both "guidance" and "auto". - outputs = llm.generate( prompts=("Give an example JSON object for a grade " "that fits this schema: " @@ -161,21 +162,9 @@ def test_guided_json_unsupported_schema( parsed_json = json.loads(generated_text) assert isinstance(parsed_json, dict) - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_grammar_ebnf( - monkeypatch: pytest.MonkeyPatch, - sample_sql_ebnf: str, - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 4: Generate SQL statement using EBNF grammar + # sampling_params = SamplingParams( temperature=0.8, top_p=0.95, @@ -205,21 +194,9 @@ def test_guided_grammar_ebnf( print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_grammar_lark( - monkeypatch: pytest.MonkeyPatch, - sample_sql_lark: str, - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 5: Generate SQL statement using Lark grammar + # sampling_params = SamplingParams( temperature=0.8, top_p=0.95, @@ -254,20 +231,9 @@ def test_guided_grammar_lark( print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_grammar_ebnf_invalid( - monkeypatch: pytest.MonkeyPatch, - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 6: Test invalid grammar input + # sampling_params = SamplingParams( temperature=0.8, top_p=0.95, @@ -281,21 +247,9 @@ def test_guided_grammar_ebnf_invalid( use_tqdm=True, ) - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_regex( - monkeypatch: pytest.MonkeyPatch, - sample_regex: str, - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 7: Generate text based on a regex pattern + # sampling_params = SamplingParams( temperature=0.8, top_p=0.95, @@ -319,21 +273,9 @@ def test_guided_regex( assert re.fullmatch(sample_regex, generated_text) is not None print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_choice_completion( - monkeypatch: pytest.MonkeyPatch, - sample_guided_choice: str, - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 8: Generate text based on a choices + # sampling_params = SamplingParams( temperature=0.8, top_p=0.95, @@ -353,33 +295,9 @@ def test_guided_choice_completion( assert generated_text in sample_guided_choice print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") - -class CarType(str, Enum): - sedan = "sedan" - suv = "SUV" - truck = "Truck" - coupe = "Coupe" - - -class CarDescription(BaseModel): - brand: str - model: str - car_type: CarType - - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("guided_decoding_backend", - GUIDED_DECODING_BACKENDS_V1) -@pytest.mark.parametrize("model_name", MODELS_TO_TEST) -def test_guided_json_completion_with_enum( - monkeypatch: pytest.MonkeyPatch, - guided_decoding_backend: str, - model_name: str, -): - monkeypatch.setenv("VLLM_USE_V1", "1") - llm = LLM(model=model_name, - max_model_len=1024, - guided_decoding_backend=guided_decoding_backend) + # + # Test 9: Generate structured output using a Pydantic model with an enum + # json_schema = CarDescription.model_json_schema() sampling_params = SamplingParams( temperature=1.0, @@ -403,3 +321,41 @@ def test_guided_json_completion_with_enum( print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") output_json = json.loads(generated_text) jsonschema.validate(instance=output_json, schema=json_schema) + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("model_name", MODELS_TO_TEST) +def test_structured_output_auto_mode( + monkeypatch: pytest.MonkeyPatch, + unsupported_json_schema: dict[str, Any], + model_name: str, +): + monkeypatch.setenv("VLLM_USE_V1", "1") + + llm = LLM(model=model_name, + max_model_len=1024, + guided_decoding_backend="auto") + + sampling_params = SamplingParams( + temperature=1.0, + max_tokens=1000, + guided_decoding=GuidedDecodingParams(json=unsupported_json_schema)) + + # This would fail with the default of "xgrammar", but in "auto" + # we will handle fallback automatically. + outputs = llm.generate(prompts=("Give an example JSON object for a grade " + "that fits this schema: " + f"{unsupported_json_schema}"), + sampling_params=sampling_params, + use_tqdm=True) + assert outputs is not None + for output in outputs: + assert output is not None + assert isinstance(output, RequestOutput) + generated_text = output.outputs[0].text + assert generated_text is not None + print(generated_text) + + # Parse to verify it is valid JSON + parsed_json = json.loads(generated_text) + assert isinstance(parsed_json, dict) From 8427f70493ed67bf26cb9e7fa98ac202b991c37d Mon Sep 17 00:00:00 2001 From: cyyever Date: Sat, 29 Mar 2025 12:11:51 +0800 Subject: [PATCH 096/593] Use numba 0.61 for python 3.10+ to support numpy>=2 (#15692) Signed-off-by: cyy --- requirements/common.txt | 2 +- requirements/cuda.txt | 3 ++- requirements/rocm.txt | 3 ++- requirements/test.in | 4 +++- requirements/test.txt | 8 +++++--- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/requirements/common.txt b/requirements/common.txt index 14084b79121bb..dfa20f5e3f08e 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,7 +1,7 @@ cachetools psutil sentencepiece # Required for LLaMA tokenizer. -numpy < 2.0.0 +numpy requests >= 2.26.0 tqdm blake3 diff --git a/requirements/cuda.txt b/requirements/cuda.txt index ad7198081e0fa..9be7a868f56e0 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -1,7 +1,8 @@ # Common dependencies -r common.txt -numba == 0.60.0 # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding +numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding +numba == 0.61; python_version > '3.9' # Dependencies for NVIDIA GPUs ray[cgraph]>=2.43.0, !=2.44.* # Ray Compiled Graph, required for pipeline parallelism in V1. diff --git a/requirements/rocm.txt b/requirements/rocm.txt index 345c84b0f6cf2..5d5fea2d0e57e 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -1,7 +1,8 @@ # Common dependencies -r common.txt -numba == 0.60.0 # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding +numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding +numba == 0.61; python_version > '3.9' # Dependencies for AMD GPUs awscli diff --git a/requirements/test.in b/requirements/test.in index 3df5e32cd59e1..a7dd54151dee8 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -38,7 +38,9 @@ buildkite-test-collector==0.1.9 genai_perf==0.0.8 tritonclient==2.51.0 -numpy < 2.0.0 +numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding +numba == 0.61; python_version > '3.9' +numpy runai-model-streamer==0.11.0 runai-model-streamer-s3==0.11.0 fastsafetensors>=0.1.10 diff --git a/requirements/test.txt b/requirements/test.txt index b0ae479604a1e..aed6a5653e2ad 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -219,7 +219,7 @@ libnacl==2.1.0 # via tensorizer librosa==0.10.2.post1 # via -r requirements/test.in -llvmlite==0.43.0 +llvmlite==0.44.0 # via numba lm-eval==0.4.4 # via -r requirements/test.in @@ -262,8 +262,10 @@ networkx==3.2.1 # via torch nltk==3.9.1 # via rouge-score -numba==0.60.0 - # via librosa +numba==0.61.0 + # via + # -r requirements/test.in + # librosa numexpr==2.10.1 # via lm-eval numpy==1.26.4 From 5b800f0932c0d6661cb3aa85a62d89265197a034 Mon Sep 17 00:00:00 2001 From: Jinzhen Lin Date: Sat, 29 Mar 2025 12:12:26 +0800 Subject: [PATCH 097/593] [Bugfix] set VLLM_WORKER_MULTIPROC_METHOD=spawn for vllm.entrypoionts.openai.api_server (#15700) Signed-off-by: Jinzhen Lin --- vllm/entrypoints/cli/main.py | 28 ++------------------------- vllm/entrypoints/openai/api_server.py | 4 +++- vllm/entrypoints/utils.py | 26 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/vllm/entrypoints/cli/main.py b/vllm/entrypoints/cli/main.py index 13f2761b0db06..aa54bd66bed67 100644 --- a/vllm/entrypoints/cli/main.py +++ b/vllm/entrypoints/cli/main.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # The CLI entrypoint to vLLM. -import os import signal import sys @@ -9,11 +8,9 @@ import vllm.entrypoints.cli.benchmark.main import vllm.entrypoints.cli.openai import vllm.entrypoints.cli.serve import vllm.version -from vllm.logger import init_logger +from vllm.entrypoints.utils import cli_env_setup from vllm.utils import FlexibleArgumentParser -logger = init_logger(__name__) - CMD_MODULES = [ vllm.entrypoints.cli.openai, vllm.entrypoints.cli.serve, @@ -30,29 +27,8 @@ def register_signal_handlers(): signal.signal(signal.SIGTSTP, signal_handler) -def env_setup(): - # The safest multiprocessing method is `spawn`, as the default `fork` method - # is not compatible with some accelerators. The default method will be - # changing in future versions of Python, so we should use it explicitly when - # possible. - # - # We only set it here in the CLI entrypoint, because changing to `spawn` - # could break some existing code using vLLM as a library. `spawn` will cause - # unexpected behavior if the code is not protected by - # `if __name__ == "__main__":`. - # - # References: - # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods - # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing - # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors - # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders - if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ: - logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'") - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - - def main(): - env_setup() + cli_env_setup() parser = FlexibleArgumentParser(description="vLLM CLI") parser.add_argument('-v', diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 18d75a04ab0f3..2a61259896a37 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -82,7 +82,8 @@ from vllm.entrypoints.openai.serving_tokenization import ( from vllm.entrypoints.openai.serving_transcription import ( OpenAIServingTranscription) from vllm.entrypoints.openai.tool_parsers import ToolParserManager -from vllm.entrypoints.utils import load_aware_call, with_cancellation +from vllm.entrypoints.utils import (cli_env_setup, load_aware_call, + with_cancellation) from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager from vllm.transformers_utils.config import ( @@ -1106,6 +1107,7 @@ if __name__ == "__main__": # NOTE(simon): # This section should be in sync with vllm/entrypoints/cli/main.py for CLI # entrypoints. + cli_env_setup() parser = FlexibleArgumentParser( description="vLLM OpenAI-Compatible RESTful API server.") parser = make_arg_parser(parser) diff --git a/vllm/entrypoints/utils.py b/vllm/entrypoints/utils.py index 773f52fa38f88..b88c2b3a080fd 100644 --- a/vllm/entrypoints/utils.py +++ b/vllm/entrypoints/utils.py @@ -2,11 +2,16 @@ import asyncio import functools +import os from fastapi import Request from fastapi.responses import JSONResponse, StreamingResponse from starlette.background import BackgroundTask, BackgroundTasks +from vllm.logger import init_logger + +logger = init_logger(__name__) + async def listen_for_disconnect(request: Request) -> None: """Returns if a disconnect message is received""" @@ -108,3 +113,24 @@ def load_aware_call(func): return response return wrapper + + +def cli_env_setup(): + # The safest multiprocessing method is `spawn`, as the default `fork` method + # is not compatible with some accelerators. The default method will be + # changing in future versions of Python, so we should use it explicitly when + # possible. + # + # We only set it here in the CLI entrypoint, because changing to `spawn` + # could break some existing code using vLLM as a library. `spawn` will cause + # unexpected behavior if the code is not protected by + # `if __name__ == "__main__":`. + # + # References: + # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods + # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing + # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors + # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders + if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ: + logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'") + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" From da461f3cbf8be4094a6f14a1eaf89b5931f3625f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Sat, 29 Mar 2025 05:13:06 +0100 Subject: [PATCH 098/593] [TPU][V1][Bugfix] Fix w8a8 recompiilation with GSM8K (#15714) Signed-off-by: NickLucche --- .buildkite/run-tpu-v1-test.sh | 10 ++++------ .../layers/quantization/kernels/scaled_mm/xla.py | 3 ++- vllm/v1/worker/tpu_model_runner.py | 14 ++++++++------ vllm/v1/worker/tpu_worker.py | 4 ++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.buildkite/run-tpu-v1-test.sh b/.buildkite/run-tpu-v1-test.sh index 2c356b8fe5274..89252000f4003 100755 --- a/.buildkite/run-tpu-v1-test.sh +++ b/.buildkite/run-tpu-v1-test.sh @@ -28,16 +28,14 @@ docker run --privileged --net host --shm-size=16G -it \ && echo TEST_3 \ && pytest -v -s /workspace/vllm/tests/entrypoints/llm/test_accuracy.py::test_lm_eval_accuracy_v1_engine \ && echo TEST_4 \ - && python3 /workspace/vllm/examples/offline_inference/tpu.py \ + && pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py \ && echo TEST_5 \ - && pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py \ + && python3 /workspace/vllm/examples/offline_inference/tpu.py \ && echo TEST_6 \ + && pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py \ + && echo TEST_7 \ && pytest -s -v /workspace/vllm/tests/v1/tpu/test_sampler.py" \ # TODO: This test fails because it uses RANDOM_SEED sampling # && VLLM_USE_V1=1 pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \ - -# TODO: Re-enable this after fixing recompilation in quantization. -# && echo TEST_4 \ -# && pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py \ diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/xla.py b/vllm/model_executor/layers/quantization/kernels/scaled_mm/xla.py index 0bf090d7fab3c..089314071d39e 100644 --- a/vllm/model_executor/layers/quantization/kernels/scaled_mm/xla.py +++ b/vllm/model_executor/layers/quantization/kernels/scaled_mm/xla.py @@ -97,7 +97,8 @@ class XLAScaledMMLinearKernel(ScaledMMLinearKernel): block_size=-1, int4_weight=False, quantize_activation=True) - + # `quantized_matmul` output is fp32, cast it down to bf16 for perf + out = out.to(x.dtype) # Explicitly capture control flow to make dynamo happy. # https://pytorch.org/docs/main/generated/exportdb/index.html#cond-branch-class-method # noqa: E501 return cond(bias is None, self.no_add_bias, self.add_bias, [out, bias]) diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index 695e31f715b4d..773cd971103ae 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -80,6 +80,7 @@ class TPUModelRunner: self.enforce_eager = model_config.enforce_eager self.pin_memory = is_pin_memory_available() self.dtype = self.model_config.dtype + self._hidden_states_dtype = self.dtype self.is_multimodal_model = model_config.is_multimodal_model self.sliding_window = model_config.get_sliding_window() @@ -771,10 +772,11 @@ class TPUModelRunner: torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0) with set_forward_context(attn_metadata, self.vllm_config, 0): - self.model(input_ids=input_ids, - positions=position_ids, - kv_caches=kv_caches, - inputs_embeds=inputs_embeds) + out = self.model(input_ids=input_ids, + positions=position_ids, + kv_caches=kv_caches, + inputs_embeds=inputs_embeds) + self._hidden_states_dtype = out.dtype def capture_model(self) -> None: """Compile the model.""" @@ -800,7 +802,7 @@ class TPUModelRunner: num_reqs_to_sample = MIN_NUM_SEQS dummy_hidden = torch.randn((num_tokens, hsize), device=device, - dtype=torch.bfloat16) + dtype=self._hidden_states_dtype) # Compile for [8, 16, .., 128,.., `self.max_num_reqs`] while True: indices = torch.zeros( @@ -823,7 +825,7 @@ class TPUModelRunner: num_reqs_to_sample + 1, self.max_num_reqs) xm.wait_device_ops() end = time.perf_counter() - logger.info("Compilation finished in in %.2f [secs].", end - start) + logger.info("Compilation finished in %.2f [secs].", end - start) # Record the number cached XLA graph after warming up, this will be # used for checking there is no additional graph compilation during # runtime execution. diff --git a/vllm/v1/worker/tpu_worker.py b/vllm/v1/worker/tpu_worker.py index c8691ee87fe6a..b51bd20f6f118 100644 --- a/vllm/v1/worker/tpu_worker.py +++ b/vllm/v1/worker/tpu_worker.py @@ -105,8 +105,8 @@ class TPUWorker: # Increase the cache size limit, which is the maximum number of # dynamo graphs that can be compiled. - # NOTE(woosuk): Usually, we compile 10-15 graphs for prefill and - # 30-40 graphs for decode. 128 is an arbitrary safe number. + # TODO (NickLucche) On gsm we compile 80+ graphs. + # Re-evaluate limit, with MM we may get close to this limit. torch._dynamo.config.cache_size_limit = 128 # Use persistent cache to avoid XLA recompilation. # NOTE(woosuk): Set per-rank cache path since different ranks From 7c1f7600248a0a0497a5c512ef0ee262577c5f7a Mon Sep 17 00:00:00 2001 From: yarongmu-google <150371854+yarongmu-google@users.noreply.github.com> Date: Fri, 28 Mar 2025 21:13:15 -0700 Subject: [PATCH 099/593] [Kernel][TPU][ragged-paged-attn] vLLM code change for PR#8896 (#15659) Signed-off-by: Yarong Mu --- requirements/tpu.txt | 12 ++++---- vllm/v1/attention/backends/pallas.py | 43 ++++++++++++++-------------- vllm/v1/worker/tpu_model_runner.py | 11 ++++--- vllm/v1/worker/tpu_worker.py | 8 +++--- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/requirements/tpu.txt b/requirements/tpu.txt index 35d5db6c46006..1930eacb61ad6 100644 --- a/requirements/tpu.txt +++ b/requirements/tpu.txt @@ -17,9 +17,9 @@ ray[data] --find-links https://storage.googleapis.com/libtpu-releases/index.html --find-links https://storage.googleapis.com/jax-releases/jax_nightly_releases.html --find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html -torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250319-cp39-cp39-linux_x86_64.whl ; python_version == "3.9" -torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250319-cp310-cp310-linux_x86_64.whl ; python_version == "3.10" -torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250319-cp311-cp311-linux_x86_64.whl ; python_version == "3.11" -torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250319-cp39-cp39-linux_x86_64.whl ; python_version == "3.9" -torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250319-cp310-cp310-linux_x86_64.whl ; python_version == "3.10" -torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250319-cp311-cp311-linux_x86_64.whl ; python_version == "3.11" +torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250328-cp39-cp39-linux_x86_64.whl ; python_version == "3.9" +torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250328-cp310-cp310-linux_x86_64.whl ; python_version == "3.10" +torch @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-2.8.0.dev20250328-cp311-cp311-linux_x86_64.whl ; python_version == "3.11" +torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250328-cp39-cp39-linux_x86_64.whl ; python_version == "3.9" +torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250328-cp310-cp310-linux_x86_64.whl ; python_version == "3.10" +torch_xla[tpu, pallas] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.8.0.dev20250328-cp311-cp311-linux_x86_64.whl ; python_version == "3.11" diff --git a/vllm/v1/attention/backends/pallas.py b/vllm/v1/attention/backends/pallas.py index 14d3664db0d64..2f86920e2773a 100644 --- a/vllm/v1/attention/backends/pallas.py +++ b/vllm/v1/attention/backends/pallas.py @@ -41,7 +41,7 @@ class PallasAttentionBackend(AttentionBackend): num_kv_heads: int, head_size: int, ) -> tuple[int, ...]: - return (num_blocks, block_size, num_kv_heads * head_size) + return (num_blocks, block_size, num_kv_heads * 2, head_size) @staticmethod def swap_blocks( @@ -132,7 +132,7 @@ class PallasAttentionBackendImpl(AttentionImpl): query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - kv_cache: tuple[torch.Tensor, torch.Tensor], + kv_cache: torch.Tensor, attn_metadata: PallasMetadata, output: Optional[torch.Tensor] = None, ) -> torch.Tensor: @@ -142,14 +142,13 @@ class PallasAttentionBackendImpl(AttentionImpl): query: shape = [num_tokens, num_heads * head_size] key: shape = [num_tokens, num_kv_heads * head_size] value: shape = [num_tokens, num_kv_heads * head_size] - kv_cache = ([num_blocks, block_size, num_kv_heads * head_size], - [num_blocks, block_size, num_kv_heads * head_size]) + kv_cache = [num_blocks, block_size, num_kv_heads * 2, head_size] attn_metadata: Metadata for attention. Returns: shape = [num_tokens, num_heads * head_size] """ # For determine_available_memory case. - if kv_cache[0].numel() == 0: + if kv_cache.numel() == 0: if output is None: output = torch.ones_like(query) return output @@ -158,15 +157,13 @@ class PallasAttentionBackendImpl(AttentionImpl): num_tokens, hidden_size = query.shape query = query.view(num_tokens, self.num_heads, self.head_size) - key_cache, value_cache = kv_cache - if kv_cache[0].numel() > 0: + if kv_cache.numel() > 0: slot_mapping = attn_metadata.slot_mapping - write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping) + write_to_kv_cache(key, value, kv_cache, slot_mapping) output = torch.ops.xla.ragged_paged_attention( query, - key_cache, - value_cache, + kv_cache, attn_metadata.context_lens, attn_metadata.block_tables, attn_metadata.query_start_loc, @@ -183,23 +180,27 @@ class PallasAttentionBackendImpl(AttentionImpl): def write_to_kv_cache( key: torch.Tensor, value: torch.Tensor, - key_cache: torch.Tensor, - value_cache: torch.Tensor, + kv_cache: torch.Tensor, slot_mapping: torch.Tensor, ) -> None: """ Write the key and values to the KV cache. Args: key: shape = [num_tokens, num_kv_heads * head_size] - value: shape = [num_tokens, num_kv_heads * head_size] - k_cache = [num_blocks, block_size, num_kv_heads * head_size] - v_cache = [num_blocks, block_size, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + kv_cache = [num_blocks, block_size, num_kv_heads * 2, head_size] """ - torch.ops.xla.dynamo_set_buffer_donor_(key_cache, True) - torch.ops.xla.dynamo_set_buffer_donor_(value_cache, True) + _, _, num_combined_kv_heads, head_size = kv_cache.shape + num_kv_heads = num_combined_kv_heads // 2 - key_cache = key_cache.flatten(0, 1) - value_cache = value_cache.flatten(0, 1) - key_cache.index_copy_(0, slot_mapping, key) - value_cache.index_copy_(0, slot_mapping, value) + key = key.view(-1, num_kv_heads, head_size) + value = value.view(-1, num_kv_heads, head_size) + + kv = torch.cat([key, value], axis=-1).reshape(-1, num_combined_kv_heads, + head_size) + + torch.ops.xla.dynamo_set_buffer_donor_(kv_cache, True) + + kv_cache = kv_cache.flatten(0, 1) + kv_cache.index_copy_(0, slot_mapping, kv) diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index 773cd971103ae..ea5a17016eb6b 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -861,12 +861,11 @@ class TPUModelRunner: kv_cache_spec.num_kv_heads, kv_cache_spec.head_size) dtype = kv_cache_spec.dtype - tpu_k_cache = torch.zeros(kv_cache_shape, - dtype=dtype, - device=self.device) - tpu_v_cache = torch.zeros_like(tpu_k_cache) + tpu_kv_cache = torch.zeros(kv_cache_shape, + dtype=dtype, + device=self.device) - kv_caches[layer_name] = (tpu_k_cache, tpu_v_cache) + kv_caches[layer_name] = tpu_kv_cache else: raise NotImplementedError @@ -893,7 +892,7 @@ class ModelWrapperV1(nn.Module): self, input_ids: torch.Tensor, positions: torch.Tensor, - kv_caches: list[tuple[torch.Tensor, torch.Tensor]], + kv_caches: list[torch.Tensor], inputs_embeds: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Executes the forward pass of the model. diff --git a/vllm/v1/worker/tpu_worker.py b/vllm/v1/worker/tpu_worker.py index b51bd20f6f118..9add8cee02e5b 100644 --- a/vllm/v1/worker/tpu_worker.py +++ b/vllm/v1/worker/tpu_worker.py @@ -136,10 +136,10 @@ class TPUWorker: # Use an empty tensor instead of `None`` to force Dynamo to pass # it by reference, rather by specializing on the value ``None``. - tpu_k_cache = torch.tensor([], dtype=dtype, device=self.device) - tpu_v_cache = torch.tensor([], dtype=dtype, device=self.device) - - kv_caches[layer_name] = (tpu_k_cache, tpu_v_cache) + tpu_kv_cache = torch.tensor([], + dtype=dtype, + device=self.device) + kv_caches[layer_name] = tpu_kv_cache else: raise NotImplementedError From 73aa7041bfee43581314e6f34e9a657137ecc092 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Sat, 29 Mar 2025 12:27:22 +0800 Subject: [PATCH 100/593] [doc] update doc (#15740) Signed-off-by: reidliu41 Co-authored-by: reidliu41 --- docs/README.md | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/README.md b/docs/README.md index 74e05ce02636b..dcd5e759dfa88 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,19 +2,42 @@ ## Build the docs -```bash -# Install dependencies. -pip install -r ../requirements/docs.txt +- Make sure in `docs` directory -# Build the docs. +```bash +cd docs +``` + +- Install the dependencies: + +```bash +pip install -r ../requirements/docs.txt +``` + +- Clean the previous build (optional but recommended): + +```bash make clean +``` + +- Generate the HTML documentation: + +```bash make html ``` ## Open the docs with your browser +- Serve the documentation locally: + ```bash python -m http.server -d build/html/ ``` -Launch your browser and open localhost:8000. +This will start a local server at http://localhost:8000. You can now open your browser and view the documentation. + +If port 8000 is already in use, you can specify a different port, for example: + +```bash +python -m http.server 3000 -d build/html/ +``` From 4965ec42d28830f0c30756dea19e14b45cdbe5b1 Mon Sep 17 00:00:00 2001 From: TJian Date: Sat, 29 Mar 2025 18:33:56 +0800 Subject: [PATCH 101/593] [FEAT] [ROCm] Add AITER int8 scaled gemm kernel (#15433) Signed-off-by: tjtanaa --- tests/quantization/test_compressed_tensors.py | 76 ++++++++++- vllm/envs.py | 8 ++ .../kernels/scaled_mm/__init__.py | 4 +- .../quantization/kernels/scaled_mm/aiter.py | 119 ++++++++++++++++++ 4 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 vllm/model_executor/layers/quantization/kernels/scaled_mm/aiter.py diff --git a/tests/quantization/test_compressed_tensors.py b/tests/quantization/test_compressed_tensors.py index 133475a3e06aa..5c928f27c10dd 100644 --- a/tests/quantization/test_compressed_tensors.py +++ b/tests/quantization/test_compressed_tensors.py @@ -20,6 +20,23 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( sparse_cutlass_supported) from vllm.platforms import current_platform +# AITER only supports per-channel-per-channel INT8 gemm +# and per-tensor-per-tensor INT8 GEMM. +# It does not support mix precision MM and mix quantization scheme. +ROCM_AITER_SUPPORTED_INT8_MODEL = [ + "neuralmagic/Llama-3.2-1B-quantized.w8a8", + "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2" +] + +# TritonScaledMMLinearKernel only supports symmetric quantization. +ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL = [ + "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", + "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", + "neuralmagic/Llama-3.2-1B-quantized.w8a8", + "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", + "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2", +] + @pytest.fixture(scope="function", autouse=True) def use_v0_only(monkeypatch): @@ -57,6 +74,11 @@ def use_v0_only(monkeypatch): ) def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args): model_path, strategy, quant_type, shape_0, is_symmetric = model_args + + if current_platform.is_rocm( + ) and model_path not in ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL: + pytest.skip(f"Skip model {model_path} as it is not support on ROCm.") + with vllm_runner(model_path, enforce_eager=True) as llm: def check_model(model): @@ -123,6 +145,8 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args): ) @pytest.mark.parametrize("max_tokens", [32]) @pytest.mark.parametrize("num_logprobs", [10]) +@pytest.mark.parametrize( + "use_aiter", [True, False] if current_platform.is_rocm() else [False]) def test_compressed_tensors_w8a8_logprobs( hf_runner, vllm_runner, @@ -130,7 +154,21 @@ def test_compressed_tensors_w8a8_logprobs( model_path, max_tokens, num_logprobs, + use_aiter, + monkeypatch, ): + + if current_platform.is_rocm( + ) and model_path not in ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL: + pytest.skip(f"Skip model {model_path} as it is not support on ROCm.") + + if use_aiter: + if model_path not in ROCM_AITER_SUPPORTED_INT8_MODEL: + pytest.skip( + f"Skip model {model_path} as it is not support by aiter.") + # this will enable VLLM_ROCM_USE_AITER_LINEAR + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + dtype = "bfloat16" # skip language translation prompt for the static per tensor asym model @@ -154,6 +192,9 @@ def test_compressed_tensors_w8a8_logprobs( name_1="vllm", ) + if current_platform.is_rocm(): + torch.cuda.synchronize() + def test_compressed_tensors_no_enforce_eager(vllm_runner): model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change" @@ -177,8 +218,27 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner): ), ], ) -def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args): +@pytest.mark.parametrize( + "use_aiter", [True, False] if current_platform.is_rocm() else [False]) +def test_compressed_tensors_w8a8_dynamic_per_token( + vllm_runner, + model_args, + use_aiter, + monkeypatch, +): model_path, strategy = model_args + + if current_platform.is_rocm( + ) and model_path not in ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL: + pytest.skip(f"Skip model {model_path} as it is not support on ROCm.") + + if use_aiter: + if model_path not in ROCM_AITER_SUPPORTED_INT8_MODEL: + pytest.skip( + f"Skip model {model_path} as it is not support by aiter.") + # this will enable VLLM_ROCM_USE_AITER_LINEAR + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + with vllm_runner(model_path, dtype=torch.float16) as llm: def check_model(model): @@ -207,6 +267,8 @@ def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args): ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4), ], ) +@pytest.mark.skipif(not current_platform.is_cuda(), + reason="The tests are skipped on non-CUDA platform.") def test_compressed_tensors_wNa16(vllm_runner, wNa16_args): model, strategy, group, pack_factor = wNa16_args with vllm_runner(model) as llm: @@ -231,6 +293,8 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args): assert output +@pytest.mark.skipif(not current_platform.is_cuda(), + reason="This test is skipped on non-CUDA platform.") def test_compressed_tensors_w4a16_marlin24(vllm_runner): model_path = "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t" with vllm_runner(model_path) as llm: @@ -271,7 +335,7 @@ def test_compressed_tensors_fp8(vllm_runner): if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8): assert len(qkv_proj.input_scale.shape) == 0 - assert qkv_proj.weight.dtype is torch.float8_e4m3fn + assert qkv_proj.weight.dtype is current_platform.fp8_dtype() assert qkv_proj.weight_scale.dtype is torch.float32 assert len(qkv_proj.weight_scale.shape) == 0 @@ -281,6 +345,8 @@ def test_compressed_tensors_fp8(vllm_runner): assert output +@pytest.mark.skipif(not current_platform.is_cuda(), + reason="This test is skipped on non-CUDA platform.") def test_compressed_tensors_kv_cache(vllm_runner): model_path = "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme" with vllm_runner(model_path, kv_cache_dtype="fp8") as llm: @@ -309,7 +375,8 @@ def _test_2of4_quant_models(qkv_proj, @pytest.mark.skipif( - not current_platform.has_device_capability(90), + not current_platform.is_cuda() + or not current_platform.has_device_capability(90), reason="Sparse FP8 is not yet supported on this GPU type.", ) @pytest.mark.parametrize( @@ -356,7 +423,8 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4): @pytest.mark.skipif( - not current_platform.has_device_capability(90), + not current_platform.is_cuda() + or not current_platform.has_device_capability(90), reason="Sparse FP8 is not yet supported on this GPU type.", ) @pytest.mark.parametrize( diff --git a/vllm/envs.py b/vllm/envs.py index 5334667376b24..8a03ba329b028 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -75,6 +75,7 @@ if TYPE_CHECKING: VLLM_DISABLED_KERNELS: list[str] = [] VLLM_USE_V1: bool = True VLLM_ROCM_USE_AITER: bool = False + VLLM_ROCM_USE_AITER_LINEAR: bool = True VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_USE_AITER_FP8_BLOCK_SCALED_MOE: bool = False VLLM_ROCM_USE_AITER_RMSNORM: bool = True @@ -524,6 +525,13 @@ environment_variables: dict[str, Callable[[], Any]] = { lambda: (os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")), + # use aiter linear op if aiter ops are enabled + # The following list of related ops + # - scaled_mm (per-tensor / rowwise) + "VLLM_ROCM_USE_AITER_LINEAR": + lambda: (os.getenv("VLLM_ROCM_USE_AITER_LINEAR", "True").lower() in + ("true", "1")), + # Whether to use aiter moe ops. # By default is enabled. "VLLM_ROCM_USE_AITER_MOE": diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py b/vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py index a5967995ac88d..bedda4c2ab21b 100644 --- a/vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py +++ b/vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py @@ -3,6 +3,8 @@ import os from typing import Dict, List, Optional, Type +from vllm.model_executor.layers.quantization.kernels.scaled_mm.aiter import ( + AiterScaledMMLinearKernel) from vllm.model_executor.layers.quantization.kernels.scaled_mm.cutlass import ( CutlassScaledMMLinearKernel) from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501 @@ -17,7 +19,7 @@ from vllm.platforms import PlatformEnum, current_platform _POSSIBLE_KERNELS: Dict[PlatformEnum, List[Type[ScaledMMLinearKernel]]] = { PlatformEnum.CPU: [CutlassScaledMMLinearKernel], PlatformEnum.CUDA: [CutlassScaledMMLinearKernel], - PlatformEnum.ROCM: [TritonScaledMMLinearKernel], + PlatformEnum.ROCM: [AiterScaledMMLinearKernel, TritonScaledMMLinearKernel], PlatformEnum.TPU: [XLAScaledMMLinearKernel], } diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/aiter.py b/vllm/model_executor/layers/quantization/kernels/scaled_mm/aiter.py new file mode 100644 index 0000000000000..582b12f76562c --- /dev/null +++ b/vllm/model_executor/layers/quantization/kernels/scaled_mm/aiter.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional, Tuple + +import torch + +import vllm.envs as envs +from vllm import _custom_ops as ops +from vllm.platforms import current_platform + +from .cutlass import CutlassScaledMMLinearKernel +from .ScaledMMLinearKernel import ScaledMMLinearLayerConfig + + +class AiterScaledMMLinearKernel(CutlassScaledMMLinearKernel): + + @classmethod + def get_min_capability(cls) -> int: + return 90 + + @classmethod + def can_implement( + cls, c: ScaledMMLinearLayerConfig) -> Tuple[bool, Optional[str]]: + if not current_platform.is_rocm(): + return ( + False, + "AiterScaledMMLinearKernel requires `aiter` which is not " + + "currently supported on non-ROCm platform.") + + try: + import aiter # noqa: F401 # deliberately attempt to import aiter + except Exception: + return ( + False, + "AiterScaledMMLinearKernel requires `aiter` which is not " + + "installed on ROCm.") + # Check if rocm_aiter_gemm_w8a8_scaled_mm is enabled + if not ( + envs.VLLM_ROCM_USE_AITER_LINEAR \ + and envs.VLLM_ROCM_USE_AITER + ): + return (False, "AiterScaledMMLinearKernel is disabled. " + + "Enable by setting `VLLM_ROCM_USE_AITER=1` " + + "and `VLLM_ROCM_USE_AITER_LINEAR=1`. " + + "`VLLM_ROCM_USE_AITER_LINEAR` default is True.") + + if not c.input_symmetric: + return (False, + "AiterScaledMMLinearKernel only supports symmetric " + + "quantization.") + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + super().process_weights_after_loading(layer) + + def apply_weights(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + """ + `AiterScaledMMLinearKernel` implements a fused version of + `output = torch.mm((scale_a * a), (scale_b * b)).to(out_dtype)` + where scale_a * a and scale_b * b are implemented using numpy-style + broadcasting. + Currently only support per-tensor-per-tensor GEMM + and per-token-per-channel GEMM through AITER + w8a8 scaled gemm. `AiterScaledMMLinearKernel` also does not support + ATIER block scaled GEMM and mix-precision GEMM. + """ + w_q, w_s, i_s, i_zp, azp_adj = self._get_weight_params(layer) + + # ops.scaled_int8_quant supports both dynamic and static quant: + # * dynamic, i_s is None and x_s computed from x. + # * static, i_s is scalar and x_s is i_s. + symmetric = azp_adj is None + assert symmetric, ("AiterScaledMMLinearKernel only supports" + " symmetric quantization.") + x_q, x_s, x_zp = ops.scaled_int8_quant(x, + i_s, + i_zp, + symmetric=symmetric) + + assert x_zp is None, ("AiterScaledMMLinearKernel only supports" + " symmetric quantization.") + out_dtype = x.dtype + + assert (w_q.shape[0] % 16 == 0 and w_q.shape[1] % 16 == 0) + assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16) + assert bias is None or bias.shape[0] == w_q.shape[ + 1] and bias.dtype == out_dtype + + m = x_q.shape[0] # a + n = w_q.shape[1] # b + + per_tensor_scale_a = (x_s.numel() == 1) + per_tensor_scale_b = (w_s.numel() == 1) + per_token_scale_a = (x_s.numel() == m) + per_channel_scale_b = (w_s.numel() == n) + + # @TODO: + # Maybe broadcast the per-tensor-scale into per-channel-scale + # if one of the scale is a per-channel-scale. + # For now, it only supports: + # - per-tensor-per-tensor a8w8 scaled GEMM, and + # - per-token-per-channel a8w8 scaled GEMM + assert ((per_tensor_scale_a and per_tensor_scale_b) + or (per_token_scale_a and per_channel_scale_b)), ( + "Currently only support per-tensor-per-tensor GEMM " + + " and per-token-per-channel GEMM through AITER" + " w8a8 scaled gemm. `AiterScaledMMLinearKernel` " + + "does not support AITER block scaled GEMM.") + + from aiter import gemm_a8w8_CK + + # gemm_a8w8_CK(a, b, scale_a, scale_b, bias) expects + # a to be [M, K] + # b to be [N, K] + # CutlassScaledMMLinearKernel prepare weight `w_q` in [K, N] format + return gemm_a8w8_CK(x_q, w_q.t(), x_s, w_s, bias).to(out_dtype) From 94744ba41a2807cb195e4a41a85d4d49f6867967 Mon Sep 17 00:00:00 2001 From: wwl2755 Date: Sat, 29 Mar 2025 05:39:14 -0500 Subject: [PATCH 102/593] [V1] [Feature] Collective RPC (#15444) Signed-off-by: wwl2755 --- .buildkite/test-pipeline.yaml | 6 ++--- vllm/engine/llm_engine.py | 13 +++++++++-- vllm/entrypoints/llm.py | 4 ++-- vllm/v1/engine/core.py | 12 +++++++++- vllm/v1/engine/core_client.py | 43 ++++++++++++++++++++++++++++++++++- vllm/v1/engine/llm_engine.py | 10 +++++++- vllm/v1/serial_utils.py | 8 +++++++ 7 files changed, 86 insertions(+), 10 deletions(-) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 428b4c593c38e..62872bf8e3e18 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -150,8 +150,8 @@ steps: # TODO: create a dedicated test section for multi-GPU example tests # when we have multiple distributed example tests - pushd ../examples/offline_inference - - VLLM_ENABLE_V1_MULTIPROCESSING=0 python3 rlhf.py - - VLLM_ENABLE_V1_MULTIPROCESSING=0 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py + - python3 rlhf.py + - RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py - popd - label: Metrics, Tracing Test # 10min @@ -520,7 +520,7 @@ steps: - vllm/v1/engine/ commands: - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/test_async_llm_dp.py - - VLLM_ENABLE_V1_MULTIPROCESSING=0 pytest -v -s entrypoints/llm/test_collective_rpc.py + - pytest -v -s entrypoints/llm/test_collective_rpc.py - pytest -v -s ./compile/test_basic_correctness.py - pytest -v -s ./compile/test_wrapper.py - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py index 5682b3dabe2e8..10677878ecc8f 100644 --- a/vllm/engine/llm_engine.py +++ b/vllm/engine/llm_engine.py @@ -7,8 +7,8 @@ from collections import deque from contextlib import contextmanager from dataclasses import dataclass from functools import partial -from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable, - List, Mapping, NamedTuple, Optional) +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict, + Iterable, List, Mapping, NamedTuple, Optional) from typing import Sequence as GenericSequence from typing import Set, Type, Union, cast, overload @@ -67,6 +67,7 @@ _LOCAL_LOGGING_INTERVAL_SEC = 5 _G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup) _O = TypeVar("_O", RequestOutput, PoolingRequestOutput) +_R = TypeVar("_R", default=Any) @dataclass @@ -2123,6 +2124,14 @@ class LLMEngine: return sampling_params + def collective_rpc(self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + return self.model_executor.collective_rpc(method, timeout, args, + kwargs) + if envs.is_set("VLLM_USE_V1") and envs.VLLM_USE_V1: from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 1887caf25a30f..7c354be2d45c5 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -492,8 +492,8 @@ class LLM: It is recommended to use this API to only pass control messages, and set up data-plane communication to pass data. """ - executor = self.llm_engine.model_executor - return executor.collective_rpc(method, timeout, args, kwargs) + + return self.llm_engine.collective_rpc(method, timeout, args, kwargs) def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]: """ diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 20904cd495f91..6083eea45cd98 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -8,7 +8,7 @@ import time from concurrent.futures import Future from inspect import isclass, signature from logging import DEBUG -from typing import Any, Optional +from typing import Any, Callable, Optional, TypeVar, Union import msgspec import psutil @@ -43,6 +43,8 @@ logger = init_logger(__name__) POLLING_TIMEOUT_S = 2.5 +_R = TypeVar('_R') # Return type for collective_rpc + class EngineCore: """Inner loop of vLLM's Engine.""" @@ -280,6 +282,14 @@ class EngineCore: def pin_lora(self, lora_id: int) -> bool: return self.model_executor.pin_lora(lora_id) + def collective_rpc(self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + return self.model_executor.collective_rpc(method, timeout, args, + kwargs) + class EngineCoreProc(EngineCore): """ZMQ-wrapper for running EngineCore in background process.""" diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 8858a564d2c2b..3dc33a1284a12 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -12,7 +12,7 @@ from collections.abc import Awaitable, Sequence from concurrent.futures import Future from dataclasses import dataclass, field from threading import Thread -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Optional, TypeVar, Union import zmq import zmq.asyncio @@ -33,6 +33,8 @@ logger = init_logger(__name__) AnyFuture = Union[asyncio.Future[Any], Future[Any]] +_R = TypeVar('_R') # Return type for collective_rpc + class EngineCoreClient(ABC): """ @@ -117,6 +119,13 @@ class EngineCoreClient(ABC): def pin_lora(self, lora_id: int) -> bool: raise NotImplementedError + def collective_rpc(self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + raise NotImplementedError + async def get_output_async(self) -> EngineCoreOutputs: raise NotImplementedError @@ -153,6 +162,14 @@ class EngineCoreClient(ABC): async def pin_lora_async(self, lora_id: int) -> bool: raise NotImplementedError + async def collective_rpc_async( + self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + raise NotImplementedError + class InprocClient(EngineCoreClient): """ @@ -210,6 +227,13 @@ class InprocClient(EngineCoreClient): def pin_lora(self, lora_id: int) -> bool: return self.engine_core.pin_lora(lora_id) + def collective_rpc(self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + return self.engine_core.collective_rpc(method, timeout, args, kwargs) + class CoreEngine: """One per data parallel rank.""" @@ -505,6 +529,14 @@ class SyncMPClient(MPClient): def execute_dummy_batch(self) -> None: self.call_utility("execute_dummy_batch") + def collective_rpc(self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + return self.call_utility("collective_rpc", method, timeout, args, + kwargs) + class AsyncMPClient(MPClient): """Asyncio-compatible client for multi-proc EngineCore.""" @@ -636,6 +668,15 @@ class AsyncMPClient(MPClient): async def pin_lora_async(self, lora_id: int) -> bool: return await self.call_utility_async("pin_lora", lora_id) + async def collective_rpc_async( + self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + return await self.call_utility_async("collective_rpc", method, timeout, + args, kwargs) + class DPAsyncMPClient(AsyncMPClient): """Asyncio-compatible client for multi-proc, multi-engine (data parallel) diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 000de21fbe7bf..764c643b5c974 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from copy import copy -from typing import Optional, Union +from typing import Any, Callable, Optional, Union from typing_extensions import TypeVar @@ -32,6 +32,7 @@ from vllm.v1.executor.abstract import Executor logger = init_logger(__name__) _G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup) +_R = TypeVar("_R", default=Any) class LLMEngine: @@ -282,6 +283,13 @@ class LLMEngine: """Prevent an adapter from being evicted.""" return self.engine_core.pin_lora(lora_id) + def collective_rpc(self, + method: Union[str, Callable[..., _R]], + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None) -> list[_R]: + return self.engine_core.collective_rpc(method, timeout, args, kwargs) + def __del__(self): if dp_group := getattr(self, "dp_group", None): stateless_destroy_torch_distributed_process_group(dp_group) diff --git a/vllm/v1/serial_utils.py b/vllm/v1/serial_utils.py index 3f000abcde0d1..146d7d747f1a4 100644 --- a/vllm/v1/serial_utils.py +++ b/vllm/v1/serial_utils.py @@ -1,13 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 import pickle +from types import FunctionType from typing import Any, Optional +import cloudpickle import torch from msgspec import msgpack CUSTOM_TYPE_TENSOR = 1 CUSTOM_TYPE_PICKLE = 2 +CUSTOM_TYPE_CLOUDPICKLE = 3 class MsgpackEncoder: @@ -41,6 +44,9 @@ def custom_enc_hook(obj: Any) -> Any: # https://gist.github.com/tlrmchlsmth/8067f1b24a82b6e2f90450e7764fa103 # noqa: E501 return msgpack.Ext(CUSTOM_TYPE_TENSOR, pickle.dumps(obj.numpy())) + if isinstance(obj, FunctionType): + return msgpack.Ext(CUSTOM_TYPE_CLOUDPICKLE, cloudpickle.dumps(obj)) + return msgpack.Ext(CUSTOM_TYPE_PICKLE, pickle.dumps(obj)) @@ -49,5 +55,7 @@ def custom_ext_hook(code: int, data: memoryview) -> Any: return torch.from_numpy(pickle.loads(data)) if code == CUSTOM_TYPE_PICKLE: return pickle.loads(data) + if code == CUSTOM_TYPE_CLOUDPICKLE: + return cloudpickle.loads(data) raise NotImplementedError(f"Extension type code {code} is not supported") From 6fa7cd3dbcf3e78e36431ca31abd973e5617dd27 Mon Sep 17 00:00:00 2001 From: shangmingc Date: Sat, 29 Mar 2025 19:01:46 +0800 Subject: [PATCH 103/593] [Feature][Disaggregated] Support XpYd disaggregated prefill with MooncakeStore (#12957) Signed-off-by: Shangming Cai --- .../disagg_examples/disagg_proxy_demo.py | 450 ++++++++++++++++++ .../kv_transfer/kv_connector/factory.py | 5 + .../kv_connector/mooncake_store_connector.py | 216 +++++++++ .../kv_transfer/kv_lookup_buffer/base.py | 87 +++- .../kv_lookup_buffer/mooncake_store.py | 160 +++++++ 5 files changed, 907 insertions(+), 11 deletions(-) create mode 100644 examples/online_serving/disagg_examples/disagg_proxy_demo.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/mooncake_store_connector.py create mode 100644 vllm/distributed/kv_transfer/kv_lookup_buffer/mooncake_store.py diff --git a/examples/online_serving/disagg_examples/disagg_proxy_demo.py b/examples/online_serving/disagg_examples/disagg_proxy_demo.py new file mode 100644 index 0000000000000..a701636f357a8 --- /dev/null +++ b/examples/online_serving/disagg_examples/disagg_proxy_demo.py @@ -0,0 +1,450 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +This file provides a disaggregated prefilling proxy demo to demonstrate an +example usage of XpYd disaggregated prefilling. +We can launch multiple vllm instances (2 for prefill and 2 for decode), and +launch this proxy demo through: + python3 examples/online_serving/disagg_examples/disagg_proxy_demo.py \ + --model $model_name \ + --prefill localhost:8100 localhost:8101 \ + --decode localhost:8200 localhost:8201 \ + --port 8000 + +Note: This demo will be removed once the PDController implemented in PR 15343 +(https://github.com/vllm-project/vllm/pull/15343) supports XpYd. +""" +import argparse +import ipaddress +import itertools +import json +import logging +import os +import sys +from abc import ABC, abstractmethod +from typing import Callable, Optional + +import aiohttp +import requests +import uvicorn +from fastapi import (APIRouter, Depends, FastAPI, Header, HTTPException, + Request, status) +from fastapi.responses import JSONResponse, StreamingResponse + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60) +logger = logging.getLogger() +logging.basicConfig(level=logging.INFO) + + +class SchedulingPolicy(ABC): + + @abstractmethod + def schedule(self, cycler: itertools.cycle): + raise NotImplementedError("Scheduling Proxy is not set.") + + +class Proxy: + + def __init__( + self, + prefill_instances: list[str], + decode_instances: list[str], + model: str, + scheduling_policy: SchedulingPolicy, + custom_create_completion: Optional[Callable[[Request], + StreamingResponse]] = None, + custom_create_chat_completion: Optional[Callable[ + [Request], StreamingResponse]] = None, + ): + self.prefill_instances = prefill_instances + self.decode_instances = decode_instances + self.prefill_cycler = itertools.cycle(prefill_instances) + self.decode_cycler = itertools.cycle(decode_instances) + self.model = model + self.scheduling_policy = scheduling_policy + self.custom_create_completion = custom_create_completion + self.custom_create_chat_completion = custom_create_chat_completion + self.router = APIRouter() + self.setup_routes() + + def setup_routes(self): + self.router.post( + "/v1/completions", + dependencies=[ + Depends(self.validate_json_request) + ])(self.custom_create_completion if self. + custom_create_completion else self.create_completion) + self.router.post( + "/v1/chat/completions", + dependencies=[ + Depends(self.validate_json_request) + ])(self.custom_create_chat_completion if self. + custom_create_chat_completion else self.create_chat_completion) + self.router.get("/status", + response_class=JSONResponse)(self.get_status) + self.router.post("/instances/add", + dependencies=[Depends(self.api_key_authenticate) + ])(self.add_instance_endpoint) + + async def validate_json_request(self, raw_request: Request): + content_type = raw_request.headers.get("content-type", "").lower() + if content_type != "application/json": + raise HTTPException( + status_code=415, + detail= + "Unsupported Media Type: Only 'application/json' is allowed", + ) + + def api_key_authenticate(self, x_api_key: str = Header(...)): + expected_api_key = os.environ.get("ADMIN_API_KEY") + if not expected_api_key: + logger.error("ADMIN_API_KEY is not set in the environment.") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Server configuration error.", + ) + if x_api_key != expected_api_key: + logger.warning("Unauthorized access attempt with API Key: %s", + x_api_key) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Forbidden: Invalid API Key.", + ) + + async def validate_instance(self, instance: str) -> bool: + url = f"http://{instance}/v1/models" + try: + async with aiohttp.ClientSession( + timeout=AIOHTTP_TIMEOUT) as client: + logger.info("Verifying %s ...", instance) + async with client.get(url) as response: + if response.status == 200: + data = await response.json() + if "data" in data and len(data["data"]) > 0: + model_cur = data["data"][0].get("id", "") + if model_cur == self.model: + logger.info("Instance: %s could be added.", + instance) + return True + else: + logger.warning("Mismatch model %s : %s != %s", + instance, model_cur, self.model) + return False + else: + return False + else: + return False + except aiohttp.ClientError as e: + logger.error(str(e)) + return False + except Exception as e: + logger.error(str(e)) + return False + + async def add_instance_endpoint(self, request: Request): + try: + data = await request.json() + logger.warning(str(data)) + instance_type = data.get("type") + instance = data.get("instance") + if instance_type not in ["prefill", "decode"]: + raise HTTPException(status_code=400, + detail="Invalid instance type.") + if not instance or ":" not in instance: + raise HTTPException(status_code=400, + detail="Invalid instance format.") + host, port_str = instance.split(":") + try: + if host != "localhost": + ipaddress.ip_address(host) + port = int(port_str) + if not (0 < port < 65536): + raise HTTPException(status_code=400, + detail="Invalid port number.") + except Exception as e: + raise HTTPException(status_code=400, + detail="Invalid instance address.") from e + + is_valid = await self.validate_instance(instance) + if not is_valid: + raise HTTPException(status_code=400, + detail="Instance validation failed.") + + if instance_type == "prefill": + if instance not in self.prefill_instances: + self.prefill_instances.append(instance) + self.prefill_cycler = itertools.cycle( + self.prefill_instances) + else: + raise HTTPException(status_code=400, + detail="Instance already exists.") + else: + if instance not in self.decode_instances: + self.decode_instances.append(instance) + self.decode_cycler = itertools.cycle(self.decode_instances) + else: + raise HTTPException(status_code=400, + detail="Instance already exists.") + + return JSONResponse(content={ + "message": + f"Added {instance} to {instance_type}_instances." + }) + except HTTPException as http_exc: + raise http_exc + except Exception as e: + logger.error("Error in add_instance_endpoint: %s", str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e + + async def forward_request(self, url, data, use_chunked=True): + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + headers = { + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}" + } + try: + async with session.post(url=url, json=data, + headers=headers) as response: + if 200 <= response.status < 300 or 400 <= response.status < 500: # noqa: E501 + if use_chunked: + async for chunk_bytes in response.content.iter_chunked( # noqa: E501 + 1024): + yield chunk_bytes + else: + content = await response.read() + yield content + else: + error_content = await response.text() + try: + error_content = json.loads(error_content) + except json.JSONDecodeError: + error_content = error_content + logger.error("Request failed with status %s: %s", + response.status, error_content) + raise HTTPException( + status_code=response.status, + detail= + f"Request failed with status {response.status}: " + f"{error_content}", + ) + except aiohttp.ClientError as e: + logger.error("ClientError occurred: %s", str(e)) + raise HTTPException( + status_code=502, + detail= + "Bad Gateway: Error communicating with upstream server.", + ) from e + except Exception as e: + logger.error("Unexpected error: %s", str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e + + def schedule(self, cycler: itertools.cycle) -> str: + return self.scheduling_policy.schedule(cycler) + + async def get_status(self): + status = { + "prefill_node_count": len(self.prefill_instances), + "decode_node_count": len(self.decode_instances), + "prefill_nodes": self.prefill_instances, + "decode_nodes": self.decode_instances, + } + return status + + async def create_completion(self, raw_request: Request): + try: + request = await raw_request.json() + + kv_prepare_request = request.copy() + kv_prepare_request["max_tokens"] = 1 + + prefill_instance = self.schedule(self.prefill_cycler) + try: + async for _ in self.forward_request( + f"http://{prefill_instance}/v1/completions", + kv_prepare_request): + continue + except HTTPException as http_exc: + self.remove_instance_endpoint("prefill", prefill_instance) + raise http_exc + + # Perform kv recv and decoding stage + decode_instance = self.schedule(self.decode_cycler) + + try: + generator = self.forward_request( + f"http://{decode_instance}/v1/completions", request) + except HTTPException as http_exc: + self.remove_instance_endpoint("decode", decode_instance) + raise http_exc + response = StreamingResponse(generator) + return response + except Exception: + import sys + + exc_info = sys.exc_info() + print("Error occurred in disagg proxy server") + print(exc_info) + + async def create_chat_completion(self, raw_request: Request): + try: + request = await raw_request.json() + + # add params to request + kv_prepare_request = request.copy() + kv_prepare_request["max_tokens"] = 1 + + # prefill stage + prefill_instance = self.schedule(self.prefill_cycler) + try: + async for _ in self.forward_request( + f"http://{prefill_instance}/v1/chat/completions", + kv_prepare_request): + continue + except HTTPException as http_exc: + self.remove_instance_endpoint("prefill", prefill_instance) + raise http_exc + # Perform kv recv and decoding stage + decode_instance = self.schedule(self.decode_cycler) + + try: + generator = self.forward_request( + "http://" + decode_instance + "/v1/chat/completions", + request) + except HTTPException as http_exc: + self.remove_instance_endpoint("decode", decode_instance) + raise http_exc + response = StreamingResponse(content=generator) + return response + except Exception: + exc_info = sys.exc_info() + error_messages = [str(e) for e in exc_info if e] + print("Error occurred in disagg proxy server") + print(error_messages) + return StreamingResponse(content=iter(error_messages), + media_type="text/event-stream") + + def remove_instance_endpoint(self, instance_type, instance): + if (instance_type == "decode" and instance in self.decode_instances): + self.decode_instances.remove(instance) + self.decode_cycler = itertools.cycle(self.decode_instances) + if (instance_type == "prefill" and instance in self.decode_instances): + self.prefill_instances.remove(instance) + self.prefill_cycler = itertools.cycle(self.decode_instances) + + +class RoundRobinSchedulingPolicy(SchedulingPolicy): + + def __init__(self): + super().__init__() + + def schedule(self, cycler: itertools.cycle) -> str: + return next(cycler) + + +class ProxyServer: + + def __init__( + self, + args: argparse.Namespace, + scheduling_policy: Optional[SchedulingPolicy] = None, + create_completion: Optional[Callable[[Request], + StreamingResponse]] = None, + create_chat_completion: Optional[Callable[[Request], + StreamingResponse]] = None, + ): + self.validate_parsed_serve_args(args) + self.port = args.port + self.proxy_instance = Proxy( + prefill_instances=[] if args.prefill is None else args.prefill, + decode_instances=[] if args.decode is None else args.decode, + model=args.model, + scheduling_policy=(scheduling_policy if scheduling_policy + is not None else RoundRobinSchedulingPolicy()), + custom_create_completion=create_completion, + custom_create_chat_completion=create_chat_completion, + ) + + def validate_parsed_serve_args(self, args: argparse.Namespace): + if not args.prefill: + raise ValueError("Please specify at least one prefill node.") + if not args.decode: + raise ValueError("Please specify at least one decode node.") + self.validate_instances(args.prefill) + self.validate_instances(args.decode) + self.verify_model_config(args.prefill, args.model) + self.verify_model_config(args.decode, args.model) + + def validate_instances(self, instances: list): + for instance in instances: + if len(instance.split(":")) != 2: + raise ValueError(f"Invalid instance format: {instance}") + host, port = instance.split(":") + try: + if host != "localhost": + ipaddress.ip_address(host) + port = int(port) + if not (0 < port < 65536): + raise ValueError( + f"Invalid port number in instance: {instance}") + except Exception as e: + raise ValueError( + f"Invalid instance {instance}: {str(e)}") from e + + def verify_model_config(self, instances: list, model: str) -> None: + model_suffix = model.split("/")[-1] + for instance in instances: + try: + response = requests.get(f"http://{instance}/v1/models") + if response.status_code == 200: + model_cur = response.json()["data"][0]["id"] + model_cur_suffix = model_cur.split("/")[-1] + if model_cur_suffix != model_suffix: + raise ValueError( + f"{instance} serves a different model: " + f"{model_cur} != {model}") + else: + raise ValueError(f"Cannot get model id from {instance}!") + except requests.RequestException as e: + raise ValueError( + f"Error communicating with {instance}: {str(e)}") from e + + def run_server(self): + app = FastAPI() + app.include_router(self.proxy_instance.router) + config = uvicorn.Config(app, port=self.port, loop="uvloop") + server = uvicorn.Server(config) + server.run() + + +if __name__ == "__main__": + # Todo: allow more config + parser = argparse.ArgumentParser("vLLM disaggregated proxy server.") + parser.add_argument("--model", + "-m", + type=str, + required=True, + help="Model name") + + parser.add_argument( + "--prefill", + "-p", + type=str, + nargs="+", + help="List of prefill node URLs (host:port)", + ) + + parser.add_argument( + "--decode", + "-d", + type=str, + nargs="+", + help="List of decode node URLs (host:port)", + ) + + parser.add_argument( + "--port", + type=int, + default=8000, + help="Server port number", + ) + args = parser.parse_args() + proxy_server = ProxyServer(args=args) + proxy_server.run_server() diff --git a/vllm/distributed/kv_transfer/kv_connector/factory.py b/vllm/distributed/kv_transfer/kv_connector/factory.py index 7336c54ec8a30..e37ce6dc75b03 100644 --- a/vllm/distributed/kv_transfer/kv_connector/factory.py +++ b/vllm/distributed/kv_transfer/kv_connector/factory.py @@ -53,3 +53,8 @@ KVConnectorFactory.register_connector( "LMCacheConnector", "vllm.distributed.kv_transfer.kv_connector.lmcache_connector", "LMCacheConnector") + +KVConnectorFactory.register_connector( + "MooncakeStoreConnector", + "vllm.distributed.kv_transfer.kv_connector.mooncake_store_connector", + "MooncakeStoreConnector") \ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_connector/mooncake_store_connector.py b/vllm/distributed/kv_transfer/kv_connector/mooncake_store_connector.py new file mode 100644 index 0000000000000..c5135dab23eba --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/mooncake_store_connector.py @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +MooncakeStore Connector for Distributed Machine Learning Inference + +The MooncakeStoreConnector transfers KV caches between prefill vLLM workers +(KV cache producer) and decode vLLM workers (KV cache consumer) using a +database-style KVStore. +""" +import hashlib +from typing import TYPE_CHECKING, List, Tuple, Union + +import torch + +from vllm import _custom_ops as ops +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase +from vllm.logger import init_logger +from vllm.sequence import IntermediateTensors + +if TYPE_CHECKING: + from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata + +logger = init_logger(__name__) + + +class MooncakeStoreConnector(KVConnectorBase): + + def __init__( + self, + rank: int, + local_rank: int, + config: VllmConfig, + ): + self.config = config.kv_transfer_config + self.tp_size = config.parallel_config.tensor_parallel_size + + self.local_tp_rank = local_rank + + # Init kv_store + if self.config.kv_connector == "MooncakeStoreConnector": + # Check if MOONCAKE_CONFIG_PATH is set + import os + use_mooncake_store = os.getenv('MOONCAKE_CONFIG_PATH') is not None + + if not use_mooncake_store: + raise ValueError( + "To use MooncakeStoreConnector, you need to pass the ENV: " + "'MOONCAKE_CONFIG_PATH=/path/to/mooncake_config.json'.") + else: + from vllm.distributed.kv_transfer.kv_lookup_buffer.mooncake_store import ( # noqa: E501 + MooncakeStore) + logger.info( + "Initializing KVStoreConnector under kv_transfer_config %s", + self.config) + self.kv_store = MooncakeStore(config) + else: + logger.error("Can not find %s", self.config.kv_connector) + + assert self.kv_store is not None + + def close(self) -> None: + """Close the buffer and release resources. + This method is responsible for cleaning up resources related to the + connector when it is no longer needed. + Raises: + NotImplementedError: This method must be implemented in subclasses. + """ + self.kv_store.close() + + def send_kv_caches_and_hidden_states( + self, + model_executable: torch.nn.Module, + model_input: "ModelInputForGPUWithSamplingMetadata", + kv_caches: List[torch.Tensor], + hidden_or_intermediate_states: Union[torch.Tensor, + IntermediateTensors], + ) -> None: + input_tokens_tensor = model_input.input_tokens + seq_lens = model_input.attn_metadata.seq_lens + slot_mapping_flat = model_input.attn_metadata.slot_mapping.flatten() + start_layer = model_executable.model.start_layer + end_layer = model_executable.model.end_layer + + model_config = model_executable.model.config + num_heads = int(model_config.num_key_value_heads / self.tp_size) + hidden_size = model_config.hidden_size + num_attention_heads = model_config.num_attention_heads + head_size = int(hidden_size / num_attention_heads) + + for idx, slen in enumerate(seq_lens): + start_pos = sum(seq_lens[:idx]) + end_pos = start_pos + slen + + current_tokens = input_tokens_tensor[start_pos:end_pos] + store_key_prefix = self.tensor_hash(current_tokens) + keys, values = [], [] + + for layer_id in range(start_layer, end_layer): + kv_cache = kv_caches[layer_id - start_layer] + + key_cache = kv_cache[0].reshape(-1, num_heads, head_size) + value_cache = kv_cache[1].reshape(-1, num_heads, head_size) + + current_slot_mapping = slot_mapping_flat[start_pos:end_pos] + + keys.append(key_cache[current_slot_mapping].unsqueeze(0)) + values.append(value_cache[current_slot_mapping].unsqueeze(0)) + + keys = torch.cat(keys, dim=0) + values = torch.cat(values, dim=0) + kvcache_to_sent = torch.stack((keys, values), dim=0) + store_kvcache_key = f"{store_key_prefix}_{self.local_tp_rank}" + self.kv_store.put(store_kvcache_key, kvcache_to_sent) + + hidden_key = f"{store_key_prefix}_hidden_{self.local_tp_rank}" + self.kv_store.put(hidden_key, + hidden_or_intermediate_states[start_pos:end_pos]) + + logger.debug("[rank%d]: KV send DONE.", torch.distributed.get_rank()) + + def recv_kv_caches_and_hidden_states( + self, model_executable: torch.nn.Module, + model_input: "ModelInputForGPUWithSamplingMetadata", + kv_caches: List[torch.Tensor] + ) -> Tuple[Union[torch.Tensor, IntermediateTensors], bool, + "ModelInputForGPUWithSamplingMetadata"]: + bypass_model_exec = True + input_tokens_tensor = model_input.input_tokens + seq_lens = model_input.attn_metadata.seq_lens + num_prefill_tokens = model_input.attn_metadata.num_prefill_tokens + slot_mapping = model_input.attn_metadata.slot_mapping.flatten() + start_layer = model_executable.model.start_layer + end_layer = model_executable.model.end_layer + hidden_or_intermediate_states_for_one_req = [] + + for idx, slen in enumerate(seq_lens): + start_pos = sum(seq_lens[:idx]) + end_pos = start_pos + slen + + if start_pos >= num_prefill_tokens: + # This can happen during inflight batching. See: + # vllm/worker/model_runner.py::_prepare_model_input_tensors: + # - input_tokens[:num_prefill_tokens] contains prefill tokens. + # - input_tokens[num_prefill_tokens:] contains decode tokens. + logger.warning("You should set --enable_chunked_prefill=False " + "and --max_num_batched_tokens " + "should be equal to max_seq_len_to_capture") + bypass_model_exec = False + assert start_pos == num_prefill_tokens + break + + current_tokens = input_tokens_tensor[start_pos:end_pos] + + # get roi for current seq + load_key_prefix = self.tensor_hash(current_tokens) + load_kvcache_key = f"{load_key_prefix}_{self.local_tp_rank}" + remote_kv = self.kv_store.get(load_kvcache_key) + hidden_key = f"{load_key_prefix}_hidden_{self.local_tp_rank}" + hidden = self.kv_store.get(hidden_key) + + if remote_kv is None or hidden is None: + # didn't find any match. + bypass_model_exec = False + continue + + num_computed_tokens = current_tokens.shape[0] + + # update the end position based on how many tokens are cached. + end_pos = start_pos + num_computed_tokens + + # call self.kv_store to get kv layer by layer + for layer_id in range(start_layer, end_layer): + layer = model_executable.model.layers[layer_id] + # get kvcache object + kv_cache = kv_caches[layer_id - start_layer] + key_cache, value_cache = kv_cache[0], kv_cache[1] + # get remote kvcache + + remote_k, remote_v = remote_kv[0][layer_id], remote_kv[1][ + layer_id] + # use ops.reshape_and_cache_flash to put kv into kvcache + ops.reshape_and_cache_flash( + remote_k.to(key_cache.device), + remote_v.to(value_cache.device), + key_cache, + value_cache, + slot_mapping[start_pos:end_pos], + layer.self_attn.attn.kv_cache_dtype, + layer.self_attn.attn._k_scale, + layer.self_attn.attn._v_scale, + ) + + hidden_or_intermediate_states_for_one_req.append(hidden) + + if not bypass_model_exec: + logger.warning( + "[rank%d]: Failed to receive all KVs and hidden " + "states, redo model forwarding.", torch.distributed.get_rank()) + hidden_or_intermediate_states = None + + else: + logger.debug( + "[rank%d]: Successfully received all KVs and hidden " + "states, skip model forwarding.", torch.distributed.get_rank()) + hidden_or_intermediate_states = torch.cat( + hidden_or_intermediate_states_for_one_req, dim=0) + + return hidden_or_intermediate_states, bypass_model_exec, model_input + + @staticmethod + def tensor_hash(tensor: torch.Tensor) -> int: + """Calculate the hash value of the tensor.""" + tensor_bytes = tensor.clone().detach().cpu().numpy().tobytes() + hash_object = hashlib.blake2b(tensor_bytes) + hash_hex = hash_object.hexdigest() + return int(hash_hex[:16], 16) diff --git a/vllm/distributed/kv_transfer/kv_lookup_buffer/base.py b/vllm/distributed/kv_transfer/kv_lookup_buffer/base.py index 845da7c501e88..bea42846e9e41 100644 --- a/vllm/distributed/kv_transfer/kv_lookup_buffer/base.py +++ b/vllm/distributed/kv_transfer/kv_lookup_buffer/base.py @@ -1,11 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """ -This file contains a new class `KVLookupBufferBase` that allows developers to -think of KV cache operations as inserting new KV cache entries (`insert`) -into the lookup buffer and querying existing KV caches (`drop_select`) +This file contains a new class `KVLookupBufferBase` that allows developers to +think of KV cache operations as inserting new KV cache entries (`insert`) +into the lookup buffer and querying existing KV caches (`drop_select`) from the lookup buffer. -All distributed communications are abstracted behind this class. +This file also contains a new class `KVStoreBufferBase` that allows developers +to manage the KVCache buffer as a simple key-value storage buffer with basic +put/get operations. + +These classes above are abstracted behind class `KVCacheBufferBase`. """ from abc import ABC, abstractmethod @@ -14,9 +18,27 @@ from typing import List, Optional import torch -class KVLookupBufferBase(ABC): +class KVCacheBufferBase(ABC): """ - Abstract base class for a lookup buffer. + Abstract base class for a KVCache buffer. + """ + + @abstractmethod + def close(self) -> None: + """Close the buffer and release resources. + + This method is responsible for cleaning up resources related to the + KVCache buffer when it is no longer needed. + + Raises: + NotImplementedError: This method must be implemented in subclasses. + """ + raise NotImplementedError + + +class KVLookupBufferBase(KVCacheBufferBase): + """ + Abstract base class for a KVCache lookup buffer. This class provides an abstraction for a key-value (KV) cache lookup buffer. @@ -96,12 +118,55 @@ class KVLookupBufferBase(ABC): """ raise NotImplementedError - @abstractmethod - def close(self) -> None: - """Close the buffer and release resources. - This method is responsible for cleaning up resources related to the - lookup buffer when it is no longer needed. +class KVStoreBufferBase(KVCacheBufferBase): + """ + Abstract base class for a KVCache storage buffer with key-value semantics. + This class provides a simple key-value storage buffer abstract with basic + put/get operations, which enables flexible KVCache transfer granular + control. + + The functionality is similar to a distributed key-value store, where: + - Key: A unique string identifier for the cached entry + - Value: + - Tensor to be stored and retrieved + - None (indicating deletion or empty value) + """ + + @abstractmethod + def put( + self, + key: str, + value: Optional[torch.Tensor], + ) -> None: + """Store a key-value pair in the buffer. + + Args: + key (str): Unique identifier for a tensor, this tensor could be the + key cache tensor, value cache tensor, or hidden state tensor + generated during model forwarding. + + value (Optional[torch.Tensor]): Tensor to be stored. + + Raises: + NotImplementedError: This method must be implemented in subclasses. + """ + raise NotImplementedError + + @abstractmethod + def get( + self, + key: str, + ) -> Optional[torch.Tensor]: + """Retrieve a value from the buffer by key. + + Args: + key (str): Unique identifier for a tensor, this tensor could be the + key cache tensor, value cache tensor, or hidden state tensor + generated during model forwarding. + + Returns: + Optional[torch.Tensor]: Stored tensor if exists, None otherwise. Raises: NotImplementedError: This method must be implemented in subclasses. diff --git a/vllm/distributed/kv_transfer/kv_lookup_buffer/mooncake_store.py b/vllm/distributed/kv_transfer/kv_lookup_buffer/mooncake_store.py new file mode 100644 index 0000000000000..7fd5967293f26 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_lookup_buffer/mooncake_store.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +This file contains a new class `MooncakeStore` that allows developers to +think of KV cache transfer operations as putting new KV cache entries +into a remote KVStore-based lookup buffer and getting existing KV caches +from this remote lookup buffer. +""" +import json +import os +from dataclasses import dataclass +from typing import Optional + +import torch +from safetensors.torch import load as safetensors_load +from safetensors.torch import save as safetensors_save + +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_lookup_buffer.base import ( + KVStoreBufferBase) +from vllm.logger import init_logger + +DEFAULT_GLOBAL_SEGMENT_SIZE = 3355443200 # 3.125 GiB +DEFAULT_LOCAL_BUFFER_SIZE = 1073741824 # 1.0 GiB + +logger = init_logger(__name__) + + +@dataclass +class MooncakeStoreConfig: + local_hostname: str + metadata_server: str + global_segment_size: int + local_buffer_size: int + protocol: str + device_name: str + master_server_address: str + + @staticmethod + def from_file(file_path: str) -> 'MooncakeStoreConfig': + """Load the config from a JSON file.""" + with open(file_path) as fin: + config = json.load(fin) + return MooncakeStoreConfig( + local_hostname=config.get("local_hostname"), + metadata_server=config.get("metadata_server"), + global_segment_size=config.get("global_segment_size", + DEFAULT_GLOBAL_SEGMENT_SIZE), + local_buffer_size=config.get("local_buffer_size", + DEFAULT_LOCAL_BUFFER_SIZE), + protocol=config.get("protocol", "tcp"), + device_name=config.get("device_name", ""), + master_server_address=config.get("master_server_address"), + ) + + @staticmethod + def load_from_env() -> 'MooncakeStoreConfig': + """Load config from a file specified in the environment variable.""" + config_file_path = os.getenv('MOONCAKE_CONFIG_PATH') + if config_file_path is None: + raise ValueError( + "The environment variable 'MOONCAKE_CONFIG_PATH' is not set.") + return MooncakeStoreConfig.from_file(config_file_path) + + +class MooncakeStore(KVStoreBufferBase): + + def __init__( + self, + config: VllmConfig, + ): + + try: + from mooncake_vllm_adaptor import MooncakeDistributedStore + except ImportError as e: + raise ImportError( + "Please install mooncake by following the instructions at " + "https://github.com/kvcache-ai/Mooncake/blob/main/doc/en/build.md " # noqa: E501 + "to run vLLM with MooncakeConnector.") from e + + try: + self.store = MooncakeDistributedStore() + self.config = MooncakeStoreConfig.load_from_env() + logger.info("Mooncake Configuration loaded successfully.") + + self.store.setup(self.config.local_hostname, + self.config.metadata_server, + self.config.global_segment_size, + self.config.local_buffer_size, + self.config.protocol, self.config.device_name, + self.config.master_server_address) + + except ValueError as e: + logger.error("Configuration loading failed: %s", e) + raise + except Exception as exc: + logger.error( + "An error occurred while loading the configuration: %s", exc) + raise + + def close(self): + # MooncakeDistributedStore will automatically call the destructor, so + # it is unnecessary to close it manually. + pass + + def put( + self, + key: str, + value: Optional[torch.Tensor], + ) -> None: + # A message queue needs to be introduced before making it asynchronous. + if value is not None: + self._put_impl(key, value) + + def get( + self, + key: str, + ) -> Optional[torch.Tensor]: + # A message queue needs to be introduced before making it asynchronous. + value = self._get_impl(key) + return value + + def _put_impl( + self, + key: str, + value: torch.Tensor, + ) -> None: + """Put KVCache to Mooncake Store""" + device_id = value.device.index if value.device.type == 'cuda' else -1 + device_tensor = torch.tensor(device_id, dtype=torch.int32) + value_bytes = safetensors_save({ + "tensor": value, + "device_id": device_tensor + }) + try: + self.store.put(key, value_bytes) + except TypeError as err: + logger.error("Failed to put value into Mooncake Store: %s", err) + raise TypeError("Mooncake Store Put Type Error.") from err + + def _get_impl( + self, + key: str, + ) -> Optional[torch.Tensor]: + """Get KVCache from Mooncake Store""" + try: + data = self.store.get(key) + except TypeError as err: + logger.error("Failed to get value from Mooncake Store: %s", err) + raise TypeError("Mooncake Store Get Type Error.") from err + + if data: + loaded_tensors = safetensors_load(data) + tensor = loaded_tensors["tensor"] + device_id_tensor = loaded_tensors["device_id"] + device_id = int(device_id_tensor.item()) + device = torch.device( + 'cuda', device_id) if device_id >= 0 else torch.device('cpu') + return tensor.to(device) + + return None From c67abd614fe670b1cc771097658dd7efe4a33747 Mon Sep 17 00:00:00 2001 From: Roger Wang <136131678+ywang96@users.noreply.github.com> Date: Sat, 29 Mar 2025 06:30:09 -0700 Subject: [PATCH 104/593] [V1] Support interleaved modality items (#15605) Signed-off-by: Roger Wang --- .buildkite/test-pipeline.yaml | 1 + tests/conftest.py | 39 +++++---- .../vision_language/test_interleaved.py | 77 ++++++++++++++++++ tests/multimodal/test_utils.py | 80 +++++++++++++++---- vllm/multimodal/utils.py | 72 ++++++----------- vllm/v1/engine/processor.py | 51 +++++------- 6 files changed, 205 insertions(+), 115 deletions(-) create mode 100644 tests/models/decoder_only/vision_language/test_interleaved.py diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 62872bf8e3e18..99358d5579919 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -431,6 +431,7 @@ steps: - pytest -v -s models/encoder_decoder/audio_language -m core_model - pytest -v -s models/encoder_decoder/language -m core_model - pytest -v -s models/encoder_decoder/vision_language -m core_model + - pytest -v -s models/decoder_only/vision_language/test_interleaved.py - label: Multi-Modal Models Test (Extended) 1 # 48m optional: true diff --git a/tests/conftest.py b/tests/conftest.py index cc48fceb8eff0..6627ab638bf55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -747,30 +747,27 @@ class VllmRunner: videos: Optional[PromptVideoInput] = None, audios: Optional[PromptAudioInput] = None, ) -> list[TextPrompt]: - if images is not None: - assert len(prompts) == len(images) - if videos is not None: - assert len(prompts) == len(videos) + if any(x is not None and len(x) != len(prompts) + for x in [images, videos, audios]): + raise ValueError( + "All non-None multimodal inputs must have the same length as " + "prompts") - if audios is not None: - assert len(prompts) == len(audios) + inputs = [] + for i, prompt in enumerate(prompts): + multi_modal_data = {} + if images is not None and (image := images[i]) is not None: + multi_modal_data["image"] = image + if videos is not None and (video := videos[i]) is not None: + multi_modal_data["video"] = video + if audios is not None and (audio := audios[i]) is not None: + multi_modal_data["audio"] = audio - inputs = [TextPrompt(prompt=prompt) for prompt in prompts] - if images is not None: - for i, image in enumerate(images): - if image is not None: - inputs[i]["multi_modal_data"] = {"image": image} - - if videos is not None: - for i, video in enumerate(videos): - if video is not None: - inputs[i]["multi_modal_data"] = {"video": video} - - if audios is not None: - for i, audio in enumerate(audios): - if audio is not None: - inputs[i]["multi_modal_data"] = {"audio": audio} + inputs.append( + TextPrompt(prompt=prompt, + multi_modal_data=multi_modal_data + if multi_modal_data else None)) return inputs diff --git a/tests/models/decoder_only/vision_language/test_interleaved.py b/tests/models/decoder_only/vision_language/test_interleaved.py new file mode 100644 index 0000000000000..8804497ae616f --- /dev/null +++ b/tests/models/decoder_only/vision_language/test_interleaved.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from vllm.assets.image import ImageAsset +from vllm.assets.video import VideoAsset + +models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"] + + +def base_prompt(modalities_str: str) -> str: + return f"<|im_start|>user {modalities_str}\nDescribe what you see from these items.<|im_end|><|im_start|>assistant\n" # noqa: E501 + + +INTERLEAVED_PROMPT = base_prompt("