From b368382964913312d41c670b4166f4c83eed49aa Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Mon, 27 Oct 2025 16:43:00 +0800 Subject: [PATCH 001/127] [Model] Deprecate `merge_by_field_config=False` (#27551) Signed-off-by: DarkLight1337 --- vllm/multimodal/utils.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index e97bab250ed19..7f259dad08f90 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -18,6 +18,7 @@ from PIL import Image, UnidentifiedImageError import vllm.envs as envs from vllm.connections import HTTPConnection, global_http_connection +from vllm.logger import init_logger from vllm.utils.jsontree import json_map_leaves from .audio import AudioMediaIO @@ -25,8 +26,6 @@ from .base import MediaIO from .image import ImageEmbeddingMediaIO, ImageMediaIO from .video import VideoMediaIO -_M = TypeVar("_M") - if TYPE_CHECKING: from .inputs import ( BatchedTensorInputs, @@ -38,11 +37,15 @@ else: MultiModalKwargsItem = Any MultiModalPlaceholderDict = Any +logger = init_logger(__name__) + global_thread_pool = ThreadPoolExecutor( max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT ) atexit.register(global_thread_pool.shutdown) +_M = TypeVar("_M") + class MediaConnector: def __init__( @@ -413,14 +416,21 @@ def group_mm_kwargs_by_modality( "`merge_by_field_config` arg, please update your model runner " "according to https://github.com/vllm-project/vllm/pull/25676." ) + if merge_by_field_config is False: + logger.warning_once( + "The legacy code for batching multi-modal kwargs is deprecated and " + "will be removed in v0.12. Please update your model with " + "`merge_by_field_config=True` to use the new code defined by " + "`MultiModalFieldConfig`. You can refer to " + "https://github.com/vllm-project/vllm/issues/26149 " + "for some examples on how to do this." + ) from vllm.multimodal.inputs import MultiModalKwargs, MultiModalKwargsItems for modality, items in groupby(mm_kwargs, key=lambda item: item.modality): items_lst = list(items) - # TODO: Deprecate `merge_by_field_config` once - # we have migrated all in-tree models if merge_by_field_config: mm_kwargs_group: BatchedTensorInputs = dict( MultiModalKwargsItems.from_seq(items_lst).get_data( From 2d631d28c697e995f555a214a46558e949875bda Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Mon, 27 Oct 2025 17:02:10 +0800 Subject: [PATCH 002/127] [Doc] Slight improvement to M2 and beyond (#27554) Signed-off-by: Jee Jee Li Co-authored-by: Roger Wang --- docs/features/reasoning_outputs.md | 9 +++++---- docs/models/supported_models.md | 1 + tests/models/registry.py | 1 - vllm/model_executor/models/minimax_m2.py | 14 -------------- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/docs/features/reasoning_outputs.md b/docs/features/reasoning_outputs.md index 302d1161c9025..dc2b2315182a9 100644 --- a/docs/features/reasoning_outputs.md +++ b/docs/features/reasoning_outputs.md @@ -14,11 +14,12 @@ vLLM currently supports the following reasoning models: | [DeepSeek-V3.1](https://huggingface.co/collections/deepseek-ai/deepseek-v31-68a491bed32bd77e7fca048f) | `deepseek_v3` | `json`, `regex` | ❌ | | [ERNIE-4.5-VL series](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-PT) | `ernie45` | `json`, `regex` | ❌ | | [ERNIE-4.5-21B-A3B-Thinking](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-Thinking) | `ernie45` | `json`, `regex` | ✅ | -| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ | -| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ | -| [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ | -| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `json`, `regex` | ✅ | | [GLM-4.5 series](https://huggingface.co/collections/zai-org/glm-45-687c621d34bda8c9e4bf503b) | `glm45` | `json`, `regex` | ✅ | +| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `json`, `regex` | ✅ | +| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ | +| [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) | `minimax_m2_append_think` | `json`, `regex` | ✅ | +| [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ | +| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ | !!! note IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`. diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 9039c0480547e..5da561c83b2cc 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -390,6 +390,7 @@ th { | `MiMoForCausalLM` | MiMo | `XiaomiMiMo/MiMo-7B-RL`, etc. | ✅︎ | ✅︎ | | `MiniCPMForCausalLM` | MiniCPM | `openbmb/MiniCPM-2B-sft-bf16`, `openbmb/MiniCPM-2B-dpo-bf16`, `openbmb/MiniCPM-S-1B-sft`, etc. | ✅︎ | ✅︎ | | `MiniCPM3ForCausalLM` | MiniCPM3 | `openbmb/MiniCPM3-4B`, etc. | ✅︎ | ✅︎ | +| `MiniMaxM2ForCausalLM` | MiniMax-M2 |`MiniMaxAI/MiniMax-M2`, etc. | | ✅︎ | | `MistralForCausalLM` | Mistral, Mistral-Instruct | `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ | | `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ | | `MPTForCausalLM` | MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter | `mosaicml/mpt-7b`, `mosaicml/mpt-7b-storywriter`, `mosaicml/mpt-30b`, etc. | | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index d0ee6187b4df1..17b1d7b527f6b 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -344,7 +344,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "MiniMaxM2ForCausalLM": _HfExamplesInfo( "MiniMaxAI/MiniMax-M2", trust_remote_code=True, - is_available_online=False, ), "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"), "MixtralForCausalLM": _HfExamplesInfo( diff --git a/vllm/model_executor/models/minimax_m2.py b/vllm/model_executor/models/minimax_m2.py index 550a9f937c8fd..dadb8a19c004e 100644 --- a/vllm/model_executor/models/minimax_m2.py +++ b/vllm/model_executor/models/minimax_m2.py @@ -551,20 +551,6 @@ class MiniMaxM2ForCausalLM(nn.Module, SupportsPP): logits = self.logits_processor(self.lm_head, hidden_states) return logits - def make_empty_intermediate_tensors( - self, batch_size: int, dtype: torch.dtype, device: torch.device - ) -> IntermediateTensors: - return IntermediateTensors( - { - "hidden_states": torch.zeros( - (batch_size, self.config.hidden_size), dtype=dtype, device=device - ), - "residual": torch.zeros( - (batch_size, self.config.hidden_size), dtype=dtype, device=device - ), - } - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) From 9932ed6a8352fcc2230990c9631bcb6e77a93f70 Mon Sep 17 00:00:00 2001 From: Danielle Robinson Date: Mon, 27 Oct 2025 02:05:24 -0700 Subject: [PATCH 003/127] [Kernel] Adding split_K implementation for fused_moe_lora (#27291) Signed-off-by: Danielle Robinson Signed-off-by: Danielle Robinson Co-authored-by: Danielle Robinson Co-authored-by: Jee Jee Li --- tests/lora/test_fused_moe_lora_kernel.py | 2 ++ vllm/lora/ops/triton_ops/fused_moe_lora_op.py | 30 +++++++++++++------ vllm/lora/punica_wrapper/punica_gpu.py | 1 + .../layers/fused_moe/fused_moe.py | 16 ++++++---- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/tests/lora/test_fused_moe_lora_kernel.py b/tests/lora/test_fused_moe_lora_kernel.py index f9a66d4d02ead..0ae992ad1110c 100644 --- a/tests/lora/test_fused_moe_lora_kernel.py +++ b/tests/lora/test_fused_moe_lora_kernel.py @@ -154,6 +154,7 @@ def use_fused_moe_lora_kernel( "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, + "SPLIT_K": 1, } mul_routed_weight = False @@ -175,6 +176,7 @@ def use_fused_moe_lora_kernel( config["BLOCK_SIZE_N"], config["BLOCK_SIZE_K"], config["GROUP_SIZE_M"], + config["SPLIT_K"], mul_routed_weight, ) diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py index d8746ebc8e75b..2031ade64b5fc 100644 --- a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py +++ b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py @@ -80,11 +80,13 @@ def _fused_moe_lora_kernel( BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, + SPLIT_K: tl.constexpr, ): pid = tl.program_id(axis=0) slice_id = tl.program_id(axis=1) lora_idx = tl.program_id(axis=2) max_loras = tl.num_programs(axis=2) + grid_k = tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K) # calculate pid_m,pid_n num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) @@ -102,7 +104,7 @@ def _fused_moe_lora_kernel( # get the expert_id to process curr shard ind = lora_idx * stride_el + pid_m - expert_id = tl.load(expert_ids_ptr + ind) + expert_id = tl.load(expert_ids_ptr + ind, ind < max_loras * stride_el, -1) if expert_id == -1: return @@ -117,7 +119,7 @@ def _fused_moe_lora_kernel( offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) token_ind = stride_tl * lora_idx + offs_token_id offs_token = tl.load( - sorted_token_ids_ptr + token_ind, token_ind < max_loras * stride_tl, 0.0 + sorted_token_ids_ptr + token_ind, token_ind < max_loras * stride_tl, 0 ) token_mask = offs_token < num_valid_tokens @@ -135,17 +137,18 @@ def _fused_moe_lora_kernel( # accumulator accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + for k in range(0, grid_k): + k_remaining = K - k * (BLOCK_SIZE_K * SPLIT_K) a = tl.load( a_ptrs, - mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), + mask=token_mask[:, None] & (offs_k[None, :] < k_remaining), other=0.0, ) - b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < k_remaining, other=0.0) accumulator += tl.dot(a, b) # Advance the ptrs to the next K block. - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk + a_ptrs += BLOCK_SIZE_K * SPLIT_K * stride_ak + b_ptrs += BLOCK_SIZE_K * SPLIT_K * stride_bk if MUL_ROUTED_WEIGHT: moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0) @@ -156,7 +159,10 @@ def _fused_moe_lora_kernel( offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) c_ptrs = cur_c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] c_mask = token_mask[:, None] & (offs_cn[None, :] < N) - tl.store(c_ptrs, accumulator, mask=c_mask) + if SPLIT_K == 1: + tl.store(c_ptrs, accumulator, mask=c_mask) + else: + tl.atomic_add(c_ptrs, accumulator, mask=c_mask, sem="relaxed") @torch.inference_mode() @@ -179,6 +185,7 @@ def _fused_moe_lora( block_size_n: int, block_size_k: int, group_size_m: int, + split_k: int, mul_routed_weight: bool = False, ) -> None: assert len(lora_a_stacked) == len(lora_b_stacked) > 0 @@ -206,6 +213,7 @@ def _fused_moe_lora( "BLOCK_SIZE_N": block_size_n, "BLOCK_SIZE_K": block_size_k, "GROUP_SIZE_M": group_size_m, + "SPLIT_K": split_k, } w1_lora_a_stacked = lora_a_stacked[0] @@ -237,7 +245,9 @@ def _fused_moe_lora( b_ptr = _get_ptr(lora_a_stacked, device) grid = lambda META: ( - triton.cdiv(EM, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + split_k + * triton.cdiv(EM, META["BLOCK_SIZE_M"]) + * triton.cdiv(N, META["BLOCK_SIZE_N"]), len(lora_a_stacked), lora_a_stacked[0].shape[0], ) @@ -286,6 +296,8 @@ def _fused_moe_lora( -1, a_intermediate_cache1.shape[3] ) + # Set split_k = 1 for expand calls + config["SPLIT_K"] = 1 grid = lambda META: ( triton.cdiv(EM, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), len(lora_b_stacked), diff --git a/vllm/lora/punica_wrapper/punica_gpu.py b/vllm/lora/punica_wrapper/punica_gpu.py index c2c26a01ee039..0cbf294cf410a 100644 --- a/vllm/lora/punica_wrapper/punica_gpu.py +++ b/vllm/lora/punica_wrapper/punica_gpu.py @@ -385,5 +385,6 @@ class PunicaWrapperGPU(PunicaWrapperBase): config["BLOCK_SIZE_N"], config["BLOCK_SIZE_K"], config["GROUP_SIZE_M"], + config.get("SPLIT_K", 1), mul_routed_weight, ) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 89e92edc8d2b3..5f9bfd6d9cf7d 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -121,6 +121,7 @@ def fused_moe_kernel_gptq_awq( BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, + SPLIT_K: tl.constexpr, MUL_ROUTED_WEIGHT: tl.constexpr, top_k: tl.constexpr, compute_type: tl.constexpr, @@ -356,6 +357,7 @@ def fused_moe_kernel( BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, + SPLIT_K: tl.constexpr, MUL_ROUTED_WEIGHT: tl.constexpr, top_k: tl.constexpr, compute_type: tl.constexpr, @@ -646,7 +648,6 @@ def invoke_fused_moe_kernel( bit, ) return - fused_moe_kernel_gptq_awq[grid]( A, B, @@ -686,6 +687,7 @@ def invoke_fused_moe_kernel( ) else: config = config.copy() + config["SPLIT_K"] = 1 BLOCK_SIZE_K = config.pop("BLOCK_SIZE_K") if block_shape is not None: BLOCK_SIZE_K = min(BLOCK_SIZE_K, min(block_shape[0], block_shape[1])) @@ -983,6 +985,7 @@ def get_default_config( "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 8, + "SPLIT_K": 1, } return config @@ -996,6 +999,7 @@ def get_default_config( "BLOCK_SIZE_N": block_shape[0], "BLOCK_SIZE_K": block_shape[1], "GROUP_SIZE_M": 32, + "SPLIT_K": 1, "num_warps": 4, "num_stages": 3 if not current_platform.is_rocm() else 2, } @@ -1006,19 +1010,20 @@ def get_default_config( bit = 4 if dtype == "int4_w4a16" else 8 use_moe_wna16_cuda = should_moe_wna16_use_cuda(M * topk, block_shape[1], E, bit) if use_moe_wna16_cuda: - config = {"BLOCK_SIZE_M": min(16, M)} + config = {"BLOCK_SIZE_M": min(16, M), "SPLIT_K": 1} elif M <= 20: - config = {"BLOCK_SIZE_M": 16, "GROUP_SIZE_M": 1} + config = {"BLOCK_SIZE_M": 16, "GROUP_SIZE_M": 1, "SPLIT_K": 1} elif M <= 40: - config = {"BLOCK_SIZE_M": 32, "GROUP_SIZE_M": 1} + config = {"BLOCK_SIZE_M": 32, "GROUP_SIZE_M": 1, "SPLIT_K": 1} else: - config = {"BLOCK_SIZE_M": 64, "GROUP_SIZE_M": 1} + config = {"BLOCK_SIZE_M": 64, "GROUP_SIZE_M": 1, "SPLIT_K": 1} elif M <= E: config = { "BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, + "SPLIT_K": 1, } else: config = { @@ -1026,6 +1031,7 @@ def get_default_config( "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 8, + "SPLIT_K": 1, } return config From 7c2bdb83dca6d71f46a5dd123402242d650df553 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Mon, 27 Oct 2025 17:05:40 +0800 Subject: [PATCH 004/127] [Misc] Clean up utils (#27552) Signed-off-by: DarkLight1337 --- docs/mkdocs/hooks/generate_argparse.py | 4 +- tests/utils.py | 4 +- .../{test_utils.py => test_argparse_utils.py} | 93 +-------- tests/utils_/test_serial_utils.py | 2 +- .../kv_connector/v1/decode_bench_connector.py | 2 +- .../v1/lmcache_integration/vllm_v1_adapter.py | 3 +- vllm/entrypoints/anthropic/api_server.py | 3 +- vllm/entrypoints/cli/serve.py | 2 +- vllm/entrypoints/openai/api_server.py | 3 +- vllm/lora/punica_wrapper/punica_gpu.py | 2 +- .../layers/quantization/mxfp4.py | 2 +- vllm/utils/__init__.py | 187 +++--------------- 12 files changed, 45 insertions(+), 262 deletions(-) rename tests/utils_/{test_utils.py => test_argparse_utils.py} (78%) diff --git a/docs/mkdocs/hooks/generate_argparse.py b/docs/mkdocs/hooks/generate_argparse.py index a4da5b933e159..99d9a7bec3994 100644 --- a/docs/mkdocs/hooks/generate_argparse.py +++ b/docs/mkdocs/hooks/generate_argparse.py @@ -65,7 +65,9 @@ ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand") CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand") cli_args = auto_mock("vllm.entrypoints.openai", "cli_args") run_batch = auto_mock("vllm.entrypoints.openai", "run_batch") -FlexibleArgumentParser = auto_mock("vllm.utils", "FlexibleArgumentParser") +FlexibleArgumentParser = auto_mock( + "vllm.utils.argparse_utils", "FlexibleArgumentParser" +) class MarkdownFormatter(HelpFormatter): diff --git a/tests/utils.py b/tests/utils.py index fb7614dd7fbce..af4ce6ebaeda2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -45,9 +45,7 @@ from vllm.entrypoints.cli.serve import ServeSubcommand from vllm.model_executor.model_loader import get_model_loader from vllm.platforms import current_platform from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.utils import ( - FlexibleArgumentParser, -) +from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.mem_constants import GB_bytes from vllm.utils.network_utils import get_open_port from vllm.utils.torch_utils import cuda_device_count_stateless diff --git a/tests/utils_/test_utils.py b/tests/utils_/test_argparse_utils.py similarity index 78% rename from tests/utils_/test_utils.py rename to tests/utils_/test_argparse_utils.py index 08dc7632b74b6..51684edcc8a30 100644 --- a/tests/utils_/test_utils.py +++ b/tests/utils_/test_argparse_utils.py @@ -4,23 +4,15 @@ import json import os -import tempfile -from pathlib import Path -from unittest.mock import patch import pytest -import torch import yaml from transformers import AutoTokenizer -from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config from vllm.transformers_utils.detokenizer_utils import convert_ids_list_to_tokens -from vllm.utils import ( - FlexibleArgumentParser, - bind_kv_cache, -) -from ..utils import create_new_process_for_each_test, flat_product +from vllm.utils.argparse_utils import FlexibleArgumentParser +from ..utils import flat_product # Tests for FlexibleArgumentParser @@ -256,87 +248,6 @@ def test_duplicate_dict_args(caplog_vllm, parser): assert "-O.mode" in caplog_vllm.text -def test_bind_kv_cache(): - from vllm.attention import Attention - - ctx = { - "layers.0.self_attn": Attention(32, 128, 0.1), - "layers.1.self_attn": Attention(32, 128, 0.1), - "layers.2.self_attn": Attention(32, 128, 0.1), - "layers.3.self_attn": Attention(32, 128, 0.1), - } - kv_cache = [ - torch.zeros((1,)), - torch.zeros((1,)), - torch.zeros((1,)), - torch.zeros((1,)), - ] - bind_kv_cache(ctx, [kv_cache]) - assert ctx["layers.0.self_attn"].kv_cache[0] is kv_cache[0] - assert ctx["layers.1.self_attn"].kv_cache[0] is kv_cache[1] - assert ctx["layers.2.self_attn"].kv_cache[0] is kv_cache[2] - assert ctx["layers.3.self_attn"].kv_cache[0] is kv_cache[3] - - -def test_bind_kv_cache_kv_sharing(): - from vllm.attention import Attention - - ctx = { - "layers.0.self_attn": Attention(32, 128, 0.1), - "layers.1.self_attn": Attention(32, 128, 0.1), - "layers.2.self_attn": Attention(32, 128, 0.1), - "layers.3.self_attn": Attention(32, 128, 0.1), - } - kv_cache = [ - torch.zeros((1,)), - torch.zeros((1,)), - torch.zeros((1,)), - torch.zeros((1,)), - ] - shared_kv_cache_layers = { - "layers.2.self_attn": "layers.1.self_attn", - "layers.3.self_attn": "layers.0.self_attn", - } - bind_kv_cache(ctx, [kv_cache], shared_kv_cache_layers) - assert ctx["layers.0.self_attn"].kv_cache[0] is kv_cache[0] - assert ctx["layers.1.self_attn"].kv_cache[0] is kv_cache[1] - assert ctx["layers.2.self_attn"].kv_cache[0] is kv_cache[1] - assert ctx["layers.3.self_attn"].kv_cache[0] is kv_cache[0] - - -def test_bind_kv_cache_non_attention(): - from vllm.attention import Attention - - # example from Jamba PP=2 - ctx = { - "model.layers.20.attn": Attention(32, 128, 0.1), - "model.layers.28.attn": Attention(32, 128, 0.1), - } - kv_cache = [ - torch.zeros((1,)), - torch.zeros((1,)), - ] - bind_kv_cache(ctx, [kv_cache]) - assert ctx["model.layers.20.attn"].kv_cache[0] is kv_cache[0] - assert ctx["model.layers.28.attn"].kv_cache[0] is kv_cache[1] - - -def test_bind_kv_cache_pp(): - with patch("vllm.utils.torch_utils.cuda_device_count_stateless", lambda: 2): - # this test runs with 1 GPU, but we simulate 2 GPUs - cfg = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=2)) - with set_current_vllm_config(cfg): - from vllm.attention import Attention - - ctx = { - "layers.0.self_attn": Attention(32, 128, 0.1), - } - kv_cache = [[torch.zeros((1,))], [torch.zeros((1,))]] - bind_kv_cache(ctx, kv_cache) - assert ctx["layers.0.self_attn"].kv_cache[0] is kv_cache[0][0] - assert ctx["layers.0.self_attn"].kv_cache[1] is kv_cache[1][0] - - def test_model_specification( parser_with_config, cli_config_file, cli_config_file_with_model ): diff --git a/tests/utils_/test_serial_utils.py b/tests/utils_/test_serial_utils.py index 7f2c1bdacf90e..51b2e4de02693 100644 --- a/tests/utils_/test_serial_utils.py +++ b/tests/utils_/test_serial_utils.py @@ -14,7 +14,7 @@ from vllm.utils.serial_utils import ( @pytest.mark.parametrize("endianness", ENDIANNESS) @pytest.mark.parametrize("embed_dtype", EMBED_DTYPE_TO_TORCH_DTYPE.keys()) -@torch.inference_mode +@torch.inference_mode() def test_encode_and_decode(embed_dtype: str, endianness: str): for i in range(10): tensor = torch.rand(2, 3, 5, 7, 11, 13, device="cpu", dtype=torch.float32) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index 17c00b9c3d0ef..ca251cd0c6ebd 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -42,7 +42,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import ( ) from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata from vllm.logger import init_logger -from vllm.utils import cdiv +from vllm.utils.math_utils import cdiv if TYPE_CHECKING: from vllm.attention.backends.abstract import AttentionMetadata diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py index 1f42b598bc9c8..3f60fbd6455a2 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py @@ -44,7 +44,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.lmcache_integration.utils impo ) from vllm.distributed.parallel_state import get_tensor_model_parallel_rank, get_tp_group from vllm.sampling_params import SamplingParams -from vllm.utils import cdiv, get_kv_cache_torch_dtype +from vllm.utils import get_kv_cache_torch_dtype +from vllm.utils.math_utils import cdiv from vllm.v1.core.sched.output import SchedulerOutput from vllm.version import __version__ as VLLM_VERSION diff --git a/vllm/entrypoints/anthropic/api_server.py b/vllm/entrypoints/anthropic/api_server.py index 249a7ee0121ad..b575dcdc8e773 100644 --- a/vllm/entrypoints/anthropic/api_server.py +++ b/vllm/entrypoints/anthropic/api_server.py @@ -51,7 +51,8 @@ from vllm.entrypoints.utils import ( with_cancellation, ) from vllm.logger import init_logger -from vllm.utils import FlexibleArgumentParser, set_ulimit +from vllm.utils import set_ulimit +from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import is_valid_ipv6_address from vllm.version import __version__ as VLLM_VERSION diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index e4ba66024135f..dc6f3df5a68ec 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -18,7 +18,7 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG from vllm.logger import init_logger from vllm.usage.usage_lib import UsageContext -from vllm.utils import FlexibleArgumentParser +from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import get_tcp_uri from vllm.utils.system_utils import decorate_logs, set_process_title from vllm.v1.engine.core import EngineCoreProc diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 29306e45bcf05..1a785e49df2bf 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -108,7 +108,8 @@ from vllm.entrypoints.utils import ( from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager from vllm.usage.usage_lib import UsageContext -from vllm.utils import Device, FlexibleArgumentParser, set_ulimit +from vllm.utils import Device, set_ulimit +from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import is_valid_ipv6_address from vllm.utils.system_utils import decorate_logs from vllm.v1.engine.exceptions import EngineDeadError diff --git a/vllm/lora/punica_wrapper/punica_gpu.py b/vllm/lora/punica_wrapper/punica_gpu.py index 0cbf294cf410a..d9590769778ea 100644 --- a/vllm/lora/punica_wrapper/punica_gpu.py +++ b/vllm/lora/punica_wrapper/punica_gpu.py @@ -13,7 +13,7 @@ import torch from vllm.lora.layers import LoRAMapping from vllm.triton_utils import HAS_TRITON, triton -from vllm.utils import round_up +from vllm.utils.math_utils import round_up if HAS_TRITON: from vllm.lora.ops.triton_ops import ( diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 6823fa02a32d7..6ffaa558887a1 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -48,9 +48,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_s from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform from vllm.scalar_type import scalar_types -from vllm.utils import round_up from vllm.utils.flashinfer import has_flashinfer from vllm.utils.import_utils import has_triton_kernels +from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import is_torch_equal_or_newer logger = init_logger(__name__) diff --git a/vllm/utils/__init__.py b/vllm/utils/__init__.py index 9cedea3461926..55efeb41fe53b 100644 --- a/vllm/utils/__init__.py +++ b/vllm/utils/__init__.py @@ -12,12 +12,10 @@ import signal import sys import tempfile import threading -import traceback import uuid import warnings -import weakref from collections.abc import Callable -from functools import cache, partial, wraps +from functools import partial, wraps from typing import TYPE_CHECKING, Any, TypeVar import cloudpickle @@ -28,34 +26,6 @@ import vllm.envs as envs from vllm.logger import enable_trace_function_call, init_logger from vllm.ray.lazy_utils import is_in_ray_actor -# Import utilities from specialized modules for backward compatibility -from vllm.utils.argparse_utils import ( - FlexibleArgumentParser, - SortedHelpFormatter, - StoreBoolean, -) -from vllm.utils.math_utils import ( - cdiv, - next_power_of_2, - prev_power_of_2, - round_down, - round_up, -) -from vllm.utils.platform_utils import cuda_is_initialized, xpu_is_initialized - -__all__ = [ - # Argparse utilities - "FlexibleArgumentParser", - "SortedHelpFormatter", - "StoreBoolean", - # Math utilities - "cdiv", - "next_power_of_2", - "prev_power_of_2", - "round_down", - "round_up", -] - _DEPRECATED_MAPPINGS = { "cprofile": "profiling", "cprofile_context": "profiling", @@ -84,12 +54,8 @@ def __dir__() -> list[str]: if TYPE_CHECKING: - from argparse import Namespace - from vllm.config import ModelConfig, VllmConfig else: - Namespace = object - ModelConfig = object VllmConfig = object @@ -149,37 +115,35 @@ class Counter: self.counter = 0 +class AtomicCounter: + """An atomic, thread-safe counter""" + + def __init__(self, initial=0): + """Initialize a new atomic counter to given initial value""" + self._value = initial + self._lock = threading.Lock() + + def inc(self, num=1): + """Atomically increment the counter by num and return the new value""" + with self._lock: + self._value += num + return self._value + + def dec(self, num=1): + """Atomically decrement the counter by num and return the new value""" + with self._lock: + self._value -= num + return self._value + + @property + def value(self): + return self._value + + def random_uuid() -> str: return str(uuid.uuid4().hex) -def update_environment_variables(envs: dict[str, str]): - for k, v in envs.items(): - if k in os.environ and os.environ[k] != v: - logger.warning( - "Overwriting environment variable %s from '%s' to '%s'", - k, - os.environ[k], - v, - ) - os.environ[k] = v - - -@cache -def is_pin_memory_available() -> bool: - from vllm.platforms import current_platform - - return current_platform.is_pin_memory_available() - - -@cache -def is_uva_available() -> bool: - """Check if Unified Virtual Addressing (UVA) is available.""" - # UVA requires pinned memory. - # TODO: Add more requirements for UVA if needed. - return is_pin_memory_available() - - # TODO: This function can be removed if transformer_modules classes are # serialized by value when communicating between processes def init_cached_hf_modules() -> None: @@ -212,47 +176,6 @@ def enable_trace_function_call_for_thread(vllm_config: VllmConfig) -> None: enable_trace_function_call(log_path) -def weak_bind( - bound_method: Callable[..., Any], -) -> Callable[..., None]: - """Make an instance method that weakly references - its associated instance and no-ops once that - instance is collected.""" - ref = weakref.ref(bound_method.__self__) # type: ignore[attr-defined] - unbound = bound_method.__func__ # type: ignore[attr-defined] - - def weak_bound(*args, **kwargs) -> None: - if inst := ref(): - unbound(inst, *args, **kwargs) - - return weak_bound - - -class AtomicCounter: - """An atomic, thread-safe counter""" - - def __init__(self, initial=0): - """Initialize a new atomic counter to given initial value""" - self._value = initial - self._lock = threading.Lock() - - def inc(self, num=1): - """Atomically increment the counter by num and return the new value""" - with self._lock: - self._value += num - return self._value - - def dec(self, num=1): - """Atomically decrement the counter by num and return the new value""" - with self._lock: - self._value -= num - return self._value - - @property - def value(self): - return self._value - - def kill_process_tree(pid: int): """ Kills all descendant processes of the given pid by sending SIGKILL. @@ -303,13 +226,6 @@ def set_ulimit(target_soft_limit=65535): ) -# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/utils.py#L28 # noqa: E501 -def get_exception_traceback(): - etype, value, tb = sys.exc_info() - err_str = "".join(traceback.format_exception(etype, value, tb)) - return err_str - - def _maybe_force_spawn(): """Check if we need to force the use of the `spawn` multiprocessing start method. @@ -327,6 +243,8 @@ def _maybe_force_spawn(): os.environ["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address reasons.append("In a Ray actor and can only be spawned") + from .platform_utils import cuda_is_initialized, xpu_is_initialized + if cuda_is_initialized(): reasons.append("CUDA is initialized") elif xpu_is_initialized(): @@ -356,55 +274,6 @@ def get_mp_context(): return multiprocessing.get_context(mp_method) -def bind_kv_cache( - ctx: dict[str, Any], - kv_cache: list[list[torch.Tensor]], # [virtual_engine][layer_index] - shared_kv_cache_layers: dict[str, str] | None = None, -) -> None: - # Bind the kv_cache tensor to Attention modules, similar to - # ctx[layer_name].kv_cache[ve]=kv_cache[ve][extract_layer_index(layer_name)] - # Special things handled here: - # 1. Some models have non-attention layers, e.g., Jamba - # 2. Pipeline parallelism, each rank only has a subset of layers - # 3. Encoder attention has no kv cache - # 4. Encoder-decoder models, encoder-decoder attention and decoder-only - # attention of the same layer (e.g., bart's decoder.layers.1.self_attn - # and decoder.layers.1.encoder_attn) is mapped to the same kv cache - # tensor - # 5. Some models have attention layers that share kv cache with previous - # layers, this is specified through shared_kv_cache_layers - if shared_kv_cache_layers is None: - shared_kv_cache_layers = {} - from vllm.attention import AttentionType - from vllm.model_executor.models.utils import extract_layer_index - - layer_need_kv_cache = [ - layer_name - for layer_name in ctx - if ( - hasattr(ctx[layer_name], "attn_type") - and ctx[layer_name].attn_type - in (AttentionType.DECODER, AttentionType.ENCODER_DECODER) - ) - and ctx[layer_name].kv_sharing_target_layer_name is None - ] - layer_index_sorted = sorted( - set(extract_layer_index(layer_name) for layer_name in layer_need_kv_cache) - ) - for layer_name in layer_need_kv_cache: - kv_cache_idx = layer_index_sorted.index(extract_layer_index(layer_name)) - forward_ctx = ctx[layer_name] - assert len(forward_ctx.kv_cache) == len(kv_cache) - for ve, ve_kv_cache in enumerate(kv_cache): - forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx] - if shared_kv_cache_layers is not None: - for layer_name, target_layer_name in shared_kv_cache_layers.items(): - assert extract_layer_index(target_layer_name) < extract_layer_index( - layer_name - ), "v0 doesn't support interleaving kv sharing" - ctx[layer_name].kv_cache = ctx[target_layer_name].kv_cache - - def run_method( obj: Any, method: str | bytes | Callable, From a3e8611da5744b1f64f3c4be063bf4a7aed952f0 Mon Sep 17 00:00:00 2001 From: Shanshan Shen <467638484@qq.com> Date: Mon, 27 Oct 2025 18:16:20 +0800 Subject: [PATCH 005/127] [Bugfix] Limit the default value of `max_model_len` when it is not specified by users (#27556) Signed-off-by: shen-shanshan <467638484@qq.com> --- vllm/config/model.py | 17 +++++------------ vllm/platforms/interface.py | 7 +++++++ vllm/platforms/tpu.py | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index f81d324d8f804..b32d820edd7b5 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -2112,20 +2112,13 @@ def _get_and_verify_max_len( if encoder_config and "max_seq_length" in encoder_config: derived_max_model_len = encoder_config["max_seq_length"] - # If the user specified a max length, make sure it is smaller than the - # derived length from the HF model config. + # If the user didn't specify `max_model_len`, then use that derived from + # the model config as a default value. if max_model_len is None: max_model_len = int(derived_max_model_len) - if current_platform.is_tpu(): - logger.warning( - "--max-model-len is not specified, " - "it's currently using model's default length %s, " - "which might be too large." - "Please input with --max-model-len based on your " - "request input length and output length, to avoid " - "unnecessary degradation.", - max_model_len, - ) + max_model_len = current_platform.check_max_model_len(max_model_len) + # If the user specified a max length, make sure it is smaller than the + # derived length from the HF model config. elif max_model_len > derived_max_model_len: # Some models might have a separate key for specifying model_max_length # that will be bigger than derived_max_model_len. We compare user input diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 1fb3aba9b1f79..4462829564391 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -608,6 +608,13 @@ class Platform: """ return None + @classmethod + def check_max_model_len(cls, max_model_len: int) -> int: + """ + Check max_model_len for the current platform. + """ + return max_model_len + class UnspecifiedPlatform(Platform): _enum = PlatformEnum.UNSPECIFIED diff --git a/vllm/platforms/tpu.py b/vllm/platforms/tpu.py index ab752f438f727..0a14ee011f7f2 100644 --- a/vllm/platforms/tpu.py +++ b/vllm/platforms/tpu.py @@ -251,6 +251,22 @@ class TpuPlatform(Platform): def use_sync_weight_loader(cls) -> bool: return True + @classmethod + def check_max_model_len(cls, max_model_len: int) -> int: + """ + Check max_model_len for the current platform. + """ + logger.warning( + "--max-model-len is not specified, " + "it's currently using model's default length %d, " + "which might be too large." + "Please input with --max-model-len based on your " + "request input length and output length, to avoid " + "unnecessary degradation.", + max_model_len, + ) + return max_model_len + try: from tpu_inference.platforms import TpuPlatform as TpuInferencePlatform From a4fc21895ed279930e5998eff4b9480da9d8442a Mon Sep 17 00:00:00 2001 From: Chauncey Date: Mon, 27 Oct 2025 19:06:43 +0800 Subject: [PATCH 006/127] [Bugfix] Fixed when return_token_ids=False, the first event still contains prompt_token_ids. (#27561) Signed-off-by: chaunceyjiang --- tests/entrypoints/openai/test_return_token_ids.py | 14 ++++++++++++-- vllm/entrypoints/openai/serving_completion.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/openai/test_return_token_ids.py b/tests/entrypoints/openai/test_return_token_ids.py index 60a80210fb768..feef48a36dfa1 100644 --- a/tests/entrypoints/openai/test_return_token_ids.py +++ b/tests/entrypoints/openai/test_return_token_ids.py @@ -27,8 +27,12 @@ def server(): @pytest.mark.asyncio -async def test_basic_completion_with_emoji(server): +@pytest.mark.parametrize("return_token_ids", [True, False, None]) +async def test_basic_completion_with_emoji(server, return_token_ids: bool | None): """Test basic completion with emoji to verify token_ids field.""" + extra_body = None + if return_token_ids is not None: + extra_body = {"return_token_ids": return_token_ids} async with server.get_async_client() as client: # Test with return_token_ids enabled completion = await client.completions.create( @@ -37,7 +41,7 @@ async def test_basic_completion_with_emoji(server): max_tokens=10, temperature=0, logprobs=1, - extra_body={"return_token_ids": True}, + extra_body=extra_body, ) # Check the raw response to see the structure @@ -45,6 +49,12 @@ async def test_basic_completion_with_emoji(server): # Verify prompt_token_ids field is present in the completion response assert "prompt_token_ids" in completion_dict["choices"][0] + if not return_token_ids: + # If return_token_ids is False, token_ids should not be present + assert completion_dict["choices"][0].get("token_ids") is None + assert completion_dict["choices"][0].get("prompt_token_ids") is None + # Skip further checks + return assert isinstance(completion.choices[0].prompt_token_ids, list) # Check against the expected prompt token IDs diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index 44211201d49a6..62bc932f8b844 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -399,7 +399,7 @@ class OpenAIServingCompletion(OpenAIServing): # has_echoed[i] is reused here to indicate whether # we have already returned the prompt token IDs. - if not has_echoed[i]: + if not has_echoed[i] and request.return_token_ids: prompt_token_ids_to_return = prompt_token_ids has_echoed[i] = True From a663f6ae64c88f0a708d3cb66b9918a659a6e868 Mon Sep 17 00:00:00 2001 From: Fadi Arafeh <115173828+fadara01@users.noreply.github.com> Date: Mon, 27 Oct 2025 11:14:55 +0000 Subject: [PATCH 007/127] [cpu][perf] Fix low CPU utilization with VLLM_CPU_OMP_THREADS_BIND on AArch64 (#27415) Signed-off-by: Fadi Arafeh --- cmake/cpu_extension.cmake | 21 +++++++++++++++++---- cmake/utils.cmake | 38 ++++++++++++++++++++++++++++++++++++++ vllm/platforms/cpu.py | 30 ++++++++++++++++++++++++++++-- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 4b8f0daacb007..192d349b30099 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -212,11 +212,24 @@ if ((AVX512_FOUND AND NOT AVX512_DISABLED) OR (ASIMD_FOUND AND NOT APPLE_SILICON # Build ACL with scons include(ProcessorCount) ProcessorCount(_NPROC) + set(_scons_cmd + scons -j${_NPROC} + Werror=0 debug=0 neon=1 examples=0 embed_kernels=0 os=linux + arch=armv8.2-a build=native benchmark_examples=0 fixed_format_kernels=1 + multi_isa=1 openmp=1 cppthreads=0 + ) + + # locate PyTorch's libgomp (e.g. site-packages/torch.libs/libgomp-947d5fa1.so.1.0.0) + # and create a local shim dir with it + include("${CMAKE_CURRENT_LIST_DIR}/utils.cmake") + vllm_prepare_torch_gomp_shim(VLLM_TORCH_GOMP_SHIM_DIR) + + if(NOT VLLM_TORCH_GOMP_SHIM_DIR STREQUAL "") + list(APPEND _scons_cmd extra_link_flags=-L${VLLM_TORCH_GOMP_SHIM_DIR}) + endif() + execute_process( - COMMAND scons -j${_NPROC} - Werror=0 debug=0 neon=1 examples=0 embed_kernels=0 os=linux - arch=armv8.2-a build=native benchmark_examples=0 fixed_format_kernels=1 - multi_isa=1 openmp=1 cppthreads=0 + COMMAND ${_scons_cmd} WORKING_DIRECTORY "$ENV{ACL_ROOT_DIR}" RESULT_VARIABLE _acl_rc ) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index f6a0d2b75be1a..c2181d4549236 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -129,6 +129,44 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG) set(${OUT_GPU_FLAGS} ${GPU_FLAGS} PARENT_SCOPE) endfunction() +# Find libgomp that gets shipped with PyTorch wheel and create a shim dir with: +# libgomp.so -> libgomp-.so... +# libgomp.so.1 -> libgomp-.so... +# OUTPUT: TORCH_GOMP_SHIM_DIR ("" if not found) +function(vllm_prepare_torch_gomp_shim TORCH_GOMP_SHIM_DIR) + set(${TORCH_GOMP_SHIM_DIR} "" PARENT_SCOPE) + + # Use run_python to locate vendored libgomp; never throw on failure. + run_python(_VLLM_TORCH_GOMP_PATH + " +import os, glob +try: + import torch + torch_pkg = os.path.dirname(torch.__file__) + site_root = os.path.dirname(torch_pkg) + torch_libs = os.path.join(site_root, 'torch.libs') + print(glob.glob(os.path.join(torch_libs, 'libgomp-*.so*'))[0]) +except: + print('') +" + "failed to probe torch.libs for libgomp") + + if(_VLLM_TORCH_GOMP_PATH STREQUAL "" OR NOT EXISTS "${_VLLM_TORCH_GOMP_PATH}") + return() + endif() + + # Create shim under the build tree + set(_shim "${CMAKE_BINARY_DIR}/gomp_shim") + file(MAKE_DIRECTORY "${_shim}") + + execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f "${_shim}/libgomp.so") + execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f "${_shim}/libgomp.so.1") + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink "${_VLLM_TORCH_GOMP_PATH}" "${_shim}/libgomp.so") + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink "${_VLLM_TORCH_GOMP_PATH}" "${_shim}/libgomp.so.1") + + set(${TORCH_GOMP_SHIM_DIR} "${_shim}" PARENT_SCOPE) +endfunction() + # Macro for converting a `gencode` version number to a cmake version number. macro(string_to_ver OUT_VER IN_STR) string(REGEX REPLACE "\([0-9]+\)\([0-9]\)" "\\1.\\2" ${OUT_VER} ${IN_STR}) diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 699a56be5cc4d..8c1d46564f6f6 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import glob import json import os import platform @@ -301,8 +302,8 @@ class CpuPlatform(Platform): os.environ["VLLM_DISABLE_SHARED_EXPERTS_STREAM"] = "0" # Intel OpenMP setting - ld_prealod_str = os.getenv("LD_PRELOAD", "") - if "libiomp5.so" in ld_prealod_str: + ld_preload_str = os.getenv("LD_PRELOAD", "") + if "libiomp5.so" in ld_preload_str: # The time(milliseconds) that a thread should wait after # completing the execution of a parallel region, before sleeping. os.environ["KMP_BLOCKTIME"] = "1" @@ -313,6 +314,31 @@ class CpuPlatform(Platform): os.environ["KMP_PLAIN_BARRIER_PATTERN"] = "dist,dist" os.environ["KMP_REDUCTION_BARRIER_PATTERN"] = "dist,dist" + if ( + platform.system() == "Linux" + and Platform.get_cpu_architecture() == CpuArchEnum.ARM + and not ("libomp" in ld_preload_str or "libgomp" in ld_preload_str) + ): + # We need to LD_PRELOAD PyTorch's libgomp, otherwise only + # one core will be properly utilized when we thread-bind + # See: https://github.com/vllm-project/vllm/issues/27369 + # TODO: Remove once: + # https://github.com/pytorch/pytorch/issues/166087 is fixed + + # We need to find the location of PyTorch's libgomp + torch_pkg = os.path.dirname(torch.__file__) + site_root = os.path.dirname(torch_pkg) + torch_libs = os.path.join(site_root, "torch.libs") + pytorch_libgomp_so_candidates = glob.glob( + os.path.join(torch_libs, "libgomp-*.so*") + ) + if pytorch_libgomp_so_candidates: + pytorch_libgomp_so = pytorch_libgomp_so_candidates[0] + if ld_preload_str: + ld_preload_str += ":" + ld_preload_str += pytorch_libgomp_so + os.environ["LD_PRELOAD"] = ld_preload_str + # To hint IPEX uses shared memory based AllReduce os.environ["LOCAL_WORLD_SIZE"] = str( vllm_config.parallel_config.tensor_parallel_size From f4e81540769848bbac8f22c356a121d3b3eb5038 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Mon, 27 Oct 2025 19:48:37 +0800 Subject: [PATCH 008/127] [Kernel] Enable moe LoRA kernel support FP16 (#27468) Signed-off-by: Jee Jee Li --- tests/lora/test_fused_moe_lora_kernel.py | 23 ++++++++++++++----- vllm/lora/ops/triton_ops/fused_moe_lora_op.py | 19 ++++++++------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/tests/lora/test_fused_moe_lora_kernel.py b/tests/lora/test_fused_moe_lora_kernel.py index 0ae992ad1110c..b724e112b9dd3 100644 --- a/tests/lora/test_fused_moe_lora_kernel.py +++ b/tests/lora/test_fused_moe_lora_kernel.py @@ -204,6 +204,11 @@ def use_torch( return torch.stack(outputs, dim=0) +DTYPES = [torch.float16, torch.bfloat16] +DEVICES = [f"cuda:{0}"] +SEED = [42] + + @pytest.mark.parametrize("num_tokens", [100]) @pytest.mark.parametrize("top_k_num", [6, 12]) @pytest.mark.parametrize("num_experts", [64]) @@ -212,6 +217,9 @@ def use_torch( @pytest.mark.parametrize("K", [2048]) @pytest.mark.parametrize("max_lora_rank", [16, 32, 64]) @pytest.mark.parametrize("block_size", [16]) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("seed", SEED) def test_fused_moe_lora_kernel( num_tokens, top_k_num, @@ -221,9 +229,12 @@ def test_fused_moe_lora_kernel( K, max_lora_rank, block_size, + dtype, + device, + seed, ): - torch.set_default_device("cuda:0") - current_platform.seed_everything(42) + torch.set_default_device(device) + current_platform.seed_everything(seed) # the number of randomly generated sentences. num_sequences = 10 # generate data @@ -240,7 +251,7 @@ def test_fused_moe_lora_kernel( max_lora_rank, K, ), - dtype=torch.bfloat16, + dtype=dtype, ) ] lora_b_stacked = [ @@ -251,7 +262,7 @@ def test_fused_moe_lora_kernel( N, max_lora_rank, ), - dtype=torch.bfloat16, + dtype=dtype, ) ] hidden_states = torch.rand( @@ -259,11 +270,11 @@ def test_fused_moe_lora_kernel( num_tokens, K, ), - dtype=torch.bfloat16, + dtype=dtype, ) # fused_moe_lora_kernel output - output = torch.zeros((num_tokens, top_k_num, N), dtype=torch.bfloat16) + output = torch.zeros((num_tokens, top_k_num, N), dtype=dtype) use_fused_moe_lora_kernel( topk_ids, topk_weights, diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py index 2031ade64b5fc..e681f3882908e 100644 --- a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py +++ b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py @@ -2,9 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -import triton -import triton.language as tl +from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import direct_register_custom_op _LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {} @@ -110,7 +109,7 @@ def _fused_moe_lora_kernel( # get a_ptr,b_ptr,c_ptr cur_a_ptr = a_ptr + (slice_id % num_slice_a) * slice_a_size - cur_b_ptr = tl.load(b_ptr + slice_id).to(tl.pointer_type(tl.bfloat16)) + cur_b_ptr = tl.load(b_ptr + slice_id).to(tl.pointer_type(c_ptr.dtype.element_ty)) cur_c_ptr = c_ptr + (slice_id % num_slice_c) * slice_c_size offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N @@ -154,7 +153,7 @@ def _fused_moe_lora_kernel( moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0) accumulator = accumulator * moe_weight[:, None] - accumulator = accumulator.to(tl.bfloat16) + accumulator = accumulator.to(c_ptr.dtype.element_ty) # Write back the block of the output offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) c_ptrs = cur_c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] @@ -205,6 +204,10 @@ def _fused_moe_lora( assert output.shape[0] == topk_weights.shape[0] assert top_k_num == topk_weights.shape[1] + for lora_a, lora_b in zip(lora_a_stacked, lora_b_stacked): + assert lora_a.dtype == lora_b.dtype == output.dtype == qcurr_hidden_states.dtype + assert lora_a.dtype in [torch.float16, torch.bfloat16] + device = qcurr_hidden_states.device num_slices = len(lora_a_stacked) @@ -227,9 +230,9 @@ def _fused_moe_lora( num_tokens = M * top_k_num w1_output_dim_size = w1_lora_b_stacked.shape[2] - lora_intermediate_cache1 = torch.zeros( + lora_intermediate_cache1 = torch.empty( (num_slices * M * top_k_num * (max_lora_rank + w1_output_dim_size)), - dtype=torch.bfloat16, + dtype=output.dtype, device=device, ) @@ -288,10 +291,6 @@ def _fused_moe_lora( K = max_lora_rank N = w1_output_dim_size - # a_intermediate_cache1 = a_intermediate_cache1.view( - # M, -1, a_intermediate_cache1.shape[3] - # ) - a_intermediate_cache1 = a_intermediate_cache1.view( -1, a_intermediate_cache1.shape[3] ) From 9273754222d4e4ada346cdff4f0f4cd967155e28 Mon Sep 17 00:00:00 2001 From: Asaf Joseph Gardin <39553475+Josephasafg@users.noreply.github.com> Date: Mon, 27 Oct 2025 15:05:20 +0200 Subject: [PATCH 009/127] [Hybrid] Added supports_mamba_prefix_caching Protocol (#27339) Signed-off-by: asafg <39553475+Josephasafg@users.noreply.github.com> --- vllm/config/model.py | 4 +++ vllm/model_executor/models/bamba.py | 17 +++++++++-- vllm/model_executor/models/config.py | 11 +------- vllm/model_executor/models/falcon_h1.py | 17 +++++++++-- .../model_executor/models/granitemoehybrid.py | 17 +++++++++-- vllm/model_executor/models/interfaces.py | 28 +++++++++++++++++++ vllm/model_executor/models/mamba2.py | 10 +++++-- vllm/model_executor/models/nemotron_h.py | 2 ++ vllm/model_executor/models/registry.py | 3 ++ vllm/model_executor/models/zamba2.py | 4 +-- 10 files changed, 93 insertions(+), 20 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index b32d820edd7b5..adb0dd9ac9f5c 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1656,6 +1656,10 @@ class ModelConfig: def has_inner_state(self): return self._model_info.has_inner_state + @property + def supports_mamba_prefix_caching(self) -> bool: + return self._model_info.supports_mamba_prefix_caching + @property def use_mla(self) -> bool: return self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE diff --git a/vllm/model_executor/models/bamba.py b/vllm/model_executor/models/bamba.py index 1a06f0659235e..151fb3b6acc46 100644 --- a/vllm/model_executor/models/bamba.py +++ b/vllm/model_executor/models/bamba.py @@ -37,7 +37,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors -from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP, SupportsQuant +from .interfaces import ( + HasInnerState, + IsHybrid, + SupportsLoRA, + SupportsMambaPrefixCaching, + SupportsPP, + SupportsQuant, +) from .utils import ( AutoWeightsLoader, is_pp_missing_parameter, @@ -394,7 +401,13 @@ class BambaModel(nn.Module): class BambaForCausalLM( - nn.Module, HasInnerState, SupportsLoRA, SupportsPP, IsHybrid, SupportsQuant + nn.Module, + HasInnerState, + SupportsLoRA, + SupportsPP, + IsHybrid, + SupportsQuant, + SupportsMambaPrefixCaching, ): packed_modules_mapping = { "qkv_proj": [ diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index d2f9f1b0b5c06..493b74bddda7a 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -295,17 +295,8 @@ class MambaModelConfig(VerifyAndUpdateConfig): # override by prefix caching logic later) cache_config.mamba_block_size = model_config.max_model_len - # TODO(@tdoublep) find a better way to do this than whitelist - MAMBA2_MODELS = [ - "BambaForCausalLM", - "FalconH1ForCausalLM", - "GraniteMoeHybridForCausalLM", - "Mamba2ForCausalLM", - "NemotronHForCausalLM", - "Zamba2ForCausalLM", - ] if cache_config.enable_prefix_caching: - if model_config.architecture in MAMBA2_MODELS: + if model_config.supports_mamba_prefix_caching: logger.info( "Warning: Prefix caching is currently enabled. " "Its support for Mamba2 layers is experimental. " diff --git a/vllm/model_executor/models/falcon_h1.py b/vllm/model_executor/models/falcon_h1.py index 4e0b6b52fc647..8bf700b474a41 100644 --- a/vllm/model_executor/models/falcon_h1.py +++ b/vllm/model_executor/models/falcon_h1.py @@ -37,7 +37,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors -from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP +from .interfaces import ( + HasInnerState, + IsHybrid, + SupportsLoRA, + SupportsMambaPrefixCaching, + SupportsPP, +) from .utils import ( PPMissingLayer, is_pp_missing_parameter, @@ -495,7 +501,14 @@ class FalconH1Model(nn.Module): return hidden_states -class FalconH1ForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP, IsHybrid): +class FalconH1ForCausalLM( + nn.Module, + HasInnerState, + SupportsLoRA, + SupportsPP, + IsHybrid, + SupportsMambaPrefixCaching, +): packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], diff --git a/vllm/model_executor/models/granitemoehybrid.py b/vllm/model_executor/models/granitemoehybrid.py index 1bb7f4e9b8023..bac64eec8c558 100644 --- a/vllm/model_executor/models/granitemoehybrid.py +++ b/vllm/model_executor/models/granitemoehybrid.py @@ -34,7 +34,14 @@ from vllm.sequence import IntermediateTensors from .granitemoe import GraniteMoeMoE from .granitemoeshared import GraniteMoeSharedMLP -from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP, SupportsQuant +from .interfaces import ( + HasInnerState, + IsHybrid, + SupportsLoRA, + SupportsMambaPrefixCaching, + SupportsPP, + SupportsQuant, +) from .utils import ( AutoWeightsLoader, is_pp_missing_parameter, @@ -584,7 +591,13 @@ class GraniteMoeHybridModel(nn.Module): class GraniteMoeHybridForCausalLM( - nn.Module, HasInnerState, SupportsLoRA, SupportsPP, IsHybrid, SupportsQuant + nn.Module, + HasInnerState, + SupportsLoRA, + SupportsPP, + IsHybrid, + SupportsQuant, + SupportsMambaPrefixCaching, ): packed_modules_mapping = { "qkv_proj": [ diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 1bc5f5ae5419f..e133206c27a8b 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -697,6 +697,34 @@ def has_noops( return getattr(model, "has_noops", False) +@runtime_checkable +class SupportsMambaPrefixCaching(Protocol): + """The interface for models whose mamba layers support prefix caching. + + This is currently experimental. + """ + + supports_mamba_prefix_caching: ClassVar[Literal[True]] = True + + +@overload +def supports_mamba_prefix_caching( + model: object, +) -> TypeIs[SupportsMambaPrefixCaching]: ... + + +@overload +def supports_mamba_prefix_caching( + model: type[object], +) -> TypeIs[type[SupportsMambaPrefixCaching]]: ... + + +def supports_mamba_prefix_caching( + model: type[object] | object, +) -> TypeIs[type[SupportsMambaPrefixCaching]] | TypeIs[SupportsMambaPrefixCaching]: + return getattr(model, "supports_mamba_prefix_caching", False) + + @runtime_checkable class SupportsCrossEncoding(Protocol): """The interface required for all models that support cross encoding.""" diff --git a/vllm/model_executor/models/mamba2.py b/vllm/model_executor/models/mamba2.py index 5eb21b966e187..8ba8af66635b3 100644 --- a/vllm/model_executor/models/mamba2.py +++ b/vllm/model_executor/models/mamba2.py @@ -25,7 +25,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm.model_executor.models.interfaces import HasInnerState, IsAttentionFree +from vllm.model_executor.models.interfaces import ( + HasInnerState, + IsAttentionFree, + SupportsMambaPrefixCaching, +) from vllm.sequence import IntermediateTensors from .utils import ( @@ -189,7 +193,9 @@ class Mamba2Model(nn.Module): return loaded_params -class Mamba2ForCausalLM(nn.Module, HasInnerState, IsAttentionFree): +class Mamba2ForCausalLM( + nn.Module, HasInnerState, IsAttentionFree, SupportsMambaPrefixCaching +): @classmethod def get_mamba_state_dtype_from_config( cls, diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index f31579e5cfa82..457d3910d0e57 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -62,6 +62,7 @@ from vllm.model_executor.models.interfaces import ( IsHybrid, MixtureOfExperts, SupportsLoRA, + SupportsMambaPrefixCaching, SupportsPP, SupportsQuant, ) @@ -695,6 +696,7 @@ class NemotronHForCausalLM( IsHybrid, SupportsQuant, MixtureOfExperts, + SupportsMambaPrefixCaching, ): hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={"backbone": "model"}, diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index e8212ef6d72d8..0027954ac2771 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -39,6 +39,7 @@ from .interfaces import ( is_attention_free, is_hybrid, supports_cross_encoding, + supports_mamba_prefix_caching, supports_multimodal, supports_multimodal_encoder_tp_data, supports_multimodal_raw_input_only, @@ -496,6 +497,7 @@ class _ModelInfo: is_attention_free: bool is_hybrid: bool has_noops: bool + supports_mamba_prefix_caching: bool supports_transcription: bool supports_transcription_only: bool @@ -518,6 +520,7 @@ class _ModelInfo: has_inner_state=has_inner_state(model), is_attention_free=is_attention_free(model), is_hybrid=is_hybrid(model), + supports_mamba_prefix_caching=supports_mamba_prefix_caching(model), supports_transcription=supports_transcription(model), supports_transcription_only=( supports_transcription(model) and model.supports_transcription_only diff --git a/vllm/model_executor/models/zamba2.py b/vllm/model_executor/models/zamba2.py index 2610aa253b575..a6cfcf509776f 100644 --- a/vllm/model_executor/models/zamba2.py +++ b/vllm/model_executor/models/zamba2.py @@ -45,7 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors -from .interfaces import HasInnerState, IsHybrid +from .interfaces import HasInnerState, IsHybrid, SupportsMambaPrefixCaching from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix @@ -824,7 +824,7 @@ class Zamba2Model(nn.Module): return loaded_params -class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid): +class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixCaching): """Zamba2 model with causal language modeling head. This class wraps the core Zamba2 model and adds: From 4f882be4a0d319551ce2a0eadcace0e76f3432cd Mon Sep 17 00:00:00 2001 From: Yu Jiaqi <54204033+piood@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:57:37 +0800 Subject: [PATCH 010/127] [Model] Siglip2 Model Support (#27566) Signed-off-by: piood <2477084691@qq.com> --- docs/models/supported_models.md | 2 +- tests/models/multimodal/pooling/test_siglip.py | 2 +- vllm/model_executor/models/siglip.py | 8 +++++--- vllm/transformers_utils/config.py | 12 +++++++++++- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 5da561c83b2cc..4d50c809d1966 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -775,7 +775,7 @@ The following table lists those that are tested in vLLM. | `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | | | `LlavaNextForConditionalGeneration`C | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ | | `Phi3VForCausalLM`C | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ | -| `SiglipModel` | SigLIP | T / I | `google/siglip-base-patch16-224` | | | +| `SiglipModel` | SigLIP, SigLIP2 | T / I | `google/siglip-base-patch16-224`, `google/siglip2-base-patch16-224` | | | | `*ForConditionalGeneration`C, `*ForCausalLM`C, etc. | Generative models | \* | N/A | \* | \* | C Automatically converted into an embedding model via `--convert embed`. ([details](./pooling_models.md#model-conversion)) diff --git a/tests/models/multimodal/pooling/test_siglip.py b/tests/models/multimodal/pooling/test_siglip.py index f681b4787b697..3345b10c099ac 100644 --- a/tests/models/multimodal/pooling/test_siglip.py +++ b/tests/models/multimodal/pooling/test_siglip.py @@ -19,7 +19,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts( } ) -MODELS = ["google/siglip-base-patch16-224"] +MODELS = ["google/siglip-base-patch16-224", "google/siglip2-base-patch16-224"] def _run_test( diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 694e06f9fc811..e363be523dcce 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -174,9 +174,11 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]): @cached_property def image_token_id(self) -> int: tokenizer = self.info.get_tokenizer() - dummy_token_id = 0 - - assert dummy_token_id not in tokenizer.all_special_ids + dummy_token_id = next( + token_id + for token_id in range(tokenizer.vocab_size) + if token_id not in tokenizer.all_special_ids + ) return dummy_token_id diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 7802cece6075a..13de5939356e9 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -26,7 +26,10 @@ from huggingface_hub.utils import ( ) from transformers import GenerationConfig, PretrainedConfig from transformers.models.auto.image_processing_auto import get_image_processor_config -from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES +from transformers.models.auto.modeling_auto import ( + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, + MODEL_MAPPING_NAMES, +) from transformers.models.auto.tokenization_auto import get_tokenizer_config from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME @@ -616,6 +619,13 @@ def get_config( model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type] config.update({"architectures": [model_type]}) + # Architecture mapping for models without explicit architectures field + if not config.architectures: + if config.model_type not in MODEL_MAPPING_NAMES: + raise ValueError(f"Cannot find architecture name for {config.model_type}") + model_type = MODEL_MAPPING_NAMES[config.model_type] + config.update({"architectures": [model_type]}) + # ModelOpt 0.31.0 and after saves the quantization config in the model # config file. quantization_config = config_dict.get("quantization_config", None) From 5d3be3ba4c109f3ce5f0ffb78245296f3b9aa5ac Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Mon, 27 Oct 2025 10:32:50 -0400 Subject: [PATCH 011/127] [Bugfix][LoRA][FusedMoE] Select MxFP4 Backend based on LoRA Enablement (#27487) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath --- .../model_executor/layers/fused_moe/config.py | 2 ++ vllm/model_executor/layers/fused_moe/layer.py | 15 +++++++++--- .../layers/quantization/mxfp4.py | 23 ++++++++++++++++--- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 5403d4e62f85e..2394053329802 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -825,6 +825,8 @@ class FusedMoEConfig: is_act_and_mul: bool = True + is_lora_enabled: bool = False + def __post_init__(self): if self.dp_size > 1: logger.debug_once( diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index c144aa23e46e4..9b826f05fe307 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -982,6 +982,7 @@ def maybe_roundup_hidden_size( act_dtype: torch.dtype, quant_config: QuantizationConfig | None, moe_parallel_config: FusedMoEParallelConfig, + is_lora_enabled: bool, ) -> int: """ Given layer hidden size and MoE configurations, round up hidden_size @@ -992,6 +993,9 @@ def maybe_roundup_hidden_size( act_dtype: Data type of the layer activations. quant_config: Fused MoE quantization configuration. moe_parallel_config: Fused MoE parallelization strategy configuration. + is_lora_enabled: True if the engine is enabled with LoRA. This + is used in the case of mxfp4 quantization in selecting the + MxFP4Backend. Return: Rounded up hidden_size if rounding up is required based on the configs. @@ -1015,7 +1019,7 @@ def maybe_roundup_hidden_size( get_mxfp4_backend, ) - current_mxfp4_backend = get_mxfp4_backend() + current_mxfp4_backend = get_mxfp4_backend(is_lora_enabled) if ( current_mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16 or current_mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS @@ -1139,7 +1143,11 @@ class FusedMoE(CustomOp): # Round up hidden size if needed. hidden_size = maybe_roundup_hidden_size( - hidden_size, moe_in_dtype, quant_config, self.moe_parallel_config + hidden_size, + moe_in_dtype, + quant_config, + self.moe_parallel_config, + is_lora_enabled=self.vllm_config.lora_config is not None, ) # For smuggling this layer into the fused moe custom op @@ -1270,8 +1278,9 @@ class FusedMoE(CustomOp): max_num_tokens=envs.VLLM_MOE_DP_CHUNK_SIZE, has_bias=has_bias, is_act_and_mul=is_act_and_mul, + is_lora_enabled=vllm_config.lora_config is not None, ) - self.moe_config = moe + self.moe_config: FusedMoEConfig = moe self.moe_quant_config: FusedMoEQuantConfig | None = None self.quant_config = quant_config diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 6ffaa558887a1..597ee1b6bafe1 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -73,8 +73,24 @@ class Mxfp4Backend(Enum): TRITON = 6 -def get_mxfp4_backend(): +def get_mxfp4_backend_with_lora() -> Mxfp4Backend: + """ + Not all MXFP4 backends support LoRA. Select backends that are known to + have LoRA support. + """ + if not current_platform.is_cuda(): + return Mxfp4Backend.NONE + + logger.info_once("[get_mxfp4_backend_with_lora] Using Marlin backend") + return Mxfp4Backend.MARLIN + + +def get_mxfp4_backend(with_lora_support: bool) -> Mxfp4Backend: # Backend Selection + + if with_lora_support: + return get_mxfp4_backend_with_lora() + if current_platform.is_cuda(): if ( current_platform.is_device_capability(90) @@ -183,13 +199,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): super().__init__(moe) self.topk_indices_dtype = None self.moe = moe - self.mxfp4_backend = get_mxfp4_backend() + self.mxfp4_backend = get_mxfp4_backend(moe.is_lora_enabled) self.max_capture_size = ( get_current_vllm_config().compilation_config.max_cudagraph_capture_size ) assert self.mxfp4_backend != Mxfp4Backend.NONE, ( - "No MXFP4 MoE backend (FlashInfer/Marlin/Triton) available." + f"get_mxfp4_backend(with_lora_support={moe.is_lora_enabled}) found" + "no compatible MXFP4 MoE backend (FlashInfer/Marlin/Triton)." "Please check your environment and try again." ) self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {} From 23ad820553845c546ecbe9914abd799e92e45ce6 Mon Sep 17 00:00:00 2001 From: tingtinggithub Date: Mon, 27 Oct 2025 07:34:01 -0700 Subject: [PATCH 012/127] fixing mm placeholder replacement issue with gemma3 (#27538) Signed-off-by: tingtingtang1992 --- vllm/model_executor/models/gemma3_mm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 7c628fe93ce36..748605b4ed5ac 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -403,7 +403,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]): def get_repl_toks(tok: int) -> list[int]: if tok == newline_3: - return [newline_1, newline_2] + return [newline_2, newline_1] if tok == newline_4: return [newline_2, newline_2] From 3b96f85c36991d9384e8ea5f3d103ef96d932205 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Mon, 27 Oct 2025 11:06:25 -0400 Subject: [PATCH 013/127] [Chore]: Stream tokens vs characters in tool call parser tests (#26513) Signed-off-by: Ben Browning --- .../openai/tool_parsers/conftest.py | 12 +++++++ .../test_llama3_json_tool_parser.py | 8 ++--- .../test_llama4_pythonic_tool_parser.py | 24 +++++++------- .../tool_parsers/test_olmo3_tool_parser.py | 32 ++++++++++++------- .../tool_parsers/test_pythonic_tool_parser.py | 24 +++++++------- .../entrypoints/openai/tool_parsers/utils.py | 21 ++++++++++++ 6 files changed, 80 insertions(+), 41 deletions(-) create mode 100644 tests/entrypoints/openai/tool_parsers/conftest.py diff --git a/tests/entrypoints/openai/tool_parsers/conftest.py b/tests/entrypoints/openai/tool_parsers/conftest.py new file mode 100644 index 0000000000000..f2ac5e5b9a8fa --- /dev/null +++ b/tests/entrypoints/openai/tool_parsers/conftest.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +from transformers import AutoTokenizer + +from vllm.transformers_utils.tokenizer import AnyTokenizer + + +@pytest.fixture(scope="function") +def default_tokenizer() -> AnyTokenizer: + return AutoTokenizer.from_pretrained("gpt2") diff --git a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py index c7a8ef83cf71d..2b68a653f4600 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py @@ -2,17 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import Llama3JsonToolParser +from vllm.transformers_utils.tokenizer import AnyTokenizer @pytest.fixture -def parser(): - # Use a small tokenizer for testing - tokenizer = AutoTokenizer.from_pretrained("gpt2") - return Llama3JsonToolParser(tokenizer) +def parser(default_tokenizer: AnyTokenizer): + return Llama3JsonToolParser(default_tokenizer) def test_extract_tool_calls_simple(parser): diff --git a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py index 94277980f229f..d297432eab644 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py @@ -11,6 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import ( ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager +from vllm.transformers_utils.tokenizer import AnyTokenizer # Test cases similar to pythonic parser but with Llama4 specific format SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]" @@ -63,10 +64,9 @@ PYTHON_TAG_FUNCTION_OUTPUT = ( @pytest.mark.parametrize("streaming", [True, False]) -def test_no_tool_call(streaming: bool): - mock_tokenizer = MagicMock() +def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer): tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( - mock_tokenizer + default_tokenizer ) model_output = "How can I help you today?" @@ -205,11 +205,13 @@ TEST_CASES = [ @pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES) def test_tool_call( - streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall] + streaming: bool, + model_output: str, + expected_tool_calls: list[FunctionCall], + default_tokenizer: AnyTokenizer, ): - mock_tokenizer = MagicMock() tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( - mock_tokenizer + default_tokenizer ) content, tool_calls = run_tool_extraction( @@ -222,10 +224,9 @@ def test_tool_call( assert actual.function == expected -def test_streaming_tool_call_with_large_steps(): - mock_tokenizer = MagicMock() +def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( - mock_tokenizer + default_tokenizer ) model_output_deltas = [ "<|python_start|>[get_weather(city='LA', metric='C'), " @@ -245,11 +246,10 @@ def test_streaming_tool_call_with_large_steps(): @pytest.mark.parametrize("streaming", [False]) -def test_regex_timeout_handling(streaming: bool): +def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer): """test regex timeout is handled gracefully""" - mock_tokenizer = MagicMock() tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( - mock_tokenizer + default_tokenizer ) fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2 diff --git a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py index 224196b9a0b2e..13cff9a8ebf1e 100644 --- a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py @@ -11,6 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import ( ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager +from vllm.transformers_utils.tokenizer import AnyTokenizer # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" @@ -68,9 +69,10 @@ ESCAPED_STRING_FUNCTION_CALL = FunctionCall( @pytest.mark.parametrize("streaming", [True, False]) -def test_no_tool_call(streaming: bool): - mock_tokenizer = MagicMock() - tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer) +def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer): + tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( + default_tokenizer + ) model_output = "How can I help you today?" content, tool_calls = run_tool_extraction( @@ -183,10 +185,14 @@ TEST_CASES = [ @pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES) def test_tool_call( - streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall] + streaming: bool, + model_output: str, + expected_tool_calls: list[FunctionCall], + default_tokenizer: AnyTokenizer, ): - mock_tokenizer = MagicMock() - tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer) + tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( + default_tokenizer + ) content, tool_calls = run_tool_extraction( tool_parser, model_output, streaming=streaming @@ -199,9 +205,10 @@ def test_tool_call( assert actual.function == expected -def test_streaming_tool_call_with_large_steps(): - mock_tokenizer = MagicMock() - tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer) +def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): + tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( + default_tokenizer + ) model_output_deltas = [ "get_weather(city='San", " Francisco', metric='celsius')\n" @@ -221,10 +228,11 @@ def test_streaming_tool_call_with_large_steps(): @pytest.mark.parametrize("streaming", [False]) -def test_regex_timeout_handling(streaming: bool): +def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer): """test regex timeout is handled gracefully""" - mock_tokenizer = MagicMock() - tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(mock_tokenizer) + tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( + default_tokenizer + ) fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2 diff --git a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py index d7b4051ea572a..fcd3df16e5cfa 100644 --- a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py @@ -11,6 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import ( ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager +from vllm.transformers_utils.tokenizer import AnyTokenizer # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" @@ -60,10 +61,9 @@ ESCAPED_STRING_FUNCTION_CALL = FunctionCall( @pytest.mark.parametrize("streaming", [True, False]) -def test_no_tool_call(streaming: bool): - mock_tokenizer = MagicMock() +def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer): tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( - mock_tokenizer + default_tokenizer ) model_output = "How can I help you today?" @@ -165,11 +165,13 @@ TEST_CASES = [ @pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES) def test_tool_call( - streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall] + streaming: bool, + model_output: str, + expected_tool_calls: list[FunctionCall], + default_tokenizer: AnyTokenizer, ): - mock_tokenizer = MagicMock() tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( - mock_tokenizer + default_tokenizer ) content, tool_calls = run_tool_extraction( @@ -183,10 +185,9 @@ def test_tool_call( assert actual.function == expected -def test_streaming_tool_call_with_large_steps(): - mock_tokenizer = MagicMock() +def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( - mock_tokenizer + default_tokenizer ) model_output_deltas = [ "[get_weather(city='San", @@ -207,11 +208,10 @@ def test_streaming_tool_call_with_large_steps(): @pytest.mark.parametrize("streaming", [False]) -def test_regex_timeout_handling(streaming: bool): +def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer): """test regex timeout is handled gracefully""" - mock_tokenizer = MagicMock() tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( - mock_tokenizer + default_tokenizer ) fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2 diff --git a/tests/entrypoints/openai/tool_parsers/utils.py b/tests/entrypoints/openai/tool_parsers/utils.py index 7489a406224a5..38899f2632554 100644 --- a/tests/entrypoints/openai/tool_parsers/utils.py +++ b/tests/entrypoints/openai/tool_parsers/utils.py @@ -11,6 +11,7 @@ from vllm.entrypoints.openai.protocol import ( ToolCall, ) from vllm.entrypoints.openai.tool_parsers import ToolParser +from vllm.transformers_utils.tokenizer import AnyTokenizer class StreamingToolReconstructor: @@ -110,12 +111,32 @@ def run_tool_extraction_nonstreaming( return tool_parser.extract_tool_calls(model_output, request) +def split_string_into_token_deltas(tokenizer: AnyTokenizer, text: str) -> list[str]: + # Split a string into a series of deltas using the provided tokenizer. Each + # delta will be the string equivalent of a single token. + token_ids = tokenizer.encode(text, add_special_tokens=False) + previously_decoded_text = "" + deltas = [] + for i in range(1, len(token_ids) + 1): + current_tokens = token_ids[:i] + current_text = tokenizer.decode(current_tokens) + new_text = current_text[len(previously_decoded_text) :] + previously_decoded_text = current_text + deltas.append(new_text) + return deltas + + def run_tool_extraction_streaming( tool_parser: ToolParser, model_deltas: Iterable[str], request: ChatCompletionRequest | None = None, assert_one_tool_per_delta: bool = True, ) -> StreamingToolReconstructor: + if isinstance(model_deltas, str): + model_deltas = split_string_into_token_deltas( + tool_parser.model_tokenizer, model_deltas + ) + request = request or ChatCompletionRequest(messages=[], model="test-model") reconstructor = StreamingToolReconstructor( assert_one_tool_per_delta=assert_one_tool_per_delta From 6ebffafbb609963d696d27c5e334fd0b9fe0add7 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Mon, 27 Oct 2025 23:30:38 +0800 Subject: [PATCH 014/127] [Misc] Clean up more utils (#27567) Signed-off-by: DarkLight1337 --- requirements/docs.txt | 2 + tools/pre_commit/check_pickle_imports.py | 1 - vllm/config/model.py | 23 +++ vllm/config/vllm.py | 28 ++- vllm/engine/arg_utils.py | 3 +- vllm/entrypoints/anthropic/api_server.py | 2 +- vllm/entrypoints/api_server.py | 3 +- vllm/entrypoints/openai/api_server.py | 4 +- vllm/platforms/__init__.py | 2 +- vllm/platforms/cuda.py | 2 +- vllm/utils/__init__.py | 246 +---------------------- vllm/utils/argparse_utils.py | 26 +-- vllm/utils/import_utils.py | 43 ++++ vllm/utils/system_utils.py | 122 ++++++++++- vllm/v1/engine/coordinator.py | 3 +- vllm/v1/engine/utils.py | 2 +- vllm/v1/executor/multiproc_executor.py | 8 +- vllm/v1/executor/uniproc_executor.py | 2 +- vllm/v1/serial_utils.py | 28 +++ vllm/v1/utils.py | 2 +- vllm/v1/worker/gpu_model_runner.py | 7 +- vllm/v1/worker/gpu_worker.py | 2 +- vllm/v1/worker/tpu_worker.py | 2 +- vllm/v1/worker/worker_base.py | 34 ++-- 24 files changed, 282 insertions(+), 315 deletions(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index d1c546398780a..00c314874016f 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -13,6 +13,8 @@ ruff # Required for argparse hook only -f https://download.pytorch.org/whl/cpu cachetools +cloudpickle +py-cpuinfo msgspec pydantic torch diff --git a/tools/pre_commit/check_pickle_imports.py b/tools/pre_commit/check_pickle_imports.py index c9256cd91a4ee..b96a6701333de 100644 --- a/tools/pre_commit/check_pickle_imports.py +++ b/tools/pre_commit/check_pickle_imports.py @@ -39,7 +39,6 @@ ALLOWED_FILES = { "vllm/v1/executor/multiproc_executor.py", "vllm/v1/executor/ray_executor.py", "vllm/entrypoints/llm.py", - "vllm/utils/__init__.py", "tests/utils.py", # pickle and cloudpickle "vllm/v1/serial_utils.py", diff --git a/vllm/config/model.py b/vllm/config/model.py index adb0dd9ac9f5c..c335c5c25e9e2 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1618,6 +1618,29 @@ class ModelConfig: """Extract the HF encoder/decoder model flag.""" return is_encoder_decoder(self.hf_config) + @property + def uses_alibi(self) -> bool: + cfg = self.hf_text_config + + return ( + getattr(cfg, "alibi", False) # Falcon + or "BloomForCausalLM" in self.architectures # Bloom + or getattr(cfg, "position_encoding_type", "") == "alibi" # codellm_1b_alibi + or ( + hasattr(cfg, "attn_config") # MPT + and ( + ( + isinstance(cfg.attn_config, dict) + and cfg.attn_config.get("alibi", False) + ) + or ( + not isinstance(cfg.attn_config, dict) + and getattr(cfg.attn_config, "alibi", False) + ) + ) + ) + ) + @property def uses_mrope(self) -> bool: return uses_mrope(self.hf_config) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 916f258d6586c..597cf57939636 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2,12 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy +import getpass import hashlib import json import os +import tempfile +import threading import time from contextlib import contextmanager from dataclasses import replace +from datetime import datetime from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING, Any, TypeVar @@ -17,7 +21,7 @@ from pydantic import ConfigDict, Field from pydantic.dataclasses import dataclass import vllm.envs as envs -from vllm.logger import init_logger +from vllm.logger import enable_trace_function_call, init_logger from vllm.transformers_utils.runai_utils import is_runai_obj_uri from vllm.utils import random_uuid @@ -206,6 +210,28 @@ class VllmConfig: # i.e., batch_size <= self.compilation_config.max_cudagraph_capture_size return self.compilation_config.bs_to_padded_graph_size[batch_size] + def enable_trace_function_call_for_thread(self) -> None: + """ + Set up function tracing for the current thread, + if enabled via the `VLLM_TRACE_FUNCTION` environment variable. + """ + if envs.VLLM_TRACE_FUNCTION: + tmp_dir = tempfile.gettempdir() + # add username to tmp_dir to avoid permission issues + tmp_dir = os.path.join(tmp_dir, getpass.getuser()) + filename = ( + f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}" + f"_thread_{threading.get_ident()}_at_{datetime.now()}.log" + ).replace(" ", "_") + log_path = os.path.join( + tmp_dir, + "vllm", + f"vllm-instance-{self.instance_id}", + filename, + ) + os.makedirs(os.path.dirname(log_path), exist_ok=True) + enable_trace_function_call(log_path) + @staticmethod def _get_quantization_config( model_config: ModelConfig, load_config: LoadConfig diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 617c464cff257..24f9d18dc958a 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -73,7 +73,7 @@ from vllm.config.utils import get_field from vllm.logger import init_logger from vllm.platforms import CpuArchEnum, current_platform from vllm.plugins import load_general_plugins -from vllm.ray.lazy_utils import is_ray_initialized +from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized from vllm.reasoning import ReasoningParserManager from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3 from vllm.transformers_utils.config import ( @@ -82,7 +82,6 @@ from vllm.transformers_utils.config import ( maybe_override_with_speculators, ) from vllm.transformers_utils.utils import check_gguf_file -from vllm.utils import is_in_ray_actor from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.mem_constants import GiB_bytes from vllm.utils.network_utils import get_ip diff --git a/vllm/entrypoints/anthropic/api_server.py b/vllm/entrypoints/anthropic/api_server.py index b575dcdc8e773..df877f99b084f 100644 --- a/vllm/entrypoints/anthropic/api_server.py +++ b/vllm/entrypoints/anthropic/api_server.py @@ -51,9 +51,9 @@ from vllm.entrypoints.utils import ( with_cancellation, ) from vllm.logger import init_logger -from vllm.utils import set_ulimit from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import is_valid_ipv6_address +from vllm.utils.system_utils import set_ulimit from vllm.version import __version__ as VLLM_VERSION prometheus_multiproc_dir: tempfile.TemporaryDirectory diff --git a/vllm/entrypoints/api_server.py b/vllm/entrypoints/api_server.py index 184cc47ceb834..154cdeb42a3ea 100644 --- a/vllm/entrypoints/api_server.py +++ b/vllm/entrypoints/api_server.py @@ -26,8 +26,9 @@ from vllm.entrypoints.utils import with_cancellation from vllm.logger import init_logger from vllm.sampling_params import SamplingParams from vllm.usage.usage_lib import UsageContext -from vllm.utils import random_uuid, set_ulimit +from vllm.utils import random_uuid from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.system_utils import set_ulimit from vllm.version import __version__ as VLLM_VERSION logger = init_logger("vllm.entrypoints.api_server") diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 1a785e49df2bf..632bd741290bd 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -108,10 +108,10 @@ from vllm.entrypoints.utils import ( from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager from vllm.usage.usage_lib import UsageContext -from vllm.utils import Device, set_ulimit +from vllm.utils import Device from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import is_valid_ipv6_address -from vllm.utils.system_utils import decorate_logs +from vllm.utils.system_utils import decorate_logs, set_ulimit from vllm.v1.engine.exceptions import EngineDeadError from vllm.v1.metrics.prometheus import get_prometheus_registry from vllm.version import __version__ as VLLM_VERSION diff --git a/vllm/platforms/__init__.py b/vllm/platforms/__init__.py index f64d7a010b5f0..badf72de4a90f 100644 --- a/vllm/platforms/__init__.py +++ b/vllm/platforms/__init__.py @@ -60,7 +60,7 @@ def cuda_platform_plugin() -> str | None: is_cuda = False logger.debug("Checking if CUDA platform is available.") try: - from vllm.utils import import_pynvml + from vllm.utils.import_utils import import_pynvml pynvml = import_pynvml() pynvml.nvmlInit() diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index 637f35a4920e0..66cffde9503da 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -16,7 +16,7 @@ from typing_extensions import ParamSpec import vllm._C # noqa import vllm.envs as envs from vllm.logger import init_logger -from vllm.utils import import_pynvml +from vllm.utils.import_utils import import_pynvml from vllm.utils.torch_utils import cuda_device_count_stateless from .interface import DeviceCapability, Platform, PlatformEnum diff --git a/vllm/utils/__init__.py b/vllm/utils/__init__.py index 55efeb41fe53b..eaa78839cf3f6 100644 --- a/vllm/utils/__init__.py +++ b/vllm/utils/__init__.py @@ -1,34 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import contextlib -import datetime import enum -import getpass import inspect -import multiprocessing -import os -import signal -import sys -import tempfile import threading import uuid import warnings -from collections.abc import Callable -from functools import partial, wraps -from typing import TYPE_CHECKING, Any, TypeVar +from functools import wraps +from typing import Any, TypeVar -import cloudpickle -import psutil import torch -import vllm.envs as envs -from vllm.logger import enable_trace_function_call, init_logger -from vllm.ray.lazy_utils import is_in_ray_actor +from vllm.logger import init_logger _DEPRECATED_MAPPINGS = { "cprofile": "profiling", "cprofile_context": "profiling", + # Used by lm-eval "get_open_port": "network_utils", } @@ -53,12 +41,6 @@ def __dir__() -> list[str]: return sorted(list(globals().keys()) + list(_DEPRECATED_MAPPINGS.keys())) -if TYPE_CHECKING: - from vllm.config import ModelConfig, VllmConfig -else: - ModelConfig = object - VllmConfig = object - logger = init_logger(__name__) # This value is chosen to have a balance between ITL and TTFT. Note it is @@ -83,13 +65,7 @@ STR_FLASH_ATTN_VAL: str = "FLASH_ATTN" STR_INVALID_VAL: str = "INVALID" -# ANSI color codes -CYAN = "\033[1;36m" -RESET = "\033[0;0m" - - T = TypeVar("T") -U = TypeVar("U") class Device(enum.Enum): @@ -144,195 +120,6 @@ def random_uuid() -> str: return str(uuid.uuid4().hex) -# TODO: This function can be removed if transformer_modules classes are -# serialized by value when communicating between processes -def init_cached_hf_modules() -> None: - """ - Lazy initialization of the Hugging Face modules. - """ - from transformers.dynamic_module_utils import init_hf_modules - - init_hf_modules() - - -def enable_trace_function_call_for_thread(vllm_config: VllmConfig) -> None: - """Set up function tracing for the current thread, - if enabled via the VLLM_TRACE_FUNCTION environment variable - """ - - if envs.VLLM_TRACE_FUNCTION: - tmp_dir = tempfile.gettempdir() - # add username to tmp_dir to avoid permission issues - tmp_dir = os.path.join(tmp_dir, getpass.getuser()) - filename = ( - f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}" - f"_thread_{threading.get_ident()}_" - f"at_{datetime.datetime.now()}.log" - ).replace(" ", "_") - log_path = os.path.join( - tmp_dir, "vllm", f"vllm-instance-{vllm_config.instance_id}", filename - ) - os.makedirs(os.path.dirname(log_path), exist_ok=True) - enable_trace_function_call(log_path) - - -def kill_process_tree(pid: int): - """ - Kills all descendant processes of the given pid by sending SIGKILL. - - Args: - pid (int): Process ID of the parent process - """ - try: - parent = psutil.Process(pid) - except psutil.NoSuchProcess: - return - - # Get all children recursively - children = parent.children(recursive=True) - - # Send SIGKILL to all children first - for child in children: - with contextlib.suppress(ProcessLookupError): - os.kill(child.pid, signal.SIGKILL) - - # Finally kill the parent - with contextlib.suppress(ProcessLookupError): - os.kill(pid, signal.SIGKILL) - - -# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L630 # noqa: E501 -def set_ulimit(target_soft_limit=65535): - if sys.platform.startswith("win"): - logger.info("Windows detected, skipping ulimit adjustment.") - return - - import resource - - resource_type = resource.RLIMIT_NOFILE - current_soft, current_hard = resource.getrlimit(resource_type) - - if current_soft < target_soft_limit: - try: - resource.setrlimit(resource_type, (target_soft_limit, current_hard)) - except ValueError as e: - logger.warning( - "Found ulimit of %s and failed to automatically increase " - "with error %s. This can cause fd limit errors like " - "`OSError: [Errno 24] Too many open files`. Consider " - "increasing with ulimit -n", - current_soft, - e, - ) - - -def _maybe_force_spawn(): - """Check if we need to force the use of the `spawn` multiprocessing start - method. - """ - if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") == "spawn": - return - - reasons = [] - if is_in_ray_actor(): - # even if we choose to spawn, we need to pass the ray address - # to the subprocess so that it knows how to connect to the ray cluster. - # env vars are inherited by subprocesses, even if we use spawn. - import ray - - os.environ["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address - reasons.append("In a Ray actor and can only be spawned") - - from .platform_utils import cuda_is_initialized, xpu_is_initialized - - if cuda_is_initialized(): - reasons.append("CUDA is initialized") - elif xpu_is_initialized(): - reasons.append("XPU is initialized") - - if reasons: - logger.warning( - "We must use the `spawn` multiprocessing start method. " - "Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. " - "See https://docs.vllm.ai/en/latest/usage/" - "troubleshooting.html#python-multiprocessing " - "for more information. Reasons: %s", - "; ".join(reasons), - ) - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - - -def get_mp_context(): - """Get a multiprocessing context with a particular method (spawn or fork). - By default we follow the value of the VLLM_WORKER_MULTIPROC_METHOD to - determine the multiprocessing method (default is fork). However, under - certain conditions, we may enforce spawn and override the value of - VLLM_WORKER_MULTIPROC_METHOD. - """ - _maybe_force_spawn() - mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD - return multiprocessing.get_context(mp_method) - - -def run_method( - obj: Any, - method: str | bytes | Callable, - args: tuple[Any], - kwargs: dict[str, Any], -) -> Any: - """ - Run a method of an object with the given arguments and keyword arguments. - If the method is string, it will be converted to a method using getattr. - If the method is serialized bytes and will be deserialized using - cloudpickle. - If the method is a callable, it will be called directly. - """ - if isinstance(method, bytes): - func = partial(cloudpickle.loads(method), obj) - elif isinstance(method, str): - try: - func = getattr(obj, method) - except AttributeError: - raise NotImplementedError( - f"Method {method!r} is not implemented." - ) from None - else: - func = partial(method, obj) # type: ignore - return func(*args, **kwargs) - - -def import_pynvml(): - """ - Historical comments: - - libnvml.so is the library behind nvidia-smi, and - pynvml is a Python wrapper around it. We use it to get GPU - status without initializing CUDA context in the current process. - Historically, there are two packages that provide pynvml: - - `nvidia-ml-py` (https://pypi.org/project/nvidia-ml-py/): The official - wrapper. It is a dependency of vLLM, and is installed when users - install vLLM. It provides a Python module named `pynvml`. - - `pynvml` (https://pypi.org/project/pynvml/): An unofficial wrapper. - Prior to version 12.0, it also provides a Python module `pynvml`, - and therefore conflicts with the official one. What's worse, - the module is a Python package, and has higher priority than - the official one which is a standalone Python file. - This causes errors when both of them are installed. - Starting from version 12.0, it migrates to a new module - named `pynvml_utils` to avoid the conflict. - It is so confusing that many packages in the community use the - unofficial one by mistake, and we have to handle this case. - For example, `nvcr.io/nvidia/pytorch:24.12-py3` uses the unofficial - one, and it will cause errors, see the issue - https://github.com/vllm-project/vllm/issues/12847 for example. - After all the troubles, we decide to copy the official `pynvml` - module to our codebase, and use it directly. - """ - import vllm.third_party.pynvml as pynvml - - return pynvml - - def warn_for_unimplemented_methods(cls: type[T]) -> type[T]: """ A replacement for `abc.ABC`. @@ -376,31 +163,6 @@ def warn_for_unimplemented_methods(cls: type[T]) -> type[T]: return cls -# Only relevant for models using ALiBi (e.g, MPT) -def check_use_alibi(model_config: ModelConfig) -> bool: - cfg = model_config.hf_text_config - return ( - getattr(cfg, "alibi", False) # Falcon - or ( - "BloomForCausalLM" in getattr(model_config.hf_config, "architectures", []) - ) # Bloom - or getattr(cfg, "position_encoding_type", "") == "alibi" # codellm_1b_alibi - or ( - hasattr(cfg, "attn_config") # MPT - and ( - ( - isinstance(cfg.attn_config, dict) - and cfg.attn_config.get("alibi", False) - ) - or ( - not isinstance(cfg.attn_config, dict) - and getattr(cfg.attn_config, "alibi", False) - ) - ) - ) - ) - - def length_from_prompt_token_ids_or_embeds( prompt_token_ids: list[int] | None, prompt_embeds: torch.Tensor | None, diff --git a/vllm/utils/argparse_utils.py b/vllm/utils/argparse_utils.py index 0007c72f1e389..3d105a3685b37 100644 --- a/vllm/utils/argparse_utils.py +++ b/vllm/utils/argparse_utils.py @@ -10,37 +10,21 @@ from argparse import ( ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError, + Namespace, RawDescriptionHelpFormatter, _ArgumentGroup, ) from collections import defaultdict -from typing import TYPE_CHECKING, Any +from typing import Any import regex as re import yaml from vllm.logger import init_logger -if TYPE_CHECKING: - from argparse import Namespace -else: - Namespace = object - logger = init_logger(__name__) -class StoreBoolean(Action): - def __call__(self, parser, namespace, values, option_string=None): - if values.lower() == "true": - setattr(namespace, self.dest, True) - elif values.lower() == "false": - setattr(namespace, self.dest, False) - else: - raise ValueError( - f"Invalid boolean value: {values}. Expected 'true' or 'false'." - ) - - class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter): """SortedHelpFormatter that sorts arguments by their option strings.""" @@ -487,12 +471,8 @@ class FlexibleArgumentParser(ArgumentParser): ) raise ex - store_boolean_arguments = [ - action.dest for action in self._actions if isinstance(action, StoreBoolean) - ] - for key, value in config.items(): - if isinstance(value, bool) and key not in store_boolean_arguments: + if isinstance(value, bool): if value: processed_args.append("--" + key) elif isinstance(value, list): diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 65f588b52e5e6..409a5a6cd302d 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -19,6 +19,49 @@ import regex as re from typing_extensions import Never +# TODO: This function can be removed if transformer_modules classes are +# serialized by value when communicating between processes +def init_cached_hf_modules() -> None: + """ + Lazy initialization of the Hugging Face modules. + """ + from transformers.dynamic_module_utils import init_hf_modules + + init_hf_modules() + + +def import_pynvml(): + """ + Historical comments: + + libnvml.so is the library behind nvidia-smi, and + pynvml is a Python wrapper around it. We use it to get GPU + status without initializing CUDA context in the current process. + Historically, there are two packages that provide pynvml: + - `nvidia-ml-py` (https://pypi.org/project/nvidia-ml-py/): The official + wrapper. It is a dependency of vLLM, and is installed when users + install vLLM. It provides a Python module named `pynvml`. + - `pynvml` (https://pypi.org/project/pynvml/): An unofficial wrapper. + Prior to version 12.0, it also provides a Python module `pynvml`, + and therefore conflicts with the official one. What's worse, + the module is a Python package, and has higher priority than + the official one which is a standalone Python file. + This causes errors when both of them are installed. + Starting from version 12.0, it migrates to a new module + named `pynvml_utils` to avoid the conflict. + It is so confusing that many packages in the community use the + unofficial one by mistake, and we have to handle this case. + For example, `nvcr.io/nvidia/pytorch:24.12-py3` uses the unofficial + one, and it will cause errors, see the issue + https://github.com/vllm-project/vllm/issues/12847 for example. + After all the troubles, we decide to copy the official `pynvml` + module to our codebase, and use it directly. + """ + import vllm.third_party.pynvml as pynvml + + return pynvml + + def import_from_path(module_name: str, file_path: str | os.PathLike): """ Import a Python file according to its file path. diff --git a/vllm/utils/system_utils.py b/vllm/utils/system_utils.py index dd18adf55e1fc..5968884e232a4 100644 --- a/vllm/utils/system_utils.py +++ b/vllm/utils/system_utils.py @@ -4,19 +4,21 @@ from __future__ import annotations import contextlib +import multiprocessing import os +import signal import sys from collections.abc import Callable, Iterator from pathlib import Path from typing import TextIO -try: - import setproctitle -except ImportError: - setproctitle = None # type: ignore[assignment] +import psutil import vllm.envs as envs from vllm.logger import init_logger +from vllm.ray.lazy_utils import is_in_ray_actor + +from .platform_utils import cuda_is_initialized, xpu_is_initialized logger = init_logger(__name__) @@ -75,14 +77,66 @@ def unique_filepath(fn: Callable[[int], Path]) -> Path: # Process management utilities +def _maybe_force_spawn(): + """Check if we need to force the use of the `spawn` multiprocessing start + method. + """ + if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") == "spawn": + return + + reasons = [] + if is_in_ray_actor(): + # even if we choose to spawn, we need to pass the ray address + # to the subprocess so that it knows how to connect to the ray cluster. + # env vars are inherited by subprocesses, even if we use spawn. + import ray + + os.environ["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address + reasons.append("In a Ray actor and can only be spawned") + + if cuda_is_initialized(): + reasons.append("CUDA is initialized") + elif xpu_is_initialized(): + reasons.append("XPU is initialized") + + if reasons: + logger.warning( + "We must use the `spawn` multiprocessing start method. " + "Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. " + "See https://docs.vllm.ai/en/latest/usage/" + "troubleshooting.html#python-multiprocessing " + "for more information. Reasons: %s", + "; ".join(reasons), + ) + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + + +def get_mp_context(): + """Get a multiprocessing context with a particular method (spawn or fork). + By default we follow the value of the VLLM_WORKER_MULTIPROC_METHOD to + determine the multiprocessing method (default is fork). However, under + certain conditions, we may enforce spawn and override the value of + VLLM_WORKER_MULTIPROC_METHOD. + """ + _maybe_force_spawn() + mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD + return multiprocessing.get_context(mp_method) + + def set_process_title( - name: str, suffix: str = "", prefix: str = envs.VLLM_PROCESS_NAME_PREFIX + name: str, + suffix: str = "", + prefix: str = envs.VLLM_PROCESS_NAME_PREFIX, ) -> None: """Set the current process title with optional suffix.""" - if setproctitle is None: + try: + import setproctitle + except ImportError: return + if suffix: name = f"{name}_{suffix}" + setproctitle.setproctitle(f"{prefix}::{name}") @@ -114,10 +168,62 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None: def decorate_logs(process_name: str | None = None) -> None: """Decorate stdout/stderr with process name and PID prefix.""" - from vllm.utils import get_mp_context - if process_name is None: process_name = get_mp_context().current_process().name + pid = os.getpid() _add_prefix(sys.stdout, process_name, pid) _add_prefix(sys.stderr, process_name, pid) + + +def kill_process_tree(pid: int): + """ + Kills all descendant processes of the given pid by sending SIGKILL. + + Args: + pid (int): Process ID of the parent process + """ + try: + parent = psutil.Process(pid) + except psutil.NoSuchProcess: + return + + # Get all children recursively + children = parent.children(recursive=True) + + # Send SIGKILL to all children first + for child in children: + with contextlib.suppress(ProcessLookupError): + os.kill(child.pid, signal.SIGKILL) + + # Finally kill the parent + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + + +# Resource utilities + + +# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L630 +def set_ulimit(target_soft_limit: int = 65535): + if sys.platform.startswith("win"): + logger.info("Windows detected, skipping ulimit adjustment.") + return + + import resource + + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + logger.warning( + "Found ulimit of %s and failed to automatically increase " + "with error %s. This can cause fd limit errors like " + "`OSError: [Errno 24] Too many open files`. Consider " + "increasing with ulimit -n", + current_soft, + e, + ) diff --git a/vllm/v1/engine/coordinator.py b/vllm/v1/engine/coordinator.py index 39d8655ff8587..953342cdd5d05 100644 --- a/vllm/v1/engine/coordinator.py +++ b/vllm/v1/engine/coordinator.py @@ -10,9 +10,8 @@ import zmq from vllm.config import ParallelConfig from vllm.logger import init_logger -from vllm.utils import get_mp_context from vllm.utils.network_utils import make_zmq_socket -from vllm.utils.system_utils import set_process_title +from vllm.utils.system_utils import get_mp_context, set_process_title from vllm.v1.engine import EngineCoreOutputs, EngineCoreRequestType from vllm.v1.serial_utils import MsgpackDecoder from vllm.v1.utils import get_engine_client_zmq_addr, shutdown diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index ca416dbc0df9e..bdc124b0571c0 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -20,8 +20,8 @@ from vllm.config import CacheConfig, ParallelConfig, VllmConfig from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.ray.ray_env import get_env_vars_to_copy -from vllm.utils import get_mp_context from vllm.utils.network_utils import get_open_zmq_ipc_path, zmq_socket_ctx +from vllm.utils.system_utils import get_mp_context from vllm.v1.engine.coordinator import DPCoordinator from vllm.v1.executor import Executor from vllm.v1.utils import get_engine_client_zmq_addr, shutdown diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 1b4b9c4550f74..4c58d5771c39b 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -35,13 +35,17 @@ from vllm.distributed.parallel_state import ( ) from vllm.envs import enable_envs_cache from vllm.logger import init_logger -from vllm.utils import _maybe_force_spawn, get_mp_context from vllm.utils.network_utils import ( get_distributed_init_method, get_loopback_ip, get_open_port, ) -from vllm.utils.system_utils import decorate_logs, set_process_title +from vllm.utils.system_utils import ( + _maybe_force_spawn, + decorate_logs, + get_mp_context, + set_process_title, +) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.executor.abstract import Executor, FailureCallback from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput diff --git a/vllm/v1/executor/uniproc_executor.py b/vllm/v1/executor/uniproc_executor.py index 0d072172fdf3e..f17d3c3092701 100644 --- a/vllm/v1/executor/uniproc_executor.py +++ b/vllm/v1/executor/uniproc_executor.py @@ -12,11 +12,11 @@ import torch.distributed as dist import vllm.envs as envs from vllm.logger import init_logger -from vllm.utils import run_method from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType from vllm.v1.executor.abstract import Executor from vllm.v1.outputs import AsyncModelRunnerOutput +from vllm.v1.serial_utils import run_method from vllm.v1.worker.worker_base import WorkerWrapperBase logger = init_logger(__name__) diff --git a/vllm/v1/serial_utils.py b/vllm/v1/serial_utils.py index 39147a67d6cf3..102357ca7c642 100644 --- a/vllm/v1/serial_utils.py +++ b/vllm/v1/serial_utils.py @@ -5,6 +5,7 @@ import dataclasses import importlib import pickle from collections.abc import Callable, Sequence +from functools import partial from inspect import isclass from types import FunctionType from typing import Any, TypeAlias @@ -429,3 +430,30 @@ class MsgpackDecoder: return cloudpickle.loads(data) raise NotImplementedError(f"Extension type code {code} is not supported") + + +def run_method( + obj: Any, + method: str | bytes | Callable, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Any: + """ + Run a method of an object with the given arguments and keyword arguments. + If the method is string, it will be converted to a method using getattr. + If the method is serialized bytes and will be deserialized using + cloudpickle. + If the method is a callable, it will be called directly. + """ + if isinstance(method, bytes): + func = partial(cloudpickle.loads(method), obj) + elif isinstance(method, str): + try: + func = getattr(obj, method) + except AttributeError: + raise NotImplementedError( + f"Method {method!r} is not implemented." + ) from None + else: + func = partial(method, obj) # type: ignore + return func(*args, **kwargs) diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index 789a74cc6c4a5..a401f6d74cdd5 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -25,8 +25,8 @@ from torch.autograd.profiler import record_function import vllm.envs as envs from vllm.logger import init_logger from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message -from vllm.utils import kill_process_tree from vllm.utils.network_utils import get_open_port, get_open_zmq_ipc_path, get_tcp_uri +from vllm.utils.system_utils import kill_process_tree if TYPE_CHECKING: import numpy as np diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 6759fe630e625..a110ad54a05ea 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -69,10 +69,7 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingType from vllm.sequence import IntermediateTensors from vllm.tasks import GenerationTask, PoolingTask, SupportedTask -from vllm.utils import ( - check_use_alibi, - length_from_prompt_token_ids_or_embeds, -) +from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.utils.jsontree import json_map_leaves from vllm.utils.math_utils import cdiv, round_up from vllm.utils.mem_constants import GiB_bytes @@ -266,7 +263,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): self.hidden_size = model_config.get_hidden_size() self.attention_chunk_size = model_config.attention_chunk_size # Only relevant for models using ALiBi (e.g, MPT) - self.use_alibi = check_use_alibi(model_config) + self.use_alibi = model_config.uses_alibi self.cascade_attn_enabled = not self.model_config.disable_cascade_attn diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 3ed9cab42a14f..29b6532e4366f 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -72,7 +72,7 @@ class Worker(WorkerBase): if self.model_config.trust_remote_code: # note: lazy import to avoid importing torch before initializing - from vllm.utils import init_cached_hf_modules + from vllm.utils.import_utils import init_cached_hf_modules init_cached_hf_modules() diff --git a/vllm/v1/worker/tpu_worker.py b/vllm/v1/worker/tpu_worker.py index f1885f9b34a13..e867e3c07caa5 100644 --- a/vllm/v1/worker/tpu_worker.py +++ b/vllm/v1/worker/tpu_worker.py @@ -89,7 +89,7 @@ class TPUWorker: if self.model_config.trust_remote_code: # note: lazy import to avoid importing torch before initializing - from vllm.utils import init_cached_hf_modules + from vllm.utils.import_utils import init_cached_hf_modules init_cached_hf_modules() diff --git a/vllm/v1/worker/worker_base.py b/vllm/v1/worker/worker_base.py index d912589ef73a4..9162e2e85a517 100644 --- a/vllm/v1/worker/worker_base.py +++ b/vllm/v1/worker/worker_base.py @@ -13,14 +13,11 @@ from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import worker_receiver_cache_from_config -from vllm.utils import ( - enable_trace_function_call_for_thread, - run_method, - warn_for_unimplemented_methods, -) +from vllm.utils import warn_for_unimplemented_methods from vllm.utils.import_utils import resolve_obj_by_qualname from vllm.utils.system_utils import update_environment_variables from vllm.v1.kv_cache_interface import KVCacheSpec +from vllm.v1.serial_utils import run_method if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput @@ -182,19 +179,20 @@ class WorkerWrapperBase: """ self.rpc_rank = rpc_rank self.worker: WorkerBase | None = None - self.vllm_config: VllmConfig | None = None - # do not store this `vllm_config`, `init_worker` will set the final - # one. TODO: investigate if we can remove this field in - # `WorkerWrapperBase`, `init_cached_hf_modules` should be - # unnecessary now. - if vllm_config.model_config is not None: - # it can be None in tests - trust_remote_code = vllm_config.model_config.trust_remote_code - if trust_remote_code: - # note: lazy import to avoid importing torch before initializing - from vllm.utils import init_cached_hf_modules - init_cached_hf_modules() + # do not store this `vllm_config`, `init_worker` will set the final + # one. + # TODO: investigate if we can remove this field in `WorkerWrapperBase`, + # `init_cached_hf_modules` should be unnecessary now. + self.vllm_config: VllmConfig | None = None + + # `model_config` can be None in tests + model_config = vllm_config.model_config + if model_config and model_config.trust_remote_code: + # note: lazy import to avoid importing torch before initializing + from vllm.utils.import_utils import init_cached_hf_modules + + init_cached_hf_modules() def shutdown(self) -> None: if self.worker is not None: @@ -231,7 +229,7 @@ class WorkerWrapperBase: assert self.vllm_config is not None, ( "vllm_config is required to initialize the worker" ) - enable_trace_function_call_for_thread(self.vllm_config) + self.vllm_config.enable_trace_function_call_for_thread() from vllm.plugins import load_general_plugins From 921e78f4bbe56d6c9a0e9e609b9b3cfe97a4a51b Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Mon, 27 Oct 2025 12:22:33 -0500 Subject: [PATCH 015/127] [ROCm] Update AITER branch for ROCm base docker (#27586) Signed-off-by: Micah Williamson --- docker/Dockerfile.rocm_base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index 5479eebaf7953..19f7fa7e1468d 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -7,7 +7,7 @@ ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git" ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" -ARG AITER_BRANCH="eef23c7f" +ARG AITER_BRANCH="9716b1b8" ARG AITER_REPO="https://github.com/ROCm/aiter.git" FROM ${BASE_IMAGE} AS base From 69f064062ba78a0ac44962f55a46a9d79cfb9ce0 Mon Sep 17 00:00:00 2001 From: usberkeley <150880684+usberkeley@users.noreply.github.com> Date: Tue, 28 Oct 2025 01:50:22 +0800 Subject: [PATCH 016/127] Code quality improvements: version update, type annotation enhancement, and enum usage simplification (#27581) Signed-off-by: Bradley --- docs/deployment/docker.md | 4 ++-- vllm/multimodal/profiling.py | 2 +- vllm/v1/core/sched/scheduler.py | 10 ++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/deployment/docker.md b/docs/deployment/docker.md index d07358b85a5e4..1c639f3533d47 100644 --- a/docs/deployment/docker.md +++ b/docs/deployment/docker.md @@ -41,11 +41,11 @@ You can add any other [engine-args](../configuration/engine_args.md) you need af create a custom Dockerfile on top of the base image with an extra layer that installs them: ```Dockerfile - FROM vllm/vllm-openai:v0.9.0 + FROM vllm/vllm-openai:v0.11.0 # e.g. install the `audio` optional dependencies # NOTE: Make sure the version of vLLM matches the base image! - RUN uv pip install --system vllm[audio]==0.9.0 + RUN uv pip install --system vllm[audio]==0.11.0 ``` !!! tip diff --git a/vllm/multimodal/profiling.py b/vllm/multimodal/profiling.py index f55bad569e166..b864c52dfbc8b 100644 --- a/vllm/multimodal/profiling.py +++ b/vllm/multimodal/profiling.py @@ -368,7 +368,7 @@ class MultiModalProfiler(Generic[_I]): self, seq_len: int, mm_counts: Mapping[str, int] | None = None, - ): + ) -> Mapping[str, int]: """ Returns the maximum length of the multimodal (image placeholders+text) tokens, including any break/text tokens in-between image embeddings. diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 7afee15a2da6b..14bdf295317d7 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -113,14 +113,12 @@ class Scheduler(SchedulerInterface): # req_id -> Request self.requests: dict[str, Request] = {} # Scheduling policy - if self.scheduler_config.policy == "priority": - self.policy = SchedulingPolicy.PRIORITY - elif self.scheduler_config.policy == "fcfs": - self.policy = SchedulingPolicy.FCFS - else: + try: + self.policy = SchedulingPolicy(self.scheduler_config.policy) + except ValueError as e: raise ValueError( f"Unknown scheduling policy: {self.scheduler_config.policy}" - ) + ) from e # Priority queues for requests. self.waiting = create_request_queue(self.policy) self.running: list[Request] = [] From 53a56e658b9ab1eabb7339d3001dc3fd9178dd21 Mon Sep 17 00:00:00 2001 From: Andrew Xia Date: Mon, 27 Oct 2025 16:15:49 -0700 Subject: [PATCH 017/127] [gpt-oss][2/N] Support input_messages in responsesRequest (#26962) Signed-off-by: Andrew Xia Co-authored-by: Andrew Xia --- .../openai/test_response_api_with_harmony.py | 129 +++++++++ .../openai/test_serving_responses.py | 22 ++ tests/entrypoints/test_harmony_utils.py | 254 ++++++++++++++++++ vllm/entrypoints/harmony_utils.py | 39 ++- vllm/entrypoints/openai/protocol.py | 8 +- vllm/entrypoints/openai/serving_chat.py | 36 +-- vllm/entrypoints/openai/serving_responses.py | 10 + 7 files changed, 481 insertions(+), 17 deletions(-) create mode 100644 tests/entrypoints/test_harmony_utils.py diff --git a/tests/entrypoints/openai/test_response_api_with_harmony.py b/tests/entrypoints/openai/test_response_api_with_harmony.py index 4251d06435c11..dea8d2d28f61a 100644 --- a/tests/entrypoints/openai/test_response_api_with_harmony.py +++ b/tests/entrypoints/openai/test_response_api_with_harmony.py @@ -535,11 +535,17 @@ def get_place_to_travel(): return "Paris" +def get_horoscope(sign): + return f"{sign}: Next Tuesday you will befriend a baby otter." + + def call_function(name, args): if name == "get_weather": return get_weather(**args) elif name == "get_place_to_travel": return get_place_to_travel() + elif name == "get_horoscope": + return get_horoscope(**args) else: raise ValueError(f"Unknown function: {name}") @@ -828,3 +834,126 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server): assert response.status == "completed" assert len(response.input_messages) > 0 assert len(response.output_messages) > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +async def test_function_call_with_previous_input_messages( + client: OpenAI, model_name: str +): + """Test function calling using previous_input_messages + for multi-turn conversation with a function call""" + + # Define the get_horoscope tool + tools = [ + { + "type": "function", + "name": "get_horoscope", + "description": "Get today's horoscope for an astrological sign.", + "parameters": { + "type": "object", + "properties": { + "sign": {"type": "string"}, + }, + "required": ["sign"], + "additionalProperties": False, + }, + "strict": True, + } + ] + + # Step 1: First call with the function tool + stream_response = await client.responses.create( + model=model_name, + input="What is the horoscope for Aquarius today?", + tools=tools, + extra_body={"enable_response_messages": True}, + stream=True, + ) + + response = None + async for event in stream_response: + if event.type == "response.completed": + response = event.response + + assert response is not None + assert response.status == "completed" + + # Step 2: Parse the first output to find the function_call type + function_call = None + for item in response.output: + if item.type == "function_call": + function_call = item + break + + assert function_call is not None, "Expected a function_call in the output" + assert function_call.name == "get_horoscope" + assert function_call.call_id is not None + + # Verify the format matches expectations + args = json.loads(function_call.arguments) + assert "sign" in args + + # Step 3: Call the get_horoscope function + result = call_function(function_call.name, args) + assert "Aquarius" in result + assert "baby otter" in result + + # Get the input_messages and output_messages from the first response + first_input_messages = response.input_messages + first_output_messages = response.output_messages + + # Construct the full conversation history using previous_input_messages + previous_messages = ( + first_input_messages + + first_output_messages + + [ + { + "role": "tool", + "name": "functions.get_horoscope", + "content": [{"type": "text", "text": str(result)}], + } + ] + ) + + # Step 4: Make another responses.create() call with previous_input_messages + stream_response_2 = await client.responses.create( + model=model_name, + tools=tools, + input="", + extra_body={ + "previous_input_messages": previous_messages, + "enable_response_messages": True, + }, + stream=True, + ) + + async for event in stream_response_2: + if event.type == "response.completed": + response_2 = event.response + + assert response_2 is not None + assert response_2.status == "completed" + assert response_2.output_text is not None + + # verify only one system message / developer message + num_system_messages_input = 0 + num_developer_messages_input = 0 + num_function_call_input = 0 + for message_dict in response_2.input_messages: + message = Message.from_dict(message_dict) + if message.author.role == "system": + num_system_messages_input += 1 + elif message.author.role == "developer": + num_developer_messages_input += 1 + elif message.author.role == "tool": + num_function_call_input += 1 + assert num_system_messages_input == 1 + assert num_developer_messages_input == 1 + assert num_function_call_input == 1 + + # Verify the output makes sense - should contain information about the horoscope + output_text = response_2.output_text.lower() + assert ( + "aquarius" in output_text or "otter" in output_text or "tuesday" in output_text + ) diff --git a/tests/entrypoints/openai/test_serving_responses.py b/tests/entrypoints/openai/test_serving_responses.py index 263b076db1835..cf21a5116ddfb 100644 --- a/tests/entrypoints/openai/test_serving_responses.py +++ b/tests/entrypoints/openai/test_serving_responses.py @@ -125,6 +125,28 @@ class TestInitializeToolSessions: # Verify that init_tool_sessions was called assert mock_context.init_tool_sessions_called + def test_validate_create_responses_input( + self, serving_responses_instance, mock_context, mock_exit_stack + ): + request = ResponsesRequest( + input="test input", + previous_input_messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is my horoscope? I am an Aquarius.", + } + ], + } + ], + previous_response_id="lol", + ) + error = serving_responses_instance._validate_create_responses_input(request) + assert error is not None + assert error.error.type == "invalid_request_error" + class TestValidateGeneratorInput: """Test class for _validate_generator_input method""" diff --git a/tests/entrypoints/test_harmony_utils.py b/tests/entrypoints/test_harmony_utils.py new file mode 100644 index 0000000000000..8d1764d411572 --- /dev/null +++ b/tests/entrypoints/test_harmony_utils.py @@ -0,0 +1,254 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from openai_harmony import Role + +from vllm.entrypoints.harmony_utils import parse_input_to_harmony_message + + +class TestParseInputToHarmonyMessage: + """Tests for parse_input_to_harmony_message function.""" + + def test_assistant_message_with_tool_calls(self): + """Test parsing assistant message with tool calls.""" + chat_msg = { + "role": "assistant", + "tool_calls": [ + { + "function": { + "name": "get_weather", + "arguments": '{"location": "San Francisco"}', + } + }, + { + "function": { + "name": "search_web", + "arguments": '{"query": "latest news"}', + } + }, + ], + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 2 + + # First tool call + assert messages[0].author.role == Role.ASSISTANT + assert messages[0].content[0].text == '{"location": "San Francisco"}' + assert messages[0].channel == "commentary" + assert messages[0].recipient == "functions.get_weather" + assert messages[0].content_type == "json" + + # Second tool call + assert messages[1].author.role == Role.ASSISTANT + assert messages[1].content[0].text == '{"query": "latest news"}' + assert messages[1].channel == "commentary" + assert messages[1].recipient == "functions.search_web" + assert messages[1].content_type == "json" + + def test_assistant_message_with_empty_tool_call_arguments(self): + """Test parsing assistant message with tool call having None arguments.""" + chat_msg = { + "role": "assistant", + "tool_calls": [ + { + "function": { + "name": "get_current_time", + "arguments": None, + } + } + ], + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].content[0].text == "" + assert messages[0].recipient == "functions.get_current_time" + + def test_tool_message_with_string_content(self): + """Test parsing tool message with string content.""" + chat_msg = { + "role": "tool", + "name": "get_weather", + "content": "The weather in San Francisco is sunny, 72°F", + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].author.role == Role.TOOL + assert messages[0].author.name == "functions.get_weather" + assert ( + messages[0].content[0].text == "The weather in San Francisco is sunny, 72°F" + ) + assert messages[0].channel == "commentary" + + def test_tool_message_with_array_content(self): + """Test parsing tool message with array content.""" + chat_msg = { + "role": "tool", + "name": "search_results", + "content": [ + {"type": "text", "text": "Result 1: "}, + {"type": "text", "text": "Result 2: "}, + { + "type": "image", + "url": "http://example.com/img.png", + }, # Should be ignored + {"type": "text", "text": "Result 3"}, + ], + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].author.role == Role.TOOL + assert messages[0].content[0].text == "Result 1: Result 2: Result 3" + + def test_tool_message_with_empty_content(self): + """Test parsing tool message with None content.""" + chat_msg = { + "role": "tool", + "name": "empty_tool", + "content": None, + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].content[0].text == "" + + def test_system_message(self): + """Test parsing system message.""" + chat_msg = { + "role": "system", + "content": "You are a helpful assistant", + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + # System messages are converted using Message.from_dict + # which should preserve the role + assert messages[0].author.role == Role.SYSTEM + + def test_developer_message(self): + """Test parsing developer message.""" + chat_msg = { + "role": "developer", + "content": "Use concise language", + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].author.role == Role.DEVELOPER + + def test_user_message_with_string_content(self): + """Test parsing user message with string content.""" + chat_msg = { + "role": "user", + "content": "What's the weather in San Francisco?", + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].author.role == Role.USER + assert messages[0].content[0].text == "What's the weather in San Francisco?" + + def test_user_message_with_array_content(self): + """Test parsing user message with array content.""" + chat_msg = { + "role": "user", + "content": [ + {"text": "What's in this image? "}, + {"text": "Please describe it."}, + ], + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].author.role == Role.USER + assert len(messages[0].content) == 2 + assert messages[0].content[0].text == "What's in this image? " + assert messages[0].content[1].text == "Please describe it." + + def test_assistant_message_with_string_content(self): + """Test parsing assistant message with string content (no tool calls).""" + chat_msg = { + "role": "assistant", + "content": "Hello! How can I help you today?", + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].author.role == Role.ASSISTANT + assert messages[0].content[0].text == "Hello! How can I help you today?" + + def test_pydantic_model_input(self): + """Test parsing Pydantic model input (has model_dump method).""" + + class MockPydanticModel: + def model_dump(self, exclude_none=True): + return { + "role": "user", + "content": "Test message", + } + + chat_msg = MockPydanticModel() + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].author.role == Role.USER + assert messages[0].content[0].text == "Test message" + + def test_message_with_empty_content(self): + """Test parsing message with empty string content.""" + chat_msg = { + "role": "user", + "content": "", + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].content[0].text == "" + + def test_tool_call_with_missing_function_fields(self): + """Test parsing tool call with missing name or arguments.""" + chat_msg = { + "role": "assistant", + "tool_calls": [ + { + "function": {} # Missing both name and arguments + } + ], + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert messages[0].recipient == "functions." + assert messages[0].content[0].text == "" + + def test_array_content_with_missing_text(self): + """Test parsing array content where text field is missing.""" + chat_msg = { + "role": "user", + "content": [ + {}, # Missing text field + {"text": "actual text"}, + ], + } + + messages = parse_input_to_harmony_message(chat_msg) + + assert len(messages) == 1 + assert len(messages[0].content) == 2 + assert messages[0].content[0].text == "" + assert messages[0].content[1].text == "actual text" diff --git a/vllm/entrypoints/harmony_utils.py b/vllm/entrypoints/harmony_utils.py index fe581e5484e1f..97f95a97ee304 100644 --- a/vllm/entrypoints/harmony_utils.py +++ b/vllm/entrypoints/harmony_utils.py @@ -38,11 +38,14 @@ from openai_harmony import ( ToolDescription, load_harmony_encoding, ) +from openai_harmony import Message as OpenAIHarmonyMessage +from openai_harmony import Role as OpenAIHarmonyRole from vllm import envs from vllm.entrypoints.openai.protocol import ( ChatCompletionToolsParam, ResponseInputOutputItem, + ResponsesRequest, ) from vllm.utils import random_uuid @@ -228,7 +231,7 @@ def parse_response_input( return msg -def parse_chat_input(chat_msg) -> list[Message]: +def parse_input_to_harmony_message(chat_msg) -> list[Message]: if not isinstance(chat_msg, dict): # Handle Pydantic models chat_msg = chat_msg.model_dump(exclude_none=True) @@ -279,6 +282,40 @@ def parse_chat_input(chat_msg) -> list[Message]: return [msg] +def construct_harmony_previous_input_messages( + request: ResponsesRequest, +) -> list[OpenAIHarmonyMessage]: + messages: list[OpenAIHarmonyMessage] = [] + if request.previous_input_messages: + for message in request.previous_input_messages: + # Handle both OpenAIHarmonyMessage objects and dictionary inputs + if isinstance(message, OpenAIHarmonyMessage): + message_role = message.author.role + # To match OpenAI, instructions, reasoning and tools are + # always taken from the most recent Responses API request + # not carried over from previous requests + if ( + message_role == OpenAIHarmonyRole.SYSTEM + or message_role == OpenAIHarmonyRole.DEVELOPER + ): + continue + messages.append(message) + else: + harmony_messages = parse_input_to_harmony_message(message) + for harmony_msg in harmony_messages: + message_role = harmony_msg.author.role + # To match OpenAI, instructions, reasoning and tools are + # always taken from the most recent Responses API request + # not carried over from previous requests + if ( + message_role == OpenAIHarmonyRole.SYSTEM + or message_role == OpenAIHarmonyRole.DEVELOPER + ): + continue + messages.append(harmony_msg) + return messages + + def render_for_completion(messages: list[Message]) -> list[int]: conversation = Conversation.from_messages(messages) token_ids = get_encoding().render_conversation_for_completion( diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 9782641296d62..0778e4d787905 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -47,6 +47,7 @@ from openai.types.responses import ( from openai.types.responses.response_reasoning_item import ( Content as ResponseReasoningTextContent, ) +from openai_harmony import Message as OpenAIHarmonyMessage from vllm.utils.serial_utils import ( EmbedDType, @@ -383,10 +384,15 @@ class ResponsesRequest(OpenAIBaseModel): default=False, description=( "Dictates whether or not to return messages as part of the " - "response object. Currently only supported for non-streaming " + "response object. Currently only supported for" "non-background and gpt-oss only. " ), ) + # similar to input_messages / output_messages in ResponsesResponse + # we take in previous_input_messages (ie in harmony format) + # this cannot be used in conjunction with previous_response_id + # TODO: consider supporting non harmony messages as well + previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None # --8<-- [end:responses-extra-params] _DEFAULT_SAMPLING_PARAMS = { diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 3bf887c659dc7..934ff78b2c710 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -27,8 +27,8 @@ from vllm.entrypoints.harmony_utils import ( get_stop_tokens_for_assistant_actions, get_streamable_parser_for_assistant, get_system_message, - parse_chat_input, parse_chat_output, + parse_input_to_harmony_message, render_for_completion, ) from vllm.entrypoints.logger import RequestLogger @@ -1351,11 +1351,13 @@ class OpenAIServingChat(OpenAIServing): index=output.index, message=message, logprobs=logprobs, - finish_reason="tool_calls" - if (tool_call_info is not None and tool_call_info.tools_called) - else output.finish_reason - if output.finish_reason - else "stop", + finish_reason=( + "tool_calls" + if (tool_call_info is not None and tool_call_info.tools_called) + else output.finish_reason + if output.finish_reason + else "stop" + ), stop_reason=output.stop_reason, ) choices.append(choice_data) @@ -1522,11 +1524,13 @@ class OpenAIServingChat(OpenAIServing): index=output.index, message=message, logprobs=logprobs, - finish_reason="tool_calls" - if auto_tools_called - else output.finish_reason - if output.finish_reason - else "stop", + finish_reason=( + "tool_calls" + if auto_tools_called + else output.finish_reason + if output.finish_reason + else "stop" + ), stop_reason=output.stop_reason, token_ids=( as_list(output.token_ids) if request.return_token_ids else None @@ -1685,9 +1689,11 @@ class OpenAIServingChat(OpenAIServing): should_return_as_token_id, ), logprob=max(step_token.logprob, -9999.0), - bytes=None - if step_decoded is None - else list(step_decoded.encode("utf-8", errors="replace")), + bytes=( + None + if step_decoded is None + else list(step_decoded.encode("utf-8", errors="replace")) + ), top_logprobs=self._get_top_logprobs( step_top_logprobs, num_output_top_logprobs, @@ -1764,7 +1770,7 @@ class OpenAIServingChat(OpenAIServing): # Add user message. for chat_msg in request.messages: - messages.extend(parse_chat_input(chat_msg)) + messages.extend(parse_input_to_harmony_message(chat_msg)) # Render prompt token ids. prompt_token_ids = render_for_completion(messages) diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 1fdb6997bc0ac..d43bc00a49d36 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -63,6 +63,7 @@ from vllm.entrypoints.context import ( StreamingHarmonyContext, ) from vllm.entrypoints.harmony_utils import ( + construct_harmony_previous_input_messages, get_developer_message, get_stop_tokens_for_assistant_actions, get_system_message, @@ -248,6 +249,13 @@ class OpenAIServingResponses(OpenAIServing): ), status_code=HTTPStatus.BAD_REQUEST, ) + if request.previous_input_messages and request.previous_response_id: + return self.create_error_response( + err_type="invalid_request_error", + message="Only one of `previous_input_messages` and " + "`previous_response_id` can be set.", + status_code=HTTPStatus.BAD_REQUEST, + ) return None async def create_responses( @@ -941,6 +949,8 @@ class OpenAIServingResponses(OpenAIServing): instructions=request.instructions, tools=request.tools ) messages.append(dev_msg) + messages += construct_harmony_previous_input_messages(request) + else: # Continue the previous conversation. # FIXME(woosuk): Currently, request params like reasoning and From a8d2e326ecb70876aa73dce70dbe2434c64b710a Mon Sep 17 00:00:00 2001 From: Roger Wang Date: Mon, 27 Oct 2025 17:48:32 -0700 Subject: [PATCH 018/127] [Bugfix][CI] Fix config resolving logic with remote models (#27610) --- vllm/transformers_utils/config.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 13de5939356e9..34c0429a80679 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -622,9 +622,14 @@ def get_config( # Architecture mapping for models without explicit architectures field if not config.architectures: if config.model_type not in MODEL_MAPPING_NAMES: - raise ValueError(f"Cannot find architecture name for {config.model_type}") - model_type = MODEL_MAPPING_NAMES[config.model_type] - config.update({"architectures": [model_type]}) + logger.warning( + "Model config does not have a top-level 'architectures' field: " + "expecting `hf_overrides={'architectures': ['...']}` to be passed " + "in engine args." + ) + else: + model_type = MODEL_MAPPING_NAMES[config.model_type] + config.update({"architectures": [model_type]}) # ModelOpt 0.31.0 and after saves the quantization config in the model # config file. From 255e34ca50db3fc1f0ef8b66193b7c2fe47ca672 Mon Sep 17 00:00:00 2001 From: Kuntai Du Date: Mon, 27 Oct 2025 18:32:23 -0700 Subject: [PATCH 019/127] [Stability fix] turn off HMA allocator when connector is set (#27592) Signed-off-by: KuntaiDu Signed-off-by: Kuntai Du --- vllm/config/vllm.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 597cf57939636..a7f7f3b45abea 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -597,6 +597,20 @@ class VllmConfig: if not current_platform.support_hybrid_kv_cache(): # Hybrid KV cache manager is not supported on non-GPU platforms. self.scheduler_config.disable_hybrid_kv_cache_manager = True + if self.kv_transfer_config is not None: + # NOTE(Kuntai): turn HMA off for connector for now. + # TODO(Kuntai): have a more elegent solution to check and + # turn off HMA for connector that does not support HMA. + logger.warning( + "Turning off hybrid kv cache manager because " + "`--kv-transfer-config` is set. This will reduce the " + "performance of vLLM on LLMs with sliding window attention " + "or Mamba attention. If you are a developer of kv connector" + ", please consider supporting hybrid kv cache manager for " + "your connector by making sure your connector is a subclass" + " of `SupportsHMA` defined in kv_connector/v1/base.py." + ) + self.scheduler_config.disable_hybrid_kv_cache_manager = True if self.kv_events_config is not None: # Hybrid KV cache manager is not compatible with KV events. self.scheduler_config.disable_hybrid_kv_cache_manager = True From 61fbfe52742bb34ce8a04d3fb734582268bdcd10 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Tue, 28 Oct 2025 10:18:08 +0800 Subject: [PATCH 020/127] [Bugfix] fixed inconsistent finish_reason handling between V0 and V1 engines (#27555) Signed-off-by: chaunceyjiang --- vllm/v1/core/sched/utils.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vllm/v1/core/sched/utils.py b/vllm/v1/core/sched/utils.py index 8af8a7d278064..82166dc978396 100644 --- a/vllm/v1/core/sched/utils.py +++ b/vllm/v1/core/sched/utils.py @@ -42,13 +42,6 @@ def remove_all(lst: list, items_to_remove: set) -> list: def check_stop( request: Request, max_model_len: int, pooler_output: torch.Tensor | None = None ) -> bool: - if ( - request.num_tokens >= max_model_len - or request.num_output_tokens >= request.max_tokens - ): - request.status = RequestStatus.FINISHED_LENGTH_CAPPED - return True - if request.pooling_params: if pooler_output is not None: request.status = RequestStatus.FINISHED_STOPPED @@ -70,4 +63,10 @@ def check_stop( request.status = RequestStatus.FINISHED_STOPPED request.stop_reason = last_token_id return True + if ( + request.num_tokens >= max_model_len + or request.num_output_tokens >= request.max_tokens + ): + request.status = RequestStatus.FINISHED_LENGTH_CAPPED + return True return False From 5b3c35a68e049a93bd312c0e2012514b51875ed8 Mon Sep 17 00:00:00 2001 From: vllmellm Date: Tue, 28 Oct 2025 13:00:50 +0800 Subject: [PATCH 021/127] [ROCm] [Doc] Update ROCm installation docs (#27327) Signed-off-by: vllmellm --- .../installation/gpu.rocm.inc.md | 123 +++++++++++------- .../installation/python_env_setup.inc.md | 2 +- 2 files changed, 75 insertions(+), 50 deletions(-) diff --git a/docs/getting_started/installation/gpu.rocm.inc.md b/docs/getting_started/installation/gpu.rocm.inc.md index 8abc5ac1c5c71..f546e0f0e5052 100644 --- a/docs/getting_started/installation/gpu.rocm.inc.md +++ b/docs/getting_started/installation/gpu.rocm.inc.md @@ -1,6 +1,6 @@ # --8<-- [start:installation] -vLLM supports AMD GPUs with ROCm 6.3 or above. +vLLM supports AMD GPUs with ROCm 6.3 or above, and torch 2.8.0 and above. !!! tip [Docker](#set-up-using-docker) is the recommended way to use vLLM on ROCm. @@ -28,57 +28,63 @@ Currently, there are no pre-built ROCm wheels. # --8<-- [end:pre-built-wheels] # --8<-- [start:build-wheel-from-source] +!!! tip + - If you found that the following installation step does not work for you, please refer to [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). Dockerfile is a form of installation steps. + 0. Install prerequisites (skip if you are already in an environment/docker with the following installed): - [ROCm](https://rocm.docs.amd.com/en/latest/deploy/linux/index.html) - [PyTorch](https://pytorch.org/) - For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm6.4.3_ubuntu24.04_py3.12_pytorch_release_2.6.0`, `rocm/pytorch-nightly`. If you are using docker image, you can skip to Step 3. + For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm7.0_ubuntu22.04_py3.10_pytorch_release_2.8.0`, `rocm/pytorch-nightly`. If you are using docker image, you can skip to Step 3. Alternatively, you can install PyTorch using PyTorch wheels. You can check PyTorch installation guide in PyTorch [Getting Started](https://pytorch.org/get-started/locally/). Example: ```bash # Install PyTorch pip uninstall torch -y - pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/rocm6.4 + pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm7.0 ``` -1. Install [Triton for ROCm](https://github.com/triton-lang/triton) +1. Install [Triton for ROCm](https://github.com/ROCm/triton.git) - Install ROCm's Triton (the default triton-mlir branch) following the instructions from [ROCm/triton](https://github.com/ROCm/triton/blob/triton-mlir/README.md) + Install ROCm's Triton following the instructions from [ROCm/triton](https://github.com/ROCm/triton.git) ```bash python3 -m pip install ninja cmake wheel pybind11 pip uninstall -y triton - git clone https://github.com/triton-lang/triton.git + git clone https://github.com/ROCm/triton.git cd triton - git checkout e5be006 + # git checkout $TRITON_BRANCH + git checkout f9e5bf54 if [ ! -f setup.py ]; then cd python; fi python3 setup.py install cd ../.. ``` !!! note - If you see HTTP issue related to downloading packages during building triton, please try again as the HTTP error is intermittent. + - The validated `$TRITON_BRANCH` can be found in the [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). + - If you see HTTP issue related to downloading packages during building triton, please try again as the HTTP error is intermittent. -2. Optionally, if you choose to use CK flash attention, you can install [flash attention for ROCm](https://github.com/Dao-AILab/flash-attention) +2. Optionally, if you choose to use CK flash attention, you can install [flash attention for ROCm](https://github.com/Dao-AILab/flash-attention.git) - Install ROCm's flash attention (v2.7.2) following the instructions from [ROCm/flash-attention](https://github.com/ROCm/flash-attention#amd-rocm-support) - Alternatively, wheels intended for vLLM use can be accessed under the releases. + Install ROCm's flash attention (v2.8.0) following the instructions from [ROCm/flash-attention](https://github.com/Dao-AILab/flash-attention#amd-rocm-support) - For example, for ROCm 6.3, suppose your gfx arch is `gfx90a`. To get your gfx architecture, run `rocminfo |grep gfx`. + For example, for ROCm 7.0, suppose your gfx arch is `gfx942`. To get your gfx architecture, run `rocminfo |grep gfx`. ```bash git clone https://github.com/Dao-AILab/flash-attention.git cd flash-attention - git checkout 1a7f4dfa + # git checkout $FA_BRANCH + git checkout 0e60e394 git submodule update --init - GPU_ARCHS="gfx90a" python3 setup.py install + GPU_ARCHS="gfx942" python3 setup.py install cd .. ``` !!! note - You might need to downgrade the "ninja" version to 1.10 as it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`) + - The validated `$FA_BRANCH` can be found in the [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). + 3. If you choose to build AITER yourself to use a certain branch or commit, you can build AITER using the following steps: @@ -92,11 +98,13 @@ Currently, there are no pre-built ROCm wheels. ``` !!! note - You will need to config the `$AITER_BRANCH_OR_COMMIT` for your purpose. + - You will need to config the `$AITER_BRANCH_OR_COMMIT` for your purpose. + - The validated `$AITER_BRANCH_OR_COMMIT` can be found in the [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). + -4. Build vLLM. For example, vLLM on ROCM 6.3 can be built with the following steps: +4. Build vLLM. For example, vLLM on ROCM 7.0 can be built with the following steps: - ??? console "Commands" + ???+ console "Commands" ```bash pip install --upgrade pip @@ -109,31 +117,48 @@ Currently, there are no pre-built ROCm wheels. scipy \ huggingface-hub[cli,hf_transfer] \ setuptools_scm - pip install "numpy<2" pip install -r requirements/rocm.txt - # Build vLLM for MI210/MI250/MI300. - export PYTORCH_ROCM_ARCH="gfx90a;gfx942" + # To build for a single architecture (e.g., MI300) for faster installation (recommended): + export PYTORCH_ROCM_ARCH="gfx942" + + # To build vLLM for multiple arch MI210/MI250/MI300, use this instead + # export PYTORCH_ROCM_ARCH="gfx90a;gfx942" + python3 setup.py develop ``` This may take 5-10 minutes. Currently, `pip install .` does not work for ROCm installation. !!! tip - - Triton flash attention is used by default. For benchmarking purposes, it is recommended to run a warm-up step before collecting perf numbers. - - Triton flash attention does not currently support sliding window attention. If using half precision, please use CK flash-attention for sliding window support. - - To use CK flash-attention or PyTorch naive attention, please use this flag `export VLLM_USE_TRITON_FLASH_ATTN=0` to turn off triton flash attention. - The ROCm version of PyTorch, ideally, should match the ROCm driver version. !!! tip - For MI300x (gfx942) users, to achieve optimal performance, please refer to [MI300x tuning guide](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) for performance optimization and tuning tips on system and workflow level. - For vLLM, please refer to [vLLM performance optimization](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html#vllm-performance-optimization). + For vLLM, please refer to [vLLM performance optimization](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/vllm-optimization.html). # --8<-- [end:build-wheel-from-source] # --8<-- [start:pre-built-images] The [AMD Infinity hub for vLLM](https://hub.docker.com/r/rocm/vllm/tags) offers a prebuilt, optimized docker image designed for validating inference performance on the AMD Instinct™ MI300X accelerator. +AMD also offers nightly prebuilt docker image from [Docker Hub](https://hub.docker.com/r/rocm/vllm-dev), which has vLLM and all its dependencies installed. + +???+ console "Commands" + ```bash + docker pull rocm/vllm-dev:nightly # to get the latest image + docker run -it --rm \ + --network=host \ + --group-add=video \ + --ipc=host \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --device /dev/kfd \ + --device /dev/dri \ + -v :/app/models \ + -e HF_HOME="/app/models" \ + rocm/vllm-dev:nightly + ``` !!! tip Please check [LLM inference performance validation on AMD Instinct MI300X](https://rocm.docs.amd.com/en/latest/how-to/performance-validation/mi300x/vllm-benchmark.html) @@ -144,29 +169,29 @@ docker image designed for validating inference performance on the AMD Instinct Building the Docker image from source is the recommended way to use vLLM with ROCm. -#### (Optional) Build an image with ROCm software stack +??? info "(Optional) Build an image with ROCm software stack" -Build a docker image from [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base) which setup ROCm software stack needed by the vLLM. -**This step is optional as this rocm_base image is usually prebuilt and store at [Docker Hub](https://hub.docker.com/r/rocm/vllm-dev) under tag `rocm/vllm-dev:base` to speed up user experience.** -If you choose to build this rocm_base image yourself, the steps are as follows. + Build a docker image from [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base) which setup ROCm software stack needed by the vLLM. + **This step is optional as this rocm_base image is usually prebuilt and store at [Docker Hub](https://hub.docker.com/r/rocm/vllm-dev) under tag `rocm/vllm-dev:base` to speed up user experience.** + If you choose to build this rocm_base image yourself, the steps are as follows. -It is important that the user kicks off the docker build using buildkit. Either the user put DOCKER_BUILDKIT=1 as environment variable when calling docker build command, or the user needs to set up buildkit in the docker daemon configuration /etc/docker/daemon.json as follows and restart the daemon: + It is important that the user kicks off the docker build using buildkit. Either the user put DOCKER_BUILDKIT=1 as environment variable when calling docker build command, or the user needs to set up buildkit in the docker daemon configuration /etc/docker/daemon.json as follows and restart the daemon: -```json -{ - "features": { - "buildkit": true + ```json + { + "features": { + "buildkit": true + } } -} -``` + ``` -To build vllm on ROCm 6.3 for MI200 and MI300 series, you can use the default: + To build vllm on ROCm 7.0 for MI200 and MI300 series, you can use the default: -```bash -DOCKER_BUILDKIT=1 docker build \ - -f docker/Dockerfile.rocm_base \ - -t rocm/vllm-dev:base . -``` + ```bash + DOCKER_BUILDKIT=1 docker build \ + -f docker/Dockerfile.rocm_base \ + -t rocm/vllm-dev:base . + ``` #### Build an image with vLLM @@ -181,7 +206,7 @@ It is important that the user kicks off the docker build using buildkit. Either } ``` -[docker/Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm) uses ROCm 6.3 by default, but also supports ROCm 5.7, 6.0, 6.1, and 6.2, in older vLLM branches. +[docker/Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm) uses ROCm 7.0 by default, but also supports ROCm 5.7, 6.0, 6.1, 6.2, 6.3, and 6.4, in older vLLM branches. It provides flexibility to customize the build of docker image using the following arguments: - `BASE_IMAGE`: specifies the base image used when running `docker build`. The default value `rocm/vllm-dev:base` is an image published and maintained by AMD. It is being built using [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base) @@ -189,16 +214,16 @@ It provides flexibility to customize the build of docker image using the followi Their values can be passed in when running `docker build` with `--build-arg` options. -To build vllm on ROCm 6.3 for MI200 and MI300 series, you can use the default: +To build vllm on ROCm 7.0 for MI200 and MI300 series, you can use the default: -```bash -DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.rocm -t vllm-rocm . -``` +???+ console "Commands" + ```bash + DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.rocm -t vllm-rocm . + ``` To run the above docker image `vllm-rocm`, use the below command: -??? console "Command" - +???+ console "Commands" ```bash docker run -it \ --network=host \ diff --git a/docs/getting_started/installation/python_env_setup.inc.md b/docs/getting_started/installation/python_env_setup.inc.md index 06794f8d3120e..ba78c329723ed 100644 --- a/docs/getting_started/installation/python_env_setup.inc.md +++ b/docs/getting_started/installation/python_env_setup.inc.md @@ -1,4 +1,4 @@ -It's recommended to use [uv](https://docs.astral.sh/uv/), a very fast Python environment manager, to create and manage Python environments. Please follow the [documentation](https://docs.astral.sh/uv/#getting-started) to install `uv`. After installing `uv`, you can create a new Python environment using the following commands: +On NVIDIA CUDA only, it's recommended to use [uv](https://docs.astral.sh/uv/), a very fast Python environment manager, to create and manage Python environments. Please follow the [documentation](https://docs.astral.sh/uv/#getting-started) to install `uv`. After installing `uv`, you can create a new Python environment using the following commands: ```bash uv venv --python 3.12 --seed From bdb01a38fe20d9eb97a2d219dcc5506cf24cccad Mon Sep 17 00:00:00 2001 From: Eric Yue Date: Tue, 28 Oct 2025 13:58:06 +0800 Subject: [PATCH 022/127] [Hardware][AMD][Model] Triton MoE tuning configs for GLM-4.6 for MI300X (#27323) Signed-off-by: minatoaquaMK2 --- ...N=192,device_name=AMD_Instinct_MI300X.json | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=AMD_Instinct_MI300X.json diff --git a/vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=AMD_Instinct_MI300X.json b/vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=AMD_Instinct_MI300X.json new file mode 100644 index 0000000000000..38034fe2ddae7 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=AMD_Instinct_MI300X.json @@ -0,0 +1,201 @@ +{ + "triton_version": "3.4.0", + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "3072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + } +} From d34f5fe939eafa1eb4c6ecbe84020f09ebe09284 Mon Sep 17 00:00:00 2001 From: "Li, Jiang" Date: Tue, 28 Oct 2025 14:25:44 +0800 Subject: [PATCH 023/127] [Bugfix][CPU] Fallback oneDNN linear to torch linear to fix half gemm support on legecy platforms (#27526) Signed-off-by: jiang1.li Co-authored-by: Isotr0py --- docker/Dockerfile.cpu | 2 +- vllm/model_executor/layers/utils.py | 30 ++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index f3fd1ee3e32be..adaf8a3c5b084 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -79,7 +79,7 @@ RUN echo 'ulimit -c 0' >> ~/.bashrc ######################### BUILD IMAGE ######################### FROM base AS vllm-build -ARG max_jobs=2 +ARG max_jobs=32 ENV MAX_JOBS=${max_jobs} ARG GIT_REPO_CHECK=0 diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index e6b6a70afd979..da5eea02d120d 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -8,9 +8,12 @@ import torch from vllm import _custom_ops as ops from vllm import envs +from vllm.logger import init_logger from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import direct_register_custom_op +logger = init_logger(__name__) + def shuffle_weight(w: torch.Tensor) -> torch.Tensor: # Shuffle weight along the last dimension so that @@ -178,19 +181,28 @@ def dispatch_cpu_unquantized_gemm( ) if remove_weight: layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False) + return elif ( ops._supports_onednn and current_platform.get_cpu_architecture() != CpuArchEnum.POWERPC ): - origin_weight = layer.weight - if remove_weight: - layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False) - handler = ops.create_onednn_mm(origin_weight.t(), 32) - layer.cpu_linear = lambda x, weight, bias: ops.onednn_mm(handler, x, bias) - else: - layer.cpu_linear = lambda x, weight, bias: torch.nn.functional.linear( - x, weight, bias - ) + try: + origin_weight = layer.weight + handler = ops.create_onednn_mm(origin_weight.t(), 32) + layer.cpu_linear = lambda x, weight, bias: ops.onednn_mm(handler, x, bias) + if remove_weight: + layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False) + return + except RuntimeError as e: + logger.warning_once( + "Failed to create oneDNN linear, fallback to torch linear." + f" Exception: {e}" + ) + + # fallback case + layer.cpu_linear = lambda x, weight, bias: torch.nn.functional.linear( + x, weight, bias + ) def cpu_unquantized_gemm( From b46e4a06f1aa8f243d3ed8d3b30d14ae56863d3d Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Tue, 28 Oct 2025 01:13:10 -0700 Subject: [PATCH 024/127] [Core][Bookkeeping Optimization] Update against numpy view of is_token_ids tensor (#27618) Signed-off-by: Jialin Ouyang --- vllm/v1/worker/gpu_input_batch.py | 3 ++- vllm/v1/worker/gpu_model_runner.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 476c3edefb84a..bc7578cbd97cd 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -108,9 +108,10 @@ class InputBatch: pin_memory=False, ) self.token_ids_cpu = self.token_ids_cpu_tensor.numpy() - self.is_token_ids = torch.zeros( + self.is_token_ids_tensor = torch.zeros( (max_num_reqs, max_model_len), device="cpu", dtype=bool, pin_memory=False ) + self.is_token_ids = self.is_token_ids_tensor.numpy() # Store prompt embeddings per request to avoid OOM from large upfront # allocation if max_model_len is big. # Maps req_index -> tensor of shape (num_prompt_tokens, hidden_size) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index a110ad54a05ea..129d7e54466ad 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -1103,7 +1103,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): out=self.input_ids.cpu[:total_num_scheduled_tokens], ) if self.enable_prompt_embeds: - is_token_ids = self.input_batch.is_token_ids.flatten() + is_token_ids = self.input_batch.is_token_ids_tensor.flatten() torch.index_select( is_token_ids, 0, From 0291fbf65cec9082dba4358f33f1d89ef595172c Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Tue, 28 Oct 2025 01:58:11 -0700 Subject: [PATCH 025/127] [CI/Build] Fix amd model executor test (#27612) Signed-off-by: zhewenli --- .buildkite/test-amd.yaml | 2 +- .../fastsafetensors_loader/test_fastsafetensors_loader.py | 6 ++++++ .../fastsafetensors_loader/test_weight_utils.py | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index ad4030e1208f5..524d2e121a10f 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -561,7 +561,7 @@ steps: - label: Model Executor Test # 23min timeout_in_minutes: 35 - mirror_hardwares: [amdexperimental] + mirror_hardwares: [amdexperimental, amdproduction] agent_pool: mi325_1 # grade: Blocking source_file_dependencies: diff --git a/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py b/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py index afd411ff4874e..f154df6dfc232 100644 --- a/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py +++ b/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest + from vllm import SamplingParams +from vllm.platforms import current_platform test_model = "openai-community/gpt2" @@ -15,6 +18,9 @@ prompts = [ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0) +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="fastsafetensors requires CUDA/NVIDIA GPUs" +) def test_model_loader_download_files(vllm_runner): with vllm_runner(test_model, load_format="fastsafetensors") as llm: deserialized_outputs = llm.generate(prompts, sampling_params) diff --git a/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py b/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py index cc899b77b5e9a..bd216f0e41a47 100644 --- a/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py +++ b/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py @@ -5,6 +5,7 @@ import glob import tempfile import huggingface_hub.constants +import pytest import torch from vllm.model_executor.model_loader.weight_utils import ( @@ -12,8 +13,12 @@ from vllm.model_executor.model_loader.weight_utils import ( fastsafetensors_weights_iterator, safetensors_weights_iterator, ) +from vllm.platforms import current_platform +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="fastsafetensors requires CUDA/NVIDIA GPUs" +) def test_fastsafetensors_model_loader(): with tempfile.TemporaryDirectory() as tmpdir: huggingface_hub.constants.HF_HUB_OFFLINE = False From 2fa90bda27870467c7922547b826c3065bfad556 Mon Sep 17 00:00:00 2001 From: wangln19 <96399074+wangln19@users.noreply.github.com> Date: Tue, 28 Oct 2025 19:11:50 +0800 Subject: [PATCH 026/127] Fix a robust parsing issue in KimiK2ToolParser that causes IndexError (#27565) Signed-off-by: wangln19 Co-authored-by: wangln19 --- tests/tool_use/test_kimi_k2_tool_parser.py | 6 +++--- .../openai/tool_parsers/kimi_k2_tool_parser.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/tool_use/test_kimi_k2_tool_parser.py b/tests/tool_use/test_kimi_k2_tool_parser.py index 43feae4d865ed..43b8c70acbfc3 100644 --- a/tests/tool_use/test_kimi_k2_tool_parser.py +++ b/tests/tool_use/test_kimi_k2_tool_parser.py @@ -37,11 +37,11 @@ def assert_tool_calls( assert actual_tool_call.type == "function" assert actual_tool_call.function == expected_tool_call.function - # assert tool call id format - assert actual_tool_call.id.startswith("functions.") + # assert tool call id format: should contain function name and numeric index + # Format can be either "functions.func_name:0" or "func_name:0" assert actual_tool_call.id.split(":")[-1].isdigit() assert ( - actual_tool_call.id.split(".")[1].split(":")[0] + actual_tool_call.id.split(":")[0].split(".")[-1] == expected_tool_call.function.name ) diff --git a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py index 98a52ddd60d68..3fff3b371dbe3 100644 --- a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py @@ -96,8 +96,8 @@ class KimiK2ToolParser(ToolParser): tool_calls = [] for match in function_call_tuples: function_id, function_args = match - # function_id: functions.get_weather:0 - function_name = function_id.split(".")[1].split(":")[0] + # function_id: functions.get_weather:0 or get_weather:0 + function_name = function_id.split(":")[0].split(".")[-1] tool_calls.append( ToolCall( id=function_id, @@ -254,7 +254,7 @@ class KimiK2ToolParser(ToolParser): ) if current_tool_call_matches: tool_id, tool_args = current_tool_call_matches.groups() - tool_name = tool_id.split(".")[1].split(":")[0] + tool_name = tool_id.split(":")[0].split(".")[-1] current_tool_call["id"] = tool_id current_tool_call["name"] = tool_name current_tool_call["arguments"] = tool_args @@ -264,7 +264,7 @@ class KimiK2ToolParser(ToolParser): ) if current_tool_call_name_matches: (tool_id_str,) = current_tool_call_name_matches.groups() - tool_name = tool_id_str.split(".")[1].split(":")[0] + tool_name = tool_id_str.split(":")[0].split(".")[-1] current_tool_call["id"] = tool_id_str current_tool_call["name"] = tool_name current_tool_call["arguments"] = "" From 7a865f2325b948f9b6bc6523b1ab4dfe2aa267a0 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 28 Oct 2025 04:17:45 -0700 Subject: [PATCH 027/127] [V0 Deprecation] Remove vestigial V0 logits_processors.py file (#27601) Signed-off-by: Nick Hill --- vllm/entrypoints/openai/logits_processors.py | 92 -------------------- 1 file changed, 92 deletions(-) delete mode 100644 vllm/entrypoints/openai/logits_processors.py diff --git a/vllm/entrypoints/openai/logits_processors.py b/vllm/entrypoints/openai/logits_processors.py deleted file mode 100644 index dedbc23ec83fa..0000000000000 --- a/vllm/entrypoints/openai/logits_processors.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable -from functools import lru_cache, partial - -import torch - -from vllm.sampling_params import LogitsProcessor -from vllm.transformers_utils.tokenizer import AnyTokenizer - - -class AllowedTokenIdsLogitsProcessor: - """Logits processor for constraining generated tokens to a - specific set of token ids.""" - - def __init__(self, allowed_ids: Iterable[int]): - self.allowed_ids: list[int] | None = list(allowed_ids) - self.mask: torch.Tensor | None = None - - def __call__(self, token_ids: list[int], logits: torch.Tensor) -> torch.Tensor: - if self.mask is None: - self.mask = torch.ones( - (logits.shape[-1],), dtype=torch.bool, device=logits.device - ) - self.mask[self.allowed_ids] = False - self.allowed_ids = None - logits.masked_fill_(self.mask, float("-inf")) - return logits - - -@lru_cache(maxsize=32) -def _get_allowed_token_ids_logits_processor( - allowed_token_ids: frozenset[int], - vocab_size: int, -) -> LogitsProcessor: - if not allowed_token_ids: - raise ValueError("Empty allowed_token_ids provided") - if not all(0 <= tid < vocab_size for tid in allowed_token_ids): - raise ValueError("allowed_token_ids contains out-of-vocab token id") - return AllowedTokenIdsLogitsProcessor(allowed_token_ids) - - -def logit_bias_logits_processor( - logit_bias: dict[int, float], - token_ids: list[int], - logits: torch.Tensor, -) -> torch.Tensor: - for token_id, bias in logit_bias.items(): - logits[token_id] += bias - return logits - - -def get_logits_processors( - logit_bias: dict[int, float] | dict[str, float] | None, - allowed_token_ids: list[int] | None, - tokenizer: AnyTokenizer, -) -> list[LogitsProcessor]: - logits_processors: list[LogitsProcessor] = [] - if logit_bias: - try: - # Convert token_id to integer - # Clamp the bias between -100 and 100 per OpenAI API spec - clamped_logit_bias: dict[int, float] = { - int(token_id): min(100.0, max(-100.0, bias)) - for token_id, bias in logit_bias.items() - } - except ValueError as exc: - raise ValueError( - "Found token_id in logit_bias that is not " - "an integer or string representing an integer" - ) from exc - - # Check if token_id is within the vocab size - for token_id, bias in clamped_logit_bias.items(): - if token_id < 0 or token_id >= len(tokenizer): - raise ValueError( - f"token_id {token_id} in logit_bias contains out-of-vocab token id" - ) - - logits_processors.append( - partial(logit_bias_logits_processor, clamped_logit_bias) - ) - - if allowed_token_ids is not None: - logits_processors.append( - _get_allowed_token_ids_logits_processor( - frozenset(allowed_token_ids), len(tokenizer) - ) - ) - - return logits_processors From 44b5ce956d3cf28841615a58c1c0873af87bcfe2 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Tue, 28 Oct 2025 08:00:56 -0400 Subject: [PATCH 028/127] [Bugfix] In LongRoPE, decide short vs long based on max_model_len (#27431) Signed-off-by: Matthew Bonanni --- .../openai/test_default_mm_loras.py | 2 +- vllm/config/model.py | 12 ++++++- .../phi3_long_rope_scaled_rope.py | 36 ++++++++++++++----- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/tests/entrypoints/openai/test_default_mm_loras.py b/tests/entrypoints/openai/test_default_mm_loras.py index 336bda81a9ef2..818ee2644b547 100644 --- a/tests/entrypoints/openai/test_default_mm_loras.py +++ b/tests/entrypoints/openai/test_default_mm_loras.py @@ -29,7 +29,7 @@ def multimodal_server(): # noqa: F811 "--dtype", "half", "--max-model-len", - "12800", + "4096", "--enforce-eager", # lora config below "--enable-lora", diff --git a/vllm/config/model.py b/vllm/config/model.py index c335c5c25e9e2..e3d158c1c60f7 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -2142,8 +2142,18 @@ def _get_and_verify_max_len( # If the user didn't specify `max_model_len`, then use that derived from # the model config as a default value. if max_model_len is None: - max_model_len = int(derived_max_model_len) + # For LongRoPE, default to original_max_position_embeddings to avoid + # performance degradation for shorter sequences + if rope_scaling is not None and rope_scaling["rope_type"] == "longrope": + max_model_len = int( + getattr( + hf_config, "original_max_position_embeddings", derived_max_model_len + ) + ) + else: + max_model_len = int(derived_max_model_len) max_model_len = current_platform.check_max_model_len(max_model_len) + # If the user specified a max length, make sure it is smaller than the # derived length from the HF model config. elif max_model_len > derived_max_model_len: diff --git a/vllm/model_executor/layers/rotary_embedding/phi3_long_rope_scaled_rope.py b/vllm/model_executor/layers/rotary_embedding/phi3_long_rope_scaled_rope.py index 2a42e3bd00ec8..e58c9783479bb 100644 --- a/vllm/model_executor/layers/rotary_embedding/phi3_long_rope_scaled_rope.py +++ b/vllm/model_executor/layers/rotary_embedding/phi3_long_rope_scaled_rope.py @@ -5,8 +5,13 @@ import math import torch import torch.nn as nn +from vllm.config import get_current_vllm_config +from vllm.logger import init_logger + from .common import rotate_neox +logger = init_logger(__name__) + class Phi3LongRoPEScaledRotaryEmbedding(nn.Module): """Phi3 family of models scaled rotary embedding. @@ -43,6 +48,22 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module): self.short_factor = short_factor self.long_factor = long_factor + # Force long factors if max_model_len (runtime max length) exceeds + # original_max_position_embeddings to prevent KV cache invalidation when + # sequences cross this threshold during generation + max_model_len = get_current_vllm_config().model_config.max_model_len + self.use_long_rope = max_model_len > original_max_position_embeddings + if self.use_long_rope: + logger.warning_once( + "Using LongRoPE scaling factors. This enables longer " + "contexts (%d tokens vs original %d tokens) at the cost of " + "some performance degradation for shorter sequences. If " + "this is not desired, set `max_model_len` to be at most %d.", + max_position_embeddings, + original_max_position_embeddings, + original_max_position_embeddings, + ) + scale = self.max_position_embeddings / self.original_max_position_embeddings if scale <= 1.0: scaling_factor = 1.0 @@ -112,15 +133,12 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module): query = query.view(*query.shape[:-1], -1, self.head_size) key = key.view(*key.shape[:-1], -1, self.head_size) - k = self.original_max_position_embeddings - long_prompt_offset = ( - torch.any(positions > k).float() * torch.full_like(positions, k) - ).long() - idx = ( - torch.add(positions, long_prompt_offset) - if long_prompt_offset is not None - else positions - ) + if self.use_long_rope: + k = self.original_max_position_embeddings + long_prompt_offset = torch.full_like(positions, k).long() + idx = torch.add(positions, long_prompt_offset) + else: + idx = positions idx = torch.add(idx, offsets) if offsets is not None else idx cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx) From f58d9b64044e465b85e9280882c738e11b59b2d6 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Tue, 28 Oct 2025 20:20:46 +0800 Subject: [PATCH 029/127] [Misc] Separate out `utils.counter` and move `utils.Device` to engine (#27588) Signed-off-by: DarkLight1337 --- vllm/engine/protocol.py | 7 +++- vllm/entrypoints/llm.py | 7 ++-- vllm/entrypoints/openai/api_server.py | 3 +- vllm/entrypoints/openai/serving_models.py | 2 +- vllm/utils/__init__.py | 44 ---------------------- vllm/utils/counter.py | 45 +++++++++++++++++++++++ vllm/v1/engine/async_llm.py | 3 +- vllm/v1/engine/llm_engine.py | 2 +- 8 files changed, 59 insertions(+), 54 deletions(-) create mode 100644 vllm/utils/counter.py diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 20b8eb57f7438..959a0342817c2 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import enum from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, Iterable, Mapping from typing import Any @@ -15,13 +16,17 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask from vllm.transformers_utils.tokenizer import AnyTokenizer -from vllm.utils import Device from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.processor import Processor logger = init_logger(__name__) +class Device(enum.Enum): + GPU = enum.auto() + CPU = enum.auto() + + class EngineClient(ABC): """Protocol class for Clients to Engine""" diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index c15b70a06809e..ce5cf0aae3a37 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -31,6 +31,7 @@ from vllm.config.model import ( TokenizerMode, ) from vllm.engine.arg_utils import EngineArgs +from vllm.engine.protocol import Device from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, @@ -75,8 +76,8 @@ from vllm.transformers_utils.tokenizer import ( get_cached_tokenizer, ) from vllm.usage.usage_lib import UsageContext -from vllm.utils import Counter, Device from vllm.utils.collection_utils import as_iter, is_list_of +from vllm.utils.counter import Counter from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.llm_engine import LLMEngine from vllm.v1.sample.logits_processor import LogitsProcessor @@ -1490,8 +1491,8 @@ class LLM: def stop_profile(self) -> None: self.llm_engine.stop_profile() - def reset_prefix_cache(self, device: Device | None = None) -> bool: - return self.llm_engine.reset_prefix_cache(device) + def reset_prefix_cache(self, device: Device | None = None) -> None: + self.llm_engine.reset_prefix_cache(device) def sleep(self, level: int = 1): """ diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 632bd741290bd..71939d6c41dfa 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -40,7 +40,7 @@ from typing_extensions import assert_never import vllm.envs as envs from vllm.config import VllmConfig from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.engine.protocol import EngineClient +from vllm.engine.protocol import Device, EngineClient from vllm.entrypoints.launcher import serve_http from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args @@ -108,7 +108,6 @@ from vllm.entrypoints.utils import ( from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager from vllm.usage.usage_lib import UsageContext -from vllm.utils import Device from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import is_valid_ipv6_address from vllm.utils.system_utils import decorate_logs, set_ulimit diff --git a/vllm/entrypoints/openai/serving_models.py b/vllm/entrypoints/openai/serving_models.py index 9b7deb40b93f6..24b9587010cad 100644 --- a/vllm/entrypoints/openai/serving_models.py +++ b/vllm/entrypoints/openai/serving_models.py @@ -19,7 +19,7 @@ from vllm.entrypoints.openai.protocol import ( from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry -from vllm.utils import AtomicCounter +from vllm.utils.counter import AtomicCounter logger = init_logger(__name__) diff --git a/vllm/utils/__init__.py b/vllm/utils/__init__.py index eaa78839cf3f6..549827a927d9a 100644 --- a/vllm/utils/__init__.py +++ b/vllm/utils/__init__.py @@ -3,7 +3,6 @@ import enum import inspect -import threading import uuid import warnings from functools import wraps @@ -68,54 +67,11 @@ STR_INVALID_VAL: str = "INVALID" T = TypeVar("T") -class Device(enum.Enum): - GPU = enum.auto() - CPU = enum.auto() - - class LayerBlockType(enum.Enum): attention = "attention" mamba = "mamba" -class Counter: - def __init__(self, start: int = 0) -> None: - self.counter = start - - def __next__(self) -> int: - i = self.counter - self.counter += 1 - return i - - def reset(self) -> None: - self.counter = 0 - - -class AtomicCounter: - """An atomic, thread-safe counter""" - - def __init__(self, initial=0): - """Initialize a new atomic counter to given initial value""" - self._value = initial - self._lock = threading.Lock() - - def inc(self, num=1): - """Atomically increment the counter by num and return the new value""" - with self._lock: - self._value += num - return self._value - - def dec(self, num=1): - """Atomically decrement the counter by num and return the new value""" - with self._lock: - self._value -= num - return self._value - - @property - def value(self): - return self._value - - def random_uuid() -> str: return str(uuid.uuid4().hex) diff --git a/vllm/utils/counter.py b/vllm/utils/counter.py new file mode 100644 index 0000000000000..c2dce32e97e13 --- /dev/null +++ b/vllm/utils/counter.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import threading + + +class Counter: + def __init__(self, start: int = 0) -> None: + super().__init__() + + self.counter = start + + def __next__(self) -> int: + i = self.counter + self.counter += 1 + return i + + def reset(self) -> None: + self.counter = 0 + + +class AtomicCounter: + """An atomic, thread-safe counter""" + + def __init__(self, initial: int = 0) -> None: + """Initialize a new atomic counter to given initial value""" + super().__init__() + + self._value = initial + self._lock = threading.Lock() + + @property + def value(self) -> int: + return self._value + + def inc(self, num: int = 1) -> int: + """Atomically increment the counter by num and return the new value""" + with self._lock: + self._value += num + return self._value + + def dec(self, num: int = 1) -> int: + """Atomically decrement the counter by num and return the new value""" + with self._lock: + self._value -= num + return self._value diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index fd0a9b395e5f8..cf458a8f074c0 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -14,7 +14,7 @@ import torch import vllm.envs as envs from vllm.config import VllmConfig from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.engine.protocol import EngineClient +from vllm.engine.protocol import Device, EngineClient from vllm.entrypoints.utils import _validate_truncation_size from vllm.inputs import PromptType from vllm.logger import init_logger @@ -29,7 +29,6 @@ from vllm.tracing import init_tracer from vllm.transformers_utils.config import maybe_register_config_serialize_by_value from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs from vllm.usage.usage_lib import UsageContext -from vllm.utils import Device from vllm.utils.async_utils import cancel_task_threadsafe from vllm.utils.collection_utils import as_list from vllm.utils.func_utils import deprecate_kwargs diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 9d69ed93ed378..0cd5e1ff3944e 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -14,6 +14,7 @@ from vllm.config import ParallelConfig, VllmConfig from vllm.distributed import stateless_destroy_torch_distributed_process_group from vllm.distributed.parallel_state import get_dp_group from vllm.engine.arg_utils import EngineArgs +from vllm.engine.protocol import Device from vllm.inputs import PromptType from vllm.logger import init_logger from vllm.lora.request import LoRARequest @@ -26,7 +27,6 @@ from vllm.tasks import SupportedTask from vllm.tracing import init_tracer from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs from vllm.usage.usage_lib import UsageContext -from vllm.utils import Device from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.core_client import EngineCoreClient from vllm.v1.engine.output_processor import OutputProcessor From 0484b6424894d785fb70f3e39c47aaee489340e3 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 28 Oct 2025 08:44:05 -0400 Subject: [PATCH 030/127] [Bug] Fix shape issue for eplb expert weights (#27589) Signed-off-by: yewentao256 Co-authored-by: Cyrus Leung --- vllm/model_executor/layers/fused_moe/layer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 9b826f05fe307..294dddade6cc1 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -1959,6 +1959,8 @@ class FusedMoE(CustomOp): if name not in NON_EXPERT_WEIGHTS and weight.shape != torch.Size([]) and not name.startswith("_shared_experts.") + # exclude parameters from non-expert submodules (e.g. gate/shared) + and not name.startswith("_gate.") ] def set_eplb_state( From 259504e147c2818d8a2baae82c59752ab1b9a558 Mon Sep 17 00:00:00 2001 From: Zhengxu Chen Date: Tue, 28 Oct 2025 08:46:03 -0400 Subject: [PATCH 031/127] [compile] Add enable_prompt_embeds to compile hash. (#27285) Signed-off-by: zhxchen17 Co-authored-by: Cyrus Leung --- vllm/config/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index e3d158c1c60f7..a488258c56da2 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -163,7 +163,7 @@ class ModelConfig: specified by the server file system. This is a security risk. Should only be enabled in trusted environments.""" allowed_media_domains: list[str] | None = None - """If set, only media URLs that belong to this domain can be used for + """If set, only media URLs that belong to this domain can be used for multi-modal inputs. """ revision: str | None = None """The specific model version to use. It can be a branch name, a tag name, @@ -345,6 +345,7 @@ class ModelConfig: factors.append(self.rope_scaling) factors.append(self.rope_theta) factors.append(self.video_pruning_rate) + factors.append(self.enable_prompt_embeds) # hf_config can control how the model looks! try: From 05181cc57f7939cb6ad44348ee7c99f4a0cee900 Mon Sep 17 00:00:00 2001 From: Asaf Joseph Gardin <39553475+Josephasafg@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:54:24 +0200 Subject: [PATCH 032/127] [Hybrid] Add mamba_block_size to Engine Args (#27289) Signed-off-by: asafg <39553475+Josephasafg@users.noreply.github.com> --- vllm/config/cache.py | 16 +++++++++++++--- vllm/engine/arg_utils.py | 5 +++++ vllm/model_executor/models/config.py | 12 +++++++----- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/vllm/config/cache.py b/vllm/config/cache.py index cf2977622a0b0..1734f6b15d4af 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -5,7 +5,7 @@ import hashlib from dataclasses import field from typing import TYPE_CHECKING, Any, Literal -from pydantic import Field, SkipValidation, field_validator +from pydantic import Field, SkipValidation, field_validator, model_validator from pydantic.dataclasses import dataclass from vllm.config.utils import config @@ -90,8 +90,10 @@ class CacheConfig: mamba_page_size_padded: int | None = None """ Optional override for mamba page size; used by hybrid mamba/attention models to ensure exact alignment with attention page size.""" - mamba_block_size: int | None = None - """Size of a contiguous cache block in number of tokens for mamba cache.""" + mamba_block_size: int | None = Field(default=None, gt=0) + """Size of a contiguous cache block in number of tokens for mamba cache. + Can be set only when prefix caching is enabled. + Value must be a multiple of 8 to align with causal_conv1d kernel.""" mamba_cache_dtype: MambaDType = "auto" """The data type to use for the Mamba cache (both the conv as well as the ssm state). If set to 'auto', the data type will be inferred from the model @@ -183,3 +185,11 @@ class CacheConfig: raise ValueError("Too large swap space. " + msg) elif cpu_memory_usage > 0.4 * total_cpu_memory: logger.warning("Possibly too large swap space. %s", msg) + + @model_validator(mode="after") + def validate_mamba_block_size(self) -> "CacheConfig": + if self.mamba_block_size is not None and not self.enable_prefix_caching: + raise ValueError( + "--mamba-block-size can only be set with --enable-prefix-caching" + ) + return self diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 24f9d18dc958a..ede470b08476a 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -535,6 +535,7 @@ class EngineArgs: calculate_kv_scales: bool = CacheConfig.calculate_kv_scales mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype + mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size") additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config") @@ -893,6 +894,9 @@ class EngineArgs: cache_group.add_argument( "--mamba-ssm-cache-dtype", **cache_kwargs["mamba_ssm_cache_dtype"] ) + cache_group.add_argument( + "--mamba-block-size", **cache_kwargs["mamba_block_size"] + ) # Multimodal related configs multimodal_kwargs = get_kwargs(MultiModalConfig) @@ -1390,6 +1394,7 @@ class EngineArgs: kv_sharing_fast_prefill=self.kv_sharing_fast_prefill, mamba_cache_dtype=self.mamba_cache_dtype, mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype, + mamba_block_size=self.mamba_block_size, ) ray_runtime_env = None diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 493b74bddda7a..ac5949cda9de9 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -291,9 +291,8 @@ class MambaModelConfig(VerifyAndUpdateConfig): model_config = vllm_config.model_config cache_config = vllm_config.cache_config - # Set mamba block size to max_model_len (this may get - # override by prefix caching logic later) - cache_config.mamba_block_size = model_config.max_model_len + if cache_config.mamba_block_size is None: + cache_config.mamba_block_size = model_config.max_model_len if cache_config.enable_prefix_caching: if model_config.supports_mamba_prefix_caching: @@ -333,6 +332,8 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): if not envs.VLLM_USE_V1: return + # Save the user input before it gets modified by MambaModelConfig + mamba_block_size = vllm_config.cache_config.mamba_block_size # Enable FULL_AND_PIECEWISE by default MambaModelConfig.verify_and_update_config(vllm_config) @@ -386,7 +387,7 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): # With prefix caching, select attention block size to # optimize for mamba kernel performance - # mamba SSD kernel uses a chunk_size, e.g. 256 + # Mamba2 SSD kernel uses a chunk_size, e.g. 256 # Align the block to the kernel: use lowest multiple of chunk_size # of attention tokens that would fit mamba_page_size: # e.g. for mamba page size = 788kB @@ -404,7 +405,8 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): def lcm(a, b): return a * b // gcd(a, b) - base_chunk_size = model_config.get_mamba_chunk_size() + base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size() + attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token) chunk_size = lcm(base_chunk_size, kernel_block_alignment_size) From a00d6254e998be472d8df9dc590784d6facf8d85 Mon Sep 17 00:00:00 2001 From: Zhengxu Chen Date: Tue, 28 Oct 2025 08:58:12 -0400 Subject: [PATCH 033/127] [compile] Disable dynamo guards check for AOT compilation. (#27288) Signed-off-by: zhxchen17 Co-authored-by: Cyrus Leung --- vllm/compilation/decorators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 4a4903035cf9e..376039e4f133e 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -302,6 +302,7 @@ def _support_torch_compile( start_monitoring_torch_compile(self.vllm_config) loaded_fn = torch.compiler.load_compiled_function(f) _verify_source_unchanged(loaded_fn.source_info(), self.vllm_config) + loaded_fn.disable_guard_check() self.aot_compiled_fn = loaded_fn except Exception as e: if os.path.exists(aot_compilation_path): From 446912d1cb94247815ee9a62a4f2f18628358a84 Mon Sep 17 00:00:00 2001 From: wangln19 <96399074+wangln19@users.noreply.github.com> Date: Tue, 28 Oct 2025 21:12:34 +0800 Subject: [PATCH 034/127] fix: allow HuggingFace standard chat template params via **kwargs (#27622) Signed-off-by: wangln19 Signed-off-by: wangln19 <96399074+wangln19@users.noreply.github.com> Co-authored-by: wangln19 Co-authored-by: Cyrus Leung --- tests/entrypoints/test_chat_utils.py | 33 ++++++++++++++++++++++++++++ vllm/entrypoints/chat_utils.py | 25 ++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 378c2624f7d9f..ca87b3e76b3f4 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -1882,6 +1882,39 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa ) assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs + # Additional test: Verify HF base parameters work with **kwargs tokenizers + # This validates the fix for tokenizers like Kimi K2 that use **kwargs + # to receive standard HuggingFace parameters instead of declaring them explicitly + from vllm.entrypoints.chat_utils import _get_hf_base_chat_template_params + + hf_base_params = _get_hf_base_chat_template_params() + # Verify common HF parameters are in the base class + assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset( + hf_base_params + ), f"Expected HF base params not found in {hf_base_params}" + + # Test with a mock tokenizer that uses **kwargs (like Kimi K2) + class MockTokenizerWithKwargs: + def apply_chat_template(self, conversation, **kwargs): + return "mocked_output" + + mock_tokenizer = MockTokenizerWithKwargs() + mock_kwargs = { + "add_generation_prompt": True, + "tools": tools, + "continue_final_message": False, + "unknown_param": "should_be_filtered", + } + resolved_mock = resolve_chat_template_kwargs( + mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False + ) + # HF base params should pass through even with **kwargs tokenizer + assert "add_generation_prompt" in resolved_mock + assert "tools" in resolved_mock + assert "continue_final_message" in resolved_mock + # Unknown params should be filtered out + assert "unknown_param" not in resolved_mock + # NOTE: Qwen2-Audio default chat template is specially defined inside # processor class instead of using `tokenizer_config.json` diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 4c73e94fb72b9..09641aaff3066 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import inspect import json from abc import ABC, abstractmethod from collections import Counter, defaultdict, deque @@ -1515,6 +1516,24 @@ def _resolve_chat_template_kwargs( _cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs) +@lru_cache +def _get_hf_base_chat_template_params() -> frozenset[str]: + # Get standard parameters from HuggingFace's base tokenizer class. + # This dynamically extracts parameters from PreTrainedTokenizer's + # apply_chat_template method, ensuring compatibility with tokenizers + # that use **kwargs to receive standard parameters. + + # Read signature from HF's base class - the single source of truth + base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template) + # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders + return frozenset( + p.name + for p in base_sig.parameters.values() + if p.kind + not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL) + ) + + def resolve_chat_template_kwargs( tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, chat_template: str, @@ -1538,7 +1557,11 @@ def resolve_chat_template_kwargs( if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False) } template_vars = _cached_resolve_chat_template_kwargs(chat_template) - accept_vars = (fn_kw | template_vars) - unexpected_vars + + # Allow standard HF parameters even if tokenizer uses **kwargs to receive them + hf_base_params = _get_hf_base_chat_template_params() + + accept_vars = (fn_kw | template_vars | hf_base_params) - unexpected_vars return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars} From 2abbd351ef6436860cb3c64f66b1452c1d941fe4 Mon Sep 17 00:00:00 2001 From: 22quinn <33176974+22quinn@users.noreply.github.com> Date: Tue, 28 Oct 2025 06:52:47 -0700 Subject: [PATCH 035/127] [Core] Enable async scheduling for external_launcher mode (#27394) Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com> Co-authored-by: Zhuohan Li --- vllm/engine/arg_utils.py | 7 ++++--- vllm/v1/engine/llm_engine.py | 4 +--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index ede470b08476a..e8f8e3f8c2b51 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1553,11 +1553,12 @@ class EngineArgs: ) if self.async_scheduling and ( - parallel_config.distributed_executor_backend not in ("mp", "uni") + parallel_config.distributed_executor_backend + not in ("mp", "uni", "external_launcher") ): raise ValueError( - "Currently, async scheduling only supports `mp` or `uni` " - "distributed executor backend, but you choose " + "Currently, async scheduling only supports `mp`, `uni` or " + "`external_launcher` distributed executor backend, but you choose " f"`{parallel_config.distributed_executor_backend}`." ) diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 0cd5e1ff3944e..486dacb2e5d9c 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -306,9 +306,7 @@ class LLMEngine: self.engine_core.abort_requests(processed_outputs.reqs_to_abort) # 4) Record stats - if self.logger_manager is not None: - assert outputs.scheduler_stats is not None - + if self.logger_manager is not None and outputs.scheduler_stats is not None: self.logger_manager.record( scheduler_stats=outputs.scheduler_stats, iteration_stats=iteration_stats, From b186149e8e9d067c17b5067f73c420b2d8317580 Mon Sep 17 00:00:00 2001 From: Junpu Fan Date: Tue, 28 Oct 2025 07:02:43 -0700 Subject: [PATCH 036/127] [Bugfix][Frontend] validate arg priority in frontend LLM class before add request (#27596) Signed-off-by: Junpu Fan --- tests/entrypoints/llm/test_generate.py | 20 ++++++++++++++++++++ vllm/entrypoints/llm.py | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/tests/entrypoints/llm/test_generate.py b/tests/entrypoints/llm/test_generate.py index e9993fd840619..34465b7d27080 100644 --- a/tests/entrypoints/llm/test_generate.py +++ b/tests/entrypoints/llm/test_generate.py @@ -71,6 +71,26 @@ def test_multiple_sampling_params(llm: LLM): assert len(PROMPTS) == len(outputs) +def test_multiple_priority(llm: LLM): + # Generate works when priority is None + outputs = llm.generate(PROMPTS, sampling_params=None, priority=None) + assert len(PROMPTS) == len(outputs) + + # Generate works when length of priority is same as the len(PROMPTS) + outputs = llm.generate(PROMPTS, sampling_params=None, priority=[0] * len(PROMPTS)) + assert len(PROMPTS) == len(outputs) + + # Exception raised, if the length of priority does not match the length of prompts + with pytest.raises(ValueError): + outputs = llm.generate( + PROMPTS, sampling_params=None, priority=[0] * (len(PROMPTS) - 1) + ) + + # Exception raised, if the priority list is empty + with pytest.raises(ValueError): + outputs = llm.generate(PROMPTS, sampling_params=None, priority=[]) + + def test_max_model_len(): max_model_len = 20 llm = LLM( diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index ce5cf0aae3a37..758e16c89e694 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -1565,6 +1565,12 @@ class LLM: raise ValueError( "The lengths of prompts and lora_request must be the same." ) + if priority is not None and len(priority) != num_requests: + raise ValueError( + "The lengths of prompts " + f"({num_requests}) and priority ({len(priority)}) " + "must be the same." + ) for sp in params if isinstance(params, Sequence) else (params,): if isinstance(sp, SamplingParams): From 936643a86837db51f05974c71a05a9b89941b014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=84=8D=F0=9D=95=A0=F0=9D=95=9D=F0=9D=95=9D=F0=9D=95=A0?= =?UTF-8?q?=F0=9D=95=A8=20=F0=9D=95=84=F0=9D=95=92=F0=9D=95=9F?= Date: Tue, 28 Oct 2025 16:22:28 +0200 Subject: [PATCH 037/127] [BugFix] Also consider RAY_EXPERIMENTAL_NOSET_* when storing compilation cache (#27294) Signed-off-by: Hollow Man --- vllm/envs.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/vllm/envs.py b/vllm/envs.py index 0c45f93ec0572..73bb2678ea85e 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1544,6 +1544,29 @@ def compute_hash() -> str: factors = [environment_variables[key]() for key in environment_variables_to_hash] + ray_noset_env_vars = [ + # Refer to + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/nvidia_gpu.py#L11 + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/amd_gpu.py#L11 + # https://github.com/ray-project/ray/blob/b97d21dab233c2bd8ed7db749a82a1e594222b5c/python/ray/_private/accelerators/amd_gpu.py#L10 + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/npu.py#L12 + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/hpu.py#L12 + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/neuron.py#L14 + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/tpu.py#L38 + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/intel_gpu.py#L10 + # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/rbln.py#L10 + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES", + "RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES", + "RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS", + "RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR", + "RAY_EXPERIMENTAL_NOSET_RBLN_RT_VISIBLE_DEVICES", + ] + factors.extend([os.getenv(var) for var in ray_noset_env_vars]) + hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest() return hash_str From 05e034f085b0255b171ff494938a775f2fe80ee1 Mon Sep 17 00:00:00 2001 From: Samuel Shen <102553648+sammshen@users.noreply.github.com> Date: Tue, 28 Oct 2025 07:40:55 -0700 Subject: [PATCH 038/127] [nit]: Fix import for the lmcache integration (#27600) Signed-off-by: Samuel Shen Co-authored-by: Samuel Shen --- .../kv_transfer/kv_connector/v1/lmcache_connector.py | 8 +++++--- .../kv_connector/v1/lmcache_integration/__init__.py | 5 +++++ .../kv_connector/v1/lmcache_integration/utils.py | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py index a5240adab4386..7232d947030cb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py @@ -13,9 +13,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorMetadata, KVConnectorRole, ) -from vllm.distributed.kv_transfer.kv_connector.v1.lmcache_integration import ( - vllm_v1_adapter as _adapter, -) from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput @@ -37,6 +34,11 @@ class LMCacheConnectorV1(KVConnectorBase_V1): ) if use_native: logger.info("Initializing native LMCache connector") + # lazy import + from vllm.distributed.kv_transfer.kv_connector.v1 import lmcache_integration + + _adapter = lmcache_integration.vllm_v1_adapter + cls = _adapter.LMCacheConnectorV1Impl else: logger.info("Initializing latest dev LMCache connector") diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/__init__.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/__init__.py index 208f01a7cb5ee..3c73a1c09e58d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/__init__.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/__init__.py @@ -1,2 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + + +from . import vllm_v1_adapter + +__all__ = ["vllm_v1_adapter"] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py index e0282c1552484..0e87dea59d232 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py @@ -131,7 +131,7 @@ def create_lmcache_metadata( # First Party from lmcache.config import LMCacheEngineMetadata - from vllm.utils import get_kv_cache_torch_dtype + from vllm.utils.torch_utils import get_kv_cache_torch_dtype config = lmcache_get_or_create_config() # Support both vllm_config object and individual config parameters From e88bdd60d9a25d985168c9f4a60ab10095236d7c Mon Sep 17 00:00:00 2001 From: Zhiyuan Li Date: Tue, 28 Oct 2025 22:56:28 +0800 Subject: [PATCH 039/127] [FLA] Introduce Kimi Delta Attention(KDA) to VLLM (#27654) Signed-off-by: lizhiyuan --- vllm/model_executor/layers/fla/ops/chunk.py | 2 +- .../layers/fla/ops/chunk_delta_h.py | 110 +- .../layers/fla/ops/chunk_scaled_dot_kkt.py | 24 +- .../layers/fla/ops/fused_recurrent.py | 22 +- vllm/model_executor/layers/fla/ops/kda.py | 1351 +++++++++++++++++ 5 files changed, 1451 insertions(+), 58 deletions(-) create mode 100644 vllm/model_executor/layers/fla/ops/kda.py diff --git a/vllm/model_executor/layers/fla/ops/chunk.py b/vllm/model_executor/layers/fla/ops/chunk.py index b046a6d3919e9..4c8bf9f439972 100644 --- a/vllm/model_executor/layers/fla/ops/chunk.py +++ b/vllm/model_executor/layers/fla/ops/chunk.py @@ -36,7 +36,7 @@ def chunk_gated_delta_rule_fwd( g = chunk_local_cumsum(g, chunk_size=64, cu_seqlens=cu_seqlens) # obtain WY representation. u is actually the new v. A = chunk_scaled_dot_kkt_fwd( - k=k, beta=beta, g_cumsum=g, cu_seqlens=cu_seqlens, output_dtype=torch.float32 + k=k, beta=beta, g=g, cu_seqlens=cu_seqlens, output_dtype=torch.float32 ) A = solve_tril(A=A, cu_seqlens=cu_seqlens, output_dtype=k.dtype) w, u = recompute_w_u_fwd( diff --git a/vllm/model_executor/layers/fla/ops/chunk_delta_h.py b/vllm/model_executor/layers/fla/ops/chunk_delta_h.py index 1c14f84c2b895..f0b78b65c4a32 100644 --- a/vllm/model_executor/layers/fla/ops/chunk_delta_h.py +++ b/vllm/model_executor/layers/fla/ops/chunk_delta_h.py @@ -14,14 +14,15 @@ from vllm.triton_utils import tl, triton from .index import prepare_chunk_indices, prepare_chunk_offsets from .op import exp -from .utils import is_nvidia_hopper, use_cuda_graph +from .utils import use_cuda_graph -NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8, 16] +NUM_WARPS = [2, 4, 8, 16] @triton.heuristics( { "USE_G": lambda args: args["g"] is not None, + "USE_GK": lambda args: args["gk"] is not None, "USE_INITIAL_STATE": lambda args: args["h0"] is not None, "STORE_FINAL_STATE": lambda args: args["ht"] is not None, "SAVE_NEW_VALUE": lambda args: args["v_new"] is not None, @@ -35,7 +36,7 @@ NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8, 16] for num_stages in [2, 3, 4] for BV in [32, 64] ], - key=["H", "K", "V", "BT", "USE_G"], + key=["H", "K", "V", "BT"], use_cuda_graph=use_cuda_graph, ) @triton.jit(do_not_specialize=["T"]) @@ -45,6 +46,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( w, v_new, g, + gk, h, h0, ht, @@ -58,6 +60,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( BT: tl.constexpr, BV: tl.constexpr, USE_G: tl.constexpr, + USE_GK: tl.constexpr, USE_INITIAL_STATE: tl.constexpr, STORE_FINAL_STATE: tl.constexpr, SAVE_NEW_VALUE: tl.constexpr, @@ -88,12 +91,12 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( b_h4 = tl.zeros([64, BV], dtype=tl.float32) # calculate offset - h += (boh * H + i_h) * K * V - v += (bos * H + i_h) * V - k += (bos * Hg + i_h // (H // Hg)) * K - w += (bos * H + i_h) * K + h += ((boh * H + i_h) * K * V).to(tl.int64) + v += ((bos * H + i_h) * V).to(tl.int64) + k += ((bos * Hg + i_h // (H // Hg)) * K).to(tl.int64) + w += ((bos * H + i_h) * K).to(tl.int64) if SAVE_NEW_VALUE: - v_new += (bos * H + i_h) * V + v_new += ((bos * H + i_h) * V).to(tl.int64) stride_v = H * V stride_h = H * K * V stride_k = Hg * K @@ -145,92 +148,115 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( ) tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1)) - p_v = tl.make_block_ptr( - v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) - ) - p_v_new = ( - tl.make_block_ptr( - v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) - ) - if SAVE_NEW_VALUE - else None - ) - b_v_new = tl.zeros([BT, BV], dtype=tl.float32) p_w = tl.make_block_ptr( w, (T, K), (stride_w, 1), (i_t * BT, 0), (BT, 64), (1, 0) ) b_w = tl.load(p_w, boundary_check=(0, 1)) - b_v_new += tl.dot(b_w, b_h1.to(b_w.dtype)) + b_v = tl.dot(b_w, b_h1.to(b_w.dtype)) if K > 64: p_w = tl.make_block_ptr( w, (T, K), (stride_w, 1), (i_t * BT, 64), (BT, 64), (1, 0) ) b_w = tl.load(p_w, boundary_check=(0, 1)) - b_v_new += tl.dot(b_w, b_h2.to(b_w.dtype)) + b_v += tl.dot(b_w, b_h2.to(b_w.dtype)) if K > 128: p_w = tl.make_block_ptr( w, (T, K), (stride_w, 1), (i_t * BT, 128), (BT, 64), (1, 0) ) b_w = tl.load(p_w, boundary_check=(0, 1)) - b_v_new += tl.dot(b_w, b_h3.to(b_w.dtype)) + b_v += tl.dot(b_w, b_h3.to(b_w.dtype)) if K > 192: p_w = tl.make_block_ptr( w, (T, K), (stride_w, 1), (i_t * BT, 192), (BT, 64), (1, 0) ) b_w = tl.load(p_w, boundary_check=(0, 1)) - b_v_new += tl.dot(b_w, b_h4.to(b_w.dtype)) - b_v_new = -b_v_new + tl.load(p_v, boundary_check=(0, 1)) + b_v += tl.dot(b_w, b_h4.to(b_w.dtype)) + p_v = tl.make_block_ptr( + v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) + ) + b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v if SAVE_NEW_VALUE: - p_v_new = tl.make_block_ptr( + p_v = tl.make_block_ptr( v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) ) - tl.store( - p_v_new, b_v_new.to(p_v_new.dtype.element_ty), boundary_check=(0, 1) - ) + tl.store(p_v, b_v.to(p_v.dtype.element_ty), boundary_check=(0, 1)) + last_idx = min((i_t + 1) * BT, T) - 1 if USE_G: m_t = (i_t * BT + tl.arange(0, BT)) < T - last_idx = min((i_t + 1) * BT, T) - 1 b_g_last = tl.load(g + bos * H + last_idx * H + i_h) p_g = tl.make_block_ptr( g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) ) b_g = tl.load(p_g, boundary_check=(0,)) - b_v_new = b_v_new * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None] + b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None] b_g_last = exp(b_g_last) - b_h1 = b_h1 * b_g_last + b_h1 *= b_g_last if K > 64: - b_h2 = b_h2 * b_g_last + b_h2 *= b_g_last if K > 128: - b_h3 = b_h3 * b_g_last + b_h3 *= b_g_last if K > 192: - b_h4 = b_h4 * b_g_last - b_v_new = b_v_new.to(k.dtype.element_ty) + b_h4 *= b_g_last + + if USE_GK: + o_k1 = tl.arange(0, 64) + b_gk_last1 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k1, + mask=(o_k1 < K), + other=0.0, + ) + b_h1 *= exp(b_gk_last1)[:, None] + if K > 64: + o_k2 = 64 + o_k1 + b_gk_last2 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k2, + mask=(o_k2 < K), + other=0.0, + ) + b_h2 *= exp(b_gk_last2)[:, None] + if K > 128: + o_k3 = 128 + o_k1 + b_gk_last3 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k3, + mask=(o_k3 < K), + other=0.0, + ) + b_h3 *= exp(b_gk_last3)[:, None] + if K > 192: + o_k4 = 192 + o_k1 + b_gk_last4 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k4, + mask=(o_k4 < K), + other=0.0, + ) + b_h4 *= exp(b_gk_last4)[:, None] + b_v = b_v.to(k.dtype.element_ty) + p_k = tl.make_block_ptr( k, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1) ) b_k = tl.load(p_k, boundary_check=(0, 1)) - b_h1 += tl.dot(b_k, b_v_new) + b_h1 += tl.dot(b_k, b_v) if K > 64: p_k = tl.make_block_ptr( k, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1) ) b_k = tl.load(p_k, boundary_check=(0, 1)) - b_h2 += tl.dot(b_k, b_v_new) + b_h2 += tl.dot(b_k, b_v) if K > 128: p_k = tl.make_block_ptr( k, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1) ) b_k = tl.load(p_k, boundary_check=(0, 1)) - b_h3 += tl.dot(b_k, b_v_new) + b_h3 += tl.dot(b_k, b_v) if K > 192: p_k = tl.make_block_ptr( k, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1) ) b_k = tl.load(p_k, boundary_check=(0, 1)) - b_h4 += tl.dot(b_k, b_v_new) - + b_h4 += tl.dot(b_k, b_v) # epilogue if STORE_FINAL_STATE: p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) @@ -257,12 +283,15 @@ def chunk_gated_delta_rule_fwd_h( w: torch.Tensor, u: torch.Tensor, g: torch.Tensor | None = None, + gk: torch.Tensor | None = None, initial_state: torch.Tensor | None = None, output_final_state: bool = False, chunk_size: int = 64, # SY: remove this argument and force chunk size 64? save_new_value: bool = True, cu_seqlens: torch.LongTensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: + # This kernel is slightly different from fla to support Q/K with different head numbers. + # In fla, Q/K always have the same head number, so Hg is always equal to H. B, T, Hg, K, V = *k.shape, u.shape[-1] H = u.shape[-2] BT = chunk_size @@ -299,6 +328,7 @@ def chunk_gated_delta_rule_fwd_h( w=w, v_new=v_new, g=g, + gk=gk, h=h, h0=initial_state, ht=final_state, diff --git a/vllm/model_executor/layers/fla/ops/chunk_scaled_dot_kkt.py b/vllm/model_executor/layers/fla/ops/chunk_scaled_dot_kkt.py index 975e119af333e..7724fa513d92e 100644 --- a/vllm/model_executor/layers/fla/ops/chunk_scaled_dot_kkt.py +++ b/vllm/model_executor/layers/fla/ops/chunk_scaled_dot_kkt.py @@ -18,8 +18,8 @@ from .op import exp @triton.heuristics( { + "USE_G": lambda args: args["g"] is not None, "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, - "USE_G": lambda args: args["g_cumsum"] is not None, } ) @triton.autotune( @@ -35,7 +35,7 @@ from .op import exp def chunk_scaled_dot_kkt_fwd_kernel( k, beta, - g_cumsum, + g, A, cu_seqlens, chunk_indices, @@ -85,9 +85,7 @@ def chunk_scaled_dot_kkt_fwd_kernel( b_A += tl.dot(b_kb.to(b_k.dtype), tl.trans(b_k)) if USE_G: - p_g = tl.make_block_ptr( - g_cumsum + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) - ) + p_g = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) b_g = tl.load(p_g, boundary_check=(0,)) b_g_diff = b_g[:, None] - b_g[None, :] b_A = b_A * exp(b_g_diff) @@ -102,8 +100,8 @@ def chunk_scaled_dot_kkt_fwd_kernel( def chunk_scaled_dot_kkt_fwd( k: torch.Tensor, - beta: torch.Tensor, - g_cumsum: torch.Tensor | None = None, + g: torch.Tensor | None = None, + beta: torch.Tensor | None = None, cu_seqlens: torch.LongTensor | None = None, chunk_size: int = 64, output_dtype: torch.dtype = torch.float32, @@ -116,9 +114,8 @@ def chunk_scaled_dot_kkt_fwd( The key tensor of shape `[B, T, H, K]`. beta (torch.Tensor): The beta tensor of shape `[B, T, H]`. - g_cumsum (torch.Tensor): - The cumulative sum of the gate tensor of shape `[B, T, H]`. - Default: None + g (torch.Tensor): + The cumulative sum of the gate tensor of shape `[B, T, H]`. Default: `None`. cu_seqlens (torch.LongTensor): The cumulative sequence lengths of the input tensor. Default: None @@ -130,20 +127,21 @@ def chunk_scaled_dot_kkt_fwd( Returns: beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size. """ - + # This kernel is slightly different from fla to support Q/K with different head numbers. + # In fla, Q/K always have the same head number, so Hg is always equal to H. B, T, Hg, K = k.shape - H = beta.shape[-1] BT = chunk_size chunk_indices = ( prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None ) NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + A = torch.empty(B, T, H, BT, device=k.device, dtype=output_dtype) chunk_scaled_dot_kkt_fwd_kernel[(NT, B * H)]( k=k, + g=g, beta=beta, - g_cumsum=g_cumsum, A=A, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, diff --git a/vllm/model_executor/layers/fla/ops/fused_recurrent.py b/vllm/model_executor/layers/fla/ops/fused_recurrent.py index f3de1bfa28219..0f27504780ac4 100644 --- a/vllm/model_executor/layers/fla/ops/fused_recurrent.py +++ b/vllm/model_executor/layers/fla/ops/fused_recurrent.py @@ -57,6 +57,7 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( IS_VARLEN: tl.constexpr, IS_CONTINUOUS_BATCHING: tl.constexpr, IS_SPEC_DECODING: tl.constexpr, + IS_KDA: tl.constexpr, ): i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2) i_n, i_hv = i_nh // HV, i_nh % HV @@ -86,7 +87,12 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( p_beta = beta + (bos * HV + i_hv) * V + o_v else: p_beta = beta + bos * HV + i_hv - p_g = g + bos * HV + i_hv + + if not IS_KDA: + p_g = g + bos * HV + i_hv + else: + p_gk = g + (bos * HV + i_hv) * K + o_k + p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v mask_k = o_k < K @@ -116,14 +122,18 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32) b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32) b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32) - b_g = tl.load(p_g).to(tl.float32) if USE_QK_L2NORM_IN_KERNEL: b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6) b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6) b_q = b_q * scale # [BK, BV] - b_h *= exp(b_g) + if not IS_KDA: + b_g = tl.load(p_g).to(tl.float32) + b_h *= exp(b_g) + else: + b_gk = tl.load(p_gk).to(tl.float32) + b_h *= exp(b_gk[:, None]) # [BV] b_v -= tl.sum(b_h * b_k[:, None], 0) if IS_BETA_HEADWISE: @@ -155,7 +165,10 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( p_k += H * K p_o += HV * V p_v += HV * V - p_g += HV + if not IS_KDA: + p_g += HV + else: + p_gk += HV * K p_beta += HV * (V if IS_BETA_HEADWISE else 1) @@ -228,6 +241,7 @@ def fused_recurrent_gated_delta_rule_fwd( IS_BETA_HEADWISE=beta.ndim == v.ndim, USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, INPLACE_FINAL_STATE=inplace_final_state, + IS_KDA=False, num_warps=num_warps, num_stages=num_stages, ) diff --git a/vllm/model_executor/layers/fla/ops/kda.py b/vllm/model_executor/layers/fla/ops/kda.py new file mode 100644 index 0000000000000..a10847d347d13 --- /dev/null +++ b/vllm/model_executor/layers/fla/ops/kda.py @@ -0,0 +1,1351 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang +# +# This file contains code copied from the flash-linear-attention project. +# The original source code was licensed under the MIT license and included +# the following copyright notice: +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang +# ruff: noqa: E501 + + +import torch +import torch.nn as nn + +from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import cdiv, next_power_of_2 + +from .chunk_delta_h import chunk_gated_delta_rule_fwd_h +from .cumsum import chunk_local_cumsum +from .fused_recurrent import fused_recurrent_gated_delta_rule_fwd_kernel +from .index import prepare_chunk_indices +from .l2norm import l2norm_fwd +from .op import exp, log +from .solve_tril import solve_tril +from .utils import is_amd + +BT_LIST_AUTOTUNE = [32, 64, 128] +NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32] + + +def fused_recurrent_kda_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + inplace_final_state: bool = True, + cu_seqlens: torch.LongTensor | None = None, + ssm_state_indices: torch.Tensor | None = None, + num_accepted_tokens: torch.Tensor | None = None, + use_qk_l2norm_in_kernel: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + B, T, H, K, V = *k.shape, v.shape[-1] + HV = v.shape[2] + N = B if cu_seqlens is None else len(cu_seqlens) - 1 + BK, BV = next_power_of_2(K), min(next_power_of_2(V), 8) + NK, NV = cdiv(K, BK), cdiv(V, BV) + assert NK == 1, "NK > 1 is not supported yet" + num_stages = 3 + num_warps = 1 + + o = torch.empty_like(k) + if inplace_final_state: + final_state = initial_state + else: + final_state = q.new_empty(T, HV, K, V, dtype=initial_state.dtype) + + stride_init_state_token = initial_state.stride(0) + stride_final_state_token = final_state.stride(0) + + if ssm_state_indices is None: + stride_indices_seq, stride_indices_tok = 1, 1 + elif ssm_state_indices.ndim == 1: + stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1 + else: + stride_indices_seq, stride_indices_tok = ssm_state_indices.stride() + + grid = (NK, NV, N * HV) + fused_recurrent_gated_delta_rule_fwd_kernel[grid]( + q=q, + k=k, + v=v, + g=g, + beta=beta, + o=o, + h0=initial_state, + ht=final_state, + cu_seqlens=cu_seqlens, + ssm_state_indices=ssm_state_indices, + num_accepted_tokens=num_accepted_tokens, + scale=scale, + N=N, + T=T, + B=B, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + stride_init_state_token=stride_init_state_token, + stride_final_state_token=stride_final_state_token, + stride_indices_seq=stride_indices_seq, + stride_indices_tok=stride_indices_tok, + IS_BETA_HEADWISE=beta.ndim == v.ndim, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + INPLACE_FINAL_STATE=inplace_final_state, + IS_KDA=True, + num_warps=num_warps, + num_stages=num_stages, + ) + + return o, final_state + + +def fused_recurrent_kda( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor = None, + scale: float = None, + initial_state: torch.Tensor = None, + inplace_final_state: bool = True, + use_qk_l2norm_in_kernel: bool = True, + cu_seqlens: torch.LongTensor | None = None, + ssm_state_indices: torch.LongTensor | None = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + if cu_seqlens is not None and q.shape[0] != 1: + raise ValueError( + f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." + f"Please flatten variable-length inputs before processing." + ) + if scale is None: + scale = k.shape[-1] ** -0.5 + + o, final_state = fused_recurrent_kda_fwd( + q=q.contiguous(), + k=k.contiguous(), + v=v.contiguous(), + g=g.contiguous(), + beta=beta.contiguous(), + scale=scale, + initial_state=initial_state, + inplace_final_state=inplace_final_state, + cu_seqlens=cu_seqlens, + ssm_state_indices=ssm_state_indices, + num_accepted_tokens=None, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + ) + return o, final_state + + +@triton.heuristics( + { + "STORE_RESIDUAL_OUT": lambda args: args["residual_out"] is not None, + "HAS_RESIDUAL": lambda args: args["residual"] is not None, + "HAS_WEIGHT": lambda args: args["w"] is not None, + "HAS_BIAS": lambda args: args["b"] is not None, + } +) +@triton.jit +def layer_norm_gated_fwd_kernel( + x, # pointer to the input + g, # pointer to the gate + y, # pointer to the output + w, # pointer to the weights + b, # pointer to the biases + residual, # pointer to the residual + residual_out, # pointer to the residual + mean, # pointer to the mean + rstd, # pointer to the 1/std + eps, # epsilon to avoid division by zero + T, # number of rows in x + D: tl.constexpr, # number of columns in x + BT: tl.constexpr, + BD: tl.constexpr, + ACTIVATION: tl.constexpr, + IS_RMS_NORM: tl.constexpr, + STORE_RESIDUAL_OUT: tl.constexpr, + HAS_RESIDUAL: tl.constexpr, + HAS_WEIGHT: tl.constexpr, + HAS_BIAS: tl.constexpr, +): + i_t = tl.program_id(0) + + o_d = tl.arange(0, BD) + m_d = o_d < D + + p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) + b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32) + if HAS_RESIDUAL: + p_res = tl.make_block_ptr( + residual, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0) + ) + b_x += tl.load(p_res, boundary_check=(0, 1)).to(tl.float32) + if STORE_RESIDUAL_OUT: + p_res_out = tl.make_block_ptr( + residual_out, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0) + ) + tl.store(p_res_out, b_x.to(p_res_out.dtype.element_ty), boundary_check=(0, 1)) + if not IS_RMS_NORM: + b_mean = tl.sum(b_x, axis=1) / D + p_mean = tl.make_block_ptr(mean, (T,), (1,), (i_t * BT,), (BT,), (0,)) + tl.store(p_mean, b_mean.to(p_mean.dtype.element_ty), boundary_check=(0,)) + b_xbar = tl.where(m_d[None, :], b_x - b_mean[:, None], 0.0) + b_var = tl.sum(b_xbar * b_xbar, axis=1) / D + else: + b_xbar = tl.where(m_d[None, :], b_x, 0.0) + b_var = tl.sum(b_xbar * b_xbar, axis=1) / D + b_rstd = 1 / tl.sqrt(b_var + eps) + + p_rstd = tl.make_block_ptr(rstd, (T,), (1,), (i_t * BT,), (BT,), (0,)) + tl.store(p_rstd, b_rstd.to(p_rstd.dtype.element_ty), boundary_check=(0,)) + + if HAS_WEIGHT: + b_w = tl.load(w + o_d, mask=m_d).to(tl.float32) + if HAS_BIAS: + b_b = tl.load(b + o_d, mask=m_d).to(tl.float32) + b_x_hat = ( + (b_x - b_mean[:, None]) * b_rstd[:, None] + if not IS_RMS_NORM + else b_x * b_rstd[:, None] + ) + b_y = b_x_hat * b_w[None, :] if HAS_WEIGHT else b_x_hat + if HAS_BIAS: + b_y = b_y + b_b[None, :] + + # swish/sigmoid output gate + p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) + b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) + if ACTIVATION == "swish" or ACTIVATION == "silu": + b_y = b_y * b_g * tl.sigmoid(b_g) + elif ACTIVATION == "sigmoid": + b_y = b_y * tl.sigmoid(b_g) + + # Write output + p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) + tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.heuristics( + { + "STORE_RESIDUAL_OUT": lambda args: args["residual_out"] is not None, + "HAS_RESIDUAL": lambda args: args["residual"] is not None, + "HAS_WEIGHT": lambda args: args["w"] is not None, + "HAS_BIAS": lambda args: args["b"] is not None, + } +) +@triton.jit +def layer_norm_gated_fwd_kernel1( + x, # pointer to the input + g, # pointer to the gate + y, # pointer to the output + w, # pointer to the weights + b, # pointer to the biases + residual, # pointer to the residual + residual_out, # pointer to the residual + mean, # pointer to the mean + rstd, # pointer to the 1/std + eps, # epsilon to avoid division by zero + D: tl.constexpr, # number of columns in x + BD: tl.constexpr, + ACTIVATION: tl.constexpr, + IS_RMS_NORM: tl.constexpr, + STORE_RESIDUAL_OUT: tl.constexpr, + HAS_RESIDUAL: tl.constexpr, + HAS_WEIGHT: tl.constexpr, + HAS_BIAS: tl.constexpr, +): + i_t = tl.program_id(0) + x += i_t * D + y += i_t * D + g += i_t * D + if HAS_RESIDUAL: + residual += i_t * D + if STORE_RESIDUAL_OUT: + residual_out += i_t * D + + o_d = tl.arange(0, BD) + m_d = o_d < D + b_x = tl.load(x + o_d, mask=m_d, other=0.0).to(tl.float32) + if HAS_RESIDUAL: + b_x += tl.load(residual + o_d, mask=m_d, other=0.0).to(tl.float32) + if STORE_RESIDUAL_OUT: + tl.store(residual_out + o_d, b_x, mask=m_d) + if not IS_RMS_NORM: + b_mean = tl.sum(b_x, axis=0) / D + tl.store(mean + i_t, b_mean) + b_xbar = tl.where(m_d, b_x - b_mean, 0.0) + b_var = tl.sum(b_xbar * b_xbar, axis=0) / D + else: + b_xbar = tl.where(m_d, b_x, 0.0) + b_var = tl.sum(b_xbar * b_xbar, axis=0) / D + b_rstd = 1 / tl.sqrt(b_var + eps) + tl.store(rstd + i_t, b_rstd) + + if HAS_WEIGHT: + b_w = tl.load(w + o_d, mask=m_d).to(tl.float32) + if HAS_BIAS: + b_b = tl.load(b + o_d, mask=m_d).to(tl.float32) + b_x_hat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd + b_y = b_x_hat * b_w if HAS_WEIGHT else b_x_hat + if HAS_BIAS: + b_y = b_y + b_b + + # swish/sigmoid output gate + b_g = tl.load(g + o_d, mask=m_d, other=0.0).to(tl.float32) + if ACTIVATION == "swish" or ACTIVATION == "silu": + b_y = b_y * b_g * tl.sigmoid(b_g) + elif ACTIVATION == "sigmoid": + b_y = b_y * tl.sigmoid(b_g) + + # Write output + tl.store(y + o_d, b_y, mask=m_d) + + +def layer_norm_gated_fwd( + x: torch.Tensor, + g: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + activation: str = "swish", + eps: float = 1e-5, + residual: torch.Tensor = None, + out_dtype: torch.dtype = None, + residual_dtype: torch.dtype = None, + is_rms_norm: bool = False, +): + if residual is not None: + residual_dtype = residual.dtype + T, D = x.shape + if residual is not None: + assert residual.shape == (T, D) + if weight is not None: + assert weight.shape == (D,) + if bias is not None: + assert bias.shape == (D,) + # allocate output + y = x if out_dtype is None else torch.empty_like(x, dtype=out_dtype) + if residual is not None or ( + residual_dtype is not None and residual_dtype != x.dtype + ): + residual_out = torch.empty(T, D, device=x.device, dtype=residual_dtype) + else: + residual_out = None + mean = ( + torch.empty((T,), dtype=torch.float, device=x.device) + if not is_rms_norm + else None + ) + rstd = torch.empty((T,), dtype=torch.float, device=x.device) + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BD = min(MAX_FUSED_SIZE, next_power_of_2(D)) + if D > BD: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + # heuristics for number of warps + + if D <= 512: + BT = 32 + layer_norm_gated_fwd_kernel[(cdiv(T, BT),)]( + x=x, + g=g, + y=y, + w=weight, + b=bias, + residual=residual, + residual_out=residual_out, + mean=mean, + rstd=rstd, + eps=eps, + T=T, + D=D, + BD=BD, + BT=BT, + ACTIVATION=activation, + IS_RMS_NORM=is_rms_norm, + num_warps=4, + ) + else: + layer_norm_gated_fwd_kernel1[(T,)]( + x=x, + g=g, + y=y, + w=weight, + b=bias, + residual=residual, + residual_out=residual_out, + mean=mean, + rstd=rstd, + eps=eps, + D=D, + BD=BD, + ACTIVATION=activation, + IS_RMS_NORM=is_rms_norm, + num_warps=4, + ) + # residual_out is None if residual is None and residual_dtype == input_dtype + return y, mean, rstd, residual_out if residual_out is not None else x + + +def rms_norm_gated( + x: torch.Tensor, + g: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + activation: str = "swish", + residual: torch.Tensor | None = None, + prenorm: bool = False, + residual_in_fp32: bool = False, + eps: float = 1e-6, +): + x_shape_og = x.shape + # reshape input data into 2D tensor + x = x.contiguous().reshape(-1, x.shape[-1]) + g = g.contiguous().reshape(-1, g.shape[-1]) + if residual is not None: + assert residual.shape == x_shape_og + residual = residual.contiguous().reshape(-1, residual.shape[-1]) + residual_dtype = ( + residual.dtype + if residual is not None + else (torch.float if residual_in_fp32 else None) + ) + y, _, _, residual_out = layer_norm_gated_fwd( + x=x, + g=g, + weight=weight, + bias=bias, + activation=activation, + eps=eps, + residual=residual, + residual_dtype=residual_dtype, + is_rms_norm=True, + ) + y = y.reshape(x_shape_og) + return y if not prenorm else (y, residual_out.reshape(x_shape_og)) + + +class FusedRMSNormGated(nn.Module): + def __init__( + self, + hidden_size: int, + elementwise_affine: bool = True, + eps: float = 1e-5, + activation: str = "swish", + device: torch.device | None = None, + dtype: torch.dtype | None = None, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + + self.hidden_size = hidden_size + self.elementwise_affine = elementwise_affine + self.eps = eps + self.activation = activation + + if self.activation not in ["swish", "silu", "sigmoid"]: + raise ValueError(f"Unsupported activation: {self.activation}") + + if elementwise_affine: + self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs)) + else: + self.register_parameter("weight", None) + self.register_parameter("bias", None) + + def forward( + self, + x: torch.Tensor, + g: torch.Tensor, + residual: torch.Tensor | None = None, + prenorm: bool = False, + residual_in_fp32: bool = False, + ) -> torch.Tensor: + return rms_norm_gated( + x, + g, + self.weight, + self.bias, + self.activation, + residual=residual, + eps=self.eps, + prenorm=prenorm, + residual_in_fp32=residual_in_fp32, + ) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[ + triton.Config({"BK": BK}, num_warps=num_warps, num_stages=num_stages) + for BK in [32, 64] + for num_warps in [1, 2, 4, 8] + for num_stages in [2, 3, 4] + ], + key=["BC"], +) +@triton.jit(do_not_specialize=["T"]) +def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter( + q, + k, + g, + beta, + A, + Aqk, + scale, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + K: tl.constexpr, + BT: tl.constexpr, + BC: tl.constexpr, + BK: tl.constexpr, + NC: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + i_i, i_j = i_c // NC, i_c % NC + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + if i_t * BT + i_i * BC >= T: + return + if i_i <= i_j: + return + + q += (bos * H + i_h) * K + k += (bos * H + i_h) * K + g += (bos * H + i_h) * K + A += (bos * H + i_h) * BT + Aqk += (bos * H + i_h) * BT + + p_b = tl.make_block_ptr( + beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,) + ) + b_b = tl.load(p_b, boundary_check=(0,)) + + b_A = tl.zeros([BC, BC], dtype=tl.float32) + b_Aqk = tl.zeros([BC, BC], dtype=tl.float32) + for i_k in range(tl.cdiv(K, BK)): + p_q = tl.make_block_ptr( + q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0) + ) + p_k = tl.make_block_ptr( + k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0) + ) + p_g = tl.make_block_ptr( + g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0) + ) + b_kt = tl.make_block_ptr( + k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1) + ) + p_gk = tl.make_block_ptr( + g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1) + ) + + o_k = i_k * BK + tl.arange(0, BK) + m_k = o_k < K + # [BK,] + b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0) + # [BC, BK] + b_g = tl.load(p_g, boundary_check=(0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) * exp(b_g - b_gn[None, :]) + # [BK, BC] + b_gk = tl.load(p_gk, boundary_check=(0, 1)) + b_kt = tl.load(b_kt, boundary_check=(0, 1)) + # [BC, BC] + b_ktg = b_kt * exp(b_gn[:, None] - b_gk) + b_A += tl.dot(b_k, b_ktg) + + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_qg = b_q * exp(b_g - b_gn[None, :]) * scale + b_Aqk += tl.dot(b_qg, b_ktg) + + b_A *= b_b[:, None] + + p_A = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0) + ) + tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1)) + p_Aqk = tl.make_block_ptr( + Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0) + ) + tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[triton.Config({}, num_warps=num_warps) for num_warps in [1, 2, 4, 8]], + key=["BK", "BT"], +) +@triton.jit(do_not_specialize=["T"]) +def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra( + q, + k, + g, + beta, + A, + Aqk, + scale, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + K: tl.constexpr, + BT: tl.constexpr, + BC: tl.constexpr, + BK: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + if i_t * BT + i_i * BC >= T: + return + + o_i = tl.arange(0, BC) + o_k = tl.arange(0, BK) + m_k = o_k < K + m_A = (i_t * BT + i_i * BC + o_i) < T + o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC + + p_q = tl.make_block_ptr( + q + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT + i_i * BC, 0), + (BC, BK), + (1, 0), + ) + p_k = tl.make_block_ptr( + k + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT + i_i * BC, 0), + (BC, BK), + (1, 0), + ) + p_g = tl.make_block_ptr( + g + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT + i_i * BC, 0), + (BC, BK), + (1, 0), + ) + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_g = tl.load(p_g, boundary_check=(0, 1)) + + p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h + b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None] + + p_kt = k + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k + p_gk = g + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k + + for j in range(0, min(BC, T - i_t * BT - i_i * BC)): + b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32) + b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32) + b_ktg = b_kt[None, :] * exp(b_g - b_gk[None, :]) + b_A = tl.sum(b_k * b_ktg, 1) + b_A = tl.where(o_i > j, b_A, 0.0) + b_Aqk = tl.sum(b_q * b_ktg, 1) + b_Aqk = tl.where(o_i >= j, b_Aqk * scale, 0.0) + tl.store(A + o_A + j, b_A, mask=m_A) + tl.store(Aqk + o_A + j, b_Aqk, mask=m_A) + p_kt += H * K + p_gk += H * K + + +def chunk_kda_scaled_dot_kkt_fwd( + q: torch.Tensor, + k: torch.Tensor, + gk: torch.Tensor | None = None, + beta: torch.Tensor | None = None, + scale: float | None = None, + cu_seqlens: torch.LongTensor | None = None, + chunk_size: int = 64, + output_dtype: torch.dtype = torch.float32, +) -> tuple[torch.Tensor, torch.Tensor]: + r""" + Compute beta * K * K^T. + + Args: + k (torch.Tensor): + The key tensor of shape `[B, T, H, K]`. + beta (torch.Tensor): + The beta tensor of shape `[B, T, H]`. + gk (torch.Tensor): + The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`. + cu_seqlens (torch.LongTensor): + The cumulative sequence lengths of the input tensor. + Default: None + chunk_size (int): + The chunk size. Default: 64. + output_dtype (torch.dtype): + The dtype of the output tensor. Default: `torch.float32` + + Returns: + beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size. + """ + B, T, H, K = k.shape + assert K <= 256 + BT = chunk_size + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None + ) + NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + + BC = min(16, BT) + NC = cdiv(BT, BC) + BK = max(next_power_of_2(K), 16) + A = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype) + Aqk = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype) + grid = (NT, NC * NC, B * H) + chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter[grid]( + q=q, + k=k, + g=gk, + beta=beta, + A=A, + Aqk=Aqk, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + K=K, + BT=BT, + BC=BC, + NC=NC, + ) + + grid = (NT, NC, B * H) + chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra[grid]( + q=q, + k=k, + g=gk, + beta=beta, + A=A, + Aqk=Aqk, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + K=K, + BT=BT, + BC=BC, + BK=BK, + ) + return A, Aqk + + +@triton.heuristics( + { + "STORE_QG": lambda args: args["qg"] is not None, + "STORE_KG": lambda args: args["kg"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=num_warps, num_stages=num_stages) + for num_warps in [2, 4, 8] + for num_stages in [2, 3, 4] + ], + key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"], +) +@triton.jit(do_not_specialize=["T"]) +def recompute_w_u_fwd_kernel( + q, + k, + qg, + kg, + v, + beta, + w, + u, + A, + gk, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + STORE_QG: tl.constexpr, + STORE_KG: tl.constexpr, + IS_VARLEN: tl.constexpr, + DOT_PRECISION: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) + b_b = tl.load(p_b, boundary_check=(0,)) + + p_A = tl.make_block_ptr( + A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) + ) + b_A = tl.load(p_A, boundary_check=(0, 1)) + + for i_v in range(tl.cdiv(V, BV)): + p_v = tl.make_block_ptr( + v + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + p_u = tl.make_block_ptr( + u + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + b_v = tl.load(p_v, boundary_check=(0, 1)) + b_vb = (b_v * b_b[:, None]).to(b_v.dtype) + b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION) + tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1)) + + for i_k in range(tl.cdiv(K, BK)): + p_w = tl.make_block_ptr( + w + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_k = tl.make_block_ptr( + k + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_kb = b_k * b_b[:, None] + + p_gk = tl.make_block_ptr( + gk + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_gk = tl.load(p_gk, boundary_check=(0, 1)) + b_kb *= exp(b_gk) + if STORE_QG: + p_q = tl.make_block_ptr( + q + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_qg = tl.make_block_ptr( + qg + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_qg = b_q * exp(b_gk) + tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1)) + if STORE_KG: + last_idx = min(i_t * BT + BT, T) - 1 + + o_k = i_k * BK + tl.arange(0, BK) + m_k = o_k < K + b_gn = tl.load( + gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0 + ) + b_kg = b_k * exp(b_gn - b_gk) + + p_kg = tl.make_block_ptr( + kg + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1)) + + b_w = tl.dot(b_A, b_kb.to(b_k.dtype)) + tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1)) + + +def recompute_w_u_fwd( + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + A: torch.Tensor, + q: torch.Tensor | None = None, + gk: torch.Tensor | None = None, + cu_seqlens: torch.LongTensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + B, T, H, K, V = *k.shape, v.shape[-1] + BT = A.shape[-1] + BK = 64 + BV = 64 + + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None + ) + NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + + w = torch.empty_like(k) + u = torch.empty_like(v) + kg = torch.empty_like(k) if gk is not None else None + recompute_w_u_fwd_kernel[(NT, B * H)]( + q=q, + k=k, + qg=None, + kg=kg, + v=v, + beta=beta, + w=w, + u=u, + A=A, + gk=gk, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + K=K, + V=V, + BT=BT, + BK=BK, + BV=BV, + DOT_PRECISION="ieee", + ) + return w, u, None, kg + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[ + triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages) + for BK in [32, 64] + for BV in [64, 128] + for num_warps in [2, 4, 8] + for num_stages in [2, 3, 4] + ], + key=["BT"], +) +@triton.jit(do_not_specialize=["T"]) +def chunk_gla_fwd_kernel_o( + q, + v, + g, + h, + o, + A, + cu_seqlens, + chunk_indices, + scale, + T, + H: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_tg = i_t + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + NT = tl.cdiv(T, BT) + else: + NT = tl.cdiv(T, BT) + i_tg = i_b * NT + i_t + bos, eos = i_b * T, i_b * T + T + + m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :] + + b_o = tl.zeros([BT, BV], dtype=tl.float32) + for i_k in range(tl.cdiv(K, BK)): + p_q = tl.make_block_ptr( + q + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_g = tl.make_block_ptr( + g + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_h = tl.make_block_ptr( + h + (i_tg * H + i_h) * K * V, + (K, V), + (V, 1), + (i_k * BK, i_v * BV), + (BK, BV), + (1, 0), + ) + + # [BT, BK] + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_q = (b_q * scale).to(b_q.dtype) + # [BT, BK] + b_g = tl.load(p_g, boundary_check=(0, 1)) + # [BT, BK] + b_qg = (b_q * exp(b_g)).to(b_q.dtype) + # [BK, BV] + b_h = tl.load(p_h, boundary_check=(0, 1)) + # works but dkw, owing to divine benevolence + # [BT, BV] + if i_k >= 0: + b_o += tl.dot(b_qg, b_h.to(b_qg.dtype)) + p_v = tl.make_block_ptr( + v + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + p_o = tl.make_block_ptr( + o + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + p_A = tl.make_block_ptr( + A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) + ) + # [BT, BV] + b_v = tl.load(p_v, boundary_check=(0, 1)) + # [BT, BT] + b_A = tl.load(p_A, boundary_check=(0, 1)) + b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype) + b_o += tl.dot(b_A, b_v, allow_tf32=False) + tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + + +def chunk_gla_fwd_o_gk( + q: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + A: torch.Tensor, + h: torch.Tensor, + o: torch.Tensor, + scale: float, + cu_seqlens: torch.LongTensor | None = None, + chunk_size: int = 64, +): + B, T, H, K, V = *q.shape, v.shape[-1] + BT = chunk_size + + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, chunk_size) + if cu_seqlens is not None + else None + ) + NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + + def grid(meta): + return (cdiv(V, meta["BV"]), NT, B * H) + + chunk_gla_fwd_kernel_o[grid]( + q=q, + v=v, + g=g, + h=h, + o=o, + A=A, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + scale=scale, + T=T, + H=H, + K=K, + V=V, + BT=BT, + ) + return o + + +def chunk_kda_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: torch.LongTensor | None = None, +): + chunk_size = 64 + g = chunk_local_cumsum(g, chunk_size=chunk_size, cu_seqlens=cu_seqlens) + # the intra Aqk is kept in fp32 + # the computation has very marginal effect on the entire throughput + A, Aqk = chunk_kda_scaled_dot_kkt_fwd( + q=q, + k=k, + gk=g, + beta=beta, + scale=scale, + cu_seqlens=cu_seqlens, + output_dtype=torch.float32, + ) + A = solve_tril(A=A, cu_seqlens=cu_seqlens, output_dtype=k.dtype) + w, u, _, kg = recompute_w_u_fwd( + k=k, + v=v, + beta=beta, + A=A, + gk=g, + cu_seqlens=cu_seqlens, + ) + del A + h, v_new, final_state = chunk_gated_delta_rule_fwd_h( + k=kg, + w=w, + u=u, + gk=g, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + ) + del w, u, kg + o = chunk_gla_fwd_o_gk( + q=q, + v=v_new, + g=g, + A=Aqk, + h=h, + o=v, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_size=chunk_size, + ) + del Aqk, v_new, h + return o, final_state + + +def chunk_kda( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float = None, + initial_state: torch.Tensor = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, + cu_seqlens: torch.LongTensor | None = None, + **kwargs, +): + if scale is None: + scale = k.shape[-1] ** -0.5 + + if use_qk_l2norm_in_kernel: + q = l2norm_fwd(q.contiguous()) + k = l2norm_fwd(k.contiguous()) + + o, final_state = chunk_kda_fwd( + q=q, + k=k, + v=v.contiguous(), + g=g.contiguous(), + beta=beta.contiguous(), + scale=scale, + initial_state=initial_state.contiguous(), + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + ) + return o, final_state + + +@triton.autotune( + configs=[ + triton.Config({"BT": bt}, num_warps=nw, num_stages=ns) + for bt in BT_LIST_AUTOTUNE + for nw in NUM_WARPS_AUTOTUNE + for ns in [2, 3] + ], + key=["H", "D"], +) +@triton.jit +def kda_gate_fwd_kernel( + g, + A, + y, + g_bias, + beta: tl.constexpr, + threshold: tl.constexpr, + T, + H, + D: tl.constexpr, + BT: tl.constexpr, + BD: tl.constexpr, + HAS_BIAS: tl.constexpr, +): + i_t, i_h = tl.program_id(0), tl.program_id(1) + n_t = i_t * BT + + b_a = tl.load(A + i_h).to(tl.float32) + b_a = -tl.exp(b_a) + + stride_row = H * D + stride_col = 1 + + g_ptr = tl.make_block_ptr( + base=g + i_h * D, + shape=(T, D), + strides=(stride_row, stride_col), + offsets=(n_t, 0), + block_shape=(BT, BD), + order=(1, 0), + ) + + y_ptr = tl.make_block_ptr( + base=y + i_h * D, + shape=(T, D), + strides=(stride_row, stride_col), + offsets=(n_t, 0), + block_shape=(BT, BD), + order=(1, 0), + ) + + b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32) + + if HAS_BIAS: + n_d = tl.arange(0, BD) + bias_mask = n_d < D + b_bias = tl.load(g_bias + i_h * D + n_d, mask=bias_mask, other=0.0).to( + tl.float32 + ) + b_g = b_g + b_bias[None, :] + + # softplus(x, beta) = (1/beta) * log(1 + exp(beta * x)) + # When beta * x > threshold, use linear approximation x + # Use threshold to switch to linear when beta*x > threshold + g_scaled = b_g * beta + use_linear = g_scaled > threshold + sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled))) + b_y = b_a * sp + + tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1)) + + +def kda_gate_fwd( + g: torch.Tensor, + A: torch.Tensor, + head_k_dim: int, + g_bias: torch.Tensor | None = None, + beta: float = 1.0, + threshold: float = 20.0, +) -> torch.Tensor: + """ + Forward pass for KDA gate: + input g: [..., H*D] + param A: [H] or [1, 1, H, 1] + beta: softplus beta parameter + threshold: softplus threshold parameter + return : [..., H, D] + """ + orig_shape = g.shape[:-1] + + g = g.view(-1, g.shape[-1]) + T = g.shape[0] + HD = g.shape[1] + H = A.numel() + assert H * head_k_dim == HD + + y = torch.empty_like(g, dtype=torch.float32) + + def grid(meta): + return (cdiv(T, meta["BT"]), H) + + kda_gate_fwd_kernel[grid]( + g, + A, + y, + g_bias, + beta, + threshold, + T, + H, + head_k_dim, + BD=next_power_of_2(head_k_dim), + HAS_BIAS=g_bias is not None, + ) + + y = y.view(*orig_shape, H, head_k_dim) + return y From 02af36df36cbea2e8c97ef4b473d607ad19e8319 Mon Sep 17 00:00:00 2001 From: Kero Liang Date: Tue, 28 Oct 2025 23:01:24 +0800 Subject: [PATCH 040/127] [Bugfix] Fix allocation & free logic of SingleWriterShmRingBuffer (#27117) Signed-off-by: Kero Liang Signed-off-by: Roger Wang Co-authored-by: donglu Co-authored-by: Roger Wang --- tests/distributed/test_shm_buffer.py | 65 +++++++++++++++++++ .../shm_object_storage.py | 14 ++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/tests/distributed/test_shm_buffer.py b/tests/distributed/test_shm_buffer.py index c6ceab181ff55..9fe409edc3ca2 100644 --- a/tests/distributed/test_shm_buffer.py +++ b/tests/distributed/test_shm_buffer.py @@ -4,6 +4,8 @@ import traceback import unittest +import numpy as np + from vllm.distributed.device_communicators.shm_object_storage import ( SingleWriterShmRingBuffer, ) @@ -113,6 +115,69 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): self.assertEqual(self.ring_buffer.data_buffer_start, 0) self.assertEqual(self.ring_buffer.data_buffer_end, 0) + def test_allocation_cycles(self): + buffer_size = 100 + ring = SingleWriterShmRingBuffer(data_buffer_size=buffer_size, create=True) + + # tracking allocations for assertions + allocated_bitmap = np.zeros( + (buffer_size,), dtype=np.bool_ + ) # addr -> is_allocated + allocation_map = dict() # monotonic_id -> (addr, size) + + def count_allocated(bitmap) -> int: + return np.sum(bitmap).item() + + def is_free_fn(a, b) -> bool: + return True + + def mark_allocated_with_assertion(id, addr, size): + addr = addr % buffer_size + self.assertEqual(count_allocated(allocated_bitmap[addr : addr + size]), 0) + + allocated_bitmap[addr : addr + size] = True + allocation_map[id] = (addr, size) + + def mark_freed_with_assertion(id): + self.assertTrue(id in allocation_map) + + addr, size = allocation_map.pop(id) + addr = addr % buffer_size + self.assertEqual( + count_allocated(allocated_bitmap[addr : addr + size]), size + ) + + allocated_bitmap[addr : addr + size] = False + + def ring_free(free_size=None): + freed_ids = ring.free_buf(is_free_fn, free_size) + for freed_id in freed_ids: + mark_freed_with_assertion(freed_id) + + def ring_allocate(allocate_size): + allocate_size_with_md = allocate_size + ring.MD_SIZE + try: + addr, monotonic_id = ring.allocate_buf(allocate_size) + mark_allocated_with_assertion(monotonic_id, addr, allocate_size_with_md) + except MemoryError: + # free 2x size for enough space if wrapping happened + ring_free(allocate_size_with_md * 2) + + # retry allocating + addr, monotonic_id = ring.allocate_buf(allocate_size) + mark_allocated_with_assertion(monotonic_id, addr, allocate_size_with_md) + + # 1. allocation & free cycles + for _ in range(33): + # will consume 2 + 8 = 10 bytes per allocation + ring_allocate(2) + + # 2. free all allocations + ring_free() + + # 3. try allocate the largest possible buffer + ring_allocate(buffer_size - ring.MD_SIZE) + def main(): """Main function demonstrating usage and running tests""" diff --git a/vllm/distributed/device_communicators/shm_object_storage.py b/vllm/distributed/device_communicators/shm_object_storage.py index 080bc03e39137..2ec33afb87839 100644 --- a/vllm/distributed/device_communicators/shm_object_storage.py +++ b/vllm/distributed/device_communicators/shm_object_storage.py @@ -127,9 +127,7 @@ class SingleWriterShmRingBuffer: if create: # we are creating a buffer - self.metadata = { - self.monotonic_id_end: self.data_buffer_end - } # monotonic_id -> start address + self.metadata: dict[int, int] = {} # monotonic_id -> start address self.shared_memory = shared_memory.SharedMemory( create=True, size=self.data_buffer_size, name=name ) @@ -288,7 +286,15 @@ class SingleWriterShmRingBuffer: self.monotonic_id_start = ( self.monotonic_id_start + 1 ) % self.ID_MAX - self.data_buffer_start = address + if self.monotonic_id_start in self.metadata: + # pointing to the start addr of next allocation + self.data_buffer_start += ( + self.metadata[self.monotonic_id_start] + - self.data_buffer_start + ) % self.data_buffer_size + else: + # no remaining allocation, reset to zero + self.data_buffer_start = self.data_buffer_end = 0 freed_bytes += metadata[1] else: # there are still readers, we cannot free the buffer From a8c02fb5bf2e3191ec93af7c671697583a2e0c56 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad Date: Tue, 28 Oct 2025 23:42:05 +0800 Subject: [PATCH 041/127] [Bugfix][CI] Fix v1 attention backend tests and add CI coverage (#26597) Signed-off-by: Mohammad Miadh Angkad Signed-off-by: Mohammad Miadh Angkad Co-authored-by: Ye (Charlotte) Qi --- .buildkite/test-pipeline.yaml | 9 ++++ tests/v1/attention/test_attention_backends.py | 34 ++++++++------ tests/v1/attention/test_mla_backends.py | 3 +- vllm/v1/attention/backends/flex_attention.py | 45 ++++++++++++++----- 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 6cbc25b4b3bff..03268beecfc0b 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -313,6 +313,15 @@ steps: - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine +- label: V1 Test attention (H100) # 10min + timeout_in_minutes: 30 + gpu: h100 + source_file_dependencies: + - vllm/v1/attention + - tests/v1/attention + commands: + - pytest -v -s v1/attention + - label: V1 Test others (CPU) # 5 mins source_file_dependencies: - vllm/ diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index 351cff246d614..6659b3eb1e98f 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -428,6 +428,7 @@ def _test_backend_correctness( # [num_blocks, 2, block_size, num_kv_heads, head_size] # Select the appropriate KV cache format for each backend kv_cache_for_backend = kv_cache + reset_kv_cache_layout = False if backend_name in (_Backend.FLASHINFER, _Backend.TRITON_ATTN): kv_cache_for_backend = kv_cache.transpose(0, 1) @@ -437,20 +438,27 @@ def _test_backend_correctness( kv_cache_for_backend.transpose(2, 3).contiguous().transpose(2, 3) ) set_kv_cache_layout("HND") + reset_kv_cache_layout = True + elif backend_name == _Backend.TRITON_ATTN: + kv_cache_for_backend = kv_cache_for_backend.contiguous() - backend_output = run_attention_backend( - backend_name, - kv_cache_spec, - ["placeholder"], - vllm_config, - device, - common_attn_metadata, - query_vllm, - key_vllm, - value_vllm, - kv_cache_for_backend, - sliding_window=sliding_window, - ) + try: + backend_output = run_attention_backend( + backend_name, + kv_cache_spec, + ["placeholder"], + vllm_config, + device, + common_attn_metadata, + query_vllm, + key_vllm, + value_vllm, + kv_cache_for_backend, + sliding_window=sliding_window, + ) + finally: + if reset_kv_cache_layout: + set_kv_cache_layout(None) # Check shape and dtype consistency assert backend_output.shape == sdpa_output.shape, ( diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 1a256a6e192ad..1b17532884841 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -155,7 +155,7 @@ def create_and_prepopulate_kv_cache( scale_tensor = scale_tensor.to(device=device, dtype=torch.float32) else: # Create MLA KV cache: (num_blocks, block_size, head_size) - kv_cache = torch.empty( + kv_cache = torch.zeros( num_blocks, block_size, head_size, dtype=dtype, device=device ) kv_cache_flat = kv_cache.view(-1, head_size) @@ -212,6 +212,7 @@ def create_and_prepopulate_kv_cache( start = start_block_idx end = start + num_blocks_for_seq block_table[i, :num_blocks_for_seq] = inv_perm[start:end] + block_table[i, num_blocks_for_seq:] = 0 start_block_idx += num_blocks_for_seq # Create a realistic slot mapping that corresponds to the block table diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index e12cc581dd1a7..c16a77c093cfb 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Attention layer with FlexAttention.""" +import math from dataclasses import dataclass import torch @@ -592,9 +593,10 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat self.headdim = self.model_config.get_head_size() self.block_size = kv_cache_spec.block_size self.kv_cache_spec = kv_cache_spec - self.direct_build: bool = is_torch_equal_or_newer("2.9.0.dev0") - self.q_block_size: int = 16 if is_torch_equal_or_newer("2.9.0.dev0") else 128 - self.kv_block_size: int = 16 if is_torch_equal_or_newer("2.9.0.dev0") else 128 + supports_small_blocks = is_torch_equal_or_newer("2.9.0.dev0") + self.direct_build: bool = supports_small_blocks + self.q_block_size: int = 16 if supports_small_blocks else 128 + self.kv_block_size: int = self.block_size if supports_small_blocks else 128 def build( self, @@ -867,6 +869,22 @@ def get_kernel_options( kernel_options: dict[str, int | bool] = { "FORCE_USE_FLEX_ATTENTION": True, } + + def ensure_divisible(candidate: int, block_size: int) -> int: + """Pick a kernel block size that divides the logical block.""" + if block_size <= 0: + return candidate + candidate = min(candidate, block_size) + if candidate <= 0: + return block_size + if block_size % candidate == 0: + return candidate + + candidate = math.gcd(candidate, block_size) + if candidate <= 1: + return block_size + return candidate + if vllm_is_batch_invariant(): kernel_options["BLOCK_M"] = 16 kernel_options["BLOCK_N"] = 16 @@ -877,17 +895,22 @@ def get_kernel_options( kernel_options["BLOCK_N"] = block_n return kernel_options else: - kernel_options["BLOCK_M"] = 64 - kernel_options["BLOCK_N"] = 64 - if query.dtype == torch.float32: - kernel_options["BLOCK_M"] = 32 - kernel_options["BLOCK_N"] = 32 - # if current_platform.is_cuda(): + preferred_block = 32 if query.dtype == torch.float32 else 64 + block_m_candidate = ensure_divisible(preferred_block, block_m) + block_n_candidate = ensure_divisible(preferred_block, block_n) + if torch.cuda.is_available(): device_props = torch.cuda.get_device_properties() max_shared_memory = device_props.shared_memory_per_block_optin if max_shared_memory < 144 * 1024: - kernel_options["BLOCK_M"] = kernel_options["BLOCK_M"] // 2 - kernel_options["BLOCK_N"] = kernel_options["BLOCK_N"] // 2 + block_m_candidate = ensure_divisible( + max(1, block_m_candidate // 2), block_m + ) + block_n_candidate = ensure_divisible( + max(1, block_n_candidate // 2), block_n + ) + + kernel_options["BLOCK_M"] = block_m_candidate + kernel_options["BLOCK_N"] = block_n_candidate return kernel_options From f5710ef02a230c55827c5843218801718f0b19df Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Wed, 29 Oct 2025 00:23:35 +0800 Subject: [PATCH 042/127] [Misc] Make `LayerBlockType` a `Literal` instead of `Enum` (#27658) Signed-off-by: DarkLight1337 --- vllm/config/model.py | 26 ++++++++++---------------- vllm/utils/__init__.py | 6 ------ vllm/v1/worker/tpu_model_runner.py | 3 +-- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index a488258c56da2..e22c218c769da 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -41,7 +41,6 @@ from vllm.transformers_utils.config import ( ) from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri from vllm.transformers_utils.utils import maybe_model_redirect -from vllm.utils import LayerBlockType from vllm.utils.import_utils import LazyLoader from vllm.utils.torch_utils import common_broadcastable_dtype @@ -91,6 +90,7 @@ LogprobsMode = Literal[ ] HfOverrides = dict[str, Any] | Callable[[PretrainedConfig], PretrainedConfig] ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"] +LayerBlockType = Literal["attention", "linear_attention", "mamba"] _RUNNER_TASKS: dict[RunnerType, list[TaskOption]] = { "generate": ["generate", "transcription"], @@ -1433,11 +1433,11 @@ class ModelConfig: def get_num_layers_by_block_type( self, parallel_config: ParallelConfig, - block_type: LayerBlockType = LayerBlockType.attention, + block_type: LayerBlockType = "attention", ) -> int: # This function relies on 'layers_block_type' in hf_config, # for w/o this attribute, we will need to have workarounds like so - attn_block_type = block_type == LayerBlockType.attention + attn_block_type = block_type == "attention" is_transformer = ( not self.is_hybrid and not self.has_noops and not self.is_attention_free ) @@ -1469,9 +1469,7 @@ class ModelConfig: ) else: return self.get_num_layers(parallel_config) - return sum( - t == block_type.value for t in layers_block_type_value[start:end] - ) + return sum(t == block_type for t in layers_block_type_value[start:end]) # Hybrid model Minimax attn_type_list = getattr(self.hf_config, "attn_type_list", None) @@ -1481,19 +1479,16 @@ class ModelConfig: # Hybrid model Qwen3Next layer_types_value = getattr(self.hf_config, "layer_types", None) if layer_types_value is not None: - if getattr(block_type, "value", block_type) == "attention": + if block_type == "attention": return sum( t == "full_attention" for t in layer_types_value[start:end] ) - elif getattr(block_type, "value", block_type) == "linear_attention": + elif block_type == "linear_attention": return sum( t == "linear_attention" for t in layer_types_value[start:end] ) else: - return sum( - t == getattr(block_type, "value", block_type) - for t in layer_types_value[start:end] - ) + return sum(t == block_type for t in layer_types_value[start:end]) if ( layers_block_type_value is None @@ -1501,10 +1496,9 @@ class ModelConfig: and layer_types_value is None ): raise ValueError( - "The model is an hybrid without a" - "layers_block_type or an attn_type_list, or a layer_types " - "in the hf_config, cannot determine the num of " - f"{block_type.value} layers" + "The model is an hybrid without a layers_block_type or an " + "attn_type_list, or a layer_types in the hf_config, " + f"cannot determine the num of {block_type} layers" ) def get_mamba_chunk_size(self) -> int | None: diff --git a/vllm/utils/__init__.py b/vllm/utils/__init__.py index 549827a927d9a..b5a7fea2c3571 100644 --- a/vllm/utils/__init__.py +++ b/vllm/utils/__init__.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import enum import inspect import uuid import warnings @@ -67,11 +66,6 @@ STR_INVALID_VAL: str = "INVALID" T = TypeVar("T") -class LayerBlockType(enum.Enum): - attention = "attention" - mamba = "mamba" - - def random_uuid() -> str: return str(uuid.uuid4().hex) diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index ce769e8575ffd..5d7b181989ce5 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -53,7 +53,6 @@ from vllm.multimodal.inputs import ( from vllm.multimodal.utils import group_mm_kwargs_by_modality from vllm.sequence import IntermediateTensors from vllm.tasks import GenerationTask, PoolingTask, SupportedTask -from vllm.utils import LayerBlockType from vllm.utils.math_utils import cdiv, prev_power_of_2 from vllm.utils.platform_utils import is_pin_memory_available from vllm.v1.attention.backends.pallas import ( @@ -212,7 +211,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): # Model-related. self.num_attn_layers = model_config.get_num_layers_by_block_type( - parallel_config, LayerBlockType.attention + parallel_config, "attention" ) self.num_query_heads = model_config.get_num_attention_heads(parallel_config) self.num_kv_heads = model_config.get_num_kv_heads(parallel_config) From e3d81866662e0ce398a2985b82e59075f327f51f Mon Sep 17 00:00:00 2001 From: Zhengxu Chen Date: Tue, 28 Oct 2025 12:54:26 -0400 Subject: [PATCH 043/127] [compile] Add fallback path to AOT compile when serialization fails. (#27350) Signed-off-by: zhxchen17 Co-authored-by: Cyrus Leung --- vllm/compilation/decorators.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 376039e4f133e..69fb93601f935 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -403,8 +403,17 @@ def _support_torch_compile( output = self.aot_compiled_fn(self, *args, **kwargs) assert aot_compilation_path is not None assert cache_dir is not None - os.makedirs(cache_dir, exist_ok=True) - self.aot_compiled_fn.save_compiled_function(aot_compilation_path) + try: + os.makedirs(cache_dir, exist_ok=True) + self.aot_compiled_fn.save_compiled_function( + aot_compilation_path + ) + except Exception as e: + logger.warning( + "Cannot save aot compilation to path %s, error: %s", + aot_compilation_path, + str(e), + ) else: output = self.compiled_callable(*args, **kwargs) return output From 130aa8cbcf6f14a642b8d0594a3d33b8214b0f6f Mon Sep 17 00:00:00 2001 From: Matvei Pashkovskii Date: Tue, 28 Oct 2025 19:49:15 +0200 Subject: [PATCH 044/127] Add load pattern configuration guide to benchmarks (#26886) Signed-off-by: Matvei Pashkovskii Signed-off-by: Matvei Pashkovskii Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- .../contributing/load-pattern-examples.png | Bin 0 -> 591044 bytes docs/contributing/benchmarks.md | 67 ++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 docs/assets/contributing/load-pattern-examples.png diff --git a/docs/assets/contributing/load-pattern-examples.png b/docs/assets/contributing/load-pattern-examples.png new file mode 100644 index 0000000000000000000000000000000000000000..9f356dc24fa3a8e8c2b2609748dfdb8b8926d97c GIT binary patch literal 591044 zcmeFZXH=D0)-{Snl~gIAEX9BXEfEDH0+K-s6%Y_4sYC_Ik~0Wc1~395l7lGefPe@{ z1{D#>N;+gja+aLo&TaSmwbpmX?Z5ZO>loc-IGppGz1LoA&bj8=ca+XZGpu7-M?*uy zAS-i1nTBS|SsI%4_kLZAzX|*NwG#gku{(L*PQ}vD&he6s0gb{XJF6>}c2`XFb~qT= z*qT^c@Eznmd{A)z4r4nzD_fC6hs^)aD-K%P7#)&N-KC0m`OQk^f-Mcr!CmBkYn);v z9cb3j(8!)RuIdyv+-4u9YOuOGKI`z}xZr;6^)i|KGV5;J#r^aL&(ok^^@hxU|8vKV z9o!GfzyoDSch39<8F(J!=iCvJEYkv<@B(t>nU| z{+C}eJGH{w{>y8~KQFOK)!}LX%dhCY^k?b*%WG(8e4idZ!S-KYb?olj-|heFt7y2_ z{qKYQdzJoo#r_j6{~KffS&RQ)oXQvHXGhz5zI@T^H+pw_LzTsLJfF$CU_K-IF==fr zt+2ME%JEm{C$_mCNYc!#YBHz&&-3GV*Z5zzH8K3~SO4>e=%X7pY-mn1@^Khxj?pji z9$1>66!BQeOV%w+{_sKOr=+>{=)aCFk#zMDsg87SOX>8To|?;c{kyHUnA2Zydg=` zNGMr8_=suK%b1?!t!lXr=1IDR5~a@Xz1b31mKQZYeky9qvX;uW?Nhh2v-7hJmvGaV zpXiZuQDHoy_wH8}Ybj6qvkOxL2S$E*txKI3u^$xLUlD{Rxcu)d$*}2sw=EnvxYH|iKgl>-ob6I)2qs&OL8p}`*X{J`S(rcSg}o)KRrF! zoa>m=AMMF7 zC=c)uix9Nb5wL96stgr8H`1KZgJ1D8-&cTZs<(Cyez!t2+^?~l@(fD!>(-s-m8z?p z>cCEJ;yuyE2XACEVs896G)kx%(Lbs^pOVXQ)^Ax}fu&cUkDHHw+HZ~s~u#4FWR z;zPyv+F$K4+A92jO;jCG@HjFuQX}0s>iCVd*yPgz)Ste8ROOAH@jnFEX5p)IFWgs` z9qW@csS~Zg{q`G+phZ}Z^O_w3GBPqBvBw$qL+bg{4Vr_Esn6P-``DOyR6jH&#cbVm z>i%I?zRMqe_hJ>ZUYs#^o2(S{?5Ys^R~cmG=Hfcl_5ML^|EFj6`0r(k^H{rhWtgxE zpHY=+eS(^R)7Og`7Ht}%qf32HzJElgg|8`w-~R>8`Ul(i-tUyE3*3L$ZwKdg$?=5t1ZKRiS9Zf_QcmF3wE&4`UO%4tDC zTiuq%OA>VQbE+aG$A%rgz4#c*hHcv@YiQTTpM7@f4BM0M&k@bFAA{yv%pZUJ!SBB0 zBxlPm=A;xA6&29?^R;xn5^py0uHGDyqr>X|dRHFD`3Zyi=NDA4jssNZAK&&f4Wc!JvszJdIU4aSXS3!QIfx?5b6q}F7XR9g=s(Y;=%_6( zOfzrZ-23n4_@8%Nx|f?qa_N75)jSOsbu=AnNJ>zSxjVpm=gyq~27S6~amBYc?5fK! zJ2yW!C-gs0`gS)LR{|bD#Calm_x#e*QeCv{MzY2+a)Ii9|MZJLavfg37uf+>HpglF zDfznMt~18iAdNzIaXOP9R?X-2KQ`hx*8frT&cS`^&tF_|Vxkd}Qd07*3KO1e6mT4E z{WLv9&A{3xX=I!(T$;G|^w~27YTniPm+mX`M)^}UaRvDHZIt2vy zn;zu|F=y46mt`OCJ9h^8PX%8;yk-$q#(t{*EStECDwe7q(YI;y=2*GFg9F%?yWZaF zs;d474(2Ue_TXWiot?RN>^OGe<>j-lt~9B}EA3=HprOjVPhGX?rNM#iM-3VVYU8F` zEDKGFJQ+l6x^ExU$&1f*94l*U(_UFwDSNcX=d}OcD2)tLitHnXFsH9M1$F~|$y(V@KYaK=mon1d zZ=9W-U0Pl)hj_%YqH#@3bTwGj= z4?spb_vZR9YS}gh0v4_3k^4@+YX53S-piX^oTP$*x3|`>HcqyB6t;HEHeNZ3#U@{E z+mXvk_;7zdqchCh$|s8NZ8_Id_E;_5_-rmuoS>y`-~qB-5<(acB-Id*&TQg(Lsb@cSA$PcFj z4<>~Om?bz*_Tq2Szg`egcXG-i2MaIlA8pIEv9^9#SXemGSDo_Lt##t|H_oaI<l;45}da!GmXxhAK!%NK8ymyS)&s2p65b5&87wy#!T? za#^Hgp_aCG_=K49y3op4?OX>FfDjVVWKqnh$piRMruNxi)^EnEW9pt9rxm!;sD#}D zkoBFS8oeKB$&}3UCdls6rAs-^ld)uHwr@|odGlvySJ!f^JU*GMgiTLb4E6|LM2kl? z-WNL@qgQ-actpZ#qFdIyEk})n#&}l=$FS|FN3T7;Xi7@TPqeg;fya8PBXt2?&*9vt zkt}G45!|Yu9Mv#C+;<1i#NTGG~5N3@E2tUYV0LE5+O4zL>O@f#vcACY<8hExK1P?TxkU zCAHdV>v`C|2SWqlnWW3FvSuA?GQdWKmN05qV*a1}#eTGJJc zT!$o5QmM413 zn{(_F@$U)vl0zS(q;g!Q6QWpW>CSXy+e-IUN3K?PVQUj~3k!6vj-Sk*`*N>Tgk~=e zU*y$^JAeK47rTqoaqt`DLc8ffH9%=OWXQn8#KaIm%j8Q%o)n~^(uSehcSS`~mW9i& zavaAjcvf(}6FL*rXMv?(& zNIDZPLb>?n`uoU=Rl$GLg4nLr10E|&p87fVLo=V*>rpBcLXTFybg!UiUio~3iC^#C zZO;09WR}=7;R%RuD2z!Z?3XVZD2h4^f5g^B-CDneM?zvHDz&q- za~%^C6%n4HSG?}QR^Gor#6D?S=EsnOdgr(|<@kvcH*qwcJd^fexmzC07stpdbkU{o z4+I#}2+4e&Ur2_{cb(-ykSS`pF3pY8&E8y;avN*UH;dP~d^!5hUAydu8V-bmOG*0R zMHi|iSM!=*nI{4PYPctT2s{)CAY-hJ}3%L34-l?NQCPtlcp%&qc%XFiu#O2I;z-E$mf z0D-<2YX&{eJwMO8JYK?q^*(|xxFrsZR=lxev?)y*#aRipx=nc0qYrR33V>bGPFE$% z>QaVj{b@E)2l)U=A3zPW!8!^lRVd*T@1|!9ALZE(1#pT_*j6k{xhAM3#**hoR-Txc zc$q%Cs65pE>v#C<^@X`Z-)jZO5^CZUIf{6(nJ=EZ>*+lOx-$@}lJAgdVmW$cFYrok zgEy#~(x^#oENWtFa)u(y@Av(8@$#OPTwO}LyNUZL@|-ag07YO#PLNMh#&ldq1W;k+ z4izrFeEat8{=vCJhw`WVY`Q;0AwmWaDJsUs@ttc1XKo|%YGUL-T?KneA4!iDt`=xZ zmHK@F=5y)1wFRk6JLy7NZxn4E$8LP)y)B2-K~CzkZH*vH6#X48k&j|)=t5J6qQS8^ zhaI3y_IU%vdT!wj<*&`8&DGwb(#PI^@vjCcgOc5AD`>1 z3MY^Rdo%VBkJ&Rf-bJzuV#vbDioZfO@L+gvMer_5OG_zh6xhLz!oq=^vAjudeE)+7 z4=$jhjj%k?0U9Alur0^F7VnUuU&2^KTQ@>L*YZ@YB0=5+h#=fqnr{ecT)mpt?l!9} zK3T!H&=aUjpTEZk)gD}qqLc5EV>eKvPemFyM^>Fj{dqi6AxqB6`}glnC%Q{{&OW{Q z{=s%5M=hI1z!(xVBsr0jHcVqbkka?kVMHT;qHG_I_qmtDe?L+&&8MD{mL4i%l2mQa za~81re9XK#y|l*F5b4TcxG@SDF%gxE$UNlVaoEdEqfd}y2*kmE@%6P5_L_;qNS`cx zhWAO#5T)mapi%Y3Ug2pkCR} zeZ3@weVE37T8 zaG7Z`V&1wn9#!&!#;%Of2j87Uny?2ys0lmNC@8}y*+zQ}!B0KC_{7Wbb?iFd{vyRo zxZ$mJ%=;-m{<}0^T`{eVeV}c&I@iTC(oxuvoBVOajcLl3jUPUcL^gQ8iY6TmF1+sHfM- zB9_$DRF~N{M*^`S@zn2>>V_JU44M~@Mr9Pk1Hn2aLb)QRNO zy_5+Qkbnxxa`@skKs8q7q78l%+oFM{T>8Qt_m6(iz;=L$o|(zP=GKxr%`Wb8-r@7L zO2b+RE2=T4nY|c{J%WHM$+`B^qBvXr{9Ua2{_!KJkLE$%f)*9nnT52gsqBGc=&6qy zpf774Eq2Ot+HG}t)*)&AZ@+!|^h|n0qXo*%FKgE}x6y`mk=zgFHq0I!se1P8S(l-t zOw~m71KQa(yDylEl;Nb;$DiHhwm6a*(5szh_y8XhR~~psGhh5n zkEbf(lD4-mgjfJ1Ey5pxs>0f;RC=IzB%DuuiX9*EjWSr>j8HL(a6SdzV&{}pk_~Kx zBKnMB`%$3`pTf|i<{=TTqr%!bcBL=FsL9$Yfb0kI3%Wk(xX&99OzhdUmVWCKl%D4N z74WpdhNRHWoq^CI>!FN3c-6NFuwMmaVH%Mnl;j0OK%yK9Q~vIG!nARW2l0qYC1XIP zNv8D)7ew~ER0Q#|2-`#(%-;u{AuGa9ROSi7!T{R%?LQwo?He+du{b++4v{pnBJaHi zTwsuz5=rttq%VyCXo@{#MJzfB%pK(m-Iu#NJ1@moL5;gy`bHYeuO5LxU(9dTAP;b& zU}GKxXvT(At0)(+-`ZhcEMXLH-n`Lxaq;gWGcY8O^D}@@@*C)i8jJehto4{BvP*e+ z`CE&c2m-WlTmr5hbw&IVG^ydjA$#VZp`!ecP!U0S^#ufq3Idtu%C>;8+1behu>o74 z;%BzYAVDkpB}$YrwOQAF{v@c2%NG$>uOzveeJOc*vl^U*prQ6-?b??4(*e7ssP_r; z=21(ChBURu)qEdT!t{ghPQt8q&y!jNfWdv_NR8iBk1c>v5Zz5t#RK}>LxS1-NDs$) zPlG151|%Q>`O7GCLbWl^+4d#7sWs93>+t*KrOE12joE^C?@kaMAEegd^s?7HzA;g& zyz*vJbn+7db={jL@|?0D!f+FkqK7Syi4b>f7T@Id`q~93f(@#_|sp4mhI^;_~=>qWh9rr-jK2xYfP0o zp!w>m)dOcgR$-fXD1n})S~+&I$S(r5?UJZ%C06%?r8qBzgN}*zL|l1{&!vDlGiLWm zR6*cTP0|n;X-eC3VTgtBWi}DJ52{xho?nT_-U(@?iU#Q zp~fly@k6iTn(7`%2d1rAsZl{7l4iBB3bF~gmrMV+<>j^gE)eDehy{;Sw{W+BD2-@r zL5YcUUl>R=I}<691b));7%T5W;ET4|!=)=nLtUo&qwr(G89sz`pu&SQ`Kv6xc>Cn| z4N$)-&Ymo=LycEgD3GV`S zP=5fKm5~=sKRvybtdl?JD*}2p2q^cGT?fqc01>|86eAsE+@bYAATfrX!dPmVQRkjQ zlCFJk#Wu50mojpa_21vOHhzGw*s`2|pObX`tR6(O_d|z_1EXcRTMBwgYFc#8VM{1l zK*bN!S`Gu)Sd&5m{UJaxl+?U35oO_sa z8{PoP84LkF;>nSRGMjBIhHJOff6zcAh z_LhI;GSi56^-J!VVG(ypFEaal2w1oQ3I2+ZMdOQ0K5WU(<6llLz8j)P`>r2lOD4ehNrl#BuaF%8N+) z8ReK6*AcwyBeA{PS0N?kU8e9^e>rjB{EOGg8ED%nfeh03=;piF`!UL$&oF(AtYfg3 zbjoaOZTYk}%03F2eTcU@v!P>~PNBOtfp#-DE)A0oU~MWWO6H44aYax`71-pz0&?u* z;VE12^gC|avHko0Xf*#nlr3c()yKP}8KroUOjW>BDK11iLwGQJBiV>`K{Y0dS72b^ z!pw*nwHa(+09?tnU;p95haqClFSVT^3~z0GbRBf=ecg_;(hES-(=~F3gmWKowx9(L z*r(6qKmzgAmEdBwH=hCyYMZ(=+MtuOfAqaIPP-=)$LR3UnrNhLk}r|*>>T#_MFX`U zP48@Ce04s{Dr#Wa($o=uG;exY++;5F4oZ0UmoHy@BLQWD1_$t+&!Gp$P<-OlDI$)Z zmXVpO)`z&-mg^`eX7@@f+h*a*y+icH(I@b-TJ*ic+B^*)h=SA%U#3iV`ABlfvB}=O zeOo3#gs?xzHjiQgwwZM5K+tH7n{f|TM-u)pdOQOZ^QeD|VVcR8cXte%{Ww)3FS$+k>;2kUM7Ri!TMuIK+wo>Sf7u7o6W2(4s*+u=x2LCgcA1NTL!$MgSuNfJtHU__ z!Mf({4|IMfUcjP&%6fVz%Vm(c=+JTy(I2i5a+RY?M%e#oD)jI`S76)dM(-^k4uLu@+_JQZPzq8|P7|y; z-vU90LVG2B4qxBj2??=l#~V;IGc8<3v-{-x=0p!2A}8hy;02TWOQY(89v&X^P*&0U zz+xl8kv=4PX+-TdK_$`1wuu=y{#S>`YJPL3rR83(ojZ>sA@iw?NpNs*2$nWwxHjHo7~d=a#l-Ai{HGNA7ard$N~-}$bt$S9!set<9RHtUVz|-->>Pjw3 zwf>xK^c3}TF=>BI^?wSG*s==`@NDw|jlH@BR|)4OA|VPOQ6>W##=m}5Li1TQSxeZU zIzpRxDzM?{&^mJ^beCJe57G2-hE7vbBN1*6fky)e4&=6HSbb&JfmSmQm=KEcQPeb{ zjHy?K2&kdGMcxS_Ga`*jWHZ2(HUuUpZY^mdef@kK+O%TeLG2@2`wm#cK>?MS$SFM^ z9!eon@DDX0a2e=lk-|{simdB4-xG-Cez%cY@b&HbvT)JNgIZY^9ap$jVvQkdk|tl| zrIXUqX5(LqJ@pVNF(60V?*46Gp~Xbn#L3#C0>&SYAvBS{0@0xAfp!jlH3V=^!tzXn z7zJ@Lh*Bp4br{9Z!enfDah8fFRVX&Z_d!Zm1q|E7%K9Ap!2^#Dc8~!6SovT*Q(X!4 z01jy9@E5bKbp~=$u?DLNLg(gZ}JqSiHDeDPWIt^5t&k-B52i+<>L`tRU@tPR+q;- zMBEnADUE1GyDbdF5ymJcCdQs)xVp0Jf=z^5XS0d`>GeVgDy^@tSG|gFjTSy1D3Cn( znJSd5h^<}tQg{nEozCI!RU;+b$U%nYG!*XGepX)oeiZvr%l79eIh136|Kw0o5h1RL z_AtL{pM;~MBLka=>R7?z=&r~y9NId_o#kG4B}P!D5&^b@?xIa~7FBgvXKZvd8Tc_C z7>#gsNHMeTwn|D$&d<+tn>9Qqx-jA01fUUR3*W)M`@;q)@s=w`iIWGg;c9mX(4oZ3 zOoA5^_Q>j99lwQs^_9K7W3+6j(Mix{R*eYiB%;;UQ6s47p8EK|$qyxjC_=;kV0A{; zv@Iv&y$?sq{tGXc`ck1Z@&E`B&8Pk9q{25snz9UVcNR~e_Tu7kmF1Y-&J*3Apc>Z0 z0fCn0QdcU_*&viJB^*Y3IE#w1sjzyT4SXRDoW&0T4F$XU~vawiplF3&m8-c*cZWqzUZQQsq z<5JOD5RFnl_6Lw814JB#d_tIK}j40LloQ5H7F(uRO+_}?}@##FE(V_x; zC5Ku25Ot0nZs$N}SF4ZnLmuMMNRPj}`9K2Tttra>U`u8ST0#u5(~oNaw^Fd(CJ>QW zgsh(6p)P0nK)z`aKhf6j$_pPK=?Cp%`gA%{BA?WHWGW&ro7O~^-YCIN#i1)D?sSM9 zJWy$*P;Q^=uJ5wRHGj_^<;;2>r6<=Ql_UbHHM6B%owsQ#aMQ%+Ws3Hq{(91F&NQtr z`>LcIZ2_q@-RyP_#Hb)%{i%`64vD$8h3TOY_JY<_eQBaO+C^H0L7nng%4c~f>S$r! z_~JNHkjHQi(F=jo$Qn^Dg#q`}e<{8PXs}Pp8tL6-d727rAGbusyWvGD}*qbAh9G>}t~D*vj8!mkFr zmti-cTq{g^RrK4Ah*@i+A?1R@y;I6_uVRE)?9j3{YFGv+BpyT#p>}lBBKVQG!)+Ic z$LzxVtrfpdqp$cW`R{Qb=h=JHw3MwS&{6{U3#F;lMU4tsANz$f%3IsGz#)H~9M7RLW zP(e;%W@9stw?HPYD{ymtHnMW#-o1N7L?g)-=}l2NdC{h~f;d!2Q`ZG*9z$^ix@BNR zsvyn>28!7s=+*-6jDK)_fFIWJShgp?1w2DeISG8%45-=XY92~k2)Y8M4bM%4X0aEK z+-|#|U=Y_G0&7c$>_(zr4-xD4k>L;I2D`nole_?$WLa{2;Wk*-o*pEsI;;nNC&&< z;w5F#tF4QX+hRXZa}w|dT|O#4a~g<~rOKf9GcyVnx=p$VG}2#~xuI$9e@IsX&9QS( za2=?^?n_yl_n-gjvu2_S2zR%X*UOdgrb08L@rem#Af+j!*~IgydI8=0QLJEQAt&L0 zZb7TkF!GQ=TGOz|@9+Cv$I%gX?xvaEGd?-#YGn+BI2(t8PX)ZSC+&vpnzDbg4ErBh zhU}&4i_wMm?%er=>O@~m^8QDcQYT%RWo#4$hq5KZ0XiL6lGE2htfOo05Iw8VM=MJi zcBnG-I9K!~uvHl&XQ6HA9tkYNB_H(l-J5nFz3yqQWr5p60HlMTo_X!2ty@jTab0Bw6Rz0I2}P87lU zJ7xsL;{gT|Qwm;Qxwha26HJJReL`mz0Nq}QhBZ17>X6}xnAn*GWYr>`HC%u)Y>W?Z zV7R`Ri&{zSG@g2XJm7=mXBpYW;vsSq8)X@sI0Q|?_x$`q8eqOD5r_b`qd@`S7I?ce z|L0$R`9_O}X{Hy61E^YJu|hQ*xH#nQ4QJB)?0iE-!@2?GY8qZXqWF92$vx&VSlAI- zxCru@4#Q0O+W6Hr;yOTtZ};YV*_ZkjG0iS~YLI0>bHTA zhfU#FZH!zT>dpw7sjF};n!;r$kUx9a*4OV@bc}1#!{g1+v<1YEEqCPQ=hp)zLqC*9 zf%l}di`xu@CSdn#0}5jbG>6~>0k;La+y-!vvk=aC{Y5!BPas@hy1Kyu0T5&ue+n95 zcUPDG-n9)!Y{WmH&1Z0f0=K*>Jmv}!Viy7r>3-8w9EX#&bJKBjiv%Q>r&Mz3;ycbk z-*B5K-Q#JtZTVo?!wUMWf;pzr`gMq1=_bI@tr~b@no7t>}YlrTtIz@Rm!OcpfYEO}z zo)D-WF7B!h!PjT`w_kr%f(%j@tFRp_*Yx$tbC2=}@m#nd7T(#zNtkNpltya2GD-t| zG4V748Zr*{54<$2WFdWWrTg$l0nFN|>s0xn!mTS|Gbq+9?CkwR zLy5?5=Z{#uLW5DZYJ4y055a8SDYVNqA{s6=KSJuM&9KpN4pcAIDoQ?vM5=9uy5?Lo z|7g)|wNkKyI0hY3s$p2#)FCrcey+MT~ z<;$6=y}g}0THBBjOb=P7VD&;}(Gxo3N7_Bm%(vcc%Cde2|AKsdnucm0}c zI*A+%1IlU9Z*}(LX6HZ z8-YWRp6TLHTW+Ql9pkPZ1ofjMmp@#jHZK;PC1hZ3&M8-qh@!RmZ&HB=GqANq)|jMl z>3Se2U%wxIyw0J>DXMR;KmYzXP%=+_!ei0H$Zhbvd2?&kAz763WQy55u#J&%{3V2od6<7~_AbNHG+J+5{=zb2Nr@qZb$5Kon#Jq`XuX#NT{~sDo?*iBv$XFW4yJ zEZpEDIuIb7iv;{8-E`47o*r(hwbFuGIR){J#rxia2WrG1A|x#%H`|Q5)DmuqA41N0 z+$j;E4q@mKoQ0iPNX`9lKOAKZ3<`>ecGQ-f0odo6EbYf{P__xS(WVRsh)kX~f$)l= zv&FxOc4%8R`-zh$NoR#|_`QT-atZBPdeWexxBz>9Ytn9ov<{C)aDq_=w8h#~{QYJJ zzXf3%53}z4y|Ntcb5T-^0*C)RF!E!wgXJI1judFso(+T^h6&duXAr&HzXc0{j@aQ% zH`0qrN~H5$reB(kKpiHAG7_!H(>O?V>ksAm`p%X$n6CFAww~6(GoT_hpyj_wQLo2TVE#2NS^2L%anSv&%jhy#}J* zTISYM2475i|D7SIuwocNEBR?jp!>Y%+vjE1a%$|)tKZ)$66 z?e?Gz;iRxny|3*>v5R{eZC=DFgdLT&)5%ykSBx&2;iih{^$@$q5^D8`T zuu{~%xj{#IMB$PiI#9163S`>BpMdp8OCKKN%Il-6)d1G}D`)v}+yjVvs>s^V(`Q*5 z+TaL$n;s$4 zs$>8FUNm-t$3Ow*uRd=e6oukusIlU*blQYxL4K~s`!f`yWm#gUkQkleY7UT-4pAZ3 z_I*i-SxvMoVb5aX;>?ikz;~9rNSah!%JXX=I;E^@7~0$1kQ-2f{i3p9&1=ek1qMUp zjwtHn&EGag2u(TQ>v%G1Ab+cZ294-2l~ABnbOwP?L8JRnTznE_LH*&MCs%C6e_glE z4~eB`IYd|}8HF)+tZOznn_Y8STUB{}VIjo(!QHzrUu^6n11!Y&O*BV9S9F8I$R494 z6BjxWcON{kZ+2u?OHh4?21}ILK&pO89ZnZvT1XeFKpU%*I{VMRcp^1(+r<1w??%l( z8uGuvC2^Gh#PWxSyS~|k0uQZz5J)_ia#XD1`L~3%YoruFmh{IVM@m>H<=azG_S)Lo#9~c6BTB6>Ysype;Khe#@eIkp1dQzNy?cD&`?w)) zMrI-1{3(mZy+0*uaov`K`=J!(y4(~Xo^tHlJ`~khTTJvB!_)wGaRl7Q!v&`_A5ax) zg7dks)-7Mo4BLKr_nQ~vA!0ya4QRP9<~$L^GQ}ke81Jd?q$nH+Pyu;1391hdyo!Ed zsPKe6Qm8jJKz^%7D<0k7BcziZmBGF{Tc?mz3w<=573v9ph?+K}3+F0{)W(87;=kEY zS*d_E{{&5kz8K{6^}oVu1O+q>N=k4U8=8<{4q|p$`54)|e&dE(j-3fHEdfWX0Z#1p zT8UmOyPUUA7R?KOu@h|{`m})&;4xBy@#&>#FH#Z}Q;SgX-bg3IAK~3L&q-Q|vh1b| zW~fk7H~=aD)Sc&1l3;C;oMIX1?2P(%hl2+&pUh_4c-T z*)up8GMuqm)&GXq!x5Quy{N1X>_*VSKEY2zxUM#cF}GV|uJq*i_$MLN7mhN&*bj0YurE;2XMg4u$MeUM1@f?zV1Tb@$(-0b4ybiPq`ka&_&IlY#; z%LA)-if5X)H2D1;jbwKgR;&H}{e{Hwvqi`0IM7oA!~! zy<4QS0gy#OGABtOiq&MqC5#o*G4i^Rj2seItOdR8(RO=$KvNyi_ah|P@HTHw{SIfb z)ul<|WTnlWm;39l>xj=-DATiVUd*Evv$gDv>-X)W*Qx*5;rod5?HV4)G2aZ^KJY9Y z1IKjev$3}q5VQKtB#_8_ATV+^Bt_yJyyD!#x(BL@)qTuE?arv|O^xdV;3jS+G*kHb ze?Axkd-xINzP`S|?q>DnTb-DZLTLFIdbVc;V!K?pk+T6cz3eqcYi z+v;*5K(qaMi?pA9`boaz$O&=#MG#styjMF8>6l8+_i+De7&J-nf;t=oHb2Q2zO-3u zdx4e6KhLL@oR;;o5znn2j`-9hm09~G{}t>-WUc+8LRW1u4MWC1K&ER@Z2U4DY+iif zd)UN!N^(6WK0bbg44Gl(lV=3#Qlc~h#V9l49cRh*Sm+> z`Jg>wyg(gAXBv%{7B}D18hPw{ykI5`y$f^D$6ZWHGQgP^o*}gjeSUMJ%3`;yVO1Cf zo4i|799aR`n1W_DVcamX>z~s)?QGt%FJpTBcUp(<#OmYkFD|(j4l&HqNvRtRO#0=O zv!tPeL@C89(*j+=wDk4iM9`fpP%K<=n4f8Xd+?!hP&Sw7qg{)Yz=}mS zp^yD{mZ7$6cDUZ%+uIuu?Ro|IUaI{NjN-(x3;t^2JX-f>_RIkja}WQJwI?(WB(7Imvy9z=c$LW$dc8iQA7u{t_)Ex zL?A3mtkXQ zLcGjdFvWw)a-rou?Qna(CK=0#8Iv^nZQZ&WG^+gJ|G1nYZq~3LHl+{AW{UqWoABd= zv2X0uXj{w21r+>cqhH^X0UQ|brJRY8)>yno0tX+)M{!wUGBE^w@j~tItYsC4c@x)i zgs}ZN6CL5PFtDdGT$HLh>i-sGM>UMuL(t@sl9~;U++%dVARs(Jj8f3=AbNrX7LG`I zFwS7$tc4#9Ikc?y$z~+ux(OsRZeHF`Pe&it1hIYX-%n zVF6e42Z>GiWkkm(t^^KMzv$zCgDEK3)1Hq|Kb0&x+m`Q2^=meq6ie=d*)V^yO4LyE zG;al?GoNVC$k_9RMTHUgLN2E@Ma(^dbs4WAb_kD^`8YDbfJw-H($q#91cL*{7Zib% zYG5)sQ10cw2M4gqVrTK!$w^y>_18E}@F)u%t1E%sVu0@rs4RxZr9GP9D&nQ|j0n@*uF7X^hor^VtJfuvT7)W=d*--115j+l~EV-$` zNBdC&wniC9q&AgI$OC2^7=Euod{qRawt4m(K@x<+Qx=l9I@q&D zOW`7 zdJj@ayu;-a`Lpe=#Cxg0OqTCUlu1R1y;O21V934P?VG(tBj{S@QHWf}E1xzLs$E!i+prK^q zN_cR!8ssYiz|}0jetzTGfDsop;7dQ!lORJZqm7`PL$|i*99FFk1{pKp8HIKbFn!VC zaC%ewD#;7PA5QEE#NmY&EBLUL-=stpc;pmjmn0o3Ypui`ZlZ*e-czhz3SJ5;ckvhR z=axeX3$K*-v#5Mb=Xvh>2bFgc~wbtP7dV zA>N+;jO`p8mI(pk6^y4*nteNIg;x$j%fhk!_ZcUw+aW0c)^Hs9Qv}_ z<;$1-hM=N-_*%fX8z1GXV#C?9{0Z%KGP0B7zMP{ygeFBYreNcd&O$^Tlaa3(>L+Gq zj<%S-EQh;@k_GsFL3tHO*I_k(NfUP=>>oJ3U^a)>kJO_LEK8&++-xC1mYpOtIMri7 zx&{%Xh*$M*4}C`Z&n?~l`Ev345OPyt6B}ny7V_O zuMgb?2p$R0Lohp^Dq0o1sENTu9PrO?*2I4L^oe{QwBhDH=R2*ovu@rT(Cq^Ctp;a~ zeLlRe_SKI=+ZO-GBWFN)R59e2OV`K+RS!7JWh8@2JTg)lrqtvl#C532M#VdD?#IJk zweBfE37Iy1 zRb>2NscROEXJXoh!G!eP5Ee2>z(>P809w^E234`2ayE?8SKHlKA<~emOgz`_Qw%yx zoO)E~!20l zHXyxF?_2u8&iN2v%IawBWgR&B{Ti}}vDqDIMis8EB5M4-&$V;a$wCxO9MZ~C zg37?lX4xC@8Q&h28~Bv=87cc4sH9`E|5lZo)u`jyxJ;$3$tUE{b?gOyy*;W14^(G z@&$cg6VmZ64!%jZI>aXf$4t;>W{haVY!fyx2ltx{phV`2XC+|ts+iE$rv^1yLVIb# z5~WRt?F0ZKPBlVs0q==Xl8irfiut%6dild4x7z<>D~6wf`z9@dT`UT}2A z{&!l3{4$oI?-f~9fX*IBWD*-81r`aDVDKn=Pe!(d#!H?)`n?fK7@@@dDDA4^j2C2G z0ilRjB*a@6W>fr#o=`ICeyo|s9oHl`;+TWJzuh|*(?hg5vN}jhL`ofSN{dXaxVm;J z7l(Hc7ew%Hf)!wk)R61+pnsJ0jO@kCxcOU!Y!MOQ)r4Jx4r0XI$$xOlu;}o}!{C($>aZ3Jpr~=xG2&#(~6ayZgv* zHsS@&5ou7U?pJ<=KH{6HOKUcA$scAuhmm3xYxOQ2vqHRnyLUD!^7I3tH?Y@Q47#|& z+3IKgypx%^%?{HZXt_;8 zHz5isuy*Z6>Yl!p8T9^Dp$}0S{fX8^B>kwZ(s^4V8-DN?UpxDcLaY8QqY8>#erN11 z*7mRY9(W$e#Y2pvYrQYp_4m0EQ3nnwy>I^il)Q<%u(&Z$6VYOUm0{sS$@00c zIwdUL7_PgG=ToRpqk?@}@>rm5!DSptUwq=kiT(_#4xbuRX;H;cONE&$xJih(>#NSJ z=nu$Dom-_DUdkbgw6q_rZdBX1h;HyhSRO5fMsd5`5&oZRZv&nY0Sk*mW|Rr#^{QJF zHUj6=Cqy@W@#XRW1z6>4a23@P2E&fcV7J*$ z`$lq|q4+aS45pkUDS{O;*0xp8Dq3PKaqq7xm*%k1^K_%S44dd zbMj<v(%Nl%O00oRgh~O(A+L}C1VYc_Zdpu zuwDWAUc_FNX{lReO`thImp#14atprE2n*s%;6WpgIL*QPH3POQ#`))YwWCKuxs;9L zb{4qHf2Y)1g-D6EugB?$Zx08PzBnj*58_Z7f6jDu!?g@Ju3aBHeXI)5L^pjU~mYvX)+gdffj7i!ai*MCEw=Eo(ODmMa&Sq#vrAJ!+e6rFrbYL9=kC&S(7 zCCD=saH~IW+(;K;P#L0zGweyX5!7Llfp{dB4GcyLDa2N4{RVbt3knS|Dhsrwta0m>K z`1T=l3SIELv#g2MHiPbVEFihy0vSUA|5JlF@))}$&3Fic&wb1)=sQXhPv8QErV{YS zXJljoA|r%EGD^eYtU$p7|NnB>S^w|TYOcQOKT6hag_f}t9LrWAt z_(JNErwvDc^W!)ExP#mugDU1UoP!#e2rY1!DdHz|AV|jwQvr0S$TO*CScL?1*9XBR zVoizdNHy#kyaHtCavES^2&Lp4TDPzYJSB_|(y7nV{TU%hCM*(Hv&c;1-YQJ0T=M!! zq#SvJ3|0~y(nst{A2JtP{f}C&Ltq3YXtD^-7ABVm$pBW zkDguZJc=wIvnwPYGKfGt-sBc8BxyxBYR-ZOkV|F>aRNrvz*Pw7cRh-xWq^ZW8h4|F zdi%hC3!$AK!ZpSSRnUe~hqhLa3*O*_3o4sOo$iN0?d0XXB~6n$@@7AfxoiH{e;hne z7=ms1)cvhdDB_&_9T9w9-BZeb~~K)mD>jAguD+v8jwF+6*pRt89|7IM7;g?Y|tQPlGJdG6@>-K zlZ@i;9~CpHnaH1k_CRDJViUzY7X`(F7(R$)qQsi5R=>(_6q36r69G? zVM%_@#=T^S0;c3&$iyOUnUYMVBd5*V+x_q+cN{08)&XAVXT1VFKv!?|ojpj3Dh!j% zfBd1B^&SorBAbx;InupGg~eQ*31L_G(r6W_q3dSRo<~7slT;((^z{i@PP_-_qNAAi zXG}*D=@-;DwxD(~6Nk|hABR`3uut;vR z1y?7x%hB1H7m}YxsUi0baFT&N)CKpi?!>@~kmJbP0*t>2#+t8Rt%w1dD0xsOkTOoI z%!cCdh+8XYwEtk+6WaKri#sF$wqSBm0@fyNzY+2lDws(5#2Oh6BATc1a0g-NLksO8rX&4yzp4T|PqIDjeFhr&BL)(akI$}2`tnCI?ss4}8&%2JwBQDU`4mRqO?aT? zeXwoNY!zV*OlQDk5Q(zF!Ph{&L!La6ZdIK5y9MIQ2&5si)&a400BtaBx&iNR<%dYs z2vFtmV2gm*EzgZL5vtZ{bXUGU&;`0ACd0L7Yp}h#AQ|hmVDb?<0r9Wf@_?AC3U|!W zs8Q{WhMn3Mtlz2$Wh^AN0P=1k(ku=O1}Hrp{n^_0z^J(7a>AB@T2=Hp${`ufqf{L4 zz&n6T8tl)G{04g#q*5;EVbd&p1_70*z;OTJHuvDK{&9{;vOBmC4d6<1Hfoeu>x6Nm zMuPSCbmkzS9K`z$E_A`kKsq%CkMryx9TKNPH0EfewQB~v%0^Y`V3@wQ@B*PqCHM|8 z&VGH5gdwlZhngI{VH(Ub*pIMU{Pr-QM)6szj19nqh5*q8xiS>&XArfBjDwYX8z+G- z0l<44lKR&nh-(g33i43}19_x{7H}$Pxr1#7?Ft69n=hfX)eaL)_%0%2*vM}LRs6tc zQTGQ0-8=!F+0MWX4mCY!!`f%~9+XEY#7vCZ6K;f0;2;sX;I8Tq<`x&GWe9h-y9m!s z8F(&EE_C7VKs*W_uh}JpSwVz89ER?WE@mMiZ9rGm;7P>}`?Ep6rgFICerU9_GXLab zIkm8mODVqbLFL$=(`@yh(`;*N3wjR~7s`+-zbxY(ix?m!fXEFiL;x-6XoR;5J)%eo zIQ9kZ%K$7Y#mp>*9|S=0?X*4-P%lIPdjvGe9Gjd>fgP#=#>iEhoJA2>(1=hrh>Ae* zXcZ#aCZND4I`3q|E(du{se)dwSg6YPbgwp?fX;`vaCdAV1qG7wGHM8@3aKBh|B++I z5JAhyS<6w0`xs!%j!EX*f^rc;z~kbYe_>UutFffJn;`6M=&9Rzk@1@Ns#J%BAhE8$%UE}Nmf zzl#}6(*Us1Gq!jC0wibQd#xuDO2Et@+{i>{RexaeMu~*J`4gjZ{*%NMe-zR{xuw6= z01JkgyyNYwJ;_M>FOpayYGR=^8Zc*w_=7rx4Qh+<6cEPMSYpZ=@=%fJ!f(Pm7Q7Uq z5lumpJP$ZMa3F-eA}_FqG+9W_2G9ZP2MwEF=QJPk-G!FOpC}f@=d|TYh%xA{LDa{(u7RH9KhCgLkiIv` zH7j7%L<>2*1s{9Y-Bt(#6=TRsX7aXxkQ;#HLLjK|5A!C&BhPGi@I2ih{|KNgVNR2S z-(^nQkGvn~%8#kI4P!YZz5I^R1SfAG!Dr*$T2M+vt}DJZ2hz54N+2MC5RqW(4qAw5 zv%$lGo@NzTe2UXzcBPlbpOgRJ^n2tF>z zi$V1YX(J;--yD>KU$IOg>gfo|3zEya-NPs(FN5HSb<>`#5b0h82Di3|0tYYwk#q}u zm5Gj$P%msrPNrl-V7?+m=XA+tN1!Z2mYNCTfJ*qW1$f#iU_9A`gue7=%LD_vqXVwtDGj}Q5;QI$r_O+6St;Gl7qXFA z2%izAY%n2u*$(ch7s8r*Gmbk@;#~neS(oUFfyP!u3KwF4!27~#833bM0%8Y^_Sb_Q z^Z)eoB7atM(hzqLOH26`2TI}x08W>NI3nNc?@csN9gtRVrC6cF(h(1NutMPz zM_Gl&qKm>>usHZ!=k?Kxb=2V$?Yv1~t`L0;Bp9902gBM9FI8)9xBSuoo#2SVEhw8^ zf_RK&5(4EOz#WPl#-$Jil(!TuNQfX{{UrgmGEx$2wUtH?8@K?niJ&6-Pe0)k#EuAT z0puE_Ik#zaR6@tHH(gB{Q2%y>@CE%W2vcRud9U>Iz(DAYB*3Wy_b+ilBRO)43$7e- zp<|C;Ldg~=O$f0e{{0$YxZM!Ta0qXtB2NVbcfmWvOz6vO0^6|&8pE)Q^>pEL4!{j4 zg7Rb&r6=h00sXkxQUfAKWiUw2_7S_t=|R5^uaGa;rV$3y5P;;(@VG8+F54rGh`_`^ zSVrNDUW+WS^%1^y`WJ}b@K7Mkf1ebZ1}g=LgAinbqpcLEy~xHv@=d54>4XbHi4j3f zz}K9E>Q~c}xGYow`CxWzMym{+p{QbNtuIC#rquq3<0 zZZzK7@0MIA0&l{my?|dGX=_u4+_D@YL_w#lUbtZKe>8JNPi*z+|3NYN(Hja$6ZFjp z((8f!2UpOt*I=)kE~2>vu?Zp!32h5Vcc>M_Nl0Xii-l=J6b=E7O@b|gM7&fk+dlA| z_&_8E5i&rEyHMk_xFinXB%<#KGx8Q3@b2^?%r4D*BQX2yn`?>467>dw9pum&BDfNy zo%@G)3FLW*b}=G6-H83*j&wc4gTO(U)eAi<2#44e(V#YT8#{oIJs>>a_F<$vc8qVE zNCE#Z+Ch|v5bFfu8?p#*1MUGScp&*4LKA?z=rpK6K?9=vtk?@^B!x0ZE{Nz`!6QQk zg0%D?uLakJP}LAIec*SS;G~Io+V6k|Ag%L=TLR)pT6??7?OV54giC>#1?h7dEFMHN z91+UxTCrCbH9#Ov2(9S+9bmg5qM7aJhgbyh0z3pcW9!+$_bAYK15ia(M&{G0L;BR9 zR|y$qFRY5yelrXXq3(DbgFTPnJ&>D{%w*DG%;{p3s)t|=qIWi@5snw4HvqX&t33s{ zdeFwZ1@e-b(Pd`I;hsR0yUj$uWDgDYB#8as$%R1nk_7Ax?al=>;IEpn-ViEd+Pr5f z1ege~0qGbAv4m@JqVApZ+6ZxzFra~GJOfi_>;o55Hsn(t0J0$bFi8gpG9IW%vjxG1 zLv-CCoPc=O0122hv$PYT&qFVG3u&(c!wwRjT7C?z6Vuaa;61g2kqaQJhEBj5u`~$X z2JAvhfiATES+*xULQr#<=q8|^6b$WZ0ucZni8AtN04ap?WX?f>vn99<1uZjR$p@a` z9blz05cOjlK}QN!k{aOAMXn2moFHll5NZ!_cSE6_Z(HU{E5_i?VZ^}J;!Vw)^*jb3uKXOGR zZG@-~=AR!qJ`Geh!3#k5By#2k(rbjs=|Ch8VTcAKm0{36WwNjZ+YLEo0g;A*P#Fgn z0olArHyR?K@;Ou%Or{1zFgR$Ce>Sf}y5xu{&&1wf|IKx`YbxfmI_K;sNBTSSx#fu#`U1lXycu{j+T zw14%0nJb3Jg0#-ToOK|d1{`$c4uL%mHU=<41U3cB3B3TwMnMj6q4Y*1!=ZL7yoK8% z^B3TR(EEHD_!tOd6F3lXsuwxJK80v;X?a-<3aK|_V*bP#AGTH5Lm zg6z_l2gf2w6<}=UU`CNR3mFws@r6Z5SrxLhva;QOr6}$OIUNSP=p@Fo}`*ggpj(8?s{LoCE;W5PTkr#j`aKQMl6&m;oaJAsQn~ zaG*#QDmlnXhoNwSU>_pT2k0X^Fs6|c5CDF@gm5GYjak60`Y}!r*nP4-2Da}<<0h=< z8q(&DYvwuznz*Av0TD!wc>#-}22lf|;+E#^t!(pWByVQ)=OJQ0wzHe1j@T{I0#bs8 zKwb3gEbiO4b6`dxL{f%0Asn!m#+Mc_9vS>O+N5yk&`_DVuy8zD6K&&m@GUofzTwN$Re25 z6OljX?kn!UpIWQT08NYX%ZSzn)Jx!uyWivc@%5|4)c?zr6@!_%5 znRfsFt?X-9K|c?K{P!X&a)jo8oY(m49Y(=0o3Koi!GK@;buQfRi|y$={YTpH``;hR zBE=414b1*u!*cxZ!@7v5DkEM0g@eB@^7HH8iueA@THdqg>~WBN1kJ@6Ac2Bd?VoQ- z*7=Vn^WVQ#-3MZG2oQ}hKEYK@{yLKI_wV<}bp7}7`HaJ6f%hV1IK)i@XF(l8eUOjU zhZDnEZ7KhC1>~EDX^#V1kOTjY{PEBK^@sOuJ^wE!m;Ukc52JsgTz~xXUqAfsh~Ixr z#sB9q(qDJ;T}K@r46kB$N4xNAhD8;#&?Qj7=}MPZQ-AQuBbA$%EJ9y5`L4t~4P{!t zSSBlT^Xtu{V^bIR{7Y$|INhj8TkG6e{14@Gx2){ zet-Snn)v@a6AoOiNQnFH7-mK+Mf#?3DuHQsBw0(KD6*t_{rfVle44_bMB%tdv0LSG zMt@0*HuE@Y1}VYVoG+c@gi6(*6PURU%@t&Hv5Iub@{~Cp)zMF zD-0f#u>?tHWCUAUCYOthkM8~Oc^{<=rI^Wx_X4)l3RU5>`g`1YQm2m|AYcM#arrcH z@v`PDZ!|??w0I)82X;$cTxRE6J!Cs&M6+Eb9EO}mISnJk*Zm2pgi)r$xLgX#$db_> z!?e+n;?^RPu^(~udwZzGR$cMWF&Q~GO*tODaNa-pq`z6n=c(n-)JqQr;vQU!UAiWg zv3}r;(E%=P^lL$E~;hfjRguDoSTvU&MfGlH$fp~?ax1Sn;F;@iZZ9}o& z*}yrjlA__;X|k*vDL2=syD-)x>6w|9;ger}%>lat$3?NNtd$|h&aVmTmqJV9M!Abi z-ya^lmp;k$>#(}WVwH1m zyTYq1!j@MuM<<>>Ev?jIp_ewy98}9z zTAQM)tkSYWTYM1TL-o#hFXS!{@83eAcJ2J5!NHrA@He%9hCKD;fS|F0s`KDnd0EMg z1@i^f0Kb}h`s~pyvxCNxsf4|0YPoi{sE_`6y<(^0yUX5hQtH}fW%nKQvH#ZsV^>K0 zTWsh3O17OWZe^}Cr~$g)UN!`In~1K5m&$PaxZ`XN9R%)r%iy(ZDZ>?Js8bKiuDP|p z7igIqvC)tDHPxC$<`>g;=NJTSD9h|XtAYap4C=udnW0@!e=@#GPijA?awpS9~NJZ3ERP8lFv(rPjBqo*C|OERXXZ+ zo_Y3z>NO`Bk2Rg;wPy!!cK^B|jlZKCam!}S(_c#544%F-t`j;XJ3H=oXyyCYP;dOK zQt_5plR;-ew5-{x*`c;mzYcDHc;-S_%S_{=S`mK!qQAE;ze{lJN@ABOk2vn!+$BFg zeOi`PVX(B<_2gS|%C7Xtct(-bQ;BrI`ycig?PZAf$Ms=^lWMdr%j9)5N9SVHIbu-rJU?IZu)z1^ z3H0PYA!Jku6h`-9^AqdS)A&!4bGov@E3A$p8`em1BcjIeWm@0bezo6M){G@>y*%{$ zr#*Rc3MqxeIJDWBmWl9+(EH1CUlba}z0(ZnGAal32^AA}2rgVVe1E?FgYwIUwAEEL zWwu}?4)`4Q>jIdreP-(x7tf&PRpB+g$4ud<_lIX{EDF;Hr+0Cu4zRsxv*3tREm65t zhUUTa2}&JW9dV|k%9r~+f;^+62XKRDsL6?v)FIco-s`l2yi=I7LU7xe2xXwYNL??$S#(N{lbns45`k#@fKu9`$eqO3%9 zx&}ps&5Mj*hTA_Nd7Ri9H67Y|ziF1q-o0C6ILpH>%uA_sPN3psf9SXVP$}cPFaJpm zZOI)zK=nxO&O&x@t!ec{nSywxxk#clGwQ*X6IW*1!5~~bc^{|d3%G~(hks1;?RTza ztAc}T`iou{F@06I*E{ZcTI0CTN%1vBzr%M>w|0vh7am09%~iP*vYQP4`hLpJZlQB< zup+$T*w2|TnwRs%cw#%bcMFQpo;b9kqAWL-g+YWRPU!CbqJ1Cg47YU0f6v2p8gX>f z)Y~>zM|Uw1`d%~TW0!;TDQ*&{Hf2e3&rEyX_w_2HlcHjms7mp%fEpQ?KJ>^O&0ZKS zwAz~)69;ZYsTylFT)@Ee^TPu!qdwus91n#yr^>HoyKFf-Hn?QxTD)m76)y?9JmJhG z`c`s$nva%@+o)K(v4lUXtK?ukZA2%QZY^VAy!zXwzIg6P)EzN?dxeg`)yE0bsnUEW zFn5xwJ@P;Q!$bVQzYTEvL%D(yQ%(4)(Qv;MA6_nH^Zn53Xb_{yz`Zlo9ZqsxQGu5F zU^tFzGl&lr>M+f-i8pRNK+(^3MMx1jx4cXnoGlmKsNeGaS~{6M`NqA3^&P`T8xYhjrUSrLJ9Yu(Y~_a5wUow;io$uo=##4qotif{VP+|RyXuBSa<2QlFmjk zc}x|3(^(!ZGa)-b04u1$;4H5_wj@8%JbFqtV_9gDj*`G%{huqdWxQMecc7Jcde6#dE;EA3gWqm z>NlS>K20s8xv=b>vg}?qC*V$?=qVereBjwVwaETeo6aYTLGEHVCh&4}6L=~1u(l@S z(#+2|)y#%WXG2Uztd-fDhbdK5l2JtmLyo*m4UwC8AUEY0%#u5t_^>uzs&p~R+QIWa z>ce;rOS+d*L(++tW~xWcc$NkT8IEPeoGK{?OU-V)dUN@-(>bb+PeIkx1cCIL$qrK= z6@uaDWBOmd{mi|O9?d8yq}f!HlXudMTq(7Q82p~s-N)L}EE5$t@j$l1MV6Cua^d{R zH^i+Jb9Mi^WGCIF&19o?H@n{KkhwMAOkvtEDe06&2ev4_R}pRUwxte31G?!|hL+Z$ z^eH}kMr>ih_4R?{b>sE<1v=yCCAZ{g+Vd*a>5Ur;#io{Wv}dohdWzb5J^O3#c*hV& zaj86+aFv*tO>97;?D$Z!YW*^GE7~B=VV;hbYK^xvIJpa&t5q+V==U)OsT*+Gd@<6e zBV)!0zPc)IOJ8d%nBzNdJ6`$g{tDGEo#u;gHW!WRFkw%!7s+(WNl+K(j%Wdw%3R%* zLxs@;gS&KXT6nGEpQFByJ(AeS>(%^|3;j(3g=>3k(=3N5Y_gWp`%8a%rYp;ilPnS>G<5`%4Gh8C=b+(PFJsN>VnIWTuxzc+P-bPD)qT z3?5sxV8*@)P_cKpYNH7YY%5*Cy2$}p*R)_7TNR%tYI+ZEJDv7?ubK;D*^SO-WLyux+k|$Ml7f z`r>Je{{-H{C99SN2KN`SG)~%oT3E{HJ*AQ@D3sQAFdQ>F+3}K~xB6-GNjT}?xl;p! zr)fnDHA?NK0$Ns0Fmp({b1UcifrMsY;jGW$L%wE>C~+f za8MAgh)qXwrhf!XBYKtmA2+zHmnd9nHl^QtT^Hd=JW_omaZDvhJ8ah z*^m0kza}pUV^2r*=9}MQy?0{u!gzG6>F1HEdR*r!zq7}E$$*@rT!f>(3u4=qlb*{_ z41MW!#$u77+^1Lm>3$!7SaBxhQ&Fd{mrjGNi)YaYb$f|Xn9D({q7J6e@wcyoI@DDc zt}Wek=f2gcEt0aezWb>_IloWMGRDr()2g3erhaR2dnMHSig3nwzD#4=V?|ZJhP2_H z;_~zo!-@<`Gv~@Mqx76QGAT7#u`A+PW@(bNvO2lPJRUXJYL)ioDn3d;KJ2`9_COPr z(8jF=HnEemmn&W=$4g|xe>MZ&X6QokAxGRpBDEiJCFPBGmyLIca~#x=?+@&}3E{@s zJJlDFlm!;Ldo=iildEp&qv#mg3&&oc-266L7(=>&EyU`1W>_h%TU>y^vZA4X*)4P=` zWg)C7%HEw`F^VzCsI6km&bnip?C4hf2^n-*Mn)&|?5R_o#zC?Tug;7%)Im(M!R=F4 zgsyN#@xFP!vN7J7DB0Pr;yx%8r)rhi-E*Jbr|aT4_VR_0tF8Szq|PW_DI+H~ftV8m zBmHbPLzkeR*!E!?X8b)Vq0rbJ{T7#8Q0+pgyF5OY4MF4c#`)@ zlGJyxpDwvKP=!5-nl(w}H1K`+Lts5E_+b$PvOvvOqfi-T`h|J%VbYXS7wD?&!x?e=9) zAGe#+NmGsuySNh0SqjZ5udjo;mU7SD1+jMB#_aA~m+^6S``}bjF@Rqgtmp-OH{Zuq z&fcmh97Er&H*J04mZ-+#tHxuIRXu{~PeM{8_Ayo{4qB4Rld?5)XWxZX+)5zcG#sp$ z{fn4MCm4=hpvslUc-GN33^aYumn3<(nSU-K{XhhimOTx_*5a;2HP4`wgJv4vSkx8N zRpVW|iL0+(at;idY%3+jE^m=MU0Tu^e>{SRlTP;@TyNBWkz|6bN7!InKpBq6pI$|W2o}a_r>M!;gn@I{MxH=I!;d;cNab-)MyYYfc;n}5` z9VqXdjwfboxa~mq`p;GZ`nmJ8%T`9hNYttr+%u5Sp%eS0Uz@GZ`&>+KO-G@KFu%Dj zs(rGjI6I~Yd)J`<#@A8N)uLx_nY-5d?8;>sH0mqY036ttWaeb+p&sS6=&9baqdVMQ zavPr!<6lC(yncu=&wBR*|5P>i*+7L^y)?{K8#~Hg{7#FYJWuOt{tI!2n7O{Ejm6Ui z*PJzsBXX?zS%MsG0~prZ0`b0Yx@cec`g^Cwx%dh66T+7U%5MyG^L}U&{?@YC8gwNb$Z^UyvcN>G2&x5snPmFP)^_^^D*KCe? zVbdJ3;4iA$FL0%^-W(@>v`6*V3P`#fS!+>F9I8m45^ilXVrb=Ao zopp`nr7gL1FGJSxJ8^;1t6w*s|3%@qgI-rG9B7bx10G~@%$w98Q!Ph2J*6(1At8>w zCDGNH=<2%CJS%j_G{--%dMj=8$W*n#nU=gbvH2`z7Jm&Cu()7LsFz5i};xijvhJ4@ha zR2ID(yBvhJcG=QqTXH}DW!rP~GFmZSf*VcjlKQksT5p>dYIsFcv&=gsVffnQ>D+Ep z_1PPEiQVO3A8%&)pEgoEuFAt!A4TAF?D(hYGz)os7x7#FRWxocavpEl?padw8IFpq z+^JjHcDuFfcJC8U^45de7e!Ux-Zw|*W4guSoIOTMT-+HPDL<9zi$wHE&@*+q5?h`y zv#@dFKA5O8c@T@dS2J9&RhbUG4FeJLGFR2Wu6JfN>70OtM1U%&-Og2b{KD$*1@y~ zG;v2L@d|F*)?zD-`}=Fn zr>Be}TidZ#@+Ms(jI`~G?GFc@ke4S}MP!H-Vt8X3@2}2%KLT4-ko*PrMZHf}muNdx z_>xt4%Flg#=E}dSeXyJm2NFXdgIWw_OR(%rV2h~%H{opb2;aeEwfXz z5_Ffx1|;YhJB-nOq3rozZ}Ek6pCb7=3y8k;>_xFg7~fqofOFOax>ml!0=@AiK;%>q*Ji$1CijWHg?LfEv^}f$OsU-s zbiS2kiH~@y`V4zM(A+zl+5bR405g9rDL3#~v&eNcXXBZp#3o19l@y-8y1CbiJ8qzz zO%mpI+McnlvYNdLJEl_og^X8?_(aX`Ok;R8`OnqPuJX6hu=n?bm2_pAyET&spXRcZ z#jXVH3a#&1h4@Kg*YW&u&zuEipX!x9A5JlnlRK$;%KL@`9SPLp|c;1p}%vc%w? z)iQdkl>TZIF`zoq;308RDfY^m!-hS)dgJ!_vtA{8XRTzpSV_wgbj-7bWfz&ewTV0H zO$$CVuNQ~!uwN(LlYS+G*?4+7Anf1M-8N8iUgqm3uf6TXgx zM^&n|OH|rjiIT+jt~CDRF@xe<)NPY16>S(!nsBfD%4B5*hR&qgv4YaZDKcX7A~R`Z zdD@gA!SFM_oOYr>d;faPqt{38ziN`u{Kk(hFSHd%;Vi4G+B)7gwQ{X3F3#~G@d+se zxwzSx1n)8x_Dp3)hVESvd&B&>CDdSMiz=GctHMr|=~_362ZldN7u(51SkN%#AIbcQ zWm_4Z3+a?;_J}X2HSGiUD6ih}F;by<#){P^dAX~O`eSTB?fCSn{lcT*y|dK#IL01V zC5ber7hflR1y09x`_TKSjxXYLYy_z&@t0Y>-$m4W!ceMIZE~z4+AOMMX42bV^OP;M zu19^(TcTRuU()UR$<{$R@nqIB6fH-6@$R=79rVHw8Jw)dEpR-l9rhvW_4qWIJG$Dn z8dkjqGVDWAZRWe5a0k-wRvkPaLL@e($sK%$;)s@=(MyZJQ_%cED4huql}N5QJK|!^j&IPPCI>GT@vSIxQk!8Dn`YFy5X3u^U7**6Ze`io<44U zQM1BlRw6U4HpFpYEua=n%VFH{)NwI#XKCn#ux~#`{-2d;v!J#V(VweW-G6GT6Ri~c zTAXBobM@&vOujR+9{6>p=lK5ZY&)74}~#AY)&ABrIQH}M{pI}Lh-DxwCij(WO;@Ptw> z>Hbi~SwJIp^>bd<+`tuL;z%w(2l9aC5kqRNk zO}AR|2=C@aNTBoWS|9adL`2?2IpG#-ZYCy3NOY&=t|@WIeV2}FNQo%zi7uh6>l%=- zAZ-PaF)|&ADRHM4z<&HD6lNr3NGzobMJkjR72E@oUe$jj zPNkdi#pxcXzH=$0cD`NX)Km%Qo;?9S7a{wx3#U9vZbdFItSa6o;ANb2F|&GK zdcHkipW$w0=(DFutRdqJmBnYC-zV=q^dQdvw%oz5xd(i2-tg)2RFyU+J8oVm$I(~3 z(Y`TsP^K^XQTI@FTY>Lm#tBWHcj$m`X+K1wyRsesDZVy(LZC5S62i3gJ z(rTNhe!D@t%;=S*&g$KYmF;DzEjb<$cCn>TL#3oSvlL&$$`yH3aBDoj-EK!va>~&< zp$E5)`i4(wUv9*8)th-UPcGL_TMXUX`-L@L`-^do=H%0am`2l-1}5&|>CqhHw_etGt<*k@ z6mmjLr+z2E-PTKL*X?cP!{(KCIcj2BfrnK{>7KPrf1Rtt+(kJ2+7%Z~eg>BpSp zxhPze3%Xmn-;%oe_3hwBGjQZD+!xF^7~}iy#ett@h?|LVWheyfsK}?tDZbH(J@ruA zyqZAu2ghZthjA|r3I>~v*_(vz78T?q?<{ZawW;yw@!02>VV53D78Jf6cxbrlLi&AX zj_zWn`02NQ9cpOhv?AZUX~_L@y|v=m=JWf(pPR=kCe_VV76VsrR;1m0B66yvbv<8I z527YUKKn?rFQv^XNVs*kPBt_#?5>ogp|SBht^9{{j6y@L?mH~$*2 zk`fgaj))$6_Vn5}7I7RWZth@T+`(qs#AU?h6a-<_DWtMk2?FIq<=HuR48*JdqDX@T z^<-IWYJmkg%PA4ebG+BNi<#+DTond%Bnbw-mMJn33?F5)!+`bd+9oIdK|i&N*jpxF z)>Wq$zI#|2FIt%8NdbP-J-7C0-2PH@i{+)T_KulArGL&Xm)EGUg)i%hnk+^Y$qwkL zrDZR3Wieq33tM}wsY+` z8G|MlsRq}shuNlEsV6mML(H|yFPy>afUlWqkR120n}#E}O5PGABI%1OmAJpmP|mEC zUo3tvCO5S#LL1jHp6HN5CV#s3r^$I(S7;H^J;d?yWsA)gMEi@~YQbG>2I%6Ir3kvM z;**DE@V%s!>Tw5)&U@Pe$6wFlTL!HxO|?o{Zl5b0oqEjeqazopy+6NLW1z6%-L>Oa zUb)VO!f8@dWBC58m9Siy>g8*y`3~4q7cO?%_Cp(lDSa0qywf77wNci>S|w4K)0eM4DoY&e z#Pggk7)edMv7|SIx5%Sd&~c&@km=Ju-_t9{%L~N&vwaW4rZEvCOX1)8Ff3xEqQ!o$ z`@0WCjb{on7rM*n%_)l#zm}t?$HgG&+W(S-dfh_p#AUfN?i10H8qc0P>7c(R-_ZT+ z8ygAFhQCco5WuCTS{i8JwLtNVDNZs*AALd+;kOA(51^ndJLBG7Bs3QiD(QxMy;Bgyq)gr6@WRG8XrTa!&9uVtO0f6I6xq z!WnJH82YA&6lHUhoY6Yd z%2$|P2|%;$-W30+ke)kB`nLXYVaZa>CPqo=3`)pR^lEm#n>N5FU8Fd2%1on0wjs^x z)}ri_|L~?4{MB$RlWM9;g)Pj3pjAN{P#sMau~X^yGQ@VB7ZJ2Ub903Tnx7NTmfEHF zwo|)hm7qjtIe6O1Gwf5clh=dm<45qLX|d6?oi=gT6ddisVZ7bIW#Qknn#k+tKN-{6 z@j`r(BWJquDAek+hFkCfrPNCv9`F9Kvttn^P}rp7Rp63J6_UKT_e#p-7Ho<9NqBYx!5Wa#9*{rD5! zz?RffA||o9@DqZ6jN9Q_sQ!ZBj#r$jOsW|%sOnzqZ9Hiz)BII(FtPQ^YsP5p{xR+* zXU1xp_$zQtI(Km-FOJp7-haZlk5oF*Fi`y(t(QJh9Qo=k0nZulNBiob zfeS~l6Z{wuP)4*Icc}RRwO4Ldxz2}A7&kP407K(kd!19xdk63 z(W32}Z8yqA`kZ@qAta2fX7LcEoF8RGPIf17l2#w@>*?v_-hO<-)Tb<>T3-aMOKkAr zqx>r2rXu1qa`+{2Q=&$=NaVguUz+$K?$3Qqc#+WU#2q8mOX}m@wKlMz)i3tiyH5AC7qd2poEo(wjqJKUJ`L@DPd_kxT)nXGa4M6s zX*ZqXtk+7Xa1Z60A76M81?PYoQ#aEbu6%+Rnk<-W*V*UEy0YGoO})C^rxIP6p` zA^-5Gp*bK-Hg+@W03>A$)$!7D2}V7iNi5vr*jKq_T2IX)vpEp{vOLiSf1#gJ4}ZWCDuxWwaF0n4rT!u1kIL#?%Y+UQLsar~~3!eOF$nB-mh;|8>~wy`MNGO(e%Ngf{G zBk$2H&&7~=WKeTcrgr9%vRC6gHp}B4=!fEqvlG>Tz<__@jUTlLtgj_I`PSQWnu1_G zTQ*UaNm~I0PijtyC*Uc@)D@gZ$~avrRA|vX9xe6wGxj7WhkkH91G=N>cCjA1{gs<% zx3#V1R2Pr07LL6M#*ESiq1J6G7yTD}#aif!>?V>8&k7stknP=LbmRwLOY^C9Vgj$j zvMkzEL{yyA`Yd~PF-%Z+ZErxkVNCMqaGv7eaJ%O8v9`rf?hzH|x21jS z4cx-D-fn7}!h-YKNu|X(T@sXQ(QCpdV{$$nU#+a$zapkl3|Pbkwi&0{X9FI?Dtfsn z+(3f1%YDTUqE~qD)12j!#iRFj5k=)X6^m1H znW0yT(dW9@&ck^-T>At&#gezNe8U+y#|r$*O1v|EwKo08Bu6i+7_$7-?VKOVK-;djf;B9$jDyxw}nt# z&(4cam4FcpggKM(V)!2|}?}YX{*a3ukTkh4c=5=WdyM$gF5x zts&RD_4g||ik#)8g@$`WKCH#)W%PV=wBWlVnzxm>^V-`*vT*LUI7+(Kp($aZ;@Ye! zsZ8H|H~W*jjEN(ytO>^xBKI0G=U`9y^g1spVmNi(+M)|V@QP5I@yF`aBAya`(tJ7} z_`C!V zdswB~NG}^z?YJsJ8rw0%|6(5H{iWmEYiL9jjcuRB-oQE!5(-ChYP3@HL~=UUTd_t1 zWzD;>>7=Spm^0+tjm4aqOiXxr(6~aSwKXPpDzsN+qCMYde>*4 zwQ(;nE%lj`e1l6_fIb;87tZH@TUTYVP&0_pU*nk~aliFz73cONbb8YF-&A^|uHS#t z-7%xDujpNeUAzx3QSbZy3K2@9=6jbQtp%KZuCn#0n#)5Ep%6{G zk6t&wYt&dawNH`B`EoEt0=U&;cG51rPgwihYqzp)p%%NNq7u9*GaQogu>uvA`tpuR zcM~hG7>mzbrb!&D?kG&jy1q%~e?Tqwbdx)$rm#V{kF1AR#tM6Wgq3|nsXsw(y!6^i zN$)r2sbF=2Q~I#yg|xX?<3ug0a?3?57DhT7h6-*k=>=Ye zRFEG3K%(F!!_NJ^7&W*=e--;QerKZdqqb^VzWhQvur|3YF>v^{<=x z=Oq6AzLlNZQWm>4Fwk#Tc+w5qiCvE_J@iB>vMemJkrcvfmQ@M{qOY>PTv^(0_1v#x zzO)OccpK2@uvPue9c|0jt@wa_A$KM@3$jsNI(C55)?{<`I!0kIk)DU;Dn_&WaxOD! zbI2m>CE=`vs9QXB(=+oim(Zw*K3Au3s0wH&_1&i}A>{Qt`9y4bjdYkuwZ(L=MBe^_ z9lkg`e0PVDGfa*5#bw5Qi%$J}XJMm_lv%9RNKhBb6K9IWJdBl%JK$sk%XLrn-El_| zD201<6U#lfo|$$zdemGKEcvAKtzzuM#OI^?GSmY$K)%S&6PkQ z4*71GO+d9~Ux{_JhFEda*>&y;GBH9(bukk4qbhat*68>k79)Oaqv;D=a9TR=Y z^yE|?iPuvs~TAWp;M8ifs)8$=Hh z*;^8$ESlD=GV$G$de=haB9?WXzE}^?zw3OVz1XD4s9X9fUj8b$?M@4LQCVa&ckz0G-#~Vz(?G_4Bf&-2$^O*zk0}2+G>b z^X2jbWopEF!Q>?A6rERZsLxQyQ;J{05^R=7`|f=yQ!AegU)ow?b(gJVn%3NXHpi*40IPuOn z2|sh?=2T3~%j28A$G+vVfL=A~RGKn(a~TuO)z;lQGIJ4nXiIRK*I%@$TDfpu2EA;W zo-&y>?R~+ox2>l~D~9{I{SAY-Me!Kf$iTAoHQj~M!sRqFF72`@T5z9o>p!`FAELur zCs*%X%80mmb+tj2HZbjU;{-&(Va=1A`#uM}F!ew5mLr^6viH$(j0A6L8-^xk-9bsA zVY-sgq-Vvsj}_X!X)C$NwfDEjU~^M$W>-96G#KS;$oD)2>{YyDnsnHwGnDl12@=V4 z(Ol&Djnqdy9&~%XA~}<5SgwRKtr)%l@*rt*Wc6ry*0gcNQhClnf(^%RECYuFxlAV^ zoBzU=dPUrT0aH5}6(;>j;=7T`s`R-5m7#mytn`$6K3W8uW1puv&_0Bca9Re=eC{6x z?O|BG1-F<>1kaIN1M83%G^;IvR_rtyLT5BM;?yr<)C7<|g74F0su!auCo@*}tE0|k z=>+%o70rH_q3K%lV$NOMC>nU&`=|zT1mnd#;MhrcldjhBvccxMza%_qMj{4aY8TzK zaVHmAa;8?7kCqv-!TU2l6VzA?ycm}u3P!@#Z<}3X%w6bS_a_(jm0XVNns0~bIhQub zS!VraMzWQ}*QE^XKdZDwhYtx_Mvl6gyDAn!y{O_b7v{dvw%4fN=74^mZ9k*EUbUY^ z$oiF7|1d8f!R``WYSwYx>~&4HP!6mLFVv&%5-yPT0(qI68z(&`E5;Dt-R8P7L(a)? z-$pQA0CVMiyJ^7`^DK~HDPIN6#3Vh5j-K{+rw-s+PIws=S3?VkB`v@Eu33Q&(Av7r zQ%~jq+E(mQsXM)D<(VL-bs)32kx4)QNt5vfanFiOIpv#%is4(jUK)~x7SGFB7+$u_ zm|IT;Pg}-vIlG|;HpUOoyJ+^j8-QIr6;U9p;^C`@dO`O@Cj|BRDNpZZ=bZHj*Hei@ z-ap-sHuJzhGNsq`=*ZXc?ouKTZqc5NL?sX*-OlyFPAqfk$B|8*$&EDdDcrMMcbtvq zna6TV?^x5mef8t_ysJ;V8}49+^Wyj0^!0{|@5GBqB~SE35UrRg%8D*SY9C4TLJfAE*$sNa0n4JzYGh9wP69|R`hq^OmT zoaPaqTNe2Fh)qqB7T+}*FE-9nNb(u;2OLT0}kb(YMnw!fQk8JbJXDxr#a`)=+ zS1k62eYP#MFf$H<4ie2GG!IROjP-1=%1V^F{PX3OBL)>`Z&z-7RR(v(kd=?KzD z@AGz~>8q-^^i=J_UX#X0!JS6Qf_I$>Hij!&xzooow3D4RLHjhn7J> zo3A$lcd;M%K|$~NEKl@#f_>0G=V9Zh*SNz=c2?W)vy(uQW?LBZ;q&Rq;vU53&s0+m zoin|B4Rbu|_C&$p>SesZta2;8TI+-NTWL_rIv9KeiVGeh)(WY>9bNhS%yf8kh#B=_ zZ5wN0xhssF#5)W)mlgw{hQLjv$_(YQIf+(AqV{)?7d+4IC z6*-1I`zlx=uRP$|pT%k%^CH^!!?t#ImqD+$rI!2apLD|y(LG--4ZC6%pG*1BwtKB* zo#-^U=Y=~N6_`(GjSJ9P&=-o6%3HC%)gsAGrJrwXGkDR!2U}!^+T424kTPO8zL>-$ z3k=ItlE0Qrg-etj^)Wd+kC2sxUBjJx9vTb#b~n-8%0+#vOecJHC!v&XVN*_nquV?G zR#H8`TYii!PwQ5?TH#=uFMd70fu+Kre3l&-mdhu;FY^u>jT3hy(bwPWC1_|&68<0J z&N3{jsEhZg7>IzNfRc)I3epk^D%~AJNOulBprWFZf`G)(-8CRFq=0lYG}6pa5<|z` zi0^yjdG7ss&%>i17|uC!&OZCBz1IK#TbciT4Asu7rHQp8nJ?^TByl_6!N@yyL>jxI zF@jxF{oAGxxAFL0NKS#<*|8)t&{NVq6^ICRc2JiO-tB;V_!P1d%hV&vPsKl}72J^= z7uSIg&Ej40H$WETJ}=bK;;Oc@U;6VJmOfv)5hkwMlI;LQ`cjoUKTQZl|d($_5hcI!5>)M)jv^ezu6sv z`mNl5tvSYgjFMYS-T`^!T9-v3g-qGbX!w; zvnF4pm<~DdRK)rCOslx6ZO2L^BnNXReL=<~9ikxWJVGu$o)unr!s?FRkUpY%V$oih z>`D`1^3~>hXbGmGC;v39*?sM5El*t?tf0Bl?*>l2zM};NP(|P9vJXAAS~8GP1&$k_ zM>Z+Gf3ox|>6+SbuE6Q`Zy`H8T!fI(N$Qz)x7XIRgzQow)eCr=`A#(xkrivH>!tq4 z*|AvRlc^TF!<~bH*jSS{yI?O|gnKJTcTx}igU1mKwEDkU$)(t}yk7s2z4^iLMhA>T z_&h~6=XL^{zAwtn$DMhysNP$PW?73%=tlk6@{MMPt zvwBhR-bK#o4JrM-Z*Cj>G~wSADU&<7YAxRjXXZ0;$%P*tv2ndX7OOaSIe)Exk39*$ zYUai@2jBR1`4W1wHN8?khvRNq1XFP9`eZ-XX^Lmo4oAg3|1iU&*l+Q3vt^GSwd{%? zC+|$3>UlmyMOqdV`f+}NsJT_|obJgP9;fUy;=7DHh;dPi*^1cLLA+Ma2tQ0^eY7IQ z9sD9E2QJPlZ16W5^81`6d(AR*e#;ZKvQv7KAuzO%Q(%exT^bNqg49Own{^;MEll*0 z-L$n<#CAn$z{BU=^74f$XaeNUvv(e|K7G4S)Fi7AbF7~6Ee;P9_1qj)rx4j2xgQ%SMMeAf8Gxg5%MrlZ_osaxIN9$z^P@Jyz^`IXSCmI61JoH@`H2VgGeVTi`w!0)X@@ckt6phz>s;$( z_kVW(it{GVyJ4+}g<3L(whc$jc_{+cwRX6^|0w~U5byHiNb=4j;J?9^mqYRO7q(a} zl7_@u8%9)7FH<^QiiZpSnVF=WtJBxUgIi&HGG`lS_uo!vzAjGBw&Oo`xzYFMfUGuT z)LAf1piZX>tBG!7O22YF?H3i(8jT1u?4YCn>6G!*(qddN&T(c!c!IYg1vPxU7&vg0 z{B#dbrUeX|War>8>GqVtKY?}3lTHCn>qOMC-bH5-O0po+j@;w44erQ~muYc5C_1d{YBD71|8lV|yE;h`iRsjwK;wy0E7)Y-dlx*f%}#ez2Z- z>EV%WKYjSS#75Z87s=3mtoVtm?=tB{oFtdsY^Yb#V`M7PW~*zE>D`ABeC(C+TJjqM ziyr;HPXP?_^73dRjY#=C$OsZs*fSunm9iJu>pD=RByh|_>3EYMT`PQ%LE66N6@2Q7 zyX{X4sw9UJK0t7w$?As)!|oC6>(=OKjV6*0Q$5YEqcky3<#}o&I`C3@=2267)Hq4b z#OKadP2T9qf@#cX4iQKrK4RIX#U?8J+_AKjFSNDfUQ>h6IUYq4bR8=1EefHm84@}$ zZN2Z!5-Bp}_5>j(*=*2V}yu@xnFCSPs& zUB+8e3Gj+%>}OFhle^xfWbx$ia5-sevafXiWlscZCT!eXtI9~Mau8FBq~KUWcvR-U zWwj$vXE7W}d$O6O*qlls&B*&*-ckhK?;&jFxz4%Y({~A(kqv!`d!+S_H4|&7AvLft zIZvT$g1T6R<8Z2JFr4&r$@uLDgb&eWME-UjLin3~IsyRVUwH66q4Gv_p5koaf8j(8~Ey_?wZDH7$;P9_cyxhhj zGaI|Kc86_H1*Y*HGL&?5x_nGK(%{=xnM6Ok=XU@Qei9YJyvlNL{+5AD^9 zG``wiqaA6_jEdO|sAZ8#5fKY(7Phvhh>KxiG~~g3cgRS@Zqx$Sn=63jwYGpf1Gx&3 z5V_Whv$erbVtyIUEpY3JVHb73BMh`EH`Q69id;8%-0FUcSiE-4gIF8N+`&6LIRd&q zz8mg`5>=IzuR;j|6!wbK$1<@Rq(xb_9~`?MLQ<=K^4*q=q&sFlP?J!jmra-d5Q`II`(tW zDnzEI|F8xhFn30Gtk=-8VBBQjvh)!wWj z446VyK0n*;RO#?Fq*zC>Vd54Z$1so0?`qrov1-1mZ=s?N>D0c)eY8plvt0qNshSKJ zo!R@qLi!=aX!`=Y-Yx_|v`G5FnnF)E5?as5@L0tUxzrdl@)OPt9xAfJ_^PkmXU zBd9{svzOkL^Upa|X0xCNSa^|h0UtB6r}cG<#qZBdz`fpC zA?|8v_Ax73cyFnPOM#labUYn62%U;08g5jG`|2M8$zj;hYali0fucCPvU4RsU^+N! zVC7J_ITd|#PzdJryyL>^T;INX{NiT`%mtxDTW6IzL5A=Div4nRv}7qt`0?+bu09$8 zSpr8Bn829pW2=eU6P{xZtM@cW_DKVr{eEI0= zWj*40#DS#-LazQJ`(*wLdv~e$KS;3kRB^+(sxL zbd0#{RPF$Te^Qdo*JtU`m>i0ZwWDw+c#w_~E*C?*3|nNKGu{4+7B90d(IG?>nEPD? zt;KXa_%@%$j{%sw;l^kRa%l0+;B{3zv#sXqO=}F7>6|~PN^aoWXd$o`8};(qu+LkJ zi3Qid0PK$HGFOXh&Ge2N+wrC>*V$b|2nOtu#4sGBpGb37uQAm20KYAv_WL}!mYgmk zr4UXdjf)Yd?R>N=6DrDRwkYH|V4Anj^zQkJTi~DQlHEG>s%nP9X5{-962cyizOB;q zY0H};8;6E`$+1F`;8TSQu_zrB((y1gZMcx_++fFg%s&O&H(CV2tl#VO@^%*m{5 z9#jis*g^FO{aeHzD%VvEU$gx^Nie*JOGAst_7y(8KkMjosYR8%jgP^=&yYQRsfB#Q z9S;a>vRgEKicvLbXRs_Kr-;32lL~Zq`Pj=`yZjM|;cV^qG$tw80Gh6}Qeio`^W!f|sa7k+G1m+cW}iyv0rzA{UrVSl`+G)s`nGttLa286xSbo{0aq8r=Zt)E z=hZyxvRlg0X~{yhr;L8V6ND?xj6@-DHRI3QRR8F}3+ZIxHwyL;qotp|OtDHQYG{EQ-(w|Ih2QUobjVQ>7+V4_i1hZfB?EjW%{ zJWSAhw@^dIfPTULnxftzeP^lfSeO$VL&e5jj~vsd&ih*Z6H|QV{yRQJXM3ZYix6wW zj3}QH_{ppzaCvE_U&2kUbxhi(TOzhf>0a zW2ijwIUf+B7;rjY@SKFgESAbnRDaa=39~;BKUY0xs4j#aC^9~gHSLH) ztvQWgW1$2YYnc|E;#?KwgUm)G^&wk)9uN$nZfLO|dR4ov8MoHaTt0?m<{2vL##-wZ z8y2n{w)6Rse@0y)GcZ*R_C0?o3tCKE%$4cRLh)ncy03l9(@uZ=sCBomD{=W%fA5q0 zaz|_8;MdRFT4m|%Jgn_jN~Wq_PPUt^dBCMtVjF6q`qFT^IrfCl*&wR=E9=+hh<2Tx zU_#)(6dQFIZ4H_<*RqIgu5pn8>DArPgoGjArncpH3`=zGi$C6JFCN2$-1zbZ?Bb-q za}ADBWR>1BYE{brW`mF#}nG;8A39eiR)-hk84?0br_PM<*FOnfH0)0WdKBdp|SSYdwJ?bbO8qfkY zV$6$gwhn3_@egZFA*PPUvvG&6k=j|zshn(-wniqj=A?C4F-Hz=zuYh_UC zLm+|c%_akI8VT`$l3+q6zveDDAJ(-K4{!}9N6Lv)qf;EwuuT+oRYsMf9Zlg!j?0NX z46hF-fhQPMC6L=n`*=+tMiJexNepwbRW?K}Mm00L%K$|-vrHnba3N6s+ZQ?vVu!F} zDjFP0pTk^a&ZA(nY7J$t$N}Mz`I$DANljFied0l-p~5@GOasm7EL!>40K}~{lEV{l z6w#I86^v1^Ex*aji8fbRLmD({Pjk5uxVO(Y=2?kP+X8hgYhw(Pj{d~=R^9X*1jIOD zyQSenDW5r`%bZfNzLyzV7P=E638}(%SHJp-C?ag_s};P^4BNZP(ud=9kyBXC?95g# z!_hp~)$e5viN!st4@Z?m9wPs|bM4}5@f<5@7N$CRmzh{j*s<$X=(E16gLX-bD7T9! zKo0L)z_IVr!_$DB;72okw`#*glU6BJe0eYM$;BFv+9B z=wV0WEvI|ykwB)(Z^sIQ$pSZ=(WE_8OcoNYBx>fvKQ=ChDyG;!X*mKNju&BI=@;s! z@719fw9$({tu<4$(<|8@>RU#i{%@Q+{d4;+)|`OV88*VNkd;{bWb!Qw(hcHIQKQxE z4|BDYr4?6w{dIt6AMPe(tJ-aP=m3m3bqN%GT((w`)j^)q0ES@ikl#1=XdnYtJ0PA7 zC>MJUHd;kLy>A08(*B<^n)?a;pRFEwq^18lQep7|92*Vt-ei_-r-RR?um0NOyivky zIRZpaVt_CKNR>`cR==OlL~(3I-l(bjjRnG|_DUx+Ib27>^&vR66hq{)mv7UX85+YL z%Vq@#4pKxr7@}QQa^y9ToJ$6Erc;hq4TP^<9_O4Gag>HuZ>}izIr1LQ8TGK78aJ3v zmUh_2$I8-M&LWnso3Oxk=Zk!Kt!choD3*&TR&6oE+1U?+o13CeyfUS*w5^G~T>#Ww zTRv3cP!Oi&;GrJS(ql!>&mi-%DP5FsWphzlIC(DO%GhwP=gV<<)`E=$@8_Rw+ns6g zXMkO97> z2lff1Yr{|nXzmlo-&wOhAvidsAOAXia%w6OYG*)*chlUE`hM>kG+66ajch+8nB%Zz zIqX&cqVX_b8o>T6`>y5W422jWbVUt3SKXSOgH{ZO!;HQ4<*#aTFABAWRQ zq^+~;b8QB9jUuff_6im}Ufd+Zr86@5o%=g-)PXNu*~z}r^HqvsHV=l-o4KQNU#Hi@ zP&3NO4?dJy!Gm66)jcF+0((PP0enJg^Ye%s=$#=}*Uiw$VFQ$q|e)HOZK?8Gf>rwV7 zgnoIcwF)*oeMv1QNX&KhCEt&U9H%PIHsh^P>casJtsV?EA9qi{eYrRcOGMwzGK!{4!A`=kU)b-dvndXzDFbCidS#(QF zg+>wjO(8mNs8*=5zOPgcJKZ?r3o+DUg3$DD;nyh(mw6?xju^jhOcXMivT#*D%&iOO zw+$<|kBp5CO4>5*zww0VL)!LIu=1*E13FqRGuHDUhU2V1o zjtc2BtYQEAmzHgnF8ztepPi-z?7}H1gNwFbGz#v`&8a#R^75M2IFu5HQa(QfQ>75R zzY5w?X=%lMxd&-5cc9)b*e%C(Karb_fMC%s9rOdfkM22L&73A8C&~jXrT}puWY^XG9RzI~6eyq~gyNsp{Aiu6} z0383h-IU*Y7QRCG8A`00OId{7|a zl1{qP6ZcckFn|yYJr&L90@8Q4BLJlW()Yg4y>Ec@y_mkDDZ2|xttDc(@x*XhEqsf0 z`RzcDl?fzn>>U=!*kw)CC_pr&?=i?D_=AdCg8-9buL}HUW_7^zc%HLMh$Tbpugtqt zX(&H$vwa21*#LOQ2j|(6{_>%jE6vL=5(wZk)_{WZWM$?|717ERJ;@U1E8JRz%rU5N z)z{d%2NdO6fQxmi5;)DS-(u!&sb;4)dV;C~lDt%B4ew*HUd>QbTf|=>qhF&HO$D3* zjfLGdD|hBtMP_(~!Tx2OHwzMffr1+LdD&XgsKVN`gS5`=wjT*Ggy_XFHh*#;Rubz) zsGM!@sB}EQ)TEI=GDhCR<9asKUv7lRfh%@tUS_nb`fp}lQj<}?HK^0zJG3vTE?*N9 zw4CIk5uk~Vv%7uZgT&_dw~E`pPtgq8-*s79tR;$5s2Y2HsgHm%$mYkO1l~;1=8x|^ z0((`gI(l)gJ)-?rh`z1c?7|itJH?*;!=U93cXbW@1tY;yM&oOTZEAm{PBL~BZ8@*z z5NQMDttzWw?|~)e%Ti{)RyWt$Esq_*;W*U)jEVxP=4D3@P2*{{hF#sdslmytGZBYk z=A}^T#E_N!okrYQ!&RU_F7-0pSkH=Bn{DRd<^mFL+OfW5aj#_5mH_jB)ym3vXYSaF ztAZr@4Y~S9WU$7$S-w_4`*80mFyPG4P3LxwUnIzLA&AyOlzolCv>4V_r=g@+UR8?e zXc0ESTA>tNYppxX*`H2PO;-3U8H6-C}{f`#3(nY+o`ZNK3`%O6W2$P6@fyR z-zx0MO;S?>lJbUa*hvL+z1J@RK$_n83P%qCYl-4V+2ci{uZZiEJN+JoW{K}^#wtRD zc4-~GJH)y}T4<1hZkh)?tycwx%i9(nY?bJFo&t7Ljmki|Zb~@^8apenLV|`JX zs{c|^JeElrw5bvi;b)g+d8?``#A%lwFM-UbODpcUIfge##9*k+O zpfnJtC5K~X07Pj#*{hxp0$Ia9C+m6JD=2r}v#2YPOIqa>Hj)|#mt@FYM$!x?YyTuN z{GDar$PTO)4pEj*^w(HQ(BT1m3?zO?&!>_~qyg~~&ZQ9rC^54p*jpppw(6agW zEcC%Ay-h2+*7jb4f|wisF1xJ?4o+ti>wYsk4o-bK2(Z+1TfKS@z>1GN0}yo$helGS zS4Zj2NC;m~p(ZjWU2fs~Zd4JB356!j4E^LhcavmwScqm>!sqU4cl~mMjRtw*mQ2k17jhCAHT+NRknf~f*r3&lJ z9a1xk8&7xw=DaeesOK249Hbo`$hV;cM6!M0SpwUep%mcqs<5qw#F-P*h#FGbjF%y5 zT@=UHt)`N?pn=uP#$XH>l&@oeBYL&4Z&DPcyIWaRDFxSXsg`ooTOveUU#e5KkxNqC zbEuW!OK4B5SS=dGmK0RNLr2Gk%4+j`fin%fMFaJ_FluO&m}W=;hHgxG0YJ` zs&lP(X4bo`9mx~xSIS1Q)s*GvLrhi4-b%^n&U7_DnomJapuV=Z#;tn7ML&Pt89dfv zB~I1r*0wCUXI!p3%Uv9BQf$C$P|!Nhyn^wj>dgHH@jAP(*?MW!Z`s%_?;Uj z|JnVS8$Q>JPA#h^O(~kn3_D=&O4qcM#-Ka$8p~oyc!QaM8Nhy>Q37&O%>un;^Mj zG%a5*IP73-Y!*g_nHab2ClHXX+oWs};!Lld5_Hec$9DmzycQ+)=hB>a_q@R`1`5h_ zB41^%gN}5M8dtw}D;MK&_#0!Po*$4uLa&K{*zUIz#zerjcR2O~rJUDBIX@C6SxUyd zq61tE<2xOC<2y?zhu)7m*RGS#K}WFdySgVWLn~xSRn1Ff<2$u>YlqX`D6U^CiTMD1 zH8{^9>Tom!$IWHWKo!{ZGTD!%Q6pUH3|1x-Pxf+!cLGqZY`^uBjs&!w!RlPhcYqNI zg+)-A`Jn7`cF}hP?E39E_`aAsN5OK*jGq6?(8_ZgL zRiIT^r#Ly(WwtSHT)ILE`goOQXRg}GTJ54R9C{9@As{v3RDRX{TE}o8rP{k7()(*E zFQ`6P;|)fwR@jjxLiBU`3t@RwlYCuX+r#676wGiN^N6z4VmXHiuiRtKPyh-v^9zk= z4!eXS7f=Q&{6(yjD${vf1ba3-5&UZ|>|}k((%Qf#t#DN4QFG|8pk>7z)9ZhyN%--d zVI!VoiTixKm7-(BdBf{Dx)6{+6V^i4{W+pmJeQM*Qc$4$LI|_OLFucvf{< z9W9JxdzxkBFP_D4a@#84~X2KU@TEm_O>>x|m4>ajxz*l%)E7P`p^<=dFZSNXQ zp_|qaCaUV7@T0Cw)Ryk?8Wyz8-L(hd<%*^2tIRnS0){*7(K&xeKM7o1!H;02Fx1yv+-Q6awMr3}FPW8jOF z>0pe*OxoLRU5BbEAnglm_S+OPpGSbC5ujDp;FFeJ$pew>Mpbrh1iwR6x$W12D%qTZ^7S*hI{I_(qGm{XG`;EzTg)x^qlslcy}0}SF!xtR2@+Gc2t=& zK7WJWdjm`ys_yl;)5{BIyI+^g>oofT&Rs) zaI8;yQ!m#Nm~Mc<=43kJQ3edKLnr%HUk6camxEX-bh=EYwUHZ^>!8YY&c`gFF>xQa z8`GRaQLiCD0bOx@2n3u2Tf(beJ;VHy6$Q^j2?MYb`$4bn+y|Uep;gsVR6bblB5U$r z0q1zz?p(9a_1D9D%hsaD-!23$d17k~*Cy)@%mFML9!mk%eBT-G5R%QhlYoE-|K!%7}Uf2Y$Bn2sQuq#lUGf z%PrJ$>|!u5-<`%AgB7T^!0LIxOc(_311^BF#7Xmk!&ptobUBEee#aXGkVZFUj|;n7 zI;@xjQpG?eS_OpZ-?+4UqTNSUt+!BFnHewR%r~o+Ou)jmDksj>+FIKGE8h8seq}_q zneo+Cvqpn=vH?*m;rH@!(}AeNkgHFpbW1^*Gmi>e%ZfDK?xF}%#B>f80ZG6YK&zN9 z0X+b6u-ArYYwUJG;4uf8FOX1c zF3}OP;|Dz9t5k|WNP^ipoHu+!dOsin5B}*TFT9HxP=Yrcv%2O603h^oV9o5DbDVT0 zZtRTgkD+Vm9xbq?g-3_!nu0Lgs&LZGS| ziYhj`9MwYpv3o`Q)iwF?r${l_VU7)GTyXK`X8$h#>gIA0Cb`0<`1X!N5pf} zckt$5f4qX5uYp%}6xrPL5?dsXM789@xzZ@v`-#`%)-q9+CaF$hvLDJ!&^`b&=U!(s zH8sY1(m=}&xe&MFWC>PEJhw0>5xyg14U#^!4>YQ&F@#=-{1&dv$?^#X&QHZJbOEuq z%d`Z%S!L(9K%!>rDaefo4H5szdi2`>(a@LrxvU$qdl>Aw;)d9tz})m2T85w1U;&=j z!Fwmm>gWWdx*fEL3reUy&Slg3U330o>I`&yH>;>}oP`Egp;j)z4qe9~SJF=wlEqGp zm~F>=VJ4BMt{aExZ2w>eF#zfd96uP z`%e3IvZFqliq&dE_KCVzpj(w9`1@bdU`;f2G?q!@jv)reicR)}}j0;y>19 z<=w2lE}Y5cbD>{eUymfWoF|7DH6eEYGG>$GndR-MpYr^v0WDJON2}R6kl5E3qqFFb zKIfy=%5<|zHXTn98msNabfKsYv-g}8<(#L*t8S0J#r68Q_|}!#jrUV^%zYIW4!W-S z5daFeWLCuFum=|b-?L_*vK&*LLq)|N1jMVCm?!FWiS0Q@7u)Lk*dy2*R>Z}qOKs~i zOMUxno;lh1hK?N{CtOJz8|8|J8XAvQphr#*9xZZQ)&v@P!kTUVQrM4Yjm3b#CElbVQN*~9V~ zfOgpB*=cweJtwGT6i(2-3K>fJFdIX*#d28Y9G{{d-mzQfy0U-^~QX@#qY-R+p?jg{_Tb*t=~>t`FY zj!qKBiN;SaInI=^8b`>mNy8!crIg;xef-6A5YkIi6nf;&nww`1k zvq>u(ev_3sqK73sgw60VEtBgQ^;q8sQhnhP^MJNU9zd6bg9|h7;x+H|1MtVeH7k}s zsN=FE7_>5>-UByBF7Ui!1=1g1?sS=vc9|`et;OjImt|JD8RP$}3E(7!1JJw)=ee7$ z7H0f!k%m*mXu5S39xKyn9hW#gK_?I>e3{f(7(U=CERbpE2lWo9dN*~o;2$odUX@Zw zz9;8weUSjA8FL(p`e!QTbpW8 z*>N$3z{F+d0d$Q=8;0u_$}FIqCq9qRw4^4Tqc0iJ*oOOh+Tlv`N_gVY$)9?z(a zaWdN))p2mB7)5{u>{E*BL6xB&H%KqzD2wRgI*w=o1GD%nS9#ZBD$* zyz||Lu19A+!--xFWrPY+p*}n6`)_^&0rkr#LJTgRqx1)Gx2yVu8rAV*$_qn_6o~~n z1eUN!_e!~olc}d05%5v%361Bmn;x&yK~>o4fL=yar>FH-v_t^-c;4M}Xtr=Dql*8W zf?;0DAqmB4|JrfhI>d&aKOLl?&HJ5R4#=2o>6mSM3az?%QYltab6$lQ@EzI=&KIgk z6+H*4>&lMnIM4p}oVxfz|M!1cgJ6~ajLZM`4}nn=_;0rQe|-oiDdj(1h5lb3`~UGN z!GCX}^#8R>nW@=+VH*H09$O9UK3$2!&LninY4`Pef-Hf!AatY*W>KW;SG9lvy_nYi z;q;%_ohk1zo}+!rPLKiTz8Qed|Jw&(4z3ma?^~1gT^F~UJk3_(P@raOz98~=9t8`? zZg`>khf2RGCVUe+klEh70N*pPStuM#ETw_Mu7lsw({$jAayUyB50`kjy&LMQZf*J9 z`(N<~hiqtaTGSNa?Y-SNR2bNr_O44D<~Ci)SaV09&=v0adW~PAx&gM`*LzxLU>5?t zbTSl78C{funEP+bGl#X+!AUqaV{OB0_4}&`;3W0)9_2?IPHyL1JY_ibvj<04h_uW7 ze#fFX&yVsCFn;zAFphjr$^mcql~xNq1b@A2gj}EY*(T8}a}#j}lBV~a8xJS@5OToQ zz3~tR04!h9?tp{+zz~87@cbA>hv#EN8zJVr#Xud>-Dyo3M;|)LnK=0nBqITg0B%sn zfBjumL6xFGfQi$WtHVDAHnV)U_ZvzMPgSKt0aQeL{>z(D7E=A%@0uJfA2M6gvZ?wtDehsbrJy*0nKyt(I)XWn$z5fc>%=9O`-i)&T z?|Z@eu8ckr<|U0g@6QGdH)5F(-%Lqr--@bt26p+I=))H4gZSNM7BcNQMa5QZLuN01B#UZ~PaE|age=Cq!0xsChw$NytjZ~FK3z`KW>MH&$Mz)x(uIq!E_ z%jwQ<^lf&fD~n^-kf}&H4w{WEvYXob!y{o}ZD@705<5=nsyR%1g*J|gt$HIh+3CMM zybyWHVX7>1+)o|_hsXx_)zrf~7}iD~0W08OZ&9AvN=nw4c#43X1Ttaq_}{ zB3>M}^U~&{CSy^+b|nJr6&Qs5%a`^Y>=a%wW0$`c=-g14bsMW+n{+N6*b@g$)AZ@H zu^Lm&Y&S2CUH=8(=%65FVbS!c3bD<)X1Qvx9{wi9fI>V|TG)837ExQ8A~CrAp$1Ub zA}9TT*Rbuy3xe95mwD%V(7wOg0RjKBDP$=3PVXi5NzOHaFrbK)>U&JT>#&crCpVz~ za;ftD`=**7Y&8%2qtloY_19(J49$#ivyjA0^7n{V2qOM{M4Oc7;GLMKtMW8DPJdnX zjmFHu=TD93)x7e9hTcIsjeob$i0>EamRHJIQHXpz=wkLpBiYYf|Ff-^=6L)q0`|WL ztZ)BYnRnUY@<&oAWrChvM$O*5yhqP=m@---x$crB14H89Ew>$2$EiSkiqko+f4MKw z&!ByYyq4_0zXT|D8+PvMq8m6`v2qU%QzeZT7yg|FGdjD$5lS@(>PrfHl3wOGoPLq2DXCz_ojIz_jv(}n?`D8#G!6QiXhZLD0 z0B@{xbi8l<;=dc)7XzZQa)T2MJ^W?Y5BZ6*mjZ~m9H>7JS9wr*Y&OrL@NwTHB$Q|c zeEQF(N#_5`UqP{>!w{l~ADiFO*t#I6A!n~KYEL?#%hHlu$IfL z^xjQ{nbAJEPib?FcDJr(I?8o~EacGj#iD4xKg;+T1h=1N){W;gE9h&-KK0%xbS1Tn zv~*JkCz?K0Jf{&^aG97mU*eVu6CIsyebUgZvIYUa?%IBR^OMFmOWEoq4XzZN{xcNA z;4}qi<2XGhm#!Gl)8m=L3xfFQSQWS%7S!5sr@UvrImT4*m(AKK*ezsH{JX(@V`-=> z-g}qx^Xm;y(*=Kzu3T+2t}pg&r2W*6+28C(!Dg=h=5V+qpHTw5ET+x4(cZ}VR&h>) z#oD7cM?$}2{|zwd&(Cw4^9eXzg}U~;GQGn{a7C0_)NZ8lI8q+uP^z#HhGxqPv;Wyq zzd`%U)VR;4L;F!m|J^(KV)x}{AW8;goZBJY!ZkK(l0rGJ{$0W{Ms%63YZax6jnj-? zbch6r>FRqmr=J5d+PA34<<-B?Ol!BT5aKd9#eIAGyw$==-4mQ$KsdM0-0Yo3GeTb5 z#)g)zi*K6mbZqzi(JM zvft10vvdoa*tKBPhc@cN{O(=pGcQkRFy%Z#hDNo@vwoKQdfQx0#`Mw;f@`)VV}16; zq-Cd$A|I!;@143*cm-cmL|&rI=XbsP!?zfU9JX7R<&Tg}5hr^q0GH4Y|2?>(V!>-t zJ&BiEUQC+&JTlMRc&tND0<#CEv{Qc|a z(~b{%7H|Tu{+J6XQTJ|^NuTf0ea!5~K(fXC%Y5OPPw7GAk&1%C!3L~M#7E4?=vi&1 zS?QYryt0ap#2E>Bc^s{RU#Ry6T+`R}bl)s4ZlMq);My&2x788hUfx>jvyUE6>{dJw z(^70p^PwfUwlJ3{jq@tCMk!kEl=|lnw?yMRf1lC5Ez7c4M;9c%l05GFj)Mp}B9+9D z%*VGCAAIY65FZs5?(BHa)sX^fiHPYn<~kc0EVd*$nGrOEl+VewGX@Ol#658lux+K~cjE2BqwVB~v?zI<4ZL`+bnoHFxMSI*NKUM)J80P$@ zHLcY=nv{}i@!c@(O{U&EK@D-8U~;TC8%*u6YPr_hKjp2Gvx2a9(JBFdhD;W$`OKRV z!><2|nu7h+YFHpAYkI;4go1!Hj9w|C#V8q);o^PV)?l{c8UeM1^P{2uh0 zpG<-t{)~~WAy1&*#4V|N@uNMp`RYCiF>ZtkJr4_gIxkLrog~ z0A<#X&*;8b3cq#b_m|gvrWQkhg)%9Q} z?ELx8v*U>lo_~Xk?QR*n7HQ#ZU;y_We7YzZx`@am6sD*_gQ#M(Hf_zkgd`)ycmEDC zIJC1+eb#`9`{YNSAs;gt3x@aMTL@YOrms=+E8+oEw{%5s35#|LUlAd@9^ZnEgqFm9 zP;-)h0-OQ&wcHM|$rBfZr* z@{o>|LUIsXdfci!x1rnE@AQ#nYsuPo#bfWcRyy7u0I`Ekra|Zml|7j1?v#&2k!@|I zIb0)B3{DnL@C(aR$ZQfFBJfa?#9#F~(E8v@6&@E53Z%&zdotq0fD%* zy~U8zJwN|l<~!e^lP)I=cDZPzsCluTu=K|~I!*JlIiD64(yy$11s zVo91_+;kii_m5BVDJ;F|9dF+ZvYH-^0&1?G4g+T%oYw5sygpE@zT;QviDEnYqc~R` zk$O*#lZJPl|6ajO6|JMZPVw>?d5Yw9o4bm$U@Q5EPZ1#+npv42P6??UOZRtdtC4{ z^}8a&hb5R;sb(PDjod&360icHxz|p$T%$?q9n7WPDSD<#|GqHPP^Ct^A}-&m@Qqx*|ONqa<~@SsW|J*8^8C&YS7ZDI+M)7#`YzagpuU4?0pfQC!awCv(uUq zhU<=leC5xXOI=t{&;(Bq_FcSqCbJyv>@c8F^HU%uy5Bji^{a%cF>%D#{PxHXr@QA{3_4^k8Mm5ooGv9@+y*fkvOfH}2 z`h+-Ve~6wLCORK%oX`|`mL9q-&O6tCz^^auX&(=6{@H;V#Hn^`TdJR9@e$v~V%v_Y z9cLnw-1yw8spuy9F1bQ4Q3|v_AJ~SinmFIEPZm3MayocAb!o!GWM-Ug=<}T|PvsIj z$|7j4Gsq!Ri5vKQ@=_xDqjZ+0KQ(mq_;kY%87f74H6w$JYsFEYI;CJ&C&teA3%k<2z3J7w=N@`}f1YXKtY=}~ z`_F(KQEsYn(U@1I^5cragxr&rfG95-y^} zn`NVBDg-K_X0nvg;-L{6ig2&Dx(Ar| zW!{#zL#;R(=3aLg*S=o;d|lvRqLTR7XQJ>&m%Hom3?z=?ZpVh_e0-6jc(oSH_75%f zN~;ifw3i6&e?o^oTsN9`7NSbTo6AfUi{FznBvjPux8$B}5oYquPR*Eqh9!roh=^dC zH)9@4upY(Df%E9IYS4@(trp>5b<{KNGn`Dsc0Y=K9C_PNavQnQ*3q~nJ~sN!TUXY{ zK;?aF>y^gM(d*V!BZhWbw^XjbX)-xYksR#tdbIDybnR}3yVmHR3IW&J26^FL8Vi=K|<+}?odIxyBh&fQlv$s zOS&6`krI$@kd7gTzMFH-z27$vfA9dq{C2&u)_UK)nbn({E8hLB$Isxu1Ah1dj^3BZ zm_EN>yFzQfOJwiTTIh*&AndINPuEI2L0nSLlEuQ>2qKr8CEhCrQJ{5UD)qd25C^N< zte)DdT;Bi7GN-w*Mlp~4zo$7^;{DxrMz@~H?bR__)5C)W%{b}Pi^Vi=bHiI-qo_67 zXk#1xrBbS$!x=JsSH;guxf*u%mE-#xcP_Kqyt>DAQVErbhquNK{7W_-wkVGR`0-Uy z4-?EdInc~6`qpjuz%|RH3lmyx-!R3#zQdHDfv=<%5_n5I8JBWt5@JLf&57Va;#q9~~#9CEOntB+aZ`&!RT~|&Dgp+?q z=b{?UiZC7Ds%&Y*z1nUv{S6CflJH+=YxK)~KPx&?fr(C))YlyV|D_5neMU<#_YLPo z+rma@&}`e3+v6|PSQezmz?V_rDX)I@KqV$zgmr8wq|31i8=g)X)ggT#LYe4$EN=LP zy4TT)G77nzs%jz9x(emAmW6j(%&J1GBfH5vePxnUw9FfK@H{qJ6$5r)I~!X%QTo84 zTvJROZdp8`7p?sw);zMYIkpmPguYH=(Q#nsy%Od_ChZ!x)&o4QJ;0LX(v?@d| zcxpv9&RD*=tyF%kAt*E*4O>wERu#>YLKa>?ekJrJ265Q6utAbiUe^Hc9o)XqcIhmn zX!V!6n5lfI>v!K9cB6}jI-#C^%lgzb#JIA8USn7RC zFFDKKObJ5xGD4~&Xtm#-{*@{ zh0}37!^um9cZTWIy^^LxRh5r^1m}C{$D+Au3)Fh*SM-c^4vP}12O-Enl>4c@vtqj$ zMk16pCPV*$GZt@2iAn8HI7bOh^}-UK7(muU&C%Res>m&)B`=uJyNGwT6jtHcrY_%! zpWo-$Ef9NGXNK)gLa5ZLpN@9{pBSaXVbtu@{+p`w^e}e|Q&f~Es;OuPZo^K^Me4bq ztATbWI38`$LEouvOf>M?kptQdB8$Zbjv=rRiM=`P6L;;ej~5=AT@3D9@47d_4<4ha zc`M%;3PMbTAH?NUBc0IqmxQ)a?J{pQ7jDId!>5B_hu>elSv*U(b7#~aBoTp^BFSkyAAV_FmbG`R9O_>l)+lr0&fw0}Czg}MXp^39&n)ShEXvlvSj zhyu&w@K>*1<#z6@DA0!w;~4hZSZ~8;OEWknAd&J9ef=xY8J`b?bRwDZXLPp3;w()E zk_R+88A$CN`2t(5^J7|*2((oUi4KmEhdfGxt!ykXi;^F%42#s6rM1*QROD%3Yvtn( zrd3s4%V46v6BaW35bh*I*;~gTcbNGXZ*IGg-Ma1HOs!fa6&o?z^6|~zv{adgTdt8tLFVU=CUfHkq`D9M9N@Eh zQ6Ff@Qx4`?O%A|Czs{TPs`vyr14s+&$qGoI6fG}9fKR6kVse2+fRPw5>%wo3N=cd6 zRfvwhex+^~bo06|F1l;jp>b;);wC8`9&mRaW+##LS|ihSsq?`_=J{D2xqu!rdzK*{ zVl*x=>(0>EC?*cYg%_iSui7<|2kX%1CCs^#l+}U4TJrK8bXEVAd^(pf~TG457jf4$e;#d;vLnYj}xSzsW|2iHPitWxw7~B zohhlqau{zlQKxzFrCg^6x}~G%T%1Hp8E%tU7B7>N(35!U>4R$)c?CV+vt6!C_GPf&F?$cA#&o#S*x2g} z2}Fu)t?JT9%8x;6*J~lqtB1WqB__I0{-MoZIoc9l@5>yGj~P5S(VfTkVKCImmLUc$ znvfQ_O)*)eWJwrf3T(N!B>{03i13T;c!``P=*@qVD{&tqrwaz>^MoC?d}s z+gG06qvlw&*~svWIYzdbBHGi*8*6H*sIP?zpQI%tso(_xxIQ6cAB|VU`tW4)K~HPv zXa)JGtGPT%co~BRu%fX>9gJ5TWTe^RCJZTEkE;?80V%^~ zN_6Sot*^B^v(B39*`(%*^&EI(M!y}wd8~Nr7xsbxSgkGP=s?!4Zn8`*5Ekz&bp7To z?~RaybS0DcA0IP4@KJNG5(3_K3iUvna{bre@Im&A$~Zj@@;)m(g;y?Md%9}Vy$mla z44``0ey)i;zTxBQj&&z4j3J4qc4u|)U;=V_F%%YpNRvuhs(A3v9@be=zi!)iDMH#l zmi>k!KI(x)9PD%UoTu=K`0pF~RuLh6$H2?y3$9oJBI6;Pu_uocSZyl={>5)gEu50t z4EVmR1Z$~gq1|4VM`ODf6*L(vgzF#f`hLPV9KI`|0aL()>bs2X>D8ZhUyEC z?s-r!%qZVJrizkv1?AFef3%>9g@_2y&3v>!u{%HthmY+988xjrx~e-m@MEV7s^4D9 z6CLJpvJZPvQ6)&k!5(IdgQj#n>tG@h7?WVUltMCoLS8LLPCl1g>+|@ee}UQbKjU!< zy!0IwMe)76I=W*dYL~cq<+P{kA#g~0r*25idVhvjSKAkFOTEG0Rerv{YF8p0R`wJ3 zLJoUD%PuPEmm?{edA%;O_~`(!SpU3`M`JvppUEUc@(xWW7#UFz^BdmLST07JF#^Gh zer4o+?=jr*@}419iBaz7^dkPD@7mg<55tKPl8qYQ4Dn3%QoFMeOhTMH zFe}D0#Z5xcad zv;6%T!i7Xsf|>HaLv8iA*!tRONKP5;32)iYcgl-2-H>4jqdld5g{zjunJmB3=hx@- zlnJliNcl$t^h2ykFSve)9NbpwNJVZL0T0=?Bw#Y7N-YV!+wm&<208Wpqu|i%${Q%D z(ZDI6Kl-&x3qP$R1})8*)(KDOq$t!Ug+2|ovK*8EW@$e*;xKo?0tQB6Y2cPtVaQJ5 z*J^xMU*T)xX45gtA}dJRC+*kg=u}&OYp747`XN&XJk-Hg4~pR!Bh*KArTC{}<>1YW zy4}$)&{fH``V?v!SW^XPdhxw8)$gC5IS-zDyNZFeKaC9#E77TAsEX<-DKxuPjW?rx zIAUn2@YC_f&y~_LPr3=*fXNS9_*{=tK5rgU)Bdn26W}fmXibEy+)A#kk%=9amz=A% zNF5ab7Y*QZd>8`nyyHqs7+OCR2ub1AFkeC_9G7?b$%fs&H$FesX>Zw8t_S0@Ko7x zu-}%YmyZyfw(6qh2Fs(AxzwvWu-qx-VXs4c-+4kd-hU(XvL-+#rVG>u6G<{kRdD#hRb%{VyH?2pOv?}I? zU{=U%p|$XXySEMd)u7| zm$3e||Fefe)$Y!*N|J~{2Qu65WB(qG_|jq4e8K=xv{G2EBZH#Dv4xo z15OIA8!pT5;>!!ojhZn;?|jW*+C_s#{VPsFIAXe;E(5?&^$K7a@Do79@4QT{3J^DI z$7>1BSIYm|hbJS{Bysusj-SmYcCcptSXlV`UWS=M=W;YH{pZ(W40 zds109@yQpnJ`Mdn+#(O>hPKGve1PrJa>lHrp^UwGm?A+Uztq)-zN8!(QgN5>fQ9lD zdR4Pu$=z1r8&E@Cn<8#d_opcoMngmHP)u?+$`iyRt0b&eP@mYtsA9ql^UDUPILH>Y zXj1MxgZj!CDY_~io%oy7P~-mTWK3kz!fPkz_^=%IMo>VrzH1ff+Vk7t(g&eCf!&^m z4yXZc?i6BlNtk`dfH)+xT!BW z64A_|0TCcAXCfC((0?IkQmU?`?tZ1985@^H{;a_JEZhvaqHeIt@fwtIIO&~MNI9DT z`HaKK?6hr0^v``WB$J>L#tkrcJ2&|yBz#1#LQZ(ex&k5swJdLfv#2N<&&i)~)_j^a zgi#pT3)~oC6o_eVlT>r*8w~XT|K_G_u?shKlpQVPXY{xbo9(33Yi;~c^<;}wnW3m-CN(7v8J{B(%J;qwAZzg7d# zC&`Jw5l)NTGGjfe7@7SFg3o-*vft!xq54l9vwhd{k6;@xu>6-s;D_dX{&4d0FZbQ%6) zN`^K{-vDQlI4>Ag?VfV}VaiR)$DzNpIG_99z>kqp0wxobIHL1!RzAe1>{I-O+jQ zB}P!8NW&@6EP7OlvS0HO_DPS0x%raq_`J&ftbw_}$p~1#G*r5i)qd2*+->|vPl?3Rs&IG5{JYj9N~$OunrIC3j?!kc+dC+?=K(3k`4@jX zM=doOctxFM76J-2m`}MbCYClN&@+9?nrCrP13CTV@zRE0#0Oetu^)&}BFiJIWDs#y zIV`f?wOQgn`aDi-VUec{#$}D=c@00Z12^*A;2gfsB1}%2$20WEaJ~R+onS#Lbxp-- zngd6fu_6dGBoG}nv$X8|O`%e-J2-9o#c^JF0HWgZS&}WtQ$3*=Nw>DNetz4_=Y@%L z97W1p8YLT9=rleh&9hJbwM~2$Y+t&o##Y+A4(!WVtPD6Wzap-V*Kr|9&h7a_I2K9- zwm1-p=4a95H+gY-Dlhe1(RB_4yM-2MKKDM(UB{ixRQ?Vh9g`kdJ*pBC)i7N;@Q&~Q z>-hTFBg4P7Xy>rI0Gkj+L5_e&>5>7MLBAB_Kt5(_<%LYv%EuMk@9EW7{9Z8DdyXC> zhc-ITkvZExKv0y1AbD|eo4MBdX@YZm3J2$~53%OwsHcWqS^mpdnOV$YG21>5eE(gw zPcf(_m`K|$Ll?Q5>X>q>FZFzPPYB*>F;Fo7$pv=F!;w_t#ea7A4e3i6ZVHl(Tw6FV zd37l~692^j6!JwDFAgvBIT!POgopr7WYo(oaQ}8m=oV3@z0Mb&S2qtL4#-%(s4V5* zFg{{FSyyae<4E$#vI$Cr6@`0vX!EsR-%-uS20uk|ppTiJBI^CZv7{>FWbd7AJDYa! zOOv*<(B|CsSBZdm$fHnkKwLd5x{Xm}FYq^TGZ=Ma)gluT6>;DCvKL2&ou`nsd1M)q z@AOpDdtrL|-Rb#DmiwK_%p9Wo{lt=Rl`zX1;U~xo1^TUCJGw~e}3H{fa zf{J%7V0-@g^q&aC&y`@yP{paiI~Vi(&wI-Q6bN%l7jRW{kEh5WfP{7$S(l0(#VFRs zHxUlwp_5PvzgE-dVUb6aSJL@@a{)Q}=fsXXm4e~de}7Xs=I4`9m4KTQ^RH z*ZCCxH(FrSYr|7PDec%F-79v=#b{mR$S$8f<*bHZ=V7a|`wstrmUH%%^T_7y}K znipmUwKr=9y0S97LmUGX-GnJ5)d<##C#@V?u97<1CgcD^1+8bpxDsDU>qXlM|1r*4 z5&F@Luk{%^Hu=i_7jIE!j8X~kJF+zCO2dKm!)jG8zm$bQs<|-SKW7v%Q?F}s`(I17 z^rJ=tnoK3a5HHg5>923`n^EEL=Yo%#!_imO)A24Vhqb1#d8&Qw9lab!)`I{CW-9ug zR$28?W)ew?iKpt^JCBvWMU?fYXi86r5u+#dT$@(_T?*1gr7AsZUcWz&#JO_Ov!Z-c z_8y%j6K0Rddb+BJBNYZV@E{*N^GEX>cafGNURiAFq>gtMPOjD#>GP(tA8y8i#{tR& zPe2A$WDj2d?dIA)9RE6CN^$ENo_j;;7I7SsQ$+SNw9z;;#1+7z4p^dl+w~b6-M-* zS%P#C^5qYBc-rFv*o@_hlj82eagT&Xtk+M6(h1RB<6Jm$rX5saLUfh6FtRFp~1>w(G4P}#iJ&Q_yeAX*5v7QiKkx#%2Z2J zMPt3;QA=5wN^8Uyr6U~<^e04ti}*OWNGY7KbCsvLxkql*em-VbdFxd^b-oAHx@(xY zAE2Lx^WwwKyURE(#5Wy(F7JX4N_dx`UE^@u?~RstyM1eOE(kgvBqJkml!GyxBg z`?T7(&ac?N+~wTFWL^imDl1Jo!0i7fst4;&Sm8g;mq-fGaT4yC1)3r7`s()hKWUSv z7j&!SsJ<1zF!X9M(`Z;Qm32k|50&Y}&XV$XCujbcRUWe*?%+=` zp#cY}?S1F&YwU4vpIob!dj~Yb3cBW%wKiS#vnQ>%C>2)@Jt4Wpf^6=sa9yIiQfFv2 zL)TK6CW()dqgx9mrE{c@BZJqQ+ZFzD!%tK|%!({ViIT+g# zo>8mjicoYFPDeT#x&;UZ5%ZWqOSJs60?T64w$tk?hcKIrBZ3^E63{QQN*ewtR?A{e z9!a=$e2r>vcuhrwxnRLDA4sBI{YQZMevh_&A>`q5D`**cdj`)tMO zBBBpvYWD|Aa)5&)S_?~pem#!x-1M4U^aN^Le_(y0tSVP{A0u)n|8c-21ALo7K=k{% zpT~dhUmKq}tw+rYYdFR@^vO?tCkpl{^|7+X7F&+!$6XsAwff}?s~Si4iz4Pt!`1U& z@Md(sw34IXPm6!%3D~9mc~8k4(O6<)u_8C#o!F_BcXF{k4@%`iOrHU{6$SD?>kGbL zC7$nm!#k4Qr4`|AK)Q>@Q%%Vp+|P80Li{D49k6zon}%Ls1cZ%;|c|= z+%{bWz5*qLtV1Xo<&!*|cnz!|8Io2R3P27#W10NWw;QN;uJAWdfRN1Cfu;&YI99uI zuEb>FI{4EF4$;H)l82>o(ReyHu)A8SK|sP_cD4Fk=N42ZTgJHiVuIEEx@B|cYir=f zdWo8I+S1C+JC#^v{i7k>(Jy)-63?T>{KA(%1QQStK*k7_V~r7}23iN_pL9>1eoG@Ma8AdTi(W(&>O@vmw)Lnqif~zI-rQvgUxYt*p&Aj_E5KL1^kQrk5jQH;gXFuI0qqgvJB=uhEncLPcWj+io;b8*Sc~1UK72-4*cZAjTG7A?y zH0MwG(KvG%HYApT2jF#mvG(}-zXn1?H`3&v!c%V*J27!o!U4T5PYG#Tti<&2G~RkG zp{W%@?M^Pf;E|JU*W6Nx>tJu&vh5?FGqH5eks+lq;_mjM>pk7}z4MF)a=xt18UKk5 zIB>IbuU^GT?29vAH+x|3cLCuBKET2y{cWUK7sf}BL(aB+dW(y84r_b0eeEl#EW0j5 zA`_-vqAr2#gypyEL zQe&VjesFrnOV`^s{#SM7x95H03?_XF{yTL?cO{bp@YR*0QiFBLzZYk_gb@W@ns5?HhlgKwZ~85}V_pK@fzgl4@dI)hmXweC9I51w~$wJZy-p~sP8TfCNiddFz4%!-Trt9$QFWxR4 z+^SrA(Nbz~M9T&B(UKor0ZFUR{-@dY3;TK`pbi002=S)2X9G(TGL09i=W$xk6#%C1 z-6>yR%C9AltUbSrMIlI3OlYB}KA-PI(b&qW&#c++!gH!y+qt;W&(2w)K&JM>y>7X2 zyT<~@WYRdqr4>Wm>GHjK#TL&iI#+aysd&cm6U$ux?gwVIkcz5-_o};lx+Lj0$+&Ka z6h#X}e`L0*XA(kxG11ZRmOq7*wxJ9c_fOC8+l4=R5wv)ND~lX)tRG-^f7^NYZBE;p zm%KkKGf5CXd&hS#X&y1>!Q9Cldh5jeC_;htfnI>~-9!%Ofv7X)E6jDLq=3ru#|OjS zrBE6jJT;_1KR)*<9G!FA#hE3d@B#^O!uE29cZQix7vAh;$5W%G+|SD=O!+_cTJc8t zCuJ`^=hcgy(t9JFq#g(G4kh~71yg91e3HClbVZ+qbizcWQ`ptWXcWZX$0otN;x=_U zl56}D(Eesj0pOQD2BLzR8!~Av73Wglfu?5V*GEU>kX0VK@B*EWXzdXmbt@Cf<;_%1 zfRhp_wnhBx?+C7i7pHB*Su=DbhMP+}!p_CVLcV?T9d0FwWCrj0F=)iT{R_?atR1a- z=t^|q&r+|AK@t3_?=m#&Trv2{5m0I85pP4U{9jm7FU;tnuLf_Fc?jq%XBeF#U{zg$ ztAi3|?ivHj0ay9(+r7a~KOkTSuhAJ|Wkz@lOttO$0P6e_=;dBHO&n^4Xvcv@6cMd~ z|LC`agSniG7ebqa&%RDqdZwS0&EUnnX!i76`UaNT)DdfJ6tZo*FZ(8?CHJLePeK~s zRFUIiFG_T9)%m!`%GM|B`b}!GvNp=}^i5KEC<&c&;cL9qqjd00^pS zcz2>nTRTzRQL*G5`90~1?v#>!p?aOHe(=JD{_o3aw(sWtF`;iqHnM&;{jw3kH$FWm zH}QlnCpt?U`%&Sq-RPAk-IGDh6Y3}l``g@URP(r<1gFM}g7&sEXnW62)~RZ|d4-CA zSTAS7ia?%`KuLzXn25v>BY*V0T+>G0qh%cWE!4N>Qp+{3O5QdZo~yA0&>EVUB1h=g zvZC$8g3?6Escq{hkgc%pokgtD1OM4Bzl@63mGcm+)l~89xoOo# zlK+H?$74!6n(KO#_4%|E!^QfCS5*8)r!C;et$r8s_ZQ)5vSowg-p}t*Q$fY%nZ0va zEFPDk6+2}-A5MEF%6i?bDvR>|O_UOrr6uZtDv}_Oe_U;gLG&x#st8Ha z?4B`F)tML17uNs6i@b(3iSoH9(5N+uc|!ORXNO3qN^`VK$0h@eewskELV z%4|FcS}B-}hXW!;-6DU}hDKIFGcM|wSPIFjKM>hS&2 zclCfP8KTLja+6DNq(_HLZMbq`siCh$J+Ln<>M$#JzTzX%q2V$FYs5Dq6(Vskm+bL8 z#5>`kgRNCZKnx3hEjjhoCycCkGsH1?*D1l(^FKA?!Pw~_!rsa+V4ny{aGv;B6bnNZ?9?my;u=X@b+>tDMiRj zLrBJ`+j@{T8}Db%PX|9%pb^I_H{SZHzBJCf?wn8Txp=i^k+b$eWX(o)%QL*odE{h& zUMIkWWNB=wfCfyxcZ)ABQ1G=OqStXJ%R)V&f&s?OzZP09+RnRZS!l=9UW%iy<4pXq zS|hk=IzUtMLUs<8dSqlSsi2i~4MaGP1H6PJKsOEO(cOsj1?(}Q&`00pQ<05sO!%2Y zqkO}!*YVOf(R&`po`H33esCG`y`%--7xz(3QZ z+<%6B9QDEU9Hjga)Q>On`%_T~b{bVbgAkw5bNf<{WmpO$N#FWa{KNFr*d@t_()as3 zuL{p$K@C0H->L1Bz$ZBt)=Wn!`O@h?{C%t5)0&Gubdy8!yP=Onv2r26vmk~u`!XB>F|=udkEK#dY?~-iH1eJ(2i(wmnyWm0c@GE}qdmW?8r? zozMZq=(NeIWJ^zAVLigwExWgpNs<~uS;{asy40I%F)%qVsoStu_BySJa+9|`|IEV> zkj(iV*4=3WxWw#_~gJ_9*i zRy$>Ait0ZwN%hcc7rkGJ1i`!f(~iU8eg(BmZMhPQ7pd0`WG=>4P7;*10}YRq&#dR> znyFQBvR)Z}pOnZH7|~)cnfb=EhO34S$pG+;zDH*Y2KK%E0jwUN*328U96E0@Cc6?D zEv<(38*x9!MvUyi7VC(Izj0B1=|DT5@FNkDGVxF9$+O=X0Oo4@Me>jLGum0G{W!+>754T)YkE*e%@&m8wJNsq$t7YMS?c zm^baUD{UItAD~4Rf5^Nj(@hzRHd|!iWcyXb!HYhj)6oiz%J5>5cuT7x&bHV%J>gkj z_SmZ3>^@T?=OPf5eW;yB7@v>5hbblATOarFYNPx4ozTOM7^P4Di3HK7u8Bq-M{0lf z<11`F5{7kd0O@qRxzDD9$S9j@k0lo{anl)C&wyXm>-a`z+);S|k7W)YMrl9W=7-wO zmFU|4p>k5~6ZgGYS$#jdbsV>voJ=jd_zL&};nq^HAez|>2@l&wfvvB%Qw98LkSmt7 z(>hYv?4F9*=bdya($3h8bj$~2ew-ZgAR&orYpdmU2)u-X%~!XE*~K zK_251%whCI6khk4jG_dj#2Lb|GUqI6MUuN8xJkoQJ4s!A{3IiCt(C2^m9}ZhzBds( z7wEzNL{0l8Wu&)wRmxAH;z=<2Lava)87IPwVX%1xQ1;fni(5VN0!^Fhf8;rDdO$ayb__6$+j#Y2JTdsp5meUdQnu-H+a1V7m z3jP{C*xMf96=cTde;~WzLcPvP`4d}d`E&d{Rbcx#QxxG#T&jN;w&}6wC9OGU&V#rAY^Ky2F z3rf|2tsqO($)*2v;_Z6nZE?qFe;}9VT=wXa+Wd-zktKIgI9ahWrCQ12;&iZwgY{3a z3w~~V9c>h>ei;jL3W>R?LJF`kM6l{PNmeuevTULnZ3CO*)kZt~8vZNMd8BinC2jjP zB<}-kafzVqn2Sg`x!Q;&uFh5yPiUJhJJuJ5wthM>)1xMQSATeaczTSKwK^Bg#`nbv zZSgRf?~dUKEPd;P!7k!`_9JF>FBJRnb#|O47GQ2_*TCmAMmEqszv=C3|w%+a>nPwLYjhu)O+Q zJMuDzxp)Cd>W{-=W183BBDGsE9j1I=a^KtP-4&&LbGuUeAZ_q|lElp9XA~HJvtE)y zzUR=laBR{UMx25Dzn z$V77p{)kz|0T|_Z#CA>*(uY22uv52NEOj7v-c~ll6d5T_>Y6A-tGLTu>T|=)r8;wfkLLXh*4HSAnl^s) zdG#bAsNeI{PkUh9k&b@f^QII#f1Fh}aKQ>SO{;U^zbY&0F{`za(IFL~wFWgYG09B- zxs=mWq=SGv(D+se6Ts+R#)|_3(`S!VpR2{BJ_IsS zp-pdZ%EPO5mt=*guG3@r{B_Q*ZM?_&Nt9GbaZ7p?icmx3Dm}aB!eCw>j?*o_heTRF(LGV5O$m@LMLDlY6u5ppbYO2IS9&ocUJn0^ z1CZ}Ripbk*SB%l>>dh$$9%@Y-CA7fFS~8EA-mmiMpK&c7LHn$D!%{u5oS37(CCX~- zh1USNrnJI9XMMN?C#zP(e2jPW5#KI%R93m}ss1hr>KmQ3?%`jLvHpW@M5yLFk7ke= zwn4b~ncR0D=(sp9Ng0a#`kS&ey=slK_f2sO=6fQH|4Sb(vxA^mvsUMlIx;4=y4|d7 zS&@jt9v%rj^Sw2$<;27wr-iHAvC2R$QK%X|8hRkv3qAnU{+<@KfawHPRv@71Uv_&6 z`xl-bSPC}^8e|V`gL*t!epUC#MA*vJ{BZW5UNnHd4H;Mgm<+y7Yy>UxU(l8Ks1)a- zF&N#pepB&4`EG&KZRhto{?koYLT6a(nh=f+5U~>Khzae*lC&#h9zh`Ge&L{ zI4bMxv{)*1%y}SH>R5X6EZDcyJWieKVGpQ4sDg6^M={ZF3I5hV1LUvLr-vc{v{KU6 zAInNf8nAG0H`Cl#tfA~a=%oPpI~D_01t@x(Ptdsa5WJ^o$pF3_w(g#L*@VW-jN27F zY{Z|^<@qEad@|j^(D!*u@Pp4E9;i9Eq`WbZk+JF7w)q4+Aj>1{rPM(3;Rns09xmv{ zCY-k@?}n6E8k$Md7!w}1OfyF?qE#Y* zGD2{Bm=>BjwC>2Z*;VP)@rM#pBS)SCNCrhc?k6lKo2vFW=pw~QFDi+XJy z>W0uQp^to5Oi-0P+m}qntGS+nY+)GOI-^3$jzugOzUyoQgB;6EFAW-Ef7=!G&pZa5DXouiM zJS0aM0EytVX7f*ZGDVV??E3~kHr?uhJGAsAay%!{Ra|HB0{N3Hx5{ zlm`ILn^J#cK-{sNP)Wj35W2fClMUSsuxY_I*~(O7Q&kWG8t%^dLB%ZF6WqOl19Uc7 zkm;rEWWavd@txOrl1PugnucbkV{u@82=Fy*a2PimStjea_>utY-&m)*c}0Pd;$gn- zsQ>(w5@B1yPX%46?J6SAO2BK<6XaO@#?N6w=h*g7QjJv6Q&a810%_ROXG$EvU^HaP z_uXUv&xWK!18>_=(U24BE8l^1!hy}IwSZ$!FDDIaz9t9m4RU`1`d3>543 znfGb(R*bV~%OEH-LOkC()rLsrzsPSwYPp3s@4ReaLkoEo9b8H+hdsw2vTbY6J1uHe zp;H=_y{Yyr94Mw=d*m8i7=4P}mlGB6*Q*pwNa7;++?~9g3=%C%kG=4x4v%S@I$~nW z!rx5SezBXd1LOlR13~TGU)4@%Om4veP!dKFp5K;G^jj{A{7wJCRo!ip;s=a+ESYAv(NWbU@?`F=wz zz#x29-k=@v?iTRViu8Vz(K($`*5M082%Xu_3iwl#4(me11j`5(S~V{^)8Abb)?j)k z+4wfhB>s~xYUPJ0=e^S3<0)O!uc--i_EG#-wcwg$ONLm+M{1#|zi~fZFMl&)`2>0V z7j|@HA!^=~t+S*zU3wDcd!}d`H@u^;0+e>nTl$fC+)ym&w=Xax@vLh%_}6Q(Xjwdh zwvEifIRa3#8DvX0>Y4TR1DHt}5(;NG~u-ZqF4s*89i zB+j(2JF=h4{yUChZGLp$1|MX!QnYf`N3VrEaYMvWm@#9If2cox=R}(_mK|YiFoiIU zdw1G8;88jh^xNOx&IQHzPYO>pLx0Z=XgKfKC>>CsWDQxKDjVSK6c2U9)?G^sA(t zg2O;vWHD&^(9f&llXUfK-?}JReZ4{E`M^pQ5E#$7dZFn2UBh^15ZOyn3q)?G)~v5h%{@`bONQm<#R$WU2{*E$i;lPE1-XPUy-8K(U}qIbNhig-lYV?|`%6*1M6xc9Vb9m9*G zWRu8HF3=$4EYITnTYxsAp($U^FZ|aqo?-7pDFTsF73;kSvXuym|J}K44}LKjY_I0q zO#DGEKA!ro=LM0MXgi|>^Id>2dxSPV4kS(tO{oxv?YaX$@B?0Ui&3*7{ayCYH=p#m zJedohSqaG@AAn^em*DVAL??*Dv+m6TNo4J-YP= z;v6q#-}P~3uN91^0*fWB!{xpY>Rk&FjlT!J|6wR-`Dn^=qnD=k&l(!GKf2H`VTg+E zlT6eKB98fQTr=wbTQ(a9|MDTvi`b!xL)4^~ltD=B6giygJH{?RW@J5x$@Z&BI5%sjRQ=>6H zU&OXqOC{stSe3dj;#(%%r?im$9c&yi<5u+?P^+SNt$*d6_XteK$hOpJ#JKl8J2fMd zEFN9r8`MI02i%@NHa>a5O|6VHO5w%mVCac>~bD7Tgj5>Lda`Y-6L?*Nsm-Wd=y;gu+(zBuDxAxt$8*RvzS2g%zal5+uvy{ zR?v6-ff=IirNY2cyay^!c+AZ-VjHAT5E(Ozi91DlM}yG2r5?X^Y@SK@ z?{DcZMhwN)U`305oxPd1r{kr>t<|nuS$plUa0FDA%bHNU+;o)j9oiQvTw!}>UO_tw zG}MA&pAMDYk$xvtEI+y!DLDDdsgtI0&u?&9Pds*?7|MuG9O!0W@9rm#j3l$Tml+d6 zLbRjbRy`Pei)~+C)MJ8g+#8jB)czZ=O99mTeT#N2c={bb8Pdv}sdLGa{K$!)e=+G!bybY);Wg?C8VP0F8aTKDgb7G-~0zuZUUMAC+3_!VG!qLm7je5W7WwKVe`tkAc;NcW4FW>eh5yu$I>Ne>~#md6Ic@zva{Fntwe=wF(7} zq+iRm+Y$tn>>#B}ls7j&L0Y_KI9$eub#yCWVX_%$KYy73gOahnaT$5n#S0G38p%JS4cJrDpqNdHu z!l-{%cgUvKczFlDK!$%?7ohKCc)JXTEc{`nZ33eIo;_O+gVAMz^R!#vU&D}7KI=j; zcwV?qLE$~cK;rDG07w($8U6ppMC+}h6oOZ1bQ99&7 zrF#b`t0{TD+4zznrRURfl_Lc^XhFs)t(-EfIUbiiu!ahVTVGI6pIn^<4K%)^IQ!#~ z+>D!Ge^HjYKIcw%f1<`;mz(o?mpkA9%G&rg;OQy9fGAmnqsjb&`u#Rj+>0i!NmEq6 zKSfMy^(o%=F{G_^ptYHYEfVnKWL#D?h};kCh^2LME@lirn#dRMPzOrT`9sIBxA58O zVsY`#q0e5LLRoR7zOIYRz@r4%^YY)7Sdi9}BeZhh4L3E>Qii|BKee09bH)>&YQ-!s z;_x$d$N}C7PitZcv`vfG!%TT>hSQx(q1?@tsbtEZ)NRX>>OrvQs|Nw}XfU84eWrH| zRa8B-p6d*3{v?tfKZTKEAr+{QRm@q(GVHo(FKaU0Jt2JIJ9ghKq%Toq?VwP- z^JXu-Fd+bX@`q1D5{)XU?H=Rp(w$}0J$2e zlLqMFh_o})m9_KkRL$%pd{q%!os&E3J-Q3 zCi#12;MJdMn-0@t=SblJEx~RIWs&NRB5-l1LuEv0-zQN5&OMU;Z!BNT$F1R?c4pur z!)K<31ebH<3QLvT68rDqOdPAv-uLykeQ&@Ds4uQrYkV(=v(lE*^$s)D7SB8B&>3?0 zR1sT@S(7C#L^h9xC(2cRR$B!9&OwKDA4vb--H(m;KC?8SGJL6^XNV&+O0Hlk7~ihT z7M3c5#S_K*gJpVK4{|rn6BW>0Xmh`caj_`~w(sBk9wd`|ZB1W2N^zMDE2yHVZ0ijo zuDW21^7QSDt8@EfCKxa2z*JVBafN!`A|&w;js-iV#HF!1UP|^R2ejoBZlkP)`~Np^ zJCeI0vk|Mqk$DjZrfO=4B}fYiX-oQbF%LV4zM*7u)J<;*UMXUe*g zw|^h&V&#omjcnP=M78^VPOw50}2^8uYn#j%<`5>`)RdsL$A&uq(Q11mcrq z8$Qb2f_Hs#9w@IJeS%l26o=3G{{-X|CbY%F%A+POUr#^;n1n1CrO;Qgl`;Xqf( z*m4Uy6=?k%<`|sfYh#Y;;kivBEoBqCBEj#I`zd~>K+bM4)ex@>PQkC=!>Opq$zDey zZ1o6Jm;*Foc-sM+B90R5)Od%|t2Hde=BxaCe1qqmTm8^hubaxb*5dHm4~h?cj%-U^ z=ilW%^{-33ogd}8-XbZ&S&=Bv`5*S)J1ENSX%`(p2@({El9eQxO_Yp?1j#uk$vNjN zQ4kOi5Roida?S_>!boz+L(Vx#j%SU!_o?sP@6`U)`Qz5TRcESTrr@3TU7=U6?q2;o z9q|i=^XgtUM`K(#NLp8Ft{E$8EZQV^HY|DH5u^orux1DjP8N>L6S_w^P%1G5?O0qR z%{yP08FS*OG1$|;W7xipt=)?7Fx+vPktCuB8A2XpaMqKdzQnbuCbQ%7c9${im3qM~FWq`&?W;SPHTKfTbaVlP34y)NP?ivNCLFYw?q%Z= z6Eg`MU$*BDSLd{Y^9RhikacRl8T^#K&n}GfQ2_1oa@DEMNBG$@r?O^P{gxUFCcxdK zx42!Tynl<*rt_fe?O5hBki6zRzs}Z_(bO`5wFRf2krZTAz^6JWYGqVl%%Exa(_mEo zwI1z~=rg661cRU^OFib3@KRd@m>FTLbA*5rEapv`DG zYqgpoK-@_8aN}pPi}qHro1y*D$RZx+1X<`K3_$^cueE4qZ`FyD{27RY*pfrLX|5;K zBV>8Pvhf0~Yg#jE24pjxq zAnjjmQ;${8kD?~M^n6_16wf<2i8z~Sb|yJux8#;${C+F0e|eBH+PD(x6=nU0H_|%w zXiBE#I z6B$gNYO-*ByTjq8Jb{J9zr5B8@|-`ZYR`mrJQl`*mA7$YD5Wcuj%0Q!1rc_SoIO%z z#RU9knREC6WJ+E1R*jtilPXcZtNosV?TzyDk$|L5v+ea_GENwhibWOk<-lc)NSaz>Y#xWx?e%sUV>lW zXIH?F=74E7@TSi*p!bzLcC3*qR*cs-_joW(SdzaC0!s&$| z0iqB^bR1-eX7K{Zds*8iNWx=59BMVgtVaCJy{^x4wy9IpZhZd+Smtb6r;e+c;<0iP zC@hvcE;|Hn7H+p6m5RsJeZJ52xhq|k`XS3s%Jfo|7(GQ=lzHxOcXQgl#64)`)ywh2 zg}<6q935M~Ey|$JflW`69wpp7@%X3Q*Irm@_L$Y+a@8E5x7sU8Aq|G&pugI#{*b`U zdUGm^o_YV-p3Kqs`zMFSLXGA9uNkwQ28WCnkpzV^eJ=G!)(GXjAdt$QDi7V3r}S8g zXaSWi)|Z!I>dnB`-%ORy{=yn_>Yghpk>qBO=)@;mblD>M+w9*|X)MBta*J_Dc3_OA zNIy7TGn1@#@b!Q6s=y-S-9|HWqWq)ORo@%!A{lOLQ}!o04Of7iVtwn!fzq2-0WPA# z(T}o>Kj%s>lO5isd#nHQTIR7zz(Y4da%EFz=eUB7K<04LZGy{BKAwlNRN{|SMLPrfG3d?T|&AeRHsbO5g#qp3bq%{6&fA^K6#J$6M?_Nq+mF6lkM#OnNzOk5scG&5s@ z2B!$a{GC7!kkc`k4E6^GWm@FE^n*T8#`(HZG_NSa(!vrzN@(H+65}%##)gNBcBl&v z25Hg~-V)w|&{p1WQ496HTg(Pl4#xPV118j;M<7Jz)v>-bFw~7JRZgcP^Y+I-p9fj~ z@elHVn{;IAckR5aXqAU&VMkbcvu&S5_>BY|GJ8R>w;zKtFNoqUN5}V8XQ_w2bAA+9 zhTo_=$7~;_oF>0wXXd*Hq3~}y^aWV%@K;igaOb|D$-s{jnA(zBu`Yj#FtNeL=yBrJ zl}mpZF!I}5A&rgk2iM#WT*;HyyC^#y5z9ldYf`GeT+ds^c=T?{r}A8Qff0pKg+X+f z^1}-xk*k7pzwnhJlN6b@8ry{f^E9OMi^$m$8l(XsG(cc?;Qu?M7k+S>AirE*LhFrZ>EoY&=s0~*v<+N71`Kf0@YoVA^o3a1&kNn(=cTA)vU7`+ z^x}bz&BFC**<4e>ZQY`f(ImL>VxV<{2;>RJgSk4N+g`<}fV_+$@y6q8WI>&i$c42| z=lmVWJtd4j1(OP?*Ifu!?c$h4`+_IGCd`2Py~TsJj%TKc5bCqci zlXrMozdZmYzl+UA)g(Tlkg3?oQWtKxWK)2io1yYa9U3(EW0E6PtDc{TRElV660f z{k{yXtv5emTL%jLzT)i92q;KRrafA2>|1z!CB7=hGIy>WHh7GVT%RP9?ytN50Pd9s z;434rEKNU0H%*TyzyYH&E%}0$OC&Xc3S-9`SfTNFGOZ_r=O84K6rgS%Rd)Gf3!Uy! z=Se35f$Ax_S1co$NcL4;yh{DG{{@1$diD6pGy=`YOQ@u4CYam?yVjaoDd^Mj1J2U5 zt+*$B0?RR9 z5rBg}W4iMS>20sse*A-dh?A0HgSN$$JH6zPQTZ2BcJw5sFe$pA_^11nhes0e8;%B# zQ&3PBdxNeolezTKZNZ#68W@bAG#x?tmckM&PeE!R?X60JCl~`}|TJE-|IAh8guj zXB?nxz#svpG+cbyQ?cLr`<5uqEnEAfj*0{T2hUSLLBm;MYh5Pft5WOv7EacF744pn zW}c_qR_~06isr~{JRBM`U@P(FVhjstMF2+nX)#WB+yjx8g=1fF^Fq6&0o1T9O(@d; zS`>1+Kol5!;0#f76DV^p^@)Bbc!dA92V(DmVo0En8>T3QvnK3$kYK#I|CsGY;f5y3 zz+b4$S72}y)k1CMlR{o#wqz$X{MC;Uu}w-}dop zg_fZ#JGvl#oCvjm8-DHd(Zb-vx$^D-!sD?g zWu2x^-4*$Q4Y?BA0bXIprdQ}ZCbJ`P99+uv}k0zU1Hmn9YviVP)bBcPGAkmY;4-!d3dy@P#k#sOAST&;iC- zO&&g$!gx-IqEPy}F3#+O_g@4W2v2fQRUY`PY_W$vlJN+Y+*C)n8#`)IYNzQl3||Sv zHgd@J%=1w90c)3sTDFd>?)9=L7HGGl$)oT9!-adlH>O7r3)j)C{*87Q*?P(+@84t% zh(yb!tu!setta<~^AGBll0EK;YqDj=54@}f%PG%P@aYuz7 z=IrmaNl(@8L%w6a>d(<7REW`UGZ|)^l6+{zEPpG$*Yh!(>|MY`VKg)VIF0w>ukoZz zWo}4k0K`4J5%&sd`L7ltv#^Jzl~0D7$myD&xYH%dXAFo0>fek*yGKDH1>7h9gcyXb z0gd6%%I9aPBRP})u>d$A!bzhmf&b2K^tFGeO1y7AkAUG{;TPq-50j-AUnuU$qb@C) zkG>8%ChC2JX0Lnp;3;!Fz*eGevmQxDjb+<%7eC>gPOp{iWe^c4s;*PCb}g_o*mi!` zXXWnqjoS3W(Pvpl^|7ADjAONea?t-1i&M(5jeW~Woj8w?Qd#skS5s_^y# z3rO?zUx#PixJ0sKVA22b#+8r2ajK8oT!lmOu3YrOXyJz@|Ibr;DluqzU4JDjQ+SM% zV=rD2DHzvwM3KAElgOIAuhIqwKLDbJ*K^D3rIGII?zaptS6T&xs!z|y35?LsIxWy! zz4x(p4u@Z6kKlk4*m7|)NncK4xkPiGP8Oap_OF7>rhAe}0}C^gs4iODQs$Lwub(40 z8S_;ErPIe+-s(k=#*y`KHpVk=zsjF*tWj{>#yO+cFjzkODPXOC9N*>eC^QX0mnc5u zzo0;ece?Twy@l^D$vNCJ#7|`^u zw)c`>ut~f+9U+Rzo-qjpnP``m;_efA9tzOVNB&}_3KsyDG6mV&FCi~9>cU(G(ciAK z`1b^4dDJbIq4`V}Tsmd2o%TrFfH_UbtfzSgl=?hx`kozwd~vdR_m+^06b?o@!7WQH9OU$C zt};{1!I9g;!w?DTdoj?btDKxp@7GT2$n5mlN#c4&ORFe4KQ02=PBQKj8VkQUv9(1q6>>ul%{x@)RPU)g#UM?yB77)PJ0W zB7FQ;8=FV++^HiN^n&7qf$BB>9>5`nsZ-KG0x0`2klq26E$YteFzp%uXJHd~^4NbT ziNg`O*#4K_7=}{!Qy!cKyi1a%3AnkouNRuCSFtPotX`iA8jryq8n(Phx%lbZ?=4XE zW3Aoi4ug-lvW$1;QcyVSo;|#G*g|->h4)F&>$-!}c|cj{w^(4eRZwGG+Z`$qiA$GC z2u{*6T~HT6S_;}r0CR{Sp$1@7g6m3mWnwYE8On}Qk?E+$ltSQfoDadhirwZ-vx(a7~2w)zKyxemI1yr{t20dK9 z#{i`dxOu+P1 zAByS5v9vW^*`lBdGQ<%y4lW$w6TumG@kN`T^W~cab-W?+!R~cFRyHSq+GrAr2|`s#_SQG&jY=IOOKNsMZ(=7ajjR0`3@If5-Vj3jNftmMxtvr*;T?gF;nLv3

m?m_qRF-Av! zH4v#}?~M8Ed%QBq?5-aw*nkwxU7z(zzmGs2T%>$fh_yRz0FY=XfznI2n)i}?v)3?X z0oFantB6!R2m_GaOEZLs(_w?~Pl2W=F={EzyNLNE6E+ZQYX8KU(X}9+FoBcQgPPqB z$(BdM=8}4G;CGIU7oOX>7)LHq^fI}7p-oeq4`v~_(CMEpf|b{U9Hf4?uW9CzE(|T~ zLoLx2C*wNi8%)IF3GClg12fvY>~wk_(v6EtmPc_z{A`$@PL>DYH2z|(lRubHhiMNw z;bap4XY#?9dXc{-f`+~^^E8kca(;;I=}o`TpB;~*lg~IR{uNW=eqYvThaxX`%)lDo zYvyp;iwXFKgWSx80N+2z9|Dmkz<2p_ohH}#DZrvjy;v}x6c<0d^UIkosQa?~G-V=U zKKOt#Qse$#f|Q5zbSDe!Sj`fVvzD7*0pp_BeHV5!q^5f~b0H2HSk;MIu3ncfR7kIV zT@>qd+2gx{V2h^kU9nm(tNmGHjbIUfW~O;G6=P_{4LCcsx3uZxrI!eX(t`3Q>*<~G zeqS&zWbk7-o)F>OdDcI|_5Qm>?Koi5exiFi^u{TSpUS`LE<{-W9 zju=faX6$8Usuj}wS2Xc65c8&;i#lRKcpwpT?sTGd{uJ6>vjTr2s;QIBq!u>sI*JoO zosZT29-}tQb(c1s**lk;tr5?focZPsOwWpRznjekq6O^wDm3ZRJPPY)O}YZ1C`$aB z%h$K|#*LwsH|Jh+Gv3RC>anUXaFj)8DL%H+l)onkE z-ltzM)e=0BhZeg#mswak0ZQy3_%^8A69WKW-|L+ZT&vk1r&8 zMJcmQ@xc*4KVK^KsnXsT3VNIlB0smLe0d#?KcD{UR+`7cB%<5`;x!ue%vHV$E?0M7 z2?w7tf@bf#3wrHvBVodNHHnXdBnV`$=e`J@Yv^w%w)oN$}okjA_MbzhPR-GKWQZ^4@X3mrt>R;1_F z5IdE0Syp9DE+{hFS7Zox`aQNFIUGzLpjjycn9)Ks+dRxip&*l%c8?&<(;Mm*Rbfn7Ucf2@&9xZI{ zXB6F03q8PTVC^*F2$WIUr#!1wCm`%i9bjSH%yrTdBz@dW?OQCcR1%sU-oFbNm?G|c zx>*h-WKQF)a{!egblNAY(tep+0I4cGS?$|n`^ShHjKn`fN~tyn9XUJTJI6=tROVUwzGz-^;;3xXxO5C*h-5V8?$!J zuV%LON$oT2qEj%Mhk9;i47Hm*_WSYu42Q?AYlU{$h<0L(s)Yis=UTRrB`&}yvULr|2AM|${Z^{+TI zIIp=%MN678!^2f_B+ilt0K5PM4VU44G#pR`Nz+o-oJa4iOoM@v37}!q)Y}^-9>(ger{_ITpK{eX{q3~b}hW%%I{gGBk zbFWHRqt(|SO#q$u%d0*>YM1osWn=$d?}_nnBgF=1n6xt$U9yQ2F|?c()pgn7uW#=Z zXbEDLfz%xod&|cxRqKg`B$rM|@f!CS3RF{ML_r?ifZa?;-{^w5DBYhRYz5HtBniUZ zvA6&#vU~tY&;5=xA~7GbqWuOiYrj|{f~_y{^Ck#q9GkWt$^^jmF1;kE_hwFzt4-g3 zKisxQ+iCH({L-J`)`5LcO_K#fviIGrE{OTIzqkS2f@_=~!{LD&R{I=)G_&QZy5I1z z-OIpdVG(dYZvnu7(OJxCJ}Z3GgB&gnNQUn#s@=Z)LJ1)!Pbk6ot>TKG)$%qBnGnP< z>|%F1`p|vS^s|pdqnwz&Q*=EUeP_&q>vS#6&4n*=cL0EpPGOsqgGL5k2H!w*LOiNq z?`;^LSt=o54-Yik{9&+g-I2=T0S%p06JosqXtXK#Bb*;Y04+Qh-5d5@kcUE1Z>5Gd zINu(yDjDMXvr>{0p|yTCmng2ue|=jYuB{sY1I3)|cU98^9;E|8K21QTE2UCaE&xiJ zKN2^+H4{%q@40N9>>tyf4(L?}l1~5l{;xrG_f`AxC)B4S8YfTM@|$bogDts8n#?-< zhBZGvFD)9eaM0-1_h62u9a~&k*)DtV!XqtXWPTH^RS z;c&ep44zrGW41WB^N1FZ@MDybvUptOKFhCcVA3N1>{~$!C18ualL4~;$2A+|4*_is zZsP1pL8pifJSeEYkk*~U3Q2d>VLoU(MbQ@juBPi6_UTeCIRM%dHTj0g$HchVbz|El z7)Byb9C`V8!Q_Vvg7pW+Y%1nTd3L+f9&NAy!lAvL6`adX@GV8L{7bQwB(Z#v!ICN5 zBBltC?)X=m^3IHo zYbUzpe=+IOA<Bfu z;kXq~9M$$E$mvPrefW>`2M~xXUH#6kkGT{gO`DsKJ;Hb2z?~*X=2wkZ4zqxXH>-E=3qmCWY%+=0_&;(L37MMlpG>?_L*P<l&lFE;?AQsKZ|4Ag9JJ{%kMTI9}h?24v8spC2;||7zen0+Arlyae^q)8~vy^SVCe{?16oB!t3(V zANPoG2(Y&tu|%B~1(B8yl^+^h(T-%=*kz{~T0zIX&vbKBfNBveH=4 zA0jdfLv3tvT0FR0bnCsnv(``s2VdX^IO*51RXziJ*#0nLKBx2Ue-p{;uL1_bOD-Z0Ui$0Vo)z$yAy2H3gIma5Sz_t%sJPG{t5+O1h7!#|ui3TnjENySkuG zweEY_I3%q4y2Z}~7;H5taUXgK*YZi3Sz#^{A{O&K|F@J3}(b+6H8X*!x3JK-CT= zRuy}W#ev2eV?o?*1#@}_I{Y@xN71@SuKC;_$~tzwy5jI2^H1k<;Lmuj0mcgugF=uc zf`Hs0Wz33Rf7Zyn+K8)%%rhokU?E9abLi~dg&)sd#AkbeBS9tU-bUTViHJrkaH=4O zf;D+Ca8?1I??R{zq7FnUvin4j9?pBFX5SvPs(G2z5i*TK1jwU})G@-YO7+nGx)-xx z^Ed!H#D4xV_SNz2Z{&CM)CEf1Jss?ApA7|W0WvE?fGGuM2M#=BUk#Zf4C$ikXr3~z zMKDb-OQGZS1Un?_s38wxI?AYFw}|9S#ePI_80HNEBw);*2G&X`&v7JoqeDZW)6Kzi z3n12K3<{hf%9Kqwi0=2o0dH7wRPp#Ka|?B{lyuIdZk&OOx3w{;zQNcVkgwn|DH1uO zd8@}p-Bk}TZvrLcCNNdkKi{5vpU&~`gn+H{%|K971O3*!qzA+hFRy^Ul!T;$5>!A# z6}WCWSCAJVlLQjos5C?e#MB7NEaRI1hX#^4E=n{ouY7k=EP-iSOZ{yuPIEWi(&J;(Qr17aJh?l!#=P|v?7iG4f~0y-ChE5RE$_mF3tiIB z5I&8_r+KuNF0TAEaGpC0}0kkh^j2*cP;-*p9HX zcGO)8jksOH`%)Z7Nxvva`g#lzb=oMOHT4f!1-PX=7*m4-%Bb!cD>P;=$Ez0CRi49; z1yB)ePK+vaMp4%TyDtx^Y{K-5oU8oQtTwhr?CN3-DlY)BA{qtCnH+}Uq33dho6<-@ z1(+#`u9V(%f4$;%=W0l5N*5rRK0VGj;0z)Q0J*kJ#m?(rFs+q9S3YkY zfm8wNY;DvaDz;n{ZjWziYfBOR8G1HAgoG&NWTDN+YXXwrH9ZLXaF)03<@H32xlxEy z;dX{hFcD)6w55mRtD-f2c}3TV))!4u#b<5EjZd*;1vIGW*byKDTns7j``-1;Gl|pU zXk2^x)zy(+8L?{DN1`6vn)#&KnFzoGT*vNxn}H$ToiAo05R{=B*Mgy@)?DIw@qpqX zE5)@hnYTJ=ciFqBg%Qdk5LL}g-~;)HMU6&wrxHrgizi!+?kxajdJiB~-oH5q6(k9( zO8H#XiTFW5EPWV`Ol;s5@HQ=+N3+I6qRaOX$w*&089wD;RX{(Mb3Qntevf~0x9D~T z5rUEfG^gDZMB2tROx~V&&6hs`*ivIgl6$PR@8%XZNozAklp^Efn~AXp7+E#!ofkzr z%&HPBf82VM;q*(k$tDTT90jUB`6|wa-i`( z+=2kM8EkWP6v{iDO`Vg?`^plcVWxOA3iL3=~@e?lW|Z)^>&LSzeAboGvfLuZra|^- zJ`kqH^)y(o^1*r|;ptypeZa@->_AogaK4PJ4AA@;U8ADR?SC`sQTvFVP!jv1s-7mE zroQ;dYuT7GvS*evWtk08;x^y}<&U$J*}jeiNM27>9*D#4KYAcdL7hMLrDQB?$V#ke z`c_Jd&qJEUavE@jN4zCX4%rqe`Q$d8jk69>l+exJH2{l$*8q&z^iQhb151+~0OjBH zf`YIaT{fZsUCwh+z1y1hPu<+O#Cu5YsE0ZzTQJ%6O_GpyZcFmu98EVA#G^@CrNr~Cla5d14eiuyxr_wtw)*7!z6R==`t z**&F#-*dNuNK52M6Qf~y7?-F|kz^0EREShIzzKXu2|Kbz1+fkLR|viCs^^-u3Y z0fh8vQJ8dI7|i5Z!USq|iBEj_?f9W2!Z1V8S_Q>B7Eo z$i9i~ki6xP@)-!Lf{8ME4xr#&2+vdbX2SVxsbbKwirvDMeI3go+14Rti=AUJaP+MQ z|LfQJltV6L3=AmE>IiM+l|t%9-7N~MR(j2oWHO;kIO9Q_X@b?06z$8Cgy@MKNnc6S8YW8$;bT@ru(-y zJ%L~6qJIvc|AV(I0zg6fCrt96y;Wtg|Cb*9)3yKY2X*{kvGjjy`~Up^P(=H$dL{os z`~O^`t@L}i|FbS2u420LtLX0kw$cBOe^@*pT5_t_&0+de)%!nu6qn@h_~QTU-M@b7 z>;C^@?~Q(8zW%lG|6EY~`2ULHe90-Y~5IR3-)zQ&0;z(xHr2c!w zX!peR&`3x~=wF{~*UsY9!u9I!0N@eI7W6kiI-qoY2|U5@wMkR|svZgub&XEAqEt|iO-k*qZ~_`bFJ5AMDepGR5Nh+7unfd_ATME!m+zy4ws z#U6j&5!I}#-*Rt)D=cZ?o~!o0EFDbkW@lrgJ(^sAQ8nd2Di5~j)y#I&*wF9+o4J_p zzwbrJ*qIOJ1t`jx|6iL!+mvQmJkKp>jZtb2nsffUIm_KGK3NGdV_e1m_5U7nFzw2? zQcJ{SvxtD-#q!{B#f#j(2koD>2Z@#Zte#3LEq1y6w+Q@a?<>E-rGE?MzrX)qMf3NH z`nUK014RS0{+0jnj-YVF`Ss$dx0!ob*4qP> zghg3uIs@7RURf`THFK<*aghkhDFV>GJv>PjSQ_8@>*s}WWrrO)O(0EWPfJ=WWz6^e z)MSNkZyR>6cFZCF_6qnUhUfZk7lA_WA#@x6epTEZxnWQh{$E}+1RwHI{^c47M2eA1 z3GH96daG)Yh4ZghQ56h|lmF{gqRN@C`TzB*|2ts+4AKAZ2TZ>fQ}im4MJKm2lE}Iv z6uYpnP+3AEw84G1<<|1$hs2`w+ri%pK+4YK?C-zR@K*Tz`HY?U+Ux#Ai*F-hqZc|`iq~%TI%4ex z_q06So9ExIg1IawXz-k0-oAbN1HWrwu3SoPZ*Q+x_vLA~$Y}cYA^p{sxrnvX)!+B7 za{q6i1BoNm68?QnKrZ_uhjtbQ#pxMPgPH^xG#mbsI084XZ7ZEF^1cy{g#@B z2EEPhczO2u$u44uiw#=!W2L@MoJ=_n6l7(I-CUovni6-V`5OPzZM=W&wvbVSo@k-> zMnRs-h?)j%a3g)1N7jtrg>6D&;zk&y-=KNAZ*FPpwKf>)$vVgEH2-VQRIiPK7cw%| zdzZAdwERE6bKRoSi8y{YwvDVj0k{Z{;Ir4GS|?{dUdI#WuHa|);Jtpc*LqUxWsct& z)S&Rwr%zw3r+iPQC#x*uD`!9w0bt>q0H_N*zRAyUzxlMcV%`b{;X9|A~ zh!~{uyIS?uqW(Q)jiUcHW#go|@Y|m%BH3Cr14Y}k{O3c6+(>suM-3ZgWqDQQ;#YtIHsKkgrZ6?{yZq)IGCC8@6SsT|J_5&crCJa;>Ps7 zS4pR_ef!Cx| z^ZdCvB#!)V>4rdhqW-&w8rpkKvr?%EU7s#uY`Q)hS=pPkE~Wi^b3WE;H|&@#8KZu4 zeO2vsWF`FrsDLWNFUV2Ee`QTri@&pl7v|?HdU|?_J-%CInyDxS7H7Ygl2WXXaJu*H zYR+6sMtXWlK0dzQZ{OY;5!)r*04eKv8<^FgnwT+6AJ*yJhtRN8_Q*oKNdgPUm znp%uQ&i`0QXnNZT(f2AN>l(Co>)a!qyYW3uRFHAR{Uc4!%015#P3!gUmyLZp(b0Ev zt8sF62G8tl`lWZG2qS{49X9W7B#tOea|$qabo*r1yrJ7*9S34R<9niKepS|i1DpF2 zlyXEuKtS+;*IvK9W-&&jMX8%4G}CdhYyCBSnnTUiN$U;moUK`+={lH~AxTQ#PXodi zvS6uzLEvJ7*AU=@rMi}m$)Hb&ogc1egD+Z6)j5)IS={pJj?7JqJQmNf4ClwMy~mXC(3p#(7fy48RwG|_Tr zs{VZT<~pxfr?%nOtqyd?t}k|XO-_8z4;>rH5UsQQF5*947^U{~O!iqb2W*&$Q}0Y` zy>~!X#ZSGVqKTqb=2pNbu#^}+_?Bpvi&6R>#u74XW&my0sCR$xGZ=XVpy0U7z7vzX zPdT0s#rV;{mVTBYZtYf_(IC6oi&<47I7v;#E2J`QOHUx$4j42c4rNQybdi6_rYkph_y9Z3}vk_v*m<{Jrn|i;I$aV z`;i{~zE5WJLW>>w=2+<%=YU|gd77sL=+}^CUTUs$KVMj@KcW#w+FEK4r7LW75@O3* z9y1Ny^hTKWD%Nxt1v&Fu+)U$QNio&(+A8nna$b=I5+VHvDtXZ+ zw3Q~8!n0n#ng#>wJ`bU%mf{sg&sNh11`vy&t-s|)U%Qg7!3M5OMKv-kl}MXxR%O{R z%hsu{aJWHxwV_dxfC;gC1XZvFGE|V4_JgUkDa#gK^XL9kUWn}&%-8yS80Bcj(;$}_ z&sFY*dO~pXfMLQK8{2|tGYM{7nb`!@b!ht?tr`{>`_jq#jfCV};R!HRZ?4zV7d6aV z2bO!0GLB-G^LXGjN0&bbth?SkkY9AE@Tt<68Y|8&D^uh&`#$*Q0q1az43XS>_NPe( ze)WPBB$NmxK0Yzpt1LbA=8;86bwQRCX)RF?`zJ0?GmX!4BY8x6x{|-Z*o&}qJC00l zSZ@#Xp8oUAnxu$RA`z=@K4R#dSNlgATsKu!RI=@+8!YGA0!c)GA1r)fv%I>>>k*YB zlSKEK(q-_eTw(dnX~Swq8b*j%!+a>|A?P;0!d))8c!PHqNmw0H=*0B{I^E|-o31xk zCqWzG0u?tox)R; z=u*qeLo4h1$ViqKZVbGBv{tgu8OmsMKscj zKLsq6TUqViVM0oPT>4AG%AMQswDg{zFhd$Fm8c={me{;=0+V8HgJ@>FIS9dFRO4 z4Feb%84(87#(G}*jOFEIwRN@IwszkKZW-Qbmw3OMi&^W9;evxRx2sR{Kn!U=pRB|3 zq`61QTft?wk&(`>uIWL$fWiFqv$dIJAB`1GwBN39(Ed)I8$QAJGV!)EaN^IPOmUS z-aY~gPvvvqxxj7{^=oFz?Ys59n^upZ>3GLzhbNPdn?K9;%i2OCFn{)~ zd1-06%-S8$Lo9axe7mT$h0$_M%Wv9@Kvd^x;%vpsz7HG5@EW_uZJxgyG(D-Qmc8L1 zemih2NXb?@k!7O7^xIoYsYKf%s(Z>Bt?%ZK@-@^O8lR&C$o#o2CR-Dg<=D?%AUtVr zcWcR@6(-v&fJV%$FOG(!0Ac${DL%N9U)La(kQwmFBGex0mb*;KK?Bd9rtAyswPLi) zNc#$Q|5pi28g!piatmUqC9US$L$E1yfK3Rxtdj13U9S%eDzG~a3)sPHY%w;Qy-)0v zqoy0(lg2h(>Cm(jbF#9sB!n{Yj#4;GBVeLJ!fPZ?95TA1Nb~+O?#h>pp#Zxw75cMA zL#;%FI^Eg#uwWd1!wgk6HO(0<(JPv$G`Al8tOcwomG<|Ruv`WqAw42JmslA!#L}1I zkzwjpRfdZSy@W@6<#nmb38#sni80Ej!GAY>c%H1tv{uurIqY2b~kD= zAf9DX9p=S^ZZ3af^f{V?{sFf|p|t7XMe|re ze!i6EKyR;=a%!m2V`^&Z%+ZF`DLf!PDo>w2jgKxM)eIJU7$TP0T$00GtW_yN>9ZI3 zi5#z*lY_(MWZHeRTjaXDX@>j;-xeYUCuL0Y<1mKZ@xa?jB_(N zv$V8qSd12MIUJIVU(VK}stv2IiSciaAfdqfks1UC@U_9zaKz*d%29euqJA@P0lD{Bx`59iX>tvSkecQ*we7^ZE>~ypwOP4 zp~1{&+%9X1dd_{^=U@yo5nJDn=o6%Lp9ATr2b^8YFW@yM-q^@nEIXD}^iu;C_H}vl zp&`$gE4J1yFC3yY>Qpub@naUMgz>Wfr7a+^%neMLVCGpaIpDfJ z2Tt50U?8&`TqP#N#%3a+pjxyAAT`Xr!bI5cdfwaBg#`t%_*WPHo5i&*z>g`^DAi}x zfM3(VHt{oZ9MZF!7bzqe=Y%Eoppm>1tH{z|Sv3f`F~j4G-p8?wh>LA}OiLZslJ!0D zh-ZWPAX8u0Q9+o1kdaR-Rfo_3;j@n>!xXimhSEb%JOqL73rlsMDWHyi@(S!f$o0NX#6a25tmmfK#hGeF zh(Ed$*5L(4mdkBRGpWd0z#}#CIQZhhHueb@LfYzbWIXF-r6s1UpTHMihn|I&8++3{ zf-fr7ZPWEMu%hBe&jEJF;5!BtF273; zU>fQyT@>4oml-vzrgXMLRW9C+yj1at;OZknP+u1|xvf!ok6A6QOv8#yresH&=pZtMe4-evlFdeckJ zR8})ho^e?UUGF}DI8Yu5he>z}Es{kZo&3kOOTX8*H_B(9BGj!ie9}E%VG?7}lh!D}!8zCfcO6wG2*6bx6AY~D;7GFgDw!oG&%#{CXhY<))$ z!yC1Qz2Q+zqUi`P>5i%WJj#d-iMylaXBDw*QLelF-Z**)!~v$6F+>HxGgw_d`qdc$U6HW40AIEi*8vxPfJ!+n{w_X4HWgdtMpj zL3wh&(|P{=uXwcEr_735m4CW&>>!HUxiYc4R)IHbEa}>?^(mRc6Z?FNhSQI~*hWL4 zsA;bqlho#`(T7|L-$SL<+l@3VIyJJu0=M1c|6*9yQ$0F7ymkE1;v?94t%0Ez=XR?J z^xL`o5oCM%{a?m3+o!3OhF@L@ozMOX-EBiZC`P@s{jq2~HNNx|UW50cNJiQGu7cX_ zw;#Z^ukiF~U?Q{DBTLB=9;CGu>T|Z7RQUO`OC=3r5_;ODY(BlzOvJao zWe6AGma9F<0JpC~19o^GSiP7;tnaF_F~n^8nIT&&FfMN3;+bn@{R%(_oZDhcxQU-t z8)W}Huuj@0-`0G2A#iT7iS%pS(l$1%e6P<|BQyA|XMOfAAWXx%{vH@^8^Gan8x?R_%ji!P*aaGF z8H5QEf%lzfMjXoTzVkWrSy*qx1(k}8$Y7dddTkomz~n*TwABUD8jGB}YhpXT&aU>w8Y4;Bf>BLc_6N~93@tx-`DlqZ#Ibv`ikByN2(91_I`zVR z#xw8dY&lq~o1?`fJT~v)In@|V{m;GIuihMo{#GHnxUauSDI_VWPUc!b-- zm2--^K@o!`J)_@DlF&f|7n;s|hBj7$o#9IbRcL1R2X-@uoJy``FFO{96RnRM$0T$w zI(F^NS|heQoiWF5@;u zu{z#bejpKWW7ZN@#yRoad;t2!@4E5n#}DPzUEk>j*TZp3vz|l#+ZGH{s}{R;-H6Qr z*IwE%@`s@C^LB0y>}Ro)%mx+KH?wXPK9xMsS>95|p9j;bv%dP8nrkng7f1RJ6*E&) zKLeHkA|{&{AD{j3St!>zSvQ5Zjab`7Cy~JxSFRngj+WNMCw^PFU2H(DPOaU>Xfa$r zwPW>X&+1RaR+m#)$T*+$9XRzSA98=*igp+Yq6oESO1x_=@7*}|ysskpqpx`3|!q@6;!|A2my$LV6}2be?<$CUxX;Oxf= z3|QL@_dNLjvYA!!L*qC&K1D>VRdiE#9=ztq9;TJfWl(7KJK#E)k06UCTtn-1KilM9P{mcCT_#I#}xcBzB>XWjvmsr8##4OI32 zgRJ*}%Xy98$HT$l6o+J#vPvRlG-w}*QfZe;b*QupEo}}$M2aU$LrXiYCvAkZwa3#y zDs7FYwEx%reSJRTp<+sB?E1d7<#Z~ZI_1Q!A^MwP zmaRcRvYNAdq37JUp6eoR+){f2qsGS_0{1TQJB8~4#3R7yF}H6?jBPpCyQ`>E&CDnW z#ga{}gWX`1P0<09Q+)%9wFg6TasjHL=n?IZ`)PHh|a5dWv-%&HO_wG?B z{oz_yKOA7O`4G1YbBWdF(arulN`#ml=`#*8#Y#F>E10s7nK!x)FkM`R8-t&BG`;Ly z(x=9*JnkARJHRdLt@QK|i$?WCZ?%-Tu2kW~U{mdTpXxJ0YR+%|TFLq7rJ;sKZB*y7 zYcg+LFLYPhJ>$IiddaGOMGC(~jW9*PAcr>goPH1S_=F8>`tH75&naDvH;~TczY{-{NwO9UUEg^!RZ!csh&G zZza=^pak`b%eVJuIWJ6lE%dbVhIvlM(YnYb$nQYZH`S7ROJ#m=QnGfg?;D-3Xnaqb z(BpHibtU~8RStz|5{4`v%P+^BXOdrIGG)FxZT8zsG|_GITl;3x$zkYaw>9gvH7d+s zB)mT7{9OBJW=_2^Sv)BEMf9L~a(kwg1Igc4ADy5)h(T&tlk~7l^gWo9aP)~f*H^Hp zqnX6FcKH7MRv-GvdL+e;^Nq@Evfj)O&uuQPsL(pF*JQI2osNA?Q#ZSC?6h#70uNpVaW{;$7L@by(2L?w}wcV6FyJwI1(I zd;uWLj}on9l;h5ok{5GQ>VeMF0tZL=GR>gkgdtr{gE%(jB4A?&9KL_LLC4t^>HP^X zmM779sMY1g!Rss6QqaYR`cN-au-mlf=}NkZ4E3%c?cLkybGJAfqS&ilkV6OB*i$0@tQIVWYXR4;h9 z5GZ}0cH$?}PN!E%=`Vojd=wB+JCq!@y8TKgi-LNVZJWcpbthk)Wg(ZQss3l8n(mQ| zTBo7|kGkmb1=_uHhYmfqb=504!X$ql%XivjYVbNuH;2}!vV4&xJiDpmxPd>jZ0X0_ z^UXwmAb)Q0_u`^ajBPSF0>WB%>q!&NF&H{imKKu1Ncc+#lCK(=n! zu=1X^*0eF#1*2X2<4X}M6}ib`zC?qEA^1@Ku}t(Whj(Ib#@uG)oMz|^(pS?cU^ zdUv-u(QGPRt3o9m%MTWR9G>hi6D+JlZsApM9eKY=OH(rnw??D=!tCc@a&vL4tJg}; zZdVZNjS4_kFJmJmv^>wg_`N~y<*$`?Pckv-N=FSESFKah9an z{1}9>{A$PGSkVUtW3;Rv~0$`>T*Mok~Y`ECH*!1nuZXkE5h zjCEI_l$`F|wh*hjCo0%BCr;N&etGH&K7S#L8tzT|$k({qtzpDU_Z}}*vb%PQDSU8y zS#c-;<~8MIXbv_0J15!{k*Oy*3&HtoGMEjY=Z2$n5d-i=$LAcS5j4`^$`@di9BXc& z?=Qb$6M4cn-gzKOAz=N|_7|b9@oLZOKqkk6m=f;?PT6zER^dEQtDG*1eqWoY*Svd( z$3(MUmTd@kF9EhaNA4PZ|53Eo?)_?7;lO}33#XW7(XF+OXoWDKB;+#sO;V}@VZ-mw zWJ7=%(_>)^i5VH3Dr{a@0mnDXH|_cJ_wV_W?X}>Ts;vWFwiS5B-wtD{(?$lk8gIvh3E%8uE(m4Bp;v!$j1boT|oIMfI4P|a(NWU7P>DvafZ`vAT`O* zN`GTnn?mDQ80l;@$4%u*W^JHxEuG52)8<~pW~{fI2ke%J&oMj*Du2;~1Z zR7h>oHMnpMlez52(Q4=dXsE2Hy#+Hv_6-XYS%o}8s)11%8A5HydPI4^Bex(~Y(_Uf z15R9O_J^!OwUqXsbN3XW&54Eaj`C?D`Z;&&#z;y%EQvjo@YE)h~DmzKNLt%N7g9GsYc)F3UZAM}~s zSfdIXL>PK6|Gvg;QU2s$X?fJT9k{<&8`9%q^t7}yzWA8Z;+X-ZFvo8{6B+Th>fw`)CaR|+H`+SnBSQAF_b^{!xhS5xj#PTAD#>lb03@i+h2L_Os@_y#Cz zkIyFke$6MoHm_XXzt86z;Uh;dMh~-JW%|-JJ5@ zu=?isQ0{OaK*-wf)scj1pPKVDH8F`LawkAD^sK_U{a*?atSX-!J;c20iy`Hkb;1nL zc;Q@$MnY23Y_fU*3#p|T=05`Ike4UHvgOe6#&oXPzFUu~PfbmYy5=8ZmPLmx&S^)Z zUZ{K)R}#Q$^(p+XzE97u&w3UTxj}%6lrHoLiGV{R`{Ux`s;{q=iZ#A}^n^jKlNN+s z6K&2$AwcY>5q?81%?zZgL=N>l0lHo0GG<#7r9i)r8a+tLBgb=oEX)e1a%p8HrLkWN zh{6QTPTF^yv>{;$p+#B{o}M8&hDkwBZHtGp5$7FqMP(VDFMjb!w=MY$t{{f5;@<_VV5EyZrLy(0(`^-F(k;|GvAn zYk!OAgtl?}`-feT(SBW7DkIr${L4BX7kJEA;)GPX+J76OcQs1S{7Unt@EP!hGN1>cJB0P+m%vWrf!!GX&|L|UCL4N)^$;nnC?Krw{0g4zgr-9fv z*H`vu)ShN+N$eWYUN^#}U8Ve)YU7cXdDdT(^S-%;mg;-q9a=Yctv;UEvZ8;7P5kP3 zu{@k0PwgOmo@FxXbl~AV|3r)BcW*KPJ3dZs%(KbvJje z)4FM?xiL}mKw2kVV>0oyfYx2fHz;NaGTr9V6Ea=2ZT_=$u&@V`Q@=EG+Y{YfrzFpb^sqSN;`@Cag@q`X zGHu($9_d(RVNd z>4xqjazqwVVVJ@V0$txWTMlV%^0YAW>f=X`AnTBGh0~J*Vq`CVf6SJ5Ul!ZQ^K-Mc z%Fd?a->WM}XQ#%rpePt0m-JeiO>0c)$9}ISksJ#wdSs$MrLX}<2}No|$b?PP%fC28 ztUsY$iXqHuO2KqIn5^)&P`lomXtAzG8qe;uA573rs)^T17)Rjit5FoJhd^d>w}*=V z_;i1YqL24C-VleH(=&lU7Rk{uST5iZ=oFFEm-CrIweMUs0P)Wui%x9$Ijzi@8VZ%awGNN;ay^#Gge9|gLzzd_&5 zrXh&SYxa_XOV{Jv{NnS5o>QiTPppNk#Hl6ga@VOOJZu51&AroNz1+%SQU;u24o!no zB%b^s@apdu7x+m{>(8S&4)Ian^IKdsuP${4jZ0$1NagD<1Oeq>;ljMQuM6^?17KK) zGE820pdtZ9Sxa76x-aEBl6tK-3yfnJib$bN5TPh5L>ubsNA0h#UeUY!nyNz>cNxXH z83>FJcNtad&)?$`jL@Aw3Tt+^&teKsLRUwU$FI}Sf1cD$B;p7aCOwWOx(8-CBRg8^ zLPChQ-A6y`NT)(03$BzLGbe}43qmYm&n`BDSxPA{S2FOR*P~XhoIn{>C!VV2Khfdk zl|S-<1GM?yk;wYtPm- z{rAO(Q~|hSNkcHDanXUpfGICw|O9?Bmt3lwbJ zacgVMg_Lb}nD3Z|>NN#?8zcp!Vn}38?F;dv7nJl}GWD+=^Qvun6zQOy+n3|a-#K)f^ z;o|H2yINDe6#~ zyO1dKf>onMUtYF*EqHjKQ`pPT|EObF2YNII+;uBxi+w|14=~E3&WBwAjLyq^`In0s zx~?uR;|Vs3Biq`u2F;f_KH$wUoXPovNrv*E`y>UJ@azXYs`XR;GXD)W?d!tghEKVR z7#Ybv$R9yJ+K*BpfSUlRP^V=;bC?N5#f^@G{tl>G`f)=j2I9lJkXUw|xTiYYp3ipY z_&894Ug%8y&{E^c+zD!tCwdWGnoTdvqd)DtceqmZ)x-{|7>hofKq3Jn=z_~wH|hCu zzDL{*YUMV?M%%k*j~};kkdVcTIV1Oxo2SJGg?d`HY}q0<`f*n#iU3F^4Z6CzrjZTD zD?~eDA+1zsB!bwh-Q%(@ne*vX)dcFv;G{Fuq==x4gJ~P?hXo7juhSo8ufEVHCwz`R z`lPgq@G#=fF=k|00}0a;|m{?J_q$S`j2+2GzXim86(ym^#enGg^% zO|~_AY8(FwnQb20a8L3WE1mn|dp4c#uR8D75`aX<$U}I?ci%>cdFN0~G)4y+#uK*B zz!<4rFU{mm%;}JLl%`IBtgCJ@Z;3$iRYToW6+G+l+T+W`Gep3U?)mNE$)U^EsB4m6 zeO)~_8dz9|ixsF9a%R^sG)&gF%h$r*OL_a{Wu^3j`+kEQp>qSs|8(?$3a6N_7}MAf zIreGtwhmTD-s1B7d5$#O;`!M>$g)P<6+kia6uy3|d^tocl7gOtUd^cGm>l0;N}=$p3MjqR$Y5 z2p}Z%!W1g8{!N8HZ=z6@nsP5V;blaG#0Wav_0mi5#DFlTw{6$gh{=8nbM(VVKhH*(Eah zXcNwU9MvyIB6mb9*_fLGNNBpk&NEVa_RHE;E)E8Zu%G})28Pl$Y5P{n(rf0C4t%NC zax8Y+PP#_+10dmaI+GB%nyKVA|F(*o{DqD}I9VIYDEsXxxsLt23p+YKA0(VMGo${LuuRI6S!(iZ{Z@Bcq`97b0 z$xU%CB@uFW$L|`=9wUhi|L#yB-1r?DzvkW?IXO9^lX$kf`_|UWw;70+_g?5Q(dJff zoBy$rR5Z%d1*Fe4%Pwf0!>x8s_JDrA59(jl7pFs<*f#;9aGMyoF-CHcj8)TFt;ohk zw%-1)r2x;)l|8+_?evr2Z8q^-w4sEM1iNPSl^vD#&zZK@qm0c)8M}rYp2E~$XqfW* zdYTsNsuC)Xc=ELzMDU0$p4PdLVc6^`3jQEVZ`m9a%X??pnjJvG*2_1=dK zxmE)rXELb)zOi8@qL7dL`$;aB<7CfwSB0fOV~I1R2@Opt@()<<=eVAfrv%t=Ld~$}q&U&@(nC{F#=GM5ko{(bI-Q_);|P@cP}f{| z`--5(E*kF+Fq5+a6?=QHI~i*rfCs^1a@|>zrR?%XywfEveSH@);e{Tca&*~RiI-&^ zW8UY?p>6>Ac_uQ2K&ea+C9=iZ#H@D{iP-@*01Z}5L_o%1%N!Tq#CIj8*8+ZBkw^Wy za#J8Y&Fb*Xokue5M_ti~u-wPl!^kH6JE0sQip)=|OUt6q6qTPTa!?q(0?oku9bxUP z#P1WrR%A%)63)`rkE3mkP5gy?U+pJjOom1f8#7)`%;Sz_&(x;DAkAj+TnlRMb4M;&?Qtt2#+I;4Maq$tB`9 zsBU@g9aPo;o)dEHxu0#`$cNn?Myaimf`Tf@RO6>-jI-TbU2`y9LoNFthUpa6-AbQO zwFEhMkQ>va!6SSwKf6FajO|fbyqTWIc6D*lLMI~!+XBm;ZrbWb5%nzV6652+3B0s^ z8z%XUsAEAK3>h2Fho1YNQqBLh)6K+}QRe9md@V9Z)`{aiRsj0@rmXH$ecVfv19UZ;A-!SjY1F7j)(m$BZkmjrxX~c?R z^~`I2iYTHA!8Gtj+gHB4Qw;;Y1;A0nBpr?4X4PJp*g=SZ1fE4b> zoe#Bz`0_k*@6+8|EMC-e41L%NDU=wxX*${#`>aOaO@CaOK{t-$Jdhzhk)fhcA@Y>4 za!lv7wX{CQvsfGFS68&G=<|PeqHi7T{*WklVR7f_RHjcKu-)PiZUxJ#(5pfn9*?{7 z7x)o1PtO9hnk4-!|6}9#Z3{ndVU>9lNMUmN$d}&$u$s{?Wc=S)93E?aa?F9ZRW!1q zHu}3x?Y;Y{gWoT2|9m=$$?qGsqb52J0(Sl^cUL(ynbXsx4zZqxX!{jea1wsp zWCSYLI-ILE^5j}Zss<~cUp&Ea>Tt!z6i%iVF;sbMfVj#>+ZcG#|85DszIHO3&gYywQ1^3d$4e{L z`4%p5skQsFY3!coWjIKhCtOPwsPUmhLZCGod3i)uaRBZ?`$(vPmNe75R1~Kdj|>bD z`ZhfeUF7b@{C#%$bV|B=<#2uVu53gOVs|`? z6P11zTBQpqMvWuI+fq)4hz&ZGC+eneU|zwaKq?q)GY*Mi5{(lR`L(pQ`=B0aw^#sx z{+&O}Z77>1)DCo>@0u%{_UnC<$ef4@I34e-6ei{ItP4fI5)OiSWF;eZv7}n-CnW4M z-Mg{-R9YlW*`iqEkX3+Dk)P6*zORZMbcH2MGLeW~fnAdW%^JZXHt|_sP9Q!wV%PQ( zumYc)OaL*EgD7B$`hNZH--=&_L<2tsi(0~Hw=xdUD69X7^sSCIzhXzL_U`q5_=<}* zJRzTun2MCOC--;ZCIbi zU(u!><1D%%VXw9Dk@kDSdTI`r3rrPb23}=P`OpOEm6okmDojkO(rL330bRM0g6)I- zjTrywUaa&z-`BIn`;LTIOj5aN?GHP}AE*gOJoTbveQDH_7XT#MvJS_*aGstrO%8qR zNPT@h(VmVY55{3sPpw$U>8z}ek52|9cnZ}QQ=fkpyHMiKQ+b(R!s>3HcL1bTYS&Da zm3`A&n|SqichdW(2_E_RiHh`Goh~5A6f)^XRcPG9$ESuKGA%8QTLDdvCc;hn0;G}F zKei%$M~!X!i{*~w4h{d^%0ZgLPYzcek;OY?M47*yK@Vw(%|QSfR3?QhQWywaZd&&4 zaLau=3oYVXd_F$fpeFR0g@os?OAR&Db1eF<4s6XjwSsB8DXNcWB(FSdst-@JN_&!fKv zyvyXY<3Yi&Me-#`M=SQkwTr4!wF#=+cq&vqWxt}o8qJ8e!L-v|wP|Em3kp$JAF->tgRS=0s?!x6C6n@)Rs`+6vMP!)?DnLr^DxM9wXUDDMd-}u9W zZ7KVbUmaU3^)sBi7hF(dPneg0LLUkO&(VM|^SW3yF-+Um=Q2Q(~kc&)#p;z8qLhrObjAgaH(Wm%X!o!)7E*%v!c|0jl@gG{@<9w zIiU}zqomKq+^#C`OdoHH6R)gjokJIBIV#*YZ{2A>37A);Z>uk-j-FmTf}Xs;12R1j z+N}0V$W4(%YL83^X(}lL51Um!;%0$cd+}ELdqkmshm{v2iZ9d_8{Q8uZS)mSC5(q< z-fYpVe|5|1ec5=>UKVz2B*P-rrl-M$#`hWzKrSHvPQ=0 z@^R`g0R4(kNqtg6lTCdu7oi0T^LuTf{b)Cx-?oAGBK(poKs6F!Y&r@I(C1QZ%BVIF zgR~s2^jX;6V2H5;TEG2`B44i@=d@lxL2suUlJq$oTLIEM>&nZ^6LFi%VIO-}(OqDM zj__Gmd3pJXuw5Jf9IouH2qx6YDSslgZIe1jM@gLvcDMe9s9n_es&9{V46E66&OXPD z?<3g(Cyw2KmjvHBYW&{mpkk9ZZwX_RU&CShZF8moC?Z=SN zhe7@}gkd>~3S$&D{3aWlTp_#nqhO9X%gz3c`<`BKXcmg>PxUC~+BfAs)W8B7_BbUh zcE8zwDJ3(%HGineN#*k72a_THab%1=8&)fj=$JtpLuk<_zUcLp657|IQ~XGmW?U>}*~m7S zYH!{+Kl+5L(egv+myk{2oZyjep`0Z1C*m~p_-O`Ga+e|VL0v!3qX6Y3nR_HS1F#$g zJYq}}ZJk6YXW^&d{*osnUuDx;ThH<;*_MU@SQ2(KNQx%PLJW$h7dQ>Jtatz%c$JEi z#OJHaS_bHo$7dz&fvFExOR`Frh|TkbLC@iv9En`><{)W`ti`9xpSCG8Jva&goKgD> zwq!B}uR%$Y@pnc&r(@_MheEl2w5yE6U>{=(pv(4*Z5e6d`K8&CTW0j&+(&)!bF>4@ zPyJn^W*n^zbs8;^W*}X*4D$iD&Jc@0r06 zy6}`4X>@Lk4Jr(^KPflxUu@c+w@U8cuT1zhQbow!-PlXEc_k=w3?A~ zgszyXwUYiVts-Y8n69T*FqyI+sv>B?5DJ(TxMW8gDyxQ{bk}=kZ@;w!*YcHOXs7BlH?OvI z<9_sVKUN#!FfkX8(gTvGM3qgSa5;`n*Bx$4xt=cmQHVKkFBaSp3m*7$RsKl8z0Pb# z@fh=ONBnYApQm5baA1y;FSL&{(U5Ue)aN=q9vzKpX#|TOK1E*p!DPyp%=45bdhGGF zq3LCqt6|A|hDM%xSXq>`N*4SzJCtZtSn%c`A|iBCRPJ6s*)?i_-q{^(iw=xOZkviG^*BmdG~GGvDmjV z_)Vv4y2h?_2}_Q;ayA z?HzHXTlDfoxa|QVVi#bZD0{VY}(Z} z+Qnlo<*n{KH0Tr4?gUcbG5pW+KiDNM01>VX-j%D4aXVB1GX6s6-~En~!SsRE07`0G zNSlZeKqVFEkczOdu$%j7+!3b1j`;X#UKbAU=lm%;A+_`F6?nH63gb$~P|lo^wtf zHJ{T>CZ;H6+`#H5{DrBYsP9XR%A4iKWzV3Kn&_K$I}cF4@-lc_bhwu&1VcR`EIkD? z1dgp^5Ff<>y&`g^D9L0MpN60K^ZvJg2s%&+pYP_#`2KN`V%EKd{XPz3I#%P=lta3? zVJ9+*m5M%ntZcS<9ZClPq7itP{F%1an5{rj-+b?$jmbl-w_HDQ` z@Jo|cIx+P)fb!fr%&3)Zsli3E>+#V%PRd)V#DgV zSFc{_R{ntn4ENA8Flhc}yuzdg4nJ2Lj#GvUFoI|SOs{e}RxR>hkSZ7>ls=2pVt|&G z+=9@1L-!vWz^2{f@9^-g7!BSjCH|nQu0P##&h^E}$u?vCG6W3`vnk-o+S~tZN*E(L zR+3b?`9MnnIBXsK`QhHfD1?|IUE4dY9uSgOTwdH%I!$T9X7+Kmy01>y+3nGO8J+V& zk-ct*Y$IpO$-?%+7dfU;nN4W_Wr_fY73a-T-sH{h`PHY%UviWScu+FdlOe-n&O-XK zr)Z*k0qtIFSU^J+8y>#>T=~@{%bs;Xx1jjYHT2m!d(1tEy1uo*VF~F1T#q10n3v+t zhuo+R)88n39O-kLHq$?B-E@G3Lqy3RW&8Lo>DhCE#@oP^?l&qp>kq+YBm|gQ6-I?| zqhn+AGNXBQe`Gx}PWU)5P_*w%S(?+d^L7OSM5jxT(oJgJ1RE7H1p(AVjJ}dtxT|yqUn_KL6=bzQ&YKZLWN*aiIEP3 zMRW8$R-5b~# z`&-hr-F-6RjA1iz@#(wSU;+`M?B31;Y}s@8aLZO3aF{{x;DP*r^~0mUz<$!Y8b6Ti zR$6cSk<8KX7h*7#fs182(kW6XD?n$&h8E}@HRxk7|x#z^`s{lS$y={kt;Eb zGhwpXo4Kf|i*;|ndrbQE@)rWqBU*`>qs7vYXZo`9@XF+z8=Ew?vH3C%0sGUcJU+JPAV?!ft7ai36 zpwJMJEkZbHqDyg?>JUgVqfzIP8BY15otNgk^7&fsoF^LM=VYiBiEcumJtac!_^x_%qkhQfFbhA>k^{h*o zKDA)Ch?aH0IfrO}i2O_sZDzi)HLU?tvvKE&ynB%#{kYNap)xqxZ9DIFo4?TFU-W0- z+Rw&j>V7R9IE<)Yx-@R(=;+`kA;rC#N#-tt5}b!*uCvY+CKZF1&k~t3rUsAp7tXW| zlkVomL5OX=eim&b0t+$P*YxQI7zPk!ftasjs7v>jx$zg$^)=Co&MxJIsloP5KpLcr z$Sf2t%t-f0-T-IMAPs@~haLC;{{|@0AlW%}-DMIl%Y>dmp@syiE#9eRsotd#Z6OMon%sl{<@)Fr>?SX&c19t88v3?soeAQW3#X-C__{LJ} zMm*tilE*x+S+JKX+qGJGac_eW6qC6pWQz@GzLE*PD@X)*`Q++HmlqUkg&9$-iJJ*@cR>|E^?@IhYR*8`ZmmBN~rY;ZypGd*5(Aabd(9^1tN{H+NOw7ka^i6tE;GjvRrKft_tZj&} z3pF(Th7C%{RsFX(otjP;w_(yX*In;xQ>rmJTYW^w!`%zd&V*2SMUfF;jOfl>n2 zJm2vr$mhZ9?`LCrS?Ju`8*QPDjLuh-O;>7BiYck8xnYi-n2Pvo@qStR8>FAnt!GPy zAK2RW%I2U@T8}WdAXO{v-vTv{N+5MF` zh#~ORlfCszTt&;RIOSedye+q9iWJm@2AjC}U)D5b=)y7lxv;o|&8Q$DQ9#PnH!kvL{4*+i`V3$$|*Jg>i( zMRU*QRWMXgZR<(!h768FxO9B_arYf+$Iz3|{SW@??$EA2&T0OFRBMuuePbw96RNGD z!Ua!;zJDHic?Gilk??sHFkjFB^fTn*eN2?~Jv9rB%Q|?)12}^Okw-yv5dfhE;5$0R z1-tJ}4cd7H|M>%^Q0T{diE)B0P8;-Vdp0v$&d*K(81&3RKQQZ1d^bkoeoQWW=@eiiJ8h=n@Zs9VuBZUXDQEW5|qV!OyRM z%mFEA!1>*-Ks?$Y)XFY1CS32eWh)zUQ@j@D$?OlI2+hF@`$r*fi0o*hM3KR9FW_@T zjx{hUHljSiad?RYFK@AKa&mG5>5LE_tOFfS<064knP2Z&juYXR1BxnMY`vP=o;`eG zf3>0n@;@gZ!(DbjbHnQR^a$U&l_m@?$t;2vVp#k4U+{4%2rkC7f&0k&AmaDJ zu-GEeyY>~tg752B@Es&RB_vlGKqXInJ`mlI_FupXz&5a$nMlI_7c0 zmpvMzF>1QHgrQ=i>gmP z_5Lv47u)IT*pje&0Hq!sH)<2K2|{CM>ZJRo&Bw_FGBOH*-;A1qt^ZR&;| zh^P;Lt8&URA0QJtgPM)W29C|9fk##9+#q%!R)Pu8V)}D~sKEMZ6XKm{{RsJop&d2o z6G)3s>=A2G*oDKLYEY#7UeDN+kQJna^g0b9H6seG_*bvafUkICoC4ttY9JR^==kVx z1G54Z24Ruv_CHmj`E{KfN_Zaz)a>K#PEi6G@^Z^sDbvG zZS~E%bTbtazXNzx8Xy~N=>j5!Hl2LD2L`AKt1oQVE(2=ce|m==zFAb z`e9X%)%a8!nKYRXU7R$zy1R+0yjSpnh;o(<-}>g%Dk>~UBr|>Wpcz=QEaGr3*g8^D zN$n5m5|xA82ke(%FJwqHNG=KFGf~}O7)=8O4si@%Ler$Y&-ZMkFG34(oAy!U4uc-mf77ae@*d7em3<2csD>4PPdZ ztAz2d&$h!fTp?LH|0uBXdw1_{aB|g%U-=-hjrRP zyI82@A_Nr_zb?^jI=N}|s5JtI zPe&zB1koB_B$-TMdgu=EEj1%HDz#h8o_#qlXR|WJzB=@xbY%XQ#kp&34K1EF$3RW( zjRtFPIS(AJFkeQ0(sb+WT5+WyiEM>H1*%(4QjY@Dr;VBPj(owqAODcVX*LInvW~Py zk4hvTvilz_GlB&u+)4%cT4H$$u6-iglvom}kIzw&dp_P=9oz_W3;@twiPF&4iXjnW z5^3-hwQ>9F_OJiVfQx13L1uDTn=SU$ZfJn`4-_0NJyudA;jV0HxQk+AXeN#vRfq8!;8j)m+!06g_+!gsqPhKM5$9 zKzsIe>2gbsxOXjaoBi`LS!$IPzOr&8E#5)BR7rX9)rfaAN%&WkQk*^85(69U%F(^nLqE`8W?bS$&2CeprJOWdqo z`xW(Jj;SJ-EG7#KMASM&8&d*i9ILNByVo$a8}D<~>FTd}GwpSW4b*sb1q|(S95UJ4 zVq8qM(aoG;$gtD)o6cO34&NV1JUA-0Z~QUsE){a+>m30da)h_nKe{D59g%lj+vcTJ zt=DkjSZE)SM$s<(KvgX@wmbaIv5!?qo&rpd{G%Ep96EYoGly*_I@=bNO*kNyfy# zUH8;`L#nZTccgD!Qj4R)A*#8+{ayf>Kp?ntb3-O zj1nkDRmUjDyW8pV3-iTSFfsjUs_Wk&Xw$)EFmWXUPF*wSB|;%zdbEPYa!!nK$ncvN zOIjiBPJU1(I zEH?LjQ`A6d4DRcZ$-f?ku&!eFt{l#P;2zVf-TUt~$oPmTi8Awv>g>fZ)0iuAR>;*t zk*Wv2iYm^e?p^%RQ*$|xDEP;|XO6W8>W;Q0{39bDvtx=`_CGs9?-vH zU+GBv`#N(Gj?zEaeqYHWH`8O{I)_@)q}1=+lj@CqRBqWrmJ2HWYx<>RxB_k^Q5^xI zGKH@KkleX=20s6cPt=Ug{9>zw6{$R>p|EF9*@15lHG_Lk`G}}5=w`Wd`q!qi?IZnw z-rB|UEW)$?jwgL4(x|KqSJE%_dp)sGoAWFw`pp)}eevokh_+x5k$JVJH`|Cu2*}VW>P}oll83C z%n~Z}Y?my2oIt)l&<_L_k*hM(&*Kc zG}lS=qkUZmwb-Z|ZlA#?#hm*%e9+M(YwKLpN|yiZ{mlheCzNTi$F^Fy;ai72jSgqi z{^$ce!WQoxVZCZWz4dPlCLdaCraxO}!}9M_0j`ReTbO23e_NK@YPZ=tEzi)jzF8{X zUT4ZYnZ$eYBXjv>;ycG8ze9&2yqw;5S~LeO{yAoCm)^a*55}RxmkTX6BP%`{R4lJ3 zp6IKqiT?LUT-DCVyV5BDmyzcV4^(=&jNUq>6i3c;=43fonDaO&{`!!X@U?eW2e~Wd z^j6=#AEF=5{_jV;Jh}5>%r*#Ke*J;W>L{QE-xTc*NuP&NM>Pg=5B z9t-Q~8n%7wygyj2=6x!xVzTt-#;0oJ*$XTmfBrwWqGFYS+^hq10x%(-U#>)EZ_T&{?84|{l1|K;VC-cG*v95OzK&**bE5w9jl5t{3UGzSrnjufoLt0 z;8g}r&8A|rKZ=MY!t4NETEExvx9fiqqee#4MDOP<>_`E@*GKAYGCBD6?OSbzTPTLa zsrA2&+sI|Z{m=ABBF1``zhP%vejSs(rp-?I#dAlp0>q|k^s@EPCohBcu}7N{%0JA* zxIt$%v+m^BDAw1jTf}Le-8!ddz}Y4shsx!ImChT-M1%8rhmbSl%zxwR1+Dnkp9sZLt?-514c$O|*16G}z) zo!3pX(HDLQu(iI0D^Xi6RuBI%V322aAXdY3pN$jmEyRMTASew%5T)mO zRa~LE6o$J&R+*I+CZR6xmiEGAPqN6yDZrAEMDziS>vGZxg=uuM#7W)?^8Zz|EbHb7 z0wY5iQV-TUiI_A(?0oTsJo#QS0p)xGs#@HM3=3fo46gn{0KbfI4{+3HTlKS{a9Hox zHSDLP>v>OK*EKCKki3xT;#0Q3iIq#$U0l9Wn?Se&&g0{K;n=^7X}cWxb$;D>jFCC^ zMhB8^)F^)jMmd^5BGI6oAay)J7f2hAxd)pGJ&?jtMzDv5+k?~F2{;Io#tXG7vEmyC zz|)sER%tbXH^*0we_EJyw(HGH#8q)&C5D}_5J?`)!}pJlc}E$$Ias$HLqD0Yv#eu> zZkjuvOZWR)W5&^J3A=4-VOnxw`L1(XHVju3x1UwTT9xelwwz{KZCJ`~CNY^F07no9aT2_#|d9_VnPc1q7{c`vL z^T_df>iS#jA7UAnNtlmxVH^ zXPzrsp-}Ves68Rgi8UZ_W0~}NHH=$8wbKu9(2e?Ldnl zF%ZAOgVW?P{4Dut4PK3G@!S9O=Wh1r*H1Go^V?B-`#v85hy-6eoc?@Tx9{s@_L-xo z?BLU4bMnytIwSCYro9zzG1N#ZRo$Mmf5I!Xc3=@3>Zq3YP<`RPY2QgjKoXz4m0;No z4~jB@Nw3ioUNLw#WZ)`*{Tjf>1&`FobeC%X;ixz{)FzBY?5RQtoPd@-Tr1fF!rLQX zY=O&+6E{~9719fV&YOieQlx%0SWIiKUtMZ94T%1P^HBFGtY_5Dv{@?8mNW;^4n^{6ChlwU+%5e1 zc@oq0RA(XWz=(bdPrcL;t@rQM_i#T%K!dmaD2)Q#T7|%NBwp>6mL4gBOFHqs%=zlG zk2nbvyKXZ3g!vZ1q60x))5Mt}WJMpIRY1TcR`vue^vfAJ06U_Jc_ z+S^RivW;X6H3!0ZqWz)jU6}?n+zZcte{SNWhrU(*cW0hBY*Zo(#G_W26hOkI65+xB ztcs0bZ3c@~hU8ZNX!9lxt8m*rEkcqL)g8BqbWqnne&;&+n3;vl35w!ls2`x~4q%Mm z|4JhMH%t|-$&4ouiRQiFZVsBQZcR;ek`6F?zT5oLN5=uVk!j(Ag?TrOw$a6f;f7f& z#F(K69#}~w;|Kr%ttjloUgX8m9PT*;1|{=w0R>%MU1D2K7C^!OpEuzUp~PIsW;AV4 z5seKSOVVtbTdHLw#mo#*xL+Lf#27FewNA3=#hkcFj^;sxSmeCXi*AN_v@jxefLtL3 zU`4&2TAtcp`}k>;fmbW~yqTXK+R&e2+>Bx3G#ftio5C+O1VIPpDu(+Ky;bE_mv0Zr z{Cau;Lf57Q1p_=yKouH)gWP$*jsyHEp%S_n2b0h5giMaik%{#a2c`{cZt3SSzN}2A zM|!?etw^%kaq7}c7R~cj?}fNsSY)}1{6v!|d^P7}(p*C92IKQo+uE)vOGK6xAf zlZ^lE<559D-!CUV%;9#CsNUVjRdwGcbuK|C%g_VIWKWv=B$-+gLoobyXJPoc!4U}a zgfI(iHiw~2YYIE@^wiB=JjLPyuO@NQt%NMuG%0)Mm zyToTgB_-r=3h)7O_PFASl{+b?w6fSCE%{#0r3X;yWUEff-E0a9LUfFx$N?Y=ZXXFL;RKi_mzIS~C! zbKFHFj}J4hZgHVWfD9D02X%6t-ezr?WN$sdZ1&aJ+AGMS@AJT#Bz++q=?!V_>%H z4&+rbH7^r6E2=|8nUwe@a@r0=^ry~w@6gi8@(jV!Y)1qW7F+;V?L=wB1V>{Jo?-&} zF(F1-UE5{-d~GBK zjDFFyDIKjp&yvq(%Gl3O@yywK{C>mSQ8z?g0p0)$yM z4BZ8^ceY(F+s|-9lIKnZRM=Qbl2pTD=#SprH3z zp6zdqL;BQbMJ|NM@yZ^?W63K_Opf~2qqw-dqs)1et{8dQc8lXX#ezl@^Pe969>Ez77cBt)RF*dfjm@P!DB0YCIGx|whfh`aA*B{h`# zE3cv%YrH*Q+c(&fnmWn+?7jprwLoUBoR&~R2bvKwjnI<1(%nyt&n9$XVy>YsPJ>4Q zW=JUzyCg$pPYPzKu&OQ22)Mg%~h>E&??O1!Yo_d?PybojYcd&6T@%l_CT6=@8?;ng=X)(QU_q~LkfjWJQaUs{+{ZAL z@}G4OYf(RnWa&gDn^etimeB(?Z-V(tAM zvb|g>u01toFC+mee!A}tj){0kKcj1$J}L*LvqMs-^!&&J;$K51P*4*dJ!dq)cOO%_ z=uK{ME5L76MUwsq60Fu6pM)=fik${|mjq4wjltAT@&36z z59EWdSPX2fDo4Q@bqokI%0u?ygtYZ@_G6VWFy(`&Rrd%O&2JAk#IZQ;nz)~eNDlHN zB=q??&QZM-!aVFjsKh|S|J>%O{{|Wyf`833&|RZlrM@!*R9cIuk0Gd3*rPz@Pt)Bd zYTIfU7BESih?&A2D>}3V6iWYpBL{~IjT0MhQx!`lTpZ4N^6K;ftr*%_X6L5s3D*m9 z60TDBx5#)r)}~k|J9`E7YGtbJYW7wtdSt4fnN;U|fJK9=Nb}aXHNB#@M(k!08&S&n z5k(wS*Q;oF+5-wz!oz#uVJDRn31HBz(`h2mFOL+Z7q1lAEQ>=U+^Y)+gd|u);i&o|o5ALEeBbFka;ERMF==d9HeYtAFaVVqHBEfq{*Oa}3$^4SO$8FiBcfI?sBno`0w1#6QF$g5` zyEu$8)$2ispSYrUytlSk*AsMhCj8$h{Bed<8*wj8j#FT#V*kf)ODiwy>FGt@zZ3Sv z0JD_8j7lr^@*oMg=zuk%3xzxt=0XtRrN&+H+$H>-sK-;vg?tB^|MFb>6m4DH zJ-p@j)S|^3_S#0-BAhFrdn8^~k8;SP7dr9LAg-x=DbDIKra9}+4~0}!*5&Q{En^4B zW&2~Bx6Ek3?}HGQWUys1^T7JSfrzb=z4J)KkF|R!=f2-`$-Vti`Gtu1a!N!!E;i_$aAk}wD9!}LyTK?SkFJq~&Y zr=CQFvARv39Uhjex@&Q*L|KdXcq~u< z@}sWK7i-Fb0z`B=+RB@#2BEO?%`uXXD_cTsJy2PMjCZ6&BA^)$VL$)x5bFxxIW7oM zCVhk2#Z80?3}Zqe`5se?J&UL64^q}%ah`2cm%&VS=RxK?+x&O(;3cmE0%`U!*yOhA z>KhLOCmMGmb4-s899OyPE^uz$j0A&{hON)dT ztY08cAVlH?S;7k!s(byH3n{7eH zX9CMLI^#LmOetc$7AA9mf1U%bCXHSR@cgh|xS%~Yc5x2XD*=6b z8|zKxV{O>AwZ97}X6?A_s|#k84&>&`2~TfloRL=ARwz4I8|o1S-afEHC?|~+p0g!1 zy^+|yb3e7|48#K>G@i>~D-9t6qmSQvKyMZqe7S-Ws>q7Zz7NIRog)~R7i-q$8I^oB zkOs9>91(R!Ax%n5u4k^dM)-emXrt^!R~X;y99M8ne*kkOxRF9Lq11Mr@&mcC;)x`k9lI7L{+~z zJwv}RDbNOLJs}hmwMhA4OL`LCM1bRqBiXCSaDz13S_m{*{w=u8eWNd{tDWT**}8R(={Tt= zg7){X!z%E>&Ijec*=VbxW<4v!qjrvShoT1!FsI1Z4sjBj+D(mpzaCFP^w!R% zW!RZd`rh(B4Uqr{Fh^Qcum)2>7zboY_9fIXJZbJ8=Ey*M&m`m(;ix7c8_t5_*2^@`&^#m96UfA#j5IDzt?Jy`Kgke?0#CUf{L?wqKn7j4#k0?4Sxb42tm& zFP}{bThZ)u@$kG3OR5)Gte^V*#14u04<5mv>tgRg_=^SiZ6nl)ph$amFUskv$dLez>apq_&RD}ZDKsPe^? zwBSPK+NV0yUB3|;Tx1(P7%Sv@y{Yn|26RF}d+Z_F!1ZS`(@^B$*LS^J$`|+%#g8$q zGaR6F>YGfv7Xb)>X8M?Dmc4@p`ANUdGQaR?gHo~ zInbsB`A^r)pd(6F+b4fcs>B%=WRt3))ZT(aa z4}1Y@gL=q0GM~>DK7-Z2rTZ@GI6gYPyI)*yN>~Qd#(Pu8D~{i>FSXcN1}*JKJn=B+)Ag1UGbv z>flrMbdZ*oHgE(i4|>b1N6Ft z4R|x!-{{C~?%QQ@YfK(%i&1sJSmN`FW;i}c5k`F!yd*965``ZCNBb@|qtxy4arA*q(+4<1{zZKyFx+R0jfiCGZ3}m5`fu zJHanQTV1M%ov@js?yG!#dZYj%>r6pUXO=YRLneVdTC-Ccz<#vTpqR{&?gzGXB@lfgk%o%%o`;{RsnH*%g73hl& z8Z;rjBTG=-RQ+6o62A@1JSsoN0Ft2&v{0{Ga*ZF{XB$N)5F_fK-_PyBA|eZ7nK+yZ z7%N22gTbIO_9m$v#9fsiOF<=jEO8T9#?vkwVU_ged;NhpJ^rBT`4Iz%bkM|?8?MiQ z33eLs{{05)=(-3*W%#H!m6RxB zodwSdVXxMF7M4#jxiY>ySm$FF^pIxWKUTzulFfP<^~pKqK_?Jdf`pz7^tWL^>kuHIn1aF zvar&B9-f0dCN(uRiOJB~LF9vu+Qu9+ zrrFp+2yE-S_mNhD*pD3WSl0gC1V5$&80#-Fl`LL-O8C5Y| z!R8f#vSK40aMc?WpAiXN*k;kYfQD!68CwWFcJU1nxsHHi3{N+y{P{ZiZ5T&=0~IJo z7!J^TeFfPt;n_kZwSbk{0oxF2r~_(jpmrJzN=?7vag# zt_8ge7u@Y1BWz;t_DE_8EPvJ%;O;e4lAm!=Gf|K~aXktD2sfc@dWnXHUchVlK^Rmk z9{j}KEj)$h&W}~8fziNIyF(&2uNDh_D5fkjHr-W00`j!|#I^VLD8rlo?vnWi+alf0 zM1VgLPAdJ#eg!bplKsj|$hzI+{=nZv(2dzRICLm+B_Lc6mB<~A=j0X^qk%UlUAv<5c6m^~=&=4^ z+CV`Y><^SfP5jJC7iVVcrT8m*$y+@PIwSI%q7muKgiku&Jr>j5ZMZ&g(EnR`|NME-=hYdO ze@y=lqnhGN@kTwbC+Q!0%9m~M?I7pdhZAw&Kk|GO$f5=LW#hThq5L;qUk3e5M&&%- zQ3;E424oR&I zY!RG`zwsv3su%Wv^KX;ZRkL0>*C$s{SFhw4M?ABA7pKCqd_McBNe{YACA%| zK81zxrtQmWYQEB5G|bIRL$jZrxE9v_f9|p86~!{T-qWu|cB_hJ-k0#diA`;uY1>Ql z{_ItLe~Y;p5StkmuGgG$9ECpF=;HFDW2q3mLQSne0zkNn;I|6Q<$2Hzp9ltr(H1q) zKBS7U(lDV;!170J@f((nB2wk}x38P(c4eH6F{2pOg-r=pnGi!%-r{a&oIgcF6LA<0 zuDPL!cd=s0*`FWH7&PBL8sdtkf&<4W%hta{JkI&Ey#Jwo7IR@owt4h!&0X>jtS;ul zJ)$awfcAl0awmZ7NFlugSyc=YIiRN=X%3UYC98hXw=qVLT1uh_IV7}|)myccSv6R1 zm01sG_W5}QJr3D|F_=OI1H1L-o1r2f-hExcn_3tOD zzCqI#g^43OSlBhStoAQT`%ap(v_ku(-k1j+(|(B#!}Ytq;mqU~lR}vv;z5>DV)8R~ zety1((x*p?IqC$VMhHo&uJD3tn7>NYYx&dC7TIq9LB6`{Zkb2+O0eFMB)pEvjf~{4 z8xnGdRd@N54)4dqf3>|)d7QodWA6el!-2pjPj;dx%`zzUJjA{lmNyKpGrc8>x%?P~ z%#BiMKw%Cp^;W)esyT@;?G-6Mb(`r>xI47t^RORyt575lt$4UEWk43+RRn_^j%QP{ zr9cgm|B~g&| z=(Q|WJjSXnJx(hUfbIInCL8Y~%cGMNYG5{lMvtSKQ5rk<_s0q!CzyGU)3I6L&!=%ul@E6)O>qkUA8fi3Xrl^bP(AI#6Tx#VTa2VJW-rgl2f+tz2k zW$N+ud}|pItfd#eCi`wgtFx|=Ao^~3;rh>tGee`~U&r;{q=?G)-pH@rOuGo98(>Fdv>&CvP|jSYd15}QpE4K z%l_Nk!MR-E%Y326>gL&WCtpU16u?FsP zLuv}2M6427w3c2s&{#L{5%bbNH*RcJT0OC0`{&m0pP&&qQpi!>7B@LQ8zU&MDVmpE zK3JXOVw<(!&6=L$mGf394n?4d6F<(f+i<5({{!8kIJ#ikD{_Cn!mP7P`WjK@zbj3% zUz@+geZrC;TF;{RaK(K*jn&xey(@Zjw_fyDI*J`n{uI*6@9Z+| ziHj`Qeo$J>Y3(1@?ElVnKbm}ge}VjS(eA;OI@|c+E-7^l!h!OYFWXw%5A8d6`_6v{ zb=I>ot3p_z2GwLP@{%96sK#FZUXwsL5sTu(QpH(4A>|PnC?zNq7g#LU+psjz}POoNmr)!9y z{99WtT1`3n;H$&bg&y8RuVyl7rR^=kMmFS-R97NL^Ybdm3)X4|PSlT?ANz8&R^7r) z_0MyJkfgo>J`@%cHL(_W-R>ryLCZV#0)IRX}Qo+fQuPW)7=dc;quBFhUfOX)Q84y8-eHGgP# z<3KH!tye_s$h@RFX9kEwAGL=o<-FC-&Dm4kw&%~|N3(x>pMm;8d%N}|k&Ct}XV}Qy zse`}rvWg2eWiO@&ef#R&bE|gr`+AnOyRKcf%73^0(dSuTJf$-?^ZwHo6TM${T0MQ| zHEL#Ig4KO;JE&Aju+znsE4tJW6b~&MgPDXMvv-M&$hi%IuWw0b<0gv z0qcl~yI?Jx&84_r`s?VBL&?!W)d1TI-x$$St zspkrtFZ6MVk8-rpw&h8)ufwp{DbL`+X2N2KzOF>C2YXTtlhbs1{-xl`P<2jsR%Yry zls_M)nasgNx5pLB6PwGF>g)`y`Ek>#T3Yv#-q#<`{CMhQU0m<4(~871jLkz%^TWPJ z>7AB1e=iiTcS+jLYgxuei8{7t>Ri*{wIxij2#n!Qg@%Y_GLoew+p>fN=9EXD$ItE6DwUlM=5fyVnx zqLMWjRYP8-PRGF?L?hc{O*^-ZSLXTjTp&x8x|t2rD}-RaccoWDX6%C4_fzvy;z{c#FE6kHLEzp;eZMu}V*S|41hxFqHCc~2f) z8$;Km`)u5#kQJ z{yt{YO>n7eN69I$X}Ss>zG8N3AiyF?=6qPF&oyxWVaAkD$6P0^Ek5oDlNWy2Uf8{8 z+Vy75!D-eV&&AP&4J*>~n~sT$4AX3*%@Jbc{bcn1EX?Kt5C0i-qUMu7l>cIF3DRlRA}$;bv=p%9guN;mK<#v^8wzW*%&$Q3L+W zzBJtRu5R5Ydg9e-N#niDOkDKgGKtrtE3ZRMb<%73@@8`bs#t26n+&Di;$<5|3bQ$i zW|wFE@*NWn$~if28`n~QS!Ov~t9ENmrZG0zoQrf|Djq(;#!Orr7(>bO+*z+uLqbBX zRJn^Mbbx(EiRQ}WAK1^eu(&sx0^c{f4~aShwT}HMw~dYu{BpVt<)t-$JR@2iBg z#0LiPMN~Q{f+Z1$*N3bx$;|K8MgQQs67*xZu{45z)sq-I=e3+_y zkbK7PJ*caAErN zHj$CdQ$)r?dbif1OAmHvs9HNpHLuzGu7E$xERzfX38Yk3nkX6? z!9(*MWF&y|&)#k0A%mmkoE-@fDhoAzIHScv3?1}uQQkp&j$veL+>2iHBG?kY7Q z8_eb=+Bh=b3n*jPEzM{AyDGl?U4{^zouD63W{Y7ozbIvx^3i5|7v2vBs`Q_M%3NoW ze&gJ3#b|cYlvszvW#|CVL8wj@6`c+GMt!_#l4FYWHm$}wJT@w?lvEXaUWUa1D!8GT z(?>XTE_zgQU6zWF=s&b?qIH*QWL+>^fYE(0Rw>}@*9>VNDN#|!Q5*o;e^U`=@IChS zY1HA#0QBvw^<)#%_@gjz;}$ z1;S0^k(nh6XMwAB=DK3d4%V@dJ$}*q9=6%DUhWeNxml}eU#Rsqmur8w_xsKxJRaL< zpwj{xYCx6sJ1fD}1tlUbuC5_RN$_QFf$qhnrBBqkT3~P9bB{6pMx&zKa&rP!L027MQ);CI&wqHWsiiu0CMP_!o@;!|!WEbNz7C1!(mjK+}<&yyE zKPVutYd8i4=Az%$RA9=0C_COG&#mAlivwOZS48l|f3xe2gj}u9^lauBG(l8vAn*}X zFs(3o(Y^T8_U&6TG(AiZnky9hC7>VV9Y)mfFyvkfh!Kj-K_)hF!YUNcCU5jl$nkP} z*^mDEs*#O+KjaB@yoLY(CfOVaM{@}j9|c&0I{-uf9l0}R&axT<0IAqn${owi1Kvi^ z54nXH(0f-|ye9>XfwqbOVjTm6oRt-?1h4U&GB7W@v$`}k3hIdtSgUZ*57q}m0O)^V z!4N?e1W3N+c%Um#&b7S;AYVEZxnsX}gbOdXnYgk_u>&jVm6qtrbO6^2elFeyYa^70 z1IR58bUv+1iX1i*0oU_EZo+SFsG@*g=*wh5T&dcx`#zURs2YFmkym8c-kuUqs!B2H zH zUpvAVK~kk-WQ2oyRTE+UYW!6s?@a1SpKLKxmpR*7 zm-f+OD!mJ}9{ox+(*Rx4-k!~0ESrMt1Q($Hj>p;NYYG|oy!8ePpYAd@9D+{~)e6<| z^MU+6DP)pbRx8rb%^?F4D}SEI7BHrxGO%S^myU(gH>tFBX8cq6b2}cKF>4)E-&^WrE2} zw`5c6i1eZuTC?R=k3N5lR02}aI5z~hIOLk8jQ#$r6nUVc$Oe7}2=gfdjXxAEs)!cu z29HAIIN#*yNM)2W(h80+=bL7OLZFoZp^-{R?}3c2sOJi2=k7_g6TztijR>t;K?|ZH zBDrtgXd(?8@-SNeAzF2EZV?SUg!!9{+qKU8z<0+e@9&z>Db2rqUDc^cyJbxiCn;{ab#2tzCk=d;tD*p)5X9n&q+x$&qwj(W?{K!O_xTdu= z`1Z5dN`$WrKDkTm7FpVNQ1|goHMGQ6ViWmEinF#nt}An1e6c!dyIn@02QTxjfg;%9 zwG6S&_Xd`~9_O5zf((Whbd9)oGV&5AzLW!M@x6Mz=;kn~ZvU?>4Ve?bJ zl2{XH9(Th4XHXP=5$E;rW-U_-sdiKWDof0~LmSLSwk5pA)6Z4TzqUE5pb0Cq#n zAQpvK*3`&Lf#ed}c3MKQZBR|*z+a%JL!KZN$^3e*J__|LcJLCK5Wo!WEO`;s5b9%4 zd4yAQNf6cy_qia!q;^&kRRwZ58GxagK++a>!7>DbuozCAng`9AEwI#Pd`-t(%bIK( zV=pL7kOkAq=_oip8>Hx-9WR#)gq(v~KI6|^bXv}Cn+b>#r5$_T4tY>bCa z!E4hI4-bzCV~STaJTuR$D*7klO8}p{l0s-_Av!q%^j#svf_6y?)n%PT3}s?DL{F->LbpdK z2V0RTD6h~VJR0#U{9=~#O(hf9vq!+*ga*nNas!OuD*jq06xVJAUXalvo=CO|rA-p} zGIhi3k7kYsCY4*o=h?6+e^C+n-0a;ObAg9IcD=A0eUXpX^7bp#I+mVE#Fb;dmX^=3 zt6M6aFi4-KbnI4wINpRy6|en;;}Z0t_oJTYjfvrsk`i=Z1E(M-$gX5$j`=OzEm+jE zGc)fhj-xXP2;&*NQv^VnW1)lAnbt*0uq-R%UhAsvligWhVt76H0SQTY4V+RslWSn1y%uElh2O}djq{TT8W@`On~l-;XCIHpk_o@&ShZp zvG1%~IekYY$f#}Z1|}opFn{nds${jUz@ZY5?Hd-EO!CGzj>A5v70wcehLp?5S3Zn7 z`w_!3*BnGvR@%Ek^oKbZD|q*^FrPueH54TVUGBZ%MYRyK0B&R%rX!HL_bKpsH^f|g z3-y~zW76XRfDqAh=P8>T6yQlfvlWNLv1m)wKi~zt5%uP4Q|Q4*=8rsHzRgZ>zgD9C zmrtOFssz000{T&dF~C@5IWf`3wwaBur!9&PQk=q?Q-s~b@2s(_1y8XLA-I8IZf5)3 zjDkG>COk^8hcqU^u7Y}&C@W9vP$ss{VsyBbT8wM(bWOgm2~8z~QNURaujOrC%X2fk z9#TOMfdc!Gthp>&z7-=E(#tBnHaFM$Lh%I2qU*<4JwBTPVIOlHe##slopdm!3BqJM z9GbfPz93CDJmm*%jNy1`1hhB`tnm(g*~0e0_4iHcOhk_dS&>()m>c)9Ol5$2N z2#9Tf)G60!tRz%3ik7Th(mx+mSN4Y+czeVBoJUhe;FP29gi#r&ZdUE4m0N&@p9B8? zw%;~ABeh{=MmepUOQD-*u^rz*c$;=d}o{`09!1f6RCMvZ9Yn~#@Z_8#W(Zw>#nSK1as`sICxS)fh0Uy@40WDzxD1> zUv6i`1tr_wV+uJqlKL*r!=u=?a8$FtO2p3;*G_&ap7iVCP6n-qUFZn zZo)0VP3g4&%>JkYWQ&3>yIbY?`3@R{O}MaDkVb$_2OO(7m7ASNyS!{zSf11~2W8zj_Ax|(f#nwZha|Tb49i;I(#Zy`kjH|oHt*i@woH1F;tZ@zQ1uBud7{QtX{Dm#C&7*VMmc;nB&u#16M-# zL<)=Ft=T4S|5f5*mV3cX*$Uw&%t=CKG2@ey=eqlK(_GchnO0W(T!V0)&M}Lg3UE-H z=%e^~+**a5PXF-IjdPiB7ktYZXG1(H(ti5DPRloB8F>WU_k(PS$-+H-rcg(R7wG}i z1eVI)Zg?#*m=eJkQ)ctr)A|M0u^uB}pbU6|MCR+IL5yE$6Z!GkKsynmQYTFyb}YhO zUKgi8Erz9AJG2a|I~~)+`r;7vfX4GZkTInL>z#kmTzTShL}d4%uVmno@-xTo$w9M%;CIt51by%X{jrD7`Z_P@-X%` zjs2NJuM%b3@R3LMU6=y>BHj_jcyu zpJR$STWpl?wnhHh>BdY1dP2MwjzBaIkrwbswKR`pwUqvQ4L5i>#svWGJr z(#P6q#XT2T#hp4n#2E_v5&S*=-uX~M`@I07coj-@;+QwUzm!+fAJ)h&f=h_l4|+r- z+KRO+XDlcCtsc{B?~Uz0sxKif?tZBf7+y3F*pXTLHnF<} zfh;^%y`?FAf-TtR!d^|7UpCL)baKHh^O>8eb>Npd{^D3-Rx1F9(aITnn|OJ6b{0{4 zZuB8}McoXJr0xZI`*E8nictnJ4?0A}O)5?I6OV9a02AS`2W2ElBBc3MT%DnSUs>*% zwF6IP272h=e!++b>YUxQ3hb}#A9<~VQtDVDs5Z+@N4~^f$og54lVcriZfk4n;Z5wP z{daA3(jEYiz>#Jv9VmJy6e81_E^iv#6^czcm!;@$vu}Rod;jIy=oz9N_|5*Tc=yf# zb{`V~gl6C=751%Cx&Sd*aSPrEL4{&XZu-cWV}iX`F;Q;fNMGPObV*$}W7y5{z?m_u zqX!K3RN4V!j|tuj3KQ-Ua|)VIQvkrB^KQci(5lj-e2W=S#0nTI-~h zoH$ca-8Y98_h|T`RjLyt((@2Os*E*(O!X^7cZ#lPZSW})!Nn_dVsf%~_=fqH#A6*Q z6cT*{ZNrBpOEI0DhCTQE_@|cn8K$8q(V5%u2HTJ>g6-TQg$)izAZn?@MlIE-sjz4? zysD4CI&KD(S>h}DRV)na!iSWdgMo8`k2!kk8-=Si?U;DRt%3vj`H>ee{-~2N zytUR>tdi$uUTn2f1nex{Ub*b0pl~S80YBs^Rg0$$89DKEYIOHi5Np2=Xl}@g=BK4> zh!WRsQ*qpYzJ?XOgLZbBE<8)iT_sw@i;`Xjo_c#@PpcBeBHJ&L#uj~+WKZR~{`m4M zPVXes!*Tdwcfg127G_Q{FzjuD`5{7;^Vo||?w;2H$6px{_FG$G#X-RcyrB;zl_G9M zR>rHx?Z+H+HCIisMbO(mlsw%~Pl^0^6ayW7HU)X;cHgq>4e;~8?^h@?TXSv9NyvM7 z;q~8!4^pn7{W5Y!2;4~lYQ0o8X}V8fYI=H_Bu1>&)z?30Bjr1`&OkiX!j?c;EoXU* zo$%d02<8I_>@FmcCf66wF^4WryxQ3CMe}FP!(vs#rs9(E*oEW5i-xo=-qeO_9crjz z8O?``cyXt1_Mu^2I%0j(YSI3U=P_Nl?AYTWIOYyhX=qdrqj^SJ`RUdgt;rFI#nzoZ zYs_;%!{gQW^%RZQL^sm3cjSjdzeaq zGD{yf>wV|rlhcaE<9k%0?Ld3AtM>J@#8oO7NXeY7^$D+xsIW}GFn@AQ2?AJYXKzHQ_{T#A2>Kf^Xyypzw34u@h) zHq3Carhym0z8)wr$?o1?0bkI#FAcKG_O`bDO7?J4z)mwDx0WsNc`f0qCN_lmN9Uu0 z%lB(Ur(;;Y;P*xu(EioNbP_mm+RwBpVu`8D=#tNaqJfCuYPfLL8frCb;d0o9nE#DXiB6ZTCJ@FN)Q3kyOiJqp6qm`7m_5bbEG6sZ z9`ME>G}VMkDQv+Pi6#ycQAragM@IT2v|#+)F1QzC5ZFa7;NLBW=`buDS*sqV2Ni8Cm$vbTE&isMvPQJx)L2;JozdZTBqe z#PZ<$d)6fX8q7&o%#z5&sB9c6V`+FvU>>4kb~Q&hPI<1xrH+uE{Y-) zzN1)}XTp1X<@L%w$GtCdj+0r=u8A1XlXxv_or)++)?Q5&AKix9)^$$cy5EX!kJyIp zkOy@3=dlsSeDL*#iC zxQBpTxuH2H9^mn20hv=n`Si*@#RiR@S|OgM%d|CBVTLL=>t@T9V922lS?OHoPuf z8c_a)RVYGXY}UWk!q^}%kig*1dC>s_|^|MtWPF6x*8|}VaEVxNBEK7 zYU|ThDxTt(T6xhiMf#FkCZ~L=w=P_0vE~+-fUM7y-_b z31QE#5VpXNEq<$!#fVaI7*(-vEVp*}ADejX`>^St0aGCJvUAu*#w!pKJ?GvH3X+;J zPMW^X13JDN$tSz39Byjf+Y5!(JJ8Z>5Z6-f0o;kWDA@GkViCl9mS9?i(p(#APi#8m zP)xwq&XHRgt)u}`0Omzai%1oX>8}53w47^+61GwQv^XAji;F$S@~|#1uc=<;{BQ$L zQen6Zv-?ds*;{;Zmy9gG$XtD@8kbdn&yHY2Q~%(Qn_0{Y`Lq>(=ruyd7dS*6JB;KG zh%{|jWCMc=<}DB^jUx|q)~AfAAX1nXsBvQSg&F$19T^Z@)C_?zgZxydgoK2Z&kEA1 zEFHRz3AR1S}O57S4%nK)^#Y@=Ay9@uH{C*H>afHJ|Cz zfOT*{IT(ahegn(={>v)JZ$|>-2{J=BuveP^#i!0iWdT#}YiXKOfEZ%aRXNkh>(d_P zQX@$9B6q@=T?sHs@!2mIKvm&u`?TtWXPt!>3c{H~pec}LbFPO{ft*felPE_ zd|rhQ_0jotqW2%wk(c%Fxm4|Y5}dZEo|sRVd|TBo<5@Ody|y2?C<~yIf=T_Ck_dlR0p^oGwCUj5_^wZw^4s}O7?F|Mg1w5=yNA5!)~Y6Y z?C%Eu!Bgbq6NuSi)){1}W!SzJHElHein-^tmZQhTy2V{wmR#Q-Tc{qwPRIP~(c$yb zOWL|tFa(3SnvYA%oXWcJgRi6dNo-K*z?GbBXP+Xf_G@ndP+Ors#3StCHx!I4Hc^%l zNS>M@A1V)una{J5Ab=YOiwA&Hje_z)^g#URCbRV6;5Y)JVQS}5LixS(NHZormL7f) zvbF5{%5)Ky78F;B%$5*^KE?NA&vMh|nmQ3$fPw8-kn#IE48jiN^Y+eVga@M8poM{S z7ZA~MGyAO$1hm{p@ca>ry6Wsos1e}?od#qDiWQV65EtSQ0cW|A0OQXFMR7vUOIeS6 zV?lj$PJIRak`h(NHi2}K5Ve$cuXN;T2_#Qdj=o}9A5}oXUcjx{1}qThZJ4&8&?;n6 z1ZKZM@@Y1i|2o*v%5W4?WSa#pdRWb?LV*YM~Aa($= z-_orNEp9Z;!ft*QCI1}wAcMA2$#YtjStlgA=j^<39qcO`M~2MqVKjX&rNU+~U(lI- z8Jh>ifZpN#Cl%ULULrvQF)h>YGa`tapZH&6QEB;*n|XR1Lk>}@^FScz0jt#xWLfr( zWvSfa#BYcE-Tyfg?q@kkCQUG1c~)kQVMt}QSGhYx&uxeCIkh8W#|Lg~m<6W!*si9w z!$;lM62ObgtQ52m72eC(Il|@&0g;Y^7UmClp%knGAWUQWS`PQx4LUS)o{b1cL`fp> zCT6$~DER;-A)y6FsRhKK87P26V1tci`Puo3pVid;Ej*p~B`0iM^j`pg)*Lmh*Y%-J zmHa?ZJ~Aby?1GY*kWkH@>Q;RE%~G#Ur(^r%T-Uxlx=1X$Wwm#kG6v?o$1%{2+W^7k zxHow<(V=u;XHij+5*DgBun+B+nyUa|{xyA5-=eQ@%cVofFc(4QRS;yFF~Dtcb9Wy{ zg^N4=i5H(x-qsjQaNkc?#6Q>r8dkvMvIx+_pms~3baxfUqhOXMrKbD|p5ref3u{or zgW^Rh->cRSzoAP2Hehp1DClg;fcQ3_??J?+7X{xzftn~XZWP$7x`2&?kEJ{P+jWa6 zkTwA*G&W(nSylsiiQ&XC2B20Q}Ze+>R)w}jm5!`kkgl?SY9QLTxuDrYx({u z19un-9}Fsd_HUI)W=r?qvgJgO!ppP{I|~bP0?;#qkF-rcEiUyheVS~o{$aq=c$xTQWW(FDxvK5&R$N_5})`XKWqNUT7}l`KwLP3u$~$LsG&D zJd(MnJ~b@1i>2zmz(p8Q51^k3nT06#16@YcjRGnup4#Xmt=H>UrtUjmdKn!XQi9`w zGXM)W0P~0{GY5B61s;RVT=3$J5dpSr zs<;Qz-PV)2CZHI7g>CNK4NJDBU4SdhO`SZsB4mJ z(S6~h6sb8CL>+~E;XEp|z<`N48qS~7O`9Xl*!);gXJ4)E#OpY(z>6^$iyooh$@CnjT$+{;cZQbKt?_27dy~nJ-1j5bAn&a~wV%4-G+VmSew=*E6xPOz zBu&W)-mzsEROXo5`Dyhp(*`p;qEfV=ZX(K3$u&=*A>9o1L>{_j$eu+n??wvbcPV30 zN;Klxu&j307RJbp!a;$m05+XH32KHt9oavwSsK_pD`gE#aN1j*f5zbk7%=l()l|#R zCFPCpab_wx=eAAu(+%6H>8IOOn+F#C@~Z{?i1+;$g~;f4saw^o5wkW>kZ z_xjn2YH5<&@Xx=wFhZ4-Suy@JbYO7PchO|9NmiuEjHynZH`~cZDwLoLHz2QJf)0`rO zYukIpi!GZyt1v@e1xoMY-X7E>LOjfsM0#Doe#y{;Cfu*!ikJzJnEnRk!c92W{DB3D z-LFfQRLw6A%wn4F=J{q4Je`OlwXNI5M|hupZE33?J*4~igUBoCRyJbN4jQ&*#Z5;F zk};wuD{u<&h2d$G@Kwin%`G-eAZi4Pa05B|OjzhB)B>(2c^Q14v{Ja=YYe!AG>0@@ zs(Gm8^A%rYqvmdH53t#2XJ#H{Y9pQXi1W?Hyx8n_Md7+EIILi%T~V^@%F3%MV0p8Z zC0#s}@<+oI5OxPT9G2Zib3O{VGNPSQ!zWXNe*c|@eqGB%-qFpjwX9gaDC_$j zd`3!vJm^wv<>z+MAI&|;PL74Y7*l$m&^*8Ig^rPp`9|bOV7}^Jy*xGxi ziMWY0loZ<)U6Z}lm>-)i2G`{%{+toh<#&J7PXP&Dy#07@QO9q%U)S&8`Tt0fxPMuU zF9WJq;7|czs%}KP0D8jE>%HPzFSnmo_y|?y9GyeC(_5mXmW4&rg|4hQY8DH6+mi&{ zcZb}lz7D`0A{&a;oc$r5yI)M=%P(?Oh`=%Q9`ssinDr%g%oLkgd#M=5KF*`f^Z+XF znd{jW`=S}x)R^@Na5sgEY$_NBBZk}_3_wk=^}$KdC9zZy<$?4ZPWav~aI`XO!A^dD z;2BSE3he~~R1(+hiN{GHq<<7$6-}uWSM))%&-W7x-R%!0_BS7XoV6IGs+^Dj$ufsB z6Gi>~tO1hTdhM4;bey*_y7@dW>xYhBC12toOOuz4$|;ag=I;L6WsQk#N?+yAZ3LNR z^Q+Ns=8jc)nfJ?!Z9no3dMd|60P$01JD&Jll`u?iL&7WBh&Y0f$c~Dvk3`nCf^7o#!5{GU}?(%~qxDiy zOQ@k8$c_K~gETZ81FJ3dkyiq|JWAT&zGNEldYUv{vVHVsInY)@Jl#qsPNquI?av5* z5{w$~UTK<*IeF=)C8c`2L#py5c>`4sW%j?S2kqtP`<40`Iw~<9e&YS^+xT|vcoS^} zp4}RYcZQ9sNKQ%x{{vMwv7Bm?ueEmQvuMnX&3<;OA7{JGe4Dk-`$I0Tz8@>Q5UU!? zvrQn%Zj&50wP}Qg$YedCq=L`%O1LOXvIAY-&9VQ3i8pntMqXEZO6r(^IVFx6N1s;$ zpAtCjAf0@UE50!*=Ln7Yo~;Ej!qzT=*+-+hkh}9b=c8oToJ*JcoPrjgUj5)M#64B+ zXl=Wb#*E(FL^?pLb3=MyVev2wg8(ffa%7u5P9#S#@%)$JrsIvj4qk1aUH$k@Z1Yzh zVf%PZOiWC2J-~mwlZ!3|TXXmQ5_aw? z&de}6A^945vAvx7J-tr)S}lvhFUQ(LFfe*M!iomn%yfgRRZ)(>9eIA1%VKaf8MXkn zvtf#gp;Da0rPPRh1!3$UKQfmVFvi~07vfwBKd863e)>kAA;2EkNLrfr_W&$?uc4Bm zrJU6&w_3dEh?*!G%7jAqi3O4<@2~*}F}Y-Nr2Fd!M*ra-OGbA<{7@Nr^XTJussu~~ z-W@KA2vk1D?1Ap1v6q^%{J!5N$m8swCCA_EZrr;yJm@T@ZrBcwgcul{JbA~*@iOZ; z#BQK#av_)N>`-aB%(nLjK*I5)b3rW<9D6=5Y}}@!*pdK#}*{S3(D$J&K~T|jhwNtXDwpJt~4 zD0R@*d*Sdus5-;ktK`Su%Gz`2l^G-n2^f~I~Ll*wh5^HEo=%919SMm`$7f(BaK(e&xAHK(+=d(X(R zK;^81kFw>YAty5L^_aYfU6}rODt_YI2qb)m zbDyFO6fVDug}Fdat>G+v6$K0HjFLJq4aVGjgTMwvl07ibLez7& z7ZnypoJ(=QI2&{h2Q*Iuj;U3lpPecwkCLaM+=8!tti4r^(S4&71;iH$Z>JHco`emaTn`{(h>_$5fI?Kx8Je_4Wc-slr4-Ido!9~j?pvAT z{K_kHtdhc$1_@@^Dn|~5ViKeOy!d%FJSKvw6;WydJD1^0A#1(s6xc|jsvG{h?YVEe zfDJe)D}(vV`RIu0e!#nOk=7OlDh4W?Z_Be{TcXquDY~x)&&#B8HywIcGm$T z!6d}#Ntut?mwz3P3jA}N&oCW`|0kwUx@Ch7ZNM9|)y*2fXdl=0z!%cLj2ToOw`Tkg zIIYRormr%8VeRnglCM?0s;@73yIX&oL7cvUTmbN8=!?O}qU7WEEYuc>&D|z&+zv<_ zz#tY53%CZ~XEvn7p1w+np04yJff$7@cxNb)g1c{WC~?|TYL!vJ*%(}Ib`imzTfQJi zJ8Sy&(%8a-TAJ5cc|{U)(?Dy~5%c}>0^NrvL5}zhkG^6i5LI#i*eMe){*HTj>)FIv zhE(nOixir0((UL;4&ZWBrhaWNYJa$LXC%^Jc?w z#^x57u<%p-O2R3xuQSUi>3=oOROT?Ak9IqlRm$gl7{HwK-u9{7o?Fdg8bw$ z*yNw@{@vKQRrhb@uqg9t{ugKAB%sV3C%##hvvHT=iP%ZrOuU z=_ox}wH=h=utVKonx<_jhi%kl?mND?^8U`$;rQ+qNfW9(C=TR7wN+GkkTmP@mgaC< zWeqrw_aVg;9F^k!&iM|yyKyL!8D)zHEg<2A&^vN0oNd6yOkUQrqgcT!ieA!dk}TOD>vwzSiZfGrMPlD}ZfIl2`4VlL4W=($f~bN4GWEbD>P;E-GLj_VRrg{e~@JTm@& z*!l{%DAxAvwG~B+Izogtlz`<8fePrnC*6$yGeOOO>)iXMRw_>M zpLA>4pHyV~09}%3-|YYKdz>2r%v9_@fCbVgQvHD8_OOHWxECVv6YqiO|3O3b22~`@ zQX3BZ=L1;lc6+%f5jubnKR3O&qDv-HBDBRUFfn{6y6(Ck=isB~7mo5Ahrs!&yzmsG zP=)6yG>{uWpZ}op0q)c*mOZfu1HjzMileV(cN$!_=S1{;BO<8rHsr+U^xoGI!n}5} z);ia^iooq3Q1+@o3gad9CkLbf7LGel8KqC!r&CSglR3YmOi{AoJz@6S%~L2aB@4XHoI4j(@Rz-3VC3dkJH zv-oUxxXsBys*?U$3i7FN=!qy6fQxpIAq+3MC}B0^>f6Zl}7*3zv+KhAMkk+4L9?OYxi%)BVSENFKy5->%TQa&|A3g zZ?{Hbsf8V`<60@fE6kxK5D3JUya+&b*%6TOh&GtW@Tz>2Q=geIJenFbD@6r9+bb(_C3UVFV$fnDK!H5EjA zio0v6koNGzJQ6NLqfwjPA`8)2gVZv@a`LIUi(q3R3pG}1T_Ur8S$oSq+5~gQJMvmk z(|rVv59tKDCmoA^;#&_r#@G#5ZH?jXROIga#2Z0&g~S^fYi#gW`f95>Rq7q{i|$t* zn13odBxj2BfJU0W92RW`?Lf#`Ng$c5__kCaDol?!B#F!R1$WSG_aliMHIr7?Gr z>4UUlo&xT`#U)Xdv>>piRi=PR_PKchY4ZiBqmyL6f~hf=pctYr9HkD}R!C}%u6xE9 zLGJ#l$0!5+&I1B-x1)3{Sk7E}d6TZKQuRORC`d?YsKO2ir_Oks)tMFfaCbKJ#p`4~v@DEV zKkwULBeYQrdz9pBJl-5BrXe(`e7muR>s-TB4Bd{w(n-iSO0W?xNBXmret%92oW2G< zWN-C~>MhrD{rTaWEjKlTnyS=DolG4)7oB^~D0kRd*x^%CQ(K`AfCtif4QX8MJUN!q zVFn4|Hc04n1GY2;IQDrPd9Y@y-@0Y6r}e8_DVx;N{wx6%BC@h{vICE&5Y()YJ@c3E z6Pn;X%iaXBCcss!4NQooARf*OeNV`FgFjK=k~3ek$N1)>Dt90 z$Ye9#$_Nv0t*In1F!3#s=Fn@Fw`-O(*~%7z&+; zJ{L*?UEX)9+n38`AOS^~jS_Wxvh@lX1NV`!r=1G0@xsp#Cg($50JVkA@g(tnJr>dP z^PY-a*|slIkE6;EQf#%R3d|6Mm;qITbU?cEh=SIbDNPu>%w?YnVw7~MLH$w)YaWkjz%yYL}>$PdKK-yfJU4I zq|y20)4QKXF6#;dOA?_kAwxpwMwi@Sj(G(yQuFw^Lc8@s; z`+@;<_d^QFgtj-H`K^tA>cqFM(Vt8>9y8s}T|0>ol)*VqM^cqwX4#z4FgU7xS4DDe zDd^Qv$4jGT$j;*>!ek=_m3E}J&bUOUfHRc}!fgwah?`ba!wTjJD-y5 z*CUaqHI8}!Y|BVamV}}v1H$YTK!iIwJRpR+)>L4i}3|5;D&=U}& zydPkKDPb7&Yo>ZUtmJvgHwmGi5m&g6$U@x#I#h5uQ&-0}+~b8Uo6}N&j}JDvkoOnh zc~G0*GX!r97O4XiwS9$T6uy_**-cGJCW3O3R~t4i?izHyVTDVaO(4e5C=q{o846KG z(M3#L^)w80b}vC>^V(ZjeN?<)JAt9AT+)bI-sW2yIxJ-2=DP)^ zANvAi)Z@QYh{)_0om1Fm@E|kR5ovyEFN6dPfRxB$mJL;$k$I{qw3A|RCrlh zppM930YvYE>d%}L%AXI_kGie`u5|1gDbuzV^z&-L{q{c3im%V+(?lbnoO=9OT1XNM zZ2R!4=1k@)l|SvKkZRQB!B*qF+pRFfh{QypFgbZ&q< zJCddVUB4_G5%1e)_I3^6D|+RB_;8XhUy$x&m;)(|)k=^}8VtLa!}X_XAm0vBdVwNm z11+2?F0y&WebD3GKKbOYEHw*GiNUbl(a9tK-bY&JI#-rrd`;H8B2AmjzGioD{M+k%ODr(L{9f0ggFqPAXL> zIQZnV6ce|WrjTHhq_^;iB2=`1YAV!Eh+oM{Yh^htKWJ9{_p|m~2DRTTwX3vyQ`d7B z+{PyNT6wV73EJH|*~08Bl60oJ_1`xSlm6r4D0`9ied~Zi|4}Ls+S1zh9gn0|D}tqh z-rwCG4s2|qKfhkSrdXV$vr!lyw1bL`TC)S)-PHX3<`B-2JCd;9H3jl-Q?? zwDch!)uDO0W>JaJQ$JSCM%4ofL@PF~-*dGdD1MOnIe7Qmzi%}$Xnv}#3t(lTBNxBB z*)h71g_j}UZ8)Q3T8fN1p{8J@@!w|V)n|WsuvSTBjx6m>1`!t|(FGM*<0QnQ#owxl z+nH*heKl>`QhQuv)6MY-vh>yJI z7_{o^8{|aqXo8;m4^2=>$?OqR^`_6cIC6|J?AOGfkWeTpNbPTw4q(&OnC->58;wdY z*vaW(yA!Rusi=+kAPahjpD@Nsv;9iI&MDl>Q@+IP=(PXwR5mPhnVMbrY}%{Ben5k4 zcN=C`Qw^%r-#NKDXX&9)*_O$y`l`ryicA384L_itR2INHvAkpL*%JDe$IiEHCOvh_ zZ0=b=Cj!Soj%rQgEL=mfqxbB~6UrYG87)SsZdVfQmGGixM!tMW6;=jAqsUB@B12A> za52upQgVRQ!Cy(v?*oaJpvc)D+*Wu0t6EQ74Uz&U9+GcmC3z)5= z@tnj9sjDzF#V8;-H{v0o{7=rpM3Iiee0UT%rXL12NC}3LRb~N=VIHzFNfnTb?A}af z`#7+z?+riryfZobQ~`4-S${1RXx#+_HFnvKj@mh$3G9F8KP~7wv9xDY*t!Jrulavv zIVD^~hlkP&ElW)eyrtNJ49?;P8=E4-Z03U9A+-d zNVu`EdG2b3K;5C;oks7^5ihO75;HflaVl%OaMji;`3;pnmlwp4Ru(2Bq*btQqYS97FH7bDT!m1ZQGj! z_w&^*bC<;~u{WTy^FMJdA7Bvc&Eg)SR_4FKlJmw*Pld7;_*hnbVIf_a z$^mM)P%#>sc>j`CxraI>KSr%~b8$^QgYT z^vgrIQp1>fGNET#q=754S@hFTwj_e+XHdGUTxKTS6Cc!(y?-CRGh;9c(Xselo-j?L zN*SN^@4~o98WA|Ao#74mE;ZT$q3EG$izx0Sb^1^dB|k*je&_ut*RMsZ^V{?!*i%Gu zyS>fWmBS*1VCvzZN&wA6x}QSX7wM<^ z3OCn^BmeGXL7n(6IcehqVlUh!oA%=B(x1m(Kq%9rAEmcUNR78~OlwRl-mD5Upgit*Jr8G^x*>NI=M8uH6$NSJiIas@KQCnAX5#M z@*A{e;ot=!X$eHOQ|>BJw$p>t%N*nI?OCsJ>RAZ`lI@^M0$vHBzIbTTvOB%-l-1Yd z;6nAdw;(WkJURuOxhOZ8cC|l zjoKN_mdaLP#eJ7;9i1~_U6IvwYzul8tOx(P$|Nfd=)eoswl^0e`|8vERD;1Wf}!nr z6+7wqyso`FI_G~eEU2$vEtGtIy}~+F@Pm_^+k?Ld0~FPX+)CUVgZ@l-K9KW72mEE% z1=wVPnp8NIO3XF9Qg<5*tWa;qU?xu_nj{_=>*#q9&P%g;b(Eh)I6tdQu3p#AkI{SJ zErurm5S)nGXX`{!t~cw5q+~9$b2*FDpe<(ECRA-(?g$2)c1bS?a55seC3 zt9LMZLMNk3cceQ{U4Pz~H)|Aa$5c`a5J2^>uRyRlkd~ReiIy~VP#2RduN;(_pFDUr zE4W<$aJx2G)rnK#92inOfpMj<^tFesg{ikK$Fx}QJ>e!mP}HG&Tb>hh zD*(13&Y?%)_cdyybwx=Jn4nO*(iu6a2(wb|l~Vzu$k@7QgSE~;~+g&~Qu$SO=y(N;MqH?J&%rQ|+v;k(;{ zefQS`sk9)5yDvF z?tiIT4u{6&h+^gTp~#z+mO1BEriFO$I^eFaByGk5=Z6?UUWZl!X_%Wy*IN4r$QS2jN(N zB&blVJ*Z$MV;^l!BT7IxAVy7uLSHvsL@JH|JcdVaEVkF#<_XxZLS91X;Kt`Zkg*T_ zF}q=F1Y#Ns(Cc#&aJ|HL%&iG}z!?SLq{@p)-1?B{HZ1LDi4|JGJL-n39_vavAgl4& zQCbho+b5cxA2G~Qb4%BegRQAXq!TO)Zv<<-#YCR{g34#B>}dC@;ZytfQ=DM4pmcg= zzW+JH)_uQS?CP1Vh(ud*GKjLfK_VFu8k`0}#!jeoow#+u|F{}aF~54UJGvgGx(qVa z11G!2Rh28W)0imj&oFry_%SM6dnyodSb4>b{~;av*vMq)+Vk8_5aid~QKCe}K5I8a z#H#Ag$%@j4&bid|E9Y2Y_?)c~Z|(@~j;tb8#aRo>?&OE1IduODP)?8pD5=bJdvL3Z z%tWmZ?rn`<5DH$*#1`Ju;N%uPDA)2(U(5W^c5?&M?LcP1ae-9?It%dt1r!5gGF&;R zL_Y@IP;3$99KfO8TA#+_`JCr;3knO(A$7Olx=G)FkO83HfM}w5#=QTZIOzE&gna_iu+=%+az~KG%V3DYfyg0ep9ZyE=esKlS!!RXj1Sw-xz#?y zC^`1vGSfp(DWr2+wLx@sVCUUke3Ln!#?+RUeDg&iQy(s1GN{{4-ii8F zj+-)W(Cd;Hs!bI;qh!xa_bf?ffk}&%1^@^UuTgz5 zK>mQhEpzk4%n< z^~GJaWd+Otk4tgK4j7Y5{a08g=_BmN7WEeBace(2!-R863kQ;*x_XJEpGkF#x1{EQ z(fe=j6BrIE4vUS}+CGUTpaZ3K=uw7mNN4RS$iAFbWj#B&s0k45&!L@>4Gl%=+5U_w zf?@@{@d})g=@8uoCqpGj5~Pu~M|Twz%;gXpxi@ZVHf`?QGxuizy(Yza)b$`*sMJHA zCafG+qjK6*TkS@P%)Xq%&r;tggP3&(;JbD6=UK#8>p3-gK;f?{6=@`fV1*ddDk}7l zIhPGx3 zoN#8c{`c9Z*CF(eC0P|k<|&7kMNL^m@m5(n$@ODrLff}c9%Vt@N|3uXzC{E)!QyHx zQm@i3a||(JyPlm~;sb%;3cyL&O)8Ax8&$1G(2Ls=@GOB+D2)> z&HK2FAglqZVjMC-Wkgi=G##LG9*9F4x7LR0MG|v!)syZ1O?w|t{o>4$bFffhV-*Q_ zS3U5`ysqu*1Jwha0;a=wZMj#qqd!H&0c*F-tcQX;80;Tbi1qlUfCLF%Jg;}wP(rss zk&~&11`wgo7^zY^sZl!F_qT)HU_knf-PxAD0!&#mcH|$Gc;hnMzDd_%~ zo$)HzFjR!QIg?`D#i9VK=Hedgq6#vt25&@9w39!&mWfP8DiP%M|~)_4|PewV(N2 z<1A*?F7xZReB=8SDeu zr$z|VH@79Si1V$VBin)q6Av0)4;YbZOFj7-z3K+alZf&|5Z%tVkW48mQ*pJFDTCfR zJ2@GU1rD7~%iTn;%`maV=##c+>%hfFWt)eFp z;G@E~{dhDGghmjpIO1-IN{|p9{Qt$x4F2t12T?snJ9fA_8#udvx~x^4YgO#o*#El) zhT<>hBjajy`46u ze9#?v*yKQMZTki{>a;VeGS?%zifj%SF8*nh^S{Cyz>jZcWFn6_=UB%>L!Z(0%tsrt zS8cz2XuPAo2kzQy`0xSfd?Z4XH{%!M*^E$X76tw~qFc-Fwrr{Ah*Mj*YE#q=*jS6U zWsq{(ui7$NA4o|?*oX<6Ux)d0ww0gw;i-Q&3zfL|4_m%CV>Mm4|4#|NQ&IlKn%tAPTU%Gyc^+VpjsKjcY$Na-9T7{8O zZ%53f)hbTOMP{8ap-Wtux6hZ4bj(5iy?z7_Ac@wL@9$^!II)AKLB#MzG6HTuq_Gs$ z)Nt~3PL7V0R&StnsRY&-qXY_N!XhHhu1%r^j??O(%w#1ok*Qx%YAO;~4^Y7)(xPJl zP4{1i6M8KRF*wwXr!7LYyTvA`4*!lcy7bLn%r(i~pvXp(p+#C*L9>t>|{4mLqM&Yy>odaMjPGqP$rv7$3u$cE{;*8O?`4*Q)%;??KiqLco zbZ)sfeumYWew?62s8E?*Uh zv6{r?^ba|aBSp81ssyCxVDmj9zWuX*c~zJzo2#-_yNaX7wJKMq$95YfT4Lv^ z1ep)2wain{LLCyc@|kNtx~_DFAN%9RC;9yQ(y*D>BC5fgsO($)rF!B_0$q$Pktx>q zi<+LcPi1he|DAxbz37AC{r2%iExo_8xQx!!OS5u;xc%9Mz6;h;bzvm&anr9b}cr%;_{YxjhE;)>sXa7B4}YA3HFsqc}OJgb#wGx zUS2*{Wo~jMV)IY$`rXLEgK){3!fa!QJc*M}=lgHeYkWa5ppPBL1@AF_TTWxK0pALGs4R5PI- zr70@~rRltjkt=U9)3c*14h*Ud;)YvN6WA8--QSd!{d)7y!f2qy)}QGLgT;Hwv2+ij zjaa^2t&^{I@r&|WEcfuZW2SD>$P`5zdsd29azS4@(e`Xx5+07+i0 z*_P3K@xTt#P_^bhhnzEM%F*5Tw~`y-63`%KI92}?s{#gYL!ZSCw!fCD&CFIfzIsk^_46T)*6AL_#l=OAbxYJNC`00al3^(t z*S9Au(MeMw>Cp;}ED_zRy4bb6@w0o-jY^pZR!*vh z$8?yR?CQCyGc^xLF4PsWij54|5_--GwzSO&*1bRRAUa{t)~`r>BI;~}NV1)7+WsW$ z_zV>_wNd-Hv#8WyBTa(zan{z_iJ4K$)JoYGuAhz)jD36T zf~sKv!?5q{r(R7f%C^dl=#umNCj4e8JSKJBwBO85xEm=uCHkGqsLZ)h5yr{M~fai#S>ZMa$#VsWYHJPex)7kHJ z_1&)hHjmCH`tTC$s^uLf@9r}uDSVPOP|=?{v83u@bwneC(Jfe^ZYEc~Xq3!VMI%P6 zwMysD#J888na&~l9c>AlVP}v^hUJZaUjs*o#tGYWjD1GE&=B^JVhE$gMZS!ysA^$S zNOzi-UdUr~^oI*{i@D)eycRxWr06lPVPtGRBH(CjTmk?buMD8%AeP6giT^aEqg~Rw zJe9og{EpDP1p5G|?85ay%2EFC`EIMhb%*(CUsuOJ1mVsKjZdrkyE}JThUeh+eOp*q zE#LY{@kbHidH)c6$0#Q&I|)?M_@UKi=jb924UO*bG5eTs7p7(pw(*j>a&1IE0_zN_ zAJcsk02S3-6m@fR;};YxK2OM(K2t|v$(3rauJ-!)_RpC%eZdxFT9X#_kmr`2o$PB# zvQH9pwHaqz_DR~|T9Y1tCTOH*?j^k0?^? zrbd_aTdTa&uX~Q4owe#U+0z3y0oX)^!p&y+Cz?(2X?OmNfL_48vM;uN_F&%K)%!Ux z7o^l%LzcgMb`N{K*{tY(y*shpZ|2&+8Lo^!Iu>irz9}jP#3=Sw(1fC}yd61V>Kr-J zb7*@}_iF@wcAe?8^}e=NWumT+)_un>i7q(6xr+a@>Pj&{!@$5wdk?>^kKvKrERiQa z<(i+-n($aRgg2x&g%Wvrx|3I9!o$s>*>%mMw*JrL0t2x{AThA^nMF*=z&VM0Xl!iE z(c|5UX9rDrne%+t++dkO7Y}b3hBs{XkAr{g`5>+%p9+7=SC{#5qNK_%h+X z!tH;|&-;ztJ1IrjKX0xh5H(mvUgdy9qlvw#-TI$2 z%CDP0{j*urF4Xi$0i_KaD;s>uuFpH4a{Du4oKEb0T)wi_rK|Pkt9(Oaq_cs9+-)k; z?0_#W*jB2Vf^!6%&XJQe39U4~PUB1X?_$Q8R?Vk(r582~cWkVqVuAUL1@GIE{(nA5 z$FLlo8mk{(^q1~iCDt@4{oL9H7iaxuKYsP(zS}MP!WP%29v9GDaN>A|K?%S`H}5uA{5Ou_NmWoXvDvBw#(x3xa<}|C_ z4f`G(riDKCuBJ1i;f`HY1e4rwKQ^-TYg6(8D*-$K!T6?uEm`$iD(_l+xchBUzRkYt zDGPgfk+R4$9?7tCo!QslI5{qvrnY8O zx~F8u2^=+?K7Z4LwzulhpYXBz`ufAx&XcWi{EaN~k(%8Tk!72RWQ)64RkjiFjkH?E zM4ahRRTZ=GIDe2YHS>OgPb3c+W7-|%{9p$iXRK$cjbb-tDPK6(#ZNa)P^bsNq@;nR z?HHbrsLf@3=RCQh_{!Sb0<+e&Y6<)4Jt#})#zDM#{+lqn#5a+;tzpV`HR~roO5AWU zD?QWbf~}S+h&Zy_$RU+A$j}JR}aslAnt*WC_=s_S*0C6A70H4IZ?roxHrhx}=MRV{zW?KMQ@d#%7a{ zjw0X5c%zUnu0A}msLUCX%^srhsnfp1;cF-^#}Osbzmq?ZaC>N7nL6w-jv4Ph7OSsv zaF6d^^_rfvAVHydT+Cs@+|x3`CpWiX(UlKC){RZud4Uh^47rJ3*(zs4ySG{8UQ5Ny zs@Bx_=w6tw!0)kKlT9IQi7)>4YQ@Fij6P z$pRJ@*Z$+kN-oWHvOVVTdB9)xZA%-XP=x~X6_r=7q$r$?QU1Z{LH{91)2K7}5d3nO zk3L(m{_QRAvlNieVcM~j-#+>^zp-#PTh8sLHVQh-+psT#f^E9Rd8#nZ;!56CR1cqS zO}}L276FUZ8iF~xuV{L1u4`I;2&wKqSaMctM_OU0Eg81gsJuls79PvaVX zrc-U!O|DV}OHbAJ4M{o4@3;skLs|rD=>#x%0w$ssAA^WlIDD+iHW?K>lpqlZ)sVchQQM7AiS5@lF0%7 zUYo6P8$2{Ho1?U$uY2z~72-Kno}%jsT@l)cDo+2oZMrfyxaMu}+nawHwD>6|L6!LQ z;Dq?bi$Af5{t#hoXh?rE;}HRyU93(VZTOVE%80NItC^4=zh0VYu5y<{Qv8_UyTy>B z>VMkwKJ^7Q3ZyB+=mtq%o~cSwd!O-qQZS{NwS0IO+CW3Soxtys&AO{vea8fNMSXzj zDc^$s<2)L~tbRc*_<~%fDxaE9{z;~g3g>Uvw>~_;ZjEIn;G3RDD1y~VB_+0NLpeC6 zzH}aPnOBp)z;TORJ2TUiRa;5u)hJgRd&}O(b7w2jwV}-FM^LnH{vj~0BMl@F=_F|I zgxzXal2`LXCB$LU9@VIF8Ky;1Q|CU}MUJ+;~Q%nd7(6 z@po0b+>Nz@M>mE`n5MEt1yUITE$ncpoL>*G+P!v1DnSC3zsS+BgLh0$rn44?>u1ka6E$S65UHOJco~x~<Js@(eeI_ z08sVSNkPRjA*=q@=KKkj@JqJ>7`f_Q>Khxse4(j#!G?V?lX$%Fp=SQXWaIO->bMHw zo+uk-0+^cTJE<^@`7cX{6Tn`4|K>CevL{@6A`hc*-{pEUpUqbE!hGJ`1Ysju>1P*S3$Yon6wF1M6te8lWrHY=AcF1 z4bJUA`CY929NkZR+FZ&@1==tTR1}aL zax1}yhJ^SljUFG&o%ULGST2Ijv^@`>r4}gy6%*+zGYERVa_e8O$HlJCl_`M#V9+6k zv|C1b`ErByoJF3B9WFT=EB~wz2S9(qIl|`@g}NaB$nqh<;@fWa-d!hQ427iHJIadD zNBvXB^yvsonrO}>Y*%uBXKh02s^e}{7b$f=Ya(Q4hZD2npsH2h)0u>WOR~!;3uIUg zR$8XU2$gGeJB;Zl3ZYQN|Lj=q%$C=AM$(ZPE0rFf_S~j*8>`hQo5bn6m}Xhimlm$> z+f&WVutUeOJRB22qcIbeY3>U>s9<_YXDEEhbn>v!I$d!`(uQhakSIWaJRmG+$IrGj z0h~#v1IrcmbogqO>C%QI`uTNrl5PKgl!#`pP_OnuOnbzE$dF9;PZfJ`zO>PSE*{SM z^jELOM_=M0o#{2pYWFHSMwG$KQQF=rJuhT*(;Qpg#ir1KOY+bwJJ}bzPi;^2cy87< zA_DczPCoM_p~71Wtz1cCsbpdP5dD5dx?E$Rbs)>P%Fz`2MEf??&M3gQ(KsVhqPsS9 zMK(?@fbXg(TD@BbbWAYxCl0 za>KUBMyGK>IHC?(TW9^|g5{rO%RM=Y@#m2&;}sedkg_M1MTtWRh?i=Z(Ad-Y)X?#3 z32vo;aOdKnyZs`u*A{YuR6s*>8bf|Av#yD~gJb&7GK{pXToSMs&#ii)Nz5-Rx|noWcY z&Gzi^^M!T?p?8b=kNs(E&gRsn3gpv#`G<_?Ji~ZC8AKal=L~>@~~c*Ud@GKuI>$ z;{8lEZ-+VHrG8PQ=zcFvLbhydwrtM*iWU8Upzq5&gGNqy}&@#T>$IhUaO zHVu0?6X&~U{R#jeO@%lQ5Z{3p$CS-|=7P|h(?j!+m$9D54?goDJq{`-D~dMWQ;38X zMM=m6ov6y5&8xLkW+I(rS^4Tm`F$A~xIk&;cm6m$sx|xX<27GDGu7B}LZe{5|Apke zP!x&mg7UJzHSO+Y^PHA6VPYxSnzJm{Gsn)7zD@NqVdiVS6We+yBS-zY6)AyFVvIeJZ* zm(1_Fcn+@Xb>!*Lg$*#P?nUWPI6D(B;KgBFuG?NTVKQ0{n2^-b$jI_c%tBZCKYMe> zHx4zcOWO2S6z!uV%~6!0j_DDl6@9kgL^~}z`CRpeu|#+MbiLOX{@zeKKA?EbZKC48 zXOK2-?$VdHaevoka&(js6MA7NLX6#fbDWQQt1aU-aMzPW1NX zyIP@?#>l}bEh=gj>dM@)v5j(pweVtbC_{I4IQ#RfXx{jFKy&b({2e7|BRMy?ohDT$(PDIhYWyc~nc^p9K#XS#Dr_n+ z{qy^G2?2@u6mw(K9TC2mQ8`C=tItu6 z34DANS5kz2=_5f>4Wp7lC~kV~g|;d|%)5rMB@YfMy?`g@$dxFb;2aBnO-+sCNIRp*fad$mVK*n)W|^c!SDS0$ zG5Qrn;Cdcbf7+?XZ|IRb(=2|D^yE67Dk^nqmE!+RtM|2x?#cFj<*)ssKHOb<_)06g+ggM^7wN*N z!myNRZSvb#15BV)|9;8S0 znl-#{ti<&-+CN{|s#oBuYW{nAeVa!bb#8Wa1Y1-Oy0~n6o5}A~dA;arZl)h_PtD>- zvQe|!YTu~I{Pc-9i;)*)rPCisQyymJJ~?^m6DKm_CJC>VtY@PR6uc$${YH|=JIp69 zy3`+$TXmO>!ddGGjs@#`&3|z>PZLL`aO2*7-p_xjHiea+MIfEJVQyNgwT>?+q}O)~ zO~u^HQhHRG5jM8F`rQYd1o2h0@={JL)#s&5H?EpLxRrJ1J5$XN`!9Xl8%EvI4gC`0 zodP{y?MT)Ml|u(0wYBiZBy8cGHOqj+?^Ed9(q?Veh~T4i=K8)nFE?k8&^+OGOAPLP zQhPXIH5NYd4O7_}HTcXKE2pRuva2VPvn>~Wwu`nX)HgO!5fACZV|P7-jBcPaC@2XF zb_;ex_4OI=3XSfm9?YYeqcQbtL*?uvZA{wZ{!@A3uhR<7tC*VPquh`=m`!*Xr(gFY>U>7?JxUFkU&{l9UhC14ihW6uh;dQmzf-0 z#2huDdsrEpG^=S8{64OfcOT+5E`i&~P9l=$f9|zI8$?HI<~7qKY{?3{z+l77Lra6T z@&;({FDcy4R;q39BMljHMUjH%IXIK&eYgF`L~hv|IA65Erw5KRbn|a{!@9ai8?V)j z4a+#eTP5%GnY>z4?$heSGSoU1Mb)bF{5g_w{C!a^lTn|d8Z0MdQ7gZB!j7N_L)YiquZl=;*_tHnO(6yY$P)s#l^ zDV<^`#$Q7wjD@pA?)(wHpb(-VBg3ecl~hwDSd`kWrA<3qV0C&oo{)zBkXk>IwqvQ` z$G*DU|8y`5I-)ox3lfUV6D@LtYIawjf>u_<=QS;@J3X=F{_CT!9xJ(WOMkf!IzGVO zZXuVq0b&T7`6&zif9*q0G#lkq$wae3Op8oOq$(SFY{5L#l{3U|(9LGig5=84{g{!y zRkc178dDipep-tg}F^#SE{W$V=Tl9ww8U|eg=GvO?Y;*pHHliK{;vB;Tl{BKAA)wsot zMG$E4rb0YTg@o(ZO1IXU5a}Y~;S`&6vCT%*L$anv@0VB?m80=)_=RqdxP4%$YtPew z#Ea-FN%A`RCwgKRIQ@-M=|ORbC>%%5nSk8=9;%=G_<_u2XmIv{PXpAoyX??Y`j$&(S$9YuC76FxQVKbw2`o#*jgI?EK6hH9jH zTSlKa97tmjUXkuWL5@l18ZP)ixvP5l@8=8eNTL)K7I9Qtue9yXS-adzp8c=qXk})G z7WQ=CTn9<4|F~!5ivFXGkI(eAFX<6@vZtG;Jp^Xn@!a3TPVxW^?Sn?7hGQO1r7zDm zj5{uVdyHJQ5DV}F;2iEr563MB%R+0@yq54}! z6CHj$;}gfO=^be1*Nx1bTG((GRzIv`l+Q@Hfh(W|S<9!c@6X6I#BSz(Gdbiq`Y`}Vnj|U5 ztn_QCC@o`)X3v_Tn@Bf6^*oQ4a{YbB3@XQRh&zh2orh9--wlt`rxJDL`>G7^(-!FZ zozq)6YkiD8<~lKz0tKq%vMb@7bpi0{iht?vc45*dlcPx%)AQp$uBl~&*8z;epm1k@ z+W;47f>t;F{JM0g>D@a692*s?xn>wc zyr-cGN6q^!S65~E02|2l1!VF-B+X8)Q4|o#!^Zqd_>-c|*}jZu?v9Um(rr*VR)Q=Z zHS$UNLM*qc~Qe8GQbn#juxf8K^7|O=AnR;qbv8d-M-6xGSs!Y!5+jCu` zN!QnjNgL8p#6*?yPTk-hF2)*&=a&`r8j#G2SBAR$1yO#N>q9b?=ZTcb+1K4*#+YnW zZQt8nlS~jT<7?;dXQTU ztHqk>3~9eYeIxT&ZEUx1L66y$>K>L2Nj&!!gMMWG0pFbT0Uzw+FsZmWO)>;lbF%L< zTl7HTAUP<}LtBGRcAiO*puZbiOp`cQelpZy9isiR{bfsqdj=Fp-@QjaIz}i2poQwz@0`)4g*s1 z`V>dsG^iK-{RSw`sj}hVvu>9>3=S(`4I*;^DG^ku9bv6PJ3sf1%zM0xv^@HIrp9u` zoY}|~s!nRme(HL1*OFgCPn%p>!QE;AFfA6P3v2bXD-Xm;*EZB)t@DQ@q)JTu4DDI& zz8+eE%07Mo1U{U1oS1vs@S^vWDmkiu&yL+Q>~J@-)@JAQSIhEeuTMDK;Ww6$c&n=kB;F%gc(3~_+8)7j6`N)9**PS64MhL5s zEBq;Ie3sJP`J~miJ>t-I;&iV$RfzCM33}mjjb7~-QnJv?>PK&w3S@Q_l(OaYmZhJz zTU6K@^v9LzX!{J=NXeU*g$Vt;l zf`}eHx-<;YdzTydS&Qu~EI%PNLCxFN|Mh#%lX=+IM~&kfKKep>ab6bx*w&DW=Mw8c&>JC-qgfIeyn^lvVGC);)?nQj{~A5 ze$d9Bu)r77tBSD2+)}hp4ce8V8d6Ood=Dn&cwpH$&{}HLHSjofDf}g5jr(c_;Ku!x~swAufRT&A^l)tl?qT!W4+9?g?;f#Hm;!N`L`kt|#;$He% zU_!b{r;v;9y*HTCsqH38d5C+VW;6>JZ{UC`831$OFvWIPW##$npfakJ4%CUaTd0s% zaMIl;#MzFSKiCSoC7jfM!!}r(m|8QFk-(Ko6l3x_pN_s)jPVEm#X#PYlwa0n)FH_=^yaV~mRb-!usvkzwTJPH2?QNG| z8;{;lYjVlKgKc5kz@(a5C#_oFcO{!10Vu7}?$Lptl^I3uoKfr03)EA9nTlV?^Re0I zbUs*!z+THq444N~T8xkGL;lqVrtX_!oU0NheDh9At9 zFDYiY-)tD6XrAZ&(#ZTu4q*;~=D_$%6bsVIum+3!q6GRwrS0tn=MooOd2(tX8{SmX zejNY((+Nt2Y(c%kop5i2;5*JAT)taML2bgq5_PM%ji*w>+oX069z%NSj;8TP`V1@t z&R*bA@8P~Xc;z`~{Y}CzL1W>JE6i!#UaRx8v-x(BTPx$y?u36cqIS2if1GIb>?O7? zByXzLN2!2fkrCf_?%?Xtz$2dvfFmQqS84c)R3P~l#N6;`3vafdzgBM-@wk6!Se4H@ z3dSKePc3D!bIj?CRaoQN@1%Xn52-bY18L2`#Q|R|VKzfMg@5js7IAdn1fzZ9&m;W< zl(nT7nuZaE;vJ~!sqc6Md5kETWAreNGk@{p{3)R|5IM05F*9?qrPh|Qp8Rx*S)x9r zMzN;2>be>c4TJb{*tuzAN1D?juVt&j+Du!8T6I{W!Z+NI4#z}Yr&uN(6;<1aRzBaP zoMuVUP&C(0kuwKRXEO~}7B57dS@9={3k5_^uqtes|GZ}tKnGYB5VhMmm0Dqe^IpRF z0NpUPx#RdseCPWo#yd%W(m%CF*Xi%0Ko*yGFjLwqmTfb)HKZ$1o#`DQ z8wzKhD!2nJebxq&PHm@ii|XGz(ID}78>`0wDJ3CDZZx!Sfg~etWuo-|<-G%}SXbDC z55`pre?fc(#9BX8oRwCLwX-|d+x3+rDS5JWwWvg5NIZ2~oks;a#5HZ4usa$qZeZ5k zaC@7^Fuhp(#Xh6Ca&MLIZ;z6q?9I- zF&*K`Wtcc)ZNg}mGn;!K^_FCt&B%ZkqId1+{WnH_2koId&UQXBg{sUTU`^5{?kr(f zWzGflTB|8f{V)yc_44!3`6G@7DX&z4Vuvd5;5^@?2MbiA8KHP$AeU@y@bl}4_HqnMxp9oqOK4foxC%&;K|a-!#NK@SjX2W@F&BQ^h`#6_*l-h}Fn_kRP=E?G~>cr|}1ima^DKluuExxupgK zVsbv=&IQ>ZT|p|1m?nIoUdJz(FF`NC#8sfepO&(v2c!-8fK#KL9-iyvEQ;qtY?H3q zLMM+_*~saxxUd668Z-cTEy;5mmwhT%B26*r1LopHqLled0Ofr4VtyFKu;JS>SelwPK5N1)RN{ zUzg>d&XUU^$!4C)oRwl7k$fvs6RY>H96<2;`ruDA_VJxq|4cb#ZW&Ub%zhzFuufU{ zW#JBn5VuGMii%a_g^G#*R~7Rr0zM{~f2wlC(zhBAK@xtARc71y!2KrO#O?|w#hz6p z;@!(iit@C>fct=%*O8kP6m2OjBC+!<6EW-@MRq#3QH^)NjwP)S!PlXAJF`p-xtStd z+f3aGXz(HbwqnZ{zgaSd$?yDF2Tu(@J8(4om3J4?-o4{v)R*PraZ7{?K3{RL(Su1z zr)$=x@beSan`Ag3!wJrAV!swas~mZHY&h~BBN=@bnAoQ8HXD-8=8g~R4aX0k*d=kQ zjQTU`U6^Gwix{t$WoNa{c&$MCDBH^Du=`KU2SivNKcLAAmY6`y<%rgZZEJBkLfm@` zq@*{8EE}1MVVBc8C-D&)RZNWrAk`QM-W@27IWowFolg|y?AZZ=12|aO!^iu>O;(B{ zI2ShimJEhth~98g>$3&Sf;2OFarDjkkcAVu&*ECBSk}vg_AK64=W@!y_;bhQEM8*4 z8H-Y~d-dn#L$+onq7$M;U8b->q}Q0BxceCf1`4q3=gR{ym!P1H`x;;LPjs?VL&h!> zH*$cdmy2v-r6j0M0xsmHJ=na*q0qAI*k9NZ?rRhb#z9*PBIuliy&wPf?ae$zy*d}t z8yoq6PDvlc8N}o6m(mIVYoDWnuY!pEnZE)+;hMNe2KjXj&T4sSbAC+_=ddP=8`e%=!U|;sttV$4Th0*x*i0^(1av?&A|HBH-+Mo09DVW?V6`*GZ^u&Si*# z^v47F54W@#fU-ByXoLZ%ZEr{Zi@TB2pDhf&L_224wJD)LMh$(siNg7BH-Pv69bDu7 zGtOMCGE!2>qyryYmsSyitcCYgmq>hFh|pwZTK&h5J(Q=`eFgUN_KqjEy72w!ti;_y1FQ5`$B>Z))&^dLozrt% zv4eeSJCR;f$X>J5>p!S??3`UtuZ*;YgwM-It`}Tv40?&%&6>e1Y60EJ+udTwD2cdD zbnvMuKe~5?lu7x$v7Id0z8&mDYU|z!DZE15(VQsYdtf`}=1&Y*ox7h@+nF1E;5!va zO4*I^=aIaXjua9E6ldaPJ{QNwUF4!K&V}C{jDYd_6jh}sTy8hL^POlJdTs;wX^Gbn zJFw28f4gK}^gjT3RRc;kKt-8JHUp?&DK*LC*Tf8uJ2Z9N6fL>9&4ohLa^ zv?y=}o%5aA*?9k-ILni6yfUR_3xN2d^P1?k!AO3>dI2SwqYM;`UF^O0KkoB{x?ZCch>DHaDLYV5Uk z8f;tEiN7-^2?P}Vm4Wa8*j!F%*xBPazg}HXL+va7fROCanmw;Ni~T(Ct6kV14904W zl&jOW%@bwf9Uvo=O$9T;M_R#b)VP!Z#`NJea&r2kUJu?ff4IfG zfXL%+BhjEu)^|PQ-b97_it%vjR@%O;P6V_Xw$uXHu(4MZ&EchaEsY!E@fmSN$;AQ@ zRj(6j`46J=)91zHvnU`goCN>R7p&Hn>{NFXY~Ft{b56H7NvVC$>9Xp`gSG*O?~@nh zXz`Cb;A60-wg1ciX0q5`s(Gt<5QZ1Vls_&pj{MzvO$WPxsOUN!Y=H>2RZabDTyH#1fu!WmXlO5 zF;Np|dvbw_Q*YGC&0CzdeIZ@ULq~&?Fy__~&*B!pD8}_w+$R5*(~X?iZ&gW4uH^)7 zdOdGuqL^sP9*zCc;lQU8BI`akG8?0%u)w$r*wqoHfL)zBpT&p-Zv%q#G_OfN`8*mh z^lxT(dBtAI~$Qr4j2x`(69b@&u=s`kc@-!Xovn8~VMYq?*9jFRNqAU1D4DMm z_mME*Es!8pFUap!Ti78t&V)aCF}05Ov+d|;?{ReyQ5oO()q$CaIFyIuZ64zzwFYL|-i=&ALt#d$(!BVn?G4etm9 z5K3c{NQKg^!F5TMRv~ANicX5$L^`poSmByj+|gkU>9JKvcw%Vv3JJS<62^^G!xFnVWG!Sd#Jc7t5Bo)NVTra&vJpq_!|&E@2+} z?K-`D;loyU8nSFo@CFON1VzbVrdy|Uz`-0*xGfO@Nl$%w6s(+cK}n*|M$ zNRgqWK<1M${E))GWYf`Q-rw6Yz9UtzmNryL2Grg2Y45WO34c)@g;Yrk6&1{wm7B*3 zYp!?<9oPLAmOfAsm4@dYi`8n9RBv?d9u#xoe`oIM-h_0cgRne|LNvy!u zTyTNQQm(d0UG;ZouP@a$!)2X-+u7qQB7N| zpF1~rLPz{fr1FrK*Nc7U>blh2)@i~kg8$g$a^IcYJ=?Mt=1^aKj4@+;GG3$l-39soW`c&^(WD@dd!Nd^~Qr|9B+;H`Y4jW>q$G zSsVJUugO-ARK*o!Cw0A6E@7Lb9wYh_5XguO@R%%DEREb@dHV2RxxQ~k`0~nw z%QqKL{>T+N7r%$|=6+`*-FT(WtpG|(e~s1d;cEn9;)5zf^ z0|QV8otX2Z^Yb%q#)l}opZp=pKAN4rGC*ftotdf$4z5Q46fC_&aK@J*puQBux-4qC1Gp`eXjhxDir zwm?r4FnJ)2zh{YEeq2$6Co&H@sVN%;NLG75c~K`TC8Y4;gxJBOB&k_z9{D(7D?`}1^w!1S z+)$eOq5kCZq5ssoYOPT3v?5Q{MK18C(F8cjpv8dA6y+d1~CKE3VJ)9dod3v%oz(=n-tQ&wz_dy1zp^}hW42h$eCjII&uuDgx1i% zMx&gZY(4dNP5qQ8$Fltv8D@5Oj~_ITnV16srFe?UhBnK``SG(o{zx= zL{}Hyr&`jEH9R;5NrenFZ}%up-pl*qMSD=cdHBPzRtL4d)XWg`WIfN(=5u+2z&Eo< zvwCp?)QDR15xh$CE17a^2WAGyuWc3W8LatEiPah&Q~V?HFUhMX|BrhMpBCYL@)v4{ z%@n&9H#48l2`)yH)PSP@*sUB1X?o(3W`<@j!=k+Md_WbA-4s9j!=uawsOVv;4xpy^_lFNX6_Q2Qg99k6 zYQ~>-^X*#?4BYDNRh=Jb;?TT+_XnzCw?-Ug<2**2QJv0G(m{wN zG0Jkd{=w09G>@N^W5RXunmJikrKrpRqAS1DPD2husitUD{cWpY@88%-tj9yKShb_O zjz?`9bay8VZ8L}Dz`h3Pv9I-KH-aPH>fuLc^pCeSSPTmBHsWk__yveAt&i%#Hx>rt zR+dZmes*w!Fg||Qy5m}uxp>GxVk=;gM~NzcibX*ONqjV=lYjK`4UZewQM)Qo9W5Ea zZ;O=en@Jhy5BJYDSvrg!H{ZUsI~ZmNzqtba!(v>TSx zyzeC;TN#v*w&j&2=1PooH>Z|J!kf_pbM^q>?Xu`(lf9fDT`c^$wpjC0fVcj1;p&7V zuA2Uza}aZ)PX9Xj!B3M z&sWe6w-wP2;ENbG^&2qc?#cFX|5A7^+2(Tv@$N>uZr)o#2{EE@5D2iWw0i7d`HT;$ zF6$q?uMS(K5Y7S_V3RW-h_&&nA^11`KT-nr<~r>Ee~5gy)zsD-ZQ$YDU+KISqbW*! z7~};!J)Mu;>*-A|%pX3DpC35pMd%2BlyufWPpdbPK7IfMzJPwtnMvmFWDr2Tu+h%8 zu43MI7lU-j3u|aINJ_K_MsJ;{L6b>i&sY+@o^Q9FYh1_py-Q_n0@RK0s7<(dLOBwV zS5(XcGODK53vP(*1|1v%@wT+UqwpO7)&-6eJO^vuE)N`XrUTM;`{sMp_`*48vs8`R?fJP=oZq80Kj-C%qUe zDVbos0pagNnyiI}GD8%RMcIwm>SxO+$E{g@$DMv9%oz5Gv#k%e3##p;Rmgb!S@d44 zzq6{H^k&^rN^JZI;5%6@Ye$C5{2|x0|BGCc;T`80-D|Zqneys^t5w|@$Tw5GMgq8e zZGHe83DiZUsqo+V4>k`cVDs^VHe}a@G5x6|gMal<9etMM`D=lFhQA*Jf0{FM$?|_9 z1{@msW0+vHjimpH>yPT{K%UPu1GOPBMId5mH`I^sn%8vZtk=Z*j|S(SRC^i!gT%DMesKCc6hxLO5R{U12fkD)Xyx)PW!X7QVy-I?uW0K7kS_Xxb7%50C6-6S%Udxk(d{X`?U{i$^iu;Q(`seaSC%6da6EJLDJ-r_=|6pdySK!(rCA(*{+(~V? zGN$S1;_3WFEZz&d%kVbx>_5||Kpv&O@n8jy+zx=|Nr*ev()rsSLimrY42E&}zog!!~t0<>6q z5#ekG$3{6@&Dc=82!$W62E`ksjyr>Q6KoUD2Tgh4=q3@l9=!F$uK+pW?BbN>?m8_) z_~ZG3jB6h0_mO2&o;dpq%u3M3Ez%{p*l>8s35bm^p)#TT*cxK*+3$$wGqPU#Q#$#T zk2F+LN~@?rnnwSjV+9p|vd69ql74Me(vY-cphY)G0~C{htswceI!9jY4QlOW~@`rne+AA;R@rgC7Q#lM3KAAkR} zSUXz<1mUm8K)MuAP6xorV8JcRS^rS|-zY!l0m^Tzhb>Ot!{uJgVQ#m<5g()XF7tM% zRe5PCp6vcVC_j+Ut<%fg+iQ91-~veseo=M+0QhpOT34khCWv`WxmyKcdGQrQrmW9xU*Zq8RK9rK-KOu&ZfY-`WIA}84W)h=L zSdzX1j-J_R_4kCcQv=hT*Q|m;R(rsN1;($^L_su5nTk2-$R}U7ctwt*Z=~_#M;AJ!0zA>+%97 za$QKw(?KeI?lPIX-rIkL8&pUoySJ-Ag4M&n8YutV3?r_rZ?6y2zyMQ*jsI`Fk=v6` zFZPS|%`xs8NKeSG@k1lSfy!=l^YU^*1(fq2R9|ZAUknCS{GFtK<`|NMNs+-b+IDu> zrU4a9Nydi}`6D*H0%&8Y60)AY&M)2JS)sJ?1HWWVzl7QZD=P~0==zhOKj8dO7yL-~ zk4)m&bcQ13iO;q&Ftbv)Ji_!@5XS8MLne*Qxb zAa6I)5J3D=posS^4*juP#@JZXC&+6zjfCz7u~@pH`bp$1ubGjwI*F26i$r@>yZdAY zB57XnRr0%k-eT0+0;>3gt_Fo$qpMSXbGOF+)GA*;y6i;J4`LDBbmH5enmWQU(fKAYNDE~ zFEHE=MYBMP?>YXdel@X3+a5Qr`@O=^7;uQmjLjks+}Q;KX27&;0|@4`>XYTvJglWveOrjBjeX1PM?m^I1F|H7a+CcQi zzUUTt`>s=feUpFNn9`412wZ~>!hNprc>4*(V@Aw3)DO0}FUL$v-)lN5k?eWKA#1fT z5i`$5CLja>)Y<^0$!&MP<(uu~!02S$p~4k@t@#FijDhWKaHT&}{@-xKuqAXa%jsLg zkKqH429%DDf6V=g{|dzp&ZW_vlv@i-P&WG%qyzf`E@mb0h$$rM!?`!VywZaDWcAjm zvcUKuzU|4uWyB2Plk_TkoY6)WD>Uz!UY}6D*PpD#Jqh7$V7GHcbj?*7A-)p;iTp@L5SM^3#q&R5%FyX-!|=6%&{ZniP9lpJJkhc?s_s`KpQx9&0Y-!> z&Z_dKY5zP(2rp0Ez<8`h+ScS(ITI#!q_}olxJ++|3?Q0w%f$-pk=^6ZnH2c61Nv5o zVGD!M8jzp^amzK5Z1Ur|6*LqBOIv8x!siV2N7$@L9|-gTlW6$s22dxafY*b-i5zAKsln*=DZ*t!99mVpNmA#;xLRFkh z*EC~ZS^~v13Ksnp*OP1+wE@#MN|Oq13+js1o?LeJPo|@x8#EFa4glUaA_cLJ1Eo z4}7N9XFSVg{>%+t2cLQ7#)YrfWnf4xEilxz3yQ2&ZftCOtBv2cJom>RBacHkjpg<` z_kqYBUXse(YX^uO*W+7F1lsNcEnCzl68eF!U%$p+$i&d^_g3n071*C|CGyC;fBMC$ zALiXO=f27+`v|){xDpA9C(|vB9{tgLk!7~oMZHv-+#^Mtz!x}f27rBd59o~^E`>a9 z?wdgLWmm_jn(@?qsu0@_%ff0$i$eD*_LXsF#FD-RsOT|zNDzOqR-f{SXVE}^T`o?D z(;q5ogu&Sbyg)%he4yc^P<&tkO3(980Rup1Ja}I1^_JB*ZNqJ~>Cm9~Duy5~o1%vDE7N;6xQhL) zX-25W)^aasXHsD3G)`{lsf-s}x=v2clulk|RlBWYU@-Jn9DS4082yUO)5!7s!TIy; z__H;%rm7jB)LwBQj~>(4UC4b{PL;%&O_!GNGBJdj1+**dP4JS>0a5?A&%~8v6fA4X zFN*N~-cCs4RLjr)lGtljN$&aWe$0r3AY?cq3z$8>KL;d4Yxjkhv;$@}*KRa?%qs6x zzE-N^j~Nf9IsseMxE#=M!pCI*kx%k8GIZepbD;@z!SPd)Gwn#C`*4hfEPN(%TDwH^(gT--y7R;jUXuI;N%?I z@7?saq0nEkxK;lvus#bf$2A7z0y7Q^KqXU)HUjX}#dyvFJtyj19R^L9BC~^T{+@zv zY?gj(KRRI5VBWg0H46o!*~Lwgs+Uaq`%Aup!Z z0l#Z-C#Xo%l|Sx315Hm% z+r=+>JbU&)I+QMdp8?Zwf=th8oIyGSFDM1yHx|5iFKc=iG2P#_wPNMoEHb6M z-~7VV^zr^Wn84>=2S&5uB7jh%GMLf(bRK)KV<=cefj@M>TYDS!Yz-s{DwC_Mm@scB zC;R2z1E0m5gCPC`xu-}Jzfr}D-yqhXpRwws^{{jBeL1_jH=>A26} zFos_uo-}dHA?cn5I(knY_WK#}-<#zBo8i=bY)q>RD5(A|e9Y)AVGo&q`09qJtOT}+ zeW`WSRQ^LshL_8KqRu_QE#oe3GFQBrI$gVvB>b7fBUTLL`koSxV39Qh@R}CaZW~lv zzjT-X6MuRdJGr+}OLu{)6>AKPk<5p2r#99Vj^@o)qJ*@iY$42A@)9-_} zYx0D)T7@4!%svX0=R|EMQV-XAFEinyG8LBEjUDAlm7@Bpe)&}uVh!3xz=5k8A8Qv} zUGi$n*#yI7J9ni;~F76uUTYooPysy#P?3gLWMcxyuC|w5Da#?Hnz-t-snb z2bK#`{fpq8!cWh=P62SJ(6r!ZS67#;?EX_~3W~}(m+AezZNP?O6#Mnfk9=YkEEjc6 z&8QFfnm|R-^}lUvR?11?2iRO;KZs|<_`FrRIk^l5_uky-Nh$`48u&aXBLgMW>#2r^ zgM}|Pe-c5hEdchsIzikK;=4Iu>)nzm$Z!7iz5^hd*qPW+mDYK1D0 zqJd-PeYYf!oLqCxGnY;F+P^h%hqORZss2aF6IDUpGuX*$Y&M;ukW*?IO< z#Yb=*mcm$ zVEvA~8-TL}n-+lz+w3VI3%mI3lb`#{J7z{`cz8Iq7XCDc7-a${w+19M=0^K<>^1k) zI?l74e6=A^@C>gpQ~gD$!Wn~tCx|18GYvmFp1^}mI)4+y2EhUmoH!ZwKV{^UV7n|? zp7Nz7bZGlFFIzj@+Xw@;W(Wip!Zj)u&QNmoGbbiTBUSBgtIZ&IUa7lMEPmL#h_KYG zDDG$k22hu<9UEL!0o!Tn*LmR*zRlv$$UlYU&ZZk<Db2jg7|@yFjH?$EV(Em=hcriz(wSE-pjU!wDY_>P{Nk%dR$KM}D#3j4W)wgjfVA z+W}%G#aQm})*7gN`4R$7+d^fcRsAL!d=sTY6=85EO@jQg?Pzu$q~|)}AK^{1%V@M7 zRm|RiwQ{?C9}33VMun_ieQis>I6#YwAJcCsuVw_Xi!)&bjc@{YwCY|uzyq`HyyL3> za*YQFObut_=MtfRg4#t7BDp(K?3#eaV4$-5VWKX}BabUQ=E9^I)AttjwPP!-^MdF~FkRHd=17pC7o0fl&>JIh zIuHnq`1%u0EPYo>0QG%ylTEhP4opoXS)T36T8|AbzPhYlHU6fT6c`IHqGv?{iuUlY z-XGXc?Y(>gQFBlgdHN%%ga9N-48I1OGW9t*IZ>cF7wC>LS@t|ORXP+e_X-EST+{Gf zA)`cp3=F8@yzK1Onywvy-y`htF^XkYXd7M;;A@I zmg+;nc~W|X@WcEZ8fon6uT5cLAB}^Gom2Pl6+6F*v0EhW{!R-COA(|z)LPt-Bz>Dv z5yk52YBcas7F83m5sXM<`rYU$aI`Khc&{!*w3?usL1#iNaBx?!5;36fN#;d*anIga zuLY6mu+p&$ww~vh@ox*!{OnO6A?l?EL zdG#~Hg7n@D^T$(TcZHpGeKJ9ziIU=ny_>)x@*!x0>h|hcKtpMEMLHv+)DU?}F4Ax(QjTtkYkz}lX>EIX^o#0Y0prrv+sx3V@7df?OG=K$1Jqh}EP^ zt78lA4_y;W@}rH9)w~C6%xz-NJ>O~_JOLA5Jk|O*EFQ|=JO+RU48eJg=6sC( z8P1IH`#6u@eh4V$wf`;~;^&t^NRXqp_7oDN={)w#s4+55nW5{Gcce^iH-tzVzr8 za8tRM>z`}Go~O#;Auer14vr0fSAJL7!R)uap2+)o=8jR>u1EaQ}d3zObM>9`BSzw9LSDTZw{KK*Oy$M z?K5?Bad;}_<1_sfhYS0H{~5gPLp|(w!9J#bXl-wGEvLTTr&9uCezdJHpUAwGKm^>w z6uNHavtOB)6^+tINKZr8rtvAnzlMMn!cOFX=&6W?9+!wYJk+bn0s1c^iq~`mFIJkd6CHFRgL(;$(;Ub9$AHW^S{8bNE4b(DA|44vENBoLf?}& zll86UE*@fKwJ9m);n_%4ViDwC5rq!-?;v8!>ij?%OT%Z5k6+78c(OHta?Aqi~|(Ug3EW_M3kN1b#@9Kr{Y_kCi~vqe0G%EG%ai3B&U_L7{a#GJ-&ZRy(R@5 z&6dQNpn-?{WcYVvB#D&*O#)|QLCBRg)v49pvVtp+58Q{QrsubJuSA9qZvOtY&#mt+ zwtq*?E3Znaj=R`{C$vAklTo~{mv1M3@RcPg>Og$ip-lqB9gq@&0Jf6italwxN5yX{ z3C-w^QSS1q1qN-SMQ_D4wT7osH{BIA#^i=_tRa%&x;zA=a3HWu<7u|GRS+XJ*)7+! zR!W{s+o(Bnhr(@bn|Sxh7>woiWkNmxJSaxv(pS(Y`uDevv3r0hifo@>fkm_qWD*GZ zNv&5u%j_2+3?(G)xqmHph<0;}s8OGQ^QIexXI*~%EMvt-^lbBDL`YDVL2nLN&V%vK zIu3zAaC13riZ`map9Lo}f9<|z&19B&`J=e$XAP)qWkT1QnD)}RrpC}Qh!W}Fhh*;; zG*x%h+Uz5OAArWajh&N8SJUX^Eml9=x*6d@lTNEyKUAYY?&Q=zLS#HcBsVv`SR!E? zJNKR-Xrk%-`=ioZ3o)eBmWGE$7t&+0_53_`qZ~ID#X;xnG3FdW`ar1Z4tuZ<89!x` z0VSE;?d?RU$LB%qNqcjwg+52~#1RV3&XF7(@j`UxsaA~VU|Sn2%tW@Eq75oC->Q8j zB)4wsnsB{qNuNQnZP^J(jOIg$NEdxg_)^=ZC=T>4*6FG81|szeB8MD-68`Ha+bZES zjl71{>5@)t>Zjk!!K{G3vjb!N;86Cqt*AV?3;ZBa#CN29oOR|nOMJow8P~~H8`Tp7 z?Mt-M!Yid?Kw+<)(2msHU`PkI-0+Y1*2pF=C@WM6Txj98k9em`HANTtHt~Yew(K~q z@xYAy{yBF;IhA4f+EAxi!LXrWrLv&y>R^?a4gLg&sGTK%9;s(iWuVRB-1ain#xrKe z?B7@CpUe_z3R8|Na~#F;I$2Cp;(u^m{-LXIep&N$HmHOH^9*4pn|*qtnZlVFM)fn$ zLGC^>jM0k8SnaG9--^eyNP1P<+hRKeiP*6A7 zdm#$Qjz#=oF*%u&;PkBMpi>w@BfIC`1L{_a)azXNXC=6#rJC;9zpRc_JEtj_&U}6n zsbRCMr#cOGwtPYI!-4cxh%o+(AShGzGeC`FSUnU*wGkK)6vieWa#fQiZ7N=db++1a zlWOk+H9-C>i611XLX*NNe;O3vHXTFFe?#*$f4rcic&Y9rq>Rv2C8iy|URzgcQ-5(D zr%fNeu9yD=dg7E}SeXMq&YS!Dy=oKZ8aQ%kh*Cel?PXU7GZBVGF1{FA|%Eu#?< z)EAO^eK`Munx$*iJn(xCuLho3#k1D3ndxe>Z42oSp7J~qiwvj$K6`W=#o=E>^IPSw zP6_7OtHBPgY9H*JaK zdua~EDR2}I=>Tc@7kW&zicJ(%{!r~>kU+ap(N@mufsxzUYB?d12fzCeQ*<}F+&V}( zcOe@?(Ol~Bu8{*IuaeFZz8|7B^BUPbXc83B`qd|-iK5k+yq*&b&tcT#i~V@!0j`~w z1pTY$>+3G#Amu=-ff?DA>@-_Z)8MWpi(F8z1)5R_5(0oXXKeUvMk!JwVwKv&+&_fO z;%m&bJlk(S@*0NgM{cjItT^Z4k6}pRKoC9M)(aYzW_8$UF5s8O{dplzAKP8#%l7X9 z*4|2pr7-Ig<_Z0S%LUke-sMkdR1f(O9eHG6)+QWaC9anQ~lJz<2PSB!2ooDOGQ>JVI(@z=+(< zUY*8ciQYW9I65!1vYX?t;-E5(QEQFy3cF!^uSU>{qo#NzC6F`5>LY$Ud`+!AL!Oe^ z{0*=>-`m6v(U1~6A#{NB@Dwo@Pi>UkUn?O&54>Z7lkWX5uy&c z`{oPRQ&uU=!dQ<%j8rD;y*B!y>eQN|DJyiT6$8CDI|VgY%<9&I?efN|@Q;Rc+OebA z))=fJ&p(_`l?ltEp-iWhrW}q`t**CmUGlC0Folvj;qH*TY^o0j@*_9gvX$Pa$T-jd zE750>Jz<5Xk>?NwjV~(GzXTWuFX8u_kz@>yhuXNMEQXZ>7u1^I-Hks&f@~*7QeRk` z{KX39qhXy8VY8pQe^p~j*GfI;_>S@otJ#|kXPZ^~)>*eJIe`WCr81AUON^kd+MM)e zSEX(doO+qci9q+cB_0I#g|%8%{2J#xTHYl4|4zA>$i++khU%;*I5>fI7(3$uK2x6H zDnTG2tz7a7aouvMsrE0iYCE-j0=J!=zCK0aCq9P|`N&QCy=_8|=6g8Zp}!-%EDpq8 zs5q&>DQPO|sIt z6~~g*D946$cc8`RHUkT8dXKZGxrvN#ZllqlAr1DuEdn$27^jV&4BsCrrD2}bEad)F zcTQ#LT3yr0y%AuTUlzbf`s?xhXQdU}lqplZX(2XihR|qU`g$M9zz}{6rOf!|J9<7W z>}CDQ?mY=(auMp_==% zriwh^9nzi0P!+9nN(F@~cUmLos8Vm%X9daO#p?OU4|f;@5*_*AJ{7I{ z@8Gl4Z!@E`sY$YGY->e@g?9ijX>__%%X;da0lc?s?)$%WdeeF_a4-R zT*J>3JrQuA8Gd!5FND@!shUqra?(DdV7vCEn^Wlz)8mpQgcC?^de2%p^g@tkQx-{o z6TCSYHII4Mr?q8$DVu}atN1~)%y43*isrU+Cq*IH^S)!m9FvpxRaSazu|~1#@d3~U zbY|h|Jf`U$jGv0*2TF$*V&sb?PVr;j1IL z7|cTdp$Q&kJ3CXf#2Tz$aZdI=DlAfuVg3v9CW=iZ!FyqVe{3Wn8;4InHoPV*uz~Te zuoD}pz|eo-Y&?bvnTeyd%1`MNU}IxQEqYdtAI1-X{uh<&m*6$5Yu#eo6T%t?j?FivD+wfZF4DyTy)98K$C;l_EZ)v>FX`)k zlR@|iVGmVenFbjRm8u#lKGF}0zZV~68r|52i|VK$tV_Q)N6>CnJ$Hf5-W$9-p8(<5 zcP|-2jV2=^pVqmw{av7%s7|5l9-!Pd`qvkLHu3lL+(hBKSR>@jg?Bs+@Cl^^P!LH< zO3L!_i6st;ss1$+SZ2|m;U0w_5eVsg{acc2ZEea`K<7;sI57AqLr&k&m{D4at=0sG zSNzQaAl0xK*1=E4NevH?}NX0~Np@A9)F1%@mMt}wSYOuAvua` z`uNm~|M~<1_Whkier|620WZ$VZ^!Xz**CV8-~D$E3wPz2iZvwY>E1H}yHj;1f`LVC zVuBPbo!*EaaMiJGboo~G0H9DEEqBO=vK#MLD~)+*#0W7;E$4+yF1ht$?moAL!Mf*= zhT=mL+Mx0?a1RtW9>yn9=WN`T-D%S3B0n&-vYZM>KfhePvy2f==FpF-23~}(CDGm3 zUhbiTs(fCbd;8S%vnE@v42;EeIU*eBO3)ZPJ=o2u9=rh25|80|g!7o*P4M0-NksM} zKSY7vbEnAllE6Au04~^6dgmBuqUA%|I}v}-d#k`>a>)s_Pc&ZL{^m!90{OK3sf`a6 zi{}l0y@s`#sCyqIz1bh@GE4Day~+ff8EWAUxkVkBGQcd<6c(mz#gl(U1h3P=Ugz)4T4&Zx z=Y=P;up5GA_Bs`|lR>)@n;pE^;OSqaV(VY8NLvFsN3W=kHlSr~lz{C-R&w%9vHjnh=h_X`VCcvK&&RW~vw*}~&tm*U z{eafvoBksq`Bj_8QRoyL#=rtI$u|j_lL3?=3mSy;MwIKK5;!*4#H8gI(*$d8dY^)@gO-%%4>w`MvoQf!e-{0; z;=QIU=J{-}!?;yGQ#{rv29SwEkcsRrV&T*hP!|C9<}!MNoRGWjm@awf!L}C6ZMohT z^K;X|#3bm7be1lt&`aNMk%`)h=Nz_Q0*I~AsQIE=1-|{!n&*NDrpvWX72nRcx~9f# zVjJYzsyEu$Fki1Iw&<`~F)y$b4Oy;DA%KMOlJo9L#c&GxlpB8N#XPoTV88}1cqM?b zLwqN-iyoSoEPUz*T5Rn|P|?r~JOo|JR>mu=z`n&I087B4*?UQMx{q06r`2u$fLW#0 zb~@mcm>9Q!jee8!V7QFN@qT^dIidAjmQ%U)&(#yVEZ1_RTfv%ie#n(bupe6fo`%)4 zA@^?JULZ>cMujM&SMR}=6gX?(=%sqjco&@&6tEy*NDhr;@XD))Z-e!>_3~wz=O`E$ zTKQpSK@%*;P*K}1i@zgRoU`#D0yzB$_aBhE`LoOs{jJm)I;?7Si$nFCyz3lTi*fff z^qVMbR&{bo;}Om|RFJ|6KBs@Mu63L7vZ-6(kV@6j(Rn_x{Ug?HXb-o`l+9BzJ0MlS zEp`9WqK%zL>}e7b@~VxU8qjG8)5ztm44b)3SuoIT82=;4hvdZHkAyY$Z^wVo$Pf53&~L%= zG46sU=)Ra9$OYHWhd|tS4lNwtU5Nm6OT(|fmUt6#>prAuq+PaL*Gt!z=19oBmi;$Y z2j-4BE9RVa9o#h#G!{toy44P>kh(XSiZY&&}g0=R+YPz-enTV zspTk3n*$cSCMk7_xTt7eW}-#@jh)l1JOTo=0HvDo2yPUE1wGTF?2?54!N<0NqJ zuu(ocUiBcs$`Wu(+X`@%KoW!hpch=(ut6?JsCQzKiat5#IV&YqL>+%< z)?)`_JpHS%|A()yfQmYM!ycDa*FaZA1teVQ59@P!ku(E6;GZ-$FG3p?K@zkNmW5vTL8tXdOTmkW*fh@Y!6Qi(0ek&qFXIL7T(An{(l# zhsG->6LXNNO?MengbdPbGLL;>s9I~V%r?jF-LVyDPJVM2=~%9_^T|RawXCenBWXc! zWTU-g9O;Z*EHH1z>GcwB0Bxk(YdzAd*`psWZny_O`_Mgf7MU26f7O@P_5YZo=uRPz zkWjq|XRSJR4sFek^2h8y0x?YM!LUT~xY%XK^LclT9$CZpDRZtYO%srlV#3-sxKBTZ z`Dy#3tf}UB(`!!q`cIes8tDK-r z?~b^CFI0F{`)rN@`N$e>wY=WoHo0s#bg1p0uA0evwYCABcySR)Tc!h5P~bCq*<091 zZs7cxx4SRu!W1&K&e;=x-kNK+nDqW`HY7}(+V8Q0uSO?$Px_=VPrZqk7Y0}gVxRZz zN3x@^0N-WNB4+{fTdm!Vt1F9c5ZAp8zwvwR8jOAA#WxoDY0E3C^PTKrucBeEdR`Nx z(Cr@>5Wx5%ZTTKtd{Mf1@xE5g=vk?uj%6X+fs(AE8o%8gR*fnLzvb(GsZF$g`r$pj zz3E8l@|eH~>~)rrHMX`3*X?@Zcx}vmw#H+OL~J+0q-b=MA;E!|L!ro5FX(UeSzr2# zN0~jlUb|DQF!R|(Wz$fqV|FWUznf@rlZS4A4l(O^hH~W8t;C6FUify53TRC80_Sg(X2`X}ygzMiZT_ zfJXV#fx9P43|7uHDF4%0dsu2^`C%$0>z}O$Qw|+{ck2H1w~IXvU%+)?!6<|n)jODZ zq2-Xg%%#*A(Y>^qv2T&@4X?%%S>%TxOr}nZnGZ;DdXo%^-!wswk*Y%H;L{;&P{Bdp z7Vi-h6lC~V94-X@+12^A2HG9vaF8T3i!r$O$yw2stlDndZ&6Yu4LQ%O=bneB#ng;3 zctTOX^2f5s_DXkmv=ZOXNBq=&>pOyO^Xei=T6jw%J0#rmXIymuC+EwR9%(XJ#yVO+ zGO$t5@rPeB;rVgCHUGo~O8@5y_Hddy8KRd?O_7 zJc{H=277is43u_#Jl|gfwJiGI4_K^Jq7U1Wk~`0Tr)EjvZ@l&+g!XM^E9?KK0oA2Jeql=3U2adskA+Lp6?ndcwLwr{Z z&2l^m2}#tNn#HHw803&uE!1t&%e0uP?ye~9s&B9#bNuz=ofSerO6p#$t*z}gX9<37 zIpF68BSLis(R>QFp+9QM@VKwFTr;IK&@_WXuIq(SWT~b_nXBckOFGCnP}rx2J6ro{ zSkksNP#l(rA0I`z+N6K^vfiu9@)x6cPb#zFWqY;-CYeMgK&d-6gD#2P)Og2Un&TL+ z`Y+tjQEDoLGt}T#O-SlBiZYkDgM#biVOa=;t`$_E= zgXe>-HqIAvce;%S=}nA#+2L(IAOZ~-p1)=eC+i6JfP1mSy{@p3YwlZdMi=l1KAbt9 zwe;;y8l(H2rg7l7TD<21jxxMsUYE5q`1w5;zhB=`NF`s}o?*CvI(NQyYa=ygWPSZQ zSsZCUPu!9ItAu}$KerUsB%-4{=K1;EMmU0lKRXQeQd==9R;FW4pZFc=h%X|75?GqL4UuI9#H*PXX@@Rf)iB43LD08XLn&uY_ z3pNd+)5-c9Kk&8Avt=YT*<;a-Kl)*+!DQR?!uS(Xl?8~PHM9)UkrZM$qNwbQotx`b zxIUC7aVZ7q%Z(qb-_`^9By{h@=U1*f8uvXOzj`epaq}+6^WGm2Qi|pvoPlEkP0h{C;Yp=e|9W7xyRB=J@mc;T z4`BuMCy{v+Qo;HLNCrfPzVEslyXZ;R5@iZqpvYvSF1q$htB_|gX?2R#Y1fb5*BM=a zbyv$@w#F5qsA&IxOX*adm=JcVbl5E^yxMK%mlbP*H#E0;bG)oH?}DiGRdXjF^46-n zyCV7T@i@w%3KiYa?fd3kmi;TuJ;}bP=Yjjre07sln>o*l=S-dj&xJgzPmKk%*p#r> zG?XlsJWoTacOTKV>gr85X*A}??8slJzuZzB#v_t_|KPADarslf0liUylQxr@VE5`5 zdAk;_85cSvq<(V@HsSvvAXO~5MM>KXde7Uc%J?yfd`ZT{7j3fHe`e|Nr~jfBROu1B zXZ1g;m<5EgYJZcBq}^pdS=IDsn+Q_HrY~1TNhx$^Ds)Hs#@`UAngs2PDea8KPF8KN z4zev&S5;vLerzLE3M>5!CScK4lw=trGz9(XRvxDpZNv*k$tvizk2@X9CZt^{qe?Y9 zA}OoGS@R_qgzrBrK9;Q+5FRx1)yD>8(sNK+~JJg#@h1RpR*SLJo_dJZtiWe3qK zl%YnPwb!$Qj5zHVmjImzwh+ZQx+%hmKM%6Pbk zbOkx|4Z`~vPM5L){VMtXzY{|x=jCcRZQgT*)=|1NKI)7Dk_Fwjjoh!O@VxwGI(MoG z@8c6FAedvog)%&O(2X(fiiwdXs|Li3@LOkOxv85yrs1 zSm(oH|MM}VQj%}~`;%9$*OBs`n_yycP6#i#%4=bvTTzv^D)A=T><^T~qbGiMc0za8 zL$O#av>aIF+Ncw?eO0Q->9^5_!{z%KK(`_HkU{%;1MmVO$vUi;TD9t&RBS||-GGNf zNkbD&kJc%cq}}}5l)ZCxU|`@42d#+B#9b;8wuzsU@kH$040zf6)XTHQ9Z0Ut)~vF) zSWQ@1SR*UL@WI8{YZ2{%XW7O)NOwi0j1`fy?vQ}&`X(obj(4?A23ph2ndFb1g2|$8 zc8klr;X&1YA8-v;X>@h{(rSovap})@Fg~%qN!9kQXQoECmK`Yo`OQy_Ul-XSeQe&q z8*Kg_7ex36`EyrK|MHLI`0Dv^7Tg_|pu}K$mV{`QbC~T1V&SImm7S=pKxBBPl|Ew0 zaBGxd{KOkEl|#_T_{l2@3v&r@36e4-~A%3Q`eTF9h>;j#M7Ey zi_&Pmc{$!AAzwrCG4qR@X-q%0OxT-HBYU}us)~FmVd1%BJ5T2zYH4Zp`nCW3{Po$h zXZ+OmXoHeN3)tSAQcoQAU;R~@f)P>hp~hUIWm9Qg!ItC0(UqFq_tb= z_x;3E;JPGSn|oJh>Pwy>oIbnk+Ay2&FMqNpU6JpKn<87JF&IbVs+qP zDJ<7BB#Z0~J4MB}{J`^PGgO~LYzV$doWTYq?Ga~gZb(mzH@r4wXyOaJ>mtz}VnRhh zQA)2m@U>PlPdtQ#@`4{Q=e67WH^To?0mZ#Kks+6Eg=K-&Xg#|uC*I}d;kCN*rJde< z-%`U}d_QMv2Y=1|DzCM9aNEIiJ$OS|=N^?5n^s<~tSw@vI;HQ8G4hIPNlijT@&Ft!U|{(KCt&5ghe{8HP@?XT6oL_Q@&d&`Yra}5LaZbl3q^l}&q&R1%vJDWP ze*_N`QX*$KkH_gkyqAl?zt{C=X}Gkh}^C8yHB7VoTSml^v+@xD7q;kz75J zK{PHrKISvz>>#m5P1P&SrN5FC1Lzo8Ig-a>P+SV8r%2bAnmeMEb5*8xW2{%e;@046 zLc=-rGsHEp6#7<>d6TT!8ACphHX|d;c43S`kRr~|9H_+Cf)aa3 zoFV(k+orhn&PPa~hHlXLM!CBcG@0GuN+zM1z@j89qXctH0A}$q%`!Hf#LmBAOm9!^ zmBnXC@n@uR<*E`-E9L~MTogTOhK}NrJkq4mwOCU49Qe6nCdkZ?nziz!uonxP;c3)@ z_bf;1d~{(wb3^JU6TRMWC?(uSN@l3IkHz3N1-w%!r!eWkdRPnJ^=Wi;)z#Y^JrAjW z|2Mx&c1LZ}(R%4{5i~aC`1zWoxz42J;T|2+6j~x2Kg#U1u~&Pn@teHI+~yn}%YHk$A_{dcuNtMwd$hT8h?39cl{BW|igTT1MMg zt&F*1h(@j20IpL7dBqh7;tPuAH%E5ska88(+9W`bAJT3a4^;KnOz=8qQy}w?eh?rP zFyZ})f6id(I7v6GzB#Zs_s_D)_IB4=JttIcM&m`pUR-Y%HNfYGF2`F7Q;_`i-&yxURTvM8_5I69jNsto(N^quwb59i%FND z89v=GJEOEf013Qez;Q$)wGxo85cU1Q9%2Si5sb=(8b5r7UpDmiFbXfZ-EasR9ipG> zPRi%iEVc5}VyIVdrMd{9Z6Ie?(PZGajq9eYfgV~h=Dj~^p`2F&`OH=)d;9#$b?c-2 z#QrD29if+muU+Gw#I(;$W&3i$&=ddw^LxgRAFLE7$($6Xzx8EwE2vvBBCkpoJEa}J zOoFloC<@_y#+Ce$Ypx6TPYroFsGIm|mkV4g&A^SqQuo7B&-9=nnStUF+0XJP$35AO zf6cGRS)HA(me;U+9JZ+T>H4rkhYqd8_18!e_yfRcl9(<_ROiA8F=gj(h5xPlsX=!QE79<}fMDbrjLp%VZghXB$=uL!J%)ixuKs2Da zfb|{u4Q6#yG$YW{lj-D3*+RT%>qN!O~ey4=E9#v?;-Q@553GF80YndlH|~ zGhHWrFH#H8lb)2T)UQEPYUaKe{&i@&G@k)1ejJM6+#t4!-F8s22>+I35SQnb^rc?N zOKh-diE^zWHRQYzI}|8m&~sf$^}i&xMJ*qchN6K$A@YCVsp7r;)ACBU4ZllOgz$3viOo zqa|VRsLU8C$p8RQ&2f(LjvZ$GDt*gFqaf_bme}SokMZ{o16-5$Tv}g#6lMnd<0Qf9 z=eu9C^*AODO^YHBX^|f{z}5wRTz55y*DASNWNWk@%`Ww@KC#&HKSd3^~$W{zmcV(tLddi%q~7P=R~)c%L;nF+O7yL*pqO zILc%g_?l1K2Mg7stp`0FeB!`#PJ7N7tzYiROM=2aG2D4i#GXV)f4P%f?`av}TjC~7 zfJWtXxjVMVLej%;IN$SyC)gh|o9dNGP9YUPr@kYeU$RdyBs|$+y$%nN^3Q*N{A%0b zx+mMYPGR|cI#PjTEG1!=ZWR?3X+mi?>LM;$Le}<~yur*Nlrd}m@Ik6JK+B&UavSl9 zdU=JjdX{_W-k>ccIugo8?wYrH<8qA9e(M2#FG@8-g_fQ+Ha7MQ57#lfUWaf+a|#Gi zBQA5L{j5g#OFiugg02mu*E4f+%!eLp#%uF?}qJWsPg0kQ|V%KZ;P46{!i_U9ig6i0Mj2G%=z z67QEnHjaPG+eKYAJxyoDyu>Np%t1k}WbZM#K&Hh& z9tN8x+3+Y6Dt>NAJu}2>;>zxQ#?p|X{B^CSm==)$kp5%6rIN^o>5yX+wG@w=#ms|a zmosb1`>fldIW)2gzZf`vJd2-qtM~eFH>RPX!E^m*Kq=UBy0~h4&+b<%_PeY0Td4#~ z*UcgswqvqucPns#wCw;^a*@<&<1%weS@oY z_#THh2xmps=*c%a@uvzr9qUQ`q> zySgKqC6WEcDtX^iuWCk{7x*03Uo#d3@6C4hq4j{T%q59nQP3eFAs$KjuwS4yhi11Q zEJIV8T|S}(!$k@fuk5O1I5;@2&I93Ya=L8Lt{W*&rmJRAYJy$`3RSw;aGykA>^y_a zc*?wFI9m&^WFF(;#+dTSROv}q!sFqi1jx}ofc4YBaz|7Cmq-Vu{$TMQ12-oF53iIf z%aw+|2)aN}0LB-BZ*+sR-jiq$gIS6+%NHm+W6I`YXm!RLGD}2;?dUxiX*lNHlrjTCm=TXUDDwu%*3pjQrZychs%;MeWh9)Ke5PAwg6a`D3c( z>B3vgJ=nf>0rg{0#H;ZsjWEdm9;QEc~e>mr97v6;OWduFov^dtiOMfAm zslEZsf;$zY*6~R4Fw}1gQw4A?f@5{^BpaZ)hN@f%XfE)aSoa(Rt7Pp?gNNHu6kN7u z$~9pu!H=#tHfw8`H%5l3hNRZ1SBcz$oCyz~=f)nb1(p29sdByad#D4Ig2#`uoKj!L zR;QT%QO?nn;qBz)8;`_(mDRqo1PHxAm`+D5INIdkTi$cQSiHIdr$ z3?#Yo+*pu(qO3N;U!iqBcr5e#hbSI~D?{XTHcW$cYt>F{D+?J5 z6@}C3QE<{k9SVBYRpB&AZt_uSx_RnjZGu2~hQ<-R6%0t5{@M!^Qi@`(`*D2Y4y8PkmSM6~h8yg#qc0%^i(pxDw z6#H!P6rjo+BcBr}H4(;7RLwL$Ihm9Tec1kJz-D7`8DwevE%DJ1{UCp9uNuDqCj_;^pBJd2 z9H<<+D^a9YbZ<0+Nm7!#Ya*`{$9e%=CtfJ^Dk&+AZ~ypVtkGU+(UI7?8O3(w9qdUU zZe1)0tMf5l^1Xto0wDFR#^xw8nsbSlW%8i!Z)Tg^RO{IsYXk28o#q2EU_BHT0^!LO z18GxF$Y?d@htQ$*m}-}zy6`vzlBn!Pj9hqLCDBrSsTb+b?;J?AvtOhlJs*6@W}G55 zna5BlzNp$FD2Tp^ywu(CDV~LzqSjp@0|8+^!cgJ9!}#9b*@Nn3zS)_5>dRkKUli{m zG_L*v;J)zFde0O?2~e~~1#HjR8EYdRSk^DFtl)@*(d*cN{l8~ctW#Q2@@dl>%JZ?3 zl9H@Qy;|QP7``3l-?$d>P8_>-EXwRZ5VF*ki>|3HAYe4@)@N^u0y@@Pu00GZDzgy! z8zU(SC)$sYp>Vaq*p5%?jc;GB@P`H!+Kt2mYk4lE6u%lGUS%`#i)+m}NJXht4XTPJ zX-J*L)XdCuEY|&d1wBGti>tFU7#>XNeL42p56ZEqAC`MAV>ZLno5BUVFRf;RxMT&7 zj9)SW0tmO`BTtsQO_94MX>EN;EPx$TgW)xG7RWe21Rz7DYZ3JRN{?Ip=Fq`}g+4ud zbd9~OZSQl#mDi7>36|Z`fvzrp;UpE`9UlU^E7~qFSl?vq<+?`T!(O9OYxL!5Y?kdi z7>FG4o`;7-xxBs3n-DD^$VAdjE#)*c-IkEB=$ojOq*>0VKNry^fmS`>jah1fS_3!<{eTf_x;1Mo zAUWGUT>lp`4zcpa|zgV7htuYB2 zIGv~Y<_z{TJualX54$~?n0e0A9DdvK3h~gRj-8SWqxrfUWExrvy`NaFr<6(P&b7ej ztYCIkmUCI$YE85xs7_-_fm8G4zBA=6?fjF{;kl*wtx3^*B;DXS@zNs1>n^CKO^WV% z7`Xp@JWytX!R z=cnHvU6k>)#z7o}dxZ zw(1-9+ug3(-PvT@guD+TG1eW;ps_0Y^U)a>z?kM_WMoVvAu$F@S@6v5WWmj~j+w7= zx&x)$Ek8gV-M@05UZqS|C%R^G`WY4=TiQ;<{jPnqaZAj|+p|8A`ZJ6&!_|CcRjogp z_owFGkoP_uo6{-jX*^QQIz-w23`1^Ev8qmCl5GThI*kdaIDV@DAUzIIh#RoUNAuXV z1#v}*2IR+#N-iLg`P)8hqSee~^+r$8w$n2oeC1$f#2> zgFh>(a?@*{L_u)*FY|==??1a^(4eVr<*rRh6=dYhudvDZXtKBHCQ6Kih#XE1n|sL& z1DNlkBf$XF$n}a=Pf86SiG(Kje!bMMe51A?U_$K1qP?$2G%-4TALbL9^S>Djx_P#f zYY!4ht-&8r2Y2dNG5yURhAc?VAu<@gJIhHkNRmQ8K)@88u&4xnt82eO2JP;86B3D+ zTK8sq`Vscd+e?E99oW44zDQPrY`yhX%2AaxeKKq2U#0agv@RbH^_=ZRC&RX$KieNsi(+ylZF?A{GnrZI%)pL8iEr05|Njl-xtd z>WXe;dXuvNONxDc;1vh!c`6^O^`BI9+Y@hkJ-6TVI#D}s5gdy{d$GZl3v`c<2zxqI zO9q~4#UI?boq6%-ct8pOjp#leIot3fv*}*Opi66@D=&0A^WV^O8VRaI78%gH;(fZp zJL{N%D%Tvyd;zR4H#+(pTE9{iiGkrxyTow%nP&`v7nXU-23=RfvqASvH&4V&+HSHLKSUiTSIkAjV6eHaS79FOxZ2F`pnH{){8TX?^=tHZl&ts)0T=Y;QIeL;rSz=sc1jbMp95-R zmPjGJM7nqE9#u^+&_HgonyRXQ*qZd_VR4vi(os1vR>##(Q$vOpp zd10hJAkzkn9?ANyJ_d0UfJ`T2_m@^(=kE`sW2d{aE&<=3gShsiZg~8?dmqv*^ms` ziT6YR-H>DC^j*T4af3OCVBpoM2zDBETGgY|i zjC_qo3|sf_Z}uQ>4welhFsxmh__`nTv<;G6Xpt7EtyK#7`SK|EFN;_M;=6lagpN{h z+FCilRUP!~LKWzIhu^fp%t;2C;;E|2S&u?m7N0_Dn#!90>7! z*wntjIq8JUI!k;9U*N;a*CN|vG{KvT>kFAZldJ*me3AjdB;~-gsm_{BatIoBS+a&4 zhSVHEeu3~YG3t$g@mX>Ozq&hjr(5J$ESh8mxVlsCaT*|WB5l6?r~8py<{FLs0<+z$ zt_|D6@wSaq33fNnyXGmA|LK~uHv;&=*od3|oREnbJ#5t1cz?Q4-srOZb;(v!tQIG6 zkXJs64fkgxeoy@ZoGyC)aj}yhN#4;B=uh`;T36k7+tYGJgwaY&rDs+M*BJ7#==`N$ zx7k_-S|-`bip00VdGZvkb#yXVVSE*k2Houp5Gk>o+%ejn@#7e%Bz&y2f3$+~#hCQb zv)A1os=n~#Nw3rkjO={&@*I@Rg<31&Td(XD7y&AosU3;bwn7_v+U`%^=JU}Bj0q0A zW|yl3n9V1|z!CIGSl97A5!Jl<$KRlpszZuyFgHo3aD}QYgfjo&ED$FN$pc+vch==~Y zo>Px!d)ak-saNtL;Vj0eQTQj3nwQK#s643wE^s#LfM<_Q}zM zfMRHD9DVE39}IeWa2oS2)H6nOQCRfUDWb&0iJo)vuw>wU5!@IJDcJ+MvQqbuC(k9r zN|%CElqn92REAoNJ`K%_%%~Ued#GFcWXBU?HPjW0J%`ab%t4=mu7#4nQN=V>H4;zX z#K+CeEor^hg=0Mn+-=K&EV0~DsV$a##4MwDlEgHJo~5Pnu?yc0g7~FbUN0JvO5hm( zfrL5xRBDtzKkGRLVEBZr_ma0P(>FaE5mcK zM0PmSLSIWJ!)g*ClVvAV4siR&qpHNC_bsSB`4G$0W9Y$Gd6Q5mA zgT&w1Xb!IuQ7xefUy6?2>=GUXb;;H*t#qoS+j)0?i~zTjC#+z{Uixy z&nC)6SjNiiVU1~U(5I4kY7=<|v7whvBY&aZs`db_5oB%p<3n4_ixK_?r|;r11gz0F zTD4C`E46zP9x<{Wu?6vrcZ5Y{P5on)_e z=1<8=#yFDEt;)(UzPyNciuGhYYkyTzmT^Xl4IoB)m(iU)?^d*6Mn`@(xTbdB5CZ&i4ONQP8m&4}_}k+paz$ZF9UP)Td2v__dPn zQUo4BdY|zx&U1L8YJMsb!CH$|R^_eaGVzYwa+s z6ej0Vp?39q-l$KS#-X3S$y6Rm|NbGUsR9v_h9URVw+MA~CvCMPO+nf=0>g)RM|sur z0%xBWzbd7HFF+OWP$3bI48guq{;~hCb&iU&{55e;p0$*(6bv6g;*mE`ye5~#@7fOK z7=?_VcS&sX6kNGL+~mc3$3Hss%F#*hpoZmha9YaQPh$*m*;+u_vkNEIEkvNC*jG)D ze3{tUvmD?$LhfL{Jm`;!?T8sX0vtj0Q^bKZVWh%Dc2`7%1>^KOJq$~-4eY;{#EB7L z3{YJ_E_SiyW0PUU7)ofAfsz|6+E7?UG~y75#@bBQ&)Em<&b~^o?7n}OBe(=tl!&J&x$@w0 z^S7Us$56&uh@b>yKp~Sg!yp5tgepJ)VD%C4Bvi6)U;wWh0G&_$Ny&V!*NELRS-VH# zZ|0)1BBekIM5&GP#P4D72}um;LNACX0qO7Ft@Fk_?~h@KD9!yFRdAfRUmSQ+N~>@$ zw~@*o#y3;k!wBS^s9QIo9%_ZD2px|n#;e&ru* z=O435ggL2Xro8RnxoKE=#qLk01T$ATh!6@)_jhybjY7T+=$?n)Ob-iN_bDW=4+;7( zv%KLhN>Df;s&e{gLD=3uc>?fEk|o#mdW_#e7i!RhfHoBm4;q2rJpSNU>?D72&aHZ9 zEM?^hLYd?3O`;X@pE#@7@{Q%m{r`7(AQnDGzPAnV3elW?EYe-g&C>f5k~5|g=BF9K zfL;PpJ{*0wo>F@BH6Yjt^+WZ`$B~VPvQ*x?V^HSG%pdZ{3m`>&@wN2P1`umB%P?*Fey9OeMq8PUtz02g6lM=^`W!ukN)C+v{PY0=;$|f~T9p3oJX5^G%>>9az>EFE)iWme zu@H+Ay6gYQv_4L`^zo{BG*q)jJ6D$e-t5%Xu{kR;!xgROgE}s;S2RFZZ9_mQm$83VdNi*nxtdo0Vx!;##;y zeCbtfs(h_!Sm2LEX zQ$Dk|Y8!WHhx4o5gx`Qqn7fRqEg-T=AVT%>T$pXIcCE9qjpVgExO^=sDJforp#@Hr*yZE6V4v!h_4VdcgRfP) z%Um{@nqphUXhC~U2EsVaC8|_?GDJ=dYJGbGY^Y?(2gck~wM)vM!el!>n94(>9WnU) z<3RrrlqDZ=%<6-oYoSv=?>j;nxk`ySzyXOZFl9(=0c7>QGJ4$n)&8tr=TJ{y7=|%` zDhb{fpFa6x&-J&Qr6yxx^!4Y`TF}vC;{rqVov(azSZcL&{0UyOsh5v+vFN zZ(|q~B!I40HqS?Qfb^dQ%IAJA#93r}Kme#}_2~<$d4ysP$+n&Y%O#?|d ziGb_2RBNXGU^#`FEKR&5wese2t%l%h-#YD;|Ds{EWmE&{Nps(vKA~J?WCap~o{4u+ zp9$p%dDvqMZZ%g(m!T?$p=^#CD20l`DrKc)So;C}USQG$YCIX;SYQ}FA-&)Nd=~`V z@M4m8MT&6rsLY!=vAaXX;?+yvbmkVwQz6uW8!r#!?{0Vwc^AJyRKd_Nz=$llXwksaVW||#Anm{tD&~R)7x_NDJ}#M z(;g5F0fhF7WLWS+fzcl!s@1Gfor8+g8sU!Lk*V(^ESEHo`wZ3Fzgi7F`PtgfBuKZW zgmJ3|z{Z#lKc_WSS@ebH7=Uv=UDNw&Y+}x_;|NNO&Sola5?wy5yMVZEASUo@zYC$a~=>niVb+VJ)R zR|FP2dLGFpvfm)?0^<$J*3g+ruQJw|^Aj%mT{!ymDAJ{HUT#OvD$}$+LPg`Ub}`6` zQd2gcWS50~+an7iDFg!%9n(y>8s>#cL=%UcXkbPq5t`SIM}=N0t;lP)UFh))r-1TLzZlDXz)^rBT~$?t^X$=x+t@7in$P4cVVOo~!aGowlqSB(oc4qH zQ+y58paSqe5mClH^OgJO6EjO@`ra55-;a04k8(pM2#4|wrBx=TJavw6>|ZGGxN2eh@xnViN#&nyryeSly; ziLZhc(4#UlNt0*XaprlXO8k4{`60G)b-HKGbH-M`nl$2SBxS z@8zHMp1F(euwT|(!{&K!5E%f5{i~|JsrRKBtUrJ0A3}9{?~SHsXx>|Mx5gsi(DogT z_DKq<`e9JCbi=6=iU`E76B{~Md@MWrVl3f;hOGL^?#R_Nj)rd@)HCGMZi)t}eSi9} z{{HQv+@$`FHKFvBEq9G}iy)WP>|lZPYNf{5?D#V%e@SuBWrJ68-j0{MSUL6A)VGpQ zaS4=AwbyeC@5|83c7g=U-YfZ^PrMRCCQq4(|2<(pD6Ki-<7+(Jw2OTbHPDlobqA?T z5npnH;CyVa4QHqypX9sjeu#xLMGmjnpB>$v9e)yj_~Ug`!W&(GN`=)``jyYq2xWSc zv4lrKQymO5rpjr8t9bk+n%Yxsp z>b~uW_BY`*VO2st)n~hTRzmaJMOBd?$P_e@6C@;@KP?VV=2`Z(_1){uGbX+pw$CB( zMIxZH>(~26ph;MCjb`bUgAfQ3n$z$L3ctK((B)Re{jBE_pkc&JtfuUnX9r4i@Z=7xSv@-C}m?ZVR6B4=WvI7MIVl=~3+1n9|!I1Z8Sb=^e9h{Fs`G%Q}VX z>?FERe9@2%NlF~ZKsTJBWd3jix*1%YDIUq(YUcEPaNcCeTT> z=f67_^jQ!7pb;Lf7jJmXCng-OVVcUaDg7VxU>}9^y@7BmgR*&f*EC)0Uy!=7WS6@n zQ9El{|0%R@D`5`tBtdD4eDyiLbvR)l?3ME6<7Zjwl-Qsy@jnW0D%XzpVyQhgJ$3qJ zhYG5cjx=SBOTh+s>$o2zYtw_k;Qy$-S!S;Cf~M%6X5@+M^8xtTKH&j=L*;8vLHzWg z6{Z#*%@SV1YRm1~)1GJt*LSVgYSYS#6QL#!L#5oif2JX-_bnvct3akL`YrorahWLl z1}H#PyV0l4tEFpGuX!_=e_qb|O9w)~Hz|H3b>kfNSv)oXV53FEez+Ue<4O$-3E;V{SxX-1`V-%GIWC=6(BFUP2qj*nRPRf6AA!PmLh) z9X?hCRLmr&0#CA!@385^aLW>>NY!TiuAf71bxN!~hJL&OmRxoIj?l1P`r@$W4HjwP zqHvqh?dz^-pz`e%N=eA={#f7N22c>e*3~MpOae~PmuzzGgOz}ze-)it<>YhOL4>@& z#TAP+G<828} zB`ZshvYIY~3_5F_PNUJiV1GSgQ?9F3LKe9#@Lex;y%Fd2)9;iVB}+h>7EKBN;&r;T z^?ss_2bOKP68cQUBrmrhr?&spgX z4QM=6g6qP>X8qI=0zh`0(@zKLkFI?>o=YuExM!fhAN$!r1}R3afX1mR()FfYV1^yg z0f<J?NxI?pNWn%^Q*7)g}x%M(F~H4t-lKO6)bm+)g}sxuA(kyq`+ zJaaH6Bo}6nwi@i1wFX|CdN33h0VQ#s=)7vj9lf9LwR-mpW*hLb=`FWw$F*{%I}BK> zwN{^x==NNH#Ee0}DiUA01=uI<-?8wC0cDD?3sV8$Yp`T+Q8g8X%# zuFvwTk)D|Pt?^sg1XbT(pFYHj)cL$h{2W3Hg4oaOZ?#=d(^NG^Ltag z*zUo;gtVZU%jS?oK*b*3w~f4iKT*vllNjgg?w3;8($yRu9@MMGgCZcYS%H6pFmHTs zM)_=?#NRZ99COi)UP?!n|H_rc)bjEV4@#qtXmJ|#8@zfA2FRar&)P@75&x#LZ9Lg9 zAS&Y0Gk2sZf`^|U9i@SYTn3tIC=RgLEAi!l3`Gu`l8_$95ynEe)N(y z#)sH8608B}+^f9j!>5wIa@FX-w#vVRk6E%44>?327mugAcyeF#!HzYk@NCr83_GUb zW(fI?6yk2U(%V2{0#>q?_0ekPpq`La$g@;|8ZeYhByh_`x=ut73kvLNB-J>~B&}LIXM2TF*P^LBRrq|YJCy^!bZvYIhCHKWXsXNAVyb31Vm$6i zNFYppf^zjb=HW?^%mhI<+3}O2SoNxOlPi zyqkT0Lcl6Z*O$ZHG5n30f?<(l0qIP#x3M9gO2#7{SLR{rlABwHHer0mi~D;#Gwezv zf)^asGNJEy4iXX1jD+o-^rQkJ<@=!iBu28w;LA>$qx}+-IGo?vE;KR7B`M63zhWrM z2X^oc&(0v1Y=va)%SYhIOJ)X2QEok%#Kje~d6Ib+?jX zHJkYm@l7DT)1{wp(W_~Sfhj&+!N7)a8ESh{-ZOnq7b;?&`g*Tx2B+J-#?4AQA8SaCkP*)UfHZP;tG$$W@of(U6r%&^ zSumBlQHsr6l9J&z^d01t>B=~TIf@|{$)t9jFJ)iTkwGc!Nd&P|2Ncoa-?ayn`gvRS zHPIU=p`Vomg@IfPzf({upU4oT5?nt&Sw^h>b8WCua)a-y-aOB&udiPkCRdAcRut~& zAx?9mT{nmJ4pB?k7JrureX#T67}~5Hv-a2{KAh`q1Un z8O)-`)FLN3dk^dy+@YJ3aoBAJaFJ${G{vd^3E28{`9T2V?92+>_sz~1u2G(YI&x=0 zJ$AAc+Om@rZ+?C`@$HnKq7Gm~RbK>%aL-@wJN!*9^W`e>u2!(5ORjErN zPMjvAS}zL{hdw=GC#H6G3O%SH4}lc19_4(d zrr6$@wuxGt5=72R?_OX|srSz>-1F_h5?G`glU$8JT#R%&W3BMURc6Ert5~j;B@5PD zuR(Dca#&TZn_s@DAgXepES2CkoD~+<;!vM8NeDwf4d>}=!d|p}cL1wq&{wLuYtq0% zC{wX>{kAdm$0rFz9~ANIMHk;810Qqv`T`1<)jn_63zb4TziOsGRB`X2ugPamhWID7w;)1$dz@&tAWzWj zeX~b69}n?PG!@}Myj=oTmydayp)eOu;SXnnnrwpwAs=*j&>C{s&+Qi)ga8x8`>V!x zX$#Y;0gwtVNzU1!GzcBF-kZg9qEOCX3h2&h<{KguM@d9Ge&xG!$8>g1URR>9*MtlF zYjq0wFJR#ln*GJu-OArj-;KvaD1O;U7}DTJ zQ0|~5K#X|Bf)rh;n%z->%Q3C8g>a z*;!)QzME-8^6@qP`9XjI0vPbt^6ZLXPvK2J@2)JYLVAuq!s)c$Zx$#EQF9?aJ z{i4G*)ag8zIu(8FHy;l04*XYpEGEICy&b1jx1iKukkKj_#8rB};i@w^NMyAzl+Z@Y)6>(XPsbuFBSQ+vJgw%%(3|c>qWkC(-~h|2nnizJ zT0}nE&W8!A@?k{Qz|!S0a}UOZ&ePaby2G?7ij)MXVYdS2VxHd&{>s{MSA>5Hs@J^qk_rhMF0d*!{61Lul>ss6%zL?foC+|VL_MVirA}I3x&G!3IrD`v zS9imz|HS2Y^r=l;=CKL`J*97)>fumKWPlIDFg<7QSKT#T;=8cstMu(|8TVj1p{u9p z&SGdNpE>q8$M|;lbcx7XWyds!y@%XqYeGgM{Zf%;2jHZihushsx%T>XFv)qR+nERI znOW-k1OHcY40AXuUp8%}&y^{QGeKue3C8WOyRRc#(EQ+{@8`+%uu~d!FHZr8@W+2Z zgk%KUO?V=gZ-$#=bRKR zzm?>*!!OJypngO-G`CloMNGIvv##_gT7zEDRww+))@aFQX+uK)FnTTC1(8O83;B05 zpG!v@_}G11czjE*ozUbA$dpn6oQ>u% zpv$pP@h5ONrH0=e~ z_n7#tz*jZn1uk?gTyJ{Df)95Kf;e3ysd%YpxBXRN%+!-!pQ_v(i^dy>C9v?cE~Y*E6raE8RGm{pKjOLJE`(eNq<-n2fEg&E7T4rER<% zF<`i@+83M#UbVf&sh>MmxHO5{5&45miIkK` z^MWXNZB1TvtCWAQmS0PZe!sYH)3FfgZlvET008E#fm+R!y`LdygZ{`O(#SZ0#vgKv zYm)abSZ{0wxQEg9q|eXIhXo%}aMr7VJiX{MSNEcdXa5BhOvWnXzc%zE7I=FWUR3Ft zv|j<}ks&hY@1L9^a1k+9ny?C;G1s$*1q}~71uP0zkp|jw2c?H|$-i4)fiAtZ!L=P| z7q#74!IIaE>Z?kIXURMC&x?QAAvV79{IAAW#^k7DWM70x4uaCy#EvYJf_gr;7Wsw~ z)>~aDXrnnxWC`h+6PaaXQfEIs z38bf{8#)X^A`vh1+JyVVASK-vagNPl@z3l_1Zzdts&j*BDbgDLo?k7K2l@?aOCChkcSoMDa!p9Npl~Pdqk4vmC22k zC?BE#IO*3)s$JaH!+MSGp|k_?e;dZQULoZ?3;&bz%oWqO_u(y$OZLRn^xJE1!JPf_ zITqk%@YDFAjdw)EX&Ln_@YQsICqm#NQr&GYmqw(}GxT%IJ}I}js}&d;=HEkWZlnu{ z`lk!kk?%A#11|mr=UQ0E z^Xo-k`3DG2!~X=#a6P@bKz*?X)B8{)o^B|5^JRg&NQz6&C4H#sdUj9inMUgO3v#lu z3DO!RlWUi=&wihWu!>!YO}bu>16OFWTL8_KuOq^kG<0;ymAi-{2?m<(F^26xd3Rx8 z;2#QoAPPlzrdyYB1+tg0TUG!l>W%KRt^Q>hsS2lA89?>k=!;px31(hi>qHIrEn8?3rrQ%m2R1Z+rL5();h;Dk0vz zLCZLS<-HJ|EArBg=X&wF4==^zUDT`D*#1?PvfD9Yxs;M%T!>rU$*l3o=@xPA>R96~ z$xV<`Lu>2$(>(vYicza?mk$Fqq?*g>yVxB2b4Y4X>Wn_ggKmoaS@)9(WtAJvb>2+o z@|{3$!jY7-5{@AQQ`|C{go{uDoE|}B8x&9Hw-(8+WaG-rLbku24*jkKrJ@3|fh2`e zXfcg?^>p%$eazh$F+jQjLHnDS^b9DqvSNBUl+sbUt`^#Q@)xrYHze009$6|9m5z?H z4-`Xx>V%c+&h_TsWRQm|?MTd#-i6kj&k+4?YuE!cHXmDZnRPgu>>yP)h?oeI!m_6k z$Yg$ZNQ8TlCALZD=cQU)S(Ep^`M&vAS-oQ)dN4QsilWR>s3m=uuo|fmRw%IiX!4DF zRM*lMM-DjW1@W!G$)zswb)AvE?Au}f?ht|TIfj;n!(eIN2n-lm;-AD}>Je%1a9YWm zjXbS;__$5c7uHf8Ohm!iCT3Lb(D@g; zE*sa_PRsbytVdVh6nh0f!fH$cN-L*o=B^1@Uq5Y3jKaOH8O{Wz(Q-_t z+AP6-VDKVW58ALK4FN<&f(LxaSX`mZ3x76s^HJ8dj*1~$$PIA&VU;SJzsVw=fjCwI z0**D%meDZT869R)K-!!jLO~qZH;C>2qICp%q$IHJa`2^_sAg-I-KY=B3bZzA92zW~ zMY|VrQNa!`6FNlt1)S_MO*U3aRuTy zoz&;Tvbx2K2Do(OJRzBb?@S~4FE=MkBcHzaBK&;L1D#p8bkW|N(By}spM}$;nRJ$> z!4{Q0;ZpbiwfvC~Na362&C&t;dAQ%anKz)iMek#${L0$vjz_%4`IGO}+k)r3k|%2F3gs@I&G*_xlCJ?V!5m0@ z4Dl;9$RV!pR_mC^|UoJ!H^~l&CSLm2k zvo5s$WoKU&hks3*7m=c&JQB#nO7$EIU5a8c z4cj=)(P8K;@(#Du-`|hWmXT6Qjy56EH!-h^ydRM;z=(B*;xL>zZ8fh*Nxf#A2LU&} zPA(Iox%4;uF75a~EArvv*gUyRB|h++HA$1opj}NaCwk){JiAO`ZLBFm+b5$sO#D;< zI)bPU8%IlNAKv$gtUvH@{jf|lo!{tyaNb+3jXDPd7#dve9i;&)OUt!lN58^ti?669 z?_t)U+Q@PX$Zoi~S=-twMy?HLs-6Ccm~uULKsaI(1{%X@2g2sdef5QTp*vwDhCP=& z5SJtUl4}Xq%bYx6>w0m$TN-}Pe~C1RkTv=0^}5lJeP&f7{nF9E??M`4T+C-QW_-;a zt==aK`1irw0ApS-B{gzo1=0??MEB> zj1|Ot<0BXQjuq8M#B-R8Wx^^)+gWbq}z8tVSFt2PU5ok&&hi7Xr)Sb2UW?B57v~)h(Jb!ujhMlcGl9!n3xFY0M=vxnZ(uWk=(?- zX7K*USzcY8E@aGnJu0$hhY}ZQtXJaMqk@rdy8O>pa2M3G#4cOB*0^$(%aDER{=)Se zm$V%Ql3E{SgJFM4dQs+}vZK3h=8776DT|%SQC#d(il@)U=7&>?6{G0@4#o|lTv-1^ z);kT=!LsBD29~DQkM{@7S!i1$?LBa5*H2Y9bK2OpQjzb zRXPYiX$*p3mGYEf#w4VMcl|6aE!EeB1^E~8c*AHvWV$p3hfto}9+;#XP3>lVVLp3~ zm?%F1t@P|Y!W&_d@#7mnZ$&M#YU4+ z=q54qF2gOq5_+WBriM$un|bwWh5>w@zYn&*Z-upTGjR}?3%K^Z*C4d5_O8$piq7YW zQuUrZCW;rn2bU#dxe<%3rZjDHrK7u3eri`WJK79dyI5}v^75YOw$e%wY>6mR=lOZg zL#>`6S8^~Pwg-c$-jevSQp!Tuv-tVv_uuI*hS(GjHu&wR`uGFef3(Wcgh0+mn%D<# zehw=3>P7jnlrh}r&ucA0H#Hez8vI==uCi`2CYs_<1)Xcj$6@MVTv-=AYH}n~Ao%-T zKO{){^5&oY?$hE#U;V}t8C;~+)konm@7xqzP_6Mlb2WfoSh=6pzCAWbZTDFz82RX=St4ck$wvl;Ae`Nh zE$^IX?os$_4TQ#V-EShWyaaRdUPF4ZVQWg)GWApPY6cuE&qH+ z5TeVv?E_WD-Tr&7h+u1OT_e)$EAm~_XJ=aR{jF}v1s>vXEO&6;j>}4;#f`2!LgyMm zw%Z_U(eNTh@by@#a)(e&X{Hjg!E|WVCap-o69&vk5x)}nXrFvdw%<}@G3Ig#|~dr(I0#1)3^`*sksy>zj=1>4k)JDWm63*{%+&wAv7z8Lfnrer@w4VuH{D;=?;0IIk%ktAnOby zE&NkERwo9hRJ<@=rsxkzYoi45<)_yxSYkcW;DN}fNWO&>AN$dU``6a8`W`3XTw}0B zF>-~Yt~7)CWrGWqpA=Qb0!U?Lc7ai+(enpxU zL#H1u@y#y}bIVhSYSf@tP-v302VGj6c!G%5PCMv?7dXA-7Ihk(tHCb$H)g`GvA4bk zgmC4@quQ?1W8DH5l;#ExgJP3m_u$xdhrtpwfo<`P(uy!VZNSz)M(+s7z3$>06_1hA z`uK^_OLU2c&+Xjts#)rmm=&6)WSLE_GQS0xLI&dgl(w~y*v38`JDb(bO1oUt=GrG9 z%2Hpq_bjl>f&sxw12k`ui~#jkUGaOqqO68gTA4k5ZQS0Kv8Eu`#J>T{u^ZQEg->kI#s#;;&aA>!$W}?_jEu~Q1_V;~ zcCw-HfbCpZBkD4bVcKF>^=+nqS^AfVO${=mTv{_ID9L*2NL&7oDj$&&tq&1q??fWx zJCoK%4iV(N>diths9{vW%Nh8kWR4HfV8=xdm4einPVfpc*K^n$zfHO&%7C*xS4B2kfF@ z7zIn{QQT>X?pG1SBJqPm^$ZoteZ&GO*$UyIp`qzRpjK!*hKD9UVNgE7ZDVPfunMK=)zrb< zjYX~NLhgayGXEWBTUzBSDT-i>CmLmkiw!cP$A0=_kgY+R^t`}=H7$m&OY&2@nD5D( zv}U2w)oW}~$4?F_v4aXuv39^-_J++JUW$mozaH1hIhN7VvB zNtoY5!#mr0_t)fg;ph;!KA8&17n9;E9l}`!rS$g<%pI?KUVU2OxJ7-Pd3!e2RTcGt z@ia=PRR27AQu@p|b zk{X@=#A5U=!q|o#=y<2LKnQ+xE?f!8)IYMoVc5l&y!!CQ=P`!?bUj!@LlgPoo!)sSePdAw_j-B?hXK-bFCSuR_ zXNqE=N&O!KiIw#A_xhWky-`sQ0vMDgKy&Gj4q``Wq=}NFSG!@fe23~}=@7l(JQ`KC zE%FY94CixuN3KG(r7@nRLxd9m-(5aUXueDHv>B~3#kM;W>v|PwFDTXj&s-1m+sP7L z5+fkL&NA1)EyiAVjNRJ|&%Wq@#P!#>4<`7CK#U|#7-Y8OPz06?8s(OO)TaZ)frJ11YF3%?e^5_c!5~O<}M9N*SGd z`1j6d%_SPHo^hYo3IPuw5)>d=xX>TiSAO2!_vXsic|H$ivk#{L?D(9Y4IW&BUpv@a z2AWi08uz>r`CkG(d>ydf*1GPpsnnzS?`){@Omw(zdyq!@`uAyWB{UqmxUt-E&A=y|jc)P${+5WUguq*;fnF^D+5#qTO z;SJ_N#%~tIs>u=T#&lZeyWmrVOGltb%yht&3O>=|?s?JOLj?93Z&u`klc@#}*Cf1R zy;VxYVx6n=$$?|XvJD*~xN4rl7m=DE-9>4sN%=-|6W5%8p~x8}ekAfC>m&t^P1w;G zi6S|QC9z^ZH@J|Pd^3H(J%r3s>*GfB~X1E?R3CIO}q(>>q34LI2 zdOY*fUfoq^Z2sxovU=|GmYQ;7S)tm#4dKrdzziV^FuQbMdsRxbA{T4bw!fawz7>?U z>}=zr(KeHHG)Y&i%_2@gaEgPl!998#BxWgz^Rj7WL?F3jk6y138(9Lj=6!XB>?*TL z+^@_ZWvQfDS^AaqW_o|^@G@?XytUH)sg%ey!>`?8QxmQEmbiX zu%uj9LybQt%jc`8aYJdXNn^b&9T8b;I!#&?WKv33sj@{EUsO8AJz(tO__5kI5Ir!$ z%1m|JJVsd&6!OdM$G(PvQo&Kul!=|fl&WJWKL&DR%LfSsf`_8ur-cJ0SJ)H(8rnaqkLcm{DJ?)3-7P$#i}hhQETME z+QWzwxCy&rpXQfvT3ocj1OjoHseWbFx{<=-U%6t;L!ab_qK6r3<;AH(u`(CAZu8QZ z{7{1>VW9p#CeJ4&7uh7osvL*s=JG;#3=k1GP}BvYkQa+zeT5E)yYA;88!Dn3R`Av# z8@gr`%4bbWAUxfGKh^)`D=sFSvo9)9puo+cLQym*T`e_HrL#AFoJnVexjNFwj9;CN zZJH^JP0?SyJK(;n6DM&3SRenrg?=oB(#HYKJ2o{HkxSi}*Vj32myEv)r zd(`&%mDtrxctxtAYYMXfT_gQq@vf;mNo6DRJXONbN|N{&z&9m-Y$F<|fWFnPt@W!h z@O=)I&8*q_@e(04S=mS+_9`GJW?9kO+E+_5kye}Qr`^zuuSwv$Ms~KNIN3WK$q3KY)|S0?tA!=>HAveT@~bgioqj#AL? z`?Ek6J7&>bsfkaYu8Ik&dre-DE0b+*(i{&}jLg6k77nMDba~n)NS+2z!{U^B#$JEw zo>qz9f}Mbfh`}k#@R?SY)B(K7zV|NlFsk1BxWCCTI13ayVxQ&|pG9HI(u>cV-Nx*z zuoFx7tFA*c#Bv5?=aSMCHu{A3`Np`dZ+UYvsn)m)5CIykAsU6#K!J_Obn2_-Eb(X` ztBR@evsp?r7A<{3936gf(&hxX;su%$3=f58=}@}7SAOLinVx~z-#gvOEqHsxN(mYx zziJ-@6M38z{8aq<;$+~}C>B@$i&8SF9kXKO&R$}aMXv{$?;+9kB!v4-7D|d2rRptC z=p$^8W)I+$#2f(eJDl+M$dxIBter?5Z{(Y_KD zkrhRC(!h?W?jeFNdc7v+za|H`NGiCa21t9-vZ0|PwU7rw^sDm_xk;qobEH3#-zV#Q zYfy-)ruOsjm2ON_O?BHzqn*=nG5KGhz@)!Qu>@zL5{UR7iKMR#CQ)q$dUD-;~l9;s>6=^dWf0P++ zRhCU&#l7G}%v$I(qS2A6*RvByY@Zuh-W3rNmk3|5V+ghFRs(pzRef>v)li#r$bG*F zFPxG)SF;?P#%?sB1hldBD)y+>fg`V>bOE!yy$$=u=H?sUskki2zVB_9>>D($iZ*8I zLT@}o+RqM}F&(cb6gv)@{AxC_2I&RDmML6}j7HV8I3->cUu(uV8kL%`fv45JX{2y{ zJUnxx@dn%3e)d&uTI%{b#|q=##%H8;bmaF;IeP6%9n@~Vo0{d)9a6fJU&BP$P-@(I zIn4-o@PUQlq@Yt(icC`cJNRAed|(Xih_LI3Hi`jFi~fkaKXMJ8AmAIxyqAs_Rg6lC z=U9E*uoDv$SQ^&OsnN-48eM5%^MC#s_hkAZgo{Q#m_NsA&O5KciLaBq8@4d0+OyNlFDks>o-|H_Udd{jnwkjJjGRHD2s-CMo>o`a*vo>@-VZ}$yqY*X2dnF^C0WhM zkDyFm1Q4DoLMlJ*fqWJDNJVJ1#?rDDk*tOXeF5X!fTu9G%^Z!9Oe9#(Z81D=z_}lQJWbL$l6{%^eyqV=m44uG z?fsnns(tKH$~+~Ff#Z&&wr!l)|6mCAA3w>>2`--vw=&I z@h287VS_*~;q8aKoL_ycODxzLMIUC!qiPFHkH<2wkFQJ#Jbd+L&vf@kKLb)y)^q8= zJt6z?)_|4j)yLU{S<)d9xodSSsc} z&1}t3H!Z&0@~N=PB6ULIZb(Rk4zpQJQDdgWL&VrQ-6U;(vA24;Z9aa#VV^3L_rL>( z66ImZIA{ldNVG~soS4l=Ek|>qDhO-G0FZS$q$)o33;Mt!BD{lRV}<@(asI+=@^27ggH$9xpy_r&8~8?he~pcBda>d|XayJ0&gH+t%Apia z1Bwd5yaMgw5qY5BRG6WP+8;nXW-7|x(Sn#~ih0Pu2(<(aBc#*yYj~#3n0RY3uQhSF zf{i=~Z6AXCgDz^EqNw6UY74sxc1Yt}%BzERc0mseDf#B(aQN2}^)I1#5i&aDXUMn*GrTcWUgX)2Rq z$=85FB_z-Kq0E>E#QEn#Th%u`p4x?6o;g0p^KtNp`O7`G{sfW6hZ$hI$;zpITMRE? z6!1j+tG$=qLOYDH+F`*(SN9EyNh9^uAu^OJC`Mk85lebrbf4(LV`J(fx~!MwIei-Gw}sa zB#7wAg*ku!r2fHqFm--F>AG=N#eyHp6g8|@n*uzg5MXD6Qzg*n+Y9oa?3f5T0HoA#KzC#o?f?QZtPlpB&P6D)a{{-7>Yw^>C2;uw!Y8Cwt{Z=PBK};Vs5R%KqKZOQ8=B!iNABA&ckJ`7dYVR#4wZUkPgG ziXn6~;n2*k6u=r#xq>cF=gb*)$Vx+;AZv5dw5NJrq4R!3L7Yz_YdXO!RacJ#^5}W} zId{M0pW9TFy-OpHws9S1gA~dO{2Qh1JvfyT&8e=IYt`s>>aV{#z{TF(6=vGXgwzX{_<=b)v#v4(#v?x$G& z>p1enq~ps4#q7t8SKe*Ok^cKpQHA%MllR+_+N=_lWFD6*zkg_;GRbbc^Wl$$c(JU= zB^Z$_mwAwTb50aeX4=6``Tx}rgZ$W9FR%hKU)bqZ9`VS87gi4~e3pRSz`byBz#>`v znMhYzJ@@a6!fgJY{H~c5$;l!|+Wd&J>ig=)Q+Z1`ZIseAVqlK^AGaxn46;x1jq-TO z`|t))9&sO;tYu6Y(67f=yfTyGLW#n;Z=H@J=tjVk(|U!s9@kY`XXlJF~hn~6cdd}=~(!G8qpJQlapV`QH5+wn-mDI`OhoFEOO0(C!?6Q50 z+0MPQr3rV5q*mGU^-Tw=KC1w7oDp74IFXqz^pGK@Mas^l;N% z>v(pj&3DLBD}s6kI<_AbjG~zfeEmS#8wy#{9-j3zIh&pFfYCVaq;Gb8B*4JTsEzWE zcg)>x_~aUoistcY*po8NY1$-w03e)B0~=dK5zlMCgq@^BeD&CaA4|1|$=t5C;dSrB>6o=2PXNRqM0 zXvm;5dPN``>s6wKGFcM`h%-z`J1ugF-Rgv*%$aA^&#&zd$lHzWxVbNh5ST^P{QcoH zrJ}zvO|OPmHW>x{SclS z*2~xPLtWBd3d2HfwO21cPa@en!ux|zO&#=~#M1Scia9?^MZhaP>VwtSq-b{HsEn-a zIcn+%K5;s#obHaKO_{B_Xh@?I6d{e?i#$pyF$~IQ`n5Hc1i7uc-D}c zpkb`x8P?RMlbeRH(^s}Nxivg!GgXoc-E@{j&>tdmwW9+jw%?7vyx2wf>FXa&m=*q4+1d=}Av3Aki*ifF#;$F1=to zUv9aI?Vw^eC;0N5A;c{`L`+=d4m zrM{lmWw>iV3km)!udLJMh|5v46?!?O%;VoTOpM+a4(T7g9{*Z3lBA70)Xt78=ghNX z!O50a_*5V{mCI)g=j32P<<&m9s&4Y>d}+VT1thu2k*sL59p-Xp8#pk1ldVeq;nAyp zM`w1mPYi@Yzl5VhZEPI69!y7DuHtuWpC|q5ewiY#%)Ln0vs&vz=@EYoF&P|34m=}k z>Km6r!@?LWl6&+S2tr$*%Ny$Kr2Cm0 zEm4!)-KpIE#T2vMg<}4bZS3dskLf_OSaits%H}xIVi2+?0b+Jnw^L-4|6E~KtbNFi?Y!;VyWJWAsBjjuJ$${*V%jhYcp^K-OGCRVNT{!VC(n!_8&ix88{iQ;F zW!H+E8{{-B{N1}w=>kxHtZ){a%+$Wlc z4*nh)P6-cUjXuEU((ZVr@bhXICQ4k_X{u6@yeQ+VZtMQ_@{9Y1@l~3(B2#!yapRcw3x8IX4!e;Nh|{ zM6j2PMy-Y2W~ZYZ_HV*?M>P3N*&sUNX)T9Gg0fJUOCYygWXZM9gf@Hp0p7dbia z-nk7Yb9$yiZoEc=mlf@yG@D`=cL!z}9Hv;{qlzA5K`%}6#5~yQ{HP{Ft3nkDdX7A@ z#^}9Y^Vd(CopYrHX04mP|Hr<+klBbF0hE_YbN(DdDJ zl&6p;wKV;D`NY<^hZKesTjayC^1kKqodL$gy}{<>2?8C0+K`R_O4oo|Kp?eT|BaW z%THxjn1yAZ(PTC$>gdSIhDDsgWWF_5b{wx%xdHth ze#s-7FMpFSn<(b>h!Uq0sa_!_KHakR_0)N5zgemJAlR%*i>?ygbA#~^$y|OvOM@Vq zQvO{%2;d}|5mfz}EashY;G=H^o#;V^=lGVvi*EPjbnB7C=)qyy;A=udef)VJ1zjZU z+bu2cGM05-$Uh2CP}WOZ6q;5OGd4_Xus8JkNMxdb=c#$(koYzJ^keJc>_W})>_{m4 zg6PK0&?rC`-#k7XbbG-FI$Ru(sb6?5?DqRk22j3}a94-4LD95YYOU0JQ{c1iI!J5W z)aT~bXJlmjCV%>)Hju={6MJUtx(0wG?hQ-}=p3w0Of#0p7cJRXxnJQ>=~l%{{jIz7 zcXHXHwQ`)B8~fc5SjdoNtv;RHKa{Q{>T?baG$X*E500i~J$4n;pQj`^63c@tL=s*e`0%A*D*>d+Q{Pcocn)@epdB55gl*)GGX z$#)loP!wqeD`7{ zrWqQ|r8@|8Mmpld6R5F0>7#vXMyr!+TUoSnxTUzEqJ&-S5=APqTfoMq)V>qAI3b#| zM3-bVFtP*6a-ni-HKBgq%ZU!c9l4}mK_c=h%(Ex2Z9)-4`%-68boXWQ(8F&R2+Se^ zFK~UP1xn(rk)m9a|wf=n9G zIlqSlQ0X+iXmSP&r?OfuUc4ZCaFDqeC2ANiA#}C*E`c7|PT+_g>BlB5j_~EYdQdgD zBT>tqJvAnh*C6fIyFpB_!B3=Kx5b{IY0VLcRI!8Y;9mZ1m;h&ht8#V$@Bq8gVkN&| z)oSu=Ec}+`7Adv<1%ydZu8RFZiEsGD@6y^URN5F8pbRw^Mqeniy_?|vXE*~|sE+h~ zdI~Jzdv~_ytT%lN6|ZQ|FUAU9yX0spm28#FoLg8pnMpEmQgk!Fq$onZueB$Hgz!AC zSzK-u#(qqM%5MsJ9pvDD36AOcoLqs(nLSzoGvZWm zbjJ8DA{;PokLN9do1=kMEHHI!zTic(ti8R`YxOJrQ`~f3dlgy*-J7Im?YVZyj_~}I%|!jT z(iGL%b9(FwyjMCp%2G-IeKirwFgKrG-4V^eIE?e=%@QiOac_Q42kSo}D4O8~0H=Qs ztDe0IX|lRDXJ74q2--BIFY2I_MXI21`>aPMfgZkuRJ%=`V<6nQ=>)Gj8}3!Uf_SWg zo!WibNQfVROcnXARPaTCM~fST4%S6lTB$gbti?~(n!MStyzNyg@FC_e@BYs#7ivXH ztWw)dNobecJpMqKg)`YnIGKD{0HRT=*?aoiqZNGx+8P`>OT_U7z#=f_fs`bDLZAP9 zYR@FL4z?bBI}7Hck>VUTi=kuabK7pAXu#xE}UzFRzal0S_crT&YTK= zZ{?j_*_aK*$gGSdo1Ex4rV*fPYw7}BTgA(7_xZq>+3{0ZH3E8EDr(a4aj=z?4vu_(OIA>Rh)sien@lNaD`Fh8pB`@?_;?~^s3t5a%G?5W z=gCrZOxJ=c=g92u8>dBVUhnzTVO}ldrz|s7sP9;;J#_Nqz~TQRoV1%#g(wQjlF+?9*&z1OK($&5rBNY2QFYzO z(XvWXTUOTY+i`pLxEx2-3f7sfy@cCrG5oFuOQZ)v{!Oz{hFBz5SIs~K7`*2sL{#6X zSnQ6M5t{pYc?@G4EL9LFo_lYHbc(5rNId7<+ugI?=5C4~=Tu7yqH}ex^CVu3c`So# zD2}d)$E{O_MHs)r=$7bBW#)b3*#6I&m{e@YeaLrqR8pV-hG9kRjh7EcAF2pd7AU>3}dV2yXtC^yX6Q4 zb9ZOM4(gAxH58!07P3Cy z>iHRQcqg4i){?A`Ft6NfHAG+YB`Jtn((u3_N=Zr<+waa`n4`R~{e%!UKjLTA4aNeC9(l!`w>HGt+s8Ynx z-kMpaG3#*uHyF5__k3sNCd}Q^Egem1e*C;c?^x(3I#}Ob-HW{`O{=?`-&M*lKA64e z>C-8pnB8X+3cv)JQh4PDkxitj{QhS6eh-EJ<}LBvwf+^crAK2PXf*beqvylD`R+Ys zP>Snft)E?6E8hb;jt=r`J=fgK>Mn_$d%JP_D<*58FTOMW2RQLxm~zC`!qu_~p9iua zBvvaF^20p(cJ=(G3)9w#TPTF$&~Y4{EIgLlfz!Lpq2g#a9BW=X%J>VCN^r1{DBqA| zP-JGMk6YO)rLEV9U)-r^@4M&a?fVDwZ4CP<@+UIbLfxS7`ubdx4`2&)n1~2rM^J^1 znu;Z%Y45mw_jjtbQK08;WqmNdEl&$^mC}Dvg4$+R(4lZl!}Xr4k_d$b?VZ1un7u;im)^+hk*mj zA_@-d5aPfFy9+<5E`Y0b7Uhqf-B_9`SzdO|h6=D7Y0tSE^gy7IKtH5FB&Dn`O-U{- zAkrnfST%f@=wgr0fA7};kVNxfRsrf&1t_U;u*p#&V(UqnYuJACS)0E;cS-go5>AuZH@1gxIXk71zHwQLD`sQ-G%j5hI|A zl=OKttkN;7x&^{k(sbel=lNX8>oG&G`81U@k|cnG{{J!FndAympj5cUJfv5OA7gJH z)3@f1)<{Ia-L$YzE3KqLT9uBC_h+M$lQl%>B+e6ofvDJ-=Zb7&+FUvgb|;^M`+SN) zw-`h&w|(JMB&$!fKE*}nj%+V~iiTF`kD$6ds`1MaTC;)k;nqwhY~3>yvl~i~+BMVS zyxr*B9xz#I;dEn$2=+RO9bAAA$vFf4LvwaDGiln@Y=R5jnI5A={y)IrIN#RK*c=VE zGjXsIm_(40NvQ%uzgBV?S`X|oaNGiFY!ZQh>N~^k{ZVu>eR}$iLcwtf=47Y4t(oBx z1weRdj9_a1=1eE%S-R*EU{vc~OeC4eaC=7>|9o)X8koo~VQmKwSG`Whh|6~Z)y zzut|1t4@1C_jp1a&=$5+_gDS7hU8#oTGQxH^`F(GhS;H+qZuow2+W{!{lFn@0^J4v zS-iuzvzv@n3FIY8iu13d<;>beB#`+mL|^7h%1uZ`6&hVNs=g+KaPPx6ee}Z4QsuFn z=_@qhv>#^?+Wtl~?9mWJ=vGX< ztTj7bzc60mJzcPUUl3V3JwQz-PKwwuH=f*GO*5KA`ax-j8ZNV^|Ha9VDQS>>CVbi+ zYg}wEZa0{U(}?Z^fFyvjS$OLTVWZ2VK@%ae0KKO!&9v$(BCvEO7&55P-lGPXE^)Sb zt1iGlwXwFAPu=RUT5Az(P&ce{5N8{Y$x2C~uixr)9OCxhQbLNz1n_&flDz_&7W!^0 z$sK*9pQFHz&s=(tTRUf!)_WA|Uw)E67lco>@7sw-TB@o;z0G*OuQWVRPSmQOcqILB zTs%sD{Se_<7T#7xEImbDi}mARf!@t-{M?YC;Jsce?oEOEZy|hW zeO(G7ffKat^(9DokhG(lu5H<8j@y_@;czGX!<*lI#9)4IDfR;KF6c215kTnqc`$$m zy;`mPg6mCU;Nz+0AyZZGKv;q0H0$F#fdWuDznnH$~|ToN75X1aqU_{QYC^ zZ(H1hNIibyx-pSbt&{eITfK32eGE}yy*V^$8!vk8S{jsUsrxUV_jlI%;=>^>F77?U zZlaHBfI9rRdv&YMy)x%J+2b*DSZwe4hM$iUhiSz;Z%h7~rU>6jy!$GT;{MNv0^Lyi z&d?7cdmiJ%zZRGFFq4}tf;w^H-Tff4vF8;IVhCv?-`kt99)B(L?Nsh7?z=mcq$IU| zD$#@;1PU!2@B{XEL44FKs$hT0tGnNS!W(lx(=jIFV>~S~*U1Rdq{$1*<#dW}?Y)p_ zWyOgJ4M)e#c;z9g9J$Nw!Z*PNJ^MY*oq7#f+)K1WKK_-46o08)#V5bHz2ru1HgMW` zZ6z>T!nxVf@9R-39Nx%+6k9nBS{ppT+8$`LcFfXbqL0 z8&PU_=XS@sh7AQB24g1tY;%d&rPN`@_EZvIoXpx= zE;cVU1g{t^{ReTzS`o7p{EZJs9!PAA|6RX-zLv0i%O>VQRNTK(FmLSKv>k8mdnr{J zR~4un4xWo_L_)NHn8zIc#CqYTuYmfB31)wkbH@jwB>Vm6_a3LvOyny6ZMc-K-OQ+t zSPB6B(}o><8n{gSa&)RT+5|e(0B>$8da~!f3DqCEH~tMeiProD#Ae51W0N_bSfk}( z8gduyNr;OV0}0T9C#Z&_(qY2jwU~&A_qcr{w3iv?n1Yd7J@F3cqXjVJ&4V)<`B`9O z90M8Q^41)H?*U<^#??A*a%a3`?J+1DOCP4pb%a=*P&kD58|&&v2I{2uRaL}E>k<33 zVs}6ZNF2SXG$338%_cN2wU-dbt|!S|Wo8YVOFs87d~r}sHmCfr39M~+O^t(D>ZJcN z2u@wf4wJuS5{b}S(t;yqhKUZe5Sj?Yc;EBPU7Xr_+VFao2b49kuy-3KZxt%anvD*< zmgS!GOgFA2oIurhztpKfwvqjxTC^ zIDKOKHDwyoc@9P{$tX|eadqt*!ioHqp#x}c@|g&A)0r5-_#Us_WWt4iX6_oYTO(vg9Y`6g26k!)Opy#wr)wO9T|8CdzMD>+v(G<%t6A$)n%y;SgcjDuuYZ9wO_zTfTooNl5EsAZEx&Jd zPfs%?7by)p69)3BLknS1va&Cu>}#@K8=gnIT&azka+r>rZogp-w)W^f}kdXiiqgaOU!cOcOxp z2k9I$uOV5u+@d=rreKL8&poNNN^*VzqMm-EJ4rset;$ziM|TWWiO$WJra;^ZD3)h} zZu{5xF?z3KJ@#XjAP>^HQ;+n^Nt}VQ%l1_1?lBiTxT@DpW5jvcF^w4?))R*MBW2<7{d1)HvRRQZYY_uW@2x zq=@KyUi2P0NGebNUMtq*6coAWk9hqGR@2*cW}D=du)FoYDLIgvc3*w4#TINYZMzwl31@&DqQ`+HMQ}w%d@~JX)?D< z@7Y)6>v24LqB1WFxPa&oRViTbvq#*wri^`rf`f{<=x-q5Mi80AhY2bJ9{;X86~ z`MSk4U)SAv|Kw=>R8vy`_%th9bxaT%*h1D_1RAoP7jl7_Il~j(jVlaV$72&hS`?>~ zhl&yXiV2?o=vQ!RNM;%(-RB{0BV-1&`glA1}yf+lK#?2Q&RV!(U9GLae?Q5csm?8rdgHX>!|UO?B}92gSVU!BqV_H0m)~Asdg7) z`KaSc{lzi9{_e>YAc&PMBp#IxnVx_vQU@-v@*z9)@cMh2uDG`LNWqO!GW~xlYHW(} zpY)q5{s@am86L*i4X<0@^0^x!i7Z1h4qB2E*yMZUde3_BLvI(ilDa04kkseqxJ_YB z`8ln|{z9?ge=2MY2QbN#=0QW~H5)&<#?g%L3vd-Z1JK7XuI9_B(AimE5M9_kML}UV z)!;YCy+l%r;EGlA!)b1Vp1Sip`w$+)LCXiY@P~(%?|BtWtfNCo1 z-iJ{~XVkHb(ot~)MEam2y^Kn4(mO$lfHdh4x^+Z~geD#7Nbk~72QVPLgd(5<0zp6^ z0SN-%zBf4UyubIqzO3bfg*VAP=bm%+-p_vavk#Ao4~S=r)DJi2y=!(3iw>MT#j~eX zRQGOm>&E-Z(@nkS5W^__gqPJu;#8jv$Ehy#pU|nkj;_;ZVh*-3^t|gB234*bMG~x! z75+>;o21)46|i&qSnDN2)%4SS9m%KnblgV{QDh6GF5gi}kG>-@sVylcl|Iu8Xm#Cz zrUX}qK#SLif`^6$t$tqqOdg-hO2U?GNDtk<1xo0E3A4g=j*gurtuWj26VNG;8Ggpg ziDe~7itmj-XyDI>ZW|fb4WDoH7b5@8}9bi^`* zZTQaSOBPA;pNzQzJJ3N7^DNm-O`1LFQ%jsfEI2-hv?|@npR5~he8hMC4fs*u z#|=zC>5u>RYRY`CvVi3Dmfs}H#8cV>En$NFxUK_PR-~aM3%)n$xa5I_Puvp&=V9n9zq{kFc;jPWJ3Qv zw5Zu2tE$|eF_=+#QC`TB>?L$-&3hu&-DG2~yd~lLQsd-DFS$J1atpB}HNrxDQ1rw9 zeUbp<%==u7=khKce1W^6Q*`9QPQNkBS=+2BE`Fc)TP>%-YK6_WCEpiglZ(gO*x$No zG5Ji62l?QS&)~x|EaG`t^`-u~!;YZIQw3obbq$C76~#;6h#9{7G8+Ua$hv~KoQrGmW zlCah2sA+NXnWkl9w;ox;vRkU;!WiV>SNFyMy34vb)M}~nt$F+PVF7InX2Sf4yMgaU zJ7!Qv&m@zfepXg~OlU=6!?)p1+tDb7Vvg)Mr z>zEtqWJ|H+QcQXowLdx)Y4J$^~#%Xj#O1~TK z!o9bwC5mnhi>U#Krr2`7-L>l2-Dzg94Tz{2)xlsUoYay%!r(jHroFRb;pHH+;h8>+ zja|X|Br5OIEbM8gL{hqpYLM7SjO~bzC?9%=hVA+>GWw8$70zN+UzE57EHTjz(J}{d z*IEy^>!Y2%2_7LK*mMW6Y`;*`A)hS%xse=qd@}Veb0Qxmlbt6+h1(2y?EYV3AXe>V zH+R)QyhpaTmv$N`q6HNoI*pWtE!MA_)`PCWi)xGMYsVxrFq;*aKB#kXdB1hqTkoCu z@+__=l8g2b=X;1enp_U%4_It4ZS{ToT@shlv&{dhs6V4C)QM+eC%-~y8d``XPIW%Z zJEu{fQK02CZ;&ZW&hI^}l|+0lg3^=1uKp8Z{@%*1SXJhrEgzFLYU5Td2Ox+VNRuJ) zzD)Yz`~lC_*Pm*YZep$40??@x&z`bUE1?*hXvh~asmw_`%?&=;#3|Jr@nb?J5)tV` zL(6AE8u2}Bk+tQ1xAYq%P5U|%6T&)eN8?5-(%D8l)cVUO-{wdU7P^iShy-F zit{}MjHp9kbNt9w2MP2A!WUmo9#vIW=lNyt7U1EPi*>7N$T_l#-2zh!W%@h%petAv zB3L?j8!E;SO}e_j*`y^wj(DC>M1bzY-x#t^1oJ!m~Z{Afl?OrbevtpRXc8 zd##+>+RYq7RaF&%m(`o_ZbM|H@R>XuNcA9Hn6e%aK!?=JI&BR)>BQF7)Wk3HXUnfh z6k)6Q6^RQAI_$mOJ7Rkex1e^xcMkO7NIM`^n(Pbw#&a&=wGM6ibV|B~2RUyTDK@Zc zC9|76b{|_wx)}MJ-fvYN0eKb&5fk%dL|o{2UcKEnrT=SS%&iP)K0gBMj)5R5eoyOV z!-HTokbg&7-Bw*kMl7?k*a^`R>NrbZX zH_33{4)gtW_wL1H}Vw7&kggVE-^ zSi`f6&kQrJ=hWdj94Ww z904hT)3PzFsFQN+@_Fi$e;3QDFX{D%R=wj{I%gpR5}khDUxF*#vc$630~a*;eVs#k zHY6Re(-cz(GR|9{A)HC%{x8o$G;722puXup_q9rOEHH*aN1((wHTl#SkHFWNfigBl zP`{|dMCC(yThNg(;)rtyTY9k*=b{{4==Ji-0zHV<%3l;%%Q9rMK1Pk&u5%Ubn<(t| zxP~1$5qlv6Kz@GAF9)05{{N79X_JkE7DJF!AE+Rh)+WBeT!xY%O4H9$snXx>N#{td z7wu4rT7UHl|MkYBhY#t$`~Q6Ol*mU!R4eAHS-}}a_1W==eerAc`|AgxAxqD*7A4v0 z=`_-A!N1P6Pkgx^^)EeT7(CtVgs9=losqXI*LKZOO!qKXX5Yu92nGFAQ^{rG{VtZ1 zjs!~vBani{MNB_JLtf^taQTbO9kDOkfw}cW9^yK@pk_bg{wutGf)`yvEZLh)8APKO~>=Ww5i1+DXId4dQryoJokQc1jKZJMe_D$1hT za{Av4{Hc|q~+s&-SuOktYlx{ z?VbHC+9D)`ouKfW#Z=7Uev{VO(=!EqdBP+OL?etQHS;NT!e#Q>qYqJ*gRDtg_2DU+ zaBtK?JFeI)ahIuQUpdb_ejz+usOMO1EHLI)GFZN@iHeM64P2hEbOPmM1Z2my8;z%3 zH?RD%@>a%kakX$>7vl-moqzKggOR6a71qbcE?FE<7LCy)*G^F(;QxQQQKFx?z9~WO z*Ai+6IA4eTgh7)6thCPp^FEY`M~_IJwJ8IO@88)j3Cz_Pg7x7%q0{S3=XnYt&MdH0 z5l_yTrZJF@TvP;y1{UZqR67hK!*BS1d#0eUZ(o_!T*;7qHKj@BTP^!?)de8@BL)Ic z-P8yPiNqeSSK^;*EdL%D(N?Kn&e8l$?;t^g`_l{IGXqYJU;3U9+ys`)ka*54?kODH zkSZCZ8!tj|{So)@>GBk6eD4UxV-mACjL9uE;h+v%l z>eVakF&{-ZKhu+q>ai7o$(S4oIQ-zQ|5{nf+!sUOX2(C?i(gTGQsac``Xogv%z5e-5Lzs6YpkObI1#DR@996F%exh297LQYfaedT z0qs(j<_9ck_U%fz9FiC+{th+NPDesh?y#VmKER+6?NTfn?yl;8gD@TgYtFSxGb2ur zjR!D5{hhrYU3~B*ZD|x~9Gt*^W+)9bT7`tikf|{IQ!6@0k3=HPA`-Y{0tU16oHs8w zmn$@!pmqp!HUrK-lQh4|_D!dgcZrxhvL+DP19|M)xT zee&YBwbZT`G3lqzc&@ko-jA@718X3q`vbi`a*1iW@(kHA~eJanY?h_R7aCIj3Siv?5@87B2rX=pt1PJ#<DoQCi=Ou$Y*T+cFa=UYG*v*Oc!+VBZ0neW6h~hIWkeNkwPmE z1m+t&|5@QCxV`Rd=?re`bB&IY@&dDS>C~NH{G-TGmFE}wZRST@f5D>NKojKs7{Omi zP2Jz{!V-7wLQE?fJP)GcKQuP%$T^!z2be?IjD zpp`U%`DSO#k8lk;Pz|qCK?%1D1uC{5z55Rz5Fd#Bi~kjj5MH5Z)pYq?D16u{+0@u_ z@rtecK>OJ``T9lXzG%Cyo`HFR8`csPvzrJ2&#ZbaHV0`sNdPU9dC3r2Y7%u$J)kC) z5E33f?_T7Sly4r~o) zU8)dN%t3H`n)Xm$X>c8pUv02RP6}!66B*h2&FkB<_6J>hhw2{sKGAmEa|fy+|fb}kb9i!oiVaLMFz4}!@xt?a%h@D{_6OruEMwTaPHWvGMv zw?fgt`@g5P87uh6OC%oeM>hWj6LlOwzUsJ?oWOuGn|mVKxo@05H-h z(a&Fhsy2%Brvp&;DgY3xI5{~z5AxiYQD@I#EM@9A z5t#@X6Y6mczZ(Aiezg)*i7%c*{EBAtZ_m^obgm;rs&Qx$sUv_o+G;1%$v6Fq{m_l2k4Z6* zhc>D8NOG&~>{Q!YnAk2DhqPGdX81Ft#|5ImPf7s3$4HsAI(cJm&Q@}5$fDI3q~na! z0j873Te|oo=hz3v^UAVF9^)}pcezb=4QI)L!SUdQtx0n82=V+S0Un$Pie zBt$X6>MO@Gb{=cRB~Y<+^PuBYh_iS{cGLiq7&3x5R#WAKwvTsEV4NfZ1YZdBF78Nt zjy890diph_Suzkw+~zW4J9&OdE`q!?!VvmDWKf{E)O>w?m|?+>z7M2acuelR+CMPO zne25HWFP!b%ezrPS^*RhU$p!A`31|RyPx{4_!||{PZ=T{TO#a8)nTfeVln(0!azmV zZfw^%FI?O5R!B~@;zyYXxyqtKJ zk)ZnC_&wmZ`{!-#jLU6K2!C*MrXu?K4V6CHm=V$Z{F(Nuc-PRN&&C$1q69{T+`saK z?N9eRIS`BHheWe_ln~%XBn!Zsxo3{e5(jM#AP>J_ZO4aYE?_h;6Ho$xZKb3UJwIum zGf>Pxcj1QFf_)HnW7D`=$|OwCW$FS^uK$c`{})b>g8LN2XGrL&`oPlmq2Kwx;g#>O zI~=Md=Df}U?(1s|$1pUB$wPY;BONN-2qvC$qZ8Yez7s{&uAhOGiZ+2mw9&3WqibXh z2v7Ky0gj?OqkhL8!2T~~U4!tM_4Z3lp>laU5@0uCdNcGiQ@`H#ovxHb*wKg7d&LiN zN*VFsb870@kt%e_VnYzb!0$S`59MPT!DoecLN;21yMrF6(JNv%gMLt>{c_8{qo+&= zZ@@C@J@Ts5%+75-I!_nn%3;*t(IhDLiFqFnQgO?qNGM#$?L%BV8ncavX#wDofh9|H zF)b0RC|$A9P6%Q#QfqCO6pv<^)*eh5sMVbzIcP&C0@qo1i;UJCqGaHMzaV}9sL5@Y zu?51@JD$iYl^^5lGt5K!LEdBio{CU(5kp1+&KrZ(#W?Iy*tll0alFw}IVYwbcI_Y3 zZLBr0%R8CMGtV=M86L>?bP|;p_%fGGb&q$8g24o}A%tezPQ`K22Xlj+F`#PA8QCr_ z2RD8JotNVGXBe6FxQYkn_o-}*1T_)=vinuLY&#_E{zzI>Gcs@b-qcjPawm^&wyy4( zg4>tA-nPQJ*M5zCRmpGS)3WOJvFR37dvem|#R+n?Zh>K@CpeGTFD0!cppqE}tXT(S z1qB6JJgf&{-Qw-p@NET1(G9YUe?2f3u(L%z1=g}D`01LGQj|b(k?&kNPQq^5h{s9o zrIA4+e{z3{qnOr?ZO4HMAS_9kBX`wgzAL+$`Z!8SQ{v3_;=4G*AG+0g293^>D@ZNV zt|$#}MTxJXw8*KLb&mHdLI+$ASxV%=&-nV5CWSCTY%?W!niZ|{yv#^N3Sw6e1&iI( zQ=681{vW+5(5!CYVOhgZ2puTfPmEZh1YOr|YWt?*4E!_LTDe~x0-9YA0F}cOHF5Iu^Z$VcD1s2W9|)uzAP@6A9~RUoZ;Hx zmra6)nC06MbZK+bsLLepS)XQX6XF6lrJ;H@HoT%m7Z5?{JEm4c1t-V#`+-5*)_$ll zyTD-{#2yM}p2B?uX7!d{Qm6q1u)(SZ9fYDs&QRL+nDG~Q4IBNnQuvpaa$1qw11=0Tu7&)#chLpK z;OgHhWKUj{yLi!M>aAedYYM5@E}Jj@M!<`*&UEC##w)j*z9JzUbmVZ|ChSl!KwNo& zw)@eI!n+h3MYx%8B;JQQBj4i}o<_5!1J!>TTI=SBw~Yy+$6MxZ>yOkRZKeLq;i5pQ@$~LlUiI+SH|{O>N1c-yKm?k);X&K`@b}tN(b8*p~i&!^K~dbKX)Q;@-O7!eD2faAVTj$kg;vd zfz6{FF0t}g#@9s^oFz8{!s;=@t41jy8r>R3>}i#gby?!Co5{!nDL%1FnD0&vljyRe zqBrItQ@i*L6+tLw-1b`<`lr2-VS-@2heb+SSXtCH6uZenN7yamV!E6&6CxN~tI<4W zu|CJnh1OWpXUr-016{LzeLN4jH{koHK>3Plj$KGZ1vqL2MpbHvRdcK-4~m5Y5SLn! zS)Z=lum1Pj93CaK9F?Y=8(k6b+yZC#>^BwIc{L$)=c9^9KCzWQpUufxcG!PR=G+UY zC5_!hV!l(s;Jy;2e)aQAj|E;Sz=#6}2g*5!(omrQ5vJ#yGKPX{X>H{g82_#81TjmK z3(;7vArBgnOJnOpYOP>#^WMYvzJ5LCA)NM?bBxVpWZD9;KOX+$MD!MIuqZyQZx7hG z#nKR0{aOVJWlGGK0PZ_)D`8eo7X4vIS<6+{E%0;u7~F z4()%$#iD{m2J8-gHcSYf>-BGNMb3*r6p-jvSuuT1sjbij=~9#tHhmFggb4{(D{7+E z=DZ4TFwLN?PaY`p*YO;4k8Ri-Thd+f=NfoeGiZpYYa{6+L~tK|ukK`n2dQ6uzMbJ8 z4W)lGymXGe0vr**^O-V+SjIU2}Z#1}M zrE0T3f5sA%+iLRJ3>p66o^B2z5-m5ZRTM!p06EQt(}FzD9p&X8IllaOU<^B7P(vZ+ z&5Iht#R}Z}gR;KkTpEf>T;AFh=zuEH(|=Tvn){ono3FbZT^9hW{Fcl>pPAkF!t~Y2 zpIp^QVjo?^P8UA8Od?$a34iLD853Wj8y?m*8?bbyXFKjLR6l(6{vXk!fpg?bBPVN$ zvkSsfknzzFsV9l|bY0r6($TZWM#DssG8>zLmTAfj&otGC3Bl<2db{-m{eV$Y^sw4> z=`Y)mo_5oH(GyzL_P~@-sqJ(54-Jj zKWp&0qxokwkL3LR0lNcr@pcDMN>R^B#ePsCvf5e@%NExu3P+3y%@Jw|lIXWTQ&|@R zlp-H6{fHmsZt{ZzgNOxZHs#GD=qEI^0`OvNYz%dhk=DSVP?&?7t~;loZ8aFp3%&6oG6?~CF?0P%lM3nNt8@q$r4KYo-B15V#x7=Aa zW2hQKGWV#Zd}o3xD9f*`FH2l9;FCQz>h^lCyZp^A5ViQzI|*7~X}l5`aGrv%ka4{| zdxDIBCU7n%nV0C-KZ11`c#@Hio;2qtetGiIbH^J;-5tC?m1*(chy}{cdchGw!teq* zfgLkDUE@iuc=tiih8?Lc(F*q}tWH%%42OZ~ans^gEsWyJcTYOrGAlfI|LN1?SFCT9 z{^5Dc6yNdGz)XShY}r%%dtT!v1BJJzoA&Izw)Z4`>rpfNdk#&t97eVMPq0M`yj@$9 z+B%-fe7Uuthew#{P|g)|S4-MtQ{(5v2f{CgyU`VpiYkR~Pa@3{GOJ z+l@i_qYDK4mtL{TdAFbP;5iOIFDUnV^iGLrr{eM@u#15L*U@wr{CW78`-BXRPCin1)vnZ;=@(GpCTIW*l12jh^D)a8FuL^-;6R=IWCTG$zdWAfTR8{mXr?IcV0W6$@I(HS4`Uncv>A4%eD~} z2K~Gke#A?Vu}e6-&u{AMo}@$V?WSFZc~?*2m4c)0*WS!jkTgB9Xs9UKMQoNdVirRM z?vQ&2gZMa(i7PT9SQbCh{F-#Zuz`8ixltXDb9``gj zv-NSgZeO$!I#Ni)8Y%3uuOnQVI)+vy2fE|spG#rC1-7&`Us{`%P``9Db%@a$m!_1% zQaw^HIL*Xn7gCvBR#WJi>Ugzc)6_+8e0oK1wB8ke zHR?r|-vw3Qm2x*TmZw+B^H+~`5;D0GoD}I4@_MA=rzfr7THaog6R*8gJViHy!Kp=_^xFc0tiyF2&Bb%y~o2(cEO&L477B2gMm^!--zrWJn!>5KnW2l*c#k z(|mNg@0~(>WVj3PsfkgU%E26>oB+QlX1TmGl3TxE<-pd8b{>&TGy>Xh~lo(I?%2p4Y3=|48z|gi&86fN9h?l zy^T9pFK3iz-7Ph@m{P7?snY!NrWcC8-Q5UUy`)F6#d@W~+_>7=!S8SAy6t5CLmnvqYEOO1a z{&rWzNVS!X#Umtex{{j}l>-S(r&z&T!*)FT3wciK3H*fIuN^~;et-vQzhrC*W?)&g zOKb2}boe>CN&S+<{1DS&=^RYm?1@aZ#t&!UVw&!YyZe3i?4?Ujy2=dD?USo*$g9<5 ziSuhws7{Ph6LvDCp*c$pJISm4!29<0H|9JvU6Uv<0$^1XNLF_Bi#lTKM}rm!#KsCVbM)Q27jxRjp3gwf)ZQGl?OB#4fiv&Nb5_WA3u$ zeaKcw(AHAwkA30aBt0QWHVZKrKs14x;>>4Av8_kv9zPfss_$Y@gN z8P#Qp4n^+A(KQtllcW4ZcCPw;TkHC|(>L()%^cAGEIyO16eP9z`u>@?#G=jW0zsmp zca4L5ifN;@{=T>@OeezIpi!m9_@*tZmndr`*@Gme4#xD5Sj`q?MI5%*V60)2*=ma< zhw9UK(!O%+3G1G`SLSa0<#jrYe%Tu2pWg`yB^Td5AWkR zh%(nGn!Z?bZ6J06ahcqNbT#0u(S4;W! zQRWsHe%ub#`p^U(1&1uNcVrwD`q_=3?4}d@j%Cr4^aNCF(irXEcD?o1EgF-|FsfR| zPg2m0d>ZYCk-GIRW~FB5>*S$Ho3WX<-?+wHdIpGUR`ViAk$ZP_yk^>VWmSDaA*kun zm`#3P`1|12oBqG|Cr!2`n{K_Ibu^%tL7uwI=xzg#)NAkF!H?VnrdBQAEf4NhJU{?eZhEL)VHfI+|(@^d&$H- z5whbnv-bH=Xhzdy;ODTCvAzEHLAo@BW)WRpPM-AqiZ<`@7fL+>OCxDjp{BfOe;{}Z z`m&K!R_;%V43^E+#(mN+)oruPu-7%DBnkXDTpie4)B=3IMK!ogMculm(0oDnOJcrY z8&=n(!DOIf&rE4HXL`!Gcv`(tse3%cvgj+Yh5NqMnCG`gZL!MStlk)MmVw7=p@o{vp?;Ho?FHP?(@Mzdr(QNe$#)uLD z`jRR8$vI>5*v<{D2j0_f($t>x(Or)?l$F6RomXDv;+GC`@}Uj-Mzs{?KM*&)+W^(K zX<~vb;Z}3Wz4%|-aBvEk=X!Xm@Niv`i~Q5znqjqqpY{1{Or1^9MUREUx#(h}ej7`U zHtvj%3ijK_jI;InQMYbs*Ah9yVtxO#k6eh0v&39UpoQ9F z7o0h#s3=q^ur+tu)-l?MIb{Cs{?s zu*Ip{%3Pr#Qz}+Fak9h~1>wb^>;=hts?U(abLhKa&GjwdcI8O!IPEMnLVpLJ;hHWo z=o_dbRbBjC)mnYfkh$V;GMF8+vYMHx-&CWNaT#w#WtE-6=Q0#Rawi9f z=&{&!bt+xEhU@Zt1;F1)6pig9ee1uwz5T5mo@YjegrqUnAu#bA=f-(1trXq|aB6?S z+3nMHF>q3I8WGP=8jCxUh*|`(!$`;?v#LV>+0Bf2f90lT>X~*f4`+vf69wW17FJeK z$DSw+FIt(16C4_i8OV+fU#H++WoB&Vc;bLWJwC$v3p7_?a4Q4?&KNXSz?FXqN7f6k z=gX%Ai?Y68j%v7PIh>GOiK~lJ+dx6a5aV>Z!dUc^sh_oMw|Wbe``^f}BIED=(YeTG z-^r7`*N!QMOE=W9o8W?*GcK4IHgt_l7j62N*5}&*MpnL$I_k3Ha+Im<$hJq$3(SFc zhAZ|c=3>cY?Po6LhVmleFT*e9tJc06fZ2wS7;QqC>YWUwZ7!$UMDcuR z?^UHz5^=soAI}s$d4oxSL$T(TWzpK2gWWi|?ld%RfiX^%Zk@lq1vl|>H$go#SHJpr z16Eyy4KFO$!f3$pB*Vn2ku#UEl^D}MlTB)`<<|+|9t-I~*R$$XDwjj%u#1`)R-4l4 z2|bI%H=Q#zSM>Q*V%fVQLln1lo(xQiaOn!f3On9{!~7?Z{#flQvN#la^BJg-o-qs6YZ$L5?|M8?_a zRh_Za)p4Qlr<%gj8zZ`h`Z)%I``Ko2TXMnE{R76TFO@&3fMb9KdD`(En9E#2yeYr6W z`ssazNA3h0yEo>Z#<*Jr#3Dvf&SkD;3msPXb97pmm+<&EIQm$H8V+F&llW{FJ;f0o z6Y)+_lPiC`;CbjFghhOWRJID=$JqM&H}!oMH&3tH!LHB8*Jrm(OmYi(yFwB=@c@aVlLFsK)K(ZyE54C&QSZAp_Q$>BV`vUZ-L7T z_$1=p;8t6M!BsjIEWL+sE1(*t$~c!C_0x=b4x_hUJ;6QMXU=I*%|SR|gWqnRd~~s4 z{H3S90Ml&N--vHYGlDA9HW5|mX54(G44e!?U*{=vCgIXZx06k4_uEdt+_tYOJUy)Z z5ZP0g6A**zEtXMPSjKD#U0-XIi zr@RL-&0gpNcb(z=wl3*lJc#ikFgoB$rfTAWZ;|hDmn1FVy0M4-5pCp zs=xhAVDCV}61wh3P(ZGUC?V+la!ta@Z@-3iEmv-&7B5~K>+!-tTuzRwtFMAB7Hv68 zbNNPVR1TzrFtJcxTkbm3d0ri!ST}ZiyZ`0YSl*aU@8nyrWl98B*kz+-gk35&&D$=W z->{Fy&#@kHa_a0x1{S#6-Vyw)XUQ>V{PmIEy)NEcGx%GX)zuk)jI9D$&el7qW=63^ zVYE^ay1@Kt0j$h5sz+YaAbKUQHG8Gqbbh4(>|u%PN~nf@?G>|S#tHW5Km2SlJ9Cu! zk@0BTS$N{`brfo~bz|28#CzLej*m85AdN($ zYrduc0v(UG*)wyEzi@!^NwB)pR-TwVTTiaX41nnGgMY)U7J8m0H^}emsEBS;(1P%y1IzWOW zN_?GlPbbH^83pt*mJd@z2UZy^NEAZj_T5ehaEgb0?Xa4eB8F>*{|?Z4aZB$DGK3u) z74)3uoIYfrQ-14*>xfT`H1+xu`}NgS$$07Tl(WYbt{WIE+k%Q5GZVBN)B>#o$_~K7 zHyF@WpTyO?$ZF*+^U>1FJy&#=*@Ij4~6i|xB65c z8pAhk&VK^#(&Z(&uz@RWdnAkJ$edHI_2^A5GV(Af=t+Lwas)}*$Y>sp%F(+=z@DxG zZOsrtm(IDG`NF<=)Abtn6u7{w?BFN;^^|yhXH4g6RD}W9z<)f_k?4iqB)0S;$&mAe zZ~k?6mtIz+RaBRZj>--Xb@mcTzp%**5{e5#onx&?ds!(H^b zxD^($rVP54J#ak}xhffoz#Y1R!F1xBb|GuzY0{EujZxex?FPCz`JuoRZrjewWu@C- zJyR0HHCo_uiu*4%(HY1swP9=g2+HNIhxa2uxy+@@_18@_`FzJN47g4}I~|Ai#AR0{SB*P^TP=+4n+-MUK3lDoj*&vQn%9xx&|IN*MFys2j&Tg?(A~84sC1SV zj*M`8`TGW8bo3=MK}0k{NnJhF6-#Qq7nvtE$ef-Wm7T1;B3js9MX8AWl15G3GgDQq zs&6SB78CPaG#=ZH)&@*j-jZ#iP*P2*ZMrz+Bc=|f1h8HP)T#O$&o}LE>1f`u#PW8{ z#%G=Agux^#DjVaXxB`;rP1Ias zarLQ00isj5%5AB?bVAGlOJPpWw;q@tr7PN>Lsa&m#3{=$i=g#yW}6*FEdo1c<{pRwctX&= zq3b)+aP^e!`Cl(TRpOcPZeJUdDApP7Gev_1H-0o%K|?e6dvHdVzb^JeP=9z}yx0 z6e~#f!u*Qv4#b|TXseGi>R7NCpP}AyS?*`Jn(z6&<14pYz!&gvJ9g!cCUQNGZ33!S z-cOL^47a&>L0M^0<&1KLxTT?Di6Dep%z!A@IORK@9vZ28j_!Dn&p^K^P{>6MD!wyV zzspqcIDhiOGgMv~PL5MtGqH)NSyI-+@QvNMclqrD3GjQ*QAYs)w6xc!XUTC~?4$n^ zUR?_A4{$R|DP*2?d_9HSTA(ZH3J&QXI~^c*csebEUCYy@!YV;d&PDF3p*yWYqRAO3Y)mt}Bc6{U%Q#wSz`5ln>)*~Ic{Uqgf z+dJe-sIOwK?;n2dJr?jRDoY%RkfdgCd&zB~g+k)6!0pZfWF^B4XHj4qQ|9kQRtW8UEVbK%(#ql*s^aR4J`63d&={^tg{2QC|#3@G3txUdWRUk-B-fZYTnRr&aONpkpxT?eB?zJ>VrDNz;>9T z#NQyor9M$+y$rjCW`+vgNiB*C7IZcI@w;L$Pn<@@Y%_#Vna$_OZ*|sL0Gp84Gu*LU zH`BU>&PhHiYLESB-;`!FTgAz>m$^@J=lWz?w~A)NZXtvrd$`-zK`U(=8hq*jF?L(e z4~lqn7$43{g}a6Y!lC0NE6vRytJ7u{bKKYcX&V$Frofy5y67;yLj zGpzn{H-jB=K2P2GxfF%bq))lmcg;a10sj(GupaowLbo6dwHpYP$2x^~5Vd}8u^X*4 zB&JQ72M#IfI^rv63`;R9`8}ijqgnk*OyCf}QO1Fu`Tgp&t-%#T>66C2(WYDZQ!ik8 zxqE6#{guS4eigC+TS-G>-9yhoTngCPBpWr7je^q@3Uy~~Z+GR5D)WtbR$9OnVm)3r zGzSgoJ}KyGX0X3M0_3Y@+?}c#6$;(fy$(hs!%Nfs0}R%G@JG}|rd)lM;BHum8s)x5 zhP>)smh>3q%nUHmg^i}fC8wbwaK~IxH_}GnnEJapf%O#QHN(NOwnq{95$E*k>CbX$ zhVzW#EO}Um{o!QuAX~Cf6HjJa=pjwaV~mb2ZMb{k#aL=To-4+Z`?UB#Rub`ulBWaH z=T&4Ib(7isP%v>k5Wvx^Z5xUae8Z6vME z)iE3V3`ZA;dRcO8DeMg13PpF9&AuNo8*n&=^Dhg>(q{RXYNqn@3EWt(gAC);x5dQ_ zk*!Pta6lf0!*(B`E+ijsHy6`@wJ!&ov;9cKe}^;lXgDJzFxHS=yXN3D1=9ida~N$4 zbR(d-IKP~%^9Ki>$-siH_N8WK!&_#a^bEV1G*c{+lnm6hNRVu*mRNbqMp@t4o@m!T zB$4EbR_}p?UON`F#bWkXB0>gGn5I5wG(m}xS}O^kL&}JI=iC;a@f6i*u1^FsCI@VL zpBA~3YS0LVSf(#DR=KnV0M3}GsLbq;2kG?L56j?LurBxWjNG$|yiT#Q30?*CM3mdxkjY++ay*bn37i|Mzn%|@^^GI>k_(3x7atM6%V{Gqns z&74u1;!=-~c)J-zloz-Y0uB2WRyP83&-1+!cQ_apbD$Bn{vKwdSo?m2H0D9d!#cH z$x$V`qVu**>c}79&vs&cf7_x5Rso9G!O9albhmj1_J9O0nat7EH#b#MB;Z|dSgwZj zo_Ud|?5Ptxn8}M;b(wbC;S=PaS$%J0 zYUX;xz6;n8*qH_x7|Pr*(rXsd=M|Agv*RGUN3fEEPxa6WIoBcks+0i9uQk6TnSIP^ z)|a8d+332Jic75uqSXa2>s@u=CgukpR~#%djMbni#ettATaEd>>w^(L@6I0db+7x2+=_9A@N#!OpY{kO0JlJLINz5@;P@-wok8!t~Y{$ZzMY) zKt199+8^f_(&2Kqz-1TPjwL39O9rny^gm9i9_;pDCh|vPw7YiTpr_mOjER6xIl&rr zi>}{VW>XeMpEHo%2b>|=f~7|7-DRI>?zPyZmfgNFcO)JOMw%Yjz%*+?NTakkR1W2Y zbm1i)&fhiX-akl1*n$Us{+UCutgHOwqT3s^y12eL@m*ybbmijRF1iwm0wg&w&Je# zjX~!tm7~J`F=ckvW?`ehzdyLu*neZX!fgH3&#V$lqQ_2&+yDfZZ!-@$l%73yaq)b2 zmDZN~0*{nK!0%y~PaXi23}VkCS-mT|)!)#xTQ`-F4*(|HuX=i3{*;F3wbtx_zM7-W zX_v))K350J+B&Kc7gXE&H%7r68=s{vj!Py-$=mYBh)u>CC9dsj({B{|tkR!{6eTVt zP5=5>7;`a=MIPJbjh-8+WYo=pV^X_NQb7Ls%1w>z=9NQf3j4J*V^efSR;dJ7L+$pg zL_HJt=!~gCRr#<}P%f!`CBekw9SA7oAsyZUDuVhxLVthMfddD0B_^QAcfjh;<+pF& zwj6Jq33Ni@TJP(7@ znm?^*DjvG)%8;6pcU&?!eruGMn?5s+X`t}$$CDfw&@%D4b$+<@0mM=t_M+n`k9qJ% zZvd8*nw-F=fJ@BSrDrIVxB_8=xs#YLOO$qB*4>(2YdD~WrUZvBP@k_wVWQ>o^o9Pb zH=?4ZmemR??fc#;bZW0zN2jC5gN-7e&x3*KSWID1&>kpC#EWLP0uTg}y_2P&MLj|w zA{FQ%f*{vR5#IScF`EtrVN5e(?!+j>Wa1gw^s2vpA-Td1v%j^ux=NWUxn}s+RaM~X zvCYu()k|hYo>NR_?_yNlZ*?T=7Z2n(qp?O=$$)BHJJtSEkP0vxf?Ai`ra$L#T|f5b z*3K33OyXrEuU1v}eyn;b0ArfTXy@UvEDF;nMxBo^7uxw_$s#YTZ{d1lh7>>c3T^#& zq}akM-j!7Q$gOp@P1YOv&cHSF9KV6-6s`ZmRWm&7^ix;0jX$_B-b!s$8*?d34t{AXc72qRx0a22$TN2^i+h|HT^OA%nI-8V{HDYf`W zO6}vw;A!o@&NVB&9={z_6JZ&u@g*@4=i-j!U|kbRqL4fP0b9`&EB?J)eYm5msz~#% z#b_mF?&#auWC?Y#f)BZqbWiX*4gZQyH#SnAW2jgxLjPhaTvRF}w3ca6k*{3|7Qaf6zYQ>B|J zuYbf6hopcCvaFQ*Ipz@cO}jly*3jr>t99oaYwUt>%Pg{rOm`>*!eZb?bdj$t27IrW z(=&;A3s9H{uDDscKTJ$uR@~7xWC@L6^y_jO^s8s7iM%4;gO1U>l^)Ss$r-uD?}wRgEu)QD=~(msW16+Q@B3N%|*$XH#;0t5t53tnRT()JOo`AjJH!balWC*zq*WaY+@P@iS_q;$_S49sk*;+d3$o*mEc z#VgTgtGFTSHxRB|36VywZpInW*e}9IPQgJ53#$?x9F5tLPa<5rMf7p8O4g17eBzjz zCDI8JmM$f|oV_~7|g?R6~`ILL@7Wh%8A=wO;a2a9BHj_%SLD5W>z zx)DqNq@PW1%;|RPL=B|rC(0^3c#-ciqasQAjYFc6qTUMXs%NAuhNtJ+1w&(IR$6pU z_;51u?B~DTkA^UVshP_CK3FVjnkS(G|CkL(9b(sOCki4t)sssmM6;3NvBvMQJ88lV z%b_fS+~7AT|ME#6!z^96nbo5A-U&?%vj0?R`2A{KP16V?KC&k;#j%?TYk5!wAIxX7xV;_Qlb)LU293<%I_0XHULH@ogjo{eh2+P|oD z$ZO_Rq9?bimAWVolI-x?Q{NKgu;f0UM^s8rpepL?n)%V+O*oz}T-dGrCEoo){;y1`!0f90dKuye(DyW>LRHtl;A}M&pCvsbvQWTvjhniZ1 z?Od}^IXbnO@Rb=6P#{VA{fpzKU!h3*3pTr@4$L60|(Q?6qls=(DqsOypJ zhrd>%=V%fYE|7WicM0u723^H27s(G<|37mv5G?8{g zdV(ooVCE>lKvVlaoJ%+l&XHf1@+nqo5?#nLRsM^;5r!M!xx)*!Bah#dO;X13Gql%x z48j!+Fn`hxI64xiXq7KUCnwz=4U2%R#K_>K?x(}ZjpcV_DM@JMMwxCA_2!}0z`aZ! z{%n);$Xac#NbQskK1s4aIs*cU^S{!vu&HZmA9Q2Ws};zSu$W@Xi*S=8kWYL;bIMeX zJdlM%IrsmmX=dt|PamJsP+$JVcdj9`h;QTQuQp8o8+eyAVn8gDMd}>f3U$YBegp=F z`40Y)>C(TVZKJ+MuMj63cPtPQFada3TGXAR=|rP#F@xjs*e_Fiz*yWm8c_4Uad($K zIjgqZr+qM#%SO1#tzZ zE+ux(@}>Jkm3sV(vnf*xOILvJk0L4QQI86eZ{}XN;I5KkCN}b#(xprI$|cVIMA*4J z6Fpgdy$R#&nVKC5bnwOSnlBBgObwUzGhROAoQ(w0n8{|DK?N=gT5&uc|Mo32UZHJ6 zx6y+v+qPl)2BokeZ%fsIgNmGFp>Eq?Y&@_GDI%0@W5C2@5qZd+6Q6_YFd}ETVmHFt zrwv>XE1F721t7D+Xjdt#=|2mdi5es|z=Kq<1^bSGB{-cyoCK3eRQue*|Hsx_heesU z@58g}s=EgGtSBl12Av|^Sjd2Mw@SCP3~8WCx(ubX(kg|<1Z~*&=6kyODvq^sIt!={!BZ7%KqG73&`)M=oSwW{UtJ-{w z*yWznM7h~@aj3?b?`$r%9r!^GqrrRT2U>;2MV-|o&q%TmTI1g3GW^UDo;~}jE{E0% zKwTX{iCcVwylQ*zkVD&j5OpGL>V;FcQkm9j>R+%s&9Ntaa*N9s$n>BB{D}W#A!lq4 zWCP37y=Bsqsb`Q|j(!q0GwKL(a-W~v&HL<;?K@2se}INN9N%fd8j7XM+`Ib9IDSbE zNJ7$88@-vgwvsb*@Hv%N2T+^=#;5qI_OZ1s#q+8XV&1z^|4(wTn9zwEb9V2JzetZ> z3HUuJy#GCIB!16KozmwS{4X3Qb82=OXSu^1o&K=sr8XRq2GI)8F&L@-X_p3Db#L{@ zJ#h(1$9Dv%mmAvXG0XDGSBbL?O#eVOHIUi-9Yz&@yW`ctsc;)r78X%q)|0?-4 zi=E!kld_IbTMy?n-T+tKxti=d{p$0ptx`8c*TD}pygYh+XE*dVzbb|14xLpXP9+Yf zg5{C(9_-7lnz%&HQzK8N2}yX%QpP{jNWN0~#SysjmsMt)o#vj=a(5Ol?jEH_&_Gf* z5GoZ+JJULWKx{O6&PN+D=hA8e?SDneZ;R%amB*q1P2}qm7JR>+^(C;|#%F_MaG{DN zHfRyLtBh-w#r&d2f3+fPnStEMT-&zZ>a=KF1%n%T3*Kv?O4RaRo8SpL)ob{rb5gfz zu1WbgN`Qs_-RF%^FJ--9yQ6BuEN+xm{0VMzy*YfjaU9Nzk4tCRpU%0B)KO=RsGi+Z zHG?;_vipU1MkcU3BOkGK4$h)W>|4*)%54&xv}Dom!;5`xy2`T&estz zvmaL@BI9z^T&f4J!6@vnAnjTJom-JVuT>0n`l^lUrn_9Dj7$KX+MJM z`d-`Ous+vd$?R9(u8*`@WOwj?aCf%Yqd03c>tcZXYETD^lQ65f3&4W6ujCJ#c&_wc zT;TrPw-9r7Zbp8HyR#X0c-BP$dA4>6HIjW&*dvoK-%bi;PQJgq?K`bEf7n{AP1Njl z*{ay&6Dh86yj*gjiMdR1rq$$iry5qdK6rQE)SU8f?dg)^Zlv2)~+M1FiuQDUX%-C6i>?6Yj(U| zOfWE23?8d-I-c<98RzU5s2sNei9j6+8>7gW;@kAmYude)`c&5-@g1_1o zMo=8dRXys=DPo4)V_s!0wr2-@{1Bnarvn)BCBPz@Zfn7vDZZl3HA6`uk-40}fOTkP zjj(iWD8%14;yWA0pR`JyzVWbnw9=>~XZoH!DpKwe_h&ao(oji|;7uHy-4HTZx#cHc z44$7SReEB$Rz+p4hei6P&e0Cf_-_DT!^BXd({ntp&oTo!5k7-s3|Iw~c?M?Y%a^T^ z`h|n24;(iux*tMPoIl@An|R=O9HPH5?c|0AgpE)AUbL~XWRH0PJm#mvsK*>%BjZ_` zsVF%AFr-}N)gi=c#w7d)VAUX`d@K<`DOj#s z)$NJz>j2s>xs(dsN4kxeedYSLn^_(3cKEAA0L>`3HE}gk*qxi{*(-BTd#CzSMYZE( z?}+I#E~>Q;bCLr zU25Ipt&(BjDXw02=mLHHES6F5_bSu3x^~r0Pgl^(hWrKtZ4|94>k(l1jX+yCGQM&r zIvZt81UA{H1xD()#O0_-3kFwf-6-P>+G9lCuogn&hB}`hZ=IgNfz22_*(I|Q6v88| z(Rkldb6^R}R-acM8DmtdI7cnLG5swkIrpd9#T1sFWKL3Hbu&PLp>nc0LE0P|FUFI! zs?>m3AXe?@XGLx)L%A6@rqHly-J$t0R2IQaJA97)x2U7|ED;4#7rWK2y7uZ4m4-iT zo|`n}Sp7&QRl-%{w}vM7{Y)6A4}(SimDCFqjEp8TDv%WpcUll4?0c67?F+MRdb&bE zLh*u1vY6#5L>smED{jGioD=5h38jBdmp47OM0__jc9j<~a_clb!I4Q;I1(|1mO4E^ z3DS||Phqr*MV?sII$S7nP29xG+1+H5x&AY$k*~Npr6e^kl4iD`dIq!-`YEg`yP&4m zn3ie6g}9%br_*oel#juI5UuDW5x7TR^9e)1m;B*ZP2L>rd{8nml`QAzI?m?jk!;nV zPO+Aq8fjCZ=1!76CoTX_`W#BsN3cr1=Q&TK;7?&z7X=NSN>NkBPuc6fT4y)wXMseo zyv#f8^soOtTj4A^90LMO+6UKVEum;ihW4{^yd%&AO0O&;E2K#zEKp-XfpF3rws3i=fQ_t#)lXb zoA{7>Z%L`5<4UmoCbW4qsYEmC71m&NXM#9L%@|&{*fZX{s$Hn#;e{L6lpRWiFmAOz zr;}msg};w}I4X(?vlz63S8C_;!H}%~qpnV-N6~P|k(HRFL%NFYp43s)E{ZiTS9xO_ zs2myJ0|DFc5FUYaC0Bf)G232964*z~Z3bV?*&;h_W2Ty0ZQsf{l;f%eTcMNhWrV?jAz8;{I9ggcKd7xE7`tpmiSMiiE0~kW$!OMPPt|kaTY0 z()9rJx+jngAP$gR9HLNO)+!sC41vp)z(S`E?;>+gNGQZyL*)W3O8QusnBM8J;5N9* zXcA5tfX@s|;og)nf@Xuoy3`vsR@({45TbG>fj@%MqA%mztO$YYhaUVN%}hW3PiIzE zmA1jG73HM+=h-7d!RroDe#E zP88nggm<7af2{{-gTdAC{1LPo2=Ax>EW=r;g~Xyn_qXJ8XkHv)@MsnA!MU5_z={Ky zZya=>TSGWk5O$zI9;A^M(mK;>V{wG$3W&p6Ur`;lnB^!Q>5=N(#ezyRzR;2W*T_II zLX>D)pu43LSpeC#NxV1va%B(#0tSnkoE0N!>%8H@_MRaoi4l`DTC+-GCLzuMp9s`V zh&9b9efw<=u4iS~RfE|;&z*s|AUY%gEEQl0cL3sQS#zG6diK6Zf&I>4?oO1%)<7F@ z>vef&AS4$^_8)C!S;y^VAIl7JsL}_w1I(bCg)>>v6kwgnGxMK$}+7 z^r3G|GPI=|nLk7z!}9m%K7Wp(k`DbbR7anS zQ<5>rU15ns&d>hB#M(+|dGWUQ3%48&Ve)@ef5b$d>Ojlkj{KrMU5|WgWZ+Wc$a49( z@Tfy_Io6Tw)}yH>kEfarwj4ef{zhuCFfLY)EjugTwmuDcT1V=3SejfX_f97_R(4m5 z)ZZYHa{YDtgdhCvG%Y7%yga0xfT1wuy_Bzz3Tv?XUzg>5A1Q?SvvIIe4!lBI-N5H< z8M;MIr9ujH0khi(TlZAPr`^=oiX3`3bgzH2#UX(?Uw z2T$M>`=*MBgTlQSF%bfl9Ka=uE)hG(8Gf*H$o#S&e81k;#uX`v2PZ>uxjv`nM&(os znpmgxN05|G`IdgTi-BPKQ;9M&vpT3uL&cRaY&GP|*q(;qRE!7EDb%B9I@>nmB~=pl zZ?}Y@I<3zCRGC)ZePLnfPXAk-P6$bAKMxw}dJoA5pIg6N^r=Fu(CV zl>92=*+xX{C+m|H@-nU2{}@`mr`#$`PLEUcDKpNluC5jlpc{!_D2#;z-1A*uw>tbV zKHupw6utE@c7-@1t{7lEdf&c8Gqi8XRK!kiH^FPx$Ai3et}u3!BNBxon3!nD$%~1J32m~MLBXc&Uqw=GdIBV{O>8yrPO^1jO-)VN0r*$7#Zo=eWymCB@>BdezkGZ*9IZ2QfT zVf&zaF>NgQGD?$2_UdeDev+j2R&LYa9)SO5cL@Y?Vt8Nfm^+8o31hK-<%L=awn6$D zNd_a_!X`UTywV^gG3}RNoI8?xguSdQ$0pf)6Wuo!ZB}l=U;O+WplGTv+a*!g$kOl* zjgc{XTjrK(ijhP;A~7%`KxYo(;Q7(e=vFSTd-oxi&v2D#z@WSaCx-kz;T|I4q*6ul zl{$9^-h68!Bv{%9@Qc_i)L3q1^E5qIR-HmdfFWa_+nZt>*=}E(bX$q)Sjigcrcj{s zm=ko8MxMwv>ngFUx6h*VoW4N{6MwcBjsh)EZ_bdEj6qU$Vl5?N==?4@I&Do~)2?~?bkjEcc# zsP~63I)me=S)un5QtTg2I^vW5DWWQ0H_Vp>>IAF3ps?kvob%I`gTwG`kV-~#qawE? zSlH3Zsj_?Np=jxph)agNXb(*k1%VVLz>PfG+ujM$oiQ)>h1Q6ch;SPGu#iLw@Imd^B18CZkGEiTWOkNkQkZ zp32KbAQ>tztdxSnsoPe`;_l}pnUT>IlDh>QUCj z>VlNds2y+4jc$%A)q8wrujacj1~cOz_4#kI;iOAeS%LVR*nnbdo54!t+jXAUtYb32 z#_0A*VvBtGg;OJ|=1oWyHCv*`j~(M%?SmE7GQGsod@k{knhw;*#$A<;4GHk!rkM@Le-jT{w@bG?&daG zm1Nz9CJmfRR=PF&rghh(S$AkFKUp+$CQNF6QF}5-tS&h>zx$Pb_cfiz_iGCS1*ySZ zFcZBqgcybJW;~LrVhN8>UtugV3)}|R+6a<4uhn!SorrwSrpS+C6v$L^M^8U%G!kG3 z3|#Vz%F47`bIBG7KkiW@T!6SRQZ%T#(Tazxvw1!hmpdn7R;L&Tg@jC;1;-~YBoP0G zSfVHJ#_Evv7MdN<5=XW2A4l$WBHRV!2DXsy3hi_0yd~QM`J) zHK!;44D=ru<*#WtlV~lFIXY?p^FfJwfSk1W7j#~wn=ERLw~7{G2&`5GO}GDBCekzM zcD3HzitVSO(L%WeFwXzsVoiWLJtVgX4Y2&jp^P~cyM8wGJjxUe*O6RKNE&hXFsAm~qI%sJ8|x4zQg@$0Jp{Esy0{rzbt8%7Bb}@CYziJbWn6c&Ox@*M4GRG{4WV$VjRfcS8x;L5v2V$s;1g^XEe?-KF-QL{OOTxAyTM8}Gm&>|nH+I`B+DcX0~T^p2c8oj_P&p6S!#T! zAxN>&>y6ky=ttnubuELgWnsyJYlub$gB7(j`8tf>7T?vxKkIy2L4FX zH){7H|I2Xdy%uF)TlS`B?)lL-F>NN%KcCHFppWQv^56;|O~4}mS9nqbk*s_jH{ERB z~@F4|-ZzYZ&=Tfu2CpkHh)n z|BaY$SmusZC+2os{<&`L0!QwZtyiA+MZ)B#-?9I7>C_#pzL$nYn`We9AF?(+0vyVc zZYxPh2?VW(6U2ZE0A%Hmx_7Yf$dX%|yTDz@zx(t6{m;}JDPNXOm>GDo?Yti_p24s_7o5Z;-1N%4=MQD$^Ha|UVw_c`nz>BB`PX9gmMy}(?kvpw3ur|1lzw5) z-3cD3vB&^E1L0Kj8R|X-e+D~K#9xdTy?y)Lu*_6{knHv-OZYdZ!pa5$;KE89 zZCWhI+u2S1K!&6vB^Q}SFX5pD=3%r#oLCvh%j`o{sfVlKZ`oXb8F_x|zXs{!%UCvz zBa#^#ANlYeM#h05>iKU6)z{te^-omBpQAzVFX#8ImAW@-86K1>|4XIdwzIi!XZxs= z<6gFJ&P+eb(-yxXk(sXYaCAIC*(BC`(nux0)iB(7R{^0KbHSS>0Z_wsY^W50M1f~J zgPNW#meqRgp3lA0Jd$~t{|b$fu;|7f>CVRI0J`j3lLRxfq6Xy2i%R0jXx1*MXd~79 zpx(RAtPt}zPk{IE{7{La>!@2QT7+tDE&~wj7CMMX-&L2jPm&Z~c^uc+)?*N$zVl6F zpg6ou{J(Wb9`De2hdB>wqND}*`PHqg4N^1zL#YrW7e{@h z1(+>wL9tj_xowTkm;3*snJYF0;t|w$CnIHRP5D%H+IL`v_s?`2WOAKym9sT2=X(M? zbFHF{I*BUh0q6&NSA27)KhFpun$JWo4-G|$gI8A@1w=dak`7XB)Tbk= zY{@JOc}4py3qc_&^_Br`-%yiFY7X5LDSaVkC>yJrR#~b0%Mp98oTs@Yb4y&YX);QX z7x~4)KTz8Q8{;SYIjjAa+5vdq-MrAOVZ8Hh#zNaejUk5_ksb7G z_$Z(^TOd)@S9rvG!v3Ig={!;PH$&Y^i(wO6J#E)>ToNX970=3%#pG!&an_8Sq;$VGXzWQR6^ zYl;9Jxq&(eg7B&U5WL;7V#7MH?rE_!Iz_ISBXdJJJsxmKJIp+_R@oKffUpoz2Y<})pv034ze0j1oC}rdFUlQCbshQy^x5JA(^J`vMNebw>Gaf1T=Lj zak)0)2acm#EM;X0SbC#|;GVb*=$Zuac(aqKARV;g2d=auG=Y8&_H{c@c5>$oi-2@o zQre%>(m5`Mciie(OtOoUfw(U6F6th%i|k*x@o~feZE0cjor#O2N|S3o3i8rzelY?{ zS@KY}*nKNBT74phE?%eofKdVIUH9YruDcsq8vphcME2^8JdDCtvz|Fx=+`*Q?RW_x z^GqNIv05g8O3ZVsU)4V%=NH3EHlCp09He`->bP>$(z(Gr9Z&9#ZL-{m$PPkDJEY{S zV5Z5gpDH0d8V101F~Hc?4uVHd`t*7y>UUW)@p{y00O}#|81VP8t-u3O?C`l)P1DQBo_6;Y5Z&;o?EVce)7A~Xm~fOm zYT_*RDtJMHlSoBl&4U2}sAcH-+YNae6OHD{`;QsC6i!DdED6(!xmfq=BqXHdAkAc< z7|Gz)Yspe_3rTrPyFU|fLS3DOL_ql>@LgHi#R#|WU#=mf1Qsi5@Qno5N;NpOwC#FS zStRlQVZ6)l?V>5sPZE759VSwB0SztqoJN6mKILaAYu|3X1TwSBp|8eUbNwcn1c z{WN;q9$aH$P8Kp4a)0(*&=mdI{v>pWnMc2Ay|~jlgCOoVJeJ%%zqT89fgNV#G@|sv zYtmnNXPrnr*E#|jXLQS~g;2#{3#51NejJ_SQZk%GsI6XPX##)rvjv*rwv zJLp}E*hJdc_Zhfn??P`PHC|rc7CIQ06n^badV*rh9hL{n2V>1#i{)zaw^VGFRh^^I zAd62us#R;tQ8XU#3Zp?Jom&?JoyYtvc3L+mXAZoe?Gy}Fw7M7=byY6X&Ut8EVkg?l zWkxV}BwKS~T2ryfstJ|{KIA;Y3DMN&3SyZ%stw#f^dc#PXUlCO8y~2uB2N&$3cCCB zJVX%Ix^`9SOAi}Ff%&Not#{ghXjq{imySU)kk6FAx~~w zn*!U?{e#ySn12>2RW$v*-l_LuvD`d2pYCW5YQZj$nJDM33Kahs7Ead+lVxQ6r#xxa zm8+WjIE8?j1~MFTM|6J%s@)u<<2~=`ovc&(ABH|6Y3t}chK0GMPWj7K+Te9!Wx>QAOVl_i9!Z#Uy;NA7UBE)>d>X-H}_3UB$5c6YXj z?ll|kA`ia!Ae=8y*@9%dY9Af25{YySmV60Yroa8bOCvt&RlC#qCM^C=c1_}m+fem; zJgKuOHucO--AEN~t2H3pbQwh>B?O_7!n^wQ4WSJ{FA)o%iAV^DlnJMo2(qelEeq4{+*bK1O4_@`#cdy%*?#~-##Yzi?L5gN{XZEbnO3pI~0 z)a2kk8$jeT6E%(P-KZMWKaS$s_B{qZEBGSL*o4QWisx)@s}v%@Iv ziEEywOJ>bJ&kX|uFf;UGpW zEHNpVbFg4#Pg~tq$;Vf$oWZUzxYe6O8JbmrnFrR>bIHfcrNP$C!;LIJj8KDJt7LgO ziNn)Hz|BbkshdkPktKh(%zkTb z=ubnUxf#0B^OWCf2q>&@Eq&L8SRehz5I#esi{Yu?u5tFur*`2QL`Ppk*G5%ML(igT z&z^-fPezF|L2T?w9ZRR6LJ)g~i<_E(AeqY|luDQxDInTpQ7NhU(J_~@dvT|`IDpIi zo*=sR*<^m)x%JPyDtRxdk`Ws*TYtBFiFs&sH7Fi6o(540BJ&74`BTImzsIsVdS-^4 zN{YQ>%*nt`p&Cy7fv{Ov+DsC}Srv27jJ|O~|LE{M^i<-6%IAPZJi9N=sM*1Xe5noEDZRcQpcQtw+11t6WPJ79nKSK3)gF@Eh8Au)l8(Tr zF}6J=NQBNQ<}NN8WqJzk>+>Bgf@4`=iYnpe!=apr)QYBZoRxy1_YzgvtXE=b)Ya9$0&aXvZg8pya^Np_SpeX%=@W7e5lZ)y zh~(jG1zN?mJ#op`$^6sBxTVicN49~nL#}U(dxU#n_WCyC)wmAl8@f(iGPzs>`0P^! z$m^K)#r^jSQ+9Y87_mKSyVm-mMN0?S4yTAImMlO7xX1Vo3UzrKz)f4XB53jF<)~S4qi(@x`HCsEfENU_2TD zxD+4N7cb%dM#JAAe3fl(iM@_hL1kJ^+dH?8xDmdiZO6k~T_~7|BupqPxBsk~1Tc1? z;BAC*0Kzw_7n>CkJEQejbpzzXWi05j4{WNG9@~Ka- z-8Q9sARy|xm?dtgrDgt&fk;GL@;aZMRk4Nsyc**uR+SJ zV(tg$FPQ$A;K(C5CP`*2!0sxqC^0lTswByVknr3x;t*Wxcp#D(Pd_pe_6ogg{WAOQ z59d~<6A_wlgri=5qq6SXT5^1V826^&l?y;IapN`;uk~>ucRH0y6cesWsEC&+kkXe0 zsf7?8*#%xFz>-qGScdGAaZpEaP8g`@E|6vC8k1>=?8p*Ak5%%o6=v*oHCVb{Ck8Pm zD&djjV1!0$=6X4g^pi5qLBZ11&nFSFUn??ztRF%G&^c2WH)v=LN+m5E_n(iRp%-zvD>z)aKFEFHb0Ep)>SWhIc(;!v(um z8fllhFBJ97!o@C!_J?Z-$sfjMGsxA ztO7|pY71qCWk*xOfwn}2$&C*`TDE#W*}B<_Tw!*1WWS4SZgyOnV^Fk9dV0mXBYRd70O+kj&T-9*|)Xjf1l4XTeJPzIq5Y>g~dK$3cO zB{180K}&TYLV1u>ZtoN?Hpg$}g#2c-Qgg8z1qH=ytLRb~!>NEGwQjxp8WCkg`rgF^g#%WC(_g`(;WHmfU|}U27mkkQtTo^jw<)4INe)H;vg$}_A<~+Yik>Ycc(IYiV6lVc7cr%nu3_v zZ?!?smNV?E+wB!Yr)-89LEye~UYTfItxQBXqnEn66eY6(ZIHQM5gR~oBSAIyjVjPA zhU=sT$=&7E>W$?F`o2^NV)~$qNNjKX&Om&6Nr`LScn}Y%YHUkfAE8L-TVEQ>mD~)K z6kclk6*wl{@0zN-4gj)Bt8D4YFkE(Oj9K`pQk?{rw}dc9=pY^2<7YjtXt^w7T{Xumnn#O!jc)rqxMZ zMbT%w8%_(z8)-FA)H+sT7r(p0y{7^i_Vx?99$MGo*A4XEl>?%2Lb%!p7rVcStcdga z4$>l_dkkDorWa)~P9x4iQAimXMu2huNuo1{Vds*h{T*NR(`lO8M%|$+h5XT-!8w7u zJ8~P-LGI7Pi#a{hFC#BuX#B}qUCK(R`bzSq!*bSE5Kq%Xfb8ZE;#o?Y;5kM4jIg zo#Q{8o5`WJ=~F*2LXbog(moVY5%EZ1B4INwTBLkxV?xDZVfbp{FI~vDNCiZuxyLcL ztvWSxmP~(n_XMBhP?n|WwGxVUz8$G>Wshv`3D#YZ9NUQAj?Nybt*t$q+w%Ey4Dm;5 zyq6C2!qu~>076et39A`iy)z9W1VjT^9EV85+p=L-OQ5nmt6;MBZIP4a3hJ9-vVA$c z$%Xd9pWEcx#0u6os@9V6JkIo}Z!PkN_AbhN$Te)t)4n9`rRMm~_f%TV!U~jCrKJXk zq7t^f`qK(E;;pyq@g^S(;*8nZ+lXsP-VrsHm+L!+=qhs~&Yiwzjyda!_&{h{$>Hu< zwKv$(k!Us=Ze>0_?>Y=O?{=h3&9_szTB>h%E=#UlCWwI>8v9M@V(yc=gOI?yc#I$L zqLq4`#${Q)EMjv6f89gRO<32EmBTn;^0#lHZYJUT&XSxxhcNT0Bb{r7PP1w&S}Y8? z@lZvVQ=ZWgiYm^O8xb`~g_X}yUiBly3Q4i7%SRajKMHI#Kmr^^d8y9ohxK>INZYRF z65D~FcK~2J0f=2?o@168jFpv$4|^j8@#)R9tm|qTCvu02#D$*L2`F(^%qP#>tggdO+aLx z3Of~F%uSD#R`BfA?ByDbX0j3=X?LBXB?p1}v-Yi0jM;x0I*ndn>wv{2+L~LPL)Otz z1&}-~T-9E6r&PH5_T_!(C9+-i)~jGrsb**T=%A#oSmG6NdTA&CIBw!G>9G_;*nB1T z%(+WuNjezy%A|DnUuqXizS91BfG0dr3F?+6#-FMZRc}q-4}T(qB-ico-al|%`s=De z##uBIKYfBzpUgFTrfZ%@=RFV{({m4bfr@l&9gp`S&6o^TxGd;&wTbTh=xRF**p8RL zKe67O1ljeZ35r*F31kvp>kkiSEQv0~|sP@(4^qguFTJA|g^vv{KXZa9k?#^sSZ9%akF zLc9#7Mk|WJ|tVsP5w8WV$S7UFTC5hcyXtHTIvZI^^s8Z(KO!67ne zog7u<$ATZ}sc|B?5iI^W8Nn_A<677*dfx{7;d)D-mUnK!3ya2OX*Dq-nIGk5lw1ev z`DLL>wC1+yrN(ZN$~uM)IUK3SY`%EU;9&RilCaCKt&a1*#Sr@ch-zD%TBXJ4JTj1| z^0+JdB}SV0tH-G6H?4O@FB(e+F`lbS*1aje%o2(BScI7k*&0W@Dz*jo+`h>^kVx!$ zppY1f{A*q0h^yEL##l?Pt;|dw82W|4SDfLek|Zg@!${~lODeS}9*yEttE!eT3XIv% z2|hX5BHK=~T3??na(6G;*`S-(+olOpc)uB_@Ig38p&-1~!prM$yCwPCP7#Df>I7RQ zl6_^Bt6jts=n`zB*0Fu=XG0Cnr!j;CuPqL&kL6-^Id7+qpKnm8bqkLDa-FcLK+=vm zLFD`_oa$P$H6r3kax&8Q_O6j2b-XZGFCqv$T;1G`8m&-b(fj^VbLh)XxZ&ohoh|RZ zrQF#w0FRD}i>uBsD%3sSprfZ}&EdXu>e)8ouh#dl)*R~w@loQL$wIwlzsx=YVX)MN zan4<9eWd}2d*_PxHhZ2&%i9;5Dr^_dp^6}c603P>9|C4bo*9QwsKaV9)=|HNNRosP9f-@YQ(dLi z!&_&b+@kA#$8653E$CbUk+_gk{P1>U2fMqC5z=@ZK3}dQU)nB_C%5*a^Cbg+T~cn+ z!b|Bul5DSnr@QOllPK)nC*s0Q&j8Z&ZOIH^iZs+JT1@pOr4nyQuHDdFBRMYlYiDcp zt?h=(PUZtRI^m@J$B^Be(hFF5`Qb24Tmx+nUG#?@#u+K58%AX5Nc+?PB7(J(X9*g3 zr>MT3ldhGDk+!t#d(I#W1Bk1Br!W<{#XvI9j_2 zQ3kKu^rZIZ8rkl^tDQ;d{o{kCM_^oBoO&~1lz-KM%jx!K#}{RE2r*(x2{bUX!nLR> ziYJ5bmzp;`HkSMP_3O->vsz~;zxw2_ux)^5hLH#giW!rza&m$^eC#f zdtmOm^aMVUKM1Sq_sh$TSKMA%svr54HRpIgvmf}*=M(6DyNb^*87*J%jwn@$X`AMY z>6&Ih{?%QGSzKDO8*)F;FFZY&E6q*cPZCPa>lt?UkUe#h|ma6!v4P-Kl2Qx%bTEsd3?cKjPk?V%MvFaaKX|GdJBrb=IwD9cem9% zVRIB;|`_i8g<7n^>#e+Ji_7K70~)WqG*0%)kE%i8j$kkj)si`&nx0vs0MrT?~-!& zbPz-5RD-?I2y}V?ozN9;;P}0(OuoA-RWWylXx18|T_h$ZR+-WVcW~s620Q6}_m2$N z-s8skQNbBU0z<+l7~J9yr49{xojtqrV9eGeOMK%kDo~tYCD*|6#sEEY=TUY~&(3X+ za;{@t7-Idc$U*)Hl2@7D)+`!>qqxZ(kB9-Q5TRT}N%1_(SYYTWQwp7w>6JNaG*LM* zX<}>~WXxca)lpNqaBlL0lvMybib1(+mKdqX4Nd_t$4%IuN^Ht-296$IH_qE82uIIk zl_t)rXU#N2L)!OBDs|AX3@RynA_h7y&KmBnMD{tEePaJwHTK=e_9>b+^IDBo=Zd76 zGg!}$oI$#)=MT(By>RW47tGd?a9f#(GVos9MO?KgD~QWfJQ$rTFiNFzcO5Ohe!^Tv zyG2@E*oaDI#;K%JkCHu?ewMydelsVc7fo;1YFs1FvzD=JkQnJ9;i;!3(--URsAC1I z;$fgOPLI;;V7P)1`vchezvVU%_;j-3N~%u)hoM|oxK^wCL21ileeB0|_Lv2OrZ+3x zPu)_UJA%Y)FbN0Rn^Pl^IFCF>+sr`O@=@F7L$8x6{q40^r0Zp>XK+$npQgi~FShT- zy*{lD7%>b+SL~+e<%df&Z6)`OH1XBYq;W3D&Lc4*BI4X!+}<`}?x^99$K@3jea@d9 zY4Tr4kfvb;=KJ&H=qQmqcG+vFFzY5=n3K&EEk+MN91KhTa8(B*ail?N6fa%)A)%-m&) z0>6jnjbxN-DgVj7tRSt@`xH0cAvRBWtGR>fq5?i#^+moU3zxX10+2Kf*TPRSo%p7@Vz-Fs-V7r8ewwll&A{cfaE zUj}55#v41D7okDD>zRzr&bbFKdzvdOW)Xh|0|L_%nT`+bMXUxl&E-=^X8L@8dnfyn z9D)DP-fTz@TFWPP$~8V#tw*2aUd1mh{z^*xRYG14dm(Np_qNttoAb2?o1Vz(V63ild+#9Y$Y)@JI<4;-eLmNO*bDb5SMy|}bIC0OIu zvw}o0*>NMl$zoLs*W>pGeazz91Td2sz0DLFsSbWHj>SFKMTsBliIjpaecfYGAF zZL*^~9ZVP=TxTZNmKOiE|GzGe9los%m@)Se2d&&!aRYJH$8%6nDyr#EA(FDRvY(MO z_#<=HV;$e4}P> zheiXE*}8Qq=jR+xn)MO~-`&x$J2P7H4x!9|Ep_ZbMG8hl3!i1PcV#$+Y-wrwq7q8V ztYWD<(3E{S%Frxm?L*&U2qPbgg7+!jm&P z?$b|=-ztj_4HM9Wh^Ke4O%Vqd*X(h-hfe_XA5vegU*R%?~E_4PfvxK!2mTw?8__~t}Y?0e`$rGtd1huX^lCvZfB3^<0XH!#Y}= z$arG7tzCb~UWwbNye&yom1G2@%+3r@u6Qp~_l%8=t%!_5DtiuaVR}W2hT8$&uc3wE z&ZOb)WK=K_!lgWQR8WQk!J577=BjK8|QsnGCKEyWG8a+{XQap6zVi#UB+BVw+}#KK;Ab}n?qZRXcLGh|vA-NRKL z$nv#*tL!t2d)4j`W?uehJoQV1z>_WNLC+jsANe}FyKtAQJDP%`IA0Ha=fT10NpQP=#!Bk36ZWK-VcT*JQ;7S7!9X0QO<=5||ZHU;NIdJXr8hPjrV2A_z1hPf}ix5sgP z6{9_+(D@elv8ra(b34C(e@j6tlH}+2N34V=sZ#XO&PT=Vr=z>;CApab7)5d}L%u2){); z$ym7afkEcu?lCVXt+LwUbO-X9@u2YnpvKJ;OFdjq*LOcTX)=w2wd&B9Nt*FtY3Y@e zVo&G4Nj4QP(wKFXZjpb@JeT$)P$@unsUU))!yG1rVl>J3)7$AKv)Y-z%%>h?8 zzp${|yT0vi#htL-W+0KzX8R9NYzt~}xWTClQ{fL;`7BdTu^#W|6ZDp%R+EquyDBUf`QB(W<9&?xOpBp3`(*q7FxNponOJ$QWG z0~OEa_lNYQ04(0Z50n6t>!&&oq0oTuA1LSamvE6+Uwyu{;f`D>qKKpPQP4ep*;*y8o#>!|jmaX@(SX^I)TnTZ#BoVY5d|ar= zHnY_8S8;G4l6!ucEw(UDt{xS(u+MWYLPc`#Pthxsr`G#wG9zYVMduGCMZ#Rq%iI@M zL$Mod^$KOR(>AxC52>M;JWT1IrBl zm7$T5xeIpL9Y;4tYByw%;pj^DFaD~7C`?Gz7>`rugTq_g{nhG&@xlWn*{m8*3iE<| zsXD^?oXH&Tp%M@f0MI(5hrnX_92e3b!-n_^=4}TXR|il`c4!jJ&doIy{T)ma=Lt$` zN4kad@O|NAON%fan_XBryn-4y z7)|oQks`oCC<~{z^z~I2rK}P&TKe4U4B+_jX;sAtRn2_)`;a3w*Grsn1xT>ulnyV9N9tZIkYaXK-=Zw9GQ;mateahDv6_UBlxdel;(euUV z0q%DI^CtmxNB?4LM217r(j4&%mF`icNDcT_!!cX6-?I()^E2g9k%#eNzsBu-5R9c- zg|Ah8iT@v6?*Y|R7VQr+*0CUtiXbTH03sqFN)eDEDoB?WdQnumNS9zJI;be1gc7AA zy@p-_0UQ(rq!@Y&Dj+1343H2B2EKD|j5F{3-*?xnS!&L`r|y2q;+ zpPXow9FwkQyId@khL86}J{og_Fy>7I;D9&Xu`)OBQC)Z3)NNc#Y#jccRg{yXIkBQV z(fh3(w7K&Sac*PSnOS|%AcZGQQVT&lc^5Zd?_s@dl_>wXX>Rn%c?EolhyBW)pL8t) zjF+QitBhnQ+T#9bo7}0wGlgz$v^T~q>D>TbW=4$BO?KBqf zn`6ib5~WJc$9>Ij@FL$hmZ+$zW@CAzw^$Wf)Nr`xez;cSKI&fCpm^wyd+NVxs14z{R$C*u8uXj!ALb1O8oDPJ1#6;|FzvIIFyZ19VR6+yLDNEYB2 zn4DiuYEWFQIqm8533X1+Bv!bRKN}ekprDlxwyU6aE`yj zj%lAitI@(6IklUS68fc6;wZd$?%*DQKS+wD+U?TidME?d`53S(kuoX?M0^nz#Gn8~ z;ZaEbzE^xR$?1f4AGamo)JgQ+k%nWhuy(`pGR}qC2E4Ojb%CR%a9s?JV?cuiBJmyy zULaW|&!IO5vG^4EhXsHBt+^T$;MQwNC_;)&ug05np_Yv_>e1 zqA!Tw`FEkpy-vg1^HtwHA;&XFsR*CO5Kz7tNFothjenoz)R8}l+S?rGjzT%lvJ^4n z9w1+E_)cc!XZkb*o#Git(K%w3h?TDl2uxJLWST`$#XBA-L*L^_gr&rqBqJ6zbuh+o zUB?ho|G@q2qoLF}SXx$Dc+8bB9l_a`C^6d=&3|afK1LNLB84`9pi1V{TGXWs`uh5k zXxLypl<9jG2)xZ2@qmVmxQRv4UYEuoeFARiVa2zFK922oGl@H|5h~ojqb@RYYwbf~ zaHpYiPIYjG4omA7g6zB;O;_J9X`f~Mt9Lm!=c@9@~b)w zSbNyno_n?l#qWSY>a3ANldbrrIVpifS5!JYxB26cCek#WWRucw)|zr%Bx-<)bK9@o zUcieH@wj!153{B0Qf}yj7mG)0FRg=!mj);3y&(=OhLiy$TzPk5rCUy=b{7E32mz4v zE(Y~G#n`YT#R0cqszNn2L3Q~R&Ypi=xJSt4+Cj{8w_`#&ozVr>T7hB+kL z6@Ue4X01N9Luf?=MhjS)^UewZ57$WF1Zxn4O-a<3#W-_g)ZXF1|?<=z({fqxg5|=5Sb>9L3to zX`DVpz&=00QVrR|F3%zd%oR$L+qRM0k_6G{N{y`wjB2P)4!6TFRk^8cc$F(cxj66Z z-==~cycyOnKP?uk;C(hF$Q{1!YGF{*lC@XXK-U;;Q9nvD&Jd;D^7L$u8b0_Ja^}N{ zaYAqP4OX%esA)(Smto@pmb?f4+LX=TsP>eRH7|DScI0rT3)BR9n!+Nay2)=#MNPsp zIF-JA*-%Dq5a$k!ta#+Z62BF9Z~*v2Rak5MVXBgYXk~1nklW;+#T4?V7ZFDh!6(cA z)2EtVMbr&c@hI!lpwDMXy*{_ynWLz%`IDxs{RY2vP1g@*7>m+|K8m$@e3ahmO8bEy zBwgDiMk*YKUai=Ff(!_uj!ykxs$SHb&Ur`q`Bw;yiMl|ntx|>Az$;EAmX8E#obYb0 zM&a&QEk!|{28{|=1 z4mCA%GV3H$=Gv<=OrQZkk6&!t*ZM_OZ{6qp=s$Tn)_p!II$XZ(eyY4B>6n({Sfgh5 zL}0Qyznl7dsC5q?F#VXE=EhHnNPPljT1LM^@T_J|g;f-NJZG-BuGn*ovav`(poRIp z+`!SPjsI%C$$V$5SXGg>q-{~|nTggh@zfW=ILSrH*oqZYJfr9&1s*b|VJz7=k?U9xlMqaMQ#vR`{9C zn^RgQ5vLQdk_8Fx%pTM?S&l1sJ+hVY68a6C3vd|mRZns^t2b9Cs1Qc-`_iPM=hwe% zR}uziLFKaHmky0fksf`ZsvF{%YLIp73&BAqeFAH@7H0>314g<|TI<0KMGHw6#r$@$ zV*~p2X7dD13O*^5l#3ib?kk=lPZ`G2?YvEtZw#Rs@J`PP!&dGRNl*7{jRNtD*0}kk zSm6T9+!yjs1r7(Z7K}Ht_RWinUK+vY<*-IcPGv{IhD(Vp8O`GlfO_R!fqL;mabK?t zTx$+MvTnb%1nc%6pitwCxm?qY&~%{l#Tb80PD-k;ugAn^EOT$qqfg_g3C-QT(HnS) zTp$7fIG3H(<-X_d->85JydAtkS~ya6BDJ8AQNYg-fV6vGP@Dz)I|aYd9Q>W@*&Cz3x`FX^@|yv&?UYU-iKUmiDT z@;-)qmm}Z9JV5TOV5(+tMLGrSqTGO;oIW*@xQMzrv{)QXaj>K!ISt5P0M^n8 zsTk;f>(*@(gAq?+TSrG=9{KA^)F0*rN}6|jl7-|?Rb0}UgEk_hb6{XU4Z@tEg&iVn zzyHlcmhcGZ$gb}ew_Vx|pkY$(XM(>TOdX1)gUOnCoXjVuhbhFl-r%de#(6|5U3zzf z;n1N>+X>qVJG8n3tqMOlt4S9`n3a_Bynqk+kTWd*+K&sMO!7?#GkSwwhqgICMfS)f z*-Zp})14zjBs^r_CCJ2|G}i`jmMhf;B%1HYiq@2)F~LC@k$7J!@G3D{*rK1mmLxZWnw;4|K{?#3YZz~z zy^4CoP<=6$##~c(WhrE7mwz5tKbR*=aj2J$a4Ym-C)~fzPY?9%!g) z=s#zCCP+t^!GenESOWC9L-yT4mYzwVk-KM6(O+EOXtKmRP3?flvpRo8*)6EKQ*BjPuAq`%;Zl7;fsXvxz2@wO3CP@5 zoUK}c0$@UU*5sHA^Ra)X?zlbW8p=l#MAT;v1pZ*)7>EkIqqX@Z{UGB< zo3m+h-Cvm!`rT98yO|N0BW5@{zh~*~Y&jBO+u{x<7ubm{JB3 zc3fV~m0%^Mesjrfm-t@Zjo4!B@lW=nVOsip6K~i6*(u$MGUQ(}Eel_g9un*N+32Ul z9_HztS^J?Kg6#1*Z#{!Z*%=D_+80iZd+E%PA%z~mRUwCHvTLDvzmy< z*FoPf#6YIB_?LVOW*i5thc~Qc*4|%VYHYmjOC{V(Nes-Wc+TTwBq&*hoVRuP(0BC| z18umBu%Zj*4!3W`?%IMUjqv124f^I?hTc)rbPjxq1DINTod`uiMyBt+ioEal>PqO2w-;Ixx$jV4tO_Dpsuo)Lpq=4Rg!w&3VTXT1`SKWV z9##q^qp`bL#$@~zReEpl3;-+B$Bj<*HsGNBu;+(O>0QRp4l*~Ew3wQH1=7Ey8-P{oqeRt?ZhAI_&q;|5V$56qUat7R$H8*@8Z|Y0w>u7sa5xBpJzD6WR{ibjjz*;>YY%m_!wohkzDR5e1p8z$DP;5gGxBawi z1kdguJUhqxex-#IC_i=4_|yD;AOBU@dKVvPR4s{uma)IqZRh7YJv2 zj@Tr{(+dV~Ke~Qov?%{MckLBoAgGK|8d@gTcik)%cdmBz5Nu#^@2lERd$f~FE~UGx zi`Ho3wffO)iOefJ91+T+(79C)}ZJo1?|9)~YWEX+$ZXw55 zc9DU3rHR__at@j5Czvj)Ni)}OjB3E_+TIG{)@qJcxh;&0m|n({@Gw1v23d{jc``w% z_}D-tKCGh=UL%geBQTjf;TWGSRYQanI77(~N!1Gp#PLKM*g@{5Rb}JxVAQ=@%P8Xu zBTVYS4ZzSE))!-9iE-s?Opu`=P`RukE!-ub6T6-gI^I4~#XJSy+NLPzz5`F{x${EF z3gxSVOWx35ISToqJ{67U>o>f75SxqX2NMsc5o2~yi|1Bnb0`;BPzYenfF=10X0qWv zegStou}VEk6-%%l(InSXHT4Oq+Xo`TBc~ysNFpPsvlWVMh!Rl7>mC{SK zS()=X+2hZ3&ruhivK-tyM`2qt7nUho=dZKA>iz)sYn5DvPC?qrMa8yDv!2$@i?yr4HjSW-G1YF0qf>E!gjh4aicTEf8_>B#c6|zAu{J< zRN}Qs3~vw;c|{G34DfiEruaN2d1cxKQ6pf{CwlnJNo`ZTI3bCvZC4%V2FckQ4>zzg z0))}1eFaK;<)&Ci`|~noDo2*htg^X{b&uYUlKr5L%ANdoq*q2) z)SQa@f3&=MucNM6GZ%AEtVH~Z+n@ti)##xsLEdDHlrx^6FHgw3a=InFJ=|DYU&c5b zXny&JL@}y8SNDEXXT~IIP!CoM^jtH$yH%W}Pnf~h@Z9HUb4SdKiQdglGmL4(>1XIG zr89%Qw6OOMA9KKYHNKakmfjVQkl0A zy3n@xNr4XGQ-+a&!E;qO$i_zi`hc4qF_8KG^${jrv)ie{JbJ~}*63>%kZ(Y0tt5KF z8BN!HM&0Ha6r~Hfo9k;&Xg|^gjyT9od&6T0(Sh{?vIgdkNw^TeI0Z-nW3hq=5tJ|k z>VdPuY59kJ{?m2+F`d=5uouuNwF-DPz#y0Z0O5r?+LZW!)p?6?;C5$_Qa7AI7)s&X zD-h1g7*1uv|IEkF@4DBYMx#NQ={{)0fmJU48g9IjB2dH2Ksm(HK=?g6ogG?A~I2jq0{dRU{*%tWu z1o4=!7dZ9kDyWFbICmQJZX=$S3g0HhL-09DeHV!O_F4>_)N>LsDklZ z{T*w7!MpzUzUB-CUrZVw??S_DX<xL#z|n4Q?)$+h64s*HZO#Svhi=_TE%wLj zfEWR63J8YK5tw=HnDPlkQCi^F$0-BQ?<=aCE5Nhd!W%&0-WCcx=?pvI0J|;@K%5*k zf8~2Kp41*u&}tAnGky&K=JFOW_oX07Z{e`2ol?L```E6Z-sruu{+%nM-2Zy`JB z0&2e0khw4%z&(k)@azyap~%dkN~PG{pIE?Kry~#;Y=%%>OWKOu)Dz-sca&RkU>z@6 zn9?TE6)P`@P)zse83b-~eHB`(MLgYB3!MEI0E|-UIw%RkwOj)NEFd5`h-Cfx_2%UL29+}iCH<2zpRk(@ zER*kpR^bk$c?)MB8BQI+V{zYR zfmt!seYLZWO_m+3R{NKcmuAMgHeb6Z*li+TIb z`Qy$F%bCE-Z1vpT|F_P?`W1cDO5#Kxgv@&e|6BZXs{8NGIf8a z4d5BCAg^0vDli_=BPmScdW*e3YTa-P%T7AP^1ZoTBI=Zv6Ea?CgCfEN4`w_y`o*=> zADL9MU=a`m2x1S6_|THFHb0zq>(YZnZjJMb?ag}owzCLkEq-U=t=BX7$V0Ph^hTP7 z;Dm=6^r?{u{EDJum2E-QARW~I30z^kR@z5S9$YxV3d|32hn)~g#`EZw8eiroSDJ5V zOOdh5=z5t=(HYn`R+U*YzCEd+y4tMqpFNFf!RsEIML`RLL3&U{WGX;YUiLg{w_16F zm%Lr-ekhSKPz_b{P4fd`?`zuu8-pDdbt}H}}njlC;W@7TTCPT&645-5?n{8!So7 z-gS*U^-yK4@2>-t?$LOz!e^d;x@Nu`bet0@kWs@+TX`T>Dcz5eb6>`B-u?zcGO%x& z!%Ye@?F|LooQW+>ee}m6fHvv_CTG=IC`}27?*X|0Dr#zSL8fKSefwPQ-MbE93LEC?8gy)I#-#TE zOHDC3Y4@eHwF_wA>dMR<_~0(Eo6E)jZ529YROcz|qT{gQgcw7};d_8W>OLaOy(RVb z61G3|8#$ed0a`Kg(Q!`T72lb=GxX47LeEP88C%E$;5NX!VG}_GhHe||htx<_mG%CF z!l@V3<^%SdF8z$ZoF=~h*U3ojT}7XipFF0vT)y1C)30}qC6gq+fJlv!=KO)%?z!l^ zYs3$H;xjf9+NdyDgog8C*IVM3s(u!h_I*63p~pafe-=~&9Z4&&8(%a666cPmc#N#Z z*4KsE8Tk(_uj7mp=rc9D1 zgZvIYB}zG5#EWl~b_B{{VGktqR!QE?)NAOnOQ*!Jfa^bykim;tigH?Zi}C-p+pQ-B zM*iCGf!%Kv)>Af<0sr+t0@QLf!sgO(<2?r5yracl@Qy%VEl!~m=I6;x)bMZ_xB8f= z6=!=SZ}tiDup3`WP_G_Aq8d~Ss1t{5FW?E$leG$`6|{EE>XzcN2f6=NTi;r6bKXQh zDTxk{0UoM~h4M#_nhcyXX01IqAAX1HGM6s?>wtvi*a*(p%5=c|20rk4hYLYOps7m@ zx(H6Svq@+7gV^up>*JXFdTN*9KpLoPH_Taa`t-csn$V5Q!I%m!`+7l+&R*v@b+9|L4GgO5@;h%7 zRX7^(#oKb}3i7_JiHBtra3G#2$yFEH1wXKkduJ0g(@|5}n8r0G^QQRaJs61zb*^}I z9xb~dXju$j;E*fu%pZNgx}Umuc@~y8p;klMhk4BYCTMDAEqRJP8?ovCHalmvOJ}0d zZln%GLXFsPL#@E$CPp{-IOm^|@^Zy2xR8g7PSz|QE$BI?$=(Dt(Vt$~fhVllE%Avq z1L$)edo?&hI_^nwYN?_WIduh(>0RzqGCV1?Fp7DcHTbAEFmUwWt}K9oaicF{W1jl) z@li_T;9m0jzJ2YyKe-tg%&bi5PtWzhBUWe8{*g0XfpKUye7+hHcKl@#XOQEWt)HRc zLr>cK!#r6YC>_Q(`K&y#JR_~G{Y7zJ3Sz04s=%k6i{s)qNJ2s&2?B#Z`;ku2^c!->Dkhy z)%qgmEy+Mt0uIwYA=ma!-rl&u7Q*`M<9~CrN`y7Cs}y z>ye*%@}c%}S4&T!W|W-oGsdiO#n$oP05nqUo~axNjGEEWRL{ZU z3P0#I@HCk&5$B8z3})wyp96J{q^Q-UV{5CSF>np`@Nv%gemYLG8NROS*AaAVwyW|V z_<%!ihWI_TdE;`xRpGRbHdg%pTFWb3baZ{5d-p+D*X59fS3bT#Sf359tbR{5`cv#B za908QqxcqO=qu3Ev}46q5oHOoBap&HGXYe|{l*Ctb>oM7SJtIzBb=S&ATeZlW1~Nm z6iC-TA#{TE7DvgEhI>d2F5NQ#kgKEJ+&lU0_5A=5+ueG6*(x^Sm8vCWGt_7}<(~7X_S|@;gC& zOeBgoy8~cAPPyG3_Z=>*@UVl3i#~<=8)+y>s)sJ$PjuO$L1>WET?z3|(A5I@TOs`xB%?umaV1JQg?kiqKMH0xs zgi;5iLpI1wZ4p%A&TJi{{nuJD?P&JCGX#5@_HA$VzDQ7;4Y8${PBFkpq_cz_Kw(`B zx_D@9IVXj+riB*->ts~%atE;^z)WGi$H?Dyw=OPN`3f^DkU{NorJpLv=1gT1VvdRt zw*ajdy2;n42z!%dEJQi4S8Jd52ky*Dcjzp3{BZ)7=a2sZoh-}Q`<@oS@f%$r>xFW+ zA7djHDZLAoXZ|IBvIU4NO+mfp-tH9q@?d{+FF()u`S$I590<2(5LNO|itHkbi{s$? zAdVS*1-$93XJElCFMo6C`zy4i-B7>4+zHXD!)cO5+ww#k8pdT-8`pbY67SikmBOMV z^vbMTxHt0N{H`@*d=#RkClX<2cecTAIz7HNW5gH6#abIg8p1%4-f2fO^3@5i9qVn=@xPMz6zS!!K}+mT{?&c=aaU2qghRS z_?gFeGPB@d4@)4yq|L**`n#8`5k`fODSaHCR!~+rKVsfBblngwbC*D#o)&&> zNz_;NZT)bn1OZ%@6U8HhHa!cRLzwDI~uXxqjm` z1UKZoU50!96W9TIxF-#?xMr>#yDUP>(etq6-b6iqZEGKdcE0pd#WHbm;!L96BZ8&| zLZGfrKASxmHJ$qu=sHk8-dwA9w@L4}?*9;g?cseN`TBbtN*(wCG&GUR$dxbY1Ki+P zv!0RSgexGX=3v3V{~^6=plT(Lk@@ese}Wp9T!pWL+&C&`@-(id)GhLp+Q|Y^ZhjN5 zg_b_RUVsvqw;~LWXK-127o8iZD)9N)2MBx;2`>}XKeajJu>0gm!k)eSZU0kZ|5zfv zzAz%PFdPHKg~lB_%~sjcrJGJL(nZ5A(X#%toNEM#emL0?4N`5ld2zAr9SyuS+7?r#5$eCi{Q;tlH&E(ZmJ{A3iNO&6K5Xf|o*yK(@ry^d~a^M0$LP z?;zpqV|FyJtWiC0oZ!5U-8p71e!Cn>`vsC*gPMpq*zE#do8^V1jdlHQYp+z`*|{L4 zXLeJ8!p)nfa>yg{Q*=uG1~?Z!t@hpDc-XISR7J$>IjD#~>-*My_;_rrnY5&i3a6e& z*VwK{YFI^|C#&?mmD%R-Om?tl(h`7Gy%2a3KRLbq&_lI=L^EkE6Hsy47NHT~>g#Hd zO2pCCo{O{Bi^D3)n{dAS+Qg4+>c{ivy4p@^xe}F9F_7*Yf9{C0z1m2%0d50Hink)L z@JC?!QOoZh7&fj@-*GQk{l0YpsLiFu3ZBldaI%ch=4RZ?hnHnEejhf)4LO$1Yf9}I z&iPO1VYJQ^9wQBc^JXhGZPS#;nG*T1^`&YjWJ7DpUSs@S7=?+Ks(k^fUf#%E%=SfS zc#QgAZ+Z2+;n(@~smT(7in9ZQp!1D%W!YQhgraKg?!c4^e-BpmCum(NKHe|Q2LiQj zINj_AA1K~)E)@JHOagX{PyQ5qXRI)09AxE&{ZKLkti`knUp3o7wCY-WKzJyiDWD(TCU-hZgW!>fJ!9}_y|bH0$)={hYHIqW4< z>$uuQ3ce_pP13GkbhsFttLdR4QxR&)zHuQ7Hyg?dpNPJ|&Q*q1B9}hsm9^Y;GlV|h za{@kI{C5kD2%9O%=oQN~G{hz0}EkRe8kp3PK%kP`DR<5r0LUuK*;@`0K!oiEk+k{u5S#!GGT2uD|=wE94W;l+Qz> zzb7pm3siH?Y5X7*YvFakRW83hxg0-rptCi4$2kk`hxYwdkJcpdxxppA@7bdX=`=l) zeS~rTKK_&Uw&fAGsIrwAupBHp{3-1Nak^J+ zZrBcg{xQ^uQr?QI3buQG8n$FCghRZd596fEJU@E@+#mts*=^|mr2w+8d zTyY@t?k(!0Ky0)~=C8xOkYn9&&z!Nm{GOhgYUZFmR-65Nmz+&XszJ5;t(VB)M(Ws% zw*MF~MtY2SviEP4;Grs8ER~cHEet?-EKxo4{4$6=Zw|0Idkw$eoVjAPS|HVtN@BlK z(hC^T_m9rX7k$+`3>9q5HtpOm5;+X%Vy#cVb_@0)o<8NS740Ug! zT&S4>772rH5^BmzwpuF-(*G+oYF32#NYP~0MP~upPU!i(+q6LZH&RRtf$GC};JMWP z;tRf_AfHVVqV#au;?#7#>wJtY@}GYr zr6PN%Jd1U~IfpIYnq-C>oD zz?txK|7bKD)cDV_hD0rWo_Z5BxwO<05m@iCs?2GM#di1I`SQHf*x;no(McOX0z2z) zBSxz1**T@wE|)C|DrZl-SeWv31h&a<&*eXIBx{h5JQrDtmIr}RwZ}66+{`!4-``3V z7Vg}5VA@o$T)B1xgpD{~tvq?R_V8V!M$zo(LwrJ@*d5`7m))p9-)eC0KTnbBL1)-I zkTwn6T(Ct%Kf8P8Tc)M`$HxUGgjS^e&!{F-E`+}%Cjx6SaQICSHJ4gHqcv!Q5nx+& zLQZn&Ns+5l_{_xeIvWN@zL4hYT*@i7o)Xu_1H{f6DOl_>3}(u1MUqlBR)Kl+=hGu6 z`mLMH@0f}(N__Id`ivN-4u-(tTOi}w`kxS|t!o0x{*=lE1}s*WEWAVjtKERes($bT zKa*f3`AQdjfvnG5?P*A4&-)`Y<$mg2d84;Gkfh@D!w5DtXxbMm3@l{Y*MLNdx_j66 zRe*Tk1lCjf?ayc?oB?V*u5YkkWIU)2+QO|{`R&gC9A}!)0|3Tq1mrLYV_n2$!zEi< z$>HlPa!q-Owvb)A3DR57E^^xOkqN`Q5CzI)?$p2HKfe zt3bxBxnBhLGx-8$M}~o3(E~kwkn2f{ut+u<$bQk18S}L(t2+@hH=0Oq3~`!Jc@4@4 zt;SnrzWf7TKG0v&`(ue#1FeDvATmYv+fXFoi~o#>_=L=QarOdfV{r8w?M3lK$KH!S zC(ogDP5SiQ#-~;pD@#QWD<=99>L^iXIcOUB$YcJM%3f|pMO^`w`!+Orwbsv4Pl4Ew zaM1M93A=_s6hWm8he9l zGDW{VXAC(wJVOcPUjfXDSF}t6IElb*GxD#ztyNZeJw`b!VuREv)NcLtTPk~cY2{ED z$<~Y)1qs&o^;I1%U)3ViI2Qm6^DP$zC0fNXA%GGD!pHRv7wDFS=)bl50~nXYL1>2A zogAv@!3-~Pdd)?H6Tiy{YI?PtdKufthtujU39ygY;!vq-*RzMQL#`2)aiK1@|G<88DbSepSwp&h>_SfmFWJ<)%V0=kX8#(hsqN>)|f z`8nXGDjHQCm`sOq|ExytYY?wUDBuIaD_=&9^m;>?ffM0FOoS|hG|H7P-<&$hzTqVY z0XrB1?BJZQI_Bw!;lSh5ub)jtLg`uh-}IRQ+#D$I-k_-oLg0`vW$$r*H3HZ##?&)h zMaj>SOQOrx&(1XhuxNlp*bb7#dTF1xjfgG;Sl-9w7R(PHPTVdWp9kI;)-%Ka&~J>{ z&a6Q}@i;%T>e7g41|t*T?iJwVD(|<|l}&t319T+IZhbXtuG;lJ`@E%1>z|3Eb+l@j zYIPaNm$o>a;&A~mbUrbVhYV9+4;A@|Ags|LhOlpS%4(C{BZpdtY0NdQDCNA;tk)fj zt`UZ2uLiwL;^U&Fat4ej!aC4^GhU1`%BjW1)1>kw@K&N?dQ%S3TH6_${9BwEOF3q|03kz^d|tv z2^kOZPBqx=+`lH`r=${$n7FVWo3!E@!vLDD zNf1Y45IOg}*AYv;@H2#IoY;%?k2-3kl9tYM z^rOT-ev*}*PW7ZO`uws!^4q^L>6cRR(g2wd-eXH!azzj3ipF%+@oT9fn{EH22>fi^uy^M}p1e{pbL0#NI)t@C&xPbUZc^QZe-&{=QZ^a*2;KxwwE_ zLz_`n{ws^UZI&PshLrIlX~0^xA%F5mPh~U2$2nO6R?x@XOeZ_ak@i&2t8M)@R=FUb zk8RoJorMKr<%w4KBbl(ug3iQtb8VL2QG-fS3pb1ur)ZQPK^y72% zIC8DQhTr&?i)IL2Pgj&fX}{1B2#$Z{_WOY2w$gK(Cnf5)9K9%!VBXHoVcC2Si~QCWe-#L|Eq0NK-DCQ@l=8vN7S|oL$F4i%%T3tR=+OYlj{4?v8<&XW>gkLwLVd$oh2n!*xAippTaco% zi&4%Z3Jx;IOl31WcIHubmm?{mLO>T8`N73z0!>I=1D(bdCBnxDzJuLeva+r2xQnW( zcjx{(?(0CXr|t>MdJ^%kO$U2W=QeY*(ObKfrvoi4)dt(MKJc{#VB5Shztl_eWp0cx zw=xao(82#1?qnKIkWY$o( zon`IzliT={54gXl+NQ6fg1mHd+OLA(A`l*LP;W+f5U499g5IJPDh=Rj?W6S((mWRz zIc@L^&2_GVCnap{vs$x8h#=!nVcW$pIyvdp&o?%J|Jun( zMhFRJprt%}4_f$*A z4p#Ju+>S;RNT{$i=`=SjC&mH!re?_8O#6UB7Uu*q1P7`VK!`%;8eH5|grT%nA1d@E z)r@oTQ{rRILuX~!^4w6nwg_pdwKvVj**64NSuptI2F_YiCro|L6`w>T${dFzD5q=f zcSir6YGQi;wmUUxAT{-Mzfr8rory`)^GnXTNE9ktPRK4I8F}u~q`CjXLhoqQ-h;m+ zN41?xL{}eY4M_V}&l)nf^DV8%JTL|$jW$H;r8f|H#RD?(pU3SvR}keMK}{Ne;@kUK z*4hqQ;E=8nX8*89OW32?Z*T=E32(o`VliirJA(7NM*c-Wt;^Qd1uZuN040442%s0k{iyBd3v zy{RLa06+IN9GD*bYD>>#g_4X=Cv7WV-^bP*&RncN^vY)w`IPbeNgxf*cl4RvS4fn! zVN!2%j3_8YdeOsb>ZX>nIW86?=gYECA_mru$Vo{FWqc4R06{9iJ`1#zz&iiRuNiQ! z2@si4+nE3|iG8Jl60hU* zB6z(?uCxLO@IYC<4^D^3-bqZD7!@AKopI%F@1TCs5II|P<_VI~$d6SOPA&O6 zG%r+LMcJlC+ce@K;mKn}iob7By(oLv%%?%nSrBu{Q=+vkpGMp7li_oU7|_-(-}$b< zIS{vb?9n)q!nuB^2g8;Z?MJP0579pK?Mk*tYM5`J>1?H`3jU)FfJKAcz-JT5B!~`j zM$ASjYjVHh+*pSVW=-5e@R_{s8NNZEkKI5b)kQGY8{5Nl=2EVRN2Eq5XBoL=NO za=pra>$q~g9jNkhS0;c>Mu+6k!kp4l7$B%1?UBmXe<44CsK&8IM`IQE-Oufi52Q&h{yui+w`|&s zv18?{E3Qa)wg#4atYc7MZVM>7V%TENocc27tK+fR62#sF4oeZOOpwUXT6b&jqvJNB zMfYSGvwpHwmfLi|cklmDyU_I*mfBU;Lf*Xm%PD4Gr-QEe zjW{UUIk?3PB#*f6A=B#mR*c4s(82)BnWo1+oe8|vefr)KXFN7$p^5? z^IhMl23)oUGpma@GM#PyO)El=r1`J`Su>B*pk%@`D+DVd`f)bb-u>}xo}soUg1Z3N z05ZTl%qoKar*`!WBlD_b_HwrZh?+Y&`->tjob=Q;=#C_Tv`hIXmrMnv`2RvLgT!5J zIq9Ux!eb777>U_p*OZK{_{(NYTWAeJA}>QopV4G8j zfcS_RlwFfVQF?60WlOA%sNY~OTymPg8_BHSNdF_P$jJUo_- zXPfsp+`n|~vX+JuL+m+mosH58i3b^XbTYgd%?FK_=Po|}nG#YhtceV+dT-Izaem8R zq;JTY7zk+CK(n!Eg^7gw$U3is$%k9Mqs2@}b(#MAkq zaGGziA3MiUjD|0N9Q$A=e2OiG4OFSJfCA9W+;AThPz=&8<+HUlCt631Kg_0% z*V_f|8Rk?FfM*W=H}n9EM52on-F7fRLdu;Sctq$JX`;aD@xDCqn9)AgsdL*4%b__zu4mkS@2j-N!A$?2j+{`R_grCfXh+- zE5(v{N(+j-kF@lwiH$DiF!i6QU3!OSeUbEs+mWlcSD-Z*AL$i!_?X@BYk#c%w!GX|H2yR1As9%AzGNJ&w6#UX zEK+{zK9+Adt6E^VC}izlGgR1GdzrxNi|L=TUh4&8K^h^S(SQEnoC=g<;^B$#!vMOf z5DzI(>#9}X^^GNh9bE@GndF^b1|I)N=4@W4{@oDDsq{-Q8CCawilYNBjp=E$Hvl0@ zicYT>;e!RX{e-zfgWE^%p}q-P3HR(YI~aFRbOv824N!+R={CKEf%XwW-$!oX-8?o= zBK~Ev+Vclcj>|PE2|)M{=dLv;=4AX}=|i0D?~k0OlB>Q5@hyH;geiubBeWeocc8WF zj!5nUxdN0Nt=dFDnjw%9__Xy^a~%*w=X*JjMzFcuZXW3S%hFPEtp9Rf$8BfeJ~r?` zNa{QX34rYQvIV>pen}>gtN(!8Ns2xXXQ#oAApXq=I6X5HcrH}sUhmUlintr1mUSQi zGuSF3`3XvCwv=VTJz3UOYmx>MM4aOWDS&T3qIFY2as;sWHzIww?wTKm9WrUC?*$$p z&(7=;P#EsaX98~$h#8mqA1YAg&b%47G_!sC=Hfs)nqk_Ll3(X*K}3r$Rz(%tl3D4k zrx%bY+1;HpS|cXL9uU7vJqWv1_R;_jB=nSs0tnCnMV5qsPGS)ylQbyus1E{Ay-lRM zo=SB#q_JQ7?1Ey>_wv~#^aHb6tgX;L%aNr)LP7z?qYbOaX~nww-NPvJCyd9t3?S(x zWS=@jCEbm5Z!)`PcXa~9&Ou6?%!EL%PI#u8k$nWPuPw_3U*#O>F(0pVy@ow9Qjqe} zgU;^l)U(X6Fd3Qgbx?$$3s!c{_lAY}Y~g1&aQmHtS6P|=t3`+OqkhF`?4qg^_MJ|; zR&!sDcW^o4 zZ}~V*gf#fmZ8>jy7k%gbHDE4dW9RF*3PrW?k^Z^nvo^NYZm3B1uRgh&GtZDtE_u?T zf>?l~zT7`IcfK-!adRF(UlGSZj@T+J*zmym@>NX{7f4C2a^D=rjxNyKIa z8R!a$$!LJmm|0stH&iLV1XzV}9))l2%1q?sKPn&{SE7vyd#t0~KOTS(eO*lE@s}W1 z75Oc>E9)&;N2iFDhzm3e*%x~s+nxN%2;{u$&RHW>t_Adeo56pze2H5zvd{D?JMbo@ zRD;qGgk2mPA5HnCz;g^NBp&R}J*xTa|HmCP_3#8av=SF&e?^|tHe+5yeE{CqWH>8} zrJH0v85lrx(AGuibFP;^|JNlXzAj%^+>=4!7x)xi{QP|Z`?_$vCza5&jfwwBYq!`< zX=DE!Qdze-|FGqmX)TBf5s0`6!dwX2K-HNW)pOVc7)inUfGbas%kFuC?A0lpm?c!` zcNu6a23t>ngmzni2J#2cKsJ(UK{+hIhx;Y!xbpVjv0}OV(wGpnPYVOS#(tuJ}-34H1t$4A~_h1Gx#YxX)2!d+(wGo2l@1$1m% z#V>R^hvF46KgN$dXDxKmleAOE()?}9mi1OvTt-qBgMkM>7`gJ4QU2q|QTDJ$^<=}T zkxM!J9W+lSgSXL)FV?mZ(eqT-W8RUkV`tzK8qj?EVk9c^A3(1(9ezs%M_U49UhRc7 z{4T2kw9YLN5pDiD$nPgB>*wCPart!;WCiV>8}H}+ldaTnV4}NoUJMj(%NA`TgR~V| ze0(nMDE2sf(xjrgoTRmdwf=SD1gWr{IMQoMJL+iZqx()7Q^tSok^3|_HTg%)_&Wes z&_=g#n&In{gpE6f^KusYJZch3Lp6{)pz&9VPmD8I*2bX&@VZ|;dHOcUh&r3^8+N?s z!BOD*RWmYjojuv$X{QVP9K0D3rO3gJ%Q#p2YW3#*7uch8MRyc!%iej355NRj3WTNe zM8m)!+smf5^L5b`M!~1r7B4d=SuGplR83EeFF@-`h0fYP)~scaG6M)~PA9 z#8jqhXa5->`~|EkqT4 zN4kp7)Q<6ix&rQBIIo_7dJOQr^MXvPoTk+?V>NvvAUlL^HqDo)c8Wm1QhhtwPyW2e{e30uH&p>XRv^3dOeftmJ+dnFgqAwEVeq z1!aDWG8@L?qwY^2`~1KmxcSgBERDq^pW&vx&nw4_~znx-jsQJEtj1mci*CV zVCzgT4_iNHWp4v3n6~x(Qu_)k&j3iRh4>p}{qRZ5)3bYGtqzpe)S4-k;o~MPeg)*4S0zD+bqnu;B$dRe zX_t1^J_RO1B@ZVK03+%Z(-~^sCy-hOhEXk^HSzDo(FHE3|I!&b%nB5)*Z^a~o-pd% z&2z6ii+24y-XZ&F3$Jf0S6;_WsSJP(+yY^eaUjreor}}mFW_e@4*M9Bm}_=|!Q^MF zvMr~eAPmAVvu6j$b)@wBb$FUdtyOiPwc_xMsD|tjQ;@q`X?+J||B94g-;|K#Q_luA z_#k-`1nuI5PWFS4$fWk@^h9 z{11-HDU7w==7X+uANrirCOa8=j~FbS$z1|QBaj8ri&*eKa;=3HO7ozYAOnTTiQg^F zrPf*k=3R1BOZIUHTEh8CRM9@oMoOf64WL zB)RuKv2Z%Di{RvU=F)jM&ffI@W9&V{no8dPVU!itRbd-ETG`dw7hdeV zkZ`!r^l+W+FiteNG|SCTOj@pgXi?6>cA-)!YFxVaPJ(gzt(OkkT5dTCk59kjNd5QR z`Ca|-Yp6y~B7AanO7VtJg_Z7&=@nY{%)fO_2c(Y}Hn^p%o6gbV9f3-l$$-s?Bl&+8 zC1uDDCt>W_Yex*vF?-e~-C97%CQVC6M96;p!p@t@=@EIk{Oy9=CNepj-#)u|SS>j+ zG}cRahC777I+-+^H+&ZE;iS?Bz#c>-;FZ^r20##QE1jPob<#dm_H_q7h{7`##aNHQ zjfP6UUWQ+Y&01ZJX7QKW;k4roXOvywgv+TEf7VH*e~2TA^1Ha zcDf#8DW!E5CZn`D)l6wUT6T6D8WC9M=6&~?a7VC>lnQG<*w+&G14Wz6r>3eyJDLqkpKvrBPOP;ST9ndb6RqFM1_Go%7*~gr+1%Nxu1p6gw zOIk*&!G;g>ES;K{ev3|eH$!SX0qN&SnOGrA)rIp8o{ig$7fh)YD+Kz4%C4}NCXS&K zm|2`D6&3*s*x>&*dXT2lJ*GniZ<#N0gK72Ki+MRpusZ-Pt98Mwd4RVXH{<7Mh=o78 zXX0eJ2_u8ZC!|xlknFCg4B`I`p>esL1`qv%HaTU1{oWv zb9=vXZ`}%S_ZPort0-n9+|YUWCIl79Qq?My6jnB(J?NwQ%w?*^j_7YkWupcn>ZIrw zt41dP*E}vm-y(F2Yc2ychF)a&O6J)ypYwsFknn9Bw&#eB@;!#cSs73j1rleYOo=mK zWbxri^FUmSvk{Hn_Cile1` z0DnyXaX)^v?L}V#C!t3{u5ZO>yT`|C>My1_2cu39vvLo=OqNseIHfSoRd@Op5+&>U zf=$NPBc0MPdypOh4S7Oq=4T;VM|-B6Na_B}uXY^9X556zrcZCmx1HkXK5W!C!dd$>9L1P6SVn=;!FRZ5z^cxFb;JV6?LCk8>r;CGxEv>88+ zEE?V@`T$=P-rmW^qE(jX0@R#8;75u0g+Y`o-`Ef~l(9^qDW@S2 z4%?ZBK+r0fVL}NV+BWEbuFAR2bGLhgPP~L3vVjM%SkjzB3x5Ok z27YSxcZGop%I{otgO!|~1WoJHlcBEk?w0ut$~%pv3E$S3-%LDSV+BVYey_EuI;i>* zss9!ERX4;`@T@Z{(!;!jXJ)$rGD)%ts98Hsu?mD!q<}J4a`H(*d_nZ3W7z4Il>z3S z*dU?%@&JdheiWyj8ws&f>t6B5airete8@qg3@={XXw=UUPpW|dn)rQ(Gqy8V^I(PT+-KaJGRsgqg1#uR(7RrZa!F$GP;MQ$(*)`!gPEdh zedE>6#? zV`;)cPu}BO?xh%XDDwrddC?XN>`+KZ5g_b9wF^G@+%~sX1Z#h^s6OE9?z54@VMQYkNVoq*&Ipf-LGAennPZwYXL?lH&6csVV)%JH-hzJEMz!u19fU(tJ=Ql!8h!yps^sI*-kazev7+- z1`qPv`wINBH;83g@8@Y5?$+*_>2(a4BbKA}paF+Im1&=(OpOW(4E29!<=CayIFy%o zS&Dc+L{qK=_j}>Wk+-;(B|%5VcR>QzuQhUQ+ zvOb^70b``n9Qt}gXyZpvdd5?M|_1Q?BUVfwO1D^yCBPt2^DvV(44U zYnKeR3FrKc#aIOEoXHH(6FitX)ocvoV*Xj?U1Q_hdjXEFFU1v2;}lHe>_6O5ryMLY zWX9_(l`$b~xfmekwdYdnu4Bje;ggSiJqo-^#Bb7HZ%+>TqO}`nmKNuUz&(A|i#iW* zA1`8sUauCljyKyr>B#iA?aAWcbE_4Ye5}HJ zRs~U$H{5+yO`VeiQo-RcuWv9OrkF(car#`~0TMtn)48aNqAvP#yF&(qdF9g9r0WuN zws8g3Sx3yGA}oPl{fbJ^zKZPa1iR;(kG{xygNbetu#8ks>|CshN+E8{rTYK*w{C*YH}EnC{{z)|-9(s9tz=_4H7rj_-Nlz<@}bCaTO zUFFzTsT*;`C*Uxi`^@MWPNTkfWI1lU2@+x&^lPmLpijak!W`xem z19T=5W#{F(Z}y6r_^5l?IRT z%X)s3A=PUR%=od#^iv)EUoRxtEAcj(d1>F1yG^>Rb*Uw}}XutBl*31KPEg>CYUdKkon7F@H1&)M*UK&+_8K}UCD_I|MKLWJm}iGb;+>OV2T4YNNc zD2z0+?^$KNO5g&Ri{Xo?Ejcmc+WUml>6f^i6`UU;EtaWY=M!Vn%ZY?~jQgn~ApbN_ zM)?Vr!Mh9f-%;{BkiWeo;BN}`V3d+E^IwCv)5!~wl^e){(zlmD74g2qTzmn~PR7}G z5yG08W7fbxY=ngi=Kr_>nXXVX+y7svm}A)iNwW^Ao8Q0fPCEG${MVs*PFcSw6L1ek z0QUe>#NZzAK1}^3$Z71*w}9WZR5#;I(Ld#4UwOy^^QJTm(+i01PtPTnW6bE}V$FfZ)?IP#^8B*?-~4$9!*$+f^lhU= z5ISiLRX~}`{%Sc<*+B#SqGvh%&A~gz&b8W!X z(M{U7cw_;&MKo@24c<;q4_zsnxhd;GPHc72jW}3i0Gt;<*%hZ}I~YoV6;*Zs(&8D{ z7p_;IGteUn#?YcW6b<}eQi4qeDKWE{0rz;rw`i%TM&>MHpWWAM>xq`5^isAY)FZtCDzT%Vn@)s;iD|==1Hqk7-4+}n+s ztCA}s#P#?tyJ|J>`E9F?nF{P!SsP6Mof11~s*%>R^m|K9a z3@L+(zHM4&*WNDNn929bure~sp{x%SKtc8&SU_@(7K(7STN~YxWeoWP3*bn90&u@n z%RW#E(>HC$uj}9~$YdAaVxK!cBShx~*k~8mR#G%|@psKh%MTwuK#p-_;6h;vyq?NY zs~ygEE-s~|w4qXq+K1o2ng(=EPiI2j+_(>X8znKoAgHb@BTOzI^2)*-(757)NnvL( z9@x|Nikk67^<3?b+R^3z0$~tbb-;6;(oErFqHN8BAQ8LKTgT}ZkU4j8WXMgW?tJ0X z!j#KB8`6wLvRenp51CJa=}%~XD<^sdp{hE=!I$z=y@=R%!edVVl0iYZ^mRlKHLi}a zZVWBX&#@}3NfAj&-}T+^IxW{de z{f2ET;mrXdW(JI)V*qoi6S#oxQMG94GPS8c9h&gpwaB4eeSiJp!h$p#77zmlDAL{G zeHp$$mThYzc8~$H!hBgL(!h*cD(A(_DmNkW|E9n#$^|fan0?6(@GtbqRB-)Y>ggX% zlBhKFk5_1$HfuPritFLr{KkpUIag?iw(jChy6kcd=myrh&b*B9c@UThi9PlA_C`Q{ z;;l#fVx)JZM*lNMnzPU+cRY(TCfJ8)g1sD@8xIl*t_++_W`hBTl^eAt&7##{6t~0} z_mbVv+aYnN+4ed*M=8FT!zbu2u+Xtt!T79^X@5NHKx|d!y&$n$hucyvRNRkxXaTFS z^TW)QHGwqM|8g33T8O>rnx6y8{pZWJYM^!dX?`q2$J;awHB4m@&s4r=uK_CcU;m|2 z9|I1u{n0J>kf5$JC0Bh>j28`Y5nkEo?%^T6XaWf*zDW7;LOf*LqB#_MAmztXAaq4q zbPg~+B4pCt`05B5(&FqW<=gzxOeSv9(9T8|SV|0MGj5^btLu)taxnl14J4BadKI0` zgRdo^tQ=Dh@FYZ&}RZLJ4wkhG$)fyo|t%UssPM0A6@ebxzsy0K9J)?wf4 z!uZyL81%|Cx<<_l7vIKX&>mO$>MjpNF$e@^a{Tte!I=%WAhGxUV{Fpn+TEM$pZ}H} zo2*Z1V>}r!iU#kFJ%pfmrrI@eNdYOu{S?lJIgVcQ;yQNB_IwJM zH)gA?`eI^Ybw(bbgd51q20EUG9a@iIy3D== zCq96%%Nh+1e}56qz}PfYLUy!8&XxPI%lvLWc78D3IpY;6;Er<#vYI{DX?r5JM^<*1 ztlUtc62()nZDDJny&=!XGBVTpJ`*Lfl0L>pZpFVtRb*mFckY)167fB-pVqc6ayjNlbO@hY}mC|VKT zoI&^J$kS|rp=D$Qp!RsvW}P>(=}v`edAnIgu-KYoaQF56dql#I;{YP|JgwG}B)=e& zT1Dn;{WEi`Cj{v|VdwELfT3m{`2(AR-(7Q`k=H9R1%mK_c<#e&ML{sK6x3>Fx~}%W9urtWh{Y^f4jn-8$C@-H&_-HN#Kys zpf`ua3=iL@1rbdLoHrH^$;y6oeQdr}^9~0X4gN{Tgs==r|0wR~v9YyP24;QJ@g9&euruDkBA9Ma zX-LV_falv*>`*`i>#0yVkC^qoYHZ9-ByM!_(#obNHUUDDQOjEg6a>_zhhI#EENVbF z6X@d<2$KseZ1*m9R=a=;pb-wyR%-r2mfx{!o7;C7B)qksP(SlGlUJ_8&plnY+dBA5 z9099r-SEANOktRh9x&LclhZdZgvq<333&PQ7Pk-N(^sxk?>9Q5kAOdm)J@HP136&W z0^E2Xj5b4FEhc5_?V?>`2r(8LY2f^YyolI*@a2@L&;|QiqX9rH1D>vJF6ZZH&%0J* zHm)K!zwP$;?YA68n^VQ1{(n`lPm=;W7oAxnWovaBej&P->;awKpwDMEDA39;64EE3 zulO2ney_PJD~kMShExL|wsTwcVDCGL8&*PNo%cl?~#JCunnzwL(cFOu)QQr+e{3k~JmX4{{K znxQwT%R3tMM2_f^d%h#6gX<7IZmgD}s?{G0scKbSF|#Syc=zZ76*aLj$jtinLvO?3 zyr5iQzyq>cgX6MiVzU}=f~r94+-^u92Yebhz*v?XGBrAf5AdFS+oUpJzq9X^OL|Ry zkAjfS-}U76^~LJqs#LGzI1CNB>zGBn>(=2pH4f|sLyo|Ds!JW#Em&p&j$DOZs$mf~ zPHAf@z0sGbCfEBp7il|R@l)i9GhwcEgzwy6WV-hV*OyqWehTO3miDJ@#0r^e)tKnX zrJ(U3S~`|U@v-w%2@n_0uF1^@T3pIN@J@LNc1iK=mKaxe$CG}_Q$yN%+LW}>yiHdu zqq`f56v2{hbTYIrm9*+Dp8YP#*kidNyKQ+HD*>Hlq0a+Wun9Vd14G`)^-)2tvwFW+ zrD6+cz$z70Pq=(*{FV<{A$CHfKWA!uy`8aMHVi7t^EEgDqoUOsO<75A<;L9fncT#p z;>}`-OU{-jo$W@Qc{BY}5gK8flqwW1SSLxX*T~$Qo$rHKTOkx zl59TjZ~LOQGCZELK*6;8pMiWpi+Y~z3lsoaC5XT`o_byX$l=~QAmsynK1)M0S_P_p z$2PfppXIpY1$(LA)zv2WUCRcrkATTeT0<+$+J8fFrv+&U<%ERiYOqVcRb!^573{ma z*h{$EqYjf7zc0rN3IkR3dx>nhGQjMDYb~p4U;WSHx5u)lI+>>*!LH#n4`Xy zVkb8)#NAxLcEy8NDo40o9jjUvsS>r%_ei9Ha-lb*7p9W(Z=4*zD5Mv*|JVhAQfZ7V z&$7oj+@}lmr1hcWTFM0*dH-Y!$JpA=}Dc@s0rQokK#EcJe7O ze>*|iI9s`jVDYlue*9SbAyb2+`WKv(fJJoixNt6eZgCe>DmyB;>b`06+BUK8C|1`u z46j4namwYqKlg#E_g$_FY$kI<*y^TI?gvR0lGPI)sBvEm|f_!lQvuv z&S*T@LuDZ=*p?0Yo*j>8%VSqO*Z9H}xFKgnlR}GReZ)T5-t}Aw#qIpz%@}|k7%5NI zBDV!LMDel}`7KLo^!$u7d|l?CXZ*OU8qw6?b+n}($BrK-ZaMQFEqg1cJpDxBJJ5@* zTJ1uN+f^=)1+QDP=sAwT~%9iV5v4GEmffkS18;Ms+`=_!gTM! zNxhF_GHy()DZWv*@JFp!cdXZE&YaOJO3roV;d#_E1^1Rk3FL$-xT-G4TTSm&d*5Y4 z9|GmATjUI@V8p;#YqU{sajvgT!!Zjz8mMR6=s`eYZrr~*7u&2$wzud+DC_J|F>3+# zoR^yV9}mJU>cka+{9US;v%p0?>sn4*jl|&!6N!f;XM3%Zu%j z?~%DvV*)En?63kuVIovzRikrmTvv_dfk0~~c2CN?7$rBOU!kw< zYi9+oLm&l@BG4GoREyyX`K`okpeH2CTzL0u1*au>5GWP^dw}|iOPz%aXxvuJS^}LQ zjjs-P!`(YNoCG{r8IPHhsvpl&vvT3$91qVPv%u|GFM0C%_#SHYMXsYav zfL)Ar#C#dr+*U4jws1#gH$Mzev#IT}l*rqnNIQ5ytNgB%cr9BanB+sPd_7n9@^dZ% zNVC_oi#DcTL&N)dX32ND{+c<@&~FJ`e6g86{;0OLcJO?0z`fZ;gUuW5WU!WPPN%FN zKvIyzar$zG@ts&sZgO>YE@oq6i>JRw)!#{S$~YoX|L9T}zh{gLvU6_gR?3^>18$AG zTDSmW^b3lo39;XMm!fM$H|a*icfmAV#KEKdE%O_lPPEC;WG%w!x!)unr_2p;RChAo zWoT_Hp-MaJ3GZPsdI8%QY(`(dn=^a}Q|@-DCh>OGKf4|sY>qe?c`~AcE_d)c{2;G% z`CHWvo8}TT+C2FTusf(#s zYwp*3yNqPyoli*>CZ$M5NQ(?_%UFX#@tCY)Mk$5QkS_S`6Kn(sl5Q~CB$@a`I4Drc7;Sxs4sd2)eM)=98V z+E!c9aYx?X1uOEi= zCjc|CQ{2*nM>9e%7SH6K7v#5a08sT)$67uCJZk4{b!R#1mKItd&!f%S@ka!ZO%rd6 z6*imb{*I@}doH|LdddzO4cIVut*)h{B;Q#W=Sj){Sj#(GSW7MG)eCv%tURqpNZRaS zxOaq!?18Nl!x&`g*u9f%W}dfs#l!%p9s(6}4*=KA4H~<&6|s|{h}{r%MSHOEUconM z*#)5gVD{6oHxLHQq^Z$CVe_SM*+oHs6NWxRH^AcNASs%46YjfSKik^o#{qll)k^_a1rGM*TE& zC`7l-3tsdy?E--8*0nw)R=6+umPX1Y{0B`&0|M3P!TE`odJ}WYT987eR@U47*-+Ny zHvHO^GN~<~h<5yP>c{a{~?f3;oee>1_yiKy`fMS zeJ>|0M5txF=?!wL(j8s1$|u*W!32hS@<7i5jqXbX-!cp&d4N2w9VPtXWnl7$j1b)#;t zmZhz)yEK1JV|P@l8jq-tAIT|wS{odmkA^L24u=3Eq7VGAEg+Z_k9sNJl}ZfVxMKA!QFpM!t<{`>i1f zRMZn1PtgWRB`7h;ycWBgeFOP1QwE)Db#j|8uH|g3e)lDJBH0ge`Rg$bqvpO)WTsXr zcVpZ)PbXq_HZv}cRH37S!7!l+feFCZA(aO>MbWT=Q%0c&I&QqkMqu%-iDQFfG>v4g zeuZTJtq$=NhNezLc-Ara*Q{b3cZRSXN~>Q>N9)v!>4eu6{K`tc6s)NL??9d``~8Ov z=b#=lk4?3&x(+-SaIwHO15@C;xi(oG9v;rkyX?HVh#rX4kQ+82EG`ZdueTPL7bvpG zjM`)sY!cgZHy-$3(^f9f`nqidtalcGalPCL^&F%Lni5F^Ot1|TR7*5tF0m{^-knp< z_)<6eD|gp8+P_38Y+=#KVhbSIFnArD&Gqi6{{y#JIRsx>@&YV-t^YLy_VU{(a=7X& zy>t#L)z5rzIKc3gGRNpY1t9dM;Uib@RLYh7x4><8(+u0vx zYZYeN0RfxO0ckKQ@-!jAtpE0+Fp-$*dd%e<)yA{PabC< z*!GcAJj7~3Wy)GjN?(xt`WJ5A(aFhT`L#0nzPbsrP;}dU!^6o@`CYd~HGD&yigy-W>m z3G_0)M$ZtJB!Q~EUJ7?yMf8KN9Wy)M zYcUoZ#^R?Pa-j=1YB49SZ=%yO@YZvBMvuJSR|Ok1Fgym3UzaMEV+>^?+(BBX_}G=a zB8Rk_lk}elEFvICJt4hIZVDNCx{7LV&=} z45MWBLt#|JnTXaIu`{Md+^J6b<6OnXUh2{TC^&g*EgA(gEOi*nn?b%LzF43V0#JvF zSYh9R0N0Ndi=Dmz$#8k&g$1j0Z?v=90HX)!DKUk*^Gf&O9zPOMt}{{>^xIy-{}IkI zlnzNohevD5IyQR?2n!Rl@?SaoZ1}wypzzh2IbU(#s{gax_?KGrZLkWZYu%MRAWZ8O z&dbP{dcKBSsVE+3mRk{_wWO@SaIG=n7W}QmRM#YZQ$i$tvi{rS&Lgp0__m4%`*WhN z5)?pX^AG)9d37FZ#gcD2@UdX&@#Q?Osq|;%yfLGtEOeU|v3LDZI(eBy{Yl^7AB)|P z{atTRO5hm3X>I=m*DsTy`@DWE*no~SbEVA-@zfP}jH^CUHsf{!3hpx}$2Z%P_aRq5 zCrdC%vJQM8wkp^9n%Jpk)?+Q?XUzYqFN9pG;rpW^#-E>o0UpP!{5v+nF5Q2Of4omV z(<|xWji5TpQXnIyv%8f!e)5Pg%mF{QIEgmnqjm?8jL~FcDo!7r4)i*bTXIc^xyZv7bl)nU{BUWp@6vd6uj2toTECD(Qh_? zfUux4p~kLYQc5yi#{rTF^$b6;dNF7-vp$hU-L*1izKI+ID>=UQnc-r|fU&>Wij zpVgxdVSE@#GSNa6tT zIU{x($@?bxqp*kh>86`4M1t3IS&N*ULgj7_VRVN7D$|gf8Pp9)MSv-$J^#VyT z)53Y-rXUYb@4MHV4FL+G9cMj~^nG=*nR4$bmml8H*t@UgXkP?zd!#(8=Tgje zD@!Wku;H_FeKQwtSFKXj{zxe5vdjcgFlj4(1|mk(gdQoli#s8n41AEMuc^iG8}i=| z*8B>OPj|{pJ)FvWG$X*r>K4!6Xf`G?TtN_cJIab~yaYy%67Gp2irxvDEjQm*q`DVJ zC!_Od8j3y-h?-E0vx_|JNpI5w)!SV=Sc?9Fa_6__m-`O9MAl`v9ek|d8*yh@NS(d< zzNH@7Y3+T%Mh?FK$;LS#GM{-jmb%{IGW%Gd?d}0MfOpK~O`Hlr)N_k6*7p+|T{lRv ziY52Fij*9K7j`&r4eo80p1GHbN%|41=j8SFLsE0Z z`QvFEImd7*5NYA2`dI%TE_j>NLfzSea7+% z{K~3^RSGzhxHV7J+Q!n5oQyVUE`fy_j9G7xi5?8pV_ z71YLuTJ&bgPHr(W14TOI2zoZqRA@!rEIzK^I;ZK7HzuO%;TcRC8X!8Un4l=;L$i#X zC1$zJN%)V&iFOn-PP~9S6z}?#l#;^N<0p=qa;vYD%<($D;CN4$`74Ep{ju}U;_k}I zyWQ*@pZIV`5~rY_0uz0xWopQfO=PiQ1FxI;5nmT?O+G%p!5bB05zE9q*J!}AIn>Gx zw{MjZrbAa0>lu0@CR$c%Q4mc)qs7V0yLci$`qDOG$NL6mT6t2YZYbq?5s9xwvR_0q zJFTS7n0{D)Q^K{XgOfW@FO;PT%BAs(p|91Pq zY@%3)Rz)VXArAFAsCGKvcQ3b4P};-FwE%8!P5)9!0(9=H`Qb zTcG><&f&V_mHv)iDgF)*Zt+Aw1sR5yJoMuRyH4dU`4ki=WyfZzoHBJTjA~N`$gvGn zGb`s$PC!G^w?=5au~h#r8({H*HM#E6BY&I%>{&x)-aTfcJtc@pD6FZ$ww-GZr_0N? zUt!40#~RA8ORu?Kp4a^Qlf)K-tArPTwM$el6Uo*Y5b`1O|CFk^lPkPCK>kf zA;o}P0nS2f^$jy_@2dbG0mLSZFerV%9gTY_ z{l5O~yQYjGD8Fg-EpuhIR$NwgJTsp@)D2_^M^g7OvMdrH__QFUd1~fu0r6ij+@qXY zU0et+f^4cJnW$^yfZxJ_fyYr8uWk0fQbkz`33sDQ zg7~jp#SLecgVo2~S@Lf|=Jmx| zu4UyOu`bF#3dk3LOjXykQXUQHD$&uJB0_=|GR|=oqWuHK`b7D>&fBPG1B@dmnZA5| zZY$F^`j=ly*&jG}{vVmcW3bKQ7s$9o=uqvGipViFW4_2NC&cBF*)r-z{#=6u<>{dA zVPEWz@0GC(bQ4IO)>sLa`mc?ZWVgCloR)`)B2u2`V=&fsWozIa5{#7lokV^j>(iWu zQd?sNR4fZOggf>jB0es`mJmDpLxO{QC*Z$`gB1CqIvrrpf$)hbZEv@053)_>=fDi&d!u`xBrjk9PwKLCcyylt(dF9bnWLChPWA>wDW+(wf zaBr)t08BTE_l=P9)f2}eH}cpKGO=t3fbC*L0pK?A=2GgUG`A1&g&?m|VMv2qqY%X- z1*k1Bcd(8+phB+s$dEWSfGb^@?Ia^ehFH1T`;}_TFQ$EJjOZlh4Mnd1coQ!Rd=;Vw7s)IWqK zyeonstsmTC9X8=cFiu3Er3N`VM!Pkf$9|O9KCGo5O|&N`#>bfmV4y&&H6TSJc7@qx z8bqJ0F%qujegBI@2soHX-}!{2QF!i{Z(+B6`r!ePla=@ z`YFUNOR^U`Cl;jfJwc9U>+_)SvikMjEK<{RimiFAh&@n+KV@rf?q;{WfSq|kp?-U1 z0SKFR1!d$00Ntk2q1y#0J;&IkM+5TS=C=SjlCTAQ1@o{6$e%8aZujfiyyh zlal}Raj#`L*S@7MUi;k1RTWzSp9M&%HSAkSdS=+ssH?2C?I1f8Vt*Cqq-#-iSSr2q zM86E%3f_@*ExM3BoY{63hsw(_P_)D}j`ak_u_58=$zzGzR2Z4(6d0fPeXXv}^Tme0 zz^=&vcCBw5bJD(-B?j%~m%_FbSBYWvN`tzWw~JQvRSw`o?^4?fjiFC+|9W> zcCjeoa`Xe{Qv@cw_~-2Z9w=U;o~Mx-SQm?#n$RT~>jMMF;P?GD;>iFJxNR9_NCTEp zk%}e93GKL`u$8JpE9P}Q_Yje=_+w7zt^cpxtK#ZK1fLZ7jgKqOqA)|P;Jnj>9|AQO$!Bs>h5I1|%zn;Rz^W*L_kLkxy%I z32(w|xY+1J7xg{S_bi5?;+C*-g&&Wgaob2~cIEkuYo`%aJOSc67@Ld3-7i*0fRkv5 zaOZ84vZ7x~SCC9d&?RaXzW_O(AiH>u*^Mmb7Jz@>k;A%ZWbiss^+TV2nFNOsanaEk zzIFRRo|!o8DfsmJiseewt)a!mKW*PPC%^20arS68g+ofTd)IX8F?po(Zgiqgotbh0F#!E@rK!%dPjmNko%TOx$nl3%bTS zfJNs8Y;@n9d6G*0$%5X~p&uiTy5cnse5!y-NdC4;$bE-Ya4_i$PlhjEI8493!%=7_ zW`(<0ea#*X5y9=f?oni9eV-Kb=-uGq?UUDoDxjP3Rzxf9^+rwd>(Q^wriF0BissQ* z>c%!YWqXr`V>LlLF7{gYZ1H-kGFCypmz<;QTn z^lK>`7=i>Bx=kVnUT%20_rpUzGYEvIRZeia?I?xkjo@^v!u3s<#;~$^zJYB zx*Yux>y^yc^U(GQC&?sB;J|z=-{nlgUYafak->kA&bqvSCG(7ba{-vY ze+S*e_ymq#tT*XV5?05}Jd1#A#gwg=3n7_4o-`!k%7)!)11=?5Dic7k)pVJgbBV91 z(VIKsFJ`v&NHmuidL$k`YZMaC$WuE;K%4Zns&R*T&{2>~jvhYCuxND}X9$*qk~|PB zZ=&+A?n@$kZh6COvU$S4>7)vin8SyAzmWzQiX>Pd_9ptHN-mFYRL|sf@oa@uGb5GxSbRFQz;2aCM6C~rdc!rsy3p6b}u225jUXrg|aL;70 zb`Zs|XZ;JpO$6togoGZm+k5RrLV+eZH$JKVp>#Ho zwC-`&83@#S+&|eX|8QKuiaAeM9F$!W*|NI25F%K@SboAk%(}^8X0TBJh*85e=@5}< zdCw_jEr>{|&Nq$yuRtG&#qA&Whh(Opz0ja2X0t5G3?t1c$IO!Z0C~)&RI3*%nXHhI zeij8|4AZCI@+SKvZ!q73z32-! zW9T9=>r4a~2nr-c?pf!iwRrcf5h!MSW;W#x*VBz8u+myR)B{;;`#}x;uOq%70uTG2 z2t0ptol~9A2c%j?$>PkB9*PFS@>{312*@A4G^NcKUi}eZlDg-*x~FOjx579vjKvdRCT?xhGa}j+E6Sb;!c2jb>VyW5Q`v`qPpL47Y4QCXbSI z++kt+s__6LT-F!s;Y58ojd3Gxiz#P#}PmT?S142%$Lx1yk>|>Ra(uR&G`w|Ken{9 zz*SREskrEBgw=!HV%?((V@x88xE71KBPT|k$4=!gRuyOD44h&dK@USHkIHJ0fg&?# z`+crMw|0fr?I6fkwk z!w?InGRkdgB ze{ur8xDfRpg7nB&oa@b$W63>U*|F0S$w?F4ztR+&=`_V-_U`F2#gYUf5b0HcG7F#@UHWbjojDW)ka%vw+Pd0qh z2ViHp3lXCulUy07+j=02^GF162~HAES)7D)ViwzGNvJ1EQKgm9Guxdfh5=sbb|4Ig zaA$3I->~IMI=viFHv&T$JsCUR8^ow6nXB#Kqf~|v1v%`+PEs9plX^I)|T=3ILyO# zN!TocfE@Z+`=5+sLki@k8jh zUNfz&YkqQYQ2l1@#{>-+B(N0*Oj0U}<+i?~pz2K|kC|wuprkYO54Wkw10nBnVWvr| z@}*V-#DqiYyK6xF#qJJ>*2QUM%n=&uXo|5WhtOvm2hY!-{dRov#89QcWEO^}UI~Jh z2{YI~<&Y3|$+KYHR-lmZiVHEZs^Ilg`s5&2Mo_IUnJUZblbNFtNB;Yt?WZtKqS#;W z&l0;O`-}96UM0sqr=RakwsJkjD?5f1_vIMJ5yS;r0l>(e0dI}^YQwy_k~#=Md`gP1 zhY=R_V!Rn8A8`42U>jSe9ZOno*KL^`mU??NH+WG8Qg{0YPzkLeP7(t_mYT;X90smH zx;~XG0HPovnSp+D6zEi&rgQL*? zmroEU5CzzJtU$-JuIgqf2K~@pryDx5Qd5k>_XC0&-o4Air;GXUsb>M-#*7bsl}?rd z?y&%p>?z3T0F$kuF=0(qWe034@*M9n~&7xQiPAEOOfr|UP}_;-QTn%6!KD& z*kV2Nf-DJNaRVWIZa*F@zxO$uU zMNdTOj3L~DbW`{1tn3Z44KWoE&Q#Jk;yu+F?GT6JwS@V5wuS*${FFsqDs1P*5?Zj< z&qJ}d9{p0K;rZGc4H_|h>z78fYemh!%5d{>bpWaAiWyN>3(^nsDu|Nv?po!A6y~Pd z;51W@83wpNf2v(y=P?5Wh9I9cg3wL9FmL5#89DzQwmI4uOCqh8kJUP!inWmorBlDg{QvX;<#%89eb_ah zwFKky%&}>(GB$~todp3qkaE5PuW*$b&&d(s(2mV~!VYpYTVHF4Spl&kjj)`-C8ReE zBKE?g0>IP`)FBVOA5AGn$Axy|I@l@m9qcQ)qJ4>O%x4HpFfKSu#_FRAYg!`)$M*rl zU}h?Pu`7@d&(m*DNzt;wQs~QRN#A$lDWD3@SL+TU_Y?q3BFqtkNu!$11{%V5+mWc}i+IMFa55Z^OSBV?LBBCbP zfwZ>E1CU400O()r;8mb_?6bI`Ej>|(=Ge+O7eYxvn~YbD_hrQ;PeB5IZ>c?RV4&C? z0jT8My3tkM$1I#4qWI=V5ne=Yt)@#Kx3_%HT-DhBql$97j;@Ll&Yz&oOoo4sNlp>O zg>{D6XYU7ss;ZDZ>%cPkryh#^nUc4XamDE5cL2|I6xWAe@euW5?kx?cFH6E}{@Rk* zR(0n1DY4zjJ-bMnmL}1zQxF1}p69Lf^}9juD%e)adz6}sTb1b<>y!4uKqOJ+!vJKd z3j|j?zm4z>@JRHp?c?o1)cTGZGPevCg*6}!#FhnZ_=To*a(nw`RBlSU=hC(XVx98i zfnm;4)P|KiEJ6pw@Sz}vH)F=|D^Lu7U~JHuhtQ5S4Wg3b9~E)1{}c+H{ns=-)=Fu5 zJAS9^{KbdWt)nKCLcrLsr0Cc<6Jd#pN_X_%Sr1rw@7DlsxUTWyhPcY7TD!!Gx;-z(r!#&TLJ$l4uja zmgIzS^*uamS0E^0JBi7HRJMUSAfBnR-Ai*mltpqmw)~wz!UdNw*H()@ ztP_gZ&Y>^=nnNFan#(?j3#a9{7kF)2`7yG^CA*+C%<+<8VeUfjpPXjgOO+tv1}GKs z$2Ir5aFfDSL1jHf;!vF#1oH{J>$KHUmJ)8AbC4(>cyTy1Vtau|3g82-_@RD|O_87r zECBAIn8KTE`!owvV`Nsv8~;ld9SCjXy>-oUCOoP&1nTx&Sl!;u%%`kWUWJG_qj72Z z2qODYvpVfNZKK~21omI1SLa*C^g0YhE#)n3jrR=)xT_=lq)Coh(E=8O@zvz9FzN;p zfcM1RZ|BI=N~F0m8y~2)znLD^D}4VM)&pHd7Wb z_G}vG%CFY7F%GH5dqI@aeR}SdDnJ`BG`16JA&u=qJKSKpLR1wXbTM~&er8gGW#N*` zd4_{(9^FB85jdz;`hPQD+g55610_;P;NLMlV|%l6@&THO(?Q9ZrLg7Ijl2a~dl+Cu z8Z7SPXBIZ$lB?mn95?8{tpN5wsh$#l_QE;3XX?Y;e7a|Ne~3(NbY1MFe*7idpH z1T2~BJj=p4@FdHC*ob(F^itT>PDMT>zy4 ze^#W1V05m7?Crff6}WHT)V=UgQ@T2-#ZEf^98lbjh@1vs4a+P=bLxk2qz}GfqvQPc zaG1N@)bu%!1@uAy+V~ly7gGK1ULToETUy=L3t{m*3q-M!P5PCDZs#Xf8}p@#9fkj2 ztq>rQag;XSvxD|Q`w8H14*LJaoUYvm`?78-+s3FX{g3yl-DYH_D60~)28Ds(QX`-J zT=Kr7e$A}g!A+Eb;U`c*!H`#~>b8Tdz`|3ZeHfpM)c4o8Qem9x#busA<8fNz^45*Z z`|cgYYna9Li}tBjO?W5$hYbR6nBdBwJuBGw4p=XrCJ3{Kw|637Em=SGFoh{lXrG2f zEP3{(k`vV_JPrYA^sAeRLf^1wZU)9U;UdN zOnBcFI<_fmB=%qU%ez6!cbh8{r~9t z?zpC|_ka4Km9`4Bbpiqw5LA{jWdy8SH6$J!jgcXGV5~fVqAPO?W z4oeUjVIw3Vk}!VfCTRQle!su`(Y%Ve_nv#sbDr~l-p^{l3%{Fp8R4=S;&@nVX@GPr zjG0p~wfY7vnhj62y8y2igv6u&nmlqyR)iS}ztaYFQSuLzUiCrd+)(0AL+GPqP#JiM zWt;MTh&S^q@?@$?d@I+JSc0xjfQk~LT&jQpdg)^emOhS;@?X~OGdaG>e~GPmUHigD z`-FVri~Mg62;VyL@5|Nvh_!35!%2A_V3{gTKk;k5SYVTBSAn}9mpN4e&~85AtJjce zP1?vLo)V(cPC8v{X24j?(RJv!I~7>-x&l-&^zp8G&hRIL8-qe8Q8FQ#@7=z-#T*}i zyrku=C!iuuLG(M|(UTsCJXr@Lt0a|-l;m-6NP>oPwMqh`Vj==c9frB6GsGzSrqAXVkk`A76+dS^@O#glUmF*(%q6Nv<=4;1}<12p9gHBmB*3^(M)7eX5nJ6d~O1#PFuxcl?7c+Ob9mH zMStbn#EeQT|Gqq5CI#BteBXd5aDP1t)g@%rCETq%9ET^qQLg5tahXuqVJw-fWnn|3%R7ZZcI|VR9|aA$(U&v@>4_naBLe%kMH8s6KAr= zUSjKejsuNw_|ZMx?qm#9DRsHwnSk&FlnoGJdKvfT&l9S8g8xLYG|L!89S6=z3^);< z5iJK3sjl@WN-`spjXve{fhv+9x=|%OQF@`4um-FzP~8Oi8XXYu^&!4dx6Ur@PX67C zi@AiIU-RF9KN)255fUd@*c^}yBmQaNX%I^9odT}&q_gN2rcd}`atb+l_IU;%1=S6z zQ8+$X#PX6$W8{q7#(d(#0yB-?Gw^i$t01+yrmsgRQ}f7)c2I=-zd$&GfQjV&r;=Z3 zPeZ-#K%^Dtpr+{6hBI|-lY#a2JW8zak`mm|!tCQ$`bL7U7wSvaH-?lIrC+WeYij#y z4L56-8i=msjNP?J;Y~n<^i=(uO{bjthQ{{-pr~2!X^y@U{3DZ}qab$q_TX{QN9@Tb zxK6!!k@?Z}mmbpb*FhR>99;hLy2}K!1}N$Js*3#X+YWJMk(AWSiEs2PSa1LS9|(>K znfVVuVvJAecaMnFl>$jB>GKmqLi)3Wam64v3widEamD(>Fc!_)gXLeypqT@^_UfPw ztririoId?+zC^E>Sq>eQY~{2}kEh_Lg}jzpO$QA!q^^Cq)HAU;)iqodup?#u;fx(~ zRv-P5&C&gGOku2`fIs;i>Zf2th@rOZ;=;&KH9oEJ-;HkcSXeC%2gwv>a6%(HTX+V* zZ?UiNJGfqFI{8+b{#Vo0RRxq+-2a$!-@^QoRe8@awT%4qIOtBB44hcUt)|^MLGzi0 zh9CAw`Fou29ZV0-f4%v&F8>W_kS_gL(n$K1Vkj4_Bv{%sFtp{%ct)h@l=et`HDgBI zV?d+g$z0)_EG1>wizbUyk7MD}QSgRv;f_3&IYZL`3Hzc6Gu+PAl&AQZ*`t2Xs3|A z0$y-&nA=)s;=iKhE!mAq#BVQo3G!!vI)bK$z`~l#BS(LQ|z zK}^2DKQo9A`r1L0RX2GARCxP?IzAx``+V(NA^0%2xpzBRkX!*jliB*jzd80uh+}tZ z?yy!WIKcC_!+(qzIr)`0U@`?EUh>MaS;X?}sCct!wbyZltZ<2w9b6_h^YsO)`6Z=o zo$p`D;`3)4BK)ir8B5d;EfE}4?eV>@4f37))N{O{N_(JU_^`JD-1vas}>YW z@6)R+Otz2o0p-LWA^d`sK=k#S`lReKklFsq>95*kTLG|EPkz2@2-PbJpWkQbiq~>L zrCrmTOK+^aTmV9l94}#|$djO3QCJyNhPds*Qvv-6duG=8E$Y$>8>l8M9kLZ*&3N>VI?lwjMc>n=vMpraIE+_L? zp|+p*N?az4LA?L*)d>kwDaI0o@x@()9CK|b#@Oz>CM@}Obpv(ypKz+G%YY1*XnY=P zk!~=zuDFN!8~S|Ye= zQu#OH-6AVSQ=2A_1cknvVthB?))7gdI2-h#Gd!ijCqdmx5GkGPkXMoZ)@`S|kd+j% z7lA|^D6MtaPHdsRK7wgtV7LGe3hF}D$43-`URQp#E1G4>&b*^0fSw2wWAnM`IL>U~ ze{J9PSphR{1k6;8uV;pJo#)ib@~3B(DK*#~FT^_cWX$Csy=y@j^ER&@=uA+BV_vXr~asgrx`x+yPHYp9C8m%_+Sf0o$e#%?-ZCtYR>E835Z81xWPmdfi z)&IH=T_tfnVk;2)cg1%mhni+imC36cV!lc=2~9b;dnO%*z~%HCn2SwXv-#tYY)zW3Z7qIlgTwO zAwkbBf|;roqyE?S;Xp^{L9Nb5w+E>Mi{|<`g&~0IrxDj8^oV@BbH#hfS@9LKS;G9t2l2nZm0WXlpBuF% z@IWsR!QCLr>cn_~I#6?m$rl3^$+Xgm*0>e7j#1^{cAYZjQH)6y28b zuxw1FABagE1_g=JbpZ}admqCEG!ilJ^Kk5mi+ZOt3b18?L6_2Kh&O*M>r7pK#@l3j zdxZ4GMU5_nrN0F&wRp@Iuumsq_R^oy*!e3Qt~l2 zLJb$L+b*JY>L*5gr1kHUNEfd!o?f4=6{H%l@%F^UO{_D^OI*FidP_vdNu!F0Vfq^x z#_b)C-c_!KJr!S-Pr-ELdg(KPf6%z`hXHryR7#z%5#euh;XFu6X=2)i^_V&MgJ=2F-y> zQBP#jII!q1;*mydB|VhP`}SgI&YC`>(x1v?m}BOlrL@0k8&uEzEVG+kSG;S7wnE;n zI!8Z4tP^WiJ{s>YZnF?SF3zoT?zGic64!Les6E)A%aR&b(J{(U3@vryA$7}23?}Tr zx-B*aa@L?vdLtKaJ%5p5qO*1wGLBDm9QU*R_Gwj=fvTWcG-(*A{e>LpLN+hE>mp_u z>PuY~qSu3Hz7D7$??h((gc{=e8G(-OzIl!hXKG-{+kG)j1WCIDa#ISRw?%z~v9V;<~*>v;L z_3ojr*(MFhP1LLhl&N>3_}S6q1=6+ZRLdVNwOL*Yp;+}i4B<5YxQbkyawhW5LaGnf z)~aNcEy81(&xmotl>Xeqs;d(dk_m0B*1?2rLYXUCAHKGh`UAZ-=`~84bLG}dogV?g z&YS)LpPHH7`&ZVp!S9wk`p=Q=r4jWhu+TQ@6w)$a3033z;p4#WFZ7pb{^x3X<6wa-qdjS>2K&9^HQES8^+X$mB&K`lFbP?P9=7jQ_r* zLA$FBuZ%tUZg&ke3xjn$_oU0)^E&Pn*ST1a(909-(YOiayN|lHRiJy?c}D@aNU| zV@YI1;uEc%V%9VX1LAD~#&l)LS%x9wN#yghvul^hD4U*z-8D?ZhiSCA|IEPfG3ZC7 zZ>xAYyJ+sAm-!QV)XuJyByzY(>7UXlPc>1PX4CV~@(|K6ISXqlFYRS*U zfFg#a!LyMP5?4|)%QExr5!+fu1qM#%FXkeA?FW4Ar@Slk6sboFRO6uafY!xzk=K&S z3$4n>@6~@iQrkvgU)72~Hss9Vq?rRhk$>Jvo83EYc$Ic9*YL!}C_cOR7E@6dXIQ{C zM;hY8!|0_n$wkJKt$$SnOw0h=O=a(w!G(q{KrD&4qT%u1mmSjM%@ zeI#u?z-5NOS(q$$*#rZJ=bu*PEO#NqX6lQN$6c_yGuKIu*bz_8NEBsmxXozX%2evV zF8^6K9#dWDP?(~Q<+Hn!3$M)09*Vv6SVDC?K%oJ2Q2Z|iH_yoM(|-Dk&vW6`wA5}l zHMa9k;_Fj=gJ})=3He$qIo0t7aM}m)E&}$qu z8h^K}K@0C4uk3iQ9$07TfmWcE=DCJiHYo+n98Ji~JTJ@*t()~lDO0$T7jAubLWMsz zz|!Qz*r*Y1eKY@=>VW zs}%|w3Th6eQ)qtu*^By^3!T1yArmp`3W~5b0`m&>2=T2Jf$)j!kXPc)4Oz^k62KU% zeG|>H-5Yq&6>ciZHJG-ipWt*-)mcNrp~w$4x7rxKJD#JXUGmu*8YWX%&Zhr&%aT;u z1e%@46|F!C(7q#(CMzT!XyVmz{;?6HYM7cOu3jB`Ph0oD?8kjp`2~?>xzcrAXIoop z0E-=pvR(hQGi8c0vhwu}`8X+gD$g3ahRM3)gJhX-(MW$`#*`^cBlITMtcEmqs*;dP z-i4@R6cT|&d}o`44@_=^ejhdnwvzECLlwa89b79)+AmYi?y+w(E2_}T(Tv+K>$WaR z+EB77KTMSf%kj#hk36BV(KbjCmEve+RB#?|ksNeD5rV-Uo6)gp^KW&8K7Z)gL0FgF z*Tb_8xqy>Xa@^6be(Y8OR_~Om*qQY%F0_}s4Ed-E{k#ctzXOL?@%n%Ml?IL#(lZs5 zt29u=xmfH8)ws;Eq^Yt@;)olg*GZxJ^`Y>S8%eD!h|()QlQyb%Y+qX9`tHZTZ1&pT ziPGg9#PB7s>Z0$Q=xDx1%f1kQy8c|iK*FiKo6szU*^xHtZf@Zx{2Z|Gi~+LU9oyG= zNi&(I@pE;(=E=M@p8Cj6uiv{EY+TaV`m4CJnP-*PUu9hwFj~>DDkA*aE@m1!axWRe z%cU0pRu!IV;vEefz?(~5PM4%nPro>{8W-e-naDNYGm0BqV8^gq${S-DI7O+lb0tQ? zQT4lTdUlG-M?~MRFpk{}J6d?8HZ`>}IATny)c03G_^S(N=zo}@` zHD3QIt7nAK;QHF>0_>#;c10-~*|s1q-;f_ZY>E7!KQd@GqJNf{J~JtuHfVNS8d}vt zb3v}=C?DcX+*xOByTY+<05>aQjsz{(wXX1h! zX~nBN=v(my8m4f)oLg~xc5xNhOLgseWi`sN9GJ%3yrdOz1>xjkF7vEq#{Qo7#)n5r zjJeLeAM%giCO)_{ysEdxr>b{yuzli>fPBUZq^CDuXhX0O(4+Act%F<$d~@Ed@>|QQ3Yx(iH~7t5>^f^C zhnxuPl=)$BI&e{uNiUJ~Ww4fu*4CP9X$=H=jOAsuUJks{KDL@y&fwx>2|Iw&+XnU7 z=Ci=DaJ9D_5}>d|jQRS1n6%$HY_IhigSwaTbR_NTNbO#%GJ!*QvqEJUtyu3K@H6y# zCZZ*u=gq`n}=hM{&C23|wwn4j#Azz1|3NWJ3H1aCIC1Aqg;DxL4%?`e- zK{izX0<6TiBkWUUV=yswcQ>Kx(EA+*pIeWMl<0TGU>PIevy1A+tW{tQu?G2Aj7g=} z`gl(lo%?P6^QE}ocT7Snp4o8#9NGoRhuPSh_{U)^>DXgIghxm3c3JUMLgqjq!>(fz ziCUXTc=Tu+UnJL}RB5VkuHnK0%gbat+HUxFhl4hvP0+UO+v6TRkYUHT3N5|4zd=i(e2=$li#NgKdgNVL(@uJ0N-o`ZRxX zl!~XVyF_Ay)>x$AK2id~>)YL%0{O#Vb;`^E zS1@EErZ^m}w2xs`*?A}A_O-`E!O%>{t5tcga~cEsBME1NB}Xp7P4fq^rcODXn&aAW zd<~VE>2mdVP3V3SV@F7PWShWux*I=~PcT=Sb_%H8!5DTQ^P3>_i13u0R9&`1X*C4b zI2W}X4m@kLv0<;K>A*1zS>#LdFYqC)p!T?A1f~!6ZtK%6o95qWyTuVZ)JA^dHtR^& zGgXCgjFJcIZa)(lc?WYLzWhiaF*nckwnvZE9z)>`JA;$lzG+=_4n`wB7N8$K!FnMChJcZ{u-T_}5zC##_#(%>Mk*`RsF zBhal#NEta&vvhGUp207bURZCe+pD<{U+FS|e$1DTpLnsX+P`mJ!VBBpB>*VaWhM*F zb~E3ov# zJ~J-Tq55&wd+P>eja;J7c=(cC)U$Ua-zZ@EZ;{HywTejT(9ajy$n#=MA8#1I6Lr=< zkpJET^DUDrOUn}+yc9JQkGYDh&$~dZC5rVP#}}HF9lC%F6XW7+ahcf@dutDNSG87>|!Z^X<&Z-iAzwRc5 zXV%X=BFtTMI3}*uLcipi+IWD!j$m4J01fFv7T5#aix51H?_GUiM%IO`Cy3(rBQxN@ z76CtQ=KC#+t?`nL?p#^aMu=@etF)k)q&C zxye=ZMAi%pyff>4Kx^=hhmpn8aHQL#!_qwR!`sa4+0^2?c0V zf5fXdHM+*Ob>w{U2{35u`;hMV3!3VbcCb9U4Vf?GXvoJBIC&f5r^77P(s6F_8!)Fx z4EwPP~^2In3Pv(h0GI%vbz1E~}q4tSHK4P7J)ekJPDR zL%;fQ82Yn_`0vZt+G#^m7DB;VC_d=j_zW0erMZw>A-W7 zA1Z}uLY#Ttd-}LDojztGVxDmNBbsjwIZ7ioS`u&*HP#1Ao6pxw6S5KVJN3 zGl=-PC82SBVx$r3=!Sf8&w031wePeR>GbushMkxwTe+o803xyyp7*=C2-_#qzvhW`upnuzSf z99&HvI@32FlObwtVgg-oiGx>1P%z$5_+l5PM0{yOCnh)v%6tHyMZUUrAtK&yLz{l! ze>175E_O#&=I`U{abG%a_jc>*xcwx|z!maiK)lXtfJyoWdXpHSmv?ia(1F$My{^uI z7rkmr=^{Xo+$+{Lm#pMpE8FzgZ|i8~`gCu;G6Xf9@?95~dakW4W^Hywjx4P1hvyt- zR$EtcaA<6=Iz{I3Dvtg)^v$ejntFYIiBUwx_AIs_~AaRHE}&Gg37NLj1Z7C1|}VcJYRHY&V8%WfmNt zQW3K5CXH_nr~EOP@;{dN`B4@b)6P8sDTl;2m6q^W%$b)icg8b3XEWX(ASNPe4nF() z#_E9S6ezj`7F9j2%&{bT8m5%kvxhp3X>@xjuI!&Fo58Rh$HAl*1%i4N<#B*bSPPas z&=l_%(-VMS39jeHD2)y#mq*Hocx*xo`&b?Da`(XnYCPj{CEVlaJwX0}qZYAlW@D%! zsUfOSbfGNNSz^yFRP4-|UR7q5EqQ zGs`-hmenLxhI%TLIexsOC1S0_fzDjtj>_7*a=dWLxiVrn5E(x;l~Z!K4w{vwJ6(0Y z^KBm@?J3a!7}wK*alHewCor+i`zf;Ema{5tcVo0n2B$b!>qnoN=Wc`-u$}SZHH~;W z8Ww>$0j%vrSObT}29)?TGY%YoMHaQz_kYnTqpIW(wEHMvU;Wu(`fDW2NfZqzZeU3v-JD zE2#5h@OZ2|b@lFcH{ZoBz7^`+8g%0iC6Gl2IG76u8_bj(A8*Yxfh9ECRLaf+?5%;9 zU=v-6mQ(p4(J_vM0dI{3HvUkyHjz(-DpbEQm; zjTFxiLK=lv_8Lrs>N4mumB*|#?lL9WumDrT*Prf4%`|PKt@=E+f_4_jW8G}yO4wPA zNs}B)KRA|^=i0=v(#=(2m_KkU(Qx>dFx>t(KOhYy+14I4=% zPCD3!ng}MFmwA)vubBQ2i`|M<*q(2wR!(_jPku>YfaEp+l6zaXh*sCLeggF7073kC9%%Ako8k4}XhE*a*NNjWU(rHN*kVBi! z{u|4%Yd@KfU2jr24kx~D2e|fO=9Iu%DJgU5GFjr?s;`HMJ;KAv@;(DGGfu%$f!y5q zEcKCMw@|StjHL%~Dyh;C4I!jaKr4Uz=w(Fb(>}izG%MQesOIUNo~owY$B91X(<6*!&aq+%}(c0O0DgJj(a1>I6I=tJo zX{;7qCoVqT_H7W6oE${wp@W^A?c6SQK@(tn3c}9D8Qa?0bD@(>uwXP??*B2ZTJ^h&C1A)&WK0~Y$2eb5*fb_2#y4-dfYu3kGu=CA}e{NwVW{|%@IjE1)O9WoLK?uwGI_J{YOs{llLZGU|z z!S@={%1S~hQ&aOStNzA{3$Jjl(JEzI>~KE|eQa!<-Qlf;DBX-A4xboN91wPMABo3 z#?LVvIaOl{h7r~a^2v?$vU_e$1s-g#qEfHsuS7B6wLpU^3ZEHxA9A~<^D#EGzpH+G zy@~rB0cPg4Shc~y^4E;umh>#~4Y_CRQ#+oyro$JTZZbf&iWN*bpaWIoJec}JA;W2B z+_BC*mguqGNWl~Xw@z?nCa|mG`F3K8k{u>ewH_P%4hD6N04x*{6d(#VChq(3q?}Qh z$G^;ZA3Emy;wv3LFC`bSg(q!`1<|n~Q{D@3HqvO6d0yXDv!{5oX)~OQn6d(al_LI? zVHTb)=4aW;iEQ*SE!i9rJ4!w~sfGlt>;$~&1H(Qf z)e~f-eY^;G=QX-hGmPy@zw111J?N^{M4pIbD5Nq-eoKD!#wp*0pdiz*{QdV|aG^|3=I9napG$J#?kb)cz{; z-lb5_73Q6h_r`5$MSwW7yMt*mv@f;H{LD$=z`I%|@HYA129$r?rNa#nkHtzz+!sK9 zxQ~d@jB&sYtHUdkjVRB^q$KhQq0fr6e798^GagI?;BNMRnik7+dhgruD?G;!36skD4uZ ztlRiXTnjQ3=@QHHU6qrkTd~}V1nwNTGz<=$#-8Iq$*~Bwd9}y2*37sVAQeqs9!cChVUfTxmrZAdXY%-vKnNFEE)rZqF?6WEa7)n?%CHR zRxjWD8v!Qe8&aRR_uokUfPZRdYXa}q{a6Ow@V7D}kL4PRlhgvsa~$gpbe+e-%I0SF z?+dXxSUs7vqec#J2*)cM!6hh_1z7IqO2i|zEbKgcoY4ANqb7qLMVkDGy^2yJma|jX z__q2T=S2I&p~t={RNzvm8tHu29-z`GNnH+Dn>R!B>Q4U#XT%?fRA;|fVkrys7O>Gz zXorU;Y+}E?dKkN?;(6-wA93O`8cR0UTLa#Vl5Vht{Zk{ zDLBf==t=5mh-@cjn)%w-k9GPZkkK^Kz?W@Kie}0El7SQueE`=eQg2HmstloD*|}NX z?_VH2n;$qhM}OnyEY6J76jse?m<$A}@<{UfMK}M6a+&-WaSoiTfZfSb&wNGRg2E79 z@$8bNwtVU8N`ghu}R4$<_%0cwkEa>TIx4 z_tJOk-JDwzs=do=I30B0U25m}pCfd(qI4i-z7-HH4AE{ss)F7@dU@3pSk~B29jm@D zez0ywn;_WEzhy2rK&o(5>|3cb3`6T-4QoD(NJ8!>5 z5u?m&mg>!4uXn5FL#tz*b3CiDbJDYwb*B<^NeKd7v)CJ zqXL^((b8!(E+o(Ot6mrUp78V*|A3@)sSR2sakxn93p(Y(`rI^G@~an)dFS!Vl;u~; zDkIiaIj}o9bRB2h(7konv|yM`9=+pVagJ1jPV1R3Mv@m-Znw2h++CIoUxMOp<{U?< zCyfqwd8vB$mcsNN5Pn+qYPVyyv`1)Y5%@-xsizKib5eYtB3+OQ8l6jh@uJMI9c6UA zr~~`9sM8)aiA?D{UEtya& zNz5%K9P{Wt$%V6sABQeAN78^ulg)ZHEQ!H!L)A7~p}Pk?Q7SRIh92>iS| z>Y3z12hlqZL>}G*8o`FH5rUxk;o{O-Sq!`*Rtpdd;K(YX_$mc@mi}-rmp4@@x}bML z6&WSbYxW$qtpCxa;;I5;>^u&O}H zy5hkO4kx6v`AAXac}o;Ha(PnE;<(k=phKf9n4~U!Z(3641XPbnJ(ua4<%{_YX0v806tC@tCKaJuusgptN_~cu9 zfT&JLx6oWF=j_C9BOBuAULD~yll{)p+4na>C=1-}P}rKwr0n)^IfzoVizA&-{pPto zp)o=lBIXLAg&F$V`iCoBR>8oBu*$)OV%6qFU5yXWnD4@X0GbgR!q!>Q4iG?NF}fx3 zCp50?=`@HC!A!+xrV@UIh8wF2ChGG92wR9fv)%}Ls`QvmQ%77s)Jkgv{DUbp}kUSxl0YD8%0nLj!W&U?qQ6PM$ z;)SUw*mCazKrBcXoDX_*bD3n9!&s$h$Cod&EH7|-lje=GUO6ftVnP}4vgQ89pJ*iCfDWnuC)(!vK#*G3JYt21chmo)&9IyW_5{3SN+_1@s3Ko zW!eU{c%u_WnM@EEdu5{*LhuU_@(86(Bx7xP+HVw~6k^Zza%a;)NPxdrP|arpA8J5p9tB0l}f-KNoF5Va&Hqe~yS^~oz;4tJzDiOhYmr~g! zG*afFo^B45h82sAPO#WGyVT9^d8OkflkybGL%r-|MJHoLRr>m zr2IefH;QDqd@aco2R@C0CS`bN=tCI~?Hc=@U0u@tKBnR_-5b0h(Y4`PCz!gEz|Dfd zm}HCuD*dcW2S3I&-CIV*_5~=IO=MXh2|_5g(_dx`gr%e#L>{}YTBY?wtds?KTJ$bj zC8So4K?6B@{cq%H+AcvE{h`he=WgZQuNtsjjt;(#w+!7QmorZZ)XY5OI_7`6|3#ZY zRV+_ub5L>lJk+*qR7128>V@lh^kJhx^B?pen(iXe7G|l94M(jY4IvN8~NG7%<6=y z3|nExLW|Fs#A{1Od1$JEuXXAjh*OWnO@x51ynb&UBhi$4LeIoLDNi|rUoFPg(b2JG z*{nyW{(pciG%e|sa;^PT8xr8;+jG=WV_MKv6V*5pMOosPy;s)iMcm}0;RoA($Bv*g|^4_#Jg(~Pz!HD{Qh;+6m? zmZV0OgC&$vNPzgigGz zmzPBKq}T1@ngzh8c#<6r;p@D-J>X!RuWF6->H>jg?o#w%stB^_5f}0^?)uNj!!0|| z0=BHmH>YDBX3KKsM@QSWe(_f@XtKe%)NsJ<9`u*jROW76{Rk+G{d-~+O&zY=#zBMwtdcMwOV^qf`&0bD)aj?`LL zPGeyy$jq}w0VQ^?dJ4Hxzb0H#zlAAo+UgW`V5yfZacB`8~h;cgHj z1x3voYS>SoF7JG6kQfnt6M_iOM>tvs8q1PyABjYaee@LLGEER|2o}D_MCVRQ<5TAS z{=UgSR#~t|{^AePJ)#UtzQ%EsP1+9(22F%@B1ekE>m!4?GNEfx6~lHhj)R%j0)IjK5U+|}0Nv-E2! z8+dfzuktSOZ~x@r;_3pZN!DY!hL9PkS8{)Kr+PYs=7g{0+-skS6X*c*0XUSwzyhhf z$a>Xy)3^#fy8F^T;LoA16LO)}mvzpVCUhOa&rOE642k^6P(M=kJKT`k4uB{Wja+7< zD3JLgS7|!nTp26$=a5lqmg9k}g3K?$?mtprS>a)~py=_i;a-uZ@>R~X?vl{8sY!&R zqK4jir{17DAkhX`f*^p}bsd*|Bwu-@&?jK*LE7Q|_x=tX1bt=&xTCtt5ea2!Y>P7A^VEkeNF zI@Q6TS$xlQoy94-SXhcNK;v^4AcqgjE%Tn)?#)UG+Qw#+km9GIsbUpR1O=jlA=eTr{`x3%R?0vmDrus zynA3$#u@@C8WspbhF|YQW@>8Z(9n<*zRW;uowu6+5;;uHOSVo5T+CIrpAs%NfnQ9J z(w|c=k9(o&@6cIUnQ1O@^;y@EhltIOx+BAcV_s=Rn$l*mAy?8dH?!jw(x1M#nf>a? zaPy}=Lg5i=`tUV$Ip-q6Frp@SN>PS%>bVL~0#7qDhuYfoDsl;Ph!FAoD86?j*CuC~ z1SHI)crYrP7JIDC%5L&>pB72-!wuy@ zH%^Mwz?UvYf&b5rwtu)>Ifk!s z?@Fz=ouJws1pmULmA3~FdSpoM#G+H}BgU7O5Serf#r+*vj~Lblm72UAPwbM(zGfJ8 z@_Ad2r|lmT2?+`0iftQQ@dH#|{ycDVS*qQ*vpy*(!~B~SYDQ*4QN z(fyi0NZ~oxOUFF36K7b@`LyBJx$quoHrLt3;IgaUReVDY3*oJOhT<9 zp*w46=Y>5AZ|8dhVGzy%7$wPs-7%jXRXuonVV91VEjF&P0{>Y#*utvO;p$kE$8_25 zSx-LsHJq!bJXrplI)jiDfG;=s{Jfs6@WeL&V~{lAC|7Ne5yIOu3IZ|pa^zDh_2CyE z^8uT6$cxIMAoIM_}_(3i+t0oLpM9B0BB zgC`U?C2brmib*SRldfb_PFz={QmG4WvC+{PhFS%~4;uFJb5C0u4bq~zxwm*&_0e)u zse8+F&ort=0ugsT6r_S(8e}UIg8U31wZN40&?JOKu2YyTw_tz2BIc)S%2Nb?B|7?o)Z;wf@qEux-0rGl2o(_*kiC0inDhc0?76U8 zFhZVJbzsYts-vUiMS1(33>&Rr_prTw^7cFL&R)%@)c)Cb7X??(N*k|auxavkKCF4M z1P#Xfq&`~A2N3Pr+S-{U7fnq~#RV6ydn5Q|jtiZdhCKmymj+kL2oLgbA{}BnpSUGt zhnH)fSBszCCK2j08%fFY|1qSK!p--tbZwkdjl<>%y-Pnhq6!^>zRS1ZJ?y&ZM*AG> zGCBJN0qo{nm43oQj1i)$AT#si9+3{c(l3FxtF9?YLd*MUDR~~eSWR=T^3xW7TQb0Q z3(Ygn%W8@`UKlc5+qiq!s35G^4(L|kRl7EprqZ){R%=)TYta5Zls;#uEXQmI=3E75 zJwiDKc**i#a_A?;%wI44@&P*)Fwx%Vvhe8Yw#Aj$;2!-=D!xfcJaDW_dpT8Mqh3-x zWb?5@HNMC!Ihsor+ILG)z{m9Ub zi@2-&@8bA1ijt%_HR7{|Z^2IxnTK2|x3915?+@|VB{gB9_(0s&kt6XfEe60%Si*8d zkClywvXW(~!(!0;8^Y=uw7I}mtL`79e=SvHS^~sa7VHt0000_?jNr}#R<5m;>DS5% zF`BtaHY4*!vF6EE0y|Zb4~bUAifUohN1DmOx;Bc)7|TG?^vaPyYMGO@n+$tlPg{5;a9fLR$1({O{eScyEO1-IM)h{r zLBGZF=-t)DMMo?)6~7uwcpt}4Kd8ErM4{8%so|!M4)8=98hU;ymv!qn&Az51tm@pc z`qXcoW-}7S2Vx5~B@7C1`_;>$o#s+k)4Lu%B>vSREmu%-fZ11gUM^07ukX~y@4@?}l;gC0*Ma{VV|aVmRj{L;iF|K?l2WF5<#Hd}Wh=)@Q;FmL zN)%#tFi7xwHvb0yhmqI+o_I{=_Q7%be8nJ;$gP9G@tvI=@yWzT_QHVPpS`-WwKDiZ zn2Wl+^DZwlSqT?wa#wx{sI%Jtq0Vyh_L{ygJOOs{d}j|~K89H#;Sz{pD8fp6#GorE zr=19^z}#QhtL9u?>H_vrIj;(A?>3FO`Qq|3<<^#a{j<>F>z>zc!U|rw#d#g^`?7`e zV6GN$5f6Kpwhs*%_^yp7O5zlWe_7cOdNYVsI)0byQZ263%Q%oocDSJ%&Vzp}A*MwU zf*_1UW}7tsXE;bWuv2mI+HbaAe(;$d2O8!MK+fHVkqMP932|{CG$9i(xk65nX+mos zNtG@-iq{&dDVq^}g!Pp*i*!#Y?1G6&I~7R%n9L9wUAL89zp}vXRDlI7S}Hsd^iaN{(%4VWkQV$q@`PvMVkVuZRnjquO zidQiOI2srJDmVeuc<^BEc00tuK74_5#rN#e6}KT!ZN0?8a=ei0R-o)g_l@#IQwi8x zk5JQTBiz9b20pw9~gHOoOj7~-L9PXA;g$j{L`5Aw;#@Qdb&kcAp>+ zJ8f=Liz^$OOIL^WwbQzewxK&@E3ySNvIv?DR6$AwVY0`;htbiy8s5BcQ zH5BR128c9~5-EZbdKE$okmOxE&Ybf+=gj&3fA9P7Ly(ZlW(}Q$oI$3eSE6mk}QkMG&Xi& z_85&r=gHemZL4(Cyn5}LP<(oAR~YvqjETWv)s)*Q>w|N?yV{g4{CN|@^Zm+S@IVky4bCZH7hZx`HUif-OT{&o78jy!go z;tukrcolA);UPXU`HK1`&bkV=P4_~eCTAgeSX2rxUHa~t>d2#bYJvB6%OOZ_;eUO3 zP4e<7Qmn7@X^ND|Cln<+T55gSSbrm{mhLS-XWsIAE@!bU*;56)DNjh#oxkZ3LyLew zT;RW5>A!|H{Ez4K-#8o&yR=}s)U$1kSV?un*b6lkyVIxF(`lSu$I?w1ZS|GjV*%ei zfdX;i|Mj)Jh_C!WZ#2$tJ z_$B`rN;h$g_uqa!e1g97-##Mne^2^zT>ft${{LhkG`z9|FBHpd-n`jv3ZZHIOk+s} zu-m?U`z-h*B_bl?RQ!)iBL2nY|E0HhkAHTl+bws%B|ChB}Kii zv|^AuH5i$}X%#APb>sKCj_=Lpai<(>b@ey72n(2$^ONL->Jxa$iSJoJaRt(XPvxHv zRF$VrJ7oRy1h#*R;d=-E+{gHDzlREV1lC(3TWXFmD(b! z=QKG0_?J6TC=>BHAKed}ix473-I?^`|K35`?bDk1!DhO-2wK#iaaY|xqR7|o z4EnuH;lbSG!Z!>2%~;F5$3^}9*Kz4pJ%9hm4mEND_uqawE&9JF@dw7~?xcf|KwOoH z%jcj_+w2XtWX6k+h9lT`)w^CaVX=-+)H`YS zl&h=jdDD@n5-J-`b#rpuoWkp>&nYS8!GUFwKiw|V8GOvPB+(>CR$e{`%G%k+^iBH4 z_Itrd9SSVu;&i6iFE7qyqaNG?@7@lqyFae>eYSINicMoWsH*(7-uTqu1O>A^VN z2Lxwy9SwJ<&XjoHV16N}c$vIjHgc=yXdJaX0Y`aprp7JLo|VFfe3u7rI_7|)(jIU? zdUNX2I6r^?PF5t((>A9+y|1N!l9!iPpKfz3pVL^NB4pxJ7vxJP3n%4f1bO|H&c~WqRcJyeLv#`_)&0c$hU&5O^ z>qZ^3mC}k6Mxz_E#-D~dzsZaK)!AUz)~0j2z55y7o2cyf`nNnhU0oFkc^Xd7$lDaE zU}MzU^JS#3DNBwx1#NSOikh6N53xUnHpj>*V{+qLmiZMc z(>`ZAvy;W9l5@Y#wR_hdLe1Z!`}*Sq`OS1Kl~bGR#a%x0Lp@pncd;M6rE&`94`R6T zcQ>`OvrEhyYnXoLKfS;C+tsa7C$0!crE7-gX&pM$j(2ro4Y(9+-|jM?Jg}`X-KMmX zH(lA;={?Z-5qlQQ$r6f*B;pF6ju2l`XKEzpY*A5>=x7uCNbM5dtrEjmx8|-2F?H{L zTvz!1qq>UMYOGz>-Du^tE}xwlxg0|FSrW8iiBg?KE1pzOu?KPnE9`hSWhbQObh4LC1|sd6fyholeeH{>OSNcXho;`{RhV#A_KpV9_eM56-#3Iw{sVKdCKW;;o3MO!9*OTN4j;~FEvpR5PTYF zIS4399ks!PNEkQP>QhbnKP8)~TRo9;)~5LT`|G!G`&+y<#up=WYsrL%3RN9Z@sj2G z^O(GIm1S$3+$mc_%X_wZGq=VfeFjd-Pc5dq76#Pr@9e7aZceq-8oeZx`{gOG&wjL4 zUmrHPj3s%hN|hQ0^w#{_3u|w1`tpChpjj6)E`RQE{2>eZ6otT`3X)}~8y(-t zmr*Zx{iri{?}_+uHi&B?IwbV3L7=5ktiUKQM& zd#2x3Q1#&ja4j08+cXA~6{zL*UPpAO*EL|eK9XY?0fS6CGT?zD8>KSQY&1S!e+u3a zYX=Ng=QOKQ>&PNj1t~aCe8%|FjI8cYf|lmy;&hMCy}>!Z2xz{}X&z0S{xF|&Sk!ys ztJ@U595D#(4c7GXz{u&gQ*I&W1eQ=S4lYm+JC!@Pv3v9jly4 zAw<}DGDTPg96Mg)B}U=V)vH%~>DMaiVmO}PzP$2m8TX=cA^30?6f4*@hs7mkZ`7xHIaet4 z#;YxlWIK+a9;cojt-7z|m3MP$m*zf+vDB$u0!q5k&5V~NNecICiy982ux_*Jz5oZ}WLcciU4mq}q)~ zNMs_32`QiC=6iD{JsUH;j!9>Jxm077=2pz@tK>>i#>^6sckK9_ZJ;iHk zj46usVG{A}4~aH%b9;qMMy^vU#oW6d<$k*&Q-)+u-K1e75bqhT?2*B}bmik2FI$t@ zf(g~`(|ZgHDoeUXR5KDIt7~IaY!fJgj}|gCD%+a_uY9=S)toRMv8(399nP2?@4V_* zvVVEwII$v;5KgxP?(iOstIx2bs!mq6xOGQOZ2Yx%g zGl5#5N>B=G$e*S>H9lC$8E=V{@#;RnM$;_@v%WoQ`Lqw(sLiM`PPJ zEYwK4Sdz8uUm7oN#00^EiW* zb5y@wJQcfqNs&m)L=h>x>att5yeO*`_~yz*T~#djCRp(Mpo|3HSWJSw4G1NxH!;Tn zLDYdK?K?l%98M6}WwGhwL=aOw=_>g0eN9Q~xPD%bF8Nc746J>AHN*lg*;>FZZ*nNz zglerttfiLCmsuqJ^GnX&IG>8z5>A*nS<>F4`ZW2-sisZ`EWNW(Yx$rcpWU%s@!U%O zjdIzYl{p>ru|8OcSj!{5j~$)ewZtlcEs3AJ{)+%>l?75E6{U#%?HWZn$Ar#crBbF+ zv$~&!rqFoYCimV15=eXn5Fj#ZwC))HQh`Z7ui>_C`Ai2Y+hpf1Utw}sa%Krktmt=S98S5Hf1yg5Y!s!GJqvFEJtC<`Dlws>PW=v+f(1|>i)x9 zvq|WO58ISR^Ji-YrIp0buCqu9FnqFa>%=>pTsEFJ=*1c{K?t)aiYXE3`Xhi_-Vte$ zv{^|oG;_x?DtlY7dm@)y43{$%lI^6Id?n0h04NUWT~by-K_O=l3954L)~4H7^#HPL zjRC+6uzM9moT=PkgTMF46H>3!j;AHLZz_uCZO>G_`_;*+qjP9taB zJlT0yq=bpnpy3Q3i^>R<+6VI1vEr#>gPxt+#YU%BY(IZg$oSl=To$ce-}}^0l2-#+ zi7t6@8rlhlsn6VD#Wcwq^0SLqo&GvspLqF32@p4xiKKDar6$s0FOa*lv%g#tj3wh& z+tt;@jA%K%C6<7l`-~~|SON?W9NrX`P2x&7u2$(QYrMY^Ds`Ek0KM`=Vqf6W+d%&=P!H}m)gr{O+MB)2=V!z&V^6CYYUiQoZiW%qPeBH{Ff9+P{@@}gCN zF=D9`SE%>&s|ko$OTW?1=;lnnn(uwCc^qKHdUOJ1q^lLXKKAaGxZUErcOboev3>ki%h*l%l%_u2d7~Q zcF{Z~&xvnlrYr(4b@NPtqwJ=O?4*tq#xFS_FXV+TgYsaSbnds;R_KuWyK}cU58ZO> zq#w(upm&GeE*$8FH@_g^eM&sQt@qSi!%Fu$b()<(_OZVJ z!byVZcJ;jdj_Lk4K-kp{fFRf8k#blmxiV+8S?9Wv)Z?vr-G}^LGbdaNTs>PCM&5HN zCrgfW&yN(Ib0E@I4=9_e?dbsr-m67RUPH8M(OtN3p@j3HCsofQ&=IQaJri7D=Z6Bmh zo!VjvGE=FnzOw^j(PMOEKC*g&!{Syu^xjJQmh zHgo5blpL9VJY4A9<25%-$~w?+Q+&C3RO&uITO0`DQr^oHuRVnHyNi+EAy2>W@>woo z$aimIaIM9Lj>kFY`(x3y>&&YoJ5P#ppVeCFVj5FBOjyAgx$SrJDRHH!!=T{auW{6r zSD%@f$J@bKbxQ=`Ao>#q{&SXT%7B3>N?AbemaCk;Blmxs<oYfLXIn# zjKi6Ao9P|{j*+3HYnIx0ZQhFaq~)|XW`QQ7P>31^3#{y^?u>=m)q>QC*CW91HS%dj zo9x`HLp%DXNaS6S5>$Fk5DyZHdvpFVyFBOnu@`m5>oW4?LALY*_$_fd7J-c8UIFD` zf7}MUCR$ZKhm#VLCq)b~!O5!yf{k!TJSK&^2BTHoH}>Jf;pT~=t*QggK;#lTZGvT;|^?#G+gJ+InyEBuumvG+Jz|(cNX;qCTIi`SJ>| zY{ZB&$|mJ5roo{(AB2l${5(a#;H#3`m@Q|PV3f{OQ}!=QG@{2Q%#yt$18chhy(rOf zfgVjpv?Ur6i9wo(@ucrDp6bY*yE}ya-z@vAbi2&2SFvgSLZO1o%iH3Jj&f85S>2gF zovl7H5g95jNrIBEgbF4KbEiHJMttU8rc7M^W&V5_>P`XWMC|RzW2O6Ze0~;iD$tC` zqhYazW^jf>3Y4wu;@z5h>*5Vr*9*?|XT9wCP{Q$yy%p})?9$>@di1B#6Klsta#}b8 zO|DEobsP@I=uiQ@4)CNZQ6DlvB@u?N>34fi%5p@dD9Oy)uE^kyN#ciSi zLb_e^7)1FckHr-?4G6vh=k|FLIB0!52Ao^tW@uYy#A4-LmR+;5!u?et$G95^+7l+9Ys^3Wc)Xg*mZ#;FoJl%RJki4i^JYC!AXcvTl3fzxnVr8lZ6A;L)}_Sem!SZu5`wXWk-eQnc&-;1)hO1h#9;WK zrzM;T+%@V`a|OHy$Gz?DOaAa>Xd1s__ED1#29TLAHJB?<8l_`7i02a!zkx9p-ol1G z3Eh5rbZqRb7{7{I&^INl)CWF^(vk_$R8EgRp&;tRzLv^s2xDC`I+dJ)p%cs5U&-0* zAB`EnTs7qOn?#WqVL899eYL@n`lAiBN2q+B)R_kg8_e~7ww5#Ojz97w zboutn1pv@mipqt*L<9{GWfWHFzOgGpruR}o691r6&Rfv>yf(y(cZ>-rKY&_xjE3S~ zqe@x9^(;1)Oejuh={)M>5P6IN!KO_#qb}06qj{m|F^LwBpO0MI>j0}xMPoj;GHCTo znpIN9Mg_WzA%F0&?}N~CJG(&==}eM_L|wmJtnY`u-1PG1VkS{u2^y*IZ<#J{QI46F zEFk4?CKj&(KpD>AYZTDtyrs3gdUR4xyt(=d234)!Opo)pE&fv>H)a^zS8^Wn zMiZ|>y6r%~qeqXzT$ zJ#DQ5l=M;w3cEyH9ljiHmp^y?-Xl^C<7VyMZQkB5rkIEwbh2pg{P9@FPg<_Ku3o*G z0vu@U-W}xz87yMH=a-o*Ztm-zsuIrhW1+nDu&Jju5|%U!T0ZFS@j2Y3`96-nNzVvuvPz@7h zJNB=#?Et||YpdhnPyPvkgDO{^zgfNXziqjg_41=DZ>oi2H#JMf^_8sd4?av7U9?yr zyx!+?uqhE6froBiTKuTis(rGTPN-c#8);aYViC(Y>L$73ddafrAi@A&GKLjJG&cF< zimBh(kA0-ye|>Y!ku%EZjEzftPll+IuvXGmNKQ^}@)*oxzm-tEvo(LnFKguE0fJc8 zLR=|5sv>~&XgOG%Ur|!$oNKKt)MXpqR2kj#<<(_2_?oO;WPMjpa)A3J;eI6W>iY@u zbOYqfT;Jjo6cae*20h57$-cPne1cvnSt7vpo&0Nue$hs-gk$jv7js17nl)>(kov!y zUPg5&K?h{=wh)Rd^mK{dRn!P^^i{o@hl4YKcuoaD|Hj1u&JN`Ln87_Z6d$coP{Eh;%Ci}rednd5z2Sn`L5 zI{!02rbu7T($n@oR!|w!(?&#-D<4W!A`E)9T*!(@tR7S$eOsp|faGw0)%SxP1#)LL z{f{juA_6|WWU5%?u2^Hx5$Hw{C%-KEY+t;Bb@eRfr@nwjBapbmOl-=)Hmg0j-VET_ zZP247lMDPB$1#YIKK6)yR5$j(A6qM8E=7{<%aFxUgFh%)93 z4e|U(x;;eiGLQWOSvy>M+q_kE3%I=n5qzX%2OpM?eo^E2q+IH^Cj)1n0 z2V}~L_s#i8{(MP&pal4wI&0G{${!Gkg!r>$Wo3sm8I*4?XR=Eor%ReVS>@~kE}dt+ z)9P`mvje4aq8%G}`M>rlnD7zrFOXu9tS$o_z(Q&D+J>Hq&Z51b7551jW_PJklw|(s zBDy+@JVVfEA;}H=1(W>Sfc5cYLjaBAHtRmWUM8Pi?&8I8X$Owc;tw7+TUDKI)1|xX za=*;yuHqf6{sf(UR;c60ytOOnC4ef@w^72wuosA+#)C%8alF)#3F-;*MeT zo`G?@!ER})%V4uw6LpsabXYgS9qn4?Prsbs27qMr?504M_+L_7iI;cm-=>+$Bov5t z6bO^(?*-!baoVJ~@bcx#-G#L88+ZhqB8Uzm54|zhiIeyw6(cQJjy@K{=+Qhq|ZmF|m;EqZa+)6xqED-B&FH-6c- zK;p}eUvICvl;YyxO>?zx6BplhF|e(qq}SQRSbspGk`tz-IrvF=u!*g_*`n;;QaOu* zWG&d&p9(1)^d8T7u1`0W*Z;9zQd;`fUY{kWmQTSK7sO@0xOC}~BG^9j!|wThwx6>L z1J)jBg?{Kyn`Vk_cI+q{2}37ogXZ@B7p~Jo#c}wIMu#;e^&Ta?cRj<;PUP+Q9MCqJ ztJ;hOHc=hPy=&hGnbhyuq0qX+dyi%`q&yRIYr8WsUcd_%CHbH#)1DbtFfZ*`0E6{_ zveLk;ob&C0Umcv4emgJEE;RM%UyR0+51J~4T~$xaaHK=>+4_``Tw=S;KHt9#TkmBY zwS`K8@TU7toH#L!hNLgaD@3@P7;Ch9^8ESw2=B>m*U9eaQg|0>yiD$7uS!?Q+jG?Y zCY{J{vDzX7)(f;_PV)>fyt`9(u5)WP&$qJfKD)ZVNVPrX|C4ue1S>WK`f zaK@I!?kM3cJQgW3zV6V2yB>!G0-`cA6*afD-Mt>`VjuQQLs{{& zPR!aj)7dJ>v0|G*MQ)M2lHzZmgT@#H> zZ)RWfow`0uz(={{nyCBf={`$} z5B`L*?%=&&8*OuJike8XUlsW%tY)vnS~nGX!o!Cv5s_4K6RqFg%%^{whNxK_6=K>` zz!cZ8VS8VWC5b7vty8K;)*~=xQ(w6__0yAms*T`WWLEu3rE{cuYUA_=vM^Qlq))T? z^oNymYhu!x4rntVyI9+o%QMWE=S;Nvn4w)ye{@q|{{wMe?%lSA=O)8vHr0o`%En#V z@>E4Omh?8|dpoQuvG%kXn+)`~DjgR)8r|S9*uC}B;aR~l)QHWtpOQ{Q(W{;xdEz_N zpymI=N;hIBcl~-rNcPre5*85Ym&N`VTerqv`8SWgWVJrP=FMk5K7AlPXX6%hEM2>! zLRPn4yD$b#_82{~dR6z&KmU9ZYt*0Q&V9a17eK6bmNLoSg7*`Zvnac68R53B0(}+j z83TA*jai1b#ZPfwKQzsAAA+b`7~wIHRZ+Rsu2~sdylHPQ&Jn$5KaJkN#?d);0^-zw z)NygfKH2ut2ZqTf_b&GusoISA7)f5T$>=Q_T$0motcMTS7*q>UJ?+cx%So(Wt#rO` zDTx71x)zsAbT2m(glhB2x7R;`*fQAc2o;Ya%;a*=7frEa$tSN2URGLJ((7h(AXw@K zl{d+#%5fQV9jmvLROUgxzHWAF+q!h@M0UmSJDSdq_iq^H%+4ZMG`8cTqyFQc-`4m6{dbufW_=pXgd$~tD zbU;P^$hD1!Q-nXi50aGh@>?va$B$OCcehs!f%$G=eu65rKqBniuDQDZ5FvJ7-=iCY zxEW5}AMdGpx#J@fQti>dwNRdgx?<`Re!gKt(XAE7E-&8AMj2mBBoKGcIz!7K_O)LQ zhpxa)40T6GLSoNdKw4*g1w3vQw1U$gpQ>-Ms#&klaQX>9jYGw?Mk7q93YpKB4by`$ zDM#{0RqoQ`jY)~TX>z_kswwNrl`HmmbkB*e{ztkjBfXY9QS+}j<0o)5@2~j2Ca+^` zygWM0K_eI0pxbP;r`88zI5oN6B2?FUQ%&wudN=wltGMW!J}U-bReNF6s9?#EC=hp3 z32_d8b$N~&SoHhw#(b|Sl2GJsahPJfBz2ie62zpod;&0I-r7E}pI)r$HTf2Sxgyeg z@{E*eu41sX$ug=p`;_v;m&;X93Pzf=D4Z>0Si1aTQvKk#SIyPIt?%T)&X3NaSRT>; zE1?%5p;yJ~##x68BFzI3;jQ8$ry8E8^$yS@A5KvPz?S-6t(O^FEC$47UI z3G}4@GbfxtPT1)dkdI{}Arz2X(ceqH6)}<+k)FCD`MihOHR&E&hiV@EyaiEbW;}dm zP5pYe-8#b$1jOGUrkK!~DiqH*oQV$lIqso?HD=Zc2h7ad-j|dKdnfaWzv@klBrc2aCC?wr&RPbk(v%#un69jV?{&6 z`GTP}THO9>_6A8IY(_~mC|zFJ*#cmZ;l^>jW?y-_9S6cT;RzW*P3R=Q&cPW{<9%TBNGiW5ZO7 zs;BCKpUeXfwbtlp`C&%3k~Z}SJ5HZ|D^OEu!#d5dd#TySy~H_Fb7L=|Q5<_YGWEGn z-=SyeNm~Ow&HR_KyFlyxi^XWlD)7yQn@vv)*@Mg#1I8>$IF@m1_S|KALZnSvudlgv z9aFiQ3+8XR#VxZxy|EQ%)*L%@cYbeqp15<>OR_| zdpT&v?LF#Tw+$s}M{?TU7^TQu9!8dEEYKp={?DADhU)HpV3J=C&JV`iP&#@sfG3fT zB(Oh9J1DydVC#L}Isy5?ddo;Z9W51kX2DSLa>s+j>3Zbw;lr1McGfnWO)W!gha8CbPjB0PsnMka6^+D%Bo4C9@*S7JZqEfmuAJq&uJssc*1E7Ae~ z?*S?HifN6AFulbw7Sv|qx^v<)! z=y=&fxv*HC1BIA5QmsW#U$!3)b+=4+Y~75#BG<2<#`$BUQMfHjnk~tn|9YE@&yRr} z49o4BGVS{SErg4k#5yzZir23bELY?&7AZOiyBc}zeQ&PLEe+EFo+3Uk0ssz(9n1H0 zBkIMrIKjGWQ@eVWkvI=|+QQh8NCN!?klQ;1dO2#_Nfg&^d)xz)jABLj_K@v%@GuSG zoTjzRPe9NYoS$8dud29`q6UUW8}(}A)zLJKHpe-{FH#WZa}Kb zl+3#zaXRblr;wd{5x@v%{eOk?n!Ztqf~@gB#A=GZ2lETN7FDy$JxDkwE5f#6P5}aC zi_oAfQ@0UlD?&q^iF$A*-ZW3itM9_hdDqb%&8#Uhe5VFl?+>j+mHFxNd{qOv!X4WP zC>SZew&J|Oa@zEeyun=YN{djNB{S#DD^q-SLrGm%R-_)Q;c?RMQ?996k*dPX_ZIR3 zHMvOHYw0YAsQQ!3yk!n&AOY*QK}Ne3YL~x_`H7sulnJpvwtZCn+0(>sfG_j2+4;c5 zPb@1VL_AG~k7+X;_w0$*4A)rE3oA@i;#Y2_tvags#*H^AJa(mUu`T5Bcvb{K8_Xsi zWORZD!fIyszVTY+e1pWGPQak`N(O>ZAsIn<3PEU&Vkpcc+k2Fpx^CAtRcmhF#B@2# z>L~XiK^Hx`Q@7zm85b8>GLe;=l=OQ;hFxS}!%Bu;g3)3Gucz-a`Hf!jkktS(wf6A1 z9P7m21%BI9QlG;SYSRc{xb5vLT`8h&!WwSckQT*B;JCZ(r7|37cy*%4OWRC^`kNYTf3`%7wfn0}v+ig_ePj|09OwDG?E!bHg%>fzMzq_Vvk})P z6ZaOl&8Wxa$bkO_E$}L&g+pB*Yl_*;DChlPxQLR`fmFhN0s!c-L z`jEFDX0K)E&vqzjHn$S-kSoz2dwxeoJLV`h|49lL-%@P79^WanWJ%G9Yb%27``{@n zSGPI5Lc?NA#9gc&py6~==Fn`$tSmi%1+r9_Fr}%BAyu3Hf>SZ(DRn(uhOWxf0}f-h zyjfWRZO*2mQ?l>$v>BP4sWxqGhT9)roVI-djLbZ48ytz2&LBW{VGPONhU^=^O{lmk_welW=RZrBm zocSoOVwz&E9woX(!MUf#AlvDTdsDn&sx~G14lN)%qI4K6xsX_Je9bJw?Ds*WTcJCd zT_c)ib7;34{-C!uPKcCU#SZSho8rzYTK8qwnl%B0`d%Bic!DtzW+H)~J!g`XkI@$* zqyJ1C#tw(VX7dYeNoszcL*U-Umu3md-U=$Kkz9Q_TES6F-Q+f=EzGgv1`pfU4v0yj z788baOJ(d68Qg}|F%-Fa+D_@U&fy`2`M+$8OHZ&jz|#oHlqBK7j|c6L+A)64x;fI{ zxYcYL3Bp3f1)9_lW;LpK?y@cG(|!`X&@+zZTR3CR^4mjg{1r@A+`c%y1Pt;Mne9V6-qTmYZDX_&OF~Az(}1k+b9{+ zo7E=va}7Y?N5=a5HcJ*SwmEz#X;qU@z#yD4hXVs(X3jcpC%Yr%-Io%fUaLBt`sJpx zwQSPSQ?QKxO3$C%2nZN7IvsG|t7J-h|D^-cS>jjciUixAMS`*i;Ho{iQ=mpky}Y@< zU}Z=^b_D~2Sf^%V#7xRZ=I3Z$G~VB{ECV>MvpFz(qu79oB#*a$hu%EcBr@BBk9}kk zi3BmD$FnEJR<3B~>|d^Tl!tca?h{Daij_p@x7-iULB0yFFIe*!u4H)w;hHcAF@=CP zA@=b|Bl1V*#y0{3=yqTIAm%-Bf{@Z~d*tNSNZzz%(Y8>VbVJRa%i{O2x{E$lDu~x2 zB$oikA3@|TpeVnZKsogx0U*!AcA?UxZX;}9^j)Vf#ZZlf(YuI=%nz3uUv_!FZU0~e73dmY^pZTue@mY ze3eB!c#S?XXU=c(Jn<@>7W)7fM0=h&*F+R(ppV$Mxao^t(b|nruFmH0#;8@8ONx%C84)ir^;L& z4SGu$dsgsU=lrWkJG+pD^0+m#22^6KhugoW9Souesp%aR;UWtb5WaZi$-cr1GiGiK zJ2qTWsV3@X9#r$=j`M_7fSFP~fh)VsAf))=T-t5WsEh#G%H3g{lZ{wk3&`B{G3vlv zy?;cZ8Z`$+=nVgKk03OJIYoc%&emL1VPPix%?wg7-~ii)GRMUI9uVWpE?{i2=LR5E zFiRvN%M{ZqRj1mr_Xx!91Vbm43-)l{(_(+s$a3>oxZSaw8Z^ zRotBj6i;?&e?O8uW9IC)X8tM`3pYlI+aCh`_snw$4;X)MlI?V7e(h#;<4Wu0p5I;( zna%(RY|kD7&9S7ZZ;>rbzcvjqi;LghSSWWZi&8;TH>rdeU$#^$RAE#8^l<4XxbDFw zPFFo9>Y3cFQ`?1NVdosa19%Xy5}x3sRRp7_{s|8rqp(*+9SFH6Z>=}^2~Gar2=S_8 z804^>{`e)L!4e=&xYrfv&QSh{7&^%Zkxf+wYGQQyQc9(Dwb#V^gJTvG%bI;3+B7%e zD@8u~;bFSQKZ^~fS@`UA@kFR@C28V`Ecrgd%Cd|4N28_B4XT?2XpKjtMStF2BTEP| zQUS^@W<+k8Ut%drZdh_vEP~V?%BTL`8v^EENr=syr?`^b?np8W{s^KqkBPR5c#bD( z8xdQ5%>eRT6Wkv!5?H|M{7yuepS-8~^>^3qC?u+k?+`$|8EVnu3K1oz(`6AV+f{6F z499Y(4?DNf=U<2oY3uVs#r7k@fQo6Zt39zCF0RNyPBSYBSlc}W$iIwKG|lhxyk<4z z0}-|5_Isp`rAv98h5g14+MCj{t+rt-#*?uFlMk-Ghc07_d>`iUeuHqa2KSebO~A&Qz0a@Q!+7@$<*8w21(b zi|^Hj$H1#T^q1Rb32x9Bn3E4mNVEdO@)cNlxwzhT)A<1d0C2ttnW$oY-&V0&qL$D@IZEipS=OAu#YFbbJjF!2|dJ?!{3NP5O7R+d_KwyyZ*b9dw&C0Do$Q zZKlN8P2gxUQ4t9$6y19E)9>@PwcUOo`QyfO$gYJ7jb{TzVK(fvMUwL?c zQ8I~&*NoK^z>$#YHy5EcZWYBJI&|*Yqs%Iki}JQrB^oZW+~SHW@1c^b+EahrVPi>E zWxpn{o<~ ziL?72p!0o4E-HnE`6TZ6CgLggz11J@m#TWY5^+wrSd+pTSt@J39}?LP)%`sYkADRB z3V)jD2!t5)G|~QDlsMgSYO>@Ce)izR*fegi-jc)GEGu6+cDnFlo;@67aS;AQMBiUxClD-l?@k3v)iCOrD zKsA-g8~Lb2Pp;Z0;&yZ1jVq6RhYUV;1-t7AA z%O}VS%s&=tUWdxImB^K+Q9z5_VvcT9g|^~~>`5fS%@B}$E2-ELD;!!l@HP79dnD944==Ej7hHVOgAZnw=A7Sg!OV~b;xn`c?mkT2c@AA0g1;yZ( zt;8ZRA!?9xeCIDB3R1{8ErqRxKGnsD-0(ZK0*9x>w|P*Y$}nHfobL;bcJu1hgj3B~ z;dGev{OqB!s=@rXX7x$r5+c86_Qb}Wtc}&}W)ls*51u<+Z74u*M~Qz)5p6`X=3KXK zGk79>Boi}9t~u7-+Yb>kOUnO`1Fa9Pj-n&e1O%Rm&}O1lp(VB*yUtF6{oA((>F*mK z_*`8aeJFGZX)7_2Zpt%@)Fr|fkt+(SQ5(kMq$1$#LBR+j5?Vl+R_!6`&z&`DMQ}na zA20I?n;+=$hCMCb|7ilTr->gyVtSa;cWCy0Rgao? zLgal&8)DzB<@Z$;aKrZ$?N1uRp+^J|HO?f3dxM?p#%e{B`BGpSPCowXlCL$qc!!SA zP4lec!m}SKIaHBb=aM<-t53z)y}!F58K*)Q-(IzwuOxvMUcXNIG|a0aw(Cc?*P=PI zt|8s#SPb1KJ8mOmgy?Y55$;w+gZnc+o%BJYWnnAyiNvGfSh*#WgPH>GXpmrlLiGbO zEBZ?TMl*|6G5%5?NZFDI2}sv>pF1AWrv0(bEx*}g$&Y8!$#*v`A7WJX?`F!88Mb^VyXCX*JVd zWVQFi$s{(D*KhJh`P5d(>vu ztXTx4p2m=rg>IEurxjsrH2~QBD>(5jV1NAyr8a-$Zm9M0^OV26G(aCErqa3Z(`Iaz zS%PO10h~ae7{{Ll1(`P_BCezPlk$4q6?^AuaVcc)ocbwb;+M_l=w!*o*z_G) zBSz2y8ANTvQ+xJ6O(DdRNTwr$ck6JmdLF2NO1vP3md!+Ivl zF-ex|H*S1CF?~l(R9H)uZLUeXDp8=@{y;j*|J3e2pw-P*HP6G01usnBu2I>$062QP z`mxTjQ@5?vk1>1SaJ@Ey&D&BbS+{HCZRwP(fERfmtsT)E6t@ZQ?9GdozIM2f+**~I znBPJz<;_gjBS(%Lj8NXz3^t@qxtngykd8o66-k?MVHts^_he6LswGVYz~7>0 zR=*Wv(s$JEhoxHJb|jf^Cn)>-{6+IBC}C6}K~JDjeis9T!?4G{S!B!UUr9mvMN~QS zLF|>-WQM6nA>>#gv^>fcRnaYQ<#EQ%kgcMYJ)aMj&u!sN(U##vXy;x3lsg^Y&9V;5 zMn8_<%KRs)I-Jwx^keNqT<8!Qt;L`YOJIxG#L5MhD9)^3v}&YvjowjBp{Z+bga)-s5KcIeqH%UFWz7QY3}%nY?bDerlC-4cdh^?A+@-O z5RI9-uP3K)0Ftnn*T|8ubFY7j;vTlK!5H(izv^Vyn%?CrD^cW#Yy)+9yFJ{IZ}*AV z9PMxuv3YTXfMv3Zx*!0h)$Ef+!|Bar(z6l?Oqt&@`I{uynF6UeY|US?Xwh5goW9?1 zLfb)Q-a;=t@|6v)<4gJ{(5}Tul?-CcA?oM9&DCJDiYa47j zdVqN2jk9EpGHh2MZJrt#Y@vV&7+1H?5O6+}<7L2!ur`qrABw}dq37A;;JQX$ETP>W-_hXtOmgUaIL_&=)r2(`D`{ zyE&9aQYfTeSyL50NJ16Eu0N zY-NE9d-hxLRnEIL&hxTP{`*Yj!^c(v86RgG><~4Cws`Q~@2m1Bd*VnHAU)5UK0f^F z3ir$>tGI-dP<(En)l};g^PlHL9A0;be8BVHd|lYHA-mPhS zBmO@hyYZNR(f|2?!2dnz&q4YB;y~=!qkb)1mD)}A=JraxhD1PelL zBlD7U!kpbE^8Em{+jf8RbB!rk^KI{d(SnBB-p5Xo^AT!CLX0|1kRbWG>H^Z!cml^E z>{Mqn$wvba8ybW`Fp*WeuhyK~OWx_WN6K{e?qDs|@Aq<8=-qJdJVRCXe*S@fJ4{xV z(zoON*Wg-eG|7AJSgVYzj~%3TutDwIxs#|3c!#!qPxkq>g0O1%+d;Z8K(+ttUcB*q z$v^K!=5K!f>vgAbPQoFYGT3WZb+p!8y_s3gM1p|F7mNk|g^0WSljUkOQap$6e?Ly| z-JHMq*(!2U9tRTb$nOIC^@Z#j=San%ELCRzElI-J1K^$&KGe_ zRw1t+kmG%wDH@5M`w@e#;xofq#K9JqkgnlBh`!*mc>W!&^hojS`T6@) z9X?hv93yGYKa+2V&DzrF#M|vNcl5VP;M33Fy@u=K3G96;svY;ttdf>JHzGciN#oJ1+Vj%~T}FpRwqpF;Dm7?emDgXZ)2@nbT%1RMZOr-s#_vc4d=7!tu&(uh*S z^FPw7z_!A2_bFVRFBZYg#KAB`wHVE=ARZflm0REr;vhUi#K*PIZyEH`O^?<5MTwn@ zD8gHx*C9hlsZM`;LkyZ`=x7p*{t05tNq?2G>-gFBLmC5I9>oyzMt0!A^;=V`!WhZs9Z%P zxX{KRjVN+4HNuu*cdQ`56m>GD8!@Ch0IOw6bT&LY(S%zb-MRCXQYXSZBF3M>&HPB! z$5Rswl2BanJ}VnVcl*BNh@qzbnr@$n7(bNfdOX$QmM7c26=UAP{O_o;_D1 z&8jxGiT((a7y}lvbrhP4x%1~Mf(&a^7T&>AiS*0!luqX{zTwQJzy^rXrPD@awvarYeC!;DzQM^tydQPtU?tZ8Q63wx zj%PC=6@qI^suL7 zxKyGvq`j1UBbnhstVF3yq!cw+hI|tfFxb7&S>VF5+kadb2hu$V2VUaqbI;*Nm515E zxA!yFRSbmDqly_=crnONdD>J48qg-T`?C^hM4q=AYAp=%5==c$Of|X9I6Od7o`(iY z{O`Y-z~Rq9UgHq&W-Wq2!(<9wUv-G58++*|+TRDo5I>tPJjgMOl6CRRp&Cc^5Ur;N z<*y=kzb6EP_46}$LkdsKHdO$2tsNogPI%;JG7qJp1s~9%v2=792ssBuU{3ST!H1BM2v(j&58` zjHIM3O&or%6JM_}p<~^`?-wi}nj=ueKAhfaHr!T1@=|oCG~n_r@W2WD*<;9ni|aqe`sbm<_x zf`*kVDBhmS?cXj)K3RCvmq~acd0*tf$1pe(LPIhHn4E5GI{7d-a(oC;P!yXX3d60g z_5!vlu>KrEB^C(@S(Z(K%eJC#=L==ffl4ypX4GX@2-XL@5|OM(;yR6KA`I-AR-~wp zkW5837w4Es!xWjx7pR7?&rc1Ej1WX(xgjaEI#LqJlAMT!qY+jR^VDS8cY&9*or-wR z2%RFnKhidY>h8PtK=4xvv7RapWHMU=wN!6>4;!+Y;1tZ)F5d3kvj{C8AJUP=r^*s3 z7`BsE942#J+9ccDgtjs0CgiE_syUNrUufKs;;{v`Y0^kRhxZ`OpYD%MXN7oKu>}hX z3k!#!g%%%CTanhx;78m>wP5Nt{<@+5xo8gk|NO-bU5*z%sp)xlv^lnk$SqzF0|6_V z?8UD^qK>rZB8O|EavdrPBY908g!U+gQ)j`Q9=Mo#psDRASy06eBr1l7?ReB!sF%Wy zMAX-x7)2SEh=D22ysQz4N(WWytQC52!kyBX4`&E$QBX(k+H)cChm20?3lkCgis(|bi4 zd$>IYCmF4*1t9ZvBTkDegViXI#<67Qw~Ra(urDhgu&RdOI<6^lxNgzZ;bxDqcWeC3 zjSJw)jDd_3jTM(y!Ge+>JPBJ!gc6X8Hcrx{*XYO|$gh)U7zd8>KkFKpa{41bZRCFk znE`u(sSo&xG~_$p4|q3xAP|`r`PUQWa|;d=em+Bf#kYNXVi9LC=y`=~>qH(pL^M;g ziU$udN8<-K8D}PyW#OZ#G>UT3etoK!Xl&TQMvQ83+9W6?_sE{TXb|aA4&UHbgW-+X zOiYOF6*2KJ{#pLdqY_(D6E!gl3pQM<*GDqu6>}&A?WU zgL6d`;$K_eLXVI%^G9+agFl1#rQ5+Kt4OZ71+iU&W^QV10ifLIdrA6THk<6DU^ZY4 zYN0`;e=34{^yuA5>~IL)9E~`Rfd}aBw&=F;h6nT$T^}B`*Kv`*Ev5&XAMU(d1GxHL znfCSdm8g7b1)4|>jBkJj`Ww&D*MGqgdb9`jwr+!V>ip~;7Sc(k*ENB>VQ&@+mcZ@# z=Is0C?A~>ymEx#mB<^-N1C<1{(?g{wftUgQ8RWGewgjMf5;S6TLO|fKgbK8G z0=I(LZ0sp3Ca0hHnk|&j^aV;QBAXMiXxBRqzz)%h*3vIi0!A|-$iHD+1(Qz z_`S&0DG>#QOOD-TXJN}5eKXYBkqJ;GUmOYdfG1_Wi{Wg8jpbJou9KhO z)L>#`L`iIfJPu!?7UI}3-GOb`DPc+W+8pxNhe{@cQy1@d67&b@Dp1r8ekNk1zngQ_ z<ke=Vi%r=&M38K#etPgrA7GUCU=I8GUxrf%@qk-U{6g&7 z(#i-d=Yp&Mb4{xNg3#vTAO1P?v4K()urWjKGX5eLzq1aLTIYGp z=kYv_*Y=Gu2tmD%NfSG4EhCfcv6icyR&((FJ(rQR^5{u5=qY7C-v+>I8UEmvRN;aA zIhFlV?#j#lSpx*L24?_mxw5oi>2f4MPSUmG3)zS69Ta-^E880YswfG=?m&^k!BbuF z0YkySIcDwe_b4)e*|kquu7k)*9|k~hk~q9SS^@l=c?4uLSFK}TfQmK6JmgHpzk=ki z1e9sAJ0N;3;5G|E-GO*{e?l$IVNZ=67ku%(euakK>H^@Nq4t5jO28He^%GB1@diKy z#=s>9oF3qbdaYi*n92vVD63NdR*2A+%!elcJ|BOEFk(6vnf9`Wtu zbPej~1G1CV0acnnFP|s?;0Ph>0QD+wgRpOD!LooT&R`4DCaKNNozXOQ-|EOVoB|b;H#weDtueN)otKgdIpIY1kwovUTM=45SWAfQfB-v%`uI~@f7fY&mF0(THG4;etHeIB&irLTZdG4+wUyL)|zSO8RPW(=0X3O*^o==6 z7j2-@Wq@s($+Oi2l#Uz#Px4S>5`>@dSbG4v<{XqJK{guD!K(K?9G5^x$siq6H8p1d z=@)S6HJv-Y0Dh$Y*E2xMPuID_2DzTA*Ei%L*ugWNeS^FViXYvNVyJkKGocV{1m$GZ zu46`2&fftFVj8eiTnOA5#XN`w3rAg`$_E@^4#p6J7-CL=tb7hIi6gsWm=i<}p)3QW z?<_AthyXPoD0tT|c)9{>CkOn&NPZDN1gbGF7KSBVHhC!4Ns5y9JtR(BzB$TlLP>T*=zp4-^M;o&r zf{&nJyaVd(%(A+|f_Fl$Hug_I1nXkDGJP-p;BuvZOe2VbK3_nlFkQAB5I{{PL5c$r zcdc{Y4^{&LJzxl~W@j@xBAFjPSHk=^2E3j0e_oTu6Sk`gQqBaxc;dKI`7BlO zo|V3Y5@OJ%_pt-m)o>6>FK?VcwN{8txDDVpfMIwM0dz&}#0jvUK%yC?>Xf%G4kD}X zgLIsLsR6ZZ#jLvm<)jp_Nu>1!O~(`}COz#SX-(wb1o^YF)pNQ#Y2Qvj6sc}?0c!40 zV=;3%M9T*bfiM7*APORLLnu;2-!#~n68qWaf}(w(Jof+fxn!)9y#f zZ`kM;u?_eEst@8~!6;3iRYZ7Lm@z2#Kid%z_t*;slR(U=J;AmsrfBMIje`L39k&k% z^PC)ifqTOZ~E3{-v$2q zhUH$+z5-ezsR8sWk1zl-%!-vl09e<+N}KfW+P-3bv;heXP749_JfW3&MT z`pLs*=jo>DHAVc-RKXFT5Jk1>0PGsXA`&jXdd+c;xaqNE^kRZayn#E|8Sq6<_W$@! zryW!pdNQvJ(*7~PU>KT*+72OJX(HfK0%9}}l{2uu2#5)_yuS|s(9C*Z4bAxNz@UU} zFoKI4%3TW#)N0A_Ff5gM!LKT)oaHqeB|YKtog zf!dD50JRM2|J^HrNCS`-v_hQhBVZ;+g|{F&7{~<$AQJWpnERg$8g98kgAr@41BfG5 z5#~LKM4+TlXb+nCoS>>=6a=J!RU?B&se)E8my~5l`5l1sZ3hzv-aA0kK=qf8A%-i| zkVy#=D2W5mC)qLb+>Qnyv7Sl*P*2%~@I5H+gv=$xF>M2*gl~Wf9p|pK5TN1C1AD26 z0MIPd_E7@B*>=!3N6w~E&?yAr3J~W6%8hJ5RnP$#&II@00Jjqb{Eie_l@t+vsGAdN zbcgty6w45#QvxEPR!C(X&H#-eJW}9iPS0o#2SoCK7^U-LfUrsi&9=_X_U#416!cnm zXes~&4jD%TNB~~c61=zzhJ^s@*8veYAj=L-6yXNh8dPJ=bs__}DgYnDA$A?(o+O|^ zW&qE7vwj3{pyIgoJabiN&cuwkG@UtazafpAjDcS7xJee1v(yc_)u0L`lzCeUNj+~`XY73>AI z9jJljU#vc8TmTrckp1Hfz#oi6DxJFXpwbcrSVVWAS!H`AkVS^N5WwOO00!TGmG?0ZtP^BPsxKC}qILWr7;Z00=su(J&B&5u&0%cy{6tNYsy} z!HWwDiXhhaC&?+x?34^p1J@qCDsV1vf?!foIzH{*U)=;CMd{gfZJ&z2PU-QRKZwOR zR1UYkQcYuseLDqYPH6RG%p8C(z8_73^wy%gQPJdFA&ypjri#Vt8HvS*B7E z-&coPAg+ctz|TRMwhhoSX#gcrXA)FIKv~AMPYiU78$g^Ss67Dc^vDDkxLMx|N*Ywq zWDkh#3#Dq<2AKfmfpQsgKq;ek>fs&$J(7-n5EoC^dEntT=vT{bwgyo46htwr1Y*^V zmjnua)N(W~>JY*6HLe@uEkF%50VJwzP>h7yCDfMOfo>^APz&>1;Z4OrH`gKJ;(Yab zQ8=iJ$${9PHMtC;9zcX^2=_dZPXKiQ-;;R&5I}VDSeHm>ouz)!gYZ7#;lDDR==i z2Vo2S6!=}po%aA$2WIw)bqC!(K%t31edjf(AlUU=L%0aYj|u@l+sX@^9D)EqU)4h! z5Nd3L_W;}$0P?wCe*jg(=Q|Jxs2U!2lMRYL{;7roq?trJL0B7z## zAT~gLW5X@rT|$AxLtFzWi7Nr@0T!T%;I%KZP{j#M(1b>eDCjwX3qxmx`VC}cASoM5 zXNF%<%v=K48$ji|i{Xz`UPA>DTh~s^gFh?%;~EI*ViuW0z)I&>{cqlj13*e2Uh!T4 z1D6|+71Uau-G2$qZgBu_5s;IS0keT2c0IUmIAswww5REgfb+up$KF5?_y73(Lm~*` ztH6FRK~+pB_PnB{Kul1GSkn$xXAZ=uqZ{VrG^0zq~ZYHAqEDBK)MR;2vD(M`)AOt!`9{g zVFtmDW&Y=9so&~aS6m1gKM zVd)w%i!pY84Q~D0|M~fe=S{$YRDVo%6?H?0{QK8uS)<2SNb0|T`|5w61^(~sLI@E1 zKYl&B@t=dP7Un+-@mD_nvk?DTh<`T~|B;CQNW_06;@?-`KN9ip>+v5O@gIr!k3{_Y z3j9YR{(U|EV}BPY1Vv`)~1g}c^ir6W!Pxg z$VARXK^4Jm_^N3P!|3gOX>DouJGQ+>j=e*|ZkjIMzBd@ZZ#oA>4e_2{^o}$B)^OSX zUH|NBp4~4G4u(KN>FfXZzdmL>2^IOj|M>rPuth=*?CN*ex^nFaFUQ<;9FLjuX$&@} zblZua)%((Iwvg0ocP-Lc=><@UmA_zrdOXQ19A=<|kmvAK)8pN=vbbyVx>MnBbxKW$ z21}mfQgxKcrb76eR>p8<|3OOB;b8UdiqH7s;pUj;r|5qk#q{CI?`!qvwyTTQBQUxP z`|+@{TVS>#7<@R`J$<5K*q6E!mz_)thrw4ERu{icVNd#a9)EkFQE%ssv$B<-fAN6; z<=3wlY`uA^7N^P`o|fJ-Xgl2vr@wY$1OB-PTQim&I-MjROO;o6_ba=NO?H#ltka_^ zQL`oT4GTL6TdU^Bk2iVZbG&?h3P#poeoT{xTbtueO+C|WCw?2ZX@ZGiKS&p+d$F%a zGqzR|Um}w)_u;oF%~qDnPz&>m1WAwXc%Hcby~J!yGGpPVCkC;)Q-&_{y+v$b6}}y| zc47g7;GG16(b8+{V&9=9q<+8FHoGd(wJ5q?{W(9i{Rzqb=5J4w7^1X=9nHe=0NJGM zh&wvs*tXNFaL&RlI!`IrEUV^|kiey1KB+{L%={It8G6iSkl3c-U1rnBe#lDJA zwB*NZHn94!DaX#&&qHa!Pr1U6%dT^;pnw#=5Z?uZ(F#x2`91pz-}4V9i9E5%|18u^ z#k)_2e`A2x3Y6XqA(W1fQ+?xbw{S`7NHOx~d8iSrkkn9HDjTiGY#T9*t^N#!mY{zU z_sf^ib)tWX=0bE9xFPEwv*x2koF=cYGngK;T%$-Bw89T;_>uDqp@pYUnP6u~p;gXa zAVR8KN4jq&zV_Iopd_h(k!zsR>!0%wXWz)N8cITrM2;r>A07;s&yXA=LKdwsIOu+U zfd6pQ$KjSD!Iqa?ncp^~%DYhFPj;|?HmIWREzmIpz6o<+rEXode*%O$2`$MsiYGo4 zJoMX+2e34vH|T`d2qY7Ft5fw`3~PS9w*ECd_IuB7*y7vs{uhpY3ppD*HIzEv)QMF& zh@x8P%$e^bJpZCooA>EM#GPAzH(k}au1+egr?Ik#BfX}^YY|e!SW_M;);kxzV)%i# znOH_w{r1=W^=kYIpSL0R6`RlqIynL91c+&@3Ok0_@y9*KGK$L0>YX^S@K<{NXdneC zYu`-vR=0lGwoL<3{^xQF+o+AZsL_Oki#*6XRR!y-6nGr@rL#BjI5rB)4S42ve18hw zt2z6v=kkIT>u$oM_Z?fo%eNDryRhp|Xy~;Q|7)x6R=oa97DH6Lu!E7087ZJbp(q0` z+u1E8#ngfZQ=K84kJcVF;m3Xddw9iFFyq%#%q8O0Zb`mjwi+)@adEq^KW|s@Dfi;Z zbBvaW*c4j++3o#ILKkrRH|L#&LjM>C5-El?uo@a4W8XCSrxRellHencfLlLcJE+Qw zuhF{0?52;&BIC^;wv%jAH@bp%BSZPGnPby4phNaQ*jc}_Y;WRNQZSTLZ(RQP0dt!~ z0XXMm7OQyHu1HW$vBY+zS6$7o|8nRgmI1mOLC?3M2Y;@fp)T8qQ^rusMI{}hjc{`v zom~@Ib|lMRi0xg!kx_plGgu3*PU-SrPhr{`-ZJDJ-R(BwFZ}~ZQ^BL!H%<2T7%fc{ zLb#+GMF!!@n&;3t7JAoV;4Wn4CGI`n5~zC({r1Ps#6=F+UxTcxz!ZaZ*8PN_UI0!@ z6!m;_?jM!A5}*a+va&`MAJnrIIVpDVR`-9m~Rl z#X?IX5f*}ppzq2`s1brrsy)jhlk)d6@l7}=oDu_`#ARKrj$}dqe(~{SAr!COB)`6| zlXw38#E(e(z6T=TqMawc&W^8Pxw_SV{^~Mj5sI8X6i3*zR!lyG4(!C1y;^K&`%0DC zi5GJW?L$n=S;bEY|Kobi zX`wUq^=P44B;9&9po0pY{QSGboeHJnOO17+2lw2#JFy)!AK(Cgq`hhU23s&iVJ1Id zd3$*CyCFa2xJ$!iU6oQ+_1=?H{qdZa|J+w}%;w9PnrXdF)i-f-2NP|Qft~NtFn#tI zXIkhmv^0Ev5%^Y8->K$%`ri%IWeJk=xAiwDJa1kDO%qKg?y(ilOmzZg@Va>IAE=ZP-Umi@Jq&Um>|a0$>l5NHx<)wXzeD1ct^-%cS7 zPJxIzMMe(Im#%wxYhqS-6tuBWYQOOD*cbWnZXX8YV5D!}zMd_F`;b_OE_8N$mEMfl z^4Ww4Rp{(88t;v2m4Eg{`g+yv1!7&FDbZkNIhAikyLtSc{T917GA&Hw+`EI<4FwZX zV6|`Vhbb~PWzU9(aFgtsaB)T2vldxB=N*7WDIKbITJu|qPP*W+B^+8MB=0W9+FO(R=9%v)a zZIC%a8e~z$v+x^6Q30=C%nq)W^cp{S_7>rCpOEW)s1eUQsYFG_W(OFoMgucRBOJf1 zeB-ww12`U^1}#K6gejBaRQDO&(}W5iz4mDPh_H4wc22-Ykk?;ReK>yKY4??1t|>Li zYoW7MK4J>mTqO*hNDPelAhrAlaZ`RC0Yt8aMK+796$-Y~L+@~VE)|m!(NU>wvvA(|E2ezQ!ntbew$XL0~mn8PQpZ&&j*HD%j8G1fNBeT@{ zd2zuCR2 zXg}+qgrHeilU(X6tEm~klV+e#!EhEa>?TdGsPK9b+TnlfzU_$>)Vg&y4i)`=!Sl?g zXHk@tu&=;xqqe5cy(KbqR4w35CF?G;qcdbdy1CxIg+bBR7k-LL@6%rR9Ea2GPM8d= z_;ArfIjhwhA6}JeE0CUx^?4%UYk^GNwnGUXRtQS`O~gPHe&edTk1Z1;EgDWo!T`&N zw`a{*Jy{HW=X$w&GFIY(O>`EUu#mmC+K@x9pT@ZDn>;94b6Ly1!S@`$_k^OHVFbX> z2L^W>q19H$CZ{4n3$g!CTm3L>*QOxcIOuV1ux>-4WO^v!2!#FYQ z4$(Xh%n$tfsTEBdi$@z)mXJ~hIO!1bj_~axT-90_jLVuLv7BAko7iTS z8T}18JC-P^=z#cv^!A(<*@SD)WEVvifS{pq;|7yl&u()#ei=YYrE8{674BJp^U81b z5iv>0?dH4qvwQ@w&+Fy(E85OKm0s8xV1IbQXTMes{#aB{I=G~e_?B1D_VooK&v#=I zHHki5tbY)m99-a7UtZ!xWipnJkc;6gN9$6;5Q)T)jrSi)e^~w$fl)?_fBp@=#E}Hl zL=C9Pu?cx36OsHbS1E2@goyL_W6o=PF@uUb6njB)oO(mDgJOFH7y>jnA zbdBBJHO z^yB%j{^BVOqct1HrD#Ysu+cO<9BrZG?}R ziBy>1_S(39#{p3l{jM;$fD&>1&V}IY1BE= zn4zP9Q`T;CMSEe3$l{ds`iX?&jNGDIe?{Mz5K5mIajOYDUR(XkObvoe4TM!p-O84> zLP4;N7#G3WE6?uZp z`{gDaF>*XKnwI8|{Ec~n%`ZPUe$=b-?vDFeze_5*EVG4 z#5r2RM*~*TiFGOECGhUGG&Zgo4X({_3#*rZXp1G{NsKxnQf?zs1T#2g^W`GjutJ(t zM@k8$S*O1p?SpXXnzj9wamOd2);ZpBYo!2R4IARA+} z$~xWT2f?JoP5e2!Q{{Y)t(Yn8GY=1HlCCV?JYkpW=%5@YX$yQ?=RTa6hUPuF&~;4?1mPlJrWSQFr)n2M>N0vfio+?pEh@QNu11qhL0Wio?%x)n^HEe3b?e~{n8 z5rQKK?y5unxb#e^R4Zfrl56h$$j zDQFLSmF6z0?slDLblbE&r_}lUE7g}Jl>WL9@y`6fF-3A@AmLI1ZFl(aYUJ3W^K9?? z_M10z>Op4sdGuM(Jptx=i5~&;vwj;VcNqda&6$y-k7=UUy{ZpyD9cGlcat4p-3fUm zAvT-RR`LY4jaytP0jqTJ0G>|Ey2L1EIO=7%ZI}DgRgXDig>~~&vA7@g%iU?pvk7V@ z@Ik?F8K#NhSa?reQ3aM}9a9Usnu^CbZ!;=>4kWz()B`J@t$|6#IlMs-A4 z?nmP~b2f91(LAeGI6mTgpP_GKxW5*9GQGF>by&FvTE%!uWo9+Xy-$-O(@w|A<{%|` zX@+`=wyRz2221a`?u!1){ue%mhElle$?sLk4r&^0Fh&Z|4mOA` zlUmO4+q`E+p8HSOkzGp05u*jCxy$0V)Qyc4l7TmqACws*b7H|XhDE=hxp!v}zBdkk zhaZOUTXW!!e=pZZDbc??l3N{}bS6ROY07gpjQOKA_W3Z?d3E5==xtRar&aHkYy&ZubkiK3Pl2i~bP5&6nJ7&Kgl>2IShfHo`6W8iVoX zd`8A9o!wcliEX1Jf2fTkIHhcGwkpxI8uY8h6wpzGoG{=Q?j^row=Iko=MncYkLqb& zU{Qtn8~Od7@GrOaPdh^w^40e_ch)Hg8Ld0uy4+hw&05qt zOL&-e;F_MVILYix7W9-wPeoVB^4umZJNoCLSCF*vggr8yTsQ5pHJkl>WMhTx+SjCR zH^xPCNsI;GP-dUOlKR!ms=-P~!ce{x3SiAuK5FrRsW|`PIgK^^%|` z*nVq8EXsqi)0Cq5BT$a)_9!=((Jl<%%P)46ZUm1IPpij%p_P@c+c{_>BY%DKju_xW zoOaZXNs>q#uwdw$k;iLf-v8PXm_(^pSuzkeRCz`+bzQ>8;)#npT^e%!62qLER_`Qf zDr+>JT_&aby3itG=8I&olRryvi+m6BFk-Y~RciyO<)D^HC7M?t9NDe;6LadPcEDG+ z*92|M$GATHd?BZ@OGHYXG!Lqa+a2zQo=l{QgiyTepG(uP+syL&`Sua& zig?17?6?00s&Y<0Pm0Py9+=B5OxK$FOpn=298)K z7bTdc2@Q6q-Se_?uXu&{1z&9=#RA!n`66Tk1my0yjuyw^T}bc^10-iqn1M`G>=$#5 zPrz`bXid?RN#$sj`HeFc478y{#I?6?yT*+8D1RiTXsy3&)^Q^lqdwVTZq}b1z06qK zIXG%}aIU8FUl^32jL|SCC3Mm6G-RD?QEv@-X9261(Rwzvg^HH{Af^y|UF@+%>9c^Y zS;4G9iO*$4I5lf++ljit9VPEjl2hsKzk2>tb9^(WhF<1;lUId4S?9gxmzjH1MZiOj zkP*z!s1pPSlmRKk$UlAc=9ljJO(=~8-$pE0SB$zEtF(x8Et0KoN6o(;Ac^i4P1&!^ z_3pTK8Q+Uc#O*U-UF%TdgZbWyfX8n=4U^B8%df)#XmifjqQ>ok?%N68VpLh@N6)Me z(g&9O>94PBKb~1pl6`yygxv;aY zQ(>5reVo-063u;fwh@!pO|IN_b6xzouv3-BGsZS2p=FvjDMM_h5<;F=XX5vgrUK2L z^9yKxZLutDG8pr9IKxca+jKfQ+pT)~bZ21)X^e(M%9isowLCNv;rorp7UavBMrX!WxW40K z!TFQNZZAz^dXPt>AM~l89O)~*xJA@qe%iNiesaCf2zc#t6waTINxgDh@I-pren+G# zGa=h9%EUfWuR{ZAZ3#BJ#a@)jXOrrhI`;L~8Bf+G@l7p$ySjN}SF)omq{=o8xKr?ka3Q`*e((dcaSGRa#NzcGGuy1D}_c>?qv7!>rF3i(}* z4!+i8$Cjd1X+_5Q?`kvKYZY*kHc9TMwOLzJZAZ0NBzL>>Ep+%I2u%1s5LT3SIFX^e#NSDQ|d=gk~L?JEcY*p-Aurqg9@GYJ3c%kST;PpBk$T7gs6-II zBm^ZUN3QK+zw@sh+X@l20YOA{s&rWN6q;Mg?hr`EOLJkg({g=cz6JUrd#Ffj%EEZ=+j%%;mZX z_sq)EZ50znWW`sM$B*0C=wLm4Xw#QI2Y$!Wbx2fXw*hNNHmPwREw|Eh*3XwnBZonh z@`g47+E{~n&hC%);^K{f9RKokJg{ZXZNC}EIDI>5njJ$oW|0hm23j2&$IAxS*#i>} z_h5EjjP`9ADA_|pjR>EQUS5$Au*+>aR28P9R$m zO#f6zs$mCJ%TNGq%SRAtcZmV^WegowdvpuArWqaskCmpU$EYSyz>Ii=jnO8FbNUmw zam;$ouZ~}~j)i=vD^Fi8TEBfuG}9r8QDH;WSNJLJuu=NqH`CUx$Ek2>8_D4vkGB5( zAviV}es47J`(>6g!@Q#nS(D(V(Rr=Ia{EQZ})(%0z%pv{Bi{M->;!C(G zxrsLF63%{SCBd|WsK0Xs1xnb$Cs{I|pOW_|#x563VZP(fvORP!=6UWH!0eyz4%P?F zgLuT#S!?*94C|wi7|(16BZ-fV%X6e#()*{ygpe-1U)qi%$rR$_6z>&5o0-ui@2Dm6 zvUtGwQvZgZ7^VD7T2!QIN~2Lpsv&inh*+Rth30(m$dV8Ry@n*FicgLM|yj(rmBs}b4=M2Xe6LQR6@WY=T7Vicl%2HH5 zZZFO=3Xwdy^lC-5iw+Q^@Bymy*gEbXRa*WOaMZra-70i!{TAGSfsn*2Hmi-J(|g}P zTjVxWki+jcildbs{YFw}V7q)Ca~2E2?C^$LOonvl+aApXw`Ztq+F$07Ns39n{@5%b zO8I!Mn^|^yE}4XG;s?Ss`KK)RaR3>nV zINXh`LltAUbDd%D*o^~QJ>6Xs}znUdMe#Her6Bt1;;m2bUI^|UmYQ2oV0byPq3ZJ>S`pxHspa>}=| z?C?u28Rs9PrIy8lHGX zR#WQ02KM$KW{$VZW^Np4``&s5oU&0XfB3r2SHHVYJ?~%F7S6dzA4r%HCK%T$xtFSr z`^13#jw!M)VA+AAR4m{PS(4(N?XFa{qy4?gHQ0W*`_Nr4lS(p4YN{u-NR4nZbT9Xr z$8S0YJaF5=sKYl8Bu zx)4iMt+E}MT5U=9M(8HioLwJHhm?}H9L?QGnPA`zdpj|5IO9ptz1%+1ou1FxL}qT2 zOAI(Ea7%fYOn+CJ5gfvvWe5kbdfMZN8Ju%-C+sH%5hUnhVv>=FG|AVmWws|?IdYY- zOZ@nGEo(yPX=;c3JCj;Me`jpEE#Y^XejXZe{1_*wi&oQ4`zvRB7w#*y#(pQppgwt~ zcB$R!xDs)D++9un%w*c~Z2E_VGRj&w77Rb=?5MM9UP!%lJ%uh3Zw}KWAPoFKd@%`( z6~_qE!1zSQIl4x2vv+M){yuG50Ik(a%x8l9IuCf~WCHVKMZb*K#=99p)r_HxiEybMKbe_AMTHfK+Zk zZn!wD3wHR6SsB!Mz>SF!h#e-6)yHPsW5Eo#k1ML-!NN#O*0QO{a09h|JMs+mM_KM? zpBu+_bwA1TWB%)R*ojko>i5~yhkAq0!qQ=W$~icyL(XgDS<%~Ewktnj)qc@j2Ym6pCtnWvqqP%p+Kpj3LV9_ zyHeiV%7pfpi3D^WA0)F&HHc&4+vEiK*$~~!JVG2!5oFD2O7jmm50-DH*AHGT+yfKP z+;#RDX=@!XSzK9pq{KvJ>{nQV=dJg-BR#=gx9Rpqj(khr?^s zDXW?^={_YEBK|59hO&~cXiB4l+tAw;VrBMd1?^va0C!0AO<3!=aO|d*Ri~XceHfQEdievd!l?XUM zE;x!6TuFdBx#|2I^X;4ydGX`YuumkF#aj;ni{|@XsdHn-587uZXJzCZbNdrc^XAB& zvDIWpFcX0U?phGqCrvD^!H)fu8Q8jSz`7WUA!dH!`q?4&Zr*biqnlDEzU+4*u6u~> zO%a$=7PbFf2(j8Rp05DB`MNe)@&b~7fL8U2iP!1C_M6MZH99o6ld3% zjFqs2l6px^#V+ruWBKh@eZI>r>dqQnP|3Y`kt5?TmJmvq(4Kv&Ja6-s=@4s|x}L|r zh~@rTLsDwvxzq7AHE|wEM$szlzj8#pm$C&L8memGh(Xz`uYrGE?;Nzi+g3ILR1X zL^iLL^q#m+F3W=C+lO4NvC@a`;^i@{3u~hd`c>0<9MDq^521m{L&tH}aPp%rl`#E} ziVC12mqTXln5Kn$)OX3Q{N}_2fdcxur>zgS64G@45*!@6j)97ke`l-|V1!Kz@oRkP z-}Zu&Sb7~b!g|T>B9H5za$cPGz@Jv^8`P-@X=Hq!Tx`Zdt#|ag!T45;IaTjA+6+#I zELR%)Ma$Cp2cddzvkaSg0Xu8QZjQ?#Q>kIs)MV_=sP2ZWF1>h%V{VB6V5))}t~UCj zN`5cc(bG%;hTJk|;-XNTqeI{_B=?@{ulJppsfWC~JbmXd$t(0c*2GpQx^bG0I#95L z=deb(2J6%+i&%tcld0<$=QcizB4iP>-5qja`kxf# zS=weZF){6{AH|%4dfmn9iv819KYWGxLL?&=$SU_56IEOtZ~pa4S<8rX&ZV0^Y*JbW zghRkWOdm=h#b79~#hR2ALCt$VezvclDRWPE7OjGcFCiQ(nWM@B- z%uvuYS>>3QSZH>KGMRUcssQ}`i-g0I;l~yu%So}nF?F^qITQ_iL|SG)G(8B-(li_C zvMTXCv(Yy?P(h?fb5XRRp@qEZVN5*kcTQX5+6AYXT~TsqZ#p41njG~_zilfmtp4t| zh1-w2MS>J}ABUa*Mb_AGO9*%Tx@8p^vb?R7y7>O}HRMWv$*qKy!RQ&)%CB43lpcoj zR;d6qXSS*3A(pD70I#2#Q;U1uL4n&BQxZ_W%??43k# zt(h5Ey^|nGmnbKa>BT6+VUrSsAqf9yKF=-ycmUH%WXnz0qSHY2H^qf0wWXsstyMSG zXd#(?2IZ!ILqiH}$NbdX_xCukDQ9d0TA}PUc&>~m^$x*HSf?<#L8O#O5E*KMEc279 z3Aggzhb^T96D-1{*fVdB#iWtfg%|>SX~ht^rVe)YrZ8ZbFk7~pG5p9JQ}AGir0~ZI ze@o>regrR?@d{!li)q)EF5HSk^*L&p3uy=My+9t#2icmKVSv{3x!CN%+r*7@ms&@z z`tT3%*+a6^{bCf3!`Jk?6-HdM~h3y}(Rl`OI1OX5GecUtfjzkM!>F z#LM}eyn3yL&3S{tkcaNIB&Il-DBr54 z8>6c@qyn;nRG{>|{K<)6Rr~doeP$wjz~J$@hw?y@1s|1x4$&=#A6r9T-X80(^14uYpgeq z(K;@l*0SZde6QosPccFwLk1OxjEX_3$G_GUb}Y{-%q4HFo}HxeNXtH(*@D2z!>Gl5 zTGfnIbXsa5bf2JuMK>Yqo1^dg7|9Lf*;85#rG&{W`B|X-bpAo`!Ov2squEJF)Zmr5 z00#=7$}v8WBUB=F^YVkuY|ok?+i|GMR6AV+V98opqcMSm`(L{CU|?_AaL{J3voR+& zl(n?(F%dh6@k!#PNk2MPs~6sId8UW7ds#=DV%Dwbdzht$>_%6qbQY<6A)P}N`RUH( z!Rb3J+|~uo_rG=S6}djkY_fRuJYP<%B~y~L3Ao`v_f^cVTkj~_$~UblO~2h{ZoKIR zb1(U^8o9#tyx)-PgN~iw+ZTq$S|E)e?=xYFYpi-iZhq_jIM-OfeY*BJZoZ@*_k@R* z4XiSEzoy{j6QT9`-XRj9U390$J^8$qF>hKj0LOj6bH7lOhGY2Dq--G2HRjqr21(wY z2RQ{PMlcHV_0S#aPtjP(_?R~zY|y<+Yoc0DG8$4KkYUcs9_6ZRQVVd=pdz=h7o%<@ znbICF`q-Iq5Q$=6UIX5nsZ36ev7Esm#wU1>O6L5F9sbr!tAyk@7gman%g0@AvfMCN z;Yls?k`D@Yu>1i-)_09Qmtq(%4)*=wHwgMv=HvMBN+*hCl5D>Pl$-D3b|g6Quuq#! zXx%Gv)co+vln7yt5^5$CSjpE;vYqggF3WzWZP^7DwmcI)+K~AA#ft=Uwao)@I$I{* zBFAT$Q-(6EF-=nJud;6xiNa>GNf;|f^zhJPP7hUCMS`o+X{#dq-Ud3$<$+>Oesj5h z)ZI-lJ5Yw4ah9nfaSi|O-%axp?(A`!hhk5r1^!boq_PKZ3^1nXqVVSOeePLhz5_5pXe;mpDGCzW7t=K?jZ#%dS8uG_&95~l^^pV*iC z84OPka4x^k!rmu=$=+Nn!Ll8NjrbxMzSDu>^lT)DJ z(~$-QXNeur+aE6)SM3ntFCSK`=I0TwXH`?$*PsO4KWo`F7OEa)!)Q9#e?9v_RXGLq zQ?`F~^;vhu7lWG_u@k+)Mth;@GWf6 zt*l@PIQ{Ewl46{jhr7GRG;Bd*Thj!S9&Cnhw-AhXJ`KG-)s(|bAAl|QfmaiE(q)EE z2nM5Y)Zl8>XoyV#*s(&7BlHAmG{boz?bTqjwB&cXmB1<35_)RuSyM^J6LC~+(y?&3 zU&76;bn?@B@vFh>>YBLG?Knf??J51^^Cn}WL;p`q+n3kw+azQ>ExBi4;R_E6Q6Nnl zMOk!0wyhiVu$;dM{|H>{v4zczhOIo-Z6PX}YxuOW4Vy5((@m0M@x9|nMi#8XgJzEf zcSBXPX17ApoMO!>Gj8!qvHs$@*^JGXhP!4CgE6M6VwkiN?E06FqWLhyVU@^=62q9=yN)AvRyWpjMvSQ%+8-j9x{WPU;Y%`^Fj>gF(nJW^{2K!nmT#7tSCP6qTaHk zRV%rne3TK!a6k~{z8&97IR;RbWk((^%E=!K?HJhn97F z-bz$5K>BwkU)yw(+nYp7tFxeP zc@o$17-Z#@8SPQ5nfMN=9mi6*DoPlH^I3>gp|mEqUgf!@8c|>n@RNu}hee;-jW9+p zZU0aw?C{>{>!f*4z!*EXqO0^5zxTtSbQW6gH_dtqjak`-rt zA&l+}-i)7nj+Z`sDN+$JQ&HO1Z}2|2QpO)3k`pOfb#;BbQlI9_CFG+h2V65tL9tBj zuwnO;xV#-*>x|#~VO7GphRP`ZZcK7zkGY;QsC7Lv6HUgzDexcx1SpZj7@O;nMf}93Wc0uq zv?4Fq=gaJ!dnKFM1h>Fn_U4)Y>5ubM5RY%gkw1Mv;~GtivB+A->M_wc{Tg_dBAcHq z;|X+1vJOv-c&_2`0T3~LWbV~7bM#CjLM2&l!o=ubqs#LQ$?xcz;@`WD|JKaF$lo~+ zd6IWTlA+A_|HB?fJfHCja{h3ImfNn!+xhe{c@&C^Iof zu5Ge~scHZ=893dB2QQ#tonOc+$t*B8)8Kt)XvzW-(s?Iha{)9wJpJBWdStVVQBbB& zzF}|eLL)KmiEV#rYI3js|Z>Is59Sq{?$ zJpm1{DfQ8a(NX?;SnIV@Y5_HCNl-N@`OWnrB{QC*;XU%H3Eku-VYA>i&|y-Jf%e64 z<}ow(+X$0K41N{lpqdAYV*$Ve{viM1LKxMWc{JE4-1DjQ@(n4B8t)hf`)Na>Ty_Uj zJ{oNu?Wz*S?&^);xu;6ZI5^a1Fz5G$S|%0E%~nJUgNcfF$~M0SRSu|ygYuUmexlbm zL1wCqi&)N6sjX{*&*b~+vr#@z8Mmo#3~!n| z-e*j**(vc+`+#YWPsF9Mm6EIBmQFYxo$b1;ZM?wpzRTmI`HkBb)?SZF)pkaQ<^jXG zA_Fi&?AP}UVdMToO~gv23}oz^gR96wmx&t_LFj$+x_p6*_azT7OWlOLoqko|T=*;x zoR?12!q1J_G*45*AxhXiynBYEzo@Hv)dIi<%z~0;KQpiU!@y-E2#Exr-6XWqFnyVF z@ya7vo(Y-CE2my;Uox`o zI=|U&3fN&-!zB1-=Q}O2`hqeNzM=1M?XyGX$>;qGpP3NJ8!fSA^4V>l_iZPB*dlJE zjpLsEo(}8Sew8BHCgQXAbaB+W+<2ni^>_DWg|}$`2*%l=uVMoSDGBr78IdDnvL!({6WeKfF-)wS(XSJV23z4-L&pD$skaUaYHh#2>5>NNRJyyQ zQKS*1ySrOD1O%j|q@`QByIZ=uyQJ&AIp;jz-#gYrsLq6N1;`U@d%Zyett*7vCrY37y#uiTl?B1ri-d@Ttkk8yKVQ5xnLk;|@ zz}8%1L%%QMZUzjySmwYXjQ0s4gE0u~bhRAy_8-EDtErqH4a7hl+QSHeJ>+oXBP;~` zP6upYzHaeAMkh`!urCvj<3l=zV8eAH%Jd-pvq135>{cY}Pq8B8cKvqC0YhPFe6t5g zdNCXJ&m8zsC<;EF9n6i73qNyCI?W)goPOF%#}X!}F03lz*LV-$cJD->zU?qRiIp{h zN%{N1-Z<6M4T_dHIij1J=|s5kXbXMgV80uYy#OL)Ks+AbT6Ds~9TD)r4uyNhYUw{@A=^hrda>5aEcQg81IvjOn{+F0}g0 zZDg}H?!#klu>63{{|%Vq{rvWK4HnFJ9*?2oxXY6YFI6~E`A>IEq;eN-2IG+nKk0YW zX(AO|MFiw}72WRHcPM9E39Bf}uDcbJke}o;87D0`Zn~zoDR)+r36|U(O&JBf+$6a0 zo~ndGfjE0uJR)R;3ITp{xU0*Q_HWz8>BDxPOPRFzJjw@URoniq3hdXzVnCl`U*xh+s4EAc};Apq}6^1X=C@67$Jp%k&t z%{;)FR3DC_%m^|-B@Io8Oo3RA0Tl;_#RnY`^K!p&jXA)MudnLGO+qLbbY^$Ox&jdSFHd|0Mkp(H=OmtGH$0$! zimFi^x^i%+e)@LT&A|5-m~GUY#7{-xqLrYQ-4HUB^u5>!Af`;<3x`p6)rm2+kk*v= z54<(MU8b23r5-07CLdnP_XXu&-{qv2;np&zisV3kGPT!l&ckQCB1m+WrbZtzVO#GM z3kZ%{ue@e-5QU4yVCG91-=fMf)^o0_v_#|2O-Qc>2bH#B_SM}X-}M7^lz;_3*UbJa zhnn}mmtBoRO+pxt=YhidRUqnnhw+eL`_5W{;9T30MkQ96*#c^#zIk$GmJN#B2&(>b zmwNvUY3g3@U15AEQmjhnFLq&Q6qqE53q_d<3|Yx*v7gm&58Vw#V8a34cW^3_FvAJ( zD=0o_@?MGq!|A@4Bx-59`v6z7e*RkPF#o7)>(Y`r(O0je5%HbH%xY$PXGN?k#VyC3 zCl;=j5F>Zkh?|H&(7)aywgW;bos+OLpBA0iPBRX-4+mFZ7y%htOJyMt9u9Q?u2cSl z7aq!RX19*^C-quj;JW^B-xE}b)5PrwG^{^YI~kuLo$l|;ieSquToLh!?eP%7)d*tJub|l( zyzjFXuIp+vQO>txQKaNdsh5?&(P;WG4#}>}*KFl>VC;hH!lx9p&rmV8-$r=w^DH(* zzu$S}eEY!6QRNWUZ}6Q5(pQEPFA5N{E@aAwz56w zTvJv%tDKr1&JcFwg8O3nR{iWD*|8i}kaE&j6hDQB=gMtGXUNqFj4`7t`Gp2Wk=rXd6Ul3rUm=DsJH zP~$Yw6^>l)P9!1|xMyBMqA!a8v&YW61h{-?-)^X;^Elwop?IACB=L~NRacJ(F4d^s za6iQ67;#q1o}`eW;^s%9-?km-`smLZmvRuW@?7>1LW;%vIkacsL1@rjUK+UBloAm5 z0)ueNGCZ6wp-o?_)hih))?)7+xwHF**)ABpVL?451ygFf8SS}%j4v&9Giu)Z% zHc)6+2k55)^8u|l5PTO$*hiYg+tAWh4m6r z)4uXt!kkr`$I2_uv#|3SW_VO3m2d7j@4N^QU^ZvITo!@1$ZW^g9YWP|^T8g9n08b^ zRqfkt1o8NN53wr*a}$}*kgUxAfe$t*(`V`MnrPB5KmGmE53!e*3+48F7v$f&)mmQ(XKPmGC)xNn z&6hh`T#CuMR@hnv+WWV>SH`eE2q2wkulO?eP2Ph_L76S{A-Rz-^67Ij3QAqea+0&w zlO7Lhuwd(Hpe)lX!KQLnOLUF|3>I`Ioo`+xl99y9Dnx0cPnr*ZwD2;q;Tyg8)Qo|+ zvZ*BOurMVKJie0G-P$_VoyR{HPjakXG*9C>Id9f(56NFa*bT^&?+@;Y5fU&XeA2bk ztkhHnKIWN89dz);=H02E8eIuK+wZj$wcEWpD94hruedK-$mmXRscFxRB}sVxSwr_X zc-_s4?S2yyguFnZ;jlzo{XR`R%TGJ+gD@3M^Ww%iHySx zwSaB^w0m&=bY}E&fF$`k^2^IDCwIip)!7t{)dLusNl32A6XU&g3|VCU7aWdCX#VAV z9|PeG!D4YatKEu&h9()={@q@(Yg}@weYpG2L2?wbH=HjD$?Yc>5bQ~wcUe8Il$Wr{ zTM-JOPv$Z?(jDnGcDDT8oQuWhVy-L2kqV*P@twGKER?&rk#1+^iwxJH^`h@bqkS)6 z-<|%+FzT==F=}D2JVE7DPf+1K1NJ~FYwugxOC0V}^SK|KhpD)1r#+%ythou3m%meK za`tr0Relr~tCX;##jEth5kdb0q_4#qWz!!hw$|k{8B+?$j!A^YHZNQE1s3KgCPZTB z4efg`eC?&CJ%6A+fg^xYVE#!V@QMn1wG*g9qsmg zN^ujVeBfYAeGLZzGEvfU1_o1v_8vP-i;l5*UK@I`BBsu*%GC*4DA>T*VrUT} z$N(#(1HJqs4 z6uD5C@YKhTg{aR&xs-H{BwunU1|2w6`EafI-JR;TV1lq*{>}TQ9=)%D#>SnAD98ct zEVAG2%&1!@UP-!s|4PZ1DfcxXs*%z8vvR5xke&qOYK^|}k=0z#GrlK|I33i!UrKz=A_VLO$*S&8fjM@I+ zV&1tR@?xO&F}dzJ)d>uKaV@uKhJ2|)EUWJks^+~FIrWF&iiB2!UL{7EgP!ygIw>OSO6PMA7yT%CbE8f45QYg!I;?>vB@0&Lfl2StaGj*lZ>KQJYU%E z&&{y$h+*gF9hjD-j~iaiHmbVUZ~Xe?hvrGvdVTw_11TRbSSHFBPC&4kFgmo^dIak( zaJq(OiWZ_nRx2|yU)}b6@8hdPiaQ8ZGTvHpQQ=zlx26B0Z7i#8N|jlAdIjw!$EVY9 zl;oD>S#)20@19XyAof8?8FHc4RCn7Y3=6$4koDlQjdMnV+154zO(9ZBhGuT!E$Mu- zZMr9TsU?~@Vv~?(Y8vL``@TH1{wa!1+k=7G%vH>#L{y)R63^2n9IJ_r8VOt%$y*m~ zaH)ksE@jVI{9|{&veEHq)Q1si`9)OWhmoy$5n+C=(k4u|b;LdAySpwI=5!ARf^fV| zCS!dc0Y`Cn;)1Khe~qUO{d3zj>*p4GD~E)YUyF1ZeryjC{PK3wF{$8%u>FK_Y&}&z zyI!+;J9mg?8a&SL9$;VWShyP-bGjxgTUXuBSifl4bo#DwPe0@y3F+TldqPAegHe-9 zu=O6#b9c|LK%pPDdYm^s$EZ$5h#}tfMB{^-ppbh87DS3(E(?5=X}y zv>RF(XII$WuRE<}ZRWrdb8m4JL%t7T=O$ROE;_jX zu+#n0a_ny{h~jv5nCu9vDvrn?CKLbGHP3~wph~YS@b87;a4(x*%Rqw3Vq1r{@BvnqEM>>%QM4O&}>c?VdMJ+ljw@{KYbQnGO27SRCibzLV&t^jGa|l zsxg`a*$oK|ch5^c`AREo9F@`Y6nJ*{!dvnB&W4OaY@4XV1CWH`hAxa>lgQe#EQhC1_e*= zXjinR(J!fu{j>2Q%KG$$Fkv8@%Ms+hLg7hwu_Vc~rOP!djcS!%%CwgZZnFhT*IX>; zOuQYCUcsBSa&9A=kaRkuK*f!k*bj4IDFUl~l57^tAJy)V#xQ*>f?d9A0GxvssSwBRuF95zWIG>cCrdc+e zs`gO=z3cJn;ftQTM)A%kiQ|18kAdLF1ax9=>}^atnnZnVv+d9OVj9j$laI;6#`VL3BvdDn{Ta!+8a&~Rf z5<|lHg!y^m{T2FU??1z-cJntWqQCD&J5PjrCtf%OD~4=G!g|yuyM$fRVtQ-F>a?-r zmuguTt(Jw1I247I?V1i6tgb@^g#L)!N_ltpMQYL>$LltRW0ENE>GvgRs{;p(V0h{$ z@j;SlQs2&#vCoAFb*~AGUK+TD6KlI*5iq)Q{6xv;KA^X;Y9#n)NY&u|u~0@Wno~Uh zwKMrwP1Lt6xPus>-G_EwrALt5N6BFA;hT0bjtCWbJ`4#blCB#^96n58KGS z|7la|zk8<08W*$KeWzGf0}mhhAfncDs2v$7%e{fuD7l%V#I(U|_3l>Kk`PnwHL-}Qta+L38_Rs>NL@Z#5oo>UJauP$7?C!hWu2Wqq#y8xs_`K*bpk6%bbo!?psQ8v ztLoVz^YdF@?Z=}LKIQfu;!=0;9rM4XT4n%4OhV1;8*E|#r`h^rt@0#dV4$9D3`OXGV#4$u&`hHHOJeDC{h&-J8?wY( zHk!0RmH`+k9!yM#Ci>rx)o7Z1YtZRj&BY8HI&oHM&thIFE_nrrCo_$fr}uM*#5KS@ znTu&ZNvQioo-kr`Dk^*jzwgrcYKl%!nyquAz21EB9Xn*JDEw!pP>gu_^NC#aJ}brD zRSWihw5v(W&3D&4A@Drgijs3PuccX<&-pmECunI>Q&ZU3HMWb!%sSk|P<7buUI!PJ;$HVdGy+S4_8M6e%U!~ z6>Y~$Z5xrMx`O=3L&G8DJ+H>iS8we1tZil?>ayVtS}5Y6(m?Xs4<*onwz?_bvk!xo zJO>dD8R;=es!SCrGu~q6aa=^?1EuRofs%JC&sZ>#2O?ENaS1FRpKp8n;UE^`Y9Geg zC{z$6Kgn73(NWt^&h>0^D`OOY9fSILS4vsg{b@@xbn1^$2g!^xtThBMQ0If1N`f3S zli8&@=ZFfb9;uAMd8_}~8NZp8ghtu150WKDKM|92dRJnL(xM`^$ZVWbcKn`llRX&! zeYn+T3d7I0*UJoV%8F0FED&Yq=a&X29-x^leW7;?=CBeiOia9g{Mc#_2mmo+M0qgb zb-w2a5g#1%`duIcH!3F$F9V{vnd0Q0uOUFz=sMi}Y+;S}CZ@Zw7Do;wmAA%iVow%W z&9{1|=$my$`yyNAr2k5eb`7%Yv_xIG)(=sRf8rzNL+SSMZ0I*=6VJmpg_=7l@3}z+ z=F!#uf;Q=A;C|qXC1J#Yo_@zTdsw5_+=;BB->PQwvJ$Bqs&bP{G!EzUJb7?M_w@Yw zVR4x(4Q7uv-9b*Ljbdqn8^IFNg@kFP%7@biNqvVv2OZP@gbljnpHX(iMX=1V+?@$$ zb1N_jP}GjPnThLKPwJSOg$rS7F%QOHk=;Evc=!Z-KNY%RAyeIU9KSN-MCMW|K!YFs z5L}6>(tO_K)ZGIFXJ_uU8>_4Ga`tivz#Ah9N+2EIVdsZ~{v5MlRioNjsCpa*pdoUj zC?Ji3ZUB2&i<`ANy_Ln$?i&$(f~A7wq=Ix2kFBEznZ6=#)z4*7Q{B< zl#-^L&)sFX%pm5PDPov(NR};d70eva^*nnWk3n#6ktYIpDlX2~~W{zX73 z6{xtok@}YUd!NFF;h~pM{4mY<4pc&MOGJ{=Q3aON9^eSg{P;$#7CjKo-9@vQh5jL< ze@4ZFBFv!_s_AKe+kZGao-!#dvR4|;!s^M|0PEd00gVi7%Co{_1-@M2nh{WTLJ zrKyD^bn{p`Go@+ODCM^-0xqn55+(|MpAV&R(v8&Ly2(4j3r=|u*vD}TvqJ!rm<%Nr z4>tC@AYu*TQfJkT>S_S8I`kPd=3Q{*mFhi!g6@3Lfa8-g{He-5QIA|m&<;JMNH-^{ zmmIpL+b*bPz4@zKAn>0M5tUG<{8iVCeiz4{H^BMAi@5}6_mIIx9T=UUg&s2TmbId1(8jb`$V$J3D?SUTBG_%DQsZ zGHF;s(vBn+*$N&T0?28ZJ~tAEsE~{7Rv|Lk|T78 zS*x~E?Y404&}ZF{_)4#YjZa@9MapnTy&SNjvH2wXbKWFjQ@@GMsv%bT%zRrTNq@wb z{O?^&am*roz%ah+Of+sR4_6|tjHsPo%+r|wg{z)%fIeQ0tb5JzV~{BXs;zm zr0b8xo2)kzlUPdD6!nzI+Pr+Qn~7{vz-RN=I{~g9^sVgP*UR(Si|{P3rPn8dgIRRC zq8?ze7aKem;co3Or8aotbkfT_7rLdq(hc~IKOY|yvB%}=2>#B*AQfQoO&pLEXM+N! zZhf}+jxfjdNBlgUFyJq86Z1gPqD1gu4gwXItvY3IUATC^54{6&??q!(^Raf^vav@L zEDpN!f9ag3&St7~zo+3~O8@sUf5rdwHL3Aub6IE8KaM-s`5q+FJf*w|XhLan>p+4? z41o=8ITD2yuw;0nU}}gy_D5c^}3e!Aw?Z|LwOoi_TEtVubIjL!-$pRwk(3)r0s_$NidQz~?4vwS9iFdO)UT5VvA>@Q(_ zey~``jhEoHD$5t!Zh9w|Kq1_W_cJ_aZxg~d9@9osGHdLsEF0vV{4Fz4-(!`$yI!sI z3-IMZ?+7FvsNLn`DTGrQm;*!!e@%AaDOVlc(x?&^V@GI(q@fMbmrj_|4eM>7R=!R@ zng0eS_QWm$SI=>rn%cFJA`q07dB|?VRQvL!Xo$(rTqL@67dYT|c{{ASp%mvq&r3CRARSEN#kXDdpFne)S_LZ}6-#!L3~I1Gk11W=zPI+u?DT{MfA=RW~)=0&S+TuH-_ohF5X6 z_l+qLq^df3_4OPlOOr4U!z&O|Q@6HniQv-Qpl;T2{VGkvRBH@0My|9Y#?O8ZHuM|m z8%J0Eq^ehO6i^#-@4vpA$X|cbpw#}q#s8;d5x+@6jJH8o-Ff(+{Q2$Uw)OAb*S$+| z8sKFM5fL#+f?v04{+#q(L&}26xDl&fm^7d8b4WoOtPhAckR7=D;ypFwX?U~MehTgU zh>HB};+2CreB3_e%7HxR%$yAD|9gVu zG=(&VLhuCHMo>&);iK$^z4|Nxj0!D*8lF@E_jhRE7iz}S64&gnIbvF?iQ@@Z;ZVzQ zvAuUoVm*4ugB98+l z0kRBj>1XTsR-uG4x=Lu*;(8)fv8PyjP;ZqRLRrJzN$W9_;Cr^@Wi-8K`ncydY`o-@ z>vTQ%RhffHsNl_oxN4Vsbn@x@8sy^8gF~r?KU80pi2jWnAZ4u=Dh~5)8JHy0ZYZII zg@)ZK-X9I|h-1d_ej=nfa;;iic@9^ujv362MFb>32ncvp%0V_`=B^C&B2REv&`=$|H_pmsG@-Mg z`7g+u{i^^Gp&+BhGf`$x8_IYaDe?DFd9QDO{}xkVmE!bj3)5ES%Fr1ImTR&kki6#$Ubr&aiUJGzq z+yDQ^ccJ~QF-@?BiSKbv`#^utcl3;>BPB zIRv~@&W1KRilV1{Q=}FnZ?u5e0Tr`G%|e$(%=ucWNf4{~@Vb2?*_cY5X3{+mR;a0h z5G0(x$}&kWw+{h#!zd|fj~Z;6SbU>bLd>Qk-gC+C(w~2htlVC2oKL|-mSN+(Uitj% z+hxqjW!}&E?6m)YIo~u0UVGhIOw-~D6los6p#H*>G_8kGvADu#2_~93`NJpmH${7 zbC?!@ICNH7`*?`{29w?5B_+#OL_B)C(=LfzIw36hofyJ^KiJgkqT+z3if>VU!uQC9 zdLLOF5C6&^o&(UnZh8e@>tlRL2{>S~UkZ(F|L(E?XRPQJj?F40n7-!C~Nc0coCrdYXVefw^IC}e!tos!`CUAs5Zgt+$Z#dcSCjl$o7yJ5VomIzldTo4Zb)i84 z{SDw)FY{r9!!UK+MgKDfNhE+|hv@}Rfb!e5?BO2dZ!iMy8CB>(7s`ZQ5{Ic;AFOSmT~3(f)#*OGfdqu~frBI&VOLl4!Cw<0SS*Kg&|Mqr*(e(u z24r`OM61`(?HO)qQL$2xXo3h2Te|TzP4yZeLMFwMZ_}(KRgBAfE7$eLSx|jwXV7*b zu1^pB+O=M?eM*k)SFX;FQlbBGc^TFw2cIKm%sJ{#dSpY_CUTSzH_l0k5}uoF4Dg#8 zs*yY_4$Ufd^`~`sn0)x+I~LM;bt~8^C?7)OJr=jY{mN6GJH(FfUTiT^9`1L$aXb5j zNWc9L0Ndn}3@(Qj1{5Hme7X<^r*$7J6qZ*G1o2gpP{r5lX+8aiY;Qyn#c|x->P7bP zbY!=&a*8Y9h$HVgyXh&g@4jTvfYTJnc?G(m3xF#=6}OFr!`p!~o{dNYvL9_~9De<3 zNR=TRO(9k1b)GtHqj#pK1=b~$jJBmO6Zn@4TNp*Uh|>Dv@3!1QM=fvh?z)Hfs90ZB zcw*xN)5gdsFY3e8ndJg&BL6+2iS{W{mFT$eX6c_EmvGYu$tT48lr)i+M2AVaj2I}* zGE}oFwfUHw1t%Bqnoq?;q+YBa&)fT||79{drNIR!wQ5?l?gu+&DV!;>`vdW{3S8K0qq4tbjXFEK9v6lPIsl^ zwWWZpF`cZj#2FFb-elILrA_@R{|ZX}ZEOrL?17Pq@7U=*N?mBS{4gN(uSXXuLFZ?t z@#WWNrp}F1B~lx!6`lXh1ckU*bvt|#+7QQ1sDKVus=!ei`6qly+V29oB2L9P#X*qF zjwH(O`oH~jS7>;m_!^~{`7Q*^2o=f~&&AJvL?}2z3@3H$4oJE`09g3i_}$wOye78x zOk`hX7i-blAy;e_1T4z&zsLcWi&JiTHgpH)I}aB(2iq?EN3m(-j+MiHd;tnqUUI== z6>H^GLESSmHX|ea^=Oy90Pb z#wOLXdv=C-8y)aMfa)h!PLp;qy72<`^IHfG@ILkbR1Td@->=6|Xwe+)hAg%8Td_A$ zQGh0}lG=?Vi|%S7E?cUs*GTa$=z|Y~zpAjeC?*7+iUPB;`~E8BzcW87MZM%aGl_^A zuBS$2x^unI^SZHnZe)`J9YKP5#>^MZ8{Nz@R)@NWCyy; zK?Tx#$AHwjxsq2VFUUV_1|xXIdVBOqAXcj$TU1$GoT&u;(hM7fTf1CT`;V8_-;-_` zut5@Xhx*`1=X70cj%69VqkNt3y^Y*<5l5({BiR!d;0p2WK19*Cd6G3(2~CQ86%cR* z7+XiAHk?9Zp(XH%VN2^ozr4f3x}ACT@9Fg`x}-}p4k&(Pzc(|{KxaXZ$XbZr`D!q@ zJEmEKZTzk?8fF`7+^Y%M*JpXk&slEogyu)C{q5~|C30z*>-sXV;ctPE{OQvr3b;+_ zIuI`Kt4t~V!{WQQrd+gitXo%CVm;xYGkE~5WgYG9Ma;%I+DKr#=~v^_A4WO5 zXa%o0@R6vodwryd4W%y8w>B3pcoNzr)U?t2r9iww{|$&X80gYOXewEmvJk0y>vT}8 zT>X!7uRf4JC03;0CTSFA1S*gl?x~IITe)ErQ*FW3^p2L?aB#hnu*n+R_gf{u0!)K%@RU}CH3nI^uZyk+g3@Xit z?b&}Xj2+9#ragYXu4N}hE6B&aeOaBP0!KQ)K&1;BkBs@gZhSeY77j;f?K0>5w3({h z4E$IkZCVyvsAfM^X*?SH-wVD^}b|+#ZJM>Di zOyAB_Rt#S%cc|^RnrZb$S@JtzwNKr22MKudq=rv-?fvwy>cww1`gwZR2OzhY;VOi> zDus*2GA68s8-|L}n|?VhoqM*uDsAIaXh@@XO|`A*ul)#s9LmP-%R!wkRCRn*^LNyT6lxFaWul9Z5C7&}BVr({G#!0E*%}_FQBF zK^N0hdo#?5(tRgcfPDyn77%vhwGAX4)H$_NZ-uxn_x2#xwe%mqd}hn+i}S~Y;b$jq zdtO)xHq_bLkQg;PZK&C2`yrF$XZ*4~-%oR|#xaugUxEnjXSN+tf`sP^kgLt2^Z$E2 z+9%YRIbrt4c$DN+e{3LfP_@D43y2vxwQrnnB_w!gM3mUvUhUo{WfVT1&Y9o3mz&`i zNB_U?WzB))bt{V6%7`Bhcd4@10uRA+Li`zwP7pM=lUmv|iKB^$IgtmLpO=SV!H6G) zBs<>})x`27$lDQTA&zU=smhcx(As{35$ILbuaZO5C3h*~FsT=7>vIqNS>hrv{1JsR zu2L#qQyiTIOx&^&p|Fr|=k{F|crX1a#QJ}jtx_Ec5vp7f z-GrW=luJF>d8K~o0A}yxKQgRDHw!I{&&W2OTFcV*5ACPj2jr104Afyne11WpfL2}) z?8wp4&4)WF3}plsRC=ctmFw`KM0#*gPQBnX8- zcVx%&fGcVcj;-;(cC6qBDAx)h4u35Ia63$ysk#59?`gbBBsk&9S2mM?#EgJoZ!SZz zK6{eh3+QL3%j_RdnZ*Q7bEi-ke4MSJEwHIL`&3_e_k$@~qk|>hc7}9%L_Knkqv&0hM^mNPreWzszlrm926e z$={sMZf0^V7t_g7bkl<%#Zs*J2mOBo97n1>eVGXUd#NsStLHPoypq*gLqUu}GdDp1 z+uyp6YGGoBaBse7z7!8K|C8Iu>b@pE8t2+1cqm zgs2A%m-uc^4<%r3I!VtLuKEy8)1MJNci!DLw!P+t>G=8#Heo$%gkn>KB$6>V-A`J7~O*zcQ z>!)~WKT@XJsTZY7f(l*021i|@N!`kihfU2GjfL=SCfuQ+;Rp2Qf2U=mI2Ow-pXZ@X zJT-6iI9Zolu??M+vmFws5o3TQI_}gJSO^I2RXzTiA6?Z^_weJv?N~Upvn+T!R6Sc- zu5tLQGwR--*ysJ!c%p=pKjpKkma3kSW*SeBcb}xZh43P&RR4O*#rGPM6g|{EU`LW( z_C*H7jT3$c_0M2{J%hWT*9tkshiu0VJ~P97*{M4(V%>TtcFuA(vJ&M7gq|x_ndgfX zEMw9q$zDkA%X;4WoU%B#Ept&rw;f8Gj`vd66W+HwDG!pU*OC1zE4vPf8K#v!WiF`? zi8okmvFTm)$lkf11UZM0M2`I^f+=)cBo$nhfQsx9OZ3D;_nbOPJ?^F_^Q2WbWeqGN z|DBzrgCfQTm_|8&H_ho#Ha;vq;N`xlzg9RGh1+Q|1R7!<5~_i&oTN^Rt)%%E{?x9s zxq*FYO>=&#cNRl({af%Btcqe#M&;~i zn^?Xum06GWK)IQnvK$}yu(o!@Nr?B6jpU$9ynk5^E;dZhF4*PQ$JVmG?I8ETI zFnPJeOFVleeLGP)_YdsvcYm`k0rHI<(qK9ke!vZx?8} zw@Cr8KDtCa)L#VZSK}?Vl1u*`k?0km(&dxJ1NrwF+HyL7VYyV>K+Rln-PN25+VlCo zjc^6waFLeu<`qO|XWLq5Tl8B4h+QdD6+HCa3rNr%<#4XjY3_mlG>fz(;zYmxXo>z< zbIy(px`erv=+o{qOorBIK!C73+HaSmabrV61A)A~G?f16DI}{Erl1h%FIVdJq&~xz zKdvV}NmzNHk>ZF=`i%#M@2rE0!E(*~ysxG$yAi6n3A#tArmlxC)?H5rRHcI3aVK(ReJ5yzlG;h1$Vw}%7sieK zeASp0XJTKKHnFW>{zu^G3pJeSzm`5uM_kKh_Gh%YWf_8TX5FLOJFg?-O!8E(7_tpw=9Mt3rfRU`U6 zntYaFHO*lkcu+~*GXIG(+u9kJ#-R{%#jYQ8n9JF}fsK_hs{wB4AF(88kF8$pHw)q6 zq3sJr-Jo#sV7tPxT$Ah4U-tmkZPQ3p@`3)y=a%v@(1xqirUS$Fy~C|xtT(vdbcoLc zZcu}~w=SQce-dD-*bHmH6^l2G3}PON05!Y zeqwmau@O^dH~9#yo^ZxC$7Q(`uBZIhk7nw^sVnmA)N?v&u3im25}6qCBmO&zkDyru z@rXMzOg6QE0J{*9U~Kzx4+;xS)pNNM_}^$j0fXg!hs-Wnr=GN7@}>Ck%etuNu*=w4 z`@_O=H^}I=?tMW8lwa*qxT@KrHC9^r!j#Q%9&To#Xn!rUn&_#wlXPP$~-u|nFRjlAb6gx8uFf)V0BHk;oJ)E3{XDuv!18c?gUaS^D zBr56f%Ubcb7zdf7Pr~TX2nG^9&;56!H=|kz%Z-7B^JLpE_5nGQ(}%7vApHdL;|UC5 zxmaN9_~)&oD3`z{#G#9xJDaC`0K0oJLJ8F^{0UYPvfz{}0qCJ;lS(r5;jvuzc-~Va zs_1iGZ;=JTY0;j2zjY5A|C4nc+6(@dDVVTCoE`MttCNA#PYV!A6$Rym7Bd+AKvFGw z*|3gl6o?g)z0M22FIIpjG`){8Y{!Fzx~Be|V%5g2a6+g08f0k^&WKEr8_T7ZGDdK+ zJOwV$UjnRk(K?q-kZh z_ZPIj0xKvvQ~20>56dstxS10%X=+|V7 znTe(@Bd`|*-j42!0kI8)Za~eZVn+y0I8G+RtEm6`-VdkVd3LZpke>chvHcp|e0rg~ zQP`7OP1m8XlBg>ix$)^wWRbRVJ||xEh|wR?$W_{tl=bSMKx6xoM$leTqX;E2&f7}^ z{WQ`m&ps;;hwzCgTM#>P9r46!j7l7^xT_L}@IpUQ&fIn$Qh*DpcuD(jqqeIeO9K1q zA=9C_H2qUtkU4Ho{5!M^$V!%CJLS&He%h$qg-&o?><`jdyInu{TuWRIe@mnN`rjN< z&dU_>2F!lX15MvyI36Ntu~DH1{kGrvLhUBdrH!a=H~M?lRG4kgPs}g$-#!yS5ZfNK z7v`XYyN-wmC7&rjT*ISRc^rq7lzw>F0Z=1@+0ssOxW=pNF#R3}$d&iagG|ug*YE6g zhq{-W_oqJ>?#RI1L_VVrj*UUi9vMWP++MWx$BvWNeWj^?bSZy~S_sy06S0!#vc(bW zB1ZK!?!MwCfTg1D{`0q&*_D5Bg~A@UXBouoObQ2@ z3kc~d$MGn1&nDv8l{%H^vw<^QZS7!d+J^6mCw$#{XhOgwHC317=TJ^oq|R?UX(7N< z4G_O6673u61liu5DC@4D8Tg3pE{5BfeQ29edm)9M%6T&TwucQe+}UV#o{9OgJtK|} zVYsYlbdvr%AZK?}FKkkaNuS~aHWOKjJ0m~bQOxO>t=6(Vm_>sb-Ifc5rx%cWkQS^= z0X~AfLFt#Hdh*r8)?9Bvvq&v*VVdFggv~#M-?OOk#CTE{>h`2&OqA0+k9Fn4H|qDu z03d&rYZr7o0~yw{Nhe{h)#Q9v>r^Zue(sP&QJ#Zt8*@c&e4`>|_a+9iphfTfQ}ogT zM}ycws63G{;h0VZTbR*NIFTut^UH`*KK^%aThRmn>vv*8L`%W27xW7NeS#RWM0kb) ze}aWbGJ_hu^F2d#Vei&V^52`qD-|vaQ5teVu?x@8Lk_3rM~BoCo?1AV>_zA5AUI zZX1>K-GDZ7)?tM(GxJWtboy-^#vfczIG1b%lSz4SJqT{kgZd!#eAEDtET5iM$!yGB z48b!YK!M)`?6%DFT{M6zqEdtrpuhKYvVuJN!WRaO*ra>5T3bqxX8JvQwD#2DY>$iE zYe%J4G$mdEMwKVbf`G~)*q-L|#ZpFBJz0W;WbZIOBh;*HR_K{tj`#o9kc-@?`W$b| z2O3nhuCJOAKs%tY5d_eS^5P}v=@cfaHp~ZIGg-CK=hV8%^%)=4hk!StbE4U-dZxp= zqP=ZD3WE&qp&aR6L`YQvY0ne_x&Z`1v7@Uv zqtVgk3?4MH;w8o}1p`*r7-^ztau!at2{j1B;GqOiq>K7XcjBTMQsD@*(6TdmzW#kM zT$i)K_S?8`_N+(jv`5U%{+wS%1G-SgrS7}REDF^w{*tu*wmnnets6S@6D@_&hz|jT zO7q`l{o4MZPg3s1Ux14#z|Va4GM7^SQ2Q9ps${S42)Y#FW9~gG2;2bV2*8Onz0fg4 zBW<;hbjzZ0J)u4Be7ASHgOqH|)CNkwrKLS2Kn-jKd4PDi|rQQ@_^!69f z7HaoGmMq6EqO`Vd%s|)cIu$Uu;Q!&;7vPmpy|OA+VHM`+%fT4J{M&E(h32`sS{7_D z`V{2PWEV>CT>JceEh|lE$b}SUkR4xs;B7KULxJ zCyL6upV%E^xW$$8`ZE?7F^B_e3cv0}BKi`cX;h4tYyCZj9ZCP$WOod&Dj#cw1XF2uobLM-wpF z@T3_^t_=d~nOzK(8PhIDH)P&7OyEO+&<_s=>UNbiP&b=pi%!I0{wrkk6_(*b55+;f z?_4_+N%xEojMdo_icDsaxKnnb!bLBgUEhh+@ecfV@G`V7g>`ns>-pXUb#NpM6ztMtA#mBaLelb3@%&F*@tov+qF zxfwS;I_-mrP6!3w?Nd*@;GK(;cT4Lv9lv5ps9UZEHzkiPANCS$#c{%bhu3)Y~M}NBdIqXT=;4EGUK}K$>HU zikLD8FCy7#I_ISbo6=mg!{6#0!<|&u0s%+~BVH36BeIN9p%6izs}x!g5VzHf4$6qD)8jYylYV4Scr~6=j#<;!T-dOwcJ=l?OA=6L|pa{yN)-gmS+NFe0 zUPPx~m_=i&aPqPWwdMxuo+iWRUnfdoZ1XMYb}wD0ijoVW`x*-hxoR&3_Ge5v37Sf| zp509#nlejyPiA7J7n1ii?NXCXB4R6M_S%KYmi!6CZr{0vi9oMjA{`AnrxEN9g1d&) zALAMwj#gKjnz@v^>INMbV&cx#2!dPZO%@1?%b+IRRt3o>a4<9$aZ@^r z)>DW)p>)9pnyJ;ZURt5P2Dn-76zmnl;v{760kFik^dXb&bwRcA!^n9x$X`z>pC9Zx zf$-brGYJ98>e=S=9jwPSPD?m2v9Fdr6XX}t9lLN6fQAz=1cYIY+W$!#1$GQs?Ze^S zUVZb~#%DQ$^Q)|8VGT=D>%*ym;mIBgKL-e0YZaHKQ#6H^t!2T>!$(=3jefphLiXjx zJ#}|52meH)>6iXxkSX?lp$2?QwFioYEd9kCD~fL;-&T5~+lPQx?zvH8yG6Yk!37!i zx9|8>7RTPKjFy_>EzbS)@!^EdC`Gv6D3Se8NBZf#hIqiJwe5$1`np~M=+#(F%|H6F zN!b5o=|ok?#XCX2mES`NVS8#v>7C}Ud~`fH=M`#mGiE@@l}=HT$(oSA-uQJ%nPU2v zPBT2Kvb~{d3Fa|~y5hFSuUZW*Q?6MHvKz`d;2L)N^SJr<-teIIk0#pGj}1$*dbEcB zAA4^d7xlXJ4Wn#9DRoOI2pDvOgmj67ba!`mm!hDwgecuLGz>$70s_*)(B0iV#Cwgx zKIh)&Iro15f1dg1XBe2@)vLa1U2CDE$Qn4~@10O+;IS)1e-t#b(6KAQ=Fdn7-$&-* zntO#VLghiBYHPE4tb-?{7KA9s6cJ%<+WqT)3W*ggNrE(U>8Uj-b(K`p2|-G$*hl7v z#*T6DP8wJ0;nm!4`x~cdGj-+VcL!yk+siqsS(VCq7P=nksviT-dAmEtWQ*})`r zcX2>xpFdU?LYeZ**8+E1pPp^0rnl*SqZ}EWCgmFV`mJCr;KPLKU~s;3)*6>{K8}q| z)MiwTqNcIS^tcqU8}`r!i^lf$0W^2OOUo*9#o>oO9qW;<1p&J{rp>Mf$PfieichWM zMruJCRjSza{4$;=6e}wS$v73XFtlJ`OJ5cn4;LDDx3K%hP2f=J8|L|ksRiwZvx*<6 zC^lZb>#3O`S6u^+UKA?{k|kDJU4YmW=*o%jr=y^HlqyqMKI#UIi6>~s z1b>p$j`R-%zV7nbSD9h1l^?ecvLDrlUU!*LOWaYc!Y1+^tmmsYNP*(H4p#T)d-O>O zD3IK)XU5tZfL*r;-13zcMztMxdS{dMIZqi`ESYQ9$9MU<@w2VOLzFlU0uzrb$n9HA zMrk>62jHLgjutkg^BnTV&_`d*=W?N2cRZUq!d{10q$Ga(7^Iiq7>pAq2d`454}z@N z6Foa#8KAK|oIBL~f#6`(g~TPAJw8S(Wm41>*T#OIXZ&f&JXJcXzWrrAzi4S_o!`(QCDF!JL6nV4DNzC6fUQB4 zgTtVd_K^9EzykJQd*u4Ju~%OU24{$3T=XL$&jDFdh1G`)tU3jUwBbUP}!Kh}RKZS<3 z7mW(L)eApp2}Su++IElt9b8v0P5$smg(&Omo48SzDNrv$SJ$asE52quZ<>LsFd= zRA!=8_&}o(R7(gR?pTEaEZW+9G{tXhXZq5dlBbH5kUEtj-T+EAd)UX&eghp=&e<{9lv^eB@?C@3k%p( z!HKcz&^Pq&@CU)J23pUw<)Ax^D;Js2dmQflJ~(I%;BMd+s00)dy)5sT_QPvg-31#< z(i%~u+&BeF{uH$roQvj5r+r1&7?>(ICo~ktXf$Va6*VB@n}jkK>MjO7cz`xo1=;>O z5Y6TR(QecYYYdTb&TDjGw`{@Vl=hdU%72OK5NZ{GTdf>)8(Q#H3wy;_$kRv)bz;09 zhqv?7hf#}_?cQ?h`w#Xfn;EJxL45B`h_cpHU8vrcj0Jo0vw+#o{Jz@{`eIjLO4JnpD<{k1vCtAg0Rpit1c)wnW->?Jy&IU-3hWpjbQv_!Sx49RVJ zv_ERYD6MOy6n+1w&c$YWRNdOf+rc_XIPGqIi%NZ!p-2^3PQIU3HHug0^b0!78pL=* zrqqf~8a!}%P(>$U)Z?x_^CvvvGX5jM{m2`wy{pZ$Q}i~}o*MJ%eb+Q#fh{vx{0Rud zo9}*_Uo+Uyd#I+CuRnCK4ufa_if!Jq9sfc6S(|lG6SZbH!S``M+a!n8b-t}@%w-zb z`S?e1_19wK;PnBGkLxHVKPZ*Bby_qD96sF8h>Gu{);5`N);J&Z12JZHPq#kE$og{q z)??=)jC+fZPUeY+rs%Av+-iq?eMZMWzzTETDQO&4G=ciG$aGfpQnWri1ETC~(*qCr z&{3z>EzGwEEzD}1na49raI8&jzcnljFs2%@A`RcQ0_BUEpwQl_P4Ims`e?j2uarr^ zrJBNlfmw@cE=3vqM=h-^6)9go)Q`ps^Bdh@Vj^6FKxZ9BHMY}etL=z?1mUI(yRGZo=Vbxsbvf^-UqyFjD7?&APK2EP~*R@9=FLHBz zJr(7zd-R(OxynlmVWf_;P-kD< z<77nvGHW*Q`+f#(Ke?&j53VI=Hla8JX*7tQtpmcy6bk8<)@8#guo`|N1-Ig?C3s9= zz>0OiTF^*4xTegj=KGMoF?AIF-U0;Pq&Uu}!I^+_YvvOu5|g<>HVcI z-@1K#@4xiYYR3mT7nGsrAT^`raM?9FqcvRzCn9UhQ5-FtbyrZ~%_VOb+fRAi0h_LI z>Y%IL0Ye_PgjM0yIj_3fW}_c%SzxLeMhs2pQS8CRP_DfT>dC(bMa^z#-7X)o*dkah zNC#^n)=bs)0Jx}bQl}F7-My%B#sm?Vu6#^kARFGZ9I_dnu&D4hk7`g~kOK~zg0qDn ztCRvva4AS6yo1W7uayZ|*EFc}Lq0otRp7&&x_EHSj?`2One`CVm&K|r78}^VkD{?I zJ*G7I1`1i)RS5*{t)rxEy=Je|%!k)E37(azdsaE{Chm1kyF8l(SBXmtvOqz}(P?C= zogPEF4dHupKg*jJnqUJqjXAb=!n$N;CCUD<;mSDTLK7(M4en$|FCTh1WbW`y^=%td z;JepDp(TXaSXijEkar@GcdsDA`Cs32`Q1%0rse9yH;pOw>&xkz-N>)*CBmM4@f?(B z|KqIq<{M3Q_^O&|(S&)Ubrw2@ySd(vVw{a^vj+HvIS&Y8{bGBk@x%RZnuKfXR=d};+1${(z z30Z`Nxn{nVH#r&zKE}()c;2SgX;{y@D|lgQEA{!d zESMDWPc;kBU6&4Llna^gwPL5-$j%Y9xJ&UCl<+~yrEsOM>uBd2o4ym;;O?HTBkq>_ z>lfF>Agd0ofkw?!yQ_uhI3$0BlH7Rt(l^Rp=T9uO6VnY3#I`S*k!Mn`_AeS6giF7S|^C)b2+j>b$LguI-p-NZXQYav0+%qlHx zDQWmbug^P}gQ@iBMf8UCRAdtF=mDq%c7k{qh7ke2 z_^s1;J8T@sm1icMh@M_QdEb7*uL$s?bCcqb&E>ln-0Wy?8&|ZunP&c=1l%%`Z%!S3 zP#be??AAjrvVpUvKkkHFZzD^dNj%B5A2t#IC3t&;XAU1b5rYfSW3Eq&ygW8G^-4bs z%FFPABCT&7c~z9I#pvK@x@g%^d{GP5^Hr(Es92%d289@IN=jWT{;ax*;k8KK2-4i2 z><^+MKz-EE4=vZb+v~e#?~W#c@~DJ_Kjam!)O1IXk)aN1f-UX(6y)1|KcV}uZyQ(t zVVEMt#iSQd;WY;PXPm6yuAV=`OfpfQlLkL-qqXU4Vo_aWHK=Y0BqpJ)B8HgMlmJfo zG%gldH`N2p*pBahdld<)u4pu#jfO(Nz6eewWy3x;!zqYa7k}q3_@mEAX@uYR+9@jD{SNs4v+ z?WVrUL3cHH@hu%RipLH7a>VqeK+(pg*_V*sh6a|R71bd}I!!cskfX3aZXcu->^0&L ztPVE&$>4`yU+eu86ls6H5;Eoa$iGhZ;cMKimHj-lp`|iin|xvzd!^nFJD~^_nDX?A zJ0(fd-fR?qV2bT^5Rw2AGGEI&z*`1+iaR_N~yhh4|xp_(ZVq_0kkv=wjaxnQ=1rLQ78|J7 zW0P0*EG@8d^g+z4d{PJOph$lnXrI-)`EuT6?oCvsd{YNK}MzPmi(7 zO%znzc^oTERVR&#t@)Cc(2ArOyIe_7WsWXf$Tay{TCZ;}Qx6LrRvtcDK8|rd5>0gW zdw@*c*AZ%jeV1bu>!_WcL!W}M!PS+dv={|kz;5Ei>(0EUn=F_Bf5$FL4C;3{6!>+* z*FiSE)E17{IqwNTlE7PhAXA>OYPt6KeDQZUsH{!|5X+v`r0^P?9CmJGYNdpNG1NnV z>}vPb5@ap-+kx_)b_|F!jp~VD5IPBfNa5~WOf!|BUels#Jz`+BAsb)Lz71}*i!+2M zz8vRGm)B(_L}n5ih{cKD4b^Ype^=>x=fS%=Pf(5%Y3ZrJ8VyqO1ck(mdT)NS=UsmR zHja*8`wkOsm_Ef9fv82mB5mKG+-v{VQL?p!Wy%^Wn_rk5aQe!UFS@Bx?4~zE$RFK} zttHuTqM|PS0AZCeQXj)(N%EPc5B?yJu9z_kQz_jFHU)qtb#s!to|RN}W<=Z%6}`Lg z{Dmvdt;B#tJzZn9Ays0+;FqlTC3~2opFQ`0Zf7ZglIF~Dl8>4k#CCXozCj^Qd;LLA zZ*T=!PT;HQ+s7>r+&`2kQuyum-uZsd4!GSYKE2b(Y9TDegZj^4-NgUn8tOI`$h-lP z+OzgFr1hl%Pg~X(>$z(JETC)w;cJcrq0ftVW;vJGut3VBHFaW;J6KUmXz?DPXn72_*JA$W&AK{rgztI7&)TseWRZ$G$(s>htp8?kU#e3vn#9cKE# zHJFQ411pc1u>(^zYyM<$Z#Ng+DqOnO zIG=4=$0YOZWnnRZj8Bd zU~{e_M2asF?bySerm00?KIk}C$hXjw_n338Tro%q-5|p83A4y8il9?oUZt__S8RSk zIhMs^$K#F0rOQbbb7kN5DN3h8dp~j7roEi`C}Mw!&cnF{ibjxHCk_vh^r#+YYTx#t;$ERbOC` zhUCtH7-Brlu=vgm#Da7p$l9~4Zy%5B35-J!IeMY9>q;f6Vg>POMb9+oxsq_xn);(& z)!&YtpPzoLBV%*ycAWjzXA673`jPLqfF~^AFzviz<-vp6egK<$L|z z|GzE+vr+%CF)CTA)c-fI-^+x7%jkmcdm0@PkxH*ozO*&n7#JG*DJ(3EMlm~C!=_dV zo~z{G;`)zkiXzfKzP`8IH&W+S?R|0PVl`Q(<#W0nFf%*5UZ_%_sn!xoT)nZYSz#-f z_A=yWo?2+<#b6jxDl-*_`FO2_Owf^F~ zx+|VD(r9vwDz(mfoNIq=Sb>01+cEdT_Zljz*NNlt(UIn$8)2Y zbz5Ve|8p1ZJY&Qt4{xBMEsPYWrlr9lDvSZwSE6ZE26pD#@$m36w5pw#mwMB>f$F9`(RYW&#+{gi~xbMSNpr*Y~z-%OGRSG(wvpz@D z0bC9X!L?dYb0w?c5|QyLr^v6mMVp1PDQ_|*zoZreUwkkruX`C7nFBq^Klaz~04=W5 z*umuvN374qnaayFIj5EWj{yM�aO=fgCTy!GZJqk7%d0A?BEFyXkK?lpxAdw{G1! z@Uj@llMBEhW+65x9qupC;OF#s9duC~Di_{Iq(38IBa``89$vT-@Aj1G-Gv1}&kn zd|p*vlZTT&h4}R9bLPPX?>~I#J=tGfUk}8i{V_N=Xtn)&l*zi!2t$x3taiNIO5>fs ze-+~+ly}z zn_RY2zE7S$onKzgDKTo#RVy{l)v5O`h<^(E*#3KkeQ8_gQ3cMQ#@znhu}XM5U<9HJ zq`yBFRD85Q2CMSs{B&35izJ0dHCPTBh0Nf-S)2MZXCMJQ4%XT_w=~PW&)l6hC&u%c zFDJ02Ugud^<+Qr6!Gwa#0*GTEmd&EfZP%EX&r{X7l(YT(@#DuUeOY0kfTsvNM}bmw zQBq8-$78cLt1Xg}mWGC=kq>GyIIvn+Um!~ArHJPDSah)sb8hGN4%qS{$31a1Zr@cw zFytMQ%1h><8I~H~erSu_2ynveO$Q+=-5J{|hxI3aF)>^x#@fAzOzDJNP!`BgLqID+ z-@t&)=m##F<&f-C(a7LAUud({`lUFWdb^)OzyemcgOiZ%aMC^*4v263}>+4%1 z-TW7M)f-hn+qrD@Ki&fAkoTjNWe7MVN_C#c znnAsIDg;hCVC&5Lv!_3)N>W6ES+1{#hlg8W45ADMx%r=d(-ijBO?=GH)E!Rx*`)P$ z$FsQrBBMlNPWu<*)h;lYmRIW|6iwvnNYw)7~9ybCe52JD3jG$L}ir zPn$f2z4Zzn^W*s+zeFCAq-y))chi&P4G%huaLGGN{`!xH{LFFUNdbuAxbRVadwuT( z@r(1n%@uv&!4B#!*75YqSQ8F;hNLGeBCEZ)6h?5YXNpRR6M73#=l+0^zTgUxP(%;Q77C zV^q5xhAf=Yb!;9Wd6AbdM@E|g0hbW)PHX?Swf+8kZ4;4eyFE$%=;{Ay zZ97av0sTJO#7lQNKIMM@*ChY#Bq3L~F@&c((4L`wNlEM*+l}tC(^E~pm)dx8NvviO zFGC2Ov|8SWgg70s{E#s&9iDb59<~%I>A64lznsksA3`dWIYmIoS}UYr(JCDlbG^?_ zN`@^X(o0Rc%>nV513Bn#Ag`dnX54WCytub1scf8XjTtWY^$Jp();A%UFf$6IvFLUj*8^fGq_GbGAflCcRoBI z=5nO!JLkVREtO8>F|$$0krx5%sN;YK$$@}ah7r)KH=&YUJ^A-{**PSW+1cH7FwM%$ ztODiLdq2-JUkRxODA@Cjo;ufU{YdR4M`=MHum8Q_ ztIs#$VyaOEHZZZU98I&*(j)@`g2??(TY=RM-oGa3~I^Y#@3u@q@YGD*x)M_qDQbNL5vq@6# zqpvCBl@2CZ-F-kTgmLM-+TDHm z42c}Y9Qi>fEE^jeHxG|R2_DYr0K@C0a#WSI>yzS{rw&HI>`nOO6BItlld&^ z1beenaF`6)PgkGc5sUN(C67QYxXWbXpHKfhf*_F0$J^U`keJ}= zOCs<=9iM!YM7Qq~2U}Y`fC(`8eGS*U0K%p{MXOj}B&JKH#85n7%nPU;NXhf_Qnp_; z@UM5@Hy!Qc*B%`LIo^`uwKXLe^PLnN?DpBKlW3cEjM%IvCJ{1c`GQMKoDa_UQuSt? zLu)hQ=*V=W%xtA6ML4sC+vmJSxLW8P21AP?kRNA2Q^IKhRzhZb%T-JN>h$s_j36`t2PiLkOq++)#{wrY1{{tq=yINE|XrBB!4|DA_dOZ{x?iUq?S@uK2@2Nl+?bs zdwAHr^O=*)>?c_WA=^q-=v^WrNNa1W<_Ag`Q2&}+T1H3N>tgBwNFk?PPhPZXFgF=c@BOd8MX*sJiK0=G z=;l3)pYXZxLex=$z-+Kr@|E`UEDxW7gMu_uCU9=u>V*{6#V!{Ab*hGr5C{l3ySc!S z+NfM4sonlgv@%?pb+9)4R6Ov*2f4$Iai^(TZY|T<+W}6eWIO|bSxcO?O6paQ^v?|g- zDQMq&SXg@yTX%k9A@bsdSOlQ^+)wlxD}6HjaIeF0lX}60{u~9uXU4%mb6#QcKH8eb zxP9AY+3SivZJ*`>R;661ou4Tcw*odMBE2t5X0`D)e?Oqh!et+)ue(i6FJijg_CJr& zq-y6XQb*G$f#W;9zD{prvk8tbqpQ372&}fI2E#FrYV5yFbZIv~x%odiAO=NXcC|QA zZ5&UJwk;6}+%D1yTuwi>prRyhFM!DX;b`gVT7l1?na#afqd-kf9gz;yN(fLVF|e_b zI0xRnduf0-wCWc>4_+V0kJEOW$6K7dbG42f6@QXTnD%GOxvv$stRVn$5v_N39}*Om z1bUg@P2hB(Xg9bz6IT!Y_5SWl01&4WNTt~OXd3-pKmhkz7_b31#;cw7mU=BPvJcls zRWJz{fb4JWmc3e3s=bZ+3-5r_6JW`*cxIcgk3fk>D5c|Vmw6VulVJP;N)bRfDCVoP z5v1MttFyoTfWZ)$J&Cdb>VFaQxXJ>(xeOScpUmnhq+(6mBFJMoY!W@aympy|uAU3@ z@9F8WAh9I8d)H2?p37(9;3rUBSvbwI;L2{5u9Hj1nMt^b&qm>686 z1?J|DC~5=7B5PJcsKcT(05rbk7+6o#45y3(b6_>}-qjO-f2R_oO7-^JK0H(bR$zaM zFuF^?)z%1nyhln}n4L`(;04CjRkX=_IF|O{neosln3QwX`;!$ogm~m|+$HIM)sDb+ z{?#cJ4RTM1E!jP)4cj7UG|Dagy{etpS9;D@AcYUV8!t@yT)>gmz7^0$#sM{P2H*ed z;lKSr_4(h>$#Bew$VfK*rkep?hZ7!!JUlFQ4;>a0Tyg=oIXD1|y9Za%G4e?K_*9e& zS7Wm`EkH;}$o@Xt-`f1^qb!r$3*xI!j}iv8J*2pu*KExs6&0hZoY!*!m^ND1-Pfle z?2pyv7+hFb=zp7Zqi8c)MZ5Q_ANn)mXFlhioeA87(+`o%nOga(t-rq~GbR5E))nBY zKivtkuAIjocVEpXmFuM=5@wWdXBgHNj;<4jgT$aVKHGU;x&GR5WZtL$?B8 z@@NA24uhoutgOn(qBD_K191L2eqa*X)b4*eIo#Z2I?u^Ulm`;Xeuw>vYW_YN-|L-# zzyI}H+!v!Tn?V5akc$I7okpjwMo3tA9-PLbBi95TH~H=D?FXS=b00#rYt z-+cHtAbVI$N($baCR(~q@bFl(D%lFpD`IGK3xw`#0o=+xoMt1e?EE+>{Wmz^aqf7 zX9GQcX*ny&xytv%2oU1W^D_jS++h^8`pV%JIJ<=gCY+{OhmR z4RK&sDZ#9@Y7a&%O4RdDfSTzz*LpYMxt)-mogK>;Nu(|WjDTFloKzq&xRfk;xdjCU z-2vqxHM0K$ie=W9Y2K48$WSBvS1*72A$cn8-;Nyx4-b!GwX+o*K#Y@J%j@g;z@Wtk~AHUN+Q`4vtg(*NH!`Cm`tQsIImTL1KtO_u!^RsPp63%nM(>-1m$ z`qv+BVEkV(FxN`%-%}2EP=+WmGcyA{>tnBJRI~oaW_>#s)z)i~ZRtS6_24EWdjX`0 zb?{9QQ2SS2Q!}22mv?@CKBK>15$GQhz%<&x!T#Nr02)BV0PKa-1b}{2fQqZHiSA2_ zh33Z_LkcLgsEEn_`0}ebLze77)u!d*QU@G>C9ZRBc=!IXEiSoQ=i*-FX>#%-KtCYa z!=zTqrQ{Jw>@-#Y{R4O%_*;a!>ceNauJ1I?yE${l=+wB{f$vk((MdK3;sK9K>*GsT zmh8>urY0&D7I{E4fQ&Y9&?_7Ci(7%6U51V3NAy%k&ahb3kmU>bM<#C4od|!2Nies$ zSqR1&O{XTWt(}f_c}`t|LPC@b4b#B{D{*9B*b<`@RK>T;0PGbg7i*QLK)ta|yF5qX zAwX5rs&SPAD~nU9Hpja&O(jth*S+j;4El%$b_j3G=Im_b=JwU(!$|4OndU=$=vFyMfW`7yb@!?VjImKbqLhkwJ>oraTB4Xk`% zU?7tFQ`KD#q(Tlr_R>jwUPb}KbQ$Sl6p@Bd`^U*eEOGPE;Zr^Mx1leTvHZ(8XI5#C zPevS$KYsiO5K8=JmESY>m27HiS^{z~a$TNpTpvn3tS?v6OcMmBh!cY@k_R=Xy3R&F zVQ&-9+1X3a#>U9XTPO_>1P+HC*87Sb?ZV!PBogA{$O$2#Ou%{KW@vx6+~5=#9NZ6N z_L@~tCrxBCT`Bwtq|>+_-}hUielbY~A7dP|y$?JRX-RI%8qtdxW z43p22^7i)Xf=)U*)J(E3+Y;Ne!Y@6wQ&e8p_6?neQ@gY+Ey|8BU0T9--OnGbSU3K2;$Swc0u~WI#rFs6iZFFuPu5I1z8Z0_$F*1+$zrDT`qPTy8Hbmp( zqjkk;3-rPTuL-`!z_|4h-~l{5ZdLkvhRktzL_{Vw0dN0k?+fb3wvTDHRF-rM2E&cG zcaJRkwzjm0YHR1^Gg@{Sm+H^z0}FEMj+-6v zwaT>}p6VKg$-z;TTie?bQc}@v=ASs>c><(97&!zio(jJXk?hU%^mId2fm+95dkk8+ zcthR0y6kaA<(h6YW&*Nz_*Ler_IByWSQWoM{>qY>zSQv{QbLMtMJTsdTkLGC(gA7s z#y!1MMy#j%uNb|Gmd}~dM5n@#G%n4<52_``y)Smx$&i+ZRS~Z}T0uxZH8hDY zA2&@dFtCI`o*w`=G%5va8%&q%3RXI7$>n`|1i)C~#&1fwk`In$8=g;o?lk_ApBE(B zJJs)r6Zs>Gu`s>qMFOvrJW>x!iZ^FW_deM(LsE(l>ZRz#V1~eRfn_IVPnWEnYIrM} zPDuhNej8Y~2dU{d-Ho)o9u3|NM=wY;NqL;Ivy+>ho=%p)>7@SQ!v_g5sd6wjuah#Q zTjcxuX)FMa{HEs;0%Br%af#y3>o@w_mGZJ9j3gp&{t<2f6sy;Qg1PdgNZ8KklN56d z!y~QpiOX4qAFdr&^ecK?+YbbOIZd+yRFv@RsMi&%)!B}ZPb8EfxJ0x_%-|z%LOD48 z?#UNHPDTbee!D(&7&ANI9f8WbAJNpwIFEF2F>lI76SB>3ZZZa?Q>BSV;1UthT2u+G z{eBRo#F;b8xLAGdM*pj5hKwCN8d(A21}AFeywADHg_+>&43m*(W{O43%*+5OQbK|S zyV9TDHO$XYi^oLJ>oo__*1Wjo;{QkTO`sUdWtc0Gi&HB@=-A@`W*+UDqk3tC`PLXQ zrY1&|qdemwdy!FrgYt+NScvd!O-^tRUK~FNoJdy9nOwA|~gwy~TD=8UVEiBA_X-5fMiviIMb908kJ|ox2k4O(MV&V|Jk(|A~ zdl*w#m}D6t&%j&sn1<#@t>EP*ai9R@;qBYEW#8wYtfZuDVBQpXJ-@mNLGigL`<}6h zi4eY=gf9_r7M3oyMRerB(5JM4o`G=_E6Q=6`-3Lm?=gdtUAM_4q3L*Xx5*c8KZRb|bj~qqN;d1l$_iX(nBqSn+EkPb%g}{0FyZf>j zxhS?t;jeB>O0#Q_6<;}dZIyxl3aC7Q9fIw|iVzYK9wWy0eL}S-&%h7&%=jJ(PNBhz z)==OIK=G|PaM7=+7PkEWSeCp*NiA_g)FYX~nzO0`yoL}M==lZ^_Y#PUSK^=aUta?v zaRcj?z+JpM_p>o0I?N8khU$G#P%2W0xfeEWVwBS;jS(Z4TXyLSBYj1G#>H){DJ~S> zjvHFuLnY+-TY+cXe;{Xrk=)2EzJZg{0J*2$0njah6c&n7@&hO3*fg!i@etLD|4qc3Q7J2XWM}?Z_ z-o;Bl^sP`(9+|DHd+4!f&MZ@p+XYnrQnmvB@VI7ZF(PCm6G(~D`sR##=BLZ|`R0+t z#6;`hU!DaP*wK=k8b%8}@9~emN4%=gsOm-(-zU@diQ~>@XS9_pC>slAIx$I~S~%`w zm8jDFGC7WoH;*sYL?|&?;+knJQhO&3m_*vbMcJHrut>}mbJY_c5`mFcF4}iZN`w%j zxY3;*2EmOZ1HL{g9qu@kzV=-PtpWS1W$7@cG`=GUxjAS3lR95E6B^zG3_#x+Jt!B0 zB2$s+p91XK9y`?ZaU41kdlOM@E4>$bS>Lt5j583M@(}p1Y185V)@56tspz05S77?) z8SWA5LDi2pkQ*$kt7d6-xIbl%hUBaw-{*M{^FCHQtHK!Vn*_=_H&mL1I4SaLMBoj}PJ{G!tupGU9 z-|y=qzn^tBKUdc7x6TvTUYc>qH&+Me8LTENsQU&{ucP7-k`!=ZTg$f}tULKk@&VPO zt>va%Pdt~m=KAXRT+!x-814R5gO=xPbz0*mTwXJ8$;`f&WSI9=DipN5uQ6btV~ohT5QQGyx8#`KMH&D1~C|-Ktko1=w z?>dPl@vRKO3cC_%hXyh2M~fpV*LRD-Q`OuLe>aX|9Uc{)+(#*6x}HnQy0EX(Um!?K zufxQs)8!@VSCvalm^_klO}g-7SKIwZwb4QNT)46}Lu#F(yEMYVe3fCZC2qXURTRjUe*UGtyY0SaePO0o zj>CrAe`?nU156Y32rDWd0=17i#=y^05c~kQ;lrkxz_jBq* z81F)XtXu;bCAry70O`tl7{9#>)#bE&=~!+0#bB)FhBl+XoNa10YpL@(SzMA6m)G>H zA#po4%D5lmUSgx;@-FbYp+Fc32F6X>p%`ui*x5;q=Sa!(oK1R+;{t*5xoO_P7~+14rFK24ckF`}8ul!$SVEbYK^{^ZtnSz~!B! zuaRe3I!>`!sT>TbVkW$7q4f(zXKZepl9RK(fj-n^km1ZFr%T#XGd|I0$LvAKOKjIQ zIS!G)Hx$&=;`jJY#b)`=d&W~x#6Wn`uNKB0fjZf0j98?frY-98R8u(6_|uMtRKg{4P3rh>|78Bgq((A{)P02WlC(%eWm8_ z&EgT-_c`TVU^ayqM(qpWY2TOF4^nBq%v~rbDC}?VEfXGmMUP0Yzd*gFw)DMp|rhLJ>W3@2bDvMjmpY4!}Y~ge(QsE3|W87eJ zWBB4DPv8~Kx1$`u&Akkre!x$*6M#p%KQm>5M-W$D?tMgC2$@YtPz;UdXZZ4L*Mb-E z6EjyJfjGHg>k7_lIOB3z&kZy2HnYhh42k8pssiAFVDHhVg$ObQ^ISgC!^F6>%j1bM z*MFu}WjgT|2+Us7{&Y@tDI3olIanyi^`A1Ao-j#P+eA*xo=mY=P! zE2hcqnn(h*L15y2(1&817&?XTUav1#RDh&1Yt<{3XIDatIITCkFtgLv-~|;`tX6Jm zWNd71Sc})xf+M({QB+&XN1w0)s-u=uIy*!@f7 z24%{9tS`ln5Dqc1DI6|q3+YW}S{}`h^Fnk~Is}FUIT)~__@*+ju_+-A*870_gVk-v z&}-`}Iw7}n9v@=uS!WWz4m!?5pgq{Rd`1q|mq@mnyE?l|YI$I+5YZXZi{jR3$vz6> z<)IS2hC)>XjOa4wb#*D=PzF(Xa3qd;0}~&K95}d`{%YWQvO)~EQvFw{g9EgcHG!W! zu*otT#(0>(1@ZA9Xiu#B3Y7MJleTOHTiH54i%5@b3FWFWhN?pyR+X)^5pf5WlPMSH z=Ni2qKLXi~fsLuKKLUlPdU5Z2TPA;d71g?Rlm%+sAlx~$O8L{gB3NDEVKh3G5$y)n zR()KJgQ-uSLLE1FtW;}XUYwo5QW>q=7*?X*HbZ9ztl|7GLRZu+eoh zL9y9&RG%0Qbgc3fR4??+SAO)i1ZF_lMo-f4_pt(NuLM4D=V@>BD{3xa-U?o!7l%oo zIug^I_%Bl5Kg+_q_aI}0V>~^2wETH6ul-RGdx3TuF^FMqO%dkuVHf%yg?rsx{$6@?s?z1ngMr$2OIv&ZNsa0&|*^TS4)g13L#V0&GjkBcOZt_YR&?;k}mk6?Dm`$BrG>EMu=_?Qk z!qN!}4jsr73EJyB+2=altgpI*N6xRK6HTimv93|p(-1)NvlE-h`1wJ}$0W~GxqB31 z5ie`@B3|M=NS%wq?bNB$#H_#2Mh@UE0Yj@H3(%bTbM4MfVxguV(du>r$(BE&llFty z=TBrvrql_xXr0$pc9B6_(N;IvI#)|0Y;P?`)O{};coJDYy-Bsqk;hqYnll7igihaP z@5Pyn4x=`m_{mCI*q&*3q8bTDX*Yc}AAXQXkoS=wrgC1-d1HVRsG76oh3ug|>v< zXBGlpjE-7gB;}!4!{-J@9giC3*$!JCYL}|#k3g*;>=)(A9v4Fe7w2}GPqp%ut)Ql( z+O=qWnOieo3O{c2WKXamo{dn7m#P%VgL9}w;$X4L{-6$@7%#_YcfowN5VAkXA2Tq9 zILT2QE~$b#5%KmE))TU}xp$t$+-tE|$ht=+-d%Izul?{`c_bDgGHDJkX+GIZqP5tF zu^CDQNgvFXYd8r{)^x>xo_U=t*S7VMi9=KhdLk-IosPQCs$9=&-n{Wsf(Z&ct&Xc` zk*;W&?yMC6SHFs>_+FTd%n>pTAdi1~i3 z`FKvGJC3WO-gI+mFmsxlZn5fK-%iGf_x6)BuPqs^=P~77=RuW|*|W8W%ZDLEtW$N> zWa6-~33UhWBXK~XGLSOE4NKy&YEgo&B!y!AYvw(9UrzWrRjs&P?4B3G&EX?T{N}xD zg?rs8!d12mLXT5Y%e?C~`lX=Kr5Vy+qn{>^FbE9)re=*DM^e$;d!>GD1+%gs_7r9S; zF49_q2@_~vD7bTp;Gv9*m3~f;Jew6JjOyq#YU`5%;L+FVmw-< z{w`reX-K8oI2xDROghfM-?wYP8mvxS-itMZL7d3m<(!ZW*Y7_Khm6k*=NhT2dz zHIKriC?ZD&tHl?);@fP;-Z?BfPLFX~jTQ+i@SmurQA*3g_-c2#G-EmvxTI!9LzF!& z^BT{AF0fr(d~&#@6U%@0YJcgX=%rNrBcMiPg?h|w>D6v+p`1A^h4&RVwR~TJ3(Bkv zl6pH1krF@1StsH!M|=-aM>HPB?sZ;2K1OhRjn%LiHff9$xkJg8TgYGS?Li8&mCu3+ z`nl7nIw(!No;hTGuQ{m-rlwXq@JP8p$ycwU0|JooCC!O`I?mceej%mQ&!^oyX&m1} zqO-D0wBKU=VizdM?=dlN72C9slMNxKnOkn2(RcUmgU}#wT?H`VC0M=&oApmJsR!K3 zxB0gO5?fXS8SBo~8~IPfitX%h5Fszq^k-)rms;;iC335vdK~o66&obf;oB}H`SkC8 zG1*(3^x-CcWvLmDR&!dStofr*$C;T+eYM=GugYq)%(fYWK7n28+}m54bWrOFgJzM9 zhnriDYn6q>@jB37R`%CQN6HLDKb@VQZ}K~lvWf+SKCE1T0)vN~jACVSwq?KFAKQJg zPiDXSWItCfLn3sRMk)1emGfFQIYhb59BMw0DnbY%`G-RhPP@hcd@RY3SWe{LXtl#a zk^Nk|_wyAOfH?ck41Nc~cYbCT1|hS3@9#dvXu|%5zWJsyp2Z@Xj%bff0X7CqMx*MN zCrD*-|h|2(ssBQnG#g5UO^8ylOtd>-?g!T9>bvOm8L0tNeWB@V~fxm&+7ZDg>@8i zdzXqX3i)p@A^(pSnk=`Ix0lk}CJ_Kkaf0Z;f1z89)-`eB-TRwb@89)g5=;8dpz#iJ~r!e813E`7?f{rR>Ee7F)A#tEHdi$3RLoC*9c(sAsT{Y_zf zy#e=HASOxLZo~zzj3pBa?9wLWG}oXOtF!vEdxz8YtQIF&`^%Fuq>>Le_k4&hyp9-o zx3|)`)`YtF8Q7J(u?h2%yjACm_-R-qh`cqjuirsjT5YtYA$YLBo? zsn}G?Bv5c-_PeYJ`jt0ZrRjUq5DZoL&HAWvP;=d9?ng|rY%2qD%`{4Q_aBm#wxz%C zGbuA^!2yOSH8rzbArG?{Ku4<|#L%(ZUbwCPNSH&aO){FjZhR=P{P%LpYd!P`^RpC;2&gUu%oGwU3^SJbGY@K5< z@bZ_O*663E?ztYKJ1)$LAD|*H=S`%;wqx39O!^Hq@--M z>&i;G>PpmYZ6n2QK~EUQaqH|l3*8SxQV4nNgfbU(SWF5g?w^*`o+0(m@UIF)g@aO3 zvDpr=f|L1bJb+8bZgp`3&h+LXX?MadH9z1rlB6}q$|IHSpOpIWVboO5RTsMwl`I|0 zgeRX-_9f;n1D`XY(@M5|$O;C|t&ag&S*Z+~J#`7yjvE~h05;uuGJ~0sNyujYjq>=! z`!{1k!Nu-fCSfOxV2CjwjjcYs?pDjf{LiKR%m+52|8K}-#*j$88tUzJ&8u43M2qYgK{=^xPt>l%b?5ru^|buEv?zxOLT zd7av=kJamFYipYi=Y7dl$t5!|&vhn#uDeN)6rLPv0uPeCX-`MbAoTg{;>GtEIt2&N5 zbK-zwHiq>6azR_;;#h3>74lTDo)Ins1TI%CtAIv1C)GDu_lp!o0=I*@$9YA`%M)l; zVmNSAZg@`#2F!wjk2#acS%5<8zu4$J9j|qnlpGL?Y480E3Gtd$>P(EV?)2LJ(|qi7 zCl-NAJmO>2d4^OQ`DXGmy{cIeY=^ya?x~_Kz-?vLd1ghgSfEc%PEKLI8M^);lv$(0 z-CID%N-L>+p{|V{mLm_(Wg%23b7;p;;EXOmaG#BZ6LOqlhY<2J_w6z!aEJ;g5nI|^ zR2}Lj=2dpTdbQZu+Zv-2MW&Rm_KJhe>!6?dWZy9WPPD-MrPvPz?&$SZ1ISgocf6og}SERcy^YE+1|A*(5yWbB zkQ#Va0S8G6CP^9LV3UO$@Z7{hvDdJ)%uDn>SzhDj(gS(>s}LL2(ed$XMf|L+iolzw zfPoBl1eq821rfRCk4gT180`A9*2x%p6$Vj* zf`bK!jp;}_^?~#DpUqCY^qoq%j%FF@=}Qe0XIzfdi~K$Znj-NbJ*OFy#&Nr0p@H(7 z8@!GybJWMj$IGAaVLAF4ffACE{aunPYo9yuvDdf-hz9dxz};T!I%wsOPmGiRfJqp= z=o#)aOWFnEVef#baeumL`%B_Q9Ov%dw<+4=d48M?r3{|dV-EA5s)vr%GW?z}c@S^2 zS^G>`$LASiON+yz=MNd$Pu^L*Hg)vkGz#T;){Nt4hWg!;)bsD zD?%jr5gZ02@$Pf&h zgaqkV7@vd{#+Fw|0WaHNFdKldc+E<_hdm@O9nOR4c#5xAjs1UYy$4j&Y1H+LI?7lX z3u2*ID1rzoAktL^kd7cCeGusa(rdtuf`Eo5HPV|9si7r|B1Ky0EkNi!1c3xd2;37# z-}m0{ey(xpS|gC;|I~B#{_X51|K9LJwSm#;1Si=9)kcQ0tSehj17zg#y|xr8)mXo*|buoB_pgf`$gQDysOI*;veu^m*pt(&_d zMA-!Y3Mn@CUD3=l+`vMk?H+p5jP6C!eV(D|<}U1o&)mEkFmb_$zHa+%J$$t-HC$Ra z4g8;n5Y7&r%Hh>Z&x|}4K7Je-$z@|M#wEG;?D{(KLUUtEEvT*ZMZc^}C#U|ac;c^m z@RJm)mDWTmDQ7m=#n0Q@7TBAo+ERm@syC)I+%TvBO4KXFdNQ1u_OPAK@LMcXd^)=? zF(~NO+=|!mdtOAfXCC$pE$ggeq3~8>#Fdy9ld?9m8*B1B(e zZZcXbf6ru6<|*yPcD{~FBX1AcNm>odg8M+udu=FA-1@EBaZ#thOGOSlLJabdP(m~% z4cO7>Ztqbs!OEE(khCB%OmT6C^ZTS^~Gj!hPyFX>8>t*J6!u5$051or}1bwv| zLu~A=ug7io@N2p|MDR;{f}i%aFReT=Rx8gq*SS`S+<0_cX{M^uc18NjCrq0^5HD)d`F#p%i)~Xv1sX-ZM`F7~NPea|vvo z(kmY-T#<@0wK@b+Rlqyt@a-K(cc~jky+nt_s4u;yc<0&q03$2&^Nh_l$s&{U%*S#m zX{7}F#P^@apNCI~X${dIex3@b@x@+S-wS(-GV?V17SC_n79winv*2vg5(%Gk;?QDm zL%6~n?>7s*gWH>?P@TAzDDBYvO_Vbj2Oe&O?3`oErf}Y4+E=tS^Gi8iQ^tL)zBP#o zLF*r@U{x{TzOb+~5Y`eeUZ$@M8;`vorPswy5U-!S!Mrb@SMOji3zucz~h%td##;hic z^J`+KY!ZCC}nq74VML3_;OPlt&mLkNx%6M^Jci zatVNN4;)pl7e1On#X&RS`WqZLB$g;e|F4p}o4?WVmexjYDW1 z`j%$q7DeDrV4%fJTP0@((WpI_47-$KkZIj99Cw7H*~ZFFt5+tsPU4Rd?}hYs7mLp@n8mR-5#C|Hd$U^QAe`rL!sd@|1z zt8A{Zu7NlA{b*y2+giQ|(`(2pP!hZSy)-Q(ZSP`xMAn>x?ltPQIDYlhHnfR1;Ws9w;bNx+>% zZmI1#lnp{;19>i#6$Ui_{D4@>EHL~6o>9q{) zd0vE+#WQ3CvQQQWcH!JZT;eX&UeIc#7yn^xy`Sie!=`5l8`+9}42X@1iAe&hGzCtQ zk@H5@ zWKeHaRh9TydW^eU%`0R_{%+yTEkEX)Jn`eZfbU`+hI30+!ab`(?#GM9YtH%Gp(oGB zx9~&lrG>U`=|SrH?pP4lVl9!Sr;>}?lCJr>60<@+jd8}1}x4 zjaZt9ATY-w?n8>PG3L8CesO$0SHSE<3J1~O7+9uF( z^}dAdfQGJ75~JQKBr&k&WtTxyb#=y*3SNn7Y?N7mLmHM*gH+l$u6N5MfgiDs!!r6- zMCSs}D>J`e8Zy1fcKMkS71cj^YrX$Ud`pf#8kZqz7H#p~AC=oPp>F4vY2RWLSML5& z5y%wFD{H-dcz!`~VH2eL0`YvDm}8-nL+3qIYtl44Wl-SMh&y#01C@{((sLW9&(WIL zbj^w7$x2V2ai(bXrmpS)dG~qcw+%lEQ4a(gw$ZKe)6r| znz2`vpeyBZLcw>{R!K!=dVP~QWS@Msp=c}2k33Z%k$T{47JGei|4@>qY2@^)s*jSx zT7I{!9%+0!eyPEi^w>=Ob$IVG`A*ht-45A% zG-~4S)@sur8F2*FfH@IA_4W`w{r0yQ7T>TEYuKo!wBDAmi!MFhW?f{rF=o|TJt})Q z)68^;xN_6aZ!qhMl>9x5MD2W#Psd%}aHH_7-!Ap=9Ojg}Wy&*?1nZ9s8K>@c+&o)C z+_JHFknPmV-|F@!_s}NU2HZRkVcyYlh!M{VPrNT@rLSS2Q3#!yFp{rulAH>xF37Zi z=X(X+H)K?{i#~Z$zipV`R>|aztS&Yi9e>7NCp3|FsNp+yY$uyt) zmlokhd+Fvhin63G+E2jYaK8MehkDp|b;`NBwPa}ao*wXsg_juoCM1GauCuz@$+3gC zY0JZ*r|Udh#^mVv)Kp5u(ej?X&ef#lk|*#-;$+E_rYpBQsB(TL(wqXklRyC*zf0^MPrNcat}%|JipQ{LtfH zt&5i5D8rIq4&uTUVpAd<|o_Z41I|hSFe}4sL1pEN~}VI@j0LL8DB*v3=ubE$<~h{9<*l*sSX& zJlRtp7Ee972!{Y0;_wooV*Vhhq8TM;P;kZ0A%^vkuSrWBF~Mrv$isd&?hf)8KRlUUE7CxTdl;L)GRmN7i~?O!L zqhiNgxVuF1k_HR)y*%fm?TFIQWrf03*)hG^9DV(2)p22ZSH3x{=Ct)U8gyn+-*+{z zFMY*4ysmSAYEj+8&i2%Fm8?DXI&y4)gQFhu_+oHJC0hF2JDl0gM9$vi*Q+Zh$g#3b!Mj?utYCdiZ8?{T8s@rWY(H1! zgG>Xn+sxC-#l~Xi8m2eNS~KQB&b@48+_i?P4{vxS@(u7f#6nlL23>BTD3hBsJ17xW zwYoy1*n5&8pkpqVsf5AQ9zK2~rFGWnAzq`l(27LRn`-HVa{tOzVOA^fIbyk_Z8z`= z7i(?1Hp#e7v0ZyiI-RGt7z2K@WbOO6UvtlN@DU&U@g#-0T-*12iUq4O8yyF(%53m_ z{gV9Ye8@9r6s4r*8@chEPG!h=&Gl8Qa8j$UP_dtS2g&_;HS{&Xu~j?1Z}k9jvFTfD z+d`Bd^*`izpHYOM4<}(0PPrM!ml~~iA5iURbu{iOcbMAPFrql?QU*lw%h9^}l+!kh zwl3d;r~5J5sOw2~)vPp%*M%~(6CSexfzfDIXx=~Qwr9>qU}Z=nmS5QSRtw_C2TDiP z3Xp!jtIef-J~h^oa3iA8=p!g($G&WXT*Hf^)ReVjjR&XF!lCYQ^-bc~vV< z+IQ<`-mDRZYmvUOpVy<9NQ%iWma-{;e|1!EXAE`ThY}l{yX(#p$-&OJ4duPF+{Ht1 z@$#5yJKJ;xR%O`r9?wX+O=&RyReI2Wevf_m=7DB;S-0y;C8?E@r*qBu#RNnt>@wn5o=|A};lkGYb!957q;^f9Js&!cC$`K(ogH6`1_GGc!?ox?72` zLRcgbNQC`10qM|{wvpjy_{qe#(3|s5_$u4?qGKFUBcCtRJ#YC}HBC3D;}0Sfn>hSa z;gTtH%@Nuh)j1yZpE#nd(k{0O7?evZ1b~7-^Msr->)N7+uSQqx?f(JjeI^ump*~%FzxA4wto(?K_{_3y)u@-73MUMCd7VGdI4JV;G5`%(Z5Oo zBQor4BFaj$*~y9?3tfU(`hbJen-UkSFB5H5N>p4M&V_pPMb)^gUiqGjL^Nm|+7cZJ zIm%H&KAh>R4^i?)TW&_9$tU-7Sl|B}Q71|4!|>p?1E3%_>0`p8a|}+TL&;fosi(n{L;{T=U zQGHqQ6z+~?j3u1(0!k6LYHCP?-Ht5?r;AxAU|<;Jt4fqpu4UGVWAk2OU(>!|{R$V1 zGjHz7%-@@4knVdBXe2iM0=%n@41hOuwNO8NIh#x+y;#20Ijf7=pD9qrpwl)^0kYcz3IdM$P= zCRP?M96fO?t$^pP9%=`h`AQtr09IA?#3?J^rNC`)qD^j>( zn9{w)e)V{$grtuPR9v|d`{CWl<1J?!1ZnxoSc+w#zZ?`EV!}@z<4kbm0YSXr>7wrw z%Tmj_P{!?|7u1XUEk9K4c+pNSGx-~PxivH4u~mS-G}RPAUA!KHfB!+JO#ZjzZbS6) znH#k=U%NhQad|xIR@=qxeIxno+1^zsw9V1J9i>^-?1>5EA=m93+56_rq3P|brg!XZ zlr!tqpepEdS&AAN~o^G2WYgJeOe{akp`MT$(;=31130;g<8U5r({U+fn zt)-6o<+pY-m8+G9INOpGw(pM941x!FGt+Fhg4wCmqgcOb-vca9)C?V0v^Gms^L6%89rJNEf~TT&}^Np-c+^gzRBWN_PfAMh@!Ux2|RvbynN(to7 z5fhTdQMVC!G`~5VtuCxng^VLFOC1^X-rhGD{y$pKU&6NN)B;qsOsm<_z(1E~7P;AG zTAhaO<_5?`2^rrD*xEW&D{sFijhz+fQoTIhhidC{Y$35h9$Eo5)Z zz5mZgD1VU9YxAx#rOt!As)dI8du@JM7tc6{de8EOa$e-;e4X zp@H+4^o7jjd&gFR3^dLrmxieikKt{(U#5c!<#r52d=FO;qIQ&gnhHIYZ820)?&-u* z_8zRTq`rb03v_QJ(zvV_o>;EWw2?<8^l1Af@c!eoF&eyIJp# zpA5qec7t5FC<9Ppe03d&Hbcp;X-V$A(f=Lm3BQy)Q??5*c$`JO9WS5g%ByEoV2lmA z^USra8l%fIMAWjM=%~J|ZVlrjNXtl04bV8u8K^0QoQkTdDjYsy0!%iri#Ys7&{uZm zXl!i__o&{Q(z(HyGIqX$+G2E-*&U5S>)hP&bjKkN&kC<{72|78(Y%IvJ%b1T_ilS5 z8idiX1=%}SD|iIMja9O9Yxkt!tJ-{{7xenT*D!B^^~J@@Yl2vGVV^n4|my7#_|xbFvogToeB4m88rEzeZYvDbx&6Epb-Pcqj%P zoM}Jbx%Ff3?(^CwA9vRXr&39Ja8iv=5u`0XBJ`O3jQ|!Uh)-b~uFAKZqMVzeSGu=@ zKzQccY^M>%F}`$RFguihnIBQuzO80_ANR03r?@v-L_g2I+m$pJPMJ%;b5x-5L4ob? z@+9qX-(|lqF|bDu(4m~3fVD;Jje46dpOWWuQkFlsTNS2w9@!t&(V{K%K*OffK)2_C4t zyCi%5|HhAeaylHE88Lbp{REU&VwxF)_Hdy3|cG_pI=SJy`b zDE;euk!fp09|Nx;zwk~E&wb(%b>p+JfYbx26fbZRmJOcw8D1R zTF7x3iTsbv0%&*b5%KtQQ5ieOHrcZeRMX&G+r?z8i^Nu9Sqy zrpWdo5D@18i)$dmhP3E|AvkM9IHnRFKg)xO{|#} zyn@$UM%sKg@TdSlF_#hV#3X2SRr$Cv=^M!)CG5>ltPAHsukUm}|8vtnLnaY6f{-0~ z?^|=Aj+ZFqb57xFV-IYGqJ>k#g0QdkVYDmPi)4KkvJeD3(Y$u&PS;EMo6%ctk3B=_ zvquf&(Vxd^-Q@ULZhWySF=EoCE`oF%*#!)egQvu2D#GdDWR@zvc<<8;6S2ml5MTm8 zp{4Ixnw{l8%~p`S>{H7ulkr19UG`k`ZOz zh7edk)uP2EDIh6QDV;Ie(y~CPEOm6-@NY1&^)BP~+miC5cNSP$6+A@oUb?U%c=FDpn? z#oVa4nJC>)B|gq(arbo9{o2)ih3Ok14sX`;NL@dl5EYrQ&_-P<^W19>GGv6zRepYg z<)VWBdc}C-o8ZaDSrUhNSFy|Z0zII=?2dA3doq!7d<;=Yjtcq?dJ7$-xZCJ0y2qO& zpz(ZkN1bFD2?i~_G2r(KwgH|QHHAl^EoxqY1e@jyj1|($dtk2>hwC8of@uDLP#qZf zifuov!%VC$3@U~;uvfbPI+7&%A6j>i?ElS>?wRMS_38ca*x)I2eRS~U9aQ!}s;=y@Y8 zlc|+-*Ppwq|4r2iJ9D~W%0tzt=8V=>^Poc)k<69jB3zwNc4Q;j+k!A-V6S+NYDc69n~`RZ%D$b$67n z6mse5OzHij^A2t|P{Ydu%^VkFnnyoZA8z;xB8GE!IrFc2o5Y60sPCtmFHOey9QoaZ zK5a}JBz3O8#p;t;)Sa#ah)~3&Y~&~1@_Ml*zh2lD!VQ|$ zo@Iu@sXJG1So~|Ed9@buRAc{M<(edM__geTJ%%k){u^s$7slx)mG-VkO#_e+ zE|SgGA;Q*0AU;z^nbXc4zLzVZfRp~X0RzRCLl$F<(kcr(sXTFbCc1|@9x{Pv4CE`Id6Ri%n zW9|L}dtx;*4)Ew0zxz=s^kmR=b>#vaxZ*Wou z1_lBu#m2WeK}e%~^X(Gx!b<=#M7KO>DLrF$^Pi&oRz(>hoex^7RwzkL|GazYKX~vB zS?qtPp~+{Oi<+bY)hiShn>Pgb`9SW1l5>l$!8Oz9X-&k;>#-|d9vVTdiN%g9_f$I! zI&pVFYn|`sh?PKSb$`c;BeHQu=*k zSnl64O2eG{CBXyq6MXw9#+5qb#b6GPND%>Um_?gL1zHa~SB{emw^88=lj- zL`kQPygXy8KCkJXyf}r#Usos=7FY8UwWtNceBibzD4+N3Jp}x;AiYd`=WDeEqUz=k z>7co@zn2#`Z<#blFc`{sD?gxlPbDyDcX0W`p>z0_)?J8NQpP zjER|MGBqdEcX$Ax0NBY|K;6Uue=`GsTNlN}#Yx21AW*WbKXl2o;w}?!S#waD=)V#Q zlrRaOwCxNjKbxZJCPI9{#t66 zh1x!xQU8MOjU>w+)YW|(bFKya?CCnc7mAdWlw-gfIu@b{xg}u32Ru(AHwtHaEBH!n zdO^&~s*5vgK^@?YymTWDZY|T1T+TR=NYjXj$d{ey zj*N>VeOL`N|8&nGEWoKZ)q#wIONAm#r3iUFdL{Wzd2m!Are@m`Fn-6eN5!a%8Ozn+M~`Atc+QT7<*8HbcRFF(Gj0C7Mh za%+d`HNkLhSxbrAF6~#AZ)c1wguePemKB)PfLCgGSahNG0aEq=XeV5J{b?n}u&;@P zVuh>zcA-<@IOYKo&)vIOt@_rQRP{U_Z|?{~#3qVeH{IH5P}shnARv|>HajpLfS~^P zqqerzZhIJUO2)M~$#0|f5&pbt`<*-cTK{oGsHp|ll!~G3beE&%N3yfC4_U&-fJEk3 zITiIEOT=2uFV(a*!iJMx@_eOkw4qbd>>4$5ZDE_aaH2g$?b^62o% z)bL^MPM<$lxM2QnsuRIRDR#zh zF1Rs@H!B@z3&S{5#$uIv0X%Q&50|;J{H3KO9YSj&i6ohH>Jv6_6L^+9Ac8;2CVL^I zc5L-`>4l}0sU2>N5d>gsXow!(CaHkK7uRen&h$(7oQ79s(cDR#6*KU--(U*<#HzvX8>fOik<=Y(y} zrioK3X3VzL;sViJbv8e~O4e5A6(U~JJ`Fqu>c}HUj&@IPOtT}pQwGyO?<5WO)C6e> zeDKTSvh4s0V#_CwMv@HL-9F1;5CeTD0uA<+x|cCLH%vS({R=q~W(`EJ3~rzFF?G9m^C-6x_Lm^mEd< z<$P_2t9%BEY{KW#D+4a6>SbuOgZW`a_$T1x3B5ZfFhb3&u}I)sz1_N-_WW0=dXMgG z<-57~JqpGHMyG;!12X2vS>psaV zXK(7s({6wT>CDi&3Tb|Bf9{iyXXI!L`>7D?9wF|ysdCjcDki+b`zBg}@?vWctFCLb zVD`gR7HOIr>)qoGrwHKPfj&q^>EI~ z?6B&X%lcMd>(J1NpY?w+K0t#a1PUR0bDT(Tn@tD|9Vl5KC~SWg(ysesH|s3ra1DtB zw)D=nv*@|zS+SE;O#h>S63a|wwm@SpCkMBdKpPJ6rsJ3HVAUiR0<}=GQRFMDe#O;l zk-~zi(r$CbA_)p8tLI&e&q^1I;WAYA)Sa=}$=V9%RR^j( z^Q0y1Q}{8%Im=N(LefHN1W6}dur2V;CgSD0x4Hdf3FE{4dGWZ|D|$$A=K*)FKCCh2 zv}81l5Ue)prgoNViG8m`hRSx3$8z5l#1FFpJWp_&8 zKG9^|a<|;yM@d=50V-eA2YsD&p?cLE=FQYHxy1@~z3G++Ud=c$6&W^89dP;T3!5%d zXSWz6C5)T&R686mh;NQE?O1%OMKQwdV3hAJGn4DiHZy&?V~5&pjjrt1f0CrUb>IXg z#mEXyI}ASHkGb3TOSNQhGg)LH)hFtFpiZZ7b_F?^VR7nX=w5 z0=ooQ)DA)O;cxFU@Qby*0^PfYNkU_kNFF#o+LqW9nkj1!uf~`B1J<(p1-HOjW`7@( zp97dGfCfQtW3KwBz$&mFs9W&AXtwK12g18k)j6>AS`Z8dh~l!qKa&l8wPk8W;U8VW&_RJ9wLKDM*LMKjKa<{rHaDN|MH_@1zr?w(l8r~+go(I4utzb! z{^wQlu>w4sgQu2*hY@m)nv^ij)7_PpGDBU(v5k=z(PbY_kZ6cpL>_J4lGxdBR`j9A zZB^Y$5ih%qXT=E{9py!o-lq&^Z)9V`^m{CkNVd?=GMTKX8(jHE8!AG#%n|Z;kjy(0 zqwqnweNSGdTDL>i{elNTPU?91x`+lQXTEO{Hegl)nX-UG!+FLr9*)eF!#0$;qr3DA zHBNRr^xZ#eVR1EHL)we_7VL$SI*)L(j+ZFSxz1%*XoH)$8s4Il6y=^(%Gs-{6Yq}D&Q*@ zBj(A$m60nvl;IOt>er@zZp@0Q*BpTfy4i1y??lnrvY-Vs8RL}bPY$Bz=ted6N=$7f zfr;Ros%Fx9Iqz&GU(+gMnLS4jouAW_mGPiWAz9xoo?Xi=HV@cD98zOAD)D02l%lRC8Q*46A(pL|R(0K#3FGw6c`3*oN-4+(qe8 z;h8IN8d~A~pEauVDYRkE*N63oD8c2|j#3ByfK7jdgvGZ%DWu_{>~nL8Mwol3d{gFZ z2O&eH*Zf+p>&lwv%!tlkasrmTcB|W4neq0$J@496J*EbwpdQThQ^~B6@!~k+Q<9z+ zULaDL&38TM)NHP{!oS{@+jtnw@EvvwSH^A@D!=6sWuA*VJ5T|xGkTT29+yC1C?g`* zERIPR-G@p*B1dH_K7p=2PR8QdDPX6M&Z7@E8WK~ly&RnKBn4!weGZY7C=S^+-7M`fO4zwB13)X&;@bv zuDpWi3wLe-Ro(z79wa)IOhm&{lLG~va@R1(vrv}vqq^HQ0}Z*w_o{aU*+N<7cJ znaRq_%9@yM9(r}_b@|&AUWIA5NQ1&751OwM9{IR#LIA)&It6W)+1Y;Jabj0%*TjO4 zOu4m!--C~zU+DXzJNChLzRnt=I z2-6Jn@ibRZz+SOV@TCTROMdNZJ-y(LM^w1#uP~llVmqX~-vHhDd3uinKCF0*i{sq6 zV)?Pm19U(7dKTXD`RelPJIkJk?c8~lb6f+JkU$Ud*-zPuS1eh|Jqc}} z;Lxxh=iGp*40D~wug!R5wAR*UV`d9X5?OTz{K}h~IxiBgK(bioO22iHAVgFlk8zG% z)P8;wWEL^b71bAC-gIJ^?sz=zOYOG-JnVUJ31z&MUmtmHqEJ6cCN$}G$d!FFjNuuG z>%Zcbb9r)E>}z1^F|!_CQJ)Vf4=){%k_yqkW*Ter>cQ|;Z{LaU*-&i*HeQbRm4N#4 z_9evsW+okb;bVO@HZmbWIJKYGxd<$hU-li`>j5`9Z)TRzsrX^b=SgiFMboRZ^{Crw zO&iYrW&O$FC=^OL^mJ+v&&d&;Rp}zPVFVG&)bSYMyt8@jh>O?(48i3R1#3!nJAsP0 zf?VyHd2ioUR-k;yPw%kJc_1gQPH*j^xotnyAi70ZYwD+Xzj0XpFw1joYQe^+x%vZB z$y{&owtwt8{+Z&lsK|G=KcALw*lK;$#PVjtZU499&n*>i-6}-i0y?j9coK zBiIfGvpZMoqw|6*yze}{`Bca>ziPmJE-NYg0Fz83EevHkPniJ_dVvdKkA<3Y^Q+8>72HmXt;Q zD9}~r;}$d4@FoldL_D6G`Dl5%Nqhi9-$pA52`e{~g_8l+B3ACjOk<;x#+Bi{P=uX;**|=SMj6lAab=i-`ASNQ*>&GXpUa)J=&ulVaLG&#c7}| zbDqdL6?HM!t2f`b%0P7Yo$YjEGk{vX8u^>Z% z<0v=)RN02yw4;^OZ#{cnVmYbCJkctAGR;*~3A?~A0<53!v)+qNL^BY&UNwuW*Iorb zJ^3(;lRMzq+aM&NyVy=IQ8w1|dpnzCtYYtu_(VhnJJ91V`l(69{&lw6fYp|AJxuxFYZ&GwfYo|BO#Uw$pc z-hO^fO-<@%R~tH;__al~V}dKvL_dye@K^EtT0v(HcL}vrJ5Eg4|J{!ti{8$b|*h)%U(eY zt%BTtpxCn|N@DE|@LpmrWzLn@=Ng7Y+Y>gl^z+{Ti1Q1=ca03%%*Ez6ybAjAj&uU= z{!-*X7mRYa{&$G(J}TT$1{Yw z9daC}Tjw?!ZoGVHWBNx=x6k7ZsHHu&UM%wtOK7$Mp9fx@0rrpyyUP;}BK(+RrQSP^ z3ByUdqtMD=pCh=Pu4r>V^Zl8~@=e3n8afrs)U2=3P53{zLv^qF`1*VwnLVp$@%_6` z+QpQ`+4nX|O#OyzY`vr~?tkO|SiSJf%6!qI^L(Dl*v%3#&r3CN(BkK{Y42G4_P7eLIxX25i<|M@O ze;{$UON&r;G}%YWKE~nh)oXq}zTZ19nkE@++mLbNsaaI{ut2Y(*|fuFUJS<{o5iN4 z+$L-OMG3zDV_$f%fhg6L^epS+lRwXK33lAPk-jD*!MplS`N-{Sm3W>ns?o@u@sEzk zlY5(b6UF?dnw^m@sCqF0;`h3Qz-y?zL+;J)(>*0_HrFz0AGE8Bh68~x37d~I?u{^5 z%-k0xWKWt3ja#K)jH;Flxa3^_RyBw?cErd;GftwrO@LdE@#-M2;Lt_16QihA4HfTG z=D^XNqi1{+$C+7@+jC*;ZrPyladx&#`O2E7a)}>z*bk)m%t;l}4}1g0Yl~|Q!n0^x z*vYAIe!v9Rzpz~Xf$*LIpS2}Oq7R=$xd7HP^H1zpj)`Qy{eYhJ*eun zA3Aghmy5f+oLlq5CC=@diQvY>d~3pk2(f!PZ2g#)Awh%KJ=e( z^v^si`55)z?=qUa47lOX!BUi%kXCjq_gM(L-i)Jgg3>Uw|JaXmSr!8)(|biUL8OQ- z43xg0HByACt{}Qg(h@8!y`#y>0qiXHd`)AYBCj4DC-o(ed z6A_TWl%LOwcclcXk1c*>yc+z9{qJ`q$07ze_mH%Bv%=pH{!`k|aqnc(u4C7~1IPbC zM+<(>9}S7qEMB1=kluL>L03aIMhm4|&i4|jgO$J~f`g^*>Dmx=B&=DarJ*Pe9N}sN z3SQ2Hqr&FWb4XUTE>@4ckPOjp9QPuvA*hEolN$b(`q`pJ9OAcPy2o?K!<$QiF;6~? zm?_^x#;>h>nfaqd*3+!Ve_2wl{QHBe+0HK?zc8I_lTZwh>W_u9Bmf6wbPw{HKJ%jHJ9T%)_~ z_1!g!+YcT1b3$rof=KRK|00)+Rm=o=^t@z|A;cfYgllgZ)xU=k;A*s`rtnLi^lSfR zVOf5!DrR$vP(xe&?jJ2J~K*%nosNb(o=Q1;7^$>oyR>JjUkWW3_SPwldDP2s)u9?|H{gU?l z`$jnbKV87@FM~fX{Fwn-iWW$kcJ;67aqds4Vb;e0qfH^cKW}Qn#d1v6#l;0W<;WV` zJK5#0!H^Sky%>V1NhxZ-zfFTF1j^{Ry1JHz^_MurnEB4~rE(oPk^&JeQmc@>h*Ts% zye)W4JK*DfivXca8$^>LXEzo+D1EdAIPU(zvYP+^4l4{H)DvXS?59r6eGS%S2G9bB$E~jyhw$Uj|IdXAzL-U3 zv}b8}!F^84cQDc+1}aD+e7+++MDaBZ;;0IbrQo3U%Z%5t4SMk+3$Bs5GMSt*pDp7RLLnr^6d@Cfod!`^jZ6Ln#L z8bYv;E`;+@pqXbo-I}PTrG-mI4LS8ftP}+jz_KA;+Nj($y=EFXelT%L5{w9tRF($k z%z5}xz0jh~g!j^9T{P52hFC`0RJ=1Y+lL8W87+bmqrorEBB2qkqY zW=1bVh>_js*O!F^Qr#Bh;2wna(6a8W%k_Zf4_9CsKhE#|^@V5r{x1^-ZURG05FsQ= zsw0`1LJ*D#8Sap3tOyeY6pN9-&v)`&cz0HD6%u(TZkvKsf;r@YGs>$9$7i8CDKB>J z-1!hgV1|M~#ME*|J-{F|;XsB24p~^&Z@w!_8oV`!speSUk*BQgla1`E9gvC5l)k~w z%kke>hNbP^{rex78M9#!n3c<%cm|QbAgE_T{`|kB4f!bR>LSB<6q4+~Di?^|-lCt9 z_b=1n*AuzR52}oLa`kp=F_HP&AxkVOo*Q zL__u_#IxnWuw(AYAPIt@84V=?cwo#ZQrM(K)h*^F-NwGXc|;+1=f*(jJRelULc3vQ zXEuft<^1kJG6HN%`wkqq3&Ttw#wD;KF4gb+p9roC4PYibUpW*|KMuJ9wvbRr15qIb z6!bF?%nMI78X7oi(ZbmP8ZI^U!;ssONQ7eu!Nzs}mMyH1e(9sI-U2_uwU!N+xNCXl z=%T}!e5#-n)ESSy)a|x(raRwUfnb>=@1a(dxs}ze9zEy2F34vRW>CBYJh^zr=2YsxnrD0GM^UF=W21bd;b zd-0Er9|uq0&3B(AaBsu9L-UwaYuO-4k_Iu_nRrwT9A(O{M{+hSn-26=f7)C1;pokpOh3Gs?qsuBG8B0pP*Pbvo{`|dMvuI1PA4fC)v zrSrNIN^t3E9I8>=0fz~L0;GL+F4~jLQ+G%?lq<}1g`j&7?i;q~z5*4iM&fYAW|E^Z z0G%A>m+5R>bn`$zI!6>6bi6g)0%>>RB|r7{cOFKjjq$1F+zE-EyYl$6eQ(oN#roUg zTz%8SnO~je`;=RZ=jv;#hn82E?USK$ML*9+ox@x*#a{GlL2g({r)RzgIQ@;M_pnTG z!9CTUh=bCXGkH$;=j17yhXH8cy&PqP@eTEKA|?YOuHx89K`%Bhz%%BWY%BxnD!Rg* zoB_~oY0gUjS?5z-xv2KxVw<2wgA0Mi9)rX}aK{kOs?N!LuEuRr=b2A$VIfv*pun-x z(}|FrNbs_q2I`MmOhJ*ywn)_Xmv0lf2&E2n;R>_il+*4sqdKiONWQL5R{JSDvN|5~ zy-m~)qnTQEv@Sa%+=Qgki=c~Ju2r{gh2-8#9GL6JV7TWtFQ1h0+WUK7a=riH)-(R_ z)0wKM!0ZoMS*PW-wbXPsC!3^?awq2jbaHMTt)H*UCnq_jFE>~j1G}XWL_t0aIj%qP zaz8R{w^%@ro26Nc=cPcN>$ZPnsq5lzK6l#wy8L;H{o*{dbwu*c!ZnAdiWV1;aQUWZ z<pSLm2 zc92I*IVhZ~a=Q=Ae?oSHo_Y{d%=V;0KHNnDkh?;583kO$oqKh)|Gw9Gj+p%@$y9KU% zp}rI=5Au*6;I2uF^B>PCh0KkWly4fsv!A(wepnf;$a?71jyYtPbK4UFF2Cy2y;^(e z7}l$vbvPQ2X2KHb;0ig2jk{y?d62l(*cz?(^{6i+QL<=Lcr;TGa^|}(N)TXW~O#xM+BBOGXivw;;okn{#L(sTAj}? zl>W|Iu?)9cd%Eioy(v;Sd!Rcf+jjJ;T0y@%_3Lhu^y3@T`Ir75_TD?HslDqK#$(r` z=ut$FVgVJTC3S(wh(!X;LCJ^oW!|LXi@BlDne8 z=e*DJ-gk`euWyWduVZAeO?Gzn{?)bST5HZa8Lz3x9a77&YH{KzZ8ZT+P$%*?HgCXb2Bg41@^YEREg*UbdrWD+lumfua?~F6lhi&aK!E z6QL_vSMWYCBv@T0h_fWhm@nDxJ0_w8Hkk%NL^R7l*Iem!7zlcuOi3=H&Z1p@Ox(Ki zE1wAzTJ}PkUD`0&WAbY?3_>JtZuWAg*4JW}Rc1^=2h?&p&-JnWczy~&!~rX}@WEk^ z@2kuZBd@t%Fd9?HmbC<$zL?QCDc z^$fMtyubvQ6-FU}9hD~D)2|m?iLy7_ZXpXgmlv}%^%wg8N*PR3M;k0vIV`a`Ezp}a zZ0jc&7Kz5>BE+mr069$wsSzB#e6~#CAOx%8LY{s7xOFF!JHIJ@0w;fQqphClAV-p= ziI*`H>PbT`o0v)3_`Bxl3kxMzIuTKn!hE9&6)TZ*i7b+N`(ctXC-}4uF~i1bb@$}>e83{X{AniV-n|`l{05yHecy3!-}TzASy8-F$O@qx+Z_dDi*8@2 zTAC^(^i&kHcINS`t7DF>0hgj`oUlCRJ*DC=lwn&|M~2AvVrLe$Re9K~1g2<}djua_>z{U=?#(yqs9x1aw?;=*2*{ootk|Gz z4L(59JQHHmZ-U77c{P>WM0M#_Nf|80`Tik}6jUrk#NOSO2`pn|T>tFk%i&XO-L>j* z=Z!hvFTDWv$Z?gJY_Ll*4XBUYyk_vm-uzA8vMZPz{RIPpt3z9|b?^-<$w`5_Ii;`E7tYY`W$2@4a)f{VH-9nn~JEhwy?foV6hHnLwSNALrbsswAFDa$GVz;ra zYVL~-{6MsfOLTO}y}atJ>o~{eQvuZOOuWg(KA4AX)anhb>PaEjZj@0*{f>RNoVALU zW(PAa2$B@9!{a;-U^i-{^1Ux2d!Kke)fSvc}Uk<3$pdgq?WtJwcfaPbb4m1ope+rt_4?i zS=(mlR3P{LwR0PzjfUdO;O3CVDrZ@y&)ld^luvN)OA3XjO2J6w1_J$Szq(8Wmp5fe zlF(U+1c$>;V#dSEhx`{3Oph>Q0-^f}d|GwVGUUvsr2$Xo>q!}q+} zIpamdO+@4INqCkj`JnTEbXG}8i;Of0c1=IpY3CjUN}(`fFOoXcMLbpWpro5YDYyV1-7Qv?&=Tb}Bb;$Gt)MKR<%66jBwJ_lRh3Bhy$ zU*iIc;7`DPp7r&sWZs?i@4#G=8(mZQ^iYP{U|S?IIfXQulO=dl1bBzYBrL&$N-mw^ z)F{e~XOUP8Y5kdB1MBT}J#d}lIVBqI7*Wm~I+IgV!-8r;8KJ6WHY}1k5I(6C^2*Hs z4AgPtT8r4gve4V zJunO)rp;u$zDTh>##pgkumjPm^K5Oc7)^ApDBqCTe#z9btIj3c$EZ&2wt&Ty*W%31 z8|zUoF~mm88YxN-=6&_XsIX#tfIT{Arpj|@UhmVoeU+)%)J5yQYa^%C z%1)SOn8$Rld9=F`a4Ds!ot~qU%<-BL*4l9{?hU37!+#Bgu&veWi^AZH4&Fn6Bg(QP z(*y-|4$;M~#nGiU#CPcW4@KL%mGP||Ffa?H#y7xDnO=i*Alup(dNQKua{Q14i-1?F zVh*!lThtFAjWdz+jrUol+Vw6+=^E>tKp+$~uHBC8rY!ZVu7h-manrcq`MwmK57EDI z5K-jw%}2y0#sddP-MzZcv-U&@dXpw5t~-3%1}vpK%Ux(?W_M3_#cJOJ^j*&#$? zewFi(A-B1u5hZA{&b8{as&>hhkA_IJ_2l;=mpRH7)(eC&gC%c|#dB7~@9KV5g9S0D zK+NuAI;c*+Ld#&+DV8?k-5^@9CcZ2>$GIh5)$VVcgGh<<8hi-mAmt9 z5{W{tYx`C9%9up91jU)OoB!-rCSIL}`TSK?erAg)K7;pK2eI+oD?3++wMhBF0tipD z@S|-of2sM!wZFi<87#uBI^m1&QLdzr*ZNKEqw+f_kJdZE#8u2Cqq9|rm6zA$-Ta4%?^Y0FPplZqwz^-+J2=58(c?_*!uH`Uz|*%S)gB&c(`@Z&2_ zx3#D)7qpzH@-VNAf7~!QXtc%0=Z56Wyg9RulZ|ekVcYB8LT*JnJ3Cg==qrIdx)LM9 zd?`HQ+FZ=sjde;Zg!h-@S=b&p$y8To<32wy^1;hYc8}Fxvmx0l7hKc<))pN;rBVS| zqobae3mxZBbxT*JJuE5&C?D;t5W+7hmajY71NwAP+6G622>bCKCP7B}$&eMzn}v>33LWks#a)o1AkFVL++29axEZ!7lYQ>3bXMzH%@Cr_M|$Hfj)`n3FJ zC*;}qm(jDJi+zGJl(LRG0iD!m;N9LY%4m&?NgEmGGs=cOQVa(JXbnh^i=yWX1Caug zpVE|XR2pV!2)rHdJ_2A8R1Yd@yaMv4lU$!`ndp{-94r!#-S-pc17yf=JL>ie=&SJe z<1C9hc0RB)Tm7^OpLFaDV7?gBosk=8-+6K`@w-8K?j5QPxdmG*X<(=lq{FYcS8r0s z_v$qLlE($tT!#KIQV~PSZuDlviMqDkz1}rby>w$8IVoJ=JOEB#aJd(h9|e4~yM(R7 zb<8_Nm)gFxqK@ab$BWr2Ak#+9TU3x+3eH;uc3YYHC+xxIr_>8c?^r#wX%(owk=a#w zZL&QZSCpk$KPz+o{+-!1rgKd0!&lPlo7Ke*4h#bh))^gp0um1%U_V`-&jH{k zPS&&g9?r==P9;5e? zei}bKOpiDTN?5#f@H7r$F1u{e5kJ?#L7-LHLZPVTHnIT1ci8Eb`&8`R(teS(K{rr4 zKgmOK>w2d)?lhbC?m@q(X{ArDet4=T)Sop6aAtRtmzz*G_N{hhCh^|yo5;VFvbdhE zn#a3|L(luEqB9IY9;k>Q_F$%B!)|MP_n7_d;wwl23)q@5R(FW(5@RhctH`)_0`GI|vB{pHkFMh_{w`L6qkP&GufYKP z-Z{6vpY3m3lySZiv!G#fqFc5}I@Q)zcE#b$D*j{n2GEq(jxPko^*~!HFY_h29uabF zUlw-?42r9n6ml#OvY7jNlSvhszaYjUx&6rR4l5yr$_7tM>^pYJd&3nH9H3M|5hS(? zj$qgsEOrCxgN9!?Juhobh(ko%C5j;88ltu! zd#6(0lkZ+wA!?H(JPeT!{!(@j=?ZWKQV3z`@im!x)UlB@Z|=hLLDt*WbF_oMu=W+dIvq;oRVL26+4a7) z%Z;8?xi-E)VG;Kkt#Fx3D|jbBsgS7;87KooY`kTqvB$1=*oR3#<6=E#(=b$6(5(oH z^AHcqccNZpXF7WLERTx-!sj%LWX7qJMRUuf8)%rMHlwKnHH+bMh7 zg%^7hqh>^E(H(K&-Rrb!AZCB7(jxcpON_8<$cFdlB%=c7ES1pmY7EI#p}oy{UCHu- zX@*wC1FFpH1nE`T>cu8-4Jm80%HdsOx(&QG|LbWU3 zK&*InF%0tJY}n9b6#62$Qc?e+%^NUlGzpGr6Aj9Bu3%ufyL(!C1OZtuHo>>j zz(&RPBKL)hnFgf>O9`U(@fVPs`fnp=YZ8p!?ZS~GojO`!x!7Nm8svXo4q85!>}^hwjnbzU58H1Oeo zi&}WCrs<1Wg`dK^`7aL7BkXW_NZA!Rz#kxpZ{3yo@(WTRH?r(fZa+*$L1XY_j0>(= z3L1VTun3RBDcS9ti@dsoH!pa=)~zPU+2P47utwU06mbB$QKgKYBzO_tkeMk$HIYq% z`tQr4OYhIVmH^K(UF|3eHH6T8kS}I}q~a@zJrK=J*|^jwj)^|iX^yGUG66(~N%S1q zO<$^CoH(B%aGkZkv`RpRpmAD$AnM&neGwT|UWwX_hSBn zQV^{Mj5-n}t^BM)!1N55BG19IeQ~6zuSqiy{O{+dO*1~6q=&ISJO?3AAk_#iF_DuC zi3M1i1qAsg-2S^kKRE# z*%C9ZJkTE@TJYal&egBmG)6CfcAoXM9&;ZBJzR#^2BOE+!OkTTmUx4bD z5#NUCXdh1tJj|1YO2;PeR@V?Zxi_ZEZDgU5c3)ax-6t(VUBuFau&>3hQ;EdOYjf4~tR zZ)KVqv-l=8HAk1dPmdj2bLxKcIc)InwXd*@Xe_07HsDh^NO9z3Go53djtXSP72&!E zD~kb(4qgf~u8P&EiCVyOE4C1@*e?k_?Y+4Mr1iIYYxfCS?4Y+Mk40ho1*=kDm^VqV z(FAY~4G;gqpi)4CciA4zlX~w~M%QxaM`@3VNqLU84p8i=Ut8*W0>zy`&BA5X-W-)Z zP*HsKtC#WNgs^X41&orPzEO=M97q&U1Qi{RSt>r?owj>M!dx_Nht@Kh;sI(ccTC-2lwMnIUzais#(jq<|q)QaF38Z~y`!<8~0 z{R+cZ<9uJUjn<1C@y1m@`i}8xQtDh*58eqO&U^mJ_SWy1Se8P5m}GV%3-LS3u(4bO zcALRrHt;%xEa`$sxHknF~uKYMsP+XA@HuU-9C%;{vdH&I|OY3W5Y!aHw zTeb#G`w1gn4%(rUFz=m*2_nR9?J?G_I9DIYRVFVNs*5;vDT&zh->@hg(b9~X9}~H| zcs6&SO0xq=P5)SvpaluRqs#}6LkI=HRu}2D<_`V$=C6hleiUywP6Q{HHwq8NTzCcC z_g;o;iOv|}(Bmkd6Yj2cs!B@7gmdh>H__Y5M+vBr?^8`!KNHE0pJqC&CPU5w2turQ zohw7Aw08ErQHbWu0*4m`LDlp{qdTmXXM>rIzv_Z0BWlP)`H<|!O=YMLj^a1C zIQN9{@V8kML=1^9U!Ed$;}h@=t8pKR6{{QAcyG~~1gwEzl^GvHXr-5)GB&TJah~4A z$4$z8jC`(aj2SGxcN$#J_=zVs$FzTi7 zD6P69fOamQ_aLOT=~sJ~`sf!JK8Zh?2e@1#MpjM++dakzaN7}b`Ssc0m~XFLR$Ff~ z&yGbn+4Pj!4kSK%_X=VO40yl3&GoDx{U*R*i|k=Xp2=8_zlGnYh6|VieCVm&(#Rso z>9v%91f`#@bKd}W9%+*2B_7O14u>p?JijL7ME!mc_~BS!tzrM30r~dDpIu>SpYbeKU$>eo*U<-Zx%HJC2sYh zK;P()PKHP@D@XwNx5QZXVIhO+{H@r8mDIgMiqyL1=G0lDr_n!($pjeFbXcpP_*9oq3?XsI{3_d~EDnFtk{w7n6vs z=O=3znb;1$wvq+1o;Sg);+?MN%0<9F8cwQTBWVI9TqlI*xX+v+!hWV*2?FNt@6@o# zB3xmNX?s=3QS6p+33sSardn*>sc^0vh2$dV-;Y^ynG8=6q2HFA&#kj!*s*2yM9h$n z@O>l7EU~l5An-d}(FELdqlE^{(|j&}RjHxfFo zOoQp1r|&`>>nW#?Jq9s1?>aS={jFom9A9;3t5#hJ6&pVs?%!Un)Hg=v3)^?Ir3||L zO^w;-)Gu?48nI_5b8AroK-aIvTRKzS3su&~mxI_%GJbAOE5@)ndAq9lo2LKj)A?C6 z|F0i%*Lp5w9yARr=6H|$6(HVv)opdCVj*4KZN+P_!2CtUQ65D{H#clem8Upq12R~} z?0XVvU5?q$`c}TuxdR;_orf9nFo@~4r)6QLo5smH>8LXv%{;Z!aRwi8?2iuBhbDkt zb@8dppFYiXRz{%p-p<3We-_ToFTFvQmbQgRA!L+`4W*FXg2(r65vM=DgBXuNL27;# zY(-8}QfXJNcID~y{V@T1l%!X+SqrPtduK%rf}syJ&nV7=vbcqSw(m=&Zp{eP9ejXUyq7mi+G^qj^o^K8~8`XuBmX^NnFrPBKggBvG9IKAZKcrBn z_Q?Eq;@Wmqh_D=Xor>hL!oJBB_sHYex^p5_gr?@m%-7k2kpT*TP%f*XMV1?AQFhLo z!+cL&%&;hQbGz;=8f6-=AGf&hU>0#4iV}+v6DNu*s4^n<{W{mLyiioyC(#RJ?BhTN z7A3op`1$EJX<3V)4x46GCSG0Fd-6~HvaUq%vT!&792elaN~~M4#623U)U1D>htR3Z zy(u!X>6d@A>qx$Hm0j9hBfQ88-^wP9g{`O9v1W&2dor+#4mSy&gIv+@i!QBfDJB`N z(^{$T7Pelu;JckDCQCKk8Ol*LiMI#-+>JdwTGub>OpOxeRyh~0B)M% zwaZ#ja|ku1Of><0mrAao-VDYs)P1Q5Nby^D4A2`qs5 z1!R|2qf_?mltdU&l6%qj2B7k?XE;mKW&e&lGrs_0BXnIfV8M zlMs+W{`D`c9O|AYKIe~wr%sE<=6AULjUZQBV(xTc`=QdzW4W@beach^$9euWs9 z)?jq$*U<+fuOq_}VfK$JeD$=O2ZEve@3p~86r)OEX$xGdQJt&8^S^*@9-X8PalVe= za-u&5`RLpGaMVLDnwxJ2DQfEdd5=lIOrwIE0ny zP|Ecj-D70`#i_|fC)qg!HDe1#+HzYaGmqhIfEhZsf_#|QWcv+tSrd}fv9Zd!_-QZi zP~xLMth=@Rj_{A_ruY+F2&tIUsvZ*OI1J300~2-Uu^7K#YU@9u*>Rhq@Yc0vR`;=tey)#rK-B{cShl?-V`R;|q@7r*BtvMl^~vybWn7kF06{E4*AYIhrN z;TaolF0RaLjp2J*!5F7(*|JA*neUD|V63hgO+C9IG@%v&J3^t*c(1Uf^9hJonhUw^ z;oD5`i=Fz&G060KCipcDv8o*s*VJDx&tB;w(zlR-3FG3AmX>c+R1DXsY3T%^kD8w`}cY6V@;B5ZsMXk!)Mga4(p$P z2E{X_6ms#sjs#xE>3z|wdLuekhc;rmQJ5m?_`YL7(~Da+CPl0Ug>qq;G)xe+08m!D zmB)L^z1~-Kof?!PH($>REJhfY-BiSm&j%hBc&i4P zbmW1I*4UR^A{EQV{Ofb_af21wL`zJ!y=DfX$A6zhg-_Lx&m_0@SwNE&Q<^V1{5Mnwrv9uZ;Q|7SC4!See0cuVj!Ekts~%q+d0?ULi?9szcx zhRvkCRBXNhmJiz4O?DWmYBhpkhv4}ZMgx%PqNly3}?*R7}1gL!orgj}Obm6~dLBL#@#I+UJwZ9R4` z1(fUWG&`?T8@@ymJL+hW0$#1RVUrgc5;oJD9E1ez?WN~CHE($KZfsw>PkC$`O4FvA zMvx<{&@)>*j!-D&uDPuV;yJ4w8OPQ+W(ExaP3)CHSn$mpsapHE9$n>HtnYeHP)tlL z0MqnZ5Opv`-l{9!D7tz>Y##?DHM=XD8|}*gTmab`Bx^u;GZu*^2tvwp9Pjb74U-}j zpSAM3gbWEd_}jXFuRf00hgv zRbBu*54Wa)$x4;HsJVUnwo6rYZ-KEgBHu_lwE9xf^%{R(9WMT6OUnEh%M$u9doF2m zeBA&*ThrprpqQHs-`K190ec4;D0 z1E6BF!pkkN(%5}@E4jciSHFnp0A9_c;~1$^Ii>h+JqNkStbB*`1w)9dh9^)V->)m% z=Eo~_tB?NZ#bL2SOa~@nT~^s8T%dVd<6f;+thhVbd%P9qUorOwOqZ)s_V~?dPLE5k zPSIPELAt0j^@rW5{zRYNzDUsV$za_lD=WJ#1GFRb=R>uI^lclr&xK9NmsX z>e5*vi2Gm5p5xA*Kl_z72ChyzIv#-`@cGYpP=`NleFsd{o%S#D7d~8NWMz?*-%riW zjVU#|T>~Io{Ft-XN)lgJ@J8eJ&C-oM$+Q%p2(&3(2l zLP3^M(oTq=R;gSOW-Y##r(xd23khR1W!gYDTvw~}Az$eo`^%<@4Goee-f79TSB0i6 zg`zs&xjTv24oS%q8f+kwP63JtI(wh(ss)avjR4NcS}|j^6%Opw#GZ+JiW-eV=ABC) zciQ)%6cjED;)(-(V#G|s7l&=AZHwk;$Q>N~>PG&e-RHosQrFU=O6K&OY8Bk9u-cOd zX{0tVbZDAETvk6CUIOLSP5gfL0_TmPs{x(VUdyh9k2|~GUVifDdWmhP0wjdq?3wIP z0=ij)1VkR%b?UqfXL}BI+C*VTwI|h^Nl=Hc!w0gh_>YNb{Hh>W1T|L>^8<(@A&sJ} zgX)I)66}gGXilr7sci@Y=7yDBy{18MdR@&Z%ZHFf$1EX6Ua32e5wsOGppJkfm!ngz zbB;b-Mw!*6WHoY*P_w-&U6sOtA zo}6dzAlt~KikjzYr|8LIQMzTT@TIk-DpIX3FRO;s&)eM>Eu#CxqP$Q5O@0#B-d$UN zjYtfY?QFmfm098DIIB=Sk+imMH}6b0ZEYUIB)mNGbpG$q`1!^8FE2L}P8kYjAPvZ7 z5*1kNiO=g?8mSorUw$XYmp;R}N8alqUQpZGnOn#zVikJ_m(CxZqtYw1N0KEScznE9 zX4d=O0BT~t-+anzqlI_OZ|pu&9J53o zC5Ik9#132`M(IT}h<78Z>r8uj($$P4iCFinI~(@ZOBF}>3>-Ux;zZ8r+DvD6g=+}A z`b&v6vmpJXzWlC3+H(?U2D~ra&xQp42rh_Abv~;mixB1zlMNrAh)LJkmD+$W>#N?u!?Y=3fu*22sW`s9cs~4-j zRrq%2G=y95b?>c<(5txmXd#LX5RaNBKvB`_ITIq}&50~YQ~QoWnaF_S05fGFdak4+ z_Zao-#F=<4NcXs!fbJ&eAsCfozD|oKqIq>uZ%Bc^p}c26>S_TZ6EB!}PAV$Bcf39r z|L{s^5F4noQ@uM(0U<4!>PtHTbX^kclGd|HSM?;207d{aebCb91cYb_MA@#BY`Q9* zoGUMWyDbx6(lj^6i%a%1rmmQ>OL#24At&FxyST(`l3D*_7}CDOQpTsrj_46|r@a#AHk%-vw6tlwPR%VYc7C?f)EWin>F(WR|3&u93{Heb z0OOEXRTZR$#sj6^#*d7y62-mj4RwY-vs=hPOQAS*(UKf(RGx&(Y!$3%3Oavu?hOZ7 zr>sc(#22eOvJL9hN~iEJ1lmlEMf(z{vxmIJd_L{N{6&kcnfS~{<=ZdXpe$DShR3Ku z7ld4Ck&SrwyJg=|;E=tWag!FAdXO?`DJQGVZJHrhSXS}tetjQjtE|pPSJ?%}Pj$%y zA21RI)CP<2MMT}7G?@}e-b_jS_BUdJ?*&Tgap?&sp3% zwCDb+6T+w%nFrw(DzT&Q1jWT0lgJN4;{jWWy*Ukofc3|UshbyN=qg9sIO!Pe@-Td9 zk+gIl>C4Nawr3Ul%FP2mg|G{+Ewf3T?Z9b+hb6MKpiZj?;8I+$Z2lUJ^U-|Ool6S= z%J1iZEM#=kk2T~~kT`q#$W*48S3)9lh3IJZpw#MZ^QHplCMw%9I zN-^!;qw%689kTiFfnJ(~+wkL>qdwKu@IVOxoZZyw^%;NiRgWd-FU6y0K(E<@7$~^d z4MX>u!ZT~#oOGyjX@KAxa%9A1^z-ifxKedRMJuQE+Q1`SpPnBYo1C0{&3alTCVjUo z8Crt8$~Bpkbk+WLthCoRxqX}$C(=#V)>iHfmEyARqkUeeD}04=vK*!UlK&bJZXD)d z?W|houiwOp71Ej_u!85XlUbCiAX>1k_3Ps(VXXocJ?*ND^v92%m_xOO-vZv*0jf11 zE_o-257jg)D>owJHG6t}w)Mk|@sogi3;rbi^TIAh8{ND#x>{%%G%YyfX`ornai)^XKpJ7vc`zHU=*2e83MZmvd}iyUB4oC%OOEN!$21;7V_!eVNlq!Nskz=`oM@}x#Z9db zs}Hc67^#_A+1Tp;XLlV8Cb%>VreGFQLG}9DvETIs52ni6(9~~_2r-Sb*^XVjYd;o3 zMI1jng{SHXTUgiHH+3evoTG8d{rGtG=jL&v#dEP$UTj<->F?jQkUMF!0R2`kjg}0< zS-qi@-_{nM`s39ZCZG9H{?Htqob>o(3)wTo0dZwKk3rI78s3nL*sz*%a&pqd`jCY# zWq+%9I1{^nruWsIgcLD%gUMPaJcQmPN6_IWzUXxCglK^kJ)>ggWp6b*&6}7F*%fOcP>aVs^O9`pA>; zVxR;E^&60(5bnG-ARgCh=2IUIR5@qm>}Y?Z_1%B5M%JItd~@^&7EH(0x~ zQkXi@X|@kd>O4F=yvVV}|JWK(h>_QypIMxlXzM-4Mjsh7S7>BOm?rGK`4!Yf$e4K% z{QM2q^`~)K*KY41&f9di7Yc^n3J3rQ6|J(m2NKSrO314L`53UgjMfI?i1XZ&(}ML) z5NDB9-8c=Sc+PbH$4L!B#mZ$WYaZ2ey^g`dQUQn+lf?^zT$d$xS*w?M>Fr#$*eKlr z5e%2{3d?JL8)W#KkjZ7Wqlde{jCc6QT&2 z7X*n(t&EnFQe)>{>yB8c7Ei*E9U`v{^ll&3#AD(*s--=Jr<_(GD+eDJnZGc(A_xH) z)GRuRRy1^|K16a`hH@6>FAh$MAbGi}wD98ihZCU?}4nQ8P}c@5A<)Hx<*Wa{Kc&UjnxWkf=f&b)IpL76kfS zAY#=;psaV5nLFw6^~O|8$~gF5ede(MC@5d2`<;^w95fPA${e+`7$Msdzm_73GC?_I zU=+fyC_T--)%1oSJ$kSU3T+KWmRKaT`1?w+P>@yZ(Wnu(TQ+T_X^vOteVO%ufxOW= zsSAp2p@eSiv-zf*^hm1#t4npKxJyIwu|A~5>%28V-S=E3Kuuu9;X|hw-Q1Y=|EUEw z25sbZO;gsjQ98M{2^seI*8L|4%fD?$=$*i}e+eCXU*t}pE zL!q^XAFsAnrO#EDiN$v_<)wBdm<|BO0uGQMbvo+oPN5FZ4SDeI5UzBj+z%u%jJON%3 z5UN^z=t-QTkVy{_j}7UQ`9svkAAc-5pkzGCbQ9QE(95T;cCEim5{K};PNmP*t~>;u z_#I)mVNeoUj&fpVe-VNnaZ=V8d6QcpMPrarX^tbuZ6de>!5dru=uA-;Bu#WQ-9f|0 zbb^`pxpO=Bd0%_tFXOnA=?Gg&{rhCF$sQ0Q7Z3(s2K8MwH>o98=7cq}ovp|?8x(|_^mV4kxuXS1j2xBK7y@m<_RdH%akH{T1(IFS7BbRC_03)EU+?|rkzD+h z|KN!E&$E1`{5I1O{`13sz2m-zo`X6n=31F^j^Utyi7^A0;DNa~!=87CVaGD77Ok`qiZT-w_M<1okESBxw@xJB8AN0D=4=VWvS9Xf z@P8GpmjCqwzLj5P7fh+mk-fL=89&s5r1)6O>!nh;<}td>6{Q@rpdP1d+R7=XrkF5H zQg7YwwCa9rx5Cfwk3N8{QlrmIxxFhVk8h{kmL<$`lPpe*>P4luNP~FY$tD}BX9S*s zLC8UbhsR~4Px&Umr7Dwubp2`;-^v!*g(*Aicq2%KL6Vt~-e10zhVwArohl|hAM4WZ zw3&rP5|k!rYZkuSfJAnZIOjZ?71REk6Dfvd^G6Q}@lbL|@oiUvjyZ zr}^L>y1SPb(6``kc|R~t4CR%;|DI=ft6la)BZEWSL{CrWK1N2x$?B_{4bv&>WHw`G zp{EkwVuu|=jJGj-*nT%9gCcB+|SS=69exEauJadOV~B{LjW zpqfpU#c1N^rKO~H-NKB}$Mo^gg=Hv+112%sHvv2(*)h}PWJ5zH-^ZY3!6R^*#CXFi zPnB>)z_2HRn?7jVw#&-Of-TY5(y~h+IPEzs=nGJp9Z| z9x%HqhQCMUJ_;^B^7o$%c?S-~t%=E2zv)D8xhuf=>y8^(p}9uVo*pT@9)4K0ZoeI#^7Pr@9t+^RoSbiJ(cdG=%%>o={QH*zhQsGy;sqP*N*hnUl4`d5 z#l+$4qW9~{a*y!mncesguR<{Z7`S=o_Q=K4cCGH$hZ}9*NJ(?&-=+(~QicsqP_di->uN`(jat$HdY=C})k+Jm_P_c%!>{cx>@aEWpv9)G5( z`2MfEWe{(=Z96%}9o141{W=wL4iBd8|KI?n&2bV=r;CS2PASsZ`GGBDZw}qJn)gSH z_g3-Q&0mZ$E9`gq-hkVdwY33?X_1Ey9yzTz@8b5V(TqlP`M#>k$dH9TKoA?k_*oy= zgt)n{%Tr~pgh?=DJbNCZ)ARgLI2WT>VDsr$G-eSZk=XYsj7KBv=hqr`Zji}iGcy9* zXL2$F0tC3YY3&rkb}7^M)8#$8{(QY${B9dV!yd>1&umZtg6K&{Ln!sdw49U;tmNXk zbg3s|Y;y=P!4Puy@JJV!f;OsfR(!6DvZmq?Igp|TOdz98P7L4jfZ!l`qUZMY+Rtal zf!fZ?EA@&Wm*VvohacRb7of-u4Z{HQ^<@AVkIc+cxTVvR)=DozsNT0JkFto<-sBBX zhcPtb_5I^CJSphZ1D&motx6DAR#>g-<#>nyM>Wwmmx6kLY^$rQUxbG0o=~?ps*cWt zlp3=4#JfM9{K0he=vgrNA{TasnS<|o3z>YFojn)IDSElfWBBLuA1d7ykVVsZtbDyb z*ox*TMes^^t-O29cS=E8TwIYEOj7VM%<{{czT3U5{e+mo%flFi2n5T0S%j?Kz<}ZH zgL_T_XUW|Z)7S9isX2P*|Io#2GLkBwtv>~}CTPrIW zh;w)pg&udlf-3OJhqv$o#;+@DPr3zOhxjtzN^3xPu1>8{wQUi34^y}##@S&nJOogV zRXCEerzT~8%)Q(ob1QEmyW_QW5bg? z!V6R4ytUEEWo1x@*5>0==V5*v-8kU`%yI>3sS_Y(0gJL9pWU7%rF}OUckjNXprBy) z=FeZF@9?e&UhVebnt+JN$a~weGE!5|&fiTvDR|=xxD;SIzs2RW4#0doM?4g8*WqU@0=)*M&7smURpB5g16I7#H)Fyb{`mu=hm4Jl_g^L? z#T!>ZLvAy8k=LVPR(!h2TGiDhs!Wxq`!ILvUiAygd`AL0(r#-)eEIL9LDkc%dIMvo zuCK=W%a~YMNG%8Ov%h3j&!JMYg_hDJ7h@BDl z+ik^Fo0~Nu&@KrcDz_?y0N_H zufW>7W>bLlf5COg>%fzy-!?DSgN5-^EJDVG{642wApX>c$viYhiVwK+21kc2UIxTm zKjLJauK5x&s!MG3bR(YIevkRNocMZ;CLAXLFA@PnKIZdpkDdy&b*&7y740=gwg~hZ z8J#WQzphhWp5~XnZZHrv>erFhz#AA7(=60zE0pyt19`CBR5V6A`P%N^^@hxAXB5Z{ zFHdC!R#11VmGpuDCJ8aU)QJc~L%(x-OCFbV95K%>+g_+A6s0ly(iuaeE&-OppY(3u ze~lR5_e8ia;7H>U71c35BId3)aM-N$Y@-FT#ZGsR)0FrZ>z91G5Yb*PX!u@N@o-d~ zWe{`v)#&M011hJ>P4=p(587)eC_Dp>r6?kazH)AFl)X%l59R0@YeJ6ASiG!^xemEW zJSB&Fxq71(Fm7)hH4FN+Q>O3FU364uuw2E<2rGuG@Qx1i3o@#YI6;D2^7X)M5KPJA zJy`n4{@0Y8XMwUV8aJPkg2GKLt=FLM~s5M9YMD4yKC?6 zDf!fNYw2B~sKW;i{M|fOIj3^qS9>*t0erx#oA(yJ#4918f9uvQSk{mkF^yemgG_Hv zH%8A`8a;EFYp91I25beu%XRMTSp}dq{B35b-OazPRcx(oRPI>Y(4(lLGCCHw8X2SF zD3;ceS1ONs%dx9Hi-Q;qjKuV@|j+*^RoNchHalbpfLzBPrqB%DliZ2R0$h z5(u$8aIq@e*Vx!NKR0)w@ABo#fJM69xqkK;2n>x8iRWO3fMF5Aqj@7fK0b<*ZZaOn zf^0fuiW_i4s0QHMJmab_PQG42c!q}kg}t`TV^m=jon7&`Cr}vzQF}7X6r(B+ZIGve zU%dt5OMvI(G@W}F$tZt0yp3!NHYtqn@$qypPc9(o-yS^p>qLf?GdI~+b)k~0=9+k!oWKJ z$@*7sFqn18J|Z-rAO3a^@{O_*>pUfZL%&PdGP8;R71WqrrY>ng70vp9ja;gVY>ccDw5r%W-A|If+qY z@9X6JC14`bhva)MWDw52c8Q3mZ&dVey1n>sfry4v7M!xE!<9KX4F&@$8mM^?Q5zpz za}w>>(3&8ar+AC^@y7J3YP4>|(*1&8*ZZdS-$NAYct}?z%XhKHq05d*st-@o`=C)0Et6zvv^^1 zhrLXv#(&=!{38hcm*%E77wwPlAOC=qe2LetU$>$Faywm$EAk*&8@+!x_r`k=3aCND zsp`k4z1B2}-BuvS+#MlkLJ6@h?w@vZmiou_{>xz)bCi+M?|0;k=NCKFa~{rpXBD-L z^jRH40^J5T53o5d;4!)&@_o^);XrL&-MB{D@9+HE0qAr8PLzg^+#AzXV?d(FxyA?V z2--Hz#dQsA4GH)8R8T}>TT_XX(B8?&(Fv@?TA4SK26*Z2@V{La{(IKpANy6A--i!! z`;MNMi<5BC0M6(!uy(xOfM-D>Vu>V3eOeE8@8{Wt%WdW3A1+l@hBV z`JWg6^-fp<9Xg}^*CEakLhr>`;|?HJ1vOLe4jJ&=tUJHQ z8A5xb|MX@ihuvD!`m!$nSZ|-h74kt)Ct(n-fkKy?b^gZTBK0} z`)oWEx%c(M9qQ-jhn%vn{XYBO4iZ0{`WH*{@i?_OiAEp`HmENvFSi2T;r;7x_zlLJ z3;yj}zyIr7xrsv3-ctJtG{pg~KtT_=$c9>UN9*^=zYoID|7fFc+rVBFn+kmX{J~}J z->3ZBfgEKMD}oPv4_&)&>#h-xre0qp61hqp=IjhwIm_ofQyJqglGk%eoFU zI?lXYpcQRuX=&+7qkA9?rvDga4v&G0cPNSc^(v#eXAt-#IoF>5CE+yQye|Q!uG;;+ zqBb;zBq(yfZ?S~9?f0+J4~}QF#7i3h!*eHa$h!Rac3CgC`_J{A+qPvQs1%yawKlLS z^Ce^$mOH|97`=}z2lDG;tB&!KP&)n*R@nRxw(#RfH!~(ElZayTVuWsQhAr$Wb244P zDS^sW>KD6G{FTj4nU@Is2N2(Yq4_&!61>{)n3UQLcEUVjlf*;gX_wj=0Sz;aS=^D( zgU2Lbp+^c_Xp&ECebCgv-Ro+hr)FBLPeTU3w%$~FVJ4JDd2;czCINFgloG%2*cNB= zf&H7$Rl8y1U$5`<<1dWM2F7uc^qzs|2?MpS_gd*HEVSqckkJA2_@2Fcop!0HtEqMO zI(by1FE;vhZT9BgfAnUVDj+ORcI?x5_^>Qr5UL+cyymW#)bVI%zm{;Dc?aec}HlofXSPIL2j{9!%P9r3o)7qQTN5=SZ1?z z^?9KYrY^f4>ei-QxE^K|*fnP`iPB z6*rC02Z)lp6E8n0GH*Hp3gnXLYb1pAV^L>Y|Kcm3&fS zkF)a<1DJo**ba6W@-muQH5}UyYy$)3PK92ZxfcJrEk}Kz=@#C1Q(X|g2yN+f z35cqN`^p}es=cO9yWuKf_VGic_kk|SuTypYffw~%W9@Bm!(chHZiGnA8pFUp(#P?PWKeHw3s%c8Hz@fc= zSWA2oN2Qbk2|rv6m;mDb@&|#b3ry#9rb6l*+}x0rmFw)-a81~ng6;!n$b(!vyYRvI zHOBUy=`7h8sSVH8y#+3)EpE@&_m%s3{(3#9AM>Da_U^fxWn6G8?((%SnAop$3L3wk z`0b9DMfd*GVq&)GetTauqB3){(8~SMKewt;mr5X&e8MP`2_WsUbr=9S z+{$knKnIvxq`b0HvIuBhj5O%uZ1bMsZ|e_WZYR<-G%EFrt*v2wEvYYmIG6mXriSU* zv2zFnf+ky_m8AiplvX~v{U9gs50!@FTWMAVG*(7t=2=q{5}~WuwpR!i%Cgkcfc`f$ zCP=g z6V@Mmw-9;@GhiUZYKZ;1f^Y2gf5k;V@W)EIYrqu#`oGwF^LQ@v^?$r`=5+d;nrgJ8 zm{ih&Axj9&Bo$FnB-;!XWlNImX68&|O@)*_Lb8PjS!%KiNp@w%nmzl@_j%oKW|~uT ze!s`}kMCcfd7P7y?)QDa?$_&Dp3moXUHIo7_VAe1Xt}S&acZVo*Rk+?G6%L#D6ICA zm=89@+?@zt9DDyATo*e{?LZ^$b9n!!9)9~hh6W76(9)0`l@7S=%yinwWu@h!}dUKa-;rIY9F>S&*rw-a!QxAK*6(;7y0D9w$| z%K{XJv5%{_gDY6F!>cMzJDS^Zs+{rKTB?6e5$XAqUp=uhlg0?5+GR&a5fI8s zJ>imLC&s_-_9;%x)jc6oaMbtBS+BTz3u^&lzi3-8;ajpfSAKtSz{v|g4qmXZo10}c zKhuc5HYkm1VD@CRVu7f{F8$x%%=IG}?;Zsd8gC#tK@h_ZaqiTy`2#Np!G56!t3yGR zzZ)D*EX>{RjPc!o5iz#H>eL@u&-}$we-X@{3SEP)v+!1u7h4YAx;Bo12&>rG?%vsD zaK7f@cMo*4vv3r4y5;iVMB=BEkiB8vaF?i6uNDSEaOBtZT$UcG(U-IxFF#llv1Qe!**y9|s82_l`&xMNJ!aW9y!d7MOpS-@jh0FbKAfD3h6d zVYn5OGlhj!o?PyE1E77hw|1#w3tIY@xrHVx_^}W?bI)flN z&^+$n|NckjV5!u^K(SzwS z=yFUGBQKlFo+9SyRwuStcZxskfFy0sj3=rtPvf1 z@LS3f`Oa*c5?A*M96EpoEeL|qYJb+<; z2c2$l_=EY?Es(%~q$+Sp$>z=Sw90DHqR&y(_qWc*?Cc=N8Eb57e%}Y)Fij3)zb&$V z|7;OB)Ave~|5&uBD=ISR73K%-)YWa&ik?pqou=Y2Amdbm-cNKps*%cv6ts0`%MWmO z1X{f{?@XhMW=@aP1N6VKSsTQ%=3`U17LNd72g=+q%2JBsIr=xE_DVUb{qjA>yRo@? z=I8heB*oqK^NJ@O9^WXZee*@fH?QBkkXRE6H9pDcW&5tC(nFmN`;qWYe40{yJV)_5}3!FrQVNW*K*J%+k_N z&#%*BN(jzjfFfo5Q&UsIJ!YWM;&r5Hwl;Ra#=RTne)Q>vqeuoJ$;pL^Q z;V2@_VMJEyBU=CSy)z=^&zA2A$>*}|zJ*rxt1y=PaifDKW~WqP8(+-${pc>f$+O?p z5TCqklJ`hZ_t4-!7A}m^$+THExom@E%g}roNUFse^|ej`21&I;ozd}~VV*KKOU@QQ za>VPWe87INMBHzWb)qB+)tCG}i|e|0i~LL1&BW&5q^9d&23c*-9HQqpJ37v4Ma{qO1?qzX11fif&w%3k60N=5)(URlx%0($T(SU) zV2a@uBQcEa-umb**s06+ym!K^p6-W@n9kE$sJ z`1(Kg)DICDEN0W4kne8r_sAo`RhYVi*(Vy2L6JyNC=4q{9Q}hTYclzQ1;4KU_x;9& zL3RU&FiNri>NIK*ADm1Gg}{QwpW|aBcJj$u^#-J0?sic2Jcm$)k{&63W);_!e zH8{u1eQ#rx{KYzuc_5(;$h)>-2ZXF)X8820pR?=jZ2H|I&TsOkcW&*}2*m&<`^lci z93qC-VN|Z{12V91;ld7=twAq)-#no`|8p5825s^$X&b6r#>tGXG$&iO^;i44MXQn*S`@y1ef)E+M1hcS7ap7Ar%zmN zdHT0f;cVF{35lJHmKl`32)I+-Wql=Ra3-K^AV`qg`h7{jC>|~R^z*CjweNeM^qtLl z^Teubg25E`7M}~%X#o`D%jP)X73_}5Xu19Q+}r?P*&DmyXSHlP!u#80XC<0#v7py< zr<&GE6#fC#EpU1zeC7SG2REkzaIvYX>cs|Nl+Fcvu7ssq3&jFthnPFv0%;IiJ_+0|9ij~TUG z*faG_>8FL+F)#dVUJY#I^~wh3`XTEotE%$O_-fx0y4>WlRbKc`rr06uJyWfVg3WRpzWZT%FRp7*~!-_?YGarX`hCRp36`PJp_n%gq=Hc3@GHB)xVyRTz|(X_FN ziHX@Mkp3=IsZ=l#<$OYIEm@wDJsk(JN+;MI3)+bHus|S7xp$>c)qkB2#vh9_8a*@m8jzx@O0+6 ztJURZXQrF!EsxFYU#q>!79$CHnGHAf>mpfaZL2q;GhNv;J9@KcXHGu+_8D)xn!0-a zXx3^9`9iZ@8R3m%m3#La!3vPMq6!d?c?g|2JTB?cA~2>CrSWuv*(rJO7#kT z8338<#Kgp<2AISeyk-C7e1Rw3H`&VkM0e~+o7-ED0NPxE)ol)* zoiYFmBC2CQT&W~>wC{D4synx=jA{1t$U$EJMy_)mufm0ZIp3DFJU`W?J6m`9vtfiL zxnMcCt&sMK1u|D`QMS%n8Dgnz+mmcw^WD$;_4W1TrT}^P8?&`YsqSKj0 zMMbHtw=|pXhg~Z=zYO#8ikx96-gHZgX(jy@#H`EF7&>`l6~QyB5KFs~ot^Bg%-ZS6QBQ(}8}dF18gVHb*rrl(5+uUttWv*^OjcIMy4A_|%IwdjDG%;u$=qD(apU|Zv(&?4BJ!aLLYqvJlUBGGzQ0<0v$ag* z;xp+@Ya4dWf4O_S_s(n?izany62~}9x?j!k?`?AOmX_7yiMpjF@hB}p_2)63Psp?( zbxltKI5}ujWZQ?R$Uk2#gReM6*56rbQ|*v?1NSaYNERkN(-A2bgM$-SIIEiP7ON(9q(>4hbucxZd)L8=+#)ir(;0* zPNRKKdee>Ux-=VIE0m@9qnEzkR9EGnEFS)P_HI(_siYLK6wORtjxdvqX5sSdt|MJl zv$kLQI3L#tJyEJ4&r`Kimrc>AI=N_Om6&71jU~P8X}<>j{w4fkS%$`(I@+}nz|Bfv zH%HcAEZ8XY-7o*Hfyyl@eU@=>IH_m*$ZNQ#rlh3oXZbYd2 zbkhOZiDO>4O&;1>hzN9sdg{f=T!(%BJ8XebOig*5X72D&Gf40KZcAC2?wNZ3TnNzY zJCLk4z1S3-Flf4a@ESEr?%Gd|u=iub_-E#)TXy+1eBkK}PcgQt5*rv9C5PkKbM9CA z+D|Tf#M}26nZwoR>DczH#))%BypV&9$c;^EyXXDW&u%D$Q_w^oN>)tE>W!tf-F(na z59j>=r(O=5H54I<`7i(4nuCaZJ!VO|Bls5L;Zch@$p#whB7!F-NcP!|5%Il>1 zGA!~kz%=9?+8VfOYv3i7Jwq?|nHOz~yw}(=9mYM)hkSPx1yXK<9~{l|<9>c8lN^*p z9^DYv;^pMnv|_f@Qt9Vq^uaH)w?Y=4)cec9p5A5(IJ&$awuD=(00F#qM7 zsD@o;b1Gz~rj+&|+dNF`)x%pCvg&PseUNo@pf_P&c|!e6tNT*UAlU2lT3h2(UQMOf zw(!1iKZfO8t5>f^_H)C>F6AB~sA)@d{w*#2$+);u8&l%;>nHBys4pnn+oGV)!&bOc zW>m$oG%6?CQ8`>l_}$0ub*PaBsbp} zd98XR*W~e=Ti57g0asoCjFnkvxk=vLIe5C;fHuxh&ExFpp0TF2+3SE*8y-_O=_#r6 zS1Fo(S*zyb<9k3~BT*^yH({Tt0{>&hH)E^hq;DLQyJM^MYU<91t3qq!{fqRDyy`iE z>U14^y?#Tgxp|d_L3n;|YV)-i(PNG8?famfFeBe_uEVy2b)f1(N7dpBtQIdci@X*% z7)Xz>_t`7COy@UtgC*?!KdseC&HZO5w`8;Xt|mk^9)QoKd3bIKXmd~`KUM8GtzI?w zJ23{5s%@g%K$a6RiVM6U8yki}J0mdJdb!@@@tVFj@w(Z3f`Xn1N}QeV$Hsc(x~(-E zwjE#&ycgT@bzg<#TC;BK(SAXGn;jgg9^=YkziwVxxb9V$ZdL`IEIz|RiujNl2!XA3 zPeAN&F1mWFSK;p8{=HH-4>O7H#I$^vVe(g|S+u{yDaosO(wF7$leuI2c0Et&k%D!$ zi1HrWOv5ZRAyz6{S1~7J;5*?Fm1lT3e5&r9ZywWrR5KLb{uJoW7_Bs6ePd8TW>s9A zrNP*7Zi64hElyP`*jdS$BM3;{$IKij>(%1zLD}g+4bzcoP!AN8p+FW zxwxj|eUQo;w#H*^{a3GEUDNu(sxGKa#1v$T8}e$DrFy)Rr;qIE&vC!<>fRv)<2B!eO!U(*|~MW+JAv;PdT5`!u7y1G>Shc z!&CSP*-JOYe9mSW{dxD};~O6&V;c0Osp&pPvx~Cx4~DOXF5t@Zz-0L+pCwDJm*ac| zk#>dtnKNh1%lEGpO^X9>d*k5dSJ`wz_=A95r@H5LXOEYdH6bvo>z<-WGix~3?2Nnp z>|~=S92tMm2hV<*&IzVhZ{HMzteE2O@|v2OmUOb!J&!7c^N{t#k9p3HdB(KQsg>#e z_IB97v*;l)@%(Jr?k`PKf4h36KX+|dT%2i`b=2|ga5>4TO|Cma9cO3yTfB~(wVMrf z6L8HOeJj;9jx2>QYbadsj5~cWIs22?lr}MOaRT%dyAhA-13&NRht?z?VO3Jl2f%^% z4@|B!J`^dt;lO%xxx;saw@8mf>rQV+g3j)%>tMb9a?N+gbY}W?JOsLh>q+50;+%;V z=(faJIiR7TQF`$M)-dGKVSh1xz*1}2*eI664toc_`?#7~9>EH|4QUD3xJq#V_AY7R zxnghCdpb@v1C5wr14m^iCnq8%mbY46i>+G6)}$KV8vD#^xA?od@+(AlD=k5)SAME*T?TNtoKKX`ULIPLIU+(K`|X+<&;AotOyX3CHmH2E3Dxgd5ChhvRyO?q>=J{OQ6 z7rAW!e+DfQh+mcBIuI0+zw1x%w2!I$0A6s?^L=6U_eXKSn(N&Rcl!&k++g4T0=L&`p-$_VzstaF4SI}^fZX0&$$h*bdmd< zYtL?M%Jlnjao6JQT6e_O?Va97kqYJr=kh;pSYg|nX5ckA)9yI4RR@B(8YDQH(t`7s zh1ERUI@AqA_366Z7ruo~ObkDd9N?`pb~;7#83+rIc&wG`ez@=Wi4*%9v+VWZlr<}j?N!f~ z1-%K?LY-NF{Yb4j%wj&y`Fg^mC?6<)`SJ@`MzVPu%q(9n+EoY{INbE_kqg&fbIYD?k49=_}{extL=aM^@qQEc;j@|2SMbAf3;wY zPwDq*T7UWrOiv7Tk^J+w=Kpl`sK_5Q>ijoZUl+UbcK-Q0^MBg=b@?BnyZLXjzVh+; z{XR87UyJ`eW%DcX|MZmoU%BIL zWuMsUd(BT}JJ=KdAq4S10TdL$AZ`i;Em)f@Hwa0PMMj0PlHxQH}$3Xr|vJY4bih3ASK}-#DZ6PlB+|ZBrF^UU}TO= zE{}MV&;I#&O~fvv^XHj37l40zf^lu4QfWtHwxeCURHMtT(%qX@xa5chGHgVPvT)-E z8qfJxk=y*s5yUaTSYR?Dc7v`XFGN`=F52WV1$@%Wo{J?>~6p3gG!9L~_v1 zZ4LX*2CR)cr?@~{X&@&6`g*ky*ox<}{wrv}UNyK9s5kztq@ z^FJn_Pq<3?_E#H;F5wS42K*gK!wbWZw@7hQp;*#K4%+`pQ82eFKAUTvJG^mt=1J=- zA{*8bs;|RLY`SjwABLJ)4vj%b!15rMYPzLq>T)bhJ{S6p8|6=@_Cd+wLOA%$aJ*wK z0MK1oR)t%9<-ogmwI9YEQ99s09fa_E4D);X?%%(E(9^1?PBboKplGbb%)u*R24u4} z(rM26ffe)`oYuemL}*#yhnIVR=SEHEIDKT}%*bPQBQ*b5R|fR=Y;}xowhUd;J%M}r z-2oxPmmwu_5=evfpdsx@1y)HUWNaxDI2Z7LBA|52;6QpodtKic_^*H9>{zpBKC;OQ zb0ieaFLsqRG)YjBDxWlh>_%KIPAhFGJF`(^l|B*Ge%VJvp8FMS2>-yzUw>SFHirO7 zd5fWl(%sz;pQGipw)UQhcVQ*0M5kckCvQsg$z>CH+(mmyseQ|h1Hw)E*CftHm$A#( zU6Dt6x7=*_6uqD?tv+=q2e$ht^6mqk(@)>|Z<)0jxyacHQpmG`)tR=VhFEq< z9Wq}KO_i|b-x%5>jbt|DpMK%fAwrtY=7ws3=>x6Bung?zqQ5*3GG4zK9e^$k7mNAL zUmrVQayc9N-!nNNInQ0j82Mq>$8T+R=$Anb6Rll9WZo@TRWqTP`i~AUfJV0K&{V_f$Ba z+ICnk0a6N>j4(p!yRm1FB**t6LbhcJIumeZVCm5g}4qNu2mf<=qIb=)zXte)5h9M#?j5=}KkB2;P*#^2w((&gRjaEnXzaocxHCPx{( z!Oj>Pl~y!XXLCtB9lhSCr~PpKjUL&uZ&m-`6p3L}=m5D)qaUPzwvS4nYKiXQBl_xk$8Xx{P4l#qjecM%|#7enLw4)&Kcufzu z3pP*~8ifX$-abCz*#d%sX2%51w6@;hQEQh)CAlPmm%HHVaJJ*;NQI$NOc;(p@aK6P>*L#>cIN-fQXC7{)K;XDe94}!r=K7#IJ zufV4ujaBztU0pr$F2A*IuYnD$=qLCeL`O}hbTA?(_baqr+otFOPE}!_$Wg9aMXSYP z$lfgsYI;>pqYdhc`&GzN8@7GeRmuMTZB`L9l)_r?J(&?7$A0GD!s%57+AK)KH8 z93mI<>PoDE5jJ;31}o>|Et><5H`C8!9>?fYhp!+Sa+1{U#A4w*Wsm{~dBxxGc0luR z32tZWhO!`ODON~g0mYSph$dn_S#HFH^|wXonlJr6<+c*_p!yWTi93tIBE}l->x6hn zjxlr+_^;Nb^8NnW6@Y0|TW%@}=Mg}j3nQOiE4p&4vO5zN7+|}^X{0f4GwqX0xEch8 zTzB^D455BGOT`<$fCgO;7TQEa5H8awK68z_mihjnzD8MyDGS~Ebn&qX0Pe)CtDr$^ zYUZB3BB=xDn;7SQfWPGQDw(=KuJ83@A5or)3+a6%iq|ZHBlag_m9`?&c|Cv71 zcNZMhA^;oG9ye!agyHWX)w|T&AhCVN4*kjpCx}`+i7GqZ>T6uml5y}-2=8zL36@@O z_x>40(9UBa-%XJ42Uz-f2(P$~X=-wkA(HoK08chMR_^KW`WJO3mVU(hDfh0HG$+G( zygMG5fq98&bq?$Ur;D6k^avN?A_`hTzLe#FdiCOpkBV~f)v-C}Wpqtu;$154B@CJ< z%~58>oiYx#eJY&Z-C$X*ZH~ks51Qx=5F*tzkrd{~Ma!>s#XS*#>2KblJ6@zFesuQb z^^N3XPJ|dc2b+_3Vig*kmio)SjKw`{;l}XV&pgzqmd}4VeC~Prny} zcFoY(*c~on6Fx$}G(e&sLa_sI$Oyug9XoMiH5hsiNR&)ZBn|V|e)sc+Cg_N-nK$R< z0@x+Po&Yj}Z^sTd;+qlR>LZFO{bdKkk6ru}p7~`LG$FKgAesnH+6B6g@oylmcN-Y6 zOw1V5Z?51w!f4<6X>rb74t!|~$a|0CIs`P_m)y_jN0k}nZ! zQ7YLLlZt@A&p3pOjWB)4ruVLg0H1_XB&M_Q-p2g1N7wXXW|`powBpy|#~QJ}apCnoXu2{@(kdE-x<=k9| zq|j_zx0M50}BrSiZTlrO(aEY zXD72~IY~8z#};+~Yl+c{U4sFvcjg_Ta=#TSB}VRd_h&BG{^tDrN_E{0VD;@ORiq^G zB=ViAGA*=s+JpSb9KI;uM_q@Vexw01zXT{hV}cnLbZ`AJ^fjk*N?9?4EdC}TGso~` zLI`iiTDHpG5pn=clcMIz-2Tf4t_gFu&yOS`WE*J%5L;|A1=UM<1!_fHGl|UIGd)y6 zMu*8)$6onntpcr6WKhIf1r^uiTiO4=j)hTq8jSE|5+9l)Ni`l=+6yr(q-qZ-_+D@G z?k>M1LoPd|L}EtJ)_@D&Bvm7%%0(%4^Ag>$Z*00D%q~02BzOz1@5((n%W$6sw?F_H zDxmbH$-i;xY{o=qbU)J59QbT51hnJsQXI0oD<-K4u+6!|?``DdC@qD=h}j?HR8W!G7-7)|cQBT-RfY|k z5i{}Hz0yrT8jO?5gtW`h6qzIbHkp5%C_GMm=kr9Eo z<4ifehN5yk*$ql99Uw=~f#rYY@wLpU?&)l7rrm=tc7*{5=}(Tg=hs1!y5}rQHy$2# z^87eF)tl2R2MMM^WgeyO#K9rD;4B#j8nFQdZk~RAerE%%VY2feu}VK?;R``+KCE~c zbWt4!>-wKr5IWY(**GSj#A5zJuB=kjcF;s^?3u6x%3U_H{7XTQN<7Qt7~d}uIW~I~ zAH?V`!6CSthmKhf&e%LQx^d#c}7KrEy zRSrlYrX-!&9w0}goo2*jkc}}&4Ggt@k@q(9f{j0)j=3Xda*7VRrhT5jSr(%a2X!5h zkAj93XnAmrV%a5v8DEePm}JQ9GNWnk$&?p0pfZEs#)H|sk4$op`^s}*_s9j_350B)s{J|$>(;oBS*__f!L-AfQ2UsTz#{rTHX7m ze_QV8j6WJzY7hvek9x(9E3{HX3SWa2aHv7ED2>CoRGJcSX(oXvMMU&lyBo2 z6!aO(XVjZM*^h%?TB-|(jxsamJU}_V4(m~3sKq#n1+^*)zdvOMp0W-nzbPiE)Rne0 zVTmNX3;iBh zpoMjl9eh7cFrt*FS6xG0CJvB%!xTsx)a_p%4%jV3YmDGA>LAYzXNYW|{vhagt=5IF zDSFSEY8A}i5ub6h{RQKRLPS>C@X!8xr80%7XJfbFS)p+`>n=Sr)~u-|4&5bO)8Tw7 z7@^2ZDefE)94#DLc74r?hfJB6)2rj}EXPgy2~6Bow#Q58GStc~E#1f=Zo_r+Wi7>)nGEn$K<%<|*E%;XFs`%<6ISH^Rm0;H=DJ7*7)}RMB9Ku`Fr!`(&@#A(Sjn#EX(VfRULI6WF%AkKM#N4pqkp_tXh%f49j=a^8r$1S{C&Bi0ieTd zrB#;o!eFe2I;+3qh@&JKFG%8SAv4)Va%}R*bQaBFMDulLC#r}1u$fceg}UiQ23~e0 zovudlnU6%!w}k65fT9e&aUJ`$)W>)28_z^a@}oawUiXU1dw2A)*oLh< zp+Yy@ot+cqa{N}eCOmRbQY+ow+TO6LOv1LI{g>-K2HFp%w13oVl2P5+aK`meRAc*6 zf1%HQioD%9%dCphp*E@SjZY@%?Vmz?;X-V~_lvmJQQ{17{JkaVI64`0^cLpZ zfB3m5)f;(DGJEf6x@)q)Y>%0v)C{s|1NUi2-rP~^6MmvB6D0?FXp3bIj3QsdAaFH` zJ>3xX%R^B|tNhQ;*L`6SXsNabuF4$YG}Vw)7^y}bM|A`5TWkik5~f<)sYyL%&Q?Y| zf~};mh~6enz-n~Q`Hp16+OMaN;-5GFutdO_q-sGnol(1oU#o^8J0O8PzGq3ml4)A& zEs9eoRx;GjRZi1_d_`8K7^JcXSjw&^Xzs-cOA7n26DB6rx9bBNs6XgAsI_klSJH-0e z7jv~(p;OIGG8|e|*T#f?nZs2?tvDZCYbiT*uNXEkc;aX9ZCRql1Gs)-ZU4qiUAp{4i0=`;irm&fsxDG0WlGmaKwIQ??<`ItC&Ht zfb8EKm>coFs&c>h2n??PN@d*>i0+C@YxA4h{AeM4E(3c&1~^oMhF8s0402kywrRQy zIX6L$OqoSK>X!np;Xs9=B!|EBj3NOKC0BTOP}0fK+G^5!Kx1t77D)IK*NOX$Arh+F z4vLB89MH8?YQ0>3dnYi40Xd$Ou7)k?CoF4*n$G@z?0;lOc99FV#IKq=0h(bYQ%g!# z9@)c|Jy99ogkm36!nIdf%I@qYphpNjJ9Y#2(0l*uvywVTI<^#gniC}?$AAaJ)yz^X z$c^zf8uyf$Fx-O}k6qs&z+B$T2wexf*+x4mA~g)C`hJrRQ(eM1vzIA83feOS>|!rP z>gvknP>hp0T&20o@8oJfGPK117#bQnJ39j%%cX1!1vidvDBue_fJR_p`)$#&meF`w z)N&wR=!i!Kh^24(tPj@--EL0whxO(v{&S`l(<_Uj5SIFzmW-B4b> z)m!O}@YOvvt?bP6aa7kNn7&Cjtd7%88mWDW{uX|3u4ksF+k|O>&pqXRz$hv{{)l+Y z_*bSX`}WE4^gk1v;lDBVYwfCceHI%HN3?%jbD^OL~T3nyV3#6GbUy^^COhNq& z_O*%HHaxd-AP4E)kGQRExL&+-Yg26fp#K0_0->7Y$GuS4_UMjst0+NEhQZnEj*x~<8r}7=A4nzl9f&inExab~MTjAXM z$Bo9Z^cGYziP}EYG{NqjiLOG~k<2C{-;zqH(b<|&=2Jq(0BtzHvh1>FLl4z8yP8G zzk3~zIs;6~1sw$>$u4o#!$0an^KKINw{d{>dZ2~a^`CSnDh4I8;^XV{&l<92ao1KJ zvG)KH+vW;cLjDzgve*%ri#{w7u(?*&0_6o;pYdc7!n?D6r=9}QHW(nD$B zp$a~FsKwJ+t53);z4r(3ck_@&w35Hm>@}~_nb{JVK5uG&sXMP;hKmW z)|Y4A(5eS|ajcF1==kq~g-5GZBM!Tu$DKULLrj64HyaUk9Gqg?XAM5o3P`<-bTN!8Ygg)w*~nuSO_;{YsQ=Xa@4M>|*0!xG$qaJd5wLZ%;e5nca zZm%juN#Og~*h*|F`XhGgX6>QxI6Dt9G~|N4#oh}1vxt4w31d8_s_Mm6k2BQ_9L@vn zB1|iSIK`wLp@sKPc7RGjsh9bbz5Uzr(x>QgnLXJ!mM5+U-2ut>#f;*=dwz94V3@>P z^+Yu-k{$b+PCHd3j1e0{kw+*n7S57bghCV+IiQ$653Nxk<#@JmZ4S_qq+2Wl5Z4kJh-R-I^_x>dPSt0#&;T zo;{E)5ej|RYF*E@Px&}pj0MJ+d zB9`M$xqG{7z$BLd8whfgO3%UeXrYXm>O97!nds$Px7RlrBO%5QB>W;3&r#vmv(am; zOWag7OrAo^$h>0*Hmn|OD^c_uM*Ks?EOIE^O1emUS-1&a)Fis<4eSu@jB5ZHj>x0N zEOD-UKK$;>&Vio{Dd_SyP2Z+0Gu}dF#VO}Y;CyM`NLUQ1jeu z+;$7PkY?(Eqe(bjc=Ke^StXQm$Ybx45hTW9{G}>$-vbYh&IP2kD#A*yQTnr1B1y20Y*+~4`38> z-yUs1su_Xk?@`fh3b4we~jr*>imSSbueAc-t#oR)Aa!kjIZUP~xG?^&JvpodD z5nJPB!APa3jvPR$i?cJr<9xv99JKJnjdWO(`o(VaNuvkiU0MBU(Q2)g;BHHb=FsK% zlbuWz*@CCpQFEk8k?qQ1;HJFU!3yC#JWU8JOv@#>d%X^BZb}NCHg+3fC1@%t=aZ)* zgKW-+B`6AJUh*ID6F#FLziCLR7Gy&+a1(}8nEo86Hk??4P;8F`uz7h1PxanrSaY6WbTEUb`EW^S|8 z^WyOF$nZT)iD1E9*MPiU0q&DBR>;Db_VOlARauc>k?Q?Hue*WeC@GafxlJ=-cUL@H zT&1Hfv%?CO|L65f)Pwr|u}pgCWyzW=!6?8P8l|WZFR@sy53?+&F$C3|jZr%k0;j=u zY~|d>!s-rfOFreW-KVI&7M(!c8mD`$`VlMcnVG20M!j(c8I;=hGYNvv*#FEh1$PNbm26$ z3bBs-x2nJMaIaZImO{qv&jIN=I42Bdv|tt2JR!K@KUdYCs+yP;|M-{xI~BKrwal{V zXUiQPme-)GNBW7u#51bx%$`}m{g==8@}l**P-+Ga5n{o*`s2?mRRAe0_O45!V%V?qQzj%O{_Ff2-ngQIUxS?n_T+N7*R`Ti{T73YZ-Z=P7S< zY!vCkMp8JU<9DAMMNJxWV-ipMo|0j%*>WwNb@5f)b8h;U;-^Fd%nv(xeoA=4{MG+u zsgZX(U_5k1*O@}Kuv<2ytK@IU*b<3EcD@Ed*I*!PWxeT0;V$R$s3?3=59)>qfDK10!PY#^(E%=>rl{^O&n{!r<4 zdx`&AK)DC=!Tn5~ZCCae<0nn{P^4Zjf-MS4*|5A!w?txViAbX{vHx1l&F>-|rH_hf|z zQTnM5uqt^(k@*-@l-yz&&9tyN>zR9iWj#aAEv%QbN086PhFnq)P>2UKi6)nO5zqz& z*}f3U&q23m$Ohi@IfzeN&J|eJDy?HKtl<8ayF6mG@ycOHa%8715!HxEftP&irjc}Z zKL{9S=Q8_V3;zJ|(*@V5idE79TO`qokGZBd-(T&)1xh<=csrAmlRM)5C;t5+dv>}H zcg=Wp6e`zsH=wItL^e?!OLooWNB{k~GZB+@#LG^&+~Kr=j;TU-w6k(YnDdjn@IH~* zIta>&SFkR$BJ#g3U?!+Ew*F_4x}{KoO>eu+NrlJI3fwrDgV$8o5=01&zdx_Y_|VAG z-x;2ICU%Tl`1C>IGqf`{h@0Yqr0zf>w$+f!;)Rg^n?A5?)FB|U8NZSC?_h3nZ{e}O z@_#QM{a_OM7OOo5qv9{{RvD5hiknHPMT9!lkdjnt6WS=?b(0*3+7guXjylecI$tfCsT~$SyQIpeW z%be-xnggJ)*Shd*VaFKi%8g1c)qCUE1_Wx?$1^Gc=Q@eYd_4&S0!fmhqEn z_4p#Rr|8k&V1+gnbT3cURQ_rQJWLQPmz&-PtiT=FubAx{5*#nCeWXpXTQDdibLdg1 zU_7(d8YVgcb<6pVsF^;ys-Q4#QKLWiF!%UBPVyA3bQyc0oH-cyPx`!vOaJoOr!@V~ zhbk9KBUA1V7qs7oSTmEqR2!(>6yy)cTdtrpo!aqWh5bMNRT=L&zx%$|iTvwFDaI}p zPLXa?khViO@x&=pj|@smq52x|-a>ct@qgjUD~DH@VPY`@<}|DO5A7i=amE;jr2?W1O-Xhbx%iHdXxItIrKCs9(j74`>gQI2Pk>8uoMWQFqyPSIYmcUcs;Vn#?_#MoB!Hk_)8nHMuJGU~m3!nt zTlK0RJt+Mv=USwuT>Z=A;j!9ogq_A1!LuCX{m(Tb(x{zF}7ra^-v3R3F4n94iU4h%u!ekw8&5F%2%L zn*?R3ln+(2a;cYB|BK;AyklGnRlx>C9gKT#LWi@vO{Tj{z&+?@I{;Z(2`vc2c^rWx zX^cSl9cnSo*gsbw)}c0$?f`}A6VNjvcXq05mK0M#iA>@Y^74~$J?%`!R!cN$Z#W&-VYp;Kn=qoc+Q>T~?~An%%pxc-c=KomcO)^xP^+Um8)p zG4tHA)|cPja`id2_+LfIe^*r5b!PEV-WweSFW9rtRW$8P@s-kAgFgEahs-KTqn`HK z#=b$j5#Iwh9#7Qf9+*oE^;eGzd}~XTvGbKaPT@r$mZ`A9Vt=gSzJ1TtO3G%NWoI^# zk7nzWu6CS!?8ra`ln>MEBLbO$83}V5J?tm4P2Q7h1JQwMou?qHfGAPQBfb53$*)F6 zM(9T+wqsaC4eLBnui3~ge!yN^+q+U+?&jta{@DB8;VQ>;)qRt7+p9(biaR-ZDqmg~ zGEK-#oQgT8oj5%zQnRKVg)^lE3t3-&Qjsv230RW0Y2J?Cm`}lOu+q}*lZX!bkPS>& z0tFvXKe*$Bs%p}S=kMr*f@!tvS_5T{Gb*uoo2QAc%Rb(Vf!*j( z#rpD1Q~-!DKRe$ATlUR|W}D{I^i6XVctWDx+K18vAxsRDVLy4O>k+xgQcPmb_c|hz zcH*<)QK)6NG4BTrw!DRA5$9S(M|3lr&|6GnK8e#}B}lHGQ%v&vZT?~iMF?7@)*zb? z>6M4>7czUBVdwk$fpMSze!;Ysn^Yueb;S%yev(2RQyY2K59QCZ@%4?=RX=`QB>L8XtE(ZOpxafoR}E6? zNMyam2Ww5EL}l9Vb^jRAv9$FJKRScem-$s~^>;k-V~_do$Ba%J-%l25?!o_Bd}rr8 zIx?SWMV=g-MaNR6zYR6MYl1iNpl_S_=qc_7Ix(kneo%>FFgGl6+Mu)r2%j&yT`NItmWc6`atz~`}9l*j{j}Z`s+eE zuqh<%ma4upVrSZG(0}KR$EB6@wNw z5f}38hlRunn3o{4SVyX(DTF`n7+}%*fXRnK_EK~JDj)Evf+sXTn-8rZeJ91EgV^(o0n zOkD2ZLI?*!%7)g-*^$TDXWW1|K`triR7140oyo(SbYzaR+b@|$id z3e7l@#iB+DnnMYy zN~59z8%&HF^(5{~BAWffOVjV2UL_)Yk8Oi%zxA^M^PO*w-RS$fp*5oiqjZ`Gy^b`Z zs!8#V(9M9v^(4t5)wc2542kBq_ybe$LCxdhikbcJ;U|U2QFjIX42YKHFh$bu5!nCC z)zoRhgHv3d1PfGT3E0Y*73#NK=k46mMEnU=p%M#(giNVS5alCiftX%myA1no48D0S zIx7^Mxbxv>F~9c?p{$d=8HFFQjv)XtV@1Hq?DP6`64}35{WA1wm<7 zykqz>r-pMUlwRusgHP><@NW{&9#;asdO3QOh)~mAzh?e+m9vQ&gE>3P*iYo3CaC%s zpEYsBL{p-71)|7iQmnlI9z~A?gM?7cHirxhR`Rm}ZC+;CV!c0hSE~kRQ42spo^V?! zR+8ARQ^sXvC#j<2Hl%-~Yt_)UrA`Z=m_<5+kcpjkpNn7p&-5gwr>CQ3VA(n*e%(C0 z^v32x7K%G0520@}I^s_zn<|w_MwvO^`_4_Wi}RFe{0fKCwBycY629#_)v3Tb%Xl{H zU-eA^HLUxLL z&26AwgfooFlOUqZ*%ldavrmP2YQDcG+A+T=*pM~{&IhZM%yM#YFeO630V3KV2W$fC zXY^VUPy^G!n&~b3^`Z@{&|XiS&@?P8F@Mw?lj;svLB^Z?YLs@7%yh?*eCql^ecY;K zDmGX(u%QKzB(H_6s?D&$ZnR_`%Pu!k?`4t!z%L%UnL`cyZ}G88!4z(Oydk^? zXQSiNx07JT-~|qWHc4o-)2WS}0}N0el^i2{aW1oK+a-kNb<{)z|54dDk+$Y&5=(_k zOk2}LevWsT950D&lVKubMQ`ED*~M8d6C`mY$iyZ9i8qxh5FkQ@U)qbgsjd*;Qt*^r z@5i#2N?Sh;Dl4aEiKZn>s1NgRZeA+U-aWv9rUXops)&4J5V<1ghEwVWN$WuH+>?y0 zbPw01nAiq2n2otqi8&1RZiAYAge`~$mo-K#HGZJ7?Rxc_Ca#j-MuxYLso6?U~V_A5UO z+!ZIt1_f(a%iHGOVO)9se`W%7*M9;+%QlX3acKOs`1vS682iW3SEPpF||v?$ODn_ z{g@cz%lt|wbaa9ow2vJtk2zJeV>{0Yoi*E*Mm|P?WoX8oGAqT{UJYN9@`m(`m|Q9%cloNm1?;D$cg!UEYxCxjWidJ9JxppYg5pcyRiuJ8Sv>( z3V$=rv3;h8{`M8k$e7NC9j4u}fKTHS2IIvni7`k|32QghfZC@ffc-^=7b;&-$*B9) z9uhGi=xs4(ua24KYh}D@GCfA1S(4@+S6#f5*mumY~o0^1fGzx$hV!Y&bJ|Bp~f>Mx1|si4YCFTKHLY^FB46QspyL|CeF@O z%&(+l1SpkiQV!sIR3d3r4~vg<+mU|N+qKjtg*udL12}**qBNtoDgT{_FzYuEFiKXHXif|)Dju} z0Z!P&QY}`}U!P397qewJC)DeAS6ddu`+3wR2Xa=A2@UiYxIC(X+0oQpfrcnh17N*@ zmO>xpq|zSe+!U1_ROae;oN2!M za$s?~iw)GEOy78FkB_-Tn-1M%A{Jt{0%$k~dWSK49;gS&!kDVe00|2Qsz62*(WrQH zGnHK|mmWGAB+J?itCoLc?meeuCm|qS1-)V2Z=&>m89ESqbNva12@%;3lHb#rps{9D zMxG&HH~|@vvok@ZG+m${X<}qNK5km+dY|Veo(;Xw%rmJ5Lhj;otRk7C-5(#mf!PQN z^d;A$qA7~VFsx|}3~=pl!+3aeD4+5GS>`f@kI3-nFb%^8KbT?9QGk$-;JdlOqRazc zjvwnVk;$)X#@?AF8@&0DUCSnJ7aFom?VuXg%)P)*`1`}-|A)OVkLNny|8>rs&YWqc zW;CiPqh_>`(rJ+psYtX*NwPJmOc5hnwwR?EEhy1qi7X|B5VFsbCB|Ns@HIpBZ7gNY z_dZ|m&sWhhzx%lNpZmw{4`cBCtncNuJQx13Ou{{i$*B%Npg_b}L{+6;1?50k2|OqV z!n+HBf)rqUOd*q8PYhX$qDf#3YHf6GL%hH-!KcQU)XpQgAPkSuj3A>0UQ%-m1_*2J>z zk#MYP69^H(;P(>VG81wbGawQ3yIm4Hyyo$@3^BHkyG9bU0d3}N5K^auvw8y54vWiO z1hyvaWwwA3adpNW&W~8b?~N_+Wp4pD{l{+%HVHB41L*sc$lU|N>1_5oc-aM_&SDJD zhN>SiiymMl>wR&7U13o8n{~aXeDmbs`9{= zrZbjcwlHsm7!{#!pAKH+f)C3#%Q`t6`^DBkHT}aAwFn1iQh^9Ns+vmZ#_IX%CFKND zrfvaoXi$jD1hAfj#mR{b=A~PV5*yNpiiVIm6KvQ|sSj&4aGJZ1`|rFD_g$ZCdo&Z& zLW;6*I+GvbXfl)U4|WnUvSJ}FYLx*K$tJWY5j1xwKHE*UY~}l~rXVK?JUwgWVHFgA zkN0qegP<$#KBLa1u78S)69t=kum7ZWJwa74 zV#nbTo30}vbmVF){RES|_{}I$@S=Pdc5Qn^@MbqR>rWiv2lfC$3vsc}N&lVHejGZM z_H&z=t^2#iX3;A9W5I?HCZU?UyptFh*#xH7t*HmE(G4M+>I26wd+2#%)6}iu4tM6w zXiqTIi>z@ttc!hA>Sv8XN{YOF4kd;nE2HnRPznoE`%6>%ZkRvsjqZ2 zDkNa(qbpY>g<8#A#0w|Swp#(n&(zRrvj#`g*-?J>c(X7gm?_&XmoIFkxJ4zkf67&g z!qA3DP_(x`IBOCPA^Y6=4~tNtLwpH5^(Yb0=!h!et9sX6Oxv(E4b_Y zMglQ+a2@?hxwIkv^bk65rP2Ku(=%$wWI- z5n3weoY|$gcD#FJM2QFbSvCeL*O5+@9tVeJhVT>Ci1< z&A1>(=NAslIh;w5;uWVQ=(_+2ur!&wXOR!b?s@yI#va0x09>~gE#h?#5INGUtnG|P zHI(oTPI?+ek~ z9s~Ar1Yb0LNjQ^|-FmWtAPoxzCeDLy3$5~0QzuLYX^MI*b|>SydQ$Tvj3Wx&bA-vW zjVRo85DRffbJG7Y>y&ec&)hUY?C6VSB!z{2gLgC?yD&A2NuZW=u22__e=I|x8HJ#T z>#FK(z!O+WN#zwFIkTL|KY`8+ikS-5!z@Q4E&C%sqflx_Gzax#r#*9F0vtw}2d$3< z)D%1caXu2=V%sAT&`$vFONJH{JFS5|;)>3zFag7pOIB?|LHaCo(`fT4wW5aS`*Tiw z0qD{!_$nX&9Yh$iso_5};!i(xXzob#Y}Ati$rfknJ{Avf4?yHLI+}l_(~<{vSCl`# zNy_9rhFrD27)wAU!}QZJ0T@Py7=5Jj(kMd}eH4bNe?jJp#QDBi>)2PUW{k+40+Yne z&Quw#!&u$sc^gec*rHKxb6CHeQ0Qn~h~b*|JK@516xO00+YQYRIdH^_#GkP#Q{>>S zH75BpsaQo30rHdb^s>5PY?+(ALNoihiFM#*Cmk1xMn^a0U0;{p$d@)SGBlinp{SPH zSF)uWND3K67Zf@YHh=mo-%mE17?J0a;EBPnjQDow65SfbcJ(IfhILg`Xa=FKHj}cx zI79BCMXE_QaJzY>7VSw&|1s1X;s(^$vtLdT$%Y?KkG;Pb*JO!fPKxXN2E_UnLRH}e(_wj zf+olt7DbS%!o$x3ZJ!*`tu+R{ZB{+Xv9hhB(Ckses#;P{g9g0CSS^;taZ||qpA!`A zT=y`d2Bpcy=0FMQ)|#LqKdT`nI#`LBLz>;Qn!K@1HV$#A1B2%%4yh-@X;M`KY|ykb6qLd zYkqCmQ+?fy&Y@mEp0gaXSv-9T75q&jyaRhZ0Jf=LJM%~8gBB)Prnox~?hF z=#ano8srtwydsKcJ(b&j<>!{{i!@4+ls~pl^x1-n50jQ0aeE#wT3WwbWmRadxOnhd zwDW9K?}Y0G`lLaGq|2?zU$MaItDNU?(%PfCP|H}mYl%OpWdt^H*b{c8pFqTaV(PNr zBr3Tzf=FV4%k2QERt2$5?_dVfzQgBG(kX40LD$i5Iq&XQ8{>|?dL?8g#o^AmC!kqG zf?fBa1+lV`%!DB}u>EXDC2sp%H!4MZ=_Yd!)WtDi6n`!_w_FsciQR$d;Yn*B2Rm2KF6t*y` zajaZ-N)z67s@ZOI~;>tdS<<3NO3~!$9*@5!p9-tcZMg zRm{;3Fc8wwjNBC6{ikDW@k{a6!;w36b!Q<=$d5Xt<1N5=9CWUYJ@e$mL0)Wld7zzC z!!5VMzP1bTCWd=LYQ>8`B`8}(1jwknWrlH-$a zr|uSh+`pfC^}>+u5*gB6+tL!py~9%z#j%6_ASfuo9MS1%NutyTxt4eF&k4MHH2`A-i9D8*T&B+sWNGE53FgT51g+$;`w5#~)r%V)m$&2xK zT`?j9e^vl(Ei({>AU|T7$QLgQsGyWg<$px)L25$%D5R% zdb}~fDBZ}$1u{vh6em6;z0NOJLW&coJ^R(wdjD(UikC_VTOy>A7-*k9R1Ob-ElZ8$ z)JLlj5T5Hsz0q!v_puT6{Y}Eg%?*j^6N?D;@gTwsJi4hg9ng@ig{XE&G2%3^)IEF3 zNv3dlIzm14KLU8NkS09=j6g{}`VwS07Ax4K*_Dc{eC;B2p*2mh8}PV(*V?eKlL8g8 zy35(Q;*6?_mSSl=w8^qjTS+V(Ccmk3hufeT&nFV`QmuKfo`8tdYX>K;sQ>X?$!>U_ zg*7Exw@fQs+q?B6=w01*q^>ebS6PLK+wD&5z!q1Z0i)*?jp@(hqcbVG=-~C9H5w5o z6`8PX3ab#r9Dd9dwAe|*!dZ2b%JCewio2AAmngek=aNS7bc9JpHmjPlU6MxtuJP7R z(iwiBbk4yk+%TXr3`!1Zb2trp#woxBtVj>?+{E@1*V#`?wnAx6;sf z!oZ-~cg%2ol_u5$s;ksr3>-N~rxxd#UQvRG%Y6buAy+;-MRfH8Y`MX}VAk{Id8`p< zCSPqb>>@PR0yD+JubfK+D_ltuPNq>w3x}R}hz5@K-K?kJ1hgcl%iNS5aT&WPu_ULH z^nPgFB7f}_jQ~K13Swu|+y8E5$UOvYw3q2$V%U3SsrdpBV*;yCqIg4_84>j9WTffG zq-*h_dkwe=^U2ux7T|nLtrwyUSuVJVO^B^*qea2^MtxI}it{RH3mw<_f<3)RUje4k zbW)AV3;gmN5^_c}67DvvNTHf`fnEBiq)!LTq#X4`2|s0Wi+x(t>t%@}^Ge<#b+7n9 z1aO_-MB`q}ZFZnPDyrSZNZH_d)F=iV96d(B?Xutb^dJj z$ba?_F&4z+OX<=GY)T<*`fyEG7NBo4)xE&k_{sthc!Jd`4ObR2CIrDq2>1}VfUTv< z>f>_u57m$`IYEB$90(DsmCn{}Dc-~J5EjEYvjl{(1apkJd$Nv9=}sch(9&J|s4mSe@W!!$=;zS@^9 zF2Q5jgW|&$>P(G*;Ol`N=L3495IE*F%9?Bk4UWX)D@e!~sxoH@OdNQKdl?VkRb6z| zdfW=e8;LSAvtMMN1f3=iMpZ0|Lw+kSQronS8Xj0LrAShZ6qV6ISOj^zz%C9Q7u5W& zN3DEI(syC2p8V7E8wm*i)e^(hHH0*>&KVxv@xZaxUk+gS`^er%hSL(+JypSb^yS!q z@P2pUdXj3TiZj&5EV0#Qez3^@6H=Lxf8=cBHD6e`#iH;I;8~zGFQ<|)bzQtYIemnn z(05fKl_d!oSBXnmmWlHGr_^=9+&Fyu3t91CBrk+)ctEQ;@hr23sm%aR6fGLEfIACm~lf;-< zIG?(a3t_# zJ3@_qbNI^>s0BmqekNzE>Dnl9NdP7vaODg|j3G&T?KrKqv8N zir?x{&a@oC8hM7@8x`n2TX}dZF~*qKct<(VJ$-GE89W@x?POs=B5@;B)CZzBF1?e4 zY)Nr9q?4o(;taW(#HrgNR`zVFNZ5{FJ4vCXlbc2%?wR%BQ37-->yLnH3wFTeZEbA0VM|sNK>oGHcFXkd z%!AXz0MH=_BtdUzAT{H{%lezqvLPIb;M2;I5;vCc`$~6s!q-^x-%~%o|JdaVC4{*% zYPsiXd*g~=+51CLa7SAw)vBaHpNM$V^w0=QyEM5Ro$fmHxJ`={*#iiw5HY@#fu2(Q z0LL0A62aEUEq}Kz3SNgEUI(AopFVS@f%Q(3I;{<39~P>5SqpoetYKzEpF!>BS{Flx z3%QEDYcUl{Akkrx5o{?ZU>uUp8xKv70lJA&o$nl`JI8e!giAx zHZ?SbUP90X1^6oBUCk56wEiizaxXU&a-?-n+A?YY0&ucoN^r0sq&gAqD2pY{(m4_A zfd*;5#2yQJ9|@6v_=THDRdZu^-;YbyDNw1BK+KJfgg`}+f6W}=(ntif4x*NBo+tZ4 zmTG0k9?B!`W}7zOepYoU=%M+VtrP13HII%Qx-bv&s8%Q&94r&^zsECcA<% z3Qh$Tn@sWjg}VS;&SmbZF7Nxzp1{O*tp3f7F(G9sP1F>RA0i}CIVwJR#tYbQeopW{ zdMN{VB(gMS#Yt?CQZdkwBZ4Hc7?`OnX`eGmcSVE?ZlqF1tbn!JtSYB$suM8=t6V8q zUCn59jUpoai)3`e?eB1Te#IrN-+Nq;i)2Fdhw5p1Y31mlkvstlN>Gz#O;(sC=o|oe z8saq|1W*P6bX>MeHOpI+LwpX5aV@rU@uG;I7buHR&RYi{CNONkgJQJ~Y&yek(silN z(M#C(Tuxq&7Q-y2*+*$QqsbAuks&@v3!v<1GO1oczSKP#Wf3{c#;YbbvJs7tI@6uU z-rEpfk-}L$m7-KLr;hqmCff+d#|uAuCKnzxU2>Uc^CQV%WtV^t*M}6&UiE$Nrl>U# z=!@GVEeC-<{3rg$UceiCx~2s^^*K6r6I%s05t7QED~vwCAJPW>2+JR!Vqj-qgS8Dv z2qGH|0K6s&Y|{1&JE)w=ByVSTC!rlX@+(uU2IhD!xe7RR-dy+M$R5A&elQ8K92Vb^ z0r`bEFk7Y{G6Gv*$VOE))5CP>GSAIk{U-;My;A$lCJ&S+eX^?c&ia+yrP76aao4YP zJuq>@?e{0Uzeb7*I?}^aappwd5{6Iq@I|$%Pt?0_;{k5c2cx>h)IVJqE!b2xT{7BB z;TPyR(WyNeWPL>t<)<-=*=@wCjy;aC2I4Avi{R1p{_m@Aq-QANXoS%iC1C1Q`R1U7 zV|&%sOGck54HbK0olvC8rEoXTDp1DslAsTwy6N@9u0gp!?be^ijLVcLG9*q526S|$ zKo{u1g1lu*2m%Zg4JYfFeUx2b&g!v2KGw;rZ-5XDW%T)jh*PPD1PGO|u+i|7@;V^) zs~c?ca&v>nlyP#@l&00B*cED|vBPN|JveykCf{Mjs~|>+<=tUhEUDOLKZYEX4d-nKinR!E0UXzI=@r_qA^-d?c>USMM?43RidE ziyf)ncmJEONc!103b-))2nOkQ)xY@0`h`6x5M8LDXNVfFW^>o+dh9|&6ORnX&sn0@ zNffS^NAP<62(*jUA`PB!b_Y1dlMZa|c$k`y+@jYmy&@gCH@i_G&7-5E{TbP0$qi`$ zi${vIDnH?899|UD#LMROSuwy$g z&cx&13k(}-Ye)V14WYVDB*8;XbGcP@-1+uhPhGXM778ybvoK>&k4h?iw8>Ez~M@-qqs`00IZHvY~_y-F|n9|HzJN2i39 zJP!#kMNl2Av>ySHO@SfpSFP75>!BDM5PH^>=_buxWpkImY1!De$_DDtK6m#@(1ct}PdvIo> zPzHH76I=nzrR;zpL;g%dPB20bXanmF0&^XR=bmIQ(hV{k0S^-$k0HwaXs*&)0+o=) zY25PwOdtXbcHcI7v6xD$kB9MkhywW8x|2@CuE>AFUA;Z7o(B&XgUQG2o^j2AFH#N; zCdGqU^MIrVyErsf&ZlFmQJ_`S zqTV2}m^utE&R5?GOB45umZk-x>a+>SDQt;am_q z^Or8Qz{-f}Rj=Bq&ns7DtMCml!m!tB1wLiObLB(SxHMyrr8gH&CqOWopm_an&!L z(6Hi;m2(@A{K!V|gc2NIAYvs;t`_pntcMCAFDy;T>6q7iCS4l5k3R9m%*8!1Eg-dC z@jSMY67oT5wg?MNzVxb{%`LM~C#XF^Fx#V1z`<9gjbgQR%o=c;=K*nO1B+XI#H0k= z{eBHwsa!|$wtV^wqY_n--0ZR=kxTzzVVJA_+wfJQ7JyI8swd4pw3gy(+Kiw#dafI} zQbqU+C8rlZalC<^Wy`ibs=Yj4eS7~Fi1Ex<=lynw-T2o_-%>c_8}y^{KHu00+FKoe z4H3KI9A7&rw2N5P9?#^sM)YZPNx^4rsVp^^M`tx6g*AUw^!M zz(W21n)QY~fEpD}W?XvQEMp_=Hl&b=4|qgxPtL z0#-(qeq7SFM_rhZw9&+sg`+kJ$qx{2ZV<=^F#FJKZ0f42rFz~33^o+p>smN9G+VMx0AS|+#04p&f&t^D8w+E3#iO+s6iPmtG05kGT z3EXO&3T(}$#|RgH3;^8V-8`+b_Ilh^ar=IJHht%wlohkR=8HB=_le!T zSYF5Ow7FOW`sKGB%0Q5O@P5;h$JbYAL1iFTTYX*A=ZkZi{C~5$BvpB`V3kpVOYING zUI`AI(vmmP)(-1!`mSzd`Wr!?(BuN@6NPMYKXdSG)_sK9w$;dBW^IeIZTxLYVL7Z| z!{wEnoK;m-5!*!VG&S3TWQl}l|3(>Mv(IH!jo(@#frpZ?XzZ=gy6sqYNN-i5^fJ?! z#2s)7zS!Tq=99h$~Kyf9gTngzN(CO&sfG zi4{3{Py;38m&Z{9--PslkGZ;al&p-;Vaw-*v*^|J(pV%t{zyf^*bZh8>fEgQ{&Gb- z>S};zN2LXK!!Af<<$Oj(G52KtZ{;6VRR6~((^sb)z;YdZ$ImuAC zq#i+UCROhZlI`Hx(hZ>^IYoF7bzAVCTjWP@jnQ<2;oJvH+DdO0N}q~ML8wUo&KrzX z7{$gy)VYH$q_R}ED{yG{68-!CRV+u8&%6l0Y{|`?L+_6Qr4|5bN9xyrrim>b0mJ7Q zt=l|5hUFsFa`$l1nb<|xg_=;@f855-=i2_}mgem+nt>V?;SnevQzUBHl{JpDFYS{JYINz%COd|UB<8hdBuy1lbswr#d>fd$p7<3 zw4aRWo{=V(Q4`|8&g!TA9eA0_@MJoNYxP9*4R_w0Gvr?;V~03*)saY&;9Iq!_7WLa z(MzJ590PDuDotf^!eo$F-4bYOBagdslJCrj^r)q6i(qxauk9xSGPu)3mH{0i3RZIF zOry_HUEExlfpQGt&&+zSMLTzxTK+ag_NhQY;)C3Y5MJ&O%2rODR)J@kSsjSZ02x-lK!f*i zqZ0IX`IhRwNeb)ga@%UtoOH40)4eIv5Rnrgj0P zb)Uo`x^UO5BQo0_wSksFuL(FsMIMrZzL87cwji%0rsQ%U9BHL=qBIAgv;y-jqnujZ0D+4O9f_(}P?wYq=0De>dvCwhR(NxZ`thohCOg&>qFwA&?SzQe50Q zpZH3I4K{|w4Z%$sdX3o=RxW;Xij7s@shw>EQXiRZQwjgV{D-SIR%2B~(;3cd@%ewr zqNfeLPr{0Fo#DA$2kDH*TP~bMyLXSF{hx+{`1W-#`Uq{Jq9T8%19Th8Hb`)le5wRk zKiYaxHy0+m2Kdvdf%vp?!=EH*K&FvcX9~LBXX~x2_&b}GT%)BB9eoH$sLNLz1Bzj! zS#8}J=nT$d2@^RsklTI#ZXKO$Bygg&LuH9fkb!9H5F-ja=>mcLZg&f(dF69is4aBn zO=|rZ4ak{9Sh4MOLy>&Ryy^frqdEb%cMi{Miz#+o>NOF>fnbA%ZG;)d_7-z$9T!N6 z$i5{WWGg-=Qw2&PnL}X1(R3sY3kdH&4vqH!KY_^b061~%J3_0O#x-^TNc2HHJ zL>Q*qM(A`}1^Z+VMeU1^NLs*MjrusCFuUcT0l{!%R|DW+m{Av!aa~=jJR7)|`}=U1 zouJ)s?stN=lLAqF-=8vDL|7P;AC>jE*v3n3sToRafETTL%bvB4e)Y2lk-rEq++q>$ z1g2~~C0=o5e#?zFk=d=dIJwRdZ`MywoE^)1)!)7y2LfrK(Z9_z8aR~0k^Q4DPzqwG z2NF(+XM4GjxBTB9MCAdDjz!=9{2OH~!sHIV%(-O?GO~~|%xF4TCe}B!@rhsty`2P= zBzA3oLvmxl7n-xw_#@Q|JSMyuF`~$j5Q4%K2jDT7k1n`?Ei$v z5CS0CbK6{$dNvvGNz~K){+~aRUn9Xjwkvep0})%Xj?h}4TNkZoei#tip&gA_Wgw8P zt0Tl575Q7fxDV0*2|;I}f&rC(IJM;rHL=m-df=V&e3cPYyAk@zBUTUT;&NC#KWY+E ziG+|&#|%>%62HcZNEfCLJ(d=^f3T8-kXhinma8HsR&rdru_C~J-6y;~{~h`Bzjh|& zG7I#%;-l95H)>+9m@&TLvJ0z^{r=3|fU8qiyi9%&ROeA zsk)l`Y#+~KYy!C&y!#rAbkn&!lO1%zmkO^5bdqtuQ(phJ0towq|3l?+VsAE9K$nZQ zVnD4PH9HQ20Q=AB+@h!Vu6uRWDvR(9&wv~m+5Tly%nD@TCXAbO^){@f)UiHp!>-Kr zU2pKOZ}$|dI3uuPTux&7-DvWf|Ik>KeeCB;6UrMwd(&c)kK});m+~W3L~VCDq{bfD z=&k+8+@|JdX|dxZJ|(wMfacqW_Ag0eQjM3~)-gRX_~fg%0bSje@R3uZe{Gq7YW93W z+MyidNVw%?G3#QzWge;B+-6!@Pmt^Dch)LO?@)ERVdUxL{cj;ahNB?yDnlGKZ+AHJg36jNx5<$0u`Pv!cB74#VFL?1jH)h4(tR)NPjTw#EiVJf` z{&=6{f7HDwKcEd$th4iXs7;gklsh00s)0(*R6k}2Lew<(-RF9F5@PRM@1N6pPcR=q zM_p7^RPYQfoZIwBysO!U3^1->v)-+(jf{?&+!C|Z>@qij7O1_Y(!_$DiWq1ejMkl9 zwn+sLLSbQo-isOXM{TM~GFcpf!+^8V@AUVul$gQi!Fm zL#tN;oFMe5z#6rpL`#a38&kgZuE`_BHdH{M<(ZT}I^)iM<jeonW1MWs@f$t4%ep65+*sG8#B33a%|@O^m_A(ZTqBQ$J7pBTAG?~ zIu#xEJ{c+$Pc0{8*#W5GGgbLupw-Vl=0cE`aTroI8YhMzbFo9U{MQ!wAE{YJM#+Ku zSrUydlo&SS>P`pZaHBrQ-gPnF@Dd%ayAY=OwY47p3*=0H;_Hqo6m@BY0S?WC2X)m zM!9xqRN>#f17nLoz=8Y*=Kl?4A_9ZPpp}+Qg!5YR?gr(Mbh}bH?xN7r$i2NT^m$h^ zPS22}CV+FEBPweS2VTzsp|8i>xyOd{#LMgIlzpfP}=si?~x2@~B;CWAz%avgv znkUc^&@K)Q05_+5F28#&`+`uFn8U~@aI>m7DTMbV2wOx0iiyF{Kq6UKT!qNl+SD|O z)=_?~(TN?ZP^saC=%nY-E67n83zN5-{@!4R$4SX8fe6X}`!l#8kPD{=Y{s1~U$TYl zexrbJq_NjjKsN9V`Pw(!LK4j=)B~?|t(Xk2rDu^Ot)rrX@MBRrT$`C=GrN%F`OrRk zM8mt{gO{hi-qb_t|74;|a%hpWQ|YWbxQ4p&S5kdr1mrWZr{$~qu?u8D*Tr`Z9~@_< zVjBGg-HhO$#uGr@FAETV>}Rh3q~v{cu4DfkF+1I^v;378L;Exwt}pezRPrltzxF3F z`tjAmzm&8ld}l3nLQ^+LZ|CGQL7m6HvD|5Gp`W6rBJ3S%&Ku ztd?jDs_Az8_Rkw3l?$~9&{O@4IS+Sr`(AEu3N|O6G!T_KrByIh@Y$z24>j(5X z>QI8Wl$3LteJWgJOiKOpF@vOaoUN7ef>tY7l=}MCk$4Q%t_ZFLY@TO9K|ytWi?w_f zJX8sn?*GlNAF%ogF{{NFN>5pq)~Dr!K(4^u0%-=B-kiO{{GnWrhY1$MhhsiZJpxat zziud%syHJ|v7`wJJ;lq zTlMDl_H($!|0bAK0wO$j;OF<5=h%f^Y(dmCAJg2IYwgJ_t*H16y`$oyqEBdfCB8Lk z$zV)HR1~wv(p(RdBl@laltbw=#b@rXqj*QN!6Ls&oDM+fUh~{<+Bi zn+y1VmjE)DMgNnYm zUw-46R6GHDa03g;EZTxuYzH3<7vq9%fmN7xKz`p+Z^XFe%SS&kur{>bbgTWyo^DL@A6}!4M zR6*Wu;B>Ulk^l`=wS~Lxc3Bx!HCd~fgvu-QW~DvXpPpW>F8SDbd(WK(*LU|PRa@Hy ze)uBgQ`-k}^FAn;rT$I9<_%%~aR(lI1bK}~+|y%eLtV1!&u5K|jVGR7P^-J3Hs>^5 zuqC`bv!O5JuwbK+gVD)p{OvVYAF3qB%`4&cCIwh8o*ppC$>rFBE*tzvtFj=~veR_f z|3Aa}YYME6Nc0)%L}=LW&=0%yC{WDcg_Fl$R^O-pR(Lp9JuJ{9!|Drr!Gl%UQ?l(5 z(KTq#yO)sA=-ksl`mdyTL2Ni$o;wi z8!Zi+z>8~5)g>M~OUu^OG;QE_{vc*quo)`u)BIO@X-YQTiHV7+5Dp3+Hv)%$q&k}t zbknkd=c-ev9o+s)dDoV%yd8D5w`653Jbv->3;9h?DRGz2nTki6Vr?t@CdV4$5zAN2 zdhbcqaqxMqZgay$ytMnI(B=j-Q7q)mDMGvmq4iu6~8-U zZrr$W1)tPfc0C;EgTMzsQ_ZW(-|^uK(gp=&Q|kC)Hji^p3(PU*s~M!PAz41M)dFk$ z0q9k;2=BC1FF|G3u3ZGZWqUR-XFjIedoPhPr6quJGbOpUmGdcqt6+f-k`&O3vk=5y zlI+H&200&3^+W5?0J;vcu=#~i^h&N=*{aJ1R(FsiUmj-Ny=n_B&e~1H83rn&ubon}*?>x6Q6UsVGC?D>V#YNYnnS>D>I?xpKE7 zBbVC#EBv!pTi0OS8pMxw@FGQDyBo!=~dUb1OU?#$nUZuOO1VWv)VTGT6%na{Dv5T zaj13VL4`1^tINfCW{>1RE~2W4XPGFzQbrNW@7ngMMWm4jiN;iylN>)e%`7`Go15f5 z<0g3*qZyOqyt{&ef-$g7V>9iRz(%!s-$S@`;f^}n&{3OtFh8<<7Mhjo%a*7Dm7Wwm z7a%DlU@0S3NX{A}VRMjCqllDMKXw_Hf_sfQiXn*UQYdM$rftE#+z=T+-4AS&4+?#C zW1xn#8FXQOVH`qGXIYLfB9EkKvf`Y1a2vaU@ZIA^a{3{lBkk+ayAqM)Xar$f1^#iS z3#wFJTru`w-q^}VEb84nCWkdqv6K84w0y4AUYVf2(2AgRe2?tW;TU?y7i_`-aa`F%(Ojf*^ zEYFvqq8E8PZBS*!grKeC;)-Dpuz$rZ)?%cIFW_F(Bf*Pm*H?GheGnxN5>d2-iuk70 zeoT6YRNct745h5=c}W-`-M`Hp{V)CW#9zO>k2*ZDnn>@)Q|Sn&!dh`{+;Frw0RUn> z)LkE@*g_dwM3S{*g~GOTuXKM?#M$mpk}gc_Tt{7F>Tp2jBb%y(py!^Z-hw6R1f=ku zMGaJ{I|iqP6d6zgsnE2| zdk;vXG1ZhdI@(rLHMk8aM&P2_r@zxToQ!WBQOGqfbTr+LCqjD!2_c>UIU^yZD>AsV z%!9_bKqY_gJhspCu27FpOul%h@4GQ z8*5SburEgqr>5f$8!~;)f^bYx$9rL`V*Ks8Z*)9O;wlvvOLGX0lrd{muU%7Z1_A9> zF*R+oEr)=ezpObzF(_$ve&Y_18R0A^Uuq%2HzNBg^B*jj*sibNm3LCRIquz1%cftR^Q|+#Mpm28nod{uXwfL47ZSsl{p1>%eA0hE&_& zOe}m9ACCRx8348p$-K^7RSv3QJyIlh1n9Ej;*>2B-h(*sPpYa&A&SU9Vg)aLeqZSm zyqjf{ec2RoO%E$+wgkg6-eai%0FIQ=TG4W6*%7%aX=m9l$ z3+yjl&s>nUW#7H|#Wixw9r%skd3(1Wc)#LPY`Wdh6=T{QSh77KRzNW#)i&t*pf2x8 z3JV@eL(Np@h^go5mYL%o9vh^}gK*!alQbJ|dlZP?`2biHbNNmDk-!Qt;H0~1iM5T# zd723FQ-QS+JWb_8y98}3M1>O%bwV$>;vcqZSc@r}Ak6=LU^a{~Ch&yk`Q>-+kiNK5 z2v0&?W&Q5IMib{iA?hso$aZ6FFC(p-oP#SqSi}vi+@N=*<9E~ON%(AC_$dm?XDKWP zcQ8yVDcWw&Q|GpHH#ZLt4~iX)VU$haVWN$9z0VzW^7VHf)p$Xz$_olN33DdSlWsp8 zSGfk5wN zL{PyHVyWCr1m{FevkJ7mnt!-{x39*ehHPd`(E?{T$d^$Ok zG$U8Z$V^`$&F+5p=dU(=j%DgS=7tLRlu-&5sR21?I27}bvMzz0(<8%u>A9LG9;>f`Qa>tyv%Qav}1~)*et)`MXsD72fTEHW$4T&eV)BuX1 zKEpr!PM8474Q4Kp;@+?L-@jisH!_)utu({7{q>O}r}1u=cLKm<9@!0<=G6dHg;J;2 zJQC^UeY<~htIMgs8;0h=qmav3P6GJsmhrUesOH2<*Vx&z!Mw3sV}TYf?Q_}KPClDE zn4?D2$)s%5rauF0k$-9PNp5o5z4MtU=oqvM8MFYQ!qQ(r;8c)*;x=*)#}QeGU;)TR zaWB-u8(Sq11(QgBk_>O?;O~gBNiCNk=)gdqdBzd-g6_QBq{qDCUrk(f63IT%q)^X-A(sVX%2; z-k)8PIrXjrnCJbh`bv=#@%57GI`t-1i@i8~BJMwHQMDdOgp(oD9svtvNn;*6tZVF> zwbEUk&-f&nO&Q|=G!ti#Y!E#V`{)J=W$NQoo`lp4?$XiGxm(VX2a!ECG4HiUkvu37 zKJLTZvYzY&siH~kGg8(JiHlolXCxosfyE}1&{kB4B+&perxaiK&aGmX`7{V!@$@9k%{_1_mk%!I02hnLm1hLAY)8@~ejsGh z@1y*7MvRTl=HF<>Q9tMe`aQ?%$tPjqHs5={Ibv@c-ue}Htnu;f*s&Ir^Qjr0pPxU; zs2?tMXJOoZOV5g6Pv14Mr#v|XixD>7-1i##Can>{EiS;M2-uR?k;^}X$U+X&8LQGrU!jg zsL!&+pjPX1y_q(mqM~Q7a5q^Y{Km&7>=+d5U^bz+ya^(N*qKEXxa{%n^uPZg6}I1p zZoiuw!ksN6s8UHE7iZV8-}IfU09zS_g^3c5sSo$N)`;5&(Cq!22d4HS!!8xU2(v-} z=(`^6H8;rbrByKXZdXN3lytF8Rxk{nUw#(3snFDcMtNvO47(y%zS{z6PZ+JeiF0YOFwuHk&CTRFV51p-(4^a^3L7=U-)@FZWaz%e9F2nIsZ9H=)HP z(fkBy)M=nf`oDI^qCsv2-%hjHfshwv-|98!aTGLOtEedZJYcEc#*CH@n;YV*&A)cD zHvMXL^0TnC0$bZn_tIV@$6avWeABTZX|a9a#ocFy{h!Kh(YW4!^_o;)*R~2_R1(*i z%!h;K2R_yXvoRUOzFfZGh1eZ1a{!>NG+T{Y3LAf^J0_W;GOutYr9M&sWqNDSw(J~n zxRk&~%_g%{=>$hJR1ay)F|m!_K)w^<`k93{ z8b}CN^b*WYlYRs#2t#3PhrB!;`|gJep!Ws3M)>_5_t=8qny&J;sX#2sq^f>HAo$6V zk$_&-c+@Hn5cbVrio`_JS>L07{NVyDO6Sb;b)l9`xCvQU54)quCNh7A-Jum%Q&-AC zJ}^9r2R()WzleD?=}Piacar#>n{R5x^xwU{{J{QWC=c&)tSO#Vd1BZM)nVoY1_JDLvgc^yPsvs3y-_w2?1+eOJfRGh)wX_Yy zZbev!uJY$?M?@Ymr?abUsM_#m;!$%0pU(KkvgsLxA4pa@Z*TTr5LSfUNu;NLVf#6* z2ZIv!4;peN4MWMcp%(mYT>QXYBlk5(G~w8l(UIOl@@GB+j2Zq_%$k@`v%hpb(~Wwz z0x0Z0q9}_D@JBQF(Qd4#>ne8ZW$w!>D=V9^16HQYe2MZ{l5uRWh+z{BIMuUaT*$(m zKKEC&L5(5LnADBl-S$z|W0D>&S?s?YAHTY7 z=+Y~PYb*8!=6;Adv2%F{mlUXK<`v2UaQt0lJn&xh41=*|*&RdlLWq?tJd=8YrV;G~ zSU6JQ$n`@d=0$BcMnlGaU3PYfDXl&18_koC_*)8zEu7j-9OuB%mcX$geihY*`=Hqv zquOAvT4OCQ)@5JlNP44WoKAF6^w;F~>FM>JNhQD{-3f`(i)T#cPGaPvpEnn^7#clH zRvn-)vKk`o(lxMyog6!A2Y(}o)gLlni!qCe{h|YS;r9LRSfo!5{5CfWUcR5`7}#ic z4w0P#%=sAO|P;Fhh&A$A`jzB~I|I$?`UoOdX>LGzX0w(JJ zfKQ^j`S{c>7fN-C(=Eg-Ho#S?MM4YL2$1p z)Dd(pi6ukwF5NMn&PxEhFuL1BTRtF|>vB?Tf$1|ZS9Ug~kcntFvgYV3Q(1QsXX&hL zseN?p+OH0|blD>esl$6Z35G4A;R5(KQSaG+L@S*-OpV&y-PrAXJRNQc2$chdeW@t=A7D9tkG9cbrBvp9m17rco0}MgOk! zgoFe!P9+xIG~4^O$Ptv7jjFS=B-CP zj)hfrZr~J}gWO&|ws3y`HHv$Qnf8FBaY>OsQV!+BuW5q7+i)iE73~c_uo1kYgO{!K z+zUY@8hRY^x8@qC&3E10s;d!9{`wP}+(LMfFk{=&lXY?7w$`$dV_mhj-10iK@FgnB z$B-X=`*vG=yc*=I$k>HKd|_d{z`58KYl#}jTq=QDBBCA{?hzOe)`7`-wd^!O181J7 zq#@bX35|@1c+z2yBE3a*aq-R~efHvh_QQ2#L-fmEb`DnW#6}B87gWdZ#mAGSl%qdO z@?7i2A*TZUv5{sEbl0EdGG&RNz9II_emSvl@jMi9$?NeZ-bHR0JoE@V)d{C9}r#)^uC2f<1%+A1-Cfgx0o1$jAYW3Wx0d-QVniHJ8+Z3YL#|$f>%)P)DD-Pw(QRFHMV1|wSdc@Q; zPqZKEM^&cnfbZMaBeI%vh+Tcqz1NHM&DWrbb}j#~F{I=iS1l7yR+L3SY2n3|=e_Lh zD2Hj~ExUtZ(&V0KR$0V8tt`Nn(Ge+#lXB zvkaJ!-YkeE)EI`u$H(X44K28=KMh$_bD2LmBmqWgRgW?bwGP+!i0oT<@u7krmB?CK zrAHs$30n}HrWhPD?^|o<&Dz>PmucU(dpk=i>3A=iVRd6Qz7%*TCPS#peh9Q2<8C>*7BQ+QV63*K}p$^DqG1+Rfbo<8*DDoo=( zduB|5-&LG>+F`Xw>Zalg&~P2csacE+zR6%1Ldaq*_gs3oW$D!cf>N8qxu0qNiz@QV zoc{dKz8oO;!Ut%6+sQYPg9>!po3b#lEul`>zi-p4PjlZ%TteI^_*Rt^jW8)YqA-cq zSX3kNA#s5}sndKKaY#p>Toeo_8DNid^TsxviLh%XX$X^rIx>C+%n^@*iYE2^dm4>w z9&1yaX`Uwi`nj7w9D89E-$Gzzle6;+=`KBZX8OZu1SK0Jah@m-Ip4G*G!&s$o!~2Y z4$%v|n1G@MMJ5vXrYFmkJnpEfv1=~+c9MO%Mom+VcDOT}cV2N3Vb?2!HF(YIH)5&^2`#=hwy3JE z!pJ}-@%{m*{}QaH`Wa$NlGwIe(?qJ*Rpu$=#;CPi@UC2^J#Ra7J7k_-yU0TgfTI4$ z&rhzFUErNsO_#zx86VpFDl{ zwtt4u`0HS9E@)a)-Hm*ESk^}poVa0y)Gh+T5(bqTatr-ACJxMi4~bQ}ZB2+xcDxx^ znfhbnRkPLdz0ZUGn!DZT^l+-_sag{{>lbnXl?BI+-cD6Fw#R2)nh&%qBgYIbl*J&v zf-QvNby5Jz!d6erB;sZbcV)PI4(_2Pz#NNwIlK)WQt_0LCC8SQ*FPk*VC>Pn@tgmDQ)7?h|G%lRCn5Cz#_%J2 z$IDB?H_cLLltKdCqh61aJ(3AB~*_{{=OatAqNPc^K!+* zxpH5I!R*A!#UvF<^5#@WgSumEg{5sXIR(-d0=ORvR-~YYO(T<-JNF^vOp@F?B))ug z;ogC-El)meWSNW{>*hhxa3Ud6W4L2>_{R`_&;1sREajF|-;^mjAKtr-n_qR&lVTFUB@tTl)yPmBTTOxyQQ z0Y-qScEI?8M@}-qcoX8{j4KDA^O+$IKFUrJ6kRewR8GQ|tTRZ)f>O)S(w5Y0H2C&} z?zR}pl1;xLJBsdWg28*wA_Iv0@;LgW3IE%uAQf1hvp^utm_N=&ZTIAfyI6u2 z@?~PsxRW3RL==8DFPsZ`G=eqCnggx?EM9!j_=i_L5v_IChg=OAhSNMcro;bOIaf?z z7n*!yf~nWfdtT+h<9_6 zN+ab^Kg}0S1iS+&@A@9=1wgv1sj2bq7s;=@y4lWFMc!}ez!PT!d7m={TW{>IZPVbj zS(~VZ#s=8eSC>DR6Mmlbjek?g{^$=wZufr9(Iw`!^nR4K;B?0O8?+X`SUViRU5xkZ zPb~J>Z@owr?Ht`vbF)rryyp9sbXe-@)`hg})T89}`0=S1M!Ks7^rm?5N*|B^4Eu|4ESbCIr-~ojd)zHlk10HLq0jzD zp4Wrg_r8?$IeCp5#uQKDAM{8{QqsqB(1MPmO3~mx3>C)?8C3nFQ!UY(_gwL65-5jF zTrBoWp#@baYEc;*R%L3&OHIn-fIsX6u2E>BvhM_6lqF}Ov&g$285vn+3Q~?O9z!ye zZE1*ewFpwaHHVzDDC~v^lvU2ruf} zjX!0&J^2S1HD3MlHI>488Lf>c-wRo3FkSe$oaF!b!XR2dLr}g@R+$*aoj0lau?a=p z8bjkMxl>1(q9BooJ(&m~-qUwaD}5p&B4@ALqcd~FzVvZN@ztm^1(QuFxG7=I0#^#zB>7DqfC3)VMg;dSYzq8cA+O_VqLk@m>^~z+39r zx$xD|!dFe)0nHAPGbdu>>ic0~3jm$5b3BQn6^S>>&H<1QYDZCjKc7(sre-NGo-%{P zCMx$@I05nuz}w=@tH+&L&@mJS zqxwMf$c)xnadAS@lT-sN=~e*5X4Q*r{&z}nCXY{p7N44sH7M`QhdVC4W&2tZV7^zn zz3NY6qQ+H&IXh|l45q3?q$MgJsH9fryugmn&OtyhG@j4_b5Q{WhL`&NRQYfnt~JDN zP&sV#Ym@%gY()oa;`0(THnk0tznciWD5KoN*4v-jq)2>AqDOh{e6`X9-I@6EicV3H z6cq0|GSfz9y1g%*aN&51pAl>csHX*2#u83kN$!r<_646vRLsi*v9&$_`Xq^yp?k-o zEpk^U<^#BrYAO#v*r`+Ovfq)~pVsAWEpOm<|J8fb?0#(n-G|1jT(F3Me>QHg$Hw$^-Zoh^*ki+BLG&apT!tw zXVIESPRs$ad|%V3(`fvQEMbj)%{6Zw4&ImF8t$Hdt<9JP(1)#j6Ztl%&9R@4w$WT+ zVRv`;?wYQXb}5Z)bSihh7VUC3^K+-dxR^JT)D+Q*_7~lOhUIp-25;#mTs*_P+I0X= zdlm&_DHaC0D!x^Gw5b{o#J(Pk?{5+t1Bm{Qu5R@JRrX-88NsdQuGc26!rY}a??NMz z*pv3!%S$KH~W)wn&?1E@9!T=4WqvCC<4(&3*=aRZ=&kPRa1Irelx{lXwHb zbxCox*&yYKlbt*3kT*KRPu&-7^FG+QLsOJqiGE zP;}p88+&WF^}!XBh7J1(hh@Df!RofZSU_68{`AyB^~ISWf&&#J(x0T?(P=ny2*4P( zaljaPlgtRQ>EFL)ky8$Q6$JCX4kjJw2@L9IYx1t4uy60^m46uB&O$xyV`iov1())# zuj_g)ze`wDc9lwU{C}s^3=!l_Vcrwo$(qWXO`)Y}J^DWXTUUlbp6PM^O8N^~x$%#V zeIHc!H^J|zx2Y!=-6WctMJg5rBr&krk>;|PGq>Niya*+eC?wnJ&7+Ia%xUm-B=5%4 zKz>=97C`#cc&M)>$r39%2r=T`^kb+1MBMIA)z)sh(YiSi(hX(_pLFMH zAoxxDX^Ok>d!`@88U{XT!~4ssYFZ+5ccGSci=lnQ8D7)Be4Z*TIsqi0*RcK#Tf68F z-vk3bdGhN>TF^3y!mcmgcqW<&3CB+GN=iy12P@4IbRhjUTV^rPN{>Ti03{7Zsh&(?lt%wPtF|+>O+R?ID=o6dCB+Ww|Ck(C zerhD`1BH5y6$}vAb*%#3r1U7UmMWKv6Y@HCaf=HQ+;esA>@AKQByQH$g%?F{mbO$t zuz9i<>upGUG%;FapK?`dIKOe~L(yeM>8X0XiD@nfc1crc4ySiFS#1I#iF*n`JLJGq z>CGY)jKs!efSBenPK0aS+ZQ9Mwh+(PTp!KrXxPS?7m`XaZ-9b{>%nZhne?`bZ_!O+ zB*7BT%dUe3!TI@z;|On*Y6|%9(!DHp4*Ks~7bAbvjT9_KNbxTbv^3desBT;mr9LsS zn!Y>C9-zbax7}W$954mOf8Xxmu&|m(cXzxoGF+8R9)<5@{IRYlBx(dqPQ&&leRN1j zbCuD_sR&}2#~>5!Nw1acxC7=Ef|CX~l1)k*BCrC{#Q6A3hyd8#r!9&qD+ zffWK2&@Q^)7L`)>{oIlL)Vo(9irZHBbduQz!9&n=Az@X*=H>@r*Zv(?yM7}Ufu0Y4 ztxU(=ZTl@k&Aj*V+E38BEllB|Uj$DQGXSs5QvTfv9WBFZfLaiy{4e(211idMZ5t(V zM|UhS8apbASU|vn2I&|ppduwJ6dML>p`w>n{gUTQV?0_B@01 zl~D!!9L`w)SaK)nAq{gzhU;Qce~jkBucNTTyrvFlZ6IeXNHf0kuDY`rI2uAvKn$%Z5d9?2%zw_G0^UT!sk}5z zMq1yXX5O3T-k@4nOaWD%jMkn5t9O`;)LrxS3{W;aaUT?UU;?`hJJbuG3hxmzDv_s2Ck9BGRKu%uVeDqi4oiKK}i)&u)dp zXG1@(m8Y~mKsf}*+kX7BSMmPg&gOtSmrYQoRq2_YJSqOuzgc5qSMZJuQk_lx5%64L zkuM+a^*zUL?g}PWB%_{i3@Bd!jdHdl<$E!RQ@>Hi^rDECE!#FoT)wVz)=?TEKZa$yCnEvuKfYJ_UQc0bP+`wxS zXpe;RaG)?>{`sSKrys7D(7np)#+*u6GW}gin(1uapEr_9e|m4ApRT0}FtoRHtcX$o zZ9?WHbbXHi7XHnyL`ML$s2;V4agc^yI(uZUS$XOFYl$b_+<8gpR!y;)<9JEvgxN;v z8Hvm{?#1UytF)!3%=tb@Grik?_LcvUcog#CkD*-rb8xsbC@T`H(+*-PPgF5g^GG?# zaCO!TjckbV+JEEHrwOU@`IMOVSR{|QeVSU^Of+R>*S{{htQvMR;;dt8LdFAT#k;CY zk#hG+(#y&#^_SoB;mF*L_x;zNTDE1}a;|P}UCYi48 zZYLKOmxH`(Hc7f67fl6bB%C+BOIvnu^ktv?XZPfQJ6c!a&yh{)h{*MmP08SlZ0o4^ zA-wGKoR{9=v^2Jzjbs^0a$hq*MM*X#NVzWi(t0GLkm>}I%M==Zd-m$;!#_>C{N)h%u$Z=EwBmVQ zTS$o3*qMPp(+OO(A|{`9GuH3`EuVj=brkJC3&`n(w7D59NH)drxa!~?)EsO$iCB|62cS_U&{->-Bdgt6n($ zm%yD%`ht4;$7jT_sy6)c&-|zJ{_)cU!O1JW^%4h-kmRsZs}^%T|2k~}>Op~+aavFGvO`_uY&SJ-~}j|BuA zKOV(?_$z_je~*dT@$q=+|GhtQdree+T}f7p67{)b>ppv)X9RwsOyC`W^6f9RYIwgg zQq_zdoeBSCLrumb(A}!3bpbbL=)h-SyBTNSf5hCiAjeVjBer17Ir6?eeKX>};-cOa zdw**iz?^fV1PxpsGNIh+K7TG*uwX*SoV37}1Z^&5fIQ8*?mTnq=#pFuRo7Xgl)KCb_yjIq!9*)vR|LOrK8Z+IrCF z(*4|d$GxRXgiS+_FKUpuXS!|7-B3<^f7@3-Y^_d~B4_|nmtRGvw!5XIr2M*oOL8=( z(aE>fbBG#CnV<5^nfuk$)J|3C8RJ?LF7reC})MdqQu;#B7;6w~hqG zU(eD04H+YS^p>*im(#pvX)oD1Zm|W5K~dYW<;z2X-5R4fSb=;!Nxut-$~-~I8tnl_X=qCV+n#tzD6kW@U->l9E06q*8a+o1R1yQ?Z3~o4@USDu zKn&5cI`#0Y2_&q^6S5?WOP)_B)Ly!J$@iL6lqL7@^&-Yxy{AeMZhHKmm1XY5#B}r~ zlHv*y{zVj+p8!!`IXpll4oXGP0tsdI6ox1%Q5v(bEgtw>d+2*qDK3Pl7c3;>m{GYBZw!f91TwBkmkaPT_ zysic`^C<}6X2JdACRUHIa9NCj5bW3u^p|wk&V!){<`%fsFbpEma79*$0oGvENGi&N zmV+x-F7Yht(%g+1u`rG&2qB*Cb_uwgA zi^-6e*r$>D^rL%!KjZIof!(fsker;n{mRF07sNPSyxp_TDJd3gGVAaj28+V;`pAD)uh^Om5marfhT>iSAE6`lJ_RbmTruRfBtm%v+li}X6F znW;=qUYMTKWfnSVx4UYme)$gz>ZEHXO-N}U834+a{KUK^5*N%?Fa;PD%o-{ydQZPa)%PNOz&4 z6&1k2*JB2%{ExhT?ENKO;Z34*r_iH#LRz0Xbp!j!>GcvG4(G%wV(0^_eTno!GWv@w znucniyE>OV`ksq@mJ99C(EK~J)DZUV<)brDjH3r013)6q!Q*N!fIe<8vfrYkQ>DmV z0AK-6f&{yOzwKw~+|!u%$C>Mi_Apd1vs{uKIjMCEhyX=w`!f+AE+YTCMSAZ$LNoG6 zBP9n9;454PH)4t*hNo6%z@msJC^}FYPRL!AyPvK3?IG{?)-7vkumE%oR7T2Um-vcn zPp4inC5MR?bqkSm6BeAP6a*KiDL4GC#4SI&!%s*4K9$qt^h%Neq^S|h7ugzf7N>81 z+!M|XghS*Lpafqj1XnAMjh38Z}HmX{n_*1(VsYkJaZb7^EvBk zfQA>PEV=|@ho_}WlriM?JOBD^YOlix=?ara{G+j+hQ^^c%Rhh*@UGAm$~p_lG*Z`; zGrPWUS}cq$`-<%{exMF2(OE_Jk6EyxDjzkBuirtTz?17EJy-1G`}b#)&@s98B1LW- zs=)?KtDYpDZflEuxGK7dHa5!|F{v8m4nh<8RU zP0|q)7N%Vd&)@}Org7J4cTA=2&LdF?R*xI(%tX)NgJ3@;_?6^r#_63jOrFOL;&fO|aXDbwlJPleyt62AhQ z&$#yf$>9C0tWk}5C!-$nxtk^ZTFNqziDEPYybju_6Zeln{q5HMU6w0wB4)0R8{rOz*!>rWyn7`mMhw$yfA z`$?h9lp~JV)mn9eq)We(u-1Ne%djE&dl|LMJ054K_G7aM(qrmM^WW&SAxUzJsFWxI zE8z|}%DdNdpkh=Yh*CadQG(4;fO{5Xz*oALyv2E{fva3PFLbt9xXMwY;}SLIIaSrO zmCL{${f!N~pyj5qhtLM`nKPrz)`TA3#t zRkomocwZ;&~`II-74uh>fOS=qegPzj6b;kqW7nN7(*H^>MU+8nmN!v1G>;H+-lImg-l?|gJ_ zbNm{6S;hfhExKfLNxXYxIqxcZ@|YgV{>N0jk4gMFRd<=nbhFL@?unh=$5#HyQOUU9 zEpD=Vo@x*6(|XL&JLqQI@jDx#R6#Kt3Yc?+3`DX`?^dM&B93gPN*Y~g9W17-2%`5Ic9*z?48ZwWHR{%ls2nBW2D zu84?7!z&8h+jeTO?kgNr__}VJv7}*6Q$%@b+>sGMyG=4on=K`?e!C`Em6Lq#Sb^r| ze=&AlU`Wnq6py#ko1@n#^GH5COMdp&O2#sUM&CC>#{KKp7AbpXr)>2#1Ael6DgV?q z|Lfz&bs9DM31?ece*{3=uNd_(ChWa?JKdQ?s0l8Pli>8HA|-`U(}gAO8-x}6{+yak zwiPr}BIJPcHA?Df2L)zl<+0JcoIY>&cfCUiMWYv$_n)E2+qail&||$&2w0%I1xI>c zpk-WfcwCQrAP_2~;YBD&t||P@W@o%T^K03dIMsMF;^V!iF&ulhMRj?3dHoSoQSk*w zZnxdJ7=2yv6yN@QYqkiPV^XS2r7IOs|5* z${c#>KFyiE+qCuXR?e-GPTH*IJ>%Vju_9XZ4Z<8$3EQ4Fz63fcF^(=(e ziByV-E#cn*cX^ZFk|i%jcg;T>{M;~j(vmj$%fXff?}p z)WKz6ldq#53gjxMGUG<)n$Kj-hNf`4#shQLcO z>d{NJb*!W84*dHUyg*D4(mQb(3@8h9k(ROKp3!HE{&4lrnln>BoF1Cv@uCIEZIF26ZO=w2|H8GR#Z$q6Fl41o`siq?%soGMlwl6ZixY=)>WAmRAgf1d? zY)Ge0^31!ux5K6fGWJ7CN?(2q!Kn_np8(t#GIfg3-^FL`~$NtJ1lN&W&|1?~+ zA*J)6{ybS#^s;!qXCy~Y>S2@N5 zQ)Fud0ReJxp$>f7)(%I9s;b&qBQ&IUxjWi7q!Z8o<>Q#Ip}BA06P`#6JXGh6bc_sT z_ObxCC7GKA?NaE#_u=X+1U6IkUSWBPjp@mnl!O8^S_RY|18zYQ=Jb0b#yrNV>RaA- zD*2b1jO35l!iw%QzwZavYV8%ZCzojO`oMa^^+&=G%F2#qzib$oJmoO5UALtBr%Ae zNu+i{Ng#LQ!HaTsdWUYSa&0{#bb-`jrcH}QdVc_lR6LKvljYw-iGpfW@CDA1-hO8e z+-xuy@faQ0Oza?V4Cq#Cp$^uqkI|4DXujD1nb3rD4LGI3n!NRI?f3C5XrfW(DX^4g zroxko(8;v`6R%BpQfg@^Om16*RT4e?vr6 zs5F7J1fw3@6yQ-MjkIgZN=l|9-(}Qi95Xw9J~SaJC-ZIMe34|``-%5wwAMK+DzYi5 zoTXB=&?Yglr!G0)KKQv0Iv7$W$Masv`$e=D+t_tnYf3q?wno7@J$R-=u;|V+!-@j3 z!X`s8`#R1jJbUIGkN<}Cx}I|8=;>F!MKERfE59)1L( zJ5 zRY+igSNyORWwPkH~*3w`|B(bNFw=T6lj??`29v@1IjU@HMbFCqiIMsz@ z7(*BfO2O73`~aOV^@^siBz(8Vv@9}hh$smUFqFNpPl4X*>6K5aU;g_)6$B2nrq&i1 zv`(mfJ|Tvl{kvz|sthu5z~sYv4(a;4|0B! zu$X5kDguxTummS6qSAKXmMz^YpZSWMSrHSuufg)vU*}$~vCRGB*ZCI8MK-27BApJ! zmYX7$XlXxN6)x7o*rl<@%Eh67LZT5G9`ww=6uZ;eTJ`FM!Q(5spF{Q%pdDcx( zHfe}-z49*okL9utEPP+3|4~}2t~I{b&XRu<#(q`!nYf6AJK=`v5~;H1?gh_#}7RTo|X)%P=k-MjO$XLl-j(D(CTos+uhL5Ga}K~ zw>&oeuJfSki{d+XPPAKd^B$iK)tDvljWxwPq!(2=@OE&6OvS92MfA7VH+`Jt(Z4+n z3sFi^Jb2kq%gY^^`5n{BTE*Io**ts6*XXQsrft1InM@ft1pK&G$tbpKVDOR_U6Tog|`i-=?qD>h6nw zDCh8)qwrYmr*s18c^0)^z~FF?h!`wW#r)+ww)o{dYvnZiQ?+#qNXm|+ld~eIsy^NR zrNQQ$Kd2!?M3i&qNWzL-)#zb`-8avX_h5n;DuUjM4ggWl6Ip}A7w@a_QjwI6pPzuh zbrI^?xy@YJn_82KjxY6#j^NP|-N(zJ-+c+x`0#W$b7YW7(jmM-71s250L~?oAbCVq z3fbY1!lTLjo(NuP^IGDgY>(~`&hIO1R^!luYKbBzb4B!t;~f8vo;YU;kMCg#+XL=Jfw(6XG!+ z|G%F!_G0{ZTymc;Zl*X+th!dwyh>x{70n8BZO5P6-u^m&d0XhT(3F{5Hy{w!mQ>I#3rV{iE1===ZZLVUo~2!tBFh%nUhG`qZP zX6l$x5%2AXUH?v^)HXs&&!?2dhMJ%M*T!U!1z6!0yDw;Wo6da?r80A3X!{jyfIF=$aI~6 zLjAn4_`8P-Kk1o{Ii`of{|(zK?UcLc6O##@S#c;>3#Qt(xlMzqF|k_E<{&GSQ2}5d z1=D`s3HYQ&MqgR3GM%6Tq(@JPrD@Y92s!;sRHdWvb4T<2SOR|<7*o(3&nq~tbAh&% zqE00_mBZ3a-MWv&8&3`Qvva6*1tmY#WLe%PJRDW~ZBmGyUL+|fG&vWdCM)2k5(fw^KoU9J8NZKr zm@CxE_~M2u?6mb)8i4Q}=+JzZCj`~eL&hs)sX}J-02`-eI%3#dkCyf&SjhI8n(?oc zxwgybDMPH!_%F;=FY-(j6UfjS`ZGKQY!YAY1oip}i9C_Ue`Qhysx0EQVD9KWim2JW zKeCZ`X1m+t()PR1eL*Q$nS)LiPx3tbl3+cY!%V3-gH_y|y1^#t?tr^W=tXr-9eF!- zXg@#ed7C1xWXd}_2lXF%rYC)Zjez*PAO=}xu2C5O(48S|DjMq#^}L5h+|TD#Yp3Im z@ZAJXTlzI7bdfcW3%tP-J?`DR$D3=DT@k9hl3@qT-<>dMbtjk-MqM?-1A2BfGfByE zAdJ5_4kf)ms)4OIDmzjEMt89?Cp^$nRTr)DAafusg0EEo8dV(`SdU4`DKAPY)U))>}k$`{?7*5;wrV9K_L8D+l4ZI~(|;wr9hR7MUELP6b?XV)?Ff z%E<-I2m48 zTYxd{n`dkA@cBvWBSLIE!H+(p&#JO7E1#oS4CI`n)8v*u-kUwme7^`*Tto${JW`(F z>{x4$Gr%0%!TKUjOwyx5kPr$8O7D(^@Z<$T(0#_3pxsI$9Zd0g=t&(Hnu_5Q@TQK= zY3naRZgZvJEi;cSdtoulWc#C!@5SCax_zU&^wTC8R^_ZImoBx{X9omq7vr++^nJLx z;Y;sz2Ocde@qEFMT(1zsZm*UrvdJ+@;SN|Ttb8Yb+1;>#alKmF*x}I1cbve6m5o4h zwmK_Rt+)SKzB}L7JJp=%mKN2erTfr!m=#)dWbPJZ9Kwn_l9qXczt$(3C&eW#pzEy3xp_1bzXJ6>;a3<;MTxR?0LeczC5=i^1a zx9uCxXx)_fP0cWCzt@GnDtEtIvF)dSogWj(mi4>8r9L{Stv=|t@!PzshUTvA4*afL zcSBd;l%|x`Dfw1>P{zgk-8&reC7yE&x=f864tcY?^5YYn4qeY}j{hAWQ|I+lOh;zr z1BLyWDUvZS6;d04x;f;0B<;XnEzkNXp~*aqYG=#Tfz z#fz=(Hn;2AaWala%Po6e^PJV9eS__99U8>`#k$VAZTmJ?UAgg(*ntyd8SQ-z`!zJR zG3l^8SistxzPT-Z4ydTl>yJ!cDE|P^i)`XKlhRO#dnbl*nM@l`=Q{8MjGyR+Kb9Z; ze-3@W_k_{B*LP!Pr=FfcKCD0+Tr~9i>W``Xfbs_}=L!S<<l3=Gn~D9S)r9I z@Ba>kHvDg73#G!g-WFZj=xJiwue{&bMxl=XtQ=q2m@s&9_~6t57|^fb2~8{%sy{uh z%k_C@agdx`1s^T(TJ-PK1aZbBc{Jinx9}2{-2(`~8jd6T3lT!lB4l0@^Gc{~RjVwHjbE)VSUA54q4I|{%J z-~&ne+~wNO0cIVsB4Kw*!$pGsdrFloG6~&Sy#2R8uiq{6w4rBB*RQRfHj*uo5SVxxl&=9gGFb??GP2^$m^JVi5oU{G z8cljK{-Ln+v$M(Joz1U^xpQ(&`xl7ja2LI~JmyM{e!QxJNn@O^0Wj{bg^*O{>lB(n zjR&L+&g(gV5kQOm=;2wsK>7OWs0&%X9zTpiU-FA#ls1o2V7}_%1Xti_1{r$=+3eaX z^HpdZ6B>&&`1F}~a7p?Knp>4Q00OPCX)gU*lm921-|x=&BsQWA@D2`o zRj{Y?mR7{~!hQuaL;26rhtzroh5$=jFG>~X3M#=f8ceOE-j7i$k^Db-pTrw6qMSM^ ziA`zVk;+LVcb18J_>Z6G-x+kqa~$#uxK~38t1^dNTj0xVZCXdZ%N9;$+9osO_@tjj zWA;6g9ZTyZ$O1Mvt_uesR-Hj0G+J0+Ook2+9v2`DMvR_S(R%LCPJf=j=v0MwySesc zfQ77Qc*Qi{{mBm)yD~iF+8>Y$byZGPl7$D*xQF2*@$=&;^mr(`6zJ4WC)JE&pQYqM zxK!y3qU;xz%t^$9}Jats`<9!#VhBQPPfNjW?uk6z2!Zk^# zo6YoqPUwxH1VR8qHk-!R1X~&&Ac76ax*crfeZF$VheW9BU_beWlNekTdYeU^b;DJz z7c87xbRZJt-pdk%ljrSg=5FNV3iea=@Bj^ri17L4gnRAJ&|%Mk?S>bpl3X}=PGdxP zgc8jYLPWxS-W?d*`xa6V#3E0pBN+IIYVpw|jag%JC$dYO@6z`Y`9^j&&-;gDL2gT@ zi@M7+T_v0#uNJ^Bd)s)zVK1k;mRo$5mvgW!jGH-gCQo>dru1p=;V!%i48{t99J?jk z+%M8-4pK%}48wS9V|VaJbrOi{IXucJ^f+o;2r5i|Fo0=zv*b}Z0x#BiR%viCyxMAF zMvLOqzcXG(qB~j>j8q+32lj9S5dtbg4`ua($87-4){`%AeZ;+JH|(vUIt>D=fgT}W z!>;Vz$1Hi9!pHIaE2C=(gVqm(youWN3EfPk_Vh2}C@6kdzUv~@`&3brZq`iFwm4fw zc1BcIKX`rf0x8JeE%h~SSM!;u{eDYX+O^}Zr?KjJ`EiqXpzK`j`CzK|hmb$!I8^sa zs(6Q%Z}H4%Ebt@!Jj0q4n^+(v<|sPZlg|(tj1zcil#`Gj{P)v_O$|mfuK=m>pj4jU zTOcGO85m=++{FLB*jwOLC%pUf(U-Mh|J#V zFK2rcBI-hj@gGQcup!>$ck*u^+CLVo{hpLT2}=1+w^6lt`=&C>M3^HH?T`e@NE31^ z(21KjQ|3lq+jO`)AYpi&U5(8IqJv91k&;PJmPgP#I4Z-f!TBPj(yp_`Oyb6)WUNeo zyS}Vn4Qxu^$fFWFqs`uUt}pRPNYfr!r5Cp~AkJ5s`L(KFukG`6z6MPC;fc3c7fEpGxB>?7sQW(#3cd^c&PV;cPC- zgCgS607R0ral(QbJP%v6{kC45qw&_tcaqF4gQcFQ;O-UUjmR!C?b*u^9DM_6Eym$6 z_q8C!(z1lC(lT%sTJ9%8Lu5H|m@iXq{zeyyOuLAU7>dXIareIc`)IK;M6 zXmM@4D#eO_XDn)h>|zHYEsOv;$Ri;CU=k5NdBov<7+sgKyCs3!@&{8ooIMXhv|ZRg zHjwq%!0@tyeKJk05I=2u-qQ+3h2zUQr}G7>#h|fxla$QPVnF?k(6gu+-~_KbzQ+>a zQxV|$o1Z6ZL2}XQ^(Gi9K3bi1|DOfE0%0O6 zIgiq{*{gOoUX^Y+lH~*VStH1*5xZ{j9I$}^4FLKe)UitF>J{N#xBPp-x9Ad`PrsL?j@dA#AjPw`n|;KioBf`*1RRe&fE8f$0%&~ zKH1n$rxH>&&mz62%q7O_WWkrX@1o8;&DmWhw82zDMpC!rkk9HmTPANkoe7A zEyDosk~Dmt2c*YC!2u1aHG8u%Cr|=>d>4=5cq5N3X6-Qz3IoSj>E@GA}$l4bXs8t3_|JgVT~g)@{jfcJwAt-(Nw# zXDth~4P1BSD%`gl{Kwb+FUrfWlDA50@5nn~ws*BnNJbaw@_dL#)I^&>vL%ly0-I=< zKa1ihn#&k~jCotQa;v^^NeYV!v}(F`?w!#Z6P`^uxwqs2d)up{enW8v+o3EoU&bWb z5E>-a+q+j~5oe-G7U7&{E)M5{DTt_%iwO?E#t1juM#uGf#lPl1cG1*x^L%>_a zk@H9a3rC?b5DVu`zF}Epzu#@k;((Te193jmEh9MW_@=qa_#sK8Q~bKm6{|A&NdhaT zXs7zMi54X|scr&!=3L%YV-N^NF_-|aGNS4m68|#cDm^D36Vya{b9zYw?Gy&2F4Rq9>(s9-sE?k*J#OA$jG)y z*NT=&1;pqwg4MfbZUy+G&sK@F-LNdu_7+iasB;`)AR!7#17B^3)1j!$Mcs!KeIdj{RmLt*S&NqGc`dJ>bC~t&ayS~GokR*Lz&~$yq700r5yS`d< zf%AxUfR;`$EuFj*v~+?knOHh0*v7DQ?p4#$!4JHpvk!lUr8B25&VH5S&H&SXpWh>! zWZG!yl<}6%_>4$n85KBiZ91N77@SHj7Ym}N4OHGrGhWXOxbs|4yQtd~2<$7=VqS36 z7L1!b44y7u-X$yQ4;wYb%o$og*qrY2YUKJeYy8|TnH3=;gY1RQt~dHkJAAA#J*Z&X z2882eMTlD9$BF|U8{JItH!PVIGqLVO-aI((U~Fu>_IOFC5mNCy7%lpw!6bGEu#Rdv+n>Ri_ZO8;X8TsM9 zdPWtbw_GdPkH3P%SRPXAiM>EKC$mp>SO}AYG%pM%jmU$aKGf5zrn=v%?8SsGXe)JR zI}hy+L~6B?6PO$vhLuyrov@bVu^XULvI{HT%ZL@N$Ea!tf4LvpIf2N7PK9vZW)8DP zntNs7%{t;7GE@gev$pPQ&O};iKsAYTLg_rqA@xSJRU+q6F2)m;w^}M_FP!;q$S)$Y zQCvlTY z*4E-V$MrIbOulvuM_Om{Ll2Ue22{qo;H?(Ykj`O|QF}l587WNN1w@DHc^Zo~=-q05 zQ(sHVFAWz=9k}rh5z3=z)2WC?-?X2MR?`|KEGkSg6uZ1UX0$5!Mc`Q)ChE!?>zWGZ z`&`5m7xNVhu$QxFraKpNpV~Df5K2mUyMV~Q&+NrWKm}P)@cXRbXXfvmTY*-+SnBWk z7-c}ihmaZ}w|di%_k;H5n}Z4~Am^bd9T4Nvb1%p|6Hhv&MvN*~+{RYFN>aH_>qaKI ze5zvKq3|m3fe(e$L_D={OprXRCh1Wd|E&gp<~$4!vjn^c>5@CR~>v zPQB37n)z1GQ>3D}!c-_%sljuTH%Yta!CB|pAm)r}IBp-}urt=b>)CWF+GI$rSeIGi znH8AcJ8mv9S?Ca)6*rW5!2x$EMdI$}d8kp(UYP5M_*}_W(1UArP_P|X;d$WVdnToy z{Z{XCytCjPQV##des-(q1%%+5Nhe5E)1$W^#Fjuv=?cdr0_AICS~u0{uW<}+5LCnS zppzIANQL^(}7d>*)m*LdHKCuCD(J1vJ-lx0lE;>3K}lSPA89E6-mF91M^Ch_q5|`iD1%XB6~8R)}PWf zkPph)vK&~*+HvPJZY@rsa>fW9owgh<6yIMeR!*{PV*ieYc`8~4OVMd7s^UBci%2nZ zb%qoeKifensfaQw@Cuw6ok#<%f@D>%@N-7R$RZP&hK<9Dhd9U>8rr1baBqaN<(P09O0IJTope=0+g{k|);ZJiPID%i2_h>NI1`7;s3F&j!G*u9bB&7y4$$cb zuiwSlt!PoGlflc&x|v<$`mwguN7oh|CAMJ;hqO8vj`sFT=n@+YaugV=1ST;t|4*;` z#+{l(b5UGy6_Ja$#Ouy(*fH)3Zk&lTncl97NyLz*b zU>?6*Sa1f&Z{#M-VuQ9DO_CVw}%fD5B!sCU1Ikm%6bG(@}Z`h}#D| zT)id*i=}5pIsK=MAa(<&WWEs6@k%=A`qLxoxB7wnQ1}aDUt@K3{n5I}JR)N+TlvD?w zNbW&JN6+}$if)&A)AAr`SF?&Osu?WoElQg$RA0J>!`s^}4%vhE=9@%UUH55aFn|?l z^%$Tp(@8Ri<~)66Z*t{wZ``N7rJyP!HFI%2FcO7TXah59Dr2g1E6g*_zc$b=t{9%t zH5xr^QArw%lQ9YMS8{dH*MnLRw>OoEETXHoKgpGwWFFMQvlek7Qm#S4DizDbYvaX+ z?#-}DgkpEvvK7<3AiWz|;7o3n4Q3-=9!{Y;>=bg0?3E+4@y=PQ-jQv$Pfb+r&+kfq zD(m{jE2$>ZqbF0`#LP9af2w#~;t`4Gr43m4yOVm*?m(jk%qD*5>pX!=|^!^P0lGWdLS0iap!zwjb>M&KZ?l7Gi z7i_Y#^v(H5c0alz#d0~B!!4P!2Itf6m=V%n?fk|wzP+OR0Nqlbz@A!&0sEk_lh#ch z1)_Zn)n5X8ir^sYZ5K>2Z^$zEmtDmvw&-Ri2;GW{Tg!kekc zZ7?U;MKwcj+wxt3SHWqX(H!NYAkz{=^&}+)ayby_(0*#=lj3%IG9~EXIp1)0dNOU> zlh%s()P#<2X6t%NXP~J)@qqRdyw+bj7}~9q7sy&?a;JcTN`ePztdxi-kOg_$4Qu$N z(C`hN_3i03RK}H~8zz6=ydj>4o8JlG)Gos1uj4?M6A9v7GN>ujf%wb@6LA2OezjU>sW&%Y=#lA zW>A||dZolFUfXE&=1P}%H%5kKB}gnluSk_be7s88_PwG_Me(OkdaoB4pMtv?8C9}Z zyIAVoKGCKMSyhHumN+*f*=foX`8xPg5)bfPc~DJ<6IY+zgOEuo+1*TU_b| z>*i34r;M>|XTS`y7U9}sz(X1|wyDKP074fR$f#Z5_X>Br$p}JQr`Eo}S3=J`lQSQ} z*ehcg%39R6PqL2#b&AYNQ?-xxWAutwUpRN{qDS434_x6TYX=@?Ra-3SKA|bibrUL*!SqzGk)d3HZ2+*D;;iGk85ThS_R88>9VBMZN;*76~=z#h_}R$!#Zp* zP~zv{G})Fn4^6GHERbj$(bLqNiJp>q5;JQ!PrGCA?OCNlL07apE~1O^Yw)d7NcrBT zY|o;)M(8iMb8|k`SJ~JMQUOIyeb$e}HtZxzITdf~2WBW5Z&|fD%QsxKHd=3T+!FmY zJVL5#On3uU4``|q0)WA#&pwEz7X}w`0SW7p6T>#~ABF+fAM@rN7sJyx%X4SkcwWxqpP=cm8B__`Cc%;%g(L|i&;*LFOfg@%-^^o^;;r3z5Wlrn8 zihEg|#`jwYq=M6eHcxN)kt0Z36z=#AdBq1iV!e^H6(c4HEsJktY-OHp4%(p&7ICdT z(1#fMcC>L6^)28NmooHt$+T(cb)70deAdQs1YPDP=Vx^!)vh$+bhhfK&3CFQ``H_c z0&SM5ctiBwc#!QnY(m}XSa_+E^NgeQWZQxh#U)1H4TQ+Fqs)7WtG&?S^^LU7Va12~ zZdRV_!#3n76lt44+p21|GWE*)(CeIqM{4Ef^Ty#btZZv+SdEuBoT_isPJI`5X9hbt zT)QQ|o2ewkQ{g6UEXrAaJXyZ`-1M(;FrwDgQl&J&= z-YdTU6g8PghU-S;&@9ai^7zGMQDgslBFo|KiP0o98!e-0a4&I3BI%;Es=}{PeuR|5 z1=Z1#7PeIq97JPNke(;1`3d+$6R7z(c^dZepJ-MuE;Nn4?`n0G)$eh$^%5q?JP6LguV} z#SM{c$PEiMULtQq8gxvwY_$ZR+CTPlfElw+pk($am`vRyp{GZ8X#Iq+pNeHU6rxE0 z0?jO=w4Nx7OL%x?Ztn`3M>-}nN>Bt)H5;Ir%nvB?-mT{G6zB;Z3sSifoz8kpItC(Im~!QShdb)wKSSW zMqEQ?cz^%(xBr^+PYLDM-=_#KD%Tdhc{6U0P0cr@dZ$F2-A=dN`}K|x>pQVU!V_Qr zdSJ!Z!s)-w`1Y5lYKyE(E|dK1{I%gCQZh@eiW!R0ExNNgdDWb1DTc}KX-!63;o30k z__Ld5o@8aI7#iLrQT{md8dX0N5S^}|5#(s3_jV`#orRv=)`t&JA zzje=^Jt9(4dnNVLRzOT=tzEk=H6lbrMO88eyVQ~jzY#cOepE4}Gud%uW6+<1ccfV8 zgVs14F`VWF%&RJ{hYsuI)I0HURev1*ts4Af?`zjgojcCV)?e{lerSPBQ#mY*j7Els zgoXFOww!^9z?2w4mWD}jr`@LZO+hnb)6Y9VqkW6$(h+p4m%~og6J3&xh#iZ+SeM^T2@>s5?DD`_F}D=1M%K57K~Gm_BWq zMw1Lz&LENuA96AKj%sUdJ*lFu?uUf70F}%777q^(zYv#!yQOB9oDt?p50C3*s+L_1 zD&Yol)~`Kr8}#rHNdn<;4NuMN--8>z6<9r0Dr8b#|IpN)ruqF>9sQHb(#{{b#?~+` zKBa4=srvx7oGL0RYt5@SctlRfDDZ3U4`{1Ol5-kC$`WVaX{eoM z9}CpIKI|nFNU2{zcJ|c^{w@8Oo&_YM$($Fg;Mf37R$r`4-%7&EuC;%As3P4lDGiD! z=KenjJPa&!QmnV)4F4wkoAhoIV>_I4>a}T*i77R!=l;-Km<+v{T;Q4_+a5joDcPw% zuC*@H04NG!jO$8UQ(rzlTYyD3nYrfs-6p+2tqWUZ4kaciFuIJQJszBswQ|oCv=nNN z+cae zx{g!wR;!u|*Z24LccGO~04EnCD7WpowRMU|WDcv}FbHdCht;zZueWc1Mmz2fIZW(H zcj(!ZV~v&5)zu}keEGIVj~-3FRL{J#o+V+Bp$i@8C3UVN>D5U87V6CwcsTTTBs0y; zc_(ER#rPrTN9L_E-N&bj=D6O&%0cYXeY{@ah*NCsgIRxF!yD1c$@jy`$?%MNq+6)V zJgqUjpIKY1acE)2fr6vl^tnwhSN06HCDrAlt)XO!L`ad2F)3z3Mv+!?#T|5L@-P_@ z!9dLyB29iA>%W#FXqisE{~+5ib~_xSBc*qfrzj;bIMSr;2`65NELpM%ZhzAL3XX|J z#>U#*bckAbdwMQHzh9|l?J9xO{a5>)vf1rJ6Kuv?TrbYY4WQ>vH^n+{K>w$oewD{l zZ9ccIZnsG>B%-#GnC%`vQ)rx#?p%uRlzpZC`}OsMDIth=sm*ZYw0A+MXRiLL`v`w; z?KTbUHraAAz7lNk0hOl3ab{INwyZrnRmd0V!8Ww(B{59@ zQC`b+YtUS2=K<@TsfZVQq7JU7)wHou;j6#qRsFU}Tk*~`M21&ajT?(Tte^_h(mP&* zNGg4SbfZ;^Gf|i0hiaXV^ zdT2yy+mZ3ICyMl=4kj4Etbw+cid4H8-3LE!#IZM^D33e-e3}2ckz+9v1#tn==Bz_%9|La0SfM>!S(7rTZb*o@k8`i z_U=kAs@CgHTD|;4(L&bO0uPg}_Uj&PO8ztO&fLBqTq8{E8dtpCIlQlc^TvrvzCz)0v1&2(vUv0W$s*SGek){q~MqwM;R`(T8|0piLL$SZ% zhsQ1dQfMw*=qladQmQ?zDzdk7O6hOE{@Q?^J8y3RZY9*BE~6yL!eFkPZ6xa5EwwIO zv!eCq?%!WGSH?2*C1XIYx$r8=+yXQ*!x73 zb#1TI^=!{bvzwYz+hp=OWco}C=r--?n8{9Lu0T{7JLqlWw^y$Jtr1}9x zQh>0(R6=4WWbAIgaqWA1m|8v*u&*W;r=73&H_GS978X8nNFM8BjSaT=7g)R$neJ@N)^PtoxOf;BK=#5XGJ69A<14|Hia=~I* z4A6=%PV!AvQq1iURANS$v9(u=XTDqoXC-sOR)l^Po3?JGhyw76IVn;M2-!NAeoP?E&q17sIaHCb*mU+t~Ul!szI_g^Ig*Q(;d26Vlq#d z|MSY_%a2bj5E#7b&opsruXWvR@^(BeRM{lNsa(u+)^$t~Qq;vPwZ5x-I6tyZZJHIc zEdT(ld;nZmVAJvegD2JIq^n_5M_3X&(XK6 zm0bPh; zcmS4PleYL#c4wx^-_cYrr?b#%@J8{(Nt2BKdiYD6ZQH>X$3Or4vkPr!6;PgCi}}D$sKrrLH1*okKb8EGgM%(fAqP8u{=8_c#mSTC z5oHSoaHz~MA3ssRCEu@@R*{{?=qlQf1RQ<=5azr19DO>tAZ}o zwU@cOXh;-|KSaa~Q-x_W-$y}x8$ zFV>|KPMsPvYvhlaZhen9C4w7*8MADf=DC{;^lb`Kz!Y(D|3y*i};o z z=*2?MnX@xOY9G?#SkL-dS1Mbm`O;vW^lnq*ob>|rgEuJG6Qo~Jt{3w0;*gA)|4q}Y zsR_ElnbMCb|MOdA;ERi|Q>EbW&J*2)ClW^<)15Dfz@Tm|u0@d}=J1yT(o6Mu$wP>dS5O?E-~KEiLtSZT-%p!1O^KQ6u3-}Wvg7x6 zYq~jc@~(9=rV-+A*~X1wbuKb^_l1tFYl<8*hkCF5qCKG0ydx8(bv_t@>pL(_xL;WR z3!=^DlT*L*_xHc)vvyrJwm!A_@sTHUV)Z;0I(*nNn*y+T^s_RD!x$DoCW5uBwAv~N z*|!w)X0h($&K*JqG4W7X*Qd?B$Pox5FEpC=apoXX#6}p&@ZhCy(Ri+6C)I8UuITQO zwQ2xJ-3MS3jUS!@3E~lF{booMlALBWVN?N!n;cv)FU53p3HkuBhFkif#a_Ku za?sDerD7&7oG&h%P8A+>>c3kCPD3ix(|Rm$SCH(RjEV#H_7ALE%Qd=dQJ>YUl@>Ui z+*i_nky*o)Y@S@J9Q5Y0ukSK&9Q>O8G(cKKRcO&~K6tYC+am4g!m?LI;uh60bHpJt$j~LF>zv|W+WaYl5^J!=Q;{M;s!hvnM}@=!@8>5pOa_Xs zuGtiCL0JZ{f#Tive=TcX?S0> zC^is9zyeYVC}5#Wk=~0@X`3chI))}7pj7E+UK>rG=Pl!WV|>5P zIWNW-3A^3*T34CZoO1;P2d`S$3=XZ2rKaZNlWhh&ivUzDaCv&^Tx29ubu3Fq*)TE{ zZNGT;#MtnvNtZjBqW|3DoG;-QGc*lN)4tr84v8@55{6*q?uY2x4o_6BHG8LGCB{no zq0cUU(KS2=Kg`=v9_og}HI@4tP+|EB_4Z8pS95Xy7LokgB(XlI+KF}|${p5z9~S+4 zzpvvi%PELQ<5?@^qG7%!M|{FI86cJ2e7cb{KDILlk1fG{UHq^2^*HXU6YIWubyW-O z_^IHP7bAIn!#Lg7O6x<4ie+_T6X5GSK=w?Q5*mM`-c^0{x8Hs%O}DvJXrrn4#Q`lH zlrXP)(g8AZu6&N{?vwD&WEKkrEB;!SKJ}!iX<@l~$DzmD*FyI11UAv1quLPC>q8W~ z$ic}G1+6%qu@9hv(a;ABGW5}JozrrOdR`>I;n3qV=|*SKR(D5Ey|O6xM!oQ~^oZde z%|c@n^BUx#{l|`Z_DrIaFWbk@3+$jgO#SS2kF&n}?s&#ii~I&WmZ)XF&E zwn3;^@y(>X<-o@yuXW!KUDzg3YvHX=H9~;}cOZ#x=d24Q-@$0wStvKV_MrVQ5|FFg4*bgwNwE#Qah~5qQtuy$S~p z-fcsZ!`8I(@1DF1J^9vudh(nLO0qf;=*b&*MD!?D`PV_~ZFH;I<0yhZj>-A6Cc}C#Y_*K-iAz)PZ&frf2#R0B z$!Rpq9nC06Sv-DAeK=a8t6&*A15QUqpfjIl73*@R-e~rg$v9?+b>r?Z z+3)W&Oo87D#ZSIeP*nFUofgK}`(G^DeyVNe%n`}4jVKy0#`Oxa140?h9PN_XvdW>p zc&~Bc(bZLGO}z&-_ZRj07o<#!= zOm>!DrS&3|h1hLI4a~f}tzvsEEfa#Y5O=-W=df+r80$y7&S7hp43dw-)1uX>m%ltC z@YbjMjcSawX|0J^0fi|V#G9=-U4 zkl0d42tlr^Pfb+A$rT56KS-W?wn)_ouD?MhEk;KHWtU4CtrZh1@0pEZ<()m~f{(T( z6Zlegt|H9i^+vv1NePQrM=&j=5Gi98$_Xr=zfo^qfd&h4?FZ~~UaBm|tMe=v`-H{Z zQpq1}Am9kyj77)ekNGmW#l>Dg*gUs$g@_)n<|;<+*B0vEvQsBa^!WClCZeM2=Mk_s zK5kpD+dn00hppDq-mXYM0BftAEU+dZ|8V&u7W~Cv!C$>krr|H0FxqN+Ba&}0*s{N> zq$gix%xFdYTnMz`$vAFl+ve&`1#iO#7r0*+u88HN+$UTSvsr4;Mznxexs&tCE1fQH z2LF}3fgOKke?u$dRZW!ys2ibx8d7yxWA#~ z>F$Ltvup^oEPZiYSjh%q)n{?J2Pq+Zh9VLhHsMA3-q&Gj6_y&+9g}V|Y?*QvjS#Ph zNNLTX0|(ws4BW=iyLponV-ejUA$%PT>8U+dbALiu#z?I)4gl3!d~vCJvBKee1iR*v^7HY%%&3YrJnSvb@SHs-d5CcsX<57*0W}|=*kVvC*5F=S-tTaM|0^$R z5f(zUwBMKV<2km0MR#K+rYaUn)HPkK#*F`r-x523PxZew;&J-e<&_CI?UAXgGxqF<--QgpQ$PX(Ww`QoA z?gc7t`>*xxKiFo5O0hv9VVf$D{@QR2Os#dj8q;gPIEMw@B*U(L-lB+_3wf7{YiJagvE*|crjv0FC><531`*#Wx> z>llX2B+23&rWPf4bwX8mEnp0-GNuK{5xL2DvTn;Ne1kwVFV?Mw`&Q!`mh#=zfmNh_OZD)?ov&bi#kbYw$aZu1v zaIce1)_^|x?(P&6M_ru~z|E90F#}67IW&1qa&u~0np6dj3M)+0#>B(S*G+w7m#@cL z=qSX@Q>P}!6g4M-2fP!I#Xv!;YXoFiu?gUuP0k%-yr^5TfGfg->EH4V3#1BGiO3Z0U*>x@0f8< z(!3CN;i$6meFH5BN(XXNY&k8CZF{`$-cxKqR?J=&J&}F^7*9%jb(D^*QX?c3AA45u zg~UFn=88g&I2v@}lna`bi-g^dTTfeD|feBQqJ zmmGB29p+_95*{^%;OYCwRD>N>1^9&Vm)&R_Mw324jl!AZ&unV(zK?CgFeR{FI0E8g zbQ$AUyt~5J_lzsAX2pnnWY?_*k+DjTwuK_Dr1n4Y)6uwhd1_) zjv~`D{GUAeb#hD8>~i($G`Q+m=8%@iYdG#86KiaFLJlYU5mYdI(Z%R{o(e3#V=?m6 zvP~h>D88pj3NJ6OQDr~oC#mMDzySrwvUH1D9nrvRxx3w98 zf_H8Iuynab^?A+Bt3<14O+2m4M-tFSRkV3N3S|yF4*_)jXsHVduqF`UH6Qd9mDBLm zP_&(I3%6*Mnfhgl7H*m4qEo|;DwC?k3d`z4QUDG4B*&xQNeiAMEMhK=34XKB;77Zr zZe%vSL?x712H#68^^V&v8~-fSVP2KooN}uLCx?qoT)ld=9IZ>UbD<+@CLFW-YhlWg zUw)BJ5YpM_%b8Zw0zoQaWS2)e<-(PQB! zo*I_TT!dLI+9ENB;;}{2uZ#>L?1R-y;g|q^{1=fqv=pbIYwXIE3 za~md7kX0(i+F}}Q5wTHujOO-gOoaw&X|#Lo=etU)J>ed_^xex93#Nt5<#g@Y=Vs5CG#*oG~QM zT)cXFNq2wW#w*%}1m2F~tcZn!givdpBH${qh}rlsx(Hxf03d>=&Ro*QXHMdVEItZq zN1=6RWyA{+QBlS2F<=yoQHzfyziFjie)k~;EvqrKC_b$K6?zAz%=wmljEBN-$si)w z3&gAXt`WS_QIzxs(&EPD$|y_3#dO#lcRc)}d3hng;2#I90S0r4R%sl_9k)CDq-tf3 zc&`Y9nr(l%C~=H8yg0k=F=kBimoV@LThmyijU?FWtNP zmF&Z4Pv*MkR~)wJN_)+rrPdbF?l94%(}~vfMnW^xZm}ne3DdV(@Kq+ZGFC-ljkLAe zbtfmrC=MSp_?7| z7DI?cVPFzuml(G6_w|)yyznWZ1Nd%<^xh{w4oGgEU-Ud1Hy8P-f&HU}Wg%vOp&ef6 z21G#*;x<`R>%R-Lr6vWaODBtgXK22G$qoqei3nEHCf@kM}!CVSIH2r;9q*&nco2+_217Oh715 z205UzsQ~;h#x!Y2QwQ*)6svju{7Wn}-yVxOVRDhN3~p=O)u*Gy4wKJmEE}nr{ClD% z(lafaqX^ZX5_qMg9!r+J3b*j|@CZcV)cIL|Y#=MOCxzJ&gVu)6LTh!gl<@toyUpIi z4~PrBAf&|{!f7sCuo^G{V#{t76v1crgDBYbKYdA?8!eIR-H z^OMNI*)e!OaNOr}efF{&Y6VpcI*wFVS6ebMYB?&1c?2=|ENf|&=;7m|(JAx8MS~Zy zh_;*YhzP7V*Zu|9o>O28ru8YTPW@D2nC}ixizamuoXKxhRsAKtYKLa=;v+0;E?p zWPO|rT&FpR)%)Vj8~Ic#nidR?&SbX;0=pyx!?h=zUC=+7dS_qNr4nCFS^3m(%Rbj< zpJUK>hat6YLQmh*1-vk2R&Wg0g2TR&`j@VzCiDPKdx2p?^_tmxHGKS45eXx|R!j9>tOVW{-Nuqt@%rv#VZ@xkQv>BMJj+*-TbWAbXqI*94r zYWQ?(>?a@+41UVmWbq7%L(z7;>(_@MG=xFMdjOQ~f}UH?-d6Pkcj~Ce37l>s|G(C? z0oL^?tZPfG>r+_QQ3Z=Kim|S>v9AC4mvy~UvJF+Ey7;-e6BfWd!}K=wr6C(z#+$zvb&mfB zk*HqNzJ~i(8G{gU`_3J64IK>Zj&X8w zrX*~K}igNZ7vpM0$VGPJLR6<&x$Diq{&BV@z0%b!=feuRz_f^~)7o z5FVC=SY&@GNG(t^Y#p4(NeP4c0%_v$RG?ydZeu(9oNuz(mkD@as)6>FGCZT;sA170 z-@e|>AaQ^nlXh_IG-5L~C&MXJ7bDOx<09pjg~Ydg%I3G$rd$s3=(}{7>9iRHnLibz zXct1!&dwm5qTSLEM7s#vhoJy7Toj6JEfnCMe06uQ zK&kx5F@rEeAScHkhI%qZDUqupkxLxecfsIAdkohjjPc81#B&tj3uEVOnZmP!nwmg% zJ}|E1M8OYpJK27A{6?KxSJH)Pr_H`6m7Eesh9l)7ME6M?sT#vfQH5ACBX(tuauKmH zFSWSwZDHlEyP2!{=%-F%o^kTl;+4^MN3j4WQ0(Yy9RKxMQ6!F0$PsInhK-Xx*L>$M zdFviR`qHn4d*sV?hG*Tn3eanx9`ctlIiq=;(lKVk&l_y}3zzt7Ys2SI+^BRtzNLHS z>`BmwRZ*0y0nc4j^~=(w0f3DIL6GSwwVtW|q@o%qQUz#uQIZy&0bPrEckkMzhAOWD zly#Fi&fq48+dh0+ls~pu1ZU7hVno<^F^!7;8MEew0Eb-3wQA2y5UZke zqM*|X1%UqoMX=(U%~DdDgCF1Rrq5TG`{x{kkXSIu29fPTF%>*Ud+un6^3(EO2eeu# z!0juL2Gnq+6~{2|oz)5D1AlZ6@j~OxS{1x1fB85wq7t-#aEMiTD*$^#g>|&k8U?x< zK5az~w?lZOcDnHyZX()o7F~C~-#->C2ToOVpKg%Gy8dC)E5{hCtMa`#0Y=DfnXM6V zVc`>7Z<@XRyaG?11pIF@XFMu9n zt8<^ncB+QOMiIWEVOS?$pNgvoRrJUYagsYh1ENsciIX^Q>2f$wpoD@t6j;&9nz@>| z8|cBhS~qlN&(WY_uXSBXXMyY8y9>mv2hJ>8v!?r^7J|T0WN0g^c0E*Bn)oK1Wuu!8b1pKpQ3-vckNjrajiyoM1N zUBu%>JEq*AI=IRpjEFAid0bYbs5s;&#g75ba&}h$9X-Rc!@r#_YqNbMlIcJaH+IMO~3G z74ZRfn9h`Q+)_4OrY*gRz;#TfK!ZgS*f}%0a(kqQzg)P)$(E@G_M7q9;*MCURH-@4 ze=}XFafni-Mi~v2+T8rB);BV@s0w-=-#csSbddF2P5mJ2qTJm3SRVN_j)y{m_+rFX zz{kIJ@16$)xBG$BrAu!7x?Hp(HUhjRCPa`e=-VFn`U-;LKm^AQoXH8|wMSPS;_UC0 z(~A>SsVe|LXB+#jk+s(x>aN{rWMpJEIc}AgpZ@?KGgLD<2H?SNA~}*SD9W@1s3 z)|C^il!ZH4ozG;l7@3GRtq3hkR<7IxEYS<#;(od~mMJqV<{tj*-_Bh7H?ydJ{&Mj8 z$@*!Z)qlSCpZ{yiGQ{=&{O$knoh&s}w*TRK|HWTbKm31tV~=o<#Q*N=f8N+XSLopN z|G8q{ug3o!Yi#yQR2R42{?2~qEf1pR=H8CWZ-Goj)Sit#q4(FDTdmT(uZh)DeFJ7z zUc>G)^TCr_FCv5m1^vCnhy zC6U)#ZwVl~<14E!w=b{n^lEjPy1WHB>tm)(o7v3QF*h=uF6fP%-zd7;Ew^t*RwCF# z^nfj=z(O=@E8t{j@2H7?F@R|F*l>1I8*mj*q_#UZb16lE6Ux{4G$;C*d}kFpu}%B+f@=r=CNh{aa;CE2?$2`+)NtGzk?732=NNv`6oZ`|d8_R&mcu*S202)7>+kkl?67CrgWGZBe(Nl` z>iey)bzG<2=ugS$jj$R#kk7F|6Ju+_{^gC%%8Q^g0GFAap$(Ty5s`n1C~Ns z7X^13m-AI8n+L~BW2R1npo@d|TwFNVU4g3R(+IBkmq?;J^99Pw2LR;(gRj@jkG)A7 z$CAE;k(nqfgCQ>wfYpF$Y@Z+MK77e#tz%wV95bGUE8u|T#`@-|LwX}Uzj<=6z?Sv6>Zos3dsR{I(Ci@Q`K8T#kVzUAgzem0WBcA)jqC%yUZ)L!GeKyCg+`b8Jb(xtn z@ItJwaZdG&sddbKzyA8|5<22y4m;?L{4kqt$D^zHHZ1edQ-JSy z<}j{3J28yfriC;$In~oo$mVE|2QS5gr@w5M;pOIhe>patJ)SOdzu|ghI8DcGIewrS z`V*J#EEhyU7P{JS*5yN}^8#_PD=oj&eD{46#e016{XMcZy~yoj=lgyI*}iUyv7P-B z-Dc_O+x$cas0*tXH+QMU8Z}edSF#VumY+~d_(D)Hrc!^|=6Z&X7panmQQC%zg z(~D!4w~`EN7p>m@E9N`yz@Ks{L}6?2t|SNK{#W+74FC(&ft) z5Tv1_$ct#djZ2YB^&niH%!WRx^9u;LfBblcig~D66g4$9y|W7o3fxh(G2cDkM+N2* zE-s-97cP|I_~QeI4p}nm?hvmbTq9lql`tg-$ zo<1KB4@Qd~Aefk^u>w{WTa2w>%=iP>*4@IaKWf)8g=tgL*Ofa0I(Gyz_4GzcrvDhq z?Nj`=wdI<4m{mMXVNP|K0oETG(lO(u6ynnMBJ8+?0>W9s-o*f#k>Uk^;K;!VfL)n&0hgpx61Hg-02E%7>)v+tV$>x8rz|l z`*T+nad|6nT8NW*eS*m&1FjV%W+lp1(CJ;~5a?-I=>IhU!+W-Nyw^(h#{oxXe*=RN*e zvf`~*H^!BN6C{_g=qP>LQTz#kGnmFV656LvKOKdB@00U|D*X3pV<<{H!85#(s1*8a z)E!N|bAz$l&d$dmr*W6W0QAGi=0KnuSA3vPf^F_2we#oC->NME z??YTaf2l=J-3~2X2r#tju}d~dN!4{E0afn=zOIIzuTt8u)<2O!;{ph;kI>-Lsq6OL zyJ~QRxS9e+$A<}bZ|_jZH+KXNu3El4XcBzo7a%9`1?F2Gl(gy#um(RK^QMEL?IEBj z4njQ~N^-X0Z3VtNkBesG9iO@^%UXm@id3pd$D8Pl=#A{n=F1RJ&p?H;C%12K?Q{gs z8ouA^)@oy1OFww;?h#w{7~AIDzB~6@yO4KITy#jYe%rnorhsB-LZSFrSAAv!0Dq%wn4YG6 zu#?oyUQoCKrwv_u4Wc~+Q$~#vAS`u5A6TXN!{t=|KzD0oe2eZ{Y~c_ref6CDf&w+b zQRNe&r^SA?fvLjd8|R$0mu*s0Mv@XunZJ;jmKJsxY~B#!Lxc*0pgioLg){(gm7HQG zQF0A!Ydz>AQf8J7=L5j_?OI-(n~4ur9i6~u3h zb6UXp*ZCd$#$+~a%GWpy#gGR8ky^o6unegDCMvJb@h3h~RxxRqc2`;geW83~W)>q4 z`im!2qycq!LBet{`z-~dVg#Im2%V@S&u_#rstzHqog0n?yVBuA`Fo= zN<&pQl8En9#xdmZuHhhr+$Bo`rj8#BbcY{~8bA;|i75mRquPWBF_W6|p0Ag{W~7pq zbf+m*7KIp*w2;CAqnbnV@&Gym!1pOd=s%#Ku;k9$XX^I2aUvH#Dr@QbZ@;kXTx$2W z>P&~swuz>uQ%0DqLMWxp8i!_pjbt`DU{v4V2np~2=mC(Thgiu92ozX9#(3XX3?>o) zZ7p0Fn}vi9nTJnc4#t$4E{v}@e$S$MBi631UHe!Y;qY{1k~7Nfi$_-7i*H78sAr&* zoc_g%eeO5@yN3oHw;wh|+ccHj=U#w{W7#jiJeQY8rcks)=;#Di=+mmIqVmb_7ze;> zELwdDOAw4TQZ9zl@0FqkcY}&C1rOITA21teDa6pGS~@)z#ep%%GL}poFsbYS&vxEe zb*@A}7LK_~hMl_I)<+F4UbN^KO~h|a zPt_Y~A=PIyoLBF7W;IjWZG+Ndp`DOtKp_(ZE0DF~=4}TpFd}tg`bygzg9pYw%frjM z%YqMw_Ww8#thWK!Z%K)A*Dq`i6NAS+{QXU}wgCsV`})BN%%Ov2y|_omqcSH`;wPYM zsOUN2=IZ*CXcZlX;0{2!txz8X))Hg%`i&c5G&=2WNUuFtjpKX7&%WP+5mY$S>Dsjr zKzAQ&ugoPtSTqWd{>L7hPw%&OWn>Jkgkrk}VCn1DV&9#?%LO$8h?x+k7OK*zDl%^| zS2$;?N}vKOr{ZsMCoNIdhT+gOpVsHko+;R+oJCDEsCluQJ}($ht)FeoNBZ>RiMIid z;oYcFLf`mD4L8^n_Z)DhsT(Y9j$pg{Y=tL*@vt5f*Ds8=<#q>TYfRlvj-iFB?L_bH z)9S)48wpoCV~HqKC82Ky`Scwgha(pHY`^fVzRc}XcTJ$xvkTBkUgUOzD1xgnh6Zft zSyWI}TQb|)+QRUSuqtgEC9s1^OH@VHL@4&U-DGw9pZBm5bbTf(X@6hR-(&qeb$j+0 z+hNmHtawi`pY@Bi;?XSXToY)VsHF|@&#%R7$_o6{B|6SaUh-MnCf=mBQpkO?kgRND zj~5}_o=IHKKirQX?i~>eR|uyMJZR%E+lSj2glb=G0*>dM7p-c49?-Z+vb){=;>HNX zoqxT(lj0?dh6hvexT39?r0MJFxxde;Fo|7wIVzpY_VaJ4rxMp&&uYL*(TMfKaW7>#DIh(;`8+zOQkNPVH$XGYnVmP#(;_&B3Cz6uL03QdmhkoQ*I8wMjN3^jy1 z(=Z$4jWmVdu@A|3F5Y5BfnjXU{wta#&xYT4G5|O^ttSIz*fFpKE{qPs&f!cp(ccx6lmsp6sL~kquslK{N}pD&iBK3=g)&xw z$x@D?>uURM!0nbCKO)Dd{AINs0#{5)&Ww`NC2!HIq4sW#CmJ;wqbN)wJIWGG8<%)Z z!3jZ(>2&>isE)%#{2Ljn@?QHZ8Ri6_{ULf*ml;y$5e$8KBcCTB@poz;;}4vi=yhPU zi9M@3X3ceV)B5 z)H48~pa9Hc{hGn1Fb9VnZ;2r22uW`vnypJI2f>x0OYJj^gZ~U&iWry#QuE2M0TD{X ze*7}@1iV9((P2=m2p#OyRbtqcq#)D_qSh1LcKQ$GTpEQ`rrUzOs}I6fCB;zma56nqtdGu2PbI*QlR#}5PF@4V7bv6$T^}L{`vXX3W3OvfzBlSCR2=vZg9|Eo%5^00g3V`Z46DY1oH6G zJP;U_$1lXB%md%vIbK7ZC3^KrOQe0AnHlk>e?KX#p#=2NUoND{H#Xc`CC=^o%)AZh zU9tW+kvA|CI$4Y1baZ9Nx`g%*6iASO6P8qcfqid2tnWf~EBAn(u^xN8h6{cX^78T@ zdl~}{{8G=neX~4VUA0UG`}XN1D{jnQ5@TzlH&VnZ>e0YXCsH205&ScqN)c7K)HBRk z3kOYX6hI;Mj34Nq>1LOAEi(~|Yc(xPYGhe?8fy^$EM_yv#+j&WTcR-}Qp8$N)QZF| zD-AoknPCH35{%`SxfFK65iWi*6%?gD+}y67TGT$th|Uq+WQah7nhY3bxDzRDI3=xP zsMgdA)#-^^m0|7%xza=@?eY zhpW3g)wOX6_j<Ml|uhK0ciT593holj})5 zjgy(=A=xPzX@x#foI|{=*gq;YiVG{hBJR%AW3Qmgj{H2lb36CKOYOwuc>iI>7*4FH zT-Kv43P+W$Ob?YlgtncUkzb>=<@Hchr@XEQU`Rq>2sq(8(2=se0_B?(QmwcyYF(w_ z6!WyuQ}GC*@H;~(E_@66+@)kvIB zXWc%wDp%V6UUvmDe^9sYK!8}s%ybs;|AuF8M?l%*sUs3Ij{6{bqXX}#Q)$~-n z@Oty+&5D)JzX}(U-2}?^>YdkburYeJbM8qsR*uwJ7ApXM}HrpPGD`A5Pu~ zwYx35w(Q-C9!zY{OeDUu<30n*FOKq5$BzX7=cJ4-Up#X@UK$0{BClAx zt8Gty`{Uw02PKHw#ZLG2W=@?8aWJBq)984Z+JQ?twZRLDAa52lM~UVyX7YS29Gh_o z&=8ZIHHLtUaMpnWW9RzyZ%R~U6#KB`6aWv#YXu)#lZ*<$V%UL*65V)AN6*R1{;-?< zjX>y2DN0q`I;UEZNO?x|C7R>n0JklLD-JL6|CPR$m$xRNWD1 zT*12PfA@0RvANKgVYZ;2p^ZNKKK0of@r#9Q(mX;i#HPnYgEa{V zKq1Y$V~6^1A7C<{gkXF2ljx@S0>e$D53h&#m#91c2LjOcx?xIz@j57d>&sfsYr$HIV9=@< zl-^}=00dq@kZC8ZmC|y4NA3A{6}sdU61rn=@+SCKm2K-=UxNlpwTug@R#+EDzVTlr zeY-c7=Kp|gK5R9viZq!>e6IT$fRwoGK9U(+`!f<5>yIEioVI%V?eFjEr#w%ddna|o z94`hwz>61AuA`+k+Jg4)ROJKU0+Uj8T8~N%W9L_Jw7t(IZGHJuUKS!xc^o>hPS-)! zJ#4}~>_#G{W`z2Vt$!y}R;oUK_j|Nt^(?D)Q&r~w_jW^q{%;%(hM z|ED`Jb2@d#O1ic!Z_;`ppf89j_OO#m=RT}?6{sveU%ltfa-)nyS^-+8nsUKUpcfHA zcYq&*!55B{J;ir({(95}Rq5f~!%UBiBIg4MIZJDm3q!76otX?0B#QF4ffpWcHIqt$ zPfsfelY#T!zDYQD+E5LHLOMlF0qls>i`_LxfftKH=>A3g~!A z9CJz9PIG(;rBrON8>2{?NES+DM$f|oeH0nrR9>qrS`jLo-d+Kx52f0nj+j9HN%`v1 zgO9f#nY&=cDHCD1AAv0-1fp7(3aG3SzK@uygP&fI`P5TaSsC0g%Q399^OiT_neGCD#Do#vzZ_ZhM zd{lDh2CL48e?>Z0Z3`(XD%w`=0Gb~>04R^P~ooZI`hTKn3ad-uepZkYJfTKnm;^pV#I90OKiGKS#N(|9;*7HO$fDon2c2v=3`Kr04e8asS))%SvMn1h>5TSjIibCm>KBuL*uv zZI#JpI7(3`mYVe(>xw1-gi%l-#KXe_JW8`)u^(h7Jp?y>ji?v3ZTIZfsVrSdLMMnP z*7ESsWCrSNMTA9hR?AG4F=Hpj95mAW>pdVIGZ2H{Ur(h81DW>KB}JpU-LIz?fY^6t zXaZ9XeW>w-&aKJcZN;TJaMRhjG7d}iY={Fmu{6UVCkQe!3kV9T5p?tS7lz`%i!3sb zR9nGFa|Q6Gh{KBJrs_*aXRe;(l~Ah=`Bq{?FRFJROZBU}Yi0Ik3is%xN?ZHQ)qMP} zn_J@C<2oo=`*4s66E*4@86_MJ@|vuRyd^Ed;?BI=Mb1FUsPz{{^Yntx9~DX#Dd75l|wL=Q?yg41sX5F&(B{u1eM^gKq|Ztp4}yb z;kAFT{nP1fR3BnYt6od~CD*Pc4KWlT4}4J;6Jyu+7p%t?y0C)+1oY)TC2q8<`5*%-)o%M_e%<7R zEg}~Q$D@6tB&z4ep?$m2rs~<93I2C0-l#hYv3wj-tq1*EjB&-L9bo#`$Iz^a0uqi= z;34^cj|Ft_yUHR}gPlKEu?DA(eY+|9Pa+IqY(z=pdzF@im5u@&5eCC1R^#_C_XMT8 z4rWW(5Z~LnNEU%8>pn0(rBpiqK3h4V--Y#{+~WC=GQk&MU#1H#e%! z`0L6%uKPG4t#qYAfuHnA)CB=Qw&EBaqlrY!z|dHEN%r7qZ?SGvRN6${WE{E@(>rkl zi+4i{hK>bqx{swYhOpdSD(u$vA;;!#eeyqMDY`S^sYp%x|oNd_r}$Ll)%BbF&c*Xu$UX_cl z$Rmr@cx(OKz6g=YX9x?wew*c&kvlgxysun`VPU`B5A)}{k$Z!n`R4hx&9ykv4+4A8 z9BSKCK=i7MC!UgBXZ*e_#i?CvO%og;F2@CIOPUbu${pNSaTx zbkMGgkJTKJ-a%7?9grf%113D4SiF#R`N#x)Yo8wLBCu+;>5g@pn$@@$m(MDOb@^Dc z`DrX-dX8y|67H^)I$ikrte$qd6Q1Xol||Ls4|1psD&opYOBW2{gtfat(q=9DlR+9## zDCi!1HnK`f*)iaVtRyaT}F6~8_>f1%O#v%yVI8tu)w|VljOu{Y!cW|-SzLI7c{W5)f1hZW$q*{yNFL6jP`!PFPxZiIIw3|)Ivs{gmDpYn9|mK0L1z7I2{}@q|((wvqbbzo!U7V zIJ839Xhy*BtHU^CsKxV;xV|&c1A!a{GO4<&#O+vyjG5}KcgQ>C1 zDL3?uc_bf?f|Hlh6s`^qLCK&_aI^vaRsjCD$mu_4BjsF3YCw=?;&d`S^y1{=3c`{H zuq$hWbbL3sjy1ofXUv@1cP5f`NolHVo<~ z*HQy4pb?k*!9Eu(Kni+zwaw_=Jy33uKb&IPeu}|`ZgklDTKoVRgy#YVF-jaqX41#m z@O?DDf?B!jq7IELsZO&pMHyg*^F8uVO?79F!xtd{Pih8pqpmSTH4n7+Ssiuhq0o!? zG}p&(87DI*P)g2t)?>y-HkaCVgGbrf>)V}CYK=e?yN)a&u|)GwBn!eq%QERKOva@K z%T=k4Dx^!%sHo+lq5z2cj0uLv2XM_S3a1ZDPeQ|t#6RlV`dW+XCX5>tzl9LiVq>9* zwAffEvc1taYTgf0ED##t@#0H_dI9zY}8XNBjSA#943Y_oe)1P zBTxu;@=e6B9SpTrpyLW})BNNCfM%4m4_sZ>qUuA4KGi);D;42F9~E4r>CgFvup!)t z;cXvKlM{`Az3+@et?Gel7^5p^cgc6M^q4hRU?kBf3~ zP}=tEOUz*QP-_9|E2`)Kq!{EbD6IkgFtRm#M&i{jgJtQ&UMZ*c^4p75MI&oyWj-^Uk4jCG;_pDt)?pUTPE# z+cD_mC3F!XP;y+q4DMGDsxs5;4~L-DB>P!v@e9Y$1jX14pCgw4B2nAVcDs~Jg2hFnj#dfi~Qr`fUC=J!YiWL8V+IGUU%dM?xjCy?< zr^@f!$7m1+-4c;lU08n2_VZb%lU6NTre9U1#pN)50oJkc33r?x?8RIHuw#sF&LRSj zkcx7I>h3{dL}Nd6^opxDgC>+Q(zc4ycK6jLm@n-h3TjQ<0ZdCbV9_snUH|_5`z5>e z*j>?z!Fyj%qm+<7qGfN7M`BL{;g^GD)Dzh-N_n`ZDdMZ{Wdk3 zqEeFuJ^>7&$q5K{lr*O+@lVKAgcng#LLL`_XK%Y$xOos#eGKXtoL=>mC{8L*cU@oN z1y+bB7`k~#AH=>TUVc){k6iD9lvcsU>CQKn(;6YzGO%192@(=5mTGS0^w|gU5)Gd% zdogToIK4v62Za;sqJ{?hZHiuQn$od>CRc-@e|+Xf6!63yCj;eYUHtB3F$;1N&a4f3 zj7A*%%x+h!3W%zRAHo;|BfLcO?D-bVC+{P=>JXuC^W{kO*dVHz;A^vbQger><);(a z#crrΝKrB_kuF4W=a@N?>0CH;!k$4_6l@E)vV?7B|WLt2qZ;tO$_BvGDVdPC}OC zK)R1pS=U#mSiC?xOiuQXg-)XRwEFUvgPZ^{ZtzlR*W@CjX@cTN#EEg(YN212FW*49 zNI%DU4G9YBbr-`af4RVkU(whlfm-yky!G{;f6{5uZbZ$KnHW$-q~ni249wIPR>YM< z6uN0rExnIbMG^FcsKClXG^M$8<2wf9D2LI^x7rvhBgWQ{k{hf0{Xy-&d{EQFJz6$z zegR;x3M?)f)=@~Jd_e4MV0R>wL;9kbHJrHM6B7z2j&?o7U^9T=B>w{^jn3exBc&|| z$MJ@;eg@6a=(CXe51<|ec?zgRS3iDp&eZwwuNMe70MU)y&t}h=L(K>;Izp&BH-Xjs z(&o=+!Jq1qRmph8PN9e9{$%$QnW4asg3T52k_ptJ1s5GTi)HN1%`HrfHULS<`X}+a z-&y38Y{`fV!b&2Ulb>HGjANKx+d-2JKbd$4f?+`@%L&!NdZUDhohZ~_rhplyJ^*^+ z)$c$LN!)8VYl@jR)aiam52l=qo@xWQJ?Mjmou%pa#LGh>z zdiqA-Z)7@3>PHkr`!P!Y(BBQvJmT>7MexA}#!UyiFwLihCT-IZ-wD%D_4}=lprv5l z`7O|XP*EbE+sC{r(75_rym$1Tu4C5j^3*}bXd{)%=jGqTev%Z4$@!k0JsEw9J{qo8 zsdZU)NWQ$#7dhR$Io`}Yb^ox;NB}fHte0V91AX?j|2IVo8{6l3kh2(m3ps3T<4{{* zGVzbJgH5u#KJ_>~Aqy8HI1xNX#*X5j&evm+^MNg}gq;pupnHe^;ng4h?RB!YL(sDM z8%_v;F=E%`?Qijm%`p>do=j5nG%izXz#7Gu&O6*d&aJ;+i=V(buI?lhQdY=ABsgQ@ z!X@o<{%;u#&p$#SHnyh2=qg@PrgIF5gd)orecx zs0VOz^z09JjCmw#=uB15GIm#d$~>g%01yFMrQq1X`r_M(C;TtePptpxn2B}=zwt3x zJcDnK`EUPtsGw|h_`m(X|NM{JFsAwEZ~xtY{lB3n+_|LXIUC!oHwWZ@Q~MwN>fdYO qe}uJvUkbMWH#Wlm>5G44iY;HW!Te{>`akH_9oTb7K62NotN#mv77tMX literal 0 HcmV?d00001 diff --git a/docs/contributing/benchmarks.md b/docs/contributing/benchmarks.md index 89524ed3bc632..e8b58dbbc93e6 100644 --- a/docs/contributing/benchmarks.md +++ b/docs/contributing/benchmarks.md @@ -321,6 +321,73 @@ The following arguments can be used to control the ramp-up: - `--ramp-up-start-rps`: The request rate at the beginning of the benchmark. - `--ramp-up-end-rps`: The request rate at the end of the benchmark. +##### Load Pattern Configuration + +vLLM's benchmark serving script provides sophisticated load pattern simulation capabilities through three key parameters that control request generation and concurrency behavior: + +###### Load Pattern Control Parameters + +- `--request-rate`: Controls the target request generation rate (requests per second). Set to `inf` for maximum throughput testing or finite values for controlled load simulation. +- `--burstiness`: Controls traffic variability using a Gamma distribution (range: > 0). Lower values create bursty traffic, higher values create uniform traffic. +- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure. + +These parameters work together to create realistic load patterns with carefully chosen defaults. The `--request-rate` parameter defaults to `inf` (infinite), which sends all requests immediately for maximum throughput testing. When set to finite values, it uses either a Poisson process (default `--burstiness=1.0`) or Gamma distribution for realistic request timing. The `--burstiness` parameter only takes effect when `--request-rate` is not infinite - a value of 1.0 creates natural Poisson traffic, while lower values (0.1-0.5) create bursty patterns and higher values (2.0-5.0) create uniform spacing. The `--max-concurrency` parameter defaults to `None` (unlimited) but can be set to simulate real-world constraints where a load balancer or API gateway limits concurrent connections. When combined, these parameters allow you to simulate everything from unrestricted stress testing (`--request-rate=inf`) to production-like scenarios with realistic arrival patterns and resource constraints. + +The `--burstiness` parameter mathematically controls request arrival patterns using a Gamma distribution where: + +- Shape parameter: `burstiness` value +- Coefficient of Variation (CV): $\frac{1}{\sqrt{burstiness}}$ +- Traffic characteristics: + - `burstiness = 0.1`: Highly bursty traffic (CV ≈ 3.16) - stress testing + - `burstiness = 1.0`: Natural Poisson traffic (CV = 1.0) - realistic simulation + - `burstiness = 5.0`: Uniform traffic (CV ≈ 0.45) - controlled load testing + +![Load Pattern Examples](../assets/contributing/load-pattern-examples.png) + +*Figure: Load pattern examples for each use case. Top row: Request arrival timelines showing cumulative requests over time. Bottom row: Inter-arrival time distributions showing traffic variability patterns. Each column represents a different use case with its specific parameter settings and resulting traffic characteristics.* + +Load Pattern Recommendations by Use Case: + +| Use Case | Burstiness | Request Rate | Max Concurrency | Description | +| --- | --- | --- | --- | --- | +| Maximum Throughput | N/A | Infinite | Limited | **Most common**: Simulates load balancer/gateway limits with unlimited user demand | +| Realistic Testing | 1.0 | Moderate (5-20) | Infinite | Natural Poisson traffic patterns for baseline performance | +| Stress Testing | 0.1-0.5 | High (20-100) | Infinite | Challenging burst patterns to test resilience | +| Latency Profiling | 2.0-5.0 | Low (1-10) | Infinite | Uniform load for consistent timing analysis | +| Capacity Planning | 1.0 | Variable | Limited | Test resource limits with realistic constraints | +| SLA Validation | 1.0 | Target rate | SLA limit | Production-like constraints for compliance testing | + +These load patterns help evaluate different aspects of your vLLM deployment, from basic performance characteristics to resilience under challenging traffic conditions. + +The **Maximum Throughput** pattern (`--request-rate=inf --max-concurrency=`) is the most commonly used configuration for production benchmarking. This simulates real-world deployment architectures where: + +- Users send requests as fast as they can (infinite rate) +- A load balancer or API gateway controls the maximum concurrent connections +- The system operates at its concurrency limit, revealing true throughput capacity +- `--burstiness` has no effect since request timing is not controlled when rate is infinite + +This pattern helps determine optimal concurrency settings for your production load balancer configuration. + +To effectively configure load patterns, especially for **Capacity Planning** and **SLA Validation** use cases, you need to understand your system's resource limits. During startup, vLLM reports KV cache configuration that directly impacts your load testing parameters: + +```text +GPU KV cache size: 15,728,640 tokens +Maximum concurrency for 8,192 tokens per request: 1920 +``` + +Where: + +- GPU KV cache size: Total tokens that can be cached across all concurrent requests +- Maximum concurrency: Theoretical maximum concurrent requests for the given `max_model_len` +- Calculation: `max_concurrency = kv_cache_size / max_model_len` + +Using KV cache metrics for load pattern configuration: + +- For Capacity Planning: Set `--max-concurrency` to 80-90% of the reported maximum to test realistic resource constraints +- For SLA Validation: Use the reported maximum as your SLA limit to ensure compliance testing matches production capacity +- For Realistic Testing: Monitor memory usage when approaching theoretical limits to understand sustainable request rates +- Request rate guidance: Use the KV cache size to estimate sustainable request rates for your specific workload and sequence lengths + #### 📈 Offline Throughput Benchmark From 141e6a0505968f022cf3b33b543357518b856d86 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 29 Oct 2025 01:55:10 +0800 Subject: [PATCH 045/127] [Misc] Make reorder batch also separate extends (#27367) Signed-off-by: Lucas Wilkinson --- tests/v1/attention/test_batch_reordering.py | 111 ++++++++++++++++++++ vllm/v1/attention/backends/utils.py | 88 +++++++++------- 2 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 tests/v1/attention/test_batch_reordering.py diff --git a/tests/v1/attention/test_batch_reordering.py b/tests/v1/attention/test_batch_reordering.py new file mode 100644 index 0000000000000..b271409b92955 --- /dev/null +++ b/tests/v1/attention/test_batch_reordering.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass + +import numpy as np +import pytest + +from vllm.v1.attention.backends.utils import reorder_batch_to_split_decodes_and_prefills + + +class MockInputBatch: + def __init__(self, req_ids, num_computed_tokens_cpu): + self.req_ids = req_ids + self.num_computed_tokens_cpu = num_computed_tokens_cpu + + def swap_states(self, i, j): + self.req_ids[i], self.req_ids[j] = self.req_ids[j], self.req_ids[i] + self.num_computed_tokens_cpu[i], self.num_computed_tokens_cpu[j] = ( + self.num_computed_tokens_cpu[j], + self.num_computed_tokens_cpu[i], + ) + + +class MockSchedulerOutput: + def __init__(self, num_scheduled_tokens): + self.num_scheduled_tokens = num_scheduled_tokens + + +@dataclass +class ReorderTestCase: + requests: list[tuple[int, int]] # (num_scheduled_tokens, num_computed_tokens) + expected_order: list[int] + expected_modified: bool + decode_threshold: int = 1 + + +# Test cases for batch reordering +REORDER_TEST_CASES = { + "all_decodes": ReorderTestCase( + requests=[(1, 10), (1, 20), (1, 30)], + expected_order=[0, 1, 2], + expected_modified=False, + ), + "all_prefills": ReorderTestCase( + requests=[(100, 100), (200, 200), (300, 300)], + expected_order=[0, 1, 2], + expected_modified=False, + ), + "mixed_interleaved": ReorderTestCase( + requests=[(100, 100), (1, 10), (200, 200), (1, 20)], + expected_order=[3, 1, 2, 0], # Only swap 0↔3, keep 1 and 2 in place + expected_modified=True, + ), + "already_ordered": ReorderTestCase( + requests=[(1, 10), (1, 20), (100, 100), (200, 200)], + expected_order=[0, 1, 2, 3], + expected_modified=False, + ), + "single_request": ReorderTestCase( + requests=[(1, 10)], + expected_order=[0], + expected_modified=False, + ), + "higher_threshold": ReorderTestCase( + requests=[(2, 10), (3, 20), (5, 30), (6, 40)], + expected_order=[0, 1, 2, 3], + expected_modified=False, + decode_threshold=4, + ), + "decodes_at_end": ReorderTestCase( + requests=[(100, 100), (200, 200), (1, 10), (1, 20)], + expected_order=[2, 3, 0, 1], + expected_modified=True, + ), + "decode_extend_prefill": ReorderTestCase( + requests=[(100, 100), (10, 50), (1, 10)], + expected_order=[2, 1, 0], + expected_modified=True, + ), + "extend_prefill_only": ReorderTestCase( + requests=[(100, 100), (10, 50), (200, 200), (20, 75)], + expected_order=[3, 1, 2, 0], # Only swap 0↔3, keep 1 and 2 in place + expected_modified=True, + ), +} + + +@pytest.mark.parametrize( + "test_case", REORDER_TEST_CASES.values(), ids=REORDER_TEST_CASES.keys() +) +def test_reorder_batch_to_split_decodes_and_prefills(test_case: ReorderTestCase): + req_ids = [f"r{i}" for i in range(len(test_case.requests))] + num_computed_tokens = np.array([r[1] for r in test_case.requests], dtype=np.int32) + num_scheduled_tokens = {f"r{i}": r[0] for i, r in enumerate(test_case.requests)} + + input_batch = MockInputBatch(req_ids, num_computed_tokens) + scheduler_output = MockSchedulerOutput(num_scheduled_tokens) + + modified = reorder_batch_to_split_decodes_and_prefills( + input_batch, scheduler_output, decode_threshold=test_case.decode_threshold + ) + + expected_req_ids = [f"r{i}" for i in test_case.expected_order] + + assert modified == test_case.expected_modified, ( + f"Expected modified={test_case.expected_modified}, got {modified}" + ) + assert input_batch.req_ids == expected_req_ids, ( + f"Expected order {expected_req_ids}, got {input_batch.req_ids}" + ) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index a0d354df06ca3..389baf1488be0 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -795,51 +795,59 @@ def reorder_batch_to_split_decodes_and_prefills( Returns: True if the batch was modified, False otherwise. """ - # We now want to reorder the batch so that the "decode" requests are at - # the front and the "prefill" requests are at the back using the least - # amount of swaps possible. (NOTE for now we loosely use "decode" to mean - # requests where attention is likely memory-bound and "prefill" to mean - # requests where attention is likely compute-bound, TODO(lucas): figure out - # a better naming here) - decodes = [] - prefills = [] - num_decode_tokens = 0 - num_prefill_tokens = 0 + # We now want to reorder the batch into decode → extend → prefill order + # where: + # decode: request with num_scheduled_tokens <= decode_threshold + # extend: non-decode request with existing context + # prefill: non-decode request with no existing context + # NOTE for now we loosely use "decode" to mean requests where attention is + # likely memory-bound and "prefill" to mean requests where attention is + # likely compute-bound, + num_reqs = len(input_batch.req_ids) + num_scheduled_tokens = [ + scheduler_output.num_scheduled_tokens[id] for id in input_batch.req_ids + ] + num_scheduled_tokens_np = np.array(num_scheduled_tokens) + num_computed_tokens_np = input_batch.num_computed_tokens_cpu[:num_reqs] - for i, req_id in enumerate(input_batch.req_ids): - num_tokens = scheduler_output.num_scheduled_tokens[req_id] - if num_tokens <= decode_threshold: - decodes.append(i) - num_decode_tokens += num_tokens - else: - prefills.append(i) - num_prefill_tokens += num_tokens + is_decode = num_scheduled_tokens_np <= decode_threshold + is_extend = (~is_decode) & (num_computed_tokens_np > num_scheduled_tokens_np) + is_prefill = (~is_decode) & (num_computed_tokens_np == num_scheduled_tokens_np) - # We hope that this is fairly minimal since decodes - # should be around for a number of iterations so hopefully they are - # relatively stationary (and new request are generally appended to the - # persistent batch so already should be at the back) - # To achieve this we loop over the decodes in descending order and - # the prefills in ascending order. We swap decodes from the "back" - # i.e. past where the last decode should be in the reodorered with - # prefills from the front of the batch. - # `decodes` and `prefills` are already in ascending order just based on - # the above loop - num_decodes = len(decodes) - num_prefills = len(prefills) - modified_batch = False + # Desired order: decode → extend → prefill + req_regions = np.zeros(is_decode.shape, dtype=np.int32) # 0 = decode by default + req_regions[is_extend] = 1 + req_regions[is_prefill] = 2 - for i in range(1, min(num_decodes, num_prefills) + 1): - # If the decode is at the "back" of the batch, i, we can swap it - # with the prefill closest to the front of the batch - decode_idx = decodes[num_decodes - i] - if decode_idx < num_decodes: - break + num_decodes = int(is_decode.sum()) + num_extends = int(is_extend.sum()) - input_batch.swap_states(prefills[i - 1], decode_idx) - modified_batch = True + target_regions = np.zeros(num_reqs, dtype=np.int32) + target_regions[num_decodes : num_decodes + num_extends] = 1 + target_regions[num_decodes + num_extends :] = 2 - return modified_batch + needs_swap = req_regions != target_regions + + if not needs_swap.any(): + return False + + # Extract indices that need swapping and sort by target region + swap_indices = np.where(needs_swap)[0] + sorted_order = np.argsort(req_regions[needs_swap], kind="stable") + dest_indices = swap_indices[sorted_order] + + src_dest_map = {int(src): int(dst) for src, dst in zip(swap_indices, dest_indices)} + + for src in src_dest_map: + dst = src_dest_map[src] + while src != dst: + input_batch.swap_states(src, dst) + # Mark dst as done by updating its destination to itself + next_dst = src_dest_map.get(dst, dst) + src_dest_map[dst] = dst + dst = next_dst + + return True def reshape_query_for_spec_decode(query: torch.Tensor, batch_size: int) -> torch.Tensor: From 6afc28a9ba236b35bbe77f96d8e7c499d1b0323a Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:51:35 -0400 Subject: [PATCH 046/127] [Test] Batch Invariant: Unit test using parameterized backend (#27478) Signed-off-by: yewentao256 --- tests/v1/generation/test_batch_invariance.py | 454 +++++++++--------- vllm/model_executor/layers/batch_invariant.py | 2 +- 2 files changed, 230 insertions(+), 226 deletions(-) diff --git a/tests/v1/generation/test_batch_invariance.py b/tests/v1/generation/test_batch_invariance.py index 8e59b695ed571..f05fac2478d8a 100644 --- a/tests/v1/generation/test_batch_invariance.py +++ b/tests/v1/generation/test_batch_invariance.py @@ -17,16 +17,10 @@ skip_unsupported = pytest.mark.skipif( @pytest.fixture(autouse=True) -def enable_batch_invariant_mode(): +def enable_batch_invariant_mode(monkeypatch: pytest.MonkeyPatch): """Automatically enable batch invariant kernel overrides for all tests.""" - old_value = os.environ.get("VLLM_BATCH_INVARIANT") - os.environ["VLLM_BATCH_INVARIANT"] = "1" + monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1") yield - # Restore original value after test - if old_value is None: - os.environ.pop("VLLM_BATCH_INVARIANT", None) - else: - os.environ["VLLM_BATCH_INVARIANT"] = old_value def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str: @@ -76,7 +70,13 @@ def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str: @skip_unsupported @pytest.mark.timeout(1000) -def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(): +@pytest.mark.parametrize( + "backend", + ["FLASH_ATTN", "FLASHINFER", "FLASH_ATTN_MLA", "FLASHINFER_MLA", "TRITON_MLA"], +) +def test_v1_generation_is_deterministic_across_batch_sizes_with_needle( + backend, monkeypatch: pytest.MonkeyPatch +): """ Ensures that the same request (the 'needle' prompt) yields identical output whether run alone (bs=1) or mixed into a larger batch (e.g., bs=64), @@ -101,6 +101,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(): seed = int(os.getenv("VLLM_TEST_SEED", "12345")) random.seed(seed) + monkeypatch.setenv("VLLM_ATTENTION_BACKEND", backend) # Allow overrides from environment (useful for CI tuning) # "facebook/opt-125m" is too small, doesn't reliably test determinism model = os.getenv("VLLM_TEST_MODEL", "Qwen/Qwen3-1.7B") @@ -220,11 +221,15 @@ def _extract_step_logprobs(request_output): @skip_unsupported -@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER"]) +@pytest.mark.parametrize( + "backend", + ["FLASH_ATTN", "FLASHINFER", "FLASH_ATTN_MLA", "FLASHINFER_MLA", "TRITON_MLA"], +) @pytest.mark.forked -def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(backend): - backend = os.getenv("VLLM_ATTENTION_BACKEND", backend) - os.environ["VLLM_ATTENTION_BACKEND"] = backend +def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN( + backend, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("VLLM_ATTENTION_BACKEND", backend) seed = int(os.getenv("VLLM_TEST_SEED", "12345")) random.seed(seed) @@ -435,11 +440,16 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(backend): @skip_unsupported -def test_simple_generation(): +@pytest.mark.parametrize( + "backend", + ["FLASH_ATTN", "FLASHINFER", "FLASH_ATTN_MLA", "FLASHINFER_MLA", "TRITON_MLA"], +) +def test_simple_generation(backend, monkeypatch: pytest.MonkeyPatch): """ Simple test that runs the model with a basic prompt and prints the output. Useful for quick smoke testing and debugging. """ + monkeypatch.setenv("VLLM_ATTENTION_BACKEND", backend) model = os.getenv("VLLM_TEST_MODEL", "Qwen/Qwen3-1.7B") llm = LLM( @@ -481,9 +491,14 @@ def test_simple_generation(): @skip_unsupported -@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER"]) +@pytest.mark.parametrize( + "backend", + ["FLASH_ATTN", "FLASHINFER", "FLASH_ATTN_MLA", "FLASHINFER_MLA", "TRITON_MLA"], +) @pytest.mark.forked -def test_logprobs_WITHOUT_batch_invariance_should_FAIL(backend): +def test_logprobs_without_batch_invariance_should_fail( + backend, monkeypatch: pytest.MonkeyPatch +): """ This test is the inverse of test_logprobs_bitwise_batch_invariance_bs1_vs_bsN. It DISABLES batch invariance mode and expects to see non-deterministic behavior @@ -493,224 +508,214 @@ def test_logprobs_WITHOUT_batch_invariance_should_FAIL(backend): The test will PASS if we detect differences (proving batch invariance matters). The test will FAIL if everything matches (suggesting batch invariance isn't needed). """ - backend = os.getenv("VLLM_ATTENTION_BACKEND", backend) - os.environ["VLLM_ATTENTION_BACKEND"] = backend + monkeypatch.setenv("VLLM_ATTENTION_BACKEND", backend) # CRITICAL: Disable batch invariance for this test - old_value = os.environ.get("VLLM_BATCH_INVARIANT") - os.environ["VLLM_BATCH_INVARIANT"] = "0" + monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0") - try: - seed = int(os.getenv("VLLM_TEST_SEED", "12345")) - random.seed(seed) - model_name = os.getenv("VLLM_TEST_MODEL", "Qwen/Qwen3-1.7B") - tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1")) + seed = int(os.getenv("VLLM_TEST_SEED", "12345")) + random.seed(seed) + model_name = os.getenv("VLLM_TEST_MODEL", "Qwen/Qwen3-1.7B") + tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1")) - print(f"\n{'=' * 80}") - print("BATCH INVARIANCE DISABLED: Expecting non-deterministic behavior") + print(f"\n{'=' * 80}") + print("BATCH INVARIANCE DISABLED: Expecting non-deterministic behavior") + print(f"{'=' * 80}\n") + + llm = LLM( + model=model_name, + tensor_parallel_size=tp_size, + enable_prefix_caching=False, + max_num_seqs=32, + max_model_len=8192, + dtype="bfloat16", + ) + + # build ragged prompts to change shapes significantly across BS=1 vs BS=N + long_min = int(os.getenv("VLLM_MIN_PROMPT", "768")) + long_max = int(os.getenv("VLLM_MAX_PROMPT", "2048")) + prompts: list[str] = [] + options = [ + (max(long_min, 1536), max(long_max, 3072)), # very long + (max(1024, long_min), max(2048, long_max)), # long + (256, 512), # mid + (10, 20), # short + ] + + for _ in range(32): + lo, hi = random.choice(options) + prompts.append(_random_prompt(lo, hi)) + + sp = SamplingParams( + temperature=0.6, + top_p=1.0, + max_tokens=8, + seed=1234, + logprobs=5, + ) + + # BS=1: run prompts individually and collect logprobs per step. + print("\n" + "=" * 80) + print("STARTING BS=1 RUNS (each prompt individually)") + print("=" * 80 + "\n") + + bs1_logprobs_per_prompt = [] + bs1_tokens_per_prompt = [] + for idx, p in enumerate(prompts): + print(f"\n[BS=1] Running prompt {idx}/{len(prompts)} - Preview: {p[:80]}...") + outs = llm.generate([p], sp, use_tqdm=False) + assert len(outs) == 1 + step_logprobs, token_ids = _extract_step_logprobs(outs[0]) + if step_logprobs is None: + pytest.skip( + "Logits are not available on RequestOutput; " + "enable logprobs return to run this test." + ) + bs1_logprobs_per_prompt.append(step_logprobs) + bs1_tokens_per_prompt.append(token_ids) + print(f"[BS=1] Prompt {idx} generated tokens: {token_ids}") + + # BS=N: run prompts in a batch and collect logprobs per step for each prompt. + print("\n" + "=" * 80) + print(f"STARTING BS={len(prompts)} RUN (all prompts batched)") + print("=" * 80 + "\n") + + outs_batched = llm.generate(prompts, sp, use_tqdm=False) + assert len(outs_batched) == len(prompts) + bsN_logprobs_per_prompt = [] + bsN_tokens_per_prompt = [] + + print(f"\n[BS={len(prompts)}] Processing batched outputs...") + for idx, o in enumerate(outs_batched): + tokens = o.outputs[0].token_ids if o.outputs else "N/A" + print(f"[BS={len(prompts)}] Prompt {idx} generated tokens: {tokens}") + step_logprobs, token_ids = _extract_step_logprobs(o) + if step_logprobs is None: + pytest.skip( + "Logits are not available on RequestOutput; " + "enable logprobs return to run this test." + ) + bsN_logprobs_per_prompt.append(step_logprobs) + bsN_tokens_per_prompt.append(token_ids) + + # Compare step-by-step logprobs for each prompt between BS=1 and BS=N runs. + differences_found = [] + for i, (logprobs_bs1, logprobs_bsN, tokens_bs1, tokens_bsN) in enumerate( + zip( + bs1_logprobs_per_prompt, + bsN_logprobs_per_prompt, + bs1_tokens_per_prompt, + bsN_tokens_per_prompt, + ) + ): + if len(logprobs_bs1) != len(logprobs_bsN): + reason = ( + f"Different number of steps: {len(logprobs_bs1)} (BS=1) " + f"vs {len(logprobs_bsN)} (BS=N)" + ) + differences_found.append( + { + "prompt_idx": i, + "step": "all", + "reason": reason, + "prompt_preview": prompts[i][:100], + "bs1_tokens": tokens_bs1, + "bsN_tokens": tokens_bsN, + } + ) + continue + + # Check if tokens match first + if tokens_bs1 != tokens_bsN: + differences_found.append( + { + "prompt_idx": i, + "step": "sampling", + "reason": "Different tokens sampled", + "prompt_preview": prompts[i][:100], + "bs1_tokens": tokens_bs1, + "bsN_tokens": tokens_bsN, + } + ) + continue + + for t, (a, b) in enumerate(zip(logprobs_bs1, logprobs_bsN)): + if a.shape != b.shape: + differences_found.append( + { + "prompt_idx": i, + "step": t, + "reason": f"Shape mismatch: {a.shape} vs {b.shape}", + "prompt_preview": prompts[i][:100], + "bs1_tokens": tokens_bs1, + "bsN_tokens": tokens_bsN, + } + ) + break + + if not torch.equal(a, b): + max_diff = torch.abs(a - b).max().item() + print( + f"\n[EXPECTED DIVERGENCE FOUND] Prompt {i}, " + f"Token {t}: max_diff={max_diff:.6e}" + ) + bs1_tok = tokens_bs1[t] if t < len(tokens_bs1) else "N/A" + bsN_tok = tokens_bsN[t] if t < len(tokens_bsN) else "N/A" + print(f" Token IDs: bs1={bs1_tok}, bsN={bsN_tok}") + print(f" BS=1 logprob: {a.tolist()}") + print(f" BS=N logprob: {b.tolist()}") + differences_found.append( + { + "prompt_idx": i, + "step": t, + "reason": f"Bitwise mismatch (max_diff={max_diff:.6e})", + "prompt_preview": prompts[i][:100], + "bs1_tokens": tokens_bs1, + "bsN_tokens": tokens_bsN, + } + ) + break + + # Print summary + print(f"\n{'=' * 80}") + if differences_found: + success_msg = ( + f"✓ SUCCESS: Batch invariance is doing something! " + f"Found {len(differences_found)}/{len(prompts)} prompts " + f"with differences when batch invariance was DISABLED." + ) + print(success_msg) + print(f"{'=' * 80}") + for diff in differences_found: + print(f"\nPrompt {diff['prompt_idx']} (step {diff['step']}):") + print(f" Reason: {diff['reason']}") + print(f" Preview: {diff['prompt_preview']}...") + if "bs1_tokens" in diff: + print(f" BS=1 tokens: {diff['bs1_tokens']}") + if "bsN_tokens" in diff: + print(f" BS=N tokens: {diff['bsN_tokens']}") print(f"{'=' * 80}\n") - - llm = LLM( - model=model_name, - tensor_parallel_size=tp_size, - enable_prefix_caching=False, - max_num_seqs=32, - max_model_len=8192, - dtype="bfloat16", + # Test PASSES because we found differences (batch invariance matters!) + return + else: + # Test FAILS because everything matched even without batch invariance + fail_msg = ( + f"✗ UNEXPECTED: All {len(prompts)} prompts matched " + f"between BS=1 and BS=N even with batch invariance DISABLED. " + f"This suggests batch invariance might not be necessary, " + f"or the test needs more sensitive prompts." ) - - # build ragged prompts to change shapes significantly across BS=1 vs BS=N - long_min = int(os.getenv("VLLM_MIN_PROMPT", "768")) - long_max = int(os.getenv("VLLM_MAX_PROMPT", "2048")) - prompts: list[str] = [] - options = [ - (max(long_min, 1536), max(long_max, 3072)), # very long - (max(1024, long_min), max(2048, long_max)), # long - (256, 512), # mid - (10, 20), # short - ] - - for _ in range(32): - lo, hi = random.choice(options) - prompts.append(_random_prompt(lo, hi)) - - sp = SamplingParams( - temperature=0.6, - top_p=1.0, - max_tokens=8, - seed=1234, - logprobs=5, - ) - - # BS=1: run prompts individually and collect logprobs per step. - print("\n" + "=" * 80) - print("STARTING BS=1 RUNS (each prompt individually)") - print("=" * 80 + "\n") - - bs1_logprobs_per_prompt = [] - bs1_tokens_per_prompt = [] - for idx, p in enumerate(prompts): - print( - f"\n[BS=1] Running prompt {idx}/{len(prompts)} - Preview: {p[:80]}..." - ) - outs = llm.generate([p], sp, use_tqdm=False) - assert len(outs) == 1 - step_logprobs, token_ids = _extract_step_logprobs(outs[0]) - if step_logprobs is None: - pytest.skip( - "Logits are not available on RequestOutput; " - "enable logprobs return to run this test." - ) - bs1_logprobs_per_prompt.append(step_logprobs) - bs1_tokens_per_prompt.append(token_ids) - print(f"[BS=1] Prompt {idx} generated tokens: {token_ids}") - - # BS=N: run prompts in a batch and collect logprobs per step for each prompt. - print("\n" + "=" * 80) - print(f"STARTING BS={len(prompts)} RUN (all prompts batched)") - print("=" * 80 + "\n") - - outs_batched = llm.generate(prompts, sp, use_tqdm=False) - assert len(outs_batched) == len(prompts) - bsN_logprobs_per_prompt = [] - bsN_tokens_per_prompt = [] - - print(f"\n[BS={len(prompts)}] Processing batched outputs...") - for idx, o in enumerate(outs_batched): - tokens = o.outputs[0].token_ids if o.outputs else "N/A" - print(f"[BS={len(prompts)}] Prompt {idx} generated tokens: {tokens}") - step_logprobs, token_ids = _extract_step_logprobs(o) - if step_logprobs is None: - pytest.skip( - "Logits are not available on RequestOutput; " - "enable logprobs return to run this test." - ) - bsN_logprobs_per_prompt.append(step_logprobs) - bsN_tokens_per_prompt.append(token_ids) - - # Compare step-by-step logprobs for each prompt between BS=1 and BS=N runs. - differences_found = [] - for i, (logprobs_bs1, logprobs_bsN, tokens_bs1, tokens_bsN) in enumerate( - zip( - bs1_logprobs_per_prompt, - bsN_logprobs_per_prompt, - bs1_tokens_per_prompt, - bsN_tokens_per_prompt, - ) - ): - if len(logprobs_bs1) != len(logprobs_bsN): - reason = ( - f"Different number of steps: {len(logprobs_bs1)} (BS=1) " - f"vs {len(logprobs_bsN)} (BS=N)" - ) - differences_found.append( - { - "prompt_idx": i, - "step": "all", - "reason": reason, - "prompt_preview": prompts[i][:100], - "bs1_tokens": tokens_bs1, - "bsN_tokens": tokens_bsN, - } - ) - continue - - # Check if tokens match first - if tokens_bs1 != tokens_bsN: - differences_found.append( - { - "prompt_idx": i, - "step": "sampling", - "reason": "Different tokens sampled", - "prompt_preview": prompts[i][:100], - "bs1_tokens": tokens_bs1, - "bsN_tokens": tokens_bsN, - } - ) - continue - - for t, (a, b) in enumerate(zip(logprobs_bs1, logprobs_bsN)): - if a.shape != b.shape: - differences_found.append( - { - "prompt_idx": i, - "step": t, - "reason": f"Shape mismatch: {a.shape} vs {b.shape}", - "prompt_preview": prompts[i][:100], - "bs1_tokens": tokens_bs1, - "bsN_tokens": tokens_bsN, - } - ) - break - - if not torch.equal(a, b): - max_diff = torch.abs(a - b).max().item() - print( - f"\n[EXPECTED DIVERGENCE FOUND] Prompt {i}, " - f"Token {t}: max_diff={max_diff:.6e}" - ) - bs1_tok = tokens_bs1[t] if t < len(tokens_bs1) else "N/A" - bsN_tok = tokens_bsN[t] if t < len(tokens_bsN) else "N/A" - print(f" Token IDs: bs1={bs1_tok}, bsN={bsN_tok}") - print(f" BS=1 logprob: {a.tolist()}") - print(f" BS=N logprob: {b.tolist()}") - differences_found.append( - { - "prompt_idx": i, - "step": t, - "reason": f"Bitwise mismatch (max_diff={max_diff:.6e})", - "prompt_preview": prompts[i][:100], - "bs1_tokens": tokens_bs1, - "bsN_tokens": tokens_bsN, - } - ) - break - - # Print summary - print(f"\n{'=' * 80}") - if differences_found: - success_msg = ( - f"✓ SUCCESS: Batch invariance is doing something! " - f"Found {len(differences_found)}/{len(prompts)} prompts " - f"with differences when batch invariance was DISABLED." - ) - print(success_msg) - print(f"{'=' * 80}") - for diff in differences_found: - print(f"\nPrompt {diff['prompt_idx']} (step {diff['step']}):") - print(f" Reason: {diff['reason']}") - print(f" Preview: {diff['prompt_preview']}...") - if "bs1_tokens" in diff: - print(f" BS=1 tokens: {diff['bs1_tokens']}") - if "bsN_tokens" in diff: - print(f" BS=N tokens: {diff['bsN_tokens']}") - print(f"{'=' * 80}\n") - # Test PASSES because we found differences (batch invariance matters!) - return - else: - # Test FAILS because everything matched even without batch invariance - fail_msg = ( - f"✗ UNEXPECTED: All {len(prompts)} prompts matched " - f"between BS=1 and BS=N even with batch invariance DISABLED. " - f"This suggests batch invariance might not be necessary, " - f"or the test needs more sensitive prompts." - ) - print(fail_msg) - print(f"{'=' * 80}\n") - pytest.fail(fail_msg) - - finally: - # Restore original value - if old_value is None: - os.environ.pop("VLLM_BATCH_INVARIANT", None) - else: - os.environ["VLLM_BATCH_INVARIANT"] = old_value + print(fail_msg) + print(f"{'=' * 80}\n") + pytest.fail(fail_msg) @skip_unsupported @pytest.mark.parametrize("backend", ["FLASH_ATTN"]) @pytest.mark.forked -def test_decode_logprobs_match_prefill_logprobs(backend): +def test_decode_logprobs_match_prefill_logprobs( + backend, monkeypatch: pytest.MonkeyPatch +): """ Test that verifies decode logprobs match prefill logprobs. @@ -724,8 +729,7 @@ def test_decode_logprobs_match_prefill_logprobs(backend): This ensures that the logprobs from decode are consistent with what we would get if we ran prefill on each prefix. """ - backend = os.getenv("VLLM_ATTENTION_BACKEND", backend) - os.environ["VLLM_ATTENTION_BACKEND"] = backend + monkeypatch.setenv("VLLM_ATTENTION_BACKEND", backend) seed = int(os.getenv("VLLM_TEST_SEED", "12345")) random.seed(seed) diff --git a/vllm/model_executor/layers/batch_invariant.py b/vllm/model_executor/layers/batch_invariant.py index 7368bfd35fec9..208ffb30e5ed2 100644 --- a/vllm/model_executor/layers/batch_invariant.py +++ b/vllm/model_executor/layers/batch_invariant.py @@ -753,13 +753,13 @@ def override_envs_for_invariance(): curr_attn_backend = envs.VLLM_ATTENTION_BACKEND supported_backends = [ "FLASH_ATTN", # best supported backend - "FLEX_ATTENTION", "FLASHINFER", "FLASH_ATTN_MLA", "FLASHINFER_MLA", "TRITON_MLA", # Not yet supported MLA backends # "FLASHMLA", + # "FLEX_ATTENTION", # IMA issue even if we disable batch invariance ] if curr_attn_backend not in supported_backends: warning = ( From 111faf11185a75435403e3843e9a21a57cf84ccb Mon Sep 17 00:00:00 2001 From: Or Ozeri Date: Tue, 28 Oct 2025 23:01:33 +0200 Subject: [PATCH 047/127] [Core] Scheduler: Publish connector events after output (#25875) Signed-off-by: Or Ozeri --- tests/v1/kv_offload/test_cpu_offloading.py | 144 +++++++++++++++++---- vllm/v1/core/sched/scheduler.py | 34 ++--- 2 files changed, 135 insertions(+), 43 deletions(-) diff --git a/tests/v1/kv_offload/test_cpu_offloading.py b/tests/v1/kv_offload/test_cpu_offloading.py index e9c255b1ee994..b654ea4298dbb 100644 --- a/tests/v1/kv_offload/test_cpu_offloading.py +++ b/tests/v1/kv_offload/test_cpu_offloading.py @@ -1,15 +1,68 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import socket import time +import msgspec +import msgspec.msgpack import pytest +import zmq +from tqdm import tqdm -from vllm import LLM, SamplingParams -from vllm.config import KVTransferConfig +from vllm import LLM, SamplingParams, TokensPrompt +from vllm.config import KVEventsConfig, KVTransferConfig +from vllm.distributed.kv_events import BlockStored, KVEventBatch CPU_BLOCK_SIZES = [16, 48] +class MockSubscriber: + """Helper class to receive and verify published events""" + + def __init__( + self, + endpoint: str, + topic: str, + ): + self.ctx = zmq.Context.instance() + self.topic_bytes = topic.encode("utf-8") + + # Set up subscriber socket + self.sub = self.ctx.socket(zmq.SUB) + self.sub.setsockopt(zmq.SUBSCRIBE, self.topic_bytes) + self.sub.connect(endpoint) + + self.decoder = msgspec.msgpack.Decoder(type=KVEventBatch) + + def get_new_cpu_stored_events(self) -> list[BlockStored]: + cpu_stored_events: list[BlockStored] = [] + + poller = zmq.Poller() + poller.register(self.sub, zmq.POLLIN) + + timeout = 1000 # 1 second + while True: + events = dict(poller.poll(timeout)) + + if events.get(self.sub) != zmq.POLLIN: + return cpu_stored_events + + topic_bytes, _, payload = self.sub.recv_multipart() + + assert topic_bytes == self.topic_bytes + + event_batch = self.decoder.decode(payload) + assert isinstance(event_batch, KVEventBatch) + for event in event_batch.events: + if isinstance(event, BlockStored) and event.medium == "CPU": + cpu_stored_events.append(event) + timeout = 100 + + def close(self): + """Clean up resources""" + self.sub.close() + + @pytest.mark.parametrize("cpu_block_size", CPU_BLOCK_SIZES) def test_cpu_offloading(cpu_block_size: int) -> None: """ @@ -20,41 +73,80 @@ def test_cpu_offloading(cpu_block_size: int) -> None: kv_transfer_config = KVTransferConfig( kv_connector="OffloadingConnector", kv_role="kv_both", - kv_connector_extra_config={"num_cpu_blocks": 100, "block_size": cpu_block_size}, + kv_connector_extra_config={ + "num_cpu_blocks": 1000, + "block_size": cpu_block_size, + }, + ) + + port: int + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("0.0.0.0", 0)) + port = s.getsockname()[1] + + events_endpoint = f"tcp://*:{port}" + kv_events_config = KVEventsConfig( + enable_kv_cache_events=True, + publisher="zmq", + endpoint=events_endpoint, + topic="test", ) llm = LLM( model="meta-llama/Llama-3.2-1B-Instruct", gpu_memory_utilization=0.5, + kv_events_config=kv_events_config, kv_transfer_config=kv_transfer_config, - disable_hybrid_kv_cache_manager=True, ) - prompts = ["Hi " * 100] - sampling_params = SamplingParams(temperature=0, max_tokens=20) + sampling_params = SamplingParams(temperature=0, max_tokens=1) - # run generation - this should trigger saving KV cache - start_time = time.time() - llm.generate(prompts, sampling_params, use_tqdm=False) - cold_time = time.time() - start_time + events_endpoint = events_endpoint.replace("*", "127.0.0.1") + subscriber = MockSubscriber(events_endpoint, topic=kv_events_config.topic) - # run generation again - should hit the GPU prefix cache - start_time = time.time() - llm.generate(prompts, sampling_params, use_tqdm=False) - gpu_hit_time = time.time() - start_time + try: + num_times_cpu_better_than_cold = 0 + num_tests = 10 + total_cold_time = 0.0 + total_gpu_hit_time = 0.0 + total_cpu_hit_time = 0.0 + prompt_token_ids = [0] * 10001 + for i in tqdm(range(num_tests), desc="Running tests"): + prompt_token_ids[0] = i + prompts = [TokensPrompt(prompt_token_ids=prompt_token_ids)] - # reset prefix cache to avoid GPU hit. - llm.reset_prefix_cache() + # run generation - this should trigger saving KV cache + start_time = time.time() + llm.generate(prompts, sampling_params, use_tqdm=False) + cold_time = time.time() - start_time + total_cold_time += cold_time - # sleep for a sec to make sure CPU finished storing - time.sleep(1) + # run generation again - should hit the GPU prefix cache + start_time = time.time() + llm.generate(prompts, sampling_params, use_tqdm=False) + gpu_hit_time = time.time() - start_time + total_gpu_hit_time += gpu_hit_time - # run generation again - this should trigger loading from CPU - start_time = time.time() - llm.generate(prompts, sampling_params, use_tqdm=False) - cpu_hit_time = time.time() - start_time + # reset prefix cache to avoid GPU hit. + llm.reset_prefix_cache() - print("Generation times:") - print(f" Cold: {cold_time * 1000:.2f}ms") - print(f" GPU hit: {gpu_hit_time * 1000:.2f}ms") - print(f" CPU hit: {cpu_hit_time * 1000:.2f}ms") + assert subscriber.get_new_cpu_stored_events() + + # run generation again - this should trigger loading from CPU + start_time = time.time() + llm.generate(prompts, sampling_params, use_tqdm=False) + cpu_hit_time = time.time() - start_time + total_cpu_hit_time += cpu_hit_time + + if cpu_hit_time < cold_time: + num_times_cpu_better_than_cold += 1 + + print("Average times:") + print(f" Cold: {total_cold_time * 1000 / num_tests:.2f}ms") + print(f" GPU hit: {total_gpu_hit_time * 1000 / num_tests:.2f}ms") + print(f" CPU hit: {total_cpu_hit_time * 1000 / num_tests:.2f}ms") + + assert num_times_cpu_better_than_cold >= 0.8 * num_tests + finally: + subscriber.close() + del llm diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 14bdf295317d7..00b34fe4fbb98 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -646,23 +646,6 @@ class Scheduler(SchedulerInterface): meta = self.connector.build_connector_meta(scheduler_output) scheduler_output.kv_connector_metadata = meta - # collect KV cache events from KV cache manager - events = self.kv_cache_manager.take_events() - - # collect KV cache events from connector - if self.connector is not None: - connector_events = self.connector.take_events() - if connector_events: - if events is None: - events = list(connector_events) - else: - events.extend(connector_events) - - # publish collected KV cache events - if events: - batch = KVEventBatch(ts=time.time(), events=events) - self.kv_event_publisher.publish(batch) - self._update_after_schedule(scheduler_output) return scheduler_output @@ -1057,6 +1040,23 @@ class Scheduler(SchedulerInterface): if kv_connector_output: self._update_from_kv_xfer_finished(kv_connector_output) + # collect KV cache events from KV cache manager + events = self.kv_cache_manager.take_events() + + # collect KV cache events from connector + if self.connector is not None: + connector_events = self.connector.take_events() + if connector_events: + if events is None: + events = list(connector_events) + else: + events.extend(connector_events) + + # publish collected KV cache events + if events: + batch = KVEventBatch(ts=time.time(), events=events) + self.kv_event_publisher.publish(batch) + # Create EngineCoreOutputs for all clients that have requests with # outputs in this step. engine_core_outputs = { From 4fe58953611ede752e34b67ae785fed28be66465 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 28 Oct 2025 15:35:54 -0700 Subject: [PATCH 048/127] [AsyncScheduling] Make async overlap work with logprobs (#27615) Signed-off-by: Nick Hill --- tests/conftest.py | 10 ++++-- tests/v1/e2e/test_async_sched_and_preempt.py | 37 +++++++++++++++++--- vllm/v1/outputs.py | 9 +++++ vllm/v1/worker/gpu_model_runner.py | 19 +++++++--- 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ec0179b9cd5ab..91155a72b16ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -831,8 +831,9 @@ class VllmRunner: images: PromptImageInput | None = None, videos: PromptVideoInput | None = None, audios: PromptAudioInput | None = None, + return_logprobs: bool = False, **kwargs: Any, - ) -> list[tuple[list[list[int]], list[str]]]: + ) -> list[tuple[list[list[int]], list[str]]] | tuple[list, list]: inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios) req_outputs = self.llm.generate( @@ -840,18 +841,23 @@ class VllmRunner: ) outputs: list[tuple[list[list[int]], list[str]]] = [] + logprobs = [] for req_output in req_outputs: prompt_str = req_output.prompt prompt_ids = req_output.prompt_token_ids req_sample_output_ids: list[list[int]] = [] req_sample_output_strs: list[str] = [] + req_logprobs = [] for sample in req_output.outputs: output_str = sample.text output_ids = list(sample.token_ids) req_sample_output_ids.append(prompt_ids + output_ids) req_sample_output_strs.append((prompt_str or "") + output_str) + if sample.logprobs: + req_logprobs.extend(sample.logprobs) outputs.append((req_sample_output_ids, req_sample_output_strs)) - return outputs + logprobs.append(req_logprobs) + return outputs if not return_logprobs else (outputs, logprobs) @staticmethod def _final_steps_generate_w_logprobs( diff --git a/tests/v1/e2e/test_async_sched_and_preempt.py b/tests/v1/e2e/test_async_sched_and_preempt.py index 7ad9606a66df6..15a1cc2558177 100644 --- a/tests/v1/e2e/test_async_sched_and_preempt.py +++ b/tests/v1/e2e/test_async_sched_and_preempt.py @@ -6,6 +6,7 @@ import pytest import torch._dynamo.config as dynamo_config from vllm import SamplingParams +from vllm.logprobs import Logprob from ...conftest import VllmRunner from ...models.utils import check_outputs_equal @@ -32,6 +33,8 @@ def test_preempt_and_async_scheduling_e2e(monkeypatch: pytest.MonkeyPatch): # dict(min_tokens=20), dict(presence_penalty=-1.0), dict(bad_words=["the", " the"]), + dict(logprobs=2), + dict(logprobs=2, presence_penalty=-1.0), ] default_params = dict( @@ -77,29 +80,33 @@ def test_preempt_and_async_scheduling_e2e(monkeypatch: pytest.MonkeyPatch): sampling_params=SamplingParams( **default_params, **override_params ), + return_logprobs=True, ) ) if not outputs: # First check that the different parameter configs # actually result in different output. - for other_test, params in zip( + for (other_test_outs, other_test_logprobs), params in zip( results[1:], sampling_param_tests[1:] ): with pytest.raises(AssertionError): check_outputs_equal( - outputs_0_lst=results[0], - outputs_1_lst=other_test, + outputs_0_lst=results[0][0], + outputs_1_lst=other_test_outs, name_0=f"baseline params={params}", name_1=f"other params={params}", ) + assert _all_logprobs_match( + results[0][1], other_test_logprobs + ) outputs.append((test_config, results)) baseline_config, baseline_tests = outputs[0] for test_config, test_outputs in outputs[1:]: - for base_outs, test_outs, params in zip( + for (base_outs, base_logprobs), (test_outs, test_logprobs), params in zip( baseline_tests, test_outputs, sampling_param_tests ): check_outputs_equal( @@ -108,5 +115,27 @@ def test_preempt_and_async_scheduling_e2e(monkeypatch: pytest.MonkeyPatch): name_0=f"baseline=[{baseline_config}], params={params}", name_1=f"config=[{test_config}], params={params}", ) + assert _all_logprobs_match(base_logprobs, test_logprobs) print(f"PASSED: config=[{test_config}], params={params}") + + +def _all_logprobs_match(req_a, req_b) -> bool: + return ( + req_a == req_b + or len(req_a) == len(req_b) + and all( + len(seq_a) == len(seq_b) + and all(_logprobs_match(a, b) for a, b in zip(seq_a, seq_b)) + for seq_a, seq_b in zip(req_a, req_b) + ) + ) + + +def _logprobs_match(lps_a: dict[int, Logprob], lps_b: dict[int, Logprob]) -> bool: + return len(lps_a) == len(lps_b) and all( + a.decoded_token == b.decoded_token + and a.rank == b.rank + and a.logprob == pytest.approx(b.logprob, rel=1e-3, abs=1e-6) + for a, b in ((lps_a[x], lps_b[x]) for x in lps_a) + ) diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 10f97576b60af..e7122ba339681 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -59,6 +59,15 @@ class LogprobsTensors(NamedTuple): cu_num_generated_tokens, ) + def to_cpu_nonblocking(self) -> "LogprobsTensors": + if self.logprob_token_ids.device.type == "cpu": + return self + return LogprobsTensors( + self.logprob_token_ids.to("cpu", non_blocking=True), + self.logprobs.to("cpu", non_blocking=True), + self.selected_token_ranks.to("cpu", non_blocking=True), + ) + @staticmethod def empty_cpu( num_positions: int, num_tokens_per_position: int diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 129d7e54466ad..e350988456f12 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -164,6 +164,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): self, model_runner_output: ModelRunnerOutput, sampled_token_ids: torch.Tensor, + logprobs_tensors: torch.Tensor | None, invalid_req_indices: list[int], async_output_copy_stream: torch.cuda.Stream, ): @@ -176,6 +177,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): # Keep a reference to the device tensor to avoid it being # deallocated until we finish copying it to the host. self._sampled_token_ids = sampled_token_ids + self._logprobs_tensors = logprobs_tensors # Initiate the copy on a separate stream, but do not synchronize it. default_stream = torch.cuda.current_stream() @@ -184,6 +186,11 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): self.sampled_token_ids_cpu = self._sampled_token_ids.to( "cpu", non_blocking=True ) + self._logprobs_tensors_cpu = ( + self._logprobs_tensors.to_cpu_nonblocking() + if self._logprobs_tensors + else None + ) self.async_copy_ready_event.record() def get_output(self) -> ModelRunnerOutput: @@ -193,7 +200,8 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): """ self.async_copy_ready_event.synchronize() - # Release the device tensor once the copy has completed + # Release the device tensors once the copy has completed. + del self._logprobs_tensors del self._sampled_token_ids valid_sampled_token_ids = self.sampled_token_ids_cpu.tolist() @@ -202,6 +210,10 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): output = self._model_runner_output output.sampled_token_ids = valid_sampled_token_ids + if self._logprobs_tensors_cpu: + # NOTE(nick): this will need to be updated to use cu_num_accepted_tokens + # for async sched + spec decode + logprobs compatibility. + output.logprobs = self._logprobs_tensors_cpu.tolists() return output @@ -2334,11 +2346,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): cu_num_accepted_tokens[-1] + len(sampled_ids) ) - # NOTE: GPU -> CPU Sync happens here. - # Move as many CPU operations as possible before this sync point. logprobs_lists = ( logprobs_tensors.tolists(cu_num_accepted_tokens) - if logprobs_tensors is not None + if not self.use_async_scheduling and logprobs_tensors is not None else None ) @@ -2664,6 +2674,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): async_output = AsyncGPUModelRunnerOutput( model_runner_output=output, sampled_token_ids=sampler_output.sampled_token_ids, + logprobs_tensors=sampler_output.logprobs_tensors, invalid_req_indices=invalid_req_indices, async_output_copy_stream=self.async_output_copy_stream, ) From 94666612a938380cb643c1555ef9aa68b7ab1e53 Mon Sep 17 00:00:00 2001 From: Lucas Kabela Date: Tue, 28 Oct 2025 15:36:43 -0700 Subject: [PATCH 049/127] [Misc][qwen2_5_vl][torch.compile] Enable `supports_torch_compile` on generic nn.Module and demonstrate speedup on Qwen Vision model (#23207) Signed-off-by: Lucas Kabela Signed-off-by: Lucas Kabela --- tests/compile/test_multimodal_compile.py | 36 ++++ vllm/attention/ops/vit_attn_wrappers.py | 125 +++++++++++ vllm/compilation/decorators.py | 60 +++++- vllm/config/compilation.py | 2 + .../models/qwen2_5_omni_thinker.py | 8 +- vllm/model_executor/models/qwen2_5_vl.py | 200 ++++++++++-------- 6 files changed, 334 insertions(+), 97 deletions(-) create mode 100644 tests/compile/test_multimodal_compile.py create mode 100644 vllm/attention/ops/vit_attn_wrappers.py diff --git a/tests/compile/test_multimodal_compile.py b/tests/compile/test_multimodal_compile.py new file mode 100644 index 0000000000000..6c195dd93f423 --- /dev/null +++ b/tests/compile/test_multimodal_compile.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest + +from vllm.compilation.counter import compilation_counter +from vllm.config.compilation import CompilationMode + + +# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073 +@pytest.mark.forked +def test_qwen2_5_vl_compilation(vllm_runner, monkeypatch): + """Test that Qwen2.5-VL vision submodules are compiled. + + This test verifies that the 3 vision submodules (Qwen2_5_VisionPatchEmbed, + Qwen2_5_VisionBlock, and Qwen2_5_VisionPatchMerger) are properly tagged + for compilation by checking that num_models_seen increases by at least 3. + """ + # Disable multiprocessing so that the counter is in the same process + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + + with ( + # NOTE: Qwen2.5-VL has 35 models in total - the LLM backend + # Vision Patch Embed, Vision Patch Merger, and then 32 Vision Blocks + # (one for each layer) - in the future, we should fix vLLM compilation + # logic to handle this case and only compile the Vision submodules once + # and reuse the compiled code for all layers + # See https://github.com/vllm-project/vllm/issues/27590 + compilation_counter.expect(num_models_seen=35), + vllm_runner( + "Qwen/Qwen2.5-VL-3B-Instruct", + max_model_len=2048, + gpu_memory_utilization=0.7, + compilation_config={"mode": CompilationMode.VLLM_COMPILE}, + ) as _, + ): + pass diff --git a/vllm/attention/ops/vit_attn_wrappers.py b/vllm/attention/ops/vit_attn_wrappers.py new file mode 100644 index 0000000000000..f71f49a1a31b0 --- /dev/null +++ b/vllm/attention/ops/vit_attn_wrappers.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +This file contains ops for ViT attention to be compatible with torch.compile +as there are operations here not supported by torch.compile (for instance, +`to_list` in xformers attn, or `.item()` in flash attention) + +Using these ops and wrapping vision blocks with `torch.compile` can speed up +throughput in vision models by ~5% relative on H100, and improve token +latencies by ~7% (see qwen2_5_vl for example usage) + +To use these ops, you must have a recent version of PyTorch installed (>= 2.4.0) +""" + +import einops +import torch + +from vllm.utils.torch_utils import direct_register_custom_op + + +def xformers_attn_seqlens_wrapper( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, seqlens: torch.Tensor +) -> torch.Tensor: + from xformers import ops as xops + from xformers.ops.fmha.attn_bias import BlockDiagonalMask + + attn_bias = BlockDiagonalMask.from_seqlens( + q_seqlen=seqlens.tolist(), kv_seqlen=None, device=q.device + ) + context_layer = xops.memory_efficient_attention_forward( + q, k, v, attn_bias=attn_bias, p=0, scale=None + ) + context_layer = einops.rearrange(context_layer, "b s h d -> s b (h d)").contiguous() + return context_layer + + +def xformers_attn_seqlens_wrapper_fake( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, seqlens: torch.Tensor +) -> torch.Tensor: + b, s, h, d = q.shape + return torch.empty((s, b, h * d), dtype=q.dtype, device=q.device) + + +direct_register_custom_op( + op_name="xformers_attn_seqlens_wrapper", + op_func=xformers_attn_seqlens_wrapper, + fake_impl=xformers_attn_seqlens_wrapper_fake, +) + + +def vit_xformers_attn_wrapper( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, seqlens: torch.Tensor +) -> torch.Tensor: + return torch.ops.vllm.xformers_attn_seqlens_wrapper(q, k, v, seqlens) + + +def flash_attn_maxseqlen_wrapper( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + batch_size: int, + is_rocm_aiter: bool, + use_upstream_fa: bool, +) -> torch.Tensor: + if is_rocm_aiter: + from aiter import flash_attn_varlen_func + else: + if use_upstream_fa: + from flash_attn import flash_attn_varlen_func + else: + from vllm.vllm_flash_attn import flash_attn_varlen_func + q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) + output = flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_seqlen.item(), + max_seqlen_k=max_seqlen.item(), + dropout_p=0.0, + causal=False, + ) + context_layer = einops.rearrange( + output, "(b s) h d -> s b (h d)", b=batch_size + ).contiguous() + return context_layer + + +def flash_attn_maxseqlen_wrapper_fake( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + batch_size: int, + is_rocm_aiter: bool, + use_upstream_fa: bool, +) -> torch.Tensor: + b, s, h, d = q.shape + return torch.empty((s, b, h * d), dtype=q.dtype, device=q.device) + + +direct_register_custom_op( + op_name="flash_attn_maxseqlen_wrapper", + op_func=flash_attn_maxseqlen_wrapper, + fake_impl=flash_attn_maxseqlen_wrapper_fake, +) + + +def vit_flash_attn_wrapper( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + batch_size: int, + is_rocm_aiter: bool, + use_upstream_fa: bool, +) -> torch.Tensor: + return torch.ops.vllm.flash_attn_maxseqlen_wrapper( + q, k, v, cu_seqlens, max_seqlen, batch_size, is_rocm_aiter, use_upstream_fa + ) diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 69fb93601f935..0946fa69171b4 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -18,7 +18,12 @@ from torch._dynamo.symbolic_convert import InliningInstructionTranslator import vllm.envs as envs from vllm.compilation.counter import compilation_counter from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher -from vllm.config import CompilationMode, VllmConfig, set_current_vllm_config +from vllm.config import ( + CompilationMode, + VllmConfig, + get_current_vllm_config, + set_current_vllm_config, +) from vllm.logger import init_logger from vllm.sequence import IntermediateTensors from vllm.utils.import_utils import resolve_obj_by_qualname @@ -74,6 +79,21 @@ def support_torch_compile( ) -> Callable[[_T], _T]: ... +@overload +def support_torch_compile( + *, + mark_unbacked_dims: dict[str, int | list[int]] | None, +) -> Callable[[_T], _T]: ... + + +@overload +def support_torch_compile( + *, + dynamic_arg_dims: dict[str, int | list[int]] | None, + mark_unbacked_dims: dict[str, int | list[int]] | None, +) -> Callable[[_T], _T]: ... + + @overload def support_torch_compile(cls: _T) -> _T: ... @@ -82,6 +102,7 @@ def support_torch_compile( cls: _T | None = None, *, dynamic_arg_dims: dict[str, int | list[int]] | None = None, + mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, ) -> Callable[[_T], _T] | _T: """ @@ -135,6 +156,11 @@ def support_torch_compile( returns a boolean value indicating whether to compile the model or not. This is useful if you want to compile the model only when certain conditions are met. + + `mark_unbacked_dims` is a dictionary that maps argument names with a dynamic + dim to be decorated with `mark_unbacked`. This is useful if we would like to + enforce that dynamo do not specialize on 0/1 values in the case of dummy input + such as for vision model compilation """ def cls_decorator_helper(cls: _T) -> _T: @@ -172,7 +198,9 @@ def support_torch_compile( raise ValueError( f"Argument {k} not found in the forward method of {cls}" ) - return _support_torch_compile(cls, inferred_dynamic_arg_dims, enable_if) + return _support_torch_compile( + cls, inferred_dynamic_arg_dims, mark_unbacked_dims, enable_if + ) if cls is not None: # use `support_torch_compile` as a decorator without arguments @@ -212,6 +240,7 @@ def _verify_source_unchanged(source_info, vllm_config) -> None: def _support_torch_compile( cls: _T, dynamic_arg_dims: dict[str, int | list[int]], + mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, ) -> _T: """ @@ -230,8 +259,22 @@ def _support_torch_compile( setattr(cls, IGNORE_COMPILE_KEY, False) - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "", **kwargs): - old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs) + def __init__( + self, *, vllm_config: VllmConfig | None = None, prefix: str = "", **kwargs + ): + if vllm_config is None: + vllm_config = get_current_vllm_config() + + # NOTE: to support multimodal models (such as encoder), + # we may not have vllm_config so we may need to patch + # it + sig = inspect.signature(old_init) + if "vllm_config" in sig.parameters: + kwargs["vllm_config"] = vllm_config + if "prefix" in sig.parameters: + kwargs["prefix"] = prefix + old_init(self, **kwargs) + self.vllm_config = vllm_config enable_compile = enable_if is None or enable_if(vllm_config) # for CompilationMode.STOCK_TORCH_COMPILE , the upper level model runner @@ -344,6 +387,15 @@ def _support_torch_compile( "Unsupported dynamic dimensions" f" {dims} for argument {k} with type {type(arg)}." ) + if mark_unbacked_dims: + for k, dims in mark_unbacked_dims.items(): + arg = bound_args.arguments.get(k) + if arg is not None: + dims = [dims] if isinstance(dims, int) else dims + if isinstance(arg, torch.Tensor): + # In case dims is specified with negative indexing + dims = [arg.ndim + dim if dim < 0 else dim for dim in dims] + torch._dynamo.decorators.mark_unbacked(arg, dims) # here, it is the starting point of the `torch.compile` process start_monitoring_torch_compile(self.vllm_config) logger.debug("Start compiling function %s", self.original_code_object) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index c24a94091be4c..f3ed78779a995 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -684,6 +684,8 @@ class CompilationConfig: from vllm.compilation.backends import VllmBackend + # TODO[@lucaskabela]: See if we can forward prefix + # https://github.com/vllm-project/vllm/issues/27045 return VllmBackend(vllm_config) def post_init_cudagraph_sizes(self) -> None: diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 6338ea93b8c8a..677d34dea39b3 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -45,6 +45,7 @@ from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.forward_context import set_forward_context from vllm.logger import init_logger from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.qwen2_5_vl import ( @@ -759,7 +760,8 @@ class Qwen2_5OmniConditionalGenerationMixin: assert grid_thw.ndim == 2 pixel_values = image_input["pixel_values"].type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=grid_thw) + with set_forward_context(None, self.vllm_config): + image_embeds = self.visual(pixel_values, grid_thw=grid_thw) # Split concatenated embeddings for each image item. merge_size = self.visual.spatial_merge_size sizes = grid_thw.prod(-1) // merge_size // merge_size @@ -779,7 +781,8 @@ class Qwen2_5OmniConditionalGenerationMixin: assert grid_thw.ndim == 2 pixel_values_videos = video_input["pixel_values_videos"].type(self.visual.dtype) - video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw) + with set_forward_context(None, self.vllm_config): + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw) # Split concatenated embeddings for each video item. merge_size = self.visual.spatial_merge_size sizes = grid_thw.prod(-1) // merge_size // merge_size @@ -839,6 +842,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration( def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + self.vllm_config = vllm_config thinker_config: Qwen2_5OmniThinkerConfig = ( vllm_config.model_config.hf_config.thinker_config ) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index b622021e225ca..30e3d2dff97b1 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -31,10 +31,10 @@ from collections.abc import Callable, Iterable, Mapping, Sequence from functools import lru_cache, partial from typing import Annotated, Any, Literal, TypeAlias +import einops import torch import torch.nn as nn import torch.nn.functional as F -from einops import rearrange from transformers import BatchFeature, PretrainedConfig from transformers.models.qwen2_5_vl import Qwen2_5_VLProcessor from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import ( @@ -47,9 +47,15 @@ from vllm.attention.layer import ( check_upstream_fa_availability, maybe_get_vit_flash_attn_backend, ) +from vllm.attention.ops.vit_attn_wrappers import ( + vit_flash_attn_wrapper, + vit_xformers_attn_wrapper, +) +from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils +from vllm.forward_context import set_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_and_mul_fn from vllm.model_executor.layers.layernorm import RMSNorm @@ -392,8 +398,8 @@ class Qwen2_5_VisionAttention(nn.Module): x: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor, - max_seqlen: int | None = None, # Only used for Flash Attention - seqlens: list[int] | None = None, # Only used for xFormers + max_seqlen: torch.Tensor, # Only used for Flash Attention + seqlens: torch.Tensor, # Only used for xFormers ) -> torch.Tensor: # [s, b, c] --> [s, b, head * 3 * head_dim] x, _ = self.qkv(x) @@ -402,7 +408,7 @@ class Qwen2_5_VisionAttention(nn.Module): q, k, v = self.split_qkv(x) batch_size = q.shape[1] - q, k, v = (rearrange(x, "s b ... -> b s ...") for x in (q, k, v)) + q, k, v = (einops.rearrange(x, "s b ... -> b s ...") for x in (q, k, v)) if rotary_pos_emb is not None: # [2 * b, s, heads, head_dim] qk_concat = torch.cat([q, k], dim=0) @@ -410,31 +416,18 @@ class Qwen2_5_VisionAttention(nn.Module): q, k = torch.chunk(qk_rotated, 2, dim=0) if self.is_flash_attn_backend: - q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) - - output = self.flash_attn_varlen_func( + context_layer = vit_flash_attn_wrapper( q, k, v, - cu_seqlens_q=cu_seqlens, - cu_seqlens_k=cu_seqlens, - max_seqlen_q=max_seqlen, - max_seqlen_k=max_seqlen, - dropout_p=0.0, - causal=False, + cu_seqlens, + max_seqlen, + batch_size, + self.attn_backend == _Backend.ROCM_AITER_FA, + self.use_upstream_fa, ) - - context_layer = rearrange( - output, "(b s) h d -> s b (h d)", b=batch_size - ).contiguous() elif self.attn_backend == _Backend.TORCH_SDPA: # Execute attention entry by entry for speed & less VRAM. - from vllm.platforms import current_platform - - if current_platform.is_rocm(): - q = q.contiguous() - k = k.contiguous() - v = v.contiguous() outputs = [] for i in range(1, len(cu_seqlens)): start_idx = cu_seqlens[i - 1] @@ -443,34 +436,31 @@ class Qwen2_5_VisionAttention(nn.Module): k_i = k[:, start_idx:end_idx] v_i = v[:, start_idx:end_idx] q_i, k_i, v_i = ( - rearrange(x, "b s h d -> b h s d") for x in [q_i, k_i, v_i] + einops.rearrange(x, "b s h d -> b h s d") for x in [q_i, k_i, v_i] ) output_i = F.scaled_dot_product_attention(q_i, k_i, v_i, dropout_p=0.0) - output_i = rearrange(output_i, "b h s d -> b s h d ") + output_i = einops.rearrange(output_i, "b h s d -> b s h d ") outputs.append(output_i) context_layer = torch.cat(outputs, dim=1) - context_layer = rearrange( + context_layer = einops.rearrange( context_layer, "b s h d -> s b (h d)" ).contiguous() elif self.attn_backend == _Backend.XFORMERS: - from xformers import ops as xops - from xformers.ops.fmha.attn_bias import BlockDiagonalMask - - attn_bias = BlockDiagonalMask.from_seqlens( - q_seqlen=seqlens, kv_seqlen=None, device=q.device - ) - - context_layer = xops.memory_efficient_attention_forward( - q, k, v, attn_bias=attn_bias, p=0, scale=None - ) - context_layer = rearrange( - context_layer, "b s h d -> s b (h d)" - ).contiguous() + context_layer = vit_xformers_attn_wrapper(q, k, v, seqlens) output, _ = self.proj(context_layer) return output +@support_torch_compile( + dynamic_arg_dims={ + "x": 0, + "cu_seqlens": 0, + "rotary_pos_emb": 0, + "seqlens": 0, + }, + mark_unbacked_dims={"seqlens": 0}, +) class Qwen2_5_VisionBlock(nn.Module): def __init__( self, @@ -515,8 +505,8 @@ class Qwen2_5_VisionBlock(nn.Module): x: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor, - max_seqlen: int | None = None, # Only used for Flash Attention - seqlens: list[int] | None = None, # Only used for xFormers + max_seqlen: torch.Tensor, # Only used for Flash Attention + seqlens: torch.Tensor, # Only used for xFormers ) -> torch.Tensor: x_attn = self.attn( self.norm1(x), @@ -530,6 +520,11 @@ class Qwen2_5_VisionBlock(nn.Module): return x +@support_torch_compile( + dynamic_arg_dims={ + "x": 0, + } +) class Qwen2_5_VisionPatchEmbed(nn.Module): def __init__( self, @@ -556,6 +551,11 @@ class Qwen2_5_VisionPatchEmbed(nn.Module): return x +@support_torch_compile( + dynamic_arg_dims={ + "x": 0, + } +) class Qwen2_5_VisionPatchMerger(nn.Module): def __init__( self, @@ -665,13 +665,18 @@ class Qwen2_5_VisionTransformer(nn.Module): self.spatial_merge_size = vision_config.spatial_merge_size self.fullatt_block_indexes = vision_config.fullatt_block_indexes self.spatial_merge_unit = self.spatial_merge_size**2 + # TODO[@lucaskabela]: Investigate fixing this usage + # see https://github.com/vllm-project/vllm/issues/27044 + # DO NOT MOVE THIS IMPORT + from vllm.compilation.backends import set_model_tag - self.patch_embed = Qwen2_5_VisionPatchEmbed( - patch_size=patch_size, - temporal_patch_size=temporal_patch_size, - in_channels=in_channels, - hidden_size=self.hidden_size, - ) + with set_model_tag("Qwen2_5_VisionPatchEmbed"): + self.patch_embed = Qwen2_5_VisionPatchEmbed( + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + in_channels=in_channels, + hidden_size=self.hidden_size, + ) norm_layer = partial(RMSNorm, eps=norm_eps) head_dim = self.hidden_size // self.num_heads @@ -701,32 +706,35 @@ class Qwen2_5_VisionTransformer(nn.Module): f"Qwen2.5-VL does not support {self.attn_backend} backend now." ) - self.blocks = nn.ModuleList( - [ - Qwen2_5_VisionBlock( - dim=self.hidden_size, - num_heads=self.num_heads, - mlp_hidden_dim=vision_config.intermediate_size, - act_fn=get_act_and_mul_fn(vision_config.hidden_act), - norm_layer=norm_layer, - quant_config=quant_config, - prefix=f"{prefix}.blocks.{layer_idx}", - use_data_parallel=use_data_parallel, - attn_backend=self.attn_backend, - use_upstream_fa=use_upstream_fa, - ) - for layer_idx in range(depth) - ] - ) - self.merger = Qwen2_5_VisionPatchMerger( - d_model=vision_config.out_hidden_size, - context_dim=self.hidden_size, - norm_layer=norm_layer, - spatial_merge_size=self.spatial_merge_size, - quant_config=quant_config, - prefix=f"{prefix}.merger", - use_data_parallel=use_data_parallel, - ) + with set_model_tag("Qwen2_5_VisionBlock"): + self.blocks = nn.ModuleList( + [ + Qwen2_5_VisionBlock( + dim=self.hidden_size, + num_heads=self.num_heads, + mlp_hidden_dim=vision_config.intermediate_size, + act_fn=get_act_and_mul_fn(vision_config.hidden_act), + norm_layer=norm_layer, + quant_config=quant_config, + prefix=f"{prefix}.blocks.{layer_idx}", + use_data_parallel=use_data_parallel, + attn_backend=self.attn_backend, + use_upstream_fa=use_upstream_fa, + ) + for layer_idx in range(depth) + ] + ) + + with set_model_tag("Qwen2_5_VisionPatchMerger"): + self.merger = Qwen2_5_VisionPatchMerger( + d_model=vision_config.out_hidden_size, + context_dim=self.hidden_size, + norm_layer=norm_layer, + spatial_merge_size=self.spatial_merge_size, + quant_config=quant_config, + prefix=f"{prefix}.merger", + use_data_parallel=use_data_parallel, + ) @property def dtype(self) -> torch.dtype: @@ -827,15 +835,18 @@ class Qwen2_5_VisionTransformer(nn.Module): def compute_attn_mask_seqlen( self, cu_seqlens: torch.Tensor, - ) -> tuple[int | None, list[int] | None]: - max_seqlen, seqlens = None, None + ) -> tuple[torch.Tensor, torch.Tensor]: + max_seqlen, seqlens = ( + torch.zeros(1, device=cu_seqlens.device), + torch.zeros(1, device=cu_seqlens.device), + ) if ( self.attn_backend == _Backend.FLASH_ATTN or self.attn_backend == _Backend.ROCM_AITER_FA ): - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() elif self.attn_backend == _Backend.XFORMERS: - seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + seqlens = cu_seqlens[1:] - cu_seqlens[:-1] return max_seqlen, seqlens @staticmethod @@ -1233,6 +1244,7 @@ class Qwen2_5_VLForConditionalGeneration( self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config + self.vllm_config = vllm_config self.multimodal_config = multimodal_config self.video_pruning_rate = multimodal_config.video_pruning_rate self.is_multimodal_pruning_enabled = ( @@ -1248,7 +1260,7 @@ class Qwen2_5_VLForConditionalGeneration( else None ) self.visual = Qwen2_5_VisionTransformer( - config.vision_config, + vision_config=config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=self.quant_config, prefix=maybe_prefix(prefix, "visual"), @@ -1336,13 +1348,13 @@ class Qwen2_5_VLForConditionalGeneration( image_embeds = image_input["image_embeds"].type(self.visual.dtype) else: pixel_values = image_input["pixel_values"] - - if self.use_data_parallel: - return run_dp_sharded_mrope_vision_model( - self.visual, pixel_values, grid_thw_list, rope_type="rope_3d" - ) - else: - image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list) + with set_forward_context(None, self.vllm_config): + if self.use_data_parallel: + return run_dp_sharded_mrope_vision_model( + self.visual, pixel_values, grid_thw_list, rope_type="rope_3d" + ) + else: + image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list) # Split concatenated embeddings for each image item. # Using prod on grid_thw_list instead of grid_thw.prod avoids CUDA sync @@ -1396,12 +1408,18 @@ class Qwen2_5_VLForConditionalGeneration( video_embeds = video_input["video_embeds"].type(self.visual.dtype) else: pixel_values_videos = video_input["pixel_values_videos"] - if self.use_data_parallel: - return run_dp_sharded_mrope_vision_model( - self.visual, pixel_values_videos, grid_thw_list, rope_type="rope_3d" - ) - else: - video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw_list) + with set_forward_context(None, self.vllm_config): + if self.use_data_parallel: + return run_dp_sharded_mrope_vision_model( + self.visual, + pixel_values_videos, + grid_thw_list, + rope_type="rope_3d", + ) + else: + video_embeds = self.visual( + pixel_values_videos, grid_thw=grid_thw_list + ) # Split concatenated embeddings for each video item. merge_size = self.visual.spatial_merge_size From d3ab240f39219df0175ec662416f630d7bf273d8 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 28 Oct 2025 19:53:12 -0400 Subject: [PATCH 050/127] [Bug] Fix deepep low latency use nvlink by default (#27677) Signed-off-by: yewentao256 --- vllm/envs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 73bb2678ea85e..018af0e5bba8f 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -205,7 +205,7 @@ if TYPE_CHECKING: VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME: str = "VLLM_OBJECT_STORAGE_SHM_BUFFER" VLLM_DEEPEP_BUFFER_SIZE_MB: int = 1024 VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE: bool = False - VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK: bool = False + VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK: bool = True VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL: bool = False VLLM_DBO_COMM_SMS: int = 20 GPT_OSS_SYSTEM_TOOL_MCP_LABELS: list[str] = [] @@ -1362,7 +1362,7 @@ environment_variables: dict[str, Callable[[], Any]] = { # Allow DeepEP to use nvlink for internode_ll kernel, turn this on for # better latency on GB200 like system "VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK": lambda: bool( - int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK", "0")) + int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK", "1")) ), # Allow DeepEP to use MNNVL (multi-node nvlink) for internode_ll kernel, # turn this for better latency on GB200 like system From 0b51c9bd8b19cee3a494b0f966a6b0a846a40193 Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Tue, 28 Oct 2025 18:32:33 -0700 Subject: [PATCH 051/127] [Core] Early return in SlidingWindowManager.remove_skipped_blocks (#27673) Signed-off-by: Jialin Ouyang --- vllm/v1/core/single_type_kv_cache_manager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 6699fb9818cb7..575ae3d7d83b6 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -394,7 +394,13 @@ class SlidingWindowManager(SingleTypeKVCacheManager): # skipped during the attention computation. last_useful_token = num_computed_tokens - self.sliding_window + 1 last_useful_block = last_useful_token // self.block_size + if last_useful_block <= 0: + # Early return if tokens are not enough to fill the sliding window + return blocks = self.req_to_blocks[request_id] + if blocks[last_useful_block - 1] == self._null_block: + # Early return if there are no blocks to remove + return removed_blocks: list[KVCacheBlock] = [] for i in range(last_useful_block - 1, -1, -1): if blocks[i] == self._null_block: From f257544709a8d9ccb8947e6f2c1779988c448ae7 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 28 Oct 2025 19:39:15 -0700 Subject: [PATCH 052/127] Install pre-built xformers-0.0.32.post2 built with pt-2.9.0 (#27598) Signed-off-by: Huy Do Co-authored-by: Roger Wang --- docker/Dockerfile | 7 ------- requirements/cuda.txt | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index eb1453126e6f4..c5b729e03b177 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -361,13 +361,6 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist && uv pip install --system dist/*.whl --verbose \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') -# TODO (huydhn): Remove this once xformers is released for 2.9.0 -RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH' - . /etc/environment - export TORCH_CUDA_ARCH_LIST='7.5 8.0+PTX 9.0a' - uv pip install --system --no-build-isolation "git+https://github.com/facebookresearch/xformers@v0.0.32.post2" -BASH - # Install FlashInfer pre-compiled kernel cache and binaries # https://docs.flashinfer.ai/installation.html RUN --mount=type=cache,target=/root/.cache/uv \ diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 7c5bc457d45b0..9636e5b4b8015 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -10,7 +10,7 @@ torchaudio==2.9.0 # These must be updated alongside torch torchvision==0.24.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version # https://github.com/facebookresearch/xformers/releases/tag/v0.0.32.post1 -# xformers==0.0.32.post1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.8 +xformers==0.0.33+5d4b92a5.d20251026; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.9 # FlashInfer should be updated together with the Dockerfile flashinfer-python==0.4.1 # Triton Kernels are needed for mxfp4 fused moe. (Should be updated alongside torch) From 9007bf57e6a25e46c0c2408e2510d6b897400895 Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Tue, 28 Oct 2025 20:58:01 -0700 Subject: [PATCH 053/127] Revert "Install pre-built xformers-0.0.32.post2 built with pt-2.9.0" (#27714) --- docker/Dockerfile | 7 +++++++ requirements/cuda.txt | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c5b729e03b177..eb1453126e6f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -361,6 +361,13 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist && uv pip install --system dist/*.whl --verbose \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') +# TODO (huydhn): Remove this once xformers is released for 2.9.0 +RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH' + . /etc/environment + export TORCH_CUDA_ARCH_LIST='7.5 8.0+PTX 9.0a' + uv pip install --system --no-build-isolation "git+https://github.com/facebookresearch/xformers@v0.0.32.post2" +BASH + # Install FlashInfer pre-compiled kernel cache and binaries # https://docs.flashinfer.ai/installation.html RUN --mount=type=cache,target=/root/.cache/uv \ diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 9636e5b4b8015..7c5bc457d45b0 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -10,7 +10,7 @@ torchaudio==2.9.0 # These must be updated alongside torch torchvision==0.24.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version # https://github.com/facebookresearch/xformers/releases/tag/v0.0.32.post1 -xformers==0.0.33+5d4b92a5.d20251026; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.9 +# xformers==0.0.32.post1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.8 # FlashInfer should be updated together with the Dockerfile flashinfer-python==0.4.1 # Triton Kernels are needed for mxfp4 fused moe. (Should be updated alongside torch) From f6d5f5888cce05b4d069e2d26503934ddf6e25d4 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Wed, 29 Oct 2025 00:07:09 -0400 Subject: [PATCH 054/127] [Build] Revert triton_kernels requirements (#27659) --- requirements/cuda.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 7c5bc457d45b0..dd45eb832a96a 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -13,5 +13,3 @@ torchvision==0.24.0 # Required for phi3v processor. See https://github.com/pytor # xformers==0.0.32.post1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.8 # FlashInfer should be updated together with the Dockerfile flashinfer-python==0.4.1 -# Triton Kernels are needed for mxfp4 fused moe. (Should be updated alongside torch) -triton_kernels @ git+https://github.com/triton-lang/triton.git@v3.5.0#subdirectory=python/triton_kernels From d2c33c397ad30f0b0fad7296a3c80d47df0243fe Mon Sep 17 00:00:00 2001 From: liuzhenwei Date: Wed, 29 Oct 2025 12:43:29 +0800 Subject: [PATCH 055/127] [NIXL][XPU] update name of nixl wheel (#27631) Signed-off-by: zhenwei-intel --- tools/install_nixl_from_source_ubuntu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/install_nixl_from_source_ubuntu.py b/tools/install_nixl_from_source_ubuntu.py index c808b01d2e94b..742aab6b0de75 100644 --- a/tools/install_nixl_from_source_ubuntu.py +++ b/tools/install_nixl_from_source_ubuntu.py @@ -37,7 +37,7 @@ def is_pip_package_installed(package_name): def find_nixl_wheel_in_cache(cache_dir): """Finds a nixl wheel file in the specified cache directory.""" # The repaired wheel will have a 'manylinux' tag, but this glob still works. - search_pattern = os.path.join(cache_dir, "nixl-*.whl") + search_pattern = os.path.join(cache_dir, "nixl*.whl") wheels = glob.glob(search_pattern) if wheels: # Sort to get the most recent/highest version if multiple exist From 0d8161b075044168a8d6528ec4ba5ba81ee72ec3 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Wed, 29 Oct 2025 05:28:20 +0000 Subject: [PATCH 056/127] [Model] Fix Qwen3VL and Qwen3Omni after torch.compile changes (#27705) Signed-off-by: Lukas Geiger Signed-off-by: Roger Wang Co-authored-by: Roger Wang --- vllm/model_executor/models/qwen2_5_vl.py | 6 ++---- .../models/qwen3_omni_moe_thinker.py | 14 ++++++++------ vllm/model_executor/models/qwen3_vl.py | 13 +++++++------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 30e3d2dff97b1..c68115729c425 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -836,10 +836,8 @@ class Qwen2_5_VisionTransformer(nn.Module): self, cu_seqlens: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - max_seqlen, seqlens = ( - torch.zeros(1, device=cu_seqlens.device), - torch.zeros(1, device=cu_seqlens.device), - ) + max_seqlen = torch.zeros([], device=cu_seqlens.device) + seqlens = torch.zeros(1, device=cu_seqlens.device) if ( self.attn_backend == _Backend.FLASH_ATTN or self.attn_backend == _Backend.ROCM_AITER_FA diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index f3b6ad495db42..efcd003fbbda7 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -223,8 +223,8 @@ class Qwen3_VisionBlock(nn.Module): x: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor, - max_seqlen: int | None = None, # Only used for Flash Attention - seqlens: list[int] | None = None, # Only used for xFormers + max_seqlen: torch.Tensor, # Only used for Flash Attention + seqlens: torch.Tensor, # Only used for xFormers ) -> torch.Tensor: x = x + self.attn( self.norm1(x), @@ -488,12 +488,13 @@ class Qwen3Omni_VisionTransformer(nn.Module): def compute_attn_mask_seqlen( self, cu_seqlens: torch.Tensor, - ) -> tuple[int | None, list[int] | None]: - max_seqlen, seqlens = None, None + ) -> tuple[torch.Tensor, torch.Tensor]: + max_seqlen = torch.zeros([], device=cu_seqlens.device) + seqlens = torch.zeros(1, device=cu_seqlens.device) if self.attn_backend == _Backend.FLASH_ATTN: - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() elif self.attn_backend == _Backend.XFORMERS: - seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + seqlens = cu_seqlens[1:] - cu_seqlens[:-1] return max_seqlen, seqlens def forward( @@ -1114,6 +1115,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + self.vllm_config = vllm_config # needed for torch compile forward context thinker_config: Qwen3OmniMoeThinkerConfig = ( vllm_config.model_config.hf_config.thinker_config ) diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 10c0eb4eb65ea..d611580c71821 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -231,8 +231,8 @@ class Qwen3_VisionBlock(nn.Module): x: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor, - max_seqlen: int | None = None, # Only used for Flash Attention - seqlens: list[int] | None = None, # Only used for xFormers + max_seqlen: torch.Tensor, # Only used for Flash Attention + seqlens: torch.Tensor, # Only used for xFormers ) -> torch.Tensor: x = x + self.attn( self.norm1(x), @@ -512,15 +512,16 @@ class Qwen3_VisionTransformer(nn.Module): def compute_attn_mask_seqlen( self, cu_seqlens: torch.Tensor, - ) -> tuple[int | None, list[int] | None]: - max_seqlen, seqlens = None, None + ) -> tuple[torch.Tensor, torch.Tensor]: + max_seqlen = torch.zeros([], device=cu_seqlens.device) + seqlens = torch.zeros(1, device=cu_seqlens.device) if ( self.attn_backend == _Backend.FLASH_ATTN or self.attn_backend == _Backend.ROCM_AITER_FA ): - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() elif self.attn_backend == _Backend.XFORMERS: - seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + seqlens = cu_seqlens[1:] - cu_seqlens[:-1] return max_seqlen, seqlens def forward( From a4a4f0f61790895f376afbc35ab41e18a796d52a Mon Sep 17 00:00:00 2001 From: Shaoting Date: Tue, 28 Oct 2025 22:38:37 -0700 Subject: [PATCH 057/127] [KV Connector] Update lmcache connector with latest compatibility (#27681) Signed-off-by: Samuel Shen Co-authored-by: Samuel Shen --- .../v1/lmcache_integration/vllm_v1_adapter.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py index 3f60fbd6455a2..ad907c75a244b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py @@ -44,8 +44,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.lmcache_integration.utils impo ) from vllm.distributed.parallel_state import get_tensor_model_parallel_rank, get_tp_group from vllm.sampling_params import SamplingParams -from vllm.utils import get_kv_cache_torch_dtype from vllm.utils.math_utils import cdiv +from vllm.utils.torch_utils import get_kv_cache_torch_dtype from vllm.v1.core.sched.output import SchedulerOutput from vllm.version import __version__ as VLLM_VERSION @@ -389,7 +389,7 @@ class ReqMeta: def need_gpu_interm_buffer(lmcache_config: LMCacheEngineConfig): - return lmcache_config.enable_pd + return not lmcache_config.enable_pd def _calculate_mtp_layers(vllm_config, model_config): @@ -403,6 +403,20 @@ def _calculate_mtp_layers(vllm_config, model_config): num_mtp_layers = getattr( model_config.hf_config, "num_nextn_predict_layers", 0 ) + + elif vllm_config.speculative_config.use_eagle(): + try: + draft_model_config = vllm_config.speculative_config.draft_model_config + num_mtp_layers = draft_model_config.get_num_layers( + vllm_config.parallel_config + ) + logger.info("EAGLE detected %d extra layer(s)", num_mtp_layers) + except Exception: + logger.info( + "EAGLE detected, but failed to get the number of extra layers" + "falling back to 1" + ) + num_mtp_layers = 1 return num_mtp_layers @@ -1208,6 +1222,10 @@ class LMCacheConnectorV1Impl: if the CacheManager this allocated blocks for us. """ + # Clear local status in lookup client when a new request is + # successfully scheduled. + self.lookup_client.clear_lookup_status(request.request_id) + kv_transfer_params = ( request.kv_transfer_params if hasattr(request, "kv_transfer_params") From 83fd49b1fc6d3af6a4c77ef8bfa6451c39402239 Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Tue, 28 Oct 2025 23:27:30 -0700 Subject: [PATCH 058/127] [CI/Build][Bugfix]Fix Quantized Models Test on AMD (#27712) Signed-off-by: zhewenli --- .buildkite/test-amd.yaml | 2 +- tests/models/quantization/test_bitsandbytes.py | 6 ++++++ vllm/platforms/rocm.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 524d2e121a10f..dceec159a9daf 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -908,7 +908,7 @@ steps: - label: Quantized Models Test # 45 min timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental] + mirror_hardwares: [amdexperimental, amdproduction] agent_pool: mi325_1 # grade: Blocking source_file_dependencies: diff --git a/tests/models/quantization/test_bitsandbytes.py b/tests/models/quantization/test_bitsandbytes.py index 5e0421af1c17b..24220978534ca 100644 --- a/tests/models/quantization/test_bitsandbytes.py +++ b/tests/models/quantization/test_bitsandbytes.py @@ -9,10 +9,16 @@ import pytest from transformers import BitsAndBytesConfig from tests.quantization.utils import is_quant_method_supported +from vllm.platforms import current_platform from ...utils import compare_two_settings, multi_gpu_test from ..utils import check_embeddings_close, check_logprobs_close +pytestmark = pytest.mark.skipif( + current_platform.is_rocm(), + reason="bitsandbytes quantization not supported on ROCm (CUDA-only kernels)", +) + models_4bit_to_test = [ ("facebook/opt-125m", "quantize opt model inflight"), ( diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 059ed4430e367..e67a7a7e70f7d 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -413,7 +413,7 @@ class RocmPlatform(Platform): "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ" " is not set, enabling VLLM_USE_TRITON_AWQ." ) - envs.VLLM_USE_TRITON_AWQ = True + os.environ["VLLM_USE_TRITON_AWQ"] = "1" @classmethod def get_punica_wrapper(cls) -> str: From 8b62495076fde4a73c4a397f5307f49c93dd7c6e Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Wed, 29 Oct 2025 00:00:15 -0700 Subject: [PATCH 059/127] [Bugfix] Fix non-contiguous tensor error in `rocm_unquantized_gemm_impl` (#27605) Signed-off-by: zhewenli --- .buildkite/test-amd.yaml | 2 +- vllm/model_executor/layers/utils.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index dceec159a9daf..0a7767b8ccc75 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -286,7 +286,7 @@ steps: - label: Engine Test # 25min timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental] + mirror_hardwares: [amdexperimental, amdproduction] agent_pool: mi325_1 #grade: Blocking source_file_dependencies: diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index da5eea02d120d..925f9ac0a16ea 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -119,17 +119,17 @@ def rocm_unquantized_gemm_impl( if use_skinny is not True: return torch.nn.functional.linear(x, weight, bias) - x_view = x.view(-1, x.size(-1)) + x_view = x.reshape(-1, x.size(-1)) n = x_view.shape[0] m = weight.shape[0] cu_count = current_platform.get_cu_count() if m > 8 and 0 < n <= 4: out = ops.wvSplitK(weight, x_view, cu_count, bias) - return out.view(*x.shape[:-1], weight.shape[0]) + return out.reshape(*x.shape[:-1], weight.shape[0]) elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None: out = ops.LLMM1(weight, x_view, 4) - return out.view(*x.shape[:-1], weight.shape[0]) + return out.reshape(*x.shape[:-1], weight.shape[0]) return torch.nn.functional.linear(x, weight, bias) From 413ef7a3b4d8722b8677f15e2320604a5a3d3b69 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Wed, 29 Oct 2025 03:54:21 -0400 Subject: [PATCH 060/127] [Speculators] Move tests + fix integration (#27308) Signed-off-by: Dipika Sikka Signed-off-by: Rahul Tuli Signed-off-by: rahul-tuli Co-authored-by: Rahul Tuli Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- tests/v1/e2e/test_spec_decode.py | 80 +++++++++++++++++++ .../spec_decode/test_speculators_eagle3.py} | 4 - vllm/engine/arg_utils.py | 28 ++++--- 3 files changed, 97 insertions(+), 15 deletions(-) rename tests/{speculative_decoding/speculators/test_eagle3.py => v1/spec_decode/test_speculators_eagle3.py} (94%) diff --git a/tests/v1/e2e/test_spec_decode.py b/tests/v1/e2e/test_spec_decode.py index 7dbdf0ca07105..45b48e5858934 100644 --- a/tests/v1/e2e/test_spec_decode.py +++ b/tests/v1/e2e/test_spec_decode.py @@ -121,6 +121,86 @@ def test_ngram_correctness( cleanup_dist_env_and_memory() +@pytest.mark.parametrize( + "model_path", + [ + "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3", + "RedHatAI/Qwen3-8B-speculator.eagle3", + ], + ids=["llama3_eagle3_speculator", "qwen3_eagle3_speculator"], +) +def test_speculators_model_integration( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_path: str, +): + """ + Test that speculators models work with the simplified integration. + + This verifies the `vllm serve ` use case where + speculative config is automatically detected from the model config + without requiring explicit --speculative-config argument. + + Tests: + 1. Speculator model is correctly detected + 2. Verifier model is extracted from speculator config + 3. Speculative decoding is automatically enabled + 4. Text generation works correctly + 5. Output matches reference (non-speculative) generation + """ + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + # Generate test prompts + test_prompts = get_test_prompts(mm_enabled=False) + + # First run: Direct speculator model (simplified integration) + spec_llm = LLM(model=model_path, max_model_len=1024) + spec_outputs = spec_llm.chat(test_prompts, sampling_config) + + # Verify speculative config was auto-detected + assert spec_llm.llm_engine.vllm_config.speculative_config is not None, ( + f"Speculative config should be auto-detected for {model_path}" + ) + + spec_config = spec_llm.llm_engine.vllm_config.speculative_config + assert spec_config.num_speculative_tokens > 0, ( + f"Expected positive speculative tokens, " + f"got {spec_config.num_speculative_tokens}" + ) + + # Verify draft model is set to the speculator model + assert spec_config.model == model_path, ( + f"Draft model should be {model_path}, got {spec_config.model}" + ) + + # Extract verifier model for reference run + verifier_model = spec_llm.llm_engine.vllm_config.model_config.model + + del spec_llm + torch.cuda.empty_cache() + cleanup_dist_env_and_memory() + + # Second run: Reference without speculative decoding + ref_llm = LLM(model=verifier_model, max_model_len=1024) + ref_outputs = ref_llm.chat(test_prompts, sampling_config) + del ref_llm + torch.cuda.empty_cache() + cleanup_dist_env_and_memory() + + # Compare outputs + matches = sum( + 1 + for ref, spec in zip(ref_outputs, spec_outputs) + if ref.outputs[0].text == spec.outputs[0].text + ) + + # Heuristic: expect at least 66% of prompts to match exactly + assert matches >= int(0.66 * len(ref_outputs)), ( + f"Only {matches}/{len(ref_outputs)} outputs matched. " + f"Expected at least {int(0.66 * len(ref_outputs))} matches." + ) + + @pytest.mark.parametrize( ["model_setup", "mm_enabled"], [ diff --git a/tests/speculative_decoding/speculators/test_eagle3.py b/tests/v1/spec_decode/test_speculators_eagle3.py similarity index 94% rename from tests/speculative_decoding/speculators/test_eagle3.py rename to tests/v1/spec_decode/test_speculators_eagle3.py index 19ba32d8dee4c..5ce6e1593b5c1 100644 --- a/tests/speculative_decoding/speculators/test_eagle3.py +++ b/tests/v1/spec_decode/test_speculators_eagle3.py @@ -22,10 +22,6 @@ from vllm.model_executor.models.interfaces import supports_eagle3 "nm-testing/Speculator-Qwen3-8B-Eagle3-converted-071-quantized-w4a16", id="qwen3-eagle3-speculator-w4a16-verifier", ), - pytest.param( - "nm-testing/random-weights-llama3.1.8b-2layer-eagle3", - id="llama3-eagl3-multiple-layers", - ), ], ) def test_eagle3_speculators_model( diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index e8f8e3f8c2b51..f13ce935ec4b6 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -81,7 +81,7 @@ from vllm.transformers_utils.config import ( is_interleaved, maybe_override_with_speculators, ) -from vllm.transformers_utils.utils import check_gguf_file +from vllm.transformers_utils.utils import check_gguf_file, is_s3 from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.mem_constants import GiB_bytes from vllm.utils.network_utils import get_ip @@ -1305,20 +1305,26 @@ class EngineArgs: device_config = DeviceConfig(device=cast(Device, current_platform.device_type)) + # Check if the model is a speculator and override model/tokenizer/config + # BEFORE creating ModelConfig, so the config is created with the target model + # Skip speculator detection for S3 models since HuggingFace cannot load + # configs directly from S3 URLs. S3 models can still use speculators with + # explicit --speculative-config. + if not is_s3(self.model): + (self.model, self.tokenizer, self.speculative_config) = ( + maybe_override_with_speculators( + model=self.model, + tokenizer=self.tokenizer, + revision=self.revision, + trust_remote_code=self.trust_remote_code, + vllm_speculative_config=self.speculative_config, + ) + ) + model_config = self.create_model_config() self.model = model_config.model self.tokenizer = model_config.tokenizer - (self.model, self.tokenizer, self.speculative_config) = ( - maybe_override_with_speculators( - model=self.model, - tokenizer=self.tokenizer, - revision=self.revision, - trust_remote_code=self.trust_remote_code, - vllm_speculative_config=self.speculative_config, - ) - ) - # * If VLLM_USE_V1 is unset, we enable V1 for "supported features" # and fall back to V0 for experimental or unsupported features. # * If VLLM_USE_V1=1, we enable V1 for supported + experimental From 4fb8771cc0bb8b0127d7b88d790a8ed38d9b39cc Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Wed, 29 Oct 2025 16:04:33 +0800 Subject: [PATCH 061/127] [CI/Build] Move pre-commit only scripts to `tools/pre_commit` (#27657) Signed-off-by: DarkLight1337 --- .buildkite/test-amd.yaml | 2 +- .buildkite/test-pipeline.yaml | 2 +- .pre-commit-config.yaml | 18 +++++++++--------- .../pytorch_nightly_dependency.sh | 2 +- tests/tools/test_config_validator.py | 2 +- .../check_init_lazy_imports.py | 5 ++--- tools/{ => pre_commit}/check_spdx_header.py | 0 tools/{ => pre_commit}/check_triton_import.py | 0 tools/{ => pre_commit}/enforce_regex_import.py | 0 .../generate_nightly_torch_test.py | 0 tools/{ => pre_commit}/png-lint.sh | 0 tools/{ => pre_commit}/shellcheck.sh | 0 .../update-dockerfile-graph.sh | 0 tools/{ => pre_commit}/validate_config.py | 0 vllm/config/utils.py | 2 +- 15 files changed, 16 insertions(+), 17 deletions(-) rename tools/{ => pre_commit}/check_init_lazy_imports.py (96%) rename tools/{ => pre_commit}/check_spdx_header.py (100%) rename tools/{ => pre_commit}/check_triton_import.py (100%) rename tools/{ => pre_commit}/enforce_regex_import.py (100%) rename tools/{ => pre_commit}/generate_nightly_torch_test.py (100%) rename tools/{ => pre_commit}/png-lint.sh (100%) rename tools/{ => pre_commit}/shellcheck.sh (100%) rename tools/{ => pre_commit}/update-dockerfile-graph.sh (100%) rename tools/{ => pre_commit}/validate_config.py (100%) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 0a7767b8ccc75..56e7b1083b17e 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -38,7 +38,7 @@ steps: - label: Pytorch Nightly Dependency Override Check # 2min # if this test fails, it means the nightly torch version is not compatible with some # of the dependencies. Please check the error message and add the package to whitelist - # in /vllm/tools/generate_nightly_torch_test.py + # in /vllm/tools/pre_commit/generate_nightly_torch_test.py mirror_hardwares: [amdexperimental] agent_pool: mi325_1 # grade: Blocking diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 03268beecfc0b..16f87e98afb7f 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -38,7 +38,7 @@ steps: - label: Pytorch Nightly Dependency Override Check # 2min # if this test fails, it means the nightly torch version is not compatible with some # of the dependencies. Please check the error message and add the package to whitelist - # in /vllm/tools/generate_nightly_torch_test.py + # in /vllm/tools/pre_commit/generate_nightly_torch_test.py soft_fail: true source_file_dependencies: - requirements/nightly_torch_test.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fbfd8016cb76f..bcd40e7f8ab39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: - id: format-torch-nightly-test name: reformat nightly_torch_test.txt to be in sync with test.in language: python - entry: python tools/generate_nightly_torch_test.py + entry: python tools/pre_commit/generate_nightly_torch_test.py files: ^requirements/test\.(in|txt)$ - id: mypy-local name: Run mypy locally for lowest supported Python version @@ -78,12 +78,12 @@ repos: stages: [manual] # Only run in CI - id: shellcheck name: Lint shell scripts - entry: tools/shellcheck.sh + entry: tools/pre_commit/shellcheck.sh language: script types: [shell] - id: png-lint name: Lint PNG exports from excalidraw - entry: tools/png-lint.sh + entry: tools/pre_commit/png-lint.sh language: script types: [png] - id: signoff-commit @@ -100,12 +100,12 @@ repos: stages: [commit-msg] - id: check-spdx-header name: Check SPDX headers - entry: python tools/check_spdx_header.py + entry: python tools/pre_commit/check_spdx_header.py language: python types: [python] - id: check-root-lazy-imports name: Check root lazy imports - entry: python tools/check_init_lazy_imports.py + entry: python tools/pre_commit/check_init_lazy_imports.py language: python types: [python] - id: check-filenames @@ -119,11 +119,11 @@ repos: pass_filenames: false - id: update-dockerfile-graph name: Update Dockerfile dependency graph - entry: tools/update-dockerfile-graph.sh + entry: tools/pre_commit/update-dockerfile-graph.sh language: script - id: enforce-import-regex-instead-of-re name: Enforce import regex as re - entry: python tools/enforce_regex_import.py + entry: python tools/pre_commit/enforce_regex_import.py language: python types: [python] pass_filenames: false @@ -131,7 +131,7 @@ repos: # forbid directly import triton - id: forbid-direct-triton-import name: "Forbid direct 'import triton'" - entry: python tools/check_triton_import.py + entry: python tools/pre_commit/check_triton_import.py language: python types: [python] pass_filenames: false @@ -144,7 +144,7 @@ repos: additional_dependencies: [regex] - id: validate-config name: Validate configuration has default values and that each field has a docstring - entry: python tools/validate_config.py + entry: python tools/pre_commit/validate_config.py language: python additional_dependencies: [regex] # Keep `suggestion` last diff --git a/tests/standalone_tests/pytorch_nightly_dependency.sh b/tests/standalone_tests/pytorch_nightly_dependency.sh index cb531e13ecb81..fd93ad76bed0f 100644 --- a/tests/standalone_tests/pytorch_nightly_dependency.sh +++ b/tests/standalone_tests/pytorch_nightly_dependency.sh @@ -37,6 +37,6 @@ if diff before.txt after.txt; then else echo "torch version overridden by nightly_torch_test.txt, \ if the dependency is not triggered by the pytroch nightly test,\ - please add the dependency to the list 'white_list' in tools/generate_nightly_torch_test.py" + please add the dependency to the list 'white_list' in tools/pre_commit/generate_nightly_torch_test.py" exit 1 fi diff --git a/tests/tools/test_config_validator.py b/tests/tools/test_config_validator.py index 22d838d272643..d6104dc6d2eb1 100644 --- a/tests/tools/test_config_validator.py +++ b/tests/tools/test_config_validator.py @@ -5,7 +5,7 @@ import ast import pytest -from tools.validate_config import validate_ast +from tools.pre_commit.validate_config import validate_ast _TestConfig1 = """ @config diff --git a/tools/check_init_lazy_imports.py b/tools/pre_commit/check_init_lazy_imports.py similarity index 96% rename from tools/check_init_lazy_imports.py rename to tools/pre_commit/check_init_lazy_imports.py index 8b3a0b2a71be0..ab2ef8b3aa5ba 100644 --- a/tools/check_init_lazy_imports.py +++ b/tools/pre_commit/check_init_lazy_imports.py @@ -6,13 +6,12 @@ i.e: appears only within the `if typing.TYPE_CHECKING:` guard, """ import ast -import pathlib import sys from collections.abc import Iterable +from pathlib import Path from typing import Final -REPO_ROOT: Final = pathlib.Path(__file__).resolve().parent.parent -INIT_PATH: Final = REPO_ROOT / "vllm" / "__init__.py" +INIT_PATH: Final = Path("vllm/__init__.py") # If you need to add items to whitelist, do it here. ALLOWED_IMPORTS: Final[frozenset[str]] = frozenset( diff --git a/tools/check_spdx_header.py b/tools/pre_commit/check_spdx_header.py similarity index 100% rename from tools/check_spdx_header.py rename to tools/pre_commit/check_spdx_header.py diff --git a/tools/check_triton_import.py b/tools/pre_commit/check_triton_import.py similarity index 100% rename from tools/check_triton_import.py rename to tools/pre_commit/check_triton_import.py diff --git a/tools/enforce_regex_import.py b/tools/pre_commit/enforce_regex_import.py similarity index 100% rename from tools/enforce_regex_import.py rename to tools/pre_commit/enforce_regex_import.py diff --git a/tools/generate_nightly_torch_test.py b/tools/pre_commit/generate_nightly_torch_test.py similarity index 100% rename from tools/generate_nightly_torch_test.py rename to tools/pre_commit/generate_nightly_torch_test.py diff --git a/tools/png-lint.sh b/tools/pre_commit/png-lint.sh similarity index 100% rename from tools/png-lint.sh rename to tools/pre_commit/png-lint.sh diff --git a/tools/shellcheck.sh b/tools/pre_commit/shellcheck.sh similarity index 100% rename from tools/shellcheck.sh rename to tools/pre_commit/shellcheck.sh diff --git a/tools/update-dockerfile-graph.sh b/tools/pre_commit/update-dockerfile-graph.sh similarity index 100% rename from tools/update-dockerfile-graph.sh rename to tools/pre_commit/update-dockerfile-graph.sh diff --git a/tools/validate_config.py b/tools/pre_commit/validate_config.py similarity index 100% rename from tools/validate_config.py rename to tools/pre_commit/validate_config.py diff --git a/vllm/config/utils.py b/vllm/config/utils.py index 5e7e7580c5a9e..7e0878d96bbd6 100644 --- a/vllm/config/utils.py +++ b/vllm/config/utils.py @@ -33,7 +33,7 @@ def config(cls: ConfigT) -> ConfigT: `pydantic.TypeAdapter(ConfigT).validate_json(cli_arg)` which treats the `cli_arg` as a JSON string which gets validated by `pydantic`. - Config validation is performed by the tools/validate_config.py + Config validation is performed by the tools/pre_commit/validate_config.py script, which is invoked during the pre-commit checks. """ return cls From 8df98c2161e28387391b667201f8458c2bdf29f4 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Wed, 29 Oct 2025 16:12:54 +0800 Subject: [PATCH 062/127] [perf] Enable concurrent execution of "shared_experts" and "selected_experts" in qwen3-next (#27578) Signed-off-by: zjy0516 --- vllm/model_executor/models/qwen3_next.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index e81ad5f68d8f3..f452ba871582d 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -159,6 +159,7 @@ class Qwen3NextSparseMoeBlock(nn.Module): self.experts = SharedFusedMoE( shared_experts=self.shared_expert, + gate=self.gate, num_experts=self.n_routed_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, @@ -181,11 +182,17 @@ class Qwen3NextSparseMoeBlock(nn.Module): if self.is_sequence_parallel: hidden_states = sequence_parallel_chunk(hidden_states) - # router_logits: (num_tokens, n_experts) - router_logits, _ = self.gate(hidden_states) - final_hidden_states = self.experts( - hidden_states=hidden_states, router_logits=router_logits - ) + if self.experts.is_internal_router: + # In this case, the gate/router runs inside the FusedMoE class + final_hidden_states = self.experts( + hidden_states=hidden_states, router_logits=hidden_states + ) + else: + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts( + hidden_states=hidden_states, router_logits=router_logits + ) if self.shared_expert is not None: final_hidden_states = final_hidden_states[0] + final_hidden_states[1] From 1891cf605ac015016cca38acc9e1950f70586f2e Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Wed, 29 Oct 2025 04:14:33 -0400 Subject: [PATCH 063/127] [Bugfix] Fix modular kernel tests (#27707) Signed-off-by: Bill Nell --- .buildkite/test-pipeline.yaml | 2 ++ tests/kernels/moe/modular_kernel_tools/common.py | 1 + 2 files changed, 3 insertions(+) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 16f87e98afb7f..e166f320f9c3f 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -498,6 +498,8 @@ steps: - tests/kernels/moe - vllm/model_executor/layers/fused_moe/ - vllm/distributed/device_communicators/ + - vllm/envs.py + - vllm/config commands: - pytest -v -s kernels/moe --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT parallelism: 2 diff --git a/tests/kernels/moe/modular_kernel_tools/common.py b/tests/kernels/moe/modular_kernel_tools/common.py index c517e5c026b46..1d925dc1bea8f 100644 --- a/tests/kernels/moe/modular_kernel_tools/common.py +++ b/tests/kernels/moe/modular_kernel_tools/common.py @@ -138,6 +138,7 @@ class Config: } backend = self.all2all_backend() + vllm_config.parallel_config.all2all_backend = backend if backend is not None: env_dict.update({"VLLM_ALL2ALL_BACKEND": backend}) From 3c7fefdeba183e5c5e575f668b797549530f5a3d Mon Sep 17 00:00:00 2001 From: Alec S <10566873+alecsolder@users.noreply.github.com> Date: Wed, 29 Oct 2025 05:42:44 -0400 Subject: [PATCH 064/127] [Frontend] [gpt-oss] Tool json call parsing error retry (#27675) Signed-off-by: Jialin Ouyang Signed-off-by: Alec Solder Signed-off-by: Ye (Charlotte) Qi Co-authored-by: Jialin Ouyang Co-authored-by: Alec Solder Co-authored-by: Ye (Charlotte) Qi --- vllm/entrypoints/context.py | 39 +++++++++++++++++++++++++++++-- vllm/entrypoints/harmony_utils.py | 19 ++++++++++++++- vllm/envs.py | 7 ++++++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/vllm/entrypoints/context.py b/vllm/entrypoints/context.py index 8886d7c42d8a6..0041db822080a 100644 --- a/vllm/entrypoints/context.py +++ b/vllm/entrypoints/context.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Union from openai.types.responses.tool import Mcp from openai_harmony import Author, Message, Role, StreamState, TextContent +from vllm import envs from vllm.entrypoints.harmony_utils import ( get_encoding, get_streamable_parser_for_assistant, @@ -109,6 +110,28 @@ class ConversationContext(ABC): raise NotImplementedError("Should not be called.") +def _create_json_parse_error_messages( + last_msg: Message, e: json.JSONDecodeError +) -> list[Message]: + """ + Creates an error message when json parse failed. + """ + error_msg = ( + f"Error parsing tool arguments as JSON: {str(e)}. " + "Please ensure the tool call arguments are valid JSON and try again." + ) + content = TextContent(text=error_msg) + author = Author(role=Role.TOOL, name=last_msg.recipient) + return [ + Message( + author=author, + content=[content], + recipient=Role.ASSISTANT, + channel=last_msg.channel, + ) + ] + + class SimpleContext(ConversationContext): def __init__(self): self.last_output = None @@ -339,7 +362,13 @@ class HarmonyContext(ConversationContext): if isinstance(tool_session, Tool): return await tool_session.get_result(self) tool_name = last_msg.recipient.split(".")[1] - args = json.loads(last_msg.content[0].text) + if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: + try: + args = json.loads(last_msg.content[0].text) + except json.JSONDecodeError as e: + return _create_json_parse_error_messages(last_msg, e) + else: + args = json.loads(last_msg.content[0].text) result = await tool_session.call_tool(tool_name, args) result_str = result.content[0].text content = TextContent(text=result_str) @@ -420,7 +449,13 @@ class HarmonyContext(ConversationContext): if isinstance(tool_session, Tool): return await tool_session.get_result(self) tool_name = last_msg.recipient.split(".")[1].split(" ")[0] - args = json.loads(last_msg.content[0].text) + if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: + try: + args = json.loads(last_msg.content[0].text) + except json.JSONDecodeError as e: + return _create_json_parse_error_messages(last_msg, e) + else: + args = json.loads(last_msg.content[0].text) result = await tool_session.call_tool(tool_name, args) result_str = result.content[0].text content = TextContent(text=result_str) diff --git a/vllm/entrypoints/harmony_utils.py b/vllm/entrypoints/harmony_utils.py index 97f95a97ee304..8888a5aeb6b1a 100644 --- a/vllm/entrypoints/harmony_utils.py +++ b/vllm/entrypoints/harmony_utils.py @@ -340,7 +340,24 @@ def parse_output_message(message: Message) -> list[ResponseOutputItem]: if len(message.content) != 1: raise ValueError("Invalid number of contents in browser message") content = message.content[0] - browser_call = json.loads(content.text) + # We do not need to check the VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY + # env variable since if it is not set, we are certain the json is valid + # The use of Actions for web search will be removed entirely in + # the future, so this is only necessary temporarily + try: + browser_call = json.loads(content.text) + except json.JSONDecodeError: + # If the content is not valid JSON, then it was + # caught and retried by vLLM, which means we + # need to make note of that so the user is aware + json_retry_output_message = ( + f"Invalid JSON args, caught and retried: {content.text}" + ) + browser_call = { + "query": json_retry_output_message, + "url": json_retry_output_message, + "pattern": json_retry_output_message, + } # TODO: translate to url properly! if recipient == "browser.search": action = ActionSearch( diff --git a/vllm/envs.py b/vllm/envs.py index 018af0e5bba8f..0786d5d9ddcbd 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -199,6 +199,7 @@ if TYPE_CHECKING: VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False VLLM_TUNED_CONFIG_FOLDER: str | None = None VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False + VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True @@ -1331,6 +1332,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool( int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0")) ), + # Enable automatic retry when tool call JSON parsing fails + # If enabled, returns an error message to the model to retry + # If disabled (default), raises an exception and fails the request + "VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool( + int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0")) + ), # Add optional custom scopes for profiling, disable to avoid overheads "VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool( int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0")) From ab2eb27b74da0e6846749c11a68179d27a78d963 Mon Sep 17 00:00:00 2001 From: Alec S <10566873+alecsolder@users.noreply.github.com> Date: Wed, 29 Oct 2025 06:01:32 -0400 Subject: [PATCH 065/127] [Frontend] [gpt-oss] Mcp type bug (#27689) Signed-off-by: Jialin Ouyang Signed-off-by: Alec Solder Signed-off-by: Ye (Charlotte) Qi Co-authored-by: Jialin Ouyang Co-authored-by: Alec Solder Co-authored-by: Ye (Charlotte) Qi --- .../openai/test_response_api_mcp_tools.py | 87 +++++++++++--- .../openai/test_serving_responses.py | 50 +++++++- tests/entrypoints/test_harmony_utils.py | 14 ++- tests/test_envs.py | 108 ++++++++++++++++++ vllm/entrypoints/harmony_utils.py | 10 +- vllm/entrypoints/openai/serving_responses.py | 32 ++++-- vllm/envs.py | 36 ++++-- 7 files changed, 293 insertions(+), 44 deletions(-) diff --git a/tests/entrypoints/openai/test_response_api_mcp_tools.py b/tests/entrypoints/openai/test_response_api_mcp_tools.py index 653d44f20b440..0dc2430caef7c 100644 --- a/tests/entrypoints/openai/test_response_api_mcp_tools.py +++ b/tests/entrypoints/openai/test_response_api_mcp_tools.py @@ -26,6 +26,8 @@ def mcp_disabled_server(monkeypatch_module: pytest.MonkeyPatch): with monkeypatch_module.context() as m: m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1") m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv") + # Helps the model follow instructions better + m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1") with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server @@ -37,7 +39,9 @@ def mcp_enabled_server(monkeypatch_module: pytest.MonkeyPatch): with monkeypatch_module.context() as m: m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1") m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv") - m.setenv("GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container") + m.setenv("VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container") + # Helps the model follow instructions better + m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1") with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server @@ -56,18 +60,15 @@ async def mcp_enabled_client(mcp_enabled_server): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -@pytest.mark.skip(reason="Code interpreter tool is not available in CI yet.") async def test_mcp_tool_env_flag_enabled(mcp_enabled_client: OpenAI, model_name: str): response = await mcp_enabled_client.responses.create( model=model_name, - # TODO: Ideally should be able to set max tool calls - # to prevent multi-turn, but it is not currently supported - # would speed up the test input=( - "What's the first 4 digits after the decimal point of " - "cube root of `19910212 * 20250910`? " - "Show only the digits. The python interpreter is not stateful " - "and you must print to see the output." + "Execute the following code: " + "import random; print(random.randint(1, 1000000))" + ), + instructions=( + "You must use the Python tool to execute code. Never simulate execution." ), tools=[ { @@ -77,26 +78,47 @@ async def test_mcp_tool_env_flag_enabled(mcp_enabled_client: OpenAI, model_name: "server_url": "http://localhost:8888", } ], + extra_body={"enable_response_messages": True}, ) assert response is not None assert response.status == "completed" - assert response.usage.output_tokens_details.tool_output_tokens > 0 + # Verify output messages: Tool calls and responses on analysis channel + tool_call_found = False + tool_response_found = False + for message in response.output_messages: + recipient = message.get("recipient") + if recipient and recipient.startswith("python"): + tool_call_found = True + assert message.get("channel") == "analysis", ( + "Tool call should be on analysis channel" + ) + author = message.get("author", {}) + if ( + author.get("role") == "tool" + and author.get("name") + and author.get("name").startswith("python") + ): + tool_response_found = True + assert message.get("channel") == "analysis", ( + "Tool response should be on analysis channel" + ) + + assert tool_call_found, "Should have found at least one Python tool call" + assert tool_response_found, "Should have found at least one Python tool response" + for message in response.input_messages: + assert message.get("author").get("role") != "developer", ( + "No developer messages should be present with valid mcp tool" + ) @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -@pytest.mark.skip(reason="Code interpreter tool is not available in CI yet.") async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI, model_name: str): response = await mcp_disabled_client.responses.create( model=model_name, - # TODO: Ideally should be able to set max tool calls - # to prevent multi-turn, but it is not currently supported - # would speed up the test input=( - "What's the first 4 digits after the decimal point of " - "cube root of `19910212 * 20250910`? " - "Show only the digits. The python interpreter is not stateful " - "and you must print to see the output." + "Execute the following code if the tool is present: " + "import random; print(random.randint(1, 1000000))" ), tools=[ { @@ -106,7 +128,34 @@ async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI, model_nam "server_url": "http://localhost:8888", } ], + extra_body={"enable_response_messages": True}, ) assert response is not None assert response.status == "completed" - assert response.usage.output_tokens_details.tool_output_tokens == 0 + # Verify output messages: No tool calls and responses + tool_call_found = False + tool_response_found = False + for message in response.output_messages: + recipient = message.get("recipient") + if recipient and recipient.startswith("python"): + tool_call_found = True + assert message.get("channel") == "analysis", ( + "Tool call should be on analysis channel" + ) + author = message.get("author", {}) + if ( + author.get("role") == "tool" + and author.get("name") + and author.get("name").startswith("python") + ): + tool_response_found = True + assert message.get("channel") == "analysis", ( + "Tool response should be on analysis channel" + ) + + assert not tool_call_found, "Should not have a python call" + assert not tool_response_found, "Should not have a tool response" + for message in response.input_messages: + assert message.get("author").get("role") != "developer", ( + "No developer messages should be present without a valid tool" + ) diff --git a/tests/entrypoints/openai/test_serving_responses.py b/tests/entrypoints/openai/test_serving_responses.py index cf21a5116ddfb..788a1e9121825 100644 --- a/tests/entrypoints/openai/test_serving_responses.py +++ b/tests/entrypoints/openai/test_serving_responses.py @@ -6,10 +6,19 @@ from unittest.mock import MagicMock import pytest import pytest_asyncio +from openai.types.responses.tool import ( + CodeInterpreterContainerCodeInterpreterToolAuto, + LocalShell, + Mcp, + Tool, +) from vllm.entrypoints.context import ConversationContext from vllm.entrypoints.openai.protocol import ErrorResponse, ResponsesRequest -from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses +from vllm.entrypoints.openai.serving_responses import ( + OpenAIServingResponses, + extract_tool_types, +) from vllm.entrypoints.tool_server import ToolServer from vllm.inputs.data import TokensPrompt as EngineTokensPrompt @@ -62,6 +71,45 @@ def mock_exit_stack(): return MagicMock(spec=AsyncExitStack) +def test_extract_tool_types(monkeypatch: pytest.MonkeyPatch) -> None: + tools: list[Tool] = [] + assert extract_tool_types(tools) == set() + + tools.append(LocalShell(type="local_shell")) + assert extract_tool_types(tools) == {"local_shell"} + + tools.append(CodeInterpreterContainerCodeInterpreterToolAuto(type="auto")) + assert extract_tool_types(tools) == {"local_shell", "auto"} + + tools.extend( + [ + Mcp(type="mcp", server_label="random", server_url=""), + Mcp(type="mcp", server_label="container", server_url=""), + Mcp(type="mcp", server_label="code_interpreter", server_url=""), + Mcp(type="mcp", server_label="web_search_preview", server_url=""), + ] + ) + # When envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS is not set, + # mcp tool types are all ignored. + assert extract_tool_types(tools) == {"local_shell", "auto"} + + # container is allowed, it would be extracted + monkeypatch.setenv("VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "container") + assert extract_tool_types(tools) == {"local_shell", "auto", "container"} + + # code_interpreter and web_search_preview are allowed, + # they would be extracted + monkeypatch.setenv( + "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,web_search_preview" + ) + assert extract_tool_types(tools) == { + "local_shell", + "auto", + "code_interpreter", + "web_search_preview", + } + + class TestInitializeToolSessions: """Test class for _initialize_tool_sessions method""" diff --git a/tests/entrypoints/test_harmony_utils.py b/tests/entrypoints/test_harmony_utils.py index 8d1764d411572..6fa051a678d68 100644 --- a/tests/entrypoints/test_harmony_utils.py +++ b/tests/entrypoints/test_harmony_utils.py @@ -3,7 +3,10 @@ from openai_harmony import Role -from vllm.entrypoints.harmony_utils import parse_input_to_harmony_message +from vllm.entrypoints.harmony_utils import ( + has_custom_tools, + parse_input_to_harmony_message, +) class TestParseInputToHarmonyMessage: @@ -252,3 +255,12 @@ class TestParseInputToHarmonyMessage: assert len(messages[0].content) == 2 assert messages[0].content[0].text == "" assert messages[0].content[1].text == "actual text" + + +def test_has_custom_tools() -> None: + assert not has_custom_tools(set()) + assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"}) + assert has_custom_tools({"others"}) + assert has_custom_tools( + {"web_search_preview", "code_interpreter", "container", "others"} + ) diff --git a/tests/test_envs.py b/tests/test_envs.py index 023767505f108..841d7945f9120 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -10,6 +10,7 @@ import vllm.envs as envs from vllm.envs import ( enable_envs_cache, env_list_with_choices, + env_set_with_choices, env_with_choices, environment_variables, ) @@ -257,3 +258,110 @@ class TestEnvListWithChoices: with patch.dict(os.environ, {"TEST_ENV": "option1,option1,option2"}): env_func = env_list_with_choices("TEST_ENV", [], ["option1", "option2"]) assert env_func() == ["option1", "option1", "option2"] + + +class TestEnvSetWithChoices: + """Test cases for env_set_with_choices function.""" + + def test_default_list_returned_when_env_not_set(self): + """Test that default list is returned when env var is not set.""" + env_func = env_set_with_choices( + "NONEXISTENT_ENV", ["default1", "default2"], ["option1", "option2"] + ) + assert env_func() == {"default1", "default2"} + + def test_empty_default_list_returned_when_env_not_set(self): + """Test that empty default list is returned when env not set.""" + env_func = env_set_with_choices("NONEXISTENT_ENV", [], ["option1", "option2"]) + assert env_func() == set() + + def test_single_valid_value_parsed_correctly(self): + """Test that single valid value is parsed correctly.""" + with patch.dict(os.environ, {"TEST_ENV": "option1"}): + env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"]) + assert env_func() == {"option1"} + + def test_multiple_valid_values_parsed_correctly(self): + """Test that multiple valid values are parsed correctly.""" + with patch.dict(os.environ, {"TEST_ENV": "option1,option2"}): + env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"]) + assert env_func() == {"option1", "option2"} + + def test_values_with_whitespace_trimmed(self): + """Test that values with whitespace are trimmed correctly.""" + with patch.dict(os.environ, {"TEST_ENV": " option1 , option2 "}): + env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"]) + assert env_func() == {"option1", "option2"} + + def test_empty_values_filtered_out(self): + """Test that empty values are filtered out.""" + with patch.dict(os.environ, {"TEST_ENV": "option1,,option2,"}): + env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"]) + assert env_func() == {"option1", "option2"} + + def test_empty_string_returns_default(self): + """Test that empty string returns default.""" + with patch.dict(os.environ, {"TEST_ENV": ""}): + env_func = env_set_with_choices( + "TEST_ENV", ["default"], ["option1", "option2"] + ) + assert env_func() == {"default"} + + def test_only_commas_returns_default(self): + """Test that string with only commas returns default.""" + with patch.dict(os.environ, {"TEST_ENV": ",,,"}): + env_func = env_set_with_choices( + "TEST_ENV", ["default"], ["option1", "option2"] + ) + assert env_func() == {"default"} + + def test_case_sensitive_validation(self): + """Test case sensitive validation.""" + with patch.dict(os.environ, {"TEST_ENV": "option1,OPTION2"}): + env_func = env_set_with_choices( + "TEST_ENV", [], ["option1", "option2"], case_sensitive=True + ) + with pytest.raises(ValueError, match="Invalid value 'OPTION2' in TEST_ENV"): + env_func() + + def test_case_insensitive_validation(self): + """Test case insensitive validation.""" + with patch.dict(os.environ, {"TEST_ENV": "OPTION1,option2"}): + env_func = env_set_with_choices( + "TEST_ENV", [], ["option1", "option2"], case_sensitive=False + ) + assert env_func() == {"OPTION1", "option2"} + + def test_invalid_value_in_list_raises_error(self): + """Test that invalid value in list raises ValueError.""" + with patch.dict(os.environ, {"TEST_ENV": "option1,invalid,option2"}): + env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"]) + with pytest.raises(ValueError, match="Invalid value 'invalid' in TEST_ENV"): + env_func() + + def test_callable_choices_resolved_correctly(self): + """Test that callable choices are resolved correctly.""" + + def get_choices(): + return ["dynamic1", "dynamic2"] + + with patch.dict(os.environ, {"TEST_ENV": "dynamic1,dynamic2"}): + env_func = env_set_with_choices("TEST_ENV", [], get_choices) + assert env_func() == {"dynamic1", "dynamic2"} + + def test_callable_choices_with_invalid_value(self): + """Test that callable choices raise error for invalid values.""" + + def get_choices(): + return ["dynamic1", "dynamic2"] + + with patch.dict(os.environ, {"TEST_ENV": "dynamic1,invalid"}): + env_func = env_set_with_choices("TEST_ENV", [], get_choices) + with pytest.raises(ValueError, match="Invalid value 'invalid' in TEST_ENV"): + env_func() + + def test_duplicate_values_deduped(self): + """Test that duplicate values in the list are deduped.""" + with patch.dict(os.environ, {"TEST_ENV": "option1,option1,option2"}): + env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"]) + assert env_func() == {"option1", "option2"} diff --git a/vllm/entrypoints/harmony_utils.py b/vllm/entrypoints/harmony_utils.py index 8888a5aeb6b1a..7958d0317739a 100644 --- a/vllm/entrypoints/harmony_utils.py +++ b/vllm/entrypoints/harmony_utils.py @@ -61,15 +61,19 @@ _harmony_encoding = None # they are available and requested by the user. # Tool args are provided by MCP tool descriptions. Output # of the tools are stringified. -BUILTIN_TOOLS = { +MCP_BUILTIN_TOOLS: set[str] = { "web_search_preview", "code_interpreter", "container", } -def has_custom_tools(tool_types: list[str]) -> bool: - return not set(tool_types).issubset(BUILTIN_TOOLS) +def has_custom_tools(tool_types: set[str]) -> bool: + """ + Checks if the given tool types are custom tools + (i.e. any tool other than MCP buildin tools) + """ + return not tool_types.issubset(MCP_BUILTIN_TOOLS) def get_encoding(): diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index d43bc00a49d36..2ee8de5fba07a 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -48,6 +48,7 @@ from openai.types.responses.response_output_text import Logprob, LogprobTopLogpr from openai.types.responses.response_reasoning_item import ( Content as ResponseReasoningTextContent, ) +from openai.types.responses.tool import Tool from openai_harmony import Message as OpenAIHarmonyMessage from vllm import envs @@ -106,6 +107,23 @@ from vllm.utils import random_uuid logger = init_logger(__name__) +def extract_tool_types(tools: list[Tool]) -> set[str]: + """ + Extracts the tool types from the given tools. + """ + tool_types: set[str] = set() + for tool in tools: + if tool.type == "mcp": + # Allow the MCP Tool type to enable built in tools if the + # server_label is allowlisted in + # envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS + if tool.server_label in envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: + tool_types.add(tool.server_label) + else: + tool_types.add(tool.type) + return tool_types + + class OpenAIServingResponses(OpenAIServing): def __init__( self, @@ -879,7 +897,7 @@ class OpenAIServingResponses(OpenAIServing): return messages def _construct_harmony_system_input_message( - self, request: ResponsesRequest, with_custom_tools: bool, tool_types: list[str] + self, request: ResponsesRequest, with_custom_tools: bool, tool_types: set[str] ) -> OpenAIHarmonyMessage: reasoning_effort = request.reasoning.effort if request.reasoning else None enable_browser = ( @@ -927,17 +945,7 @@ class OpenAIServingResponses(OpenAIServing): messages: list[OpenAIHarmonyMessage] = [] if prev_response is None: # New conversation. - tool_types = [tool.type for tool in request.tools] - # Allow the MCP Tool type to enable built in tools if the - # server_label is allowlisted in - # envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS - if envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS: - for tool in request.tools: - if ( - tool.type == "mcp" - and tool.server_label in envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS - ): - tool_types.append(tool.server_label) + tool_types = extract_tool_types(request.tools) with_custom_tools = has_custom_tools(tool_types) sys_msg = self._construct_harmony_system_input_message( diff --git a/vllm/envs.py b/vllm/envs.py index 0786d5d9ddcbd..ca1f84bba419d 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -198,6 +198,7 @@ if TYPE_CHECKING: VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False VLLM_TUNED_CONFIG_FOLDER: str | None = None + VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set() VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False @@ -209,7 +210,6 @@ if TYPE_CHECKING: VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK: bool = True VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL: bool = False VLLM_DBO_COMM_SMS: int = 20 - GPT_OSS_SYSTEM_TOOL_MCP_LABELS: list[str] = [] VLLM_PATTERN_MATCH_DEBUG: str | None = None VLLM_DEBUG_DUMP_PATH: str | None = None VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE: bool = True @@ -354,6 +354,24 @@ def env_list_with_choices( return _get_validated_env_list +def env_set_with_choices( + env_name: str, + default: list[str], + choices: list[str] | Callable[[], list[str]], + case_sensitive: bool = True, +) -> Callable[[], set[str]]: + """ + Creates a lambda which that validates environment variable + containing comma-separated values against allowed choices which + returns choices as a set. + """ + + def _get_validated_env_set() -> set[str]: + return set(env_list_with_choices(env_name, default, choices, case_sensitive)()) + + return _get_validated_env_set + + def get_vllm_port() -> int | None: """Get the port from VLLM_PORT environment variable. @@ -1328,6 +1346,15 @@ environment_variables: dict[str, Callable[[], Any]] = { ), # Allows vllm to find tuned config under customized folder "VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None), + # Valid values are container,code_interpreter,web_search_preview + # ex VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter + # If the server_label of your mcp tool is not in this list it will + # be completely ignored. + "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": env_set_with_choices( + "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", + default=[], + choices=["container", "code_interpreter", "web_search_preview"], + ), # Allows harmony instructions to be injected on system messages "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool( int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0")) @@ -1379,13 +1406,6 @@ environment_variables: dict[str, Callable[[], Any]] = { # The number of SMs to allocate for communication kernels when running DBO # the rest of the SMs on the device will be allocated to compute "VLLM_DBO_COMM_SMS": lambda: int(os.getenv("VLLM_DBO_COMM_SMS", "20")), - # Valid values are container,code_interpreter,web_search_preview - # ex GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter - "GPT_OSS_SYSTEM_TOOL_MCP_LABELS": env_list_with_choices( - "GPT_OSS_SYSTEM_TOOL_MCP_LABELS", - [], - ["container", "code_interpreter", "web_search_preview"], - ), # Enable max_autotune & coordinate_descent_tuning in inductor_config # to compile static shapes passed from compile_sizes in compilation_config # If set to 1, enable max_autotune; By default, this is enabled (1) From 7ba6aa8f564320b136f8ee17d40ea656a12df7e5 Mon Sep 17 00:00:00 2001 From: Yue Zhang <81500899+KevinCheung2259@users.noreply.github.com> Date: Wed, 29 Oct 2025 18:03:54 +0800 Subject: [PATCH 066/127] [Fix] import get_kv_cache_torch_dtype error in LMCacheConnector integration (#27670) Signed-off-by: KevinCheung2259 <2651309292@qq.com> From 1a33aacf8212472d5c218e3e7b1e8c0f8729cece Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Wed, 29 Oct 2025 18:06:42 +0800 Subject: [PATCH 067/127] [Misc] Raise error for missing video metadata in `MultiModalDataParser` (#27664) Signed-off-by: Isotr0py Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- vllm/multimodal/parse.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 1ae2c7408a66f..2fa3f6ebcc114 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -506,6 +506,11 @@ class MultiModalDataParser: for data_item in data_items: video, metadata = self._get_video_with_metadata(data_item) if self.video_needs_metadata: + if metadata is None: + raise ValueError( + "Video metadata is required but not found in mm input. " + "Please check your video input in `multi_modal_data`" + ) new_videos.append((video, metadata)) metadata_lst.append(metadata) else: From 5e72216d176e7f47119263d45f42129ecb1d5b18 Mon Sep 17 00:00:00 2001 From: Eugene Khvedchenya Date: Wed, 29 Oct 2025 12:24:52 +0200 Subject: [PATCH 068/127] Feature/video support in random mm dataset (#25963) Signed-off-by: Eugene Khvedchenia Signed-off-by: Eugene Khvedchenya Co-authored-by: Roger Wang --- tests/benchmarks/test_random_dataset.py | 123 ++++++ .../test_random_multimodal_dataset_video.py | 398 ++++++++++++++++++ vllm/benchmarks/datasets.py | 105 +++-- 3 files changed, 601 insertions(+), 25 deletions(-) create mode 100644 tests/benchmarks/test_random_multimodal_dataset_video.py diff --git a/tests/benchmarks/test_random_dataset.py b/tests/benchmarks/test_random_dataset.py index 68e4afdcbe521..57f6893061825 100644 --- a/tests/benchmarks/test_random_dataset.py +++ b/tests/benchmarks/test_random_dataset.py @@ -359,3 +359,126 @@ def test_random_mm_bucket_config_not_mutated( assert len(mm_data) >= 1 for it in mm_data: assert it.get("type") == "image_url" + + +@pytest.mark.benchmark +def test_random_mm_video_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None: + """Test video sampling functionality in RandomMultiModalDataset.""" + ds = RandomMultiModalDataset(random_seed=42) + + # Test with video bucket configuration + bucket_config = { + (64, 64, 1): 0.3, # Images + (64, 64, 8): 0.7, # Videos + } + + limit_mm_per_prompt = {"image": 2, "video": 2} + + samples = _collect_mm_samples( + ds, + hf_tokenizer, + num_requests=5, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + ) + + assert len(samples) == 5 + + # Check that we have both images and videos + video_count = 0 + image_count = 0 + + for s in samples: + mm_data = cast(list[dict[str, Any]], s.multi_modal_data) + assert len(mm_data) == 1 + + item = mm_data[0] + if item.get("type") == "video_url": + video_count += 1 + # Verify video URL format + url = item.get("video_url", {}).get("url", "") + assert url.startswith("data:video/mp4;base64,") + elif item.get("type") == "image_url": + image_count += 1 + # Verify image URL format + url = item.get("image_url", {}).get("url", "") + assert url.startswith("data:image/jpeg;base64,") + + # Should have some videos due to 0.7 probability + assert video_count > 0 + assert image_count > 0 + + +@pytest.mark.benchmark +def test_random_mm_video_only_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None: + """Test sampling with only video buckets.""" + ds = RandomMultiModalDataset(random_seed=42) + + bucket_config = { + (64, 64, 8): 1.0, # Only videos + } + + limit_mm_per_prompt = {"image": 0, "video": 1} + + samples = _collect_mm_samples( + ds, + hf_tokenizer, + num_requests=3, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + ) + + assert len(samples) == 3 + + for s in samples: + mm_data = cast(list[dict[str, Any]], s.multi_modal_data) + assert len(mm_data) == 1 + + item = mm_data[0] + assert item.get("type") == "video_url" + url = item.get("video_url", {}).get("url", "") + assert url.startswith("data:video/mp4;base64,") + + +@pytest.mark.benchmark +def test_random_mm_video_deterministic_sampling( + hf_tokenizer: PreTrainedTokenizerBase, +) -> None: + """Test that video sampling is deterministic with same seed.""" + seed = 123 + ds_a = RandomMultiModalDataset(random_seed=seed) + ds_b = RandomMultiModalDataset(random_seed=seed) + + bucket_config = { + (64, 64, 8): 1.0, # Only videos + } + + limit_mm_per_prompt = {"image": 0, "video": 1} + + a = _collect_mm_samples( + ds_a, + hf_tokenizer, + num_requests=3, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + ) + + b = _collect_mm_samples( + ds_b, + hf_tokenizer, + num_requests=3, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + ) + + fa = [_mm_fingerprint_sample(s) for s in a] + fb = [_mm_fingerprint_sample(s) for s in b] + assert fa == fb diff --git a/tests/benchmarks/test_random_multimodal_dataset_video.py b/tests/benchmarks/test_random_multimodal_dataset_video.py new file mode 100644 index 0000000000000..db19a169e359c --- /dev/null +++ b/tests/benchmarks/test_random_multimodal_dataset_video.py @@ -0,0 +1,398 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import base64 +import os +from tempfile import NamedTemporaryFile +from typing import Any, cast + +import cv2 +import pytest +from transformers import AutoTokenizer, PreTrainedTokenizerBase + +from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest + + +@pytest.fixture(scope="session") +def hf_tokenizer() -> PreTrainedTokenizerBase: + """Use a small, commonly available tokenizer.""" + return AutoTokenizer.from_pretrained("gpt2") + + +@pytest.fixture +def video_dataset() -> RandomMultiModalDataset: + """Create a RandomMultiModalDataset instance for testing.""" + return RandomMultiModalDataset(random_seed=42) + + +@pytest.mark.benchmark +def test_generate_synthetic_video_different_seeds(): + """Test that different seeds produce different videos.""" + dataset1 = RandomMultiModalDataset(random_seed=123) + dataset2 = RandomMultiModalDataset(random_seed=456) + + width, height, num_frames = 64, 48, 8 + + video1 = dataset1.generate_synthetic_video(width, height, num_frames) + video2 = dataset2.generate_synthetic_video(width, height, num_frames) + + # Videos should be different due to different seeds + assert video1["bytes"] != video2["bytes"] + + +@pytest.mark.benchmark +def test_map_config_to_modality(video_dataset: RandomMultiModalDataset): + """Test modality mapping for different configurations.""" + # Test image configuration (num_frames = 1) + assert video_dataset.map_config_to_modality((256, 256, 1)) == "image" + assert video_dataset.map_config_to_modality((720, 1280, 1)) == "image" + + # Test video configurations (num_frames > 1) + assert video_dataset.map_config_to_modality((256, 256, 8)) == "video" + assert video_dataset.map_config_to_modality((720, 1280, 16)) == "video" + assert video_dataset.map_config_to_modality((64, 64, 32)) == "video" + + # Test invalid configurations + with pytest.raises(ValueError, match="Invalid multimodal item configuration"): + video_dataset.map_config_to_modality((256, 256, 0)) + + with pytest.raises(ValueError, match="Invalid multimodal item configuration"): + video_dataset.map_config_to_modality((256, 256, -1)) + + +@pytest.mark.benchmark +def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset): + """Test generating multimodal items for video configurations.""" + # Test video item generation + video_config = (64, 48, 8) # height, width, num_frames + result = video_dataset.generate_mm_item(video_config) + + # Check the result structure matches OpenAI API format + assert isinstance(result, dict) + assert result["type"] == "video_url" + assert "video_url" in result + assert "url" in result["video_url"] + + # Check that the URL is a data URL with base64 encoded video + url = result["video_url"]["url"] + assert url.startswith("data:video/mp4;base64,") + + # Decode and verify the video content + base64_data = url.split(",")[1] + video_bytes = base64.b64decode(base64_data) + assert len(video_bytes) > 0 + + # Verify the video can be decoded + with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: + temp_path = temp_file.name + temp_file.write(video_bytes) + + try: + cap = cv2.VideoCapture(temp_path) + assert cap.isOpened() + + frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + assert frame_count == 8 + assert frame_width == 48 + assert frame_height == 64 + + cap.release() + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) + + +@pytest.mark.benchmark +def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset): + """Test generating multimodal items for image configurations.""" + # Test image item generation + image_config = (64, 48, 1) # height, width, num_frames=1 + result = video_dataset.generate_mm_item(image_config) + + # Check the result structure matches OpenAI API format + assert isinstance(result, dict) + assert result["type"] == "image_url" + assert "image_url" in result + assert "url" in result["image_url"] + + # Check that the URL is a data URL with base64 encoded image + url = result["image_url"]["url"] + assert url.startswith("data:image/jpeg;base64,") + + +@pytest.mark.benchmark +def test_generate_mm_item_invalid_config(video_dataset: RandomMultiModalDataset): + """Test error handling for invalid configurations.""" + with pytest.raises(ValueError, match="Invalid multimodal item configuration"): + video_dataset.generate_mm_item((256, 256, 0)) + + +@pytest.mark.benchmark +def test_sample_with_video_buckets( + video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase +): + """Test sampling with video bucket configurations.""" + # Configure bucket with video probability > 0 + bucket_config = { + (64, 64, 1): 0.3, # Images + (64, 64, 8): 0.7, # Videos + } + + limit_mm_per_prompt = {"image": 5, "video": 3} + + samples = video_dataset.sample( + tokenizer=hf_tokenizer, + num_requests=5, + base_items_per_request=2, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + assert len(samples) == 5 + + # Check that samples contain both images and videos + video_count = 0 + image_count = 0 + + for sample in samples: + assert isinstance(sample, SampleRequest) + assert sample.multi_modal_data is not None + assert isinstance(sample.multi_modal_data, list) + + mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) + assert len(mm_data) == 2 # base_items_per_request + + for item in mm_data: + if item["type"] == "video_url": + video_count += 1 + # Verify video URL format + url = item["video_url"]["url"] + assert url.startswith("data:video/mp4;base64,") + elif item["type"] == "image_url": + image_count += 1 + # Verify image URL format + url = item["image_url"]["url"] + assert url.startswith("data:image/jpeg;base64,") + + # Should have some videos due to 0.7 probability + assert video_count > 0 + assert image_count > 0 + + +@pytest.mark.benchmark +def test_sample_video_only_buckets( + video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase +): + """Test sampling with only video buckets.""" + bucket_config = { + (64, 64, 8): 1.0, # Only videos + } + + limit_mm_per_prompt = {"image": 0, "video": 2} + + samples = video_dataset.sample( + tokenizer=hf_tokenizer, + num_requests=3, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + assert len(samples) == 3 + + for sample in samples: + assert isinstance(sample, SampleRequest) + assert sample.multi_modal_data is not None + assert isinstance(sample.multi_modal_data, list) + + mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) + assert len(mm_data) == 1 + + item = mm_data[0] + assert item["type"] == "video_url" + url = item["video_url"]["url"] + assert url.startswith("data:video/mp4;base64,") + + +@pytest.mark.benchmark +def test_sample_respects_video_limits( + video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase +): + """Test that sampling respects video limits per prompt.""" + bucket_config = { + (64, 64, 8): 1.0, # Only videos + } + + # Set very low video limit + limit_mm_per_prompt = {"image": 0, "video": 1} + + samples = video_dataset.sample( + tokenizer=hf_tokenizer, + num_requests=3, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + assert len(samples) == 3 + + for sample in samples: + mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) + assert len(mm_data) <= 1 # Should respect video limit + + +@pytest.mark.benchmark +def test_sample_mixed_buckets_with_zero_probability( + video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase +): + """Test sampling with mixed buckets including zero probability entries.""" + bucket_config = { + (64, 64, 1): 0.5, # Images + (64, 64, 8): 0.5, # Videos + (128, 128, 16): 0.0, # Zero probability videos (should be ignored) + } + + limit_mm_per_prompt = {"image": 2, "video": 2} + + samples = video_dataset.sample( + tokenizer=hf_tokenizer, + num_requests=4, + base_items_per_request=2, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + assert len(samples) == 4 + + # Should only see 64x64 videos, not 128x128 videos + for sample in samples: + mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) + for item in mm_data: + if item["type"] == "video_url": + # Decode video to verify dimensions + url = item["video_url"]["url"] + base64_data = url.split(",")[1] + video_bytes = base64.b64decode(base64_data) + + with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: # noqa + temp_path = temp_file.name + temp_file.write(video_bytes) + + try: + cap = cv2.VideoCapture(temp_path) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + cap.release() + + # Should be 64x64, not 128x128 + assert frame_width == 64 + assert frame_height == 64 + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) + + +@pytest.mark.benchmark +def test_sample_deterministic_with_videos(hf_tokenizer: PreTrainedTokenizerBase): + """Test that sampling with videos is deterministic with same seed.""" + dataset1 = RandomMultiModalDataset(random_seed=123) + dataset2 = RandomMultiModalDataset(random_seed=123) + + bucket_config = { + (64, 64, 1): 0.3, # Images + (64, 64, 8): 0.7, # Videos + } + + limit_mm_per_prompt = {"image": 2, "video": 2} + + samples1 = dataset1.sample( + tokenizer=hf_tokenizer, + num_requests=3, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + samples2 = dataset2.sample( + tokenizer=hf_tokenizer, + num_requests=3, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + assert len(samples1) == len(samples2) + + # Compare multimodal data + for s1, s2 in zip(samples1, samples2): + assert s1.multi_modal_data == s2.multi_modal_data + + +@pytest.mark.benchmark +def test_sample_different_seeds_produce_different_videos( + hf_tokenizer: PreTrainedTokenizerBase, +): + """Test that different seeds produce different video content.""" + dataset1 = RandomMultiModalDataset(random_seed=123) + dataset2 = RandomMultiModalDataset(random_seed=456) + + bucket_config = { + (64, 64, 8): 1.0, # Only videos + } + + limit_mm_per_prompt = {"image": 0, "video": 1} + + samples1 = dataset1.sample( + tokenizer=hf_tokenizer, + num_requests=2, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + samples2 = dataset2.sample( + tokenizer=hf_tokenizer, + num_requests=2, + base_items_per_request=1, + num_mm_items_range_ratio=0.0, + limit_mm_per_prompt=limit_mm_per_prompt, + bucket_config=bucket_config, + input_len=20, + output_len=5, + ) + + # Video content should be different + for s1, s2 in zip(samples1, samples2): + mm_data1 = cast(list[dict[str, Any]], s1.multi_modal_data) + mm_data2 = cast(list[dict[str, Any]], s2.multi_modal_data) + + assert len(mm_data1) == len(mm_data2) == 1 + + url1 = mm_data1[0]["video_url"]["url"] + url2 = mm_data2[0]["video_url"]["url"] + + assert url1 != url2 # Different video content diff --git a/vllm/benchmarks/datasets.py b/vllm/benchmarks/datasets.py index 55e24bd5d9d31..b1aa8530eb026 100644 --- a/vllm/benchmarks/datasets.py +++ b/vllm/benchmarks/datasets.py @@ -27,8 +27,10 @@ from copy import deepcopy from dataclasses import dataclass from functools import cache from io import BytesIO +from tempfile import NamedTemporaryFile from typing import Any, cast +import cv2 import numpy as np from PIL import Image from transformers import PreTrainedTokenizerBase @@ -498,9 +500,13 @@ class RandomDataset(BenchmarkDataset): num_requests, range_ratio, input_len, output_len, tokenizer ) - # Generate prefix once - prefix_token_ids = self.get_prefix(tokenizer, prefix_len) vocab_size = tokenizer.vocab_size + prohibited_tokens = tokenizer.all_special_ids + all_tokens = np.arange(vocab_size) + allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens))) + + # Generate prefix once + prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len) requests = [] token_mismatch_total = 0 @@ -513,6 +519,7 @@ class RandomDataset(BenchmarkDataset): input_len=int(input_lens[i]), offset=int(offsets[i]), index=i, + allowed_tokens=allowed_tokens, ) token_mismatch_total += token_mismatch requests.append( @@ -553,13 +560,17 @@ class RandomDataset(BenchmarkDataset): return requests def get_prefix( - self, tokenizer: PreTrainedTokenizerBase, prefix_len: int + self, + allowed_tokens: np.ndarray, + prefix_len: int, ) -> list[int]: """ Get the prefix for the dataset. """ return ( - self._rng.integers(0, tokenizer.vocab_size, size=prefix_len).tolist() + allowed_tokens[ + self._rng.integers(0, len(allowed_tokens), size=prefix_len) + ].tolist() if prefix_len > 0 else [] ) @@ -623,6 +634,7 @@ class RandomDataset(BenchmarkDataset): input_len: int, offset: int, index: int, + allowed_tokens: np.ndarray, ) -> tuple[str, int, int]: """ Returns (prompt, total_input_len). @@ -636,8 +648,11 @@ class RandomDataset(BenchmarkDataset): To avoid uncontrolled change of the prompt length, the encoded sequence is truncated before being decoded again. """ - # Build the inner sequence by sampling sequentially from the vocab - inner_seq = ((offset + index + np.arange(input_len)) % vocab_size).tolist() + # Build the inner sequence by sampling + # sequentially from the allowed tokens + inner_seq = allowed_tokens[ + (offset + index + np.arange(input_len)) % len(allowed_tokens) + ].tolist() token_sequence = prefix_token_ids + inner_seq # Decode, then re-encode and truncate to preserve token count invariants @@ -772,7 +787,7 @@ class RandomMultiModalDataset(RandomDataset): Status: - Images: supported via synthetic RGB data. - - Video: not yet supported (TODO: implement video generation method). + - Video: supported via synthetic RGB data. - Audio: not yet supported. Sampling overview: @@ -782,7 +797,7 @@ class RandomMultiModalDataset(RandomDataset): The maximum is further clamped to the sum of per-modality limits. 2) Each item’s modality and shape is sampled from `bucket_config`, a dict mapping (height, width, num_frames) → probability. We treat - `num_frames`=1 as image and and `num_frames` > 1 as video. + `num_frames`=1 as image and `num_frames` > 1 as video. Entries with zero probability are removed and the rest are renormalized to sum to 1. 3) Per-modality hard caps are enforced via `limit_mm_per_prompt`. @@ -797,8 +812,7 @@ class RandomMultiModalDataset(RandomDataset): """ IS_MULTIMODAL = True - # NOTE: video sampling is WIP. Setting it to 0. - DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 0} + DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 1} DEFAULT_BASE_ITEMS_PER_REQUEST = 1 DEFAULT_NUM_MM_ITEMS_RANGE_RATIO = 0.0 @@ -828,12 +842,47 @@ class RandomMultiModalDataset(RandomDataset): ) return Image.fromarray(random_pixels) - def generate_synthetic_video(self, width: int, height: int, num_frames: int) -> Any: + def generate_synthetic_video( + self, width: int, height: int, num_frames: int + ) -> dict: """Generate synthetic video with random values. - TODO: Finish this method. + Creates a video with random pixel values, encodes it to MP4 format, + and returns the content as bytes. """ - raise NotImplementedError("Video sampling is WIP.") + random_pixels = self._rng.integers( + 0, + 256, + (num_frames, height, width, 3), + dtype=np.uint8, + ) + + # Create a temporary video file in memory + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + fps = 30 # frames per second + + with NamedTemporaryFile(suffix=".mp4", delete_on_close=False) as temp_file: + temp_path = temp_file.name + + # Create video writer + video_writer = cv2.VideoWriter( + temp_path, fourcc=fourcc, fps=fps, frameSize=(width, height) + ) + + if not video_writer.isOpened(): + raise RuntimeError("Failed to create video writer") + + for frame in random_pixels: + video_writer.write(frame) + + video_writer.release() + temp_file.close() + + # Read the video file content + with open(temp_path, "rb") as f: + video_content = f.read() + + return {"bytes": video_content} def map_config_to_modality(self, config: tuple[int, int, int]) -> str: """Map the configuration to the modality.""" @@ -1044,16 +1093,6 @@ class RandomMultiModalDataset(RandomDataset): enable_multimodal_chat: bool = DEFAULT_ENABLE_MULTIMODAL_CHAT, **kwargs, ) -> list[SampleRequest]: - # NOTE: Video sampling is WIP. Raise error if video is in bucket config - # and probability is non-zero. - if any( - self.map_config_to_modality(cfg) == "video" and p > 0 - for cfg, p in bucket_config.items() - ): - raise NotImplementedError( - "Video sampling not implemented; set its probability to 0." - ) - # Get the sampling parameters for the dataset input_lens, output_lens, offsets = self.get_sampling_params( num_requests, range_ratio, input_len, output_len, tokenizer @@ -1071,9 +1110,24 @@ class RandomMultiModalDataset(RandomDataset): bucket_config, ) - # Generate prefix once - prefix_token_ids = self.get_prefix(tokenizer, prefix_len) vocab_size = tokenizer.vocab_size + # Can't use tokenizer.all_special_ids since + # it returns ONLY ids from special_tokens_map.json + # We want to exclude placeholder tokens and all + # tokens that indicate start/end of image as it + # may break prompt replacement logic. + prohibited_tokens = list( + tok_id + for tok_id, token in tokenizer.added_tokens_decoder.items() + if token.special + ) + all_tokens = np.arange(vocab_size) + allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens))) + logger.debug( + "Sampling from %d out of %d (vocab size)", len(allowed_tokens), vocab_size + ) + # Generate prefix once + prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len) # Add synthetic multimodal items to each request mm_requests = [] token_mismatch_total = 0 @@ -1086,6 +1140,7 @@ class RandomMultiModalDataset(RandomDataset): input_len=int(input_lens[i]), offset=int(offsets[i]), index=i, + allowed_tokens=allowed_tokens, ) token_mismatch_total += token_mismatch # Get multimodal item iterator for a given request From 3481e4074322cebf0c22aacbffc2fac4163220da Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 29 Oct 2025 03:29:49 -0700 Subject: [PATCH 069/127] [chore] Remove models weight on S3 logic (#27725) Signed-off-by: kevin --- vllm/engine/arg_utils.py | 10 --- vllm/test_utils.py | 129 --------------------------------------- 2 files changed, 139 deletions(-) delete mode 100644 vllm/test_utils.py diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index f13ce935ec4b6..b31e4931f2295 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -75,7 +75,6 @@ from vllm.platforms import CpuArchEnum, current_platform from vllm.plugins import load_general_plugins from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized from vllm.reasoning import ReasoningParserManager -from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3 from vllm.transformers_utils.config import ( get_model_path, is_interleaved, @@ -1126,15 +1125,6 @@ class EngineArgs: if check_gguf_file(self.model): self.quantization = self.load_format = "gguf" - # NOTE: This is to allow model loading from S3 in CI - if ( - not isinstance(self, AsyncEngineArgs) - and envs.VLLM_CI_USE_S3 - and self.model in MODELS_ON_S3 - and self.load_format == "auto" - ): - self.model = f"{MODEL_WEIGHTS_S3_BUCKET}/{self.model}" - if self.disable_mm_preprocessor_cache: logger.warning( "`--disable-mm-preprocessor-cache` is deprecated " diff --git a/vllm/test_utils.py b/vllm/test_utils.py deleted file mode 100644 index 91dcc2fd84e17..0000000000000 --- a/vllm/test_utils.py +++ /dev/null @@ -1,129 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -MODELS_ON_S3 = [ - "adept/fuyu-8b", - "ai21labs/AI21-Jamba-1.5-Mini", - "ai21labs/Jamba-tiny-random", - "ai21labs/Jamba-tiny-reward-dev", - "allenai/Molmo-7B-D-0924", - "allenai/OLMo-1B-hf", - "allenai/OLMoE-1B-7B-0924-Instruct", - "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test", - "AMead10/Llama-3.2-1B-Instruct-AWQ", - "hmellor/Ilama-3.2-1B", - "BAAI/bge-base-en-v1.5", - "BAAI/bge-multilingual-gemma2", - "BAAI/bge-reranker-v2-m3", - "bigcode/starcoder2-3b", - "cross-encoder/ms-marco-MiniLM-L-6-v2", - "cross-encoder/quora-roberta-base", - "deepseek-ai/deepseek-vl2-tiny", - "distilbert/distilgpt2", - "facebook/bart-base", - "facebook/bart-large-cnn", - # "fixie-ai/ultravox-v0_5-llama-3_2-1b", - "google/gemma-1.1-2b-it", - "google/gemma-2-2b-it", - "google/paligemma-3b-pt-224", - "h2oai/h2ovl-mississippi-800m", - "HuggingFaceM4/Idefics3-8B-Llama3", - "internlm/internlm2-1_8b-reward", - "intfloat/e5-mistral-7b-instruct", - "intfloat/multilingual-e5-small", - "jason9693/Qwen2.5-1.5B-apeach", - "llava-hf/llava-1.5-7b-hf", - "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", - "llava-hf/llava-v1.6-mistral-7b-hf", - "llava-hf/LLaVA-NeXT-Video-7B-hf", - # "meta-llama/Llama-2-7b-hf", - "meta-llama/Llama-3.2-1B", - "meta-llama/Llama-3.2-1B-Instruct", - "meta-llama/Meta-Llama-3-8B", - "microsoft/phi-2", - "microsoft/Phi-3-mini-4k-instruct", - "microsoft/Phi-3-small-8k-instruct", - "microsoft/Phi-3-vision-128k-instruct", - "microsoft/Phi-3.5-MoE-instruct", - "microsoft/Phi-3.5-vision-instruct", - # "mistralai/Mistral-7B-Instruct-v0.1", - "mistralai/Mixtral-8x7B-Instruct-v0.1", - "mistralai/Pixtral-12B-2409", - "mistral-community/Mixtral-8x22B-v0.1-AWQ", - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head", - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse", - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue", - "ModelCloud/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit-10-25-2024", - "neuralmagic/Llama-3.2-1B-quantized.w8a8", - "neuralmagic/Meta-Llama-3-8B-Instruct-FP8", - "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV", - "nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama", - "nm-testing/llama2.c-stories42M-pruned2.4-compressed", - "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t", - "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test", - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing", - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing", - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing", - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing", - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym", - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym", - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym", - "nm-testing/Phi-3-mini-128k-instruct-FP8", - "nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV", - "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V", - "nm-testing/tinyllama-oneshot-w4a16-channel-v2", - "nm-testing/tinyllama-oneshot-w4a16-group128-v2", - "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", - "nm-testing/tinyllama-oneshot-w8a16-per-channel", - "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2", - "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym", - "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", - "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym", - "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_int8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_fp8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_int8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_fp8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_int8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_fp8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_int8-BitM", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing", - "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing", - "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme", - "nvidia/NVLM-D-72B", - "openai-community/gpt2", - # "openai/whisper-large-v3", - "openbmb/MiniCPM-o-2_6", - "openbmb/MiniCPM-V-2_6", - "OpenGVLab/InternVL2-1B", - "parasail-ai/GritLM-7B-vllm", - "Qwen/Qwen1.5-MoE-A2.7B-Chat", - "Qwen/Qwen2-7B-Instruct", - "Qwen/Qwen2-Audio-7B-Instruct", - "Qwen/Qwen2-VL-2B-Instruct", - "Qwen/Qwen2.5-1.5B-Instruct", - "Qwen/Qwen2.5-Math-PRM-7B", - "Qwen/Qwen2.5-Math-RM-72B", - "Qwen/Qwen2.5-VL-3B-Instruct", - "royokong/e5-v", - "sentence-transformers/all-roberta-large-v1", - "sentence-transformers/stsb-roberta-base-v2", - "allenai/OLMo-2-0425-1B", - "shuyuej/Llama-3.2-1B-Instruct-GPTQ", - "ssmits/Qwen2-7B-Instruct-embed-base", - "stabilityai/stablelm-3b-4e1t", - "stabilityai/stablelm-zephyr-3b", - "state-spaces/mamba-130m-hf", - "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", - "zai-org/glm-4v-9b", - "TIGER-Lab/Mantis-8B-siglip-llama3", - "TIGER-Lab/VLM2Vec-Full", - "tiiuae/falcon-40b", - "tiiuae/falcon-mamba-7b-instruct", - "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "upstage/solar-pro-preview-instruct", -] - -MODEL_WEIGHTS_S3_BUCKET = "s3://vllm-ci-model-weights" From ad3ec89532b7e0ef676fd319b061adde360f5c68 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Wed, 29 Oct 2025 20:19:37 +0800 Subject: [PATCH 070/127] [VLM] Add Qwen3-VL generation test (#25185) Signed-off-by: Isotr0py Signed-off-by: Roger Wang Co-authored-by: Roger Wang --- .../multimodal/generation/test_common.py | 22 +++++++++ .../generation/vlm_utils/builders.py | 33 +++++++++++-- .../generation/vlm_utils/case_filtering.py | 3 ++ .../generation/vlm_utils/model_utils.py | 48 +++++++++++++++++++ .../generation/vlm_utils/runners.py | 1 + .../multimodal/generation/vlm_utils/types.py | 4 +- vllm/assets/video.py | 2 +- 7 files changed, 108 insertions(+), 5 deletions(-) diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index f11f75418e7d7..4c79ac318ffbe 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -159,6 +159,28 @@ VLM_TEST_SETTINGS = { image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)], marks=[pytest.mark.core_model, pytest.mark.cpu_model], ), + "qwen3_vl": VLMTestInfo( + models=["Qwen/Qwen3-VL-4B-Instruct"], + test_type=( + VLMTestType.IMAGE, + VLMTestType.MULTI_IMAGE, + VLMTestType.VIDEO, + ), + needs_video_metadata=True, + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501 + max_model_len=4096, + max_num_seqs=2, + num_logprobs=20, + auto_cls=AutoModelForImageTextToText, + vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output, + patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner, + image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)], + marks=[ + pytest.mark.core_model, + ], + ), "ultravox": VLMTestInfo( models=["fixie-ai/ultravox-v0_5-llama-3_2-1b"], test_type=VLMTestType.AUDIO, diff --git a/tests/models/multimodal/generation/vlm_utils/builders.py b/tests/models/multimodal/generation/vlm_utils/builders.py index 6252f33bdfad7..47852453c0585 100644 --- a/tests/models/multimodal/generation/vlm_utils/builders.py +++ b/tests/models/multimodal/generation/vlm_utils/builders.py @@ -4,7 +4,9 @@ from collections.abc import Callable, Iterable from pathlib import PosixPath +from typing import Any +import numpy.typing as npt import torch from vllm.multimodal.audio import AudioResampler @@ -236,6 +238,7 @@ def build_video_inputs_from_test_info( video_assets: VideoTestAssets, size_wrapper: ImageSizeWrapper, num_frames: int, + needs_video_metadata: bool, ) -> list[PromptWithMultiModalInput]: if test_info.prompt_formatter is None: raise ValueError("Prompt formatter must be set to build video inputs") @@ -248,7 +251,10 @@ def build_video_inputs_from_test_info( ) sampled_vids = [ - sample_frames_from_video(asset.np_ndarrays, num_frames) + sample_frames_with_video_metadata( + (asset.np_ndarrays, asset.metadata), + num_frames, + ) for asset in video_assets ] @@ -259,12 +265,33 @@ def build_video_inputs_from_test_info( return [ PromptWithMultiModalInput( prompts=[prompt for _ in size_wrapper.data], - video_data=[video_scaler(video, size) for size in size_wrapper.data], + video_data=[ + ( + video_scaler(video, size) + if not needs_video_metadata + else (video_scaler(video, size), meta) + ) + for size in size_wrapper.data + ], ) - for video, prompt in zip(sampled_vids, model_prompts) + for (video, meta), prompt in zip(sampled_vids, model_prompts) ] +def sample_frames_with_video_metadata( + video_with_meta: tuple[npt.NDArray, dict[str, Any]], + num_frames: int, +) -> tuple[npt.NDArray, dict[str, Any]]: + video, meta = video_with_meta + video = sample_frames_from_video(video, num_frames) + + meta["do_sample_frames"] = meta["total_num_frames"] == num_frames + meta["total_num_frames"] = num_frames + meta["fps"] = meta["duration"] / num_frames + meta["frames_indices"] = list(range(num_frames)) + return video, meta + + def apply_image_size_scaling(image, size: float | tuple[int, int], size_type: SizeType): """Applies a size scaler to one image; this can be an image size factor, which scales the image while maintaining the aspect ratio""" diff --git a/tests/models/multimodal/generation/vlm_utils/case_filtering.py b/tests/models/multimodal/generation/vlm_utils/case_filtering.py index 77e478e53c1fd..d42150bcbf672 100644 --- a/tests/models/multimodal/generation/vlm_utils/case_filtering.py +++ b/tests/models/multimodal/generation/vlm_utils/case_filtering.py @@ -100,6 +100,9 @@ def get_parametrized_options( # num_frames is video only if test_type == VLMTestType.VIDEO: iter_kwargs["num_video_frames"] = ensure_wrapped(test_info.num_video_frames) + iter_kwargs["needs_video_metadata"] = ensure_wrapped( + test_info.needs_video_metadata + ) # No sizes passed for custom inputs, since inputs are directly provided if test_type not in (VLMTestType.CUSTOM_INPUTS, VLMTestType.AUDIO): diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 0685a01da58ff..87cd5c3cd3554 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -905,6 +905,54 @@ def qwen2_5_omni_patch_hf_runner(hf_model: HfRunner) -> HfRunner: return hf_model +def qwen3_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: + """Patches and returns an instance of the HfRunner to use for GLM4.1V.""" + hf_processor = hf_model.processor + + def processor(*args, videos=None, **kwargs): + if videos is not None and is_list_of(videos, tuple): + # batched multi videos + do_sample_frames = {video[1]["do_sample_frames"] for video in videos} + assert len(do_sample_frames) == 1 + if kwargs.get("do_sample_frames") is None: + kwargs["do_sample_frames"] = do_sample_frames + video_metadata = [ + [ + VideoMetadata( + **{k: v for k, v in video[1].items() if k != "do_sample_frames"} + ) + ] + for video in videos + ] + videos = [[video[0]] for video in videos] + elif videos is not None and isinstance(videos, tuple): + # single video + do_sample_frames = videos[1]["do_sample_frames"] + if kwargs.get("do_sample_frames") is None: + kwargs["do_sample_frames"] = do_sample_frames + video_metadata = [ + [ + VideoMetadata( + **{ + k: v + for k, v in videos[1].items() + if k != "do_sample_frames" + } + ) + ] + ] + videos = [[videos[0]]] + else: + video_metadata = None + + return hf_processor( + *args, videos=videos, video_metadata=video_metadata, **kwargs + ) + + hf_model.processor = processor + return hf_model + + def tarsier_patch_hf_runner(hf_model: HfRunner) -> HfRunner: from vllm.model_executor.models.tarsier import get_vision_encoder_info diff --git a/tests/models/multimodal/generation/vlm_utils/runners.py b/tests/models/multimodal/generation/vlm_utils/runners.py index c91ae117b5589..218339ef1dffb 100644 --- a/tests/models/multimodal/generation/vlm_utils/runners.py +++ b/tests/models/multimodal/generation/vlm_utils/runners.py @@ -117,6 +117,7 @@ def run_video_test( video_assets, test_case.size_wrapper, test_case.num_video_frames, + test_case.needs_video_metadata, ) core.run_test( diff --git a/tests/models/multimodal/generation/vlm_utils/types.py b/tests/models/multimodal/generation/vlm_utils/types.py index fe02f71884324..5c1bc6ac28fe3 100644 --- a/tests/models/multimodal/generation/vlm_utils/types.py +++ b/tests/models/multimodal/generation/vlm_utils/types.py @@ -154,7 +154,8 @@ class VLMTestInfo(NamedTuple): dtype: str = "auto" distributed_executor_backend: str | None = None # Only expanded in video tests - num_video_frames: int = 16 + num_video_frames: int | tuple[int] = 16 + needs_video_metadata: bool = False # Fixed image sizes / image size factors; most tests use image_size_factors # The values provided for these two fields will be stacked and expanded @@ -212,5 +213,6 @@ class ExpandableVLMTestArgs(NamedTuple): size_wrapper: ImageSizeWrapper | None = None # Video only num_video_frames: int | None = None + needs_video_metadata: bool = False # Custom inputs only custom_test_opts: CustomTestOptions | None = None diff --git a/vllm/assets/video.py b/vllm/assets/video.py index 8818b59970046..d025368cbd43d 100644 --- a/vllm/assets/video.py +++ b/vllm/assets/video.py @@ -94,7 +94,7 @@ def video_get_metadata(path: str, num_frames: int = -1) -> dict[str, Any]: metadata = { "total_num_frames": num_frames, - "fps": fps, + "fps": duration / num_frames, "duration": duration, "video_backend": "opencv", "frames_indices": list(range(num_frames)), From 9a0d2f0d92908c7fcc17eab63a3dfc998aae53a4 Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Wed, 29 Oct 2025 05:55:51 -0700 Subject: [PATCH 071/127] [CI/Build] Skip cpu offloading test on AMD (#27690) Signed-off-by: zhewenli --- tests/v1/kv_offload/test_cpu_offloading.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/v1/kv_offload/test_cpu_offloading.py b/tests/v1/kv_offload/test_cpu_offloading.py index b654ea4298dbb..a5cb23c4ef0f2 100644 --- a/tests/v1/kv_offload/test_cpu_offloading.py +++ b/tests/v1/kv_offload/test_cpu_offloading.py @@ -12,6 +12,7 @@ from tqdm import tqdm from vllm import LLM, SamplingParams, TokensPrompt from vllm.config import KVEventsConfig, KVTransferConfig from vllm.distributed.kv_events import BlockStored, KVEventBatch +from vllm.platforms import current_platform CPU_BLOCK_SIZES = [16, 48] @@ -63,6 +64,9 @@ class MockSubscriber: self.sub.close() +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="CPU offloading only supported on CUDA" +) @pytest.mark.parametrize("cpu_block_size", CPU_BLOCK_SIZES) def test_cpu_offloading(cpu_block_size: int) -> None: """ From ecca3fee761c9dd710daf3acb2e646d10fb631d7 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Wed, 29 Oct 2025 20:59:48 +0800 Subject: [PATCH 072/127] [Frontend] Add `vllm bench sweep` to CLI (#27639) Signed-off-by: DarkLight1337 Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- docs/cli/.nav.yml | 2 +- docs/cli/bench/sweep/plot.md | 9 + docs/cli/bench/sweep/serve.md | 9 + docs/cli/bench/sweep/serve_sla.md | 9 + docs/contributing/benchmarks.md | 6 +- docs/mkdocs/hooks/generate_argparse.py | 34 +- setup.py | 2 +- tools/profiler/visualize_layerwise_profile.py | 12 +- vllm/benchmarks/sweep/cli.py | 38 +++ vllm/benchmarks/sweep/plot.py | 312 ++++++++++-------- vllm/benchmarks/sweep/serve.py | 15 +- vllm/benchmarks/sweep/serve_sla.py | 22 +- vllm/entrypoints/cli/__init__.py | 2 + vllm/entrypoints/cli/benchmark/base.py | 2 +- vllm/entrypoints/cli/benchmark/latency.py | 2 +- vllm/entrypoints/cli/benchmark/serve.py | 2 +- vllm/entrypoints/cli/benchmark/sweep.py | 21 ++ vllm/entrypoints/cli/benchmark/throughput.py | 2 +- vllm/profiler/layerwise_profile.py | 7 +- 19 files changed, 340 insertions(+), 168 deletions(-) create mode 100644 docs/cli/bench/sweep/plot.md create mode 100644 docs/cli/bench/sweep/serve.md create mode 100644 docs/cli/bench/sweep/serve_sla.md create mode 100644 vllm/benchmarks/sweep/cli.py create mode 100644 vllm/entrypoints/cli/benchmark/sweep.py diff --git a/docs/cli/.nav.yml b/docs/cli/.nav.yml index 6c2c09d566a3a..d2d2905703ec5 100644 --- a/docs/cli/.nav.yml +++ b/docs/cli/.nav.yml @@ -5,4 +5,4 @@ nav: - complete.md - run-batch.md - vllm bench: - - bench/*.md + - bench/**/*.md diff --git a/docs/cli/bench/sweep/plot.md b/docs/cli/bench/sweep/plot.md new file mode 100644 index 0000000000000..f29bffb64655c --- /dev/null +++ b/docs/cli/bench/sweep/plot.md @@ -0,0 +1,9 @@ +# vllm bench sweep plot + +## JSON CLI Arguments + +--8<-- "docs/cli/json_tip.inc.md" + +## Options + +--8<-- "docs/argparse/bench_sweep_plot.md" diff --git a/docs/cli/bench/sweep/serve.md b/docs/cli/bench/sweep/serve.md new file mode 100644 index 0000000000000..5b5f91a951ed0 --- /dev/null +++ b/docs/cli/bench/sweep/serve.md @@ -0,0 +1,9 @@ +# vllm bench sweep serve + +## JSON CLI Arguments + +--8<-- "docs/cli/json_tip.inc.md" + +## Options + +--8<-- "docs/argparse/bench_sweep_serve.md" diff --git a/docs/cli/bench/sweep/serve_sla.md b/docs/cli/bench/sweep/serve_sla.md new file mode 100644 index 0000000000000..5f8ab6005e50b --- /dev/null +++ b/docs/cli/bench/sweep/serve_sla.md @@ -0,0 +1,9 @@ +# vllm bench sweep serve_sla + +## JSON CLI Arguments + +--8<-- "docs/cli/json_tip.inc.md" + +## Options + +--8<-- "docs/argparse/bench_sweep_serve_sla.md" diff --git a/docs/contributing/benchmarks.md b/docs/contributing/benchmarks.md index e8b58dbbc93e6..be3e32a73a332 100644 --- a/docs/contributing/benchmarks.md +++ b/docs/contributing/benchmarks.md @@ -1061,7 +1061,7 @@ Follow these steps to run the script: Example command: ```bash -python -m vllm.benchmarks.sweep.serve \ +vllm bench sweep serve \ --serve-cmd 'vllm serve meta-llama/Llama-2-7b-chat-hf' \ --bench-cmd 'vllm bench serve --model meta-llama/Llama-2-7b-chat-hf --backend vllm --endpoint /v1/completions --dataset-name sharegpt --dataset-path benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json' \ --serve-params benchmarks/serve_hparams.json \ @@ -1109,7 +1109,7 @@ For example, to ensure E2E latency within different target values for 99% of req Example command: ```bash -python -m vllm.benchmarks.sweep.serve_sla \ +vllm bench sweep serve_sla \ --serve-cmd 'vllm serve meta-llama/Llama-2-7b-chat-hf' \ --bench-cmd 'vllm bench serve --model meta-llama/Llama-2-7b-chat-hf --backend vllm --endpoint /v1/completions --dataset-name sharegpt --dataset-path benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json' \ --serve-params benchmarks/serve_hparams.json \ @@ -1138,7 +1138,7 @@ The algorithm for adjusting the SLA variable is as follows: Example command: ```bash -python -m vllm.benchmarks.sweep.plot benchmarks/results/ \ +vllm bench sweep plot benchmarks/results/ \ --var-x max_concurrency \ --row-by random_input_len \ --col-by random_output_len \ diff --git a/docs/mkdocs/hooks/generate_argparse.py b/docs/mkdocs/hooks/generate_argparse.py index 99d9a7bec3994..ea89108f01fc2 100644 --- a/docs/mkdocs/hooks/generate_argparse.py +++ b/docs/mkdocs/hooks/generate_argparse.py @@ -56,15 +56,20 @@ def auto_mock(module, attr, max_mocks=50): ) -latency = auto_mock("vllm.benchmarks", "latency") -serve = auto_mock("vllm.benchmarks", "serve") -throughput = auto_mock("vllm.benchmarks", "throughput") +bench_latency = auto_mock("vllm.benchmarks", "latency") +bench_serve = auto_mock("vllm.benchmarks", "serve") +bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs") +bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs") +bench_sweep_serve_sla = auto_mock( + "vllm.benchmarks.sweep.serve_sla", "SweepServeSLAArgs" +) +bench_throughput = auto_mock("vllm.benchmarks", "throughput") AsyncEngineArgs = auto_mock("vllm.engine.arg_utils", "AsyncEngineArgs") EngineArgs = auto_mock("vllm.engine.arg_utils", "EngineArgs") ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand") CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand") -cli_args = auto_mock("vllm.entrypoints.openai", "cli_args") -run_batch = auto_mock("vllm.entrypoints.openai", "run_batch") +openai_cli_args = auto_mock("vllm.entrypoints.openai", "cli_args") +openai_run_batch = auto_mock("vllm.entrypoints.openai", "run_batch") FlexibleArgumentParser = auto_mock( "vllm.utils.argparse_utils", "FlexibleArgumentParser" ) @@ -114,6 +119,9 @@ class MarkdownFormatter(HelpFormatter): self._markdown_output.append(f"{action.help}\n\n") if (default := action.default) != SUPPRESS: + # Make empty string defaults visible + if default == "": + default = '""' self._markdown_output.append(f"Default: `{default}`\n\n") def format_help(self): @@ -150,17 +158,23 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool): # Create parsers to document parsers = { + # Engine args "engine_args": create_parser(EngineArgs.add_cli_args), "async_engine_args": create_parser( AsyncEngineArgs.add_cli_args, async_args_only=True ), - "serve": create_parser(cli_args.make_arg_parser), + # CLI + "serve": create_parser(openai_cli_args.make_arg_parser), "chat": create_parser(ChatCommand.add_cli_args), "complete": create_parser(CompleteCommand.add_cli_args), - "bench_latency": create_parser(latency.add_cli_args), - "bench_throughput": create_parser(throughput.add_cli_args), - "bench_serve": create_parser(serve.add_cli_args), - "run-batch": create_parser(run_batch.make_arg_parser), + "run-batch": create_parser(openai_run_batch.make_arg_parser), + # Benchmark CLI + "bench_latency": create_parser(bench_latency.add_cli_args), + "bench_serve": create_parser(bench_serve.add_cli_args), + "bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args), + "bench_sweep_serve": create_parser(bench_sweep_serve.add_cli_args), + "bench_sweep_serve_sla": create_parser(bench_sweep_serve_sla.add_cli_args), + "bench_throughput": create_parser(bench_throughput.add_cli_args), } # Generate documentation for each parser diff --git a/setup.py b/setup.py index 990fe4cde3ca7..83a4e3eea57c8 100644 --- a/setup.py +++ b/setup.py @@ -709,7 +709,7 @@ setup( ext_modules=ext_modules, install_requires=get_requirements(), extras_require={ - "bench": ["pandas", "datasets"], + "bench": ["pandas", "matplotlib", "seaborn", "datasets"], "tensorizer": ["tensorizer==2.10.1"], "fastsafetensors": ["fastsafetensors >= 0.1.10"], "runai": ["runai-model-streamer[s3,gcs] >= 0.14.0"], diff --git a/tools/profiler/visualize_layerwise_profile.py b/tools/profiler/visualize_layerwise_profile.py index a049dc0425dd6..ed4bf0beb716b 100644 --- a/tools/profiler/visualize_layerwise_profile.py +++ b/tools/profiler/visualize_layerwise_profile.py @@ -141,7 +141,7 @@ def attempt_to_make_names_unique(entries_and_traces): """ -def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame: +def group_trace_by_operations(trace_df: "pd.DataFrame") -> "pd.DataFrame": def is_rms_norm(op_name: str): if "rms_norm_kernel" in op_name: return True @@ -370,12 +370,12 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame: def plot_trace_df( - traces_df: pd.DataFrame, + traces_df: "pd.DataFrame", plot_metric: str, plot_title: str, output: Path | None = None, ): - def get_phase_description(traces_df: pd.DataFrame, phase: str) -> str: + def get_phase_description(traces_df: "pd.DataFrame", phase: str) -> str: phase_df = traces_df.query(f'phase == "{phase}"') descs = phase_df["phase_desc"].to_list() assert all([desc == descs[0] for desc in descs]) @@ -438,7 +438,7 @@ def main( top_k: int, json_nodes_to_fold: list[str], ): - def prepare_data(profile_json: dict, step_keys: list[str]) -> pd.DataFrame: + def prepare_data(profile_json: dict, step_keys: list[str]) -> "pd.DataFrame": def get_entries_and_traces(key: str): entries_and_traces: list[tuple[Any, Any]] = [] for root in profile_json[key]["summary_stats"]: @@ -449,8 +449,8 @@ def main( return entries_and_traces def keep_only_top_entries( - df: pd.DataFrame, metric: str, top_k: int = 9 - ) -> pd.DataFrame: + df: "pd.DataFrame", metric: str, top_k: int = 9 + ) -> "pd.DataFrame": df.loc[df.nsmallest(len(df) - top_k + 1, metric).index, ["name"]] = "others" return df diff --git a/vllm/benchmarks/sweep/cli.py b/vllm/benchmarks/sweep/cli.py new file mode 100644 index 0000000000000..108cd75690864 --- /dev/null +++ b/vllm/benchmarks/sweep/cli.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import argparse + +from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG + +from .plot import SweepPlotArgs +from .plot import main as plot_main +from .serve import SweepServeArgs +from .serve import main as serve_main +from .serve_sla import SweepServeSLAArgs +from .serve_sla import main as serve_sla_main + +SUBCOMMANDS = ( + (SweepServeArgs, serve_main), + (SweepServeSLAArgs, serve_sla_main), + (SweepPlotArgs, plot_main), +) + + +def add_cli_args(parser: argparse.ArgumentParser): + subparsers = parser.add_subparsers(required=True, dest="sweep_type") + + for cmd, entrypoint in SUBCOMMANDS: + cmd_subparser = subparsers.add_parser( + cmd.parser_name, + description=cmd.parser_help, + usage=f"vllm bench sweep {cmd.parser_name} [options]", + ) + cmd_subparser.set_defaults(dispatch_function=entrypoint) + cmd.add_cli_args(cmd_subparser) + cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format( + subcmd=f"sweep {cmd.parser_name}" + ) + + +def main(args: argparse.Namespace): + args.dispatch_function(args) diff --git a/vllm/benchmarks/sweep/plot.py b/vllm/benchmarks/sweep/plot.py index 92485c09b4164..9947d6170d891 100644 --- a/vllm/benchmarks/sweep/plot.py +++ b/vllm/benchmarks/sweep/plot.py @@ -8,16 +8,24 @@ from dataclasses import dataclass from functools import partial from pathlib import Path from types import TracebackType +from typing import ClassVar -import matplotlib.pyplot as plt -import pandas as pd -import seaborn as sns from typing_extensions import Self, override from vllm.utils.collection_utils import full_groupby +from vllm.utils.import_utils import PlaceholderModule from .utils import sanitize_filename +try: + import matplotlib.pyplot as plt + import pandas as pd + import seaborn as sns +except ImportError: + plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot") + pd = PlaceholderModule("pandas") + seaborn = PlaceholderModule("seaborn") + @dataclass class PlotFilterBase(ABC): @@ -40,7 +48,7 @@ class PlotFilterBase(ABC): ) @abstractmethod - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": """Applies this filter to a DataFrame.""" raise NotImplementedError @@ -48,7 +56,7 @@ class PlotFilterBase(ABC): @dataclass class PlotEqualTo(PlotFilterBase): @override - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": try: target = float(self.target) except ValueError: @@ -60,28 +68,28 @@ class PlotEqualTo(PlotFilterBase): @dataclass class PlotLessThan(PlotFilterBase): @override - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": return df[df[self.var] < float(self.target)] @dataclass class PlotLessThanOrEqualTo(PlotFilterBase): @override - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": return df[df[self.var] <= float(self.target)] @dataclass class PlotGreaterThan(PlotFilterBase): @override - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": return df[df[self.var] > float(self.target)] @dataclass class PlotGreaterThanOrEqualTo(PlotFilterBase): @override - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": return df[df[self.var] >= float(self.target)] @@ -103,7 +111,7 @@ class PlotFilters(list[PlotFilterBase]): return cls(PlotFilterBase.parse_str(e) for e in s.split(",")) - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": for item in self: df = item.apply(df) @@ -127,7 +135,7 @@ class PlotBinner: f"Valid operators are: {sorted(PLOT_BINNERS)}", ) - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": """Applies this binner to a DataFrame.""" df = df.copy() df[self.var] = df[self.var] // self.bin_size * self.bin_size @@ -147,7 +155,7 @@ class PlotBinners(list[PlotBinner]): return cls(PlotBinner.parse_str(e) for e in s.split(",")) - def apply(self, df: pd.DataFrame) -> pd.DataFrame: + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": for item in self: df = item.apply(df) @@ -396,135 +404,177 @@ def plot( ) -def add_cli_args(parser: argparse.ArgumentParser): - parser.add_argument( - "OUTPUT_DIR", - type=str, - default="results", - help="The directory containing the results to plot, " - "i.e., the `--output-dir` argument to the parameter sweep script.", - ) - parser.add_argument( - "--fig-dir", - type=str, - default="", - help="The directory to save the figures, relative to `OUTPUT_DIR`. " - "By default, the same directory is used.", - ) - parser.add_argument( - "--fig-by", - type=str, - default="", - help="A comma-separated list of variables, such that a separate figure " - "is created for each combination of these variables.", - ) - parser.add_argument( - "--row-by", - type=str, - default="", - help="A comma-separated list of variables, such that a separate row " - "is created for each combination of these variables.", - ) - parser.add_argument( - "--col-by", - type=str, - default="", - help="A comma-separated list of variables, such that a separate column " - "is created for each combination of these variables.", - ) - parser.add_argument( - "--curve-by", - type=str, - default=None, - help="A comma-separated list of variables, such that a separate curve " - "is created for each combination of these variables.", - ) - parser.add_argument( - "--var-x", - type=str, - default="request_throughput", - help="The variable for the x-axis.", - ) - parser.add_argument( - "--var-y", - type=str, - default="p99_e2el_ms", - help="The variable for the y-axis", - ) - parser.add_argument( - "--filter-by", - type=str, - default="", - help="A comma-separated list of statements indicating values to filter by. " - "This is useful to remove outliers. " - "Example: `max_concurrency<1000,max_num_batched_tokens<=4096` means " - "plot only the points where `max_concurrency` is less than 1000 and " - "`max_num_batched_tokens` is no greater than 4096.", - ) - parser.add_argument( - "--bin-by", - type=str, - default="", - help="A comma-separated list of statements indicating values to bin by. " - "This is useful to avoid plotting points that are too close together. " - "Example: `request_throughput%1` means " - "use a bin size of 1 for the `request_throughput` variable.", - ) - parser.add_argument( - "--scale-x", - type=str, - default=None, - help="The scale to use for the x-axis. " - "Currently only accepts string values such as 'log' and 'sqrt'. " - "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html", - ) - parser.add_argument( - "--scale-y", - type=str, - default=None, - help="The scale to use for the y-axis. " - "Currently only accepts string values such as 'log' and 'sqrt'. " - "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="If set, prints the information about each figure to plot, " - "then exits without drawing them.", - ) +@dataclass +class SweepPlotArgs: + output_dir: Path + fig_dir: Path + fig_by: list[str] + row_by: list[str] + col_by: list[str] + curve_by: list[str] + var_x: str + var_y: str + filter_by: PlotFilters + bin_by: PlotBinners + scale_x: str | None + scale_y: str | None + dry_run: bool + + parser_name: ClassVar[str] = "plot" + parser_help: ClassVar[str] = "Plot performance curves from parameter sweep results." + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + output_dir = Path(args.OUTPUT_DIR) + if not output_dir.exists(): + raise ValueError(f"No parameter sweep results under {output_dir}") + + curve_by = [] if not args.curve_by else args.curve_by.split(",") + row_by = [] if not args.row_by else args.row_by.split(",") + col_by = [] if not args.col_by else args.col_by.split(",") + fig_by = [] if not args.fig_by else args.fig_by.split(",") + + return cls( + output_dir=output_dir, + fig_dir=output_dir / args.fig_dir, + fig_by=fig_by, + row_by=row_by, + col_by=col_by, + curve_by=curve_by, + var_x=args.var_x, + var_y=args.var_y, + filter_by=PlotFilters.parse_str(args.filter_by), + bin_by=PlotBinners.parse_str(args.bin_by), + scale_x=args.scale_x, + scale_y=args.scale_y, + dry_run=args.dry_run, + ) + + @classmethod + def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + parser.add_argument( + "OUTPUT_DIR", + type=str, + default="results", + help="The directory containing the results to plot, " + "i.e., the `--output-dir` argument to the parameter sweep script.", + ) + parser.add_argument( + "--fig-dir", + type=str, + default="", + help="The directory to save the figures, relative to `OUTPUT_DIR`. " + "By default, the same directory is used.", + ) + parser.add_argument( + "--fig-by", + type=str, + default="", + help="A comma-separated list of variables, such that a separate figure " + "is created for each combination of these variables.", + ) + parser.add_argument( + "--row-by", + type=str, + default="", + help="A comma-separated list of variables, such that a separate row " + "is created for each combination of these variables.", + ) + parser.add_argument( + "--col-by", + type=str, + default="", + help="A comma-separated list of variables, such that a separate column " + "is created for each combination of these variables.", + ) + parser.add_argument( + "--curve-by", + type=str, + default=None, + help="A comma-separated list of variables, such that a separate curve " + "is created for each combination of these variables.", + ) + parser.add_argument( + "--var-x", + type=str, + default="request_throughput", + help="The variable for the x-axis.", + ) + parser.add_argument( + "--var-y", + type=str, + default="p99_e2el_ms", + help="The variable for the y-axis", + ) + parser.add_argument( + "--filter-by", + type=str, + default="", + help="A comma-separated list of statements indicating values to filter by. " + "This is useful to remove outliers. " + "Example: `max_concurrency<1000,max_num_batched_tokens<=4096` means " + "plot only the points where `max_concurrency` is less than 1000 and " + "`max_num_batched_tokens` is no greater than 4096.", + ) + parser.add_argument( + "--bin-by", + type=str, + default="", + help="A comma-separated list of statements indicating values to bin by. " + "This is useful to avoid plotting points that are too close together. " + "Example: `request_throughput%%1` means " + "use a bin size of 1 for the `request_throughput` variable.", + ) + parser.add_argument( + "--scale-x", + type=str, + default=None, + help="The scale to use for the x-axis. " + "Currently only accepts string values such as 'log' and 'sqrt'. " + "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html", + ) + parser.add_argument( + "--scale-y", + type=str, + default=None, + help="The scale to use for the y-axis. " + "Currently only accepts string values such as 'log' and 'sqrt'. " + "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="If set, prints the information about each figure to plot, " + "then exits without drawing them.", + ) + + return parser -def main(args: argparse.Namespace): - output_dir = Path(args.OUTPUT_DIR) - if not output_dir.exists(): - raise ValueError(f"No parameter sweep results under {output_dir}") - - curve_by = [] if not args.curve_by else args.curve_by.split(",") - row_by = [] if not args.row_by else args.row_by.split(",") - col_by = [] if not args.col_by else args.col_by.split(",") - fig_by = [] if not args.fig_by else args.fig_by.split(",") - - plot( - output_dir=output_dir, - fig_dir=output_dir / args.fig_dir, - fig_by=fig_by, - row_by=row_by, - col_by=col_by, - curve_by=curve_by, +def run_main(args: SweepPlotArgs): + return plot( + output_dir=args.output_dir, + fig_dir=args.fig_dir, + fig_by=args.fig_by, + row_by=args.row_by, + col_by=args.col_by, + curve_by=args.curve_by, var_x=args.var_x, var_y=args.var_y, - filter_by=PlotFilters.parse_str(args.filter_by), - bin_by=PlotBinners.parse_str(args.bin_by), + filter_by=args.filter_by, + bin_by=args.bin_by, scale_x=args.scale_x, scale_y=args.scale_y, dry_run=args.dry_run, ) +def main(args: argparse.Namespace): + run_main(SweepPlotArgs.from_cli_args(args)) + + if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Plot performance curves from parameter sweep results." - ) - add_cli_args(parser) + parser = argparse.ArgumentParser(description=SweepPlotArgs.parser_help) + SweepPlotArgs.add_cli_args(parser) main(parser.parse_args()) diff --git a/vllm/benchmarks/sweep/serve.py b/vllm/benchmarks/sweep/serve.py index a06d4d6d60985..45ac446a7aedf 100644 --- a/vllm/benchmarks/sweep/serve.py +++ b/vllm/benchmarks/sweep/serve.py @@ -7,13 +7,19 @@ import shlex from dataclasses import dataclass from datetime import datetime from pathlib import Path +from typing import ClassVar -import pandas as pd +from vllm.utils.import_utils import PlaceholderModule from .param_sweep import ParameterSweep, ParameterSweepItem from .server import ServerProcess from .utils import sanitize_filename +try: + import pandas as pd +except ImportError: + pd = PlaceholderModule("pandas") + @contextlib.contextmanager def run_server( @@ -257,6 +263,9 @@ class SweepServeArgs: dry_run: bool resume: str | None + parser_name: ClassVar[str] = "serve" + parser_help: ClassVar[str] = "Run vLLM server benchmark under multiple settings." + @classmethod def from_cli_args(cls, args: argparse.Namespace): serve_cmd = shlex.split(args.serve_cmd) @@ -401,9 +410,7 @@ def main(args: argparse.Namespace): if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Run vLLM server benchmark under multiple settings." - ) + parser = argparse.ArgumentParser(description=SweepServeArgs.parser_help) SweepServeArgs.add_cli_args(parser) main(parser.parse_args()) diff --git a/vllm/benchmarks/sweep/serve_sla.py b/vllm/benchmarks/sweep/serve_sla.py index 6159aba4bbb5a..0403d1ddfd6c1 100644 --- a/vllm/benchmarks/sweep/serve_sla.py +++ b/vllm/benchmarks/sweep/serve_sla.py @@ -7,17 +7,23 @@ import math from dataclasses import asdict, dataclass from datetime import datetime from pathlib import Path -from typing import Literal, get_args +from typing import ClassVar, Literal, get_args -import pandas as pd from typing_extensions import assert_never +from vllm.utils.import_utils import PlaceholderModule + from .param_sweep import ParameterSweep, ParameterSweepItem from .serve import SweepServeArgs, run_benchmark, run_server from .server import ServerProcess from .sla_sweep import SLASweep, SLASweepItem from .utils import sanitize_filename +try: + import pandas as pd +except ImportError: + pd = PlaceholderModule("pandas") + def _get_sla_base_path( output_dir: Path, @@ -399,6 +405,9 @@ class SweepServeSLAArgs(SweepServeArgs): sla_params: SLASweep sla_variable: SLAVariable + parser_name: ClassVar[str] = "serve_sla" + parser_help: ClassVar[str] = "Tune a variable to meet SLAs under multiple settings." + @classmethod def from_cli_args(cls, args: argparse.Namespace): # NOTE: Don't use super() as `from_cli_args` calls `cls()` @@ -419,7 +428,8 @@ class SweepServeSLAArgs(SweepServeArgs): def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: parser = super().add_cli_args(parser) - parser.add_argument( + sla_group = parser.add_argument_group("sla options") + sla_group.add_argument( "--sla-params", type=str, required=True, @@ -431,7 +441,7 @@ class SweepServeSLAArgs(SweepServeArgs): "the maximum `sla_variable` that satisfies the constraints for " "each combination of `serve_params`, `bench_params`, and `sla_params`.", ) - parser.add_argument( + sla_group.add_argument( "--sla-variable", type=str, choices=get_args(SLAVariable), @@ -476,9 +486,7 @@ def main(args: argparse.Namespace): if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Tune a variable to meet SLAs under multiple settings." - ) + parser = argparse.ArgumentParser(description=SweepServeSLAArgs.parser_help) SweepServeSLAArgs.add_cli_args(parser) main(parser.parse_args()) diff --git a/vllm/entrypoints/cli/__init__.py b/vllm/entrypoints/cli/__init__.py index 211e157fc7c82..9dff68236fe94 100644 --- a/vllm/entrypoints/cli/__init__.py +++ b/vllm/entrypoints/cli/__init__.py @@ -2,10 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from vllm.entrypoints.cli.benchmark.latency import BenchmarkLatencySubcommand from vllm.entrypoints.cli.benchmark.serve import BenchmarkServingSubcommand +from vllm.entrypoints.cli.benchmark.sweep import BenchmarkSweepSubcommand from vllm.entrypoints.cli.benchmark.throughput import BenchmarkThroughputSubcommand __all__: list[str] = [ "BenchmarkLatencySubcommand", "BenchmarkServingSubcommand", + "BenchmarkSweepSubcommand", "BenchmarkThroughputSubcommand", ] diff --git a/vllm/entrypoints/cli/benchmark/base.py b/vllm/entrypoints/cli/benchmark/base.py index 3263459fd6810..d8543822cf6e1 100644 --- a/vllm/entrypoints/cli/benchmark/base.py +++ b/vllm/entrypoints/cli/benchmark/base.py @@ -6,7 +6,7 @@ from vllm.entrypoints.cli.types import CLISubcommand class BenchmarkSubcommandBase(CLISubcommand): - """The base class of subcommands for vllm bench.""" + """The base class of subcommands for `vllm bench`.""" help: str diff --git a/vllm/entrypoints/cli/benchmark/latency.py b/vllm/entrypoints/cli/benchmark/latency.py index 548ddf4d603e7..60f2b03341b1c 100644 --- a/vllm/entrypoints/cli/benchmark/latency.py +++ b/vllm/entrypoints/cli/benchmark/latency.py @@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase class BenchmarkLatencySubcommand(BenchmarkSubcommandBase): - """The `latency` subcommand for vllm bench.""" + """The `latency` subcommand for `vllm bench`.""" name = "latency" help = "Benchmark the latency of a single batch of requests." diff --git a/vllm/entrypoints/cli/benchmark/serve.py b/vllm/entrypoints/cli/benchmark/serve.py index b085f52afb3b3..6616305c7472f 100644 --- a/vllm/entrypoints/cli/benchmark/serve.py +++ b/vllm/entrypoints/cli/benchmark/serve.py @@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase class BenchmarkServingSubcommand(BenchmarkSubcommandBase): - """The `serve` subcommand for vllm bench.""" + """The `serve` subcommand for `vllm bench`.""" name = "serve" help = "Benchmark the online serving throughput." diff --git a/vllm/entrypoints/cli/benchmark/sweep.py b/vllm/entrypoints/cli/benchmark/sweep.py new file mode 100644 index 0000000000000..c385207690a15 --- /dev/null +++ b/vllm/entrypoints/cli/benchmark/sweep.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import argparse + +from vllm.benchmarks.sweep.cli import add_cli_args, main +from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase + + +class BenchmarkSweepSubcommand(BenchmarkSubcommandBase): + """The `sweep` subcommand for `vllm bench`.""" + + name = "sweep" + help = "Benchmark for a parameter sweep." + + @classmethod + def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: + add_cli_args(parser) + + @staticmethod + def cmd(args: argparse.Namespace) -> None: + main(args) diff --git a/vllm/entrypoints/cli/benchmark/throughput.py b/vllm/entrypoints/cli/benchmark/throughput.py index c25be75ec11e2..2097f9ea0781a 100644 --- a/vllm/entrypoints/cli/benchmark/throughput.py +++ b/vllm/entrypoints/cli/benchmark/throughput.py @@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase): - """The `throughput` subcommand for vllm bench.""" + """The `throughput` subcommand for `vllm bench`.""" name = "throughput" help = "Benchmark offline inference throughput." diff --git a/vllm/profiler/layerwise_profile.py b/vllm/profiler/layerwise_profile.py index 1c0fce702b3fa..829b63d8a79d0 100644 --- a/vllm/profiler/layerwise_profile.py +++ b/vllm/profiler/layerwise_profile.py @@ -7,7 +7,6 @@ from collections.abc import Callable from dataclasses import asdict, dataclass, field from typing import Any, Optional, TypeAlias -import pandas as pd from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent from torch.autograd.profiler import FunctionEvent @@ -21,6 +20,12 @@ from vllm.profiler.utils import ( event_torch_op_stack_trace, indent_string, ) +from vllm.utils.import_utils import PlaceholderModule + +try: + import pandas as pd +except ImportError: + pd = PlaceholderModule("pandas") @dataclass From d6704dd099b71a9881613dee74b440d78594fa8f Mon Sep 17 00:00:00 2001 From: Roger Young <42564206+rogeryoungh@users.noreply.github.com> Date: Wed, 29 Oct 2025 21:01:05 +0800 Subject: [PATCH 073/127] Fix MiniMax-M2 rmsnorm precision and remove useless code (#27627) Signed-off-by: xuebi Co-authored-by: xuebi --- .../model_executor/layers/mamba/linear_attn.py | 2 +- vllm/model_executor/models/minimax_m2.py | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/vllm/model_executor/layers/mamba/linear_attn.py b/vllm/model_executor/layers/mamba/linear_attn.py index fd4567ee47018..0a2742ff49a44 100644 --- a/vllm/model_executor/layers/mamba/linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear_attn.py @@ -77,7 +77,7 @@ class MiniMaxText01RMSNormTP(CustomOp): if self.tp_world > 1: variance = tensor_model_parallel_all_reduce(variance) / self.tp_world x = x * torch.rsqrt(variance + self.variance_epsilon) - x = x.to(orig_dtype) * self.weight + x = (x * self.weight).to(orig_dtype) return x def forward( diff --git a/vllm/model_executor/models/minimax_m2.py b/vllm/model_executor/models/minimax_m2.py index dadb8a19c004e..21ed428a05d0f 100644 --- a/vllm/model_executor/models/minimax_m2.py +++ b/vllm/model_executor/models/minimax_m2.py @@ -263,23 +263,6 @@ class MiniMaxM2DecoderLayer(nn.Module): # with the layer's index. layer_idx = int(prefix.split(sep=".")[-1]) - # TODO: support MTP - attn_window_size = getattr(config, "attn_window_size", None) - if attn_window_size is not None: - if isinstance(attn_window_size, list): - attn_window_size = attn_window_size[layer_idx] - elif isinstance(attn_window_size, int): - attn_window_size = attn_window_size - else: - raise ValueError(f"Invalid attn_window_size: {attn_window_size}") - attn_window_size = None if attn_window_size <= 0 else attn_window_size - - # different rope theta for full layer and swa layer - swa_rope_theta = getattr(config, "swa_rope_theta", -1) - # default to full rope theta - swa_rope_theta = rope_theta if swa_rope_theta <= 0 else swa_rope_theta - rope_theta = swa_rope_theta if attn_window_size is not None else rope_theta - self.layer_idx = layer_idx self.self_attn = MiniMaxM2Attention( hidden_size=self.hidden_size, @@ -288,7 +271,6 @@ class MiniMaxM2DecoderLayer(nn.Module): rotary_dim=config.rotary_dim, rope_theta=rope_theta, rope_scaling=rope_scaling, - attn_window_size=attn_window_size, max_position_embeddings=max_position_embeddings, rms_norm_eps=config.rms_norm_eps, qkv_bias=getattr(config, "attention_bias", False), From ded24e3e547dcc4ac1418316d81b5209b64c3d92 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Wed, 29 Oct 2025 22:44:03 +0800 Subject: [PATCH 074/127] [ROCm][Platform] Add MI308X device id in _ROCM_DEVICE_ID_NAME_MAP (#27623) Signed-off-by: Xiake Sun --- vllm/platforms/rocm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index e67a7a7e70f7d..d3535c9781c48 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -72,6 +72,7 @@ _ROCM_DEVICE_ID_NAME_MAP: dict[str, str] = { "0x74a0": "AMD_Instinct_MI300A", "0x74a1": "AMD_Instinct_MI300X", "0x74b5": "AMD_Instinct_MI300X", # MI300X VF + "0x74a2": "AMD_Instinct_MI308X", "0x74a5": "AMD_Instinct_MI325X", "0x74b9": "AMD_Instinct_MI325X", # MI325X VF "0x74a9": "AMD_Instinct_MI300X_HF", From 0f95a1c3f2d7b1b5e2ab8193cfb679dd8f64de63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Wed, 29 Oct 2025 16:10:35 +0100 Subject: [PATCH 075/127] [CI] Fix flaky `test_two_responses_with_same_prev_id` test (#27745) Signed-off-by: NickLucche --- tests/v1/entrypoints/openai/responses/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/v1/entrypoints/openai/responses/conftest.py b/tests/v1/entrypoints/openai/responses/conftest.py index ad7594a3dd6dd..032ed42f43d1b 100644 --- a/tests/v1/entrypoints/openai/responses/conftest.py +++ b/tests/v1/entrypoints/openai/responses/conftest.py @@ -6,7 +6,7 @@ import pytest_asyncio from tests.utils import RemoteOpenAIServer # Use a small reasoning model to test the responses API. -MODEL_NAME = "Qwen/Qwen3-0.6B" +MODEL_NAME = "Qwen/Qwen3-1.7B" @pytest.fixture(scope="module") From 5522fb274bf384f838c015798be891f9fbf7172f Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 29 Oct 2025 12:05:09 -0400 Subject: [PATCH 076/127] [Chore] Optimize P2PNCCLEngine `http_address` (#27488) Signed-off-by: yewentao256 --- .../kv_connector/v1/p2p/p2p_nccl_engine.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_engine.py b/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_engine.py index 3ef287817c39c..0e748db666e64 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_engine.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_engine.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json import logging import os import threading @@ -96,19 +97,30 @@ class P2pNcclEngine: # Each card corresponds to a ZMQ address. self.zmq_address = f"{self._hostname}:{self._port}" - # The `http_port` must be consistent with the port of OpenAI. - self.http_address = ( - f"{self._hostname}:{self.config.kv_connector_extra_config['http_port']}" - ) - # If `proxy_ip` or `proxy_port` is `""`, # then the ping thread will not be enabled. proxy_ip = self.config.get_from_extra_config("proxy_ip", "") proxy_port = self.config.get_from_extra_config("proxy_port", "") if proxy_ip == "" or proxy_port == "": self.proxy_address = "" + self.http_address = "" else: self.proxy_address = proxy_ip + ":" + proxy_port + # the `http_port` must be consistent with the port of OpenAI. + http_port = self.config.get_from_extra_config("http_port", None) + if http_port is None: + example_cfg = { + "kv_connector": "P2pNcclConnector", + "kv_connector_extra_config": {"http_port": 8000}, + } + example = ( + f"--port=8000 --kv-transfer-config='{json.dumps(example_cfg)}'" + ) + raise ValueError( + "kv_connector_extra_config.http_port is required. " + f"Example: {example}" + ) + self.http_address = f"{self._hostname}:{http_port}" self.context = zmq.Context() self.router_socket = self.context.socket(zmq.ROUTER) From 1da3309acefd156a1a834446b828d5e4065822df Mon Sep 17 00:00:00 2001 From: Braulio Dumba Date: Wed, 29 Oct 2025 12:32:01 -0400 Subject: [PATCH 077/127] [Core] Exposing engine sleep & wake_up state as prometheus metrics (#24176) Signed-off-by: Braulio Dumba --- tests/entrypoints/openai/test_sleep.py | 49 +++++++++++++++++++++++ vllm/v1/engine/async_llm.py | 6 +++ vllm/v1/engine/llm_engine.py | 6 +++ vllm/v1/metrics/loggers.py | 54 +++++++++++++++++++++++++- 4 files changed, 114 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_sleep.py b/tests/entrypoints/openai/test_sleep.py index e07436f89d2d2..5f94ac6da2c25 100644 --- a/tests/entrypoints/openai/test_sleep.py +++ b/tests/entrypoints/openai/test_sleep.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import requests +from prometheus_client.parser import text_string_to_metric_families from ...utils import RemoteOpenAIServer @@ -31,12 +32,28 @@ def test_sleep_mode(): assert response.status_code == 200 assert response.json().get("is_sleeping") is True + # check sleep metrics + response = requests.get(remote_server.url_for("metrics")) + assert response.status_code == 200 + awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response) + assert awake == 0 + assert weights_offloaded == 1 + assert discard_all == 0 + response = requests.post(remote_server.url_for("wake_up")) assert response.status_code == 200 response = requests.get(remote_server.url_for("is_sleeping")) assert response.status_code == 200 assert response.json().get("is_sleeping") is False + # check sleep metrics + response = requests.get(remote_server.url_for("metrics")) + assert response.status_code == 200 + awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response) + assert awake == 1 + assert weights_offloaded == 0 + assert discard_all == 0 + # test wake up with tags response = requests.post(remote_server.url_for("sleep"), params={"level": "1"}) assert response.status_code == 200 @@ -59,3 +76,35 @@ def test_sleep_mode(): response = requests.get(remote_server.url_for("is_sleeping")) assert response.status_code == 200 assert response.json().get("is_sleeping") is False + + # check sleep metrics + response = requests.get(remote_server.url_for("metrics")) + assert response.status_code == 200 + awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response) + assert awake == 1 + assert weights_offloaded == 0 + assert discard_all == 0 + + +def _get_sleep_metrics_from_api(response: requests.Response): + """Return (awake, weights_offloaded, discard_all)""" + + awake, weights_offloaded, discard_all = None, None, None + + for family in text_string_to_metric_families(response.text): + if family.name == "vllm:engine_sleep_state": + for sample in family.samples: + if sample.name == "vllm:engine_sleep_state": + for label_name, label_value in sample.labels.items(): + if label_value == "awake": + awake = sample.value + elif label_value == "weights_offloaded": + weights_offloaded = sample.value + elif label_value == "discard_all": + discard_all = sample.value + + assert awake is not None + assert weights_offloaded is not None + assert discard_all is not None + + return awake, weights_offloaded, discard_all diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index cf458a8f074c0..761c37504d80a 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -689,9 +689,15 @@ class AsyncLLM(EngineClient): await self.reset_prefix_cache() await self.engine_core.sleep_async(level) + if self.logger_manager is not None: + self.logger_manager.record_sleep_state(1, level) + async def wake_up(self, tags: list[str] | None = None) -> None: await self.engine_core.wake_up_async(tags) + if self.logger_manager is not None: + self.logger_manager.record_sleep_state(0, 0) + async def is_sleeping(self) -> bool: return await self.engine_core.is_sleeping_async() diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 486dacb2e5d9c..0fce343702e0a 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -332,9 +332,15 @@ class LLMEngine: def sleep(self, level: int = 1): self.engine_core.sleep(level) + if self.logger_manager is not None: + self.logger_manager.record_sleep_state(1, level) + def wake_up(self, tags: list[str] | None = None): self.engine_core.wake_up(tags) + if self.logger_manager is not None: + self.logger_manager.record_sleep_state(0, 0) + def is_sleeping(self) -> bool: return self.engine_core.is_sleeping() diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index c5d7885eefb79..055da5d856b25 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -9,6 +9,7 @@ from typing import TypeAlias from prometheus_client import Counter, Gauge, Histogram +import vllm.envs as envs from vllm.config import SupportsMetricsInfo, VllmConfig from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorLogging from vllm.logger import init_logger @@ -56,6 +57,9 @@ class StatLoggerBase(ABC): def log(self): # noqa pass + def record_sleep_state(self, is_awake: int, level: int): # noqa + pass + def load_stat_logger_plugin_factories() -> list[StatLoggerFactory]: factories: list[StatLoggerFactory] = [] @@ -384,8 +388,33 @@ class PrometheusStatLogger(AggregateStatLoggerBase): self.gauge_scheduler_waiting = make_per_engine( gauge_scheduler_waiting, engine_indexes, model_name ) + if envs.VLLM_SERVER_DEV_MODE: + gauge_engine_sleep_state = self._gauge_cls( + name="vllm:engine_sleep_state", + documentation=( + "Engine sleep state; awake = 0 means engine is sleeping; " + "awake = 1 means engine is awake; " + "weights_offloaded = 1 means sleep level 1; " + "discard_all = 1 means sleep level 2." + ), + labelnames=labelnames + ["sleep_state"], + multiprocess_mode="mostrecent", + ) + + self.gauge_engine_sleep_state = {} + sleep_state = ["awake", "weights_offloaded", "discard_all"] + + for s in sleep_state: + self.gauge_engine_sleep_state[s] = { + idx: gauge_engine_sleep_state.labels( + engine=idx, model_name=model_name, sleep_state=s + ) + for idx in engine_indexes + } + + # Setting default values + self.record_sleep_state() - # # GPU cache # # Deprecated in 0.9.2 - Renamed as vllm:kv_cache_usage_perc @@ -1010,6 +1039,25 @@ class PrometheusStatLogger(AggregateStatLoggerBase): } self.gauge_lora_info.labels(**lora_info_labels).set_to_current_time() + def record_sleep_state(self, sleep: int = 0, level: int = 0): + awake = 1 + discard_all = 0 + weights_offloaded = 0 + + if sleep == 1: + awake = 0 + if level == 1: + weights_offloaded = 1 + elif level == 2: + discard_all = 1 + + for engine_idx in self.engine_indexes: + self.gauge_engine_sleep_state["discard_all"][engine_idx].set(discard_all) + self.gauge_engine_sleep_state["weights_offloaded"][engine_idx].set( + weights_offloaded + ) + self.gauge_engine_sleep_state["awake"][engine_idx].set(awake) + def log_engine_initialized(self): self.log_metrics_info("cache_config", self.vllm_config.cache_config) @@ -1131,6 +1179,10 @@ class StatLoggerManager: engine_idx=engine_idx, ) + def record_sleep_state(self, sleep: int = 0, level: int = 0): + for logger in self.stat_loggers: + logger.record_sleep_state(sleep, level) + def log(self): for logger in self.stat_loggers: logger.log() From 7568a282b90f012e199ff71e7813f186a51addec Mon Sep 17 00:00:00 2001 From: JartX Date: Wed, 29 Oct 2025 17:55:35 +0100 Subject: [PATCH 078/127] [FIXBUG] Qwen3VL hallucinations without Contiguous on Torch.SDPA (#27744) Signed-off-by: JartX Co-authored-by: Lukas Geiger --- vllm/model_executor/models/qwen2_5_vl.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index c68115729c425..41cb7084057dd 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -428,6 +428,14 @@ class Qwen2_5_VisionAttention(nn.Module): ) elif self.attn_backend == _Backend.TORCH_SDPA: # Execute attention entry by entry for speed & less VRAM. + from vllm.platforms import current_platform + + # Never remove the next contiguous logic + # Without it, hallucinations occur with the backend + if current_platform.is_rocm(): + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() outputs = [] for i in range(1, len(cu_seqlens)): start_idx = cu_seqlens[i - 1] From a9fe0793f2697928a0eacba9f8664646c547941c Mon Sep 17 00:00:00 2001 From: Boyuan Feng Date: Wed, 29 Oct 2025 10:08:54 -0700 Subject: [PATCH 079/127] `use_aot_compile` should respect `VLLM_DISABLE_COMPILE_CACHE` (#27698) Signed-off-by: Boyuan Feng --- vllm/envs.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index ca1f84bba419d..0548f01fc8cdf 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -247,10 +247,19 @@ def maybe_convert_bool(value: str | None) -> bool | None: return bool(int(value)) +def disable_compile_cache() -> bool: + return bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0"))) + + def use_aot_compile() -> bool: from vllm.utils.torch_utils import is_torch_equal_or_newer - default_value = "1" if is_torch_equal_or_newer("2.10.0.dev") else "0" + default_value = ( + "1" + if is_torch_equal_or_newer("2.10.0.dev") and not disable_compile_cache() + else "0" + ) + return os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1" @@ -963,9 +972,7 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_LOG_BATCHSIZE_INTERVAL": lambda: float( os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1") ), - "VLLM_DISABLE_COMPILE_CACHE": lambda: bool( - int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0")) - ), + "VLLM_DISABLE_COMPILE_CACHE": disable_compile_cache, # If set, vllm will run in development mode, which will enable # some additional endpoints for developing and debugging, # e.g. `/reset_prefix_cache` From f7a66828728db28bfbbacdb1552d78cb620006c3 Mon Sep 17 00:00:00 2001 From: 22quinn <33176974+22quinn@users.noreply.github.com> Date: Wed, 29 Oct 2025 10:26:06 -0700 Subject: [PATCH 080/127] [CI/Build] Test torchrun with 8 cards (#27548) Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com> --- .buildkite/test-pipeline.yaml | 22 ++++- .../offline_inference/torchrun_dp_example.py | 82 +++++++++++++++++-- 2 files changed, 94 insertions(+), 10 deletions(-) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index e166f320f9c3f..d556073cd1049 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -205,6 +205,24 @@ steps: - VLLM_ALLOW_INSECURE_SERIALIZATION=1 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py - popd +- label: Distributed Tests (8 GPUs) # 4min + timeout_in_minutes: 10 + gpu: h100 + num_gpus: 8 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - examples/offline_inference/torchrun_dp_example.py + - vllm/config/parallel.py + - vllm/distributed/ + - vllm/v1/engine/llm_engine.py + - vllm/v1/executor/uniproc_executor.py + - vllm/v1/worker/gpu_worker.py + commands: + # https://github.com/NVIDIA/nccl/issues/1838 + - export NCCL_CUMEM_HOST_ENABLE=0 + # test with torchrun tp=2 and dp=4 with ep + - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep + - label: EPLB Algorithm Test # 5min timeout_in_minutes: 15 working_dir: "/vllm-workspace/tests" @@ -401,7 +419,7 @@ steps: --ignore=lora/test_deepseekv2_tp.py \ --ignore=lora/test_gptoss.py \ --ignore=lora/test_qwen3moe_tp.py - + parallelism: 4 - label: PyTorch Compilation Unit Tests # 15min @@ -1126,7 +1144,7 @@ steps: - tests/weight_loading commands: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt - + - label: NixlConnector PD accuracy tests (Distributed) # 30min timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" diff --git a/examples/offline_inference/torchrun_dp_example.py b/examples/offline_inference/torchrun_dp_example.py index 295d1637528cd..eb7ed969ea4bf 100644 --- a/examples/offline_inference/torchrun_dp_example.py +++ b/examples/offline_inference/torchrun_dp_example.py @@ -9,10 +9,76 @@ To run this example: ```bash $ torchrun --nproc-per-node=2 examples/offline_inference/torchrun_dp_example.py ``` + +With custom parallelism settings: +```bash +$ torchrun --nproc-per-node=8 examples/offline_inference/torchrun_dp_example.py \ + --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep +``` """ +import argparse + from vllm import LLM, SamplingParams + +def parse_args(): + parser = argparse.ArgumentParser( + description="Data-parallel inference with torchrun" + ) + parser.add_argument( + "--tp-size", + type=int, + default=1, + help="Tensor parallel size (default: 1)", + ) + parser.add_argument( + "--pp-size", + type=int, + default=1, + help="Pipeline parallel size (default: 1)", + ) + parser.add_argument( + "--dp-size", + type=int, + default=2, + help="Data parallel size (default: 2)", + ) + parser.add_argument( + "--enable-ep", + action="store_true", + help="Enable expert parallel (default: False)", + ) + parser.add_argument( + "--model", + type=str, + default="microsoft/Phi-mini-MoE-instruct", + help="Model name or path (default: microsoft/Phi-mini-MoE-instruct)", + ) + parser.add_argument( + "--max-model-len", + type=int, + default=4096, + help="Maximum model length (default: 4096)", + ) + parser.add_argument( + "--gpu-memory-utilization", + type=float, + default=0.6, + help="GPU memory utilization (default: 0.6)", + ) + parser.add_argument( + "--seed", + type=int, + default=1, + help="Random seed (default: 1)", + ) + return parser.parse_args() + + +args = parse_args() + + # Create prompts, the same across all ranks prompts = [ "Hello, my name is", @@ -30,15 +96,15 @@ sampling_params = SamplingParams(temperature=0.8, top_p=0.95) # all ranks have the same random seed, so that sampling can be # deterministic across ranks. llm = LLM( - model="microsoft/Phi-mini-MoE-instruct", - tensor_parallel_size=1, - data_parallel_size=2, - pipeline_parallel_size=1, - enable_expert_parallel=False, + model=args.model, + tensor_parallel_size=args.tp_size, + data_parallel_size=args.dp_size, + pipeline_parallel_size=args.pp_size, + enable_expert_parallel=args.enable_ep, distributed_executor_backend="external_launcher", - max_model_len=4096, - gpu_memory_utilization=0.6, - seed=1, + max_model_len=args.max_model_len, + gpu_memory_utilization=args.gpu_memory_utilization, + seed=args.seed, ) dp_rank = llm.llm_engine.vllm_config.parallel_config.data_parallel_rank From 5b0448104fa16ca28711e715f4866d9b2805f171 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:29:20 -0400 Subject: [PATCH 081/127] [Bug] Raise error explicitly if using incompatible backend (#27424) Signed-off-by: yewentao256 --- vllm/platforms/cuda.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index 66cffde9503da..cc06f034fba32 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -261,6 +261,21 @@ class CudaPlatformBase(Platform): from vllm.attention.backends.registry import _Backend if use_mla: + # explicitly reject non-MLA backends when MLA is enabled to avoid + # silently selecting an incompatible backend (e.g., FLASHINFER). + if selected_backend in { + _Backend.FLASHINFER, + _Backend.FLASH_ATTN, + _Backend.TRITON_ATTN, + _Backend.TREE_ATTN, + _Backend.XFORMERS, + }: + raise ValueError( + f"Attention backend {selected_backend} incompatible with MLA. " + "Please use one of the MLA backends: FLASHINFER_MLA, CUTLASS_MLA, " + "FLASHMLA, FLASH_ATTN_MLA, or TRITON_MLA. Alternatively, set " + "VLLM_MLA_DISABLE=1 to disable MLA for this model." + ) if not use_v1: raise RuntimeError( "MLA attention backends require the V1 engine. " From accb8fab07de05b2df43ccd97619100526cb8695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Wed, 29 Oct 2025 19:44:49 +0100 Subject: [PATCH 082/127] [KVConnector] Add metrics to Prometheus-Grafana dashboard (#26811) Signed-off-by: NickLucche Signed-off-by: Mark McLoughlin Co-authored-by: Mark McLoughlin --- .../kv_transfer/kv_connector/v1/base.py | 22 ++- .../kv_transfer/kv_connector/v1/metrics.py | 89 ++++++++++- .../kv_connector/v1/multi_connector.py | 110 +++++++++++--- .../kv_connector/v1/nixl_connector.py | 141 +++++++++++++++++- vllm/v1/metrics/loggers.py | 18 ++- vllm/v1/metrics/ray_wrappers.py | 14 ++ 6 files changed, 365 insertions(+), 29 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index 2562eb9ce70e4..2ed0fe592e373 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -50,7 +50,12 @@ if TYPE_CHECKING: from vllm.attention.backends.abstract import AttentionMetadata from vllm.config import VllmConfig from vllm.distributed.kv_events import KVCacheEvent - from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats + from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, + ) from vllm.forward_context import ForwardContext from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.request import Request @@ -471,3 +476,18 @@ class KVConnectorBase_V1(ABC): which can implement custom aggregation logic on the data dict. """ return None + + @classmethod + def build_prom_metrics( + cls, + vllm_config: "VllmConfig", + metric_types: dict[type["PromMetric"], type["PromMetricT"]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[str]], + ) -> Optional["KVConnectorPromMetrics"]: + """ + Create a KVConnectorPromMetrics subclass which should register + per-connector Prometheus metrics and implement observe() to + expose connector transfer stats via Prometheus. + """ + return None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/metrics.py b/vllm/distributed/kv_transfer/kv_connector/v1/metrics.py index 21002fe572c52..d6ea4f1ab4cfc 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/metrics.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/metrics.py @@ -1,13 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass, field -from typing import Any +from typing import Any, TypeAlias, TypeVar -from vllm.config.kv_transfer import KVTransferConfig +from prometheus_client import Counter, Gauge, Histogram + +from vllm.config import KVTransferConfig, VllmConfig from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.distributed.kv_transfer.kv_transfer_state import has_kv_transfer_group from vllm.logger import init_logger +PromMetric: TypeAlias = Gauge | Counter | Histogram +PromMetricT = TypeVar("PromMetricT", bound=PromMetric) + logger = init_logger(__name__) @@ -102,3 +107,83 @@ class KVConnectorLogging: # Reset metrics for next interval self.reset() + + +class KVConnectorPromMetrics: + """ + A base class for per-connector Prometheus metric registration + and recording. + """ + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[str]], + ): + self._kv_transfer_config = vllm_config.kv_transfer_config + self._gauge_cls = metric_types[Gauge] + self._counter_cls = metric_types[Counter] + self._histogram_cls = metric_types[Histogram] + self._labelnames = labelnames + self._per_engine_labelvalues = per_engine_labelvalues + + def make_per_engine(self, metric: PromMetric) -> PromMetric: + """ + Create a per-engine child of a prometheus_client.Metric with + the appropriate labels set. The parent metric must be created + using the labelnames list. + """ + return { + idx: metric.labels(*labelvalues) + for idx, labelvalues in self._per_engine_labelvalues.items() + } + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """ + Record the supplied transfer statistics to Prometheus metrics. These + statistics are engine-specific, and should be recorded to a metric + with the appropriate 'engine' label. These metric instances can be + created using the make_per_engine() helper method. + """ + raise NotImplementedError + + +class KVConnectorPrometheus: + """ + Support for registering per-connector Prometheus metrics, and + recording transfer statistics to those metrics. Uses + KVConnectorBase.build_prom_metrics(). + """ + + _gauge_cls = Gauge + _counter_cls = Counter + _histogram_cls = Histogram + + def __init__( + self, + vllm_config: VllmConfig, + labelnames: list[str], + per_engine_labelvalues: dict[int, list[str]], + ): + self.prom_metrics: KVConnectorPromMetrics | None = None + kv_transfer_config = vllm_config.kv_transfer_config + if kv_transfer_config and kv_transfer_config.kv_connector: + connector_cls = KVConnectorFactory.get_connector_class(kv_transfer_config) + metric_types = { + Gauge: self._gauge_cls, + Counter: self._counter_cls, + Histogram: self._histogram_cls, + } + self.prom_metrics = connector_cls.build_prom_metrics( + vllm_config, + metric_types, + labelnames, + per_engine_labelvalues, + ) + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + if self.prom_metrics is None: + return + self.prom_metrics.observe(transfer_stats_data, engine_idx) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index c1a2ac012415a..d56f30bd11e5b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -9,13 +9,19 @@ import torch from vllm.config import VllmConfig from vllm.config.kv_transfer import KVTransferConfig +from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, KVConnectorRole, ) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.outputs import KVConnectorOutput @@ -72,6 +78,27 @@ class MultiKVConnectorStats(KVConnectorStats): self.data[connector_id] = stats +class MultiKVConnectorPromMetrics(KVConnectorPromMetrics): + def __init__( + self, + vllm_config: "VllmConfig", + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[str]], + prom_metrics: dict[str, KVConnectorPromMetrics], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + self._prom_metrics = prom_metrics + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + for connector_id, stats_data in transfer_stats_data.items(): + assert connector_id in self._prom_metrics, ( + f"{connector_id} is not contained in the list of registered connectors " + f"with Prometheus metrics support: {self._prom_metrics.keys()}" + ) + self._prom_metrics[connector_id].observe(stats_data["data"], engine_idx) + + class MultiConnector(KVConnectorBase_V1): """ A wrapper for using multiple KVConnectors at the same time. @@ -84,19 +111,13 @@ class MultiConnector(KVConnectorBase_V1): def __init__(self, vllm_config: "VllmConfig", role: KVConnectorRole): super().__init__(vllm_config=vllm_config, role=role) + self._connectors: list[KVConnectorBase_V1] = [] self._ktc_kv_transfer_config = [] - ktcs = self._kv_transfer_config.kv_connector_extra_config.get("connectors") - assert ktcs is not None - for ktc in ktcs: - temp_config = copy.copy(vllm_config) - engine_id = ktc.get("engine_id", self._kv_transfer_config.engine_id) - temp_config.kv_transfer_config = KVTransferConfig( - **ktc, engine_id=engine_id - ) - self._connectors.append( - KVConnectorFactory.create_connector(temp_config, role) - ) + for connector_cls, temp_config in self._get_connector_classes_and_configs( + vllm_config + ): + self._connectors.append(connector_cls(temp_config, role)) self._ktc_kv_transfer_config.append(temp_config.kv_transfer_config) # A mapping from request id to the index of the connector chosen to @@ -109,6 +130,32 @@ class MultiConnector(KVConnectorBase_V1): # Propagated from scheduler to worker side via the connector metadata. self._extra_async_saves: dict[str, int] = {} + @classmethod + def _get_connector_classes_and_configs( + cls, vllm_config: "VllmConfig" + ) -> list[tuple[type[KVConnectorBaseType], "VllmConfig"]]: + assert vllm_config.kv_transfer_config is not None + ktcs = vllm_config.kv_transfer_config.kv_connector_extra_config.get( + "connectors" + ) + assert ktcs is not None + ret: list[tuple[type[KVConnectorBaseType], VllmConfig]] = [] + for ktc in ktcs: + temp_config = copy.copy(vllm_config) + engine_id = ktc.get("engine_id", vllm_config.kv_transfer_config.engine_id) + temp_config.kv_transfer_config = KVTransferConfig( + **ktc, engine_id=engine_id + ) + ret.append( + ( + KVConnectorFactory.get_connector_class( + temp_config.kv_transfer_config + ), + temp_config, + ) + ) + return ret + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): for c in self._connectors: c.register_kv_caches(kv_caches) @@ -295,18 +342,12 @@ class MultiConnector(KVConnectorBase_V1): None if the connector does not require a specific layout. """ assert vllm_config.kv_transfer_config is not None - ktcs = vllm_config.kv_transfer_config.kv_connector_extra_config.get( - "connectors" - ) - assert ktcs is not None layouts: set[str] = set() - temp_vllm_config = copy.copy(vllm_config) - for ktc in ktcs: - kv_transfer_config = KVTransferConfig(**ktc) - temp_vllm_config.kv_transfer_config = kv_transfer_config - connector_cls = KVConnectorFactory.get_connector_class(kv_transfer_config) + for connector_cls, temp_config in cls._get_connector_classes_and_configs( + vllm_config + ): required_kvcache_layout = connector_cls.get_required_kvcache_layout( - temp_vllm_config + temp_config ) if required_kvcache_layout is not None: layouts.add(required_kvcache_layout) @@ -372,3 +413,28 @@ class MultiConnector(KVConnectorBase_V1): stats_by_connector = MultiKVConnectorStats() stats_by_connector[c.__class__.__name__] = stats return stats_by_connector + + @classmethod + def build_prom_metrics( + cls, + vllm_config: "VllmConfig", + metric_types: dict[type["PromMetric"], type["PromMetricT"]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[str]], + ) -> KVConnectorPromMetrics: + prom_metrics: dict[str, KVConnectorPromMetrics] = {} + for connector_cls, temp_config in cls._get_connector_classes_and_configs( + vllm_config + ): + connector_prom = connector_cls.build_prom_metrics( + temp_config, metric_types, labelnames, per_engine_labelvalues + ) + if connector_prom is not None: + prom_metrics[connector_cls.__name__] = connector_prom + return MultiKVConnectorPromMetrics( + vllm_config, + metric_types, + labelnames, + per_engine_labelvalues, + prom_metrics, + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py index 72fcb5cd5bb78..275a8c734058b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py @@ -30,7 +30,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorMetadata, KVConnectorRole, ) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -254,6 +259,18 @@ class NixlConnector(KVConnectorBase_V1): else NixlKVConnectorStats() ) + @classmethod + def build_prom_metrics( + cls, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[str]], + ) -> KVConnectorPromMetrics: + return NixlPromMetrics( + vllm_config, metric_types, labelnames, per_engine_labelvalues + ) + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: assert self.connector_worker is not None assert isinstance(self._connector_metadata, NixlConnectorMetadata) @@ -1960,3 +1977,125 @@ class NixlKVConnectorStats(KVConnectorStats): @property def num_successful_transfers(self) -> int: return len(self.data["transfer_duration"]) + + +class NixlPromMetrics(KVConnectorPromMetrics): + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[str]], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + + buckets = [ + 0.001, + 0.005, + 0.01, + 0.025, + 0.05, + 0.075, + 0.1, + 0.2, + 0.3, + 0.5, + 0.75, + 1.0, + 5.0, + ] + nixl_histogram_xfer_time = self._histogram_cls( + name="vllm:nixl_xfer_time_seconds", + documentation="Histogram of transfer duration for NIXL KV Cache transfers.", + buckets=buckets[1:], + labelnames=labelnames, + ) + self.nixl_histogram_xfer_time = self.make_per_engine(nixl_histogram_xfer_time) + nixl_histogram_post_time = self._histogram_cls( + name="vllm:nixl_post_time_seconds", + documentation="Histogram of transfer post time for NIXL KV" + " Cache transfers.", + buckets=buckets, + labelnames=labelnames, + ) + self.nixl_histogram_post_time = self.make_per_engine(nixl_histogram_post_time) + # uniform 2kb to 16gb range + buckets = [2 ** (10 + i) for i in range(1, 25, 2)] + nixl_histogram_bytes_transferred = self._histogram_cls( + name="vllm:nixl_bytes_transferred", + documentation="Histogram of bytes transferred per NIXL KV Cache transfers.", + buckets=buckets, + labelnames=labelnames, + ) + self.nixl_histogram_bytes_transferred = self.make_per_engine( + nixl_histogram_bytes_transferred + ) + buckets = [ + 10, + 20, + 30, + 50, + 75, + 100, + 200, + 400, + 1000, + 2000, + 4000, + 10000, + 20000, + 50000, + ] + nixl_histogram_num_descriptors = self._histogram_cls( + name="vllm:nixl_num_descriptors", + documentation="Histogram of number of descriptors per NIXL" + " KV Cache transfers.", + buckets=buckets, + labelnames=labelnames, + ) + self.nixl_histogram_num_descriptors = self.make_per_engine( + nixl_histogram_num_descriptors + ) + counter_nixl_num_failed_transfers = self._counter_cls( + name="vllm:nixl_num_failed_transfers", + documentation="Number of failed NIXL KV Cache transfers.", + labelnames=labelnames, + ) + self.counter_nixl_num_failed_transfers = self.make_per_engine( + counter_nixl_num_failed_transfers + ) + counter_nixl_num_failed_notifications = self._counter_cls( + name="vllm:nixl_num_failed_notifications", + documentation="Number of failed NIXL KV Cache notifications.", + labelnames=labelnames, + ) + self.counter_nixl_num_failed_notifications = self.make_per_engine( + counter_nixl_num_failed_notifications + ) + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + for prom_obj, list_item_key in zip( + [ + self.nixl_histogram_xfer_time, + self.nixl_histogram_post_time, + self.nixl_histogram_bytes_transferred, + self.nixl_histogram_num_descriptors, + ], + [ + "transfer_duration", + "post_duration", + "bytes_transferred", + "num_descriptors", + ], + ): + for list_item in transfer_stats_data[list_item_key]: + prom_obj[engine_idx].observe(list_item) + for counter_obj, counter_item_key in zip( + [ + self.counter_nixl_num_failed_transfers, + self.counter_nixl_num_failed_notifications, + ], + ["num_failed_transfers", "num_failed_notifications"], + ): + for list_item in transfer_stats_data[counter_item_key]: + counter_obj[engine_idx].inc(list_item) diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 055da5d856b25..3772f07066a12 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -11,7 +11,10 @@ from prometheus_client import Counter, Gauge, Histogram import vllm.envs as envs from vllm.config import SupportsMetricsInfo, VllmConfig -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorLogging +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorLogging, + KVConnectorPrometheus, +) from vllm.logger import init_logger from vllm.plugins import load_plugins_by_group from vllm.v1.engine import FinishReason @@ -339,6 +342,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): _counter_cls = Counter _histogram_cls = Histogram _spec_decoding_cls = SpecDecodingProm + _kv_connector_cls = KVConnectorPrometheus def __init__( self, vllm_config: VllmConfig, engine_indexes: list[int] | None = None @@ -358,12 +362,15 @@ class PrometheusStatLogger(AggregateStatLoggerBase): model_name = vllm_config.model_config.served_model_name max_model_len = vllm_config.model_config.max_model_len - spec_decode_labelvalues: dict[int, list[str]] = { + per_engine_labelvalues: dict[int, list[str]] = { idx: [model_name, str(idx)] for idx in engine_indexes } self.spec_decoding_prom = self._spec_decoding_cls( - vllm_config.speculative_config, labelnames, spec_decode_labelvalues + vllm_config.speculative_config, labelnames, per_engine_labelvalues + ) + self.kv_connector_prom = self._kv_connector_cls( + vllm_config, labelnames, per_engine_labelvalues ) # @@ -962,6 +969,11 @@ class PrometheusStatLogger(AggregateStatLoggerBase): scheduler_stats.spec_decoding_stats, engine_idx ) + if scheduler_stats.kv_connector_stats is not None: + self.kv_connector_prom.observe( + scheduler_stats.kv_connector_stats, engine_idx + ) + if mm_cache_stats is not None: self.counter_mm_cache_queries[engine_idx].inc(mm_cache_stats.queries) self.counter_mm_cache_hits[engine_idx].inc(mm_cache_stats.hits) diff --git a/vllm/v1/metrics/ray_wrappers.py b/vllm/v1/metrics/ray_wrappers.py index b845852a0c0d5..a319ffb1d2573 100644 --- a/vllm/v1/metrics/ray_wrappers.py +++ b/vllm/v1/metrics/ray_wrappers.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorPrometheus from vllm.v1.metrics.loggers import PrometheusStatLogger from vllm.v1.spec_decode.metrics import SpecDecodingProm @@ -141,6 +142,18 @@ class RaySpecDecodingProm(SpecDecodingProm): _counter_cls = RayCounterWrapper +class RayKVConnectorPrometheus(KVConnectorPrometheus): + """ + RayKVConnectorPrometheus is used by RayMetrics to log Ray + metrics. Provides the same metrics as KV connectors but + uses Ray's util.metrics library. + """ + + _gauge_cls = RayGaugeWrapper + _counter_cls = RayCounterWrapper + _histogram_cls = RayHistogramWrapper + + class RayPrometheusStatLogger(PrometheusStatLogger): """RayPrometheusStatLogger uses Ray metrics instead.""" @@ -148,6 +161,7 @@ class RayPrometheusStatLogger(PrometheusStatLogger): _counter_cls = RayCounterWrapper _histogram_cls = RayHistogramWrapper _spec_decoding_cls = RaySpecDecodingProm + _kv_connector_cls = RayKVConnectorPrometheus @staticmethod def _unregister_vllm_metrics(): From fcb1d570bb8f95f5b7ded716a52fec902c535f0e Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 29 Oct 2025 14:50:39 -0400 Subject: [PATCH 083/127] [Bug] Fix DeepEP low latency `assert self.batched_router_logits.size(-1) == full_router_logits.size(-1)` Bug (#27682) Signed-off-by: yewentao256 --- vllm/model_executor/layers/fused_moe/layer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 294dddade6cc1..7dbe4bc543941 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -1135,6 +1135,7 @@ class FusedMoE(CustomOp): ) self.global_num_experts = num_experts + num_redundant_experts + self.logical_num_experts = num_experts self.zero_expert_num = zero_expert_num self.zero_expert_type = zero_expert_type @@ -1998,13 +1999,12 @@ class FusedMoE(CustomOp): moe = self.moe_config - # Note here we use `num_experts` which is logical expert count if self.vllm_config.parallel_config.enable_dbo: states_shape = (2, moe.max_num_tokens, self.hidden_size) - logits_shape = (2, moe.max_num_tokens, moe.num_experts) + logits_shape = (2, moe.max_num_tokens, self.logical_num_experts) else: states_shape = (moe.max_num_tokens, self.hidden_size) - logits_shape = (moe.max_num_tokens, moe.num_experts) + logits_shape = (moe.max_num_tokens, self.logical_num_experts) self.batched_hidden_states = torch.zeros( states_shape, dtype=moe.in_dtype, device=torch.cuda.current_device() From d4aa14434397b46a562f93d0371719e62d9bd62d Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Wed, 29 Oct 2025 13:16:52 -0700 Subject: [PATCH 084/127] [BugFix] Fix handling of resumed reqs in `SharedStorageConnector` (#27719) Signed-off-by: Nick Hill --- .../v1/shared_storage_connector.py | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py index d0cd4b07c51de..fc277630603aa 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py @@ -336,36 +336,34 @@ class SharedStorageConnector(KVConnectorBase_V1): cached_reqs = scheduler_output.scheduled_cached_reqs for i, req_id in enumerate(cached_reqs.req_ids): + resumed_from_preemption = cached_reqs.resumed_from_preemption[i] + if not resumed_from_preemption or req_id not in self._requests_need_load: + continue + num_computed_tokens = cached_reqs.num_computed_tokens[i] num_new_tokens = scheduler_output.num_scheduled_tokens[req_id] new_block_ids = cached_reqs.new_block_ids[i] - resumed_from_preemption = cached_reqs.resumed_from_preemption[i] - # NOTE(rob): here we rely on the resumed requests being - # the first N requests in the list scheduled_cache_reqs. - if not resumed_from_preemption: - break - if req_id in self._requests_need_load: - # NOTE(rob): cached_req_data does not have the full - # list of token ids (only new tokens). So we look it - # up in the actual request object. - request = self._requests_need_load[req_id] - total_tokens = num_computed_tokens + num_new_tokens - token_ids = request.all_token_ids[:total_tokens] + # NOTE(rob): cached_req_data does not have the full + # list of token ids (only new tokens). So we look it + # up in the actual request object. + request = self._requests_need_load[req_id] + total_tokens = num_computed_tokens + num_new_tokens + token_ids = request.all_token_ids[:total_tokens] - # NOTE(rob): For resumed req, new_block_ids is all - # of the block_ids for the request. - assert new_block_ids is not None - block_ids = new_block_ids[0] + # NOTE(rob): For resumed req, new_block_ids is all + # of the block_ids for the request. + assert new_block_ids is not None + block_ids = new_block_ids[0] - meta.add_request( - token_ids=token_ids, - block_ids=block_ids, - block_size=self._block_size, - is_store=False, - mm_hashes=[f.identifier for f in request.mm_features], - ) - total_need_load += 1 + meta.add_request( + token_ids=token_ids, + block_ids=block_ids, + block_size=self._block_size, + is_store=False, + mm_hashes=[f.identifier for f in request.mm_features], + ) + total_need_load += 1 assert total_need_load == len(self._requests_need_load) self._requests_need_load.clear() From b5d90f740048d43376390a61ca5b77c287505d0e Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 29 Oct 2025 16:28:27 -0400 Subject: [PATCH 085/127] [Bug] Fix DBO IMA issue for DeepEPHT (#27666) Signed-off-by: yewentao256 --- .../layers/fused_moe/deepep_ht_prepare_finalize.py | 12 +++++++++--- vllm/v1/worker/ubatching.py | 9 +++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/deepep_ht_prepare_finalize.py b/vllm/model_executor/layers/fused_moe/deepep_ht_prepare_finalize.py index 13866a5c5bf49..929cff79980c0 100644 --- a/vllm/model_executor/layers/fused_moe/deepep_ht_prepare_finalize.py +++ b/vllm/model_executor/layers/fused_moe/deepep_ht_prepare_finalize.py @@ -16,6 +16,7 @@ from vllm.utils.math_utils import round_up from vllm.v1.worker.ubatching import ( dbo_current_ubatch_id, dbo_enabled, + dbo_get_previous_event, dbo_switch_to_comm, dbo_switch_to_compute, dbo_switch_to_compute_sync, @@ -110,6 +111,10 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize): # for the other ubatch before the dispatch kernel starts. dbo_yield_and_switch_from_compute_to_comm() + # capture a DeepEP event and pass it as previous_event so + # DeepEP honors the dependency internally. + previous_event = dbo_get_previous_event(self.buffer.capture) + ( num_tokens_per_rank, num_tokens_per_rdma_rank, @@ -119,7 +124,7 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize): ) = self.buffer.get_dispatch_layout( topk_idx=rank_topk_ids, num_experts=num_experts, - previous_event=None, + previous_event=previous_event, async_finish=False, allocate_on_comm_stream=False, ) @@ -148,7 +153,7 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize): # to this value. expert_alignment=1, config=self._get_dispatch_config(), - previous_event=None, + previous_event=previous_event, async_finish=self.async_prepare and not dbo_enabled(), allocate_on_comm_stream=False, ) @@ -339,13 +344,14 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize): assert fused_expert_output.dtype == torch.bfloat16, ( f"Expected fused_expert_output bfloat16, got {fused_expert_output.dtype}" ) + previous_event = dbo_get_previous_event(self.buffer.capture) combined_x, _, event = self.buffer.combine( # HT combine only supports BF16 x=fused_expert_output, handle=handle, topk_weights=None, config=self._get_combine_config(), - previous_event=None, + previous_event=previous_event, async_finish=do_async and not dbo_enabled(), allocate_on_comm_stream=False, ) diff --git a/vllm/v1/worker/ubatching.py b/vllm/v1/worker/ubatching.py index 6edcb78486380..9f16b1e6d03ee 100644 --- a/vllm/v1/worker/ubatching.py +++ b/vllm/v1/worker/ubatching.py @@ -185,6 +185,15 @@ def dbo_register_recv_hook(recv_hook): next_ctx.recv_hook = recv_hook +def dbo_get_previous_event(func, *args, **kwargs): + if len(_THREAD_ID_TO_CONTEXT) > 0: + ctx_idx = _THREAD_ID_TO_CONTEXT[threading.get_ident()] + ctx = _CURRENT_CONTEXTS[ctx_idx] + # execute callable on the ubatch compute stream to record/wait events there + with torch.cuda.stream(ctx.compute_stream): + return func(*args, **kwargs) + + def make_ubatch_contexts( num_micro_batches: int, compute_stream: torch.cuda.Stream, From 48eb8eba581f0e45272f4e763bf5ec342f77091a Mon Sep 17 00:00:00 2001 From: Chenheli Hua Date: Wed, 29 Oct 2025 16:17:48 -0700 Subject: [PATCH 086/127] [Temp fix] Disable torch.compile for Qwen2.5 VL's VisionBlock temporarily. (#27760) Signed-off-by: Chenheli Hua Signed-off-by: Roger Wang Co-authored-by: Roger Wang --- vllm/model_executor/models/qwen2_5_vl.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 41cb7084057dd..dfaeb663bbe2f 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -460,15 +460,17 @@ class Qwen2_5_VisionAttention(nn.Module): return output -@support_torch_compile( - dynamic_arg_dims={ - "x": 0, - "cu_seqlens": 0, - "rotary_pos_emb": 0, - "seqlens": 0, - }, - mark_unbacked_dims={"seqlens": 0}, -) +# (FIXME): Enable this after dynamic slicing is fixed +# See https://github.com/vllm-project/vllm/pull/27760 +# @support_torch_compile( +# dynamic_arg_dims={ +# "x": 0, +# "cu_seqlens": 0, +# "rotary_pos_emb": 0, +# "seqlens": 0, +# }, +# mark_unbacked_dims={"seqlens": 0}, +# ) class Qwen2_5_VisionBlock(nn.Module): def __init__( self, From b798e39f931ad42354e0223de3d49e24523b79af Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Thu, 30 Oct 2025 09:43:13 +0800 Subject: [PATCH 087/127] [XPU][bugfix] fix rope for llama4 and deepseek (#25145) Signed-off-by: Yan Ma --- .../layers/rotary_embedding/base.py | 17 +++++++++++++- .../rotary_embedding/deepseek_scaling_rope.py | 4 ++-- .../rotary_embedding/llama4_vision_rope.py | 11 ++-------- .../layers/rotary_embedding/mrope.py | 22 ++----------------- 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/vllm/model_executor/layers/rotary_embedding/base.py b/vllm/model_executor/layers/rotary_embedding/base.py index 711902f0cc67e..91276320df4d0 100644 --- a/vllm/model_executor/layers/rotary_embedding/base.py +++ b/vllm/model_executor/layers/rotary_embedding/base.py @@ -14,7 +14,7 @@ from .rocm_aiter_rope_ops import ( @CustomOp.register("rotary_embedding") -class RotaryEmbedding(CustomOp): +class RotaryEmbeddingBase(CustomOp): """Original rotary positional embedding.""" def __init__( @@ -86,6 +86,21 @@ class RotaryEmbedding(CustomOp): ): self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype) + +class RotaryEmbedding(RotaryEmbeddingBase): + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: float, + is_neox_style: bool, + dtype: torch.dtype, + ) -> None: + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + def forward_native( self, positions: torch.Tensor, diff --git a/vllm/model_executor/layers/rotary_embedding/deepseek_scaling_rope.py b/vllm/model_executor/layers/rotary_embedding/deepseek_scaling_rope.py index 2e5efec066634..d9134f05fddff 100644 --- a/vllm/model_executor/layers/rotary_embedding/deepseek_scaling_rope.py +++ b/vllm/model_executor/layers/rotary_embedding/deepseek_scaling_rope.py @@ -7,7 +7,7 @@ import torch from vllm.platforms import current_platform -from .base import RotaryEmbedding +from .base import RotaryEmbeddingBase from .common import ( rotate_gptj, rotate_neox, @@ -22,7 +22,7 @@ def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: return 0.1 * mscale * math.log(scale) + 1.0 -class DeepseekScalingRotaryEmbedding(RotaryEmbedding): +class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase): """RotaryEmbedding extended with YaRN method. Credits to Peng et al. github.com/jquesnelle/yarn diff --git a/vllm/model_executor/layers/rotary_embedding/llama4_vision_rope.py b/vllm/model_executor/layers/rotary_embedding/llama4_vision_rope.py index 6241cb5abbc8e..9fdac309df7ee 100644 --- a/vllm/model_executor/layers/rotary_embedding/llama4_vision_rope.py +++ b/vllm/model_executor/layers/rotary_embedding/llama4_vision_rope.py @@ -5,10 +5,10 @@ import math import torch -from .base import RotaryEmbedding +from .base import RotaryEmbeddingBase -class Llama4VisionRotaryEmbedding(RotaryEmbedding): +class Llama4VisionRotaryEmbedding(RotaryEmbeddingBase): def __init__( self, head_size: int, @@ -78,10 +78,3 @@ class Llama4VisionRotaryEmbedding(RotaryEmbedding): key: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: return self.forward_native(query, key) - - def forward_hip( # type: ignore[override] - self, - query: torch.Tensor, - key: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - return self.forward_native(query, key) diff --git a/vllm/model_executor/layers/rotary_embedding/mrope.py b/vllm/model_executor/layers/rotary_embedding/mrope.py index d269733083d83..3c184ce9d6316 100644 --- a/vllm/model_executor/layers/rotary_embedding/mrope.py +++ b/vllm/model_executor/layers/rotary_embedding/mrope.py @@ -7,7 +7,7 @@ import torch from vllm.triton_utils import tl, triton -from .base import RotaryEmbedding +from .base import RotaryEmbeddingBase from .common import apply_rotary_emb_dispatch from .yarn_scaling_rope import YaRNScalingRotaryEmbedding, yarn_get_mscale @@ -199,7 +199,7 @@ def apply_interleaved_rope(x: torch.Tensor, mrope_section: list[int]) -> torch.T return x_t -class MRotaryEmbedding(RotaryEmbedding): +class MRotaryEmbedding(RotaryEmbeddingBase): """Rotary Embedding with Multimodal Sections.""" def __init__( @@ -357,24 +357,6 @@ class MRotaryEmbedding(RotaryEmbedding): key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) return query, key - def forward_xpu( - self, - positions: torch.Tensor, - query: torch.Tensor, - key: torch.Tensor | None = None, - offsets: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - return self.forward_native(positions, query, key, offsets) - - def forward_cpu( - self, - positions: torch.Tensor, - query: torch.Tensor, - key: torch.Tensor | None = None, - offsets: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - return self.forward_native(positions, query, key, offsets) - @staticmethod def get_next_input_positions( mrope_position_delta: int, From d7fb10c574a3a9cbf596bec086bf02603b71c5c8 Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Wed, 29 Oct 2025 19:39:57 -0700 Subject: [PATCH 088/127] [Bugfix] mamba-block-size is set for vision language model (#27773) Signed-off-by: Chen Zhang --- vllm/config/cache.py | 10 +--------- vllm/config/vllm.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 1734f6b15d4af..d743d5aa9dd29 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -5,7 +5,7 @@ import hashlib from dataclasses import field from typing import TYPE_CHECKING, Any, Literal -from pydantic import Field, SkipValidation, field_validator, model_validator +from pydantic import Field, SkipValidation, field_validator from pydantic.dataclasses import dataclass from vllm.config.utils import config @@ -185,11 +185,3 @@ class CacheConfig: raise ValueError("Too large swap space. " + msg) elif cpu_memory_usage > 0.4 * total_cpu_memory: logger.warning("Possibly too large swap space. %s", msg) - - @model_validator(mode="after") - def validate_mamba_block_size(self) -> "CacheConfig": - if self.mamba_block_size is not None and not self.enable_prefix_caching: - raise ValueError( - "--mamba-block-size can only be set with --enable-prefix-caching" - ) - return self diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index a7f7f3b45abea..c46f409edab61 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, TypeVar import torch -from pydantic import ConfigDict, Field +from pydantic import ConfigDict, Field, model_validator from pydantic.dataclasses import dataclass import vllm.envs as envs @@ -943,6 +943,20 @@ class VllmConfig: f"compilation_config={self.compilation_config!r}" ) + @model_validator(mode="after") + def validate_mamba_block_size(self) -> "VllmConfig": + if self.model_config is None: + return self + mamba_block_size_is_set = ( + self.cache_config.mamba_block_size is not None + and self.cache_config.mamba_block_size != self.model_config.max_model_len + ) + if mamba_block_size_is_set and not self.cache_config.enable_prefix_caching: + raise ValueError( + "--mamba-block-size can only be set with --enable-prefix-caching" + ) + return self + _current_vllm_config: VllmConfig | None = None _current_prefix: str | None = None From b5bae42f913efebef6d5239291418df8fb73b555 Mon Sep 17 00:00:00 2001 From: Kunshang Ji Date: Thu, 30 Oct 2025 11:17:13 +0800 Subject: [PATCH 089/127] [XPU] Update latest IPEX 2.8 release (#27735) Signed-off-by: Kunshang Ji --- .../scripts/hardware_ci/run-xpu-test.sh | 7 +++++-- .../installation/gpu.xpu.inc.md | 4 +++- requirements/xpu.txt | 2 +- vllm/_ipex_ops.py | 21 +++++-------------- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-xpu-test.sh b/.buildkite/scripts/hardware_ci/run-xpu-test.sh index 250a64fdd071c..27ed67c4517e2 100644 --- a/.buildkite/scripts/hardware_ci/run-xpu-test.sh +++ b/.buildkite/scripts/hardware_ci/run-xpu-test.sh @@ -20,7 +20,10 @@ trap remove_docker_container EXIT # Run the image and test offline inference/tensor parallel docker run \ - --device /dev/dri \ + --device /dev/dri:/dev/dri \ + --net=host \ + --ipc=host \ + --privileged \ -v /dev/dri/by-path:/dev/dri/by-path \ --entrypoint="" \ -e "HF_TOKEN=${HF_TOKEN}" \ @@ -42,7 +45,7 @@ docker run \ pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py pytest -v -s v1/structured_output - pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py + pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py --ignore=v1/spec_decode/test_speculators_eagle3.py pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector.py --ignore=v1/kv_connector/unit/test_shared_storage_connector.py pytest -v -s v1/test_serial_utils.py ' diff --git a/docs/getting_started/installation/gpu.xpu.inc.md b/docs/getting_started/installation/gpu.xpu.inc.md index 9156df9db6df3..620a660a240ed 100644 --- a/docs/getting_started/installation/gpu.xpu.inc.md +++ b/docs/getting_started/installation/gpu.xpu.inc.md @@ -56,8 +56,10 @@ docker build -f docker/Dockerfile.xpu -t vllm-xpu-env --shm-size=4g . docker run -it \ --rm \ --network=host \ - --device /dev/dri \ + --device /dev/dri:/dev/dri \ -v /dev/dri/by-path:/dev/dri/by-path \ + --ipc=host \ + --privileged \ vllm-xpu-env ``` diff --git a/requirements/xpu.txt b/requirements/xpu.txt index d14b631aa9364..e69a98b86036e 100644 --- a/requirements/xpu.txt +++ b/requirements/xpu.txt @@ -15,4 +15,4 @@ torchaudio torchvision --extra-index-url=https://download.pytorch.org/whl/xpu -intel-extension-for-pytorch @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.8.10.post0%2Bxpu-cp312-cp312-linux_x86_64.whl +intel-extension-for-pytorch @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.8.10.post1%2Bxpu-cp312-cp312-linux_x86_64.whl diff --git a/vllm/_ipex_ops.py b/vllm/_ipex_ops.py index e773e1d13f0b8..60ee0124c3d9c 100644 --- a/vllm/_ipex_ops.py +++ b/vllm/_ipex_ops.py @@ -151,7 +151,9 @@ class ipex_ops: def rms_norm( input: torch.Tensor, weight: torch.Tensor, epsilon: float ) -> torch.Tensor: - return ipex.llm.functional.rms_norm(input, weight, epsilon) + out = torch.empty_like(input) + torch.ops.torch_ipex.rms_norm_vllm(out, input.contiguous(), weight, epsilon) + return out @staticmethod def fused_add_rms_norm( @@ -160,10 +162,7 @@ class ipex_ops: weight: torch.Tensor, epsilon: float, ) -> None: - tmp = ipex.llm.functional.add_rms_norm( - residual, input, weight, None, epsilon, True - ) - input.copy_(tmp) + torch.ops.torch_ipex.fused_add_rms_norm_vllm(input, residual, weight, epsilon) @staticmethod def varlen_attention( @@ -296,16 +295,6 @@ class ipex_ops: num_splits=0, s_aux: torch.Tensor | None = None, ): - if cu_seqlens_k is None: - # cu_seqlens_k is not used in ipex kernel. - cu_seqlens_k = torch.cumsum(seqused_k, dim=0) - cu_seqlens_k = torch.cat( - [ - torch.tensor([0], device=seqused_k.device, dtype=torch.int32), - cu_seqlens_k, - ] - ).to(torch.int32) - real_window_size: tuple[int, int] if window_size is None: real_window_size = (-1, -1) @@ -318,7 +307,7 @@ class ipex_ops: k, v, cu_seqlens_q, - cu_seqlens_k, + seqused_k, max_seqlen_q, max_seqlen_k, softmax_scale, From 2ce5c5d3d65a53e81b5117867f5ce9c873e68334 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Wed, 29 Oct 2025 21:04:25 -0700 Subject: [PATCH 090/127] [BugFix] Handle unscheduled requests properly when async scheduling (#27756) Signed-off-by: Nick Hill --- tests/v1/tpu/worker/test_tpu_model_runner.py | 4 +- tests/v1/worker/test_gpu_model_runner.py | 6 +-- .../kv_connector/v1/offloading_connector.py | 2 +- .../kv_connector/v1/p2p/p2p_nccl_connector.py | 4 +- .../v1/shared_storage_connector.py | 2 +- vllm/v1/core/sched/output.py | 32 +++++++++++---- vllm/v1/core/sched/scheduler.py | 39 ++++++++++--------- vllm/v1/worker/gpu_model_runner.py | 15 +++---- vllm/v1/worker/tpu_model_runner.py | 2 +- 9 files changed, 63 insertions(+), 43 deletions(-) diff --git a/tests/v1/tpu/worker/test_tpu_model_runner.py b/tests/v1/tpu/worker/test_tpu_model_runner.py index 1aa0709696c41..18aa599f1aaf7 100644 --- a/tests/v1/tpu/worker/test_tpu_model_runner.py +++ b/tests/v1/tpu/worker/test_tpu_model_runner.py @@ -212,10 +212,12 @@ def test_update_states_request_resumed(model_runner): # resume req cached_req_data = CachedRequestData( req_ids=[req_id], - resumed_from_preemption=[False], + resumed_req_ids={req_id}, new_token_ids=[[]], + all_token_ids={req_id: scheduler_output.scheduled_new_reqs[0].prompt_token_ids}, new_block_ids=[([],)], num_computed_tokens=[0], + num_output_tokens=[0], ) scheduler_output = SchedulerOutput( diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index c2c34ee95ad5f..9007436350be4 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -259,10 +259,10 @@ def test_update_states_request_resumed(model_runner, dist_init): # resume req cached_req_data = CachedRequestData( req_ids=[req_id], - resumed_from_preemption=[False], + resumed_req_ids=set(), new_token_ids=[[]], - resumed_req_token_ids=[None], - new_block_ids=([[0]],), + all_token_ids={}, + new_block_ids=[([0],)], num_computed_tokens=[0], num_output_tokens=[0], ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py index 6d4ffc152de97..19344e5784c23 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py @@ -494,5 +494,5 @@ def yield_req_data( yield from zip( cached_reqs.req_ids, cached_reqs.new_block_ids, - cached_reqs.resumed_from_preemption, + (req_id in cached_reqs.resumed_req_ids for req_id in cached_reqs.req_ids), ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_connector.py index e47cde2614fc2..780dd12fccda3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_connector.py @@ -415,10 +415,10 @@ class P2pNcclConnector(KVConnectorBase_V1): for i, req_id in enumerate(cached_reqs.req_ids): num_computed_tokens = cached_reqs.num_computed_tokens[i] new_block_ids = cached_reqs.new_block_ids[i] - resumed_from_preemption = cached_reqs.resumed_from_preemption[i] + resumed_from_preemption = req_id in cached_reqs.resumed_req_ids if self.is_producer: - num_scheduled_tokens = (scheduler_output.num_scheduled_tokens)[req_id] + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] num_tokens = num_scheduled_tokens + num_computed_tokens assert req_id in self.chunked_prefill assert new_block_ids is not None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py index fc277630603aa..9c230d7d0d2f4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/shared_storage_connector.py @@ -336,7 +336,7 @@ class SharedStorageConnector(KVConnectorBase_V1): cached_reqs = scheduler_output.scheduled_cached_reqs for i, req_id in enumerate(cached_reqs.req_ids): - resumed_from_preemption = cached_reqs.resumed_from_preemption[i] + resumed_from_preemption = req_id in cached_reqs.resumed_req_ids if not resumed_from_preemption or req_id not in self._requests_need_load: continue diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 035394f045301..cc6b89e2bf3f1 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -2,8 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass +from functools import cached_property from typing import TYPE_CHECKING +from typing_extensions import deprecated + from vllm._bc_linter import bc_linter_include if TYPE_CHECKING: @@ -96,16 +99,16 @@ class NewRequestData: @dataclass class CachedRequestData: req_ids: list[str] - # If resumed_from_preemption is False, new_block_ids will be appended to - # the request's block IDs. If True, new_block_ids will be used as the + # For request ids not in resumed_req_ids, new_block_ids will be appended to + # the request's block IDs. For those in the set, new_block_ids will be used as the # request's block IDs instead of appending to the existing block IDs. - resumed_from_preemption: list[bool] + resumed_req_ids: set[str] # NOTE(woosuk): new_token_ids is only used for pipeline parallelism. # When PP is not used, new_token_ids will be empty. new_token_ids: list[list[int]] - # If resumed_from_preemption is True, propogate the token ids to the - # connector, otherwise will be empty. - resumed_req_token_ids: list[list[int] | None] + # For requests not scheduled in the last step, propagate the token ids to the + # connector. Won't contain requests that were scheduled in the prior step. + all_token_ids: dict[str, list[int]] new_block_ids: list[tuple[list[int], ...] | None] num_computed_tokens: list[int] num_output_tokens: list[int] @@ -114,13 +117,26 @@ class CachedRequestData: def num_reqs(self) -> int: return len(self.req_ids) + @cached_property + @deprecated("use resumed_req_ids field") + def resumed_from_preemption(self) -> list[bool]: + return [req_id in self.resumed_req_ids for req_id in self.req_ids] + + @cached_property + @deprecated("use all_token_ids field") + def resumed_req_token_ids(self) -> list[list[int] | None]: + return [ + self.all_token_ids[req_id] if req_id in self.resumed_req_ids else None + for req_id in self.req_ids + ] + @classmethod def make_empty(cls) -> "CachedRequestData": return cls( req_ids=[], - resumed_from_preemption=[], + resumed_req_ids=set(), new_token_ids=[], - resumed_req_token_ids=[], + all_token_ids={}, new_block_ids=[], num_computed_tokens=[], num_output_tokens=[], diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 00b34fe4fbb98..c794886bc24c8 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -71,6 +71,7 @@ class Scheduler(SchedulerInterface): self.finished_req_ids_dict: dict[int, set[str]] | None = ( defaultdict(set) if include_finished_set else None ) + self.prev_step_scheduled_req_ids: set[str] = set() # Scheduling constraints. self.max_num_running_reqs = self.scheduler_config.max_num_seqs @@ -444,14 +445,9 @@ class Scheduler(SchedulerInterface): # `request.num_prompt_tokens` to consider the resumed # requests, which have output tokens. num_new_tokens = request.num_tokens - num_computed_tokens - if ( - 0 - < self.scheduler_config.long_prefill_token_threshold - < num_new_tokens - ): - num_new_tokens = ( - self.scheduler_config.long_prefill_token_threshold - ) + threshold = self.scheduler_config.long_prefill_token_threshold + if 0 < threshold < num_new_tokens: + num_new_tokens = threshold # chunked prefill has to be enabled explicitly to allow # pooling requests to be chunked @@ -620,6 +616,11 @@ class Scheduler(SchedulerInterface): structured_output_request_ids, grammar_bitmask = self.get_grammar_bitmask( num_scheduled_tokens.keys(), scheduled_spec_decode_tokens ) + + # Record the request ids that were scheduled in this step. + self.prev_step_scheduled_req_ids.clear() + self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys()) + scheduler_output = SchedulerOutput( scheduled_new_reqs=new_reqs_data, scheduled_cached_reqs=cached_reqs_data, @@ -691,14 +692,12 @@ class Scheduler(SchedulerInterface): req_ids: list[str] = [] new_token_ids: list[list[int]] = [] new_block_ids: list[tuple[list[int], ...] | None] = [] - resumed_req_token_ids: list[list[int] | None] = [] + all_token_ids: dict[str, list[int]] = {} num_computed_tokens: list[int] = [] num_output_tokens: list[int] = [] + resumed_req_ids = set() - # Because resumed_reqs is usually empty, it is more efficient to do - # in-place appending so that we don't need to allocate a new list. - resumed_from_preemption = [False] * len(running_reqs) - resumed_from_preemption += [True] * len(resumed_reqs) + num_running_reqs = len(running_reqs) for idx, req in enumerate(itertools.chain(running_reqs, resumed_reqs)): req_id = req.request_id req_ids.append(req_id) @@ -715,12 +714,14 @@ class Scheduler(SchedulerInterface): req.num_computed_tokens : req.num_computed_tokens + num_tokens ] new_token_ids.append(token_ids) - resumed_token_ids = None - if resumed_from_preemption[idx]: - resumed_token_ids = req.all_token_ids[ + scheduled_in_prev_step = req_id in self.prev_step_scheduled_req_ids + if idx >= num_running_reqs: + assert not scheduled_in_prev_step + resumed_req_ids.add(req_id) + if not scheduled_in_prev_step: + all_token_ids[req_id] = req.all_token_ids[ : req.num_computed_tokens + num_tokens ] - resumed_req_token_ids.append(resumed_token_ids) new_block_ids.append( req_to_new_blocks[req_id].get_block_ids(allow_none=True) ) @@ -731,9 +732,9 @@ class Scheduler(SchedulerInterface): return CachedRequestData( req_ids=req_ids, - resumed_from_preemption=resumed_from_preemption, + resumed_req_ids=resumed_req_ids, new_token_ids=new_token_ids, - resumed_req_token_ids=resumed_req_token_ids, + all_token_ids=all_token_ids, new_block_ids=new_block_ids, num_computed_tokens=num_computed_tokens, num_output_tokens=num_output_tokens, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index e350988456f12..1fe749c614ccf 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -706,7 +706,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): req_state = self.requests[req_id] num_computed_tokens = req_data.num_computed_tokens[i] new_block_ids = req_data.new_block_ids[i] - resumed_from_preemption = req_data.resumed_from_preemption[i] + resumed_from_preemption = req_id in req_data.resumed_req_ids num_output_tokens = req_data.num_output_tokens[i] # Update the cached states. @@ -754,16 +754,17 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): # Replace the existing block IDs with the new ones. req_state.block_ids = new_block_ids - if self.use_async_scheduling and num_output_tokens > 0: - # We must recover the output token ids for resumed requests in the - # async scheduling case, so that correct input_ids are obtained. - resumed_token_ids = req_data.resumed_req_token_ids[i] - assert resumed_token_ids is not None - req_state.output_token_ids = resumed_token_ids[-num_output_tokens:] if req_index is None: # The request is not in the persistent batch. # The request was either preempted and resumed later, or was not # scheduled in the previous step and needs to be added again. + + if self.use_async_scheduling and num_output_tokens > 0: + # We must recover the output token ids for resumed requests in the + # async scheduling case, so that correct input_ids are obtained. + resumed_token_ids = req_data.all_token_ids[req_id] + req_state.output_token_ids = resumed_token_ids[-num_output_tokens:] + reqs_to_add.append(req_state) continue diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index 5d7b181989ce5..0ced138b940d0 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -483,7 +483,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): req_state = self.requests[req_id] num_computed_tokens = req_data.num_computed_tokens[i] new_block_ids = req_data.new_block_ids[i] - resumed_from_preemption = req_data.resumed_from_preemption[i] + resumed_from_preemption = req_id in req_data.resumed_req_ids # Update the cached states. req_state.num_computed_tokens = num_computed_tokens From 17d055f527d2bd5d39a1352e5161ed82345466ac Mon Sep 17 00:00:00 2001 From: Benjamin Bartels Date: Thu, 30 Oct 2025 04:09:10 +0000 Subject: [PATCH 091/127] [Feat] Adds runai distributed streamer (#27230) Signed-off-by: bbartels Signed-off-by: Benjamin Bartels Co-authored-by: omer-dayan Co-authored-by: Cyrus Leung --- docker/Dockerfile | 2 +- docs/models/extensions/runai_model_streamer.md | 9 +++++++++ requirements/nightly_torch_test.txt | 2 +- requirements/rocm.txt | 2 +- requirements/test.in | 2 +- requirements/test.txt | 6 +++--- setup.py | 2 +- .../model_loader/runai_streamer_loader.py | 10 ++++++++-- vllm/model_executor/model_loader/weight_utils.py | 15 ++++++++++++++- 9 files changed, 39 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index eb1453126e6f4..42a830cb605ad 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -495,7 +495,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ else \ BITSANDBYTES_VERSION="0.46.1"; \ fi; \ - uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.14.0' + uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.15.0' ENV VLLM_USAGE_SOURCE production-docker-image diff --git a/docs/models/extensions/runai_model_streamer.md b/docs/models/extensions/runai_model_streamer.md index c2cf107263a03..fc9d5eec3803e 100644 --- a/docs/models/extensions/runai_model_streamer.md +++ b/docs/models/extensions/runai_model_streamer.md @@ -45,6 +45,15 @@ vllm serve s3://core-llm/Llama-3-8b \ You can tune parameters using `--model-loader-extra-config`: +You can tune `distributed` that controls whether distributed streaming should be used. This is currently only possible on CUDA and ROCM devices. This can significantly improve loading times from object storage or high-throughput network fileshares. +You can read further about Distributed streaming [here](https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/usage.md#distributed-streaming) + +```bash +vllm serve /home/meta-llama/Llama-3.2-3B-Instruct \ + --load-format runai_streamer \ + --model-loader-extra-config '{"distributed":true}' +``` + You can tune `concurrency` that controls the level of concurrency and number of OS threads reading tensors from the file to the CPU buffer. For reading from S3, it will be the number of client instances the host is opening to the S3 server. diff --git a/requirements/nightly_torch_test.txt b/requirements/nightly_torch_test.txt index dea1926bbd695..63c1908f024b3 100644 --- a/requirements/nightly_torch_test.txt +++ b/requirements/nightly_torch_test.txt @@ -42,6 +42,6 @@ tritonclient==2.51.0 numba == 0.61.2 # Required for N-gram speculative decoding numpy -runai-model-streamer[s3,gcs]==0.14.0 +runai-model-streamer[s3,gcs]==0.15.0 fastsafetensors>=0.1.10 pydantic>=2.12 # 2.11 leads to error on python 3.13 diff --git a/requirements/rocm.txt b/requirements/rocm.txt index d9743f0446438..6f1cca90e5e2b 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -12,6 +12,6 @@ tensorizer==2.10.1 packaging>=24.2 setuptools>=77.0.3,<80.0.0 setuptools-scm>=8 -runai-model-streamer[s3,gcs]==0.14.0 +runai-model-streamer[s3,gcs]==0.15.0 conch-triton-kernels==1.2.1 timm>=1.0.17 diff --git a/requirements/test.in b/requirements/test.in index a79ec839dbec1..b1ab599ff16e5 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -50,7 +50,7 @@ tritonclient==2.51.0 numba == 0.61.2 # Required for N-gram speculative decoding numpy -runai-model-streamer[s3,gcs]==0.14.0 +runai-model-streamer[s3,gcs]==0.15.0 fastsafetensors>=0.1.10 pydantic>=2.12 # 2.11 leads to error on python 3.13 decord==0.6.0 diff --git a/requirements/test.txt b/requirements/test.txt index bc007ccf10bbb..e54bb49fde684 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -965,11 +965,11 @@ rsa==4.9.1 # via google-auth rtree==1.4.0 # via torchgeo -runai-model-streamer==0.14.0 +runai-model-streamer==0.15.0 # via -r requirements/test.in -runai-model-streamer-gcs==0.14.0 +runai-model-streamer-gcs==0.15.0 # via runai-model-streamer -runai-model-streamer-s3==0.14.0 +runai-model-streamer-s3==0.15.0 # via runai-model-streamer s3transfer==0.10.3 # via boto3 diff --git a/setup.py b/setup.py index 83a4e3eea57c8..8139d0d62b8ac 100644 --- a/setup.py +++ b/setup.py @@ -712,7 +712,7 @@ setup( "bench": ["pandas", "matplotlib", "seaborn", "datasets"], "tensorizer": ["tensorizer==2.10.1"], "fastsafetensors": ["fastsafetensors >= 0.1.10"], - "runai": ["runai-model-streamer[s3,gcs] >= 0.14.0"], + "runai": ["runai-model-streamer[s3,gcs] >= 0.15.0"], "audio": [ "librosa", "soundfile", diff --git a/vllm/model_executor/model_loader/runai_streamer_loader.py b/vllm/model_executor/model_loader/runai_streamer_loader.py index 079e3168647bb..93da07c550195 100644 --- a/vllm/model_executor/model_loader/runai_streamer_loader.py +++ b/vllm/model_executor/model_loader/runai_streamer_loader.py @@ -27,9 +27,16 @@ class RunaiModelStreamerLoader(BaseModelLoader): def __init__(self, load_config: LoadConfig): super().__init__(load_config) + + self._is_distributed = False if load_config.model_loader_extra_config: extra_config = load_config.model_loader_extra_config + if "distributed" in extra_config and isinstance( + extra_config.get("distributed"), bool + ): + self._is_distributed = extra_config.get("distributed") + if "concurrency" in extra_config and isinstance( extra_config.get("concurrency"), int ): @@ -92,8 +99,7 @@ class RunaiModelStreamerLoader(BaseModelLoader): """Get an iterator for the model weights based on the load format.""" hf_weights_files = self._prepare_weights(model_or_path, revision) return runai_safetensors_weights_iterator( - hf_weights_files, - self.load_config.use_tqdm_on_load, + hf_weights_files, self.load_config.use_tqdm_on_load, self._is_distributed ) def download_model(self, model_config: ModelConfig) -> None: diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 5a9faefa4d894..3dbe803f99860 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -657,10 +657,22 @@ def multi_thread_safetensors_weights_iterator( def runai_safetensors_weights_iterator( hf_weights_files: list[str], use_tqdm_on_load: bool, + is_distributed: bool = False, ) -> Generator[tuple[str, torch.Tensor], None, None]: """Iterate over the weights in the model safetensor files.""" with SafetensorsStreamer() as streamer: - streamer.stream_files(hf_weights_files) + is_cuda_alike = current_platform.is_cuda_alike() + device = ( + f"cuda:{current_platform.current_device()}" + if is_distributed and is_cuda_alike + else "cpu" + ) + + streamer.stream_files( + hf_weights_files, + device=device, + is_distributed=is_distributed, + ) total_tensors = sum( len(tensors_meta) for tensors_meta in streamer.files_to_tensors_metadata.values() @@ -672,6 +684,7 @@ def runai_safetensors_weights_iterator( desc="Loading safetensors using Runai Model Streamer", bar_format=_BAR_FORMAT, disable=not enable_tqdm(use_tqdm_on_load), + mininterval=2, ) yield from tensor_iter From b8c48c5d722298656074c559d0e8d702a6c28da1 Mon Sep 17 00:00:00 2001 From: Fardin Hoque Date: Wed, 29 Oct 2025 21:10:34 -0700 Subject: [PATCH 092/127] kernels/moe test pruning (#27053) Signed-off-by: Fardin Hoque Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- tests/kernels/moe/test_batched_moe.py | 25 +++++++++++++------ tests/kernels/moe/test_block_fp8.py | 14 ----------- tests/kernels/moe/test_block_int8.py | 10 +------- tests/kernels/moe/test_cutlass_moe.py | 3 --- tests/kernels/moe/test_deepep_deepgemm_moe.py | 1 - tests/kernels/moe/test_deepgemm.py | 2 -- tests/kernels/moe/test_flashinfer.py | 2 -- tests/kernels/moe/test_flashinfer_moe.py | 4 +-- tests/kernels/moe/test_grouped_topk.py | 2 +- .../moe/test_modular_kernel_combinations.py | 8 ++++++ tests/kernels/moe/test_moe.py | 11 +++----- tests/kernels/moe/test_nvfp4_moe.py | 4 +-- .../moe/test_silu_mul_fp8_quant_deep_gemm.py | 4 --- 13 files changed, 34 insertions(+), 56 deletions(-) diff --git a/tests/kernels/moe/test_batched_moe.py b/tests/kernels/moe/test_batched_moe.py index 2dce099770f08..62704bbcbbc79 100644 --- a/tests/kernels/moe/test_batched_moe.py +++ b/tests/kernels/moe/test_batched_moe.py @@ -24,23 +24,16 @@ from vllm.triton_utils import tl MNK_FACTORS = [ (1, 128, 128), - (1, 128, 2048), (1, 512, 512), - (1, 1024, 128), (1, 1024, 2048), (32, 128, 128), (32, 512, 512), (32, 1024, 2048), - (45, 128, 128), (45, 128, 2048), - (45, 512, 512), (45, 1024, 128), - (45, 1024, 2048), (64, 512, 512), (64, 1024, 2048), - (222, 128, 128), (222, 128, 2048), - (222, 1024, 128), (222, 1024, 2048), ] NUM_EXPERTS = [8, 64] @@ -117,10 +110,19 @@ def test_batched_mm( block_shape: list[int] | None, per_act_token_quant: bool, ): + """Note: float8_e4m3fn is not supported on CUDA architecture < 89, + and those tests will be skipped on unsupported hardware.""" current_platform.seed_everything(7) use_fp8_w8a8 = dtype == torch.float8_e4m3fn + if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability( + 89 + ): + pytest.skip( + "Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89" + ) + if (per_act_token_quant or block_shape is not None) and not use_fp8_w8a8: pytest.skip("Don't test blocking for non-quantized types.") @@ -244,10 +246,19 @@ def test_fused_moe_batched_experts( block_shape: list[int] | None, input_scales: bool, ): + """Note: float8_e4m3fn is not supported on CUDA architecture < 89, + and those tests will be skipped on unsupported hardware.""" current_platform.seed_everything(7) use_fp8_w8a8 = dtype == torch.float8_e4m3fn + if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability( + 89 + ): + pytest.skip( + "Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89" + ) + if topk > e: pytest.skip("topk > e") diff --git a/tests/kernels/moe/test_block_fp8.py b/tests/kernels/moe/test_block_fp8.py index 60f9f14b7f6f1..cd34617ee0fc4 100644 --- a/tests/kernels/moe/test_block_fp8.py +++ b/tests/kernels/moe/test_block_fp8.py @@ -42,57 +42,43 @@ DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32] # and its hidden size is 7168. MNK_FACTORS = [ (1, 128, 128), - (1, 512, 512), (1, 128, 7168), (1, 1024, 7168), (1, 4608, 128), - (1, 4608, 512), (1, 4608, 7168), (83, 128, 128), (83, 512, 512), - (83, 1024, 7168), (83, 4608, 512), (83, 4608, 7168), - (128, 128, 128), (128, 512, 512), (128, 1024, 7168), - (128, 4608, 512), (128, 4608, 7168), (2048, 128, 128), (2048, 1024, 7168), (2048, 4608, 512), (2048, 4608, 7168), (8192, 128, 128), - (8192, 512, 512), (8192, 128, 7168), (8192, 1024, 7168), - (8192, 4608, 512), (8192, 4608, 7168), ] MNK_FACTORS_DG = [ (128, 128, 128), - (128, 512, 512), (128, 128, 7168), (128, 1024, 7168), (128, 4608, 128), - (128, 4608, 512), (128, 4608, 7168), - (192, 128, 128), (192, 512, 512), (192, 1024, 7168), - (192, 4608, 512), (192, 4608, 7168), (1335, 128, 128), (1335, 1024, 7168), (1335, 4608, 512), (1335, 4608, 7168), (2048, 128, 128), - (2048, 512, 512), (2048, 128, 7168), (2048, 1024, 7168), - (2048, 4608, 128), - (2048, 4608, 512), (2048, 4608, 7168), ] diff --git a/tests/kernels/moe/test_block_int8.py b/tests/kernels/moe/test_block_int8.py index 74cc943714dd9..3799e60f1294a 100644 --- a/tests/kernels/moe/test_block_int8.py +++ b/tests/kernels/moe/test_block_int8.py @@ -21,36 +21,28 @@ vllm_config = VllmConfig() vllm_config.scheduler_config.max_num_seqs = 128 vllm_config.scheduler_config.max_model_len = 8192 -DTYPES = [torch.half, torch.bfloat16] +DTYPES = [torch.bfloat16] MNK_FACTORS = [ (1, 128, 128), - (1, 512, 512), (1, 128, 7168), (1, 1024, 7168), - (1, 4096, 128), (1, 4096, 512), (1, 4096, 7168), - (33, 128, 128), (33, 512, 512), (33, 128, 7168), (33, 1024, 7168), (33, 4096, 128), - (33, 4096, 512), (33, 4096, 7168), (128, 128, 128), - (128, 512, 512), (128, 1024, 7168), (128, 4096, 512), (128, 4096, 7168), - (222, 128, 128), (222, 512, 512), (222, 1024, 7168), - (222, 4096, 512), (222, 4096, 7168), (2048, 128, 128), (2048, 1024, 7168), - (2048, 4096, 512), (2048, 4096, 4096), ] diff --git a/tests/kernels/moe/test_cutlass_moe.py b/tests/kernels/moe/test_cutlass_moe.py index 4330eda251f75..5512ccce47b05 100644 --- a/tests/kernels/moe/test_cutlass_moe.py +++ b/tests/kernels/moe/test_cutlass_moe.py @@ -26,16 +26,13 @@ TOP_KS = [6, 8] MNK_FACTORS = [ (2, 1024, 1024), - (2, 1024, 1536), (2, 3072, 1024), (2, 3072, 1536), (7, 3072, 1536), (64, 1024, 1024), (64, 1024, 1536), (64, 3072, 1024), - (64, 3072, 1536), (224, 1024, 1024), - (224, 1024, 1536), (224, 3072, 1024), (224, 3072, 1536), (32768, 1024, 1024), diff --git a/tests/kernels/moe/test_deepep_deepgemm_moe.py b/tests/kernels/moe/test_deepep_deepgemm_moe.py index d46f453488a98..9d039b81690a1 100644 --- a/tests/kernels/moe/test_deepep_deepgemm_moe.py +++ b/tests/kernels/moe/test_deepep_deepgemm_moe.py @@ -393,7 +393,6 @@ def _test_deepep_deepgemm_moe( MNKs = [ (8, 128, 128), (8, 128, 512), - (8, 512, 512), (3, 1024, 2048), (32, 128, 1024), (45, 512, 2048), diff --git a/tests/kernels/moe/test_deepgemm.py b/tests/kernels/moe/test_deepgemm.py index cad0085d5ba6e..9b1054f7d0ab8 100644 --- a/tests/kernels/moe/test_deepgemm.py +++ b/tests/kernels/moe/test_deepgemm.py @@ -130,10 +130,8 @@ def run_single_case(m, n, k, topk, num_experts, block_size): # Note: N <= 512 will disable the deepgemm path due to performance issues. MNKs = [ (1024, 768, 128), - (1024, 768, 512), (2048, 768, 512), (512, 1024, 1024), - (512, 2048, 2048), (4096, 4096, 1024), ] diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 0780232a82640..f985f9ac7ca67 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -34,8 +34,6 @@ TOP_KS = [1] MNK_FACTORS = [ (256, 8192, 5120), - (256, 4096, 5120), - (127, 8192, 5120), (127, 4096, 5120), (10, 8192, 5120), (10, 4096, 5120), diff --git a/tests/kernels/moe/test_flashinfer_moe.py b/tests/kernels/moe/test_flashinfer_moe.py index 18cfd4f79092d..be3e36865d1a4 100644 --- a/tests/kernels/moe/test_flashinfer_moe.py +++ b/tests/kernels/moe/test_flashinfer_moe.py @@ -34,10 +34,8 @@ if not has_flashinfer_cutlass_fused_moe() or not current_platform.has_device_cap MNK_FACTORS = [ (2, 1024, 1024), - (2, 1024, 1536), (2, 3072, 1024), (2, 3072, 1536), - (64, 1024, 1024), (64, 1024, 1536), (64, 3072, 1024), (64, 2048, 1536), @@ -49,7 +47,7 @@ MNK_FACTORS = [ @pytest.mark.parametrize("m,n,k", MNK_FACTORS) @pytest.mark.parametrize("e", [40, 64, 256]) @pytest.mark.parametrize("topk", [1, 6, 8]) -@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) @torch.inference_mode() def test_flashinfer_fp4_moe_no_graph( m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype diff --git a/tests/kernels/moe/test_grouped_topk.py b/tests/kernels/moe/test_grouped_topk.py index 3f4f142be7674..662e0723b7583 100644 --- a/tests/kernels/moe/test_grouped_topk.py +++ b/tests/kernels/moe/test_grouped_topk.py @@ -27,7 +27,7 @@ from vllm.platforms import current_platform @pytest.mark.parametrize("topk_group", [2]) @pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"]) @pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) def test_grouped_topk( monkeypatch: pytest.MonkeyPatch, n_token: int, diff --git a/tests/kernels/moe/test_modular_kernel_combinations.py b/tests/kernels/moe/test_modular_kernel_combinations.py index a46b0053e75a3..e3b8621b452fa 100644 --- a/tests/kernels/moe/test_modular_kernel_combinations.py +++ b/tests/kernels/moe/test_modular_kernel_combinations.py @@ -295,6 +295,8 @@ def test_modular_kernel_combinations_singlegpu( world_size: int, pytestconfig, ): + """Note: float8_e4m3fn is not supported on CUDA architecture < 89, + and those tests will be skipped on unsupported hardware.""" config = Config( Ms=Ms, K=k, @@ -309,6 +311,12 @@ def test_modular_kernel_combinations_singlegpu( world_size=world_size, ) + if ( + quant_config is not None and quant_config.quant_dtype == torch.float8_e4m3fn + ) and not current_platform.has_device_capability(89): + pytest.skip( + "Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89" + ) verbosity = pytestconfig.getoption("verbose") run(config, verbosity > 0) diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index 2c802ff4e6bd6..014df1fa111f2 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -66,8 +66,6 @@ FUSED_MOE_MNK_FACTORS = [ (1, 128, 128), (1, 2048, 128), (33, 2048, 128), - (222, 1024, 1024), - (32768, 128, 128), (32768, 2048, 511), (40000, 1024, 1024), ] @@ -76,7 +74,6 @@ FUSED_MOE_WN16_MNK_FACTORS = [ (1, 128, 128), (1, 1024, 1024), (32, 2048, 128), - (32, 1024, 1024), (222, 2048, 1024), ] @@ -512,8 +509,8 @@ def marlin_moe_generate_valid_test_cases(): e_list = [4, 12] topk_list = [2, 3] ep_size_list = [1, 4] - dtype_list = [torch.half, torch.bfloat16] - group_size_list = [-1, 16, 32, 128] + dtype_list = [torch.bfloat16] + group_size_list = [-1, 32, 128] act_order_list = [True, False] quant_type_list = [ scalar_types.float4_e2m1f, @@ -885,10 +882,10 @@ def test_batched_moe_align_block_size_opcheck(): ) -@pytest.mark.parametrize("m", [1, 33, 64, 222]) +@pytest.mark.parametrize("m", [1, 33, 222]) @pytest.mark.parametrize("topk", TOP_KS) @pytest.mark.parametrize("k", [128, 511, 1024]) -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) @pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm") def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype): input = torch.randn((m, topk, k), device="cuda", dtype=dtype) diff --git a/tests/kernels/moe/test_nvfp4_moe.py b/tests/kernels/moe/test_nvfp4_moe.py index dae19c0b2b31b..aa544fe0e0f63 100644 --- a/tests/kernels/moe/test_nvfp4_moe.py +++ b/tests/kernels/moe/test_nvfp4_moe.py @@ -26,9 +26,7 @@ MNK_FACTORS = [ (2, 1024, 1024), (2, 1024, 1536), (2, 3072, 1024), - (2, 3072, 1536), (64, 1024, 1024), - (64, 1024, 1536), (64, 3072, 1024), (64, 2048, 1536), (224, 1024, 1024), @@ -39,7 +37,7 @@ MNK_FACTORS = [ @pytest.mark.parametrize("m,n,k", MNK_FACTORS) @pytest.mark.parametrize("e", [40, 64, 256]) @pytest.mark.parametrize("topk", [1, 6, 8]) -@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) @torch.inference_mode() def test_cutlass_fp4_moe_no_graph( m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype diff --git a/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py b/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py index 92e78ec2396dd..97a55c37b9a3e 100644 --- a/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py +++ b/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py @@ -19,20 +19,16 @@ CASES = [ (32, 64, 256, fp8_dtype), (17, 31, 768, fp8_dtype), (1, 1, 128 * 1, fp8_dtype), - (1, 1, 128 * 2, fp8_dtype), (1, 1, 128 * 3, fp8_dtype), (1, 1, 128 * 4, fp8_dtype), (8, 16, 128 * 1, fp8_dtype), (8, 16, 128 * 2, fp8_dtype), (8, 16, 128 * 3, fp8_dtype), - (8, 16, 128 * 4, fp8_dtype), (8, 64, 7168, fp8_dtype), (8, 128, 7168, fp8_dtype), - (8, 256, 7168, fp8_dtype), (8, 512, 7168, fp8_dtype), (8, 1024, 7168, fp8_dtype), (256, 8, 7168, fp8_dtype), - (256, 16, 7168, fp8_dtype), (256, 32, 7168, fp8_dtype), (256, 64, 7168, fp8_dtype), # Only add a few fnuz tests to help with long CI times. From b5d70751d82c272a72f105299ef24ae316c41ded Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Oct 2025 12:39:34 +0800 Subject: [PATCH 093/127] [BugFix] Reordering extend logic fix (#27739) Signed-off-by: Lucas Wilkinson --- tests/v1/attention/test_batch_reordering.py | 21 ++++++++++++++++++--- vllm/v1/attention/backends/utils.py | 10 +++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/tests/v1/attention/test_batch_reordering.py b/tests/v1/attention/test_batch_reordering.py index b271409b92955..e37219454222b 100644 --- a/tests/v1/attention/test_batch_reordering.py +++ b/tests/v1/attention/test_batch_reordering.py @@ -53,7 +53,7 @@ REORDER_TEST_CASES = { expected_modified=True, ), "already_ordered": ReorderTestCase( - requests=[(1, 10), (1, 20), (100, 100), (200, 200)], + requests=[(1, 10), (1, 20), (100, 100), (200, 0)], expected_order=[0, 1, 2, 3], expected_modified=False, ), @@ -74,15 +74,30 @@ REORDER_TEST_CASES = { expected_modified=True, ), "decode_extend_prefill": ReorderTestCase( - requests=[(100, 100), (10, 50), (1, 10)], + requests=[(100, 0), (10, 50), (1, 10)], expected_order=[2, 1, 0], expected_modified=True, ), "extend_prefill_only": ReorderTestCase( - requests=[(100, 100), (10, 50), (200, 200), (20, 75)], + requests=[(100, 0), (10, 50), (200, 0), (20, 75)], expected_order=[3, 1, 2, 0], # Only swap 0↔3, keep 1 and 2 in place expected_modified=True, ), + "complicated_mixed_interleaved": ReorderTestCase( + requests=[ + (1, 20), + (1, 50), + (374, 0), + (300, 20), + (1, 20), + (256, 0), + (1, 5), + (27, 0), + (1, 4), + ], + expected_order=[0, 1, 6, 8, 4, 3, 2, 7, 5], + expected_modified=True, + ), } diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 389baf1488be0..07d62e9849e00 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -811,8 +811,8 @@ def reorder_batch_to_split_decodes_and_prefills( num_computed_tokens_np = input_batch.num_computed_tokens_cpu[:num_reqs] is_decode = num_scheduled_tokens_np <= decode_threshold - is_extend = (~is_decode) & (num_computed_tokens_np > num_scheduled_tokens_np) - is_prefill = (~is_decode) & (num_computed_tokens_np == num_scheduled_tokens_np) + is_extend = (~is_decode) & (num_computed_tokens_np > 0) + is_prefill = (~is_decode) & (num_computed_tokens_np == 0) # Desired order: decode → extend → prefill req_regions = np.zeros(is_decode.shape, dtype=np.int32) # 0 = decode by default @@ -832,11 +832,11 @@ def reorder_batch_to_split_decodes_and_prefills( return False # Extract indices that need swapping and sort by target region - swap_indices = np.where(needs_swap)[0] + orig_indices = np.where(needs_swap)[0] sorted_order = np.argsort(req_regions[needs_swap], kind="stable") - dest_indices = swap_indices[sorted_order] + src_indices = orig_indices[sorted_order] - src_dest_map = {int(src): int(dst) for src, dst in zip(swap_indices, dest_indices)} + src_dest_map = {int(src): int(dst) for src, dst in zip(src_indices, orig_indices)} for src in src_dest_map: dst = src_dest_map[src] From 8bff831f0aa239006f34b721e63e1340e3472067 Mon Sep 17 00:00:00 2001 From: Kuntai Du Date: Wed, 29 Oct 2025 21:43:37 -0700 Subject: [PATCH 094/127] [Benchmark] Cleanup deprecated nightly benchmark and adjust the docstring for performance benchmark (#25786) Signed-off-by: KuntaiDu --- .../benchmark-pipeline.yaml | 184 ------- .../nightly-benchmarks/nightly-annotation.md | 28 -- .../nightly-descriptions.md | 39 -- .../nightly-benchmarks/nightly-pipeline.yaml | 196 -------- .../scripts/download-tokenizer.py | 26 - .../scripts/generate-nightly-markdown.py | 97 ---- .../scripts/get-lmdeploy-modelname.py | 9 - .../scripts/nightly-annotate.sh | 78 --- .../scripts/run-nightly-benchmarks.sh | 464 ------------------ .../scripts/summary-nightly-results.py | 82 ---- .../scripts/wait-for-image.sh | 23 - .../README.md | 54 +- .../performance-benchmarks-descriptions.md | 0 .../scripts/compare-json-results.py | 0 .../convert-results-json-to-markdown.py | 2 +- .../scripts/launch-server.sh | 0 .../scripts/run-performance-benchmarks.sh | 2 +- .../tests/genai-perf-tests.json | 0 .../tests/latency-tests-cpu.json | 0 .../tests/latency-tests.json | 0 .../tests/nightly-tests.json | 0 .../tests/serving-tests-cpu-snc2.json | 0 .../tests/serving-tests-cpu-snc3.json | 0 .../tests/serving-tests-cpu.json | 0 .../tests/serving-tests.json | 0 .../tests/throughput-tests-cpu.json | 0 .../tests/throughput-tests.json | 0 .github/mergify.yml | 2 +- docs/contributing/benchmarks.md | 13 +- 29 files changed, 10 insertions(+), 1289 deletions(-) delete mode 100644 .buildkite/nightly-benchmarks/benchmark-pipeline.yaml delete mode 100644 .buildkite/nightly-benchmarks/nightly-annotation.md delete mode 100644 .buildkite/nightly-benchmarks/nightly-descriptions.md delete mode 100644 .buildkite/nightly-benchmarks/nightly-pipeline.yaml delete mode 100644 .buildkite/nightly-benchmarks/scripts/download-tokenizer.py delete mode 100644 .buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py delete mode 100644 .buildkite/nightly-benchmarks/scripts/get-lmdeploy-modelname.py delete mode 100644 .buildkite/nightly-benchmarks/scripts/nightly-annotate.sh delete mode 100644 .buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh delete mode 100644 .buildkite/nightly-benchmarks/scripts/summary-nightly-results.py delete mode 100644 .buildkite/nightly-benchmarks/scripts/wait-for-image.sh rename .buildkite/{nightly-benchmarks => performance-benchmarks}/README.md (69%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/performance-benchmarks-descriptions.md (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/scripts/compare-json-results.py (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/scripts/convert-results-json-to-markdown.py (99%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/scripts/launch-server.sh (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/scripts/run-performance-benchmarks.sh (99%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/genai-perf-tests.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/latency-tests-cpu.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/latency-tests.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/nightly-tests.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/serving-tests-cpu-snc2.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/serving-tests-cpu-snc3.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/serving-tests-cpu.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/serving-tests.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/throughput-tests-cpu.json (100%) rename .buildkite/{nightly-benchmarks => performance-benchmarks}/tests/throughput-tests.json (100%) diff --git a/.buildkite/nightly-benchmarks/benchmark-pipeline.yaml b/.buildkite/nightly-benchmarks/benchmark-pipeline.yaml deleted file mode 100644 index 4259514940d3f..0000000000000 --- a/.buildkite/nightly-benchmarks/benchmark-pipeline.yaml +++ /dev/null @@ -1,184 +0,0 @@ -steps: - - label: "Wait for container to be ready" - key: wait-for-container-image - agents: - queue: A100 - plugins: - - kubernetes: - podSpec: - containers: - - image: badouralix/curl-jq - command: - - sh .buildkite/nightly-benchmarks/scripts/wait-for-image.sh - - label: "Cleanup H100" - agents: - queue: H100 - depends_on: ~ - command: docker system prune -a --volumes --force - - - label: "A100" - # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing" - agents: - queue: A100 - depends_on: wait-for-container-image - if: build.branch == "main" - plugins: - - kubernetes: - podSpec: - priorityClassName: perf-benchmark - containers: - - image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:$BUILDKITE_COMMIT - command: - - bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh - resources: - limits: - nvidia.com/gpu: 8 - volumeMounts: - - name: devshm - mountPath: /dev/shm - env: - - name: VLLM_USAGE_SOURCE - value: ci-test - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB - volumes: - - name: devshm - emptyDir: - medium: Memory - - - label: "H200" - # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing" - agents: - queue: H200 - depends_on: wait-for-container-image - if: build.branch == "main" - plugins: - - docker#v5.12.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:$BUILDKITE_COMMIT - command: - - bash - - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh - mount-buildkite-agent: true - propagate-environment: true - ipc: host - gpus: 4,5,6,7 - volumes: - - /data/benchmark-hf-cache:/root/.cache/huggingface - environment: - - VLLM_USAGE_SOURCE - - HF_TOKEN - - #- block: "Run H100 Benchmark" - #key: block-h100 - #depends_on: ~ - - - label: "H100" - # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing" - agents: - queue: H100 - depends_on: wait-for-container-image - if: build.branch == "main" - plugins: - - docker#v5.12.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:$BUILDKITE_COMMIT - command: - - bash - - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh - mount-buildkite-agent: true - propagate-environment: true - ipc: host - gpus: all # see CUDA_VISIBLE_DEVICES for actual GPUs used - volumes: - - /data/benchmark-hf-cache:/root/.cache/huggingface - environment: - - VLLM_USAGE_SOURCE - - HF_TOKEN - - # Premerge benchmark - - label: "A100" - # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing" - agents: - queue: A100 - depends_on: wait-for-container-image - if: build.branch != "main" - plugins: - - kubernetes: - podSpec: - priorityClassName: perf-benchmark - containers: - - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - command: - - bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh - resources: - limits: - nvidia.com/gpu: 8 - volumeMounts: - - name: devshm - mountPath: /dev/shm - env: - - name: VLLM_USAGE_SOURCE - value: ci-test - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB - volumes: - - name: devshm - emptyDir: - medium: Memory - - - label: "H200" - # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing" - agents: - queue: H200 - depends_on: wait-for-container-image - if: build.branch != "main" - plugins: - - docker#v5.12.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - command: - - bash - - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh - mount-buildkite-agent: true - propagate-environment: true - ipc: host - gpus: 4,5,6,7 - volumes: - - /data/benchmark-hf-cache:/root/.cache/huggingface - environment: - - VLLM_USAGE_SOURCE - - HF_TOKEN - - #- block: "Run H100 Benchmark" - #key: block-h100 - #depends_on: ~ - - - label: "H100" - # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing" - agents: - queue: H100 - depends_on: wait-for-container-image - if: build.branch != "main" - plugins: - - docker#v5.12.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - command: - - bash - - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh - mount-buildkite-agent: true - propagate-environment: true - ipc: host - gpus: all # see CUDA_VISIBLE_DEVICES for actual GPUs used - volumes: - - /data/benchmark-hf-cache:/root/.cache/huggingface - environment: - - VLLM_USAGE_SOURCE - - HF_TOKEN diff --git a/.buildkite/nightly-benchmarks/nightly-annotation.md b/.buildkite/nightly-benchmarks/nightly-annotation.md deleted file mode 100644 index 466def07b6f1f..0000000000000 --- a/.buildkite/nightly-benchmarks/nightly-annotation.md +++ /dev/null @@ -1,28 +0,0 @@ -# Nightly benchmark annotation - -## Description - -This file contains the downloading link for benchmarking results. - -- [benchmarking pipeline](artifact://nightly-pipeline.yaml) -- [benchmarking results](artifact://results.zip) -- [benchmarking code](artifact://nightly-benchmarks.zip) - -Please download the visualization scripts in the post - -## Results reproduction - -- Find the docker we use in `benchmarking pipeline` -- Deploy the docker, and inside the docker: - - Download `nightly-benchmarks.zip`. - - In the same folder, run the following code: - - ```bash - export HF_TOKEN= - apt update - apt install -y git - unzip nightly-benchmarks.zip - VLLM_SOURCE_CODE_LOC=./ bash .buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh - ``` - -And the results will be inside `./benchmarks/results`. diff --git a/.buildkite/nightly-benchmarks/nightly-descriptions.md b/.buildkite/nightly-benchmarks/nightly-descriptions.md deleted file mode 100644 index 2ef36089b6afb..0000000000000 --- a/.buildkite/nightly-benchmarks/nightly-descriptions.md +++ /dev/null @@ -1,39 +0,0 @@ - -# Nightly benchmark - -This benchmark aims to: - -- Provide performance clarity: Provide clarity on which one (vllm, tensorrt-llm, lmdeploy and SGLang) leads in performance in what workload. -- Be reproducible: one can run the exact same set of benchmarking commands inside the exact same docker by following reproducing instructions. - -Latest results: [results link](https://blog.vllm.ai/2024/09/05/perf-update.html), scroll to the end. - -Latest reproduction guide: [github issue link](https://github.com/vllm-project/vllm/issues/8176) - -## Setup - -- Docker images: - - vLLM: `vllm/vllm-openai:v0.6.2` - - SGLang: `lmsysorg/sglang:v0.3.2-cu121` - - LMDeploy: `openmmlab/lmdeploy:v0.6.1-cu12` - - TensorRT-LLM: `nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3` - - *NOTE: we use r24.07 as the current implementation only works for this version. We are going to bump this up.* - - Check [nightly-pipeline.yaml](nightly-pipeline.yaml) for the concrete docker images, specs and commands we use for the benchmark. -- Hardware - - 8x Nvidia A100 GPUs -- Workload: - - Dataset - - ShareGPT dataset - - Prefill-heavy dataset (in average 462 input tokens, 16 tokens as output) - - Decode-heavy dataset (in average 462 input tokens, 256 output tokens) - - Check [nightly-tests.json](tests/nightly-tests.json) for the concrete configuration of datasets we use. - - Models: llama-3 8B, llama-3 70B. - - We do not use llama 3.1 as it is incompatible with trt-llm r24.07. ([issue](https://github.com/NVIDIA/TensorRT-LLM/issues/2105)). - - Average QPS (query per second): 2, 4, 8, 16, 32 and inf. - - Queries are randomly sampled, and arrival patterns are determined via Poisson process, but all with fixed random seed. - - Evaluation metrics: Throughput (higher the better), TTFT (time to the first token, lower the better), ITL (inter-token latency, lower the better). - -## Known issues - -- TRT-LLM crashes with Llama 3.1 8B [issue](https://github.com/NVIDIA/TensorRT-LLM/issues/2105). -- TGI does not support `ignore-eos` flag. diff --git a/.buildkite/nightly-benchmarks/nightly-pipeline.yaml b/.buildkite/nightly-benchmarks/nightly-pipeline.yaml deleted file mode 100644 index 199517e8b067c..0000000000000 --- a/.buildkite/nightly-benchmarks/nightly-pipeline.yaml +++ /dev/null @@ -1,196 +0,0 @@ -common_pod_spec: &common_pod_spec - priorityClassName: perf-benchmark - nodeSelector: - nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /root/.cache/huggingface - type: Directory - -common_container_settings: &common_container_settings - command: - - bash .buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh - resources: - limits: - nvidia.com/gpu: 8 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: VLLM_USAGE_SOURCE - value: ci-test - - name: HF_HOME - value: /root/.cache/huggingface - - name: VLLM_SOURCE_CODE_LOC - value: /workspace/build/buildkite/vllm/performance-benchmark - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - -steps: - - block: ":rocket: Ready for comparing vllm against alternatives? This will take 4 hours." - - - - - label: "A100 vllm step 10" - priority: 100 - agents: - queue: A100 - plugins: - - kubernetes: - podSpec: - <<: *common_pod_spec - containers: - - image: vllm/vllm-openai:v0.6.2 - <<: *common_container_settings - - - - - label: "A100 sglang benchmark" - priority: 100 - agents: - queue: A100 - plugins: - - kubernetes: - podSpec: - <<: *common_pod_spec - containers: - - image: lmsysorg/sglang:v0.3.2-cu121 - <<: *common_container_settings - - - label: "A100 lmdeploy benchmark" - priority: 100 - agents: - queue: A100 - plugins: - - kubernetes: - podSpec: - <<: *common_pod_spec - containers: - - image: openmmlab/lmdeploy:v0.6.1-cu12 - <<: *common_container_settings - - - - - - label: "A100 trt llama-8B" - priority: 100 - agents: - queue: A100 - plugins: - - kubernetes: - podSpec: - <<: *common_pod_spec - containers: - - image: nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3 - <<: *common_container_settings - env: - - name: VLLM_USAGE_SOURCE - value: ci-test - - name: HF_HOME - value: /root/.cache/huggingface - - name: VLLM_SOURCE_CODE_LOC - value: /workspace/build/buildkite/vllm/performance-benchmark - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - - name: TEST_SELECTOR - value: "llama8B" - - - - label: "A100 trt llama-70B" - priority: 100 - agents: - queue: A100 - plugins: - - kubernetes: - podSpec: - <<: *common_pod_spec - containers: - - image: nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3 - <<: *common_container_settings - env: - - name: VLLM_USAGE_SOURCE - value: ci-test - - name: HF_HOME - value: /root/.cache/huggingface - - name: VLLM_SOURCE_CODE_LOC - value: /workspace/build/buildkite/vllm/performance-benchmark - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - - name: TEST_SELECTOR - value: "llama70B" - - - # FIXME(Kuntai): uncomment this after NVIDIA gives us their test docker image - # - label: "A100 trt benchmark" - # priority: 100 - # agents: - # queue: A100 - # plugins: - # - kubernetes: - # podSpec: - # <<: *common_pod_spec - # containers: - # - image: nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3 - # <<: *common_container_settings - - - # FIXME(Kuntai): uncomment this after TGI supports `--ignore-eos`. - # - label: "A100 tgi benchmark" - # priority: 100 - # agents: - # queue: A100 - # plugins: - # - kubernetes: - # podSpec: - # <<: *common_pod_spec - # containers: - # - image: ghcr.io/huggingface/text-generation-inference:2.2.0 - # <<: *common_container_settings - - - wait - - - label: "Collect the results" - priority: 100 - agents: - queue: A100 - plugins: - - kubernetes: - podSpec: - <<: *common_pod_spec - containers: - - image: vllm/vllm-openai:v0.5.0.post1 - command: - - bash .buildkite/nightly-benchmarks/scripts/nightly-annotate.sh - resources: - limits: - nvidia.com/gpu: 8 - volumeMounts: - - name: devshm - mountPath: /dev/shm - env: - - name: VLLM_USAGE_SOURCE - value: ci-test - - name: VLLM_SOURCE_CODE_LOC - value: /workspace/build/buildkite/vllm/performance-benchmark - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - - - block: ":rocket: check the results!" \ No newline at end of file diff --git a/.buildkite/nightly-benchmarks/scripts/download-tokenizer.py b/.buildkite/nightly-benchmarks/scripts/download-tokenizer.py deleted file mode 100644 index 8532ff7ef798c..0000000000000 --- a/.buildkite/nightly-benchmarks/scripts/download-tokenizer.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import argparse - -from transformers import AutoTokenizer - - -def main(model, cachedir): - # Load the tokenizer and save it to the specified directory - tokenizer = AutoTokenizer.from_pretrained(model) - tokenizer.save_pretrained(cachedir) - print(f"Tokenizer saved to {cachedir}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Download and save Hugging Face tokenizer" - ) - parser.add_argument("--model", type=str, required=True, help="Name of the model") - parser.add_argument( - "--cachedir", type=str, required=True, help="Directory to save the tokenizer" - ) - - args = parser.parse_args() - main(args.model, args.cachedir) diff --git a/.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py b/.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py deleted file mode 100644 index 053fd52c35ae9..0000000000000 --- a/.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import argparse -import json -from pathlib import Path - -import numpy as np -import pandas as pd -from tabulate import tabulate - - -def parse_arguments(): - parser = argparse.ArgumentParser( - description="Parse command line arguments for summary-nightly-results script." - ) - parser.add_argument( - "--results-folder", - type=str, - required=True, - help="The folder where the results are stored.", - ) - parser.add_argument( - "--description", type=str, required=True, help="Description of the results." - ) - - args = parser.parse_args() - return args - - -def get_perf(df, method, model, metric): - means = [] - - for qps in [2, 4, 8, 16, "inf"]: - target = df["Test name"].str.contains(model) - target = target & df["Engine"].str.contains(method) - target = target & df["Test name"].str.contains("qps_" + str(qps)) - filtered_df = df[target] - - if filtered_df.empty: - means.append(0.0) - else: - means.append(filtered_df[metric].values[0]) - - return np.array(means) - - -def get_perf_w_std(df, method, model, metric): - if metric in ["TTFT", "ITL"]: - mean = get_perf(df, method, model, "Mean " + metric + " (ms)") - mean = mean.tolist() - std = get_perf(df, method, model, "Std " + metric + " (ms)") - if std.mean() == 0: - std = None - success = get_perf(df, method, model, "Successful req.") - if std is not None: - std = std / np.sqrt(success) - std = std.tolist() - - else: - assert metric == "Tput" - mean = get_perf(df, method, model, "Input Tput (tok/s)") + get_perf( - df, method, model, "Output Tput (tok/s)" - ) - mean = mean.tolist() - std = None - - return mean, std - - -def main(args): - results_folder = Path(args.results_folder) - - results = [] - - # collect results - for test_file in results_folder.glob("*_nightly_results.json"): - with open(test_file) as f: - results = results + json.loads(f.read()) - - # generate markdown table - df = pd.DataFrame.from_dict(results) - - md_table = tabulate(df, headers="keys", tablefmt="pipe", showindex=False) - - with open(args.description) as f: - description = f.read() - - description = description.format(nightly_results_benchmarking_table=md_table) - - with open("nightly_results.md", "w") as f: - f.write(description) - - -if __name__ == "__main__": - args = parse_arguments() - main(args) diff --git a/.buildkite/nightly-benchmarks/scripts/get-lmdeploy-modelname.py b/.buildkite/nightly-benchmarks/scripts/get-lmdeploy-modelname.py deleted file mode 100644 index ddea1d2b1b1ed..0000000000000 --- a/.buildkite/nightly-benchmarks/scripts/get-lmdeploy-modelname.py +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from lmdeploy.serve.openai.api_client import APIClient - -api_client = APIClient("http://localhost:8000") -model_name = api_client.available_models[0] - -print(model_name) diff --git a/.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh b/.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh deleted file mode 100644 index 69b6b146b3549..0000000000000 --- a/.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -set -ex -set -o pipefail - - -main() { - - (which wget && which curl) || (apt-get update && apt-get install -y wget curl) - (which jq) || (apt-get update && apt-get -y install jq) - (which zip) || (apt-get install -y zip) - - if [ ! -f /workspace/buildkite-agent ]; then - echo "buildkite-agent binary not found. Skip plotting the results." - exit 0 - fi - - # initial annotation - #description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md" - - # download results - cd "$VLLM_SOURCE_CODE_LOC/benchmarks" - mkdir -p results/ - /workspace/buildkite-agent artifact download 'results/*nightly_results.json' results/ - ls - ls results/ - - # upload benchmark results - zip -r results.zip results/ - /workspace/buildkite-agent artifact upload "results.zip" - - # upload benchmarking scripts - cd "$VLLM_SOURCE_CODE_LOC/" - zip -r nightly-benchmarks.zip .buildkite/ benchmarks/ - /workspace/buildkite-agent artifact upload "nightly-benchmarks.zip" - - cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/" - # upload benchmarking pipeline - /workspace/buildkite-agent artifact upload "nightly-pipeline.yaml" - - cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/" - /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly-annotation.md - - - - # The figures should be generated by a separate process outside the CI/CD pipeline - - # # generate figures - # python3 -m pip install tabulate pandas matplotlib - - # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py \ - # --description $description \ - # --results-folder results/ - - - # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/plot-nightly-results.py \ - # --description $description \ - # --results-folder results/ \ - # --dataset sharegpt - - # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/plot-nightly-results.py \ - # --description $description \ - # --results-folder results/ \ - # --dataset sonnet_2048_128 - - # python3 $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/plot-nightly-results.py \ - # --description $description \ - # --results-folder results/ \ - # --dataset sonnet_128_2048 - - # # upload results and figures - # /workspace/buildkite-agent artifact upload "nightly_results*.png" - # /workspace/buildkite-agent artifact upload $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-pipeline.yaml - # /workspace/buildkite-agent artifact upload $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/tests/nightly-tests.json - # /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly_results.md -} - -main "$@" diff --git a/.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh b/.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh deleted file mode 100644 index a00de940cbbb8..0000000000000 --- a/.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash - -set -o pipefail -set -x - -check_gpus() { - # check the number of GPUs and GPU type. - declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l) - if [[ $gpu_count -gt 0 ]]; then - echo "GPU found." - else - echo "Need at least 1 GPU to run benchmarking." - exit 1 - fi - declare -g gpu_type="$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')" - echo "GPU type is $gpu_type" -} - -check_hf_token() { - # check if HF_TOKEN is available and valid - if [[ -z "$HF_TOKEN" ]]; then - echo "Error: HF_TOKEN is not set." - exit 1 - elif [[ ! "$HF_TOKEN" =~ ^hf_ ]]; then - echo "Error: HF_TOKEN does not start with 'hf_'." - exit 1 - else - echo "HF_TOKEN is set and valid." - fi -} - - -upload_to_buildkite() { - # upload the benchmarking results to buildkite - - # if the agent binary is not found, skip uploading the results, exit 0 - if [ ! -f /workspace/buildkite-agent ]; then - echo "buildkite-agent binary not found. Skip uploading the results." - return 0 - fi - # /workspace/buildkite-agent annotate --style "success" --context "benchmark-results" --append < $RESULTS_FOLDER/${CURRENT_LLM_SERVING_ENGINE}_nightly_results.md - /workspace/buildkite-agent artifact upload "$RESULTS_FOLDER/*" -} - - -get_current_llm_serving_engine() { - - if which lmdeploy >/dev/null; then - echo "Container: lmdeploy" - export CURRENT_LLM_SERVING_ENGINE=lmdeploy - return - fi - - if [ -e /tgi-entrypoint.sh ]; then - echo "Container: tgi" - export CURRENT_LLM_SERVING_ENGINE=tgi - return - fi - - if which trtllm-build >/dev/null; then - echo "Container: tensorrt-llm" - export CURRENT_LLM_SERVING_ENGINE=trt - return - fi - - if [ -e /sgl-workspace ]; then - echo "Container: sglang" - export CURRENT_LLM_SERVING_ENGINE=sglang - return - fi - - if [ -e /vllm-workspace ]; then - echo "Container: vllm" - # move to a completely irrelevant directory, to avoid import vllm from current folder - export CURRENT_LLM_SERVING_ENGINE=vllm - - return - fi -} - -json2args() { - # transforms the JSON string to command line args, and '_' is replaced to '-' - # example: - # input: { "model": "meta-llama/Llama-2-7b-chat-hf", "tensor_parallel_size": 1 } - # output: --model meta-llama/Llama-2-7b-chat-hf --tensor-parallel-size 1 - local json_string=$1 - local args=$( - echo "$json_string" | jq -r ' - to_entries | - map("--" + (.key | gsub("_"; "-")) + " " + (.value | tostring)) | - join(" ") - ' - ) - echo "$args" -} - -kill_gpu_processes() { - pkill -f '[p]ython' - pkill -f '[p]ython3' - pkill -f '[t]ritonserver' - pkill -f '[p]t_main_thread' - pkill -f '[t]ext-generation' - pkill -f '[l]mdeploy' - # vLLM now names the process with VLLM prefix after https://github.com/vllm-project/vllm/pull/21445 - pkill -f '[V]LLM' - - while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do - sleep 1 - done -} - -wait_for_server() { - # wait for vllm server to start - # return 1 if vllm server crashes - timeout 1200 bash -c ' - until curl -s localhost:8000/v1/completions > /dev/null; do - sleep 1 - done' && return 0 || return 1 -} - -ensure_installed() { - # Ensure that the given command is installed by apt-get - local cmd=$1 - if ! which "$cmd" >/dev/null; then - apt-get update && apt-get install -y "$cmd" - fi -} - -run_serving_tests() { - # run serving tests using `vllm bench serve` command - # $1: a json file specifying serving test cases - - local serving_test_file - serving_test_file=$1 - - # Iterate over serving tests - jq -c '.[]' "$serving_test_file" | while read -r params; do - # get the test name, and append the GPU type back to it. - test_name=$(echo "$params" | jq -r '.test_name') - - # if TEST_SELECTOR is set, only run the test cases that match the selector - if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then - echo "Skip test case $test_name." - continue - fi - - # prepend the current serving engine to the test name - test_name=${CURRENT_LLM_SERVING_ENGINE}_${test_name} - - # get common parameters - common_params=$(echo "$params" | jq -r '.common_parameters') - model=$(echo "$common_params" | jq -r '.model') - tp=$(echo "$common_params" | jq -r '.tp') - dataset_name=$(echo "$common_params" | jq -r '.dataset_name') - dataset_path=$(echo "$common_params" | jq -r '.dataset_path') - port=$(echo "$common_params" | jq -r '.port') - num_prompts=$(echo "$common_params" | jq -r '.num_prompts') - reuse_server=$(echo "$common_params" | jq -r '.reuse_server') - - # get client and server arguments - server_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_server_parameters") - client_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_client_parameters") - client_args=$(json2args "$client_params") - qps_list=$(echo "$params" | jq -r '.qps_list') - qps_list=$(echo "$qps_list" | jq -r '.[] | @sh') - echo "Running over qps list $qps_list" - - # check if there is enough GPU to run the test - if [[ $gpu_count -lt $tp ]]; then - echo "Required num-shard $tp but only $gpu_count GPU found. Skip testcase $test_name." - continue - fi - - if [[ $reuse_server == "true" ]]; then - echo "Reuse previous server for test case $test_name" - else - kill_gpu_processes - bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \ - "$server_params" "$common_params" - fi - - if wait_for_server; then - echo "" - echo "$CURRENT_LLM_SERVING_ENGINE server is up and running." - else - echo "" - echo "$CURRENT_LLM_SERVING_ENGINE failed to start within the timeout period." - break - fi - - # prepare tokenizer - # this is required for lmdeploy. - cd "$VLLM_SOURCE_CODE_LOC/benchmarks" - rm -rf /tokenizer_cache - mkdir /tokenizer_cache - python3 ../.buildkite/nightly-benchmarks/scripts/download-tokenizer.py \ - --model "$model" \ - --cachedir /tokenizer_cache - cd "$VLLM_SOURCE_CODE_LOC/benchmarks" - - - # change model name for lmdeploy (it will not follow standard hf name) - if [[ "$CURRENT_LLM_SERVING_ENGINE" == "lmdeploy" ]]; then - model=$(python ../.buildkite/nightly-benchmarks/scripts/get-lmdeploy-modelname.py) - fi - - # iterate over different QPS - for qps in $qps_list; do - # remove the surrounding single quote from qps - if [[ "$qps" == *"inf"* ]]; then - echo "qps was $qps" - qps="inf" - echo "now qps is $qps" - fi - - new_test_name=$test_name"_qps_"$qps - - backend=$CURRENT_LLM_SERVING_ENGINE - - if [[ $backend = "trt" ]]; then - backend="tensorrt-llm" - fi - - if [[ "$backend" == *"vllm"* ]]; then - backend="vllm" - fi - - if [[ "$dataset_name" = "sharegpt" ]]; then - - client_command="vllm bench serve \ - --backend $backend \ - --tokenizer /tokenizer_cache \ - --model $model \ - --dataset-name $dataset_name \ - --dataset-path $dataset_path \ - --num-prompts $num_prompts \ - --port $port \ - --save-result \ - --result-dir $RESULTS_FOLDER \ - --result-filename ${new_test_name}.json \ - --request-rate $qps \ - --ignore-eos \ - $client_args" - - elif [[ "$dataset_name" = "sonnet" ]]; then - - sonnet_input_len=$(echo "$common_params" | jq -r '.sonnet_input_len') - sonnet_output_len=$(echo "$common_params" | jq -r '.sonnet_output_len') - sonnet_prefix_len=$(echo "$common_params" | jq -r '.sonnet_prefix_len') - - client_command="vllm bench serve \ - --backend $backend \ - --tokenizer /tokenizer_cache \ - --model $model \ - --dataset-name $dataset_name \ - --dataset-path $dataset_path \ - --num-prompts $num_prompts \ - --sonnet-input-len $sonnet_input_len \ - --sonnet-output-len $sonnet_output_len \ - --sonnet-prefix-len $sonnet_prefix_len \ - --port $port \ - --save-result \ - --result-dir $RESULTS_FOLDER \ - --result-filename ${new_test_name}.json \ - --request-rate $qps \ - --ignore-eos \ - $client_args" - - else - - echo "The dataset name must be either 'sharegpt' or 'sonnet'. Got $dataset_name." - exit 1 - - fi - - - - echo "Running test case $test_name with qps $qps" - echo "Client command: $client_command" - - eval "$client_command" - - server_command="None" - - # record the benchmarking commands - jq_output=$(jq -n \ - --arg server "$server_command" \ - --arg client "$client_command" \ - --arg gpu "$gpu_type" \ - --arg engine "$CURRENT_LLM_SERVING_ENGINE" \ - '{ - server_command: $server, - client_command: $client, - gpu_type: $gpu, - engine: $engine - }') - echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands" - - done - - done - - kill_gpu_processes -} - -run_genai_perf_tests() { - # run genai-perf tests - - # $1: a json file specifying genai-perf test cases - local genai_perf_test_file - genai_perf_test_file=$1 - - # Iterate over genai-perf tests - jq -c '.[]' "$genai_perf_test_file" | while read -r params; do - # get the test name, and append the GPU type back to it. - test_name=$(echo "$params" | jq -r '.test_name') - - # if TEST_SELECTOR is set, only run the test cases that match the selector - if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then - echo "Skip test case $test_name." - continue - fi - - # prepend the current serving engine to the test name - test_name=${CURRENT_LLM_SERVING_ENGINE}_${test_name} - - # get common parameters - common_params=$(echo "$params" | jq -r '.common_parameters') - model=$(echo "$common_params" | jq -r '.model') - tp=$(echo "$common_params" | jq -r '.tp') - dataset_name=$(echo "$common_params" | jq -r '.dataset_name') - dataset_path=$(echo "$common_params" | jq -r '.dataset_path') - port=$(echo "$common_params" | jq -r '.port') - num_prompts=$(echo "$common_params" | jq -r '.num_prompts') - reuse_server=$(echo "$common_params" | jq -r '.reuse_server') - - # get client and server arguments - server_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_server_parameters") - qps_list=$(echo "$params" | jq -r '.qps_list') - qps_list=$(echo "$qps_list" | jq -r '.[] | @sh') - echo "Running over qps list $qps_list" - - # check if there is enough GPU to run the test - if [[ $gpu_count -lt $tp ]]; then - echo "Required num-shard $tp but only $gpu_count GPU found. Skip testcase $test_name." - continue - fi - - if [[ $reuse_server == "true" ]]; then - echo "Reuse previous server for test case $test_name" - else - kill_gpu_processes - bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \ - "$server_params" "$common_params" - fi - - if wait_for_server; then - echo "" - echo "$CURRENT_LLM_SERVING_ENGINE server is up and running." - else - echo "" - echo "$CURRENT_LLM_SERVING_ENGINE failed to start within the timeout period." - break - fi - - # iterate over different QPS - for qps in $qps_list; do - # remove the surrounding single quote from qps - if [[ "$qps" == *"inf"* ]]; then - echo "qps was $qps" - qps=$num_prompts - echo "now qps is $qps" - fi - - new_test_name=$test_name"_qps_"$qps - backend=$CURRENT_LLM_SERVING_ENGINE - - if [[ "$backend" == *"vllm"* ]]; then - backend="vllm" - fi - #TODO: add output dir. - client_command="genai-perf profile \ - -m $model \ - --service-kind openai \ - --backend "$backend" \ - --endpoint-type chat \ - --streaming \ - --url localhost:$port \ - --request-rate $qps \ - --num-prompts $num_prompts \ - " - - echo "Client command: $client_command" - - eval "$client_command" - - #TODO: process/record outputs - done - done - - kill_gpu_processes - -} - -prepare_dataset() { - - # download sharegpt dataset - cd "$VLLM_SOURCE_CODE_LOC/benchmarks" - wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json - - # duplicate sonnet by 4x, to allow benchmarking with input length 2048 - cd "$VLLM_SOURCE_CODE_LOC/benchmarks" - echo "" > sonnet_4x.txt - for _ in {1..4} - do - cat sonnet.txt >> sonnet_4x.txt - done - -} - -main() { - - # check if the environment variable is successfully injected from yaml - - check_gpus - check_hf_token - get_current_llm_serving_engine - - pip install -U transformers - - pip install -r requirements/dev.txt - which genai-perf - - # check storage - df -h - - ensure_installed wget - ensure_installed curl - ensure_installed jq - # genai-perf dependency - ensure_installed libb64-0d - - prepare_dataset - - cd "$VLLM_SOURCE_CODE_LOC/benchmarks" - declare -g RESULTS_FOLDER=results/ - mkdir -p $RESULTS_FOLDER - BENCHMARK_ROOT="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/" - - # run the test - run_serving_tests "$BENCHMARK_ROOT/tests/nightly-tests.json" - - # run genai-perf tests - run_genai_perf_tests "$BENCHMARK_ROOT/tests/genai-perf-tests.json" - mv artifacts/ $RESULTS_FOLDER/ - - # upload benchmark results to buildkite - python3 -m pip install tabulate pandas - python3 "$BENCHMARK_ROOT/scripts/summary-nightly-results.py" - upload_to_buildkite - -} - -main "$@" diff --git a/.buildkite/nightly-benchmarks/scripts/summary-nightly-results.py b/.buildkite/nightly-benchmarks/scripts/summary-nightly-results.py deleted file mode 100644 index fb3b9d5e34e03..0000000000000 --- a/.buildkite/nightly-benchmarks/scripts/summary-nightly-results.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import datetime -import json -import os -from pathlib import Path - -import pandas as pd -from tabulate import tabulate - -results_folder = Path("results/") - -# serving results and the keys that will be printed into markdown -serving_results = [] -serving_column_mapping = { - "test_name": "Test name", - "gpu_type": "GPU", - "completed": "Successful req.", - "request_throughput": "Tput (req/s)", - "mean_ttft_ms": "Mean TTFT (ms)", - "std_ttft_ms": "Std TTFT (ms)", - "median_ttft_ms": "Median TTFT (ms)", - "mean_itl_ms": "Mean ITL (ms)", - "std_itl_ms": "Std ITL (ms)", - "median_itl_ms": "Median ITL (ms)", - "mean_tpot_ms": "Mean TPOT (ms)", - "std_tpot_ms": "Std TPOT (ms)", - "median_tpot_ms": "Median TPOT (ms)", - "total_token_throughput": "Total Token Tput (tok/s)", - "output_throughput": "Output Tput (tok/s)", - "total_input_tokens": "Total input tokens", - "total_output_tokens": "Total output tokens", - "engine": "Engine", -} - -if __name__ == "__main__": - # collect results - for test_file in results_folder.glob("*.json"): - with open(test_file) as f: - raw_result = json.loads(f.read()) - - # attach the benchmarking command to raw_result - with open(test_file.with_suffix(".commands")) as f: - command = json.loads(f.read()) - raw_result.update(command) - - # update the test name of this result - raw_result.update({"test_name": test_file.stem}) - - # add the result to raw_result - serving_results.append(raw_result) - continue - - serving_results = pd.DataFrame.from_dict(serving_results) - - if not serving_results.empty: - serving_results = serving_results[list(serving_column_mapping.keys())].rename( - columns=serving_column_mapping - ) - - serving_md_table_with_headers = tabulate( - serving_results, headers="keys", tablefmt="pipe", showindex=False - ) - # remove the first line of header - serving_md_table_lines = serving_md_table_with_headers.split("\n") - serving_md_table_without_header = "\n".join(serving_md_table_lines[2:]) - - prefix = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - prefix = prefix + "_" + os.environ.get("CURRENT_LLM_SERVING_ENGINE") - - # document benchmarking results in markdown - with open(results_folder / f"{prefix}_nightly_results.md", "w") as f: - # document results with header. - # for those who wants to reproduce our benchmark. - f.write(serving_md_table_with_headers) - f.write("\n") - - # document benchmarking results in json - with open(results_folder / f"{prefix}_nightly_results.json", "w") as f: - results = serving_results.to_dict(orient="records") - f.write(json.dumps(results)) diff --git a/.buildkite/nightly-benchmarks/scripts/wait-for-image.sh b/.buildkite/nightly-benchmarks/scripts/wait-for-image.sh deleted file mode 100644 index 50e1ab0242202..0000000000000 --- a/.buildkite/nightly-benchmarks/scripts/wait-for-image.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -TOKEN=$(curl -s -L "https://public.ecr.aws/token?service=public.ecr.aws&scope=repository:q9t5s3a7/vllm-ci-postmerge-repo:pull" | jq -r .token) -if [[ "$BUILDKITE_BRANCH" == "main" ]]; then - URL="https://public.ecr.aws/v2/q9t5s3a7/vllm-ci-postmerge-repo/manifests/$BUILDKITE_COMMIT" -else - URL="https://public.ecr.aws/v2/q9t5s3a7/vllm-ci-test-repo/manifests/$BUILDKITE_COMMIT" -fi - -TIMEOUT_SECONDS=10 - -retries=0 -while [ $retries -lt 1000 ]; do - if [ "$(curl -s --max-time "$TIMEOUT_SECONDS" -L -H "Authorization: Bearer $TOKEN" -o /dev/null -w "%{http_code}" "$URL")" -eq 200 ]; then - exit 0 - fi - - echo "Waiting for image to be available..." - - retries=$((retries + 1)) - sleep 5 -done - -exit 1 diff --git a/.buildkite/nightly-benchmarks/README.md b/.buildkite/performance-benchmarks/README.md similarity index 69% rename from .buildkite/nightly-benchmarks/README.md rename to .buildkite/performance-benchmarks/README.md index e6f5c8b60f459..332142ba5d170 100644 --- a/.buildkite/nightly-benchmarks/README.md +++ b/.buildkite/performance-benchmarks/README.md @@ -2,40 +2,23 @@ ## Introduction -This directory contains two sets of benchmark for vllm. - -- Performance benchmark: benchmark vllm's performance under various workload, for **developers** to gain clarity on whether their PR improves/degrades vllm's performance -- Nightly benchmark: compare vllm's performance against alternatives (tgi, trt-llm and lmdeploy), for **the public** to know when to choose vllm. - -See [vLLM performance dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm) for the latest performance benchmark results and [vLLM GitHub README](https://github.com/vllm-project/vllm/blob/main/README.md) for latest nightly benchmark results. +This directory contains a benchmarking suite for **developers** to run locally and gain clarity on whether their PR improves/degrades vllm's performance. +vLLM also maintains a continuous performance benchmark under [perf.vllm.ai](https://perf.vllm.ai/), hosted under PyTorch CI HUD. ## Performance benchmark quick overview -**Benchmarking Coverage**: latency, throughput and fix-qps serving on A100 (the support for FP8 benchmark on H100 is coming!) and Intel® Xeon® Processors, with different models. +**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100 and Intel® Xeon® Processors, with different models. **Benchmarking Duration**: about 1hr. **For benchmarking developers**: please try your best to constraint the duration of benchmarking to about 1 hr so that it won't take forever to run. -## Nightly benchmark quick overview - -**Benchmarking Coverage**: Fix-qps serving on A100 (the support for FP8 benchmark on H100 is coming!) on Llama-3 8B, 70B and Mixtral 8x7B. - -**Benchmarking engines**: vllm, TGI, trt-llm and lmdeploy. - -**Benchmarking Duration**: about 3.5hrs. - ## Trigger the benchmark -Performance benchmark will be triggered when: - -- A PR being merged into vllm. -- Every commit for those PRs with `perf-benchmarks` label AND `ready` label. - -Manually Trigger the benchmark +The benchmark needs to be triggered manually: ```bash -bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh +bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh ``` Runtime environment variables: @@ -47,10 +30,6 @@ Runtime environment variables: - `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string. - `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string. -Nightly benchmark will be triggered when: - -- Every commit for those PRs with `perf-benchmarks` label and `nightly-benchmarks` label. - ## Performance benchmark details See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases. @@ -152,26 +131,3 @@ Here is an example using the script to compare result_a and result_b with Model, A comparison diagram will be generated below the table. Here is an example to compare between 96c/results_gnr_96c_091_tp2pp3 and 128c/results_gnr_128c_091_tp2pp3 image - -## Nightly test details - -See [nightly-descriptions.md](nightly-descriptions.md) for the detailed description on test workload, models and docker containers of benchmarking other llm engines. - -### Workflow - -- The [nightly-pipeline.yaml](nightly-pipeline.yaml) specifies the docker containers for different LLM serving engines. -- Inside each container, we run [scripts/run-nightly-benchmarks.sh](scripts/run-nightly-benchmarks.sh), which will probe the serving engine of the current container. -- The `scripts/run-nightly-benchmarks.sh` will parse the workload described in [nightly-tests.json](tests/nightly-tests.json) and launch the right benchmark for the specified serving engine via `scripts/launch-server.sh`. -- At last, we run [scripts/summary-nightly-results.py](scripts/summary-nightly-results.py) to collect and plot the final benchmarking results, and update the results to buildkite. - -### Nightly tests - -In [nightly-tests.json](tests/nightly-tests.json), we include the command line arguments for benchmarking commands, together with the benchmarking test cases. The format is highly similar to performance benchmark. - -### Docker containers - -The docker containers for benchmarking are specified in `nightly-pipeline.yaml`. - -WARNING: the docker versions are HARD-CODED and SHOULD BE ALIGNED WITH `nightly-descriptions.md`. The docker versions need to be hard-coded as there are several version-specific bug fixes inside `scripts/run-nightly-benchmarks.sh` and `scripts/launch-server.sh`. - -WARNING: populating `trt-llm` to latest version is not easy, as it requires updating several protobuf files in [tensorrt-demo](https://github.com/neuralmagic/tensorrt-demo.git). diff --git a/.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md b/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md similarity index 100% rename from .buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md rename to .buildkite/performance-benchmarks/performance-benchmarks-descriptions.md diff --git a/.buildkite/nightly-benchmarks/scripts/compare-json-results.py b/.buildkite/performance-benchmarks/scripts/compare-json-results.py similarity index 100% rename from .buildkite/nightly-benchmarks/scripts/compare-json-results.py rename to .buildkite/performance-benchmarks/scripts/compare-json-results.py diff --git a/.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py b/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py similarity index 99% rename from .buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py rename to .buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py index a7544aeef4c74..80bb4d846a226 100644 --- a/.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py +++ b/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py @@ -392,7 +392,7 @@ if __name__ == "__main__": json_file = "benchmark_results.json" with open(results_folder / md_file, "w") as f: results = read_markdown( - "../.buildkite/nightly-benchmarks/" + "../.buildkite/performance-benchmarks/" + "performance-benchmarks-descriptions.md" ) results = results.format( diff --git a/.buildkite/nightly-benchmarks/scripts/launch-server.sh b/.buildkite/performance-benchmarks/scripts/launch-server.sh similarity index 100% rename from .buildkite/nightly-benchmarks/scripts/launch-server.sh rename to .buildkite/performance-benchmarks/scripts/launch-server.sh diff --git a/.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh b/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh similarity index 99% rename from .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh rename to .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh index 5a47576483bbf..9447ceffd7e22 100644 --- a/.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh +++ b/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh @@ -469,7 +469,7 @@ main() { ensure_sharegpt_downloaded declare -g RESULTS_FOLDER=results/ mkdir -p $RESULTS_FOLDER - QUICK_BENCHMARK_ROOT=../.buildkite/nightly-benchmarks/ + QUICK_BENCHMARK_ROOT=../.buildkite/performance-benchmarks/ # dump vllm info via vllm collect-env env_output=$(vllm collect-env) diff --git a/.buildkite/nightly-benchmarks/tests/genai-perf-tests.json b/.buildkite/performance-benchmarks/tests/genai-perf-tests.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/genai-perf-tests.json rename to .buildkite/performance-benchmarks/tests/genai-perf-tests.json diff --git a/.buildkite/nightly-benchmarks/tests/latency-tests-cpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/latency-tests-cpu.json rename to .buildkite/performance-benchmarks/tests/latency-tests-cpu.json diff --git a/.buildkite/nightly-benchmarks/tests/latency-tests.json b/.buildkite/performance-benchmarks/tests/latency-tests.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/latency-tests.json rename to .buildkite/performance-benchmarks/tests/latency-tests.json diff --git a/.buildkite/nightly-benchmarks/tests/nightly-tests.json b/.buildkite/performance-benchmarks/tests/nightly-tests.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/nightly-tests.json rename to .buildkite/performance-benchmarks/tests/nightly-tests.json diff --git a/.buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc2.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-snc2.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc2.json rename to .buildkite/performance-benchmarks/tests/serving-tests-cpu-snc2.json diff --git a/.buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc3.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-snc3.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc3.json rename to .buildkite/performance-benchmarks/tests/serving-tests-cpu-snc3.json diff --git a/.buildkite/nightly-benchmarks/tests/serving-tests-cpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/serving-tests-cpu.json rename to .buildkite/performance-benchmarks/tests/serving-tests-cpu.json diff --git a/.buildkite/nightly-benchmarks/tests/serving-tests.json b/.buildkite/performance-benchmarks/tests/serving-tests.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/serving-tests.json rename to .buildkite/performance-benchmarks/tests/serving-tests.json diff --git a/.buildkite/nightly-benchmarks/tests/throughput-tests-cpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/throughput-tests-cpu.json rename to .buildkite/performance-benchmarks/tests/throughput-tests-cpu.json diff --git a/.buildkite/nightly-benchmarks/tests/throughput-tests.json b/.buildkite/performance-benchmarks/tests/throughput-tests.json similarity index 100% rename from .buildkite/nightly-benchmarks/tests/throughput-tests.json rename to .buildkite/performance-benchmarks/tests/throughput-tests.json diff --git a/.github/mergify.yml b/.github/mergify.yml index de1a8314a4ecd..18d4a2e83144b 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -108,7 +108,7 @@ pull_request_rules: - files~=^benchmarks/ - files~=^vllm/benchmarks/ - files~=^tests/benchmarks/ - - files~=^\.buildkite/nightly-benchmarks/ + - files~=^\.buildkite/performance-benchmarks/ actions: label: add: diff --git a/docs/contributing/benchmarks.md b/docs/contributing/benchmarks.md index be3e32a73a332..dca01eab5b426 100644 --- a/docs/contributing/benchmarks.md +++ b/docs/contributing/benchmarks.md @@ -9,7 +9,6 @@ vLLM provides comprehensive benchmarking tools for performance testing and evalu - **[Benchmark CLI](#benchmark-cli)**: `vllm bench` CLI tools and specialized benchmark scripts for interactive performance testing - **[Parameter sweeps](#parameter-sweeps)**: Automate `vllm bench` runs for multiple configurations - **[Performance benchmarks](#performance-benchmarks)**: Automated CI benchmarks for development -- **[Nightly benchmarks](#nightly-benchmarks)**: Comparative benchmarks against alternatives [Benchmark CLI]: #benchmark-cli @@ -1167,7 +1166,7 @@ docker run -it --entrypoint /bin/bash -v /data/huggingface:/root/.cache/huggingf Then, run below command inside the docker instance. ```bash -bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh +bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh ``` When run, benchmark script generates results under **benchmark/results** folder, along with the benchmark_results.md and benchmark_results.json. @@ -1185,7 +1184,7 @@ For more results visualization, check the [visualizing the results](https://gith The latest performance results are hosted on the public [vLLM Performance Dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm). -More information on the performance benchmarks and their parameters can be found in [Benchmark README](https://github.com/intel-ai-tce/vllm/blob/more_cpu_models/.buildkite/nightly-benchmarks/README.md) and [performance benchmark description](../../.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md). +More information on the performance benchmarks and their parameters can be found in [Benchmark README](https://github.com/intel-ai-tce/vllm/blob/more_cpu_models/.buildkite/nightly-benchmarks/README.md) and [performance benchmark description](../../.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md). ### Continuous Benchmarking @@ -1210,11 +1209,3 @@ The benchmarking currently runs on a predefined set of models configured in the #### Viewing Results All continuous benchmarking results are automatically published to the public [vLLM Performance Dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm). - -## Nightly Benchmarks - -These compare vLLM's performance against alternatives (`tgi`, `trt-llm`, and `lmdeploy`) when there are major updates of vLLM (e.g., bumping up to a new version). They are primarily intended for consumers to evaluate when to choose vLLM over other options and are triggered on every commit with both the `perf-benchmarks` and `nightly-benchmarks` labels. - -The latest nightly benchmark results are shared in major release blog posts such as [vLLM v0.6.0](https://blog.vllm.ai/2024/09/05/perf-update.html). - -More information on the nightly benchmarks and their parameters can be found [here](../../.buildkite/nightly-benchmarks/nightly-descriptions.md). From ded8ada86a3962477433054debbcef1d45161850 Mon Sep 17 00:00:00 2001 From: Bram Wasti Date: Thu, 30 Oct 2025 01:28:45 -0400 Subject: [PATCH 095/127] Add more dims for batch invariant shims (#27489) Signed-off-by: Bram Wasti Signed-off-by: Bram Wasti Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- vllm/model_executor/layers/batch_invariant.py | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/layers/batch_invariant.py b/vllm/model_executor/layers/batch_invariant.py index 208ffb30e5ed2..5706786bccb1d 100644 --- a/vllm/model_executor/layers/batch_invariant.py +++ b/vllm/model_executor/layers/batch_invariant.py @@ -478,9 +478,48 @@ def matmul_batch_invariant(a, b, *, out=None): elif a.ndim == 3 and b.ndim == 3: # Handle batched case like bmm return bmm_batch_invariant(a, b, out=out) + elif a.ndim == 3 and b.ndim == 2: + # Handle 3D x 2D: common for linear layers + # (batch, seq, hidden) @ (hidden, out) -> (batch, seq, out) + # Reshape to 2D, do mm, reshape back + batch, seq, hidden = a.shape + a_2d = a.reshape(-1, hidden) + result_2d = matmul_persistent(a_2d, b) + result = result_2d.reshape(batch, seq, -1) + if out is not None: + out.copy_(result) + return out + return result + elif a.ndim == 2 and b.ndim == 3: + # Handle 2D x 3D: (M, K) @ (B, K, N) -> (B, M, N) + # By broadcasting `a` to 3D, we can reuse the batched matrix + # multiplication logic. + a_expanded = a.unsqueeze(0).expand(b.shape[0], -1, -1) + return bmm_batch_invariant(a_expanded, b, out=out) + elif a.ndim == 4 and b.ndim == 4: + # Handle 4D attention tensors: [batch, heads, seq, dim] + # Reshape to 3D, process, reshape back + batch, heads, seq_a, dim_a = a.shape + _, _, dim_b, seq_b = b.shape + + # Reshape to [batch*heads, seq_a, dim_a] + a_3d = a.reshape(batch * heads, seq_a, dim_a) + b_3d = b.reshape(batch * heads, dim_b, seq_b) + + # Do batched matmul + result_3d = bmm_batch_invariant(a_3d, b_3d) + + # Reshape back to [batch, heads, seq_a, seq_b] + result = result_3d.reshape(batch, heads, seq_a, seq_b) + + if out is not None: + out.copy_(result) + return out + return result else: raise ValueError( - f"matmul_batch_invariant currently only supports 2D x 2D and 3D x 3D, " + f"matmul_batch_invariant currently only supports 2D x 2D, 3D x 3D, " + f"3D x 2D, 2D x 3D, and 4D x 4D, " f"got shapes {a.shape} and {b.shape}" ) @@ -667,7 +706,8 @@ def rms_norm_batch_invariant( def linear_batch_invariant(input, weight, bias=None): - output = mm_batch_invariant(input, weight.t()) + output = matmul_batch_invariant(input, weight.t()) + if bias is not None: output = output + bias return output From 31b55ffc62189b32dac15fb7c00dba20e3573168 Mon Sep 17 00:00:00 2001 From: yitingdc <59356937+yitingdc@users.noreply.github.com> Date: Thu, 30 Oct 2025 15:47:36 +0800 Subject: [PATCH 096/127] use stringData in secret yaml to store huggingface token (#25685) Signed-off-by: yiting.jiang --- docs/deployment/k8s.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/deployment/k8s.md b/docs/deployment/k8s.md index 54031ec368b5c..abffb7bc5f948 100644 --- a/docs/deployment/k8s.md +++ b/docs/deployment/k8s.md @@ -49,11 +49,14 @@ First, create a Kubernetes PVC and Secret for downloading and storing Hugging Fa metadata: name: hf-token-secret type: Opaque - data: - token: $(HF_TOKEN) + stringData: + token: "REPLACE_WITH_TOKEN" EOF ``` +Here, the `token` field stores your **Hugging Face access token**. For details on how to generate a token, +see the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens). + Next, start the vLLM server as a Kubernetes Deployment and Service: ??? console "Config" From 5be1bed79058ddc1016f2639c52dfb5b597bf39c Mon Sep 17 00:00:00 2001 From: Huamin Li <3ericli@gmail.com> Date: Thu, 30 Oct 2025 00:50:56 -0700 Subject: [PATCH 097/127] [CI/Build]Add eval config for Qwen3-235B-A22B-Instruct-2507-FP8 (#27113) Signed-off-by: Huamin Li <3ericli@gmail.com> --- .../configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml | 14 ++++++++++++++ .../lm-eval-harness/configs/models-large-h100.txt | 1 - .../configs/models-large-hopper.txt | 1 + .../lm-eval-harness/test_lm_eval_correctness.py | 14 +++++++++++--- .buildkite/test-pipeline.yaml | 13 +++++++++++++ 5 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 .buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml delete mode 100644 .buildkite/lm-eval-harness/configs/models-large-h100.txt create mode 100644 .buildkite/lm-eval-harness/configs/models-large-hopper.txt diff --git a/.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml b/.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml new file mode 100644 index 0000000000000..514c15d6098ed --- /dev/null +++ b/.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml @@ -0,0 +1,14 @@ +model_name: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8" +tasks: + - name: "mmlu_pro" + metrics: + - name: "exact_match,custom-extract" + value: 0.82 +limit: 250 # will run on 250 * 14 subjects = 3500 samples +num_fewshot: 5 +enforce_eager: false # we use false to speed up the eval process +kv_cache_dtype: fp8 # we use fp8 to speed up the eval process +max_model_len: 40960 +apply_chat_template: true +fewshot_as_multiturn: true +gen_kwargs: "temperature=0,top_p=1,top_k=0,max_gen_toks=5632,until=<|ENDANSWER|>" diff --git a/.buildkite/lm-eval-harness/configs/models-large-h100.txt b/.buildkite/lm-eval-harness/configs/models-large-h100.txt deleted file mode 100644 index 4fb0b84bc4d81..0000000000000 --- a/.buildkite/lm-eval-harness/configs/models-large-h100.txt +++ /dev/null @@ -1 +0,0 @@ -Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml diff --git a/.buildkite/lm-eval-harness/configs/models-large-hopper.txt b/.buildkite/lm-eval-harness/configs/models-large-hopper.txt new file mode 100644 index 0000000000000..5552391d9eaba --- /dev/null +++ b/.buildkite/lm-eval-harness/configs/models-large-hopper.txt @@ -0,0 +1 @@ +Qwen3-235B-A22B-Instruct-2507-FP8.yaml diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py index f10de82b1d8e8..3627b760eddcf 100644 --- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py +++ b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py @@ -21,10 +21,13 @@ def launch_lm_eval(eval_config, tp_size): max_model_len = eval_config.get("max_model_len", 4096) batch_size = eval_config.get("batch_size", "auto") backend = eval_config.get("backend", "vllm") + enforce_eager = eval_config.get("enforce_eager", "true") + kv_cache_dtype = eval_config.get("kv_cache_dtype", "auto") model_args = ( f"pretrained={eval_config['model_name']}," f"tensor_parallel_size={tp_size}," - f"enforce_eager=true," + f"enforce_eager={enforce_eager}," + f"kv_cache_dtype={kv_cache_dtype}," f"add_bos_token=true," f"trust_remote_code={trust_remote_code}," f"max_model_len={max_model_len}," @@ -37,8 +40,13 @@ def launch_lm_eval(eval_config, tp_size): limit=eval_config["limit"], # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help # text models. however, this is regressing measured strict-match for - # existing text models in CI, so only apply it for mm. - apply_chat_template=backend == "vllm-vlm", + # existing text models in CI, so only apply it for mm, or explicitly set + apply_chat_template=eval_config.get( + "apply_chat_template", backend == "vllm-vlm" + ), + fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", False), + # Forward decoding and early-stop controls (e.g., max_gen_toks, until=...) + gen_kwargs=eval_config.get("gen_kwargs"), batch_size=batch_size, ) return results diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index d556073cd1049..339e3aab6c031 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -1186,6 +1186,19 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4 +##### H100 test ##### +- label: LM Eval Large Models (H100) # optional + gpu: h100 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + commands: + - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4 + ##### H200 test ##### - label: Distributed Tests (H200) # optional gpu: h200 From e806178d2a9b65ebd536342d58097a825d066b9e Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Thu, 30 Oct 2025 00:54:44 -0700 Subject: [PATCH 098/127] [BugFix][VL] Fix FA selection on Qwen2.5-VL (#27790) Signed-off-by: zhewenli Co-authored-by: Roger Wang --- .buildkite/test-amd.yaml | 2 +- vllm/model_executor/models/qwen2_5_vl.py | 30 +++++++++++++++--------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 56e7b1083b17e..35bd4c99adb78 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -318,7 +318,7 @@ steps: - label: V1 Test entrypoints # 35min timeout_in_minutes: 50 - mirror_hardwares: [amdexperimental] + mirror_hardwares: [amdexperimental, amdproduction] agent_pool: mi325_1 # grade: Blocking source_file_dependencies: diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index dfaeb663bbe2f..3d67653726bd8 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -43,10 +43,7 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import ( ) from vllm.attention.backends.registry import _Backend -from vllm.attention.layer import ( - check_upstream_fa_availability, - maybe_get_vit_flash_attn_backend, -) +from vllm.attention.layer import maybe_get_vit_flash_attn_backend from vllm.attention.ops.vit_attn_wrappers import ( vit_flash_attn_wrapper, vit_xformers_attn_wrapper, @@ -318,6 +315,7 @@ class Qwen2_5_VisionAttention(nn.Module): use_data_parallel: bool = False, attn_backend: _Backend = _Backend.TORCH_SDPA, use_upstream_fa: bool = False, + attn_backend_override: _Backend | None = None, ) -> None: super().__init__() # Per attention head and per partition values. @@ -358,8 +356,14 @@ class Qwen2_5_VisionAttention(nn.Module): maybe_get_vit_flash_attn_backend( self.attn_backend, self.use_upstream_fa, + attn_backend_override=attn_backend_override, ) ) + # On ROCm with FLASH_ATTN backend, upstream flash_attn is used + from vllm.platforms import current_platform + + if current_platform.is_rocm() and self.attn_backend == _Backend.FLASH_ATTN: + self.use_upstream_fa = True self.is_flash_attn_backend = self.attn_backend in { _Backend.FLASH_ATTN, _Backend.ROCM_AITER_FA, @@ -484,6 +488,7 @@ class Qwen2_5_VisionBlock(nn.Module): use_data_parallel: bool = False, attn_backend: _Backend = _Backend.TORCH_SDPA, use_upstream_fa: bool = False, + attn_backend_override: _Backend | None = None, ) -> None: super().__init__() if norm_layer is None: @@ -499,6 +504,7 @@ class Qwen2_5_VisionBlock(nn.Module): use_data_parallel=use_data_parallel, attn_backend=attn_backend, use_upstream_fa=use_upstream_fa, + attn_backend_override=attn_backend_override, ) self.mlp = Qwen2_5_VisionMLP( dim, @@ -698,13 +704,14 @@ class Qwen2_5_VisionTransformer(nn.Module): dtype=torch.get_default_dtype(), attn_backend_override=attn_backend_override, ) - if ( - self.attn_backend != _Backend.FLASH_ATTN - and self.attn_backend != _Backend.ROCM_AITER_FA - and check_upstream_fa_availability(torch.get_default_dtype()) - ): - self.attn_backend = _Backend.FLASH_ATTN - use_upstream_fa = True + + self.attn_backend, self.flash_attn_varlen_func = ( + maybe_get_vit_flash_attn_backend( + self.attn_backend, + use_upstream_fa, + attn_backend_override=attn_backend_override, + ) + ) if self.attn_backend not in { _Backend.FLASH_ATTN, @@ -730,6 +737,7 @@ class Qwen2_5_VisionTransformer(nn.Module): use_data_parallel=use_data_parallel, attn_backend=self.attn_backend, use_upstream_fa=use_upstream_fa, + attn_backend_override=attn_backend_override, ) for layer_idx in range(depth) ] From af826e082045e8bcd3ab2ea3129bcf91da7d58de Mon Sep 17 00:00:00 2001 From: wangxiyuan Date: Thu, 30 Oct 2025 17:42:49 +0800 Subject: [PATCH 099/127] [V0 deprecation] Remove VLLM_USE_V1 usage in config module (#27784) Signed-off-by: wangxiyuan --- vllm/config/lora.py | 5 ----- vllm/config/model.py | 25 ++----------------------- vllm/config/speculative.py | 7 ------- vllm/config/vllm.py | 34 +++++++--------------------------- 4 files changed, 9 insertions(+), 62 deletions(-) diff --git a/vllm/config/lora.py b/vllm/config/lora.py index 2f9d638542b65..84e92eef40077 100644 --- a/vllm/config/lora.py +++ b/vllm/config/lora.py @@ -9,7 +9,6 @@ from pydantic import ConfigDict, Field, model_validator from pydantic.dataclasses import dataclass from typing_extensions import Self -import vllm.envs as envs from vllm.config.utils import config from vllm.logger import init_logger from vllm.platforms import current_platform @@ -106,10 +105,6 @@ class LoRAConfig: return self - def verify_with_cache_config(self, cache_config: CacheConfig): - if cache_config.cpu_offload_gb > 0 and not envs.VLLM_USE_V1: - raise ValueError("V0 LoRA does not support CPU offload, please use V1.") - def verify_with_model_config(self, model_config: ModelConfig): if self.lora_dtype in (None, "auto"): self.lora_dtype = model_config.dtype diff --git a/vllm/config/model.py b/vllm/config/model.py index e22c218c769da..2151939d5a9f6 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -32,7 +32,6 @@ from vllm.transformers_utils.config import ( get_pooling_config, get_sentence_transformer_tokenizer_config, is_encoder_decoder, - is_interleaved, try_get_dense_modules, try_get_generation_config, try_get_safetensors_metadata, @@ -442,15 +441,12 @@ class ModelConfig: self.enforce_eager = True # Set the default seed to 0 in V1. - # NOTE(woosuk): In V0, we set the default seed to None because the - # driver worker shares the same process as the user process, and thus - # setting a seed affects the user process as well. - # In V1, we use separate processes for workers (unless + # NOTE(woosuk): In V1, we use separate processes for workers (unless # VLLM_ENABLE_V1_MULTIPROCESSING=0), so setting a seed here # doesn't affect the user process. However, without a consistent seed, # different tensor parallel workers would sample different tokens, # leading to inconsistent results. - if envs.VLLM_USE_V1 and self.seed is None: + if self.seed is None: self.seed = 0 if not envs.VLLM_ENABLE_V1_MULTIPROCESSING: logger.warning( @@ -703,23 +699,6 @@ class ModelConfig: revision=self.revision, ) - # Interleaved attention is not supported by some backends in V0 - if ( - not self.disable_sliding_window - and is_interleaved(self.hf_text_config) - and not envs.VLLM_USE_V1 - and (backend := envs.VLLM_ATTENTION_BACKEND) in ("XFORMERS", "FLASHINFER") - ): - logger.warning_once( - "%s has interleaved attention, which is currently not " - "supported by the %s backend. Disabling sliding window and " - "capping the max length to the sliding window size (%d).", - self.hf_text_config.model_type, - backend, - self.hf_text_config.sliding_window, - ) - self.disable_sliding_window = True - self.original_max_model_len = self.max_model_len self.max_model_len = self.get_and_verify_max_len(self.max_model_len) # Init multimodal config if needed diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 4c7b7369ed4b5..903b9a26fab88 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -9,7 +9,6 @@ from pydantic import Field, SkipValidation, model_validator from pydantic.dataclasses import dataclass from typing_extensions import Self -import vllm.envs as envs from vllm.config.parallel import ParallelConfig from vllm.config.utils import config from vllm.logger import init_logger @@ -366,12 +365,6 @@ class SpeculativeConfig: # Replace hf_config for EAGLE draft_model if self.method in ("eagle", "eagle3"): - if self.enable_chunked_prefill and not envs.VLLM_USE_V1: - raise ValueError( - "Chunked prefill and EAGLE are not compatible " - "when using V0." - ) - from vllm.transformers_utils.configs import SpeculatorsConfig from vllm.transformers_utils.configs.eagle import EAGLEConfig diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index c46f409edab61..f592a708a02b5 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -130,7 +130,6 @@ class VllmConfig: from vllm import __version__ vllm_factors.append(__version__) - vllm_factors.append(envs.VLLM_USE_V1) if self.model_config: vllm_factors.append(self.model_config.compute_hash()) else: @@ -306,7 +305,6 @@ class VllmConfig: self.cache_config.verify_with_parallel_config(self.parallel_config) if self.lora_config is not None: - self.lora_config.verify_with_cache_config(self.cache_config) self.lora_config.verify_with_model_config(self.model_config) if self.quant_config is None and self.model_config is not None: @@ -332,18 +330,9 @@ class VllmConfig: # we use the default mode. The default mode depends on other # settings (see the below code). if self.compilation_config.mode is None: - if envs.VLLM_USE_V1: - if ( - self.model_config is not None - and not self.model_config.enforce_eager - ): - self.compilation_config.mode = CompilationMode.VLLM_COMPILE - else: - self.compilation_config.mode = CompilationMode.NONE - + if self.model_config is not None and not self.model_config.enforce_eager: + self.compilation_config.mode = CompilationMode.VLLM_COMPILE else: - # NB: Passing both --enforce-eager and a compilation mode - # in V0 means the compilation mode wins out. self.compilation_config.mode = CompilationMode.NONE else: assert self.compilation_config.mode >= CompilationMode.NONE @@ -371,10 +360,7 @@ class VllmConfig: # if cudagraph_mode is not explicitly set by users, set default # value if self.compilation_config.cudagraph_mode is None: - if ( - envs.VLLM_USE_V1 - and self.compilation_config.mode == CompilationMode.VLLM_COMPILE - ): + if self.compilation_config.mode == CompilationMode.VLLM_COMPILE: # default to full and piecewise for most models self.compilation_config.cudagraph_mode = ( CUDAGraphMode.FULL_AND_PIECEWISE @@ -428,7 +414,7 @@ class VllmConfig: # override related settings when enforce eager self.compilation_config.max_cudagraph_capture_size = 0 self.compilation_config.cudagraph_capture_sizes = [] - elif envs.VLLM_USE_V1: + else: self.compilation_config.cudagraph_num_of_warmups = 1 self._set_cudagraph_sizes() @@ -535,14 +521,11 @@ class VllmConfig: current_platform.check_and_update_config(self) # Do this after all the updates to compilation_config.mode - if ( - envs.VLLM_USE_V1 - and self.compilation_config.mode == CompilationMode.VLLM_COMPILE - ): + if self.compilation_config.mode == CompilationMode.VLLM_COMPILE: self.compilation_config.set_splitting_ops_for_v1() # final check of cudagraph mode after all possible updates - if envs.VLLM_USE_V1 and current_platform.is_cuda_alike(): + if current_platform.is_cuda_alike(): if ( self.compilation_config.cudagraph_mode.has_full_cudagraphs() and self.model_config is not None @@ -587,10 +570,7 @@ class VllmConfig: if not self.instance_id: self.instance_id = random_uuid()[:5] - if ( - envs.VLLM_USE_V1 - and not self.scheduler_config.disable_hybrid_kv_cache_manager - ): + if not self.scheduler_config.disable_hybrid_kv_cache_manager: # logger should only print warning message for hybrid models. As we # can't know whether the model is hybrid or not now, so we don't log # warning message here and will log it later. From c7d2a554baf8694503e6865b5df300650b6c6b6b Mon Sep 17 00:00:00 2001 From: Huamin Li <3ericli@gmail.com> Date: Thu, 30 Oct 2025 03:13:03 -0700 Subject: [PATCH 100/127] [CI Failure] fix test_default_mm_loras (#27795) Signed-off-by: Huamin Li <3ericli@gmail.com> --- tests/lora/test_default_mm_loras.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/lora/test_default_mm_loras.py b/tests/lora/test_default_mm_loras.py index 1a5b9ba3641d3..dfc45e78e464f 100644 --- a/tests/lora/test_default_mm_loras.py +++ b/tests/lora/test_default_mm_loras.py @@ -30,7 +30,8 @@ VLLM_RUNNER_BASE_KWARGS = { "enable_lora": "True", "max_num_seqs": 2, "max_lora_rank": 320, - "max_model_len": 12800, + # Keep these LoRA tests on short-RoPE for determinism post-LongRoPE change. + "max_model_len": 4096, "gpu_memory_utilization": 0.8, "limit_mm_per_prompt": {"audio": 1}, "enforce_eager": True, From c01f6e525f457133cfb00127a89c09e5247e563c Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 30 Oct 2025 07:32:17 -0400 Subject: [PATCH 101/127] [CI] Fix mypy for `vllm/v1/core` and `vllm/v1/engine` (#27108) Signed-off-by: yewentao256 --- tools/pre_commit/mypy.py | 14 +++++++++++- vllm/config/vllm.py | 9 ++++---- vllm/engine/protocol.py | 1 + vllm/v1/core/sched/scheduler.py | 16 ++++++++------ vllm/v1/engine/async_llm.py | 21 +++++++++++------- vllm/v1/engine/core.py | 1 + vllm/v1/engine/core_client.py | 14 ++++++------ vllm/v1/engine/detokenizer.py | 13 +++++++++--- vllm/v1/engine/llm_engine.py | 16 ++++++++------ vllm/v1/engine/output_processor.py | 10 +++++++-- vllm/v1/engine/parallel_sampling.py | 4 ++-- vllm/v1/engine/processor.py | 33 ++++++++++++----------------- 12 files changed, 91 insertions(+), 61 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index a3aa546347255..8d04848f8f780 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -36,12 +36,15 @@ FILES = [ "vllm/transformers_utils", "vllm/triton_utils", "vllm/usage", + "vllm/v1/core", + "vllm/v1/engine", ] # After fixing errors resulting from changing follow_imports # from "skip" to "silent", move the following directories to FILES SEPARATE_GROUPS = [ "tests", + # v0 related "vllm/attention", "vllm/compilation", "vllm/engine", @@ -50,7 +53,16 @@ SEPARATE_GROUPS = [ "vllm/model_executor", "vllm/plugins", "vllm/worker", - "vllm/v1", + # v1 related + "vllm/v1/attention", + "vllm/v1/executor", + "vllm/v1/kv_offload", + "vllm/v1/metrics", + "vllm/v1/pool", + "vllm/v1/sample", + "vllm/v1/spec_decode", + "vllm/v1/structured_output", + "vllm/v1/worker", ] # TODO(woosuk): Include the code from Megatron and HuggingFace. diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index f592a708a02b5..1acac70c32b03 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -84,7 +84,9 @@ class VllmConfig: default_factory=StructuredOutputsConfig ) """Structured outputs configuration.""" - observability_config: ObservabilityConfig | None = None + observability_config: ObservabilityConfig = Field( + default_factory=ObservabilityConfig + ) """Observability configuration.""" quant_config: QuantizationConfig | None = None """Quantization configuration.""" @@ -170,10 +172,7 @@ class VllmConfig: vllm_factors.append(self.structured_outputs_config.compute_hash()) else: vllm_factors.append("None") - if self.observability_config: - vllm_factors.append(self.observability_config.compute_hash()) - else: - vllm_factors.append("None") + vllm_factors.append(self.observability_config.compute_hash()) if self.quant_config: pass # should be captured by model_config.quantization if self.compilation_config: diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 959a0342817c2..24fcd9fe1cab9 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -77,6 +77,7 @@ class EngineClient(ABC): lora_request: LoRARequest | None = None, trace_headers: Mapping[str, str] | None = None, priority: int = 0, + truncate_prompt_tokens: int | None = None, tokenization_kwargs: dict[str, Any] | None = None, ) -> AsyncGenerator[PoolingRequestOutput, None]: """Generate outputs for a request from a pooling model.""" diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index c794886bc24c8..ad6fbee2ec083 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -167,7 +167,7 @@ class Scheduler(SchedulerInterface): self.kv_cache_manager = KVCacheManager( kv_cache_config=kv_cache_config, max_model_len=self.max_model_len, - enable_caching=self.cache_config.enable_prefix_caching, + enable_caching=bool(self.cache_config.enable_prefix_caching), use_eagle=self.use_eagle, log_stats=self.log_stats, enable_kv_cache_events=self.enable_kv_cache_events, @@ -407,13 +407,13 @@ class Scheduler(SchedulerInterface): # Get externally-cached tokens if using a KVConnector. if self.connector is not None: - num_external_computed_tokens, load_kv_async = ( + ext_tokens, load_kv_async = ( self.connector.get_num_new_matched_tokens( request, num_new_local_computed_tokens ) ) - if num_external_computed_tokens is None: + if ext_tokens is None: # The request cannot be scheduled because # the KVConnector couldn't determine # the number of matched tokens. @@ -421,6 +421,8 @@ class Scheduler(SchedulerInterface): skipped_waiting_requests.prepend_request(request) continue + num_external_computed_tokens = ext_tokens + # Total computed tokens (local + external). num_computed_tokens = ( num_new_local_computed_tokens + num_external_computed_tokens @@ -905,13 +907,13 @@ class Scheduler(SchedulerInterface): outputs: dict[int, list[EngineCoreOutput]] = defaultdict(list) spec_decoding_stats: SpecDecodingStats | None = None - kv_connector_stats = ( + kv_connector_stats: KVConnectorStats | None = ( kv_connector_output.kv_connector_stats if kv_connector_output else None ) if kv_connector_stats and self.connector: - stats = self.connector.get_kv_connector_stats() - if stats: - kv_connector_stats = kv_connector_stats.aggregate(stats) + kv_stats = self.connector.get_kv_connector_stats() + if kv_stats: + kv_connector_stats = kv_connector_stats.aggregate(kv_stats) failed_kv_load_req_ids = None if kv_connector_output and kv_connector_output.invalid_block_ids: diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 761c37504d80a..dc61d45015682 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -6,7 +6,7 @@ import socket import time from collections.abc import AsyncGenerator, Iterable, Mapping from copy import copy -from typing import Any +from typing import Any, cast import numpy as np import torch @@ -131,10 +131,9 @@ class AsyncLLM(EngineClient): self.output_processor = OutputProcessor( self.tokenizer, log_stats=self.log_stats ) - if self.observability_config.otlp_traces_endpoint is not None: - tracer = init_tracer( - "vllm.llm_engine", self.observability_config.otlp_traces_endpoint - ) + endpoint = self.observability_config.otlp_traces_endpoint + if endpoint is not None: + tracer = init_tracer("vllm.llm_engine", endpoint) self.output_processor.tracer = tracer # EngineCore (starts the engine in background process). @@ -266,7 +265,9 @@ class AsyncLLM(EngineClient): if engine_core := getattr(self, "engine_core", None): engine_core.shutdown() - cancel_task_threadsafe(getattr(self, "output_handler", None)) + handler = getattr(self, "output_handler", None) + if handler is not None: + cancel_task_threadsafe(handler) async def get_supported_tasks(self) -> tuple[SupportedTask, ...]: return await self.engine_core.get_supported_tasks_async() @@ -314,7 +315,10 @@ class AsyncLLM(EngineClient): priority, data_parallel_rank, ) - prompt_text = prompt if isinstance(prompt, str) else prompt.get("prompt") + if isinstance(prompt, str): + prompt_text = prompt + elif isinstance(prompt, Mapping): + prompt_text = cast(str | None, prompt.get("prompt")) if is_pooling or params.n == 1: await self._add_request(request, prompt_text, None, 0, queue) @@ -436,6 +440,7 @@ class AsyncLLM(EngineClient): # Note: both OutputProcessor and EngineCore handle their # own request cleanup based on finished. finished = out.finished + assert isinstance(out, RequestOutput) yield out # If the request is disconnected by the client, generate() @@ -653,7 +658,7 @@ class AsyncLLM(EngineClient): return self.tokenizer async def is_tracing_enabled(self) -> bool: - return self.observability_config.otlp_traces_endpoint is not None + return self.observability_config.otlp_traces_endpoint is not None # type: ignore async def do_log_stats(self) -> None: if self.logger_manager: diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 85cab32ebfb85..6cbd986b3cd32 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1075,6 +1075,7 @@ class DPEngineCoreProc(EngineCoreProc): local_dp_rank = vllm_config.parallel_config.data_parallel_rank_local assert dp_size > 1 + assert local_dp_rank is not None assert 0 <= local_dp_rank <= dp_rank < dp_size if vllm_config.kv_transfer_config is not None: diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 7b554ca991b9b..9b440505bd9dc 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -385,10 +385,11 @@ class BackgroundResources: with contextlib.suppress(Exception): task.cancel() - if in_loop(loop): - close_sockets_and_tasks() - elif loop and not loop.is_closed(): - loop.call_soon_threadsafe(close_sockets_and_tasks) + if loop is not None: + if in_loop(loop): + close_sockets_and_tasks() + elif not loop.is_closed(): + loop.call_soon_threadsafe(close_sockets_and_tasks) else: # Loop has been closed, try to clean up directly. del tasks @@ -1044,6 +1045,7 @@ class DPAsyncMPClient(AsyncMPClient): return assert self.stats_update_address is not None + stats_addr: str = self.stats_update_address assert len(self.engine_ranks_managed) > 0 # NOTE: running and waiting counts are all global from # the Coordinator include all global EngineCores. This @@ -1054,9 +1056,7 @@ class DPAsyncMPClient(AsyncMPClient): async def run_engine_stats_update_task(): with ( - make_zmq_socket( - self.ctx, self.stats_update_address, zmq.XSUB, linger=0 - ) as socket, + make_zmq_socket(self.ctx, stats_addr, zmq.XSUB, linger=0) as socket, make_zmq_socket( self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=False, linger=0 ) as first_req_rcv_socket, diff --git a/vllm/v1/engine/detokenizer.py b/vllm/v1/engine/detokenizer.py index 5f66e36893bf3..b7a24096bf15f 100644 --- a/vllm/v1/engine/detokenizer.py +++ b/vllm/v1/engine/detokenizer.py @@ -69,14 +69,21 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC): # Stop strings params = request.sampling_params assert params is not None - self.stop = stop = params.stop + stop_list: list[str] + if params.stop is None: + stop_list = [] + elif isinstance(params.stop, str): + stop_list = [params.stop] + else: + stop_list = params.stop + self.stop = stop_list self.min_tokens = params.min_tokens self.include_stop_str_in_output = params.include_stop_str_in_output # Number of chars to hold back when stop strings are to be excluded # from streamed output. - if stop and not self.include_stop_str_in_output: - self.stop_buffer_length = max(len(s) for s in stop) - 1 + if self.stop and not self.include_stop_str_in_output: + self.stop_buffer_length = max(len(s) for s in self.stop) - 1 else: self.stop_buffer_length = 0 self._last_output_text_offset: int = 0 diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 0fce343702e0a..c2ca9579d55ea 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -4,7 +4,7 @@ import time from collections.abc import Callable, Mapping from copy import copy -from typing import Any +from typing import Any, cast import torch.nn as nn from typing_extensions import TypeVar @@ -112,10 +112,9 @@ class LLMEngine: self.output_processor = OutputProcessor( self.tokenizer, log_stats=self.log_stats ) - if self.observability_config.otlp_traces_endpoint is not None: - tracer = init_tracer( - "vllm.llm_engine", self.observability_config.otlp_traces_endpoint - ) + endpoint = self.observability_config.otlp_traces_endpoint + if endpoint is not None: + tracer = init_tracer("vllm.llm_engine", endpoint) self.output_processor.tracer = tracer # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs) @@ -259,7 +258,10 @@ class LLMEngine: trace_headers, priority, ) - prompt_text = prompt if isinstance(prompt, str) else prompt.get("prompt") + if isinstance(prompt, str): + prompt_text = prompt + elif isinstance(prompt, Mapping): + prompt_text = cast(str | None, prompt.get("prompt")) n = params.n if isinstance(params, SamplingParams) else 1 @@ -285,7 +287,7 @@ class LLMEngine: # Add the request to EngineCore. self.engine_core.add_request(child_request) - def step(self) -> list[RequestOutput] | list[PoolingRequestOutput]: + def step(self) -> list[RequestOutput | PoolingRequestOutput]: if self.should_execute_dummy_batch: self.should_execute_dummy_batch = False self.engine_core.execute_dummy_batch() diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 44e4eadce42ac..07c8113dd9b33 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -44,10 +44,16 @@ class RequestOutputCollector: if self.output is None or isinstance(output, Exception): self.output = output self.ready.set() - elif isinstance(self.output, (RequestOutput, PoolingRequestOutput)): + elif isinstance(self.output, RequestOutput) and isinstance( + output, RequestOutput + ): # This ensures that request outputs with different request indexes # (if n > 1) do not override each other. self.output.add(output, aggregate=self.aggregate) + elif isinstance(self.output, PoolingRequestOutput) and isinstance( + output, PoolingRequestOutput + ): + self.output = output async def get(self) -> RequestOutput | PoolingRequestOutput: """Get operation blocks on put event.""" @@ -408,7 +414,7 @@ class OutputProcessor: within the loop below. """ - request_outputs: list[RequestOutput] | list[PoolingRequestOutput] = [] + request_outputs: list[RequestOutput | PoolingRequestOutput] = [] reqs_to_abort: list[str] = [] for engine_core_output in engine_core_outputs: req_id = engine_core_output.request_id diff --git a/vllm/v1/engine/parallel_sampling.py b/vllm/v1/engine/parallel_sampling.py index 2a47befec25f1..26ee10d2b9bbf 100644 --- a/vllm/v1/engine/parallel_sampling.py +++ b/vllm/v1/engine/parallel_sampling.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from copy import copy -from typing import Optional +from typing import Optional, cast from vllm.outputs import CompletionOutput from vllm.sampling_params import RequestOutputKind, SamplingParams @@ -37,7 +37,7 @@ class ParentRequest: self.child_requests = set() self.output_aggregator = ( - [None] * sampling_params.n + [cast(CompletionOutput, None)] * sampling_params.n if (sampling_params.output_kind == RequestOutputKind.FINAL_ONLY) else [] ) diff --git a/vllm/v1/engine/processor.py b/vllm/v1/engine/processor.py index de15677aeea91..c49fd1bde8b98 100644 --- a/vllm/v1/engine/processor.py +++ b/vllm/v1/engine/processor.py @@ -3,7 +3,7 @@ import time from collections.abc import Mapping -from typing import Any, Literal +from typing import Any, Literal, cast from vllm.config import VllmConfig from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs @@ -208,9 +208,9 @@ class Processor: enc = prompt.get("encoder_prompt") dec = prompt.get("decoder_prompt") if enc is not None: - _validate_single_prompt(enc) + _validate_single_prompt(cast(dict | str, enc)) if dec is not None: - _validate_single_prompt(dec) + _validate_single_prompt(cast(dict | str, dec)) else: _validate_single_prompt(prompt) # type: ignore[arg-type] @@ -332,7 +332,7 @@ class Processor: if not mm_data: return None - mm_uuids: MultiModalUUIDDict = {} + mm_uuids: dict[str, list[str | None] | str] = {} for modality, data in mm_data.items(): n = len(data) if isinstance(data, list) else 1 mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)] @@ -384,7 +384,9 @@ class Processor: # if provided. self._validate_multi_modal_uuids(prompt) if isinstance(prompt, dict): - mm_uuids = prompt.get("multi_modal_uuids") + mm_uuids = cast( + MultiModalUUIDDict | None, prompt.get("multi_modal_uuids") + ) else: mm_uuids = None @@ -410,20 +412,13 @@ class Processor: encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) self._validate_model_inputs(encoder_inputs, decoder_inputs) - # Mypy does not always properly infer the types of some elements of - # discriminated unions of TypedDicts, because of how it handles - # inheritance of TypedDict. If we explicitly extract the items we want - # we can avoid type errors from using `dict.get` later in the method. - prompt_token_ids = ( - decoder_inputs["prompt_token_ids"] - if decoder_inputs["type"] != "embeds" - else None - ) - prompt_embeds = ( - decoder_inputs["prompt_embeds"] - if decoder_inputs["type"] == "embeds" - else None - ) + # Mypy can be conservative for TypedDict unions; normalize access. + if decoder_inputs["type"] == "embeds": + prompt_token_ids = None + prompt_embeds = decoder_inputs["prompt_embeds"] + else: + prompt_token_ids = decoder_inputs["prompt_token_ids"] + prompt_embeds = None sampling_params = None pooling_params = None From 74374386e27f9e7a056a37960d5e996093e45ac4 Mon Sep 17 00:00:00 2001 From: Sairam Pillai Date: Thu, 30 Oct 2025 17:27:59 +0530 Subject: [PATCH 102/127] [Bugfix] Improve GPU validation logging in Ray fallback scenarios (#25775) Signed-off-by: Sairam Pillai --- vllm/config/parallel.py | 14 ++++------ vllm/v1/executor/ray_utils.py | 50 ++++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index e8847354bb092..82d575f24690d 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -521,15 +521,11 @@ class ParallelConfig: current_platform.is_cuda() and cuda_device_count_stateless() < self.world_size ): - if not ray_found: - raise ValueError( - "Unable to load Ray: " - f"{ray_utils.ray_import_err}. Ray is " - "required for multi-node inference, " - "please install Ray with `pip install " - "ray`." - ) - backend = "ray" + gpu_count = cuda_device_count_stateless() + raise ValueError( + f"Tensor parallel size ({self.world_size}) cannot be " + f"larger than the number of available GPUs ({gpu_count})." + ) elif self.data_parallel_backend == "ray": logger.info( "Using ray distributed inference because " diff --git a/vllm/v1/executor/ray_utils.py b/vllm/v1/executor/ray_utils.py index 518f1582faeb0..382f008266e62 100644 --- a/vllm/v1/executor/ray_utils.py +++ b/vllm/v1/executor/ray_utils.py @@ -255,12 +255,33 @@ def _wait_until_pg_ready(current_placement_group: "PlacementGroup"): try: ray.get(pg_ready_ref, timeout=0) except ray.exceptions.GetTimeoutError: - raise ValueError( - "Cannot provide a placement group of " - f"{placement_group_specs=} within {PG_WAIT_TIMEOUT} seconds. See " - "`ray status` and `ray list nodes` to make sure the cluster has " - "enough resources." - ) from None + # Provide more helpful error message when GPU count is exceeded + total_gpu_required = sum(spec.get("GPU", 0) for spec in placement_group_specs) + # If more than one GPU is required for the placement group, provide a + # more specific error message. + # We use >1 here because multi-GPU (tensor parallel) jobs are more + # likely to fail due to insufficient cluster resources, and users may + # need to adjust tensor_parallel_size to fit available GPUs. + if total_gpu_required > 1: + raise ValueError( + f"Cannot provide a placement group requiring " + f"{total_gpu_required} GPUs " + f"(placement_group_specs={placement_group_specs}) within " + f"{PG_WAIT_TIMEOUT} seconds.\n" + f"Tensor parallel size may exceed available GPUs in your " + f"cluster. Check resources with `ray status` and " + f"`ray list nodes`.\n" + f"If running on K8s with limited GPUs, consider reducing " + f"--tensor-parallel-size to match available GPU resources." + ) from None + else: + raise ValueError( + "Cannot provide a placement group of " + f"{placement_group_specs=} within " + f"{PG_WAIT_TIMEOUT} seconds. See " + "`ray status` and `ray list nodes` to make sure the cluster " + "has enough resources." + ) from None def _wait_until_pg_removed(current_placement_group: "PlacementGroup"): @@ -299,6 +320,23 @@ def initialize_ray_cluster( assert_ray_available() from vllm.platforms import current_platform + # Prevalidate GPU requirements before Ray processing + if current_platform.is_cuda() and parallel_config.world_size > 1: + from vllm.utils import cuda_device_count_stateless + + available_gpus = cuda_device_count_stateless() + if parallel_config.world_size > available_gpus: + logger.warning( + "Tensor parallel size (%d) exceeds available GPUs (%d). " + "This may result in Ray placement group allocation failures. " + "Consider reducing tensor_parallel_size to %d or less, " + "or ensure your Ray cluster has %d GPUs available.", + parallel_config.world_size, + available_gpus, + available_gpus, + parallel_config.world_size, + ) + if ray.is_initialized(): logger.info("Ray is already initialized. Skipping Ray initialization.") elif current_platform.is_rocm() or current_platform.is_xpu(): From 4464723f220a74785cd1971cf62a04e3961c2846 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Thu, 30 Oct 2025 20:13:05 +0800 Subject: [PATCH 103/127] [Frontend][Doc][5/N] Improve all pooling task | Polish encode (pooling) api & Document. (#25524) Signed-off-by: wang.yuqi Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung --- docs/design/io_processor_plugins.md | 2 +- docs/models/pooling_models.md | 83 +++++++++++--- docs/serving/openai_compatible_server.md | 4 +- examples/offline_inference/pooling/README.md | 12 ++ examples/offline_inference/pooling/ner.py | 2 +- .../{ => pooling}/prithvi_geospatial_mae.py | 0 .../prithvi_geospatial_mae_io_processor.py | 0 examples/online_serving/pooling/README.md | 40 ++++++- .../openai_cross_encoder_score.py | 0 ...enai_cross_encoder_score_for_multimodal.py | 0 .../{ => pooling}/prithvi_geospatial_mae.py | 0 .../entrypoints/pooling/llm/test_classify.py | 12 +- tests/entrypoints/pooling/llm/test_reward.py | 12 +- tests/entrypoints/pooling/llm/test_score.py | 10 +- .../pooling/openai/test_classification.py | 92 +++++++++++---- .../pooling/openai/test_embedding.py | 53 ++++++++- .../entrypoints/pooling/openai/test_rerank.py | 53 +++++++-- .../entrypoints/pooling/openai/test_score.py | 16 +-- .../test_pooler_config_init_behaviour.py | 8 +- tests/test_pooling_params.py | 14 +-- vllm/config/pooler.py | 38 +++++- vllm/entrypoints/openai/api_server.py | 8 +- vllm/entrypoints/openai/protocol.py | 108 ++++++++++++++++-- vllm/entrypoints/openai/serving_pooling.py | 23 ++-- vllm/model_executor/layers/pooler.py | 4 +- vllm/model_executor/models/config.py | 4 +- vllm/pooling_params.py | 32 +++--- 27 files changed, 499 insertions(+), 131 deletions(-) rename examples/offline_inference/{ => pooling}/prithvi_geospatial_mae.py (100%) rename examples/offline_inference/{ => pooling}/prithvi_geospatial_mae_io_processor.py (100%) rename examples/online_serving/{ => pooling}/openai_cross_encoder_score.py (100%) rename examples/online_serving/{ => pooling}/openai_cross_encoder_score_for_multimodal.py (100%) rename examples/online_serving/{ => pooling}/prithvi_geospatial_mae.py (100%) diff --git a/docs/design/io_processor_plugins.md b/docs/design/io_processor_plugins.md index fb64a7bb9c8f1..2f4b17f191a5d 100644 --- a/docs/design/io_processor_plugins.md +++ b/docs/design/io_processor_plugins.md @@ -79,7 +79,7 @@ The `post_process*` methods take `PoolingRequestOutput` objects as input and gen The `validate_or_generate_params` method is used for validating with the plugin any `SamplingParameters`/`PoolingParameters` received with the user request, or to generate new ones if none are specified. The function always returns the validated/generated parameters. The `output_to_response` method is used only for online serving and converts the plugin output to the `IOProcessorResponse` type that is then returned by the API Server. The implementation of the `/pooling` serving endpoint is available here [vllm/entrypoints/openai/serving_pooling.py](../../vllm/entrypoints/openai/serving_pooling.py). -An example implementation of a plugin that enables generating geotiff images with the PrithviGeospatialMAE model is available [here](https://github.com/IBM/terratorch/tree/main/terratorch/vllm/plugins/segmentation). Please, also refer to our online ([examples/online_serving/prithvi_geospatial_mae.py](../../examples/online_serving/prithvi_geospatial_mae.py)) and offline ([examples/offline_inference/prithvi_geospatial_mae_io_processor.py](../../examples/offline_inference/prithvi_geospatial_mae_io_processor.py)) inference examples. +An example implementation of a plugin that enables generating geotiff images with the PrithviGeospatialMAE model is available [here](https://github.com/IBM/terratorch/tree/main/terratorch/vllm/plugins/segmentation). Please, also refer to our online ([examples/online_serving/pooling/prithvi_geospatial_mae.py](../../examples/online_serving/pooling/prithvi_geospatial_mae.py)) and offline ([examples/offline_inference/pooling/prithvi_geospatial_mae_io_processor.py](../../examples/offline_inference/pooling/prithvi_geospatial_mae_io_processor.py)) inference examples. ## Using an IO Processor plugin diff --git a/docs/models/pooling_models.md b/docs/models/pooling_models.md index 40651be1d4495..18bb645ea9a9c 100644 --- a/docs/models/pooling_models.md +++ b/docs/models/pooling_models.md @@ -30,11 +30,11 @@ If `--runner pooling` has been set (manually or automatically) but the model doe vLLM will attempt to automatically convert the model according to the architecture names shown in the table below. -| Architecture | `--convert` | Supported pooling tasks | -|-------------------------------------------------|-------------|-------------------------------| -| `*ForTextEncoding`, `*EmbeddingModel`, `*Model` | `embed` | `encode`, `embed` | -| `*For*Classification`, `*ClassificationModel` | `classify` | `encode`, `classify`, `score` | -| `*ForRewardModeling`, `*RewardModel` | `reward` | `encode` | +| Architecture | `--convert` | Supported pooling tasks | +|-------------------------------------------------|-------------|---------------------------------------| +| `*ForTextEncoding`, `*EmbeddingModel`, `*Model` | `embed` | `token_embed`, `embed` | +| `*For*Classification`, `*ClassificationModel` | `classify` | `token_classify`, `classify`, `score` | +| `*ForRewardModeling`, `*RewardModel` | `reward` | `token_classify` | !!! tip You can explicitly set `--convert ` to specify how to convert the model. @@ -45,12 +45,14 @@ Each pooling model in vLLM supports one or more of these tasks according to [Pooler.get_supported_tasks][vllm.model_executor.layers.pooler.Pooler.get_supported_tasks], enabling the corresponding APIs: -| Task | APIs | -|------------|--------------------------------------| -| `encode` | `LLM.reward(...)` | -| `embed` | `LLM.embed(...)`, `LLM.score(...)`\* | -| `classify` | `LLM.classify(...)` | -| `score` | `LLM.score(...)` | +| Task | APIs | +|------------------|-------------------------------------------------------------------------------| +| `embed` | `LLM.embed(...)`, `LLM.score(...)`\*, `LLM.encode(..., pooling_task="embed")` | +| `classify` | `LLM.classify(...)`, `LLM.encode(..., pooling_task="classify")` | +| `score` | `LLM.score(...)` | +| `token_classify` | `LLM.reward(...)`, `LLM.encode(..., pooling_task="token_classify")` | +| `token_embed` | `LLM.encode(..., pooling_task="token_embed")` | +| `plugin` | `LLM.encode(..., pooling_task="plugin")` | \* The `LLM.score(...)` API falls back to `embed` task if the model does not support `score` task. @@ -144,7 +146,6 @@ A code example can be found here: [examples/offline_inference/basic/score.py](.. ### `LLM.reward` The [reward][vllm.LLM.reward] method is available to all reward models in vLLM. -It returns the extracted hidden states directly. ```python from vllm import LLM @@ -161,15 +162,17 @@ A code example can be found here: [examples/offline_inference/basic/reward.py](. ### `LLM.encode` The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM. -It returns the extracted hidden states directly. !!! note Please use one of the more specific methods or set the task directly when using `LLM.encode`: - For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`. - For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`. - - For rewards, use `LLM.reward(...)` or `pooling_task="reward"`. - For similarity scores, use `LLM.score(...)`. + - For rewards, use `LLM.reward(...)` or `pooling_task="token_classify"`. + - For token classification, use `pooling_task="token_classify"`. + - For multi-vector retrieval, use `pooling_task="token_embed"` + - For IO Processor Plugins , use `pooling_task="plugin"` ```python from vllm import LLM @@ -185,10 +188,47 @@ print(f"Data: {data!r}") Our [OpenAI-Compatible Server](../serving/openai_compatible_server.md) provides endpoints that correspond to the offline APIs: -- [Pooling API](../serving/openai_compatible_server.md#pooling-api) is similar to `LLM.encode`, being applicable to all types of pooling models. - [Embeddings API](../serving/openai_compatible_server.md#embeddings-api) is similar to `LLM.embed`, accepting both text and [multi-modal inputs](../features/multimodal_inputs.md) for embedding models. - [Classification API](../serving/openai_compatible_server.md#classification-api) is similar to `LLM.classify` and is applicable to sequence classification models. - [Score API](../serving/openai_compatible_server.md#score-api) is similar to `LLM.score` for cross-encoder models. +- [Pooling API](../serving/openai_compatible_server.md#pooling-api) is similar to `LLM.encode`, being applicable to all types of pooling models. + +!!! note + Please use one of the more specific methods or set the task directly when using [Pooling API](../serving/openai_compatible_server.md#pooling-api) api.: + + - For embeddings, use [Embeddings API](../serving/openai_compatible_server.md#embeddings-api) or `"task":"embed"`. + - For classification logits, use [Classification API](../serving/openai_compatible_server.md#classification-api) or `task":"classify"`. + - For similarity scores, use [Score API](../serving/openai_compatible_server.md#score-api). + - For rewards, `task":"token_classify"`. + - For token classification, use `task":"token_classify"`. + - For multi-vector retrieval, use `task":"token_embed"` + - For IO Processor Plugins , use `task":"plugin"` + +```python +# start a supported embeddings model server with `vllm serve`, e.g. +# vllm serve intfloat/e5-small +import requests + +host = "localhost" +port = "8000" +model_name = "intfloat/e5-small" + +api_url = f"http://{host}:{port}/pooling" + +prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", +] +prompt = {"model": model_name, "input": prompts, "task": "embed"} + +response = requests.post(api_url, json=prompt) + +for output in response.json()["data"]: + data = output["data"] + print(f"Data: {data!r} (size={len(data)})") +``` ## Matryoshka Embeddings @@ -265,3 +305,16 @@ Expected output: ``` An OpenAI client example can be found here: [examples/online_serving/pooling/openai_embedding_matryoshka_fy.py](../../examples/online_serving/pooling/openai_embedding_matryoshka_fy.py) + +## Deprecated Features + +### Encode task + +We have split the `encode` task into two more specific token wise tasks: `token_embed` and `token_classify`: + +- `token_embed` is the same as embed, using normalize as activation. +- `token_classify` is the same as classify, default using softmax as activation. + +### Remove softmax from PoolingParams + +We are going to remove `softmax` and `activation` from `PoolingParams`. Instead, you should set `use_activation`, since we actually allow `classify` and `token_classify` to use any activation function. diff --git a/docs/serving/openai_compatible_server.md b/docs/serving/openai_compatible_server.md index 1414718a697d5..e331b3422ea64 100644 --- a/docs/serving/openai_compatible_server.md +++ b/docs/serving/openai_compatible_server.md @@ -638,7 +638,7 @@ Usually, the score for a sentence pair refers to the similarity between two sent You can find the documentation for cross encoder models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html). -Code example: [examples/online_serving/openai_cross_encoder_score.py](../../examples/online_serving/openai_cross_encoder_score.py) +Code example: [examples/online_serving/pooling/openai_cross_encoder_score.py](../../examples/online_serving/pooling/openai_cross_encoder_score.py) #### Single inference @@ -819,7 +819,7 @@ You can pass multi-modal inputs to scoring models by passing `content` including print("Scoring output:", response_json["data"][0]["score"]) print("Scoring output:", response_json["data"][1]["score"]) ``` -Full example: [examples/online_serving/openai_cross_encoder_score_for_multimodal.py](../../examples/online_serving/openai_cross_encoder_score_for_multimodal.py) +Full example: [examples/online_serving/pooling/openai_cross_encoder_score_for_multimodal.py](../../examples/online_serving/pooling/openai_cross_encoder_score_for_multimodal.py) #### Extra parameters diff --git a/examples/offline_inference/pooling/README.md b/examples/offline_inference/pooling/README.md index cd9717122b16b..ad78be38716b6 100644 --- a/examples/offline_inference/pooling/README.md +++ b/examples/offline_inference/pooling/README.md @@ -38,6 +38,18 @@ python examples/offline_inference/pooling/multi_vector_retrieval.py python examples/offline_inference/pooling/ner.py ``` +## Prithvi Geospatial MAE usage + +```bash +python examples/offline_inference/pooling/prithvi_geospatial_mae.py +``` + +## IO Processor Plugins for Prithvi Geospatial MAE + +```bash +python examples/offline_inference/pooling/prithvi_geospatial_mae_io_processor.py +``` + ## Qwen3 reranker usage ```bash diff --git a/examples/offline_inference/pooling/ner.py b/examples/offline_inference/pooling/ner.py index b2dffdd6c5ee9..34c80e7ccffd3 100644 --- a/examples/offline_inference/pooling/ner.py +++ b/examples/offline_inference/pooling/ner.py @@ -33,7 +33,7 @@ def main(args: Namespace): label_map = llm.llm_engine.vllm_config.model_config.hf_config.id2label # Run inference - outputs = llm.encode(prompts) + outputs = llm.encode(prompts, pooling_task="token_classify") for prompt, output in zip(prompts, outputs): logits = output.outputs.data diff --git a/examples/offline_inference/prithvi_geospatial_mae.py b/examples/offline_inference/pooling/prithvi_geospatial_mae.py similarity index 100% rename from examples/offline_inference/prithvi_geospatial_mae.py rename to examples/offline_inference/pooling/prithvi_geospatial_mae.py diff --git a/examples/offline_inference/prithvi_geospatial_mae_io_processor.py b/examples/offline_inference/pooling/prithvi_geospatial_mae_io_processor.py similarity index 100% rename from examples/offline_inference/prithvi_geospatial_mae_io_processor.py rename to examples/offline_inference/pooling/prithvi_geospatial_mae_io_processor.py diff --git a/examples/online_serving/pooling/README.md b/examples/online_serving/pooling/README.md index 3b6da20d5f0fe..b76ad21f04818 100644 --- a/examples/online_serving/pooling/README.md +++ b/examples/online_serving/pooling/README.md @@ -3,65 +3,95 @@ ## Cohere rerank usage ```bash +# vllm serve BAAI/bge-reranker-base python examples/online_serving/pooling/cohere_rerank_client.py ``` ## Embedding requests base64 encoding_format usage ```bash +# vllm serve intfloat/e5-small python examples/online_serving/pooling/embedding_requests_base64_client.py ``` ## Embedding requests bytes encoding_format usage ```bash +# vllm serve intfloat/e5-small python examples/online_serving/pooling/embedding_requests_bytes_client.py ``` ## Jinaai rerank usage ```bash +# vllm serve BAAI/bge-reranker-base python examples/online_serving/pooling/jinaai_rerank_client.py ``` ## Multi vector retrieval usage ```bash +# vllm serve BAAI/bge-m3 python examples/online_serving/pooling/multi_vector_retrieval_client.py ``` ## Named Entity Recognition (NER) usage ```bash +# vllm serve boltuix/NeuroBERT-NER python examples/online_serving/pooling/ner_client.py ``` -## Openai chat embedding for multimodal usage +## OpenAI chat embedding for multimodal usage ```bash python examples/online_serving/pooling/openai_chat_embedding_client_for_multimodal.py ``` -## Openai classification usage +## OpenAI classification usage ```bash +# vllm serve jason9693/Qwen2.5-1.5B-apeach python examples/online_serving/pooling/openai_classification_client.py ``` -## Openai embedding usage +## OpenAI cross_encoder score usage ```bash +# vllm serve BAAI/bge-reranker-v2-m3 +python examples/online_serving/pooling/openai_cross_encoder_score.py +``` + +## OpenAI cross_encoder score for multimodal usage + +```bash +# vllm serve jinaai/jina-reranker-m0 +python examples/online_serving/pooling/openai_cross_encoder_score_for_multimodal.py +``` + +## OpenAI embedding usage + +```bash +# vllm serve intfloat/e5-small python examples/online_serving/pooling/openai_embedding_client.py ``` -## Openai embedding matryoshka dimensions usage +## OpenAI embedding matryoshka dimensions usage ```bash +# vllm serve jinaai/jina-embeddings-v3 --trust-remote-code python examples/online_serving/pooling/openai_embedding_matryoshka_fy.py ``` -## Openai pooling usage +## OpenAI pooling usage ```bash +# vllm serve internlm/internlm2-1_8b-reward --trust-remote-code python examples/online_serving/pooling/openai_pooling_client.py ``` + +## Online Prithvi Geospatial MAE usage + +```bash +python examples/online_serving/pooling/prithvi_geospatial_mae.py +``` diff --git a/examples/online_serving/openai_cross_encoder_score.py b/examples/online_serving/pooling/openai_cross_encoder_score.py similarity index 100% rename from examples/online_serving/openai_cross_encoder_score.py rename to examples/online_serving/pooling/openai_cross_encoder_score.py diff --git a/examples/online_serving/openai_cross_encoder_score_for_multimodal.py b/examples/online_serving/pooling/openai_cross_encoder_score_for_multimodal.py similarity index 100% rename from examples/online_serving/openai_cross_encoder_score_for_multimodal.py rename to examples/online_serving/pooling/openai_cross_encoder_score_for_multimodal.py diff --git a/examples/online_serving/prithvi_geospatial_mae.py b/examples/online_serving/pooling/prithvi_geospatial_mae.py similarity index 100% rename from examples/online_serving/prithvi_geospatial_mae.py rename to examples/online_serving/pooling/prithvi_geospatial_mae.py diff --git a/tests/entrypoints/pooling/llm/test_classify.py b/tests/entrypoints/pooling/llm/test_classify.py index 96f634ee0a8c7..1063c3b6b755c 100644 --- a/tests/entrypoints/pooling/llm/test_classify.py +++ b/tests/entrypoints/pooling/llm/test_classify.py @@ -37,15 +37,17 @@ def llm(): @pytest.mark.skip_global_cleanup def test_pooling_params(llm: LLM): - def get_outputs(activation): + def get_outputs(use_activation): outputs = llm.classify( - prompts, pooling_params=PoolingParams(activation=activation), use_tqdm=False + prompts, + pooling_params=PoolingParams(use_activation=use_activation), + use_tqdm=False, ) return torch.tensor([x.outputs.probs for x in outputs]) - default = get_outputs(activation=None) - w_activation = get_outputs(activation=True) - wo_activation = get_outputs(activation=False) + default = get_outputs(use_activation=None) + w_activation = get_outputs(use_activation=True) + wo_activation = get_outputs(use_activation=False) assert torch.allclose(default, w_activation, atol=1e-2), ( "Default should use activation." diff --git a/tests/entrypoints/pooling/llm/test_reward.py b/tests/entrypoints/pooling/llm/test_reward.py index 81058dbad891b..0255704cecd94 100644 --- a/tests/entrypoints/pooling/llm/test_reward.py +++ b/tests/entrypoints/pooling/llm/test_reward.py @@ -37,15 +37,17 @@ def llm(): def test_pooling_params(llm: LLM): - def get_outputs(activation): + def get_outputs(use_activation): outputs = llm.reward( - prompts, pooling_params=PoolingParams(activation=activation), use_tqdm=False + prompts, + pooling_params=PoolingParams(use_activation=use_activation), + use_tqdm=False, ) return torch.cat([x.outputs.data for x in outputs]) - default = get_outputs(activation=None) - w_activation = get_outputs(activation=True) - wo_activation = get_outputs(activation=False) + default = get_outputs(use_activation=None) + w_activation = get_outputs(use_activation=True) + wo_activation = get_outputs(use_activation=False) assert torch.allclose(default, w_activation, atol=1e-2), ( "Default should use activation." diff --git a/tests/entrypoints/pooling/llm/test_score.py b/tests/entrypoints/pooling/llm/test_score.py index 2df973dd7863b..b69c6a47c1913 100644 --- a/tests/entrypoints/pooling/llm/test_score.py +++ b/tests/entrypoints/pooling/llm/test_score.py @@ -34,21 +34,21 @@ def llm(): def test_pooling_params(llm: LLM): - def get_outputs(activation): + def get_outputs(use_activation): text_1 = "What is the capital of France?" text_2 = "The capital of France is Paris." outputs = llm.score( text_1, text_2, - pooling_params=PoolingParams(activation=activation), + pooling_params=PoolingParams(use_activation=use_activation), use_tqdm=False, ) return torch.tensor([x.outputs.score for x in outputs]) - default = get_outputs(activation=None) - w_activation = get_outputs(activation=True) - wo_activation = get_outputs(activation=False) + default = get_outputs(use_activation=None) + w_activation = get_outputs(use_activation=True) + wo_activation = get_outputs(use_activation=False) assert torch.allclose(default, w_activation, atol=1e-2), ( "Default should use activation." diff --git a/tests/entrypoints/pooling/openai/test_classification.py b/tests/entrypoints/pooling/openai/test_classification.py index 92d40efad21cb..671bb948780ae 100644 --- a/tests/entrypoints/pooling/openai/test_classification.py +++ b/tests/entrypoints/pooling/openai/test_classification.py @@ -7,7 +7,7 @@ import torch import torch.nn.functional as F from tests.utils import RemoteOpenAIServer -from vllm.entrypoints.openai.protocol import ClassificationResponse +from vllm.entrypoints.openai.protocol import ClassificationResponse, PoolingResponse MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach" DTYPE = "float32" # Use float32 to avoid NaN issue @@ -163,20 +163,24 @@ async def test_invocations(server: RemoteOpenAIServer): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_activation(server: RemoteOpenAIServer, model_name: str): +async def test_use_activation(server: RemoteOpenAIServer, model_name: str): input_text = ["This product was excellent and exceeded my expectations"] - async def get_outputs(activation): + async def get_outputs(use_activation): response = requests.post( server.url_for("classify"), - json={"model": model_name, "input": input_text, "activation": activation}, + json={ + "model": model_name, + "input": input_text, + "use_activation": use_activation, + }, ) outputs = response.json() return torch.tensor([x["probs"] for x in outputs["data"]]) - default = await get_outputs(activation=None) - w_activation = await get_outputs(activation=True) - wo_activation = await get_outputs(activation=False) + default = await get_outputs(use_activation=None) + w_activation = await get_outputs(use_activation=True) + wo_activation = await get_outputs(use_activation=False) assert torch.allclose(default, w_activation, atol=1e-2), ( "Default should use activation." @@ -191,18 +195,7 @@ async def test_activation(server: RemoteOpenAIServer, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -def test_pooling(server: RemoteOpenAIServer, model_name: str): - # pooling api uses ALL pooling, which does not support chunked prefill. - response = requests.post( - server.url_for("pooling"), - json={"model": model_name, "input": "test", "encoding_format": "float"}, - ) - assert response.json()["error"]["type"] == "BadRequestError" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model_name", [MODEL_NAME]) -def test_score(server: RemoteOpenAIServer, model_name: str): +async def test_score(server: RemoteOpenAIServer, model_name: str): # score api is only enabled for num_labels == 1. response = requests.post( server.url_for("score"), @@ -217,7 +210,7 @@ def test_score(server: RemoteOpenAIServer, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -def test_rerank(server: RemoteOpenAIServer, model_name: str): +async def test_rerank(server: RemoteOpenAIServer, model_name: str): # rerank api is only enabled for num_labels == 1. response = requests.post( server.url_for("rerank"), @@ -228,3 +221,62 @@ def test_rerank(server: RemoteOpenAIServer, model_name: str): }, ) assert response.json()["error"]["type"] == "BadRequestError" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +async def test_pooling_classify(server: RemoteOpenAIServer, model_name: str): + input_text = "This product was excellent and exceeded my expectations" + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "input": input_text, + "encoding_format": "float", + "task": "classify", + }, + ) + poolings = PoolingResponse.model_validate(response.json()) + assert len(poolings.data) == 1 + assert len(poolings.data[0].data) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: str): + # token_classify uses ALL pooling, which does not support chunked prefill. + task = "token_classify" + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "input": "test", + "encoding_format": "float", + "task": task, + }, + ) + assert response.json()["error"]["type"] == "BadRequestError" + assert response.json()["error"]["message"].startswith( + f"Task {task} is not supported" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"]) +async def test_pooling_not_supported( + server: RemoteOpenAIServer, model_name: str, task: str +): + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "input": "test", + "encoding_format": "float", + "task": task, + }, + ) + assert response.json()["error"]["type"] == "BadRequestError" + assert response.json()["error"]["message"].startswith( + f"Task {task} is not supported" + ) diff --git a/tests/entrypoints/pooling/openai/test_embedding.py b/tests/entrypoints/pooling/openai/test_embedding.py index b3f12283fdbdf..e971b23e8f1a0 100644 --- a/tests/entrypoints/pooling/openai/test_embedding.py +++ b/tests/entrypoints/pooling/openai/test_embedding.py @@ -562,12 +562,40 @@ async def test_normalize(server: RemoteOpenAIServer, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_pooling(server: RemoteOpenAIServer, model_name: str): +async def test_pooling_embed(server: RemoteOpenAIServer, model_name: str): + task = "embed" input_text = ["The chef prepared a delicious meal."] response = requests.post( server.url_for("pooling"), - json={"model": model_name, "input": input_text, "encoding_format": "float"}, + json={ + "model": model_name, + "input": input_text, + "encoding_format": "float", + "task": task, + }, + ) + + poolings = PoolingResponse.model_validate(response.json()) + + assert len(poolings.data) == 1 + assert len(poolings.data[0].data) == 384 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str): + task = "token_embed" + input_text = ["The chef prepared a delicious meal."] + + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "input": input_text, + "encoding_format": "float", + "task": task, + }, ) poolings = PoolingResponse.model_validate(response.json()) @@ -575,3 +603,24 @@ async def test_pooling(server: RemoteOpenAIServer, model_name: str): assert len(poolings.data) == 1 assert len(poolings.data[0].data) == 11 assert len(poolings.data[0].data[0]) == 384 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +@pytest.mark.parametrize("task", ["classify", "token_classify", "plugin"]) +async def test_pooling_not_supported( + server: RemoteOpenAIServer, model_name: str, task: str +): + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "input": "test", + "encoding_format": "float", + "task": task, + }, + ) + assert response.json()["error"]["type"] == "BadRequestError" + assert response.json()["error"]["message"].startswith( + f"Task {task} is not supported" + ) diff --git a/tests/entrypoints/pooling/openai/test_rerank.py b/tests/entrypoints/pooling/openai/test_rerank.py index e43148d25feeb..1d85190c12a19 100644 --- a/tests/entrypoints/pooling/openai/test_rerank.py +++ b/tests/entrypoints/pooling/openai/test_rerank.py @@ -125,8 +125,8 @@ def test_invocations(server: RemoteOpenAIServer): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_activation(server: RemoteOpenAIServer, model_name: str): - async def get_outputs(activation): +async def test_use_activation(server: RemoteOpenAIServer, model_name: str): + async def get_outputs(use_activation): query = "What is the capital of France?" documents = [ "The capital of Brazil is Brasilia.", @@ -139,16 +139,16 @@ async def test_activation(server: RemoteOpenAIServer, model_name: str): "model": model_name, "query": query, "documents": documents, - "activation": activation, + "use_activation": use_activation, }, ) outputs = response.json() return torch.tensor([x["relevance_score"] for x in outputs["results"]]) - default = await get_outputs(activation=None) - w_activation = await get_outputs(activation=True) - wo_activation = await get_outputs(activation=False) + default = await get_outputs(use_activation=None) + w_activation = await get_outputs(use_activation=True) + wo_activation = await get_outputs(use_activation=False) assert torch.allclose(default, w_activation, atol=1e-2), ( "Default should use activation." @@ -163,7 +163,25 @@ async def test_activation(server: RemoteOpenAIServer, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_pooling(server: RemoteOpenAIServer, model_name: str): +async def test_pooling_classify(server: RemoteOpenAIServer, model_name: str): + input_text = "This product was excellent and exceeded my expectations" + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "input": input_text, + "encoding_format": "float", + "task": "classify", + }, + ) + poolings = PoolingResponse.model_validate(response.json()) + assert len(poolings.data) == 1 + assert len(poolings.data[0].data) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: str): input_text = ["The chef prepared a delicious meal."] response = requests.post( @@ -176,3 +194,24 @@ async def test_pooling(server: RemoteOpenAIServer, model_name: str): assert len(poolings.data) == 1 assert len(poolings.data[0].data) == 11 assert len(poolings.data[0].data[0]) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"]) +async def test_pooling_not_supported( + server: RemoteOpenAIServer, model_name: str, task: str +): + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "input": "test", + "encoding_format": "float", + "task": task, + }, + ) + assert response.json()["error"]["type"] == "BadRequestError" + assert response.json()["error"]["message"].startswith( + f"Task {task} is not supported" + ) diff --git a/tests/entrypoints/pooling/openai/test_score.py b/tests/entrypoints/pooling/openai/test_score.py index ef213ab0ea18b..b8f796d47efaa 100644 --- a/tests/entrypoints/pooling/openai/test_score.py +++ b/tests/entrypoints/pooling/openai/test_score.py @@ -218,8 +218,8 @@ class TestModel: # TODO: reset this tolerance to 0.01 once we find # an alternative to flash_attn with bfloat16 - def test_activation(self, server: RemoteOpenAIServer, model: dict[str, Any]): - def get_outputs(activation): + def test_use_activation(self, server: RemoteOpenAIServer, model: dict[str, Any]): + def get_outputs(use_activation): text_1 = "What is the capital of France?" text_2 = "The capital of France is Paris." response = requests.post( @@ -228,7 +228,7 @@ class TestModel: "model": model["name"], "text_1": text_1, "text_2": text_2, - "activation": activation, + "use_activation": use_activation, }, ) if response.status_code != 200: @@ -238,9 +238,9 @@ class TestModel: return torch.tensor([x["score"] for x in outputs["data"]]) if model["is_cross_encoder"]: - default = get_outputs(activation=None) - w_activation = get_outputs(activation=True) - wo_activation = get_outputs(activation=False) + default = get_outputs(use_activation=None) + w_activation = get_outputs(use_activation=True) + wo_activation = get_outputs(use_activation=False) assert torch.allclose(default, w_activation, atol=1e-2), ( "Default should use activation." @@ -252,8 +252,8 @@ class TestModel: "w_activation should be close to activation(wo_activation)." ) else: - get_outputs(activation=None) + get_outputs(use_activation=None) # The activation parameter only works for the is_cross_encoder model - response = get_outputs(activation=True) + response = get_outputs(use_activation=True) assert response.status_code == 400 diff --git a/tests/models/language/pooling/test_pooler_config_init_behaviour.py b/tests/models/language/pooling/test_pooler_config_init_behaviour.py index 55663ee3f1b41..deb5de984d909 100644 --- a/tests/models/language/pooling/test_pooler_config_init_behaviour.py +++ b/tests/models/language/pooling/test_pooler_config_init_behaviour.py @@ -24,7 +24,7 @@ def test_classify_models_using_activation( model, max_model_len=512, dtype=dtype, - pooler_config=PoolerConfig(activation=False), + pooler_config=PoolerConfig(use_activation=False), ) as vllm_model: wo_activation_out = vllm_model.classify(example_prompts) @@ -32,7 +32,7 @@ def test_classify_models_using_activation( model, max_model_len=512, dtype=dtype, - pooler_config=PoolerConfig(activation=True), + pooler_config=PoolerConfig(use_activation=True), ) as vllm_model: w_activation_out = vllm_model.classify(example_prompts) @@ -104,7 +104,7 @@ def test_reward_models_using_activation( model, max_model_len=1024, dtype=dtype, - pooler_config=PoolerConfig(activation=False), + pooler_config=PoolerConfig(use_activation=False), ) as vllm_model: wo_activation = vllm_model.reward(example_prompts) @@ -112,7 +112,7 @@ def test_reward_models_using_activation( model, max_model_len=1024, dtype=dtype, - pooler_config=PoolerConfig(activation=True), + pooler_config=PoolerConfig(use_activation=True), ) as vllm_model: w_activation = vllm_model.reward(example_prompts) diff --git a/tests/test_pooling_params.py b/tests/test_pooling_params.py index e73d7efc1483a..7812562c8948c 100644 --- a/tests/test_pooling_params.py +++ b/tests/test_pooling_params.py @@ -17,7 +17,7 @@ EMBEDDING_MODELS = [ ), ] -classify_parameters = ["activation"] +classify_parameters = ["use_activation"] embed_parameters = ["dimensions", "normalize"] step_pooling_parameters = ["step_tag_id", "returned_token_ids"] @@ -88,13 +88,13 @@ def test_embed_dimensions(model_info: EmbedModelInfo): def test_classify(task): model_config = MockModelConfig(pooler_config=PoolerConfig(pooling_type="CLS")) - pooling_params = PoolingParams(activation=None) + pooling_params = PoolingParams(use_activation=None) pooling_params.verify(task=task, model_config=model_config) - pooling_params = PoolingParams(activation=True) + pooling_params = PoolingParams(use_activation=True) pooling_params.verify(task=task, model_config=model_config) - pooling_params = PoolingParams(activation=False) + pooling_params = PoolingParams(use_activation=False) pooling_params.verify(task=task, model_config=model_config) invalid_parameters = embed_parameters + step_pooling_parameters @@ -137,13 +137,13 @@ def test_token_classify(pooling_type: str): pooler_config=PoolerConfig(pooling_type=pooling_type) ) - pooling_params = PoolingParams(activation=None) + pooling_params = PoolingParams(use_activation=None) pooling_params.verify(task=task, model_config=model_config) - pooling_params = PoolingParams(activation=True) + pooling_params = PoolingParams(use_activation=True) pooling_params.verify(task=task, model_config=model_config) - pooling_params = PoolingParams(activation=False) + pooling_params = PoolingParams(use_activation=False) pooling_params.verify(task=task, model_config=model_config) invalid_parameters = embed_parameters diff --git a/vllm/config/pooler.py b/vllm/config/pooler.py index 0590f74aa4c93..6bece8d0785bd 100644 --- a/vllm/config/pooler.py +++ b/vllm/config/pooler.py @@ -7,6 +7,9 @@ from typing import Any from pydantic.dataclasses import dataclass from vllm.config.utils import config +from vllm.logger import init_logger + +logger = init_logger(__name__) @config @@ -48,7 +51,15 @@ class PoolerConfig: """ ## for classification models - activation: bool | None = None + softmax: float | None = None + """ + softmax will be deprecated, please use use_activation instead. + """ + activation: float | None = None + """ + activation will be deprecated, please use use_activation instead. + """ + use_activation: bool | None = None """ Whether to apply activation function to the classification outputs. Defaults to True. @@ -59,11 +70,6 @@ class PoolerConfig: """ ## for reward models - softmax: bool | None = None - """ - Whether to apply softmax to the reward outputs. - Defaults to True. - """ step_tag_id: int | None = None """ If set, only the score corresponding to the `step_tag_id` in the @@ -77,6 +83,10 @@ class PoolerConfig: `math-shepherd-mistral-7b-prm` model. """ + def __post_init__(self): + # raise deprecated warning for softmax and activation + self.use_activation = get_use_activation(self) + def compute_hash(self) -> str: """ WARNING: Whenever a new field is added to this config, @@ -94,3 +104,19 @@ class PoolerConfig: factors: list[Any] = [] hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest() return hash_str + + +def get_use_activation(o: object): + if softmax := getattr(o, "softmax", None) is not None: + logger.warning_once( + "softmax will be deprecated, please use use_activation instead." + ) + return softmax + + if activation := getattr(o, "activation", None) is not None: + logger.warning_once( + "activation will be deprecated, please use use_activation instead." + ) + return activation + + return getattr(o, "use_activation", None) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 71939d6c41dfa..f3aa5351e5302 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -107,6 +107,7 @@ from vllm.entrypoints.utils import ( ) from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager +from vllm.tasks import POOLING_TASKS from vllm.usage.usage_lib import UsageContext from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import is_valid_ipv6_address @@ -1748,12 +1749,7 @@ async def init_app_state( log_error_stack=args.log_error_stack, ) ) - if ( - any( - task in supported_tasks - for task in ["token_embed", "token_classify", "plugin"] - ) - ) + if any(task in POOLING_TASKS for task in supported_tasks) else None ) state.openai_serving_embedding = ( diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 0778e4d787905..d0061f9d5b40f 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -49,6 +49,8 @@ from openai.types.responses.response_reasoning_item import ( ) from openai_harmony import Message as OpenAIHarmonyMessage +from vllm.config.pooler import get_use_activation +from vllm.tasks import PoolingTask from vllm.utils.serial_utils import ( EmbedDType, EncodingFormat, @@ -1669,8 +1671,58 @@ class EmbeddingChatRequest(OpenAIBaseModel): EmbeddingRequest: TypeAlias = EmbeddingCompletionRequest | EmbeddingChatRequest -PoolingCompletionRequest = EmbeddingCompletionRequest -PoolingChatRequest = EmbeddingChatRequest + +class PoolingCompletionRequest(EmbeddingCompletionRequest): + task: PoolingTask | None = None + softmax: bool | None = Field( + default=None, + description="softmax will be deprecated, please use use_activation instead.", + ) + activation: bool | None = Field( + default=None, + description="activation will be deprecated, please use use_activation instead.", + ) + use_activation: bool | None = Field( + default=None, + description="Whether to use activation for classification outputs. " + "If it is a classify or token_classify task, the default is True; " + "for other tasks, this value should be None.", + ) + + def to_pooling_params(self): + return PoolingParams( + truncate_prompt_tokens=self.truncate_prompt_tokens, + dimensions=self.dimensions, + normalize=self.normalize, + use_activation=get_use_activation(self), + ) + + +class PoolingChatRequest(EmbeddingChatRequest): + task: PoolingTask | None = None + softmax: bool | None = Field( + default=None, + description="softmax will be deprecated, please use use_activation instead.", + ) + activation: bool | None = Field( + default=None, + description="activation will be deprecated, please use use_activation instead.", + ) + use_activation: bool | None = Field( + default=None, + description="Whether to use activation for classification outputs. " + "If it is a classify or token_classify task, the default is True; " + "for other tasks, this value should be None.", + ) + + def to_pooling_params(self): + return PoolingParams( + truncate_prompt_tokens=self.truncate_prompt_tokens, + dimensions=self.dimensions, + normalize=self.normalize, + use_activation=get_use_activation(self), + ) + T = TypeVar("T") @@ -1686,6 +1738,7 @@ class IOProcessorRequest(OpenAIBaseModel, Generic[T]): """ data: T + task: PoolingTask = "plugin" encoding_format: EncodingFormat = "float" embed_dtype: EmbedDType = Field( default="float32", @@ -1749,14 +1802,27 @@ class ScoreRequest(OpenAIBaseModel): ), ) - activation: bool | None = None + softmax: bool | None = Field( + default=None, + description="softmax will be deprecated, please use use_activation instead.", + ) + activation: bool | None = Field( + default=None, + description="activation will be deprecated, please use use_activation instead.", + ) + + use_activation: bool | None = Field( + default=None, + description="Whether to use activation for classification outputs. " + "Default is True.", + ) # --8<-- [end:score-extra-params] def to_pooling_params(self): return PoolingParams( truncate_prompt_tokens=self.truncate_prompt_tokens, - activation=self.activation, + use_activation=get_use_activation(self), ) @@ -1783,14 +1849,27 @@ class RerankRequest(OpenAIBaseModel): ), ) - activation: bool | None = None + softmax: bool | None = Field( + default=None, + description="softmax will be deprecated, please use use_activation instead.", + ) + activation: bool | None = Field( + default=None, + description="activation will be deprecated, please use use_activation instead.", + ) + + use_activation: bool | None = Field( + default=None, + description="Whether to use activation for classification outputs. " + "Default is True.", + ) # --8<-- [end:rerank-extra-params] def to_pooling_params(self): return PoolingParams( truncate_prompt_tokens=self.truncate_prompt_tokens, - activation=self.activation, + use_activation=get_use_activation(self), ) @@ -1958,14 +2037,27 @@ class ClassificationRequest(OpenAIBaseModel): ), ) - activation: bool | None = None + softmax: bool | None = Field( + default=None, + description="softmax will be deprecated, please use use_activation instead.", + ) + activation: bool | None = Field( + default=None, + description="activation will be deprecated, please use use_activation instead.", + ) + + use_activation: bool | None = Field( + default=None, + description="Whether to use activation for classification outputs. " + "Default is True.", + ) # --8<-- [end:classification-extra-params] def to_pooling_params(self): return PoolingParams( truncate_prompt_tokens=self.truncate_prompt_tokens, - activation=self.activation, + use_activation=get_use_activation(self), ) diff --git a/vllm/entrypoints/openai/serving_pooling.py b/vllm/entrypoints/openai/serving_pooling.py index 568896ccbf1b7..0eade272111f1 100644 --- a/vllm/entrypoints/openai/serving_pooling.py +++ b/vllm/entrypoints/openai/serving_pooling.py @@ -170,15 +170,24 @@ class OpenAIServingPooling(OpenAIServing): pooling_params = request.to_pooling_params() pooling_task: PoolingTask - if "token_embed" in self.supported_tasks: - pooling_task = "token_embed" - elif "token_classify" in self.supported_tasks: - pooling_task = "token_classify" - elif "plugin" in self.supported_tasks: - pooling_task = "plugin" + if request.task is None: + if "token_embed" in self.supported_tasks: + pooling_task = "token_embed" + elif "token_classify" in self.supported_tasks: + pooling_task = "token_classify" + elif "plugin" in self.supported_tasks: + pooling_task = "plugin" + else: + return self.create_error_response( + f"pooling_task must be one of {self.supported_tasks}." + ) else: + pooling_task = request.task + + if pooling_task not in self.supported_tasks: return self.create_error_response( - f"pooling_task must be one of {self.supported_tasks}." + f"Task {pooling_task} is not supported, it" + f" must be one of {self.supported_tasks}." ) try: diff --git a/vllm/model_executor/layers/pooler.py b/vllm/model_executor/layers/pooler.py index 145f18f235662..7dd02e32ff211 100644 --- a/vllm/model_executor/layers/pooler.py +++ b/vllm/model_executor/layers/pooler.py @@ -607,7 +607,7 @@ class ClassifierPooler(Pooler): pooled_data -= self.logit_bias pooling_params = get_pooling_params(pooling_metadata) - flags = [p.activation for p in pooling_params] + flags = [p.use_activation for p in pooling_params] if len(set(flags)) == 1: scores = self.act_fn(pooled_data) if flags[0] else pooled_data @@ -681,7 +681,7 @@ class TokenClassifierPoolerHead(nn.Module): if self.logit_bias is not None: scores -= self.logit_bias - if pooling_param.activation: + if pooling_param.use_activation: scores = self.act_fn(scores) # scores shape: [n_token, num_labels] diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index ac5949cda9de9..3bd02121f018e 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -53,8 +53,8 @@ class JambaForSequenceClassificationConfig(VerifyAndUpdateConfig): @staticmethod def verify_and_update_config(vllm_config: "VllmConfig") -> None: pooler_config = vllm_config.model_config.pooler_config - if pooler_config.activation is None: - pooler_config.activation = False + if pooler_config.use_activation is None: + pooler_config.use_activation = False class JinaRobertaModelConfig(VerifyAndUpdateConfig): diff --git a/vllm/pooling_params.py b/vllm/pooling_params.py index 090d924144659..72a8320cc1bf8 100644 --- a/vllm/pooling_params.py +++ b/vllm/pooling_params.py @@ -2,16 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from copy import deepcopy -from typing import TYPE_CHECKING, Annotated, Any, Optional +from typing import Annotated, Any, Optional import msgspec +from vllm.config import ModelConfig, PoolerConfig +from vllm.config.pooler import get_use_activation from vllm.sampling_params import RequestOutputKind from vllm.tasks import PoolingTask -if TYPE_CHECKING: - from vllm.config import ModelConfig, PoolerConfig - class PoolingParams( msgspec.Struct, @@ -25,10 +24,12 @@ class PoolingParams( Set to -1 to use the model's default truncation size. Set to k to keep only the last k tokens (left truncation). Set to None to disable truncation. - normalize: Whether to normalize the embeddings outputs. dimensions: Reduce the dimensions of embeddings if model support matryoshka representation. - activation: Whether to apply activation function to + normalize: Whether to normalize the embeddings outputs. + softmax: softmax will be deprecated, please use use_activation instead. + activation: activation will be deprecated, please use use_activation instead. + use_activation: Whether to apply activation function to the classification outputs. """ @@ -44,7 +45,9 @@ class PoolingParams( ## for classification, scoring and rerank # --8<-- [start:classification-pooling-params] + softmax: bool | None = None activation: bool | None = None + use_activation: bool | None = None # --8<-- [end:classification-pooling-params] ## for step pooling models @@ -59,16 +62,16 @@ class PoolingParams( @property def all_parameters(self) -> list[str]: - return ["dimensions", "normalize", "activation"] + return ["dimensions", "normalize", "use_activation"] @property def valid_parameters(self): return { "embed": ["dimensions", "normalize"], - "classify": ["activation"], - "score": ["activation"], + "classify": ["use_activation"], + "score": ["use_activation"], "token_embed": ["dimensions", "normalize"], - "token_classify": ["activation"], + "token_classify": ["use_activation"], } def clone(self) -> "PoolingParams": @@ -84,6 +87,9 @@ class PoolingParams( msg = f"You cannot overwrite {self.task=!r} with {task=!r}!" raise ValueError(msg) + # raise deprecated warning for softmax and activation + self.use_activation = get_use_activation(self) + # plugin task uses io_processor.parse_request to verify inputs, # skipping PoolingParams verify if self.task == "plugin": @@ -168,8 +174,8 @@ class PoolingParams( raise ValueError("Dimensions must be greater than 0") elif self.task in ["classify", "score", "token_classify"]: - if self.activation is None: - self.activation = True + if self.use_activation is None: + self.use_activation = True else: raise ValueError(f"Unknown pooling task: {self.task}") @@ -197,7 +203,7 @@ class PoolingParams( f"task={self.task}, " f"normalize={self.normalize}, " f"dimensions={self.dimensions}, " - f"activation={self.activation}, " + f"use_activation={self.use_activation}, " f"step_tag_id={self.step_tag_id}, " f"returned_token_ids={self.returned_token_ids}, " f"requires_token_ids={self.requires_token_ids}, " From 1994de99ea0bf8dd84257a19800f4f62526a7edf Mon Sep 17 00:00:00 2001 From: Huamin Li <3ericli@gmail.com> Date: Thu, 30 Oct 2025 05:27:53 -0700 Subject: [PATCH 104/127] [CI Failure] Fix test_kv_cache_model_load_and_run (#27717) Signed-off-by: Huamin Li <3ericli@gmail.com> --- tests/quantization/test_fp8.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 7f863a169d5f9..bb3572752d9e2 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -49,7 +49,18 @@ def test_model_load_and_run( KV_CACHE_MODELS = [ # AutoFP8 format using separate .k_scale and .v_scale - "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V", + # The original checkpoint below was removed from the Hub. To unblock CI and + # until a small replacement with split K/V scales is found, skip this case. + # See PR #27717 for context. + pytest.param( + "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V", + marks=pytest.mark.skip( + reason=( + "Checkpoint removed from HF; temporarily disabling this " + "AutoFP8 split K/V case (PR #27717)." + ) + ), + ), ] From 4e68cc9b6aa2b9cfe8d799c2b1cd156a01bca438 Mon Sep 17 00:00:00 2001 From: Zhiyuan Li Date: Thu, 30 Oct 2025 21:02:27 +0800 Subject: [PATCH 105/127] [Model] Introduce Kimi Linear to vLLM (#27809) Signed-off-by: lizhiyuan Signed-off-by: Zhiyuan Li --- docs/models/supported_models.md | 1 + tests/models/registry.py | 3 + vllm/config/compilation.py | 1 + vllm/config/model.py | 1 + vllm/model_executor/layers/fla/ops/kda.py | 2 +- vllm/model_executor/layers/kda.py | 426 +++++++++++ .../layers/mamba/mamba_utils.py | 41 ++ vllm/model_executor/layers/mla.py | 7 +- vllm/model_executor/models/config.py | 51 +- vllm/model_executor/models/kimi_linear.py | 663 ++++++++++++++++++ vllm/model_executor/models/registry.py | 1 + vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 2 + .../transformers_utils/configs/kimi_linear.py | 144 ++++ vllm/v1/worker/gpu_model_runner.py | 29 +- 15 files changed, 1325 insertions(+), 48 deletions(-) create mode 100644 vllm/model_executor/layers/kda.py create mode 100644 vllm/model_executor/models/kimi_linear.py create mode 100644 vllm/transformers_utils/configs/kimi_linear.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 4d50c809d1966..c9744d31f0efc 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -382,6 +382,7 @@ th { | `InternLM3ForCausalLM` | InternLM3 | `internlm/internlm3-8b-instruct`, etc. | ✅︎ | ✅︎ | | `JAISLMHeadModel` | Jais | `inceptionai/jais-13b`, `inceptionai/jais-13b-chat`, `inceptionai/jais-30b-v3`, `inceptionai/jais-30b-chat-v3`, etc. | | ✅︎ | | `JambaForCausalLM` | Jamba | `ai21labs/AI21-Jamba-1.5-Large`, `ai21labs/AI21-Jamba-1.5-Mini`, `ai21labs/Jamba-v0.1`, etc. | ✅︎ | ✅︎ | +| `KimiLinearForCausalLM` | Kimi-Linear-48B-A3B-Base, Kimi-Linear-48B-A3B-Instruct | `moonshotai/Kimi-Linear-48B-A3B-Base`, `moonshotai/Kimi-Linear-48B-A3B-Instruct` | | ✅︎ | | `Lfm2ForCausalLM` | LFM2 | `LiquidAI/LFM2-1.2B`, `LiquidAI/LFM2-700M`, `LiquidAI/LFM2-350M`, etc. | ✅︎ | ✅︎ | | `Lfm2MoeForCausalLM` | LFM2MoE | `LiquidAI/LFM2-8B-A1B-preview`, etc. | ✅︎ | ✅︎ | | `LlamaForCausalLM` | Llama 3.1, Llama 3, Llama 2, LLaMA, Yi | `meta-llama/Meta-Llama-3.1-405B-Instruct`, `meta-llama/Meta-Llama-3.1-70B`, `meta-llama/Meta-Llama-3-70B-Instruct`, `meta-llama/Llama-2-70b-hf`, `01-ai/Yi-34B`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index 17b1d7b527f6b..9a2a1eb5f1a74 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -296,6 +296,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "random": "ai21labs/Jamba-tiny-random", }, ), + "KimiLinearForCausalLM": _HfExamplesInfo( + "moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True + ), "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"), "Lfm2MoeForCausalLM": _HfExamplesInfo( "LiquidAI/LFM2-8B-A1B", min_transformers_version="4.58" diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index f3ed78779a995..6a5bd5ef4e07c 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -453,6 +453,7 @@ class CompilationConfig: "vllm::linear_attention", "vllm::plamo2_mamba_mixer", "vllm::gdn_attention", + "vllm::kda_attention", "vllm::sparse_attn_indexer", ] diff --git a/vllm/config/model.py b/vllm/config/model.py index 2151939d5a9f6..092c67e7bed8c 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1236,6 +1236,7 @@ class ModelConfig: "deepseek_v32", "deepseek_mtp", "kimi_k2", + "kimi_linear", "longcat_flash", ): return self.hf_text_config.kv_lora_rank is not None diff --git a/vllm/model_executor/layers/fla/ops/kda.py b/vllm/model_executor/layers/fla/ops/kda.py index a10847d347d13..700f287ca4569 100644 --- a/vllm/model_executor/layers/fla/ops/kda.py +++ b/vllm/model_executor/layers/fla/ops/kda.py @@ -1304,7 +1304,7 @@ def kda_gate_fwd_kernel( tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1)) -def kda_gate_fwd( +def fused_kda_gate( g: torch.Tensor, A: torch.Tensor, head_k_dim: int, diff --git a/vllm/model_executor/layers/kda.py b/vllm/model_executor/layers/kda.py new file mode 100644 index 0000000000000..c45e7546fac1e --- /dev/null +++ b/vllm/model_executor/layers/kda.py @@ -0,0 +1,426 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from einops import rearrange +from torch import nn + +from vllm.attention import AttentionBackend +from vllm.attention.backends.abstract import AttentionMetadata +from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config +from vllm.distributed import ( + divide, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from vllm.forward_context import ForwardContext, get_forward_context +from vllm.logger import init_logger +from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader +from vllm.model_executor.utils import set_weight_attrs +from vllm.utils.torch_utils import direct_register_custom_op +from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata + +from .fla.ops.kda import ( + FusedRMSNormGated, + chunk_kda, + fused_kda_gate, + fused_recurrent_kda, +) +from .linear import ( + ColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from .mamba.abstract import MambaBase +from .mamba.mamba_utils import MambaStateDtypeCalculator, MambaStateShapeCalculator +from .mamba.ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update +from .quantization.base_config import QuantizationConfig + +logger = init_logger(__name__) + + +def kda_attention( + hidden_states: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + forward_context: ForwardContext = get_forward_context() + self = forward_context.no_compile_layers[layer_name] + self._forward(hidden_states=hidden_states, output=output) + + +def kda_attention_fake( + hidden_states: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + return + + +direct_register_custom_op( + op_name="kda_attention", + op_func=kda_attention, + mutates_args=["output"], + fake_impl=kda_attention_fake, +) + + +class KimiDeltaAttention(nn.Module, MambaBase): + @property + def mamba_type(self) -> str: + return "linear_attention" + + def get_attn_backend(self) -> type["AttentionBackend"]: + from vllm.v1.attention.backends.gdn_attn import GDNAttentionBackend + + return GDNAttentionBackend + + def get_state_dtype( + self, + ) -> tuple[torch.dtype, torch.dtype, torch.dtype, torch.dtype]: + if self.model_config is None or self.cache_config is None: + raise ValueError("model_config and cache_config must be set") + return MambaStateDtypeCalculator.kda_state_dtype( + self.model_config.dtype, self.cache_config.mamba_cache_dtype + ) + + def get_state_shape( + self, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + return MambaStateShapeCalculator.kda_state_shape( + self.tp_size, self.num_heads, self.head_dim, conv_kernel_size=self.conv_size + ) + + def __init__( + self, + layer_idx: int, + hidden_size: int, + quant_config: QuantizationConfig | None = None, + cache_config: CacheConfig | None = None, + model_config: ModelConfig | None = None, + rms_norm_eps: float = 1e-5, + prefix: str = "", + **kwargs, + ) -> None: + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + self.hidden_size = hidden_size + self.model_config = model_config + self.cache_config = cache_config + if model_config is None: + raise ValueError("model_config must be provided") + kda_config = model_config.linear_attn_config + self.head_dim = kda_config["head_dim"] + self.num_heads = kda_config["num_heads"] + self.layer_idx = layer_idx + self.prefix = prefix + assert self.num_heads % self.tp_size == 0 + self.local_num_heads = divide(self.num_heads, self.tp_size) + + projection_size = self.head_dim * self.num_heads + self.conv_size = kda_config["short_conv_kernel_size"] + + self.q_proj = ColumnParallelLinear( + self.hidden_size, + projection_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_proj", + ) + self.k_proj = ColumnParallelLinear( + self.hidden_size, + projection_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.k_proj", + ) + self.v_proj = ColumnParallelLinear( + self.hidden_size, + projection_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.v_proj", + ) + + self.f_a_proj = ReplicatedLinear( + self.hidden_size, + self.head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.f_a_proj", + ) + + self.f_b_proj = ColumnParallelLinear( + self.head_dim, + projection_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.f_b_proj", + ) + self.dt_bias = nn.Parameter( + torch.empty(divide(projection_size, self.tp_size), dtype=torch.float32) + ) + + set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)}) + + self.b_proj = ColumnParallelLinear( + self.hidden_size, + self.num_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.b_proj", + ) + + self.q_conv1d = ColumnParallelLinear( + input_size=self.conv_size, + output_size=projection_size, + bias=False, + params_dtype=torch.float32, + prefix=f"{prefix}.q_conv1d", + ) + self.k_conv1d = ColumnParallelLinear( + input_size=self.conv_size, + output_size=projection_size, + bias=False, + params_dtype=torch.float32, + prefix=f"{prefix}.k_conv1d", + ) + self.v_conv1d = ColumnParallelLinear( + input_size=self.conv_size, + output_size=projection_size, + bias=False, + params_dtype=torch.float32, + prefix=f"{prefix}.v_conv1d", + ) + # unsqueeze to fit conv1d weights shape into the linear weights shape. + # Can't do this in `weight_loader` since it already exists in + # `ColumnParallelLinear` and `set_weight_attrs` + # doesn't allow to override it + self.q_conv1d.weight.data = self.q_conv1d.weight.data.unsqueeze(1) + self.k_conv1d.weight.data = self.k_conv1d.weight.data.unsqueeze(1) + self.v_conv1d.weight.data = self.v_conv1d.weight.data.unsqueeze(1) + + self.A_log = nn.Parameter( + torch.empty(1, 1, self.local_num_heads, 1, dtype=torch.float32) + ) + set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(2)}) + + self.g_a_proj = ReplicatedLinear( + self.hidden_size, + self.head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.g_a_proj", + ) + self.g_b_proj = ColumnParallelLinear( + self.head_dim, + projection_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.g_b_proj", + ) + self.o_norm = FusedRMSNormGated( + self.head_dim, eps=rms_norm_eps, activation="sigmoid" + ) + self.o_proj = RowParallelLinear( + projection_size, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + compilation_config = get_current_vllm_config().compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + + def forward( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + output: torch.Tensor, + ) -> None: + return torch.ops.vllm.kda_attention( + hidden_states, + output, + self.prefix, + ) + + def _forward( + self, + hidden_states: torch.Tensor, + output: torch.Tensor, + ) -> None: + forward_context = get_forward_context() + attn_metadata: AttentionMetadata = forward_context.attn_metadata + + if attn_metadata is None: + # V1 profile run + # Mimic the memory allocation in the real run + q = torch.empty_like(hidden_states) + k = torch.empty_like(hidden_states) + v = torch.empty_like(hidden_states) + g = hidden_states.new_empty( + hidden_states.size(0), + self.local_num_heads, + self.head_dim, + dtype=torch.float32, + ) + beta = torch.empty( + hidden_states.size(0), self.local_num_heads, dtype=torch.float32 + ) + core_attn_out = torch.empty_like(hidden_states) + return + + assert isinstance(attn_metadata, dict) + attn_metadata = attn_metadata[self.prefix] + assert isinstance(attn_metadata, GDNAttentionMetadata) + has_initial_state = attn_metadata.has_initial_state + non_spec_query_start_loc = attn_metadata.non_spec_query_start_loc + non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor # noqa: E501 + constant_caches = self.kv_cache[forward_context.virtual_engine] + + (conv_state_q, conv_state_k, conv_state_v, recurrent_state) = constant_caches + # deal with strides + conv_state_q = conv_state_q.transpose(-1, -2) + conv_state_k = conv_state_k.transpose(-1, -2) + conv_state_v = conv_state_v.transpose(-1, -2) + + q_proj_states = self.q_proj(hidden_states)[0] + k_proj_states = self.k_proj(hidden_states)[0] + v_proj_states = self.v_proj(hidden_states)[0] + + q_conv_weights = self.q_conv1d.weight.view( + self.q_conv1d.weight.size(0), self.q_conv1d.weight.size(2) + ) + k_conv_weights = self.k_conv1d.weight.view( + self.k_conv1d.weight.size(0), self.k_conv1d.weight.size(2) + ) + v_conv_weights = self.v_conv1d.weight.view( + self.v_conv1d.weight.size(0), self.v_conv1d.weight.size(2) + ) + if attn_metadata.num_prefills > 0: + q_proj_states = q_proj_states.transpose(0, 1) + k_proj_states = k_proj_states.transpose(0, 1) + v_proj_states = v_proj_states.transpose(0, 1) + q = causal_conv1d_fn( + q_proj_states, + q_conv_weights, + self.q_conv1d.bias, + activation="silu", + conv_states=conv_state_q, + has_initial_state=has_initial_state, + cache_indices=non_spec_state_indices_tensor, + query_start_loc=non_spec_query_start_loc, + metadata=attn_metadata, + ).transpose(0, 1) + k = causal_conv1d_fn( + k_proj_states, + k_conv_weights, + self.k_conv1d.bias, + activation="silu", + conv_states=conv_state_k, + has_initial_state=has_initial_state, + cache_indices=non_spec_state_indices_tensor, + query_start_loc=non_spec_query_start_loc, + metadata=attn_metadata, + ).transpose(0, 1) + v = causal_conv1d_fn( + v_proj_states, + v_conv_weights, + self.v_conv1d.bias, + activation="silu", + conv_states=conv_state_v, + has_initial_state=has_initial_state, + cache_indices=non_spec_state_indices_tensor, + query_start_loc=non_spec_query_start_loc, + metadata=attn_metadata, + ).transpose(0, 1) + else: + decode_conv_indices = non_spec_state_indices_tensor[ + : attn_metadata.num_decodes + ] + q = causal_conv1d_update( + q_proj_states, + conv_state_q, + q_conv_weights, + self.q_conv1d.bias, + activation="silu", + conv_state_indices=decode_conv_indices, + validate_data=True, + ) + k = causal_conv1d_update( + k_proj_states, + conv_state_k, + k_conv_weights, + self.k_conv1d.bias, + activation="silu", + conv_state_indices=decode_conv_indices, + validate_data=True, + ) + v = causal_conv1d_update( + v_proj_states, + conv_state_v, + v_conv_weights, + self.v_conv1d.bias, + activation="silu", + conv_state_indices=decode_conv_indices, + validate_data=True, + ) + + q, k, v = map( + lambda x: rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim), (q, k, v) + ) + + beta = self.b_proj(hidden_states)[0].float().sigmoid() + + g = self.f_b_proj(self.f_a_proj(hidden_states)[0])[0] + g = fused_kda_gate(g, self.A_log, self.head_dim, g_bias=self.dt_bias) + + beta = beta.unsqueeze(0) + g = g.unsqueeze(0) + + if attn_metadata.num_prefills > 0: + zero_idx = non_spec_state_indices_tensor[~has_initial_state] + recurrent_state[zero_idx] = 0 + initial_state = recurrent_state[non_spec_state_indices_tensor].contiguous() + ( + core_attn_out_non_spec, + last_recurrent_state, + ) = chunk_kda( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + cu_seqlens=non_spec_query_start_loc, + ) + # Init cache + recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state + else: + ( + core_attn_out_non_spec, + last_recurrent_state, + ) = fused_recurrent_kda( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=recurrent_state, + use_qk_l2norm_in_kernel=True, + cu_seqlens=non_spec_query_start_loc, + ssm_state_indices=non_spec_state_indices_tensor, + ) + + g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0] + g = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim) + core_attn_out = self.o_norm(core_attn_out_non_spec, g) + core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)") + + output[:] = self.o_proj(core_attn_out)[0] diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 91a45623582d5..831dab2fbb01c 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -80,6 +80,15 @@ class MambaStateDtypeCalculator: state_dtype = get_kv_cache_torch_dtype(mamba_cache_dtype, model_dtype) return (state_dtype, state_dtype) + @classmethod + def kda_state_dtype( + cls, + model_dtype: ModelDType | torch.dtype, + mamba_cache_dtype: MambaDType, + ): + state_dtype = get_kv_cache_torch_dtype(mamba_cache_dtype, model_dtype) + return (state_dtype, state_dtype, state_dtype, torch.float32) + class MambaStateShapeCalculator: @classmethod @@ -182,3 +191,35 @@ class MambaStateShapeCalculator: head_v_dim, ) return conv_state_shape, temporal_state_shape + + @classmethod + def kda_state_shape( + cls, + tp_world_size: int, + num_heads: int, + head_dim: int, + num_k_heads: int | None = None, + head_k_dim: int | None = None, + conv_kernel_size: int = 4, + num_spec: int = 0, + ) -> tuple[tuple[int, int], tuple[int, int], tuple[int, int], tuple[int, int, int]]: + if num_k_heads is None: + num_k_heads = num_heads + if head_k_dim is None: + head_k_dim = head_dim + + proj_size = num_heads * head_dim + proj_k_size = num_k_heads * head_k_dim + + conv_state_shape = (divide(proj_size, tp_world_size), conv_kernel_size - 1) + conv_state_k_shape = (divide(proj_k_size, tp_world_size), conv_kernel_size - 1) + recurrent_state_shape = (divide(num_heads, tp_world_size), head_dim, head_dim) + + conv_state_shape = conv_state_shape[1], conv_state_shape[0] + conv_state_k_shape = conv_state_k_shape[1], conv_state_k_shape[0] + return ( + conv_state_shape, + conv_state_k_shape, + conv_state_k_shape, + recurrent_state_shape, + ) diff --git a/vllm/model_executor/layers/mla.py b/vllm/model_executor/layers/mla.py index 34f05f2ee9624..c4c44b83ae6bf 100644 --- a/vllm/model_executor/layers/mla.py +++ b/vllm/model_executor/layers/mla.py @@ -147,9 +147,10 @@ class MultiHeadLatentAttentionWrapper(CustomOp): # Add head dim of 1 to k_pe k_pe = k_pe.unsqueeze(1) - q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb( - positions, q[..., self.qk_nope_head_dim :], k_pe - ) + if self.rotary_emb is not None: + q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb( + positions, q[..., self.qk_nope_head_dim :], k_pe + ) if self.indexer and self.is_sparse: _topk_indices = self.indexer(hidden_states, q_c, positions, self.rotary_emb) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 3bd02121f018e..b0a48a9f1d458 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from copy import deepcopy +from math import lcm from typing import TYPE_CHECKING import vllm.envs as envs @@ -8,7 +9,7 @@ from vllm.logger import init_logger from vllm.model_executor.models import ModelRegistry from vllm.utils.math_utils import cdiv, round_up from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE -from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec +from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec, MLAAttentionSpec if TYPE_CHECKING: from vllm.config import VllmConfig @@ -347,12 +348,28 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype] # get attention page size (for 1 token) - attn_page_size_1_token = FullAttentionSpec( - block_size=1, - num_kv_heads=model_config.get_num_kv_heads(parallel_config), - head_size=model_config.get_head_size(), - dtype=kv_cache_dtype, - ).page_size_bytes + # Attention backend constraints: + # - FlashAttention (FA) requires block size to be multiple of 16 + # - MLA (Multi-head Latent Attention) requires larger alignment: + # * CUTLASS_MLA backend: kernel_block_size 128 alignment + # * Other MLA backends: kernel_block_size 64 alignment + if model_config.use_mla: + use_cutlass_mla = envs.VLLM_ATTENTION_BACKEND == "CUTLASS_MLA" + kernel_block_alignment_size = 128 if use_cutlass_mla else 64 + attn_page_size_1_token = MLAAttentionSpec( + block_size=1, + num_kv_heads=model_config.get_num_kv_heads(parallel_config), + head_size=model_config.get_head_size(), + dtype=kv_cache_dtype, + ).page_size_bytes + else: + kernel_block_alignment_size = 16 + attn_page_size_1_token = FullAttentionSpec( + block_size=1, + num_kv_heads=model_config.get_num_kv_heads(parallel_config), + head_size=model_config.get_head_size(), + dtype=kv_cache_dtype, + ).page_size_bytes model_cls, _ = ModelRegistry.resolve_model_cls( model_config.architecture, @@ -372,17 +389,6 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): if mamba_page_size == 0: return - # Attention backend constraints: - # - FlashAttention (FA) requires block size to be multiple of 16 - # - MLA (Multi-head Latent Attention) requires larger alignment: - # * CUTLASS_MLA backend: 128-byte alignment - # * Other MLA backends: 64-byte alignment - if model_config.use_mla: - use_cutlass_mla = envs.VLLM_ATTENTION_BACKEND == "CUTLASS_MLA" - kernel_block_alignment_size = 128 if use_cutlass_mla else 64 - else: - kernel_block_alignment_size = 16 - if cache_config.enable_prefix_caching: # With prefix caching, select attention block size to # optimize for mamba kernel performance @@ -400,15 +406,8 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): # easily by changing the way we layout chunks in the # mamba2 kernels. - from math import gcd - - def lcm(a, b): - return a * b // gcd(a, b) - - base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size() - + base_chunk_size = model_config.get_mamba_chunk_size() attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token) - chunk_size = lcm(base_chunk_size, kernel_block_alignment_size) attn_block_size = chunk_size * cdiv(attn_tokens_per_mamba_state, chunk_size) cache_config.mamba_block_size = attn_block_size diff --git a/vllm/model_executor/models/kimi_linear.py b/vllm/model_executor/models/kimi_linear.py new file mode 100644 index 0000000000000..a60a8d764d9d1 --- /dev/null +++ b/vllm/model_executor/models/kimi_linear.py @@ -0,0 +1,663 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Iterable +from typing import Any + +import torch +from torch import nn + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig +from vllm.distributed import ( + get_pp_group, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.kda import KimiDeltaAttention +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.mamba.mamba_utils import ( + MambaStateDtypeCalculator, + MambaStateShapeCalculator, +) +from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig + +from .interfaces import HasInnerState, IsHybrid, MixtureOfExperts, SupportsPP +from .utils import ( + PPMissingLayer, + is_pp_missing_parameter, + make_layers, + maybe_prefix, +) + +logger = init_logger(__name__) + + +class KimiMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: QKVParallelLinear | None = None, + reduce_results: bool = True, + prefix: str = "", + ) -> None: + super().__init__() + + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=f"{prefix}.down_proj", + ) + if hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {hidden_act}. Only silu is supported for now." + ) + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class KimiMoE(nn.Module): + def __init__( + self, + config: KimiLinearConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + layer_idx: int = 0, + ): + super().__init__() + hidden_size = config.hidden_size + intermediate_size = config.intermediate_size + moe_intermediate_size = config.moe_intermediate_size + num_experts = config.num_experts + moe_renormalize = config.moe_renormalize + self.tp_size = get_tensor_model_parallel_world_size() + self.routed_scaling_factor = config.routed_scaling_factor + self.num_shared_experts = config.num_shared_experts + self.layer_idx = layer_idx + + if config.hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {config.hidden_act}. " + "Only silu is supported for now." + ) + + # Gate always runs at half / full precision for now. + self.gate = ReplicatedLinear( + hidden_size, + num_experts, + bias=False, + quant_config=None, + prefix=f"{prefix}.gate", + ) + + self.gate.e_score_correction_bias = nn.Parameter(torch.empty(num_experts)) + + self.experts = FusedMoE( + num_experts=num_experts, + top_k=config.num_experts_per_token, + hidden_size=hidden_size, + intermediate_size=moe_intermediate_size, + reduce_results=False, + renormalize=moe_renormalize, + quant_config=quant_config, + use_grouped_topk=config.use_grouped_topk, + num_expert_group=config.num_expert_group, + topk_group=config.topk_group, + prefix=f"{prefix}.experts", + scoring_func=config.moe_router_activation_func, + e_score_correction_bias=self.gate.e_score_correction_bias, + ) + + if self.num_shared_experts is not None: + intermediate_size = moe_intermediate_size * self.num_shared_experts + self.shared_experts = KimiMLP( + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_size = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_size) + if self.num_shared_experts is not None: + shared_output = self.shared_experts(hidden_states) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = ( + self.experts(hidden_states=hidden_states, router_logits=router_logits) + * self.routed_scaling_factor + ) + if shared_output is not None: + final_hidden_states = final_hidden_states + shared_output + + if self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + return final_hidden_states.view(num_tokens, hidden_size) + + +class KimiMLAAttention(nn.Module): + """ + Main reference: DeepseekV2 vllm Implementation + """ + + def __init__( + self, + config: KimiLinearConfig, + hidden_size: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + q_lora_rank: int | None, + kv_lora_rank: int, + rope_theta: float = 10000, + use_nope: bool = False, + rope_scaling: dict[str, Any] | None = None, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + **kwargs, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.num_heads = num_heads + tp_size = get_tensor_model_parallel_world_size() + self.num_local_heads = num_heads // tp_size + self.scaling = self.qk_head_dim**-0.5 + self.rope_theta = rope_theta + self.use_nope = use_nope + assert self.use_nope is True + assert self.q_lora_rank is None + assert rope_scaling is None + assert num_heads % tp_size == 0 + self.kv_a_proj_with_mqa = ReplicatedLinear( + self.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_a_proj_with_mqa", + ) + self.q_proj = ColumnParallelLinear( + self.hidden_size, + self.num_heads * self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_proj", + ) + self.kv_a_layernorm = RMSNorm( + self.kv_lora_rank, + eps=config.rms_norm_eps, + ) + self.kv_b_proj = ColumnParallelLinear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_b_proj", + ) + self.o_proj = RowParallelLinear( + self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + mla_modules = MLAModules( + kv_a_layernorm=self.kv_a_layernorm, + kv_b_proj=self.kv_b_proj, + rotary_emb=None, + o_proj=self.o_proj, + fused_qkv_a_proj=None, + kv_a_proj_with_mqa=self.kv_a_proj_with_mqa, + q_a_layernorm=None, + q_b_proj=None, + q_proj=self.q_proj, + indexer=None, + is_sparse=False, + topk_indices_buffer=None, + ) + self.mla_attn = MultiHeadLatentAttentionWrapper( + self.hidden_size, + self.num_local_heads, + self.scaling, + self.qk_nope_head_dim, + self.qk_rope_head_dim, + self.v_head_dim, + self.q_lora_rank, + self.kv_lora_rank, + mla_modules, + cache_config, + quant_config, + prefix, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + output: torch.Tensor, + ) -> None: + output[:] = self.mla_attn(positions, hidden_states) + + +class KimiDecoderLayer(nn.Module): + def __init__( + self, + config: KimiLinearConfig, + layer_idx: int, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + parallel_config: ParallelConfig | None = None, + model_config: ModelConfig | None = None, + prefix: str = "", + **kwargs, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + + self.is_moe = config.is_moe + + if config.is_kda_layer(layer_idx): + self.self_attn = KimiDeltaAttention( + layer_idx=layer_idx, + hidden_size=config.hidden_size, + quant_config=quant_config, + cache_config=cache_config, + model_config=config, + prefix=f"{prefix}.self_attn", + ) + else: + self.self_attn = KimiMLAAttention( + layer_idx=layer_idx, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + quant_config=quant_config, + cache_config=cache_config, + model_config=model_config, + prefix=f"{prefix}.self_attn", + config=config, + qk_nope_head_dim=config.qk_nope_head_dim, + qk_rope_head_dim=config.qk_rope_head_dim, + v_head_dim=config.v_head_dim, + q_lora_rank=config.q_lora_rank, + kv_lora_rank=config.kv_lora_rank, + use_nope=config.mla_use_nope, + ) + + if ( + self.is_moe + and config.num_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % config.moe_layer_freq == 0 + ): + self.block_sparse_moe = KimiMoE( + config=config, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.mlp = self.block_sparse_moe + else: + self.mlp = KimiMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + attn_output = torch.empty_like(hidden_states) + self.self_attn( + hidden_states=hidden_states, + positions=positions, + output=attn_output, + ) + hidden_states = attn_output + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +@support_torch_compile +class KimiLinearModel(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + + config = vllm_config.model_config.hf_text_config + model_config = vllm_config.model_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + parallel_config = vllm_config.parallel_config + self.config = config + + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + if get_pp_group().is_first_rank: + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + prefix=f"{prefix}.embed_tokens", + ) + else: + self.embed_tokens = PPMissingLayer() + + extra_kwargs = {} + + def get_layer(prefix: str): + layer_idx = int(prefix.rsplit(".", 1)[1]) + return KimiDecoderLayer( + config, + layer_idx, + cache_config, + quant_config, + parallel_config, + model_config, + prefix, + **extra_kwargs, + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + get_layer, + prefix=f"{prefix}.layers", + ) + + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + + world_size = get_tensor_model_parallel_world_size() + assert config.num_attention_heads % world_size == 0, ( + "num_attention_heads must be divisible by world_size" + ) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for _, layer in enumerate(self.layers[self.start_layer : self.end_layer]): + hidden_states, residual = layer( + positions=positions, + hidden_states=hidden_states, + residual=residual, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + ) + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class KimiLinearForCausalLM( + nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid +): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + self.model_config = vllm_config.model_config + self.vllm_config = vllm_config + self.config = self.model_config.hf_config + quant_config = vllm_config.quant_config + self.quant_config = quant_config + self.model = KimiLinearModel( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + if get_pp_group().is_last_rank: + self.lm_head = ParallelLMHead( + self.config.vocab_size, + self.config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + else: + self.lm_head = PPMissingLayer() + logit_scale = getattr(self.config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor( + self.config.vocab_size, scale=logit_scale + ) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor | IntermediateTensors: + hidden_states = self.model( + input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs + ) + return hidden_states + + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: "VllmConfig", + ) -> tuple[torch.dtype, torch.dtype, torch.dtype, torch.dtype]: + return MambaStateDtypeCalculator.kda_state_dtype( + vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype + ) + + @classmethod + def get_mamba_state_shape_from_config( + cls, vllm_config: "VllmConfig" + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + parallel_config = vllm_config.parallel_config + hf_config = vllm_config.model_config.hf_config + tp_size = parallel_config.tensor_parallel_size + num_spec = ( + vllm_config.speculative_config.num_speculative_tokens + if vllm_config.speculative_config + else 0 + ) + return MambaStateShapeCalculator.kda_state_shape( + tp_size, + hf_config.linear_attn_config["num_heads"], + hf_config.linear_attn_config["head_dim"], + conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"], + num_spec=num_spec, + ) + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + if self.config.is_moe: + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="w1", + ckpt_down_proj_name="w2", + ckpt_up_proj_name="w3", + num_experts=self.config.num_experts, + ) + else: + expert_params_mapping = [] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for args in weights: + name, loaded_weight = args[:2] + kwargs = args[2] if len(args) > 2 else {} + if "rotary_emb.inv_freq" in name: + continue + + spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) + if spec_layer is not None: + continue # skip spec decode layers for main model + if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + # We have mlp.experts[0].gate_proj in the checkpoint. + # Since we handle the experts below in expert_params_mapping, + # we need to skip here BEFORE we update the name, otherwise + # name will be updated to mlp.experts[0].gate_up_proj, which + # will then be updated below in expert_params_mapping + # for mlp.experts[0].gate_gate_up_proj, which breaks load. + if ("mlp.experts." in name) and name not in params_dict: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for idx, (param_name, weight_name, expert_id, shard_id) in enumerate( + expert_params_mapping + ): + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader( + param, + loaded_weight, + name, + expert_id=expert_id, + shard_id=shard_id, + ) + break + else: + # Skip loading extra bias for GPTQ models. + if ( + name.endswith(".bias") + and name not in params_dict + and not self.config.is_linear_attn + ): # noqa: E501 + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight, **kwargs) + loaded_params.add(name) + + +def get_spec_layer_idx_from_weight_name( + config: KimiLinearConfig, weight_name: str +) -> int | None: + if hasattr(config, "num_nextn_predict_layers") and ( + config.num_nextn_predict_layers > 0 + ): + layer_idx = config.num_hidden_layers + for i in range(config.num_nextn_predict_layers): + if weight_name.startswith(f"model.layers.{layer_idx + i}."): + return layer_idx + i + return None diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 0027954ac2771..8e4413c90cf6c 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -118,6 +118,7 @@ _TEXT_GENERATION_MODELS = { "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"), "JAISLMHeadModel": ("jais", "JAISLMHeadModel"), "JambaForCausalLM": ("jamba", "JambaForCausalLM"), + "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"), # noqa: E501 "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"), "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"), "LlamaForCausalLM": ("llama", "LlamaForCausalLM"), diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 34c0429a80679..b1f4e3e2a9831 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -79,6 +79,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( deepseek_v3="DeepseekV3Config", deepseek_v32="DeepseekV3Config", flex_olmo="FlexOlmoConfig", + kimi_linear="KimiLinearConfig", kimi_vl="KimiVLConfig", Llama_Nemotron_Nano_VL="Nemotron_Nano_VL_Config", RefinedWeb="RWConfig", # For tiiuae/falcon-40b(-instruct) diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index befe9cdae76a1..663a8e44d71dd 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -19,6 +19,7 @@ from vllm.transformers_utils.configs.eagle import EAGLEConfig from vllm.transformers_utils.configs.falcon import RWConfig from vllm.transformers_utils.configs.flex_olmo import FlexOlmoConfig from vllm.transformers_utils.configs.jais import JAISConfig +from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig from vllm.transformers_utils.configs.medusa import MedusaConfig @@ -54,6 +55,7 @@ __all__ = [ "MiDashengLMConfig", "MLPSpeculatorConfig", "MoonViTConfig", + "KimiLinearConfig", "KimiVLConfig", "NemotronConfig", "NemotronHConfig", diff --git a/vllm/transformers_utils/configs/kimi_linear.py b/vllm/transformers_utils/configs/kimi_linear.py new file mode 100644 index 0000000000000..65ddf48c5249b --- /dev/null +++ b/vllm/transformers_utils/configs/kimi_linear.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from transformers.configuration_utils import PretrainedConfig + +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class KimiLinearConfig(PretrainedConfig): + model_type = "kimi_linear" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + model_type="kimi_linear", + vocab_size=163840, + hidden_size=4096, + head_dim=None, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + rope_theta=10000.0, + rope_scaling=None, + tie_word_embeddings=False, + moe_intermediate_size: int | None = None, + moe_renormalize: bool = True, + moe_router_activation_func: str = "sigmoid", + num_experts: int | None = None, + num_experts_per_token: int | None = None, + num_shared_experts: int = 0, + routed_scaling_factor: float = 1.0, + first_k_dense_replace: int = 0, + moe_layer_freq: int = 1, + use_grouped_topk: bool = True, + num_expert_group: int = 1, + topk_group: int = 1, + q_lora_rank: int | None = None, + kv_lora_rank: int | None = None, + qk_nope_head_dim: int | None = None, + qk_rope_head_dim: int | None = None, + v_head_dim: int | None = None, + mla_use_nope: bool | None = False, + num_nextn_predict_layers: int = 0, + linear_attn_config: dict | None = None, + **kwargs, + ): + self.model_type = model_type + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.head_dim = ( + head_dim if head_dim is not None else hidden_size // num_attention_heads + ) + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.mla_use_nope = mla_use_nope + # moe config + self.num_experts = num_experts + self.num_experts_per_token = num_experts_per_token + self.moe_renormalize = moe_renormalize + self.num_shared_experts = num_shared_experts + self.routed_scaling_factor = routed_scaling_factor + self.moe_router_activation_func = moe_router_activation_func + assert self.moe_router_activation_func in ("softmax", "sigmoid") + self.moe_intermediate_size = moe_intermediate_size + self.first_k_dense_replace = first_k_dense_replace + self.moe_layer_freq = moe_layer_freq + self.use_grouped_topk = use_grouped_topk + self.num_expert_group = num_expert_group + self.topk_group = topk_group + self.num_nextn_predict_layers = num_nextn_predict_layers + + if linear_attn_config is not None: + assert linear_attn_config["kda_layers"] is not None + assert linear_attn_config["full_attn_layers"] is not None + self.linear_attn_config = linear_attn_config + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + @property + def is_mla(self): + return ( + self.q_lora_rank is not None + or self.kv_lora_rank is not None + or self.qk_nope_head_dim is not None + or self.qk_rope_head_dim is not None + or self.v_head_dim is not None + or self.mla_use_nope is True + ) + + @property + def is_moe(self): + return self.num_experts is not None + + @property + def is_linear_attn(self) -> bool: + return not ( + self.linear_attn_config is None + or ( + isinstance(self.linear_attn_config, dict) + and self.linear_attn_config["kda_layers"] is not None + and len(self.linear_attn_config["kda_layers"]) == 0 + ) + ) + + def is_kda_layer(self, layer_idx: int): + return ( + self.linear_attn_config is not None + and (layer_idx + 1) in self.linear_attn_config["kda_layers"] + ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 1fe749c614ccf..729ce462cf186 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -8,6 +8,7 @@ from collections import defaultdict from collections.abc import Iterator from contextlib import contextmanager from copy import deepcopy +from functools import reduce from itertools import product from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast @@ -4134,26 +4135,18 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): def calculate_reorder_batch_threshold(self) -> None: """ - Check that if any backends reorder batches; that the reordering - is compatible (e.g., decode threshold is the same) + Choose the minimum reorder batch threshold from all attention groups. + Backends should be able to support lower threshold then what they request + just may have a performance penalty due to that backend treating decodes + as prefills. """ - for group in self._attn_group_iterator(): - attn_metadata_builder_i = group.get_metadata_builder() + min_none_high = lambda a, b: a if b is None else b if a is None else min(a, b) - # check that if any backends reorder batches; that the reordering - # is compatible (e.g., decode threshold is the same) - reorder_batch_threshold_i = attn_metadata_builder_i.reorder_batch_threshold - if reorder_batch_threshold_i is not None: - if self.reorder_batch_threshold is not None: - if reorder_batch_threshold_i != self.reorder_batch_threshold: - raise ValueError( - f"Attention backend reorders decodes with " - f"threshold {reorder_batch_threshold_i} but other " - f"backend uses threshold " - f"{self.reorder_batch_threshold}" - ) - else: - self.reorder_batch_threshold = reorder_batch_threshold_i + reorder_batch_thresholds = [ + group.get_metadata_builder().reorder_batch_threshold + for group in self._attn_group_iterator() + ] + self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) def _find_compatible_block_sizes( self, From 0fe01404082744c955d135c3634e17de1404b00c Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Thu, 30 Oct 2025 07:10:29 -0700 Subject: [PATCH 106/127] [KV offload] Enable CPU KV offload on CUDA alike Platforms (#27770) Signed-off-by: zhewenli --- tests/v1/kv_offload/test_cpu_offloading.py | 4 ---- vllm/v1/kv_offload/cpu.py | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/v1/kv_offload/test_cpu_offloading.py b/tests/v1/kv_offload/test_cpu_offloading.py index a5cb23c4ef0f2..b654ea4298dbb 100644 --- a/tests/v1/kv_offload/test_cpu_offloading.py +++ b/tests/v1/kv_offload/test_cpu_offloading.py @@ -12,7 +12,6 @@ from tqdm import tqdm from vllm import LLM, SamplingParams, TokensPrompt from vllm.config import KVEventsConfig, KVTransferConfig from vllm.distributed.kv_events import BlockStored, KVEventBatch -from vllm.platforms import current_platform CPU_BLOCK_SIZES = [16, 48] @@ -64,9 +63,6 @@ class MockSubscriber: self.sub.close() -@pytest.mark.skipif( - not current_platform.is_cuda(), reason="CPU offloading only supported on CUDA" -) @pytest.mark.parametrize("cpu_block_size", CPU_BLOCK_SIZES) def test_cpu_offloading(cpu_block_size: int) -> None: """ diff --git a/vllm/v1/kv_offload/cpu.py b/vllm/v1/kv_offload/cpu.py index 250ed5e95af4b..f765d19ea0175 100644 --- a/vllm/v1/kv_offload/cpu.py +++ b/vllm/v1/kv_offload/cpu.py @@ -51,9 +51,9 @@ class CPUOffloadingSpec(OffloadingSpec): self, kv_caches: dict[str, torch.Tensor] ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]: if not self._handler: - if not current_platform.is_cuda(): + if not current_platform.is_cuda_alike(): raise Exception( - "CPU Offloading is currently only supported on CUDA GPUs" + "CPU Offloading is currently only supported on CUDA-alike GPUs" ) layer_names = list(kv_caches.keys()) From 9956aae4ead0906abe7a1840a503587cab2013c1 Mon Sep 17 00:00:00 2001 From: Fan Yin <1106310035@qq.com> Date: Thu, 30 Oct 2025 22:34:41 +0800 Subject: [PATCH 107/127] [Model][Ouro] Support Ouro Model (#27794) Signed-off-by: yinfan.1024 Signed-off-by: youkaichao Co-authored-by: yinfan.1024 Co-authored-by: youkaichao Co-authored-by: Jee Jee Li --- docs/models/supported_models.md | 1 + tests/models/registry.py | 1 + vllm/model_executor/models/ouro.py | 518 +++++++++++++++++++++++++ vllm/model_executor/models/registry.py | 1 + 4 files changed, 521 insertions(+) create mode 100644 vllm/model_executor/models/ouro.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index c9744d31f0efc..fd25647dce54b 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -403,6 +403,7 @@ th { | `OLMoEForCausalLM` | OLMoE | `allenai/OLMoE-1B-7B-0924`, `allenai/OLMoE-1B-7B-0924-Instruct`, etc. | | ✅︎ | | `OPTForCausalLM` | OPT, OPT-IML | `facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc. | ✅︎ | ✅︎ | | `OrionForCausalLM` | Orion | `OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc. | | ✅︎ | +| `OuroForCausalLM` | ouro | `ByteDance/Ouro-1.4B`, `ByteDance/Ouro-2.6B`, etc. | ✅︎ | | | `PhiForCausalLM` | Phi | `microsoft/phi-1_5`, `microsoft/phi-2`, etc. | ✅︎ | ✅︎ | | `Phi3ForCausalLM` | Phi-4, Phi-3 | `microsoft/Phi-4-mini-instruct`, `microsoft/Phi-4`, `microsoft/Phi-3-mini-4k-instruct`, `microsoft/Phi-3-mini-128k-instruct`, `microsoft/Phi-3-medium-128k-instruct`, etc. | ✅︎ | ✅︎ | | `PhiMoEForCausalLM` | Phi-3.5-MoE | `microsoft/Phi-3.5-MoE-instruct`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index 9a2a1eb5f1a74..7b5977ec58e53 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -369,6 +369,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "OrionForCausalLM": _HfExamplesInfo( "OrionStarAI/Orion-14B-Chat", trust_remote_code=True ), + "OuroForCausalLM": _HfExamplesInfo("ByteDance/Ouro-1.4B", trust_remote_code=True), "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"), "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"), "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"), diff --git a/vllm/model_executor/models/ouro.py b/vllm/model_executor/models/ouro.py new file mode 100644 index 0000000000000..b8dad909c5470 --- /dev/null +++ b/vllm/model_executor/models/ouro.py @@ -0,0 +1,518 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2/modeling_qwen2.py +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Ouro model compatible with HuggingFace weights.""" + +from collections.abc import Iterable +from typing import Any + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionType +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA +from .utils import ( + AutoWeightsLoader, + extract_layer_index, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) + + +class OuroMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + if hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {hidden_act}. Only silu is supported for now." + ) + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class OuroAttention(nn.Module): + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + rope_theta: float = 10000, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + rope_scaling: tuple | None = None, + prefix: str = "", + attn_type: str = AttentionType.DECODER, + dual_chunk_attention_config: dict[str, Any] | None = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.dual_chunk_attention_config = dual_chunk_attention_config + + # Get total_ut_steps from config, default to 4 if not specified + total_ut_steps = getattr(config, "total_ut_steps", 4) + + # Use total number of hidden layers instead of hardcoded 24 + total_layers = config.num_hidden_layers + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=self.rope_theta, + rope_scaling=rope_scaling, + dual_chunk_attention_config=dual_chunk_attention_config, + ) + self.attn = nn.ModuleList() + for ut_step in range(total_ut_steps): + base_layer_idx = extract_layer_index(prefix) + unique_layer_idx = ut_step * total_layers + base_layer_idx + + unique_prefix = prefix.replace( + f"layers.{base_layer_idx}", f"layers.{unique_layer_idx}" + ) + + self.attn.append( + Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + attn_type=attn_type, + prefix=f"{unique_prefix}.attn", + **{ + "layer_idx": unique_layer_idx, + "dual_chunk_attention_config": dual_chunk_attention_config, + } + if dual_chunk_attention_config + else {}, + ) + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + current_ut: int, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn[current_ut](q, k, v) + output, _ = self.o_proj(attn_output) + return output + + +class OuroDecoderLayer(nn.Module): + def __init__( + self, + config: PretrainedConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 1000000) + rope_scaling = getattr(config, "rope_scaling", None) + dual_chunk_attention_config = getattr( + config, "dual_chunk_attention_config", None + ) + + if getattr(config, "is_causal", True): + attn_type = AttentionType.DECODER + else: + attn_type = AttentionType.ENCODER_ONLY + + self.self_attn = OuroAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + cache_config=cache_config, + quant_config=quant_config, + rope_scaling=rope_scaling, + prefix=f"{prefix}.self_attn", + attn_type=attn_type, + dual_chunk_attention_config=dual_chunk_attention_config, + ) + self.mlp = OuroMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.input_layernorm_2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.post_attention_layernorm_2 = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + current_ut: int, + residual: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, hidden_states=hidden_states, current_ut=current_ut + ) + hidden_states = self.input_layernorm_2(hidden_states) + + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_attention_layernorm_2(hidden_states) + + return hidden_states, residual + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + } +) +class OuroModel(nn.Module): + def __init__( + self, + *, + vllm_config: VllmConfig, + prefix: str = "", + decoder_layer_type: type[nn.Module] = OuroDecoderLayer, + ): + super().__init__() + + config = vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + + # TODO (@robertgshaw2): see if this can be moved out + if cache_config.sliding_window is not None and hasattr( + config, "max_window_layers" + ): + assert config.max_window_layers == config.num_hidden_layers, ( + "Sliding window for some but all layers is not supported. " + "This model uses sliding window but `max_window_layers` = {} " + "is less than `num_hidden_layers` = {}. Please open an issue " + "to discuss this feature.".format( + config.max_window_layers, + config.num_hidden_layers, + ) + ) + + self.config = config + self.quant_config = quant_config + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=f"{prefix}.embed_tokens", + ) + + # Use the provided decoder layer type or default to OuroDecoderLayer + decoder_layer_type = decoder_layer_type or OuroDecoderLayer + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: decoder_layer_type( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ), + prefix=f"{prefix}.layers", + ) + + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.early_exit_gate = RowParallelLinear(config.hidden_size, 1, bias=True) + + self.total_ut_steps = getattr(self.config, "total_ut_steps", 4) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + + for current_ut in range(self.total_ut_steps): + residual = None + for layer in self.layers[self.start_layer : self.end_layer]: + hidden_states, residual = layer( + positions, hidden_states, current_ut, residual + ) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters(remove_duplicate=False)) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if self.quant_config is not None and ( + scale_name := self.quant_config.get_cache_scale(name) + ): + # Loading kv cache quantization scales + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + loaded_weight = ( + loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0] + ) + weight_loader(param, loaded_weight) + loaded_params.add(scale_name) + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if name.endswith("scale"): + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + if weight_loader == default_weight_loader: + weight_loader(param, loaded_weight) + else: + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +class OuroForCausalLM(nn.Module, SupportsLoRA): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + lora_config = vllm_config.lora_config + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = OuroModel( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + + self.logits_processor = LogitsProcessor(config.vocab_size) + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + hidden_states = self.model( + input_ids, positions, intermediate_tensors, inputs_embeds + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + logits = self.logits_processor(self.lm_head, hidden_states) + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(weights) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 8e4413c90cf6c..7eca1a09e5365 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -148,6 +148,7 @@ _TEXT_GENERATION_MODELS = { "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"), "OPTForCausalLM": ("opt", "OPTForCausalLM"), "OrionForCausalLM": ("orion", "OrionForCausalLM"), + "OuroForCausalLM": ("ouro", "OuroForCausalLM"), "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"), "PhiForCausalLM": ("phi", "PhiForCausalLM"), "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"), From eebf00cb0c925404672d407674b319ebc5ae3a84 Mon Sep 17 00:00:00 2001 From: "Li, Jiang" Date: Thu, 30 Oct 2025 23:12:05 +0800 Subject: [PATCH 108/127] [Bugfix][CPU] Fix MRoPE dispatch on the CPU backend (#27800) Signed-off-by: jiang1.li --- vllm/model_executor/layers/rotary_embedding/mrope.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vllm/model_executor/layers/rotary_embedding/mrope.py b/vllm/model_executor/layers/rotary_embedding/mrope.py index 3c184ce9d6316..0592aa8f967a6 100644 --- a/vllm/model_executor/layers/rotary_embedding/mrope.py +++ b/vllm/model_executor/layers/rotary_embedding/mrope.py @@ -357,6 +357,15 @@ class MRotaryEmbedding(RotaryEmbeddingBase): key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) return query, key + def forward_cpu( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor | None = None, + offsets: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + return self.forward_native(positions, query, key, offsets) + @staticmethod def get_next_input_positions( mrope_position_delta: int, From e5e076cad7c1c922fa6d48049c45bead505f52a6 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Thu, 30 Oct 2025 11:24:31 -0400 Subject: [PATCH 109/127] [BugFix] Stopgap - Flashinfer Autotuner + GPT-OSS + DP/TP (#27762) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath --- vllm/model_executor/warmup/kernel_warmup.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 79d1927d32103..ffa3bc8f021ef 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING import torch import vllm.envs as envs -from vllm.config import VllmConfig +from vllm.config import CUDAGraphMode, VllmConfig from vllm.logger import init_logger from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup from vllm.platforms import current_platform @@ -30,13 +30,19 @@ def flashinfer_autotune_supported(vllm_config: VllmConfig) -> bool: Record known issues with vllm + flashinfer autotune here. Return True if and only if flashinfer autotune will run through without issues. """ - return not ( - vllm_config.parallel_config.data_parallel_size > 1 - and ( - envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16 - or envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8 - ) + is_tp_or_dp = (vllm_config.parallel_config.data_parallel_size > 1) or ( + vllm_config.parallel_config.tensor_parallel_size > 1 ) + is_fi_mxfp4_backend = ( + envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8 + or envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16 + or envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS + ) or ( + current_platform.is_cuda() and current_platform.is_device_capability(100) + ) # on >=sm100, default mxfp4 backend is flashinfer + is_eager = vllm_config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE + + return not (is_tp_or_dp and is_fi_mxfp4_backend and is_eager) def kernel_warmup(worker: "Worker"): From 60f76baa6688ce265a4205f183bd42a62d8f7179 Mon Sep 17 00:00:00 2001 From: Ilya Markov Date: Thu, 30 Oct 2025 16:41:44 +0100 Subject: [PATCH 110/127] [Misc] Replace CUDA_VISIBLE_DEVICES in DP with torch.cuda.set_device for device selection on cuda-like devices (#27564) Signed-off-by: ilmarkov Co-authored-by: Tyler Michael Smith --- .../kv_connector/v1/nixl_connector.py | 12 ++++++---- vllm/v1/engine/utils.py | 11 ++++++++- vllm/v1/worker/dp_utils.py | 4 ++-- vllm/v1/worker/gpu_worker.py | 23 +++++++++++++++++++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py index 275a8c734058b..d5712bdd9feb4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py @@ -1008,11 +1008,14 @@ class NixlConnectorWorker: # Enable different block lengths for different layers when MLA is used. self.block_len_per_layer = list[int]() self.slot_size_per_layer = list[int]() # HD bytes in kv terms + self.device_id = self.tp_rank for layer_name, cache_or_caches in xfer_buffers.items(): cache_list = cache_or_caches if split_k_and_v else [cache_or_caches] for cache in cache_list: base_addr = cache.data_ptr() + if not self.use_host_buffer and current_platform.is_cuda_alike(): + self.device_id = cache.device.index if base_addr in seen_base_addresses: continue @@ -1040,7 +1043,7 @@ class NixlConnectorWorker: "All kv cache tensors must have the same size" ) caches_data.append( - (base_addr, curr_tensor_size_bytes, self.tp_rank, "") + (base_addr, curr_tensor_size_bytes, self.device_id, "") ) logger.debug( @@ -1087,7 +1090,7 @@ class NixlConnectorWorker: block_offset = block_id * self.block_len_per_layer[i] addr = base_addr + block_offset # (addr, len, device id) - blocks_data.append((addr, kv_block_len, self.tp_rank)) + blocks_data.append((addr, kv_block_len, self.device_id)) if self._use_flashinfer: # Separate and interleave K/V regions to maintain the same @@ -1098,12 +1101,13 @@ class NixlConnectorWorker: addr = base_addr + block_offset # Register addresses for V cache (K registered first). v_addr = addr + kv_block_len - blocks_data.append((v_addr, kv_block_len, self.tp_rank)) + blocks_data.append((v_addr, kv_block_len, self.device_id)) logger.debug( - "Created %s blocks for src engine %s and rank %s", + "Created %s blocks for src engine %s and rank %s on device id %s", len(blocks_data), self.engine_id, self.tp_rank, + self.device_id, ) descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type) diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index bdc124b0571c0..e74519b21aa6e 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -134,9 +134,18 @@ class CoreEngineProcManager: data_parallel = vllm_config.parallel_config.data_parallel_size > 1 try: for proc, local_dp_rank in zip(self.processes, local_dp_ranks): + # Adjust device control in DP for non-CUDA platforms + # as well as external and ray launchers + # For CUDA platforms, we use torch.cuda.set_device() with ( set_device_control_env_var(vllm_config, local_dp_rank) - if (data_parallel) + if ( + data_parallel + and ( + not current_platform.is_cuda_alike() + or vllm_config.parallel_config.use_ray + ) + ) else contextlib.nullcontext() ): proc.start() diff --git a/vllm/v1/worker/dp_utils.py b/vllm/v1/worker/dp_utils.py index 2b2a69f4af3ab..464fbf11a21ad 100644 --- a/vllm/v1/worker/dp_utils.py +++ b/vllm/v1/worker/dp_utils.py @@ -8,7 +8,6 @@ import torch.distributed as dist from vllm.config import ParallelConfig from vllm.distributed.parallel_state import get_dp_group from vllm.logger import init_logger -from vllm.platforms import current_platform from vllm.v1.worker.ubatch_utils import ( UBatchSlices, check_ubatch_thresholds, @@ -20,7 +19,8 @@ logger = init_logger(__name__) def _get_device_and_group(parallel_config: ParallelConfig): - device = current_platform.device_type + # Use the actual device assigned to the DP group, not just the device type + device = get_dp_group().device group = get_dp_group().device_group # Transfering this tensor from GPU to CPU will introduce a GPU sync diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 29b6532e4366f..54c5f81fc7e8e 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -172,6 +172,29 @@ class Worker(WorkerBase): if self.device_config.device.type == "cuda": # This env var set by Ray causes exceptions with graph building. os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None) + if ( + self.parallel_config.data_parallel_size > 1 + and self.parallel_config.data_parallel_size_local > 0 + and self.parallel_config.distributed_executor_backend + not in ["ray", "external_launcher"] + and self.vllm_config.parallel_config.data_parallel_backend != "ray" + ): + # Use local DP rank if available, otherwise use global DP rank. + dp_local_rank = self.parallel_config.data_parallel_rank_local + if dp_local_rank is None: + dp_local_rank = self.parallel_config.data_parallel_rank + + tp_pp_world_size = ( + self.parallel_config.pipeline_parallel_size + * self.parallel_config.tensor_parallel_size + ) + + # DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK + self.local_rank += dp_local_rank * tp_pp_world_size + assert self.local_rank < torch.cuda.device_count(), ( + f"DP adjusted local rank {self.local_rank} is out of bounds. " + ) + self.device = torch.device(f"cuda:{self.local_rank}") current_platform.set_device(self.device) From 33a0ea5f3264b5b2f571b8a53357e10efcc94670 Mon Sep 17 00:00:00 2001 From: Kebe Date: Fri, 31 Oct 2025 01:33:13 +0900 Subject: [PATCH 111/127] [Docs] add Shanghai Meetup - 2025/10 (#27545) Signed-off-by: Kebe Signed-off-by: esmeetu Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: esmeetu --- README.md | 1 + docs/community/meetups.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 3dcdd7dc00942..2e750ef8fc894 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Join us at the [PyTorch Conference, October 22-23](https://events.linuxfoundatio *Latest News* 🔥 +- [2025/10] We hosted [vLLM Shanghai Meetup](https://mp.weixin.qq.com/s/__xb4OyOsImz-9eAVrdlcg) focused on hands-on vLLM inference optimization! Please find the meetup slides [here](https://drive.google.com/drive/folders/1KqwjsFJLfEsC8wlDugnrR61zsWHt94Q6). - [2025/09] We hosted [vLLM Toronto Meetup](https://luma.com/e80e0ymm) focused on tackling inference at scale and speculative decoding with speakers from NVIDIA and Red Hat! Please find the meetup slides [here](https://docs.google.com/presentation/d/1IYJYmJcu9fLpID5N5RbW_vO0XLo0CGOR14IXOjB61V8/edit?usp=sharing). - [2025/08] We hosted [vLLM Shenzhen Meetup](https://mp.weixin.qq.com/s/k8ZBO1u2_2odgiKWH_GVTQ) focusing on the ecosystem around vLLM! Please find the meetup slides [here](https://drive.google.com/drive/folders/1Ua2SVKVSu-wp5vou_6ElraDt2bnKhiEA). - [2025/08] We hosted [vLLM Singapore Meetup](https://www.sginnovate.com/event/vllm-sg-meet). We shared V1 updates, disaggregated serving and MLLM speedups with speakers from Embedded LLM, AMD, WekaIO, and A*STAR. Please find the meetup slides [here](https://drive.google.com/drive/folders/1ncf3GyqLdqFaB6IeB834E5TZJPLAOiXZ?usp=sharing). diff --git a/docs/community/meetups.md b/docs/community/meetups.md index e821e2ac81149..0dfc582c7f8a7 100644 --- a/docs/community/meetups.md +++ b/docs/community/meetups.md @@ -2,6 +2,7 @@ We host regular meetups in San Francisco Bay Area every 2 months. We will share the project updates from the vLLM team and have guest speakers from the industry to share their experience and insights. Please find the materials of our previous meetups below: +- [vLLM Shanghai Meetup](https://mp.weixin.qq.com/s/__xb4OyOsImz-9eAVrdlcg), October 25th 2025. [[Slides]](https://drive.google.com/drive/folders/1KqwjsFJLfEsC8wlDugnrR61zsWHt94Q6) - [vLLM Toronto Meetup](https://luma.com/e80e0ymm), September 25th 2025. [[Slides]](https://docs.google.com/presentation/d/1IYJYmJcu9fLpID5N5RbW_vO0XLo0CGOR14IXOjB61V8/edit?usp=sharing) - [vLLM Shenzhen Meetup](https://mp.weixin.qq.com/s/k8ZBO1u2_2odgiKWH_GVTQ), August 30th 2025. [[Slides]](https://drive.google.com/drive/folders/1Ua2SVKVSu-wp5vou_6ElraDt2bnKhiEA) - [vLLM Singapore Meetup](https://www.sginnovate.com/event/vllm-sg-meet), August 27th 2025. [[Slides]](https://drive.google.com/drive/folders/1ncf3GyqLdqFaB6IeB834E5TZJPLAOiXZ?usp=sharing) From ba33e8830dceb32e9b03508bbff435e3082759b8 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 30 Oct 2025 10:22:30 -0700 Subject: [PATCH 112/127] Reapply "Install pre-built xformers-0.0.32.post2 built with pt-2.9.0" (#27768) Signed-off-by: Huy Do --- docker/Dockerfile | 7 ------- requirements/cuda.txt | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 42a830cb605ad..61ebf970fe960 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -361,13 +361,6 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist && uv pip install --system dist/*.whl --verbose \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') -# TODO (huydhn): Remove this once xformers is released for 2.9.0 -RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH' - . /etc/environment - export TORCH_CUDA_ARCH_LIST='7.5 8.0+PTX 9.0a' - uv pip install --system --no-build-isolation "git+https://github.com/facebookresearch/xformers@v0.0.32.post2" -BASH - # Install FlashInfer pre-compiled kernel cache and binaries # https://docs.flashinfer.ai/installation.html RUN --mount=type=cache,target=/root/.cache/uv \ diff --git a/requirements/cuda.txt b/requirements/cuda.txt index dd45eb832a96a..5f7d520cd3662 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -9,7 +9,7 @@ torch==2.9.0 torchaudio==2.9.0 # These must be updated alongside torch torchvision==0.24.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version -# https://github.com/facebookresearch/xformers/releases/tag/v0.0.32.post1 -# xformers==0.0.32.post1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.8 +# Build from https://github.com/facebookresearch/xformers/releases/tag/v0.0.32.post1 +xformers==0.0.33+5d4b92a5.d20251029; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.9 # FlashInfer should be updated together with the Dockerfile flashinfer-python==0.4.1 From 10042057953cd1528701234925de3d7b109e26de Mon Sep 17 00:00:00 2001 From: Mengqing Cao Date: Fri, 31 Oct 2025 01:27:39 +0800 Subject: [PATCH 113/127] [MTP] Refactor mtp predictor to avoid d2h operation (#27643) Signed-off-by: MengqingCao --- vllm/model_executor/models/deepseek_mtp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index aa176ef05fccb..3984d23970ac5 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -97,7 +97,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module): ) -> torch.Tensor: assert inputs_embeds is not None # masking inputs at position 0, as not needed by MTP - inputs_embeds[positions == 0] = 0 + inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds) inputs_embeds = self.enorm(inputs_embeds) previous_hidden_states = self.hnorm(previous_hidden_states) From 2918c1b49c88c29783c86f78d2c4221cb9622379 Mon Sep 17 00:00:00 2001 From: Roger Meier Date: Fri, 31 Oct 2025 01:36:56 +0800 Subject: [PATCH 114/127] [Model] Use the same fused_moe configs for all H200 devices (#23642) Signed-off-by: Roger Meier --- vllm/model_executor/layers/fused_moe/fused_moe.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 5f9bfd6d9cf7d..d0f5eb498127b 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -818,6 +818,9 @@ def get_config_file_name( E: int, N: int, dtype: str | None, block_shape: list[int] | None = None ) -> str: device_name = current_platform.get_device_name().replace(" ", "_") + # Set device_name to H200 if a device from the H200 family is detected + if "H200" in device_name: + device_name = "H200" dtype_selector = "" if not dtype else f",dtype={dtype}" block_shape_selector = ( "" if not block_shape or not all(block_shape) else f",block_shape={block_shape}" From ab98f6556ff84508cdcdcd6a6b1e612a7a8819d0 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Thu, 30 Oct 2025 14:52:18 -0400 Subject: [PATCH 115/127] [Bugfix] Fix 2 precommit issues - (mamba_block_size, kv_cache_config) (#27811) Signed-off-by: Tyler Michael Smith Signed-off-by: Tyler Michael Smith Co-authored-by: Nick Hill --- vllm/model_executor/models/config.py | 2 +- vllm/v1/core/sched/scheduler.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index b0a48a9f1d458..7150977e9266b 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -406,7 +406,7 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): # easily by changing the way we layout chunks in the # mamba2 kernels. - base_chunk_size = model_config.get_mamba_chunk_size() + base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size() attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token) chunk_size = lcm(base_chunk_size, kernel_block_alignment_size) attn_block_size = chunk_size * cdiv(attn_tokens_per_mamba_state, chunk_size) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ad6fbee2ec083..98c8f08b0aae8 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -13,7 +13,7 @@ from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.distributed.kv_transfer.kv_connector.v1 import ( KVConnectorBase_V1, KVConnectorRole, - supports_hma, + SupportsHMA, ) from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats from vllm.logger import init_logger @@ -93,7 +93,11 @@ class Scheduler(SchedulerInterface): ) connector_vllm_config = copy.copy(self.vllm_config) - connector_vllm_config.kv_cache_config = copy.copy(kv_cache_config) + + # We're dynamically inserting a kv_cache_config variable into the + # connector_vllm_config. This is distinct from the cache_config + # that is already in there. + connector_vllm_config.kv_cache_config = copy.copy(kv_cache_config) # type: ignore[attr-defined] self.connector = KVConnectorFactory.create_connector( config=connector_vllm_config, role=KVConnectorRole.SCHEDULER ) @@ -1327,15 +1331,15 @@ class Scheduler(SchedulerInterface): block_ids = self.kv_cache_manager.get_block_ids(request.request_id) - if not supports_hma(self.connector): + if not isinstance(self.connector, SupportsHMA): # NOTE(Kuntai): We should deprecate this code path after we enforce # all connectors to support HMA. # Hybrid memory allocator should be already turned off for this # code path, but let's double-check here. assert len(self.kv_cache_config.kv_cache_groups) == 1 return self.connector.request_finished(request, block_ids[0]) - else: - return self.connector.request_finished(request, block_ids) + + return self.connector.request_finished_all_groups(request, block_ids) def _update_waiting_for_remote_kv(self, request: Request) -> bool: """ From 4574d48bab9c4e38b7c0a830eeefc8f0980e8c58 Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Thu, 30 Oct 2025 11:52:36 -0700 Subject: [PATCH 116/127] [Core][Bookkeeping] Update cu_num_accepted_tokens for all req_index (#27629) Signed-off-by: Jialin Ouyang --- vllm/v1/worker/gpu_model_runner.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 729ce462cf186..04814b5991ebc 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2323,11 +2323,19 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): sampled_ids = [-1] if req_idx not in invalid_req_indices_set else None else: sampled_ids = valid_sampled_token_ids[req_idx] + + num_sampled_ids: int = len(sampled_ids) if sampled_ids else 0 + + if cu_num_accepted_tokens is not None: + cu_num_accepted_tokens.append( + cu_num_accepted_tokens[-1] + num_sampled_ids + ) + if not sampled_ids: continue start_idx = self.input_batch.num_tokens_no_spec[req_idx] - end_idx = start_idx + len(sampled_ids) + end_idx = start_idx + num_sampled_ids assert end_idx <= self.max_model_len, ( "Sampled token IDs exceed the max model length. " f"Total number of tokens: {end_idx} > max_model_len: " @@ -2343,11 +2351,6 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): req_state = self.requests[req_id] req_state.output_token_ids.extend(sampled_ids) - if cu_num_accepted_tokens is not None: - cu_num_accepted_tokens.append( - cu_num_accepted_tokens[-1] + len(sampled_ids) - ) - logprobs_lists = ( logprobs_tensors.tolists(cu_num_accepted_tokens) if not self.use_async_scheduling and logprobs_tensors is not None From a2981c42720a34b5abf59c4c14df701f8105d4cd Mon Sep 17 00:00:00 2001 From: cong-meta Date: Thu, 30 Oct 2025 12:10:16 -0700 Subject: [PATCH 117/127] [EP/DP][API Server] Enable DP-aware routing in OpenAI API requests (#24945) Co-authored-by: Cong Chen --- tests/entrypoints/openai/test_serving_chat.py | 76 +++++++++++++++++++ vllm/entrypoints/openai/serving_chat.py | 4 + vllm/entrypoints/openai/serving_completion.py | 4 + vllm/entrypoints/openai/serving_engine.py | 15 ++++ 4 files changed, 99 insertions(+) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index d1367b4eeaf62..1b83ed7e31e78 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -651,3 +651,79 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type): await serving_chat.create_chat_completion(req) engine_prompt = serving_chat._process_inputs.await_args_list[1].args[1] assert engine_prompt.get("cache_salt") == "test_salt" + + +@pytest.mark.asyncio +async def test_serving_chat_data_parallel_rank_extraction(): + """Test that data_parallel_rank is properly extracted from header and + passed to engine.""" + mock_engine = MagicMock(spec=AsyncLLM) + mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) + mock_engine.errored = False + mock_engine.model_config = MockModelConfig() + mock_engine.processor = MagicMock() + mock_engine.io_processor = MagicMock() + + # Mock the generate method to return an async generator + async def mock_generate(*args, **kwargs): + # Yield a fake RequestOutput + from vllm.outputs import CompletionOutput, RequestOutput + + yield RequestOutput( + request_id="test-request", + prompt="test prompt", + prompt_token_ids=[1, 2, 3], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text="test response", + token_ids=[4, 5, 6], + cumulative_logprob=0.0, + logprobs=None, + finish_reason="stop", + stop_reason=None, + ) + ], + finished=True, + ) + + mock_engine.generate = AsyncMock(side_effect=mock_generate) + + serving_chat = _build_serving_chat(mock_engine) + + # Test when data_parallel_rank is present in header + req = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "what is 1+1?"}], + ) + + # Mock request with X-data-parallel-rank header + mock_raw_request = MagicMock() + mock_raw_request.headers = {"X-data-parallel-rank": "2"} + mock_raw_request.state = MagicMock() + + with suppress(Exception): + await serving_chat.create_chat_completion(req, mock_raw_request) + + # Verify that data_parallel_rank was passed to engine.generate + assert "data_parallel_rank" in mock_engine.generate.call_args.kwargs + assert mock_engine.generate.call_args.kwargs["data_parallel_rank"] == 2 + + # Test when data_parallel_rank is not present (defaults to None) + req_no_dp = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "what is 2+2?"}], + ) + + # Mock request with no header + mock_raw_request_no_dp = MagicMock() + mock_raw_request_no_dp.headers = {} + mock_raw_request_no_dp.state = MagicMock() + + with suppress(Exception): + await serving_chat.create_chat_completion(req_no_dp, mock_raw_request_no_dp) + + # Verify that data_parallel_rank defaults to None + assert "data_parallel_rank" in mock_engine.generate.call_args.kwargs + assert mock_engine.generate.call_args.kwargs["data_parallel_rank"] is None diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 934ff78b2c710..bb770ecf03383 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -264,6 +264,9 @@ class OpenAIServingChat(OpenAIServing): if raw_request: raw_request.state.request_metadata = request_metadata + # Extract data_parallel_rank from header (router can inject it) + data_parallel_rank = self._get_data_parallel_rank(raw_request) + # Schedule the request and get the result generator. generators: list[AsyncGenerator[RequestOutput, None]] = [] try: @@ -331,6 +334,7 @@ class OpenAIServingChat(OpenAIServing): priority=request.priority, prompt_text=prompt_text, tokenization_kwargs=tokenization_kwargs, + data_parallel_rank=data_parallel_rank, ) generators.append(generator) diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index 62bc932f8b844..14dbdd4cb4c7c 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -141,6 +141,9 @@ class OpenAIServingCompletion(OpenAIServing): logger.exception("Error in preprocessing prompt inputs") return self.create_error_response(str(e)) + # Extract data_parallel_rank from header (router can inject it) + data_parallel_rank = self._get_data_parallel_rank(raw_request) + # Schedule the request and get the result generator. generators: list[AsyncGenerator[RequestOutput, None]] = [] try: @@ -224,6 +227,7 @@ class OpenAIServingCompletion(OpenAIServing): priority=request.priority, prompt_text=prompt_text, tokenization_kwargs=tokenization_kwargs, + data_parallel_rank=data_parallel_rank, ) generators.append(generator) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index af5a423134fb0..c0750cd641667 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1298,6 +1298,21 @@ class OpenAIServing: return raw_request.headers.get("X-Request-Id", default) + @staticmethod + def _get_data_parallel_rank(raw_request: Request | None) -> int | None: + """Pulls the data parallel rank from a header, if provided""" + if raw_request is None: + return None + + rank_str = raw_request.headers.get("X-data-parallel-rank") + if rank_str is None: + return None + + try: + return int(rank_str) + except ValueError: + return None + @staticmethod def _get_decoded_token( logprob: Logprob, From 4917002523db90813a47ca5aed5cd22e2edb75f4 Mon Sep 17 00:00:00 2001 From: Sumanth R Hegde <39546518+SumanthRH@users.noreply.github.com> Date: Thu, 30 Oct 2025 12:26:27 -0700 Subject: [PATCH 118/127] [Fix] Skip `record_sleep_state` logic in `PrometheusStatsLogger` if not in dev mode (#27789) Signed-off-by: SumanthRH --- tests/basic_correctness/test_cumem.py | 43 ++++++++++++++++++++++++++- vllm/v1/metrics/loggers.py | 3 ++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/basic_correctness/test_cumem.py b/tests/basic_correctness/test_cumem.py index 09f4ec03fbbb0..0c037622f5e82 100644 --- a/tests/basic_correctness/test_cumem.py +++ b/tests/basic_correctness/test_cumem.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio + import pytest import torch -from vllm import LLM, SamplingParams +from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams from vllm.device_allocator.cumem import CuMemAllocator from vllm.utils.mem_constants import GiB_bytes @@ -201,3 +203,42 @@ def test_deep_sleep(): # cmp output assert output[0].outputs[0].text == output2[0].outputs[0].text + + +@create_new_process_for_each_test() +def test_deep_sleep_async(): + async def test(): + model = "hmellor/tiny-random-LlamaForCausalLM" + free, total = torch.cuda.mem_get_info() + used_bytes_baseline = total - free # in case other process is running + engine_args = AsyncEngineArgs( + model=model, + enable_sleep_mode=True, + ) + + llm = AsyncLLMEngine.from_engine_args(engine_args) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + outputs = llm.generate(prompt, sampling_params, request_id="test_request_id1") + async for output in outputs: + pass + + # Put the engine to deep sleep + await llm.sleep(level=2) + + await llm.wake_up(tags=["weights"]) + await llm.collective_rpc("reload_weights") + free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info() + used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline + assert used_bytes < 4 * GiB_bytes + + # now allocate kv cache and cuda graph memory + await llm.wake_up(tags=["kv_cache"]) + outputs2 = llm.generate(prompt, sampling_params, request_id="test_request_id2") + async for output2 in outputs2: + pass + + # cmp output + assert output.outputs[0].text == output2.outputs[0].text + + asyncio.run(test()) diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 3772f07066a12..67b6ceaa847f6 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -1052,6 +1052,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase): self.gauge_lora_info.labels(**lora_info_labels).set_to_current_time() def record_sleep_state(self, sleep: int = 0, level: int = 0): + if not envs.VLLM_SERVER_DEV_MODE: + return + awake = 1 discard_all = 0 weights_offloaded = 0 From a8141fa649d1296488cc5de2b479fed460bb34f4 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 30 Oct 2025 15:32:39 -0400 Subject: [PATCH 119/127] [Refactor] Remove `VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK` (#27750) Signed-off-by: yewentao256 --- vllm/distributed/device_communicators/all2all.py | 2 +- vllm/envs.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 013ef3c1f5c36..c40dde26b741f 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -363,7 +363,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase): num_rdma_bytes=num_rdma_bytes, low_latency_mode=True, num_qps_per_rank=num_qps_per_rank, - allow_nvlink_for_low_latency_mode=envs.VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK, + allow_nvlink_for_low_latency_mode=True, allow_mnnvl=envs.VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL, ) diff --git a/vllm/envs.py b/vllm/envs.py index 0548f01fc8cdf..2744335ed3d38 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -207,7 +207,6 @@ if TYPE_CHECKING: VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME: str = "VLLM_OBJECT_STORAGE_SHM_BUFFER" VLLM_DEEPEP_BUFFER_SIZE_MB: int = 1024 VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE: bool = False - VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK: bool = True VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL: bool = False VLLM_DBO_COMM_SMS: int = 20 VLLM_PATTERN_MATCH_DEBUG: str | None = None @@ -1400,11 +1399,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE": lambda: bool( int(os.getenv("VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", "0")) ), - # Allow DeepEP to use nvlink for internode_ll kernel, turn this on for - # better latency on GB200 like system - "VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK": lambda: bool( - int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK", "1")) - ), # Allow DeepEP to use MNNVL (multi-node nvlink) for internode_ll kernel, # turn this for better latency on GB200 like system "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool( @@ -1566,7 +1560,6 @@ def compute_hash() -> str: "VLLM_NVFP4_GEMM_BACKEND", "VLLM_USE_FBGEMM", "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", - "VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK", "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", ] for key in environment_variables_to_hash: From 4b68c4a55b0fa5846d180532ae7e58db85101e07 Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Thu, 30 Oct 2025 12:47:30 -0700 Subject: [PATCH 120/127] [Core][Perf] Only invoke save_new_computed_blocks when computed blocks are not empty (#27799) Signed-off-by: Jialin Ouyang --- vllm/v1/core/kv_cache_manager.py | 11 ++++++----- vllm/v1/core/single_type_kv_cache_manager.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index bb8cec91f36dd..63a1ff06e4049 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -306,11 +306,12 @@ class KVCacheManager: "Computed blocks should be empty when prefix caching is disabled" ) - # Append the new computed blocks to the request blocks until now to - # avoid the case where the new blocks cannot be allocated. - self.coordinator.save_new_computed_blocks( - request.request_id, new_computed_block_list - ) + if new_computed_block_list is not self.empty_kv_cache_blocks.blocks: + # Append the new computed blocks to the request blocks until now to + # avoid the case where the new blocks cannot be allocated. + self.coordinator.save_new_computed_blocks( + request.request_id, new_computed_block_list + ) new_blocks = self.coordinator.allocate_new_blocks( request.request_id, num_tokens_need_slot, num_encoder_tokens diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 575ae3d7d83b6..8f14fb1894707 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -151,7 +151,7 @@ class SingleTypeKVCacheManager(ABC): num_tokens: The total number of tokens that need to be cached (including tokens that are already cached). """ - num_cached_blocks = self.num_cached_block[request.request_id] + num_cached_blocks = self.num_cached_block.get(request.request_id, 0) num_full_blocks = num_tokens // self.block_size if num_cached_blocks >= num_full_blocks: From e7acb200766a0f8f006f9b6fd961dfdceabd7269 Mon Sep 17 00:00:00 2001 From: Paul Zhang Date: Thu, 30 Oct 2025 16:11:29 -0400 Subject: [PATCH 121/127] [Feature] Batch invariant torch.compile (#27660) Signed-off-by: PaulZhang12 Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- vllm/config/model.py | 7 -- vllm/envs.py | 8 ++- vllm/model_executor/layers/batch_invariant.py | 71 +++++++++++++++++++ .../model_executor/layers/quantization/fp8.py | 5 +- 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index 092c67e7bed8c..082f90653f5af 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -20,9 +20,6 @@ from vllm.config.pooler import PoolerConfig from vllm.config.scheduler import RunnerType from vllm.config.utils import assert_hashable, config, getattr_iter from vllm.logger import init_logger -from vllm.model_executor.layers.batch_invariant import ( - vllm_is_batch_invariant, -) from vllm.platforms import current_platform from vllm.transformers_utils.config import ( ConfigFormat, @@ -436,10 +433,6 @@ class ModelConfig: skip_mm_profiling: bool | None, video_pruning_rate: float | None, ) -> None: - # Enable batch invariance settings if requested - if vllm_is_batch_invariant(): - self.enforce_eager = True - # Set the default seed to 0 in V1. # NOTE(woosuk): In V1, we use separate processes for workers (unless # VLLM_ENABLE_V1_MULTIPROCESSING=0), so setting a seed here diff --git a/vllm/envs.py b/vllm/envs.py index 2744335ed3d38..21237c70a45e4 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -251,6 +251,9 @@ def disable_compile_cache() -> bool: def use_aot_compile() -> bool: + from vllm.model_executor.layers.batch_invariant import ( + vllm_is_batch_invariant, + ) from vllm.utils.torch_utils import is_torch_equal_or_newer default_value = ( @@ -259,7 +262,10 @@ def use_aot_compile() -> bool: else "0" ) - return os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1" + return ( + not vllm_is_batch_invariant() + and os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1" + ) def env_with_choices( diff --git a/vllm/model_executor/layers/batch_invariant.py b/vllm/model_executor/layers/batch_invariant.py index 5706786bccb1d..39e77b935d3d5 100644 --- a/vllm/model_executor/layers/batch_invariant.py +++ b/vllm/model_executor/layers/batch_invariant.py @@ -11,6 +11,7 @@ import torch import vllm.envs as envs from vllm.logger import init_logger from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import is_torch_equal_or_newer logger = init_logger(__name__) @@ -716,6 +717,10 @@ def linear_batch_invariant(input, weight, bias=None): _batch_invariant_MODE = False _batch_invariant_LIB = None _original_torch_bmm = None +_original_fp16_reduction_precision = None +_original_bf16_reduction_precision = None +_original_cublas_workspace_cfg = None +_original_cublaslt_workspace_size = None def is_batch_invariant_mode_enabled(): @@ -724,6 +729,8 @@ def is_batch_invariant_mode_enabled(): def enable_batch_invariant_mode(): global _batch_invariant_MODE, _batch_invariant_LIB, _original_torch_bmm + global _original_fp16_reduction_precision, _original_bf16_reduction_precision + global _original_cublas_workspace_cfg, _original_cublaslt_workspace_size if _batch_invariant_MODE: return @@ -745,14 +752,75 @@ def enable_batch_invariant_mode(): _original_torch_bmm = torch.bmm torch.bmm = bmm_batch_invariant + _original_bf16_reduction_precision = ( + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction + ) + _original_fp16_reduction_precision = ( + torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction + ) + + reduced_precision_val = ( + (False, False) if is_torch_equal_or_newer("2.10.0.dev") else False + ) + torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = ( + reduced_precision_val + ) + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = ( + reduced_precision_val + ) + torch.backends.cuda.preferred_blas_library(backend="cublaslt") + + if not is_torch_equal_or_newer("2.10.0.dev"): + _original_cublas_workspace_cfg = os.environ.get("CUBLAS_WORKSPACE_CONFIG", None) + _original_cublaslt_workspace_size = os.environ.get( + "CUBLASLT_WORKSPACE_SIZE", None + ) + os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8" + os.environ["CUBLASLT_WORKSPACE_SIZE"] = "1" + def disable_batch_invariant_mode(): global _batch_invariant_MODE, _batch_invariant_LIB, _original_torch_bmm + global _original_fp16_reduction_precision, _original_bf16_reduction_precision + global _original_cublas_workspace_cfg, _original_cublaslt_workspace_size + if not _batch_invariant_MODE: + return + if _batch_invariant_LIB is not None: _batch_invariant_LIB._destroy() if _original_torch_bmm is not None: torch.bmm = _original_torch_bmm _original_torch_bmm = None + + if _original_bf16_reduction_precision is not None: + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = ( + _original_bf16_reduction_precision + ) + _original_bf16_reduction_precision = None + if _original_fp16_reduction_precision is not None: + torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = ( + _original_fp16_reduction_precision + ) + _original_fp16_reduction_precision = None + + torch.backends.cuda.preferred_blas_library(backend="default") + + if not is_torch_equal_or_newer("2.10.0.dev"): + # Set cublas env vars to previous results. If previous results are None, + # that means the env vars were not set, so we should remove them. + if _original_cublas_workspace_cfg: + os.environ["CUBLAS_WORKSPACE_CONFIG"] = _original_cublas_workspace_cfg + elif "CUBLAS_WORKSPACE_CONFIG" in os.environ: + del os.environ["CUBLAS_WORKSPACE_CONFIG"] + + if _original_cublaslt_workspace_size: + os.environ["CUBLASLT_WORKSPACE_SIZE"] = _original_cublaslt_workspace_size + elif "CUBLASLT_WORKSPACE_SIZE" in os.environ: + del os.environ["CUBLASLT_WORKSPACE_SIZE"] + + _original_cublas_workspace_cfg = None + _original_cublaslt_workspace_size = None + _batch_invariant_MODE = False _batch_invariant_LIB = None @@ -831,6 +899,9 @@ def override_envs_for_invariance(): os.environ["NCCL_NTHREADS"] = "1" os.environ["NCCL_SOCKET_NTHREADS"] = "1" + # torch.compile settings + os.environ["VLLM_USE_AOT_COMPILE"] = "0" + def init_batch_invariance(): # this will hit all the csrc overrides as well diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index e5681cb856258..f82eccb88ce09 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -363,6 +363,7 @@ class Fp8LinearMethod(LinearMethodBase): self.use_marlin = False self.use_aiter_and_is_supported = check_aiter_fp8_linear_support() + self.use_deep_gemm = is_deep_gemm_supported() self.weight_block_size = self.quant_config.weight_block_size self.block_quant = self.weight_block_size is not None @@ -545,8 +546,10 @@ class Fp8LinearMethod(LinearMethodBase): # if batch invariant mode is enabled, prefer DeepGEMM FP8 path # we will use BF16 dequant when DeepGEMM is not supported. if vllm_is_batch_invariant(): + # Call is_deep_gemm_supported() ahead of time for torch.compile + # dynamo has trouble tracing through if self.block_quant and should_use_deepgemm_for_fp8_linear( - torch.bfloat16, layer.weight, None + torch.bfloat16, layer.weight, self.use_deep_gemm ): # use group quant consistent with block size across K assert self.act_q_group_shape is not None From c9791f18138d1a11bfe68550b10673b493ec9330 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 30 Oct 2025 16:26:13 -0700 Subject: [PATCH 122/127] [BugFix] Fix broken import in initialize_ray_cluster() (#27838) Signed-off-by: Nick Hill --- vllm/v1/executor/ray_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/v1/executor/ray_utils.py b/vllm/v1/executor/ray_utils.py index 382f008266e62..9385e55b066f8 100644 --- a/vllm/v1/executor/ray_utils.py +++ b/vllm/v1/executor/ray_utils.py @@ -322,7 +322,7 @@ def initialize_ray_cluster( # Prevalidate GPU requirements before Ray processing if current_platform.is_cuda() and parallel_config.world_size > 1: - from vllm.utils import cuda_device_count_stateless + from vllm.utils.torch_utils import cuda_device_count_stateless available_gpus = cuda_device_count_stateless() if parallel_config.world_size > available_gpus: From d5d2a0fe7480fa23348ec253cb5c80901d27f952 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Thu, 30 Oct 2025 19:46:02 -0400 Subject: [PATCH 123/127] [Misc] Make all tool scripts executable (#27831) Signed-off-by: Matthew Bonanni --- tools/check_repo.sh | 0 tools/ep_kernels/configure_system_drivers.sh | 0 tools/ep_kernels/elastic_ep/install_eep_libraries.sh | 0 tools/ep_kernels/install_python_libraries.sh | 1 + tools/flashinfer-build.sh | 0 tools/vllm-tpu/build.sh | 0 6 files changed, 1 insertion(+) mode change 100644 => 100755 tools/check_repo.sh mode change 100644 => 100755 tools/ep_kernels/configure_system_drivers.sh mode change 100644 => 100755 tools/ep_kernels/elastic_ep/install_eep_libraries.sh mode change 100644 => 100755 tools/ep_kernels/install_python_libraries.sh mode change 100644 => 100755 tools/flashinfer-build.sh mode change 100644 => 100755 tools/vllm-tpu/build.sh diff --git a/tools/check_repo.sh b/tools/check_repo.sh old mode 100644 new mode 100755 diff --git a/tools/ep_kernels/configure_system_drivers.sh b/tools/ep_kernels/configure_system_drivers.sh old mode 100644 new mode 100755 diff --git a/tools/ep_kernels/elastic_ep/install_eep_libraries.sh b/tools/ep_kernels/elastic_ep/install_eep_libraries.sh old mode 100644 new mode 100755 diff --git a/tools/ep_kernels/install_python_libraries.sh b/tools/ep_kernels/install_python_libraries.sh old mode 100644 new mode 100755 index c2d8d1ed9e3d5..5ea543f4cb1e8 --- a/tools/ep_kernels/install_python_libraries.sh +++ b/tools/ep_kernels/install_python_libraries.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash set -ex # prepare workspace directory diff --git a/tools/flashinfer-build.sh b/tools/flashinfer-build.sh old mode 100644 new mode 100755 diff --git a/tools/vllm-tpu/build.sh b/tools/vllm-tpu/build.sh old mode 100644 new mode 100755 From 697f507a8ebb13d74e8c0695aec05d9baefb45a0 Mon Sep 17 00:00:00 2001 From: Jakub Sochacki <97886316+jakub-sochacki@users.noreply.github.com> Date: Fri, 31 Oct 2025 00:57:22 +0100 Subject: [PATCH 124/127] [CI/Build][Intel] Enable performance benchmarks for Intel Gaudi 3 (#26919) Signed-off-by: jakub-sochacki --- .buildkite/performance-benchmarks/README.md | 3 +- .../performance-benchmarks-descriptions.md | 6 +- .../scripts/run-performance-benchmarks.sh | 13 +++ .../tests/latency-tests-hpu.json | 55 +++++++++++++ .../tests/serving-tests-hpu.json | 82 +++++++++++++++++++ .../tests/throughput-tests-hpu.json | 61 ++++++++++++++ 6 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 .buildkite/performance-benchmarks/tests/latency-tests-hpu.json create mode 100644 .buildkite/performance-benchmarks/tests/serving-tests-hpu.json create mode 100644 .buildkite/performance-benchmarks/tests/throughput-tests-hpu.json diff --git a/.buildkite/performance-benchmarks/README.md b/.buildkite/performance-benchmarks/README.md index 332142ba5d170..6d494f64f14fa 100644 --- a/.buildkite/performance-benchmarks/README.md +++ b/.buildkite/performance-benchmarks/README.md @@ -7,7 +7,7 @@ vLLM also maintains a continuous performance benchmark under [perf.vllm.ai](http ## Performance benchmark quick overview -**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100 and Intel® Xeon® Processors, with different models. +**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100, Intel® Xeon® Processors and Intel® Gaudi® 3 Accelerators with different models. **Benchmarking Duration**: about 1hr. @@ -34,6 +34,7 @@ Runtime environment variables: See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases. > NOTE: For Intel® Xeon® Processors, use `tests/latency-tests-cpu.json`, `tests/throughput-tests-cpu.json`, `tests/serving-tests-cpu.json` instead. +For Intel® Gaudi® 3 Accelerators, use `tests/latency-tests-hpu.json`, `tests/throughput-tests-hpu.json`, `tests/serving-tests-hpu.json` instead. > ### Latency test diff --git a/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md b/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md index 8bb16bd3cf373..b9437ac5ca99a 100644 --- a/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md +++ b/.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md @@ -5,7 +5,7 @@ - Input length: 32 tokens. - Output length: 128 tokens. - Batch size: fixed (8). -- GPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. +- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. - CPU Models: llama-3.1 8B. - Evaluation metrics: end-to-end latency (mean, median, p99). @@ -16,7 +16,7 @@ - Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed). - Output length: the corresponding output length of these 200 prompts. - Batch size: dynamically determined by vllm to achieve maximum throughput. -- GPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. +- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. - CPU Models: llama-3.1 8B. - Evaluation metrics: throughput. @@ -28,7 +28,7 @@ - Output length: the corresponding output length of these 200 prompts. - Batch size: dynamically determined by vllm and the arrival pattern of the requests. - **Average QPS (query per second)**: 1, 4, 16 and inf. QPS = inf means all requests come at once. For other QPS values, the arrival time of each query is determined using a random Poisson process (with fixed random seed). -- GPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. +- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B. - We also added a speculative decoding test for llama-3 70B on GPU, under QPS 2 - CPU Models: llama-3.1 8B. - Evaluation metrics: throughput, TTFT (time to the first token, with mean, median and p99), ITL (inter-token latency, with mean, median and p99). diff --git a/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh b/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh index 9447ceffd7e22..99a5a5e334f8e 100644 --- a/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh +++ b/.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh @@ -15,6 +15,8 @@ check_gpus() { declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l) elif command -v amd-smi; then declare -g gpu_count=$(amd-smi list | grep 'GPU' | wc -l) + elif command -v hl-smi; then + declare -g gpu_count=$(hl-smi --list | grep -i "Module ID" | wc -l) fi if [[ $gpu_count -gt 0 ]]; then @@ -23,10 +25,16 @@ check_gpus() { echo "Need at least 1 GPU to run benchmarking." exit 1 fi + + declare -g arch_suffix='' + if command -v nvidia-smi; then declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}') elif command -v amd-smi; then declare -g gpu_type=$(amd-smi static -g 0 -a | grep 'MARKET_NAME' | awk '{print $2}') + elif command -v hl-smi; then + declare -g gpu_type=$(hl-smi -q | grep "Product Name" | head -n 1 | awk -F ':' '{print $2}' | sed 's/^ *//') + arch_suffix='-hpu' fi echo "GPU type is $gpu_type" } @@ -138,6 +146,10 @@ kill_gpu_processes() { while [ "$(amd-smi metric -g 0 | grep 'USED_VRAM' | awk '{print $2}')" -ge 1000 ]; do sleep 1 done + elif command -v hl-smi; then + while [ "$(hl-smi -q | grep "Used" | head -n 1 | awk '{print $3}')" -ge 1000 ]; do + sleep 1 + done fi # remove vllm config file @@ -451,6 +463,7 @@ main() { ARCH='-cpu' else check_gpus + ARCH="$arch_suffix" fi check_hf_token diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-hpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-hpu.json new file mode 100644 index 0000000000000..296380f72a668 --- /dev/null +++ b/.buildkite/performance-benchmarks/tests/latency-tests-hpu.json @@ -0,0 +1,55 @@ +[ + { + "test_name": "latency_llama8B_tp1", + "environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "parameters": { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tensor_parallel_size": 1, + "load_format": "dummy", + "num-iters-warmup": 5, + "num-iters": 15, + "max-model-len": 256, + "async-scheduling": "" + } + }, + { + "test_name": "latency_llama70B_tp4", + "environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "parameters": { + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", + "tensor_parallel_size": 4, + "load_format": "dummy", + "num-iters-warmup": 5, + "num-iters": 15, + "max-model-len": 256, + "async-scheduling": "" + } + }, + { + "test_name": "latency_mixtral8x7B_tp2", + "environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "parameters": { + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", + "tensor_parallel_size": 2, + "load_format": "dummy", + "num-iters-warmup": 5, + "num-iters": 15, + "max-model-len": 256, + "async-scheduling": "" + } + } +] diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-hpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-hpu.json new file mode 100644 index 0000000000000..8c6b34bd9fa33 --- /dev/null +++ b/.buildkite/performance-benchmarks/tests/serving-tests-hpu.json @@ -0,0 +1,82 @@ +[ + { + "test_name": "serving_llama8B_tp1_sharegpt", + "qps_list": [1, 4, 16, "inf"], + "server_environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "server_parameters": { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tensor_parallel_size": 1, + "swap_space": 16, + "disable_log_stats": "", + "load_format": "dummy", + "max-model-len": 2048, + "max-num-seqs": 256, + "async-scheduling": "" + }, + "client_parameters": { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "backend": "vllm", + "dataset_name": "sharegpt", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", + "num_prompts": 200 + } + }, + { + "test_name": "serving_llama70B_tp4_sharegpt", + "qps_list": [1, 4, 16, "inf"], + "server_environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "server_parameters": { + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", + "tensor_parallel_size": 4, + "swap_space": 16, + "disable_log_stats": "", + "load_format": "dummy", + "max-model-len": 2048, + "max-num-seqs": 256, + "async-scheduling": "" + }, + "client_parameters": { + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", + "backend": "vllm", + "dataset_name": "sharegpt", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", + "num_prompts": 200 + } + }, + { + "test_name": "serving_mixtral8x7B_tp2_sharegpt", + "qps_list": [1, 4, 16, "inf"], + "server_environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "server_parameters": { + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", + "tensor_parallel_size": 2, + "swap_space": 16, + "disable_log_stats": "", + "load_format": "dummy", + "max-model-len": 2048, + "max-num-seqs": 256, + "async-scheduling": "" + }, + "client_parameters": { + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", + "backend": "vllm", + "dataset_name": "sharegpt", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", + "num_prompts": 200 + } + } +] diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json new file mode 100644 index 0000000000000..3127bf2f6bce3 --- /dev/null +++ b/.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json @@ -0,0 +1,61 @@ +[ + { + "test_name": "throughput_llama8B_tp1", + "environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "parameters": { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tensor_parallel_size": 1, + "load_format": "dummy", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", + "num_prompts": 1000, + "backend": "vllm", + "max-model-len": 2048, + "max-num-seqs": 512, + "async-scheduling": "" + } + }, + { + "test_name": "throughput_llama70B_tp4", + "environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "parameters": { + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", + "tensor_parallel_size": 4, + "load_format": "dummy", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", + "num_prompts": 1000, + "backend": "vllm", + "max-model-len": 2048, + "max-num-seqs": 512, + "async-scheduling": "" + } + }, + { + "test_name": "throughput_mixtral8x7B_tp2", + "environment_variables": { + "PT_HPU_LAZY_MODE": 1, + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1, + "VLLM_CONTIGUOUS_PA": 1, + "VLLM_DEFRAG": 1 + }, + "parameters": { + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", + "tensor_parallel_size": 2, + "load_format": "dummy", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", + "num_prompts": 1000, + "backend": "vllm", + "max-model-len": 2048, + "max-num-seqs": 512, + "async-scheduling": "" + } + } +] From 2bf0bcc1fca422222b78a3b1f39845ecd037aecc Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:29:26 -0400 Subject: [PATCH 125/127] [CI Test] Add Scheduled Integration Test (#27765) Signed-off-by: yewentao256 --- .../deepseek_v2_lite_ep_eplb.sh | 62 +++++++++++++++++++ .../qwen30b_a3b_fp8_block_ep.sh | 61 ++++++++++++++++++ .buildkite/test-pipeline.yaml | 18 ++++++ 3 files changed, 141 insertions(+) create mode 100644 .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh create mode 100644 .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep.sh diff --git a/.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh b/.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh new file mode 100644 index 0000000000000..5302f524a0ae4 --- /dev/null +++ b/.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT] +THRESHOLD=${1:-0.25} +NUM_Q=${2:-1319} +PORT=${3:-8010} +OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled} +mkdir -p "${OUT_DIR}" + +wait_for_server() { + local port=$1 + timeout 600 bash -c ' + until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do + sleep 1 + done' +} + +MODEL="deepseek-ai/DeepSeek-V2-lite" +BACKENDS=("deepep_high_throughput" "deepep_low_latency") + +cleanup() { + if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill "${SERVER_PID}" 2>/dev/null || true + for _ in {1..20}; do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 0.5 + done + kill -9 "${SERVER_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +for BACK in "${BACKENDS[@]}"; do + VLLM_DEEP_GEMM_WARMUP=skip \ + VLLM_ALL2ALL_BACKEND=$BACK \ + vllm serve "$MODEL" \ + --enforce-eager \ + --tensor-parallel-size 2 \ + --data-parallel-size 2 \ + --enable-expert-parallel \ + --enable-eplb \ + --trust-remote-code \ + --max-model-len 2048 \ + --port $PORT & + SERVER_PID=$! + wait_for_server $PORT + + TAG=$(echo "$MODEL" | tr '/: \\n' '_____') + OUT="${OUT_DIR}/${TAG}_${BACK}.json" + python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT} + python3 - <= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}" +PY + + cleanup + SERVER_PID= + sleep 1 + PORT=$((PORT+1)) +done diff --git a/.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep.sh b/.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep.sh new file mode 100644 index 0000000000000..a5135299297e2 --- /dev/null +++ b/.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT] +THRESHOLD=${1:-0.8} +NUM_Q=${2:-1319} +PORT=${3:-8020} +OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled} +mkdir -p "${OUT_DIR}" + +wait_for_server() { + local port=$1 + timeout 600 bash -c ' + until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do + sleep 1 + done' +} + +MODEL="QWen/Qwen3-30B-A3B-FP8" +BACKENDS=("deepep_high_throughput" "deepep_low_latency") + +cleanup() { + if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill "${SERVER_PID}" 2>/dev/null || true + for _ in {1..20}; do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 0.5 + done + kill -9 "${SERVER_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +for BACK in "${BACKENDS[@]}"; do + VLLM_DEEP_GEMM_WARMUP=skip \ + VLLM_ALL2ALL_BACKEND=$BACK \ + vllm serve "$MODEL" \ + --enforce-eager \ + --tensor-parallel-size 2 \ + --data-parallel-size 2 \ + --enable-expert-parallel \ + --trust-remote-code \ + --max-model-len 2048 \ + --port $PORT & + SERVER_PID=$! + wait_for_server $PORT + + TAG=$(echo "$MODEL" | tr '/: \\n' '_____') + OUT="${OUT_DIR}/${TAG}_${BACK}.json" + python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT} + python3 - <= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}" +PY + + cleanup + SERVER_PID= + sleep 1 + PORT=$((PORT+1)) +done diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 339e3aab6c031..8d4e5ece94d19 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -1234,3 +1234,21 @@ steps: - .buildkite/scripts/run-prime-rl-test.sh commands: - bash .buildkite/scripts/run-prime-rl-test.sh + +- label: DeepSeek V2-Lite Accuracy + timeout_in_minutes: 60 + gpu: h100 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace" + commands: + - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010 + +- label: Qwen3-30B-A3B-FP8-block Accuracy + timeout_in_minutes: 60 + gpu: h100 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace" + commands: + - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep.sh 0.8 200 8020 From b2e65cb4a7ea7c000517a7b78a6e0ccd9ecb0517 Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:40:35 -0500 Subject: [PATCH 126/127] [benchmark] Make request IDs unique across clients by default (#27723) Signed-off-by: Seiji Eicher --- vllm/benchmarks/serve.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 71d136d61ceaf..4b15d8e62913c 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -26,6 +26,7 @@ import os import random import shutil import time +import uuid import warnings from collections.abc import AsyncGenerator, Iterable from dataclasses import dataclass @@ -1160,7 +1161,7 @@ def add_cli_args(parser: argparse.ArgumentParser): "--request-id-prefix", type=str, required=False, - default="benchmark-serving", + default=f"bench-{uuid.uuid4().hex[:8]}-", help="Specify the prefix of request id.", ) From 36960501d336a15cf0de7569e2662793ad9a4f3f Mon Sep 17 00:00:00 2001 From: Akash kaothalkar <61960177+Akashcodes732@users.noreply.github.com> Date: Fri, 31 Oct 2025 13:15:26 +0530 Subject: [PATCH 127/127] [Hardware][Powerpc] Fix VLLM_CPU_OMP_THREADS_BIND="auto" low CPU utilization for Power (#27734) Signed-off-by: Akash Kaothalkar Co-authored-by: Akash Kaothalkar --- vllm/platforms/cpu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 8c1d46564f6f6..4b9f4aef022d0 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -316,7 +316,8 @@ class CpuPlatform(Platform): if ( platform.system() == "Linux" - and Platform.get_cpu_architecture() == CpuArchEnum.ARM + and Platform.get_cpu_architecture() + in (CpuArchEnum.ARM, CpuArchEnum.POWERPC) and not ("libomp" in ld_preload_str or "libgomp" in ld_preload_str) ): # We need to LD_PRELOAD PyTorch's libgomp, otherwise only