From 78f4bb0ba8c70f8876311cc414938d3d68997fc6 Mon Sep 17 00:00:00 2001 From: Fadi Arafeh <115173828+fadara01@users.noreply.github.com> Date: Wed, 3 Dec 2025 11:36:58 +0000 Subject: [PATCH 01/47] [DOC] Add Arm to list of compute resouces providers (#29894) Signed-off-by: Fadi Arafeh --- README.md | 1 + docs/community/sponsors.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index abbb63158f166..5c040fe4a66d2 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ Compute Resources: - Alibaba Cloud - AMD - Anyscale +- Arm - AWS - Crusoe Cloud - Databricks diff --git a/docs/community/sponsors.md b/docs/community/sponsors.md index 8abb07caaab62..fd1c82376d086 100644 --- a/docs/community/sponsors.md +++ b/docs/community/sponsors.md @@ -18,6 +18,7 @@ Compute Resources: - Alibaba Cloud - AMD - Anyscale +- Arm - AWS - Crusoe Cloud - Databricks From f5d3d93c40417c296c20dc301100e55708a17f3f Mon Sep 17 00:00:00 2001 From: Amr Mahdi Date: Wed, 3 Dec 2025 03:41:53 -0800 Subject: [PATCH 02/47] [docker] Build CUDA kernels in separate Docker stage for faster rebuilds (#29452) Signed-off-by: Amr Mahdi --- docker/Dockerfile | 66 +++++++++++++++--- .../dockerfile-stages-dependency.png | Bin 149377 -> 177867 bytes setup.py | 14 +++- vllm/envs.py | 5 ++ 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 006481b23cb9f..8bcd7f118f1ef 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -150,8 +150,8 @@ ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0' ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} #################### BASE BUILD IMAGE #################### -#################### WHEEL BUILD IMAGE #################### -FROM base AS build +#################### CSRC BUILD IMAGE #################### +FROM base AS csrc-build ARG TARGETPLATFORM ARG PIP_INDEX_URL UV_INDEX_URL @@ -172,10 +172,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') -COPY . . -ARG GIT_REPO_CHECK=0 -RUN --mount=type=bind,source=.git,target=.git \ - if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi +WORKDIR /workspace + +COPY pyproject.toml setup.py CMakeLists.txt ./ +COPY cmake cmake/ +COPY csrc csrc/ +COPY vllm/envs.py vllm/envs.py +COPY vllm/__init__.py vllm/__init__.py # max jobs used by Ninja to build extensions ARG max_jobs=2 @@ -195,9 +198,11 @@ ARG SCCACHE_S3_NO_CREDENTIALS=0 ARG VLLM_USE_PRECOMPILED="" ARG VLLM_MAIN_CUDA_VERSION="" +# Use dummy version for csrc-build wheel (only .so files are extracted, version doesn't matter) +ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+csrc.build" + # if USE_SCCACHE is set, use sccache to speed up compilation RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=.git,target=.git \ if [ "$USE_SCCACHE" = "1" ]; then \ echo "Installing sccache..." \ && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \ @@ -223,7 +228,6 @@ ENV VLLM_TARGET_DEVICE=${vllm_target_device} ENV CCACHE_DIR=/root/.cache/ccache RUN --mount=type=cache,target=/root/.cache/ccache \ --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=.git,target=.git \ if [ "$USE_SCCACHE" != "1" ]; then \ # Clean any existing CMake artifacts rm -rf .deps && \ @@ -232,6 +236,52 @@ RUN --mount=type=cache,target=/root/.cache/ccache \ export VLLM_DOCKER_BUILD_CONTEXT=1 && \ python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \ fi +#################### CSRC BUILD IMAGE #################### + +#################### WHEEL BUILD IMAGE #################### +FROM base AS build +ARG TARGETPLATFORM + +ARG PIP_INDEX_URL UV_INDEX_URL +ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL +ARG PYTORCH_CUDA_INDEX_BASE_URL + +# install build dependencies +COPY requirements/build.txt requirements/build.txt + +# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out +# Reference: https://github.com/astral-sh/uv/pull/1694 +ENV UV_HTTP_TIMEOUT=500 +ENV UV_INDEX_STRATEGY="unsafe-best-match" +# Use copy mode to avoid hardlink failures with Docker cache mounts +ENV UV_LINK_MODE=copy + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \ + --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') + +WORKDIR /workspace + +COPY --from=csrc-build /workspace/dist /precompiled-wheels + +COPY . . + +ARG GIT_REPO_CHECK=0 +RUN --mount=type=bind,source=.git,target=.git \ + if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi + +ARG vllm_target_device="cuda" +ENV VLLM_TARGET_DEVICE=${vllm_target_device} + +# Skip adding +precompiled suffix to version (preserves git-derived version) +ENV VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX=1 + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=.git,target=.git \ + if [ "${vllm_target_device}" = "cuda" ]; then \ + export VLLM_PRECOMPILED_WHEEL_LOCATION=$(ls /precompiled-wheels/*.whl); \ + fi && \ + python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 # Install DeepGEMM from source ARG DEEPGEMM_GIT_REF diff --git a/docs/assets/contributing/dockerfile-stages-dependency.png b/docs/assets/contributing/dockerfile-stages-dependency.png index b327eb2151f50e4d682fe533fa57e12f0b6118c2..7420ca4d89441e6dd320657092aaf3e1c0491e9c 100644 GIT binary patch literal 177867 zcmbq+c_5W**Z!m4>YT=NP9sgHq9PHI%$nO2GLxaf5Q;J+GIcuDDUI8dS%b)&Aw!`_ zX3Cf%8W2L6$?&_@raJHY{rTIk&Zlh8zMuPE_gdF=t?Pci^Qy`_W=!Ro$}r3f*3PYJ z4D*`_!%RH(%VhkEI$z=({GZ7Oly+=oM(KZ{8R4D`^C!dFx=G_i;OAP0>zWN|67_Y# zT$}#pI{b^pVVz$tOp<;j{`)^WR6Spv6x=Dc;iA(YhaWGoR6ZzR`aI=-d%diP|u8k7vi#Ay4(_ql4_&=H(B5rg>m z>rqwE-aYOA`S(B3_dVC{O#k11vkm3{Kfk@`Q2zhEZQT@q(f|Fo!c+erm%Eg|m~%aM zzPgWjp6jh@Qg>Znx6E_p?)=Zyi3N*)-FLn;-M%&HaJ6W`+7x}g6f1-FjXwr3ZJ{@S4+329=u=Y z`c2Hd^mfg?AD42yIyL*iJ)QR7IR8>^ooH54EI&F>QghsBr{+Z2=3H^_C zO#Ms}x{9UEvW9z=N`hs}QjdMW558Zh|Hejbb&3gwX|#Cq_pfO!kx@29;#D7q`|3hI z_*cg44Ob6W);00gEKM*eNILZLz}K_?ImFyi4)SMtu57qGpW%_q!20F+RbnZ|X*RQ|KK7{U5F<+0LCix7xnSz_$D4j_|!HR(GSL1rKg^U+Qz< z-t}c$&VG!Hip_s?YT|m!%IMNetr)$A&xs|<9d9$#_w3p8-0S<*nHlWy$Z>8yJxeZk z&yL03QgEqG@0GFB&I$n3);W0i^`F#}?6$5S7 zZUIIkoxXlDjy7yzw=XA~f*cxq-&mJOwMMJs*9!A61OBbC4}`Xyn_1-48sp3BCvtH6 zWx2smpR-0s{G{wH{AT~0<>_Y-dV&tWa6WEcR*C$mOomfW{=qA}Y?h|bqytVH{ zMn+!DV|0Ei3s)(*8y@UmAoM`c$fY|aZ)Jcn*3!J}&ThY-lOXuAIA*oO*hk=sa^D2y zhnrQDW^nt9nX{FIGsGm`p15^(I-_*`&hCIlwvtd@twUXq&)U5!S!ZW04UW10TA(BI zo7KOsSUoj6Ziz1^OFWwOao*WkYLRmaCs9SNyu0ovon;~NtRVYvt+wfwG8YlL|0ARw@2rAwMo~biL;Dr?W=aP9}YAI zRNI!wsv8&>G#Y=u7^5_A_t=$P8KKPn_ce}9K`b_MKoervtWxLTCDHH+`}XMVKQp(p z{`4$UZ%r*0TV~*+r(}bddVU}k8_sKAp4#ep9@#eM#{Txwu`_Q~VwtjZSbDk?b|TGu zX~xMuo16*zL>$}GniAUXzcGD&k(+|oPkf}7?jF?jcB}HC`|0F8y&9Wmb5Wu z8-Hs{Dd}(<{U*9c8*k-N|=n<|iO+ z-MMpTWs$#l0p-Bnwl}^QwFz!Eq-fCV}xN=aZhtbf@!t+do z3YDp$kE`Ta?2C&eMf>X9-2VNMMn8VtZKNwi=}p!^k+s^McX5AlMYJJD3m-9Z>8UW; z^&`uuX3f2T5?8Z~GXkBBE0qJU(@ygc&fM8r8M7Y6LJavDH8H{rvlNtEWhmW|(PtD>GR-EyAD%+uhKRIrbAXS3|AUYv0Kz1tOzzfQ4|Dvy^>{m$#PQ-I{BR``Nad?G4JDK|Ff$wOG2t zc&Ps?iyd_QHH&WTF6v!cXZ{*0ZV$ganO1!)XEA^Z=lwhvs}s$d&dfK=4c&CQ6E}v$ zZJ5^-;%10J&M2O`Ci?0niMJacd`?iJ+B|cyl$Z1$%!^0kwfu#`!=cWCu(skLXf_T>nyAdqR*|`evSev=Lp8T0DlS+#NXq2&j14;<9B<=57eKPz{$l*<2i*BelFZ9Q zfv0xi?*zIGT?ZN-Vu}b~m=~Sw$nbR?9&8A-EzWZru5xQAS6AtApUs3SivdF8b?DEx zcu(o^b2As`U;mFgUv^c;{1rPc)8Oc(VKlo(3Ut+OTfrAC(-VvrcWZEjr zxxCL#f=WxAzdYv6Lk&_x=rnF+1b_9HunxYf9&U!8M{7q99djaZr^DllhtLoFsTefYhD znMZV}KS#be$l-;)$s>0zlnh(vpkvRu8!o5pjL95qvCZf` zW0KKR8E|kINI@vcHvUlZp_j{9odZoFzLH~MU+$)UXIGxwx6htiW=isgxb*6>jkD)! z7rJvz>_q68UXkt9i3rJNz6`HTE?1Kr{B)}e8Rd@K=!mXUbC@SD7P}DpEU<34-2uzS zZ>z}cW7Y3X<4nzrt@m_x{bBTlYu_p7irl8CZ(~Jdomt1jF6H~LEuw0Gufoy@6*^+d zZC-4eVjet8Rb_2Pcd2Agxq8=Y%Q%zppyP_24NbS20&SHELO2h7%`s8_g(-I!Nt(j> z7_mcBmO39rhHS3*4KR0AwTZ7m>CJ5(pMgYx*IE`;WsHt|3u|q!imO|m*IBvO+PnBz zK7Y6%+m`eDjMp1=6x$dZiZ#*kn*pWP$ye&z7Rd~K zd79S$-lGgl#hdrxihM9(^o+i@Yiy;MP`NU%v9f~jsMIxUpo)ZbvgxaJXacHP>%hzT zIQPkxFG4pBG^@DzpP#*IB}>Zg)7AY?&xVLB`s+_uTRmuF^L}`KNeS3d)U@zA5FXPqd%R9@uEj9*qu)?g%c8X7 zb_L5wvJ$YiO$ct~ZD&H>F=AT@wO|bckg|DMog?2m!nNZJ3G@SbGCgPHXZ$F{G%_}( z8=OdP;LFolp1g04q!?fb_r)yABMO1>y-=A%PkgDTG|plJy$P)fK5>8`PtTe0WmBC+ zH;vt|zv1FCCDi)?lV zpLl=XCTC2T&^a{N)9MR$$;;9Le;&enc9KpIIMqG-28NjiQl8jdsjkX%Ud${LG38T&)T`x|!Z7cI35PaBEXeRb2{| z$=jaF`A{)}-ykBSuSmZCXN3)NQPEc=hxrC}d#JigE|JC}+qJ#!+1n+1vbRvSucj+z zAh}Y{^78Jr&o3_yz?LZfy2v2iE)OY5ylRML^PJm^s)OLArlu7)QPmo$V(pfw?G9?4 zDfKCIsAP0Vr5NNj&~0RZ#m3Ya4|X?q$xLZz8X3z8Mo84Y3Du?n#5oW=#UQ5}a!MST zL$9pTo9`?-`iP6gCfy+}YsBewlDT@L2BX0JQ-1ogrRUSFE2gb6DMI4LIu5n=wqV(& zh!;OeTT?mke3{PN6$1^zZn{#9=T`;imbmsN0`(?n#oX9(?#_-YORxlXfTA{qB00vd z;`jS)Jazz@&wAIP{^?6(Hp&m=sjzu(ZNHR{bR9s1@06`PuE<%O*H`yjVV+MPMj1oIneHw_|KF^d>^n6F_z*OWz90X-#Yr4kB+0svn5v2QJ7(_11t zqN~DHLevu7v0;RvlbPJFu_+)R42+^{mDW%ivp0pZqGWsOhs?xEV6!0LKqpNtEuUT& zpnS=XSGSa+b_E`PZSL4pp-t)KBKP{0zDBN&P!~jV{EeKyZo~S8G|*P#P-iMTgS~A5 z_LaJ;uvd3?H5Qen*)|0_ymjQ3vOA=gbV#JGqp7c_Dr-c1mR$cn%7#M&C2qxc0<8Rj zhzRdjAAfVimdYUT|Vuw_b5c*;`>8g1+9VEE1wPxa!nSzTGL7a+ghrtyzT0;ty zrQ5_t=MTy*avS+>Lxlyom z*I4fN$j6-eN`FEx~z- zeQ!^i<$Lk7*huO%xCQgqNzM9KBoUm6*y2UU>IffW8N$Tgx{VHJbq&qeKe`D!s7sc; z3c2av%3as50=o$kk4!-Hnqeacw1dzk6!j+$aYYZAy^=E)ZV-e*cEtQ4 zbfu5aW~&6Y4o;jUvkq0xn)Xii?_;!V-wl84Jb<-_yew?DPT#B{zDqpf0!X#_=4a+= zO=V7wzKaAIRaVGxy3=qL?uOME^X^JY^{X#B--F_d2z!P;WWk4S~ zq7%#2luWSn0atFvyaR|wZ|wUF@K<-6@1SE|qGeph{mN7n_%8dJ4Q~#6 zf`7SLW@<)hB|pdR?v)8CK8l(irXyIK{PJlH61>*NY8%nRi*^LD5Uh1fTW z9|khsA~x*WYaCB?u8lygG>%EG^lwVHs!gwO>aflnXe=);7tB9ovfjR>oY=HjY-FT= z+{z^j565q!vb*j2n4Z7yUBJ5in}q;;c_Ak6?tD-;97{@q@YFM#iviC|`!B99zTVU) z0+P!bSE+R_QW`})DNVh3^FU@u64g+%#zH@g)4VRz#XJ*}u6yv3xZ(#;Pf02&s5hQS zQD^4MxJ;5r-ny3$h z%kwm>Q2(7xi~Kcttm7X~wmm#qbQKDwp2oN$Kkqz#Zy8txi@gZztBV-Or=>B5(9JKK zTVEtmS4i?Gn3{s*_cn#c%YS}fp=i{i$fnc>k<%{7LQ6yABfzW@Az52e@*vqKzQ3@> z6vAW)Y1+gD?4?xL(MXTeoD09adun+J8=+0ANk`}P{Ug$KmRWDTma$OBz0WKTH#~ua zhd1b1#tg{a2eY0gJ*G52T5b>W;d38^)ZjTUQ?8w}$aa_cH|$Fx(AZG@ajNT{eF$Pg z^7FNO@9VF&2Uf_DLG7zP;IY=CNCk}#h-Y=S^{1xXLCd1rA}YoPA7_SAT-PcSqnE4~ z+LL`iB*b;lJSHvOu#5T=fGLidv>hEQE#5vHa1KeqOjq_JsQ1=?R{AS_s7vo^-x$XX ztqBIPLD5;rVlO`aYMK1V*DIWOj{s>1J12+8S_cfg2AOT&vF|Ko-YqQD!^#s#I);am zC8~`*Wxjn*q%>;#@!Ag4FmJ8M=dO6l zy12Ah0fENRTWssbU>V2GCzsWQcln8iRi;9^?s7S7V$$&B@)*>M&#ggH3~PfbAAOO< zM3g}038ZYq5{2l3dY?-BF?;vV5RQzrxReD>b;kAbk^9vvwGHewp?`XU5j8-#EEb50 zRf~;=(iQ3Tr|4MEP`+G93*dvJ0z^r3v?Kt*F2% zohBceu_3)d&3lM1XuV#73dHTnFb&^ZXM)>_=f!-5iZHJ_4SS9XtNzi)lSl=(tukzWA>6Ac z70VGa&^wge(h@NG5W3yQx*Vu6_WN_ju$d0;_Dek8DRr*WtWv0yX9lZhF*8bh_FDNh z_!`NvI$vy=1=N<#nX{kUz7j#Mp{eQp976cJ3F4FcGX>VHA$l`?shkWTg}u3J=f^y5 znG;`@%l54dIILzVbKu!O+nh2deFY9F?tAS}=aN#H3f^U_uf2MX#U-=iK*^HV9tn~; ze+@R;Us@z-lX=D9NALv_Y%1+VsOn>X2>yx>O9Wb3IncR&1SlZp3;z zuqm5bU*TP=(4)Z@6(w`m`bVdrR#dxwYirWIg?!gsm*p0dh6OD$>hEZd66SMmfD(7T z;~Q7BtP!IfxNAL_e{yHMOG^y@XwqrKlW0vk^J?{31)V zsnuYCA*9ud3ni?ATS?tQ;w?Q<=hC-(TXvwlsV@EZ3!643FG6=iMiQ`c<2N z-jdgIf(6g7EFBHXA&oZ;$&<|s$d?B}Z6HQ`soT)8C4{VkM*&i!#771_7tK2}S6g3X z$5IzN^P%@1!i_i1E%9$U2JGu<{=lK*tq1R3dwYB0rycqTd0S}zKF|rwqeESvF>pUQm%&9?wq^qoL8)wax8dz4dRlchwr<)Xi3j^FTgeN%MsDhg%p=GJ+YPmr%4Y{al^i% z%Ml2AT5RuvB%BB+>hrv>#^9Uncj;XcTjJ9FQtuLf zW;4=L34Z36pP&C7riB(0Y~(esPB4ivtzgzk0MiQ{Q3>355Bc(pLTH=l;uTYIA-?7K>Bc!WFL-^)42ha}9oL<8(28r2I&|HJdpvm&(zm}>+N3K*hnNHBaWJ-m|`y|R4b@%nidE80gwwr6gC8X315WH1!a)=6@0GN#IKKo z+n1(V*JV~?jO!1-iYKN=+7fb*EwsQpsFk{wG0CK?`-+Hg_`##001$k(mt5RLcgKN2 z*)6C`{rmaRIXjTHWsKT6ES5fAxX}9HHB{%)`Ts5Gq-=AMasvb$~pn5yM^|mDfT5xtcfSh;^kJ8#?4Qhfea{6?GTh95!WAn<(;x;y#PK7!P{5EyL1ExC z;f;z;P$adCAPmVm2lC1x!x4~5L#LM0p+KcBMtTe6)dkSxj1E=12=F>n!QQPW$|%gT zDwbBJ6B;BE-GB;Edm=P{V|-j80WWCCf)|wZ4naWmA?ysoVLUQemDTt)XK4wL549FH z9C>X_<_1)L?~LBpQG%m#V+f*L^M>kb2dZr?X`5k1@R1&?bM{TFCdPSb`%=zY4E^R< zD#_HH`!!};0RfBuiNsOJ$8_{}%2;tMT{w3=h{(ma)=?%HJ#X~ar0*H?U( zS?i}n(eK{MFXI3|YEhxc6h-qzR( z1(D4wE-ucoS(GvHx37b;vPU0W`7zQjZa-R2diM|Oz77U$0+frt9T6K_OkI)k%zwR^ z?|Z}YAGku2lVkyd(6HRc62Ujvau56tW{M%3xEO^kKicdsq1u)9 zqA$*YPyCs#S^b0K%^~+}GtdN;8>%}|+0f3_@ppqSN2#mMxfZ?2I$eBlMH+C=w(LbRBF4UWZ1B2;xPITaiXt19TeZPu3T}X^xtO z7U3>mA%g^{jt%Ir=i+`cHJeZzgbZ@B=Nsw!$KfKaJ-y9kW?XOgpo^FKW6JT(zD?gz z(u*(MD4*E%+UR1B^0Io-H6;rKr&GJjk4;br^2cEwaSJa1gm5&Cn$al6b{47fhr4Ds z6l+l+9}N&a@=3+-e<}*E7VHdWU|N8$M_oHTLwH#Pkx8>eHXJ^XW{LrM{TPtc@z2)* z@c0Hgo9CcKPL#Q02~r_13v-HmTM#(r=&C>0j243m@cTWT=pMf!R2B{Y(MG5Nz5WMd zXx*Xn319jKof?d6{hPyyyOHk`5WL2oyEo*aqvUgH;*>jps4O-xtT4J#ul3hsidL;R zE8fK>HH3xsfOYH-pjE4=N;-?vQ5ZmpL z09%>&`;SN^3748)Y-bdz6$=Xs8zA{6EP;xNi3yB< zsBLd4f|lx7IKBp=5NAR|cvUYZ737=fjQtvt?B)7A9@HWUT3k5R8h zW?0eaAKh-b>5^(K9C)VDj<@!Hk)@8omn#bp?aU)IUo2^SVVDCe*NWtcMlK#Yy?tX{OZCL(vQ(LkE`ZQ3@eLGW4){7amF4 zLj@1k8iwJCr08+`_NGqMB;TtLSb?wjy%1ACk-YIXj^Hd<@E-X)bO zlLaLv4N$i{jsGc0?9@bJq2=uj1XjW2iA9ljGEU)bh6?En;jur|mSGA!cSQdP3B z@04Y^jV4)C2%rHJR<+`a3>!c)y|{;qap!ybL;HcW1CcuUIdwf_?o20(6*{8iP6Kh5 zWr+e7c7Lv34@|Ax*IwII7BV764Twxl2>Cn{zVE^71`f626kfD#WV&B$A*K61S}5>o zg$ADhV74vWEy2fvsWk}ZA7a3a`(VAW+kjGY$y6gAD_lc@55uk7pVv)*$)*W3PiJ*< zra6+N9BJlp$ohs}jc|sKp;I$Ju~dZ%>M=E*oGl>_gp>f%=!L)LObqD9 z87)f`8r#4+Snr;v2l-7?n~8rGnJw+7C7* z3A+uOmxvOn8yXt@5wOb73#-sJ8s2M*%ouB13-#g-!U-#L>FjjDYlMHH@ClI=g(gX2 zbW*7zi%k$(**<)CfDajx9f3}|kjKq>KUYs%fzb!qPgx#R4Z~Ge z-&S31{4w`haxdCQ>mUMxEDp=G61~DI#)(THN(PE?7d?A!*^J?geJ?4XM^Ka-uWp%% zPJ(TrNOMh^E#RIGc}#>zs6cWyaC-lV+br;%9D=p(3v`%|V5qsYOuw(#ZcJ4XIk`Zp zqJ|_fq*W7GA#-^wqpzVAwH{fqakJudrC0)0woS#!jew@tfL0=v?a_v8gYzU@jKoHo z4x|-%$aE3vN3g>Jp>byQ9V1tYP})Z}ZyvmhouKc=XbxX*x;J~}u3QkttGv~44I~0) z7agm`n8b~AD^XbVKA8CEHcHz9c)2uvl^EM*HPk%bGD}vBMYw>ROC;aGHxj=Qb_8*R zq#0QTd0E&cXoQ7(NzP@M@}-n7$<>EY`yBstCdZ!zrCNj>r$LtJXH|2r+owS83nH+9 zM!WIud7CW`ogg#$<{PTvJkV|;#Dl@o4I%4@S$rITvrkZ$DB9~_u+EfTNLEFNVSJ@Q zf3a+n{`zco*+a*!v6fVvEqN!1uibrKLme4M{AtpK{S=Dg+e5c5HKzBrR_#UK-fz8G zxl$>7!nr;hKwA!Z1$OhDG99mtM_(!3sfnPaggTahc`s0mt{5<42U zv9xz*7s4*PVl#G(;eRTxfzjr}8RW4kG1LJ_z7{7M8ampqaa+Yd2J zyaF+$(q2&f%;eFi5?8h{T!InF)W~^;1HzwWuzTDe5AOI@Ck!g+@rp z32kNbv=3bIS1X`ll19gg9Xii}&&!k?d4(3Ck1^UuXExtZ7N?{~zAz{yJ0T+_pfZt> z-C*ahr?O2T%oISuFtoJ7uCyN+MOSURF`OI4WCS9y*w9Ga1z6<__QUZ4Y-n#@%C#e3 z&dGnwU2$a>@ShDj9=`rUhjA(D9cJ1Okvs5;+~Dd}A7^<#s|sr*u~aDl=^cDq=$@*hOfN`5?b>0)|R7gycp@t2^0Z z)^BAbpiDYc){u=4OoPx--sr)O;-`_ZU0GMMW_r{=xoLXfP;6LQN9ev!?^2Be)ja)K z=J$SIc;tEGeSg!!m#%uISr(;*5XHAYhWvD zZ{7Or$PDFvT})%<=*Y<<)d|}WyX0~43J8dTc<~8ypbafS{LqU(v6L#Yv9T#G?HA^* zp?S0b6p}jG8_kfoIBj5oA;t;Ua>onVNtYLUW#Ok!v)CEBRH|-N7Dkd=P|EZHl zhfSBXZ7lI?{!;dl6DCZUvtR)kA|kLT+s(Kb z)w@U4*E03WUxp_#?uzGcjM*Nrk^F*WPio<4&r3Ku11-U0;*oFG%fh%a*@ug<+Od86 zS!8VSBER)HX_c8SCuu(*ql?7T6|OFtd9Js;HVRGoPx;LD+VjfP;cc zWMyQ^Z)}<30?w8Wh&@AaA`{wKe@Bmb6jVPR!_WU76C@Y~KCN)&#kkfkl7)7y^yH-^ z-16eaX{5KCfq{!oPvc^?&~h(0A+1nKP4p87AK|B+Gheh_peC znLndOvAC#cHw+aoF*%_S!&vNC!;DC%VFt5gzPN*1-hKE`=HLo0hVANfx@2get0KWf z2~AumBD(C`NC7PXTovfy38|>4oL+;Zqgu_kYSkJ!Im4!s5QkXGSU=}gvgVPTc__V(>&xH)sOz)79>5^7Nu8&g=g>lSR@4Udq{?9hB=vC$!j*yb^h zTOZS`nD;Y-F;+Nz6batl{H-IaO4#k<=f=i6Fy{#f3hqMrivSQY-N(mh9L~ACxD4I; zxl5M{2Nd0@si|3Ln4%--gb1q$Q<{HzDsJEXra`8&?keR#Rcwe$%3SV>W+?Jf$KL;4 zXqx!f>C>7SPWHE8KvBm}LTD4Yzsi@e_1)8ic%7l&PCs<4*v zQo(LjhYv6zA;GTmBa0ex1RSwARAzC+PC4YtFA{C$PW$=an(HfQq&4OFLi#VTrhz4KMp>~)<^?4sS11?Fp?4SGn2tF) zuzDg>FF_{ufmCeS$v#-l(&wt*OZ_8j%G>>4KZv^yoxp^sAo>%k55K;5@LX}Qtfk!& zMq?w&uLiC(hleK;0=)*_CH(TnuONhFz%$18bm(hCfF|DZ$Acx9?EJGB%6v)ZQy7Y> z9ICH6H#fIlqUoj2#~D+UdzN6qEPkSczVOao!9|WZYvj%|DfeUvYMJR!pYDBYZ$a zDuA<B<5&5}u=0sq+ zC7cIw52I^Wt=fUhXeldSZmEik1GDq_v=1Ov_0F#2>(j^Yg;C8M7#Ki}E%|g&@!jiW zOT}HgyoyZK4RXKe)+O4egUqY{kXQ)0D6=rjsZ}$6+isy#V1}yL8Pm%PeKY5TeKC4 zaF0UJhP>t9y#NaC8xFq;1$bzM@Hq5#26I$tLjdhx_U-uib(66hRP(N5DV~1%acojG z2{&eeroLXf-Ce9HQh1B=5#RtGDZ7YM_KW}KBlI&-Gh&Yl0@=9JQC7wvdJApBFqy2I$5x)7&)X|5~r^716QbfGGwPPV| z#VErQ?b}V)PP0zfI&%y zq#Svpr4^+~R#$uAO&*=e%%j>jS#DYA>CLehMa@BCl2olpL5(2}d7;jC4vlyV; z3?8waY`ej3a}s~CT z)vCZ}t%OWnvZXJ9%`C`5ljj1|ID;qOxO(-s7~RC}1R4>T6^Ujlw-T{aciGu-peD-+ z1ZP1m_9vgQ&77XEdNgi&g+7}zp0EJW311;kM8(H{xu01Hw2cwsB)PylMW){#WijDR z8@_nfn_fga`heXZxZIguC~kWpHmDvy-aDgp%B&?CFgs0;oWN+z!$e-VbV(E21OZqU49(&0_OLRYbmerE}T?gJl6GV;$~-X(+Pk7I7YqTOSZo zvD#ST7N&jd6)~MwOjRRI&U})Dunt#bc5)MF*EfR+SsjK(?{d*+K zm8kPPfU>JsWDa#IH7T5Z@j{iezgT|%y}tx*c6Zy2$1Nav84_wo%R{0eNV}10VTvMi zCn!{3_4B(65kmzz09YMd%%47&F;PZkq@@HT&b3I6M5i*+Fe4o+kC*B)YG4#Edxl^7 z_19k~&*x(D|3Xi{8OLK@Ak_I-aCjLXg~{5x>|ba?br-;qmzURV)aSKegXrq;-%pg4 zl}$D&n3s&|^{V(k?g`6RuH3F7H*lLb4*&o)$20rDwd@0Xpz?ft_iq1DKp%x_RUMtx z^XJdMwVp;3`hKy$%MH4wS<4*t+gIYC*kc<5+Lc;*FsnB5SapB~GBoKw|H(aY4_^^TclIk$o6=3IilqvW}N%mjKxwKC^#FyAXKZ76>3ybhPNQBVB`DZmKD3D7*(G|(#Llle_((w z#0xOZv$cJ1;iZ{6V+Nt{SkNu61AJ&gSOWm~&l5I`M$1$UZK5*BcGk4DO`ZT>04`bI z4Gpb7+k=D^34L<1CKvOkiLr468qDcO-~Gkg3>Ux)bj_#7PGG$51BmHD!}aDR%+Alx zPZcf|e0jk?KgbqSb7{WeiA~570{6Obe&wVr`Z0XRC|ehwwP_>uA&h6I>!}P0s;aqFOiLO>S^KDpMn@YiibS7KyUSzc8Fzr zdQv=~EYwlPAw_2fx9C>3MzVz_A0OXcG>a%;6$?OqyJ7aigA-(By{})7z&vtt&h(#X zi)nxEy+)g~h(FS}H;Rg?f$Rzb?u9=5)^PPKV!0VP>WoR*qq2;x(5(nG4 zih^bLf@iIVYT%w=2}O|DKMYOgc9<6SZC^tt7P`;AyeRpqEEe%`2~3!3DWy#08G`U^ zy60`%xARag%xC_!nNsds*To{LTcNIIA2&soNj{h}6QMKrjUS`>J0STQY3aQvC!$jL z)kn=(Q4%Lmy-kw!=5ll2#V7dfT6wFC6HqDda2X1GD7m~WUxzmG zffuk7%LyIjZWxZnhV#qM;ixWp>-z1;rNz?sb&9c3)ejuF2@ZatK?YGqH5Qq9rG$h8 zm1KoU_i$pU=hMs6`%ltmt11V{x#&ULeu4Ajd@L}kK}d*zgO*3RAvA@BhXa`yN%OSH7?1(jx4w82pxm@xKHZV42LraU^MH9x|$P<~kzP>)7vbFr~En;;@ zYA69M<##=~2)96_J$|?@)4)2T$A^;O`0601&D{zRAUMwhM1+@x1pBhHp`ZscEf1uS z)V;XfN*p?}|N4F#OAW^|y{=vj#hGJ{PLh8;&pG->?aaOnRz@tbqrcU%JmLn9@?SWG z6qHYymbjCvtE)tG2-FY}5fLXe)+^F%%_ua<#*eoCEgW>?(HOhj1yPqlJLGsB_%*Zu zP0;5in;ADMWdwtq8wTMa0_|t^ZB#}3_8MJWlH|sAFQ|mrO$2mNOo0nx3^V*O3MO>t z%5zWOt&>8U?H)8@B-YK3_+1|6t7*P}4)J1`?-)C4}$ThY z_$i9U=>dg5!=b$t3R({V4D3wX*tE1XFptsuB@4t3U4O?p5vbD`amzKh#r@_+r;ovfNVZCVe^OwEuQ2%_<^peH$j4b0)@ z=4a7Lqwx1dHKnGe9%{(?M=w_(i%EC5rC=YkWJaQ z(MQq7!wAYDXlg>lqjBM7QUP6~CtQA*Fq&Pgqmz>b8j((rn~)`~;tg<2rz;XR_J67Do&I_RKfuy3A2TIzOgX|g&zeYZ?0Opm5hAedmxz7Tf;`7}Tn0#N< zP63jQ9%74uEc;+d;1jwDGPpqJht{-sl}d0qNGUWLL4c__Y1j->xJC?CZxy)aKz~0t zY?PN0SLhHrRHfM-^Krl-jEFYck`goa_v=E_qXLfG>T+0%kuUt$$Wsp2rfZU%jba*P z`R?62e@WXr7M0PCd(C9v*7uh^dB1C5oR-f&xdzRVyuv~ax+jQd{{AS)|I_8PxLqFk~RtWgZhM# zB%SQDLa0T~8TrVle#M=NTh-`5Slt7*UJ(VwTMSCF7QR~wSc1jxAl(1iYcP%8v>48SfRFQ1j0fo

b|c*%190j1Yt%3y!fh zKx`>6rPjCp(69B!gUAuv?XaS?B0uRjjVom#9wUipp_+i!&^JX$MiS(W{Z&;JnRQaQ=5&8 zX5L!C87=;^?ln|xB9LKWVUkr(o<0=^H-K1M_K5TTN5arGKVh~j(9pHV16!J$2oJK^ z%jeIR3rDceiL0s6hqT=aC&NJiXD@Y#Y1$wtB=lKSMej|C1qF>k;QPE5hoB9K^?+#B zDNX{wUo<>1 z#av>e3<9uHFPm=7{STz28^~VkuR_YW2e^0_XJ5mPXP{QD;p00)a1c?n>C4`o^bKAL zVfNjh_Mj7xfivg%1qGhG*f-S61eFs{oe6z>J(7$pyl?4HS1RD`y0k!KGw=-(sHoHo z#(g=JD%zM^xBlJRQ5Q$05TeY5F@?olw9q2ir(Yc0hzj8_o~#1xZOzwK&Lk+F%X@cp z9FhCjx zDTtxJG427xchG7UmqPRs|G6C&MqzH+Z{mS2)$ob^66eDA=(9Ty0CLYxJNJK4eV+L9sl4{`_m~^xY@%_?KUCe#c>QIp!6MULl7SpxCGuL zfcZl6gmlX7+qd5fuDf?|2LXB~oT9$aqfaAI@O%2BCMjee|CswbMtKNbi{j)*j}*wc zfIfmagdF$IW^DVhra&18nmv3!E^s2z*#l}^;^oDHGXV;G7+~g2n2}nbIEx}N2t*UE z6B1Gez@nZDmdIn6XQd|#&M~8ISa9;6KE3s-gaaVFB!8BA4SfVSkH~?)h4fSZWZ}2U zkUmgeHm0Sgb0#J`9#0X0=(PitQiMbRn%QQYuqcP$n~xP79PEUtkK%4 zwu&fUu+|lBZle!c#wJhNv-Tp4f@81Hjf6W1DFFg5{T|dOY}X*7%K@HxaEj_KP(y-_ zAifCB0qS5W-T$~65)ubFO`E7X@lUcP1R+bR-UPIWLRZETW@buwpm%?w<;^KnxQFMk zbR^j|s}T;(zAzU>R>;PP$n%mVLXg6-De{d)Ym>rphywSbYGu`a#NZ93mz>LJXMFhB z(&D7;Hj_rLFkt-Or6F}Mn4uHQ8MOZ2;ga~l=hGp4Jwq>y`C{~iKreZNHg43}zkfg3Nrr}pty@1sdoagMXQjnKU#AD;;iKIg z;0r-QeRBP?sU2sZ(qA1~0spBh^~dblL6#7HiSbt4X3d^h05;PWfwWEJ9o9|hDoEU8 zSdW*LnVH!F#)AGfzED$4G=!M?nebaRL$9Ld-t7=Mb32^;cFtc-y%s!Thj0>45Eo;j z2G|JsI)b_ix(Of8hRh!+g>&~7&M|F8%Lx13)7Q7EBh#nCVz}tWbUZ1hoP<<-%5Xan zEnagy{<}!jEB~uV+zzp7fz2%%87$`#9v$el9!-ZzuoU9Q=ou}RbME&6f$TFp^Y`Cl zLh`zQer57cLBZNWQO0{0(1oBoG&K@i@mkOfzX=e^h4bh4=<5ffy?UVrr$=UEGw{pE z=pzb!Jv<5w4Dj@mThyVu76;g>iAOA+qC?>i(#OE>lj}bv3xzbVpr9D?N5ex>oD8D} zeS<&|C9BIEU!;EIfLyB-%bu1BSdI=~`v&g;T)zX-gIi?)w$L5bK6EI8JOH#ksN50m zK@-6J>A6`%y$~p67`8noG@v&%J)ImBm|t*6@+o*^+(JEvG@rIV2ERNAgFYF^4rEb_ znhCA>seXqJRpZFp*_n$Y@#sGJ;^cq+`2~+?*n$@9IvT4NsZKBwG|8jgewFvTs|!Hk zsB|p)?hwEiVA=6ny$$n4t#fpH<{ioGsm5%F-xZJON6QT`bK2Aq70yWjb*F*elc+58WgqhC1XE!td~@ z)N7L)3hhj7>}MZgqgZ@fmy`p|g^Ye^p#iEwT{Ko`Rzdp<5VQv-=(~uSvtt@&zRbUd zka_+30!Y{JDl4%+!pj1$hXu^_APt1s7>)moHjNG~g$;+W3GT_}*fEi@fMG~7#3Oeh zja!sc@2BN;aui~}92FWG3XlFTp$RyUq=`XNn{NNp$#d;J6e01#)c|!HUz%}-vJ(|K zYn*!UXU+67KnduA^|oK(p^Y8SSDXy&{uvCFHj_Wkp~IT zAr;LLBzo=)JiQgli`KO`h>A4!slWU}mN;_L(6PwTbCXnW0z;zDwvrM7I1E|GuYGIB zChuu`qYVVy0y6`2Ma4Awe*ggO@n1*-(OtHZdKO5o|0lXN#0qXSJdgkG1@Kv0h7Ouj zLDlf^Fx1bvJUjx4)zQ!#ybdg;3x|q=1^9yjy1e-Yg|KKSTZQi^)c>LjJ2z?TvW3&9nPmMvL`dI&${nU&@@1-~ynhn2{l)uC=ST5AmA??A3 zP~CPMUol2mhQ7i^LN;-E#%d6IdSJUm=8A_zLlZpL0QeU*k`EpN7{dtre{ni~{62}M zaHU9(O5&(nTyQfu&-f1D^P3qONNrR_#S4^pX3xF_9Ua>p-D=T*B1$}PtCEPFA4Z?R zcuDmXQVuvO(hCx-6To`K&27`h zlg8@xKi#V9icz@5@d#56FogqI48W=)V$W0J-R^H3;9=IhtCs!g;(_%zQUd0^4X{yM zYLWsshn2!Wm+m^8&T9_jGze-rg9>;=Kc0&cH#!h9dX1OkESx+e{X-(ME7Ct73%+_a zOw)xGzbkP2qKl0cM|iKnN^Gm$t_JTOD)9fK>rLQl&foX{gE3|>8Y4^Aj1ZGGTQW3^ z`C|azI8H_DCSrb_biBLpJ41=OXNz#T&D239h`aSOxGxPcWAOG|CK0e=> zk<^pFX|cqx7qpZ`)%F@I{E-x7v5HN;?Uv!oSV;L>(q_eYm>+|Hdl$+2W3TDER| zf+7yARrRs{)Ozr-FWyM8`yPmZtTHFSAcYJekVgW|$x@>m*~r+Kg0k@PA2r*z*Q0<( zL@D|ueUiNhkY7`pPzA6~64Xn26>Snt(XF1)tGHbYOeq>vkkJCrA1}jmM2Mv#=0K@bsK${}Hhd z>aYg-3!t#9JRdGv6)5Mn67j-bHv|LN9vkP+xZh3Dn<9h4lIRSB)C| zL?w(>;B-ymhl*bL-MBH54C)n0^`fU2g0Ki%A~cPZ)K4F9EhSU}A2qS?TX$X9BSjp?$qX!8t7NLzPY}AJFX&Km2e3 zo(!o}%Ls}hGcgkEmf(;yW35X`S@PiUZ(f~7gy5FM&`4}9H4YAqQU!-y_{Adn{OJ;# zh5bG%VKq&~ssQrXHzIJC9VS%@&;z219z;l)nVGkCe|z{VVMdB!{r`2y*yuoO@HN88 z4Fgkey;8rQ{M)Tv%(#gxFjil8*t7e;o{}~+|rr5QOm|RdeJl!6c!U)PK;ne40 z3!I#sh?dn{L4sD@fdi+|FzAK<)`GOg$EY8qWc5GsKW6XS(|AOlkWANxp97pIdeAIM zCGBVn7K7lBI`-{;Mn_diaY#BD0+iJYp7j;V>B7V2eq*Tkn{9F|`Fxju6SH7p7$Yi=5C zBx9tGvR&Y%C#u-%=-1~=e=%VcHz;Z$rJYX9}UxkY#<4drr`4O za{S`tV9N;{6)Dak%6EikSqNw(Wd=U~<+VKXqEak?(wmIPOIV!VN(pL^pgqP(8r~I1 z?Xcf8den>0r#eQTIlGXQhJ!;>#_LZ-(OfyeJHf)G__othzWn6rwH@zjD>)UnoCOxa zpccN-Qw)|066vS7agv9V;>Ibd@7}db2VO>c%DK6tB$wmVNYMbqf%2{-jScERawGG9 zyzc9EW(;0an%pyV^zlI?XHUW46am`{-5@ZcehhAF)9rc&sq7si60odWxtmZ}T2 z_cGUvoofE^ZS6JFapOid@itme-y&dC-|u(*`b)qcKbU<#*}M15zw(0Hltj1Gj96It zr*o4Z0$Vqk&~)^|o!wLqng{4=C)PjWQ@=-X)$g5(mulw*^y^WSGTkaa&NkstzMo%C z+@ZMD;Z~16h47FL#B>sK${uRz+X>1~cUG&eYP?b5bxx~n&-5=b!u5J!97)jSmKVRA zpj2($x;1l6*-Y^g&dx_W_QlRQsl2@fFq}!X9ke6~khuS&i8xnrywwP8@8H<$C{0aG zSHjhh{$(dMoas4!sLRW3Jtv=>b17urtu2eV(G?$w1%l0vj9gesMkv^#v5MgZ9ja-7 zh}{!sG|Tl<`7$wd{GLhxGH65ZjM~A_qOfw3-C0RP1mWW;uO z-ek9q3m*hO(Jw{X1)8+({fHhLr@3-9Y#&h_)BOH>ZKVKm+}yoEHPmhTNZw1+2!8{P zKVsMB6!-O$2Ya4&fps(G^t6X>gjGK71Y}uyaeCe=YOQN#1OVV&S7L1W{r88l`NPf3 zDuCKgO%77&1YsJmyHmnne*IxRPHiy+D)8k;0Lja-;I?U`1VP*_K|ANnIlQG+uS##9 zx);(L^Y8oEVE-Obh^8*{)H((Jv=B}x-G%q%dMWzb6g)XE&Y-K30;~lzKK%@PHU&+o zQ_#i8MSwR73goa)5`4{i=1l#o8Vaq}v?2(ticW?^2~VKV2Gb;^xc&>VO%endY)Nly zhzTiW60o>M#_00yUXTFht1CXZqNo1JpG&_u{jxc26ykz0 z=e~}Sz*(|of|2+RT3l(=S)J-Cyo1gEDZ_eS!RjOWH?}P>q)MgI$=Lrq53O_h%?|3B za#}s(ge6`=EA2Ry9A;2Dd#fzTL(m)5Y~FlsNH9%!!U6*qoS(W@F{F?dBr!!ONcK`z z!UPYdWNM|JJf$UMN}fov_e!RGX|fXhD|ubNhb?!;y+tIQ+>Q4_t=bS$U&;|B9kk33M zD=E$3OD?H*Fmxo?hVd@lBQkG}^CD2po{|I51L##0w9^X8Nk%0Jz{Pe^uB4(Ys17ZT)BGfjv3kv$P;`>yJ&^ObP#Q9qz4-N1+G%hD zLW_b?#^2=V@f}aY>~}z}xCr9Qd;XN_N{IDHTmsR%5QlrtiYgyvst)ZGdi^s~-+Zwg zo_-56qdcUhY+$nIf9!^H=nyfep*MsKkpH-sQkZELj(bU$d9~tvn(^!&Zr~rW-M%}D zLMATu`V){_s^vnRkoricPX2o9oD?0Qtquv(9`!mg@S^+sd5z^F=ln8yi>CBh5r@RB zAdlUFddm@{2`11%_yE*QO6zOaaU(S68%0aJEVvkD(UGB9c+xA{8CWQ&cEpRb2^Dj{)^b2D*rBM&5s_iLr5_SSIihF*8tqQ$PjIcr{dbm{R( zJ_YlsLm#UNuQ(Ya(MxTD3YMaAz>f69PTU$qv1k$sL^-*+(&)9j=M-a7q?b1QGhyL@ z9U1BA9bf2=OM?R!NCOP}Y6L~};?QP_97Cb)%eo0-#MW8fspn~pcdB`S^iK);->+zF zmjAf=1g!uPR|&~Tw(ZRjIhEgJyC}|CHf!hq@3Vi1C(|K2 zp-Or>IzPK!kJKhvFVx%m;Raac(>v&o!lw+t3{YaauQ$~7EH3F3hy_97AsqE#+;l8_ zDfn=$0vdXz_aLqkt|l%)I)tFE&z7}k<2x zMHf>YPgzHVdbFPrVr@W@CQK}K81l7ediQSKZV&VNnJG@vn@!P7Y=XuG*e1gI5l{yh zA!YpqdGY$Zc+A>}rK?T<@r^MFSho|D+ItQiI$cEx@)R=`j7ZaValUW8=5KkHe5;{B zb>w3oaMDbtt6-7D*k|}{f{U+2hE>n_8-+(!tWS3cWQ)#NM_5qye?Ua{Had$`QH}(4 zl2%>6^7OMOLRv`$SEemSe%#QwrNFJisb*bF(Iq*r+gXvbhUZQq&6D^aTm`T4@Y5mM zG3c%+AwqF*(}R zy=O`j{Q|6qfQD zU{+3u_l9qY6})%1@o!*sf7i|1d7ATCZ25S8vg4k~m(jgv6OvDiC3FbR^bEC_|F~-c zh>)92#ac#4ky5(07_2rQ7}siFGljMinuK5jLbRo|Ezx>5k)FgnAnT7eG*E1XA21dH z)8j0LV+h+A&S;omDxR|j9f-fNT%k~2SyL4*jof!KX5RuCLl3sZUac zb480im2d46%nT76b%0f5ptCE)F}65+dGYz#d)QS zxpe~vEjEJRDK9GhQ^<<-P0{Ey+#2R~zjcxIa+<)IegtS zZKi$s(x65SpS$L)w#ACXBHVyE7gt&@TxJq336W*WNwdFh*eVob{S4hqB~u5wieDi_ z2%z%GK^9$`_|!8hm4*hPh!H6WbLQR(kLz-FrRXHh%n7C zfOS4#WnX#J_@nPD-0yx=9}1qoBTF!ha5+XV<{9@CBRG(WE2btW%tWQ z4vGUsTjH!h{w(JQj2R94_ayzop(w@vQ=DAl)_H&;rz;VYR1M>-y?>W&kIL)pm2k&!{)&IJTl0kggI%)w{^g{q)T#GO2%FB&*H^@Xj%dmx_nINOmK_52uSk({yn;MEGF z-mOV$z#@{*%$Ta8lbk5x6xPyKnNK8_jijG z&R0Q%h3Uy}Pz4a)<%5en8`eva(_FGq-awMN*QURnhx-I);R!DWqo=iajkJ!B>$W-Y z=P!Vyo|M;C6lKmN_wScNJYXz2AAySAc*!Jf)A1@L5X$q}C{=*;g^8Am-@${uOutsd zEJCTr5|b|c2sPzP%4t;DtayRMKCs}4tWVQv)mbTx2*Ob{FzDysR*^H1=s^NCe4~z1 z+#;dz?b!|w7%E3>BoZ9B(8jZelVwCsXkYF8DWJ~y52lK^OJHYsS$vV?XsSV zrPctS;s}a2?jYEHO_d>gqlO|To8aU)C}S{3NfjXd9mplMI>AQ9T$4Lov+H2COUUid z5vr&4BEm_Cf)5F)bj6m6DiCF{oa}6yWdhV{yP&gZb7tE4kt)KsKnzUKxSc=B7Y=KR zZ<1DNiItVY@DSDgt&{3`+6eH(`BPfp_DfKMnKZ4b;={M-UK>xdyB(Suo+yFe|byRRAHN8XhHutBY9T46(g!2P`;D z*%+TFIC$fxO)m^>rFho4$#>t;zj4sE&Bn$pdj`Pt*{9#Go{{MVsB}Wz2AYSaAnl>R z5p!1PDXxzSpYvCJ_5gi*$d>dk-%U$Pdoh_Osc6K_@Z6P~UHsnEYR^A@@GS)=jh_)Q z^BuD6D0_;LqNdOZ6M{ABu0w+&+<2g6AGKnIrmezrwKR*tRNJW|Tmr~Et-9Cu+y#gm z4DGG!55eodZ~>^rK!GEGNPkyPQSnGRL8Qr0CF2`q;sk$>JsrWGDhltuc^?tqw`rH3 z){Fa!R-;$R&-oUO&>jD^Nlj!5mh*#$)E}Ve3phrXE(P?Ls}}X z_vDtCsaq;W)Q(e$=7w~Tu;ZSMWE+ICts~THsCj!y#$*RABi!E~6@?VCQ)Qy0Qry^S zf*)uX*$}v0p4tkN{BH>Ed|9pnOM~MYsD_Xv)dgJnY);O=78HT3q4ZdSV+y2m+Dbpz zrk+ACoJeuSs|UZR0Hz00DH}&sPoAw-@=u{fQQhCeV~Ozw=dAlXpLaF!V7vNmZ=ZHO z5O2o~Qg}vfCEL+9uo^JDQ^KQ#bqj|R5stury+zL-e_X#_wmim#BF}N!JWW)LG4YVR zT1P2pHKoR`N`{7<;3m#r_M_rl8S;Tr+#|K-gs7jT@K_E#&V^q6jMd&P$5P}Rf z@DdxVl-^FZ?KBBqOlPj}BYcg4SGaLb`gtyeC>x!ccK)bSW>k-S(?aQ>C3 zIFvup-ZYH}08i9QNeH-y%z59=)FBeLe$DOld}RAq`Rb@i;W}SDzwHy^wIc%9iYQT& z{ui7wrVI639ix0WO;6D*5nj+?43!B7U-3lXZ1_exO4(I3Rpzo#mu(8uBd&N9AJ=(} zYELc(%<~{|e2}yo7JkxL3aVIl&JV~rxA1BC-}t}T+%_8_CR3^HB26r6UJ`oduK?;S z5Xb1nU{_p|KZ)0Y9fam9n$Vcye2SC1e}_FyzH@(2l#dV{vI=*&Rjs_a|BG!($MKLleLzqaZ=Wn2vK2FYPj^7r$>4|xi}%_G{&5J z%Lb4!h$5N9smMr72{dlWp^UkMvv9m0Qpz~SzEzw;D;%}pphRrlK^H}df|y!1Mqza5 zFJgyhe~wp&HTII%%E#y_?6VWNgMAIPxFix`V(!&okM`_oxQvR++%WZfL8Y32o%dOO z!ssT3egP|Ak#Tk1rXie_fWLvsDBr4<=Y8#s?A z#=brxbktI$3XS?2O0GeJXf?gl5X ztFmEZO*8VqUZxEcF|qRN6*0rNCDc=R-Hod={>i_oUylC?V!WldP=~29`sP#1J%+eb zuw!ayUA*2g1cFyATo)1`$;4>|Sh51Gm1a``kvoi7YmG$SBc5<_z_{jMwK1Cn3gx$`WnO2)y*uvL`VVZ;Q3cW}&?3&xTU|SV3jUB{ zfKaUc-Bu5vVw%GqzZNZC8s?lw4FL&!e#_0q3SQwofUPNTh#RC1Ma{XuLj!w zS5~#pC$bS|(Uy9S8F-m`(}0P87r$|}|Ip!XcfIPMP|T!1cOCSK$l?hh92`k((E?}> zV*HTLTLvJebaOh_kPWzh|9(&F%ja~Q>>AAJAdB^SMUnm29EksZ_rH09?Gd(KkVibE z#rW~tbObBK{iYzvc``;=nB-F$Ey$R^Fj0 zG!E`}tffG26|SM(6*PFKPDYeR(hD8Mho8J`I?pd~FP_`}MDDK@6;Ik-ap&H|H3-mXkd^gvwzG?W@aY9$8M4G3p=VVl08!sE}P!$d+oz| zxfO(6J*Qtd2|4lft5>gV(*y`vok^SJ{XKe{G&&6`Yu>)SU#}{DTTE3=@bSNR^IX3mh4&3j~1xnGOCtwZ-&JD_%B8?ehB(+(&$0+eEO$5k5Jk^L8d>G zV$?-7WUtUZD-)bW;fxiXBnmO$-+cKk-f7mCm;QpK z7fdZ~1(j52a9bzCp9Q9nANp`b(07)^lD?lq9xt^dlC&IKU!i@cNs}g%=zd=D;_q*j z7Gl*C4)vlBS5oe}k8^|W!C#!ek<$lmX)yHvx||v&fCCw7-BDNAMv@f>=k(#jhhj-E ztg;wJUMr}UGI7R1`V!*TH&vlw-I+S?WU4e4Kt^hqO#RCu>ix4^*Xu1?woF=Mq-2jI zoQZUg3aBkDPVkQ}4(a^)0iT7Cb4bsiaK@V6^wxnaHXe$kN?OvTO%dRpTXU&Dv7h>* zz3JFLxp=^HSZ7jg70tNf7a8!mO+^Js?X14o{>wk1V;~Ppz^7g)**ivT9mW|*>#$Jt ztQM3puv|(@qVY+PKTS(bjo>$WkNUTYXECSEMmf=BmZH_EDzuI=oPSrd*X(F(nt9YH z7LmvXFr$<7L`y`*Do#d8SxU6jFQ(A_mi%7t;);CB@7na0(yH0cA4g9HYmS?b=1XScu(y~vOHd&3;qnn6boJ6Y`9ko*Jr1HtQg&FFl3JB1lap5F8`;<1Z-4pA@ zoFj31MRYa=)Jvq#fV+d>vJ3PlH3*|BDfV$+g}q3kA^uiyASQfXK~AmWo&WXPKEXTU znfe(7>0X+51xS`}0=)_*aF9MKHGDaAMbC`t+oVMVg>MoAxANAzOrtjbIUKr;+-egwjE_seVzTYSu}lQ66UNJz}{$2#7A!H z9D5RIC)wOvBi-8{+qSf?a(d~DOE zk_D!gauiIKbwZe3cx|9N2WlD0Jc8=pG6UFR4Y$YW4Y!wWcuxG;2ZK_?@- zcNO)X&2k&Dl{*lZ(1ork8;Ch$2&}fGk&U-Q3au~I|M<83`8FWKw%D?u!s5a~>{;BK z6Tj;XSXc9Dzz+Myt$X_wgmYzkI3*En+FpvNb=U9Z)&7KmvtBt_;FGJhMYU4?KMxB& zEP*<9J7Nyw^H)KqnzwE1YgZE&<7HF4N&?aaoWQwZsfnvK2n(;fqqV(_#tFk+YrrVS z@V62RrmEgE+odf?wTJJzNjO0IsQ64+Pct0%8Md!5lQ@D;v z!!rFDY0w~z+Sb72#j42Ty}H!4TWV`<$SKowp##BeZ2LB4 zu?{s>6qHgIcWTwD)aU?Vv%!U;Cr_R<3g5nEiwuf-{$SOUl$62AWz(l$%i1`%4X}pm zsVd>--aNZ~cmH=4?>lw>;`s_bF_=z3GUAx7N^&U*h@y%joT;rz>YoZ_v4H3fhyoW? zYm^N}a+f5Q6oLVCU>CbcNJS^fS)|@_E2xtf1ZOFQHEE^SH{~j`Nbqk5q@xMaWe+Nx`{KXLe#iEv~XN4H8S6x{F0STw1z(;};j&jVC> zOyqDDFk##k{bR4ucncsAs5&xiQ^AlyONcC`Y4~Q}5>6*^rWJYqE~l*MGa_wAe?T&k zOE>Z=te;xG7t@>Qw`K+Rcu`jLRmx}Rc6R4tjpw+hux7jKI1lIvO~-ld(K$F|*+^M) zq{J=W4yFJEBlR)I^(?0e8@m9bHA#WXi%9L7(2&{HkvtRZ}J5Bgl zpuAJ5fM29;Fq7T~ZnmsFdAeQr0eyQf@kKnTG_(O|B!l}(36R4s#B|Yni7(~7Xw_aC zZi}Z1qwR=SCy4ouFP^xrwwl+w5(pkyjmCw91d>?7ZASEm05dR$xi-mk2#UBid7D%t z866c2{wI zc8wp8cv)X@E6GRl-otkFO-n!6!C>I)gWoyOZ|eFMs!g_L8kaxiT&m+3hz;?j+Itfm_M3a-t1eVAGOE8g75i~MwjkA7L{;NmyWLJ7)WUx4 z-d($FiPAGZrP8OD`R9%uZVS?gs(g#WF4*0ODr?g-P-}G2?wwX9ML)MX4Zp?wq*>=K z0|pF84Tmrtp1%(KY7B_ic1}#Uv>Hz7Mi9`{8W3JL-AQ9?%g$G1xZ>se(-XFDUL;X+ z+jjRiGK4d-->;o;-L#dEO% z7V&Fpi^?zq+v_)M+=eC4r&8|ZM@swW)~)(NzO*4Qw2Ne20xEoSzS1iHR6g-;?z+$3 zO1o196&HxcCnQ}x{h|urmCuoos>tVTHCP}P>3_iGZqQ7Vb{tZpm(2P0Spm0UQ}cxW zVu&g0&hLE*@Cy2X-IlkjpfO1yk?X0^g*IMQz`u#b0Q-w&C|M&nA`Ly=Pn68=_?Gde%;FtaAPmbyVZD8cs1cPEad>G*RNz?X~>^i&QNF&U@#Y} zQd%X^CakvjTr^KbJxoR`g}%iaFTP)?8+#;GCv2o^#{+~fGl@cati6WS5S6_6Tq5qD zSm@Mn)CN+J6N}Gfba&U}H4|x@CAU~;he%WqhTNg|j`jxqldYzXsC?B$3h>q)StlGz zQaX(n;DyKdWm4IX7)G&cQ+VXckEf(WqD&;|3+p<=u06=3zFnLd{-sm-A%Z+;u6Die zV5UdZR_qY&1vqH;v@9Cf;_py_YPKV{!$zkScoGg_tWpL5hSYE@MaC_# zd<|f|NKkS{Q~_Liak4$auPw{o2+zW)_bSB5a7E6puBPEvq1Z}88kT*CUuxf`5?e?e zU8iNAS=OSd9vtit`ZgHqrXb_@Xg!nQ;AWLi0GocoJ>&&~M3V^B;&aKgrtrEG-mkI5 z;6#?C5(mo=oK4R3d()ic0j5nLM+>e%s!F%}9lY8NzJDaT!!o?ERKG{mehMRIk|~qI z)$9?j(X82HO2u)xZ22kSO@*&;b4wvyn?ly8Oe7vEVc<^SPgzK?NG2!J^_1Z(aSOs% zV&PvwYn^OlWOSwS-ge!)Y>9ldXX`f|#e|~$v;#|>Z`>7^`YQ#NMGTqjsjd>Bi0NH6 zD{gOl0KJrMVv<{{atARDI>_p@nJVMEMLoW4*6jZMam=D6382?amt-a`YTr{jhDZh; zz#-y^K8`OsCWTw#1@itmvmDp{4}@WTysVx>3EXIkoG(MW430{VUXvE3^<($Uf*+L- zU?#p_g=>)Br;IC*AG)ndx-nCoh?UbA26sMg^|(g1cg}mjKzsy-GN3GjfgZ*0X}4o( zLIAc5Ty8s6fXD`w&LB_a3H9QucXTj(yt;tpkrqVd#jAL@nFlCZ7a!M?#fkVG8W(ZB zeeX^~mKs6YR=PyuTm{MiH=9C7aj5>hjWTRi=!09buDQka$nfZ3@soSX;a?g^72KL4 zmfOownc11kW=*#lrVMKJz{FJ2Z^uc1q7b|uTf-5YvSl&c+p|P9E*@NK*RvCyWq|Uc z^P8(gg~zqww^XQ>OnukS0BVwJUakNwVcw^@B)Wu*G_I|Rdsr5)pD--@fy0U-k3X>E zB9PqdqrWdmvGi_pP6YvwZ4h!~n*WG(nISP(IP3xrz8TK7IwA7|tecU7b8>T+-sVx@ z4iCOnHVWcs`@)|t8ApHmR+fZMP6JvQ-=tkS%SEyYAzRrglU>xPvl_4UEU<)dBJyDI zt{mUpl;hOxoLK5a#-JX-Hru-Wy{R|W+{yAJN7-O`3oFjYodgCDn_Kq_8E? z`lS#!-wW?sAKXKXyMTpr%?D~8wZYiJMVRgOF2eib`S(6w%P0Y-!R=CK3L_$8{8O!d zSecB=%WgW&#C6iMcw+p=Dy)U5>bd4isUPaT?WrsZKV&;UpW`$hSvc#kpD*x-D76G9NKKeu9jS1De8L6l+!Aldc|dDDM#V$|O&QP7WP z0!iYz$&S))Zh3Q#g&Vf?*k9G$W!p=BkF)kL)!>m+g>?9qmfoLc)M);l^A%S&+)i|^ z&GIICXR1QcZ?0uo`})E3h8n7!tTo^o3i6^3(_`&A3>6i3|A=%7#B>M*?lc+*QswI3=|jgdui~|z-j9_cBejtp;b%ePMrFr?uP;DTv{~L|xAA_{>-|Os z^|V)`h7AK#-B|i$y0%1tsM&nZ@%b50?6;0ele?mKbR|PGh@WpAsqLqHy)Mf(CwaL{-&%ZN)iDKr}0-Q4p=sY^4DJ`PvW94cdx1vCrCi@5W_7^pOR37oLk^5x(B2V z`|rK3Fv#fj5@fNJ-n7F2A%+(ggIW3fJlk0)@yDd&^hmVxPyb-;F?*#u7X0w~>Gq4m zQcDnLaUco^vYaWb9!1V~pgl`Z&cqr?wO!H~a7t@pgkOMJM;K;XQD(jo_1A%Z!`5r;j_cke z;q!T0MphKR+mfM~=utzk5<-4)%NfOR6X6S;z_PaJiQKjNgbHqDCd^?dpa&b<3_~WP`$w;^!MaDFQFy$gayi1c2!abA0ICDVo8sEZ_SmFRfN8} zQ##)DZ^mnMQVXyW9_u&NrAs^{$XKw53|EYSHdG|Yjw(^ z%fd{EECIo$h(;iN6g;V*hN+I>jiL+J9O`kkJ4m6lQKn_A7OYP&AHn^kl^CJ(*|?i# zBmArEjX*W7+)T?{bId3E|^4)NSUX>ndo+02xSc;+@@&HaEg54zVyTadW!y-Ha-`VW|5^Ou z!`;c(fbEr~1Me3U6i7LB?N3DgL2J7%q-#<7D>JU7S5`fQV>6-VCFguf^y&)7)b62KOOCNJMbXE&)^uu7iH;&_RhW~UIt-{w)Ip>Ls!O-UIt(w zKuh|2n`r7D##CeHh9^pKMu?bhw>NMDQJ;7q($_z9H5N87H2r;}vU}g`E3J?8Fg<2Y zo2uPjyO-;RP+M7XG6Rpy>9i}4$|Cyh;N*~7^)dvbCHhyOi<7Thn(%hmNaL48oS+Xq zIMIm*Q4p>!@u}aSL5HVV)kT}OwVEjJ4-WbEfvug@GADye|J3_Dhb8oFx`t^Jh`|l& zqs>KDk70sJ@fpxP?R_Zq3<}C`_bf@jyi<eheO*Vrnp5!m0{>f!Lz ztSlZpTT2?L{{I-^)DJ_)t(^1t>}l1^!P_k<8JRS0n|_w_nfjq^nq#jwpNmZWyKhq&4=jHm{TOc~`7$#z3tclm_sNg+ zb>6F#Tl#J(fR^Y~h_apze$H61KbV&VPesa$_=n~~gh~Bdnios!5 z{>*6eo(Pz$Z%%1kY-TkX@Pv!HH-gVbQ(3gze^08}c=$l+!vq!aVFzYb(X=p5#MVw3 z=ogP@Z*PCcscXVn8(AHiFW1U&4;{LY-thdE+zTOJwd`&EB%~Unr@tcelMKDUkmVK> z$a4YF2?w_@V}{g6p^I2#45#Jhxl0?bKN0CXdGhzITTS-7)xYHYsc9W~ff#8dhFuJf z^n1{AN2o3QQZ)@rp=kdDpyZ{3SKCHL#8)TeXQmz{8tsAkI3=wDAwzDE@Jkf|OR1Ms z-3Q4gjL!L=V!YFb?_7m~MG)@bF)&nxbI1PLGjZskiN?m# z>>jtiDGm%0y`=LrkDlVY?cMhzREdF-XjcgH(p4*^OzeHZQ$O^<*sVmUqN=yYpp3cR z77b&m6LuFv*ET?pv<|S{BlDoJvI{ z`De2Qz|rlrYOD+_yz9A{ed$*-IwZAMkO3iW`}W!06DZ#m22W)mM0T5c&Zjsf^DQ9cegqOQcu=7}hlY z@78E}0}e|#PD6;rma1Pe`*Lq1+Z7jGdp;i#9#sIP_1M_W_s;mVXw&A^gALW37~`Fyrx)8V)_{Z_^-;Y`~-@40hwNo@9Pt!E+;g#AsWq&<> zYzc~x$X6#b^V4P>v=|Tym$Y)B7C*tnFLxR%ydrN6*xZSP#nZQk8ojRF@>L_|CvxNS zJi?FO2eJ}TS3vkZb8Dev?|NMhw4-&2MoU=1KJ8-xQ#y|wJJzO(wA-rp{y#yU5-;Oc zPg3T;?Fu&tMe)$q7d-oAulR#q2qpwqIbFSUnNUWW`^(EnCG*0qYyZyw_T}M!ZrPDg zsLjIXp(-%ARBcD$B-2S1*F>F9#!O|^`~sH&zG<0$5)@UqjIW;%64*ZR+P?j2-Y4vj z)!m(~QK)&|bo55)S?9B-@IZ1=cbHmdz){5O%78QAF=G{r8^D^J#Ma;5T)IC3l^8KF zQjh4q!G2Ju#wSX;N9pM3yrK@ayJTJW$nwg{OxF$La=cjhnuHb;U{$Y4@JI^Tx3UjD zi@^)AiB%JtHESk=6!Z%WIj)O2454>0!G#8HzDR)V?lXH*_ znJ&Ce1dK4QGDrYXDDiNczio1YyNeE}{KvBPX@oRQ^Z$x_>@G|Tvt8c12L=V`wd|zH zk&tn<=MSO~22N5*Zu$cUB(JQ%~u#n&~bzC#!FKzakjnV3q6oFBbhsfgh)G%zK#>FQMN{>aaTRS2baibiYd6+xyosV-~@+ z8MEve?ZY9Q^?fbRJc#sPG0V;&N)DjzH6G4+&*KB!4XdHih#GEu%O&b+QbqwGD|CRV z3a{il*HBwq`(D{hNESM=nVGYw7RG$2j$6)l*nZ60-9g+bLCSnbqOs|;c|>HF1DfBZ z#QY@M-`__E!JqC2lv@Y=yAkfv3r`8`BdIdo(&nlXhNp{!j-+$3uX%jBPs|YkrwX(- zr61cSK==8@4i3S1%&cXrF&8;l7005d7xP1k697C-rsl+($k<^H%lT^B+A@oW*CH4E zAtM{{*9oL#>IsMVr(=3lQ%oc$5aI_n;FbI<{Ymn#c)iT`Yjr<4Iy(0ETd~iP?0{YddjG-(s)xgY^|%QwYO9Ctk}C#Cf~eqqrc-|qWh#}097H& zsxakNE1Mlssg9C|C{^TD?jwYurTdjk2L?)Mo5qPuVmGxnzqpxVNxLFl4Xk@ktcLZk6BFPMF)I~+Ee0nZ*IHs^xD482xD)u5%z@a;3%G*83Oix-ob zKJ1#Jp*vw;-(P^g2Zw`KiW;1u7d7e>pNYd4I)Yyt+u!$#U0y)2Gmz88TJ2d2m)~7CWB>Q4G|` zdVchuG8U-6^bHCd} zEI;2=PI3D{2~exk9O;u0inAP68Mw<}5)meVH7XLjtn+02rE`Ag%uCb>PEl;WT`J;l z!eljcE1b-?OKwOv#hcwzXFUXOg_E&}(=n@mzkWL>mJXXd7eW(xp@ z8D^MFT(EaE5v224%eY?QEwmf?68&&kBZ0hw1Ex@(5#2!}0LU|6_~NX~OP`0h1lEcuC;vJf zDU>~d?+LVHO8PO;Dg*x4^;Ma*5CbnB8oJ`{P^u_~Uf+K6jr+S!-QH<@swQ#6(Oy7A zC9OP7#$9}hkD5)OXQ_M)`{<>a@xJ+Z0vYALtqt=YkxhBM;QlW*P1JyiUZj;5@Y`Mn%0EE3n$j-S)4 zW-V))bNWX4Cz;!+b*O6g<)xkPmBknO7O#DL5nl6@aIfaATkj2gkL+PTuVrUH9?@|i zP3l;_Y7H|k@|*ODn)AtRSMtSa&dwiZ=561)HLR3*wug!X&$v+QbHCG%n0tRm8~od2 z@`o&!39uv0A;#4B&aK7!g?*}ijn=}R=)n{(qw*^+t*foE^6egZc8YD;wJ?HjTi+tk zzAJgFR;f;dN?(LzRPI$<11L4^jAif&TClC|7UN{Jv|v|#u}T?y<@ltpp!k%}ACeHwx3M_}Z%78mOAVP>5%*eNAx@jc zC|rsh{bjBNg?`aYOWo~i*^0Dhf}>mqZS zE14JKzqWh)`$$wmi-+LTLQNIDC-Q+x9Q$sD2|)}89TQg}zB;$C@KB(KEV!l~phP0J zx~md*VF_3?-gL4%H+-TP5}BSXbUyCwWAfCN&6^*A%bJ>&7EEul$PZBf(q?3p-kDoc z62Z~P*oMc1_8-ws(}fxfB1q?kG!)37aUlVTR2n0@{N5Hw-+<(BSfquJ#YGKIJO#{C z(Txx+iVn;^B(EJttQ~f94UNQya*Gn!t_$2a!AITCijt`aDS;AxB$z%4i1vmD{cY{F z1#rBjWq$yzK2jrMew+A3KBDFEW#*g=oB=4yD7dV(3#}rNG7#bpa2E{i6~b+$F!Cd4 zDtNRhTtssT-)ZEGLBY2mrs1Aj*r)hrUruhBGeEo0<~f=JOGV69l(D#OHuKa$gT_(V z1H?K9R9*gew@afw|N85{i_(XLsG9xZKfe;!kCpGCCOPm;Up_H^Lkr!O z1`SUHv^FpFTz~I=!IwW=+FgVyMy!nzCR#x-$4u(R(xKf=bgcgSiOG@>FWVKyg{}Ra zpxo=#=lU6b`2pLP?5+hGB>gq}3rhx=^%!%bNCAE&jME{%OU|KhalOvlqA^dink!-) zhV5vmNY^+dZMivmkMndF5e~}4SMqVYg`e_gm z0jkYHk@rxmZHmbtCqVh)Flo*Kb-${SDgSkk4%O@ZeqSlfy#Ky^+gL@MGv2iR;gc^s z^c32W1au;uxj9ez^9(k8G}wX^t2TIFVTcXJPM4XPs1z$Lc$;K$)2Kfi$ zf548HSEFf-NUZn@i6iVqe~i~OLnr~z>AM?6V(@{^A9r?BP z!K#`5ld3)+uDTW&#bIq} zBU~0qlGLqRJ+obsu8D46z>I5q;iye8{Ynv2E4=s0FO2D3cY=mJrXjfCO4i2+F<>rTJ|n0K+inb@VXn&q@Jpux2Zr-X z;RauOK&?@wI~{vJXIP)J$O_=$MLC0$8;t#iDK3cG$M#r{o($;IfWs?K+JQ0jI!dDcXE&V{o{>_#5VNmVjl_J;@g-_GCBSF^ zS@>(FfWG{?z9Po~K{PXD=XGdJKQNL@<=Os5CC%I&G(!Jl5kf@W0r}WRjJz5>W{j~4 z1WX1@IB-W-yOZ53`uEfR$+lZC~xz@0ydrll$ zdH3>FJV~QN8VWOaq}dWLAFL^Aq}bQFx=3_M_H#SDvjW$3OoPnV0x1NA=5Sd^tB>0G zE?WtyUoO|?Kgwqu^W3W^5`K$3N2zoulIFpB(eEelq1gq|eFJq``OE_h2lbaD>^GgO zCZG?J#_#szoicLWhoq#;5-Yo&s!(h&p@<~Xp_q)AhY!sm#7z=Xuk5?1 zxo{g~>dUpYYZ^9cbP`G&E^SC z8$VwTv*paLQyYKSGUhR6lt`p6nk*_%?Qr_;-?VA0SDcTZ-)Sn`!cP?UCTat+tF#kJ zyN@(!&=EmP%zH1FfbF{j=-a0~;^nwO3Zv&g;6-?WaaX2PD!FiB5*?`<97I#&8cp)C zn+XUXLST}FSeXc-r*^tAbJeT<{_y8jAM>Swd~^y_yn%DBD;8&5Nqz!CXCif2F>dN* z_Z&dfBxDCJ&4+@0gdUdUXZov|jx$sBrT0UHT<;8m@Lmbd*yIsRB9pOKaME|`2*E{2 zBL*!MSUA zfZ7YS`8YgNrRw0pgR>CW5VaaX0}Q0$)w4vZACL-@ys}_PNFkVCKZR-E(vCtBFiu5b z;~1?n`}cgO@V=(*v7$^YUAu9*_6kK9S?DR~NYKUi>c~)PV$S|Nz4owd5J(F@BVBd@#9uDB5rk;IT>I2flJdeGpm{Tyg`Eo32w2ipD5j2zE9t>iX|7Cg3T(bK;4el^W27a*z?@&5h?qG@hDkLBtqT) z{9>~@JK9ke%rG|Sqk*8{>QBcf1nRVS>cdA(d6EZm2S6o?$93`3(!l@&u}RqsAj$`_ zF{jZ-g4}iowso2zm=*zJXQc!TT(rF>DmYh3P+fxvHjGu+;S;im$aeIXFdwwOyHX|} ztgWfASMTfZQX~hgR|QDV%})clB02r#8RK#uRi@9&4He3Vd@O4Ve! zHqbz2Fm&Cf8X|3B{#o+_&QDI|dv5zNLPx=r%)!r!hPslsyrC_M9GN&uoF`ILxhuyg zjA~H)?je3J&k*k)$e`B zHxvj)i;p(yICo7m#WFyn0Q|xBr~`P(3amh1!kMT2a72Q(s{t0do4*g%0WBC74MeLW4M) zDV!`6P2`=p{B{l@yop-V7ow)fOK~peuktcH4|EQ!Arn-_Vh7hAj=+)t&UeD*Gl|g@ zO1lO5hqF$1g?25K_=A8k$0^;V0`Ve_@aD<+V;rAnhscQaAVaHbU!wZRdp*jTjP^n( zr4ivmt0WS{s7o-@?F2c9?#gLss6E`%B;Vzq3XW|&cU$^xpmesWqAV^^AlTmLS8R_c zAJt=}$eSXGK=v&p7haU6lCXg^W5`U#K(ha~v&DZxMCZEfo_o4(>S{;bNG=Mv2M|l; zIIM)n;fTZoGOugmAw6Z!+0juj{zSb#@5v|BL}m6CZLpKTsC7h_hLKQySaI!qQDl=c z#+50KQKb8gdiO$GZkJpJG(md~E4VZ0X?zcLV=p!wP$M?K+zL6iq}Q z#&PQ?rGN`I+LipL4uNTCLUIEoc-xH)7wT`kaJE3#^7K9&{g03+IDYeW=bYpj_T;^0 zoF_>0{)|oY)YFK7YhK<>-azt#cH&?OSyj?aDO?r0*6zMXAmzg*nqE#(wI?3)))9#| zpg-?h^ZFT}7CDn&3ke(^ohvW+M>s&h?%p$>=GY>E;|!RE4>eUQzDO1hhh{K#MPc;t zHgVaOL)O;*lfKk*p4}(VIaaU1*FvJZ9SzG?gaJf+VRNh)k54kTiF!-CS-7|54XKE6 z;_N(nb4jIBU{vbJ>EDjx>H`lIU-!7fkXwrH1JcSI$UsF#Pcmi=zvoUIiR2@r8c7DC zdW1@uM5TZg_mZ>gD?Vg`FUfQw0ITt;M~tx&^=8nWZrvMr7DQ6v7K<-LJW-+eNMW1d z%*pbueyuPET%R}elBfcJ?qs;5^U|e#StEr)eQMy{i$n&w zx$4ZBGj(;xV!$Y^`Aur|=3HioX!uJZKATjsux7bummHQsROT-y^ zQ6aM)zgwoSh}3VkP6h{Cl;BUXH;3lqctoQ^h;2ZR-a5j_k*1?R8X5i8iX0ik(6&?k z(hnaJc}HP6`S{dRI2+>*g)&95rc}?a5eE)X!Gf@jZE*o+<+*=V`dNHQVYR5uERqRb zWUbrQ5SXRVRt}=qt_5F5Luj9#J>P`i*g)VXYYbp`@3Hm5TTNa!X)XE7^XGHlH2O&S z*bOrH^F!HaADyKjF&mG)n{1|fPVM)sv!wHCMM!SzLV7b~8sZ^)G{jC*4ZDZjhPwV7 zB)vlT;`EG+2`FJ|^v1D)rC%1<;I-vnPe^-3HbGvq<0G6q&)NsckZ~%J8wW8|&lOg0 zU(II1T+LY?7yiv6uGdzO#)!VF^-eR@|1~@@KNBxfF7h*%9pe}`N%g%4vWUSP)VOtG z<2Iso0k772H4M53YK+TIPEu?uw6mMxUlpRagAS#6(>p+1+ z&CeqOumB^l1hXD*iqeXwp?Jj&CLPZv^x(v(KiX9N@_AXNqWWxW-OU70~ha`LlR^k+u<}h(C-F z;R5OZN7_PL3Hd|jtqerEPwgFXi&H2%B}4ZW9-56mUHsr!N1+|;Q~N=yzc59O8?|_x z_fLd9I)j?f0Lcc@!sDN$pu{eiGO_UhbvEF=hUh>`y^k`wDkmq$F`n@?dyXvae8%$# z@nC3J*uB89bG(p!ou-cev53=&Ws)e%6~B%Bio+pL`alRpHX_O)^WR}r-TE6ta(Wu0 ztEI4u&23vHda6hl_&|017`jDHv17M0|8DJ;(Ek(ed^STbejmlw$6TV?(qFT8K+C@EX>cHL zOn7V!!?y0#rMHm4{bgd8m~}EKg5*r@y`*xY-Ql&XE9wdl=q^}}qKHkF2sgnl@KzZp zVm8ULp{u+AwCnb8f*!K(i3A6dDz-NyE>4ODi^=qbhHxX%| zN5m{2sgty=k`S0+_#`}Pbv%FlcL{|WZjJrd*m@on-D=6&IVjuzzc%TT0DhzsW1f$k zUhgLCM8+}roQVL8V!BYTV@o@~m606y`_;XFUcX*SRWgGP`k%e)E=NO;4*{}ExJRuj z5ENkwi{dsl_io1P*O%eBBgeGUfU|Lm0(WAyU4bLe&5UcR?Hr5^{*?{94LC2 zfEH2$waJ2dF@R=8s`PcB(dY?C5uwq!{lo$;=jOj5l94&M=V`ijtZ}2{jLb!#N0~u0A`Yx} zsb>2Qj#xK*6r23*O_HWvbdH~i5aT2F_Kzb#kbX-wL}@~7ov;)xU%uS5)-v4D3ik5^ z09e6Sh^XdS7%Oz6B+X%CbP=kcwlKxk)^_7Mw-qarWp+Fagxz{pi2=KKMyOA3vTAEC ziOM^y7TfYmWv zTgs@2%1VWXXpn3YWtEXt*((&4k<82zLUu~Fk`-lUhW_{Kot*Re-M)Xf@Auq3=Nw1x z_v`h1Uf1J#T#xH<35D@DXjNUtMCQ)%&E}LjN4wg$=w|w9jA1|+dDXNe%MuTtr8hTo zWyncWhMlSU1{2hGxr_HH9n)C;1wN*$!OVh@FIts}%w~LODAqm#vfn=J45Dg!Zs@ag z?9qb(COf!DSFT-ayE442+1W{$c#x?3OACwqyFPEd=;r7_+0$1O|01Y0&gaWX_KJQY ze;zkb_Hw~i4xF4kGy^6#{i>P#pEKYpHsGiZ8Z=pY?9idx z+-Y22b@%H+1q6*>vSP&@<`6mcht-2mp6)!iC3KeHOOo`kl$&t914H|*!0GrFq3sCE zR|E{h_K)pXHc00j=7+$jOqirhJm&A+y}Q0~6{Tt@A_`&T=!D%tw#7p3X@{DZFXFhhO}>nKEUj=JtyJ*OU%gtt^Wqq8^oz?O z|A}BJvM}s_A{ebymU{cBP*u~O{aNP9WT@=wRaXz&9@MN&n>J2UMC=OxXIxdVRcP5@ zhF6}2m|nA6h&i$sVNB|9l4c|k7N6Y|r~q3}S*S( z(9(cIH1#5wfRI27*&#{`c@k5mgfm+O$JkB|MW;>=6SABAa+dxpy|WRNfbA@uwwc0Y zNcT_F->`;1r{{DoEeeBDJb{(C*=Nbyo+IQrp!~a1o6itIj3BX&)pxQqaP5Q_E_KhL zZtLLCAc4Ij+C~|=U^C%3cXoKAOHC=Cq3?%*&^eJ8`sv%=mE8`A!^5IrfJTT}N$1*C zO&T_ADBp)35@keg$Ck<^3r>xY=yH%XH9e*`8cb*FH9c5?gdfDvhOZ@xuiSU(-7#m- zhvP>Gzy!w~E=^|QBzM6&WBZkxyoGw;L${+yZ&?(-R~*`12XIU>k53sV!(ZqX#9E|P zTfSPlE(Sl_L$qbq(5rs><9dixMnV;4e^*4dsZ}Xs-%t z*j2Eg&-v&JYlBgiqZ6~)phLYNVJf{jIVjrn@k|T!V8?aQoQ&iX4+YNH46MM`$hg4$ zUd7+n3>EU7x@rh?&BJO=W{f!4AVkcffqEFqgR5nOF6PwBU%Vv|jKAn*-ZW5FK~qR- zXlcdJ;1Jqq!wULiCjj(1kWf;;i6S94iqG073rH1-vf;(|NfXA87rTl)j#c=UEhD!! zX`-w${P#;&L*VC5A`r5BiEfOa1`7I7HW*3s7@u~PDpf+h zaZfOOiY>X6dWAwX9O~+|aqd^1jpA+vFP+RnP z|5HS-@)&>`jVzrG=(cQWKrJR1<%0+)2pEq)&xK#HW5ECL^a4leNx=k*aRxKX+Ip2f zCLyv#$1Faq@}S4=9vOi&kq;38vrOlp85SLNWW}CFV@866ngMEk%3m)(nffsvz1fR< z#(24_m(pWqz3Mh>xQA5eeQQUb)H8%ePwF%|xGK6~)kqS7n59s24@-t^$eX>@cJUm$n>d5GC0t z?hwhvLA!g|$ZV9jq4kcH~pVN*zmhvTuu~LS7FcA&=MtXAOv|ZeA=WEcH+x( z?nu9W7ggsZr77oxlQvek;YH`kw6dk*WuEbt&6~y2O2qqvR{97zNDzCyqL12_FJGjT z4_2?pJsZ(x?SoWp`Z+LjVY>8;*0K~R*1 zd3%%ik&`cpM2So8+&!50CRU-4mLUgc!(06OC0p{!Pc78~6u#;n@A7>RFS74FxiRpZ_9d$wF4cp6N8T9?8}ah; z2ZSUdT7(ngZ$2*L5bU8wOr6g6&yXcwl;FrHRR)(dyV_hkGUl>+?wn|XvJdg@cvyY? ztKiwgsHpkId1RhjoLZB{!~XzNW~E4~rJxL6ekCzB28!hPKY`SipHbY`jV)`Yn63Rc z80h{t0=|Kg70&{dh=oWQ`N|64F)p!CiBfAn`cQS&jHlVzk$k86i-?Wz4PxFrIZsk5 z31G^d_|ft0MxJ=~V-ihCQGc=hqWcv`lee;-5P1}_jUQh z@iIz<$1SIiZK?V^e4>t0dr~S-6?XMlkE!ed?hRZJNkxm@FjxAULI0vWD`D`*>AT42 z+riuOI75aEr0(=P%Q(BfD}O{V8_?yB%dg2B`#R-FDA2J4UvMx)gGtdK1aSbGho|TmIwDAtpL_5u}zI5y* zm+^g}%SJUX&-Y!%<$wbpgiHG6TeE;E;Q2OqMl|ime4s|Vx<1o;=<)mHSJNA^)PmYk z#-J18f$So6k~%2MO*WVliMa z1p%9s0%YJDYra=2%dKG=Q)rczvXI@ebg|%}|j5P5`ljR1THEWx-An?NZwj)}5 zlBXnuPG!&2^9)ck-RwBJz|f+XRO##c4QMtQDJwnLEH#-j^|xq#A%0|C%Zlqr6{hys zUMlCgbaDFIhpiyJVf5=4$tHK9qCl9c)Lmf^Sk3OJP;Pj%oZt`s%bN~MvBim*s?21a zw(yX&@kUDl28ZzC+bP*;l{hJPa=_#H=gH;t>qH_4k>957l{wd@eE;YnYBSa1LkMBx zmhNQ}LnI?rsL+b~FYxDw5R)Gl7gG|nh7Dpwl-ya?iwf5(KE2*=1s& zL!9W(1l5hz-}|m;G~!vc`nq|PZm9+6Dqn6ou5?>ZIqQX~U;|d}z@bVbJ9o@^P>*F8 zCBYH8(PDo^#5zM}@zUsUnA}c!YSB}w?|kn0F6sRc!B9Oq%P`goX|TbRw-$`Xgd~-W z%3%M=@KzSgoruj?3_r7GL=(lf&|z=q5f&m<;#c;(HsoKa?YK6&rO-_LLZ!%r@M4b( zi0KVAOi7|)AVVLj#px5%fW|^+l(8U{jPdc=)PE((na{UFM8nL-rhWPGYunFXzdj>U z|D?SnO8i}9Uz?uQ58fnF*6gCKvW7-h_vw0}p~8dPYQ>oHtER{^V00Z$Pgdr@VP2l< z%*&LzL;RU(!${m!UE&c!+)7+vM#d56OW}7LzeFuVfR=O&owuQG-e{n&=xS-4u3fR^ zOXpLx6F>YmQnKd;n3swMSJhs(XaSgc+}4tIX{CvP!ICqv$_|ud z*Wm~g{Ltl3{z*k!1HKoU41u!v&W-bWD+^m7i%!4nM2Zo~lkm&a(U|}4Sqey$LjnV3 zo`iC~6m=b?B`!=iBEIyc_0rGKKoABMero)+HN#G=I^%a11rTTuIpy`}ejEW^u}hU! zy-L%?sra~^4o)63Pq$Dm9Sn$0?@DEHhp98_w0u`mQ(M6Ka0)r4eMAXqYrBe? zS~WV@Wj{MagsUe=N^|3J*6h>N73mP;@4DN}VjMJ8a?`0IrkkYPFavMVYMizBa zt&VqYCAGGgii?a!RlJND1X&Eg8V6Zz=)$yu`Oz1`>Z|HxHNVLREV=4Z+`v4vC0 zAV{!TYrcuDqPQfYeiKVHsqfLtrOw3?SIkc-)^6Mh5Lr5yzSaJLEzuFs4YVFGAc7V# z9s7VMQzSh@ZG}(WqZ>d|rn6?9pehIb({n9qc8}tPf1EdvqDe`tbI5w9aE3ITJ$_6i z(Wxj#%Ke@wC!6<=0g|gc`E{k6xA07pfvFu#Cv5r4gd|jr)J;~c^$0_(50sXLHnarvup=76JUBffMQJbYyCT6aVLCgx2aR zZ-GDO9BR4NGE;qin+2A({hj&_$lRz?uf1p6on1~C-v1F7t)FoC=-W)MuAat&56Asl z-LS|$)qX<9Bf7npbRV*+=|RJ!o<9-OePdP= z@#krP+4#YO2Mc%i{B?Qz-kh)0kW}hi#Gy?@?!P~-OY)RvE$ej9S1M*zWZ{u&vps2-A}7pv)E}9=2n*yGh4#cZ zP%Ji!{T_oOrPZrp@-X>`sn)$K^?frb`9en@^St%+=!>$pKXAr~${6wBDNsNm+crp1d74j6#-ILrQW-^aC0eutH}gwv0s8piQP~|^9a-yY zT(?1kI9m0%oEFGpVGph$c zZiLdzN4K&1kyKpNQ>HCcQV~6_i@}eVJ&R*$ljyf?drj->`}a4Suk0qCINYt#U#Fq5 zf{Qiwt$1cNmY=Hb4BEKthR9L3> znoRGZgd)~4jB=|lzIu;PJ&G2n$Itf`7=;};zoNPlW4r6GMj|4JPT3ne>(dhxJs95- zt(*)`NPZ0Zi0a!(Hm+2ymme?uEQq1cMr^qTlP?1N31RrF_}oq z5rpNrMMX7r=L2g&&WJBKL^4G~mNjYCitFOVyK+43e}rc@EPm9SbLN2}#HDPe4SUdW zpKvx+tO7~EF`Qb}!d}p}S7$|eu3gIZ1ht2m@InoMH_Jwjlb#g?>28|-yO~YdBp@rI z68;%oXy(}^)@ssZVuZ9j3xn(3{#}<`4b^i)-??(NYVEDH$eCjj95yFX5!w!eyp1Ul ztQvRaO#1DB@B&iI63aW$0J_5#jf8==rA-1kKhb;t;lmw}jCGCZVrkD8#7(FL-{hdf zp>?K|c0-VmakD>%^x4EfZYHAyvg9-mamP?5Wr&vhAH>6dNdFVl*!hm8YQq}DmW5V#7PY>tjuWW(&bG~wWXk~{omfUk=}N47Xi?=e;8TM-@n z&Z)DLJoI}w?6BQYzX@dogWLU{$}rlm1evP~x%7cDY?1ISP=}aGQZN)?_m0=txY{MCt|+>JeeY zzTdYqUSb4h52#j|=HZeUd#_AH_xkngLjdoTlfa<%raneUx>CavuZ|%d-hpqw zQKPns{2o$efS?2xq0Bn>3m6}ZURIQ4v?0QG?;cmBWd$WN<8R&N7yO^N9yPk%kVb@A z97OdwYyvWZ(pxhkv)%9Ps1!d;7)hT;ET=FP+gYRC-Tv}OM1Ik7pX!k)5kqP1h*&}B zb)>!S4{1gsV#yRoC1UmCk3aMB@*w<&@y21VWh51mV?w7L>wH9y!L+xo7043+x+PR6 zn3gCM($5!Jj&16Sh7%^i4x!yt+?V3`)DOE=&)%h>p`lO5q9hyMXLPLkSD}p&(nrzx z!~38g!MrUPPRSlCn~FrJM>>?LW2(JJeo0Uv+&sd0E2hNiE{k3Wh_h{NmsawP6ph^+ zT~*M=tyvYc0Wo4u(n7|NqWd^HZd}o*vs;AW-D+Am=@f6OcjjkZw#;+d8!oE$rED-| z{bg39TN#x6fzPdo5Ml_4Ib|RCsDDxmJkqI`2Ftr2^86T8p0RY}CwqI!(V(TmoF1m;m^Mh0GV!&T`wo;S#9lRB# z_4>`58(}k%1|x21X#@<{?jE-zp!Z93_(4I+PQRS{WrLYd+B!g0h$+7E9Yt-z#EjU+ zrY2r+v%9giyEqNdblBmras#PX3T`s!g)S3k{lk!bh>2fUK%3ZxZC|(@^1@&<5V%Ly(LF*={kUVsmpMd@rx=1FT*T3ZA?03hwAUKblZ z>bh0Bk7m8x!lIoX^)z|a*SV8jIn32m#Rb`2=NkH^ceaTxhAu2BnTM)l%i)smWqXU$ z=83#jBmuG!Ib;Gi_!dnZV1+?=dQKpdm13wt$dT|Yd^7re8J+;0jlEb-L9;1Ks|2{= zy}|J1_J(hVqk-!FKDit*osq$lt&t6;SWRrxzI}MopfZYwW8c2PxZCQpMw-T^?BqU^ zGnf0nbm>xZX(lzUA-ySy0cIQemcC5Ct}N4xVJnF2U^YqxN9`cQ#X*m%j_n1#KX*EL z6|;U5WaUa$=Lpz@t@7%2H(S2-A6YiIU_jCM?nF$4(4vEo&e>oSa)*J`bZ`Kz z|Bq-cj|68BqKu6k*5keYe!o334>WG;tiGI}HR8ACjZR+L3Zjk@N!QYpQKwu2?^V;n zcp{3S2@S9Pq!`4i=Ydr9j5-ib2n^E%Qpqphy$251;(Qf<>f=&65xEE25)H)92^!M`ZxQiB1CWJ$39v68;%^u&(qQ{7m!lA`aJcr+*n_J-@3n0H)OdNGOu+eY*VL6d#J@FE@pq8AuFMg}U;RooO>@ne$oU*A3>s{}Xznh!oey35{e+6A3L zhckZNB;fr#wC2v^nfS~ei&$n9={M}!Sk-tmh?+Tun%a5@38|zu5mke#{;kuq z$D-qZojH7TCZTirrp{P`arnL$fs5|Hq-Q-6OS%zW?-;Kl|E6RWYslV7!tw| z(-l>2h?yztR2f0A4ue(rELLlP?%nxSl^{3a${`j9ijL&mOsY5WawKbohm}#@Kltwj zqSzZa@cLgkRC0m7PN?_m7w%HD(w|zP~;ZQxPl9xJ|k`cGUvmk7R*Gfm<@yLOyd03cej&oy73VX$A8qZjAQMW`nQHAwGLM_4PT z>2z{M^$m~`ktuJKCxO??m?Msy`)DG?+^rnt!oR>VaW(by8<|&yHw;aHNb^LZL0fr| zbdvRFBWU?U*Z1i0Sl_i1Q+7)~PFh18fpFC zokw21CmRoJHcmZf63heoRWLCy_r(@_aS|4PLWt;gRzk~D@DE|c6MDz&9)KuHi~+{~ zBQ1v#P=wbBG0lJheURwsoEw`V)d-UM=`ZVuvd|eGKHgbG%?8E&zG;-wRMi zE&!U@7k_F9CBnDs^!%H|S&u{a8l|!lNVfZlVJeN7j;$Kcy#lt7m;U_OGs`fLNXN;~ zEG4wj944{^9n3@8DW`*BMLGN<@eBzrdh}{ZE_RozNgc5|O%6dq3e!#So@m-biEKoS z726-X^9QJ@8or%zvGI%;uDx7#2q@@Grxf|RZU27V1L)aQ2wpsXA|uUIlM>=gfg_3Z ziX&>2kUwA4gNX{SdA-!%=%l4+NUjCVf$XcZ7ky0m<#B^sKU(b1d4^N9uua=8l@j#9 zVv}I|@eD(VzB_94^?MjM?B&I6V$vrKsY^F$dx{gEw1t?G`WRd!3k=ac$Q~}*Cq+(O z)?RSC#pU}9DTWC9*L1gXO4_{8P&GN&TybDP292|ciKEe?fk-&pG81togqrrt-!`#j zI=D%-NRw?p?s7yJs6@okSm*}ZkCdZRQE#Yj@&SgCxG1rfQ$QfC|6&WDJE1MbLWN&H z=aF2+u?fgmBbPR)IN{28F=ul_L=DqDJWpX+wiW*z*QZ}U$p-nBitT_V^l)Nt47 znhh1^g#+2<&a4mtDLsGM(&@O+#ZXD^He$pG=?ink3h@+?>d5qAJlzyHByl^SL(&52 zu&M~Ru3XBz#H<&SP^zic2ra^3P72AtY*&2z)VvBnJ$&QF*Cij4vuk2vvb4iG^tvUq z132%SwyLMcw@w5gy?=Wo7-q&ufdq}=t9#FF+ex}fjvPO_2cz<*lvAuznd5<8 zX-)jw$5w4*eAEg6Zm}>a5ohQvyI?QZJE%%2^KN#5Sy~x$^Wx7`l}DK|b{6y{69lsZ zrIRQ2@8AD6;)~Ff3bV_cgR?B$RByr0wg19t7)n^|zi>L}sU{jD!NJIi^Yuw&CTUuv zgwE0xA=GaECy6$KtZ=V4YUWw5i=g9p`YrVFFO5BvM{9c6s8Nrn70sMAE0%~YaVcr; zFIfbtk~)Rkyd3GMx?9Q+EmGZ0T0WZf>WvE(iI`YY^0p-ded{xD;0g*J4^?fKAHU;0 zdq)`-Ag7^#cx#(nt5Ku;k~pmA4nmfnBM=)3^!2<-+D2W0DSgc4fan?G$Q=7=oQFpq z8$vTk&$1*@IFQdX>Gu%oWHTcXTtaZ;ZZqE#f~hd2nB$>q&&)0ZN21~tg3?p%9UL4y zr?c-~ekI@gFPJFozhD-G84`vIw>8QrdaVr&2Qjncr|$#@EL1-Dz_j~1gi0Z}4i;XS||7X%kUVMx}5QytD*`*w;EYiFM>r6*iJ3we%gm*~>% zgAP-Mq6Ho~khCY2fob3R7WbwD4OOd~xi@GDrya_ro;k}^Ah-XhGv9CJ)-DLJE15Oq?eH?w=P&QfLuey~e~ z86U$u8YIB1MvI6{_Tx~`Y&7v$DrVLpA9&3weR~JbT2GR!6Qt)#Y!MS!p)1fD$i%EE z0_5zvd)bL_e&q)Kv8baMgOF8S`$RZ8v!W8C4+s*!l@i<&Vv^TQg;g(&vX|T zbDywUK?l?R=k!H(qBrbA?^_C0{t#U&4p0VqT~Y5c^=mc)MddzZFa{7UC&~uXf!FYA z$6zHRlUb@NTI!c4Po9*#(X?ZQg;QmuMHUG`0AE{CjwuciTrWj}jsl@^DxJ6vDEDO2 z*w?q4)4s#R8`&c%-bfxE9`$o+rgy=zMB2*(Dk+JYl6~O~(Y4ZR6mhx`CA&Oe{a?#q z3C!wlpvxn_+0=iqV}`=y_3r)t5aA|K^Y-_8v>@eI+8SERvfEhIVo#vC;YH8#A+^;v zn?L-wDnUpI+V}E}Ykn+tM!bp=4B8i&?{g^q^K0Z!(i}uZ_?fr6u`!XsDxJh8tQIxI zZP!y6fQX|Vb(}uJO!%O&s8RG=wYrRl46+UB+cG1w6^jo21pw?QsM?6TAK*dR5#1Ku z03bqEF30h@MF>IE=p*t_m;+ivs4Mh@eLa7$jM*$Zc3&YYdVRg?yCINYQ_)ttZ(_V*eH>}r%WkMFB*2fd)a+NYlbT7;j zl(>XZm2WLY&o;g1T8c3Nu=n0#|cf-uqq zB_Z}gZDB7t#x~$y5nu8DI{Xv&(eisyI_KOaG>T`MBp%wAv9uAecA0EdYzR4Ut5 ze#8kN;%BPEJsc{A8|agS{!f1{xxkVk2!@_9wbF8LL+6r^HjFzfC8?3c-v>j!=`%Nd z75Q5ApafL#Stpk=Nw#$vk!SOT>Ndk3H9EQ|B*aHX#BfRg!rNFcelQJb6%|=#zLpa7 zAOinoZjKnn(d1i8fpmN2tnB16GQ}pHLQzl_(vEFJJobSx^v_SYC~#E5VbH;e{|4mS zqDK_-4GNqvU6fWJXy=KiKd*((lSJ;fHa~wndXKxRyPJ;#C}RYWMm$vM2}=JAvYHK= zj-?1mNozg3{y8+FM`nNZ4Vl2(yUn_--jWnG5pnM6F#F4RDYS}bpIiaO3snOY*)M+x ziB}ddAXBFDwXs%HJO;sI%AtpKm#BCjX%^0idl*X?bu6AQ4Oty6(lA2!Kh0>!>9_og zQ0_(iPmLTFCDD;EJq%9ShA*aHCblRMnF{x)jV{ z+_y$=krtD-_HfK(eKVcD3HIiSez9=EOfSjQ3O)T{bLXB(xaO81W^fYs#VN?mnp&f8 zDUT9#uqD+jMbjNr51yge^1Mc?vic8#Ifml=o~rY*YW@0Geyh)4d;j(83|`(kqlLl2 zAIF%>O-%BlxZqC?6Y^W;=~%ZDFPAOavRAKpEU?)Rv=b3njRy!Lw-ci51V~np%?#C> z^l+BVRp(~n4Io6LYy*MHZVwqa#2IQa9bvp+9cf$Tf}EvI-VyUx1WxPMOWWW0FDlR1 zZ__42emhmX$CxpeVT82~?x||x!tb14S&4Sw0;vG*IfVP6Nw+7i{V(E8^puxkwwnJ` z)b4QP?NyCAU}qdE5pJotG7QW@`1R{}CA|zOP&Pp7y24_@N{Nk926fvlH z+ZoGjx;Rg;_TC`NNHJ@*Dm;8A2-$~bw8i?4b9{`6#CPk z|29B(+Fp;^z55M)kSr|amM3r-=<xG0?2Dc+x!fn~3&^tn*?Y@`3mOm>nq~Rd7xNl$P z9aYAO4-;=`og}==k_^u86%kQJM(iw=&qfa)Hf-$<7FQ9jxagKYp696}QCh+Tg)2Bm zKMB#`)TigGs0e2nk_=4>pBP+%vnNKhg{dO3mD~D{)VYxvt-vg#LYar7?V(s(jvc#8 zTIw=FM5wIsKoGz9auXJ|SmpV(2mNO9--`f1>ySemap*jQpC!&an>cs2qsB831J~%~ z&mT z!P!CYES9X=BHF?ZjzZS;iB%OpOei~{7#deoYzNBJ2OYy9#Ccs=`y|>233-!El;nL< zL2LWJYL@%*ET2hSB7`ZtOr2)ds&^^3YQ3(M*u<~2-B>m-r%UsboJ~h%J>H3Fb!UH= z|I5SBrPI_`tAy%|I-Y+4BE20zl=;iPB909oCg+e}8u5H-#)4)`yGeEL9ukW&xE9z# zg}_Yh!=-?JkvtWDyVz4~rkP(c>->>2Up8ZbRMzMs=@&PjaD=|{X^)8=2b`1W-lInE zUxo|h0(1!;d`ZPN?cZyaODGiG7P8RgstAH>G|yBxs z%8(L+T1gx8%(ti2i1r&ct0~!SWW@51K-x0wE$gHZ6|JdMl@PO}^qs74 z#juk|0TJsgPr&xrlYPr4SRpTf^g8C@e<&mhru{S~RbM4krBnf^?4`ZawIl%rJ!$V> zC=G@Ou2-Q4Vfclbu!@E6rfZlE#r*kl%We9mrde8F3k!wU{w!&>Y18EM3loQtwjt+t zNk(~}Qcak7GoF)(SuxdaWA*yx_5}#WuGkz`Y|DtDnnXROOmwZJ45ZKGj)#+CTlMeN z2pf)KiSZF{6we%jRk|pv#h=|DJ7DJO1`MpM95einqzeitkEVCK|G}yBd&H;hPu$8L zCWS18nT&o&4;Ho<;flBE<-ma?pE5Z(5=q9vu1c-*{EdIS|(O9^@T5kIIrqH3VLGGtG{r(tJ zcjWC0CD5Y9`C@s>pC8YDH`4UX*lP^{oEukFR)R+VgmySVbBHU^lE5J&J6tefA0ik2 zLQr@MIf$+m74~q7QQ>6erMl>plFGJ|MKc)qGq;I&C+pIsQ}w?)`qUlNy{v8AAB9DK zEmo7VlWjC7_5apXQIp+(^6v<41V(q!h`Iw6d86B>P@F-UQ7_i1^YoZx%fRnfM(bKh zZ45roND2(}_2&(>HZ>iN!geJ*W^*f%q9LU5;e!T^_}f{P(HX|C*W+BLtY2LEb~=sz zH)`BS)IUfC^aGW42CkJzppp%#zDpEfGBq9=8v3tV!nHCEVtC}2swI?2HwUPAj^{4w zm@c7}7zbiXL;OCeVkg*lc^FEH`}L&w*Eatz`LPH9?Hw5P(HwhY>pses{$WzUi*pT4 zD3Nro7jp>+T%o*m#q$T!xr10q8C59_d4BsjGqg`+LbPM5Q~}a@CTwxEY->kHN9Vz- z^%eTt6(A*5@#+_%T4f|5^rIjGDqJ>Q4gI6xvRy43e}y=txw?J_?TLhh1b<|bGTjt1 zK{^-|0iSt1;E^j)3qeF};D-mZLQfoq@aCO^G*^Z;WfHnY(T%|0J6RZQ`6K$kI3|xo zR?GKNTC5-F(L|M|mCZYlUdXx@iQ81DR$)vJ$Q}k^hrys}hyE}1(Ns;Agh-4+pjfk} zX;l#@ev1?rNX6ET#1Ku6qlse%jUzp59K+Av*rlYMe)!N?LI7OW5TMYxqs-SV*Y55J zn~~e<4$%1MDb%!5KInQ;Yip3!kz-S!VO4gRCjg`EMZd+Lb%L0OD#`$tR&CNUcb9yV zWSdT8xZfXE26}=*Qyu8o%2cCxyn#s5mX)dQBplaT;+=)|xF9ahQ>974#&D*~znjZ-UTHW6{@z7RR}xgy+{lmSMbJh$=Vxl%;vl zQZkf6OGx&Y2vcC&hEpsCr*z~+9E-+SeADG6MV}XiBC1_0A;4+K46@^PYu=dZJ+Gv* zW4l+bw=fILl&sMm(Wb6cj#TbzS;m0LNJPg!4ZyO&3m29r4A%YA@)!CHF+-_-YH2?h z0+CzNSiP^`yc_eukZaefS5z+Ki*lJDBq3VD#rS3H?t~fC(A2z8-AL~hEmF~K)0q~P z+TOiwAbg~X);AS*Kj;&yboZ6r`(kmJ+zZ{UwDklkNPmFK$a2PUhp==DQXh*UUYibG zOL@T%inl$o$?F~PXAiA0>cxRAc9pfge{Wg6d_&qD;Do893Y}{nhNBXX`Za?V;&RxN zR!Z-@^>u2r&&9u=6yL_5f6eOE+u`-ZCSz$IRE{DkmKSM_1mtdAeCrAHQ*JM_5=qQ2 zyqwSY2GDgSmu#rAgs(G-RZwULR_Vgi8?+xg>gRCk)pD@tX+5d{ezY#pCSLK$-+g8^4N5Ad|zi7MZrSP>9;?og~&mRkDB=P zn0w*-w+b(Fhi_gvdQy+MXTN{bEcqM$&CmwdE7>-ZsYV8(M6FMG(&x+vb+bHJx!r-k zKW(!fIB*yYICDM*{E>&;*UT3B3dRRSMTh13?b>#&$;+Z5>YF2l2I;ufe= z6aeK!f9BnjglmEiN-nsE$AYW`uo8h4CuR%}%*-od)22PrXaMm5U9+$#I$p7%613Fg z#HM>y_OC&cNEjLjVLccvEy{?QPM^dW2e^82UuPooET>{K>p_E(3e<=QX9_k|Oajrd z7UApKB(mwoV{>jd{rmiQvOLUsB~SLxV~)-VP>#6~jX(iHYYS^n$BMHO6n>ufjZh+A zOEZi|`1Zt`c7mQs=|eb~;G(N57FBIZf@6KaR26^s|2BjsYDl3fl8WjrRRP5EXMkGd zfUc9DEtYW#VG}W`{3o@0w{EhLN{mXynte$XS6LuQA6OnDr=~+!xR49pocYvvR0NFC zQscG<6mD$!wXduIqz|*5h;1d6_{5VXHMnh; z&G?#vfkbXG_2;)qge0@$GoZ9ZOOJR?40S+``$Vz`7#%`9zP2J{a#NK~G3UN|@ZP<9 z|EAqQ7**+%muhOKBQ;+ISVH0;VGLQoZJOw5$4dbb<8;7|iJ`;6gm_xU(0Yju3Q|B5 zfxc@)3t);5_o*si@>-6fhC7U&x#x<RepI+MF?mA7 zd&ZZaev(>m8e)ovAB|<=34Phafdi{b?dK&dzqRO6S_|mcR;h zv)zcQMm$~w7SSLOYZ>aQ-&e=;icEH`o7u{O3(Rut?ZTc>iPw0*bBQGii(PgWV_41U z*oTGk&Y^yB175+qqEq(Hrp=%}eN0?&teiJ)FaZw5RAS5_?&^vg|$#Nn46CX*O z{m`b$b|D!rCx!G66K`ca=VYQbDE*~)d@-S?YbAWN=x)TV+4`guuiFB#bdib^xY?p5 z5KV84|B_+GG7l^!WT4aK#xj7yUKvWV%+bF;)}y->F-BK{jj7Cwrp80A>_fR7%ad!+ zd2$pS=vX(Ch!#}yw^+5hSG1qx?bMmH;f1#oZ&HS!O^ur=LmKhcGo|$^O^bT%oTa)U zWHsR%&JEL*=}@lC?I*>*XPkx*Q&qtO)y7>eshlvk0tD||n8N{XZ9P%i2_PUuX5_#6 zdTMkP!JPx0tEl3lzVD1DbB@XUzDRTUQ>6hcA{!LUJpom1^YWfwoxO z%A_*%NXogBOGiMgOGQCR9P$BPOmIkYb=h?AZ5mXO4Jvqj14>cX2owI{I;k>R*Xr+T zw?FZ8A0|0AwjZ}bsDRvEjp)%Zk~>-)TL<+Ofm9mu1any!CWb1ExwwXM@4O(i{j=v) z8ZaOv-GwMETtuVa0kn1p5o9v>R5}R23o4XX9-7hNlWH9s3v}qWwBjLrq?O22)y?x$ z6GuYn^}Q##82G0`pU$Aql7E6v5#9GvLA05d46D(T+VBvn_3sYG;PD&r3LPR|O$L$-SjVDW^00f9gi zwN}zq5%yb-VvxGYqr7M}1gSD7No*P$3VI$QHrG|E@_h6cK&BpH6SGiAvb@1q8*w6S zqYW)L3=v*JM8HBZZk ziben#`vsx&TSA$Gc*)Dv$ zZgf*oeVbSW&HTYS3gmiYQNeS`}k+>A;+uq`1TcYY+qbq8nKZ83m`W#2{MOI_B&b2um&91lg5K0RQOVAuFy$=l3!_9KFBH9bX$>dq6$x#@LJen%9 z%>a+wabhEP%=eMB0Ib4bV&wp-j3;P8XvjLvT^F@y4Wy>|DL%RB>DGXcR=?hiEc_NrX3vfpopnWjv0KiSXS_WOwL_8>VKeFja7y z=)xWF2^9<%sm5Apk^SrD83uk|2JP#usv68UJEb+XV*Cus>SPBz8&Mr}`D81fZ-C6j z$;IZLhRZ@zPm+(A(HuXU-9e1IDd1IZLC9xsQikkTB{?3_!#I?#%<~Lla(tbxP;a!n zhSP(%d^znXS4Gu&ILD&YzwA$Au2ls!Xp2=CeUx4++o>q+Mal?Sy^G-dil&#yZ{b^g zr*}|}dHo9^8mD>i*7uTZBs$7(T2kf83hX_Gh;r@RdH@K;7gNQ)_SCYiWg?4OL=Ke9 z@^)Qu`)A;nw_RH=eG}N^o5jW8pjw(`DwKD(sBN!NO{?0`^VVfk`?}faEbhC!h4s(- zH?}+Lc{hBU|mo7C&K(XZo&4)16KY4ZkhygI^ zvE|nwKArrsJj&;Fadn+X_$j9#mVBRouYKp?eR_Y^Q(uz(?%hhlp$<|Bw=Mhkx1(Ja zLaDW)SJ`QWVB%`q)xdor(b0N6e!OZ`_%IfQDYS&)97e2w2 zS%6NHpH)^O;a@uuuDdolJVg7H?eD?b$+a3cUe1(!muu0nu~k?I+01Wyx1h5b84<;U zmy~?Z#oh<}KYM>GD%#2B2o`13glSs@@HW007PFc^9fey%0?j!pb-g>hqO>Zr!>ysj(>;_yaxw&Ghvvqh$ZsF5Zqbp2OiX&raFX zOl>92k=xF3ad9`Zvg(zsmsULjcyUefsU=hoP{OPrc*cl_jY#)Vs zu_uDxqbNd4ZSFCr0Torml%c(QXRJrd?1!#hRxLwnGp^Q+(xECl6-{-ncXJnYrC9+nf)m-EC$Np}6F$ z)=sq5PPvKaL&ef>TKjb04jnqUxVx9b1$DzUjOT}7`IgOOO+U!TTm07bn>OVVPS@P~ zXYr_hC{0ilhH!G^g*)5WRL610;Y>hh>T#WmRU5Ww+qODgIf*8Ove(^L9zWihfhT3k zlqoqEFI`&PtDWi*OpKu#$%V}L*lvM*;m1a59)E(PHxJPjM@RD_#C*N^49^|pg&bj^ ziCM0O0I@zCQ8>Uw6u4@1Lp{R^>ea3L6eAXbdst{7DXM4!kL}d%u}8J1LRKGSoaGkA+$4&gUxv85U54TJODm z^JX&KpPb=Wd3l#(Q+`05oZ}AHCdeYOL&Q=};e1q>GUb#14Pd}BV18QM09Axn{l&kW zam2im*bygY@ngo0Evtxn_AVy-$N4Q`m3aPbMxM~%2llnItBu?A5|D?4v=*NzR)t`x z?7TQMRKr}0azk(b&0!4*R$5wGmFHDMMp4IHOE#_@+E^OgUlla1yX+RDsf#bTb({NB zi<7P>g9i`pJiM>26K#7K>Ymwoc}sv}OQ!Yh+t+W>qIeD?XKGJExE4~e0%hB zI`h@5{&_j7uj2OSPSzeOCRMoZX((oc2GwNbAYuxNRoy{7e}8KZfS&a3ad(2F8v>~H zU1#`ic>$bNPpLbqx(l{d^WJ;R0El~8m2VeO$%xIT71GiI9)y=Xp~Z`%c1A`9@eIo< zkW_yjGEcta%(oz~Tyn0FiHZ7LL@)D%M^7_C&p>z+^BXGh?`fi*;k5HK2-aV^I9wMjxDuvisg@H`LR#}#%l zC2dnu3nidL+qG-!78Vw|`1ma2JM847RewB?`^g^?3@Zd_x-%-OJOx!Us!SB9Y5_E6 zrfoeG8L3bmA(vwzis>o>YkIo3kw>%B4#&1BC$20YoKE%~O}WgBbvM!@Ph0YM7+;kmtg{OY+T|2hJWb zT|+73PI6D2UeJJGP7-J^HMF5m6YgdX9S?IeGX+KVs;ygVthOvaa^%Ru{Cjoy-pQyT z-)N5Cuz7PR_cVu))N8;1HM%w1ckQYuzwPa#hWLpx_G3n??oBsSZnmAfU9GI=CR0i- zW#K>s`+eZY{cxI8l6g|bNaF;2_5L^=si~Tlk>Zx11=Vqm!htYRZAlNHiTXUrOkdwU zxyYS)bGUokwr!UrFZ!qi3zeQez4T5j9%TxVdY(~8y|pwBnlO7lnUt+Z#gqcmlwDA; z9NOOX_>LVqyz5XXH+Kq`$*~1o{%@eW)vHzo@z`9y(-C#?^&RzY7Qwf+re+21m0*k= zL%1RSS7P_<*?Hgqdrs^)JbDOk_l+bq4p>Zqz6gE^J8E zY6iRa?hWPhxIxD$K~zjGs1X(~U%v0?Q6ig+gLJ&}#Fj9!9%HTpH@rA_edwN~6hL{n@p`{8|27FM*WjY%>`;%> zD@Prf^=YiM_^Al@9GM9Bd;p8+0HguxUyrxeSzQ@*;=Ge5XYYH`-22Iy?%W7$lR_MQ zJiX(u6xZbKc-z0xcs{aBN%}iI7oQ8m@+O?Bpm}P&dzX*Qw|Ir#tBiipf%|k89gg8; zjE{eV`}gl3ICyaW>C@fk;tBKZ+qb6Ox}CaT)4n6A)58Nbb0r2F)Oq;l`6jD2Y*<1Y zkG<-RQCH5}w5cJZpXE6z6-}suRKIRD?B(Rt(9zLR{mPh0nYXyK=WgGw!e@;+#zo(FNgtjB7H7_qF;qaqMND}Yk==A!wyXpyq(V?Xlhm^AvIKl_2dpDH|gF)0KX$niM9?+R^bT7#CvwEaDM^w1`# zXm#|~zyWL?#bZ=-wA3zin-w7;J9X;Rv_*?ucfQjwTgXW@Z&UZ&rAw9hu)TnW3`?zA zvt|)^FB_MhMf?Ek{{26|@g_fhtgW*;=K2xS3JDHsvuDrdwY#7#B_=v9SUJhPU?(&I zt5r3*L^m@s7M?uWm9EX|b?ZWiA&l6#5I^M+DXQAE>-0AKTc6FQJ$p9cbzfxC`2EM{ z{V_z$hBv)P%Yt{ll43fWcdOOizk`DLa(x_h9T)&Qd~$l0sTl~;jArQxw0Q9@&jXOZj87O zU13*}Btn-jzeTS}08(mds?k6H1oIA`jMFXolXZ3+a~wOi1(@T*v&(hJJpX)~vQD6v ztnbxboA>5TmVf48|Ji^MdFa3ZReZgZM~=FDxr)3r#wEABs9w2pGL_!@pWml+3;f*( z`gZ2;pPje%m{WDagaZGp*+pL_IE)z4YxL+Qyth|(6Pl~3sb#-~m5tHRf4roIFf?+VC?ArG=xJX5*87BfjS=6yDI zH8QG&cd&U8pz}aNLhPigJjD;-+A0M!L)^JRNSZ97&j5E|-a~5XQ)2e?^z)~mS{MB&DssnNhzXoB3JKi}x-av3b;TlB?JQZUOvK1%79?_t>!EXJ=yvJS zfHbFJUXikAsdP9x1%CexSZlt2bW)ARjkmNe!WPtxG=V-{QP>gUg1KwQ>m!H$0^T24 z5xZyj%7l6I${=wf^ty5#6ViT9>z=wiVI{aa7`C(OMH!9$Mz+|Kzlj5nDyDKX-5GBm z_9An~?VEvgCuP!bJG*(?wl$$SJj~NSHZyl(eFUwpk27bZilSeAn*&g zHOQL=9i{4H{3pMjiA=)0>4Gg=8keIavYZb-BZZb#E?v4L$4F7YybBT%j0tV;*`_3H z@;8!w?S>7PVT91lzpyg#aRH?;U~dthFBvOV3Gshm#+}>T*hPyfpsO+nzcG?TWzcrd zo6BE@b`JPvB-k5il_Wx(fh&P!DNOHYACH}cC4v;SEkMx@M_0_F%6y-C2elw=_l!0sPO@mb*s_^o~kX!BKz_L3L^5IC7P71i$~xW(4x&|%Z9l;}?)Q4}&kX0XAK=fJl0-DXW* z?oP#1jzqQnG(9)=-VD2EN*B$o$p|MWce?1A*So`~EeS7#E(=9u8v7<`#yf4`-Ja9y zLwG^*Fb_kTv|H$xF_w7(gaL|2v*wh;b?@_jX15)Eu?2Av9y_z5i88HgYJAx99wHyE zb;)Jy*kKczmb3kG5?bVoL{#`LXM;(Onp6rO zu9-e@;od`h*U`{?v$=Zz(gV^v^X{ox?GecZfx1&JT&RE>uvV9;o(o={4)|_@H_LLG z4K8Sz)}su>qP1b8y+7XPSTyQ9xjcdoGw?A^KcxD3%;;9k`nd~*nFR=)aOdc8sqX}$ zX-|y+jEhoV26Ufczv<5z{n)*39Cg#%DGk zr}#?G@OUfgL{J~|Buq{3%)CG7vEqX28S1F;{)iq67ZPdV|}GV$BqpHK3=-(%D{9}#j1C& zAptCaghg%LHVGE!>a}Zcx=#c%0ONs*W0u1Jlv5}ig6IDFkvV&m0ugfDFY_jlWpQsE zHSKO%R6w&9ynuEv|z0~vvWI&S?oY!h@w zE~)QkX67PT=A`!4oXm{!hYufq&)~KT1fCKEVN9u535?PIh&zLEt2S<23g88|oQbvG zSI5ceL4&LrB#nW;mWPJw-v3;s>z>QB6q?=A#`BByKr5T9q!OxD0QPXhr{lKS!X6yQl-x=R-QanAxU4S6ND)$j!>N>0FpitgFk&wo zn|*f@6z@mD$mn_(@%vt(_&)N>;7n@l5(??wU3GNIN<%m!cau5Xj;ouSy-!cC6_bvq z-tD(>WSIzi;5EZcc!lHAti z(F3=u$@E{tsJ)?7zNXf_N-nW(PZ;X?mUN}op15w1QLDDyH1XSzqo4>W9Q+rJ*FK#i zr8AUA^X}anQ%+hA8nmv{hom#3&IF8j{*kxc!IJ|r6#zKI(HGCmY};U9@^-|YZX1>^ zwO(LRKd#G$*b~17v-PtfX}6+_~4tk*9CH*;BK9`$G+~j?w2E_zMcS)vfE7FUO8{ z9SRg!U3nmBW7UtYwZ-_UHxS1*ZW45JSlqVY;EJddL&-#C^iMv45mBEvgQ85mD3=Ov zA-N=DT11Em)8`a2&T(18DR5+K>+(5#(w4Va54tw4g%{w|4Dw=T^+G_~hHGsN4t7iZ z+^nDnf(Tui%P8E`eq0SwS$2ND^Sj7pOOq<4SG4i=@i~{AtT4Q<1YtNH%eBf>i zb-wN8(2N`ATf5DgPg+!hNWV~z4fg4m0L`$PI0@x^p-0)`qBZDsnkn2~&TAi9iw@Xj zmQbXs-?~G!VRm23TNxMW8{|M@0nC{0ilpd<+43Y}EqKwkRjb&^v)s3AwtMvD^$uPN zBu-yghwXsj>eZKf$46{jv!(*n>Q73{lo_U1kb~5vg>n{2Ma%q)yts~uJ>QyvYx{@HgSwTfAry15h&Odi#ut7pVxrW@A?@Ms^>_MbxI z5!z|10Cl|mij{^)+|68ouH%YKGe3e>8(>63dqIN^HTP)EHfSen(4}s z`3rQLcTRT3u}ppcCl9T&W5#(Ub4 zmBl+TB0VY|^F~T7PIt)QGpD+tl6?Jo4Qd-3NB5ENG^XCguQ~dbnlVNU2Z6dODy!D6 z<+k#(rrjS4R9R#j{PDZ@>@DdBJ^IJKjQ=+B3fF(h7m69Z`!udo=u$>)Egs5;50Bf{ zq2KZuH*Kn*bm+xnNI+1YGo>rpizZJ4pf|0K@{}KGB`m~@WJIpbK0KVx?NLFU zTr!uEX5Ps1AIvRu@24;9l9AC9N4Y!3X=(POryNWFsrhE#UEjq^%V=iWF&laBpEB8G z+gG^1)cPk9#*3#^m!}MIU6GSWGvfTpy*{ftVr4xjXU}J?a?6ueiXXf%-;gk8$+HF( zCeqZ*q3I~)wJel=cydPk{o~I*td~;PTg|@b&SfiBU_%iTBPe3|(3%pUYoBfoC%y!+ zy<&!i(dIqPBJ*v2OorT>ymuj;3k@Veb>UOzIIUPyO7HL=%@Ue|M8&jxIW8>qz&5`l z>9j3e!Pqn^QjljGp^51{T!lGptVWJ(jQ+9KeN*3u$$`bcTHbm|Y`b*rS|75Q(d1{B zcD4JkFB%g1R=b)5!iQb>4dv(SXVdZ86I3@#reFAh(99w4S%6m{VnV&!_oJzWjcDem zS`!?0@udGrZ}C-Dd=p^KlONRvW=;>?R zG*f)nd>RrkU@P;tk5^i;8mi(GL%D4~!U#n%F_rOvR+Q$=oAcBJG;amb2bSc4}pR5zMo0%)?%?_s_=$Cv}rOT^ZyEnx2X7r)YTXq1KGM7B2 z3a_0CTVZ{5^@a^2c25}V;bC=n+SIAujR&vG4q5sM;Mf1K!JTd4;mtokj9&l)zogm% z;SmKq_HPTJWZ!+R+}AhGWWA4f`$2gM#l$Q)GSbD|DPhXgJKosPes@~J#2@Zqe3zKb zL*h>`cFKt@s~C0YFg|1cpZiFh)DLihGeAgSq2MUk^x;3)qPo-PEZoJ?wnOiWe`Xdp zFF`Ky*C8X|?WAo9WRO|3l%Y0dlJDPdz+vu1I$<`eERE34!vq)hjxR?bQDer?6U(Wl zwBQs5tQI03h>44Pii9(4_>zfwaMbV_l?$Njo;`nV)$t3&fqFC9VMnK6?!Menr)D;r zHV1Aa!ZXRgS9OIR-pbxxDX(51A6x#a|+j~J1|ZAj^Ju$2EiF|8V0#rh6j>Hv&q&c>>XI*`Ut z%ZKd{1W4CZxhs)h&i0!9=7Xw3f-Pj4t@4Bf2UYEwI*NrRJ0`fhH_uGuNyYxawZ&@l zfp~hg+THyv3lk0>?k$<%3k9Gw!8bP1pFMzqX-Rdk9AW;O_nvAns*(b7Iyr)LF@UDIP<7AdwcrNee@S=b9v!$hn4ZnHXmRh*$%SVZc{?LB zQx(a$dajh{Bux<^CY&-<6N)EaV`xt~rXpi4wKA<89rf}?J2=dVbBUY=#O56wY#(d_ z2DRniCASsYDv_T~m4Z`0icr5?XJuFI>({Ow$*pL*aLBfxgoGC>>f7XyiZPh{#*JQb|Wn9V#?aiGV7~;wne7Nu;9`QmElK15TNK7cHyqwyR+dQ zl8Vq48Q}O2oFR&-HlUIOE@00+%{s~%9o}mdsgk*=DKsixSQc(X3t5zy<2QvmMx*@} z%DKcl4IV#x_RJl(l(2r#An)boTE(9Q%9o(svy=sP52|C{v?e$>*u0!GF5aKOJpJ&zDF?`RxsslxTS^=EO6xDG#<^iJAsmd3 zryeqyXZwyF_V?$2?xYryMbE^<@Xl!|{(+b%9qw&Eabo_F$=vjH+JVP=?7ryYKKbr| zYKpY*XzeLaq)AT+|9rgq*VE~Pr`^EpkdyVyis(`JgJbvty-)9>oS7XX1R*Za+ z-lK;|+@2FnGgaH9;(K@gSW(yoRg|mPPVJd`Hit9&BS_2z_5-{g5&>_zPhxE`e&btv z4pCxXz+9X0vu4d&@9C2(9bw=~7|E$z*P(YOcn*0$TJlF$^eNRgC^K4(z z$8;JO7|2JLE^EP5lY7Ufs#UTdQtus@1{Y*9_|%jNZtl#-=bBTv9ywk?zOldV?#=Ci z0cPf%5*}C1p2XIL2D5!V5bJw$Bnyel+SWGu_{e$l3|CbA_&2QZ^=rNHc@g2T!ZoSr zC2dv|g6bM!g^bwcIJJUi_SM~e8o`87GE2R;Y_Zw>UKl@H4tK~*^i0SGpKzWtrydiV zz%d3+``l&R$=Xyw8FXG|3?F>RQb$8zR{{9}hgZWK-jSo*k&+eU_~gmI>4J551S5xr`>0W4AkOQsl?7YB0Xwp_8 zrmRXx=!LM=uj{$8DkuYjv(HA2nK`qQ0L7RdS*8cQauTD&g(|>Pc1HL zBZ&g-NnmNy^e?BDvcyuMNyO5_BEys6J7p^yYH4}11{*+ExCk!gPte&hs~3Ni?`Pd6 z{J!GmitB}>W4i64-0DylS_xAFrXl-?Le zpnRgUb8BEyL08Y;ys5!zRXj;&R&t1QXP&_fUtE-5o!{ZE)MhlD ze0AaOC{lVwl@yI2j@;I4oS9WqI8iLRw%iFgh3RbA=Myxj%Ga@|85IRcX6fyLuV{8?0jdTLgS}B4dA_&Z`JmIfy zH(sdEiKrcabl&m)EnvjC<37wV`_`~}Ovi+Oq~?u<1q+S8E^dx#T;4HsLK~a)wzYgw z2X+9(A6#l9mA7Pdi^W`X-Xm5v4)NqIE&Nmo8RIFlAitddcCkgqeg2v zoz8=GiF~;1=ccHr4lBwFd&#oJrmgH~WiD$oPy%{*KO~TDF_um?qyR*Auf32xoF@R@ zT|7g$P=sCPJ(d2X#^?LFxvB-YA9=kHKdWQkMKsiSUwF9f^rkcnb7oIQ%bj|Ly{hP` zSuBgc;~?f7sh^O^Sj9W-hcdE(+%F|Py@nzTV7`|buWtUp)ThoT162cX7P7Bhh&(j1aG&83HC*wK1G)rBuUtbVqRd7CZf<3_i? z-qSoyd*a#<{piM-rgKhmpPaFaO3k_YAa42^cIl-BO`TT$=*h{`nK|h8_FB`XO(SGV zR}}L@j+=>e2%UlTT<;!jP6v9**uShxUuFjA4uwomPay%ykbq7Lw3f$$1qsu(Uy@)45-(uKLLIa z$!;t|03-y%NOL?sr4}{y+aSB~jU>K+ig}fiE*gMJhCRfCl*jXgaF-LELk_@)_9DswDWEBhsJaG>8r6VCh^L=l9hj`1C?D@G>jRy_ zXvrrXIj0d&3p<75r2cwyHfL25+NXp_uJG81&oo9fgr^r1(-o%>cXXvf)+HKeWo3D; zUtddDgUpsmxq0*WqmnDRj4Q6+purm2_fH53r{=$^i#{rp#Rp;x4DAA(1OgbOADX`~ z3nu_MjiB@a-lVZCVFy-GN{ppZk1t$R@DyQ6K_E}zcwZxh7p+g)8(9)i_2d{;_0)#B!z{vD zPd}pW-(GA>>XA;>3;}s3bHy5~YRA~x+ow_Kb(@`Ml(hV-2HkT>b|ptR5t7+6r>IpM zHx99bbkMbALSY&4Bo^FGF(ok|0(G%BT7_ZO$JSmjgE(Pz)h~V2#}ZYtC+a*>bD=iT zT7J8DlbZw8K1~8u=ZWA)^5HzZR1T735WW>pmVlVQY|$CthV2s4Xb?fc zhkHZc!YlCni{Ui2g+VObFQE+{IB?)omm#STW4r=2jvP4>MrQcRH}c&3M`zXO-q2_z zFVGN-m-1ukx#lFhQEaNa^&n}O_4)hnhEN|3kLbc zy;dRB9_@w9xGte@{Qpw7@eqLoppk9K50*k$WLd@DbEue;Y&QyYl!q+Ll)8Uc0-&4jpru7%i7u3%+a>73Oh=OFa}= zlek!`kUaPHndTH{J#g!GCF3@a{vEnj`91PTxpY`r@Oo40nNNq z5GWL(PKdM2?pLAdQY!-BV*H0#)RXm+9_zY>@#{3i108eB=vLb2FcE(J|pNzT@ zRrseae*g15*S$W1I?j&*Ka5@{R24nWViH(Y#d_4J3qc)5Nu4BL5n#ldbA4CG$`~_7 zucAxB;p7Xn?B*nVs%RRsr-7`!IVMw=G_rOea|Spxfx|+7%!kDNLRMCNtT4s0WEKB= zCGld7A;X4UWZRSMt;SJ+o_Kv@V`I8U>97bOyFYl|@-^ElpE^b&J`7b6@%8N4Q&Mf> zSO5NHPpWRz?iu~4yg#i>8c2$#i0h-5KvwflNipSWp0GzQUB5nPL&MIYM~>8ixX(_! zK#EJ+a>!x0-%~E-oCvP|5&HQ zxWy)io~0kEE7DV<#9UU8c9?x)~^&mr`Hu-i`t6O)gE6MkSXm=Kh zH(1=qw7tT}#*ucf>cEzSlEKfxQBvHiuSq(%QvckD* z+e1QJ;Bk=}dC|+*{Zf*=->;>2G5UPQ2V{U3Zr-`GW2`im1dT$c><91TQk0qesM|26 zahU<`^EIANP$ihQMjlALR$C!iil^_#iFKej*UM1q369W1^LC?`Vd<@9B{{PcQyVl2 zeRpyHZ#|!o?8qPdp-YYq5$@(`uRj*d$Qfyxo>MlB;pdOX=@0 z)`=Y;v~3nWMN*1n|Ik_{L;kb9JGu)5kyEo~%osQMF+V(+$4vlR_z}D@w-0^yd^hPOd6Y9+Z0LuEGV*iBUk#G3 z>-)6hfo9Hp&r!er@faaf6h453Q!}i#LU!EtmbK4a&en-RiF2P;5#|m(Agcm_gNZ<` zJ9VnfJxsw#VSVrUFHJfA!<{BqR746`NS%n}zap@&y(g#D`VraE=$N+Fer0J zv+|u$2iqCQ2<2ZVp*;KEjfF}I(oBxl;H?|X+psz8d#A%J=Zvmlz365ElKVsNdZC6` zUXtSwyp@S&H7Uqu1Wy*P&Ce-;zAJA5K#iXzPfSgvhVr)fOFZjMqY$KThqs~K^21i+ z7*U~(0Z!NN!*OG>aCeWjTQz|_d4cm?QHV|bppCtCT==Ys;gp(G$Dtg8A8a4#k%Znm zv#=v_dR2};;nS0Rno$_Nn4rDoDv^+8fC}uEZDq^qDkJywJr5r61Qp1DH8rFg*R5N( zNcVHHkjS*riiuRKQ1tpe*MmleN{>|L(QQOhRx zH^9Uz9lmlm5JtMGwMixfd!r@Lv09V;2M(lynUhM}?%q{ujE|Wbwsl>@9wHott#v*x zuNlyZDbP1?qU=W&pC}4u-CipF4Ajas3qQ?W!zvA@z?S0-@fT;bNIW1H90o3rh$&x&eH*kXT;yCLuFhwVfyNy&%ibR-so1!`)!vpwyNqM#y`Xl53s2@Uy#iLDnL^m(LnsbOJTPq>w_H8*2c3e9W9j(fqdRpx* zpgud&^2e%~Y4x{~nC-|ZJ>%y4SWS`bFQU0P*g3UH)agA9VC%59gefiw3#hx)zFe=E zPv(uNE!|xB;a;s8)1NgmHm+>Lbn7Nv*!Pcn-MqT~ySy%13h!=wFWj1S`2G#=JGBie8N?4_4_KlX>;3rkRV(%(&ZXTXfR5 z@jh=c#aE&G_OZCw9R8`G%)^)$c3^;=`6ua+L4{NypHIUb!g=acZw`}Ze09mW$k{h? z9|^MONp$e4?0N3}{CV><_WZ-NX$_)BI;3sqDO0K;DCZV~!9FG84WRZGOoHPzpCf|S zS=6FOz(gW!MH*hsx-{<&WM&$yGCf!nC$zrtai~X(F&{m;p@`NqHdVA-_N95lw49%H zzF%1#8o5sEaBQfJ`tYDo^`@&MHD{#w=-SwHy6CE*xmU+!-RfBLwlB8~o?M4OYdT!> zyp_xOE`P5MwzLVk>Aa-kNk-CD+eHqmRKBk`x}hNC#QbS5A5BGv>BT&ydiUs0@tD{?yPnS)** z&v$qUbfKv=<=C-fo(v0dopvR)N;Nc0CjJ&rLdCg*~+X+k%2SbvO?Oa?(mK7=3bNQbjlh8~*fFe7n4kkBbwjy@-t{ zMCGKEgA;O+tU>_0mHQ^!x2I8p>V9ih3z|4AV~3*|uJ?|$7F&W+Zkzqm7BA-Yrh~3~ zrf@>!@v%)}Hh`wv45;zXKd)SdB=b3gIQ_B@j%+F1I&c-7F>13#&0qQWwlMJ(=gr7g zGW^R+K-q$LB>10qZQ3M}sfO~gsAvF)R%WM}qin>yD0^OZ(cvC&ur#*LnmN-vrscvT z+;Z`FI6c3;^t7HXA`XM>Ns&6UpjPQP^B&at#QCtdd)5c^s*#2$)46^Lbm*LUjFuEWaADjZ$E3xvOI1e2H)ztN!}8v@ zaQPp$!?3y(fS5qt4pWbMA!i5454cY3iXTg}jcJJsWY`$GAEE})vf-u(aN2_RwjaBE z1lT(SkAGqdROk>lZrbz$a3O0{8vC@gfHgLPhHgD$g26*rD~Y+3$>tyS7UUdS%YNQR z{@9ot1f9sZndwIYXf98%+x(R9+#vm{8FAm7RpUHsmLG7Gq-Ibt1mi!u?SIqxZFtF5 z$0a@;@WlJ^UM*}f+fo(R<)D*E$l*05GFa-<*At;L4NA$%7BEol?Pii)-$ak{lgGvz z8HmHi*`}G-?IPPqCf<6W&$l`PnYxl5 zCpzXYv7tTGm`KpnBmsA&4kuE_(wP#9UziZ#f>qbbS#jr1L$*~JO&NQu7SUEBZ_T?w zdC7Y=d9k8)+S@YTZPfHcQiQfRlP}nr-zd4~@To~%M3KU9qPS($&J+||bCdCs4X>h0 z;tC2qB6fvVzZC*r7#TUbj$~}7-R1#Xkb=BkBZG@TgQ7G?4cDM07IW^K6?a3AuaC)Q zy~qiPC+9vu>}bCJhy8#h6ANxnA#2du$^q;TQnu}W1yEi1N#nG7zCUj$^?fbRc24^d z*`Pt_it)nWyFNgzVJB0HwcExm?B{?&v7`-zj!8#VI{&0+-nX6Aqn?8hEQaZDrITw< zD7`4;R12v$zkgU+G3v|*@_NmVm#ppBg0YCT!kX8w-%7vuK;*G?jjXM^MIQcg16(Ve zrekn=85&0j2@bA|%y`Da>=>U6=J+}m6=ri1hoBZ~|HE7&58y)T!-*tKE4JHtTq?^Z z_z!Cb2}cXjp%@_0)QoKXE~@e}Sc<2I__zYBuj` zRsKV?DLKP(!u1%^q)(}BHKA+Z<&RI-wp1-d`DR-VyMDv8t^SY=2#dioP*F}tK`6v5 z%6Ci0f1h7cr+KAT$Z2Ed;E-^27MQcmbf+piLPEm1YgXl)6vG^U(%*sPeQ-h3)8o5b ztywQ!u|-!BxQa(e8Z)bmB(wNVxXhTtqzS zylC9;u9?{IhzFm|1$0&Ob4n|BvMKa89(dO&*%UymyV-w&T^NrNT+q{BO>(KbHxoTU$^c;c{H z6SqX$0!O>~9R0gYR<@A!h*74X?;JE+$Hl6L`+h=_6EybNu`CqMy4v;R*oY{H&Aaht ze`vyxhBR}ho~`$do8n9j#($7~Tn;w07rOf1e!6_!x0Y0en{GEhn z0WXv+s3Xi_JbrNnuxs?kA0n+ZwZiL6^i&|T~ zwHdVg)sP~Cb9lL+>m6Fyk|Qi>genceaY1?GgJq_yS(`u(7>=oq&yA2itd%ZJ#djgd zATXpj)mylet%1E&qr?gBHHt&_8%}IQGC+S%Ggtifqc;1aV3MX3)d-fkyNp%=R5To# zc{H-<8C>)5{O)p5QDqzRYu}*1pswY=xKZ(k$4(LM%rsj?#p{{XpHa|OwYCIDB*iRr z3jxl+0j<_78gC-pd$Y(wElx27ReGRB_D6h8P18D_kiLR9+mvLDQ^q~S;-YJajp>G)lgceR-s z-O!>507gvQNDv6~Lf{J923q3~nn@i)_o-4z1g|2T#6Y(+nR9&)9?Ubh7AFLQd2^Pi z&LXLTmJk&;i}-V;v;U{i?WeXggd$FFAOk0k^<|!)M!8hW+-$RP?E;@-aOBXT9h?3; zztS)1EQd?DOMTJ?{q_^Iv5~e98ql0B@8-)>%*>X6H17C;@I(Kt)8cUpkC9;u2xvN> z`Yo2s=zFomrXm{TLcuc~u=*Vhcp+n9$fH@%b=#F#Jeh>^dhblgu1OOJcvM^KrfGsH zjsDsg){3UDq2`{w8%8Hr*Ps2JyH`J+Q#j1_Ds3ayv9LxxSBH1tR(`OvJrezF8kwa< zp=1YGT4i2cZ1d=@anJSRy9nE??f1PSLwXE(QDBg!bDO$0c+;VK2}!;$_0spxF49Z; z8IeYXzp_^KB;gMLsYNbq)&Y{kp&l3d1@C43N*P5&kRumKw&U#Tmg^pErBgx&^?>bT zzzbjB5NO8R9D@c-u(%NmcotE5A|ZjB-bc~Y^ajWL?RVitW)$g>$FTJzqKxZKJS@jM zMuI1p+4!(6iE)}8pQF+I^RgR7644rwKJ2gykREi$whz3FE!7&TqGqHt_AkGv$j{ov zK)1n4b8Px01^#p6O)#vMpxK5%!+2Nl^Lz`p^dp`y3_;UadFe)7pjLGkfD z$*(1OQL5K^OqnE9o~*=w=k##9T@xHne0l`Iq?&h~F=H~VTyZu%wl`$FUov6OLhYeq zedz5ZFnJ5fBU~(+c=|RmdGz=D)F~zMGzJfLIGx&0Ye-JfF}4_@Q?2wMRs${;k-OfF z$=1&wd+>lX9VtrcDUJItHSVOfJZ&)Cko?N6uDkrUY5=4sw<)R;_H9u3GpjlZG%S8OOirbIkE!0 z`7#tTt&ua3B}dF2RqPQJ4gn~iJY_K`mgnj(B7GO^#9;BqNdnm6RFdXv9{dvi2s!25 zMab#GODV|4{0~!po!c&InwgK1nt-!*6w$-jyEJ)3wFT#!3FHyx>Xm@OLKku^l1L>5 zNJeN*Lz@w-?Cq%zD{DcWjmUr2?YCx$1@*X<)Fy3A+bdQFNE%FjHioifUP+IBa()d? z0oqdxcSJe?Wy0UB8rn)dp)dBe>vG>-T{L0h`c+eVQ(be&<6Us5^kim>3fd}tmzCBw zJ%9Cc#I*%`cJ1oq=CG~F`Hr7Ydz5`_O?@b;y7-e-gG@#$NP{ee7+2^q^o6*tFr;J+ z1_+!#b%6i)Wf80Bss`V%8N6GXKv8HThp5gzP(lCupvXD3@w_-Qfenu_PPGeC*tGMS&c*mhKL10F*n9Z-3HlH=o5>SoZB(wAP%kUQ0|y z{)t?~B(2LY=8ws|Upyx>#XE94Z-Cg;cB!$zFQRWRu4j3s7+P{@hz# zT`Mpkpw0K|4)D+5RA~&B_Y6KVRPI&(vgG_J)3U`bKc&=O)g+*sdlS}Rs@>NKMI z-ZrDQZ}(Z01c#DlS+u{44bfz#J((E}p{Fl};cxO>r_waB!WU)a@7i;mqhnBRSEfJS z9=%I9DcW}V?40%EcE2hG4%psr+21%isAyiD*O$ytb``CyfuxGQ9o|8mDT zkLlH|+*W5Uo)FY?P3#%X4XK)b5lI=FGS?*@uzj^Fs?5Y;&!+MR(Orgo4!+E-tyFr! zwiV}r!i*vAcN(PW+&!*WIx8nScx*1-jP4htWKiq?I-)01tQ6+3O4^^)Z2_bYm5S=r*5zi-R+mO!P$E@pF$baP&kEkWm0f^N+F*-rDH@m)3!bfc+`4yrll=9eD7JUFXcuGJ)rV$JN&2vAp$NH?GSzsh%6C8q#T>#3}Zghj+T=A!BK4 z$i&Fs+78gRsN&VNz2U?Z0j)e?WUNi@7rKzC$A`5W$ZqUt*kg!&dVyKdE}_&>B?Xnc zXD<70y>Xem>Eq;;y$205Evo39Rxu>nSm;_;YvMNLPs~rYdFsn4zb@C?yS`o6JqNOH zT%RoCD(GN{iDuy`WRPMay#CH!+#EIBiV{Bg zpJ~%1K@?%r$(i?UXSO823onJjF1{}FHUx-Qe%Y(9ja~ zov7YK{$E5CR5q@8rwjz$hn+g4`Bp2)eV+jdWG=q1HVQtvDa`*< z3lsg0v+b|$HED?qJ0|W~)Y6Ha{fyK4X=c<YS?N08T!(VA1mP>c8VUc{+rLq$7p!v?plB*?NP zSFKr76<{6EwDm$ub2pokZ}k)^0`o!);MeEEWxRLJBQa}sZd?cU`Xl}R_Klk~c}g4o zqV%AJUO}@kwWO^go0;5gEtGrLK4~pCMbCdIiSX~2-R`q1erU5&gla2nrEFtPfZu#w z@Viwo;mmC+Qh$Tg0BP4_A<(8x zhe(qbbL)&+P~$VvSjJ_$mY0`TRWMHySB>`>vu_=!Lk;7-2=Di91!oK)v%Y9CGK@Oj zH&-U-oPLjo_b|s6-6omt=;%jU!FHmW zZF4cRnNH)gwcEBYpY0~(A=spOOYZ3jkp(b5ixfHPf-PlBBac-xQDHjw(29`RxZ77U1ZUM=~*LMOdHF#{2g>Siymi=?1 zG1|CC(2HoOh`sIMaS88_96e815t46Mb0#Z%T6mF0vlqSLS=dmP`#rP14F+Pn$zHHX z)#~j$R~P76fbOIaJM4d|)vQ^6mI+3s8s#=6p^zLn$J+n61X{}rWW4+ZFc0Z{E;&_ z5EXG9c7yJ>ettMbkRzz0%z<5za{m0P>c3<(dDCZ9f(7bg#}nL58-&LS4`-<`!wTQL z>EPUh)ua>g?>W6SqBd#rH$n*Z+G6rXmVrofM3iUdKY{TB0*bp{c`*+e{K$w|6=ZL! z>uc%l`F=QaSbnt5;f7QwLC0n~JKIciVj+^tTQAtfy^3D23l2&O{-Z2N-4%MaiHm00 zB)SHS7*lR%LAhhm-C-~?{JzUhMcOWg3rf)JP#Pu;(d*pK<`0U(A)}wob z_syus>1!jBa{Y`>+pf?*sKal714buJu(1A;q|Iomc?RJ(Bflkwm2KB=H>0605VJ3B zMDz7|FM5yIs>5?-$5&A#X@=i+#-7E zm-iE} zq9U}L{!YdEUEU+w=Qe!|3~X-CX2yk8IYgI(J=lT55U29B`r&Zcv9-QYF+3p`4~`1m zr7c&Gl*AvM-n>~e9;pg;CaOQ){jslSWxbvyZc>jQx5R~M0IYg3HigXYEJ^7xG0P`O}fj^y~gWay2hQ9`wJ<2)bWNfVZZ+I8SyqI-!70r?F9j5x+1da_Z-5nI9V`q3b zCnt^32KyK6BpF_p3o~55`Jva7?l+Oj6;2wd2p^-RBunK#Xe-J zF<8lq{X5ut8*+Wu57HfCX=&YY$3guKAmNne(x8#TS42j%e5VviUzFuZfAL)A1K`b} zj$&Y7<=r#-1o`HrDTdfD}8qiYo|TK zu1Qr94-LwI@zz+3iRrg0De6t7V&xeh7>9F51 z?geKujIpSjg}&*tF|mdY9W-OSoyYBHdo|$WduNyRq9y_V2fg`=uRgT7w$14+S6&#c z_#G?RxiWgcI2XGV`F)Tj**z?7SBZu~;do5Jvd5 z3Q3N3;ZYW01UJyDNNjW1f@`cENXqg1Wxtg#e*qRASlL`Q5#utuWe?+FiKtK9mB^ZW zXj+qBw{^c*xwwwvk9kp+Q=BY}n{$fSuZw(nX36JvRJ|&i!_v>dkNkFeQVGliCE z#Kl+&nGFHK@NW^#jeq$!5m1Ue&(&kBhg#XqW(>&dGUyok^Y^k>HP0z_X^f&W@4gYH z`=~d6a|G+R+}CFB!Em~lP*~OI1kRK(vV|Q}xu9;?chl%F1>Hnx<%PS%=@!p8eZ^IT z!4X3J;Ua=hlES56cN$a*+PBNvXun7uC}@Lc2X5V3gCTU{6)8v#xs$zblz!CfyVWag zpQyiG%PBMC9QBDY@`B0zF(RY^)9J(wn$2FXsw*&ku5_Bzn%GyGgTMw0UCp%)#1%zFf)}q(mzhVJ1(RJ|s({;u5tOF$9D+_i z<}OE%Z+tilcrvK;GR!J`TC#p3E3tZ);F{egyI>g!n0^|PpbpCq9OHL_Ps(Zd`xWgf z905kgZ{xZ#Y-+4y_{k2@&H+o%8K0)jCkM1weOgIE{5eC%K#^ku}RF+*<0> zQK^&|_W2GaaIS4XDYcqyFboY%J&6K6>)gtU{u7f|)PZJ))o`fUrU?f}G++Ls1^vbYQcedxE}?d z%*Kj&f2H9lX*jQj#=0pg-O9nikN)+CwQkr$j31^{}v7*vmiv5OX+VPbgz>) zWv197l4Nj9rQn&h7I0!~fq}>pu*sKR5O0{uMtu6})xn!5A(%RzoH=I4`YW7~o=BW4 zP2+iTqoYhOQ&-Jpk|Msnt@ZU)!KF2>J)hXzuDGCz0`Tku%A{JzUXw~^YXA1o0?GjG zRWxW=@g#~AWm#|=X)!aWj$WuvQaR|Fz;^jLB8PX zc4FoQ4auT(O5C~4BUfkbFjAJ_d%NJ*$89(GI#qB;L+sr$RIa-!j1cS#<-R0DsHA z&ALuq!jT12y$BaiQP8(-TUGdGYuElo8Q(=e-#Uakbo{BgDgf!DJEK21q;-q*$-BuL zar#F>n-A(`X=!PS!iT%3-msw$Ye3DrDh)~lymoEBlp{J-z2Od{o|`NP5y7h!iZ9M; z>c>6;H3B%pBZz9&-MZV*LqJYhkB&K4#e~RLnmxa)31zy)itj~(h1vw*m5N}{E;oUl z&X4I^bt?VzDHWEHh&`rEnIaQR@Ty6z`1!?T*xvrlA(*t5K$E|Q*U#O3O39vU`A{4iA&_zxFmUaf&FK zhgCxvvT*N{+)Z+6d&y6zaUZv83*Y< z6SW7<_0qSZhv(c^XX!Q;y$A_Jz-NFDu&{w360oLZLXLoXl2bPx%&u4q_7b$T4hW*I zzS|dyjmasVD$>bzK_xPOei1fUf`!0-ZB4B=ahm$U0Ajy0Kg0^SK)9EDo^EX|Z$F@% zweY2D{b0>aaC6pCm}yi63$?a+2XX5Yi!vef0gBorx}DG0~UI=mnpVXYx%l>Q0w3l!qV3&e^^~k{4GJ`O|jS^hxG$@ zWeM4*5}qQD(;%n^`uCT0w;iv5!FZ%?-DQlCPEWt@#}AD8Zh|}wIv^=spFU)6r}L^V z5e&Dq_@;Q>bHr6q1Q?`2VU=*7shu~g4O6ybr&;Nu*h>8aQC|Os*)mSWQ;CTk3-6xu zSl0Uf$ZS4J1RCq6eI9)qtQ&?IA!P945t_Y?jq~C3)KJ7(9F$_T_BK3AaA34Z_<_MC z;`bAqpWB8Ytx~u>;XB@J_$I9)VcO~Fm|M1CEp-)|X^$=zRHj}SkQ2Ex-_eAXwI|ZU z=`IVfH6|y@Oxi2H^w4^MyP$!_YLtr#2*QEhqGfalgvR#v-Cgf9|9+?oIjot{Txq-L z#f}_u;}%8@$DoD6P}+8SCF#)OrAu9Sn!49(_E}!`(d9(eBk+L{BSwVZcVxOV0dd?* zyh?I)lfGJKkR?pQCi?9+lJI;2mHo10Um9y4$95gUSqT=WtADi{j0nR(KRSAA#0j%IkuHLrK?&2}K(~-q#z~_*&e-1j) zCFxpOm>y;-z#i$6jgJW$x-TNaTV$>TP2F}6Y4p`5U$xpaX2DF4ad2pOpfF52#c!f% zMPL;DE1)JliADvtLI+gc#XuR;`(GwY*Oxd%sFtC`Zw=6atbxO}VcO-R@5)=J#b4(^ zd*`d^Zn->#w5bW-!m4mSsSG_VQ$1Dd0FHX`Mu8*Oq30t^gvJ(^iPMh}zD-&pmn#5(NM?C%dG`<4u}2 zZA(KJczf#4ks85k$x(vGexD|!8HUwifkaevc)8-+mk(kQQjEHNx+ zc7O}wP=dLh`&C-9x_sSsag6T~AOl9SnfE3~Iv1l3J0-V3FKV6JNN;h=VgRhR`ubMW zAG4^RCRhCEih#^6x0lV5r)9q_f!Uv=bUYuK7>B0ht4qPPYuBW=I~ig9oRZz`E0g@G zJ9^C#j8f73eiXR-bNG+}Z24)3LE5mPyqEhEEgUSn=<>t%eL8+E9Hd^!D-cJh$3|hL z!>IL<#{6vfC^@3yetc{f17wFRp?FSW=02!R(QgT`+%$^{W zKmZV?ux!jcQHvx%8;@WW6vvdMHJwZ6u=nWEPGa|xw`4R;Q`59F|NQY|rEf>$#)P<< zH2+Oud~q{{S&8*$E$ZNi>N-|80Y8}$)(Y5+uQnJN3x`)2!b~*E3&_q$s52P6CwWqQ z=?}EilP?G8is=6M@#ErTY1rW38NT40vf8k65XlT*(Li|433BJfHuhP19f6F=jVwou z7{pIZF>C_P2CK5FbTxs+<`l2WV|_}85j;`tL=8PCp7!>|1>u|F!83ti4GTrcLvlL7 z7KGo^dRKRti7M>J;J}!nwtJodcGYM%a?eIE+I-sfV&4JHV$1gJHBXQCXNl~i5h25& zcJH1LJoHrHdGR8}6S%yd#@Kn0H9sg>^zq6h$W6ByP^*7Bpxt@G(55BcE1) zL5hvML}n;Xk^yH{Z=%SQ8I}wPk`V?PG7m;5S0tn`7|oeyhkeMw9{0CLn^1p74Qv%{ zE8CyqwGP~`N~IxejMfP5$+4vMiG%ByT{ZBj+bwjeSJ<$);r475Z%(GkSVefZ|d6HlJfq3=d)C^5&B zpXFJK^BNNwggHVC5=)M6rG*;^+77Pj!s7k?3J);1uy7^^5Ea}nA_x&$bZ~Sktde(D z3Z$dmFhG($BRnywtbFP=_eGhTlwEBuo&SFLJvIW5GaKMTM)!;ATdT(>`|arlc!Kag ziYTxq4%?hd-Xjtpil~9m&sc3w)~ZWBQVm|G`QX9(pxhZT3gPzCq|ZjT02|KM@^2}B zUitd4mF$KtoUnxis>mN%ANNl>&#Pq!)Pq`srDj^Eh+J3}82Fx|kmII93RYoPu5SyV z`-y2Vb$E2pm8zMZzvuy_aTVwI#Vv#}8?9K|p>J!xekh}|!>qOwb=3Z|N0*>j`Lb^- z#R||P!BN&nwlP4A7!_j}4lOZ2&rEJ@_@}9pCV8^^!s&x7W-QUAAqwgO{Z`Moj7Sc8zQ*oqKaEMOn!Dllu+^op;WosSGKDR3XIFs0-Yb1SS{x z@FQ(0-A&I=xyVsv!4ahEzmfYHtsU%Z2v+P&R-K%>IxOT0m;i8q?b-p;BDERI`D;di zJR$pGPdA;!;zq~c%d?`Q??;c8>grKzo>hWIa^W`DKGQdSmv_1WtQ<8A_&bd~7!~zL zmnZXLO`3`eak%ykHaxlGv6Cl7AV@fJc60OPi=>`EKj7_0u>Cp=6q(F+mU?(|U|>~6 zRP3T*AXOgPOa>y@Tj*;|6%yChTL2fG8p}gzvos-z_M&1PGqKG9P(Wq*x=?odFsu%flTv;AX?GAw$p*^YC^q*O--Pw!j>8ZaE zY0;NpwBx;6P(?U3QXO!+YhxACkol*4(7&VEL^3Y+*@+wdt6G2Ta|kBKK3K0c)zkuw zjJZm0IQ_Uy>1$4r77S$picZJP5DM32?zgq^39n`AZF0OaI<#5Zl=_EEu1ZRHZF`IC zDY?LEmW^xi3>tywi@u6KsNBNxubgY-zcT8EjLIM1tIDWC4D=n9Vib7HfQzQQs z?rP@ygJ69Kblay-AD93x=+Cv9H<0&NKU($!T3{A5HA= zx8@Py@=*EP;%C8h#Flmoe)Z3b>&i&UV^?*qN1y*sWg!zEY!)O*cI&l;h4brWGls9y zjjAusHCu{64XxLIV@<_V{>`b!56Kj>hE&%|iDC3wC<~HCNWY9piz*5a+!5SPpkmS# zk(+}bYYetBFP;tdoDSkC@rHiQ6O3BdjPLT7LP)B?u?DSXEgXW2Tj=52EKHp!1rsW?<%>4(Hp{_S!Cz;mz7ib(m4x@O-e1E_eFA0ivqHu-$N;- zO^3NyfSzn7Zr1R2f#-<;lu|hVq!SvL)FLIaUatoR#zA+jF`b&=<7E2T1+u!LntCb5g7YdC#!P9Ed4j)zyTM69K@mq z0(H#%A70=-Kw1rROZnz39fpO4;jIW>B>fb)d7@W))MR)_k5xF2i4!5GN(28EwEeaak9Su*#c)1xj2;k;_Jwv zFg9)kw~OFGx3oW_E!x?5RX&Xm2e7Lnj;K)GNU@5SAo6`*nr>paBHlH(TWhH2)I@(O z{&UDPIW7c<7b8=0B0%jux1Y)_suND@b8pv$^()`Ec=l<`J62#2ORG_GHW>yQQFr~% zk$CfpxGOHM2laZ}cI|3FFt%OGd0@YBe3!MX8t|E#a1kKz3MH%Zk1F5Nym*8q1I3Am z(%vT6MBMu(iY%P&*a6@7RI0WjeXVd&r5h%&9{5+(h1PANN7MQYi;1a6mLyA+BvA_F z$dO>~lPGx@`$r2uj1aA1Ax8Z@FIqMHJq-6Uw=Trq^-w5^^GUGQ6;Hr`NfaKHG*0_# zl6cTOR2cDWF?a5T8?bb9On@i>RD=<3pp7HrBcSj^Zk zSW!Ij)j>K`Nof9cNjBGtnMNo&AQ|gNEiz`N2OlF0R)QSGsKe@-eySiQRonOL})p!wni%hU*6LLx_A{YoXZtO9pT7%6%GQpD0)G+w{9k0cnxTg8;k zJr~KGoPNzwfcI<2i1)luq6 zG{p?fu_d`XmEm8E&TfptnL$0B2JGiv*;&y2{`IwkXI|RenoM6sK~W@*!IgbRH~RG# z7?4v#L1=LvPH(Hg%l~^_-vX|@kkABk5NoUPomz*|N*7uy=Cw@1ZetU~r&PXr4#p_} z`%al7OEKHFOP8Y;R|x5O_wMG>u>hoSoj%=u-hO`C&bu;{$b|*J17*O0QoJKU+r_X) zoVF`BqQ-hAEK_(&`K-uSrcmx&iHZLD$edf$=`!pS4s{z`Q{qNY2gzOLFsG$71zP{$ zaJ@f^%D zg7<5URwDU;DGK)Sy<@Wt`7tM2|Hz5P) zoK}3T4$#~12)Kj@w!zf|mWLoR^uOB}8A&u~!(EWQ%4kE7nxXIRLTq9C%U)$mlH*1` zuiOkH z{mb~xf#KGH*C!L{>89zHVxiZB0$i0OnP7uU?7@Jxa&D{I_P1yvk|x?0iUNQH6W8j} zl;a6x#FejTAG!t3PgA`WPlQQ=%K*Zxy=X&NvOsL54!u)+gMZCrU%YJD(Eo`*Ik=j5 z^QfMooa$WqZ25nzOn_(~ps4?rM)m_SIa|GEjVup{LqS2|41?YvBu+ekXbgF>$f{K) zfW15H4X@3#QIUthaZ(XG=ko~1xUWTb)a8ul+H$0bl9WwBhOiENg~eI7ay4t|GWJ$< z50$mP_`cJFEF0+lJMgq=Q=N{xJaL)CPWI$tk~#-ZEkR^3rU?aF)Ufz{uZt`0C4T<6 z=lb`GXEE>8#;n_%n`S%It-k%ZMm8=(-c3Jf=^LPJarO31n*(RFvTt7Rwfbc0ZO7Gr z>uH!-*}Zc+bT+7=uEz29TQ7YozYx?%U@I!z23~%jDl?Pm|SBAz0EB#H*#~wLMO?FVV8rRaRlfk&|Q>fcR}%T@TJ(dFLWI(&uBuY@x15jblyZA+;bC>ck$KkH{?349bsEM zFDyL5i81udmxjr#zENI(RbG&ib+%~%HJX;Tb~HQYO+v!sp7iE&I$xbCpB&T;v=@id%_Mp~nL%Ch{^5hymU?Zl%7MLdlZruS2nwoPnptQ$ z=gc)!Ob6^nril|Bl-zAPLwdU1&cb7(nATj=KM$kZ9dfo}WcuTrJW7J8KR=aVfE|sO zhBlUO6Bj?NPPx_O&ll3ofncy-2IjFjeEt0?gzr?H%mII!?Bgq(DcXT8owZb6~t{UyJ8<*Ec<{2Gk|9fB+ z_6-EQr|IF3es`5K2L0WV` z$xezE#i+3x}Wq0iu~7&fIUY@QJ#1<;qZs zp#6qMyq|TZ2V|LVu(^t9d~VN$02=|74D;Iblqq^6_@ofdU>rb>!Xp?G056zx;1Fkj z<6U)?kI6u8qlmj%un#A>F~R08rU6~ZoaH-LzS0U{>DwR@M$)+X!-@o_B}+|+;mG3l0m{rU*D)n>Hk82N;TZu_~x*$DKBbB_l zmGNXjw@ga09x!F7AVYGY8y{hZMf1(&f%LsfP2CfTS7)bldBsa+gUq8Et<$o9-jfBD zm#df-VcX#^9bNVw{@qsGsIT81=H(Sx|Q3

rwUbI3$WCGkar|)# z-(bj@r6tiK6YWgD6V;SKeDcRswXvt>hH;ChOfs&#Y9(|r>DO4uK-u__hS80GRnWax zFU*s+(I=8PEBp9Q>^SuS5eVUJ8z*EO<1fVBw_bAxhaaMIrD^C4-}_@S~J7FU7lXp8{>y@5jrxSucUL&_LP3p#=*U^86n z>*2BSR#bN#?4#|5JbLu#EyLZ|{*hdwb^{G1 z^BAGgm7`l06FvJDiyt`QrqPp3#t#d2lJ6Q3zk;9Q`-pr-;t5nAJU?#&taEA8h9_8fNb0+Co`;e{5nDOc%7jE1bNijBZdB1Y^mX-h1 z`~QA1a>%_+7tC4aBi{Mqy8``Jy0 z=ylFpv}h{?vwAhFrc4GEcHl=*5aE-5kgRhew4Qp+$L$+bhaEg#3`clY8+!AsN7ckD(J zI7RzkI)u19!E81sWg&eD;4T5(wBJ-WSyq8}qlud_7|V|RbzPoq z#5T$e=zr|-DeI4pKOGQr1O0v1v&8|A7C)=Qu{x3p^1uDDmpFrP;KiFaQEW~psI%R) zT!F{eMje|@n#-C-vE7CE(CXaC82eCJqO!G67@gn+ePS-YMceo3u#q$Gn}6RI7m*7#EwjM zRNRGX??{BLO-#Jj)z>~koz|=(c&pCJ9-thhj(36kvKTeWBl`@&C7Ggu6b+Pf4L&G7 zvS;)XZdgz7EkE7$Zjruosu+hHIy6EJ&5RpFPf0X*oYfDNXEv65-afl90buYR20qS( z^*&f;zp&iQBOk_qZe2JJ> z_I=KPztDqoIXI1Dv$qe<6!HpilZh=KNk!OPBWJTuZOA@QyZ=6={3nzUY=8Di2IoPE zL1=wYr5MKl9{BQSK9Q{cZ?CmwZ)WmvK1x!?n}Hk*rj5*Q&YX8urXdGUqmCifO6FjL z-s;mf-2K(f*1fR80RNMK-LAQ!WK}lhS9w_$U~|!nWaaN&-G>nV)d;jYO7hx*Eb-#& zvL8}>r}Lj4(d+m>I}quxWG7x;3L)Ep7Ov`g)yn=t@+r<_19A^JmKe`DY}nQRqu{%; zb1rc?2bP8>C>7q{h|R>w;gpdQ@FYN0{*Zib(wE&6qX&dq&(i*R|NbbofGw-Hr(yY* z@KXOnG?$GfqDBlRY7~O2YT-bLyy9cAPT56S-r%W(P*BNP0-lfp-O{H25&kd-H*tt< z>aIz_ri~E!N1;2|dy`}}il2!J}j0%JWft6O$WzHQNU*wTth!o0BHgVVq_R(CbD z^l|nZEt?u4eI^qFTykG!WrcI0C|0dw7ZE8vP|>#N>0%ieRZUTSRVA0bo7W4JB1bqv zF_|z^1BT?kJO6$La9uPSt-h@*euz{#5}hXUDrcw#8y;M&CuAFm+z5C*c^>@Lx)uZy zc|Es+%eA6dV#MoO@Gp8;^Z$DS=R7@s8j5jZKraC72x{3Mb4t)79>LxtoZV1uJk^O+ z#jd>a8u4z60ZWcSrZ~?a?CckjHFea_tXwriOfFFYNOl0IGvIbQLY8V(t7dQR^UTsu z->xDr$6o~ftybg42dQwQU|0|(|64&77UqQfIeFQe7l+9X?obDQT|Z5VK-x))o$?6q zZ7TiT0x@iXe45Z*xKrnre?6iLC1Va`c^^4oozdmRcYxv3vkQi!TDr{x1s4i|vrHfJ za#DpBU_~)fh-F!RV?Se&NiA;wGj3RF%DF-D#%7&C2#|cM5=KJ^Epnd>|erCre^Uk?ttR^+q#QKPgcGu)L zpVJ`-?;f8voSZd`f}HdoyoQ7}0z@)rUYo6NAxcDb-lxy$%Efi4dNu}?Lo9})*%-2c z5DV;ngiI(BVOu27)z4z`!x`AX6LhVXMmQP@suZJ-O9Rx^YS5spjw(-VOw0k4!>EuY z^pl2Lj2N-}{|NmjFd*IBs^7=s_Yd1g?ZIwUXYSnhtFP8je7k8nm)#GqJkHB}o(t_d z4(jQyuHm$;jAf580L}YnkSjVse4mXoB}g4A^ruzkp8D(LNk1Nt6cvIF2_T_a_Le5* z7*It~q|^C55hC(rSI5|*+w*wtek2Mw@kRm98KsP8bG1N;C*3&IBwD&mzA5?{*(}5k zG?_7xG>Y{+DH$zok)R5FDx!#2y0qeQt0SoT4BEBJSmlYXmXs&wPo zOX}v*mnvO^|3p|9@)a_j>?*}TIF@oe*J&O^vD188#WQl)@h`MTF`Intn(D;a{rmQb zCNr0MWdS~McpHfd$B*-Mqlk&;CPRB;3btYG)0P_YTjMOuUy%)l{@ z>VVi6#8H#~_VHMd*Plm?90P@hcrv5p>E5@gUuV$kYPcV-tF_*N=itZpOGaRa3}ZgKIyje=SPW; ztSXHPS|fyHSYC!VXk6$8L8;?-2w4~T?RHW++sv7j6pLydGIVv?noQbJY)0IeolNiHkcTBO`yV=eFV`OJN&5 z?p_4#iC7W`lD}(SU#569+FTKBRFg0!CrDR+%_sLf32f$yVA(cA@^@Z8OlU%9fWlFN zkt+pK;oGQboagu8homB>k14gHY*0P~kjXo61RC7yg-B8k&K&KMg7u)(n-ZmLi(+~L;DE*G(L3Fb>gOYa zkAfLl@JXODUdb0o^?T8t3GL$AF)J?UN<1S|b6@q3jMiF+_Jk4~LKEPiTJkYQ{E3%iwLNQ!&#W&# zv$gbp{N`xq_lc-7M$OjJx4n%{D^kq;_<#NV@`G{OUyo~l)pBPDro28(ht|r<$%$aw zG`YSE=>bPMh2ail^R~yQn%d!HO<9F{!zW_w(IcFwasp7Q@JKrEoTymt#8^I=)1C!w z#U@}|{N^5P+~ZDab~#XCGDr$1=;iy=%e|;!DL%&{%N@%qM%6ojZPv1NLoP+7MJ98E z160#x3jzeK>~<#w4FRvHmJDa__8QSh|igRRR8N&yPTL0`+$<1xQ+;omfRphL!u!<@huef+pph=t#D zyfwf(=?_V!4;Lv%gk|7rPWG(KOkHXdo>wxB#q4_da~x-<z^3FJHXS;rmL>RQifB<7Yj2p)yI4#Am+Sh@i|gcEl3z-vDcnHokVBEIjJS7u3EMp^Be7B zONXs5e`Adi%a)zH%BTd&Pn!woJp^AHD09zB5pRoA&~1Vngtpv$AhT{f`?2%wx|aBR zFhcuAW5;ghFp72nHNYnsm)4U96kopoYu~<_1)J+)BC%lA+;>xr-#Hn-n^yY2|JQE} zvOUW#BOsGmk2{#~7{#K=KD{-44}1V}ycwy%8_B0TW3vUwe=Q}&ijt_hUUje3S5&kl zK{i}H074M?%WDp-`o(S1$rd3nQqxO*EC$4GvCUrHbH~rev4b|7gA@f!jxtw6ne?RF zY?Qyz{=L7I$Di`s;$=Q-)?p~w!=$CJW}apvfv3LqU){v8no<<&?^$S0L39@$M=%f; z(XK+`T3X!7mB7VO@!tFVm7l?C(WGGY!26NpIeNhAx#Ymg_fm@^%uHL9a=l0I#HOmLs8{V2RCnayV3O zB2-Iy_AK_y3(JaP!x_Nih;RZBr1}O1Bh;DM*WE6{S_{V7b?B6-*ge-tQr zA!$?h{*6j+ipklp{v}1j<=ceL*i z@>CKReVVynY#8+O$f!>j{eEDLU{==dM@2X-;}Kx)r|T|bNunKDS5UvlCvKD@2O<<#nh@EjXg#>B;z~%SSdQXX0N{hd2MZvdJ?PCI|%)C%_Y_2+uI`0SyRsW`L^aH9}7<5PDh=&HL_*7 z381L*0~#Z!T_>+BUf8;?n~6G;tOEUj&GB`E4e!Jp`>p&6Eg}iRxfMHQHVLDggm~!J zEe&m%i}6rk1zu&k!P!>_WU+NByLW?;S?8|IpkUDgt-?&FkSh1X>}z=fTzvY zzxDwcjstL==5)oY$5l=M1N+11jNa;+qp%-~YIq@^J9xsS&fMdpw|^^71T4<7&1T5m z5o!+9U?y8}&%ouUU>8fDtk=gkAk!`IdX~XP=J0^GrAwY1q3j}58<6+0!T$NaxELw} z(jd9w!G??l@db}wNV~JSnhKZIB45mT9ETC{%pLWsxoA5yTedW4*>dmd(F5-90W0+yzI4}xiwK4sc;$q16j_^2W~t$)qPr%U6+|fl8o-58Mc}%c&{D-ocb~& zn&hpkdH;DY*8a#py0ZDcvWO7y85jP0Gml>QP5+vVpYuRjTzbcD(l9{|YQxE{YmUnF zHo5X7)dHJ}bD>}}0kae34|2iiQ>S+73eWuU^JiNId6uF9mTd%20=rb}a&rWg6z=Mf zECI`LwCnd|Ow^sn=LCdr-Fmj0K7Y>DroN_i`VJdQ&wR|ld|w8sF1T~SOYw^S8{SE} ztI^`>y3%{dr`xT16;c5verNzlprN6WO4)Qb%&NV)HTP6@mT<-liLc&~T-0Rgc8IO_ zUCs)Z%2{7$B!8#`s0J{^koGvWZW7cv+p|YsNk;kk*9(}%iP8LsQoKVDQ>kG4YAFGI zsS<8{c4%vkh{+zT_E~X)j*tB0OJXXaJgv$Y~yi7%06Gh_ooAkUvSZ@ca!rVa2UtFbOpjCn=Rpqjvd{E%+a z-Mc5_OV%`#ublK%R5VQ3kyn7NM!kCX;7+=gE_Bklx}&QqhN)yscCer3sf6DXR7o8V zZ2I}x?Q{UH;i3GwU zaT`R-z&884<=mJ+WF>L{b2G{g;L`6&ST>_$Vk7y9t~pSku=}im0A0*D{7Zw42#r{( z{w{6)is{(pRmI<0fa*4t;`uX!ymvh!Ymjux0Nh5spY_I#n#2 zHiA=gSYlm94Q^m0ZmU(hqPH<@8@l+_J6GuP8PP3 zHN)O6RzPFrUqyAkN!N)k{XH??sXMZ=q6!Dgn%aNzu_+g)IgY8w)B+mNWI9$bc>q9N zb4eL$)w1GFxs%2GL~P36(eex3M_|S_YAJOh>1dSg;5!t9QX9$xJ##qzJ0uF)xyycD zw0AJOA9d9Zq=GmSb}{e2o5%`YRPp{PnhU>OBYVWtKm2_+61s8nX)rKO>- z;sf|+fO{dp%*aF#zL-_!iW0Jl^|JMHtTgR}L5^%!h-r?P^3*wB2|7iO2K3oZq2S+t54~`=F|M$VA zW)YD^#d18{W14$Wx_|qU+CLxI)T!kU40G-OePH>yhyJez)_)?AP)${T zP}D~Iswbtc+57JM(0p9G|C7Wq^upQ-ar8kvZs8To zOCK6;MnG8q`_Ozf0bgjI4%RfU^ow@Wq zbh-}tE;IzkkLKLA0ox{F-@&Q5%*Bz_2(vW)#>OdC$@DS78F=D7ILDQJ)lHnDPn^JT z|0KtZhyoA3l(j+ps|m>Et4p=gVqev*FY$IkZSi)C*=kE;QX$tL&ZP+lN6>h@O;l1=Qp<=@<)g8(}&ejTcbvwbE~< zByXcPYaExV)+PP7APW4_w_$01`rD0NflwmmsRl=5f~aea@>am+kW3#V1RcUxWe|Oc zwj{EA2|>V6`3Moe3=oe%)zM+Xgr@hlel&xvI1H|l%{K?dQG=r48XL|I6zqfme{WN_ zSkCS9Q~%D7On6pb*(MSJSH@5D$7PVKJ|z4u@E_0wdOk}c-rbm`3&2H2iMht{nVJNb zbs#b)K^jGGb&lyrKuoq#<9o@olJP2>@3lRiQs!FIyxV*Ka#W{e58komNM##;@(h}= z6z889s8^$S7BqTxDG__QJvRY38A+T@r77gZw8VN+pdsoQmMHmB!w zv_3RcM6uXIGPgN|XD6kj=n+@FS~zcB($@}otd(p=D{fs;%zYKLz@#|ONldkhprmqm zKGItTjD)^~R7^V4_R3-IhtS5lod6a9k=kCwzy#b;wSoI<$pL&bRXJvW%Gn?^!)l7S;^F;{fl zS7Ot>&te@+s`brtrEox~fS}4P)`6IzkXNJ^FZ2OJ~MU-l!? zY*EHo5{YRx*}TO&{U<+TxmwHx}l!lPz`2f2o5%S z^p+GWix}?yOzI(ZFoqqZXggD^w#bIE6nhiXHUAyKE_pJEeP7sv-a#tKNnSm89z=CWt5#M2G~M`G*{4`20RU`8#oi zMbcymLBH(fnOpMXeaQ*hcFe6AZGt!`m?NddK8-l%IS4h&Uf%@6!73M#hX``aY^a;x|?i~(C-7CPa)ZDH`qKtG&2^xH>-BW{!pYF%~xXb)G zglOZ>xrYmN#(z&n-%sH!(lj}+08Dlt5!zu?pB`F%FCNraOlcc)q{-f%Tn$1`b?{|T zJIFb(pPB3k0==Xqy66po(Ibd&v=!_zD*7;F_ulmLy>JMzn|mRDCsHVj22IbpR%ijz zT5J{W?Q@d>=F3nXU~Zucs9^%kxhyCA(C0@pkGiG=nMT-#tAm%%y}q~CO#4;$87X{p zA=1vux5~?TCH_*;mslA~vqt@V>rYTq6^L9QA4`fz91GOJ;8KuBVO*I+<}xmO+J&Nz z*K_V1+R7Fa4_L&P&$7+b!SpAD0YC}dFC-t*-h=J4KBb#Bm$Llj>VSCCx}a56YCwuk zqb|ND2n7^V)0`HRzoNs&9`EbE87=O*2+Z)be=X^NKQd5$k3mc?kW4#8Ol(dNVdBigjZ# zPH#Q6;*b#RrABSjUrYfCa~(@&^mqL0x6PQ?7hay(QCHW6#i!OCGv-#u1LZ;uaLG@n z5?VQ_SLM`9ceZt zjGVdSV^^G|F+;z0?M;Ix9mIujzPo#ozqDcOd!O;(@Zrv)BlO-Qy#|D>?)y=NO`v=! z`E(BP#-1(EA)m2&Z8GDCqgJA);zgzl(9*{B4QICV9M6(3LQD)u-Fwn!+U7(_j8Jm# zx`|CriQ5VgE)fIXS!!l>t?|)f8s-t%4uQ6zHd^*#Y5C?=3I?KAxC=#+FWN-Lxp0;n z_HWc?bwx#NQj5_Ar4B$P_lwf812VL*FtOO{Ro5F3@q z;e3Z2s?GAZpo}y4BX#iNXG_=bol=qNs}d_j2#RkbfRQ_N1omRk$(>6!?EM{N%|z5P zxlcgwcS7Z;M5(m9*3`H{%GYc9wQVcrs=9TiPEQ4-W;pP z-t0m(Kd1lPJE4EKpY-+dJQO=yPtHL~Y4k0#vAH!1L;GyAa>WH$+1}2ZE9l(kDdhoJ zlzk%3z;f^kCfckvk)5#T^mrTxr&0R__)7l6QxKsK`VTAeMgSI}J@0Zx0178(Z>T2P zFv6ieZJJn&S#b@(t8M85J~=m?9hko?xQ}7eP^e~rd>85mIYdM(LPeSk>;`Ww`<;sG z2%kMLr3itu1SVaPf&sA=`QI;sDwp0IM14q=djW+ru^U!VK*YIh5x^>~bL zxQd!~28@avTnqDFm?Gu84rR&XA4@8F7fup#bYNZ)-{jF>zi$P-521SmTcMexPqy@M z*s?M2_$QIfj?1k7>OR^8vsWu8Qq_OE8Fl!u#=;*Zo^8jfLG;>udqgJlntUnr6NkS@ z{=9dN(qH;LZ#W({!dt#Ya>dXy0p9ufG&CQQ?aB(#TC+maH2=Fnw{Bz5`Q4@LmK>O; zkc)y?f~^4kzaLmSj6`KDxD{o8@wf%QJ39+e&b9vOS&&f!a;_7)9GFpS0v9aX1m6Db zs`YfhznIcvln^NBoy@IadB}3|ujO(X>jhY+aPKg3*N5y?V=IPRjqra6H{}}F?-hOkh^X+syICT8pB)Zq!4wsvKUw-?> z^24884R=ldt=WWYJrXWIs6I+J+MrQm>+J)z0?k5375~V7{B7ZpBUuw|$L5_$pEKw5 zr*k)R7f!yFdv5fheV>*MbPStxzBj@94+vc`iwY`4b|E7^XlrZ0xLI_^IFE=)0cG|O zZtL|fUjVQeg)7D!RQ^{W5T`gf$=HLcJf_urt<0gPlf`Eml_Q(38A7yeJ9n-l{vG6I zi+_w@CHMXc(E$LuOtK@6ryNVX;!{%_u|ozUo-l-!p!fB{;v%s*<{-n0eJxwtgc;fZ zrbFdNIPgy%#-^EJb$bMzlwryU< zKMJZ42nVr7Hx`TM1}3d|+ephYe|H+voG^~v;0HdU0@~H{8xpZ0!-wa&tsQeuFMJ8 znj^#@@2^Sjn~m{TgiYQ=V)Rmm8ZC>7j`0NNez9KLvw=EEbL{6&G=Ht9110|Q<*HQn zmtght-QO{C_0H75PK`o8F40Dc@C)*&wx>5U;hfrAOycMP-1G%;uyn`s4h7uF zV{q`zTC_+WcoQrbPd(cg`QXfbQ69{G$={&(s;p4RXV95=`WX;H9&pm1OP;b2A*HrG z{s&+^}*=>6i3H&>y1_#$kTCN8~ZL6 zy`v6JYq&tF=G`iyoDzd%L`&kn{RjES5)bGv8J`vAi2(x4s@NUSYTX>O)zJ@Q0!v;| z#)T4Reh2Itb53^xk5|5^gzP_I3y`VvL47yvvmLW#ctt*1<>6wh{m@S)$ejXnR6 zVcPr|HP~gIpcN^Xc^*~TG?LOY^zCf)xZ?j7M6(HHu$R2$+PADl8Gri3h>gDPOM1F~A4`sBiUO#u7jz0DzP1G#gh(!RhoO!NBcD(b6JX zOfl5Z;iNhA;bDw#LHN{fXKYsedq<~&(BP&S54PZ4s{;y6H+Od#j+FWAbLnh#>wh?v z#M}@c+bZz$Y|ZK<4`L)i_29jX$2pv4*Xr#(`VbTnI3T^();V|XoD9XH|6vUw7rp%y z;O=98{dEbU&N0rw9$Tfp>AU1ZTMFwbIPp?>Orb9fhl(pK*W>6%#FU!m3dM^&>3QVQ zSoPj!Cc9Cz)Ow8iWbC;2tPuGKd2Gy&5yMKJoOq4^a?AsZhfJtK4zAV6aKN(9HA$N> z7`^uEipn7}kf8vOc^84NwW)%PVH3(4iW$;#(t}5Z&g@ zyU4*U?~G4DyS8l=@uIg+UW(g#EMcNi(kP?pR0BcL(VyN-y4JxSTO9Qh``%9RY^f1? zp~+Wo-KtL(KZuO<7$pMtM7l;F(5b|Ax7TgMa?qy*U$>9zpn1*k0A zYl;;Idu|_tWl<@vVaPKRnZ&h;NAps8RKGPBBQPF+lV=koQoG*iAkT^8{E`G^w2Zdl z*40=&;%6v+&S%X#>dTCbizD#d+TAlqdj~%RuzwpoS}U;r!%BCo)kifu9mssa zb?Q~Oxc&(vnRd;0F+CC2MP$c>q_S7U08+$C2%6mAKWi=aeLUlacoG7@qONPHFpqpa zU5R40g}+Jy1rDa88-XEpFqK3Ab5Y!~$?j?;g5xgSZ8XAo<&ObmVLW5t(FB&W8?^fX z=3Wp)#vPwp4f&s<+2Or*U$?fRlv?B{w~F9}DernVt5~U##F#RkRcyjL?`lAs4>>Nb zmk|U0p^6Zn`gZNsFl8F$!deQe=M)PvGmS34wgVh#bh^w%dRL9f5#v=(rrRWA%wu^; z7}~&K8aB0e)kZuJJRbH0zgkb^Y1(9noMP+{al;(2qCluKv1t(HZ+3R}b8g}TOy3b; zP1JGdZ#Q|QOm-`K*>2=uazdtriB41dWq&6puU@~t2Fc1+5*3%egXJzqxB&WNuUM16 zG6hm9$b+T0+S|JthlfH;!Z3&_cr|Iaxgb?vkV-JWehvZI-Y;}2XtliZu zt!uAbxzg#w7;HkN$>7@q5#-j7)PhSWWmX>qy431CFsp{5Zmcs~$031G$Pfx|Z9LS^w z-V+)DZVGem7i>cmc+9h7t%=8Wt$ z-1Y%faop)S`-6>aJKdUjZ~#J^;gl!X+14kcvc-b^%(~6^ebT9*>3I)CPK^J_r6;z3 zKMhI4;Jv8dH+^{;v$Qd~6}cI*I_0`}va&~E#l3)*G!|B=Tv@YTTAi9Tw|y7eXVbX* zn@VyiUM+9kyJx?on9EQ<`x7Yf_CFq;P0eLeYsTXC3WhQfsrl6Y9W5G6n?AiNzPm|p zDdC1XcUrRu5M~Q$f*cZnj;U)(4>s8og<8f~y z0l#O|h5Eyjc8pBpU$6q|@@K&&3CH}KY;Z2h^gDxSTZ%a^ncfio?=)qK0!qYG7lqn) zoj5cG1OzBDTA0e$n^^yxo)#03< z#Xb9WC2AR*?z9eUYV!1xF*WnC4}-BqV{wmo(w>t1$cJwjGjBN1K5pr!>XL3_%rNbxx6lPrf8bQwanX{#{T4wxmaeSn}w$8A51`7$tr~BD6PD&uOJ2$K0?>D$A4v9lW+K z*ucK#_QZSlnvjp})EweK#@Za_ZS6-&kzhB!Ik*FMrmMs3X5bON*%W92jjy>3+N2i3 z$g7ef)1FZmSTB77_?{rr&e5ZH^_tlPz4WtAMx)No&+^%Jql!W#=(IE_#umg+({C=l zrDtd7eQKf5|0cOdCf6o&WMaR+`_+3P0YUfNO$1l1DN1dZzDbSs)Ee_41&Al)l{L7S z_fS=`H~}NWAAFWti1aZip^%v0(4z^3@dnhG5677P$y_v{IEn^Z{`K1^agrb0uS=Jx z8`ZbOZsi1>Lhg4Aw)Ff;60w9uRH&R2$p$AhvJhT3I`#I=o7=!B7Wapz#6Qu=(M1aa zFd34#gPvIj&Pf!Nc>JR!v0qts!ttwi3!cWA^XF@4ai~4#$*UhBkgq~hhfGHl&6It+ z*uN2@Lo+k5(mKz50SPJEX;2t_@O7f559A zy?bAk6YoilE|lYyNj^BKDyc?J-4DuyH+OvOT2wJSIrN?<1cjIqQ;$_tZ|gaRhU ztYe`m3MeOITW7R05@_-g;>e+SDvLR7$B?x%jN-_LrHd%)WK3jWD@{#Xo!4u0M3dq_ zyow3OS9fT|z!*#@Vm!XvPtCO5-u>UYKcz+jI}6fxFoQiIJ8CK7AdVDf0~v{D&&a>T z8Y<=d%=Fj`3AMH;9}rn=Oor{&$gM|z(zxPE3MTK7(IwoXRz@qDDIZYdh8^y9mlR$y z?!d!`4t3l!%V6t~BVB3Hw&+sbE_mcrcQ>~N_J%by#va-kR8aL5sC`n@C#J!s=86xsHFG>86A^#>EDkKly0(?ErU0td#>-P?Ut^fBN|G z!_&p-8B|y@;+{Q}Hh!b&9nj}v81MuRfV)l6?;jFEV_;AvN9(a6#I)ZwX6kECv#uWC zR&DuG^Tj?`Ej9JH=~Mlt*4Rh)kG?Rsk$He^CM2JnK{6MV*#+3gW_RDqNT&*7kdX~y=PVph4##$kxTdYomZKX%emF=WsHBiO%tE@_m&(4E$W27dGA+Wyf7qn zRJ=EyWVeqP=rDTe=XT34bp3HUL^C?yPtSQtabYh}6)PsBQ<4^*EY$BtF9l^(=k13} z9meFBu){I}rDt9`eiF`mc>)=+%krjI(@FyW$?7mp^&&f4rN0t-!v!~>ghFz?rhapu zoXPX7lw4CoqtVm%i|_$vJ^7sq2lm1+$9q0a!I{6`Py|aNJyb>g!qQ2(kih|-6x9^+ z*utwAdLRMp^!0ISuFa%THM0m1Z5ae(ku3)Gl39i<^fA6OYRs5FPpDQCiMcDb_k;7@WL_P0=lHPmtCcBlj!D#-{iK+-6KzVA1o7RlFNNx}H<$3}`UxJqLDH@)w8` zJ}Mp>AnV=A?_Z*}eW;S)lu6Z$%<7kj8#lOhlJHSLt{ zh&ICeEwy}fokI!cBStiiYSWuJf++!alT&LbRz0*7{Te07>eiNBmdCuHr(qRdTbGCf z9CI`BZ4OZP*QG4Kfdj~$zzQqll;M&kf*v$UhT*pCw{#!{>n(p{aTDh&+umyz+t@j2 zWu?3Ybxj7Wa%=+yVD$}9>3x%3m7_PFfECj_nadosuozSmNgiL|CY)O%1;-k&b`}}l zbHE9`oMP~~ppQ_qa*^jorG#Q=J7bi-RKej5v&-_K-5u8C_g3TD&;7U#xJ!d~2J zw|R4;I{ztLK39TAyh@Xd%+Ya*en^s-66>*?dO)LEwQ4Q4V061-gMJWES^w!#00;>0 zz{4}DeUkZlwhV+hnpz05TSckUWcXXO|GT z$H;_z?A>d$-J|Kay;ugMo}SZM5yTfJe{Eo()9(CN85xy0$^+kIWrg&7cpdR;!$ys) zU*x|?EEosT7Ik@P=!Q4FqE9p$U~z%dcF#wak+|ctFYUhlI-EY{W*Pl!YW`SZzW>3b zqVe15Kj_qav}x$9YDZrr=d@CFiTE+OlE$rXIuVVQuP`HN$)NGH&X?BM{7~lVPtrE^ zaqOeCg<**L5}mK5msP8F?I9*6&~I@}I}r_rz45;XDPoJO8`@2oQ`O`Ok zU!mn4vHb7oTN~}}{^{|$O+cTLHA^;$nBQMZj<@hv)d5eQD$Y$* zy}05<0u_%bzARq`m`jBS*DiBs)X#2xpP245=1mPR6hcXBS{dkszgl(fmC4c_^Xqo| zaeAF^`k&f9Wc6Ca6~*#Qp@Tz_QKzj&KG*6^#ooX^^1_?N-MV!e!>lJta}7l>KXU!F zKh(03<&}MjaV8={l8gMr09Fsc*^`InmA_dNB9Wfmj7K^V<{VnU8IDP!C zL%|*Nk&v9;#jqbws-wk6S%co@F3xsQl+vl@ce>N`%#@xRc<#NGULg|>KnpyW)cpW=(S*X zon5Y&dIEQkC_dw?hnG6oX({6c+6w6IJ1!*ju>0%OYS7gQ0Fm=Z-5^sF5)*4+nxmJ) zsqMs)laa})PM13RudwcVJoYGjpiF^vz5os}6^h&}fW}-eFQ4>rQOS zW%7%hcj^OlvzAi{iWg@Fw~{3^j}ySJkyDeYz8l14*=TYF==V^}tTIu1-%h3R`wS*hu6a8Kej5sg|!E*!KGmnU^wL%4N==A$+m! z$PHvt9h?-+kO_zlajQ-4;Xe!fZRpSl8k%ODSo!d-k;M=?lf=l@XX@3~B?CN*#-+v5tC`heu4%x{VAqHKB`*w`YORdweAE#&$`y!A87D1s+CY-d ztSpHQ8x0Z&4qFxHl9BuRHYD-i@!g2$q?g~x`%T+~jw#FqhH;BSUM*_)Ft`pK0j>_-`GdAy7?wPtR`}tgj;4HLL;N0PvZf^C9)&%b zwLR41Rc7WCHX=aX@h6$XqVu3RT96rpw8>pRG`XLNRwCXH+HvTwP5{)R^^d+<;F!wf{2AuWbDB;UG~2+0IB0j>0aw&LJdzMq|(tjk^J$%D3zq|lp( zM~je+rF0@=MByG~SQmlf5`Q~w{Gc6j)|hF*N)(ZYCj_4+7ucxoQECZoNYi|e-OqoY zGpDQm^&>X|sSf5^{Mt(Ku3`h;yEd9KKxl*r56v;jw{c$e&X+WgpB`_gJ80?Dni(OKJ`;J$C=}p3(Qs9{PuWJmZ;UgDDo@q&keNhyZ%ed9neD zspj|kt^2PfrX|qTn1)o!+LpoQoloxK@!HQwv^LE-X|`bEWUe~wKPDZ9JME2lsa8qh^51HSKoK8iir%g`b+^9W}N$D#k zaVkg^(5}{j?;Nw-Nn!0xoK|NtetY~wK97MUgQ9vvWH$EOWUTF{Lo2E65%dbly>kYP z@hC1l88*wwsh$4yt4D8sD)f0yoB}Fqn2ec+)8KVmHIL5txu&$4Kh|aBo}RS;rq^A# zFwiT(9J9VQ(vam6-$O1Z0p|T<)KIAAfH}9iAUOR%E#ME{RYT>l5aC-oFTFC?$ zy&Q7qw%D?LjEgJzF(EA$_-%e}*^}c}k1*Kvj?0E&A<;sE3lNu+PHx?woBH0qIFb(Ci^LY~Zq&%%L+v;gP8`E`k?;mXAl;X!e?`Yc( znopqUrCH0h#eZ>-%baL3$e9Mbn!CgCS#o0D9u^99hYy#htgV&;^})@u$t{ZJ=r2{g z*hGHg{*F0u0&L8-YpjoV4>&u78A~13+D`4&C_Hl8h}Qe&BMftzkgVUDmE71b#B-Q( z8(bE#MF8c~{I4yWHLLl!dGqG_TRZ;ReBAEE#dF?2Iom#bvb}v1i1fAdU006r8^rhq zJag7jT2@kE8mf_Rhs+jz2cs@X6F%0o3TFeA;&k=F1=pmwjHXjN^WxHI4wq5eaM`0V% zW;z1agmqqCPxH{muT~A2&$X%1FJ|95KkHa2RqA|1M2hzs2ga~gq2BS%XSzxT7si2a z7iJXf^7LFUlE}3JZUrx*4s7Iy2ftOsyni3{VF*BjS#lg8dsGz8eAtDlFiD}?9~emw z>neO8EW4^y@#OS+Coe2|yC>`kt;FVUaVOU^o}hmqvyEzn{kVOziqMz}$>TmtAr7}h z?!TEFwf3tCOZCUQ1#Li2o7@`|g8W()xy@|Iu8Kht6fy08pEAXZ=1?IZWU`S$%2%4Q zp?^S`6rNF2rd4DQ!QQ-Sicd>aD~Yi4Ycnfd+0olqoJ^TfP+Rf}vpt=cLXY*>6kcG_ zR#~ex-F9i==kVZVS9bKE#b}gsF@p9D9eOvNdlY!G4h2(nMaDO0QTeJw7@XA|j6y$r zuA--|0wHB@{cCu7)T&jL+!rqNo{zgxaMBa&xI;c&Ij1O#>*DiJSCP2_GAG-m^8rry zODN>nVQTxp8eSYBwW?f2o%B8B$iyAY<*^tYXhy%6s+0IuFtf%W!sg!nZUKgqJ2k$? zTuG8YpN=(ELG+JfMi97$?9Z2R-eJC>N+6oh8^S8mO>oX7Pn7%jlE6pe#J$jT) zA8o?9JxjEm162;Mzn}PXQq~ARIa2l)(gv==b%;Y>6zH`<6n6AfP6Xw*^CK9Q0-rP+ z1ZTD(0%)BAgRq7FSECw9I^}D9EA|1#W5(gKl(B8+4%<4T|>l zZhc_D(ja@aTSTk>tc4AVrcvjOfBd0oU$=`&CF7}BL}PU(l*aMx=g4X=`}n>oY)MnqZ%-oI3TUc(djy* z6OBhj(?QZ7NxR=Eb4FcpaOi!tTG8ltDg;qLobX(<=+vWH1Lr)8sf7-sFLP?y*;Lgw zksb~L(W9CXMMi*|*F9PnN1WWsnx2OKP$vHI-@L18TTeX?=99dP3S49+UI|)DV44kH z7XaqUAVMJ&2Tj&qjqc3Gc|QpECi}PZ7A!EZIz(oFMa(n~@7TN7p#NN>;?Fi`^XUev zOL`UfCO6lh(>ieLn&tzi*QgzPG&-iYexqj1#!c0Qpg8~LZ|`1a1DVzx$4 znKi5PEVQ$J541HkVUlVPJXNjMQtQ|_{R;N`T4Nli?oQuvCil?v)7)E`p@|1!kA&P# zegA5b?{Mv$$-fV0X7qT6xvUI}TxAif9?8p z(Ss0Zdp3KzJ`(vz(5qMya((|OVdp6i2A^erWv7Mj$TaH12oOA&MhCUy7y-Aj}VQ-;p6ng{|J=AxF}AusDisb=o_4-i98-5Irs|(-@m`hw*vZ} z8v5}tju^TQ_VhHOZAGtcTYWWJA8csq0rm_7MuIjJN+{{=^}&pfk+I^`F^aeUMOeNO z7yJ=(WKeucNi%L-Gu&M}pM4AE=G)<@Q2HW}pnqr(xFT?QX*K_nJ=};D$xtv;Din0m zZsb2n^XzO1OQHfrDW`SsEK(jA%=W$zl@|L-Odd{-^LdQR!8XmI+eO7ngPEeBC+Yj^ zuh*k~XY8*uR;nDa^%5tJ{;o#2xS8FYMp6K%7XH17uYN9?2~O?;xf87N#Va8s`0%yS z*aUI7mUJ2z+Tq!7hcH09WBa#lvz%H&@$LNo6=kRt)LpT}wq8Q4Lp*lZ{(T zHBC2l6+)TQZ@Sok89AAp_fd*i%5iUCA}uq zARfA$HO3l6G6~nc@Y^CQ82nH~{t0uJ=EWhLLu_xmr+{szK^YU(;1=|}UZUe30UTLw zwX)z+odB#fa?w7#lnQa{?XZ2XUHU6!(VSl+mY{DC>=adTJ7!RFWbeZ7!>#cJQk!zX zO+S#+KZx@n$gjPScyTLBdc2=7e0U886^VF3hPrXpQl?0;K~sX@ZBhzQdjlAEL^7Cy zqMDRhM_)f;zp@W2TbwXXD-eL^HDpGg^vL+eRZ=s*hpp|kM@zjzQT+nQx zcu><1VWvg!`*1Tl*f#3cb6$AU+{#xsNad=bv>LFa0S8UzyPl+;ZipsQX6EP`b4%8u zC&&Ck^enXVYLVqK-)1;cI?edoKJ{9kF1S3CA-^gU4t1BT1IgDIhQy?4 z5xT*_GZWklX+|CNPVycQ8UUK!mt~ygYUHvBmwO_W)PKiL*=6WdnKs;wY>z;;98;#_ zk-5*KOIxNhGr0PUWja0XuQT(gEnjeh$DN#^B|Hk_xRGDQbJyshz%8H}nu&%jTW*r# zURSp&H>VhM>TKGM&%XI5tq#DnPo=+*;pufSh^x=}vTM&C^e9dHF8HSomhyQRL@M5K zjoI3^NBV}40ylwDJ+`vMQ}M2)b!yI2A6g6pAwYm}TS=M`VCY~D7?auN(v>UK81>n} zzQcFan8>Yu`TXW#W6I4T9OMvdYe{Ei%n|7BU*pyV9?|{7Yw<*(kaxlZ*w0 zM3+ElTlIG|3e8zYH4vLIqQ0&ujL^T-BISP)w^Dogmp3w|*?W017K6-Cy^#8Gx+86rK(XyDqggT80Oh;XB+z1U$oREE}=4Skjyk0%l?6m z_u%xsFSbwSB$lo|y~!&%ia7P`M*moZCW^Q%L+!j9XHwc%qH~liMI>_i(`EoSyJ#&_ z5T!RHb&@H&2D4{UX^5(jBx~?|uNTl>^AKYN2M0?@h*jEpCUP@qpzHXMVLqSflbCq6 z5ypB6B~>)%k(#ZpRvTRB$cf2NNIbQ0_q*SPl9o}^$j&4)J-d(|pg_0Exhx`;as z97|>|GuK5M2mYy^F#Oq9W)W@V2R3!^x_&Su#GAxI_)l(igB!n@2|u$JSpcfe#0rcd@`k zQ`s{T6R9A4+n~znJuYTRck%8!F&Q4r2gYX_2bFMl!F6Wh=x`YipshrOh{LrNSgm*j zN+oM|B>5Le#T& zAAA>KnFN3@#l+MgZjz$l%G}OrTMv>UDWfmgTUvUR6u)2DclnpY35(#bWD2aPP~tN5fv8e9 z%c5>{$>w2gq{L6(w?<w?2 z1d3}f-2+0jk$Tz{K4E&^mvMjnR#%$xJN-a|AbMByAa)L?x%uO0!cACyjgI5N(bG)< zXS^9J*JgkT*p1y6#+Lp#eYu$zbC5uhHXO)JwxJONoVtks?@!idNs|qWPVxkf;IKcc zsYcFmw7(T;WKW-p2`MYsyaN~U(sV06u3cg1>^=d9k&S9`F8ls46YyJGR__#G_ z(+u{TP2K|>ehd-DK6FZ=oWUZ;aQhv^9dc!?Kv7R4XOQS%v2}6bXICeak$ma9qw!wr z(4hk@Kyn;I*NKz3PWSE&X=PdqH%1tdA?>JBO+K!jL>5QHs|GjFcixMtlADk>aIggt z@g&}(F_0mlu0baTU>_9EW|3gLk4zena46>!3&IlREaYGgAdu zHlv)7L6_3*!O(ijtw6R43QSP)$r}-lD}+I=O^%k$VIY#>T45Q5Xhq#WE6^9z6+@QeWAlG1I{* z0A1H*Pz>Z9t3{vCwQD_|8|BAZKo`=`b?ABnNNc4e9sjk1^d~~Cw}^t_N7Jl%vwgGPy{pGJC4twWXAzxR!pUgvn6%{~ z+`n1s` z^R(A88moD1?eauIM@QljwClEGUni-@-Z@WaTrK%XCQJxthuyWP7SXQ?-!P!%!bOYL z9X$B**RKboqBaAkPQl{f!C$}Ezv*wh6>&U(MHNJ5!-@8CScx%Q)bI0o+n*V-6&{mEAAm!U1?ddaji`022ns49h8uK@G z`8*E&1tx|P7q_n?a@F8BTy;E&#F5u-rC-38LCaMXSgD?^q}4=TR#zk)at&s&mENjqQilksGjh%L3Q?e4 zE7Kh$K%QL=FB?5HmEPlnFm(xW>`Ny{?2=uy8~fj0kNUv$!EgVPkvge0vB2SZ2arf9 zUJwVOhvzIdg)=Ds!UZ%Dw4aq8x6-lv!>ml_BPVe>Ur^Gr{N8Rb`i(tXUBOCj<2!$@ zsMCBnDyQ?=UpHv}M-DhNL?#8&Y|#kTNtyjNVZt^JJufIS?rG0bfvMFM`y!{g3_+^i zzI0g!`2-h~)K6=!DHvuOAArqVbr%tQLRp+<57$Uz8ZTV8~s5HS+xX`iu#^ zfSr;2QBZPF!TU`wl00nxL-6Y8|AGb1(CO24t5ZM6oZ@&tG4p=Ck{{oXR&QUrgm!W2 z)0$pML7y>v^5g^S%BC#qjKT>a!OvivKW3=j4yl51Kb0s9hg(wij zj}HbGQv&!Sn#;C7+4-NP95#{-pTQoX%wKc-cz303hYr=rzt^5V-G|D!SF3iVn+^%h zX8w95h31f5*BF(?OZwhf3Q9cv!S&!W>z1-!a`$b!qpb^MeJEa-(`7PW z!XlyNn%0!n&daJ=&nfYW(|5A4(nq6z4&2KL*4-iM)PL7kR;a8mQE`>6ug5#e*OwS8 zXJpIU>#yV6UPSMpNJnJ5A;zU2;IRUA`8-0G;>C1;V;_I=##(s4>^heEjW1|rX&E5# zH%w<4Y_OsI!-Ry&qBf5l_@97ibr!A36;WLAW~I%IQtFF-6(b|)x!Z1QEs*lzvbes%%%*acDtY7&ig%<@PHt}L-7SqVK|UdTwd_YH4V1PdC78UWNuaiE zh|5;fQn-?R+>DVacMtq$4Vbm3_*(@?@CUx3OT@rTMxjHoMqa!oH>2k*tZ5&7`TBLZ zr)O;SgJlQFDEQ?r(f}Gb00X|d!-mKO^Rk^MmN*Zd;elAX_I*?o<5WLd$pBK;-<$pS zLA02L^8%el7io(n3YV@H(KjFg$jltVH!5_&yCQ)(L!6XC_xGn1gcruiwYu2{ribwEv!iuKP%m`B2fXC?@13 zNP5tziV z36_(YXDD|r$cLc{BDvH|W-jT{Lofe(0d_S}!gLaphsgyd6V89Q=S-nXgY`V}?6Q;C z_J=?zfNF{fr5J~Ac=?(L8{kTn2XrI-)a1(HWk;x7h5iYu?t`q0(l+Y0%&xt}cd;@R zM4d=qNx>0jHHS*fop~o{kGV6vZHo zy5tH!x~~72+V$;ox%{%%GcmMo$e?v0^M9^UlEPlin#;HOCVp-i4DD0!`l^O$R#5qk zsd$48`k_vjei(mwH*j)pzx0xl1ySx>bkv{Pu4$-cg%D@ZmW?CB?fz*yZEyIT$8~0Z zYJIoOqKAKd9T~aXX~y2;;jMPqsQwt{qaN3=@%)-YZt6I=pNU;r;PL0{SH3gyU%SPL zHA(I=NB>*jKb@K5Z=8Jc+~mj!qcP^uO``fgfbOK}#HuD%l+`;I+#1*i*m#7x=y6=} zm5&IV0?B@`(1b=#ZZO%SlU3-3pchk_ACr=!dDKE!P znzWyJD;*PC|38$y2UL}J)9$_17!yl;6I*Pkaoam6ilUfe*>=T-1+al&2P}Y;#1=Iw zZYzjj#ex)3RFD=;1XMslz=EABB1#bvgzq<-BIfy1&c=-_e{<>bN@4R{$@>xM}TYHrDIe|0Io;^E- zRAA!cCCU?TjVR^W9ln!zX^Ps zT1oI3)$4MKni_x>bLvI}TRmEKIjxI9u}VF$tPNzttnUUra~|-(>YzzSjp)|7qBW0K zCquvwfP}`pu2w6Hp4hLTcJ^?#?^WMtl~a59ElDkq@3f26%Wzk!R3&Y1maKVVo1u(8zH68p zxwF&DE*eb+vTKGfe}0pEf4Mnmuy1uV1?lMeN)EOcG)%b{mk!%eR!sf zCV!?)zv=p1`={Q;rp>B;lk@n52}5%B4taB>BiYz-d08#ida{0h+)~VJt+npi@9sA7 zssVM@z9+|!vMRFe1BwWfU#)IfF=9=3@zpoMOORa%!_^JKvZ&N44BfM!q|Z0>{lh=9 zHNR!pwC?)?5P~|UBY)KGR`UA7MEU58*eg|l>EC})9n^`WQqL<5ofG+9D|9|jL9yDF ze=6Fegqnz=`ZOR$_tCk+?0s^xWwS4zD$SbLHhux4c^PEh)Yb>uOgCTkrqUlizux$(Oj&?$Q!k$@&zk5o z1_+cNQKxa!IrSpW(xo~7yCJXr-F|O~6oVd8Io1i|2PP&B->lqPdCy0lq{^lxlNfzN z@@|Uepepa4;lz`uWVF;?IGX81#w$B0B7)!XAJaneV{c)uX4oL5zcc^_+!I17<3M=n zq9=|e%oz}mZv4h`g*;xT2JkM_(! z+uWXs<5o#V=4nhl8>gn0NT2<@4P(yo9G(^GXmn<3_rzBfhpv7eQeP*?q`MAtu#1_+ z>slvHo3RMv*VDhD;>a56{PcmEF(J13Q$WctcMj=o@bT`h(zZyzC+)_QreX@>D9O1X zV*(00?i}=!Y!qI`*i6=n3&g>lBHN(Hxl4dMs-3MEt%fGm zh1iecc|+nea!z^JSvr=NJUs;&cfL)r{suY`d0PxhJ77>_xmB|-ggr1jM9WBT5r5i>+c%7mlSmMK2BCM0QEpH8H52 zrgC&MMT7GrerTg=@yLq0k?Qi*nk8>**tG|%PG5K&|3xl9_ zsN%ZiYVHv<(lgud=)6PCR%=$sd9{yN2G(T4;H(n`*~xt>AJRCXf>?e38GV(hZNF0PeJY_Io|A zN(&?q4yMoyx!Uq(O5xEapOTRJIs;9mJ5iFg-?~`ycW3 zaVWw<=n1;$2^JkRLFRDuK(_X7#$!^eLqvXc-VJ17?r`puEyPoG}%2dsFRWo{9|ps8EC zc+5QR)pl)4KcvS#anA9C>v3e>4vv5;x_?W~O}@^Rym+I$t2Zpsn^7sdoYFPNdSl7E zZZ*&Qtb`H)tPx(zC@r$3$snICNyi*@8(hzHgG`Ph+usx3@|IjYR0o?KE||8tUKl&rZG6j+sOO$=_n@nxO z0dJ5eF7q?4Udmo{m)1jyY|38?^uNncak~0 z&TTQraD^4%1DmyiL8HbsK&xWXqe`^)p?=Mkn7uq`QH4jJx25fVQojrFZ*^N}V2$QY z6oE(2P3^vK+fL1&?oaAD)4x5r!B|e3Ws*2K?Cv5EOb@(v_1DZ>oA$dyCf0`Z&+(uOzR4DIkQwnqO}EYkvw!3ZObUgiB*)abI=d zGdM%%A~f@!Dt^qg*N@YUZ8VWm#OAY@SX<2U6gtf=thwH3;0ag=!58A?s=gu~npRA_ ziD-pE?PUe(NW+T7?Vr8eRcT8+W_5qwmPMErqpSd6+1`>g)* zzFB!WW(EA)eD*?;D?yMc7t;36?yQn12Hwk@8IDH5HWptj#DSV8R(B2c@bf`_+8}?o zKp2OGaykCH5MdFY7ozTv+z{(G__fbR!d*`XPwhng_w1D#cUbX|Z*6NDyz$Y$HXgbc zE3t@@iUCW+jzW|M@Vw1y-uco!)%S3z<#&TKkLc$`b^N>%Fym>Uias?D><9DB6^E*+ zDV9q<)r#Y@(A(z{Q&Kqbn9+4)<@N%?n$OW#--_px5PymcD&Xr-0FuW2m=5kbc(rBB zED7n>yuMF(bGGJzy?^5KHW^;dj~!x;uw%=9^M4buYo#k5SK?~|Sg)E!;=WC#q4d{+ zc#ESJLz~`{*XJX(RBAj(AN~+6;RW%^shW1|e!-6w2G(Co?`=dK9qMv`0DfjUn_Dj* z4Xb0t>1$DQ?G9;Gct}N(4%MKxnCX2`MCR2CL9LExMFtS8{n6m_Q8G=KG`;)2F(XE4 zR^n~3G4OIKQ97zL-bO<*k8IFJ1nz$=MP~IJ`3glVF1x4W3NnwQ>xa;qvcq3}+}NsOB|Z5k?7 zCKYCae6?)a7eBG85oj%=+is%QH-i+mt1OFXmD-B!q?+OI(b){OKnZQ;{ zh>NK<0#y_Wlc$1bfm5j~oJwZLJp8#{+r1dflhW+CspdP)t9ESXJPxmU*o{qHL1)@- z;@VCsEs)x4%uLInb~fVHtG@i7*Yr{|bRnv7Wy;A8q?vL&dDNJ+`^zKIM1R^t~s4I(#yEznP zF<`hs?~3=9CvH1j*GB5KuJ_}>$;xu%Oc@?M{?A`1@DgH}PeK*I@oonVw> zr_WwCt>#;uORn}&!*eWwh%7@q<)vr~HSC#AF+IQYB0@0PW zStk#~12L(&Xiss}s+#7~zfAK=W{0dESJeW5Vw(v3H%DPWFa6*GwesYYZWRCB?y#ke z8nx&z?*7d1syn>ZfqT)9}c?lp~I{ekB~GAlg)Hfz>7l^q*TMeUL#q0w+4>2ioy&8x9nT9q`DX_I91 z&*hR(h+!dWb*bRDn~W&;pvff_;o;sjFUhufl|$!LYGiWtChEWYO*1j0+rBX^QLG39 z7-vu@+51YB)H%O0DYU{T{;@;qe)FzA{p6b|ki)d2F1X+|(fA1Sh>=U#4cH-vZL!YduR#O~o0Gxk z_!&{5Fq@u*!<00BVJ)-&sdj2J4FZ=x9;{;?JFa@g_#ty=l&<)6%R?pzW&l1h;9dnY5kalDJx5KHk zJRqMZAxza*#H<;{l_Ne{Z(er!u`UsVy_g~hlrAbQ2;_^)9_eBPzF5(R=9N3@O=`O* zlDAPzmUHI@4OHcE3e2ed5T^18(e)R8Bd7AZJNx;74aNOol_l6yMvou(c&{uWEX9|< zs_O7~pR8D_+XNY|1aN?s-g6Ev*fmsFem@b_9EhpvuX8OcJ=@HV1Yb7Xb}Rg6XB7@y zc4I^1VS||5IQ5Hb^s(~7o9JwADFG5z& z0$Zp>VQ?jHtNCruZdKxs;XamxcgtA~akHo&jZsrHFc5lnG&YE}tZDh)7FRytktEK` zNytNuQ>R~`lZrVi*HN14E7l04Vf4jf?5}hj=X8W6rnc1;hVm2#OJxN-JH<%}0y))$ z%+C8NA24EYBKSls5$EOs52{xBg}$G)6KY#yoE-`#fANUlR-THk|4bg$b|0sr&DxYz zD)p2zduibEf`POxeHU5tHz}%qU=qxeBu5s3n=SaCdZAczu+9`CBUd0m%{MxvUG-B_ zp_#c!2jKrs5>TtuuU@^9b)YvOnrmLuftngFs=QR$n^rhv&lZxn`gXv43O68Ze9hb8 z->nk1I2VqjSs=ZET7-Kah?SqSQ7iQq;4*Gc8E?!{K}3?0~ajQSV9>AX&bhj%WV{{OHR z3(Tt4BD88PVrfO%d%!90?t57bEME%z%NL7rzG^WV{la2sumA9t&g^dN*<5uhWrA_v zc5!#P5IeXBN!xBU{FC#J9hC=g>7=lt(A-6|%jrGe*hJ%|1xpBwgXcL{^XBVzuDbbu z!`3ddf)164N^>%VXBkaE@c(l%TGy#u#QW{5?g*n8WNS4^GL$n8F4(EpS-7E^=XPmk z)!pB439Og_nRA;=pJ!g)#9{BlNW2Ma`@hzsL)FqZGOk(=4}Cq1lf(agJsRz-OI9j)=dau(kpKcp!^m4$UJ;5<}y% z=us~q(bf|9?x7>F`5aBgQe)8~KE_u49!oN2abziUZ1Ylu=vJu*K3UhXi^sb8k`F?= z^h8bTnKYwnpH+j1D}}skGEu*uYD)Kg+5nL@q_obQJ=?toba5`MUOw$hoQ)LDMi9hd z0tARM8UW-f?pZyycQnL*{Tc|`)M0+*p+Dp1O1JyY(qgeVIoxVTd4G-1-+cIlKB0E=)u+g66$wNh`EnbH9Ts z??&qtsVDYKxM^g5cps{uH8*Sc?y7~~v-Y3wN1fokyBxNeqVXhvbnzru)S4vEk*<}i zH(c#=`N?3W%UNqORtS@*@*t9j{-2IT@Fjqg!vFkfU*A~vYDVz~$-@58p!aFpVJy2W zLyxwGjDAH=j_Y?Rx>mh5E@XOF>?<-M-jGbAI4YpwgiD|h*(M7nIc8{ulanaTa4Iv8 z$rmW3$4q5|qWeIE{+Cm%#>V7>U5AkcHc8V}m?mgDmVW3-HvZJ%HM1DpI+;Y``jfIX zM>c$S6U6EvKi3W)BV>Uj^WpRtUBclFct|Ll8@Dzr|8U^c3gS+V;qYZ*p^`LC03W(< zm*_&mMcnonS~5270rRcJXi4_f#4bg)M64Baaj%&hFQL()Ebl(|7 zt7{1qrpv_2SpE(D>j5R{l+}+=r6i>gq%>#U^GyTds9$o2pw56CxJ3{dkJL@+o3-J$ zcqW6QQjpOA0R{9y$pw)yK|^vb8hVgbA}`aAxV@NfJ_N_v2gazp#d}8lb(QogHug&V zM*&%UMB&({ttnv>4i6^OgT$GHsN($+DEn<=|N6WIkm;_i;%!UNM2t&^U-J&Z$;x(MDhBLPe$o%(us|d7v_Ug zVCj+fnii8SjPcGvFaGRl95P5KS5Be_*}wJavp6bIz&2}!FFTrbz0P&sPXo^>b_I-Z z!!mS~!WmfcS384YsMAfvH6~EcLvgDPtMehF?9*3@Z8qFj0W8JQ6CKs~i{2`2D0m5- zC5sGEU9`_$@(b`;MjUjtWSWYL2 z_9%<_j&QRT-zc7yUm!zBu|_k!SEr8$kee|RU=17+_RT^uNy?ytKsVJMW>~RPb`7H% zR#3XVcW_rpky_+Tboyv00Cv`r8fP{};A3kSGWENcEJJe!y1mQC67?syhbzVaGB&B=Mh$s$AJ^J2IlXSj7-{*}#XN8TN=* za>sPrbM{Z$mNKl*r{E#&dP~Cm?_&dCC*E2Yk@*@LuOVp&!-_T~zoLasol0KG+vsYR z*P%S3`;%1gJ-|Ir&mFH&oI57`rLYiUIwjx`KfvU*?Xk6;Uk4@|OHe{ptGi$fv+xWL z`*^`zM07pcUS0R@h5M=-Esbb<$#_!;pDwv}kYC`FM%8yCB3XjOrNgDlz%3H+J2xwk z9`8i`<%wq#54^cRlCx8B>cGh?Xr}cpJ)G+W$>3qbHCbF`P`cX^_4>j7waA&+!k*{= zFogmYH$(7qVde8{$1W)DZ(N3xm8yp~7?{`o*T2J`KV7dQjtPa^J>07HDl0^uGDwyd z$A^+*+DjHT(N0sBQt_}Ro+hgrDFRKQnfaio16rCCJ&EGtStVtKeoj8Nr8z5{>`x{e zI~J$8MN%?T8t*__>#AF1pU{GM^RM-lp6i2BF~!kBB?ARhGmr0z6@$8zlP6iH@Vo|{ zNWBY5FA-ua8C)6&`T==Vv}p(*-}La;5uGy;-;NV5LLQjek>k}+!<~_HyV=<3!}Y)L zmiyl86{T2POVWZs=WNWJ>h)HHb;jr)X|7KY)sp@$USp4EY3hq&utB%USFD1{PaN-x}m(F&A` z2uZStofWbts^hM4l3k@P$}#Qf7W@IdqDH>#p3R#Sz4DKK`q&s#V;1w06s%)bV0rPawwS)>EwqXr=Q<7)#LM^jh7`n)RjH|5 zDKQBin0*5{I>n~8;#>fZB|OP*G6i|ua5EadvZ#RPllD^=md=h0oJ8rJ7xAp@q%dpZ zua|h1^Xdzc%?|qTV*8S#pSdgNM4>G1{Xxf$Pydvn?3w_fDv%i@7bjQIyJh8WPG$_L)MFC}_ff?khIT(p`M#$gw60(@^3s zjeQJyv8_iuPQ-o9LyI$~PmiV_W&4CWrcM00t?BTyCO$_SssvI<0oX0r*h_1!e_rgF zy#Qwsc3uS9EsWRAQ!k{Q+!oMomF;smouZW}fS24-$whhU`2pnz>^jiXDDPzQ2Nq?C z7L&44$L`9(&Tt*~CGDk+M7Y9p~6|s3CPgWGK&Fd{49^GpWHP_Tq;nS*}Ng%Z$Nh{LZts zAy|Swp*IY|%i=N|JV?!@V@zWqgxsmib7=OoZeR`;st~pb+E!8yCykESpK=k9ptvdlp9blYc#gz^YxbJt8)jqGRHb*zyoecK4lc? zw{_xf(-9o-z|6C3au^Srr0>W*4!05G*;^$WMeAfi4?m~B>$W&&rWbe;k%19i6cOF+ zxjdE*U9b1P`dUqcbE@D2(EK#--TT);)8Bv2Aet8L zZP6B!?j|4`Gp^zZK{IZH~7cuNJsYB;k2A4kIrQ+*uV?@;EoixUXV( zuSXmXp~D4^*p?M%=%BM(r=_CnZ3IlDQp+cCD2dtEB*#pB-c*h_=OjfWUwAO19e1ct ztbYFO()5Ah(X=V&F}rmz*h=!)mc|A=3K-LU>WmrYq#USADl&gh*ZsJdmyq>~4%_&h z!7Zd=?R;#9t$Y?i&iD0_oPLxmN#qpM-pwyFJ3Z42Af#;T!_TE}S2J>AHkzVB?_ZOeoE ze?53zT5>(@aNxYw`43(@UhSP+Xcua7Dl#Kq%i_JGqvF}O^t4Qdt4c*ykL>K};lJ1R z+WD2j86{F98P`lx)Xvx8jE{4G@3xat$%RhU>rwG9`Uiu~aJZb$funa#Ui@5U+x_<2 z#(wFksV77YjV5Q|?@4ubLaj2CQ0Em*9VlIjbX~LFb7rE}kzuG?OEKJ+iPJ}^s?UUb zpNbF|lX`k0HCLbPtJPtOkSr;i(X+e3O|l_sZ8P-qk*;`>md7O`w*8CQagf(tR5Fqi zxkGoeQi{^xn!3U%qV1N-NMqU&W zcGcIPLoBri?aVY^P|j;Z zD~e2;Ck~i~8=MM4ZdlZF>S>qV;aJ$lF_X^C?6<&?chbrK;eroqe8Whc`s|rY+EC;5 z??x<`yC&(Wy;G+%kkLjUPsPI9G|`e8Po~i1EZL14_eifr^=Ga%W#p46E1Sx+csfmF zx;(;6J-T(vI`2_P?d&UxdlShNwaX6s&r7cBd*L2E>xNq*5*8qlV)v;qxs!?$6<-d9 za2=#ce@EP-vf-HIHiM=ISuK*$- zg=gvHzlc--Bkj47ugySNSD^LSV z#*7=+|L6bpF&k=pj5flt%n(0LD&{nYEt4KJnH~-k+R+`l^iuNu~K|}){V>+iU z?fLVgc*a}kIU!vTK%3~#^A%BtrWhKoMuAI=5skx1ToPHrOr{XtMJr>6ZN2Z>xOjSc zk_q)8^Oi+kYA@p?zTNZ1ZbT&Ps@_QF{*1PidoMM%=x>bH5!+Ds`hg*bY#O~Ha%qq4 zRlK!9&qVtn|8c-WMg-o4vpvOed42jZa{=7M^susZQc1&)mA)%q&@%fjK^G-ZtI10rg7iF`qlZK;o_mNuYMDB2tQ%LcT z<_HDyO)&@i{mNO~FWniVq0y!>EOcu^^)ASRMVY-=ce8C&_m~O zpG$M(6K$IqSX2?w#S!dCB9csx!yIO-1^YUX_3S*Mt|CgcBCObp8{bP-ZaRgj&04jx zd@;4=GbvmyPpaMu4=d*83Hp}(-3~q{Khlb^;uWf_%OP#lgfmd>#$w(|N9#UzMZ^S;WGB0fBwnDT+Ns?$p!0s z0t^Zi8Y4ggkw_XD^CAl6&f@HC7>mZhcyIC$c9fsqJHKEQkoiNH`6Zm+ zs@9K^q38H3zeFu zT_A60c$!)EGRm*`-vKuFel5e68`A>9>ox`TB6@Z>vo3@q&AdokGnhhhLI+Z&# zVvZOlTs5#c6L?7rEFl8Ty205vMil&tqUyZ!sFzITWpLx? z$O%qF*liC+QF(@TK#lc#n=0al+!%=`r=m4m1zAx@ks!dsQvMEaLo(aL!;ZRx@$TO! zOvmsGTR9`X10w^_mrt9uM6`-FW~pO7dG$;0k*uK3GTH>!j)*($k9{ISQ{wM8ZA5I?_D zyV3D@JG2nUFO6qE4T-11KD=l!H^^!PqJX-jWOYVSlsg%fU#RFa5MUjyhc;=_q!1oY zr4}_){`Oq>WLtWpA2mg9RyB)K-_4u)@hW-8NgZ&mOfdddYu6q?h03nzjSoqRLxWo> z^rs;vYf%~=hLEyINR1sE)~TVQ_%V-(u8l8IUyR4l*-XA=r2NToZL91|#S&TdvNo)T z$6m_bj{JEu8`2GwMha&-p%08)9}B8bJ~AAofE>N|=-oeMC82DT(lx3tDlyAau1v<; zO~aQDV3feJ$HCJ8vO z5-b+5foEycG+y_-ZZG4InUtD$9`}u6;x0ntY+$qc1boeDg0apMzEVUrLM-wWoYiPG z(6c{!CB!{7&xv!2pkC9TL=J($Y|3bq@u|-}uwlcmo2_Xo^`)lJ>*_G%CRJ08{<5in z?1)!ua)jsCuV1fJfjN`0?g8!zKs)h2cTEu9cY6oe_hu4ug-<|P$l?ZKyF1#w@?mV2 zBqq}u+D=1d<8JSn{ztyu!}FwS;dkzFO?^lg<9J6RJ+nv2ta#KLN28e*ysbqaU%F6 z1P<|k>7kWbw*$JKLVEUpNx0L0Sk^lB5gZ#eeRIe&9Ks$s0!v_jZp;)z_ZMyGk|ij5 ze2W=-{?7fMo@czd6*L*X>i z731kf4b63>%dWcw+!ae=*^ z-IKM(V1XU)si(TIey0*0hE_RH7d9^?DyA*(B5Ib!B_-Zsex{uGQ{qWng-Q*1+;-@rlL%a^uYX(14VB7wgJo)aPwL`3W;mn z%0)cL@H={BEMd5}?b?;hbL|t~^AH_l%-{*$I18$%fBK61{y#%x zVMCufm|#yLHtgkA!@zl9%!Y2-yxEeQJu7T|5Y@J|TModi107m|S>AJx)VSkg?FMYed9iwS_yw4Pk)A7qwB-9m?lXsiUt(FE{l_P2U2w^ zLbw0kxY24;lQ(&Vg#+qq_g|heQ~WNLZ2-Nu5*A=rWD`(U&9zSl{2Zf5Yb;p@hOQq_ zTCigEzxn14ebo}I9RE>r$`{`D1|OuD5OE7KKRnD~7J_`lQrD**`|aBW`p9fUt=Z9> zJ?+v`qecwf1;%L4!yp_MofXc(Xf>Nu9x;C>Cf=Jq0n101e3V%Tt=?rW?!p(peQ{D< zhG-)9)EV5ZeKr~OOnUXYP!{=)Q~%c*sHWT@aY(!|yM>n3DG2-}Z5SgfahGm_RhzB( zJ$^u&#oTi@J!>y?)6WWZq)yGv^+P1lEc2A6eXBV!byjeKP10PMy}!G3tB3BCO?&t~WcV$o}qUx!99a>o!F>+djqN$MI@}h_nV3 zz4;KjM7yJ|?sO>=zyc^$gmg&&GRLRIVi7Su&46rhT4U5Qi9zbAv*wg$GvFHI`CITG z4b=-5Ejo$eyvbzQE0++jRH=alxb=Nr?Kda|ydr>dhO2%IAniaO{ztM!2|0720gulm z?c<7OHquliC;Iw6?KOP2o7-8KBS;!cL9+L*9srJck>H4a{CF-L;43I=ys-JhxA@tj=8LYBe47bNdV+YB`IOVw zuMd&HOisqV-CrEd_Q^+qvIip`Z;4b!$7|a=HSN-=Q_CFtb~odq9_y&p>b-7mZf|~e zP)T?af_rjAKLxkctaa-iSrg`MVYAMH>5GP*`fckEZVXzcc6i>LLqqK^VL`MD-CWM& zW}d8l9d2U44cn`Pdzc9Jyt#5bK6EOiFAJ1-u-pVCoRN>1D$D#1p|L2x-H~0?yy;fv zsEAm{XkkjDdW`Xd*EcfW^Ari)q46KFopLR9T^xnm3uHA$J?yBhowqv)ZyMdT_MW*M z{ApAqYFRXX6WNxg&Q%6w|9-l%q2%4Vrfe9I(G2(~x_k%tF!Kqb^B#6}ANsE<>G?wf z`ZOTp)~#D(h2$kU7GL`5YlVjyGIr8YqC7(pXw%b-xOPD;Q1SpmDq?g8Gk=u>z4U*f z<%&c4WXprx=0l^-V)a?*?fMH-xpg<({;ID%daYLfhTndxTKhOQb^>(69fpJh+7q{= zx6)qtC+Zk&Vb{-c<}9Ct6#ts6KY`n6h#Jl21(f^j-?AmLPVyGyNQpXI?u3WT6eW_> zzlbft==vnki{rxDvSu`Tr*H;zlv;^d4h!2mRNw=a4g zWS?9*gETV*!0lAQ`FFw)_933}!u;GyNzqRTVleuS;_1_;yBZf=5!GZg0@rU|z|Jq&T zth!G<{dRQr*u7ofMYS9w|J2jk=i`&p(VSs*4ykQYb`81G&Z0=OSaSS6P|`3?&Kq}j zH9*Sd<%n%fCxbMEp1?c*vVZ^j244*?dkZu9V-7aLwePoz_I_APue-ynQvVcF*~IG$ z8pb(gOP3!M5x=paCkY*2fzpG?Jn+){PZkypo3xgTQWCqJyTt`oM#seDgyjRUy7%eh z*K<+S4wT7VSU$P(cra~S}Gm9OAh0iQu(^3gq#N)^e<&lZ9mr(%Uf8r)x-&?bXSqWhcaa?I0K_RC2a{JAEAe>&bqnb!4Sw;(+16{Tzitxz zs1Q&@fAKMN1Bc;-l-**I@4aNEZs&mmFGmKSQsNKBva4@}mFn_l!STOwB~gowyyLRwfSm{;iGm%9TeXOWU~>>h zpGXzmew&P1R)mUON@~CRN1P$nJ_%eWih;z5XJG?01g!&#OTq>y&tNfLr3790Ho@!( z4!7A4Kg=(nQtfcs&#4YS*nVs)(F7sD5C<{{P zYKoLGP5-@i-ErpuScsAqSwvs%i3x~vcXw~vrp?lV*8LYBfq_W0 z`TdIxQVb9Kx5;-3mj+F{PaVy*N~BkRHLARG!`2&NB)J%S$Iv~}puzTfzAGT*$Y(cg z*6d_{MqH0!!;E|P?%jR%rPp z)dhF#KbW8^a0fI(;Cj&K!ccvElomP@=e^<#+%SejR`1)F=#VV@8a}x@!f0;UP)0Mj zB%GG{b9ziV;{2`<6?C7aX}2Z5Xl*X{e(e4tYEe$Js9lBEj?VjN?-V|I3WX6DLa<s%jEkIXp4hGLm-s>!X1E=0#d(ew$hJE*=a-TO4<2lm)xi;&@#BjlUf$zr zBg4#%r~1y(W6yifUcw}uc4Ie#=)^p@dX#fo&DIJ3IdN?acwl+U_P4XA$`fnCA z6@lZ2kTLTE;#ROQB(cYP4x1}%%ae64cl7jJ#Vp3qx7Qj7^QJt5E{m`T1dR*azlT4k z2xu{BCh>u^;H^e^TzaYrG}THrO)53i(#wYLYi*tq+FJ#l&eBG~`*GKhmCWk^XGoxT zc1}xE^ZuGcuQ=toY{LzpjluRAtRNPble6;HaFg%C2MNowL1g!<4DwsgM#Ew9!oYYDu zCnqxQ(K8KbtAPqC4bvlH>ng0+9&c}Nr3H+GbO(D0kyi7o@6hU#&wnnyE@fDRfXB<> zIZpM6TIok&!fAp@KIm3KOUmh!p#}abgAhz;w(!afjCLWihChCSr+vjuXlZu-V(H~1 zDK7Fjqc3m-z?k~_^*J(P3oSnvhHD-ns^5`%rG0aMvUrlCr%t3(TPLJuuRi@W+i4)FVFOH z6&8(t4)qBj(ivzs*PCBt8ZeJ7Zf%)Z?VZ>4C)qM4A^h?;*RD71F?}2}32ie&s87%t zU(QQY`4oGvB}+DvnXpD|tf3_0Vp7*s)LP zK;>!6=8G8lB11*Up>-m;ul?vhRd~)8pELK0)GHe|ZtU*>A6wpJldG$dSGojm<7Gk= z44mt*4CA`>n?}72CP_ix*ijACrVIgS&@brf)z;P&!3-ecRmIDBJN%#R`|@RC|Ngtf zcmpOJB}o2Gdq8e@-5)7NJO}x*@T2?fR{HDwC-41PJ*>RcI>uEeGIPbFOXFGfDahp~ zQnnoFIPA;AvCgVSQIJCa6J=ou1t?#*Wd%Yd-o6(CO7zM?CYU36ZXK1xL~eDCw7{ROh5;WM#Q}IpJT%`88_L;MTnm;uN!zhN{6U zpEv|+AhG?MqaxLCAWH{;>qp(vgUl@(H)c{t7klp^d8}FZ zC5`s(P@6DD!UKlQ3z%|pj{7aSJazwdZ=a3Js4Cg0+?=2mpI9Qj*|n~{q>iJKw)K2W2d?t!^9Igdf#mf8kbSM4KqfL8s$x`3*3(63BBTk`_R}# zRGUu`CU+xfyIxGW)g%^UHL9wrcsfX?ofy4)+~H|7#D&tHVDVmk0^5D_#*NMiu3zrx z@HdMrUR6I+`*OK_IP#`Vn<`ZTn+i!^oR$0#nrV!a@gZ|0(MbIU4H9@h1K|r9LE^3K zrmZ~%Z*;>EumNVB=B7h4^J!*^ON|p*FE^M_4pbtgwTZrSXG=)Uc=pkw|CCsT1Fee3 zufX~g^$ZoI?{!7y?7E! z?d;R}ScyHjO_HWMF~sv>W@>7|%=f^(bq|p_&m^NiLt-bg(5{2&;8^!peL@uRu-K?O zK=6Hg_e#g5=;TkS$th0J>ii;{fjE}27@NH~Gt4o8ZkgQ2uXvo^I; zUP?}e46afN9MI;6MgGbvmim6F8#+tElfIn?O@n+K>kqCYH&OKFl-hzW|sRN`3ObqaI>HpIHd~ASEE6pO26l_b#t6 z5rjr&$7?6|O=}McElMUWM8kxn$8kZKxTM|C$50qV6^2qVO>L=$&57dJUJr#ve`7xZ zTIV8D79`(-B+~{V-~ptN3B>;r7C(9N#FA{$C6ny{COX596Mw4W=>R&M6Iy-BHUBdF z&99Tce%1j;6Tkk|tdS6xN5mIVjKIH63Vugx+>iF6k4j;=<>nH1q_7SSTp0B~V%vYr zCm09zC{@t)FlWc=H}83H`7U-hONw-CGMc;uSsdCrW9zA(8OR|+-Bfn;L28$NT&JGk zp?`*Kk4gIgg9B~l}@337XPWSJ6)>-S;6t#wFD#s)H&m%jC-=nJoXb-I)tjOqtBk1%6MVY zcMqPlmXmgYlLjmrF#hMwN1$0uD^)zSoONjkg)-iR6s?BZU7WvQ zLGF+hfN@G@k_m;iu6T0l$b2hRfYs^ni}09^HfG0q#o)JHa**IAQ*kbKz|2wF4*~PQ3`;Br%CTe?29g}JbiDcIAz=8}tM%Z4v z_KQ@3QgUVey3Mo6C`OH?6pYyYgv3BEGafv6aE4%$B#oP-_gJP=Ee!~hqy>89 zRjcm8Lj>9tT5>p5>dFm~p~9fQ&e7i~g7$HAlss1SKUCbJ&<91)bN)%8$wFL6gjUlpPM>=MtYI-c-$Z&q&OGzsx zW?Wc13K}_LyeDLUzH*9_5PaU$D<8YYS^T0rt zBc?A&}680sE()0HZzmRpS|q@OUPu=6oa-V$GU1J%Pg5 zbh72saa>bb^nG0?|AHO8xiXh6Xqin|f;sKKKpL*c>ZIr7_)xk0WMIb2rcl-EXRX2v zctp9b<-{%q5J}wt=CzXwlkOsQ`6Lieb2*HqUbOrum3mz3u`w%Y3pDPEg)AQO_-^*og*V~i&Puc&Y)!m)jT5S8D0ln+juAN`9!E@JN z`dttFUN5z9*7m^n`H$_Lip&l2KgH_bcFVOt#cIK)s(K^tacp~%)7uU0POy=iHoZeN zVByZ8@9)3H02uk3{;W~QLmcrIG8ci5b2HY4m&|2&!$p1C!1dnWU!YN)MtclQE?w|< z{qkv-b*F&lS~ycb&iUukV(W9sk2whlgr%$r-?h-8>#u6OM~}!h*0;Y4O5ZHvO_~N_ z5u{qiMTrLTd%%M&ooqrcBj)NhG?OPRvCI#+Eut18 z_CuueICTJZbm<~PE}aI%d>7!`0OMUSoXdfWdvJHB@b;3G$@5v^u;jv4UVt&%27Tee}tWL$oUFW1?TW*P-P% zFNYQSFbK;2s)@CE0QqOpIT=f-TRyHTgWN?NO1OLI>P|g#%3}=4NfDG6xp7hY71=ZV zQ}fvJ1kbX)J^Vtnqd)p3a|kd250ysk{27{fN0j_zx<_wn+h{Lco?KEKyEbf(G?j~z z1AfI-=?YwxZd}n3NuFapo^q!{h_PJL^DHmT$Vby7?CnPpSzxZiL8WMX7+>)mFG|ST zD4h556}@XwztRmV-#rSwS&@PvgiCo*b~%aOk^}N&K%h904Wq12#<;{dm1Q`I)QPAR zY39`!eE7R@UQnFF@(UmJ(eMyGPo^1bgOqL&8V`H>L8PZi@@z_e`svNu{Loy7KB_#1 zH}p$On_aZvvb5TB+VVsQ8S+`BMpj5>jIwO>=$LSd*JrMQNZBtpp1ul#9+E$|xTx1*=fSU_4GdX`J+c)rQ? zNuY>Vrv3z>xS>1bs|L@GpCFI6DF?Wp#{T!2H$d^aK09r6|GjlJ&Nf}}OUol)vtv3U zbGJtcASpNN^WjNz5yYas$OA-^fvCRdgkLThu(q?P0%vfP zykr0_EFH`tS`yxyv>Ue8?TM0(;3&sJZ3&hdRGG0NBaLo*8W?azV;FTac|md;y10Fb{(BAc<5AE+dEU9GoD~~C z#o>D<`0~aXV!1?^3Mh~!ZTSY}#c`UMBUX$NO6y@#JVrD|MCS+xc@s|C;NnxPUq?dZ zXzH@_amg!kA9b~a!xphU(Lw&jbQHsu_NB4V(b&2CEzUArim-Q9* z2Z&9PzY)pqW%#IA(c#!}?c}8-8A3rG(j2KvQHoSA#GuI4TOOR{C0*nLtlB!y{rBoT zM^zs0Fr$IZSI+ZwOCM)k4L7OGHW9uI{iUU81G%WM1l>Z6DY4{G?x(O&blD$p-K=2} zX3QQG8Y&XBE#|EJnUHo5inXgQH(0dJX}CZMwGv6DNrCU-G7KQS>;nIN0>F>)EGsyr zj5eox-SYQ_tv~K6|FCNu>Te|2qy0DpwAA}|UUAe7p`6pR+F!qlL7#Hp5&m^a|Gm$f zcz*Z|=JwKukj8co-vUuZF2FH!)qtFTnoPWn!25ot6lIH^&2}woGzrs0l|uwPvEXJG zN|w{ArcUML`tF`}`5p})RwX%?SLBt9fek$y>Vb-`y3qOsAf>lmEJ#oWI+z1dgE+L? zHft{(6bVW@2sq8a5Bcgw>aO>L-8x9RXT^b4BUZKUz1(32#?y+Pr4@$ZK%m=VwS8pp z#iNiTtHC3#T+fj?kAmB;g=4gR`**#8mIhp}1VS{QH)~m1F?(T?OK?EpG25FsqI6}> ze@%QSW2{gUS$62713#sMe3s=hbiK@_?SI*;wfG)atntVkQXFNv4(YZ$d4uhz2HHHd zICT=VGyCz1`HmsCO4scwe&^9v&w#FhO`^&tl1fK&c;Q$M_`qVnQm(=U$0C~7nR@b-B- zyE(u^D@sIHOn#dCB$DfAh`t`q%IEB=B_d5n9Y)Hz1Rs9%bDsRm)rW2D*E)8}jX9sw&$sJx)S_Ak&b5l}^eE)?8u*UZz4{Jg^)B1q zEf#+&2&Q%5itJr%|Labuqx%*=!I5g9~M zF<5)=Kr(FW54AWvk;{Zt%o>?PE!0QdUV}ZEMRd1l9k7F+Pbr-lSA7h0FN!&4s9N+I zr&%QJLUX+4vgTdvIk82!Mvew{r- zg7q4N??Pit#M)^tk>jXidGBYFpdldY9zyH(hG9p^l)v@x}`f?TL{YX`)L1Fu~#SmhruI-d{QS6JF`^ zl12mOU3b|ba^?Q&!g*g49WucBU){p|HxB4aE&}9j>KAyt&6@l32o+<^G!j#EYITmT zw<|}f5woYre*Dgy7}o`@!&>iLgS;=&QA;!4<$s)yPdP2>Yh?)Pf@ zNM*TnY50)HOOaQO@$@&A-CKja{&MSZ=#pD_kI6pEt_Ot%;#2p4_Lf;2(#tBKtyOsEh}g4q3T%2C%S(K_NMX8%42#MAH&hya+oL*-ao_ga zMb676Vd-R;n-M*2vN0@gPr<|plh4Yz!oh2HX<);8>%toqVOZ8)u>6pcG3-E{#`=9> zWTFH*_6phHlgy!DW{<<6k5^C}Tyom2R|2gN*WTm9bZt{0E9!3#deZKP^FbZzx9+We z`asmVI8C3LdkX#6lY&K!Ej@+$5svo3;>k~ZUp=0*xz2Zkjwq9#s@wk=!`{d=1?7Nt z&H+PjH2XKyY1OLndTDzNFO3by&Y#|0_famX-u#?AW4ariu7*Z($RGCpl=RR1ocZ4k zEx6Ol$p4cg9Lj^`x#x?|dlbLZ%6~a@h@GQMCt%c+(Q-m3xluqm$y)ewie(1Gjcd_j z3CIdeZ;(c4GHp5PS!DYINBY+C@1aLhOZqN(gT4rZIZ;-dh*r_XdyBX;ws>!Af=8uF z+<+`qC*SIr$TUN_x0YoD+ohQt8QsP6wy62Z^e!Nm)4cM?Veijhdta|tya`$Wcddtw zra4R0@$lJ$VG$OD6`bs|?BBMpjfcJ4If3(-&pRT{ zgk;vd=OsJ+Pd0cc-+bcOvH9Em@_$}%0R^ji?bff91Vky*)ly!BtsRMxH&8Ft|9Z_x zF`DnmF!V%du1@3{fUtbLa)6V`qXlznDX#irMZE$7v}M{DsN;YP=7Maw#n=riI*!sA z6(jN|mx9Z=y(!E>MBH#}0R73zY=b@T8L`C)a6m1?&n>u_^V~6VWo2?44o=LEtg~nt zp6uPg`S5LOK@XFI?xLbvFXSiRdN~S3vyD;<&>u7WPqXSX{n~w__V7ksd-S+PymJVt z0&`{owA5WX6HqYi{!q` zl~GU{Wg~BsPu=N1>eFg*Ut)31^kwgyM_^64b8-(wr}A#LKKI(_^76Sc&C6>+kVqqg#UDPT}0Li~U2oqbf4Wg5m`thO)vGPUJ+klY3pOen2TQR`G3 z9aL;}g@{qGNZmjQ5L=25&mOU3da?$AOpsCu`4CA>G!pP+DH@ut$%L|g1Y$&DkOGRh zziVKS`m6u9`3Gk_@Xq_Z&vSoV_kCX%6RVI*y_UZ|u|tWjqyep$TLF-AKJ<*)E-0#_bnke zE{7ynfh-IseRI5gieJk1G$wjXY*~AxmW7yV6$v@vW^w}K=AxBNF_pDfMoN}&(c(^G z=ok_nx1h2!H&E)RX$28}M<`#1|C>k$?|1ONuyi!u=T^qP21MFIMROlBFJbbrC&>-?mT3}37nl& zAeQr@=eEYz`<#};EWS`pbnI^}2re%C%f;G_O)RJ%2g_;_ksAGX(Zn^ca))-tQIefk z8B|jp7jS82{)w+odtSqhn&LPe(ce72Uuq@o*PD{ESO#&+$3dod3|2^&CDS$PK*F9< z;JQ*-Db>FZ*ol z>W;=7*OH1pCxk0^dP3f%J@bfI1)BH;c^XB7)Yk;GmMXj!>sWF1$ZWjJ0kl;nUb^uF zL$Zh5b26*|8%i|IR=~76LY!Cla}%5fpGgpXBT{z5%y!jYZWxQdG;^;qL9m*wm)^v< zxfS$uPK4GE!wGz=KPiC^b2M3NJs6iqqYn^UGtXoqoHmE)F4iH)X6W%p3RH}ZO8sOg ziBqN0>iCv*r%#;=r>$u$14&zRgWC_BBKjuWJ$wEf))}-_z_+y!4V?vW#l1> z(&~CTp|>n>l|bI%F+Z#_Rlr=khI)Js#cby>uZP0Nvi{Bohf0dsr*S<<6#<)=SS;u%(A33H#b|b%ObW7Ip^oJv}P9ZmxvvSvV_5y#fB~hNx5HPgn>S# zH_wOe+b&b9vMf2qIKEuX(PE=$${1))qu~%VnHk2djj@_hAI1?Xaw_V{PgrCWFt}PMdkXoD4P~ zox{6uTN?IQEW*y-i!6!Pe+rq;c-;5WP2_GB{0YnRA9Hq@@Hjtwg!i~47~V&$+A?jz z(PCO^lV8DWT-h7xZ#|N5F-CSJ@t2h$l6VZFm6cmbF$raw_-n_9@^Kx&1c#ZIwNsS~ z6xz^ol6&=S-7~@N9)ahrVE{nE-%XPFZT;oq%#XaQl<29Rv+i<2{dI059aleElBkJf z*Yl{*#I;mMKXryIC zML=>`#8104t`zWG?$<+&_%)gVP)$j|43u7DIVeJ+R3vJ`%C$=5i z@Dq769@;x;ztqF2X1JLZdBy|jxkvdvn|*BHPqdNRPcbCt z`jWYa7-l+f?Mv6r_xK}pILvY1K~Ch(@bF?fgWjiFaXhf>>UYwPW8!1@tlTw|K{{jL z6?HQ!#z6{r!@dD~3pWBA+&{*~_at$7L=fqU{63+Rf+>;vkN;(#2r1Gxyd;dPpf{w%`zWeO*LAQ=FB0k3S*5m3CKwRhTCFm~fa-YUTjNXhp!WOVM zm*4C1Qg2^&&$G^k&9f9P?Rto1zX73+#;2rxo-uM%gsn)qFqyzX0&Gh|v|%h6q`oRd zSVU;d`7xQ&+g<|!tK>H}yld7qY<^j)Kj z*N$na2+QVS#=5kzh*tv-GG}Y)8O10soYXa3RlaBwZhAL#=+d7zRbW|KlD*=G+-Q0F z#>{gEjm>xJmsJ;_+&t8*D3HMC6FP0&3i~=9ZDbxsF0OLIi7A#LhHDtuu^xazV@!^lS*=&vfC@CRsVheGY$@AiPz1efm6`CD zEAEP6GIm%{*?~8Ji$p@+Liz0pP_B_t$wtIx+_CauO*sqlmOg@&AnV==71R1UkJ_|v zK+my>U)=rFAAWoAn+sG`mC>Tif))=vH|^Mo3@VrPtfntda@IKyrzMeGsgkM1|Bx(Y z#GVLJNz-Z~gRlp#bnAV>O}G`pDAM&hv(dME)Xsdut6>iVyDe2^olnw0GJMC8OdoH6 z=|lmCeO<;qK?{=S+*G3^i=*eMO5jU7Bi0gki9>sMqtB0@JTx<9IR)8clAjJ+iypYW#Z>x{HWRwH~D; z?0&Ugx>(%1Ru+zti`^&9aNP_ZubQ@KeOC?g_LO?sU`G2m1lwN!hVFu9yh2)$-_q5e z@FykMp#`8S=_P3pk67B8x?d#&99lYQ$dh;ph^K3FJLrLAO2x6S!Y{vEufirB->MtW zNWZJ-(Dx$rCrXfjIXc?UYY(dMO(mbYr$=m@;}S;6NXIokl`=S+VKSuhPgS=PvdyPE z#%=Rmp;aQ9ZNv&^d-fC_m;P-TU9`Mj*eJ3pVIQCOSPIU_Jp5wMMY{h{et7clWF8&4#Boe>j@u2StCK{sVtaSwH7;K^s;I^SGH8#WX7^-&-It6= z6(aJxSLpnI%xUP;wqUGM7ErNN`vyDnNCALuwy0+uucbB52NAc<%3bOB#PQQs(eY70 z?g+o{%?-W!IV|QV;}g<8u7+w?W9iE58j$l`_3~l R8}*-NU;pO|pAG%@{{Wg+^oIZd literal 149377 zcmbTecU;fw|3Ch;j&sa|j8Ji;5(@2|nHMb$?LjJrx}7;?94 zxw1T@6lw+Uq3jsCY|ri$vUCZkh0n%sz#4ZjW^eY2M8X~Te#5V z<&G7H4)5O^zDnxQdFoKGWJK}AR`U*)+Eq)cmCtJ#$Jb8?O-v3ujppq5R^PV6yP(Aa zW0~`%>UTtiAz;o2_@nN@kk^0w@=@FhjtldD`6Oz=G5=papgLVwv;Xo*j&Sh*%Vi|> zOXvN(jw^TT)JS8=L~nM`)urnVKL2y|aNfML@|>KUA3|mPo}VjY>#j?YE4aV=mit1= zWZ^x*pT8T-te$M5#(eS9C28M%+sne_N^Udq>@)wE@mO!3$)4mov-GaC&X^k0l!kq0 z-g++EuxHDijeMNK>!`R(vlCUib&_vTnEf;LR%Q_7B_hm>o>qg<2mx zH>xla3tLF>Z<)IY^7m{TodopW+|G}6nd;o7k!<3fmt1QUxoi{vu2C(!6P;BVLJD>OaYl$g)v$8c{obv9@2 zoa#D7#>5nF_tB(V77MZ8TF)J$W&R3f*1>ACZ?o3Oi z7gM}jp6qm!{keG5rdu+9>k#P#e8f4Lr4 zkuMMXg5Q>f?>>M3*i*6uzWk;SF|q}FEt=nok~tZCf6$Ay#X<QGFxj5IY^P#uv9shV8tndWSFe%=zX5zXJ0Z@gzW(7K(LX4OvrV+~u# zaUZTl>`XR3TJxiQ)%>h3w~>sa~W3@w&_yE{VGGXY-+Zz6dROJZm_9i&BuB4$g-{2m-n1PZIZDMs~vvI zqHu2>gZ~mLT<(`C-#u34GF3G>R^lStKjS85T9>?Aa4psOVTDMSr$bLFyPsLwIeN3p zbeoGs?pl@E3vmB>fzhCuu^>&X9F5*urnTm#?d3PLM``L$#V{5_BwGZ zD=RMwn@_og&YcEptNIIh`pB#_daujOgiuHL6Sw6HCAX5GL12i{`u|zV-8?x^!Y#Op zl7Iex!ee7Z!%)sG2m3MahCW(0G$48 zTJf{YQpG)&@Xe! zCpu8bla%K)KDcAg9~g;5mQM9cMv9;7mqptc=rmkY+cdi|7P?-0EakzRl!u>)yKv%{ zncCwp+{H>mibjf4IKPqFcsaoKW9slIb`=*c|G%?@nu&xbB zwM_sj>mDqmR2zRC>SU&4S(rx*P7A4O!BM|$KLUXK;r|zknNZ`y z-F^zs?%|*PSAJW|WEddQ{k+Ub#~ia!3NZU=&D;nf*ZhA(fJ*C_$Z!Am zL0Q?mh`(-^R=Q1*O7?gKsYHx>bR~~H{UGdUO;K^v_ujQwq<;PL`zh}<`Z#pf!c)JQ`0 zM_xZy9$spw?a=5n(lXIFS5rWR2A$NmSXNKT!f9&2m!JhVC0HbD^83q)k>N7W1sg2<(I!=5*8~6u%>zP)gdG?f1 z)l6@!T;0`y@y^(i8p9whhv|W!8ByE%3sMC35v;LW{rS<}kD2&*EGQKhS5Q;)WhEG@ zShzDvu4t5drk`6a{`6t`nZgHqQ@auhK4iPhJU43)2J(JKHqYksIrB)-l@)FeDRIoK z(s6O>?i3sMciRX?)Hd4W5i7f6fb7*;97`V`4iZYFd9l?v z^k!zCHKC~R5zbpHzK=3a9JRM4I_2XrdKBs$kh3h^5jT*nOE&G~b?9%--{Dh0qa&|} z;2P(uVziQL4E)lrf0zFwj{y&i%`%57E1VednA|@fbLhihd)Ywt%tZFs*C!r? zb0{e*>(?acr+&J;^aFC0a8Y+-_Do)-d1}jj#{s{_sN>CD*)!7!s1{CR0?fI9e1(g| zP!#(C*h5f)&ec9Y_fWt*kV`AA*(ZBaKcE#zk~JT(t30ttfc7C)CmTRubg5X1VM$=p z_F^D2QLOJ+hf22k#3-&uFAflCLx9rg^;Tsv1kYNu`WhOHe*63^YVfWNMe&mT`1OcHaG0vl}SSru;ELbV(StchjrPnDjmM_<#^u+|bmOVyA zMJ2Ycuq1(B+;b9<+k)Uw*N--D%(4!g=+v1J&YtQvFN;u)Bq5DNEfaY%x}RmHkA>|2 zfVKr-`LB?ePYU(cCUvfNHoLxp>7hHA5Ng0wiF38N~ha&e?WTW&bYD?A#vOGqn)Am0Er>&`=O0R_7mMH63N3! zu}-@^MEj3A4?bbh?=3H1n|1F)#$Fi1){E7IaMN8L(XmW)qaYpym zzr3=z4ZJ)OxWTlJU*;h%&4){?W3@nlh;1=~{A3toP%H3|4!b40T%e{y#>KubQL6EO z?K~M3K2V?CdwIY^J;iM5S-Q9L2~Z&OEG%PHU!zTo|jVXEfr#|U&$KsvLSjQ)4GHw#%rx6G$p zAOBE9a*lj?UM5>WB0_B_Gk4KwGRu7D5`lJh)h-9cvttmR4IFhCm{jm z0nHTf*Wn*2wBs3WM|NC30^p%LlZXN6UQ}7e?v(m|J!7z-=(0jq5`C6mA6>7sTakON zF1ee%Cp<0GrbgdqvqM+BuYkGsN4vHF!Xe3@3Hb|4EfF1%&g%D$B?8A=Jh~sUpD;;V z?!p}$B98MZNv7w* zCkwLAG{3!TJk+}rv+xtJN{S`owuXo?wS+UY?poWdfdFhD59lPzSj0s&Op}y}(gOS$ zQT$YRl=aGQ%ijg>zfw6~kLn`E$H!;a8J*G(0NoOw#~hg%L{zM()Q_1Cu@=)Ek+CsV z69Pu%99W<8*OqU(AM4a@Y&TdctimU0 zy1>VaPl_$2K-eK3aXmS5XLic~%{)b#w+>v|c^t$%fjmIL?iA@~N)cL(27aa;ytXax zyrBWq+Si!=PEj3C#GS@NtUD`{+K|nxjEs!3z72rozK7O=e=;Wf3nE2rh-RV0sUPGt zJXSA+GEfABko~R}A=ZD@k^+%F^Mi_qKhZC^-*R=m_FJN+U^x0YuJf!PreKhvp`mQp za4}O!5LsfMK7W6HIT4VqEk8~tn~msLo6Ixr0)}PDTF#7DWrxZIiPPwamVICr$5YH2 z`sHQ@<=U%abpk9GP&wlt!cRQmbN;x}`P&suq$C<0*@hjlu1jQx5~rTaiC9y9rEoOdzA&4VYNCU-LRG+%u z0yQskpo>I*qCwHSzyDa%Pm(!6r9FUA);fh(*PdX~+v`&;hmoRr0sLeWwK1aFCcw5< zAnZcEDoro^xss0!F`z=2dw1W5r^h9RM`Z*Wl+yKDG2+a3dSQX~Mnqe5nC_jK?$!Ak z2!^G7f0vqqNK;v%p8)qt(Qi3z#eQNA!<=pD-N|)f+7lkJDQRiK$W*E=eW=Hg!$W}< z?{)+^n4`Y0|4{~Xmt!>x{e^e{RAgmjWV*+(JW9Hpx-us5x$6k3OoUMK^x*XsNu2<^ zyZ5ZcII~B~J->I2( zk`kRm^Rb``YF6I^{cN}m!~-x>ks8QdiTH1rMv7T`O`>6GylX`{iv3_`RqQZ9M|SN` zeL6J1_>0(@xr+@~4}LA(h6$4PU>3s+`|ui-hsWHV2cB_fv(P>jZ99E>GuF45+qv%+ zNim>Ug)c5N86AHJne1p=ae&0|Brvfk(o<1;SWqCuMG{=Igb)z3T-uy~K0 z*pO08T1Uj>^hmZG-3me~vg1J8bkqa@%61H*kkQcp?xTa(4UnW|ut$kVOaz0}Z~rVE zfathX<}5(SeQupV?M!=ep~HEX)Pd3$Gg-;^fX&NghN9%y zAya=qRg=qXMRCZdu?O7Co*s$~awxg7LYM=y+rpV%JM(1i&Xdh4*;A%mXr}sceM!VSuT- z)Sq*p+~-4<^dTT%uuQJxq=xBGswIKGB0Vp*HRXAfls{AC!OHAXOJI#wPkJ|8XC4&m zJQO(wEHZJ`?Y@fuc62@ThcWOE|7<*vU2o<{g5y~KbCa5VbtZ{2gyRxzyvAi_O2v5b z1hPR3QKvzw?vkWLOUPZPamGVz_$`?+=c#e0&uVd8b~BJY=szDc3z5=_Wz^D-hRo~B zV<@}-rt6(tq**#6k3sqLLx?nikZ?iVL%yCxmE-tK#j*W6Usn31-<=g*tvch_OhR8# zYr<@MKenUp+0>p3gFUh75%=A5j*BbPe&`r3G|$H!@Alxkzj@`;0IjC>LEUUb{&`I0}RP` z?yS+%WC}nD(=c^$r2w*rNw6bBtYcc_`Ws}#wQ5<8cJwpwVRl1vi4*@pXXMS7Z5E76 zew~U%3~`y6t^wsOL+!cjHLsinRl(8*7-ci)Dvb`%c?MtB|lwPlR-&e&}61&dcm1DCqFFL=KNIu1X9cmw`7*RUIaIbqhF zD&L7x)t*yMi@U(wpU#3?NLQ}oW_XySJQo+1c@+POwTBmfPYSQT;r3yuyRtmceIIP| z5LLLeCYT~lP8GH-KoXbJ#BdQwQu$u&>zS5P9Zw0CqLJ9nKrLGoHZOSV#m`sQ8+pu3 zi_fP_Hvgz7s@s4lOPce&EZ;C1h9JrawItDrD2w}z2y@zhustbH%zlt#Rti)8q2st-GXyuNfkKdQCZDcc83qLuA-(?I z5?8(o$OiT$7g<(=5&*~ zQD6u|H7eN&-!D=n*0KG#4kwDAam0nDH?y<~(2SLAcZ->+k)5pza>7R6sbncQOwN+q z|D@cG^p1`W;sw$Ox3%i7O@c_9Gp_WD>iYhhW2Z*rJ2pQPI;jZngZI|}MiY0Diu(w~ zg3AsH4GAk0`Vya7>f5vF!7vGZ)Ej9!p~!^*LJH^qqIAVJ{&YSnZ>;p2_swqIE>%yR zsSXuJJ|m@n^)@9dR!Y^F=r(OZF5=KXeFZO^b{G6|gXpJncbG(yhcANpLb#>e{uu)xwMUX&@Jz}x-=6g}{K~8|K zsqhuOSV_8_w|KKW8LE@wcNw($DE6iVNT4+#s#C*tv0nvYtCGr6+eB;zvXPJ6-f$N% zBsxclinGC<+e7RxdNUx|+d^NSrLqepod#hBZbd}c(3|0A7NZht|Idwb08+GS;8>Qy zBP10*6=`eW+pCqpQXh8+Z=^Q8IL?*ITBT_zbHshKuMB?;D=h&E`uyEh& zH3O)x+i7GYEMpdZfo&>z_n7{S{eQJnp1UYeOoRYgFd++M^|efksgA2GQ(vW0zqsub zPj-eZs}B`-=ayB}WDya$u-w~OoCZ(8bm4)2-vwU^W{Iv?I@0Gr3G87S9lD4pk72R& z#BldgS@=_De}EfFF;b?9mkmN#jAW|Ff3f2bm-*GuvCdr zx8#RWCm-YwzK}Zl<(7(CB(N}I!wX0WNtWseBLXp!fHXF}JHXQ8CQ33QGLi>f4YS!U z0h?7HEOde#5_;L7bZVcQ;m0#JzQi05*(KkyyFkjh$_yqNjomc^UTO9H0Q zS+V~v0>EE``1pccYBy0`FzkW1+dNol#BLxv6s+^$?4`~;j~)2oaq%0)W>4Fbx|o3^ z1(2Y-AfZHjqB+#hEMm7Qky%JaCj^jK(AU5&cmQe*O)n(^8vqB=v1z~U{Q-vnoVy9^ z^fe6h1@&%OEHSztlYIiUfd5Y-G_XlO@VviUgN?=}>;Hf_z##J6$a&U%cy8i35+9I0 ztE<2*3jz~Z>%R=H;E$?bNhg)gUoigb1}eu$75aZndQTcF4FTee8g&MQq5#uKx+l&H zQ~V1FW+(T8?8ZhjD0=8kY;{D`F5EOS5=uArW+8&c2jrNImS)ra5GjBc%xM?EqrA=! z2v%N-WwQj)3k3@l`sBKxKWD*82Btr#(aYF)WU8C7zC}mUfTALs}q<(#TtDkrWDC-4;PL#=tM1NJK zOmwHS6zhnPRGWlu1F`Sz9kmDkq=pLU+mBoHq_%~UPKSN^$LJxCAO9sWGge}Wuzd=k zwupTpJkwun^5CuNNCzZdXK984r#ERi@<;t3&qEqQ_>>hDz1)^GdaTSz8-Q-l&mQDx zK}?(4BzRDJiTt<%Ju{@Yrfi-kD=AhLDmnqA4+g~~QVUWv2vzIDE}-^Bp$n*uo~6@I zSN)4Hb(cdB{)=4jA-YPy?}ZV(CRp$A9C)Vwq$16#W<+Zfg8yh-MM40zb{CR~9vGHR z3&MEG6{$?=@(sxwA>2n1nKN_|Eh5smRiS~?8!>D$}k zS(FkIeAbm25M;8MT_PXO*7JAv;4J&l*+Y~%7&fyVJjZ^I`|JQPyAL~+Kp7$xA!V2N9!DP5Jh)he4E~QMi{Mck9 z#Srxxd;D-773@yX9g!P|Bn1Pv4A9rwxn#YL<6Z)q%(BK>Q7>!*8dth1L&@8X%ykC& zl!a_#4%FE9-+pW(h&nQ4iN=RIeR`dG9fU^eEz5vL0Se=;5hsUa;|Q_X@bKBjF&K>y zvv@HM89X2H9f@&41VALHhs08Xq;{?$&6>aFj2E#!0W)aiC(&_6q-atXx|8tp_{a2Q zsx9O-QjjqH(t)BvWFUY;FS1;gQv4TDvW^nh6rC2gH|+fj7u_kmnulj`q_<9erxq!8k0kiEZAz)Tak zPIrMkb{R;q22Mm!{mCbQlcU5)vWJnvmvusFtN0?!{OduO;p^xu4vbL0uw+E^^iP*xhT zdjauV?`xO=0Rk5Az{@qttZxG@wFl8;1jwaam*4o_>*IQ-`1Q>-1e+)+jegX?6~!Bg z_kxNOl{iMC;lW-#qE$pEI^dh~H-6dR-x!ey5-Lt=v_Krx?o32FZL7B^*iG6_s>Wfu zQcDDCtKR)=Udu65g;0wJ&#G*LZ`D`?t>+P$__%qIyME~P6cuC!!5s$U32xUR)9J}E z!o%L&+2n&pw`Q_>#)-%{0g*9joWzK&s+GpAI@o6tzLR)LUJmHkkZ4E56VsXevi=<8 z0Ma5#LbaeeUXKTDNYZ}%yIU;mq8s%_8f;5|IQ^lj#17TqYUL7hGTBf$>9KsK7-j(c zPMS2QC$eWmAkf+Z$n$RiEH5OJMv4;%A4~t6P3DeYtclHqOv__j86^r3ZEO__7ix{H z;8qw#C&(V~sm6_Xe{_vXUICDWvF+wQkBTeC_{k0=@`|?eSc_w~u}%SWAb(P0NWK(i zmb@}ZJ16G*6-x4G5#JJBhLaXKdfLBe&n?Tv~uj66HDe#LYy&e!@4&dZ)d? zWCj0EZ*5jMB#pLvin95meNAO}V1QRD%92P~uw(qLGuv~}K0t^Tof<4V_c6l`;B2Ei zUc&x5rLK@ldz1rJjC{$etiSTp>gzewq+KWUOPa=$+o7HgV&bj-x2uu zUcJjeiqA1b8$bp2!1`>t!2xIe_(InS|!K-Z_AbNUQ+>S}}ZR zZi7mXTwHDp!cG)tk}B&_tHHab`)^j6BGnV{vk$$S%b0|DAy$sp*G!EMO&mzS0A-06 zJWiH)x5yPy!^Rkj?v*`h2|uz+P#Xy2+f`DV?~*j-Jp?bw<)qb;Xc5f^c{+!XHc!ipy@q;)9Uoa8YA)H!BmiCLI0>6=T4R0rsvzF+$ZZ-8^WZ) zrb!8b_#DZ%iR@30AN$k3SEF(KB&jm2G(Zc>my6an!ydS+MVj^aEg&VRqm}d?VM7Rp zBDb|W#chaaPFgw0&49Xfi76o5FaT6AR%p?BSBlve?0gprV^o#Cd#GQl+s+s=gYlI;pcRr=1m;d}80s7mq>^EcDxDl}B%QsRyhs3G zr1{vXmU&FJL>~;gh$d1~AJT04`o=0oVXLzbdCw)t_5+2HCxt3&*R9@|@hn|TmVF(}wp)i3!M@A1A6#M&)U12w}fxW)N}pY^B8?> z>ZONMb-7Hyy^+o|3oVhv5yz2_MQkFCHK}comy;$ZC(F~Ujrq>L&-r`VynBZP%DgCGkFi*ZU3P)7dgc!u+#m>Y^}zIjg8z;O1DKmPBnLsnK+`??Ek zY-~m-h_&gqvDN`TK0Y!h!h(W=IF-WXJ=^DVyaAwj>goCC)oa&`01(qG+hiYZU}tAf zGHXzO`SN8)T}rG`MTGu=|L!9u{By<0oh0`@w;wVZ`dWIBJ_r`|3`!Qc5hVF1P-_nE z*s+7r%s5`PJb3>CT(MTY9Fu-rNJM14(m3T7ckdnpljz0w>Mv3ujaxNkWMq`5S{g3; z?Y9UlDyK?}<~&1keua|E@;kEXKS7B69bXIwA$KdFth3Ulr>E7^ZK8*=!oxX?_qRbF zc#a+~zCw0hJ$?L3{MK8+t_4S?mf-)bY9C@-uRN#>HRm?53c<+r=faL6J*T(kr5S`-bs#Q7HQEK6@x101XBa5T zQ92O&69jR~#VFthW29+%2$D&JMaw%o`Zm{pc8iN^*QJ=N96fr&d9qJR>&5x44KIYP zJHi2CmMmL#FFQM1Q%h^9pyKRwFM0ZFLC(#vX-?5`2{KTujrDwy7 zL!IM$t;aKanrmumGGM18C3b+LR(eQGheObN7=X4c$(yAiv^A+|X`EZPZml}(Z>RR> zk{rkU{Ctw!4jep~bUJs*6darOH_8!8w0ZO9Em^WeK~j>k{1;9px{~{Ux zaZ8Dd+rxM6!=d8hVgsmpl>8Ub(N9-MSITB4rnO8=y9ESNHs2x7k1=rW-Fu?QPe4Ug zRh8Lr$(HQg*RKhJgn)4Kz<~n{jEuf!ny~N~Hf#w0^hr0#wC?1J6)P6`@0h(`?i=2@ zS>vrO8YuRN5M^z3B&LwulJ4cT7LiHRE~%y$FJ8dPI3jG(d;{{1z;;&Fw-pt99*K)Q zt7L=3qlY@Hc4CUrVXk=k^y!Biq@<*h&XpZO0NcZ4uZTN!aLqnk9JTu5;~R5YGRZ^l z)zQ)E>;|cCdvQZCP2*4d=C`Z1Z{OaI`f86O83TVErtCPjZ$G?vC9@LZluYbwWDFJe!RK$>x>bfY9TI;? zv|3q?LJcnv$UFAzkte&ZI{t}&K*0C7KgRrju2<0cS8>M1rIJ0G={7Qe(I?KH4L=#J z{`A3v2fTYR&t&A$SN(wK_HF%l^Bm*o7;VF)J?cB9X`r4_tr{kN??!wPm zUcG)@Bi$4|i;CAyh1|SgYpa6Pu$F-#1RCBm{0`fh*(6N$Jo?}16vA`{NVsUFX7y=}AU<5SMOp`Hd26}@Qf zO!W^9m*w3AFV%}!cibZpj!`*c|v3|hMipb|OZy9EQmd?&dC>sSOC2}A!Cs981<5fT0_D;L8#?=YG0UpmrY9nmzG!*w=YU49S_Ew$Q zXJxps?$w@UiC(jRnf*U`Ud>dC!%QMJ%r1!t65RY`ov_ z$U7fybH(x+%!Hkt-3}okHRJ(hO-=ue{3hYR4NqJH=gANHi#ePG?pDS|Btp#V*ROf^ zy1Tm@m4)6RWjfu~oX@Pj`oCN%Den6VL`X+Rl@v1Q3lQ9`w{joDRyf9;@$GWWIgDw` zVmh79fP@3JSk;9jnWrHkXVSFLhS%KI7KSp<`PW|uu(jOW+!UavF5$2Y{1O{sE2CQxaL?Es?H%$SRz#5T;$Mz+)i|T&| zf^7Zu{Z=$J>r9OFR^d^t0iSlI)FHEjU?@_I^RFsmZtx(FkUxl$x1L*DTA;YDP6N;f zrBeyisrkzng&bexaeS?j|8B+MFI$X3%#{G;&uqAc`kLf4VIu}T)gX){U`LzbF17BD znHo4`QVM^Z3@tY|x8`+WpqK{mGVhLqIijadog%%rfJ|?AUDm@q{S*ws$bs64^kr20 z{MjFB|K8W@#LxSBt~qhy1S+rmnKL0~A2ZI`2uo&7pidT1_Ai<}h&U3=8H*zv?{=-b zg^6ovZ-0h;^$ZbYfJ3C)0V)Knn%*oweB=n{zI`9MtmGGNXJ_w3CyDTdFMt2@>Xr52 z*Arvo<3?y9HVFHzS!D6z#huP3fOFQ^Ap}3q)?GQ#zQ6y=0aziwd%H0qOL$t_C{N30 zZEX+@_KG}>nloFT-mhODGHpmx-XvgF4snxV-MUb$vM)+7!zO?SrCaNEH)H$2+YE<_ zU}0m!FS48B=>PKN%U}U_HmQlij-zq^{PU01P=|72*?ia2udqz3R;@Zlqn$ucHm}uJ z`Jt|wL=JyZBvse>uO~NC{XQuvDQHiR0Wo>k*rh>Ud10kQtxZSzKC$gR5 z*h>nTt1GpfSyUAX`tFr0S88@(L*%d)mdwi$y?WkR_GkXhnF6{b3Pw95%MT#@Wq11e z`xU?vl%v)8>@Hq>e`*0`@d>FA+k|}Idi2{VWae!RFMvvn(INn8F$AD~iQsXHLzqiD zBYbAEO@cJy*p*{6Q|A{GC}VKSc?GrT$&I2CU@!cA_RBNm?c1Y85B45}+vjW6m}P3IL+CwL|HVr{OA(z_3Zz6x}nA=bmRMM}IqRY_8t$+F& z87ogy11ieD?*%Z%xP!m#_F)Xh1@p_J%%5D4zfk1bx%*vMlt*6+AdSp z%Fh&XUw2Wc?_Xa3kqOybJ>z10(P0|gkSFMIivWrucc5G|a-_e zDMJg~;}KXnQE6JAR+f`d-23)|8c@x@ z4Omhf>e7Khia#^rvHO4jy)YBH<2ce_bZ19pbR>E&6cF`$@k_2^gg#+?I*=VItpQQ1 zpsjCRzkWR#%B1zguxTcY_HkhR=uTK>5u~62QT+S6fGC4FY~G37K=09(CRy>}UcGDn z{{GU%F?^3w#iFq|W4_F7e?2Ghly7S`30l zJ=NkVDRG6%sQx_c_O*Zh8H`^#mgaD}6wTLj&y!nWX15xf2nxS~;rS{mJ`GU{1Hq)H^RAe-TD)Z6H(i4g+) zpiSIm#W&;tY)+!4lYv_FVwZP(czP>FJClwC8Ms^TVO(u**xjQBxgYP2m^b@y z-sqqyK)Jxl=CF`-_7kR!Sy~`}dqGa$rh@o z0wNFlYzxh(^zqeXlbXNe$YuIjPL3ogO?c$yLTFaay}jmQ`-vD{fkGj)AIQiT zey0(DaD-x*JL?XR5jpX(kS|S5WDY56HHnWMOHTii9Pbl?xOsCFAjwfWnSp4v#3M-S z`^W=1gAF|-2oY``jqW@(q=H5i%YDmc1H@tV>eU1)28h}zd}vs~A~7?W2>K>X0*i(0 zL4ZAV4;8rh+#AImU*xMHFkHuksC&D{#O5XwhU4({WBASLIcO2;NfDJXT10Lk7;cLM zg6cS-qX5j?2?ZHP$Cym<5-On$&U6`I)%K~zn+ID8d<2AcVz}rkzf1TV>hrPL!*u}X ze*X#?Z;-~01y1@0X)NOs`ImKJO5A*rr@>7$z}jkR)JVwTtW=@wQc7Nu-=vxpbbzA* z2uXw;LKh?7N_Y-cdih3PoUH8WiGic`uBxi4If|kdk;8OLR;;*xYJ-KfHCeotwl+f> zb=+wC3NraM{imG8@BA>#sbJOA3r*Lw?c29Eq`SAEacRe=g|4d!E_x_xm%vKk`<2Bj zamef*W>2ye*eC?edKGD+maWpVe=fq0qB~?92U59c$BrIF^Wqf-78Y}hljMd=aYISx z!np-J_Z>QHY_&4&mFeYSaxCaKtZcrGVt)}wVh~N-?L9YDa5~;SpAk2MDWngKc#MwL zzzAsK5B&b5av0E5br+|ucGJdTNDV?udFYR}1Ihe_Rs(0F(v;G@TjzT z)>tTOJ7_5mRHaB4qsKXTR1dlS=x*+|HmjegDEA+90R+PpR;QOoDDOmWms@u$mxP1C zaaq~>v9Ynh)Z1wUQ-G|K;GHfZr4aaorhx8Th_}LCJo71r-ymv|0LCrZX|?fs*K3na zjc3;~cRg84`~I`evU4775X|ZbRd?9;w1THa12+4BmrYuQAFS6NCfZ<(769!fBn_m)n$T= zb}abDjo-<4psW~u%t!?9=p^u{Gosx@!`pa3gw8Pu_gWlTkoG zz`CpY1l_9d%i#bqhuI`Xxh?+p4n@CA&>7VDxAy7C8f*shDGpA&y=LpIOzHRF4_Er_ zDOF0+reVfz|8)+6bzdTjd1?s-%tcUcq%^9UPmGU0B(oX*VrJmWbjd1;<7gKKlMsn+ z%*wi1Hrc#xJ38KkISAg66 zVLhyMKQq}PHZ4}a?8pR={T=N+KAnMqfoz5gD2^qZC!@}0I`vvEq?XiX&Pd>M65B$X zD9IC_T3YlonHU+#U9jf}H2l98>~Y$m5ThxiwriaiPRTMX;^N|hbmsLszn3bs?49vI z9Oh%Uxw!#_YnUgpAkaMp9t16*5_X3Df_Lu{5KSX`j&$+D+){KzZo8FB?1le+FAXS5`CoTk^>}_ zmM>rax`{Qy()=@7f6}JSzp%n}axc0ImdzxiX<>b0+0H%-;a|{%+O<6KFBR$hP%m8=6dkXTW4fc11tv!-^p>Z?0J1opukXXqYn1?taLSdcaFb)9VZakIdDF=n>UF@4wy%FyXyf{bo-G!iLsj{ z#sJEX_vNt=9`o;)ywV&6h_N*iJtCG@}95Lqr{5VD&0^O`YiLB`0aHP{vx>7yC|+jgG$e{_anB zZk;gs9}5A1mW+wE`|XQp~xDzcm$ zARQb=rhxBvpFAdx3y0j4qtQq&a7%A*EK2CR+}y+9OD7Mbo#3A4c(sAoB* zViQ>28^HVxLec3&`?=E9C2N)NhCYbasyK#z8=1jp^Bvzn@G@g`a*zOpR@U6n5rKsx z;yl52!@Z3`>>id76Umzg$jy-MP|u_D7UE?p9=v^}rKRKe10Z}D_yp`*g$S+mWYXgX z*t1(osH-7e1p<^jw8yAkt(7ZR64Ht}FC%&u6H`E4*hB6WEaFi(M|0fS*%^WU5H=(( zvz@?8NN)0H&z^O6R}6uRAclGH1Wak^=-|YaNOdFKFvE?syQj$l%ziPPKpEp>WdwpX z2)n#ml0IYz_;&;BXzwUbXGW1|Z3Hck(KiWho7q@3gu>*e?yEq(i=8 z6zgs{x&h3(px;go$ASztIn-x4)osALR6=9#u5jCW_GGcS0% zTwH<*8HUodL0em!KS&X}{2LvYDeuqxpzC|d#!Hj|X}pc?q(62sCMM<@YZEzPT@ZFj z6g@nTca0cArwHgte%2^AGDA);HJB>&nj7Q>PVqVnF9J9}B`7F(W&^RW^Z<%0_xd2C zyhu+s+@KSov|}49D{H}|mOK)_=U#IasutTZ>-w3uTT#w{06l04sjtL_IhqT zcsnG76@E<<0P_(<{JuLQBO@us+TLs8UBSq5Ec>Hgp>Tw<6(J9=iR=2n-@nQ zjg+EPOJh>?&`fn3bIGS+=Ylr|9eJkkbRSa+TD8a`#6yQ$rgy}HIpDM2W*`~J;CHl2 zKI080`-sr{Srr)T-T68Z+nx<*=G_PLC+O5|FE6jXhQ*sZFJ{k7r;|-6ZAn~)Zu{M+ z9JkSbwNYGL+(t|Vfu@I+e4CjVe)EqeiBrf%Wv~4OFZ>ge=YH$f@kG-@hyL!v3sY)_ zn9V@MLwl2=rn8p6+Jh`|N1CM*XKE5BlJ+XD5SI`S`F^MfbLlB_?Nm4(l!#L}%SFVJ z9m2xua56~(g}|tH#}A;Mh^DXvyb`%KV4^xopIpzH8rJAcBiTU2rt3YVYSL~?EEE*T zvVX2FYmlGRb%1rT0$o_;|%zEA{Gwe^ukv(ni3X_2OPw)-{WRxW^~eB z_6++AonqO9sOp0W015YMP1|D?o^0UPzg!bTd zY6DSIF*)Rf?hfc(_^M5{48zgZwL(HdgrpPk9k$%euJm*hv}Gelmtz#f&cPvH3_;87 z_H88v1v>CE>D8Xax5RJsK0&pb$J)FcYCnB{`TN=@oeHMzkeC>VG^sdJ&Yo6TMu|-^6O9$dYz7AAfXLN21aF?Ve?*ibsh;h;T(05A&*T0*RVm8Q~>~Z4gIrcPad|UX$7r3tlkcFM$fN*0Co*0hNuWNG`YWq}55? zSi&eRuEdV#7iK2dV}&n-+RS9>LVHVZ+8(R^w3kUuD%e z?53d?5D;(&BBH?tzwsn;swq_O$@$L%fZJQl*H%ms=2g>Gh2mM)W2%FYJ4j!}z;N%J zrnB{zj7FSoGD&_0UR8>a*#_xQ#soX56k@7zl%JnpAKn1NNMufI!+@SqY^k_6B0@rp z$h%3McGbqfbd`PRu`;XAu;Y1E4XZ*!#Ms`>PG7iZD%Qz9K)Ls(hlhauNRO%4K@$^` zP(2s;7yPkfco5&ud)=|aDmWBFZ}*{duf`-1DL|TKo2Y30d0X3*Hxm;Rn@ELz6DVp& z-j;HWud<|syFiZ%lOr>bmH$;Wm&IlBl%R-+h8{xFc;Ui@XGXJaPXFGXtH@#O&YoNm^N>|4Y|<3*m1{{tp5lp>b1Vm#?#&)O7`8pbMt0= zJF|M^$=yvbLfYTpYa_(VH%8AC4o#6xKKc~pqVVX8+Z=wAy09rFT1h@+qwf@ zeaRkY?52%-bX_5Hsr(5vy=8}Og8US4EsD<9Cgs|xB2jYS*5rqSp5)Btg7pKY}G#l&4w@b%T&BbBB zOV6-db(GWaTEA=7c2J{^fpZHCqnBdI(xr)9h81_Gv2G_24he-9u=470x|gi&CP({x zZLICkpgV*bq2Z$gnHExo{+J-ymPcWTQU z090NaR7-vFA|y2yDS%jS)(C0- zKR8&pCmFktcxVZHS!gRI#}htJ0a`tjaLEDzV{)QPBx5Dq#uiMRDcH5}aDjUw6hz{250e3j`lZN85c_pgwT9k zc0o&16H05|E*2NCvC29;={ODs-anUtP6C-iTq~OW0#YBvu@S<^5DvYm`gp+H#l7`Y z8_>P6cGIRv0DMWHfr1hS%VzjO-X0!u*U+BbcCROEg|F6@ys9b{6wM?qyf<;n2<>Cr zw4L@zOHrDy_IY-(;mwhpI$0TLsx{NXI}d_*ZfiY1d$;K78(_F|1LGlvFll|)g@q2| zSVWCx&e`g?Q~c|19Fs%)6v&IN)-=3VZ#w<-h@L^4>eYKwBfgovewf1gn5vHHrr(hQ zl@aAyO@^&dzX=%~M#Nzejfr0c)8<+qs6>1!hcV)|904QE@&xx#e)YM{32z2cB&M|cooC*;W9TjM~?v;OX7kLJ^(tPwAE1qgM-i8Wxdzn zCDC$arz%g!eDqf&MRKQIB5arT*S1L+HqvR%V5gbCXqH(k$8eD8Kb-*Eh&T2B+Y4~$$dL}v zZ<)j_6gaiO*Y_A-?Ric*ggQIRpXphCQIze%Yg~%*4G)j?HbG7ytsr=B&bds|PX3U` z&>dANATm$3TU(_V?_@ZMx7Jjk=O;=_Xz{WQ!RVn7m19}C=m}mXVviGRIMnY*BQMvV zMP{Ugn$P`wTu=idL+NI*&|kC&Y-^kkSUlV1xKRkUmrXsmXZPL^X$j2S)k zF3Uvgh|UAno#+5bR1-h{$rb(fA6`EdziWwT9tW1=>3-jbHn~n9#*%N-K*(Rwa8lWV zj@;S`89@Lv13SEtWg`w<+^-|BmP3?q2OKihxYdBo=-s}#WY02mROF?Z8d$DMJj2%} zEhgoJB>?enU)HpH-7V1P=)BupI$8b`ti6^qqUKOqNCzx=nbRpp$aY;1&9iWxhoH)? z;-+b#3kHj>qtqm;tyr?;orMV+0!Yxz3`orAsc;cNqR1lKd8(_HIPH~932=wV%hbEZ z-D&TpC)n8Y`1g?$OeRcHFJ?y2)*B52R1p)M>@ZP|#|lN^Z3k;X>%s>VCUeWJ34?gQ zUry@_CmUzE5jP2>WSrKB#37IxwDqFBMNHeG1YrFcL3T0mZk%&zT+Zg-4r7V@LNC+`P2~kNYYce8Q6j_onj4ewWDMX@@P(qUIsZ>%S z*`rjlRF;VTuk+zM%#6N`HHX?;iI-Jx^li^cAJC?YC(diY|0_`W*42iyGoFK0wO= zwo;6x1(_rzla57xZ?e({ixx>jP2T9Nw`ke2WoM6xpIYg#5#5GDfF1Jc8@;_kK2Kx= zFeDb|Cd6N+BdL$vuJQG3fi3OCgv^js!vv9h6wFKL2L@ULAQKEzp?FIgY8=BC`$q`Oe4 z8Mm)&Z9r6fL}eW)gA4f-9?E zYG56`+r@6_R`uL;)AHq8+rrixTF|UjtHT7t5>(zfY>elnrF#x<^9%UAZIE@MhYb&} z=dJOSQeVG(KAK9qG@xbB!wkC4Q7Y-64}o+30Zv=aW_anb<#XxX6PI7XYxa2b?86DK z?4%7iK^|-0%IVd;*m0y5XX!ZOZFE4Mnp;|_%W}Og36Fddgn+Pgq0{r3Zr>P(ax?mq zOrfL2&jHa~9MisKOaD7X&FhE9@5{|ueRq?%>(q?=DL(UAxUZ8u3g}wg;S^Q`c5MyrnWTOlsozS%1X^S zlb43yuy4nX419jS<)3+7hkr=Jc_K<+Qs4xk3EaPb-xh9d)%|Kc?d1~zk6w@gMGvxc zVeJ-Y9o|N}DiRG7EDQsvwh<39y1V}4_j1Xs^VL^gkp;n}z46LH0!{PZ0AQjYPJq@& zIqo;>CCQ}fgMotK4}aWZiu8@jw-pBd9xWAh6R3dO-Skawllt=pkCB>SwKz@5XT99 znYTXQV@CLp0=p&cyJ$#@nY=^DouU(f4}SXdyY!;Bf8Vd#>zw6q^33B~mhxJaU-~+s z=j$)FOYzf~7qfD86GJEG)l0Pv%ig>>z9R4CxxM<+)oCQ}p5WtJoSppc40XZ;mzu`K z8v6=I5lQ=_Z|h&;GVNCzBGQs18IR#IAZp2nQPD7sawAhWFS-KwXlS}2zo;mn#OVuX z!Kz-9XB-_G59=r3)fqvpa?Z5etoFdE6J9{pUK$UE);r36)hXHZ^wAi=j0)=2F*y18 za;6^QFM4wG5;6!UVv$EW)aTB&eS2vTHtI0g49FrTs{HNSO!A<2*kHF9>(s97A*EGh zhjBD*Ptp?*NPqs*S9{vG`~COri(j9(pJKT5=DFe?XYJD31~#+N)*@9NAEetaYgg6X zfz`+m`Tjn0kAz$ya@j7ML46cEC+Ro2@I!fT;bFz{nZ2f;o7!@-0WtG7FGb|!#xu&Z zZHCJ^i@@1-$UH-lK^^LfoL`}5E@fqD&zN4SH0`@2ol6Ef%tg)X(Cn7;hs;=xt90~XcO+Zul+RZXYbBeKDqo)4$`@j*LRbX z4QT~uQ%7k~W5UusIH+C@ZRuB+UWyP7b=qz616gcGr=GY}@XU<7ZN-O0f>*JZG8t>2 zRh1a9(`EC94IA{@Nn=I-qNAhZO>pJNR;Q%x4!V>Bqbl$WW&rAtk}0RXDgD}nx12^U zE1aNfTRR*fmP|lNxz_E~{rTEsr~nV0uPPfSi%Pm@;?izwHLCLI<~}s$-ETi9dzz0} zSfVoOk$3%*{2#8aUKO|~s3=($hTBrjaF=WVzZY_L!c|pGPE8#rrLySjNWsG~&Cq#^ zzHDNam-Y~(r{UBP+_vHO^cr?8D7P?_g}j_qI%m$D@`|U^ zUnBfyN7)Qvn`T;FD9XmIbuVw)XSMW82uUi3ywsuxJgy~ypMCx--^ zj_MYDZ=f`)fDIyR2eEO5P{w1*rbR;QtHAKXOvB=v``1c!2?a~6^MoKo$eQ2G1!;`% z#DGJMmu|`4WAMkwk+!WzZ}o7#cTqZxo^g`=n{FRS<`VcBoV|jR-0Uud%x{(9{h!)7aPq#YJpo7n1E156eP-El>8tz6F(;f;`LY{? zQs9+?_Fy=Co>ez_2OWm3oJx9h-0e*sMLj8YIMuptYO_7Jd>}tdSltKYy6dX)7z{tAy+$97dxyM36ip@0 zaJ8HN{`<@i9|z4s?Rsc@yH!w-=!|X=sUw})sjmk%`~%P9P0%9(?DFVz=4uwsy`yaZ z8my&(_}ALCM^i~*?!cj(Z!?_+A~k}9%o5xi2XwuvGK+l8GJR5Muay$htnVTJ zqjHMvxqAq*1L`CHSEp#nowWS=GA5(!*@2n0-*%W_7!(5ZIfMSi?dM=@5oCvzpLS6< z-J{D~_2mlz30eF`@CPZDyn|1txbOD0>3E_W~eml&b>@A{E+5dhy~wv^a-x+pcx24wyQeiMjL>< zIXgLJ;`emJ=)|bx)K#2F=AgJiX$h@=Y5jHuUkLYwpm2;8V`)lc{oTOa8B9C>B)zY7f3F za-#T?4(YqEtE=)m%`Y?Z%CA=t{1f-#`XNrYdo5bEXffC2z%nKhWRs^Yne*Vm0|)=b zh|->aB!4=tvF)!nfOVA z5puA4$RHi^$MJRRKihWh{=kW=X4@A}*i#gDXNQ`GG$g>wp-}kFjf5k{yp}%#T`{nn z(CkC?-T3k|Jxab*S7*%HG5(hpA8Lo*cf5LH)Ww9RE~9&?P0LQXm7po{O0SDFfZ!PyRZ3I_p>4_N*GY;e>TJA}Qg*{o&F_dw z^z?+~k~zD?pE)y?5UOrb{d~{T`1tq*n1!d8KKj~D^O*Fq`J8S($(4h~xi0OsudDUq zSw|8RS8ptZv+!a@^{1WXF=>}BjeG$0XKvJ~Dv~Pil#iR<2s*3KTl{bWC*gNdyf? zIxa<~Bo*@dv;O9Tj-C8eK>yWpRm@iU!VnnT7WJ`=9#vFSB!q#3-UfTato+;;J!Drj z&0{Z}vrnX2cL}KdX!S{8Kj&DW71NCl!-prP-XoVdd}ZA|vT0G?q~=tMduVZXPeXO5 zj*L%G1lxUDwZm@o==F$-3eIKT#5Lh*)s0$yudNE)L(|@JrQ_~vmh&mkcQ5WZ#4XY~ zf8wsb`>(tgVub5`VIbs8@CN_2nmvSC;C_+^JtJ>bXeK0=5%-hY6j==$)8qA{BPv3? zrSUTOG|a$Ah(gvUzTib^6nSHLuhrcR+v3Sm+vm2!PeLX|3$v+H!#K))bFV#d{FBo^ z-2vv0$XqNIJ|d@RTDucKgGNj>u1BY>HNKv?weM?1cUDs2`_VU7n$=T@n3g2Um8gk>t4Xte_@=8M=K7MHwIc34dz(&ZT!>k8RkIr34KXSV&3B)8w zA3ZC8#Mg zgvco+u7{$g{Y;Tkzy9EX4ZU2A@&;+7pprbKm(JVkJ{lw~VfG4MuYrNVpwlEruJkx2 z=aHJ&Nc>}&MRi$~6UU6PaVv}BX;3$9m0G>y!6Grvf?xh>(=vs^I%n|_XS%@C$ymI< zE;dg`K<6L#so?eN>D$gefqT^Z#K>y^jcQI#kw?r6;wm}B_Ef(fC&1G&7V(S|k5IZw zqc_sAfL>7%4Iqn^=Yxaw@%MLHB%N-AmzuGqXcI ztdQ>O=S}C8#L=8B5Sw2mR(lk3#>cxpxl|ja`1+AY)K4LCEsQv95zQuTT7FyOsP{)) z4E^qUJl)vccgK!N_IXdA&K?TG2DRa+IOk85mGddl(@VE(X(&9Lb?erJo*E8nC&qE0 zEW3@ntpbfk%e-`JKQuGAUfI0eMPE^LzTO9c97+I=dVg3pIvx?za}K}z9{FAaPl70E zoT`?iAfcwc0k#q)r0u3u`az$?z_+KGT4?rrggz0=-@h!nzZYBGfFmh(wzfjMO1W{v zH`!JiDIYzK{q>)h`~sIEQ;Kj=;V`7qRNC0W%4Z_jDjG#?>2(5_4Vd?t)GpzQ!Ceb2 zC|rfD_Qu7<(Jt6^p7jMO|ZdBT+iI>51w+Gt&g`$8gXEQuGFXsHn(hp9rz7@VqPr(ukX61HOCq96fhVPYCRN5-NnN zK;j=-e;$(w%t66oY2k~KmRnKp(L;1I?|AYNLiE5QL^Yh1E6jkDbA6pb=o!usulnVz zR*}up2cbClOq2wS63d(b1+*&Qp(zFII|9DpC)ahpWj9PE0Ax1=R&nvz`Zf?`b@HFj zYm!Nx!H#$lc1ue|2#+38Siiz90FjJjc9&a=R;|W)9|#E<04Oo}vy5&OysyyB-aY~( z{3spwV&E2aO-(am_Di>TVgJFHKR&9-lblgw#z+?Ex+v+|HMtdN%OdFZ+qqM0>bigFoTr;eV!Iuasl~yOduhe*mG}?~1C+)_j}9lVXcxZ}duo}XT_E3H7| z)Q)F;1B|zF?C6S|eyDi-m%MrNrbo;p>>4iZ*thTX83DEK7O?7;=(hhYNhEi6GZL%e zot?b=#8H>LJ2l6nvg~>2`Ssjx5@%P`@|8T8_Aut0Wy-0=!e-5yxqMk|{J8P5YK2J? z@?^)q|NdL|Q6PnI2TI(_uJ4K<|80NvYl8+y4*K>@A)d{6dVenOzQRsw-aR zX1U*p4LS^F965517>7M?n@@}Bw|_1_Ii9=lxqYJJlS>P}4vqhG|Nf%WL(nWoecDZQ z&%b-lJ?8R*2MO7Quj5}2o3{_)_*OpuQ6?+KdEdSdFETHmAdEPF*^1Rm2SU3GHHvlY z4E9YD&7=;sYuCP0HE>NoE}iLL*0FD^?CFWArGEiY-|5_GTIQg_*c%HFlw0rbhb3@I zjt4kIdcP>0;l9uQynp7cYju>Vsc8q==E3EOA!X@TX-22M9H^m@>V(OmO>E+$f;fD} z@3+Wn&;HKtSCFyq^_#bE2#Nk4x2GS3jDZMyI>&!{O79o-MQazNpBHTE-*6 z1bNgEuFEGt=UC7WQHLpoF+{adKs8ZsimgFICLvzOW-^R^2`r^KvOB2+#HnNqZqr}~ z=5M{#y*Oh85g5w2OkMugu@OR)5*!8QLZY4E_d6QL8AHFy7j8WDlaq8Ejp1Qdt?kNjEp*eqkzUa`n3z z)uG5I4?oB-RNnND38+KpZ#P_wA`us04p)kR$Sa@ ze+;%niI9?{KLmU&)FlwQ!5&p-L~O{C6s{KSi!6a+@T>D%(0dAxLEgmfVXMwc-6FV( zG$E!>-&?a*En)IDYOiY|I%bZr3R>=Fx}WLo9_4H>B}>a8wIWq%LhD<4_d*i}Q#;1$ zN=0{H_qX;voFeHjBg=7ea=OP}!UnHh;Zg(O2_N1YKz}F`bquLPvOrU(ha~oqX{!>; zCBtG%@w>jr!6^cCfIHsWm;T_B4}buLD46Lq?$4XSYd9c=N_$EwOOXcBT`v;9rZ;a) zI4T4Oen2yb0oNq zW6ME}IiARuoDKAXt>SP0N633kOr=!mem#7ws_@)>fIO0(qxV;tMr!TFW$PP!`8ss1 zT)$@uni^MgC#MsX(O;oSYEX8wL`)}5+QF5boR`s&VFe3{b7Kxe%TqO!0a_TPG@Q`L ztnr<}!#{*=>Z^PR@{g$BfB(I6QFVQ1kN7!S3WY`4=RS8lY9`wP%H!BhGqkM3CJ8Wr&dl%|8pAr%5PkN z*z}+En**jWJ}GG1hnKm3;3e@mf}F2(W!2{oO%#ryqBe&! z<*t1pQDQvAPl~q!joYg)boITK94&V_9r3I5C6r%}>nB!Ee)eI8ze3@tW?l7L@Zc9T zy2|?%GQ79!Zp^%G%70ZT{MI6~3_cL$`m;1C>;#EcFDgyLTNIY_7^SN}{y4Hm!^z;y zXFhiDoVDm|QMbZ5>4=*wOwG&=ifaHKTAsA$XEG>4;dyV3e5^{;y#IbI>cNAwcnohP zR)1Lx(2r=jgX=k18HBBsFj7;EZdmp45BSL@vr^0L=EDV7P zReP$d8zI?~|EUP;AQC3^r^^fe^U_NpDJ&g!DzCH-(>KhP>I9}$E~8T8zdWlyFAXDR zLkHAh22JqbeC5U|7D1iwop=9{*bcY{*1TsM5!IeC75c6Pj=B}3bYs1ZyZJ;;Il?sO zY}X)kV<{nQ8d!cORKtN?+V-3MH-AO+ZzU44vNI3=v)@Kzw0H!*G_ufy6{jf(8ctxr zZ54$kd_N;^+&iblE`Idz@hC;{S!5?l!7Y@L(QxNl%Wmk;k&BU<|l4$$i z{y|WD^yJC@VDN33b|acW>0eq}S_;>TZ6(IfZTK6L@nC+VpVftQ$rKLxOQ0{GJZ5>Hs-%g&!9)@_BCO&xd5th?o&u03uO)#=j z`s|gJm%pR!r;H#rWP9O@-%0=D+QS0Y1)~g&qvA7QD(#Ll_becD1y$L_Q7~aD-h;*lL$O6Lt)ZmL>7}?zd!Zjzc&dv^{>>P_ zLH}70Z6!sm!+~%3DP#555W0gGS!s%{Uog|oBp*;SgvY|bfs)>9o7K=SVz$A4*0iCw zl%#}Pd73j1!fl=m5#LW06342u81*({4TCbtCaRh~H$7uK3Hv0=i^^yu|JdU8=J5#0 z+hFpEDiK74M%fm8za3AI0m(vUweiNXs1d{9PPAi%MEwbtkH>kSnTd%UuRx{qK`&*& zow+N$V+_L#Cqr4|uo1*a9)u#hgFN8Lp&1#4(LV;7gvlJqV1}m|C>M{kFd$}JThAwD za~jB;?k{c=^J&-Y;o0R(r$l}SENs5&SAb==;m0dzPeAIavFg@5w@S~ z@`zpgVt)E{u51M9?qOyy&RYssQe!-e9x=VbGlEv%phm6x>Z4irJ}8$0U$q~zM0_W5 zj$(zY!;Knx^NzwOgRbZ0<$e1pyj|d3WVm;Y0Mis76}fyxSO@`7|ADzdd3oGV$DUPB z4UKWIE6gcKRp&E_TWDLAe|1pUp8!*C)M=PCV1I&YI4ypZ>46Zu8cwLAc)pY}_gzs| zlKYPBIrcds{S;lv+8T-oV>`P;bsBe!q%<|0%uc7AJxC(mhT#>8;8)_C;ySTKdPd<{ zo2B2p6nX2O_SZ4qkRBO@nyR66P5}3SHUGp+Uf=wBMur7?rx?!|78b>NDBITy*UUMf zRTk;y&m43DK~^JxnAZAm1tZYi0_!hTM6-Cw-v{GsTHxQF!rxHdOsUQa#at z^Q}in=LuU=9378yt0J;%)8W|PbQ^{Bn39s>t%_z=a%)&w19aPvMA^=pw}2+ARl@RE z)o^&b(f}f_lqYm&5jn6#cvi8RAq;)Ijb3z>V8??nv6N)sNKOPK!f)Q2boqrPU#B}~ z^M%Forb-L{s;spB!3NV`$3Vq7$ZHJ4GiZQ`oMLB-rKdOmk!9KEMu(zev)R3rYJl_8 z7fGtb=grZZ|sI$ADx@_OSXYV(0>C;PYgK$#-x@;?&c5rH~R6 zF$NDViv$rBQV7MXnjzJjQng6uQ{H`44%wb7s8I<{H9J&6Jghmtr?5*K@lF(;8scyj zyRE^0GQUlEiK`QZ4579v(6}38G%m$$DqO7LUZ-x|Ar|HQx~~J?B`}{2D7T31?X8MS zV**N*vtW-JS>tr?P(YAw{+kl>ca}qtfPAt`I3`rKO=< z1xSBRNjB| zRL$Km;aCExt6s_rgP6!_9`u1ho0DOiOhG3Wq0(Cs+*De)Vzb5X$-m#KO`CA&-||-a z_idvvJ4#|<&ttzw>gO_pEhoGX3YWIKShF3aEhhVnJg)FO-XJ1@ts260nye+Bd+tsP zX2G70DE_zQpFN#&nAi|AE5ERypqL?$;utk!f6WO+cl_?*{Subcam9@|p==7lN{|8K zOUhQNm0w69gz^p>Vfq2j@`XPPWKD)vaODP>C|#hK9{*vo-}b3h8EZaur~ z?E%$r-hh}qiT+l(@tZ2FFQa$bx3X-baCO)lhTbSGLV(-CNRW_O@3{P=udbRx-8rSqQy1g(4os424t|zg_)?~0EEM6?ggdz7j zvHCo+fhYH0<4A~PL}?m{8$v^4yj46?uCTx(4%_Pt-#6}uQQnzu;zdgQJhJkt5$6hp z*?94f6R<$No%r?ve%=CDFnW26T#<^Z8;+Dv3DtFV&!li-*zEN8KZH;`Wzp#cge;Kj z9&fAYatPibPr8oUFP_EoSQNVlxg0{4WIa0(e3c1(QGAF!`M<=|5%p>Wzn9PedJNNA z_*s4*Bd^$M6){7Dd@54z+!W#{)FEwaTmj6a>8e&r4IU5 zim(8Z>1TS|nDrPY?&8uZq8qw1_8pA^dJX+)VX&y%YzHk+j2530^J8`XQ>Qw&uzo^PVBp6XXd7`tz71i{*&fCRj(Na5S3%7uyjEjc|YvvOXHRhWbfYX2L(nPiq2y+Uz7IP$EHro!Y7=+s0E zqv`D&m!z1=ETk)Y^>O3#AD=dOKuaUt0o6SM#p?(&Kpw%5_lr;+Z6i&J>(H=pA!Yyj zuq5Su<>z;W&4h`23V(&9TpWJ+tC*v(T!8e(ZAoJfS%=VmBnQo6W~e%wL6&BRq7Of{ zu@d=lQ<74Tk5B3nm_({zVYB8Ip9uu+r{w`9NxTvlkn8x}nqe*=BRQ z^Mt3mA5F0=w)0S2YMa2+`@&cqbGA4O;)3hP{Hk+Yf_MUC zo=Ex06WP^#`SP(baCVrs{7o!gO-Neu$h?C1@PYb|cAM1$w9E&6`?`0)j$kh1YY zMr-3TToG|w>=5Y^PxqF>g7pwiQY8fdxkh`~#_pH*NrwdTftiQL`SMB%2OA-{uBlgJ z&PwSWVvd4)qIq#SAmvR$lx?gPQqwL(y;fq@b78TR`NGdX{#MR#j_$FP!t3WM6s>>q zvoHQjx=_GV+yi_O0cHunlMDJ$2v`s@_80`>#jF1Al^@Vk4goW&=T4w*gbF!G^cD>6 z=vhMHE7cY0wVx`)k_F(b&1Z0=WNXpP07Z=tDo^4m;l}XC+>S9MflzR>4eG5a4uv=Z zF_1QUbY7{Ev9V^X4G5xiMy(eJ4W^!6QD(L(7wSCDrKLa@jWj9~{s8C$~ zfo1AD`0m|1efGLEGH>6hYN7pUe+$nEbUt_C6q+D-+q&nRkoFhkhABN}NN1MpjZhLk zGO0%2DXcRpsZ!0FGf=7RUG}t|vfg`aMHV|BwL`kOjNq(U}k$hOK7E+Uo1*2&KyF-l$^{pZF{0P1Dj>#O!NOMfi#sOGp$0&pfp`g42i_EX`tTt&WOds~>b=&)pOcV_{}%I9+tWO%=rd z*s?90RXi@$nB13|_g+$vhuZwU3><>0^<_jUtxVD0VO(XFaz?Smh2w!=aGR-y6s^G! zjC|d*_T4JjMiqRPBV=4;dL@@JjRem}Rnl9x_=J?0S_5pTi2=!MPv?myj&sTHUT^5e zIBrm(nwyz5A9zu!;87ImLDQ*j)H@YoACS$}U~zw09j73Zu0OpkS2qF-qKZGJom$xQ zgL+S%qJ@|~>jci#2^2?D0`ipk>A04#al^6;1+)ZF84m&ScMVW77RCsVua)yKdOLgN zJdJh{8m8}~S(l1}qq2%6CH6BQ@`FB5&b`1ZMz+7IA?spEvcXM#2Ood`Zh>jt;vYnj zyhR*8{w7)njR3f@FFO}ajrh*h^5&khJ3{XoOzt!;2j*4?mk*?3?)=RqsA z^U-v&n;HaxnTc%;CsyZLf&?iP9zH=qFIVcpO4Vwh+pK9*X}F};*HsJWC=#-V%MeY? zNawXRT}=RySdlcfyY(D6(1Nugdq&Cwj=~{%rI^Ej&I#$eIr36Sb*OJQDht^V`mHV`g2KK5*SBuJvE6e!WfNGdWLO z)fkgCY4qsP;Th93TS6NwhV~$gb1iLc`hHX9#;rNq8<^EC-bN>IJQ|B^aA)wfe&0DR2L zEC2<8bYW4@k7ivD$n(rfi{xaD>+$v-#t?S^4Ufu!2{~mT!xQ;>+?Y6W5Z2OltP}zQ z0%!=HiA)n<4wn#`&Sjgv7?V*}f4Y+>*e}rzKL!W$HtjCFuA1Wcd$VRC!d}SE*iEG@ zT|`bO8BxtVgaMs8HC1eeY=3hf#~8f zO0=?@A!=&MOAOs<@ttJQx_WZM;DxjmZjs|kw~nbKd)RQ+CJ6V# z;Nli7TgnX+%QK-mV<>cDa^3(KL1KT*!!X*@%I_b#V3cnIpd6($^XB=04`q4b%3UBA zsTCR;I>_QK2Z+V@=zaGg#~ybk-Eq!eN*NF?VEuwi+23xfSg0WB(^kAIcFjdaSrr5N zw=$>Z1Ve}`PQzY-zdv91i%A#8PH?&W5zlGNoaXYFPv@a3d9hHcHTa97@fH8A@AVwT zrv?hq3l{tu$5rsgj{rR6F4TdEqpqzTlkG<1M?P_?rep`)6(=hQWF(w=s5B! zKW8~T@#)VralP`};+wI`Jwu=3*@=QPtrumqfs zUW3&W%BbuYP^ESR22NWi>7{qUi(~#QJQY`LuGEbh z2CG+{|8+JbrNM-a@KQGE0lwSv=oE1vBV}IdL}F*MbMCx(oIn$ZgK+(kqT7a-VhoM| zQSNNno0*7#%>E1lX|zPx-8)8R&x$81WwUwMVuj+X=f5rm`VVL-hmqb5vA7GhHe1pF zy=z%nrj*|%*yqGJid#r;Bk>W9l%?fQ&fCQ>7*|6JV>-SC6*50 zfL#OA4$(+eA3nS~S_kfF{l<-V0_qc)=uQ`wv=wpX+?Mmu4}C$V&}86A&Xu30^u(He zE1@23(s6+^rdM#7*@Tu(3w4TM8gPISO%A)o{1g+)bjlqIoQ+ts=XT zkm<7b|54ZQ7T7}t6vJYMoZp0uqH%vGTYK@&K#}q-hZL$YKtqww0Xte!5IJ=)KZ=Zu zOucs|E)MhS8*xwl2oe&_!+No#MK?zZg$C1a%Dtz6-T;@3tg@m+narG+P_NgHx=M}+ zr`i>d)vH&Fybl0*MyOY9MOgWdfS&Cj5+fmpayfFv#v&iUY(D#0g1IBGqI8{Ry&TUe%caU_gO=nQu zr=+L*$NZhg1jTP$09C>rL2;6we>44eJA8aXI61k$RZK|}AEI#%yqqC2!B9fTu$5;( zDhi*<+wB&tFz4uR=F+*9yMB1l3SvG&r9tiO{zo0+*xv(4h%e>IVYknQ#dK zNzn0nu%+x`Ecqj;H0B|jo7Pf@;!Jc{m9O7%ga|kETPw{&ZUe@nrs3J={?nMfe0ISr ztkfUH4SqxpgweQl&o(v87PqiblLQZVsE#U3nw&J2I%d!65<9G7ji@uvMSATxvg+8k zelKB5SWEy~NnX58^EE%V(>wqF*ILTPw>8`b=_s+Vq(&zutZT8IMQ1$|)-UDmQQz|% zpyq%P7cS_}wY7yzVs4bEu-}U8?@taB3foWcAaNyJYT1qzOL5Fyh=7txDyD`Z;;2b1 zG!8S4G+&G)0EJWUeZ)NyDCOG$C2(R5!%!GF3|lh5j_D>PW20o2XtN&+%&Zd)2GMr1 zx;PfYc+ph!J3l>twOFzhbdP`%WEG?X%H#JLlmT%hli>x_7i~N6NP{Q*38stplXdti z$~_Z-46=vS$kzLjLr8T5{mcJeZ&lSRanZM-jtPNUxn|TUkwtS$#KSM}4tbT4!yU{u zxUghy8CON|A=vEW-@AMFF=U&gli0h9esmMJxcn%ZoLV}Fi}Q^+V%jNzV4+UxLLE_+ z5XhDg!{y6#s>?02t^=WYV{Sn@0HG&@r~@gST}vyUg*Es!jf zwN436?(|}u;(dL)Z8c#n{??~o zOooszCELXa3J72V(0nG+3vf=QYga~x;B7TkrZg&}?YWfludgsrBMnn0{(x~F7SL0& z=!gzKG8FT!By&JG=`D2f^Yi_Dd`y+uG3ZC{NheW1=6}m3Df%Rn6~4UI;=N0Ht(p*^ z6?*GaEt7#kFwH6FBeQEmY-7%mG-BsHv7-hEFP%ESox=0T4~ClK7aB>z#wx5ING+3z zzyQnuSQ60d|Ex0PNhq>si#vo=KrC{j(t$cUQwXHWCDw~$Pddc>qLcIzImIKh-N-h? z*g%pW(lPNCQND=>aLmTh{_3PViZ%!YHNp#95 zLNS~xAT;gR??5{J6 zX7A6am#lc~EU6|&t3(uuj?@`%qjg9$S+XusRlpC8#(vU(oX(*yTHmvbFmISzLvbNC z?1fizgwjW)b1fHX^;5(&ggfxxsW}lN;!AAl5gec3a!=yHib6W06-;zS`9RxOe z;Cyi4j{`oqz`jSmU_M(*(ZgoK1R)`HkAKjcgSDcfVsf6)km6MCBC&l&6&}4sNIvAe zIZP>KYRM?+ey3imqX<4gGdz|&RSq!x8!dub{ZhBC3^rmzpDjI=hkwL#+0fZW|Hc%3 z8~=cSa4JByDU~^RbgtI6dcXd9lgZ`JH1BO;A{Hyqnka9wlb?3V{2kELRTu*c7e-1i zgoSAKVk#Q1$re{@c}7h6wj;O#P91pr{`aB?KXPo5*YyXHQQ|sLR0bk3aa%NzHz~%k zq)-t{tQCrs5_qV>k&to>#VGW9fF_^`Y!31qHvMO(MuauFAc{MIQRSj%(JI_@1-z}Axvg_p*^KH(5HMC{ZT?ZZ;7F6edv(4QmdnQ zobr`Qym;~I&w;8%WFFbWt1Blm7OHg+aB#4l*efnN^P~==%X(0R@mon+LtBpgatN$U zVo3wjniq1uXW{%6;oYLDN}DIZxAMFN1Vx`3wctJ~QR)3<7xJ1!82thttb`~^xE|XM z98i&omy1qo5Ud24jJ2WP!#pfCA`MwTww&?0q!+Gj|4oQY+A*ih9;+@k3|fU)5Gk!~ zZ!0Rypi`1ZKheDZ5^A0#L?-<@t$edajt8!C+$UFaKV^$Eu_~I5PQwY*X^#if~vH}sZ4!P3r3N=kx71nqVQVN-NMlohuc4ZQC-epS$5S0l6n zrtE*(^h=QbQ|S)chxhh@VILpYNKtB#rOaD^gqDZN7x0?FZy=7`Q% zp9T4F=%9#Fu}RND4sFT9^n3n$Sa=rq25q4&!E-PpW2eb0j{YF-qzPi)3Hl^1=#iG% zvQUGISrA1`j=w3;9UvyGpbGO+SlaHYXEggtdfokM{bs-RSC1HW^wi25x9yMU zn8%yvFF0*5&g;O@YdvPKyW4N}olSnft2W)Zqw@3RjQGErYE4kh_*{BBuk=RA@@|LA zCZrEKR)PS7%C;HgA0a+C(@h_-kv*^V5DYmvaNx1bevE({b&D8O@*+2;JM5K14*1PH zVO1w3Cl{@|N31(A^~CJ^XI&d;XlMZZ1Rk&~#4l)swRL;W?Nyl{Ra64f>EA%muM4bH zMKrd?Kqx#3zmzSkGHKv}$N%l;*Fr%adzjeQ=l-#6+ZqEU7`-KLzJJX=$jRoy!Z*z)MpM0xi}Sq z_D5<-`u+pJ4>-;WHqN<`mKM$!tZ+wCs4UNeT#I*o^{3p+_pjt0w0?v1+`TDxfJ!(^ zA5#Ocn7)<_YT2@-Z*QBnygSoOJ0SZR#RlWQ|Cswb3?fbTkF33^Wv>)=2}Y+k#>Ki+ ze_cfg{=**kS+o9@Qh*Y0r;jnurJ3G6TD4&ffBKyZ5 z-mt;8PoKe_7uZ`HV`8YKSSPApVH-?kg($yN699gq7e5-t`=6yQ$NjQ`L0FLBhTnp# zq;R9*+7f+#8{etle?%*UcW!#daTb!`>1Vy@i)XUm6=YW#pg>vbdpC^rm$7*e6R+@u zE7@If9OS!C%?LGWK_}|3`1lpOPie5mCazqmhknS8hhK&8&lsb{Pg!Hwhe*$7 z_Ua~0c0~SRAX8Glf6ebQY}j7Nd`&R0P`{_=^z@49nDkZGZ{2E2^_bN#kcl0>8Wz3H z``7|pD!=r4cj>YQr!vQa8*LQ46CA@fT?`*JYI+AM0d!#7!HZ%zBKj{Fh+QHG_eT1$ zhb#kmwS%y#^Dc;8g#nqQZCzb;%qb<*t?QSBs7{&Lyy9u=1q&8{ zWE!#!u{3XFu&H^!&YVgcZ2PDEw8_fa6JLL+T|2-|LRMX}i`4~#PMtbs7G>S$`zyDJ z*THeDO*47rdo}v^Ct|k>nIn(#U_3|v<>gZE@(x{Uw}OQ8M2ItL-Dm(N#oU*U~mhj>C4hmZC$z6*2daS z|9fp1`cOpw&Eq2;fa1gfP-Cs_ix7j`u=6RfOH8f`3;yEf@?^AkA2w0oT3M-EQm)#e_bRZD^E9OF^`X8kQt)FBci%LG3|eSO87U`04B zzdGG{wgVs$1d^i98#{yJaVDl7x}KiZEK&mw3@fTBoa7=T`R5ce8rcNfBBH|Slrd%WP`9+Nzj zlJpAs?gmWu9r{>AwTH9AZ1vW=Ca^-H4?z9shkqFRcO-l}?URea`EaPLrH^)o9J()2 zsm<;Yqf|A{xR-Y$x?p`gZ|c6<^_xG2+Hr^|-{v;YvMQeg2lLAyI!7)H_T4+hZPvVb zV>rm>Ga7;-p&N0ijw-e;jNs*R|)J?6B~$`*C*c(4_CQ2J~q6x4lU%u&g-j zOk+zctEZR*@1B*Fmlw!L?V@#~`O$@{J$h_lT4Z0O6kfi4cpCFndiLB1ZtKPFbS`^l znB|7$J!sALS$*6)4O_X3)CnV@zXJo?fi<75@5zdv3FN2mhF!0&?; zLw9a1U5laR?FnC;WaAA2!8GpPyJu%I`}?7r{2L9)H)UnBX?h=WU~@edI{V~Mqc;Be zbVp_KpyI~v0h>ui=1qO1qGD;daG@quV|R7+l;aKk{r#KwvhpR^nBZH7E-!r zB=qU6qLNH;kpJSvhLDi1veMa+zUt=cx&3r>bkMf$lfXw*!4=BZ#7cIjiq?!K^Oh~s z1tsp>ts82W!3QIbqUv9V*7b0N7u0-U#J9H!|5>*#Qy9&2nd#b8#ENai<2o4T5Yo)q*!}Z}D zQ(;q2sNmdv0LckR>rATfC6Q*Ew{0t=;yu(xr&g_6g8(kHGm>|7tqZKTY{iPe`U3}O zMg1=x{!Jh5Nq+txEcn&~7qu6^!<%2J^$j@d4abhXkeeF-GC9Y=fsI83*#zP3CV+im z$3mVMd8M11+q3jN?Z{6#8BWB;UQJ1<&vd%Tqep2Kk?%KcpntFCg zPjJaDI9}$S#}^D()ydIuS<(AcKaDZi15Wlkp}(eP_~1vkZ|mf=ky&*SY~s^8H1|B5zY`#JtjiboA&GD%QAoEqz& z{oW*5o!B@A11znB$EYFuLak+ajF~$1uIaX(Q;&D%LOFBuA3yfov&Uw=ZL=oHMh6ZY z;5d9n&Py`YgrRWvT6OH0&H-w9ZdpIIasCI1J!t=1tXQ$4Rx{iQuOCU*gk#k+WXMq0 z;^uI4R6BR>?A}^io9;t62QIP6x6h1@<5G?LS%%3z8hdJKdBJ$#{Ru#s^c{u@7P4qU zN+R@wSsbq`Q9eofn74D_;_38QN3sUqzJD)~woYjCCdp@^8|9LBK~|q79t0FIxgIeUS6vT23_6UDk(3o-M2*0oi121Y?nG}E!igSr`Q1s zuM|0H0!H$_0tPPbsDRuwm*E4FsO%_|9R^cp9taEt;K1DTFd&)Sq%Ai=D40BS*(s7b z)F``1XV?W))_31riSJER$fv(`kp0Jetk%Ahxv%N7^0iKGpvC?dDDKw8ub(!LiBI$C z-WZ)^O9-2XsFm8Z7{fR6trt+MT_-3Sm(Wo?#5+9+g|1HzBQ5ZsXHdT$b6OvAAgzXL z^*2OtCcT(yYHBt#?xEtHfEMNips6fP0x4B+(ENP_ZQ$(zFv5*bXO#cHRHgl!o#CC? z+RJYQ(|L*zAB?8=?%LE`qZ$gpi;12A#|J4N3!anV&n22onpC!;T4adQI&5<`@p}SK z72XK&p?10nZ4vBd9q{#gvy(XcMRrjM_B$;S0nwHF`j1Guq?m=svg~GL{ zbI9k$W0vre*AMpq1q7TT^SDK4OlfV*v({Q8Z5u2J2w+!12#oU*5)vw@-6V`e+?zOd z>|+!>ztpMIv~}yTWU=_37{8xc?(~1wy#vPn&BHpU#Zew^vcD#Ij-oSBLVhhydcTmh ztj*fU4sgqY>BEa@bb*Fm8>OC?(&|zBLC$}}ZHl38&C;QeC%M0hAayipTO(t#1O*7c z9Sua8l1y^9>(pt0KeVWS*3^)2A70Vt>6VQfpYWPyvu6%D@Jg79$$AgC5Q)eUmSmZuKYLDQWvqzyI@N~-k(n3 zQ~6*z2Zu6TIHsVHjwHR95s^kjiJ{wTyRY`d{Fg6pFob*rg+}izM&;rzUObwUl{neT{f+ljp!QmeQ1r4-lYdNdCd896s#VLaK+md3ntkRDu(0 z<_VrqPh=-4>~Vo~6fAqQhT6`tAMzgm}Ev&N{_;Bs0HAOZQe3Pf@NQEwH`1t%ok}PfxkO0|BND%BE}g?%lg! z>%D`tqRrqXJpe2Ex2VCO4hg0YJw7|gE4^Nh-OJ6;ajhmWtMlG%ATu6v`K5J!a8)H9 z=IV?aH;#ZfUVGv4<;!J`CMd%#)kF!0b|jNs&VPP2gw$y?8;LUa0MTc3Qd2IwIZW7m z6ti=nev$aq@+4)`#x``VPpCDdsDq$-|2L0`~SEA zy)~zux^eO-9{!l*G&VanUHMAA1!Z9|t>s)$%(Sz+#^XwXp?Dl@Uw4n}`PoI&u8^X- zF}QEy z9J|uMi{GjInwWKT+CW&34fOGne{ttnCz?}NP?kQn?ODzW)`n=SsME0Z1*B~am1_Xy zj~=P#Gpr31Y$0umSZNwNYH`|GQXW4iimU19wDcqrpKvR+g7h~}{<2B(-?tJsXpoWMD*BIMZ18*A3KZGC-#5(g}3DKWLX&&ifxL8L9Z9Q+C%G9yEF;>mIOF%iSt zis%;%O2MH}%OuRC3$cF`uWCyp?p*ohosQz;BhEiXIY(M4Spn`{1}=VhB`3#^LxiMv z(9&1!N#SH8?5KXt0`LFlO;50k8YZY&32Hl^j%W^KR1U#iG^s2|MdLbu=boJ3jKZX^ zZUAou)zyse7*Q@+PiwRT^*G^c1yc%3iA?g-F39+nk*%#NR{+xS>oSx3qEb*d3!5oV z#887YSWZ`_Cn@UV=S%db`qx%|y7I;ArP04^J={Z2K@6OW?~gKN_=h+?1{TQpwOuU2 zOx8dSGvyhq_1U;#tkod5m#t~*KH;e=Kv9lkg4Di4heom_R01|^q73It)q!5eEx4vc zlzN`=@k0m;msli^ZN07NX`z$Y4nAF@@#qZ?7&;}hS}!K^Q%9khVEEgW+_6RvHYB~| zXQ?|g~r8vF`eW&UEZ{LXNC<(k~_@(D>S z?4Y)sp9BiI36hI%@zMBCa3F?Pwev)mru;0QGdQn^=>+H)Zpvg&qich*TB3z)hf`Ab z)nBU=0xScCwhfXKNUPcZV8kMo{{4MPZznMbH6&zg+mCbz#2Q=S=!J7gyK0U|InDS;72}O28c`?jYWwv|dJ! zNj5=`!_()7Mi9?*SAFU)>q`0bVfW3XVSQokrM>KQ**t4aSvbp46%OZQ)oyfBoC0O2 zjib8!&R_nA*&S>QB+$aqt_iQ50Q;)~Y(uoTH75Y8NOL68I<6^ENC9wkgNNE^f*Kd* z{f;%U39ildJ~M`8oe~)Q+%l(H{K^fsy*c-1LpW!hK7R6K1ZQ>fqB*$}19iNpdKBUm zCGe7MQx-%27LJ^hWjItFk#`yxW)-VWF`RVcMng|@b|;YJ%58Aq1qXavkng$o(ezt}hcVa*5_%cs-Gskt!?Gyomf`R}v{C>4i+sXjgw4fqu_UdJc+jZs+ zd5`OuX4-do#h+wfJ`OmTc1J1Ew}<&P>WYgT2E6<^vu0hSo)!CVISitur^Oh|4DxP; zbVy{_*0*0TK9Xc5^*pJATsWTzN8cV+om^GS#S|#+c)Vd2Py)F z8jh4Wkgw-Cs_=q#yzIg}m~%K;u^QI{WTm~(l?5@H>DI=zgMblH43qyf6im}GdW#&E z7p%k=hroUR_6xBn5!1%6=&jv1xv* z>gf2B*E#}xN=%j7ph>^={{shi9R5r$3v!~ENmeLGc5*@e<~=TuZI3U3X52SzSr?v^ z%CS0Mz7ifS*MXsc>f{E|MFx$P zCp1|_OG_Fu6smuGZ%6>@d<#acmV9_+W8~Xo1PLtJDfP2F=a4xYH`uN~-?5(pP9cGt zu-95XO=~UxXA-lBVFLZC&&fiq{4I>sI``2yh!#d6kGZG`gRJCBYa_XCfd3e;wxKMF zCjH3#x6F5kT}Mkt&J0zT@^|ml0B*rB88*}%Twxgv@Ieg2%FN`i1BseX$Z$s|Et|`5 zq<^S+6q3MxdvX9!TRIMHtH3TdE1E|X8#@5{|G=ZeKMmW;f6{TYzEX+bgmkcoaetVZ znJu@KkjGao6f8T@sMV4f!TV8WApz1>pgOMkm^37EdcwWOqvTb3H|5yTrTcRmwniGQ zmHv)CeEbG}nKqjjYGH=wVUkK~WrQXXpmAW^|vqDZcwmfJM!z}D~p3T+dPb`+Ay3H0Lre^Ta@7*tK@bP1nS|2Ac!u-!L(W!{n@t)tz53p4SZkW(@jW zlo?W28cezmj&ud6rCe^-vgP`9>uOVC@5fVF)_C%JiN%zOHG0r3fw-m-$%&TyE-_R=X$6r53%gwp|2QJz=ME38<*d4x$WqVz(${yu^dPEt~8( ztTk}pVUUFjYyV0g{`KSKZ>y#303qJ|0=@J$Qp?d{*u}D!0*(x)4W~tY-5EDY`s+1l zP%D`FQW-*2W#m&wVpg3XTuEGT!V0!&78!(@^J=_0KzC9!c?3 z3PvmI{WWk?r7M#f zMp<*iijP$1J*S^L^KG%Pm;?OZtz=5_|&V|lU4hwtCD23l|?M8DCbj@_m$ic>>Lh&)G*T8u#hbhosDFtMAcO z0vH<@)DZk7;k25Hce38f^Y-)S8)C1cTujIQqNnoxVXCe$oYbU{w=q7Iwrf$#Z)aUG*9Ed+)W^ zI?r>R%gjNHCil2@lr^Wf>P#3}xj;8l}4Xwab8c!Qj;34FyTAD^6j z8SlzpwBcIx)}1pi$I2fT{=!Fg?D?gA|2;xWc!eiixQDw89G2$dy!q9eH_O}`2d7wL>NHHGuiOdJ@|yO8RzBK(7v1|rO=egtVvok>Ze&Og80@sK z*S*lozqWd3?s=tp%Zu z58r~flks?mC_f&%M`5|e;>CgE*BZiNwH#lClkB*)hzadpKYl#&ZHo7n@@IRPoZRN; zheMjb#1Spl4l)NXNJmdapIc`Mo~ma^LdLwc+_Gf?Yg0#e2?FbIp{42VIrJSk6~9^_vegM-uTa387HhF96oI*I}po+KGyZ3O><^dm; zbt-uE$_VupayG}UTPKbVIp>3k)yej6e;^PPScXVWrayVuP||hdY<;NWz^X z=-xThVwR67QTl<|u?FH@fazz zzk$+mVXL*Lu$IYx907(pVN`Qbb7N8R!I{;U=q70gt}MhoQT7-Q`S9wMJcT%>!f`R5 z-th+M)otP=sg*Up`A0@B4kEiX3D@#8tv=n8#D6VdFAA=oK5l@KCA>8cXg`*EVX!S@ z*+89}Z-tn@9Cpv9+PzOdYs8Wla&D0keI{*=-D7EDUN=?YHf9j;Q1{vVAWjf|j|pv+DA!{ujn3hxOMje;DBa^gUyV2ll~v- zsI@#*WR|M0|9G8Vi%axc{$Aji=(()zj!#NX@$&Fc<4-2o&C@Qm0pw8HIRw`HA%d>X z1}$-(q|BJ_o%hZ1E%p=W$%ZhVDxB|@aQE)T*Nu~U1p1lZ+OQmHqb`7lsB|M4ioL&^ z$%xOxz-ZW9tyipFdvP&1!Xw8qRm(PV5Sf=M*-esZ!ixR&8y*zSwh)vxAg~ymFmhQ! z_i5Aan%-)lMd26!Iq_m*NU#X8FCvspK8yM@6z$n=Fn~QkUVSyrmIBCwnY5FDWUc@M z0r|~)W^kwwziGSIG?JfPXw!=P%CG-Kl_!>uKt{(2UhQTKmkm5!Z(t2BhuMe$kBzZP zqsk=9By0~N06wJA_%SrFm}YTewaW>CCnN04vkhU_UI3s4q#uMX0K)h+n2$&ph`mCH zg}df%c;1C#l;VHYMoX-^^$2{9G-4D|_DL}($;6-lQT)$gby1iqlgKAGH=U%v`Y;)H z;E^MbcnIoS%DYqdq`ZAQBefA7(gA435FU7IBce4?fB(gc%5d>aH9bM!jt_?r66P_p zIWx$#MZuN#cb)rl3WPcu+;(>;+DBKC7m%INIo%#yJFnOp;v=&*k56j~f(4DD44JUW zTYU1*&Og0a+)XV%uODFab!vTgJ+5CdYmYp(i-Ds2>r|vg9i^m7SR;V?BYcVxW;G3h zDL`kGs}{bxH{VxiX(g@<_>viE@$7%)0$vz%&=huGt3(@2^AUkN*b=YS8UF{WY)eZom4# zr)fD^tw1E6@c>T!d=8`VYmVk3Kh%t z?FDI{H&zH>C|42F20{i0CnTtg2F>0hEoR29OwIix-PFuu0IK)3a|2w+Y(K!!$%*LH zl}#DOEkGLjfd2_mcan#0(>E((gYCS~hRDCKK|-J?lc2 zC;qdw&6IWkZ~A=JYuB%Tz~}x9abwP0p#+Jy&zFKX<^N9wAl$I!+-jj-l9FnDp9>ML zgVN8^oCC@2dy$eF6-F4>WF?ZQ914&L6&Lv(aSJSNRrbNR&zPqmk_KPNwhcnm_Ar@h z!jm8l5ZZkkmE!oDH`maHkSV1u7tCc3BC;^ay)*ifH6XeI-=?5KzIFcUKvcex>K<%kCC|*gC~Srn)jh)E*>M z^Y^)PjJ+N)_=CfAGyDn67LWBzK*mq1u{9w{r7k~a-ms_D*P>4jxcU=pKz~jix zK?7ld+ypt)Sr~&Oqm~{8`4!k*ochk44a!pLt=L#AXZFwhCZKCRLo2Z}TPPF(Dtb@f z0J^lnfZu!C9@%OCaz{ZVFOx z%Esko8w%vB3CXc-)GpOw2M;%f& zgRcSwBOkRpVZ&h|56FstM~IW^J+k)kW)WYZ4d#DEIshm+a{lI7X=34z*;rf80#Dc%46+AhlBpC-STc4d{(jV2waJ4wBCKK;u3u7gcl#Lb3PaE{_(TCns zcm8?i-x`Sn5qE@5mIEonRhvQKqt4(0-wp?M9z_%20iXGkIX!A{xeO?_3sj=yxsv_W3q*?Qfzk~us9e4(?%_TeuyVKb~q!OxD1uE1rTas4$QNXL=G0E=>DBYRjS^%DhWq*m~ z5I?$~Vj-|ap+k42-BNgtR|L~NQQDfT6r#^0C4A+zY|K6Gfh$ z!-K)jTr?k8l_&OTJpR3`Z0(Yp*RI{q%d5KW#yT32-sja5AxMXwN`G`c3iJYPuwe-^ z%3m(r^6o?W3=Qx5RUW5j_c8rHEr6F-3tLQrbEITehesAn*I7w#H8~oJSqCV;;7`f_ z{aZVvUH{F~*C!;G_*6uVsO{NfmFOvelo^Mo1z_lZU=1Rtaw?=dXd@o`N_=dN8p5ti4WTOe|_AyCs3B3i>-cxZ?rCr({?Nx(8f7Du9d z#qwSj(J1rS7&(~zi|io5I#RQ(w!Hx55sQGA2b$>yf&UxGiyM|y#7r>Ae+z;&o}n7C%S8t5n=(OOce1CgO$m(R^U75EcwiiLvcFSX#%rD7b&%OAdnRmK+XAYnO zqQeO>$A%lXR#lCbsa{C$PTuf3jsY5mlN4_Ev`du9k!HN!p}*F8G#oRG5Naez@sA%1 zz!4U{`9ZiAXk<=KATGi3ANMFOBsp14_+DR>6b8@ujA`ez{LQ3MG$UB`SWcyOZa#(E zaTqv;PL^^#2}T#Tmm!~74EdWYbR=C4{+do^FK!wPt#IMDtuHL(E$E>l@#-SFpi1N$ zyjXQ{+X0HNC65zIcR*=@mM}kUj#x#8=Lj>K%Cel<$d^`;Cks-Jif|+{wm97;z`Ev zzI~pasq=1CI#+HbT{jIF)o%^?6iotco+Z1xxA5r{LqSux_58sX7@Z~HmhVeT zwW%f}m);t^F}ZOPNUgzEO;M_$OA=3kY`>?R+yag()?kC>+HE1-|u3}c7 z4@vW zXn1sJ9z(e3LGcAcDRO^0yrhuJwgTwq%7}G*uLa&+|NN#q-UVobe?R)epgTY4*cq|+ zbOYtFfRf9u`!ZN2z;D3hgJ44%v16uAJpn6E-`70o%f9N4#hVLviV^A|s9u2rT_Ucv z8~nF&VDH{(By)At$)bkl_Sdh{N_+gRQvSbq*Jr&43^@6%=rY?|+HE7_=?rv2f3Dc_ zp8_cO~5c=~$`>d4EFINu#ORUVk%VN#4`_TYvobPit-c_O-8Y`nUV^l(5)ZS@~ig zgRotE4WaZ;OG_&se$|g!OkAuWYq7xZqM~sqJSu#RC!bcFr>Czkq!=_-DZI|!yqn}l z@L-UkdZcI;364X_HwwlVaxV;IwN`Pq-%x}3G}5vKNgj);R3u8|vPd0v`D*;fo$&ct zcpGxwT`R3P^h7u)YZ^VjMf}Sf5)Q6307Gso%l~N?yu^FHsGI%U4Nm(-64|Ey3#~P%kHJM3Y3>%5Y{b z6@x+!Zx$Kx;X&@~9@4io^7OvrR(B+17@i(e-aHL)Dd>*tmz|N_P8$yfiJ^T(2(1`# zU-!Hwi|YA4!{~lL*O=qi_kmFLPO$9`j_X{qJlCs8wC(3%qQGH2QL<-k-WQjTpsIBI zx!B8blto7m9d>l$=`rTbu86E>ic?8jV5edfAfkps{ZxPZc4K2B^g(CtNY^iSZ;(s0 zM4^IsAIQfO0LIKbedX@fO4h;#(pers!&AK&Gv&=u_1c#v31aL(gmC#gkKSYoxWj_(R z$eIy!E z&`ybz3Jff6(?@+&O%HewgouWP%^`;KkXo;%{zUvjd=rq4sFKJ*IZ4BKkQ4pudkq@& zgpvi-Xr**@uReX|QYHL6V{`!-PV)|OmYsD};q%*t!eXuhu zkD>NHiXKaR_fL>tU8H!tNUrYVt}??lw}*{C?(wp!@~87-pUpqqvUjZgKIKPM8fB3q z_x@ilCeFj&T+RO&g0ROvZ#5h>1AlHI(DU>=9=xy~I~`}yMO?lLkS#lpOa5A}wPVNBQ=0Ch z<4;apcj0EqPkwip{kp?OmtHM$vT7HeaFq?H|IS?F3dYqGVzzITG&C%ARhqG}>wKa% zhC#z)c1+?5J)HnwV|gSJJ7yT-HaQ*?>Nz^2{@56&?+?=AZBND9!^4MOA_?N}4TUyHhiY&0M=)DNAGLD*E;3l}cvGEJ1<%DK2e?!4Zme}$;&-{Au$ z=f-;n%8<@JduU}n`<;DvoG?KO^uBGqgoI-5UETTfMRNy*=or%ukF&Dqa-W7+?*NcC`F#(X z{)7SzP9u=DVDljCk!o0ebA3PI8htB);m~A@9oS*^5X4cuZZIXaZAZ_}*zI}sV7I++CLCrd)El}aT``NKcgU@0Xv4T?|7KVH2J-Y}; zk#iR=ln_*Kf@sI|r4b`j@EqDp=oyBCQ^l^l+}vlFG$ADGNTU>N_Cr36BkGrP=g+?f zT^HdphrSre?fj`7y7X^LP7~E_yeK$!W%@>P8)bd__H8xwY8;F59|rfM?`|$Tn45$s zP`19{_3KOkyogQ7+!x)e-!gfkG=#QL8}a*G7q+s=c=Q$|QJwcroj?Y=*xdYRZW5j} zK@7{vgz?iUWY^pI(T(g7;b1xl+nG$XfkS_vWK_pU9NTP)n5wY29NuHzYgAej(}K#_ zgef;+c#K~|wXKcg$aQj4HV7H=Uq*(}>t^W6h=%@r{Hxhi?mBLi$>+zuBYiR~FGA7m z-*{})Ql+VBfXJqQW-u_|@w<1Hzu(-n%bW9I?5}zzfi3Timr7g_?L4cwwR_Yzllu0r ziJ3lb%bDU-J|$<7YgXScbpe@D zWp3flmefZFvXY=s;^^z|Q7)f8ru-;${>;@EzX-x`>_=v_ilV{&(6UG`Rb z%3kvk6w=4!lbLImwHZ-bq{->;-=O;L5Z@^?G%U;}`?Thw0?lE=W={gu%bCQvm{i*s zkomi2`NbhMvn}=Yeak~w*t~=%R4dwrbP2C>Teel$(&n!{#)FA$2Eb92`^udc;n1P$ zK#ONTaZxqEw{BX_uEHDZF6RFsUGL-IPO6t#7*7&cw3(b~Xh9KsDBjm=%U5BE-RfDU zCc1;y^qQ{bYuYFOr)vKuA-41G#{~sT4%Hmdy-f)wef2$eXa)}6Hr$+MubW8-#&L;Q z58#L{X5G>nPcojaQRC;YkMkCjQ+hQ1!T87F4Y9jNJ<07Qw`|x^BPO3Kv>5_Da=M_g ziGGcUnrc!)vwx407+O9Jb@Gh{rQC_9G+R=_-^1`0{Zss)&1;e-8?f3qXzmQgbR28_ z;}&3c%0^wBKaa=6#9ZcsP4y1Z9m%=*Lz6=0(b3CQgn8b2B%E>d4rdTRrQ(aYeLG#X z2B29?WwM%B)<69}Z{G+4(EI9T*VoKnjOAPSY)<(o+W9ji85odSmm4t?)!^vi!^5Jj zl9L=aM488&_{O0Py6M*R_x5Ah()KfZfZ!Qz>10Ar>UtKP53U$z(1#x;F-QR zsg+Mejnrh2zYW4)2uZbQr!rjS1C#F?t44&>hM0Ii8Imb|1 z;Pj84IB_X{BO-vykWX4goS*2>ZozY#DnH}=^_WGgSeD2ZIB=8Z!&3!~Uw0LB9Fp$vX71ey->sUanPYL7wTq_kafJ2oWk+S&O85wkz@_4EQUpPp0B1-AS~rSN_%U$(u0lPagL?EDQ!q1D$JO{|8;J|=jlf=4QK(PVOrs4?$zId7)+l!y@{x^6!3Td? zmdRZrkxJZAEhVxU6jGo1;$E25#D+*kL*lWGp{0>K)(v($tqHDk2*60Xf@|t1f^Pp5{3OfMQnASHV9=6~PMf&KO%rwr#4713*kK4n#z2-{sEM)O# zlg8x#S#)`sC5&cfdXW1Q$xK2`sYAZ;n?sYvwkk;93VHv2ye;a}C1%6UmsD`g2wqn7 zPw<~rlN5db=2aQxwkwUxih<|MM=Q9pvktb{bYeYTn-*AfVE~-J(CU+`i(Ww&d zh8Bbzd4WwccY*XR(#^|%YBp1wJrkjQEO+t>GI}kbxgLN_=C6zL^UZ!+TwZ#sD{xN# zL;qSewRKLOKd&fMucDQhmbypaDMHuK`p0CZuO^H-oY{TQAb&va)<0_aSBMVM+5Ap? zDc-vq%yQ}m*T7AmSH2PRomuvgOaT+^B=p)zI1Xf6qo3{4Z1^@FVUH}Rj$CeDUS)0N zDwxUst_LMK4W!Jc91Ag=0=;bSx8K{_cN#GWqHmV%+&ObfsNjNbHUgjEox2bDg>==d z13`@YpuJtSF&Wb447ZThGzEpWB7xzGKJs+@jkvg;{;6;8?ObGHGuHt>*v<@L40&mc zF7GeeLX|QfVUh$P&?iY6toKsbqKjfmWM3x!6gcn_?YVHm?|}ja2X~X+nfQxznfiGm zp#1sY%>yRb4lzB^c2l}z(Y>9@c14Jsdq25rLIL0bz-MmTP+L2MEPrLGXBl^{*oXyF zybGqM|JqX5SBzB;=~IZUB50TX$6Xj7U!(l-uWTjy9j|RJGYl7VOKJUFY$HZNKvqas z0Kv->wsT`e2>*TFLcs|gObXM~KQ;@68wj6P1F4cq!-(?M*tl-9i{1 zNxIj`!Avg{x!8RYL62?XbStTT`iyN~nnt}0bDjGK99eM2R7YRVM*rx@s6SqYgX_vZ zv%~d!<(JYt(-E9PhXR=VS$sJGFWg%>!2_Cp1+i3XC+XX8PSsPht!_Gs&i`UW@{KDN*@RXZgSTO zu-EuBH;4e-`hTcgv^&ahRaDGtcmZ7 zZg+4tF-Fk2{&y6IIoB2rSEY}77Rv8|IduX)c$vWZHQ0ZHR_%Zn(qh}_hC#m(Cv>_+ zjymD{(zi2BP7!sypZ25H&2xTt67S^lX%zsD69#*=T;JXrhJ4EuvW1+x+c#3Ux2LgY zCQcAiiAB87ARf{jasJ7&u|AUFn~wQe`V#As>2CEhq7cGAbAg7(1B9r&zPlgO zTaFA*09*DiT)k#Z3iY`}4T09ZRP*kD5@yqW2%d{_!8Ymz1b`*Wk0EBD!(4FYV~Rl5|fTx5bHrCjO~h224D5__m@@lR~e?6 zuFe3t30Ey-(b}s|o3`$ovSzm*d{>=q50=%HE$HWP>cjLqyZn&scov$ZWph49+ZorB zHb{wT9640y8;$<{{yJ{JmBCM?MdOu()uv7h)l;*b~#~0G<}Y!P`0|!EhEf8m2*65g}uFffc-jZdI0Rb&qkBA zoXd}*d&r9`+gGX8x`-!L=W6>BMvH`yI;#m~YxbfTVNZLcvl)tln+6F1P~X15rR5*h z`OKbVwm)|Oqr|5j7%gliRi~(!TQmg7q&3LXaw!Uz0-koj5$BET)*WC+NU2_Gq?MH( z;8{u1?)6jM#TB!~_%nGxz>&*WEM|SdUMoU7?~rIOQ3yLmR&xjvu44t*uD<@6;Un)f zU8xF(MtB53;2|(shHNEJ@IT{(RTaZqurxa%?Kj@W19@AT6e`AwrUt9V2DSf22&>diLqxP z+yvUO-oe3|XGa#+d$?spw^@dUU1>E`h+lNs!ninl9@|?YdPKm}hxkn<`x;EvtPMkB zwn6wCB<^bL&my)Tb1K~Tg3DO~(DO^lC5eX_tyLp!;ECW%@A=WVXX`CB>sCrA`(}~w zOx&=D1-+pc?LxgL1r-Q=(y0Mwn_YI0*gI&G3$#!sE z%4Eq;GPV_HEc@RQoe8 z$-4&e$m%X#lVt*fQWJS&I34XHUDR|d)4jt-C{3j&&CZ~tbLY-78=^0z91jcg=WV4O zxJ_>K=~cSe!lGjv3(pF6U_@ySRE1vBD@#g@m>~5V*7o*N{F?QBH6~3sG;`t))R#5J zsFTck8GVjJ%FI#P^X{O{h7AX)&}npOGjoI1lfR2!XifM(EkIorN;%~v`6Czb?2!K- zv8=5N=oKgR!L5>7Ek|kvP0E3;eiRkbdZ8|-v&lPqRp-2{o9$$F2~F=j-`52N{b>bB z(eVQU5krkHTT~AX-0@BbrFAMW-XRAH8~xa*yEvRfaaDc5Df4_;ht65iU2b7w$519_ zNTnv-(T`i?r&pM&B-0S>`;~MrWv?6vb8e>Z=lYu@DtfZNbti*wcE<{W63+nBGp1+V zhySY@RXl6;aA?t^@dZi&cO`Qy8Ild8Gkku@^P_*)UcU3CJNdwU__tQ2N`XMfGO%l@ zyua}UgMKlG2K4W*a!Dl@Dm+uV#{>p?_LZ5xV8L8lTiZR#UV@q6M{gp9y}G*k5Xf|% zD~^PDdRiv>`ua)9%uVTCd&hxtnB_S3;|KT4NXMRKygUi&(E37dOzFSV8Y%GW(t@|d z&OC|6fu0^~2Zt_PBA7XLg^#B)8uiRV8yTkJ{BcV4ecQEd+p)@8xbkr$^lQ2C0_a55 zJK}MR8&6CzwrM72wGp-jW~&c$x2->DNef|2z3Q&)V7b$(k;D`0@A3gl2jx6{YM6VI z=qz*N{AknYC$7RR@ICjaWYdrB3>mUL6*@Fcl?(Q2tce{Ak4^m_!)U zWMpNPl#BqRT&daR>z*2)s$bb-_%VXnCoVrL}heo}fpwFof zaya=bjXkSC5Pw!xwF#J}mViJg^G8e#15MVF%$QoV!)3cO;P`zOLU`*5XCqkc1dQC^ zUiOgN({?uyrFt5on9<+LW?OE(srIhVFP};#f6nQWnQk<&ILW;^DOb8ry}G;1gzt%^ zH<@M0YsY6O)_*Uv9;UP?=>o_Kux#@6d*mHNw49Xv=$X$=PaT(2i{9r_THq!Q)i zgFcbFh!(GBmMESA7rWw6pwSt4Q^;~epY)LJ;fNXJ&Tp0zod9*eZ2I==hkmk?=b%fg z+1CpjntczvTleLv`e~y+MK7zw9nT)nYB;<=!+1$&t>Z8_XVA@gA3D@QA}*#eR!Ps* zUtVD|2Yooa;AfjnR&9P&m3AI$xfl|`-%Unt_SRJizX0|{S5Rx6JQuAV*N;}5?yN5-<1Cm9H@!6cfj zaiZDBL?iB(h!mrdLF(uKIvX$hsI$e_WbLBe6^?Wi6u<8sI;(0j#&pv^uXaFR5E<_IQWGPO=TE4X&d4 zu0$yV#3WepI*rr5m)N|$52Q~0{h2E%!hUc48V3h|^3@gS^00N>5BUXVV`ZDJY%=%c z4`1SR#5N`a8p`J<{`OqVx{l84%QoOL+gw| z%0kH^NMYATHc?v%I__D(x>6J8IM^;7YPv{Phc{Au@|7xF8XZ#?iF4H_|MKSeuw)Ca z$DB8>shDjIN}hkLy~IUv{_%gtp$jAI9@p>;1;B94IdRgKiDSf6Z`12%w)PAMl)0R06*`8EIni31f#&b9ef-7Jf{{4k*h9M>EHUaSVWhmW? zv2AW&)oTeKf#EU-6b#)ArZX>O4i?yep6w;Kt+FVc5jF2LPDZpPstKA+wMZAX^xs#a zqEKKe^}J^FUxL{~W)z3GF5#O9+al64lz~f<H%b?j2(OM+G}3c0!AtuOKyNyeZ=M29cx zolKUUUr@RC)Kp)$8&QKVA3T1%9~;tx2;U~7@roD)I%VkMO0DFT%K+a*+lNF8$uY6CSAZp#8I+giTxYLoXCCOgQqq|MY1;dIGm+X7{6<^FEEYsvJK7=E~|c z34-`uQ2afcK7IKjMG*s7^pG}GtKcpO_UNE7H0exE*s){B-E@%%E*2K;fcd0$yCI-b zXpiK6c?6%WhUDjb8{@1JcPrnrE`&YtQ?E&wdReFS)J1?^#~OPYR~`!r5@ytr-GPBU zqKk0?dIzc|LB!HC=oaq$p+wHHV|!qur@H&ixdpjhQY$qadPgmKVHNAVcbxC4u!3(^ zV=ZUt;)HQKDM1GWc3!rI&Sdh?ZF^~aJA+VFMW2vcU_GL>DT3X| z4bmQ>yvd_Y&DPJ+_7;~t*U!{^m>qr~eJC)!Xc3bLqUSyp-;E`sj4ZjER;*kpe_2xVw{CN0 zp~|0pn;}QFJ8DHl^@pX43GsF&PXFy6VW-2p+Fi3+KJ9!4BfSBn>tD;yVId;ioY>w z`U<}tadY${Spm`g`+11Wy-!a&Mlec!oT}KP2eS*_z7@u4IV+$2(hCLaYcKJW(`>re z`(;yKizQ2>4xKjjrHrnuT;gz5y=jC^_?qPhulBB~%%XgSZ0}S7irjzXp1jml9b|PX zR9Iaq`(jF&Li-PJu@E=WquW&shJfbwcHt@lTrf#;IMA?8_`eT7bk1ry;zI5xZ8W8^ zQ69sCT|FlpH`q_Km97vi2@-SyU13PWr>m!e|l_(Mhn!M&IF5SgFllVH4gXv?Q+W0h6pVB8zpB*ek{nR z_ayD(i$9>YwG}|muNrU8P^Vk;$!)zOwTX~mHh5lGa&yyCUEOw@lIM@Y&XQ(S` zA>g-vdu42=siM(;@&-SO$qyVB(I3wpe*C^(s>!>V{vw<2aYMF`6oa!w3G{43;mw%x zTh7*^+z@OA`Im1~5-aHHmRwsRZM^iPPRD;hxj4t=)Ji=r4PwN2PZE=Hj>Mwh;nC=@ zx$sBE<22(%RwWU=E!_tX?h2cW>rR2u&|LunySg-x(q)8;aWgGEIj+NLkCJi_J8+oFmld}nr4i{BX0gZXSucE zNCd|+BFloM3|tmw@MZJKkdWc_|23B{(pjWhFCz;U_=rn4;KT_jVZ89_+EA;AGl>YD zcs=U(c6pG&bq`;{f|Oob*epfy-ctn}*0)YTrCNvij10RbCjj zMmAF5XO1l^Ejn81TFeG}^X_QIA#qxY{W6`s5hXG(6HP1*A%Rxwp& zyQvyvp6$ykVb6|ct`7PX%~+?eFLfMfHqk5e^zxeb@rqtGi*+$LH9ufnTe}inGPXxP}{xzZ-U`J~Zk0G+kKVdAZZUBhPDk1x!oMeet3< zp#+8S{?ZEZS%1GjyP%})5dAm-)*~exYHCdR7;KpwKf?UoU4}xPt6+%6SHu4a}#zSlYFD2L{(N=DI}vptKE zf)k2b_xq)~z4dKrHQk{fuTNQk5RQ`Na_niiNj`yMeDJ8AT+Dlkz?~|HOfa;1zQcGT z>Q;cJVz^Rpyg<7D{d_*g*jKgo)3HBl@tWnI=A?^R8j*{yUwGC8Ls|)b@E-J5a>l80 z{SHma{a!b^4W15$m{_KmP4|$Z!*(#F@T^Nyg{Do`lETm^Y08{bx5Qs=X5yeR`oAb7~&X0aB^1tZd zZKS*-FRl0kk$_~Cp>@B;{;r#z$xMX42=M6p9(g&*#uhDEIy4u~`V#gjEw|-~ch;ao zMZ~cW?b8Y%)F9P4`}gl>9){kkWLWM_K50bDRI=KM8)!IdjpqOovB3hN1eQ}sH9b<^ z;MPhqdw=IWfQ8T4U(x3($Sg)HICSYZPCIwfo$2lwRtfVKEI9W2l@Jrd0ZTTGIn{+B zu+gi&oLHGZxlVt7mhL_EsHc?F6V1nxb^8BIl*|NI%U4Ql*}#9>#jvRoch(sGZYIUh z`iS)VrOI0|GQgt{Dt%#tC8*jR^UTpMLORN=3#HTn_J^%K8?HD0-60imK$lxIA<3!y z{dr+<5q_bETvz{P@F2-4fwz0P(JsnNSr?` z5VtW_&FhMZa^3yR`N{yv*H9T6$r z!Pt7q&Not4Rh8y5n--#TvStfu6+U{@mak`Ca7+ErB9+ngJ4-Wq_!pMSZWx}$By$0X z+;BUmk?vX9H%!BA$%lmLlAP&R@C5Vz6rH)0qS0~M#EG(0+|Lt!KxzvM3<+fKiK}0f zd0%7eRXvqAML*n%;L=v}A1Hc_nG`g*pMSZE20*PXPLn9EVTiR*2j(`xYT$>VvnP68 zR338GkdTm$6>x{L(7}sdH*9USC|!To1{irGpx1CW z9uke5RyNJxGj;~ujc-1Im+gU42!#m^<_CI)cYe?Qq7!m1%P+1bM!#?(EH9>e;{u9v zEwfytamKf%7XgeSAf60JiRoPtlX5;&SK&_}2{`E|pmx1ZPlR}yWKYPvL2^XjeuTbL z>Yk*cJ~KDi#DQ6MtRVGY$-}cheVA(QJx?FnAn+Y&eQxX)m!?|sYHigSjnKR5+rNiq zHBrmcX*!E>C=yXfVv%~uJpK6%`f_~DKNld~&1zdNBo&}CJr&<1b>qgts>sx0)I4K8 zCv|TC#M0jnxAh7a37RR)$nnpgq_XXpP1h1D!MuQ6QYd@YN!gy&cmL4&%1c2TuamK72>rVM9<#}Z*`-a z3!J0_bCe?fZ1yXkE$$M6mvFvOX+WLR6G^C6A;V5s!zV9AKXf@-pJ#^p`pC`iW|?U* zzgTfpT!bzCo47OHDFA5HNbx-bS|!Z&a(8i5dX1VR{A%cuTv)Na00={9Uph#H5esoB z_s?j0ZvXuR4n~NrYvH(+ewL}rpBqQ2LUYuG@RdN82-F0L2d9^MCd{Y5ZbMq1kOETY%|Hkh?qy6 zBwS81isuk^MTU(J3k&N4*cLwX5tp|S(uSDtZTLEYSH}8OHFF#Nx9EYGe!64+(Z_i3 z`;j0^S8#<1sZ3> zWH=sFd163zbnOq?z&EbEe!=&Ay$oT4iHZy*YCrS)rCCI|o+_Dk^^>7{_0nDFOHERE z4~L!U)V^IiYx?fECuZSQK-?840b6rHs)IXg+1slCckIJBXx7K9S-z#cuIzr3`SHiX z>Zi|s*Cl1-0nh-t&USY1(6+6x84_}T(WxNTitx%)dB|CN|M{~I*QlUZh@3CKT(>$_ zCOU6Epy_BIu(O9X9V)f>?KlD@(Qb{{ZvamF4~S1f%_^+RXKsiHd;ap4xV@|eO4UH~ z100MB`VmhSkYVS=>=ax~sKH~bj|S0|>^ zQA~=a252$ipigt98hy_TpIhZg?cJXsSO1fC4&TxD8|vzYq98@&lYq{`G3R+!mcGq6 zSIQP)KNeo`@uR1BEeKmKH+U=2FM@vy#kI6w-?3VoTtzQC*|hSq$az!*pfC#V7v{yp zNs9pw7T4Ew(Woo!CAkmQuST~ayT3gzBDyc|A{H#92$wUzyS!lhcy8*n)aM7u_JV2%#kd_D zvhS2l3g{-pwIHUQ2JO1|Pemjm$1bLKclxk^c9us#2>-YvKT$PMDfl#*3)%#a!kInX z1|l8WE#Ph_Qpu)XLutv{{H@@IG{H*H^q(Su;g{)7ibfM8{FC`q`Wi1t;&b4~=^_#y z&4M3?kQV(s0ARQs4G_q8{r0afx?rYhU$b~wy6Th1kFPPVhPLKT?kB0t=%d^NtS=4k z0BJ1jl)7N~$lA+KoR~i(*8U*^YI0v4j?Y-E16#L1u#J36M@Pr|JOK|EH&Sxv-{n0p zH0y}8_Ag5$*-edLfBDL7GAHX+Wk@A39+G2krLo{YAtc3_WH-|YJ&TmH60W9IM%cV7 zpQ{DFZ`#*y^_QbMbr(R=Zh>BANr&;$C=%^b zLX{t|rI^u9$f9li<}uAQ``XG>De#|56>Hb8_d{mC0$j0wuPAB@5B5Riz0;!O2|=}p zMSQFg71wF*!*yeFQ4vfb7H((b?wFtA8w%2xW)&f<3I1dJXF9E>ofrZ#%hffpps}!^ zAccQVy|8GB(*<&hx=U{Wm%3Ah^oTjdsP|6LV>i+{AAQ8Yv73j+k!Wcl{4{N9reuw! zRXe68Is-Ya^HtR5&YadRO1b)B|Sdp)y5ZR086%PnbC@75rG|v>Y6*SX}z` z!?#Uyn<3XO{6~$>DPrj=(uSa*nfCiZHmSPfFO-2Vr!Vr&MO0=tQb^bos=*x|kxW0oqT4}&bj||< zQXE|r?2?ht{I@7J4@E4L%;9)9jP=-tVkURoA?(NKiP($(eecNDA9zP6A$`3G2XNqp z^kUnys54Lzk3doFL*{UA&_y98>i|H4h4g-wc!0uwP=BEfxpwoWdBX{XTu;22q79Cc zcgzsDD9{JE{>+M9JXtSxGs2$!_6kq!^1VJUp;Vii{l-&@jywDY^5nQlGRWp{#*g>nMOx8jNKB*I-1!b%f)3 zQZ0KmtdB}refJ*5m8Y^t(>Zj+uH^?8(YN1^#Mr(jC(&hd z_=!@=BH^*&FBa;`@I5^wVo60S45AAATQQdRrtWI27Oe8D&(ljNfLBpjJ6C^^Wom82 z*xnM#n^`#3C{zdE!aGOgeV?00Stj=7&^_Qzef~83#iuDRs#mp@@VLZH2U>s8DcMEb zU~pl+zFi{akr%^_`EQQd{bqGOsRs;t^vtT5-{I;lJwUo^U436n?%75Jnjl9ES8CsN zQvb1zVY9*_LI|u$>P7Si4EnnR_U^rYy=te~_jM?%MR;T%ln^=Cg`G&Bt$_?Ve_|M#4uvby z0Qn^o=lXKDti%dajL8LZ75y+bd^F^Iv7;05l5}<^@@{B=%TD>R1iG_?*pAZD57}9w zEtV1%we3Sf10xH>&<6T$o?+u}EITAhXgavoh5QR2BB3~#sN~q&tbUhODWj7gi3VCA z@AnkLA_u{F7Ws&h>l%U1At5;&dilKH=xjM~SdssOiFyivgyt3O%S3yx$Pv&GqL%G| zte~7EBJkvyjocBa!^3%D^-ZN6sRq1|PNXo-1HI*bCdrYX)L8y#Wya?eOlPRNQ){Mm zij0YVU~5hv*Vq{_Q0(LmnI8%s`rS(d?~1m-g~BG=5rgxgJ@V*}0A-oNS@$NoM82I% zMlD3{bmsLzL&V9RNq)_ZX$1Gfbd5HYb|ZV|Cy6qa-dhN&5^@cZU;y#?bMJ)4$JR6PzFM(y;~`;^PXH31n}rk7doeN^LS7LD z?aJ6GKE9d`mtOaD>ntLpfIGxV7P%a|{XM`jxu_>ki|JNkz65bq*~(^%jYyh+a#ny< zBWscqWIg4PWG3=Hp*T>VHfO;emg--o_qgDu!vV5xY3WmU?%o}s9&uu}%ZGqQ&Q(+4g zIgi{6;3}!ci74*HwIt9}VciD>3L-0Sp)!M6Ms^w=o1IT%Wj{-2t zBW{Ksefo4D|2K^^YK4rC{Wgt~<6Qpqp|hF9J6QNtiN`Mz48E+#xmPtV2dgXlzJ`aU z)up8f!t*{9B@?O^k=)}P-vRCumA&vy7DK|t-I4z_4H!zWy|52LdM7qLP}5bbDo(jxqo`LH>MR}uS10__kNhbj7@I?}39 z0NV}4b^F9W<7c=g!Or>{7{(yM-tQ4w@#U)&CyrA{)ejXD(w3=dpZj-XY3OVGuzKgG zD~(WRE?(M_GekZE;jR7u`HAS3O9_XE03UM|VYY?Q`S)htoqhB|O^0Lo>sd(q>wal5 zc<}%8qq`vP`t`BQ$O#wvF|fM6YXAQiA2(s)h%iAQ4eH=l46HT%{yx$>Fmv_( zu6y`DrM$~F|L;#Cnm^zJK}2mO;V#|>8{DEy&Yf(j%+NIPGhe~1BtAY~W&>El=TUuI z{#ktjeeta&443LWuv4p^D_pf_V?hs%qctor2&7X!m9I^0%pCkO#)7^pM&4jXI zWWAF6;FiHPyl(v~QF(|;R&t-$0N~0yM2MPS0)mMe(~za6K)OdfOblg#F~;6wu;U$}4pY znX*}l>RKWJ^4v|&l{ktKDPlP4`$_4GaLJddJLmrI<(7p!(reik?S(ITi^>dT!1`j_ zjuKj7aKKZQ+$R$nDjpK+CcD+WsZM$Ko2uIYax8SqHfNb!0lb8Zh~x~=N(i3d|K2he z%Y^sa-c@hn_slqsPECJZB*LUnWO{8{hP--mj$1x}X*og;F&Tq)w~6*1LLa3r-&#VL z#9F(PgbX1wes>g z!U>}CU6NZ1I}thj?xv4Y07`9yCUg(Fx0YxXk@85vu;s;6w!m6z1p|W2Z8LZFFK9`n zHyS&BN46R^R6Zi&JB2|?-JAUUd~v@1{m3Q}s-+GryJ4Hty1AjK`QLKgO#J!*?VFa( z8eY~0#gx+j?Io}O_q73vjj=J7>}JZR^}eBjG?Cqh_7Hx8;+(ZClTJP@aGr%|=Ekj1 zXjv})%X7@AQC-Nhdlx_v?~fgOk@4ql>r1a5+E{q=Ny}k07B&iETB<}$LZj6<|GxON z#?~K#g*gBZZVTzrLr^g>DVNOe*|YZnp_C;L(6vj%DcE$wLSEkF3-6w=t)joA80c;; zNMXyjYx&~F@0W10k!7_SGI`HMf`3bp?)-rEXOQ4ll4S{wt(dIR@=tw*XOc)6B)gFs zpP>ozUqqK0a9F+N9Ayk42?YaPTdtX+%mltAkE^2}QrB*&ej5bJacH_`w7rBF#bE^SJj#hM{603RE_rTf1{G~Gg=JPY6~ zDnE%Bkt;+2d(LuRB1%{mF{V2JZ7Vi+?Df{r0^_z={J-^IL-F&9>4Or{@g33WHAa zhOu3E1$WYYTLMGm;%h5fwVY>0r;Y0MZ>fmQXMg>CKO5KKE)tQv=2S!O)~g4TtWd|b zk%-PvM&*bHZfwl?)vnrrD+(eXK4sua#ee%;K1+Gpw1(!`?PWO=i+q~XtzNJ0u{fZ+ z%=|)+A&>k`^o$xd?K|#km)?2F*O7N@?QUH9YU^c^m(!(sU9`=Te8acV+kD@sCCE?k z=pCb|8{Bqi(3{d4uZ>sx+oc7)S#Y%MLugZAw%eF7Wt+P`xmQ!|BJQ063}_WTU(eRa zbE|o~)=@>WI|m-@97sxZLV^Y5VLcd@86FmASG5A3l}HjEQ@_Qd{?f}WYWg#Iut!Hp zfWpVp(&4mUqNzRV<>e&?OA##l4U(3uxQ0diOj`4gDcoRKG$O2vPzf`uAL)gP7u&tg z#o5^w%kZbC)BOirB=_wb(owP^J~45+KTLq8X0Huk*)r-aZ$0xy9PriJ7W1ARjJfDy z*j5rLa*nXnU0x5vDNZBLoVj0&x$H0-8=KEO-R(_6N@h~rR&u8QS>BhQ;W?z;6EPRl z6Uj{Vqh1U8hVqW*s96?MCw^wd0eqG`iMpYjDPWd$51GM&aHsC#N2fLK^ z5N;JgM~)u7fbHynmY2Yh-a@nS_hAJ)lA|vFpRSHIyZDnWOr-7^v| z8mR&_d4>K&^#$+$7VqmeVN1EuzJ;7l2k-*%mjkXE_eoVBuc- zP)3RumP@h@nFtlv_w7eJD*&k@_m?U82*FrXtz? z{q1?kk`-qbWtE0KiinVB^x9k;4@~=Q(`s4e_JsI6K({c>=8t3C|9_Od2UJw&7WRMC z7!x&m6MHviP*AL(0@lQ696+Sm5W!x+0thNtAjyqhyN=iptRT&uR7+{TebC{<<@tm~RzbE3l8^r&~swYCzSCW9l@1PE`JNaBU?ri7RGi zmQhY36EU0DwNRA3eLVZW4GrF%1d?7v;OQg+Ny29PJjPH?33`2kdfp( z_+ZV7vipy=4IeDuHMH)g?c0M%l+#}>QX^RW@)R6QQj?pR(-ofrYfH<$NNUa|#nJ?l zOvcVX6HWo{>n?TVciC9%_UA~8rm%ssA3xrQ!wMq;k%w*TBKt@tw$S5jrXj^IrW^St zpy%DmijUhu(hZN?q~SP9O(1ZtHL;G)TM6&8f#NIXCi3O=!^b z++LopnWp`+H$T_+SA6oVL5Y{YOP`gn>WQ6xb2+Pw7?WG`U4_IO8k!~z`~eKl4hEg) z7nGP#bBiY*(;3FS4^Eh+=IN_fN5CAyLHTAx4CG6E>h~Bj#F0lDuz&ylbsPS0D`03( zSJc#CS8Bu@<%>?qB<&70I%k%dQ=kWY${FjIn}(f<8vAl)Z<9ibMn5iS)Ue zKuA*%#MQ72w2!RW$aMcJr6+M_b5N+(1afrF7bTe`Sj#!^3G3d!|2XdK*ocko4+Ao^ zYF97&>5;RqlE6TkK@?ZtJ2vIdSLaI{5Wh&0?Qufpq!qu)xIZDYh}H>M*du;#qP=!o zGl8mq{`q&+92}b1zW#?une8H7_MJQx1sW{?d{k7FcD4Tk3F^vWpOA^1^&N6ZJ(t>8 zRo`5-zsAx%gFD1Rd>amcDrGSv?ZmtEpD zyV+?VyW-sB>z031iq}YpR=K($a#KT5>x6h?*Jspn_ zCJbocPrL}#v~}Htu1DD4B#B!%yPdR$U!Fqz?m-i0d_2@q@o4}H#S(Npq4q2gK?(rg z!|En@*{(OUC$%D$GCz2;tY@NAdI^5lpM!A>*D$;iB&9O;q&PH z((rgXtZ!v7;v%}V*aRzw`(pWBeg~SH!%cuh1{B(%U@spCM$vLd!y`o~Bst-5?9O{d z6xv}R4N**G+yZ~6xCjuw4~X4~igbcA+xa#H>89=5uNdac-kS&&$Nkd_Tn1Un)$7;0 z!J+HCMNK6wq29#`Z*OZWvz!?d-ch|~+r4={tyPb+0OR0-~rHR)5yM|w+d!*DL58fSczym&Md z;#T4YmnAz7XxnJU^yzq~oBYf?sYgAtbj4&fFica(-&^9I4m^%J>9b|A-^#@(4Gmbi z^GK0v0?{tX!%Fuv`d--^_Tl+~jYW5-ZOjF>O>hPiNT!z}gjn9SzM*2Pl$MtI8O++? zu%dK^m_W~RF^&Mo(#u>Yxt8K%dZn*U!k(8@q!=slgJ&&G6&K(H3I1Lj=Nxo$j@)a_ zCB%#}S@>+7e)F{ysMdzx*o9~Avn$@~#qsN?@l2lli>(UtsNEpw3mUg=lv6BaR#8k~ z&H&;qvnT(egzR4N@$GNtQP1MC62@eY>fgUVl}U4hnYS!qP~BYi2%uRH#i4$?M$w&n z4}5)$;|M)9B3dAd+4>#;BvwL=Eu5=e7UgqyT4trp*0n31e-nybNAVyMHHRI8S8TO$ zV8MTg_}o*AIc##q35t7eFL@2>BqN~Y)XJ)wxIX!JVKgKc#(Eyj*}c>P!ob&^al~8K zIh*+>Wp^o_BiI&vD%mp8}TsO(}x*6-S#SRVODnQPsWtUj-=QM6cq!px;BZ5G4xJ$(e_&Ub1^{#(g=c?rdLKQ4A*|t8ib!jJs958$*}!HW=nU+s#%}GT zS>Wn0*cd&|P*@+eXYeGP{y4Sc>v^c;$TcD?!5R;hYi!uicjLy5+G~uaURYR|%67y> zPck(#vl%<~PTy(g&z&=52pFh=8W!!>fpxKZzh-xLJs1=;hq(b~PF(}N4_I*F;UU0- z=Zvg5ms=`H2?G)qo3(-p6H2GX!=%(;Kd!9zWTw!8m|_L%PNBb45Tu{6eekv7s}J#x1!9m~ zhU|9o$Ai}8OCKVGCJ%+EQxo|2-@h>L+qgl%qO+6Wd_*}VkbBICu;WK((u%J0-Q7=- zjrwONAaJt%^91c34ZT`Ij!U>b^W;u)vKSW-20wH$9G1T9`||GbVH_k0y{MOM4G5U$ zK2Qm9lhhKk?3M`hy0HUl)` z*v|Re?!c2i8&s-2G(5f@S^wvA?Ruk}npZ;iqqRPsp{f&(`|#^E|>na(a$_f&QvT?jE)(FI#;$ez%#k(K)xE#hF>Rc1&^U zL?X+SlZ6+KmOogSsMf4tP66W)egn5ywBmvnmgLVZwTEWEEu0lD*YfSRR^!L-z}yFk z0mn_>k?08#jFahy;QEITKESF)PMCj{*N9cxFC_|{_}InBck|}Q`W~ITba94h-Sy>5 zF!a#`5=IRID&niHc`4S~_q#Z+C^`Emm&)x{)hN{dz={kolf1@B&?J>jYUd%so zHszwXHu=t2+@w&{Rb zp$1qG0^=U06-knP9NzxNAAjcR>+{rMQlKq5+g)G2y~kKv+dx=)6ZD(2kJDei`{BB5 zb*bK_&mBuT)vRgLr%$jodo}wjfEE7rgAAHFnKI{Ox-a^o#4?WUUX$3;*#XGM*LUtb zf+(nmxi$+`ZrE8_ZAN_mDCXWw!!`*9!I85_ZZcfPLDGja=EQh?)__$A2!w&PC>RXY`FV9k;smTD*~QBTQOOhX9*}PV(wPR>$h@83e(T1)l44AW)fSJ3&MZHKWnKEtyO+}UcWrPxG*ki%Ne?weKs2)v zq7a~jAib4{>Na0HTrR-7Jc7KLrH5+;&apForUd<5uwh3%Md{V@%k4*6Q4c$D|E2t- z&*di?3U}y2TqIxZ-(?Z1a5t`Bzdn4m_RD=#EXZP~+x!3G!J>Q~4(A{Yu6+g2Wf_MB zFlE@)R^kwNDWTL%w;A=IV+nA^)ZE;buF7oKn6d;hif;uts3@6Zn>BAv1hSlOPiQiL zuq8a{;fi7+9i}I5GCy)mU!@ITn-njbVAIa33oFC6-ywdl8)fW~L@_bwltT5d^eXP;X_MAEk zDql8-@9(GD>$P1`x32V3Z51g$^#{N@L&yS^7cif5bBYK8zRq|`8F$#rN3@3zU^sSe zKW7FstXZ?BkL&>5VPBR%Mi;i1jh&s}+usduQ?H)=RYxN`&ug4Q$9m0ss zc}#Au7*UtCUo7Cz;ZVzS^*!#rT*RXoMamJB0G(}|y}dtq8$Vc^#8qdsEAUYUfzpa- zO%H_I;>F@tnl))Mt-|`bND%Mj6h?}{7JRX2>FO--RY~*WxPKl3@sqpu%bV2ra$t#Y zOeSC;c0eLu=mq*78!leFC=8-4ae7)atX;cs+W?s9lPV0Z zPeG20QPVbU!U<$eH8QY#eY*j7DxSGZA^0PdHU03^u$~v*|3W`VPgi?D88VQ=e7_h# zMEbQSZ^iFlPM+=X|0600N+>NomEgGhZf<^ler4e2dsy&7DI4gh*&L}DcL0_tf`zk< z2f&^4y*l+a6{iTqg_<>MHp$=`6mH3EDoj3I8x2R+SB6H6__E+(b*MUkhvC`qxUdn% z%vtdLQZFwiI{!W4Io^Bya-u=mGqZFE;Pc zc%eCQ2^kra2)^YK%*}n{rfbh!YNIb!H{{6wKbId<+^XN)9fLo<8KzE>COvY_3ajUn zZ%;EJsV;UOZV2W4>!xkfs8EZ!n|V@X&8;b;+50zDe4MJ+K64E@nTc5dzet$Nq4|*d zwtf5dYu2sfV%@fxpvnQz`|9r$xkbhP|FQ8}cBUVK2y0uaQz}IQTkG|KtW-1!@m62o zK%Rd4r!0PYwaI($0oyYTTsR;o0e_^+aJUp>@?vX=k+ZUzVNr+`-ptSj$v$RAZ5v^P z39i-uv+FEfKfbx#*p6ZL9Cj4b4@=T(aP9KtVbWucLOP0(d%2WWo4EC^e}36^nb)Z3 z-Iz&x>I-^pmagvb>XS9WqSnv2t z2)b{wt$1YIV6Tc}Z(G07s$uZFs*V6UT*P~bRbn}|rq8_tZVQN=DGt}x0_C`;%At+&NO-FTWW4FL(2?5R$kK^#p zF)3bm?bS{!fP_LM?tZ=Lr~+@?&UX_j1hbaf^{>D2%lhyA_ z(-L%+ zt>YhLu`B){Ad;_1Fd+Jx_3O9N#2z=@8VNRDm7%8P_grt+e*MN$tJ4Hh8kq>B-j`Yy zFWXT7njlPT831ueOrJC-x!RNH!;K7LrcaHaW9VOB!d=C(H`}@byBlB@P;~;0t54wW zyXmlbfg8{EGL|oj8HssMBk0soW8;&FLEfJShvaCgIDQ0t9DmY=uA%#%K*uGT6eypd zQ)(yT?pIcyzmQ1IUht_?hWI)}T$}-G;5FKkZPczULtJNt${e3Co8S$VL9cx-76*$c zwLBt`Bfh>(BNjY*C{;>=&hF(*MY6lFEDC=sC_Wtwj{g_)mpcO&B-J{BeluLJw_BOb z@_AN2l&xmq035NHZ8JpcAun+pG{j7%c( zkAQwm_QTJNnKQQ;y`mDnc=tc);9A>PtUTk9Kb9MCAiNP^R)XlrxeIU*zkG2JmQjSM zU4aEF4ha0q6b_-;{}a=>@i)BQR5}&a*3^O{z!8YlT4}Q)U~Sfq)C%y4!_xI^hQpIc z8d#)Z78?z#y=Tdzw@kCzzumzruCx$hDH(u1nE1B*3)t;d)rTi9?$#~$Z6@A>j~n#t zc{IhCI)8%oEaD^%M;AH^TIgt_I84n)~VEUbfHo=Z;FE~B~BOO9z zm;`!5hJ5zx&E^xYhUMxDdPI)1&2s~(6-ilPm@|z<|4^0e8A0#Pt&%#5?ZoOQH^z~Y z7YJ6izB}<7S$S}ZG1E=fY}hcC>pJB~=v;V4@HxbJB_Mv(+(Ya0&alXhcX%>En*SVFwm9@ z`)vH`^!@qCpnQjrvu%^mSc&;uZ3|+KiXZ`EaABmLguj@q=Mawxz8f~Q1%Km;34IAR z&sMks+j&r#E=Vm|ofNS6VJH;bnh2>FDU_VULL3 zy1#uKj{6@6907UQ6xetHI~f=e^I&XT_PY3VDLZ% zp2NOMqm|RFZCaqZ2?ZK6wwXk~XU?2a&7rfq^}r=>zI%Ni55AnZGi~`jvc(!3OgZy} zcIJCg3xiLlNC#L>i#6V(XsVIAHS2|?3#CFvaPC?Wxrg`*-U*hb*1Wp-c?E|lHuG1x zJD)Wj|J!_b&Z;)wG$=_}yTIuvn<4LW`Nhv&PNpvlKVC~ot@kCI0F1$IlDXlvfn)Ni zTJ!W8G+4OJWK`!ZuP&6$PYtoUJbd+nNnkdDEDkD&{V5^1r3AJ^Cmbh|O$ptCacatN z8glurpsy8%-KYLD%zkAyaRDmz=h9Xg@nCSt^r{&$EPHT^I*NT`Q!N`v?#1^EC3|-N z%?Wwt78Z99Ndm%wyC8HMurq$hDy@OZCN+8A8HKTMtW$Jh!hqsuyB35Iw~Y8r3t|j^ znv)aGPBBGFLM3(`8l`_l#HBRL0GSr528LbSJssn^$;E&Q;c#vAI(6D_o>o_}HLf;I zy?$YFUMxDx4@<;P4>IFCf4XsrDFJ+_`k=DpYV{;39+6H2QC&U_oF4lxDgUu640?%^ z_X*1k+U}4<4Nj%?z2IHd`Nz;1))jEuAc9d)4!x5e3|H)nNI%Sq=v#Nb`mf6g`38TC z>ls`DH&rAaSyhQg9*Kt_D-CQMdwUhlMwO@}%o#M(PvlS6wUTW<%uNe4Jdm=HKw~|7 zjs|3v;t_Ui-3G074+&&+Km>KyK92oKI%#RNvb40^dGtFar07e|t5{mW{6g-3UV?RS z{LEx5DNhSB=~dvK>3v98govcmp*h4&X8;+U3wMZ>&-1vPc!E(m^1v-U$o|8DS)_qm zOL-8>KEEO{8#-g_=FR(9tL^tc01%|2)R@ASaJ$d()~i0=dr!C+wY?D=yD4M-;%z2K z?ToRtEec=wza`8WUVWa?)YTMaz*U#uhEKy6^re_7l@h%8jMM$K&q*jqttl6)Lh%sn zpK;VOD-VCjAt}65Pv4f`o)y!U80og_Q$CQ2Z)lC{=|eE>PeweQR2v2nqSw#C-#Aj= zd}B#xAqJ5oCkz<8f4~$rd)~wIWghFlgLp-0DU2I@ShPacCQOEdhH~N)9xcF$(C#1% zJzhg++yTy`o*$>PRNOzzVoSn>Nhdledw;29y2;zdEJWNEYc+r@<@CH=plxOWP1?6V zLi0`qEF}>Pue69qLgpK_dtm9xJ~-hq@UynD0T#?|&&T-WJ#Os%{?EaTF;*v`*oZOf z&gLEu4QFu>^q#JKRdRK7OsAdK%Y+G{j{ z63zp%U;I#}y&xrR8eE6gq#8F%Jiv7*H5Q?FA>66#vN}rFuWDS-Z(x=Q);Wp6Uus6? zXykz%GU!aAAgHmUp&#`R$d+RDt_2rDAp<`sk`lBMwvucWD)U$q$PTJG>=_}WRPB=& z!Z0I(Y;)QhtT+PxXx}+WSB`loV?RhuxHs_kI4DGep9F>DQx)1@r{yL1HBk zpPQ;-6vm(~OqyeHmqPSy8gzXF0uGaJw4_AUwHbA5B1E#mk@P3k|K^)sCO<%(TY??| zXs@AKvYv<)sJc;)U_S~F9C?=n1gz+7{crA9 zaRI_{4xY?BBBrJfv42JOx>qxi5&@}F@sR5*Mcn=S69xIklT!FQ?BgTc1NlQrADHRQ zM~@z*B9i^`Wp{6HZ^&Uc`78MHv@vhpxG`~zGugIOP@o?Pn#o}Z0x68n9NkDs9fsAC zDyldm9DNEA)lZ0R9GF&KT?kNImV;vf+gHYE1hV$;*xZsNa5ocs7(IgVKi))os53fIlh@;!RdM{!7f$gS%oX*RL9;49`h2VEoCXRc%-bEltXq`~Z zptd-6ci0y6@#C`Y=tvT|r{M8#l>f|dvTg!j(nsfv5Cg_$>F^~!hSkpBr4YM+imPy& zAc2&ypNe6rW2o%1PyD7P3Myip^&ANLkSE8lo`$nO02ZvPuWv33S5mTg!eBV9XsjLu z1&6e1PLeCNr`1`qwFr@%iLbS}Aa#5D84x+VAYbbUw~l)A7UHE!*Bsef%Dr%j`qC)` ziFd0}XErL2dK`v@I27LBkQTvT5urgHp>@$8K0*Zh*m1dTp+u6j)y;;y%Q0|qHSJP; z)H9z6zJ(XOk(k&sZ8_a=d&fC=4riger!A*CFRm6dZ$?mgF)xCx6iPQq9u)#^ID{gc zuXA%HWCJtkx{~S>am<0#NxGXd=4idIzs`ecoHlLapb?(!si6{BcQrAwgdM>SJRkD8bfX*@!MD0} z$q7qiPlm%1vi2^$>G%|EGjB>hHOVaz-+(WWrM&HI2M?kSRVut+CQd`h1>8z#?$~5YMx7k|)9}bf#1&XnC)8 z%1{mXVxW+X;iN?7?b}C9GDUm@+aFT^=-D3V#l| zwkD8~X=*OLO9Mz)vm zu&#Ywv5UNAFxiL=Z9_%8)w4S#8lfQ4AnL3o7mkPLxZ5jFFTlw4m!$01?U){H0ck68 z-|Kb{I@UPx*TfFtT)L(WqR|3Q!?3>dAB-o6978eh$OrLGc!Q{YeMQ5NLpX=SmEbHV zOVSct#@6BvwG>MWv1-4y_xu$^YO#aa$dUgDqqk|(>Dr6=KtqbJudhl=uSZ$r0FnKy{nFq*2=m@I)ycylA4ovZKggilu5YVx zs@G3s(ga(|-AvT3Lb&QQUjxw0FrG^|Ir@wbfj6P6P^WVcW`!z_&;fOrgJjp2`Vb~( z+93qA?^99m5E(6?354Dv=iKS;v3YOtW-7I4PG{0g9_1443@MfPIjO0HC@Ta?!TnyAzV0> z9`KFum|#Q+9r%4;8cOy)^!~hDtx2D|?dsIt{j>dRXe4h2ol5k&p-d1c(C#dET%-d5&}uc4}>i~0{yk{Pi3WE z!)ZDSqV-rfLP^E(fJTYrAlqh&<&BhCb@5j8Hg8gifGuvvumY4;bOx~elu`mNUWdXUi+L{R8)eKtm_ez@z>qW5 zh=v+=2lOUbf1CH!^ZH{Yv(&y)&1UGjn6zyKY9qX7*ow@;`5BITSO|$T!I|lqj78A6 zxly5VEjiL#v2BYu)~Uj^N6~M&&5(|UH2Clu8pmvtKw2q`yf@B+=0*)+sP=O>0?zOj zB(PBB0%P3)@w_>Ad9@MPYbuo62vYqDG?gX+kan@~mb>Slec4a0LgYeKV1=UsOi~4b zQVU`yRADvQk}%J}{%2NXU5e>DNUlyiQq7U3O9>(++}9l~_a=1>!bx=!3SmWO8O`4ka3epx!CGCP6LSdqbGyBIN|VxK7AkkS(Y`#1>> z`*Rs{qFdRB+ z{N>XIPZY~hLekD7Ob)#1c+QVzRobTZdPzRy;9c&a)({;60CXYFuC%O{Cx(>gj)vB~ zO;!GC@x^3Vfzk@r>u+$w!(h$oBszZ#iOwe4B_4jCbKO!~ew8b>b(6J*IIU8X3{)O; z2|Y)YGLuCj`v3&=s917@?-1Mz9U!Ei>^_13lhKd;8AiE#$&+utTnJ?O-$>3$$5 z2FND)-j*?c9KNCwAsV{+;qsD6)bGb?QgT0^T%7Z^@9xsi*B^CR`|IX^-@E!l_XDOQ zttPfu>g+ht)G0jZ)bRM0GY+Q4|Fg(s&4!?mLAJx3ABEaY?cf{|pmO^5MC=9sM;%r~ zT~w>wetD9xI<4?+Y?iBIj(ytliwEu*Z0WWqzwkykJA7f>YDMwg~7GYgjl+nUnwIsZQHezKB_iZLIvo2u$E7kVSJ9zI5ARc zjbQZkD;|&MSfr!mzttL80hpChY_M3F-7GxR*0fxFdcW`0uET{Xzb3O`3Tt;_9rbOHr1 z#>wcYF-+FrLfUUNd$=U)yFrDw2=M?kg+iHrq2BN>fN9Jt@T-i-XDzPOQgX3l>s$8z zyJe!kaq<;IWz$6|6G&54;L_j}ZG}a~6<+AxG&myn)Opr;3>n zk^EIh$JLCcqj7Aq`{R!gz$I4Sq9r&fo1*Z9gHU$Y1mZ}YZ~&dV)K;DjCmR$~zJLFI z=?){Jq2pus36IjROsXi3igNa}G>{o&4j4U0x@D z5D32=Un!f?;O(SG9{f)e#qACuT`_GG>gaJsLl|qzU=qKI@|;gSv+{+1L#fNSqEdX$ z9l^6)^H@}}Q-5rDs$cT&RUfRhUxoa8vz`w2P&*2Wic)~GrXL@F3*0sK0Qy$8bPxZv za^(#e?_e%&NP`S$uG~G(@0@@0cY_b?J(XIBd9vl*v}mBxULP5FT)GcU^gj>Q#+|Vu`!GEhIDXWxRA5A zC2kJ4xEK2WH4kFd_G~V4*k7$*YU^p2ELegq>l0_5h$fE zOuuR3u*Za0Ub>nQIIal!2QZNBH1cV6?wSNdeg(^J)}cbZ<&(`Ka) zMwYf^b?3i&5GAAmRp_s)fABM2cu<@JuO&t=3~DLuS6-dESw3eli&gd7B3&0S9r)q+ z&Ad*$8f)p|a!eBO)>#wG8Qe%%gAr zHf1OGoP>5;b*+A3NT)Ro77`MZ%zoIsXCZy-QlgM}5;hKsRo&sk$U}CbT}bHX#6j%m zr!k9pCIv`tyFi#gy+^H)c2j(JV%0@>UYFAYFe%(lHnHi*k(O9h;bMN*ve!`R&*pxm zATX&^bt7OC1q@m`mV8niA_!l&w4;MFZ0ue}ws?wqFQZkTQ_`v5xbb%AV92rsVX8wx zD7@ZYZPB6BSP*wo5P=YX_+dY-C~|{BaihsopxssXPJ)W;lHOVNRX*>d0&5LPV+=~$SDhm4;v0QQ_TKc8&!f*o3?J0*{pqeg<`NLU zK3BNmh{JimTz~!|E-`U5h>U6SV<$peX&5}Q>>9pG`X&Kf^uoKx;2UNG4-*Gi_@%LF z$6Q!es=vpBPUTb-HshT!8OfYU1OLxF({6@_hBwPS^yI!Sa*2;0+1fh9^aDQG67&p;+N9;=}rNt;cJ>_rQsRkgaC$y1C%ymhgOG2NF8_RYUip&P|s zzux7t2gSH#!~mp+I~0YzXht5cEUe#DS5&@4EMmgUbVNYJ!a(R5baVMY{H{z~Kv-_` z7MpA3ZRW*oe2|r$9Rw)>KHuG<#a2mQt$NU?(uQ#&2gZpQBwe(ZbTjqQpOU>sZ?P_& zDnx`Iz{kfhNu4uc2haA_I|J(HA?$VGNMP5yTJ8V$L5qMeI>n>|nec2Y{c|FKrWW22 zjobSO`r^poK*Y-pcIZOcuO(un5~C6 zo9cxv|L5lrJ*oLo`)xIRA?78GX>snzdrzW3fqdW{coDz=KdL2`mFfeMxpQ+G(~1NY zM;aJJlR!R}ny8TA4rChA*+Qxvk$Wr_6Y7NHWlh6S@)H>rna3Wztn`H3CvlJ9@o>8F z85(&;pikY1R_>>7?ti;FN59+MB=pCl13+Q9lVN)ON-O9pR7~`c$JE%OSjkSkd3W;Y?sWU`ugUC4ZS%7U=U`<<|9a4A>A7i{ zB{=|uqzNk$am8F3jlelYcsA+Nop7X9`~HK6C><9{&FWl{r2;SL&89kt;VmLK{)Sv_?ZhWh-m+&8X zwr!SeEX?+D$ft?SB6c1TNtvVlzFVrG{b2zVV zk3N#qNwJBxB8~g&MA?X&{~UI$ygn(Mw#ddZCY|rGk2urj_u6$|?OKYBkbXr^z1{)ij1Ss-)*V4y-*A+|vFLbSBuxG(j^Vi+ubYIaElfq?; zLEE=s4qhmPBS&3li)#6&(j2J(wO{gjT|x3{gT*y=g%UdqIbx@|E6oJs>Z70ODiLR0_0Egd6K; z8k$k^LTn)5@lqn^JGA@FT!)}r2eQcu7EKY_{Ic*tItZj!Z`^PCl^b)S^ z9<%S4zYH;Q4I3Eggt|QbTN-ZH)s6$56itwRLtaz%Ai2CviQ9nN6+(I_dBnB%tSyfD zrr~FX2Cc7c!7_~6*+=JcZ3}hS_~ZL|vO78_QlFfc;7ywssoc!*F?tzgvm^~}lZ6j% zIsEK|oxP^~aisC($>jdGpl zki<7PY|zK%9NqVGu7R^zaZv^Vd%T8>(4cGdX?CsTbjmG$b$-UHhBE$Tpvx%Yq$GF! z3`UB8*32t3s7Fn8Mp`Pjeo(K#{Cv*Vdz@92<@Ba~q|_XW`f1~HQZ38Q&YrO@=hn^x z?2L)~yIou3V^+ll(AU?$`KPJP>(Y$#OZ#6wf4XQ*Q@#CEb#-ofva$LdS@0lobC!39 z$&ISh2*t;q2`og(jL_N6%&I;Fm1;5q$US0n6SW8(dywqYbK|v3KM7FC1183Y~oha)Enkodps1u zOm1NVh_~tY`tZQ{)EQS~mSGR(T$|9YV1jJGEW-XF_Q%WvU%R9q9MxvmouG%KHLpBI zMqm>UpYXRiFI4rQF&M$EIFz5_M^`7rOijUVkL~G#X{aMK-YA;GK9p10V zt8JwQmrtsbgIcQaFefxWZCYZo`rwG~XMGx44OuCh8YJx*A-iYS9dP3NAJ>VVS-erW zW9yMte79lNEenfRpdEf&)&UTIiYeo8+kkdT2O0sbvkfgnlP`cv2=x(=P%8*AvvjRf zMT~xW3^vCE}Cn{d+y zm)xFu{O-9HTccyFH89EN+T{R=BcyuQ@C?Uq&Nbo(a{~)=^Ay0U^~VuKUS4LuGnCNJ z?=1x-?b|oc2u><`{o9BU_uLmuG%_}hqYtU&BW8(QfAw8q&BW^IA4S1W1_P}1JKSyJ z=|k~}M#Q0huA~ri?QjvHhFL{zp6G%3N!4rGGI#~%4F)5TX^5(X+WqlYO3XtMwq|X% zxxda_E#=Cm73p-xi*Me&v#@Cxi4P%dnr>7A`%8#I^q4$cQTFnBv)=b1pJ^Y~z+4Bb z%s-9rG_AVm=nS$y))0-pA4gfuMwjtvUY=9(Jrsnc>#F6dl`He8nBH=mwCG)Q(&AH+ z!~4A@9!h?SPVc!<8TvtG4Z3e^H(+LRG|p<4aq#3zf6TcCJF#EJQd)MU%}=^OrI8m~ zIhHf|s?4-}amSP1j*}hdmQ8F>@!U`#-$_eP@s zq4;F!QX=GM&;_hm{Pm7s?Ls{)KHz+XrI?!v1H&*w7sKrG$e$MJa?%#ft7gkP%?f-q zRWJR5Q0&x{eXKPaP%>BKMJakl5#~=nBOL3_LlF zVAk&9it{QFoS^mg4%5lkl`uLp6thaE{2cHz{%uYj#m9d;+(ryUn~a812;=}(;Q~*8 zQ@JhPR2;DOW4WGUSVyh1$T#xw7Mehk*mBY`kT%@y`59xc zEQ_OK6F|K8H3G|%QLzPtwsY1PnK8Zs^E*B272$5b9K=`tLX02LNsLO)f`4c!3JF&wNHC{F-`ViKdyOD zuwigor`*rcZotx?E+nj7Vo0?^u$1-3Q=R0v&9#F9blWhCP~+aI9&}?r`{g#}$lz9t z9q%7+wEPndYk)R)4lXrIqVS8OGR96Oyf1raMA2V!hWP&^v2a8VT>+NMa1ch#MJ$j9QpC{GXw@8g zs=+HxmBMa~Y0Hp=-qDLM7ToC8dcdMXQNLH-=K+E?>IH_^gxW_}htN9JbJh;ryfEqY zgrJX}vvkS_PHa?nYap>wIY9b^1X5Ewzh7=6E;kn3D4NjWBL&dodaG1214tu9A2@#L z=XwN3o;@4f3I{VY>KRu_?v!wNWUp^)V3|XEyua!AX+P(vWfq%<%r5_TXFz^nm?7yO z*CC9d4XS0hr)0b*?OT9ZF(v4tCJ+cbgni#{&;oYJwTM)wG1x;7Qr>2|mP3I`ju}I| z?j?g>B1BeH5G=_Z8cnI#z8lwr1Ce0PSoh~2e~411G=THpB_30LO<)Le5Yjhu*3l&I zPL5RX2)WK?TCRKJfi4g-PcJl>74Nq-+aJ3TlxqZ`DyCshP~s3;U%`^P-7LdqPavSy zs07sU?Qt}Bt#$wT4|Zhxy=WTx_lp0D7c>Ov4uEBSt_h&a6EN^^)22-~KppBVGxJ+X zr7>x)Qug6P9%0wc1jgQ-#^wd$l!Rcgv(MFMPu2^@2dVU$QFha8lD4qvahmiSC}Olf zN=D3>W@`<_AW?%7c)k*P%v2|XZIF$&$>N($_cxd|pRxc&7r*)~dg!aAs}ALn&#cZ_`Sg{@x+ZHi#E+(1SMXiCm;NXlNaQdIB!>OIeNY# z&-hPlVadhM8P7f%hticUIZf|}XBO-MK;D%5k*i!@_VUcepIWy!_^%W2Bs*K^nOWPv zXSeGjWEW4kNd7LmabyfMWOU&VWhBxEm^vql8!~oBXoOS-H>t>8N0CX$V7_9hlw!8! z_fU`S%bXpWPlic{w);^ zGHW}Sc*cpE1>RCaSE)4m0Llr0(2|(N0}LC`+Y)Lw@N`i7cN<@ACVtH&jYUvl%iDW6 zVncMug)$|*^{fw=i6Mh+xnnD;83(5ie|2KKO-aBPWbW8cv?ttV_4z;@5T@HNjE0IY zlv*!!4OOx+xqQB@D8O*d|L7V0|L!7gj@{<-G2Ww z_=s3XDr;sMj$bHuF{5m-&(8NBl{bN{Am7s`(L*h8EAaoG+|LiSM>$jdnDQXK2l70D zCk+QAG?{@d$a@fVviB=yEXu&4U3VC87fJ_OzxY8OdA=h1$UV!<$_WV_dQJ8jT=v31 zUD7DR9?o}s;$2|MV*zO^r!}$34=;vI(sB*tk z{VX&)bD`b3CgKdBgUygVS%m0};9Irmh!SW z4D zA=fWm`WYx`ERIX?H@u7)GbzSl)ntwqPc{^=)+r-8g2*AhLvsn%fyI&zTiPWvXR7N7e1EjfP&>g8m3dl^N9)953pdAcMQBK6#h*ioq?v z|9*=0*O*n`|H#RciBEmoRK^3`WBYm($JL+5ZQ9~CwU46gR8M-pN2=$gGE66eR7ss= zkT~!Fn;>@T@g0@yjB|R~Ie)V8K*v9N&+gcUYG4YJ(i=W%KvNdE@Vd8@y&(I&i`-u; z3T2q2$b^K!O{xbPLVXCr|cMbzD8w zL;IZDcjOy6zv#mDGiKaId&aXq{69Wb@a-dH?Va(CO_KTGOEgTZ&)0r*;t~6({S*#w zI7OIhq*P2e@#TKdm$TzYYzB|^E^3&0yg|c;hO%tGR;Njm&sZ;5 z6WHTMEv2IYn=Ez#qdZcWhe{Y^rYXYwNdQ+x4tgEVbadB%Xx(~>&#f9k&~78h{eM8D z(osYCskkeuP{TF0dxzE{9C+HmyX^iWZN*Hhb_zhbrAaMWjmaJ0S1yD;`kF? z15x|bN}hycenfQY0_CoM>wsW zgN0!vYjot~*~FSB)fs5o@CZ#S^R&#XnGtry6tYbv3i`gy0%k+|NRxwm;q)0<*StZ(d#i!h~yzpm#nN|gBMOc3Cqx2al%)MNrH41Go9h8Cz*WuqgEu%og{UY-jl#; z_E>BO?!1_OG*@U4uXMv<>K5GOa7M)B)4Lmr@;CmA$$({hfWjDOWp0|rRNS3tnZ=U%Xfs4AS0M-xa^Lk%!q@LL0+ zx<@a}*`NxgE=FvaJ?zoVsS$REjhj{ICbDQTb2-0bpi77vWW9RzMCg-XLKZEv_e6j= zE=+W+n5mM>my5nUe>MFk*v%MHFG>QY8hU@61#+x39#0)*hf;$^o2w(EyTp+3$ogwP zvaqnwU`6wZit?F6Va{{nnARC+yhHaOCEW39-@KP35K7@ zz4S7_n!H#tCVDRp_?)D0{DHv=GpknQ3=U-uAc;mamxNsorYS6nOjij@KT-8=okb=+ z$h20lYIXIA-}HJJB4!fDt8&4Y@~EZ|M+nVnNn7%r15l`SNhh>ab&AfraLFfL6K$ta z08(WP2$@cPRZVp3__+*NLBxvGQfn%rdXDI20t@AH{V|VQ64tkm%4q}{ij4KOpB|-6 z9cAqGg=^oj1A{hg*&`&BdTP-M5aSpX9qmWI#-@n`Ek*a7kIvzgYXU(CW)nSXscCUa z$E6sOAaW12M&e#VSYc0A`jX5zSI9%qP{xr(YE+cmtVNiR%q;4XqC|D3QAyY-Y*>|A zD#Db3aEibGcD}d6uBTUHKEtikn2`W=mGIOK z3Pqq#6Q2xYWHm`sX0G!_ue9>$EV;FN#fLexuZHt^nm`B=`LDe{pCwToMVCzMcO;0? zzi1Vc3vmN$`SjQSd14EQjYm<+Ht2VkVeCgRPTgRzJ*BP8AHDG)uHK38Tich$&b9yu z>u20bnV!g=6iq4+4^z$H_nCDBMyTAv19#$u5AeBh&}}q=4CT^OmKkHI#LoKr@?}ZV zg^#SUWe)9O$BPc+rg6t-)rc+%r1esi^o6phnu8_sdKr->+%B2U=Jq#tDZ4_5MH;5{ z{V2zdGBB3R=Y^CgWxNCttJ)eUvY6$#Uv**WqW%v!fps_22v-G->Fsrf)#|uMrk{%j zc{?MQp&Hv$v#o^OC0$@{MaI>kxX-eqfNwIxfwGN`n)F0NW%c;?b(CYHS2gZ1md%D% z!U}aszdQZOt|{WDvyz*Z(}E}(Wjbm}!;uh|D5=tw`|#(^{9af4$Ya_r7)fiWuuWtB zUd(~JFml9z(R4?WGKOatiA-s#^+ znY@;;E|>34gEWc;L>IMJZqW8qP>RPKb;x+?&hpVtfriSl-k*4u;Vm4O-L9mocKiw7 zkCa3i4c^_=$jzlda}i!EhSg8|(GsM=!uUW@g~O$*mOd~pM+dbd`=vdqam~!tPlyvJ ze2x;()nOu}8^JTk1eiVY@QORpFS^ItX$11;d~NHaoWfPq3fi>GSMCPE70O-T|6#8+ zNx!XYxer)HY@58McMh`IMX&kWdt}_wjz-wls_xGa!*yu5!qaOFljJHw&cvz(izf#S zjQX%l8}j6RAq*=8AxOavmmKNS$HLKBeJMIE|BxFjE$e^Q$(d5yCtdWGU?xgNJSV#I zxDd;qQTiH3PlCC5J*9C^XjRgdfNAcM>oD#yv1}Z15zkXzG0kb}NMd@?j)M(yaMR&{ zexbZY4D!J`%19A16IE=2-F!N6uH`fH^){>}X=EO}p=FaA+~d)wU{46ePxM0a<~89$ zMKG3aZUdraZvKrj`XI!9C-%7?nnz?eQ_sC3hQkfyAy>bh68_YwzbH!#rI{8ktd3d& zJqk-=GOG6Hi%8GHgDV@G_}cd7dyy$T|`}0i8-Iva`-t zJ|ACGqY!?UB!1A25H0vIzKnj!e>nJ|JZ?YVe<^O)0FY=#?xog}VHW0Ci1ulvLCTyW zveN=85qa-^=Fy@>3n^%F@T9@U$f&8}b{h(th^R^j4m(*2tZ#SIj3O<@(l?8lUH;&9 zjkO;+59$|_evY1^Nz`zci5_QO26s{0NBzQvNDCZbiGQ$rZJ&&_Gz1XyBF{?{c}Orw zP=;uX@A8H1+OAQ?VMU2XGxxhF}9(`LYBf9EXf;BRtyC5X^UL=*V5Q1 z&a6+@w70N}q*Ysn7ciks6fgntNR6ldVu!f|r(E-&|3Ht7lokOzbltDN?rLaQbRmlA zrudI7B2pu%aFS__n6KkBx=vEqbfWA9h4ShPBjwGNP}@+AsDnbk4p~-O0Gr#Vtt+&A z61LHR_?uMHc1(+H6%RCa4nyv5OQC_`Gv(w*@*7<&h?-L&ILlleP_cdk8dci|Wr8tt z2BlY&)A<~9fePN~^vz1gM|#kDMOA^55dwPc1$8PO6?J22{wEMSp-rd6eTmYLwn?NS zdVJcPxKLWtSx}MOkx5`28dh`Y1skUQ!mt#@H9k53X1UX%z?2%#IQ9tDT?qjJPDE}} z9w11`B7Nmp8?s&=u%+;Si4!Gf>k8B;tvG2k^aw|T;Cr)*uKZlAr5E-?A_aN1KO@vj z_~pFg|;L>ltw^JpfI6nBCkTp zxcr9I&R=`|##fvo!eD=C34SYEVQ6*u+x zY!G6x=aon!pXd4ge=m~HdYLu@;4N(%8rj*RsZe#(@4a7Sct&nt`1g=Ei^F~Q*G*m5 z*ZEq(H%+F6UT?4VGtgVp@^+IM&QE@v`0Kl%(&5|Itu*!h=G$p{*T#Rnadq!4Bc@%u z*GFfLaX?|&ft%Y;%`TqjdH%>vkBmoUpOy_SDO)<<^~gdm?;*;e(CYsG$*s)&je65; z?@ng2RVCbk9w6nNDth=Pk2+YrtA|VmqZf@vMo$&0)Q}k5aD^=@3A4^7)ru73*HBjr z#({W#E-%@l!qp{Ep0H*iJ5^u;lSj#v#>$tOB{D@g=QK~kGmQ#=9?LYLoum4t2gb%} z-!r~Vbz6GO5B4sX>P7+cZBezEcgm{0_s6$$qq4URuh^F{_iCsIHP1p<=?3xuyHr?m zYeelneN6K%AGg^?yI+p=t5($VdF@G7o{V|^JD(>blK=uNHYLc24fy5ajxJHi|% zA#xzTrBVZ)ExguhR@16AIB?@?qnPOIMDI@v+vDJ-UYE|2)(Ma_FY2&~?HV4gV zA)(kxC}U1av$kS1olnHpJ-sSzq;l-xuEEQ_1eFjmN;Ly7&a7+MyHlIF$9~>iab}M7 zJK3a5csvkG0R0H(?uwMIk4FrBGXBB4y}~O0JB8X;0Knce_|p~|#+m98TEtyzv%tPG z#5?rT*J#4LMd>rPw{^Yp{nZV!zK7}%pXISGIiGxb?D*=Z)I@J&F&|Zj4vf)fSHC)B;7xNfnj(M*}wkXVOM^UfFnr(sA?oXC&L?F3BGaXfmTQ;KyZLN7a9jj00v4owy zr;XS!upo9_harR2N%a4JMB6QP=B{~Tt1fsEDx-HIwn&pH=WoBCYsts^r7yE z5N~9aU2)X=($IG7j>r^(y!4^jLKU~N&HNqo!s}Ct)vH>TSv8iWctzo^M^6(5yf)X^ zQn6)#k7c8QZ(^$K=JIO$7Gw8v|Joim$V2yNtx)?VnZKWcuUom7tM&DLBJQH#NbdVV zr53yhhdYR7486*oJig{TleUg?*eN?X?vL6z3n+QXd(ae}1VUALFnn}sewqs`|7ZRS6ZFnWMSLMdv*uNw7Et5U&Os{~0Flm%$v<0E zO|ls9@{})w z>D4@-bM{MLMpWJL&1!XhpT0?K>m6+72+PcYIsOaX*osw0SARf_qiY1kFM7guW@`SiUePt2WZKq!!@ zWw>MZh5@feQYKwfVZ)VedupuFzvT(e3h)@KT_Ylr74xIAwZm;OhIj0$H)YdF03Zv= z6I#^w2Qj(jkv3^uWi)uqt@_aEh5>9m`=db*-E$XS{C~uqiC>QC+y9%@3^ROZ##po8 z*2QuAtYogC1p*rv^?+QPR#uN zf#3alUY_r8FV}Tm=Xorj<8yqD!&z?GdigEiKbxp_TRs|LpDN&KGTtqdd#Lv#0w}}f z!8hLefqx0~Zj3>-R{n9`$u}kHH7Z{?H$M^wCB<+tY`WMuKr!j5h0uPRkTX zF*c3wo%8WjKHC#GVx_k{&Tlk_km(W_-{Q6mh25+>`6{(y(F+>pBEaoFe0jjf#}=7QD3ynvXieFKmBuR9BbP?E9qzTHIg`HL+bW-{ zvL`Gz6G@;kH7_t7jJ^UxRe>{ZCdYHzHGnc!hS;LFRfWOY8cSdTiqZEJ&Z^+r05IiPO0awG93o=w0*rbUE&`3#}oxPrNiz z-`?A}vc{3@Z#yO5mWFCPX~bbdga_PEF2yt~6`T>2NY>O)m2Y&>uNSf!seJ~5y%2YJ z1*X%_*bOp~b~5b>d6k3%+6%ozJ(upU`r147*;}y{SK=8pocFS#Ko{cI0x>P8-_YB& zm%>28{k2*$sdZI}38rLM&6K(E7IJ6jV(!3szvWe#y(*u^)7-W*gAZLX;j+tdUE|r4 zhpVnliQu71wCZ_lVfu7xQxt&p@SD|cfM=z9HV$XWi3)KK>q5V6i1zF`sy^W=EZhHp9=4cR#Cm6h1^b2+tCFt$=n z7(s41N>v;&qIS0T+q}>c>_-Nu;byW&1K9aa_5h4mqNglg(^dr5q*a8PkKIErG(JTx zEBxeO4WDywSYz6S?!0__b1;@^vc!L2KP5d!yip$7BbRz`?Fod84NySVrKL)d$@cfB z*5~A_bF|S#zh1~0Nu6^Lg0U0T6shCf#P!)#KJ|!)89OP!{rJ{y+fy&~^>)+g*mo!&ejv(S| zo89+%M<=NpW2YbDy(3(C9v8X?*s3teddh+1&bJV|@?H0i`GcY8>J3)9 zi5qDetTcm75e)Iw;ND+9s1(e~{Z&@}-0i14xp4%6481vs`PW%Amx`iN#p=e?wEs-w z+H&lzEK;7(+9-OdV#aQ|rBDWWj4z+zK$4mhC zt7P)(48o~%8wAoxoi4kVqi(F8HUYl36zMZFBCJN+cHfM{7N0RV)FxN~oNN8?25rYp zlO{b*uFM%{yNBIcFl{xYSim#eS=mpAzPj#c&c}^7{;Lk39c;Lmhvs?i4m~h#&`@n) zYWO>DFz6vBLi)x)Aj6ZBlN%nI`nej22S4>i>z3a3L}oQTILz}!1p=I3UcY&H+cejR zJcY$*mNdO@CQUK{D4F!lPelCB=H3znEFB)+fJ5FaU99Azp^}EeT;GB?sR=P4Qy`1F zI?9|W$-R)^&H%^`l07dQhSklZ`r-wqnYE>Z##D>f@LPBAaXm;#E=O-KD`yJ2KI>&u zQLQVN6J?LV$;|8-8qBe32=`!N4#jJ0zkje+?qY1`Jfo5*lI-I;RPydHQq!ztP!e6x z_fSPM?@zpM%#P4mv;s_XRT02&UytbZ`Zs1Bsyz7=X^&iyz95`MRI(cY{|p%|$Jomm zIp1m=Oq7CZ8In&xSwT`AAgb=UW09Ez(+y|Ws#LlTc_`<{^*iCO5KKx;BarI;_wn=8 zY?;j$`4e@#ILw5O#|4$=J?(z|9A>r7Wy8*-=OKamk?yq9f1U2xMhIRGvu=a2dM|i9 ze-hNf1vO;AItyD4?j_4gd=sLWee>vJv;30u=NEKyTN-|!GyYC{7Ix~2N_<;4(XJOrLD6jZNNJuH*rL(iY#$?T_fJY0 zJ)r{H?@GWb_!gdX$NZQ8aJ?8&Upb;+Wqoby1p9;+$Lgo_)6$yqXc#)O&C+Ej-#-q| zC_>Jv(UJvJZ?~}QCHwYlu-SU-)ErNaK`SmeYV-N?=U)&_3*A++3gP>6AzjFyZvC9J zS#0CM>Gp^kLPI?K?310c-8gcGiJ@9i29;M-V4qfoylo#Is#lW^i+*c93v)Nvg4nJCCX$Q(yLduUZ!1sio$5fP1!#|b*~dnwJ> zZkYV3`yKfoDBYz+O?9&jaEVFSd;L3&seyJ&A8d&{an>4LgbGp;*dm!<_YB@d!6N$F zJ2jZmkLcU|&4YkauDo9`bJ8+_4@%F($x>-dZ97TZlmAyxfr5(x9e;ib&e?g%EiP7x zsyeQFtV$NFH>aawD9!$T{f)Tlr&qVN(U@B0Ld!`^Ldo_TSd}>hZkLgn{6ffTJ{C?q zQSo!F`YiB-9zYdwVuM5Snc)OQ$jaAt(WOxdDBkO0Q6$V--NDmJu!7^au}C#3_0{#_ zN>R*hHv3vsL01<5pxp~@vFe?=y)q6mTIyE!A|R!uJ!R!Z3H8%cG1D(%SJrAN#hRm3 zx+tLp@8!$|uF5A2H>cY-#nwebBXTq~aC0KPA0`+L09Ud*R3s}A<;QW{>bP;3K4`>r zRTxI242EsakfL>st|8tR&f`=aDKRX1dV3hE>o@0@!rF6q_||&njx8{Yx-0u0zxxM3 z$?{$00&_v2@KXqujw4$6*A*=~Zx@PM^u163;6(}f5BY^@bwnj8w<;;XB>eU_499-e z?KD!?-kk2Ma~aro9Ykr6?_BoyUR098v`;nL{#4@A56{mT9*qvu=# zS=4?+;tN$_Btn6Pi|5_DA)y!=?Fyv`p3Xq{ESvbo52UP*Br&gepyC6RDn54TU~`Ss z?fdAME2CFia4!k435hf1UZITfDwk5)jFj3}S!!b?U44~eG|)i&;0FvX`pVXNBO<8y zIMcpK6^GTf`%cQrX^V3dbO21i*tlW7ikb3$9!;~vi#6D&QXrLFWzQWi9mgQY`;*)W z{l`0qe;=SO)wg0`bFLuu(PGvh0tR{D;Q98d;+Z4fS&LDNK6>Loqm_Rwkx%4UC|vFr ze`?YIBR&YZgYvy(HCdA;4dJ>50;^mI8r+XvktL6D?T@Wp5845}E`J|Mrxc z2mPS<)cFx1Im_YYnT3K9_C4&3nQ8jw*e$oLP8WUyt~mYrT=cO%lw4vL#47e%M&EFY zJ;cD?AFu6mNU3wnt$$K!cS(cJ$-RMp+O1V9CtdrNa?e=YQzd@5 zbq*o74zNMdz!k51$%EH8IBD`tHRSjDk~CO|7A`R2V^Qj&qrc+qOQDQNTGZN5o~nit@=lq=9BK$rMvJNus-X{e zHl9KcUv;i}(1r;OC$WLNUT>hzS&YlQfOS_XlAa{GfMQSVl&Ypr;^)sz7oVYiO|8MXZx1OM&zEC>kb*}jzzBpjF{CSds$5E)Ld=T z*;k)_yy19T3Z@c+{nFe(L8NZ2a!Pu9G%FMX6YO@r_b8;aHdGeo3H)s`2oZ4br^nz5yWVTeVUa@dLeiPSc8sqEAW~+Lt?jv*Uo=BGKL(WuWz#- zhtOIdOlCNy@mt;aR-daC5?0I4OO`E(f*NTg+?z`^BeZLTSX3rST9GYo#``%jpmvoz zpvO|#F2tC78*j)0ZSJYx4j>XS0-;;Vp!yeB_?fus+4NXL7W1b^XC^VQ?V}p;d}sgp zE|6v^5*nFAUYt_@F2%j#Q2Jgx(75DM(t;BiZy-uKNobB4J`5sXOd^`itgxhu+aw-E zvS+M1Jfy;2*5YKkU#c~U{BEqgEeh` zfuy&@*dxSG&Lfqd6l*_Bc^K;-5f!CPcAbrBeF1ZRAih8A`~0>8rvJ*?hA1+7M+6_X zZ#jfyFhn7ow44$*qq?*N?vvOi_n zg+4VsT^{u7iE!u*1MEDlDV`>(QFMU@1VxuXW^c-DvOUIKkm(S%BuT6Cff+9VON2mxD`ll}*|9qSzqAFUn`Q#nK>fo26b3X+ zN}1aNrVJ= zFCmySCh{0axs_OX#eQoUvj3AK$x@^8VdTeY#2G@}8G`XhDmSKs$0Q^v{0NwwI8F)V ztO9Mx$!$vr?dH9O?OiB9rtkqsce7}xep-W0dX~UAP=#BjB_jtf?1ebMpPArm=XH+V zQi=5bKl{aaY`p^pA}ef>I%!MBFF+dZnZRyW;3oQP&@>VVYXNre&GQ$ z-{lOa2~|i&?UwpUlc-N%S2twTN=5>kizOyr?*{#UG$>*)WTQo( zT+iZbVt}3mYiPE`3xa=;JO|0+yB6VfPQAn&wzggHlibRvm=xAdI4uLoNt4lVS+&C= zl#)s^VY(;^iBS5FX&d2V@L11_B|lqDucJZ|HCR;JV8}ZYISaDtMyB}y+{=_$Q8hoM z-%!rUWo)BN)!;dMNj6gaQnuzeeZ>Ey7@WhrwPU9f2q?Nq^!bV;rE8A{NAB&4*YBm8 z-*gLjR;F0_Ta_tE-AXJ$SI;h-;(BP5l%ExEBoDe(tyF=%?OY(>ML)L_FjOU}GA6MPHxVT!(krRZnxQTXBSp zrTUF~?V*$jD0!xabJrBb0;+w}X}A#>iKKoi1$+Q^cHL3i>IWp6RhBul`;TfGocV{H zZIkAj+t&V=QfT##BVnwfv9mW0uKedOa-2@}9Cn9dzQJ9e&jvyQ5a!S-=1{ueKzeJ= zC%}?OOAz0<(Aua4bwQ`+a8T>2Vpq=KdUB9+ki>La!kyClo=d*a7~9Irfp9#jm_M?_ zy?`95|<{hd&XTRoJ5 zdNE^Z!6A3KNFn6}f+|j{siqerX@M<}D!ghbSWF74K_x^K31U{>?Vxi2 z1C}*x=8DB^=9nurQdSYvWxpQe>m5{B7lvP30?sgbP8|k%8#^ubN;}?go(kM?GnXYD zn=R>h=Y`S92TQf@E{$p@@(J`mxPeobFxeuLzCp1LQU&2TUqB&r6Jd@Mpyha+@MW%Y z)$pAX_D{7)gjeXEV^CVm@AL{blrW6Azf>x|-@PZ9x-?a9$9Zoa2dRo*AV;w5-s`0B zi{}EV%TLQud3IwE>Z9q4e`6(`rC=u@6u@a-ixRrlT%=NpwVc78$vV}fQ_Y|5mf=!o zJ*bseE$dsr&JP$`k6XzWJoCDX!0*_Tq|0ps#$ep?I8HO#y~7uj@VDb+MB0)yMPiP3 zc;4tO;2Y%F>T?_3c<+A^lz(SfePI!vT%jb+K5}8HwENOdWP$||fL1GM9er(K|2 z=6=;;@`D4BG3Oa~v$Z@XsjtA#-{xrU(~VWSN^tj3A=n(s-+IKi?9n8t z<8^#jxMM(RcF?ga+}Z6!aKb37gWVtECjnb=>hBG8>}BopL}D6F*3+j)6T5zip`)JZ z7@lw47UN-47JKeRjlq=f<&hUn2}wU@wDAk0jTY+EO;7>IkF7kr)W-nSmf#MgjQGOW zp=NilJ7*-EE8oj?^`&T5@0WtjTjef{0k*?YVYxLT1A2jyE>^(6ZENW1swx&H2Y_dQ zi#L0=SvrCC?eEi1UvHU-y7Sa3n2!_s12>_JX^SR z!buXuFe-WiQf?m8G4TAB+T*f=tvqnfev(7-);__JY*mq85w>pa%n7m$K=`xo%psT} zTqz+kH&qE3oMm_a7D1Oe&&UhS%%vn!0Yjc76Sa?2V)(6N}?8m@UapRt@1qfdNf1;H7L#I zBB9>$RNyTiix@zI#f2V|#ny1FTIO_ua!B-4zwG%Z2O6&WJj}XYDlr+%u)#3v2_kaZXJ*pSK&{SpBc&%lv?Ddl0t5RgvE3b9zE*h;Ir*^ z3F#Yjtdd}=gzBHMQkmJ+aURhNkFQ7O&-JF!O{IN{m9Z?_u9TvIEtHZ~`P8+f`Vru= zUf9?2NtLK#^4HceZ%F=a_w8Ipb^o>Q7aEb*g;znGF#}9#DQT0rz5U6HRjjEL%Eehq znS-iqtKPl2+k-LhAjk#_Jfbf3+fs&i%1`lSR* zm?{()L5JlrkXnybT?id9LU_*t@;nBD+I+fD=qhq0^dQR*cf?*u5UW&)Im3pi!yxTm z!EeSrDL+Y*axovvVn(;R>yd)aYE8GXsxY{7l0->5Pjv<2hGbi#Op+>udrRc7>)A4@ zyICjFghS!t8h_O?Ygo#WBa3!@0~_RpTkIzq+K;%>3Boy-jZIm*uh;dng9QYWe3tZD zNIo#d;<|PhF#lQ;%4(TFR>3&JFt$tPAE6V3kSLV|47Kn3^vmidb=H;>+5Imd43!8w zOxpOs0E<=Q`ewpf+{O`eMr=Qjy{ks9r#RKI%{Q@ud&&Y*BzoacU*(_{)gvrZBaADm zEL6qB`{N*TdS9s?y-~rWB6n9xdnr`$YYVu&Wfx!vyT5I876>9pB(K z8djYIsge?)1WUBzIP(~M*U3*|qArUdA6978*srcb4*?daB#P)Q!HlrkWkQq;5!$RMn6+oUgz7J!6dFqI zwb}1ejq*LjI1!YZ0&(75^W}1m&|S-u$zsi|{*sGLefy9UYaFxxIFe6W=CCMQLZB7w zh1hlfKV|AW@`;CUo7!$CrR)Um+8tX|3PesV$x8+EP=;fYXuLk<@C_B)#6;# zd=sIZ{ah^|lP)ZS3xt>ywt7Xy0}DiOT*n13CI*`U+ECz@rPF%o2*9EXz}mY8j^bvT z)z?WiUCZCy`Z0!Q)$;CY1F?KP$~(GZj#OcUS6RdAlUCZRoFaC3+B~5kC=(9s9#&0& zNiO9tA*2xsuJsp=qg5vaAnMQ8gX1}0-1*ZgcOh4<=^3(({Ks>ZH`6#ZH0nc7EZ>-$ zP**>sE?M93U|aWg<1Y+2{l832$FFwGy4U`J*W0e8H~clte*S0obW?{>Ym9$AoxkgE ze{+-OXBPKJpB6ZGTS3)3$CMSv+jt(2S&`PUnmfGdabiSh?z`ILS8u$W8X+2$I(xLqE;~l$prx-1L(Pq@&uVi3lQtq41$bkEWhC1xs5I^~oUl>(U z{Pbz)lUg(u151_=06%Htj=E|_mETsxw49bk)1ntgueUu0A3nF+a z%9EZ7c@HAb(R5dVoK8aCb$}(Efikg;(5v{&UiBOJ!>6#ZIZpEt&*v)r*o+{U_I`Td zrg7365A*gK@$vcrw5p1@lzL79;@s1RHh#;q{*A()L;RWFP|#2W9NE_c!6fO!kGx&WOimgI$#sbtSMWF-yO?_Udk_|Ls{P9KM8cH^?3aoVQ()hkQp9$`LZ($o(> zH27IQed}Mk9{lGfM{!R`{K$Z1^gQd^7KyemI5&CaFjRN1oc*iuM>Q6Y`OnH^EJTl@ z3eaX22ZpZS{%JdcgCT?UlsyPQGGIWL#*e1fHtxZOzj5=&w`dI+VkL$1w(~hrP>+8> zE$&R1Q{!he+7|tHQ5oJz>lXm{Zu-=3Hq3t${g?kPYKxZt*=aueMVsQ}isjNZ%p(|> zCZMqhq@s-}P{q=QVy67+z}_1R#0)jQMm~|@23TVuu%Vi&pN3{f6rN4yQpj-Hk2g5- z+J86d;j4vf2g8O9XB_nu%4D0+n_KhshcB=A^Iu)*JOv%;{Fh~;RX5=39KEy^D^}9GJ!G3rtfMU*Z!v6`=1|eGLl*I_E0( zwd%^Wa!=C4&Kk;Jf0|@IVL#!Y{ujX%X)bfbc5_zPRHDc9{34`wpL%$?HDd#(c^)wOLb<3}1|E{m4);l#WniMGQC?(lj zRo`BK#isx2CN0q=_JgK2q3@nr!!d=D5Sg()uj>3Nxeo4m7U5yrp*cuRf?xP4WRw*q zG{bBI`RzV>;H0g;ufxgU@C2@|qbK(w+zg2;y>^afo-j6S50KN}F%1$`+q4G^5dG;% z@R6!O?Pn3$rs-qkY6%=g#!}?W)_!>Y8g-pU1&{2(Zn8V+Wevu6k@Q~Nt9r&sti$5h zH>k~A0$Qzj=J9N*^}(sXPfT52obg>;(JTyMBC_s`Ws-s1p26SHhqn_&+$f?(84Mg4 zQu_LNk;vXFlpvDzDWWK)%w(T_l`85~c+)?(c4PiVcgB_d(%E_YtfONG|#N|XN z0W$O~cCA`4Cb41a=%&J)kYSN9F87g0wbOUq1|x2UYotTUN#-^0qG)G93hMd#&Yz8Y zQ={XgNh*y%R%lY#*zcr296Q;dVgoWH&EKU5J%$+>DU<}`mP8$Au6=TNI{zdJk!TV_ z1n9mx7DrM$fi+nM1gH*y$n^DZ(s2}n%!p+2YB_uQPVN{ukdHV%K{{(@pZw_gPQ9S> z?=9hKmTVK499L_>+gf(&)KA%m*;91{CjwC+*E@+u0U@3-JSOP(iIQG;~Ml{j{)kn$rtiK^|^;{jzY_g>ypwCmd zhQxNsK@Xp(ZH9|u6e6C&&o6qtjNXU8=oRTFdZ{UOZzHKyMS=7ft3;_)RB*ALbMBlx zfm|Bq`Qp1A>jbwMNF148&d&Fnh(u^w5fP`;9H>P;t0&H>9?vA--IB`d9BBt$E8jjn#DUrDvm3bQj1!y4}sxb1PHshmq>(rL@LfM=`X-f3()k* z#Pf1M&>cIUo<0U3?=s2dE;QE%ZFFIdDtC(j${>bl$^=DJ0OwrVy$b0KkGC&piy$s# zugRnbS7=4Dt#&7_a~i)aasKUZ?F`gA^I!_9l@i*cktE_qma+D8YX%<;2nYZ^+y}yd zrphG1>4641jSo5X&(%k#!aOr5j9PQjyEgHceA#-ShI0aZC|~irC~AYmd$=T$ZjW8-P++7%|Dsbz?@2k}%OGdC4K* z%{eW{4K@4q*T2{@;ow7U9O3AaHAc$h^I8tS1!z)-?WgQz9G_A1ufFQv{mkQ1t+>)y zwwWr7W(TZ}sgOC`j$SwpsE2orM?=R^8!026{S^*rl>BTY~L9COkghwq;8xp!8ZU)urapKYPl$_6 zvE$bOkQTjuHVpvW+gtQGpbsw3PwC$H95gzwKqB~hX7o8T;p{%NLAgczx?(m&tLAoE z5ol#yqY$=-7OF8AKV%=+tFo@_U7Sz%6)!S-s)s?h-O>Ui; zeq|b{M1;7lz$XUfSN1!$oK;VfTV1#kGK&74O32?gi<4@DMd8`bppN!2EwuCG$lxp( z7I{DzAs`!zD}or@$Dx@h+V4moQ4tJBq7&R@Jb}V*Kr5)w>d!Tf`^EJDbhKiAMW9uT$YPICK3*JsevH2Al%WB(Cs64dxQ8N&gK zarnG=3^K|{CZ9AC5;c4}5GP+hCBrwEO`uev7fr5F{!EX|OAEP1nWpg;u1trM_NGM) zf%>l7Xc5_@(q9)28bTZTuAjd^D-ST#{t;5=&z+xf9|f8t&<#aAp6TOgg3FX7RBbKU zxfUtjD?}p?!|JaPnaCx3680QFetZ@o!ZK=YTgdo_kp{M7itEIGjx%ANV7R~|n*fWL z&gpd|bFQj`7jtzg1x_QW|>F4RFt7-V97)b zMY`Q&w60H~=&HxuB?J!<=N+#yOPv4eN#T_DKJf6Mm2)QNaOT?-4mMA6lbKGQ^&Qlc zReYKhAhah6e?Tgx@Bk(;mgP`1D7Far$RtKWt6ogE>CvMhk>HdrIsPo5b?0WNWXj}# z6To3I!A3C~%5xvLYVaK;w z0d}u4TRR-$_JZ``lCF0|gv%Bymx%Ys9_Cq|n}~^-QEo8ie9+u^rO4MmSox=vrpMe% zB+!#6Gkhtq@qC7=yxQ;_S1@DtZKiw;dYa3L%RFQ7Oo7tUw3@XyP~}n$s>sK_Aw%~{kb5s^H1AI`NZvc$z!u5H<1r%`t7${ zNY@hG_Il1vsrs{@+E8ccps6RKapOaW+tIDHI$NJXZ!&3t6KIZSH8K7Gwmf?V^p)Ww z<4zW@D4I{%=Ovi0$buDYjaXSK_~WXfQn$NDM95PU4$H7XeQK6}FmdX3YHZZ>Oicn;L2?M6!v+m$Ah`H1k7xG&Eg z@9bsjC~XYyo|H2C7?GqNzqm)7+;HXol?rEep|mNw+#zHr`nC;_fjL;^vmUFCKoNMi;See69JGb3w;$uY3Bryhc z24Qj`3T&s}=xEG|si!`**4{cO<%va`rCK9Kq^R?9&s;}4TieTC-p8IxdEN0?t1oXw zMhb|V_Mts_#m&w ze#w$0PP6ShbqXC+8lPU3R=fF+cSSweZimb3#+sT+8q0W(C<0!IRcO@qm*Cy4c&r$o zrP)VZ3llA%?~0!Bx%*7bh5m=y|<*2^drJ!2It<`Is3$pRz2;b>^=; zrVg)$mDG(rRlD$q>h$piD^~usdUYh1ej~v>Lgoj;11y7wmrr_yvfbRI=#^EEEY`Qx zm08UC-ZBCrUnb&Em0Gl7Y(w)+i|zP@IfzNiAfBhMlY+>HWQ~BbGIDcs{k<9&S!3+; z1xLI;GtnozmcUpgdIRX`-*8QdH@rtu`*dh-bUFj7I@`IlW=s_xUZr55k#1q2yDswD zMbAl+o+LV8XP$rgY|6&XuD-5*D%EA9;rM3yiO(BfX|jPwJh7sUNTRoUH+Dq_@t_8o zwR~x6uj1reawdpg7V}Fl` zh@i}=(>uKXBe+cQ1y9P$wS3PZs1AM3u2iz9=f~j|mD#7@zSLQ9AA*O%^aqxp zonM!G7k&=|vbXXwnPgymYyZ!{S=*i4KBKS*T-+2kjecG8yPPx0HYndU;*>?dsH=|Z4^36$K=3G& zo-2pHFDKaW{%?wg%UW`-rUS%aLis~DrLUYLV@zb9Z(HgC3ijnx+Mvo*Zp6V^K}QRZ8>~9cn{3p6k*H z=d-TG&)>4_aR}>89(>`7CY3oV4Vx)8RQ<3Uv>*mAuQVe_8K1TJI}kE|#kutK_W92q zlYKcFnLaKK%F}T;dFyx&5QyxEwGopoVr1e<8N#HriXivsZ>m3#R2PI7z<0%6k!GqIYPqkLBoBUC%37+jKNDK+3 zv2oearOTgRA;s;c)`bSe5_L@A2PI%}9<_B;)(7v=h=Vr+q-XN?wF!^Y+G3lp2B@^w zQ3GMo{S;iFrTB6(JVxAGh)%8uS7U({;VqMwE?ahcz!-3g?n)8pW=gK$aFRL_nwxP# zkDv+fxL(G$&lR7fR?QR4K@RWHDs#Qrm(MOq2FP=DxD#SlX=rE|Nr2KLLK#Xr5s9%? znZ;CcGkyB>F$SlTLG|!?rnOv0?pfKkPaBpo@o-`f&$5OuIu)6Z4juBWscSM{?q>0-|3$QXn_y;yd%auC`$Swb*l=K zf%XvB2OJ#+O<5GV5hImilnnbLsq#-sN&-Tjw`dV*@fL#0E%)M?pC{wcl9D&v+R1mk zMOVKV^=pJs|CM-3YRKtBzn!Sag^@6`a|QtLNx4Gxl4iesgzhrCiHgcgip4VV`+Vq# zl}AtXphWR4AcCqT(;M0D`+zcVDO%1P+Im|$sItIGmB`BMU z@I_}efrGPOp*1b}hzxW>mTuwFB}+2!#HgCeSS-oW8Cf^fys$k1p0w@CTHc4mdP&MS z$f=T#Sb)&Gd3ab5*D~L_iNLj;R@NNSZZ&Ruy0{X0xdpH;#^O>-7v}@+EHe{j8n&zV zC>iA!DWe?N95Rc3_I#%aCt8U{KX_M=s!Js0o_M6YlxEO#OicX-;8$ce*&xONzt5o0 z`}N!?86j0r7n($F9ACIc;w;Ze*ni*fV_H4mALSNtq*($Yas>g@>_w6J{@Je`mxHGc zns)kgQ5u#~4#v#H1j0hY%jd#DEAtVFKOFoM_Dh>{Q+DmzC3p|xWwv!;T0I-X49Wt( z9f5NQ?9ltnAAkPIaZ+sQZ9W=QW|1eQ<^?+mrOFX#x`>C5q>{ z=8qidLoZP*;K_xk;?=owgQmgmix+>>u{t;se-KwtLViev z?EtMul*u#gMlASHQqyShR|}ctbykiwd-o1M=q}0fC(BJb;*O*!TCi}S6F&idbqQi$499$%D1da%m&tNRvFgMd z=dWJ9+NQ-h?l-7WT;jUqA=(B{yv_;P1LBbhDFlm9E`Llsw}#eO4a z?#n0{%G`c-?e-4cejyiL^!)y)I7fb^co{zP22=eC2Bpk> z2_{X`-j8kCjJ>2bsq)^DeSi87`yf#EvTO||kYAK=?YjM*(chAsj>ln}5!8+!`yqqZ z(EShnVC0v!cnU4csEgcOJcFR%Koq$4W1VPKiiJ`LzvS-!#eEjUcMc{ERt?*!%LFWXg1P9KX6Mw_Cnqn9(5ZartgsJ9i^Z$4-3_ARq zD&k}Nlw3tjMytNI`<^n^j0tA?Hz95?)oZ6XKfE&4twCz`X2Vrj0CP3_Q9D zy05z;S-}tm>E*EA?ntBni}3u*=TayEhmP~_LS}{)$hxc5Q#-Jgc2ETN&_ip3@ zEF5mOVEDZIUbkNE3~4c8!i1ntP4QSz*C+9;L?6AoQVI)1s(E^yN*j{DWy2enIrYzv zw3v}wPe65~F1~zgi52Appcma0UPMLG{vQ4faggwwx0e31W=-bH=OpA(o;&P64Ud4K zn7y^+u2{jZBpfBMC2f&9-;s3XT1nRt-&3C-J24@%?RxA z=uP9*Xte0jGp-?)<*zz=!IbNdtdYjZz3>RM2uVovCXt3wq`iPB{ri{C>d!yw4p>_r zn`gIU#7cYI6}s^mS2URrV152K0tA5^#<@#_&A9Y&W|)petglI@r;Ip$@QviY@GJtt z4Tf~;%@Qw^HGO|OWj_t?1l*hEn4^Yj+)n@df@P@1_70!6*ofEIM>R+lhS60|kNH)7 zeRro_WU7E9&MJvxKo7U6r8*j~UP7R^shJfYZb$y%coM#2?(O6PwRW@mt)2fpkb=L- zIOUuh5>1XRLzZP2Xu(4D`}kwzmmkS57g001utEN%^S6+^kI5`jrD*J{r>s!FXcPhG z9c3R0V8zpwN+odl0Wl2J(c+5aD~^xnS?hV%Ecvmot(B@J4n}^T>{`er1|JT<@lGG8 zI-LV#o-4xqQfIR}zThXswl0#RUqFeUL#;1(j5rxRmmW$P!9tY=iMZW7=WXxRF?idp zW*<)Sre^2Pp52GFkfMZo%nL!-mf#&b8btp)>rjV~5R+2f0ZPncRYzS5c7uxd*4MxG zUXx9JcisoPO{;AMm7^2vqfPdE;4$MBO32`0+6nLQt@?LCMTe}q$}!nAjT3T__IMeMpu8->nvgwu^pKHI z?_ampeqdZ>d_l%WejE)dYph}@iHyXfikn{SByS!FSU4F7Y}2u0#}vZcUPgb!M-6(7 zfAIs0KczV@TtZT{AU=W|#Cn z`$8(+l+oQt{2PRB`m#cVFbZ~mP`uR)4r$r7x^Qj z?UG{+pbpVOm_**)d-fcs5}1Ry!QM!BegEFhlieEWX8-;)|2y@}KOZ*AfB&h;&i@W8X#CgIPd+@! z#=mO#{|hueod5s+gQm~_$Dhb>{0~!AV~8mwV_{zcLyK&=l(ZRv^8S^f*X#aYXOhXT z+cEd~ebkov%U#fC=P~N{-2Gh;OwOmV3dm% z=gEk#9aisEnU!$`$v3|rPT6w@RS;iigRiKXho%O>@=Bb z9u*`RLrDx7N!Co`Z-f;7fQ)aHhlgd6v%aLI&IRKp6WsAXa+Q)3K!LHCI&~*=q@!TN zIlIJ6dNsV0MrS4Y{v|Ss1BhnzW}F`MskOeoevy%p0C4?)_GKI#pX~#ZvcayK7U8lC zpOG@Mcr@V#fTc39bPQD`8P5tuxpz`*62ou=fJ&>RwLsDrPlrI^Ci$MEa#91t*z}|{qMbYC_=0Ic zrZWIxEP&e+H&SiPmt+^WDKmh%t8BL(%WozxjeGu@1||J|WA$Ac4rdpov55(2U6qNc z6GOKjUlw`<-obfk3=erW!h)2UW_zoV3;()WtHEVKxzu! zy_>p0Zv%q@=bJ!>aK~b$9tsCg9Fb@;xQ@8<%BWLW71i78H8qczpNK)(GPk_jOe;|Q z=c4AgXzGfw3aXbN-r|{M(0R#l_JJ{j=-7+#E8clRBsl-bS=Etu$*Osqh40HqhW{=$rW%j%Ce?Jhlg!cJ8PP}7u2#{|0LO+E66r-f;xmh7G+ z!{2`B5=pn$E@Co5u%{dTNV_23nK*Q&QFT?71;11Uk@CQ=Yrc2$^t^^@!x2g$(PKjH(2jNA|52^(o4O=b6_N;HQ8aM-RcB=(oLH-H6t)%2J9Y^a#<1S7nGF;M1@5) zn_J6yaYKHP;mC)L8oDu9`z8H9riU3Gtkc5WQs;ydv$P_LjpG*2i5! zxV8NfI2e~Hmh*s*`TEUfVFApdV0-{FXZCSQAwodLOhvLM+YOlh7w1$$eIfhNFJ(d5#Qj3MwpCo+ zEY>1TP1ISmvM&>$BnFnrm)w>m{0ShIy+V2?#k00~Cp8-I@Q`qW_xn^M_0R0gj}e2@ zG2qAjEcp3=exbBN?}Vcg)PdnBl+T190{2ZOhaxlZbzHZCOma$v6VK=l^f9T``2eF#*Oi5in(4uU0KE&erQtq6iqG8NnlmF{r@ zU@$7p)2;~4&6a?G{1w$&^p}qxFn;><-N1-5^R;DG8bu=)qkfoQ>Ojlx6j3#DHiF}H z1RlcleB1qpi@Y>^qy~xCidi8PTLXhaFT+#9tAl>0d1&-#0ivOq3=Uz5-i5paxv&9r z-uJu)fGX$Z4Jm=(VPr}qU`VgL%EqVf)8t>1r|zDd_w@8^X-)Yvim))x=abI{F{ltY z`C1q0#Oqf04wm!HlBJf3A>k~t4u*bE2n6&YcIjVy{`vJq-wU(q@pdf>&)^msy;y#b z%xy6%nlYc1a#x$&qWkw3A+atO%$PGk^$SbvNh2t}K028$#qcG`Y#&Tb|1&PJHu1(_ z!J8FCh`p$EwGH|3w2Uj=tv7 z`%>~IxbYcK0Xh3==tVeV_voc}38^ z+QqIzNQI0KmHDOuHDU}Oagv1PBb7adCuT83i)i0W2&H=UP}GE0tyED(B&+ne;|Omx zo=5|x)cm3DCF>Zm%e=qcpMKGFAM(kvanUyrNs$?S{JNb@PZ!;&iv%0UD=ie-KlO?$ zC$69y;Uu#yFtwL~>eEb-FR68X? zqd+0$b-IUCyG&1Pc>88Xyp0sW)Zr5jT4?f=>Zl0RI4ob>(ZV93eqqgLU-fRw z{PStklo3`>hA<4g<&b}#=G!cet_`lNn7J~yV0DwHC%kedi>wa~%il2&FoTc{+LeKg zA=PEN67mQ7^=v42p>K0^z2om*i^!UB`Rq5ZaF?rLR_hsN$^E1 z^4M-~Z^Q<#aN{0#Q$L|Xm{nLP!`>``duV1TeK*`{NB(kmZ(B!dXssOMmMZ~GAuoqf zzCf~nUZK>~x1D@KgT;@nTSri(DWg|W*P(x-73k&v)d%59lX9x2gc1P{+yzp{zahjh zU&au&${jU&bRU2_?!&xcB;7NU(pcvD%FMRx^440G+IgLd#+ecH0iy&xFrXxNm0p4I zSD#Yntb8jxxRjKXQ9H*OH+y;Z?AeF>8{C|jzgu*IO1iATw)`m7{BjCc7hWWwQE}CI z>RoR7EgCrQaoriqh68L6OUnsrrERqnTU?HbvqVJ04=k(3ANy1@bif#6$Qx}+4 zRa-~U0xhxB)xi-4Dz#a^-W9DGyu5E;F9A;|YMqcgALs6zWWNjyRwCS&T-PJWz=0tf zeSM`d^t%nqiQM%(E360&dXC9Y2$)fk?0nRGuQi`&RTTXB1tzrXq)DZC%l0RF`}gk` z)CMxOZmsK`+eO&feTX^EMtfx$-C1*E0s`Jm2FL{anEO|5UQ%tP=Zj*cj$NYgelL=& z#4_R3z5NM^Agm^Ly}$%o7&5N*DUFPv}zzTueU!ifZpjx%nmDQC$W;1QR@yJqWJ<$_AJWM zLVhFS+xlQA-!sw4$!TwTJ-rkRIi0yQhyHt!VOCzo_(D}m(JmD>qYU93IBYmxN2Wz( zxpOZ3%1_-}x7>YRlk1g5?H~akY%_Vk*|{rATNpj_!BWINd$vG%8O(=a>aKm9xW$xq zI}Gp8??}8dbhSmJ*YG+0_x~CJi_Vj8C3=jL_*L*CqFm7sb6c=LDi7#%50Zzf8U%$^ z8pK3wZHHD)ps6&=PLS`)X(~qfh0>wnZPNWee+;kbqp=`zY<<=_HQ*v z!Wpc21nbbJd-qY))P$4=y0`~8Q0-|T*fSg3S*Ggkjg1p$cAWUjvQ(av%%mni?jG*( zSky1BCVIFCHI_l|a_)~X%vIcpbm)WR{rJEC$uN3ij>paKtVxO^8!NA(-#aSfAm(QoyjW?hVn`=(tGmEVl)I&80fG>i z#E#1tMfY2e<2nlx$F$r@ho-C5eCFV@c(^3`eP?6uBpU%Fklr7PdE-Njtl6K378h6x zoGi>_Y0Bc}WCSZ-@CaN(HF>yYyLJZ%(~NHR+tw>eayAm-c17g4#jnS8ZhBfzR@3)T z;K5gOf%ZrLQ;COw){`t9dzHcLkT%QLKx>qw@XXrAE~k;xe*J)5O|^Nu!2)>m;8$_7 z6C{NfFx*$ek(QR0ZQAU+ePVh0mHX;JKK4QJWRLeRL2=odCulim&H*Vhv9HG&zx(Vh z8|y}es;#D`cJJu+w@^fi^!INm$Sj^l-!4VxeXW1JMg)u;!CHJbti#dCj86D2gD`8% zxEMu$L_tg;wRBK|Q*8zVEtm%+RSYpS6u=?y)0I3g3rvGpK$*-VJ%N2~kM9)z7$$3Q zyeD&uL6^ISpGcG8Hd#xl*iuB({WO>n6e$9$N;tlaNOCuL;XXn!;Sn%-ChD)h{xWe1 ze|i*f-vK<~it~CQD$V(Z#sV6~W$ej<$J129*=KWfDGi} zWP;1kbbj)BS@l{H@%HK9 z!nsvsc!YFOEcPZ+GINL*>rKxbjj~O++*QiTBT33H zPB-OFgo4M-n;>ntfx3P;MYBGP3>J~Lqxl)8-cQzB%jhKOTR@4xW>JK;!$~Rf_J%Dd z872e=$sz%%KAElR*PNzXS;k%t~`vVz;a9^n8siJY;p zBk$_y`fG}d*{_1Jx^3;0LYKhsIxo-JapREV z>gP!lWl;2rT)QH?dn-G4^66+#%WUJc+<8DzN4w-m3JG~Z+#GxmH_MN4%#jy$E~$;@ zq&luwoS*bb5pzTDxGPBK?(W4(h@wbjCVw5?kdix@rIDDjYlfGr-@RlWJWdt zABA~srSzU%eRf6^ZmpBY2(M(6kM16!1mZ^a&dC@@x_a#4d4bPt9)GHqduTs6=c~#R z0nY}0!7nacc!{2%sPDTQk5$#d&8LT<>rIaqorVLoEX$;@A?OEr-IIO8mlps|d{?(6 zjAjC;0;0P@2E3weAa z;l#@D7N0MEbS1N7rXT=h3l>N9?D0YQf2LHyb%6!zzGzruwEEkU@9sAi7K_)Cn%I2% z>)6XgTk{4CCoH_0K(uk_;qc=~X1T_O1_wt;Bb@3b{mo8>ccO@iXOnWIT?ZlW8tnI@ zR5Hvhz5D;L4EpQQhohxE$J`~N#jj*HL$eInoY&B7UAGW#NV8|0YyK1X%rZ>+>gLws zfUuOw@j2$J6xUF zLc%;)_XE6Xu8s0sexg0>(YUWFg;aTEDVSMK%e0f7ehD_SS)E*wIOi22KKECIKtVrR zUh=AL%W$U-aP1!7aDH`wfWYRj6Yr2@oqO>(HQTgzTefdMK)U9%e8(R#7&HAdZ&XyY zB8f@c7x6gT?3mbWEqiMpDg&0Gv1ERuw%xgwGP!9aj#J#d>hj5d&TxM6&s4oxxPIL` z4^zV_GlS=Aom~nbC{lN-Fv8wQR@5M|p?#i5)a%yAT*7UX=TF;KcU+cne`6&B%A<2U zAfu68uYV4@C>lvYWb^%=_IY$%nxR z1AFE_fAPYM#A5wI%FHxEZhO2GSDy3IDlNyg+V|ztP(1(<>48}H96&kx%eyb;)F&2Z zTB^KJf^ju071o|a(nz##&Ao$S`0GD3dPn<(2JU>0R}n3Pwna3vU7a49+&b{hH)9Wf zmB<(=Y2sRHak;|)_j<+wbSWC5t*zZJtM>xV{2(gUjz++8aCshtB>k=MeHO2vAfXsj z3IXz_ZNC!)qcX)u(1TIccmL?2Q2Z)a0737>xJDa!_vFtXVu?*!GQ_^}4f;13g@uJ} zX6AR%aU7gok+_hK94pmJYKbLt7N^s&KV=>50m7K9Ds6U{ zQvLSr0h2?2zojZnnXIs0X`W0zexfvH;gmE}7V-9lOK;ki zPHFaB&_!t%P?Sk|>8Rs}9^T{i#TAh~8dv(@>AXDMn+9d|+iqJ9?$%r9H|Rxndwg#a z>Pb7fcD+v>nuZc2XliOw!K&7@oOhFuSv##P>C}t}eDGpT>y{3&Fz=UGi+FSRbP2 zN5ljAXI2)Rs2AD&9!1wJ7iX_Kg$PA#g>PStCogZ;iTLoI@^i@1;1oCj0YWvW3Lp$y zbowlF6Fkbd0#8MM>E}y6Bz#7SPEA|1nA{MZEFM$!AcE+zOo|}{I#QUId1hlqOj&#y zEehV12On<>9iYNbg=fqjk>(+32d8lVh#M117+pvGloI|+e_@pQ#nT0Ec5z>OOZPsV z+AI8V?$P2`b$TeD2o01jiT|ta-2ZCM*EYUpj2YV=hP_P_Qwp)O9a3q88HEv1X+$CA zPzmKw357Hm3<)_)*h)!65gm$|?+ z&*wf|*L7c?pfCn>7cq?>eI(4^)8gV#L8mDv!W}aUE0>mS@DEy!y#y`Dm6xyEZ_m#h zZDrE#b^4v2M_T^qG2d^tnwpFfQ=a!Sr~0%x@^;j%9c~5t^p_uHvMsdEu;|U{F>k3x z1gX}EWV{1yq-*CWWpYt8wj1-#D1(D!CUj$-4%rQI(YUnwGv5E2s28DWFniVG_H84+ z_eQJ~pG{wknGMpD3QyG0uUIp)>U5thAKj4^`uBjzhYw>8+??; zhj`oF$*osLc6N4>?~cm8Mn*<9S90A8hgYc`B!+WQGXDq0=Np(|RE(|SUFu1%wS~#T zz)IVUv~Mrfp`g_w$DSQsNY#j4-O46xx}zA^9e_cTIGC_P4AP0hdDKACO>XN_O87aT z7gr?F68Oq_WQ}xMpf4_7R#pwM0lgxaWF1)gG^b@6)Dy{Rl0XUgn%MQx384^v*X~aU z!XWRYD96U{=y~w!zA~4}rUVA^fFp*EuP<8hy4Gcz2g9s%1-*hxc|~=jhr#(NnU$_r z9PcDEMC4j*9C0*rbq5l0> zY8xB(hlV<3{9Va>Dt_wO%j$ilpsOQNge1p5Zr$pimq(EjiV2|mC#Gt8j)(jI=EH#J z8HXg>DKnoLUUK=o@I+0&Lm|m;$F>mL7E+;&@OvBoVmif6Hc}@cR6scHET1e&9&8P+ zWj7)4H@rGYa5x%)^M_Zm4J<-*ylQ*qrvYOb<}Cik`EXWd6LT*@9e-U+jE|%o5St<< zAA`#h>Dzu6>(fxTf&rkixQxt!ev@e;^c_OH)y8d-auKpeRF(qdv9Sh#fYB5wfy7vs zNy5QP0!3^*C<8oh{^eZzE>Cov_r54?WxF3Nc~ajZT|WS$)rLAsEQ+tp7Uzk*W6N?~+RHZ2|& zN8jYNcG|6#ck?Eg2pufkc+j4P8Ofm0Wu~>1n*&q3k2E8g+mkwXxB1n;W8eDelrk#* zxtFz`0|zF80kf}?DUB=0&oYpq?+OS3QEQFRIUA&4*4bW8c_%14XZ?~I(&6*ZFLW@K zo`tlX9#>CXcGtMHFe|k@8GPY3lZQ|t20by<8k%!3=LjfFzwzV8Z_{?=VBP3EO$`nA zu8-V#AmLUahl-;w{4^S@LDab*owu)FAIDO6x%`>&Qm^I9r!u|n?ssu%n)?5oripdj zJ5ATQppW4Smp!$9t%>P`L9tx}Nc&PvEu0`JQCcm}ien;St9UX`v71}CJ)pj^ENeV@ zEWy!j?z|F1|MkV->(=N!bs{Olgb4QfZq`AqymR3sDYGp$ZFN5k>cJ7intf2f_yQpg zI`Rh?(%s0MnRc%B)FQevM7v}2rcEUaRB|=z=O2pXZVT|N60e>Blc3qe;c;8MsEC5X zpzKCX{`e!-^#yfm)!KV8jdqkR>I}pzqqx#-rli)*PWt%X2GE+uG`C1&KVs#O8I(Iv24por2CkM;d!P z3-EA1LrkCh?Jx7Lca$Mtfnto$Vu_rT z>gfy!JfA6jl--B!+uG8KLCW7arL_fVpR!bx68+{T1Oo!FJn*4I4br zb{wvHdw8RJ&C8;aCzl!6{@r*5RW$q6@~HiEQp-45cNG6mXlR*2q!W*eT{$DXD*yZq z;|A0K*dkU=-@f+A6GJU*sPr>ldF{)4v%fnqVGiaOA`07fgJTGeE?kn@T-8vtV5v{S zu$fK`UONk2wZB`P9JpS$`Hl9|%+)cRxWb(_4`NVVQxJ)%6D}Qd=ra=P@tnFA?jPAc zl=kPJ88Dv~yf9Ek6vmFVY;{HGpjLVbawg43c@K|0?}fVm-Iwlj&H!&;Yz$i>3XnZ& zop*{>$F+RlK6@;V7U++nW{a$&;^hIKzyXNv5{|^gF{zblLL}LR9CcLCD-eWABRjTo&nyeIUtUxfw zy(>VgxpdSVY~Ws}!RP@EUaT_3&}71*bnggpW>4hw7dteg)YNF2O6`du37jWRfCw_x{zP zM)MmlBZ`?3u-N--+>Hv|Ej?E~`e>VW#KFe@S#o*EV>W0m97$hug63UP@(JiY`w*>e z-E?FwqPszusl&AZ)}0_UsjqnWL%fTidY9-gfe4DuRprELy^Ze<k1C3DoH0xNrKL z6JcS~H_kZo^VY4g9XxE=p{(V&1G`z(du-1h^bZD~yk4{K<&Re%YG`T-8|^F`{;TXmMflAJu_@fodjS z&mVCau&6OGx=nFGDKXK~avBKaReD^aHgZ1qJJz zT(0CH6pf#$ic0^FGw!a}O+CuSY9YbM z;=?b_m~3Hj$3wwUAlG1{_pia!5Gz`Xkj_>31B^ywCp@~2xlki*XJh0ULobU?<8@Qh zBHRMZwC`IirYpF;!jTbefG*3J+Oi07vURN9 zo-4W};&{w(S1Mq&j~)1i+(ez-{$EaNog=rB+XF7=R3?is1IJi_~#w7guCKy3#dfT? zw~Jzl*$#K#=Klh(9zB{v^LPE_YoA`Y2U5z2PB`pB;~WG$34S3^;ilsFgAP>`mMG0q zUnU6B*?mPPG=%u0!wnB$uYj3GQ4;4zCn3se-@bi+AYEoD(Y^hS;LS2`nl+@2%=ZaP zl{1{-Od0Um?m^yAniPS`IF#U-=%kMQ@nE{v}RN}e+?(>@2_yG#19A&&yA8pvOFGB zdKd)i&5GYy_8PqV9m4O~NruV=;J0lTg)*KizUeWt;nodDa2r&3GsmV@-m!0yUW+VG$TlMDlPraYCXGe%D-ISRM z_@INnXnHY1m18`*7xu=J=$M>5nPA5KHge6-2pw6;F@~p*m)H|uMHEYQ_mWLq0l{EP zy05=CNlzS{!pk30bFn#G-hR#_p0T`pW~bL4oysyA^I2qHAD!F3Dy<)$P|j;|!5=R` zcq8sBd}jO*<#v|C0cDY{aQB!HSN~;IM(SXh3>LkdV9gU$)>8OfU2G2zTu zH1We&hn4RKi}kA+mrYm9F%!oSs_fX1VGP@{qy}ddGY$&=konEPTxtwa3=*1Rdz{2B zYlBe7N-fL9a4^1^%M*5o33!K&PU@X>FP849$JaKyLCKpP81fw~5;6cYS}W0>3mpzL z{|tB_?wQ{rGgwiu3lZketUA3i7*2Rp%PS{(v3n)SqUTlDzdGM8h0~F&Av+ReVPP=u zGCYDOVB1bl)I;3yMRn)P*|%){NVIH-SZ{e+**5kkdE+x1(;>j+`meu^Cdk-OXdCey z2bK;*qSt%DJ~$Nq{K4_nVh~$Tz|2q1WogS62=npy7I*!cmkvV!9c2Gi06KMH;TI*56c! zLKQB|??#>q#%w=D@nM#*F@PsCRjBbXCcpwW^D694O*IFKvS#L(8!IED{ehUoZ;67w zUV;TQ=rI!9Q1TF#`y|WZo#i5E=^Fivz}9BCB|wiopX_Imoq9^C+r;StMW^p0?;mDGKCivlZ=x zC5xQWmuYotk%yomv^cx^U$st4?{HAOAB=qtX{Mg_C6=1^2cW_^&PIzwAp`b1{YO8| zgmmJEAnxR$a49Od)l-RFOfpWB`^1ZC}fN z!5vHaKRb0q(m|sVbnjzNlKmyGampXbzl}pjs1%S2iHP(mK|0`($T@Fhd`TwJ9wsg@ zR9WOg8y0(&gL2|c(b!4V(J4$(s@fql)QN-`trgmI?2^e)^5M z0+HfEH}D`VLCPBj%MVRI>Y}B!RAayH4_5>CzW-&#vepl9%um086M_ySUtk8c=bGoE zN7_jsIi?+tHL*u!Ny+>h6P14$-)j6IN%=pf)f2Q3_EsO(93a74-l1ew%foo`*zMgUz{*Xjp2kCE*UtfmN%bJZSG9dZ{y@?AFV$+X-h!{{l)%7+tywj!RK1C(oh)G|&*4|hBWE)zvt5fnyvSOiS;?v+jbq)21P zs*kyNV~G>XpKt8XAFkK{EJXsJerHg7CnI@-%Ky;F`l1XG4LWP~Ha%@oU*%d~HrsQ8 zrTO|9Jr#;i9p#JwOUt;GUFGli@-N_i9S3YJ$00B%3~9wFs+|?lvj0u}#3vqF=5MN9 zJxA-Kgn&JLd~VkF`|>T>hG>zTJNeHJq(%37!5%M$=F- z|5pIH(tKh|imHQq5<)2>sGKhccs;GIUQ**Jm#qBqf3tP$A}WQHZOZTb@I!P|?` zE2{gVq7%yB_1DmySEHIomha9fq;M92U{zs~omk8|rX-5gpe9!JH-A$-8-=qc!edUO zvImuD7%UiO{8EQs{2asw+z~!TULAVJ)V^gI%bLZF`9~5#j@rA$C zqAUI%VFek5-gjQx+xov5LzI?8B~#rXoU00`Zm33cMM!j`TzO@toj>_-W2YOF_r;r^ zhN@Vfid|wW1@mrFnr4&);*Mw0k%;nBDiSdZy@44=cEn^Y8JH8rvgQC)SvLHGjB{r` zS3!OpD$ZDCt**}Kc;yI*ws~h%>7tk-%#a5C;DPlls3HyUb8lOLx2}9QPAnp(@Czgme<-9-#?+zI89im`e zGwg);`=7G@&sB#rF$6)_{Gu1HN!)fwU8+xQ;s6lBPf<<}lJXP3EpK-9Y%JC)*fF$V z$GT-EEx$HTxYqpYGrqNQwFwa-luwDtv<;@&886r17LXlo`L&RDP#=lGQC{}#0-=q45NxW9;Bfqf6es4FtdGUqnTk#LyypS%mZA^F-sY|Sg z#j*>}jribyC@;_S*Dlt-6@95(Ast7t5ajVvRA!??m$R~(Dd(ljqbsd+AbKg5P39oQ zO*)Yl2Mq`$a@1EiYwvbiTH1#d>|bD}+ITzrNKH*eDm|7ti0FQ?Y8KTav|A#Xm=>?P zya%|W1kdV4+c`k}Np!}3F$zF7-4K|GjZAVy51Hi=Rl&>+Z+2~x&nlmNVcW;&vn#(< z-QWimWJW_~#~?-ZqCc^S!^4WIS`C#SUHO^Y{`C*|e-Wls*ZhA+!Yi%qhB*G1xi9Yi RX8CRNaaP|)kD0Une*r_c+CBgP diff --git a/setup.py b/setup.py index 5b7d12bb373e3..8b2b4f7e5def7 100644 --- a/setup.py +++ b/setup.py @@ -461,14 +461,22 @@ class precompiled_wheel_utils: "vllm/cumem_allocator.abi3.so", ] - compiled_regex = re.compile( + flash_attn_regex = re.compile( r"vllm/vllm_flash_attn/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py" ) + triton_kernels_regex = re.compile( + r"vllm/third_party/triton_kernels/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py" + ) file_members = list( filter(lambda x: x.filename in files_to_copy, wheel.filelist) ) file_members += list( - filter(lambda x: compiled_regex.match(x.filename), wheel.filelist) + filter(lambda x: flash_attn_regex.match(x.filename), wheel.filelist) + ) + file_members += list( + filter( + lambda x: triton_kernels_regex.match(x.filename), wheel.filelist + ) ) for file in file_members: @@ -648,7 +656,7 @@ def get_vllm_version() -> str: if envs.VLLM_TARGET_DEVICE == "empty": version += f"{sep}empty" elif _is_cuda(): - if envs.VLLM_USE_PRECOMPILED: + if envs.VLLM_USE_PRECOMPILED and not envs.VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: version += f"{sep}precompiled" else: cuda_version = str(get_nvcc_cuda_version()) diff --git a/vllm/envs.py b/vllm/envs.py index 8b954fa14f28c..4b594e54f7219 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -78,6 +78,7 @@ if TYPE_CHECKING: MAX_JOBS: str | None = None NVCC_THREADS: str | None = None VLLM_USE_PRECOMPILED: bool = False + VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False VLLM_DOCKER_BUILD_CONTEXT: bool = False VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL: bool = False VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False @@ -462,6 +463,10 @@ environment_variables: dict[str, Callable[[], Any]] = { .lower() in ("1", "true") or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION")), + # If set, skip adding +precompiled suffix to version string + "VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool( + int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0")) + ), # Used to mark that setup.py is running in a Docker build context, # in order to force the use of precompiled binaries. "VLLM_DOCKER_BUILD_CONTEXT": lambda: os.environ.get("VLLM_DOCKER_BUILD_CONTEXT", "") From b78772c433515a22bfeeaea41f3524002609e264 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Wed, 3 Dec 2025 20:53:44 +0800 Subject: [PATCH 03/47] [Frontend] supports deepseekv32 chat template (#29837) Signed-off-by: chaunceyjiang --- vllm/config/model.py | 3 +- vllm/entrypoints/openai/serving_engine.py | 9 +- vllm/tokenizers/__init__.py | 2 + vllm/tokenizers/deepseek_v32_encoding.py | 456 ++++++++++++++++++++++ vllm/tokenizers/deepseekv32.py | 148 +++++++ 5 files changed, 616 insertions(+), 2 deletions(-) create mode 100644 vllm/tokenizers/deepseek_v32_encoding.py create mode 100644 vllm/tokenizers/deepseekv32.py diff --git a/vllm/config/model.py b/vllm/config/model.py index 5de97697698a1..655b7c995f6d2 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -84,7 +84,7 @@ TaskOption = Literal[ "transcription", "draft", ] -TokenizerMode = Literal["auto", "hf", "slow", "mistral"] +TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"] ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"] LogprobsMode = Literal[ "raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs" @@ -141,6 +141,7 @@ class ModelConfig: - "hf" will use the fast tokenizer if available.\n - "slow" will always use the slow tokenizer.\n - "mistral" will always use the tokenizer from `mistral_common`.\n + - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n - Other custom values can be supported via plugins.""" trust_remote_code: bool = False """Trust remote code (e.g., from HuggingFace) when downloading the model diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 67291f45a9251..9642024dd1e9e 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -105,7 +105,7 @@ from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput from vllm.pooling_params import PoolingParams from vllm.reasoning import ReasoningParser, ReasoningParserManager from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import DeepseekV32Tokenizer, MistralTokenizer, TokenizerLike from vllm.tracing import ( contains_trace_headers, extract_trace_headers, @@ -1128,6 +1128,13 @@ class OpenAIServing: messages=messages, **_chat_template_kwargs, ) + elif isinstance(tokenizer, DeepseekV32Tokenizer): + request_prompt = tokenizer.apply_chat_template( + conversation=conversation, + messages=messages, + model_config=model_config, + **_chat_template_kwargs, + ) else: request_prompt = apply_hf_chat_template( tokenizer=tokenizer, diff --git a/vllm/tokenizers/__init__.py b/vllm/tokenizers/__init__.py index 42487f5f51651..67a6d7c8eb3d9 100644 --- a/vllm/tokenizers/__init__.py +++ b/vllm/tokenizers/__init__.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from .deepseekv32 import DeepseekV32Tokenizer from .hf import HfTokenizer from .mistral import MistralTokenizer from .protocol import TokenizerLike @@ -21,4 +22,5 @@ __all__ = [ "get_tokenizer", "cached_tokenizer_from_config", "init_tokenizer_from_config", + "DeepseekV32Tokenizer", ] diff --git a/vllm/tokenizers/deepseek_v32_encoding.py b/vllm/tokenizers/deepseek_v32_encoding.py new file mode 100644 index 0000000000000..72f43395b192e --- /dev/null +++ b/vllm/tokenizers/deepseek_v32_encoding.py @@ -0,0 +1,456 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + + +# copy from https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py +import copy +import json +import re +from typing import Any + +# flake8: noqa: E501 +TOOLS_SYSTEM_TEMPLATE = """## Tools +You have access to a set of tools you can use to answer the user's question. +You can invoke functions by writing a "<{dsml_token}function_calls>" block like the following as part of your reply to the user: +<{dsml_token}function_calls> +<{dsml_token}invoke name="$FUNCTION_NAME"> +<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml_token}invoke name="$FUNCTION_NAME2"> +... + + +String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects). +If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example: +<{dsml_token}function_calls> +... + + +... + +{thinking_start_token}...thinking about results{thinking_end_token} +Here are the functions available in JSONSchema format: + +{tool_schemas} + +""" + +bos_token: str = "<｜begin▁of▁sentence｜>" +eos_token: str = "<｜end▁of▁sentence｜>" +thinking_start_token: str = "" +thinking_end_token: str = "" +dsml_token: str = "｜DSML｜" +system_msg_template: str = "{content}" +user_msg_template: str = "<｜User｜>{content}<｜Assistant｜>" +assistant_msg_template: str = "{reasoning}{content}{tool_calls}<｜end▁of▁sentence｜>" +thinking_template = "{reasoning_content}" + +response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" +tool_call_template: str = ( + '<{dsml_token}invoke name="{name}">\n{arguments}\n' +) +tool_calls_template = ( + "<{dsml_token}function_calls>\n{tool_calls}\n" +) + +tool_output_template: str = "\n{content}" + + +def to_json(value: Any) -> str: + try: + return json.dumps(value, ensure_ascii=False) + except Exception: + return json.dumps(value, ensure_ascii=True) + + +def tools_from_openai_format(tools): + return [tool["function"] for tool in tools] + + +def tool_calls_from_openai_format(tool_calls): + return [ + { + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + } + for tool_call in tool_calls + ] + + +def tool_calls_to_openai_format(tool_calls): + return [ + { + "type": "function", + "function": { + "name": tool_call["name"], + "arguments": tool_call["arguments"], + }, + } + for tool_call in tool_calls + ] + + +def encode_arguments_to_dsml(tool_call: dict[str, str]) -> str: + p_dsml_template = """<{dsml_token}parameter name="{key}" string="{is_str}">{value}""" + P_dsml_strs = [] + + arguments = json.loads(tool_call["arguments"]) + + for k, v in arguments.items(): + p_dsml_str = p_dsml_template.format( + dsml_token=dsml_token, + key=k, + is_str="true" if isinstance(v, str) else "false", + value=v if isinstance(v, str) else to_json(v), + ) + + P_dsml_strs.append(p_dsml_str) + + return "\n".join(P_dsml_strs) + + +def decode_dsml_to_arguments( + tool_name: str, tool_args: dict[str, tuple[str, str]] +) -> dict[str, str]: + def _decode_value(key: str, value: str, string: str): + if string == "true": + value = to_json(value) + return f"{to_json(key)}: {value}" + + tool_args_json = ( + "{" + + ", ".join( + [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()] + ) + + "}" + ) + return dict(name=tool_name, arguments=tool_args_json) + + +def render_tools(tools: list[dict[str, str | dict[str, Any]]]) -> str: + tools_json = [to_json(t) for t in tools] + + return TOOLS_SYSTEM_TEMPLATE.format( + tool_schemas="\n".join(tools_json), + dsml_token=dsml_token, + thinking_start_token=thinking_start_token, + thinking_end_token=thinking_end_token, + ) + + +def find_last_user_index(messages: list[dict[str, Any]]) -> int: + last_user_index = -1 + for idx in range(len(messages) - 1, -1, -1): + if messages[idx].get("role") in ["user", "developer"]: + last_user_index = idx + break + return last_user_index + + +def render_message( + index: int, messages: list[dict[str, Any]], thinking_mode: str +) -> str: + assert 0 <= index < len(messages) + assert thinking_mode in ["chat", "thinking"], ( + f"Invalid thinking_mode `{thinking_mode}`" + ) + + prompt = "" + msg = messages[index] + last_user_idx = find_last_user_index(messages) + + role = msg.get("role") + content = msg.get("content") + tools = msg.get("tools") + response_format = msg.get("response_format") + tool_calls = msg.get("tool_calls") + reasoning_content = msg.get("reasoning") or msg.get("reasoning_content") + + if tools: + tools = tools_from_openai_format(tools) + if tool_calls: + tool_calls = tool_calls_from_openai_format(tool_calls) + + if role == "system": + prompt += system_msg_template.format(content=content or "") + if tools: + prompt += "\n\n" + render_tools(tools) + + if response_format: + prompt += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + elif role == "developer": + assert content, f"Invalid message for role `{role}`: {msg}" + content_developer = "" + if tools: + content_developer += "\n\n" + render_tools(tools) + + if response_format: + content_developer += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + content_developer += "\n\n# The user's message is: {}".format(content) + + prompt += user_msg_template.format(content=content_developer) + if index == last_user_idx and thinking_mode == "thinking": + prompt += thinking_start_token + else: + prompt += thinking_end_token + + elif role == "user": + prompt += user_msg_template.format(content=content) + + if index == last_user_idx and thinking_mode == "thinking": + prompt += thinking_start_token + else: + prompt += thinking_end_token + + elif role == "tool": + prev_assistant_idx = index - 1 + assistant_msg = messages[prev_assistant_idx] + while prev_assistant_idx >= 0 and assistant_msg.get("role") == "tool": + prev_assistant_idx -= 1 + assistant_msg = messages[prev_assistant_idx] + + assert ( + index == 0 + or prev_assistant_idx >= 0 + and assistant_msg.get("role") == "assistant" + ), f"Invalid messages at {index}:\n{assistant_msg}" + + tool_call_order = index - prev_assistant_idx + assistant_tool_calls = assistant_msg.get("tool_calls") + assert assistant_tool_calls and len(assistant_tool_calls) >= tool_call_order, ( + "No tool calls but found tool output" + ) + + if tool_call_order == 1: + prompt += "\n\n" + + prompt += tool_output_template.format(content=content) + + if tool_call_order == len(assistant_tool_calls): + prompt += "\n" + + if index >= last_user_idx and thinking_mode == "thinking": + prompt += "\n\n" + thinking_start_token + else: + prompt += "\n\n" + thinking_end_token + + elif role == "assistant": + prev_assistant_idx = index + thinking_part = "" + + tool_calls_content = "" + if tool_calls: + tool_calls = [ + tool_call_template.format( + dsml_token=dsml_token, + name=tool_call.get("name"), + arguments=encode_arguments_to_dsml(tool_call), + ) + for tool_call in tool_calls + ] + tool_calls_content += "\n\n" + tool_calls_template.format( + dsml_token=dsml_token, tool_calls="\n".join(tool_calls) + ) + + summary_content = content or "" + + if thinking_mode == "thinking" and index > last_user_idx: + assert reasoning_content or tool_calls, ( + f"ThinkingMode: {thinking_mode}, invalid message without reasoning_content/tool_calls `{msg}` after last user message" + ) + thinking_part = ( + thinking_template.format(reasoning_content=reasoning_content or "") + + thinking_end_token + ) + + prompt += assistant_msg_template.format( + reasoning=thinking_part, + content=summary_content, + tool_calls=tool_calls_content, + ) + else: + raise NotImplementedError(f"Unknown role: {role}") + + return prompt + + +def drop_thinking_messages( + messages: list[dict[str, Any]], last_user_idx: int | None = None +) -> list[dict[str, Any]]: + messages_wo_thinking: list[dict[str, Any]] = [] + last_user_idx = ( + find_last_user_index(messages) if last_user_idx is None else last_user_idx + ) + for idx, msg in enumerate(messages): + role = msg.get("role") + if role in ["user", "system", "tool"] or idx >= last_user_idx: + messages_wo_thinking.append(msg) + continue + + elif role == "assistant": + msg_wo_thinking = copy.copy(msg) + msg_wo_thinking.pop("reasoning_content", None) + msg_wo_thinking.pop("reasoning", None) + messages_wo_thinking.append(msg_wo_thinking) + + return messages_wo_thinking + + +def encode_messages( + messages: list[dict[str, Any]], + thinking_mode: str, + context: list[dict[str, Any]] | None = None, + drop_thinking: bool = True, + add_default_bos_token: bool = True, +) -> str: + context = context if context else [] + full_messages = context + messages + + prompt = bos_token if add_default_bos_token and len(context) == 0 else "" + + if thinking_mode == "thinking" and drop_thinking: + full_messages = drop_thinking_messages(full_messages) + + for idx in range(len(messages)): + prompt += render_message( + idx + len(context), full_messages, thinking_mode=thinking_mode + ) + + return prompt + + +def _read_until_stop( + index: int, text: str, stop: list[str] +) -> tuple[int, str, None | str]: + min_pos = len(text) + matched_stop = None + + for s in stop: + pos = text.find(s, index) + if pos != -1 and pos < min_pos: + min_pos = pos + matched_stop = s + + if matched_stop: + content = text[index:min_pos] + return min_pos + len(matched_stop), content, matched_stop + else: + content = text[index:] + return len(text), content, None + + +def parse_tool_calls(index: int, text: str): + tool_calls: list[dict[str, Any]] = [] + stop_token = None + tool_calls_end_token = f"" + + while index < len(text): + index, _, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}invoke", tool_calls_end_token] + ) + assert _ == ">\n", "Tool call format error" + + if stop_token == tool_calls_end_token: + break + + assert stop_token is not None, "Missing special token" + + index, tool_name_content, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}parameter", f"\n$', tool_name_content, flags=re.DOTALL + ) + assert len(p_tool_name) == 1, "Tool name format error" + tool_name = p_tool_name[0] + + tool_args: dict[str, tuple[str, str]] = {} + while stop_token == f"<{dsml_token}parameter": + index, param_content, stop_token = _read_until_stop( + index, text, [f"/{dsml_token}parameter"] + ) + + param_kv = re.findall( + r'^ name="(.*?)" string="(true|false)">(.*?)<$', + param_content, + flags=re.DOTALL, + ) + assert len(param_kv) == 1, "Parameter format error" + param_name, string, param_value = param_kv[0] + + assert param_name not in tool_args, "Duplicate parameter name" + tool_args[param_name] = (param_value, string) + + index, content, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}parameter", f"\n", "Parameter format error" + + tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args) + tool_calls.append(tool_call) + + return index, stop_token, tool_calls + + +# NOTE: This function is designed to parse only correctly +# formatted string and will not attempt to correct malformed output +# that may be generated by the model. +def parse_message_from_completion_text(text: str, thinking_mode: str): + summary_content, reasoning_content, tool_calls = "", "", [] + index, stop_token = 0, None + tool_calls_start_token = f"\n\n<{dsml_token}function_calls" + + is_thinking, is_tool_calling = thinking_mode == "thinking", False + + if is_thinking: + index, content_delta, stop_token = _read_until_stop( + index, text, [thinking_end_token, tool_calls_start_token] + ) + reasoning_content = content_delta + assert stop_token == thinking_end_token, "Invalid thinking format" + + index, content_delta, stop_token = _read_until_stop( + index, text, [eos_token, tool_calls_start_token] + ) + summary_content = content_delta + if stop_token == tool_calls_start_token: + is_tool_calling = True + else: + assert stop_token == eos_token, "Invalid summary format" + + if is_tool_calling: + index, stop_token, tool_calls = parse_tool_calls(index, text) + + index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token]) + assert not tool_ends_text, "Unexpected content after tool calls" + + assert len(text) == index and stop_token in [eos_token, None], ( + "Unexpected content at end" + ) + + for sp_token in [ + bos_token, + eos_token, + thinking_start_token, + thinking_end_token, + dsml_token, + ]: + assert sp_token not in summary_content and sp_token not in reasoning_content, ( + "Unexpected special token in content" + ) + + return { + "role": "assistant", + "content": summary_content, + "reasoning_content": reasoning_content, + "reasoning": reasoning_content, + "tool_calls": tool_calls_to_openai_format(tool_calls), + } diff --git a/vllm/tokenizers/deepseekv32.py b/vllm/tokenizers/deepseekv32.py new file mode 100644 index 0000000000000..7466ad4076bd8 --- /dev/null +++ b/vllm/tokenizers/deepseekv32.py @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from pathlib import Path + +from transformers import BatchEncoding + +from .deepseek_v32_encoding import encode_messages +from .hf import HfTokenizer, TokenizerLike +from .registry import TokenizerRegistry + + +@TokenizerRegistry.register("deepseek_v32") +class DeepseekV32Tokenizer(HfTokenizer): + def __init__(self, tokenizer: TokenizerLike): + self.tokenizer = tokenizer + self.name_or_path = ( + tokenizer.name_or_path if hasattr(tokenizer, "name_or_path") else "" + ) + + @classmethod + def from_pretrained( + cls, + path_or_repo_id: str | Path, + *args, + trust_remote_code: bool = False, + revision: str | None = None, + download_dir: str | None = None, + **kwargs, + ) -> "TokenizerLike": + tokenizer = super().from_pretrained( + path_or_repo_id, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + download_dir=download_dir, + **kwargs, + ) + return DeepseekV32Tokenizer(tokenizer) + + def apply_chat_template(self, messages, tools=None, **kwargs): + thinking = kwargs.get("thinking", False) + thinking_mode = "thinking" + if not thinking: + thinking_mode = "chat" + messages = messages.copy() + drop_thinking = True + if tools is not None and len(tools) > 0: + messages.insert(0, {"role": "system"}) + messages[0]["tools"] = tools + drop_thinking = False + encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking) + prompt_str = encode_messages(messages, **encode_config) # type: ignore + return prompt_str + + @property + def all_special_tokens(self) -> list[str]: + return self.tokenizer.all_special_tokens + + @property + def all_special_ids(self) -> list[int]: + return self.tokenizer.all_special_ids + + @property + def bos_token_id(self) -> int: + return self.tokenizer.bos_token_id + + @property + def eos_token_id(self) -> int: + return self.tokenizer.eos_token_id + + @property + def pad_token_id(self) -> int: + return self.tokenizer.pad_token_id + + @property + def is_fast(self) -> bool: + return self.tokenizer.is_fast + + @property + def vocab_size(self) -> int: + return self.tokenizer.vocab_size + + @property + def max_token_id(self) -> int: + return self.tokenizer.max_token_id + + @property + def truncation_side(self) -> str: + return self.tokenizer.truncation_side + + def __hash__(self) -> int: + return hash(id(self)) + + def __len__(self) -> int: + # is an added token in DeepseekV32 tokenizer + return self.vocab_size + len(self.get_added_vocab()) + + def __call__( + self, + text: str | list[str], + text_pair: str | None = None, + add_special_tokens: bool = True, + truncation: bool = False, + max_length: int | None = None, + ) -> "BatchEncoding": + return self.tokenizer( + text, + text_pair=text_pair, + add_special_tokens=add_special_tokens, + truncation=truncation, + max_length=max_length, + ) + + def get_vocab(self) -> dict[str, int]: + return self.tokenizer.get_vocab() + + def get_added_vocab(self) -> dict[str, int]: + return self.tokenizer.get_added_vocab() + + def encode( + self, + text: str, + truncation: bool | None = None, + max_length: int | None = None, + add_special_tokens: bool = True, + ) -> list[int]: + return self.tokenizer.encode( + text, + truncation=truncation, + max_length=max_length, + add_special_tokens=add_special_tokens, + ) + + def convert_tokens_to_string(self, tokens: list[str]) -> str: + return self.tokenizer.convert_tokens_to_string(tokens) + + def decode(self, ids: list[int] | int, skip_special_tokens: bool = False) -> str: + return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + + def convert_ids_to_tokens( + self, + ids: list[int], + skip_special_tokens: bool = False, + ) -> list[str]: + return self.tokenizer.convert_ids_to_tokens( + ids, skip_special_tokens=skip_special_tokens + ) From 15b1511a15dfb1d56048847da755213632c07b29 Mon Sep 17 00:00:00 2001 From: ioana ghiban Date: Wed, 3 Dec 2025 13:56:47 +0100 Subject: [PATCH 04/47] [GPU Backend] [Doc]: Remove duplicate statements on missing GPU wheels. (#29962) Signed-off-by: Ioana Ghiban --- docs/getting_started/installation/gpu.rocm.inc.md | 3 --- docs/getting_started/installation/gpu.xpu.inc.md | 3 --- 2 files changed, 6 deletions(-) diff --git a/docs/getting_started/installation/gpu.rocm.inc.md b/docs/getting_started/installation/gpu.rocm.inc.md index c80ba9478f6be..21120cc6fcd98 100644 --- a/docs/getting_started/installation/gpu.rocm.inc.md +++ b/docs/getting_started/installation/gpu.rocm.inc.md @@ -5,9 +5,6 @@ vLLM supports AMD GPUs with ROCm 6.3 or above, and torch 2.8.0 and above. !!! tip [Docker](#set-up-using-docker) is the recommended way to use vLLM on ROCm. -!!! warning - There are no pre-built wheels for this device, so you must either use the pre-built Docker image or build vLLM from source. - # --8<-- [end:installation] # --8<-- [start:requirements] diff --git a/docs/getting_started/installation/gpu.xpu.inc.md b/docs/getting_started/installation/gpu.xpu.inc.md index 620a660a240ed..7e9c6a2b9de07 100644 --- a/docs/getting_started/installation/gpu.xpu.inc.md +++ b/docs/getting_started/installation/gpu.xpu.inc.md @@ -2,9 +2,6 @@ vLLM initially supports basic model inference and serving on Intel GPU platform. -!!! warning - There are no pre-built wheels for this device, so you need build vLLM from source. Or you can use pre-built images which are based on vLLM released versions. - # --8<-- [end:installation] # --8<-- [start:requirements] From 1bb17ecb396f911beaa26ab0d3926d46154c7155 Mon Sep 17 00:00:00 2001 From: ioana ghiban Date: Wed, 3 Dec 2025 14:33:50 +0100 Subject: [PATCH 05/47] [CPU Backend] [Doc]: Update Installation Docs for CPUs (#29868) Signed-off-by: Ioana Ghiban --- .../installation/cpu.apple.inc.md | 7 +++-- .../installation/cpu.arm.inc.md | 26 ++++++++++++---- docs/getting_started/installation/cpu.md | 30 +++++++++++++++++-- .../installation/cpu.s390x.inc.md | 7 +++-- .../installation/cpu.x86.inc.md | 2 ++ 5 files changed, 58 insertions(+), 14 deletions(-) diff --git a/docs/getting_started/installation/cpu.apple.inc.md b/docs/getting_started/installation/cpu.apple.inc.md index 4dc707d5f9a14..9f1f6e3821397 100644 --- a/docs/getting_started/installation/cpu.apple.inc.md +++ b/docs/getting_started/installation/cpu.apple.inc.md @@ -4,9 +4,6 @@ vLLM has experimental support for macOS with Apple Silicon. For now, users must Currently the CPU implementation for macOS supports FP32 and FP16 datatypes. -!!! warning - There are no pre-built wheels or images for this device, so you must build vLLM from source. - # --8<-- [end:installation] # --8<-- [start:requirements] @@ -20,6 +17,8 @@ Currently the CPU implementation for macOS supports FP32 and FP16 datatypes. # --8<-- [end:set-up-using-python] # --8<-- [start:pre-built-wheels] +Currently, there are no pre-built Apple silicon CPU wheels. + # --8<-- [end:pre-built-wheels] # --8<-- [start:build-wheel-from-source] @@ -78,6 +77,8 @@ uv pip install -e . # --8<-- [end:build-wheel-from-source] # --8<-- [start:pre-built-images] +Currently, there are no pre-built Arm silicon CPU images. + # --8<-- [end:pre-built-images] # --8<-- [start:build-image-from-source] diff --git a/docs/getting_started/installation/cpu.arm.inc.md b/docs/getting_started/installation/cpu.arm.inc.md index 9cae9ed1a212e..156f31f633d57 100644 --- a/docs/getting_started/installation/cpu.arm.inc.md +++ b/docs/getting_started/installation/cpu.arm.inc.md @@ -1,11 +1,6 @@ # --8<-- [start:installation] -vLLM has been adapted to work on ARM64 CPUs with NEON support, leveraging the CPU backend initially developed for the x86 platform. - -ARM CPU backend currently supports Float32, FP16 and BFloat16 datatypes. - -!!! warning - There are no pre-built wheels or images for this device, so you must build vLLM from source. +vLLM offers basic model inferencing and serving on Arm CPU platform, with support NEON, data types FP32, FP16 and BF16. # --8<-- [end:installation] # --8<-- [start:requirements] @@ -20,6 +15,23 @@ ARM CPU backend currently supports Float32, FP16 and BFloat16 datatypes. # --8<-- [end:set-up-using-python] # --8<-- [start:pre-built-wheels] +Pre-built vLLM wheels for Arm are available since version 0.11.2. These wheels contain pre-compiled C++ binaries. +Please replace `` in the commands below with a specific version string (e.g., `0.11.2`). + +```bash +uv pip install --pre vllm==+cpu --extra-index-url https://wheels.vllm.ai/%2Bcpu/ +``` + +??? console "pip" + ```bash + pip install --pre vllm==+cpu --extra-index-url https://wheels.vllm.ai/%2Bcpu/ + ``` + +The `uv` approach works for vLLM `v0.6.6` and later. A unique feature of `uv` is that packages in `--extra-index-url` have [higher priority than the default index](https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes). If the latest public release is `v0.6.6.post1`, `uv`'s behavior allows installing a commit before `v0.6.6.post1` by specifying the `--extra-index-url`. In contrast, `pip` combines packages from `--extra-index-url` and the default index, choosing only the latest version, which makes it difficult to install a development version prior to the released version. + +!!! note + Nightly wheels are currently unsupported for this architecture. (e.g. to bisect the behavior change, performance regression). + # --8<-- [end:pre-built-wheels] # --8<-- [start:build-wheel-from-source] @@ -69,6 +81,8 @@ Testing has been conducted on AWS Graviton3 instances for compatibility. # --8<-- [end:build-wheel-from-source] # --8<-- [start:pre-built-images] +Currently, there are no pre-built Arm CPU images. + # --8<-- [end:pre-built-images] # --8<-- [start:build-image-from-source] ```bash diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index 4b68cb4811789..210f720e2d92a 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -46,11 +46,25 @@ vLLM is a Python library that supports the following CPU variants. Select your C ### Pre-built wheels -Please refer to the instructions for [pre-built wheels on GPU](./gpu.md#pre-built-wheels). - When specifying the index URL, please make sure to use the `cpu` variant subdirectory. For example, the nightly build index is: `https://wheels.vllm.ai/nightly/cpu/`. +=== "Intel/AMD x86" + + --8<-- "docs/getting_started/installation/cpu.x86.inc.md:pre-built-wheels" + +=== "ARM AArch64" + + --8<-- "docs/getting_started/installation/cpu.arm.inc.md:pre-built-wheels" + +=== "Apple silicon" + + --8<-- "docs/getting_started/installation/cpu.apple.inc.md:pre-built-wheels" + +=== "IBM Z (S390X)" + + --8<-- "docs/getting_started/installation/cpu.s390x.inc.md:pre-built-wheels" + ### Build wheel from source #### Set up using Python-only build (without compilation) {#python-only-build} @@ -87,6 +101,18 @@ VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cpu VLLM_TARGET_DEVICE=cpu --8<-- "docs/getting_started/installation/cpu.x86.inc.md:pre-built-images" +=== "ARM AArch64" + + --8<-- "docs/getting_started/installation/cpu.arm.inc.md:pre-built-images" + +=== "Apple silicon" + + --8<-- "docs/getting_started/installation/cpu.apple.inc.md:pre-built-images" + +=== "IBM Z (S390X)" + + --8<-- "docs/getting_started/installation/cpu.s390x.inc.md:pre-built-images" + ### Build image from source === "Intel/AMD x86" diff --git a/docs/getting_started/installation/cpu.s390x.inc.md b/docs/getting_started/installation/cpu.s390x.inc.md index c2163139a7c5d..4984c87c17b01 100644 --- a/docs/getting_started/installation/cpu.s390x.inc.md +++ b/docs/getting_started/installation/cpu.s390x.inc.md @@ -4,9 +4,6 @@ vLLM has experimental support for s390x architecture on IBM Z platform. For now, Currently, the CPU implementation for s390x architecture supports FP32 datatype only. -!!! warning - There are no pre-built wheels or images for this device, so you must build vLLM from source. - # --8<-- [end:installation] # --8<-- [start:requirements] @@ -21,6 +18,8 @@ Currently, the CPU implementation for s390x architecture supports FP32 datatype # --8<-- [end:set-up-using-python] # --8<-- [start:pre-built-wheels] +Currently, there are no pre-built IBM Z CPU wheels. + # --8<-- [end:pre-built-wheels] # --8<-- [start:build-wheel-from-source] @@ -69,6 +68,8 @@ Execute the following commands to build and install vLLM from source. # --8<-- [end:build-wheel-from-source] # --8<-- [start:pre-built-images] +Currently, there are no pre-built IBM Z CPU images. + # --8<-- [end:pre-built-images] # --8<-- [start:build-image-from-source] diff --git a/docs/getting_started/installation/cpu.x86.inc.md b/docs/getting_started/installation/cpu.x86.inc.md index 310f179cb89ca..1fad7f4338822 100644 --- a/docs/getting_started/installation/cpu.x86.inc.md +++ b/docs/getting_started/installation/cpu.x86.inc.md @@ -17,6 +17,8 @@ vLLM supports basic model inferencing and serving on x86 CPU platform, with data # --8<-- [end:set-up-using-python] # --8<-- [start:pre-built-wheels] +Currently, there are no pre-built x86 CPU wheels. + # --8<-- [end:pre-built-wheels] # --8<-- [start:build-wheel-from-source] From 5aa9b090407d5fb9b89c05d28fab808623e3070c Mon Sep 17 00:00:00 2001 From: rasmith Date: Wed, 3 Dec 2025 08:56:35 -0600 Subject: [PATCH 06/47] [CI/Build][AMD] Skip test_shared_storage_connector_hashes in test_shared_storage_connector.py due to hipErrorLaunchFailure when calling .cpu() (#29839) Signed-off-by: Randall Smith Co-authored-by: Randall Smith --- .../kv_connector/unit/test_shared_storage_connector.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/v1/kv_connector/unit/test_shared_storage_connector.py b/tests/v1/kv_connector/unit/test_shared_storage_connector.py index e7013a794a8c6..ff4697a978255 100644 --- a/tests/v1/kv_connector/unit/test_shared_storage_connector.py +++ b/tests/v1/kv_connector/unit/test_shared_storage_connector.py @@ -3,12 +3,14 @@ from dataclasses import asdict from typing import NamedTuple +import pytest from PIL import Image from vllm import LLM, EngineArgs, SamplingParams from vllm.assets.image import ImageAsset from vllm.config import KVTransferConfig from vllm.multimodal.utils import encode_image_base64 +from vllm.platforms import current_platform MODEL_NAME = "RedHatAI/Qwen2.5-VL-3B-Instruct-quantized.w8a8" @@ -108,6 +110,13 @@ def process_prompt(processor, llm: LLM, question: str, image_urls: list[Image]): print("-" * 50) +@pytest.mark.skipif( + current_platform.is_rocm(), + reason=( + "hipErrorLaunchFailure when running this test, see issue:" + "https://github.com/ROCm/pytorch/issues/2822" + ), +) def test_shared_storage_connector_hashes(tmp_path): """ Tests that SharedStorageConnector saves KV to the storage locations From 9bcf92295a918c9579d59a9c4d003cb563f495f7 Mon Sep 17 00:00:00 2001 From: Lumis Chen Date: Thu, 4 Dec 2025 00:06:57 +0800 Subject: [PATCH 07/47] [Core] Add xxHash as a high-performance hash option for accelerating prefix caching (#29163) Signed-off-by: LuminolT Signed-off-by: Lumis Chen Co-authored-by: Russell Bryant --- benchmarks/benchmark_hash.py | 120 ++++++++++++++++++++++ benchmarks/benchmark_prefix_block_hash.py | 110 ++++++++++++++++++++ docs/benchmarking/cli.md | 29 ++++++ tests/v1/engine/test_engine_args.py | 16 +++ vllm/config/cache.py | 18 +++- vllm/utils/hashing.py | 36 +++++++ vllm/v1/core/kv_cache_utils.py | 11 +- 7 files changed, 332 insertions(+), 8 deletions(-) create mode 100644 benchmarks/benchmark_hash.py create mode 100644 benchmarks/benchmark_prefix_block_hash.py diff --git a/benchmarks/benchmark_hash.py b/benchmarks/benchmark_hash.py new file mode 100644 index 0000000000000..08cdc012d6527 --- /dev/null +++ b/benchmarks/benchmark_hash.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Micro benchmark comparing built-in hash(), SHA-256, and xxHash. + +This focuses on a single test payload shaped like the prefix-cache hash input: + (32-byte bytes object, 32-int tuple) + +Usage: + python benchmarks/hash_micro_benchmark.py --iterations 20000 +""" + +from __future__ import annotations + +import argparse +import random +import statistics +import time +from collections.abc import Callable, Iterable + +from vllm.utils.hashing import sha256, xxhash + + +def _generate_test_data(seed: int) -> tuple[bytes, tuple[int, ...]]: + """Generate a deterministic test payload.""" + random.seed(seed) + bytes_data = bytes(random.getrandbits(8) for _ in range(32)) + int_tuple = tuple(random.randint(1, 1_000_000) for _ in range(32)) + return (bytes_data, int_tuple) + + +def _benchmark_func(func: Callable[[tuple], object], data: tuple, iterations: int): + """Return (avg_seconds, std_seconds) for hashing `data` `iterations` times.""" + times: list[float] = [] + + # Warm-up to avoid first-run noise. + for _ in range(200): + func(data) + + for _ in range(iterations): + start = time.perf_counter() + func(data) + end = time.perf_counter() + times.append(end - start) + + avg = statistics.mean(times) + std = statistics.stdev(times) if len(times) > 1 else 0.0 + return avg, std + + +def _run_benchmarks( + benchmarks: Iterable[tuple[str, Callable[[tuple], object]]], + data: tuple, + iterations: int, +): + """Yield (name, avg, std) for each benchmark, skipping unavailable ones.""" + for name, func in benchmarks: + try: + avg, std = _benchmark_func(func, data, iterations) + except ModuleNotFoundError as exc: + print(f"Skipping {name}: {exc}") + continue + yield name, avg, std + + +def builtin_hash(data: tuple) -> int: + """Wrapper for Python's built-in hash().""" + return hash(data) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--iterations", + type=int, + default=10_000, + help="Number of measured iterations per hash function.", + ) + parser.add_argument( + "--seed", type=int, default=42, help="Random seed for test payload." + ) + args = parser.parse_args() + + data = _generate_test_data(args.seed) + benchmarks = ( + ("SHA256 (pickle)", sha256), + ("xxHash (pickle)", xxhash), + ("built-in hash()", builtin_hash), + ) + + print("=" * 60) + print("HASH FUNCTION MICRO BENCHMARK") + print("=" * 60) + print("Test data: (32-byte bytes object, 32-int tuple)") + print(f"Iterations: {args.iterations:,}") + print("=" * 60) + + results = list(_run_benchmarks(benchmarks, data, args.iterations)) + builtin_entry = next((r for r in results if r[0] == "built-in hash()"), None) + + print("\nResults:") + for name, avg, std in results: + print(f" {name:16s}: {avg * 1e6:8.2f} ± {std * 1e6:6.2f} μs") + + if builtin_entry: + _, builtin_avg, _ = builtin_entry + print("\n" + "=" * 60) + print("SUMMARY (relative to built-in hash())") + print("=" * 60) + for name, avg, _ in results: + if name == "built-in hash()": + continue + speed_ratio = avg / builtin_avg + print(f"• {name} is {speed_ratio:.1f}x slower than built-in hash()") + else: + print("\nBuilt-in hash() result missing; cannot compute speed ratios.") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/benchmark_prefix_block_hash.py b/benchmarks/benchmark_prefix_block_hash.py new file mode 100644 index 0000000000000..8bcd8af0d3102 --- /dev/null +++ b/benchmarks/benchmark_prefix_block_hash.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Simple benchmark to compare prefix-cache block hashing algorithms. + +Example: + python benchmark_prefix_block_hash.py --num-blocks 20000 --block-size 32 +""" + +from __future__ import annotations + +import argparse +import random +import statistics +import sys +import time +from collections.abc import Callable, Iterable, Sequence + +from vllm.utils.hashing import get_hash_fn_by_name +from vllm.v1.core.kv_cache_utils import BlockHash, hash_block_tokens, init_none_hash + +SUPPORTED_ALGOS = ("sha256", "sha256_cbor", "xxhash", "xxhash_cbor") + + +def _generate_blocks( + num_blocks: int, block_size: int, vocab_size: int, seed: int +) -> list[list[int]]: + rng = random.Random(seed) + return [ + [rng.randrange(vocab_size) for _ in range(block_size)] + for _ in range(num_blocks) + ] + + +def _hash_all_blocks( + hash_fn: Callable[[object], bytes], + blocks: Iterable[Sequence[int]], +) -> float: + parent_hash: BlockHash | None = None + start = time.perf_counter() + for block in blocks: + parent_hash = hash_block_tokens(hash_fn, parent_hash, block, extra_keys=None) + end = time.perf_counter() + return end - start + + +def _benchmark( + hash_algo: str, + blocks: list[list[int]], + trials: int, +) -> tuple[float, float, float] | None: + try: + hash_fn = get_hash_fn_by_name(hash_algo) + init_none_hash(hash_fn) + timings = [_hash_all_blocks(hash_fn, blocks) for _ in range(trials)] + except ModuleNotFoundError as exc: + print(f"Skipping {hash_algo}: {exc}", file=sys.stderr) + return None + + avg = statistics.mean(timings) + best = min(timings) + # throughput: tokens / second + tokens_hashed = len(blocks) * len(blocks[0]) + throughput = tokens_hashed / best + return avg, best, throughput + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--num-blocks", type=int, default=10000, help="Block count.") + parser.add_argument("--block-size", type=int, default=32, help="Tokens per block.") + parser.add_argument( + "--vocab-size", type=int, default=32000, help="Token id range [0, vocab_size)." + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed.") + parser.add_argument( + "--trials", type=int, default=5, help="Number of timed trials per algorithm." + ) + parser.add_argument( + "--algorithms", + nargs="+", + default=SUPPORTED_ALGOS, + choices=SUPPORTED_ALGOS, + help="Hash algorithms to benchmark.", + ) + args = parser.parse_args() + + blocks = _generate_blocks( + args.num_blocks, args.block_size, args.vocab_size, args.seed + ) + print( + f"Benchmarking {len(args.algorithms)} algorithms on " + f"{args.num_blocks} blocks (block size={args.block_size})." + ) + + for algo in args.algorithms: + result = _benchmark(algo, blocks, args.trials) + if result is None: + continue + + avg, best, throughput = result + print( + f"{algo:14s} avg: {avg:.6f}s best: {best:.6f}s " + f"throughput: {throughput / 1e6:.2f}M tokens/s" + ) + + +if __name__ == "__main__": + main() diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index 44a4c40125952..1ce6b611745b1 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -670,6 +670,35 @@ vllm bench serve \ +### 🧪 Hashing Benchmarks + +

+Show more + +Two helper scripts live in `benchmarks/` to compare hashing options used by prefix caching and related utilities. They are standalone (no server required) and help choose a hash algorithm before enabling prefix caching in production. + +- `benchmarks/benchmark_hash.py`: Micro-benchmark that measures per-call latency of three implementations on a representative `(bytes, tuple[int])` payload. + +```bash +python benchmarks/benchmark_hash.py --iterations 20000 --seed 42 +``` + +- `benchmarks/benchmark_prefix_block_hash.py`: End-to-end block hashing benchmark that runs the full prefix-cache hash pipeline (`hash_block_tokens`) across many fake blocks and reports throughput. + +```bash +python benchmarks/benchmark_prefix_block_hash.py --num-blocks 20000 --block-size 32 --trials 5 +``` + +Supported algorithms: `sha256`, `sha256_cbor`, `xxhash`, `xxhash_cbor`. Install optional deps to exercise all variants: + +```bash +uv pip install xxhash cbor2 +``` + +If an algorithm’s dependency is missing, the script will skip it and continue. + +
+ ### ⚡ Request Prioritization Benchmark
diff --git a/tests/v1/engine/test_engine_args.py b/tests/v1/engine/test_engine_args.py index e96759ed66a79..527a56ff49eec 100644 --- a/tests/v1/engine/test_engine_args.py +++ b/tests/v1/engine/test_engine_args.py @@ -9,6 +9,7 @@ from vllm.config import VllmConfig from vllm.engine.arg_utils import EngineArgs from vllm.usage.usage_lib import UsageContext from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.hashing import _xxhash def test_prefix_caching_from_cli(): @@ -48,6 +49,21 @@ def test_prefix_caching_from_cli(): args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"]) +@pytest.mark.skipif(_xxhash is None, reason="xxhash not installed") +def test_prefix_caching_xxhash_from_cli(): + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + + # set hash algorithm to xxhash (pickle) + args = parser.parse_args(["--prefix-caching-hash-algo", "xxhash"]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.cache_config.prefix_caching_hash_algo == "xxhash" + + # set hash algorithm to xxhash_cbor + args = parser.parse_args(["--prefix-caching-hash-algo", "xxhash_cbor"]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.cache_config.prefix_caching_hash_algo == "xxhash_cbor" + + def test_defaults_with_usage_context(): engine_args = EngineArgs(model="facebook/opt-125m") vllm_config: VllmConfig = engine_args.create_engine_config(UsageContext.LLM_CLASS) diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 00530846fce00..91f083a5534ba 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -30,7 +30,7 @@ CacheDType = Literal[ "fp8_ds_mla", ] MambaDType = Literal["auto", "float32"] -PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor"] +PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"] KVOffloadingBackend = Literal["native", "lmcache"] @@ -77,9 +77,21 @@ class CacheConfig: """Whether to enable prefix caching.""" prefix_caching_hash_algo: PrefixCachingHashAlgo = "sha256" """Set the hash algorithm for prefix caching:\n - - "sha256" uses Pickle for object serialization before hashing.\n + - "sha256" uses Pickle for object serialization before hashing. This is the + current default, as SHA256 is the most secure choice to avoid potential + hash collisions.\n - "sha256_cbor" provides a reproducible, cross-language compatible hash. It - serializes objects using canonical CBOR and hashes them with SHA-256.""" + serializes objects using canonical CBOR and hashes them with SHA-256.\n + - "xxhash" uses Pickle serialization with xxHash (128-bit) for faster, + non-cryptographic hashing. Requires the optional ``xxhash`` package. + IMPORTANT: Use of a hashing algorithm that is not considered + cryptographically secure theoretically increases the risk of hash collisions, + which can cause undefined behavior or even leak private information in + multi-tenant environments. Even if collisions are still very unlikely, it is + important to consider your security risk tolerance against the performance + benefits before turning this on.\n + - "xxhash_cbor" combines canonical CBOR serialization with xxHash for + reproducible hashing. Requires the optional ``xxhash`` package.""" cpu_offload_gb: float = Field(default=0, ge=0) """The space in GiB to offload to CPU, per GPU. Default is 0, which means no offloading. Intuitively, this argument can be seen as a virtual way to diff --git a/vllm/utils/hashing.py b/vllm/utils/hashing.py index edf1e9cb34e56..f01c6b074ffeb 100644 --- a/vllm/utils/hashing.py +++ b/vllm/utils/hashing.py @@ -11,6 +11,17 @@ from typing import Any import cbor2 +try: + # It is important that this remains an optional dependency. + # It would not be allowed in environments with strict security controls, + # so it's best not to have it installed when not in use. + import xxhash as _xxhash + + if not hasattr(_xxhash, "xxh3_128_digest"): + _xxhash = None +except ImportError: # pragma: no cover + _xxhash = None + def sha256(input: Any) -> bytes: """Hash any picklable Python object using SHA-256. @@ -47,6 +58,27 @@ def sha256_cbor(input: Any) -> bytes: return hashlib.sha256(input_bytes).digest() +def _xxhash_digest(input_bytes: bytes) -> bytes: + if _xxhash is None: + raise ModuleNotFoundError( + "xxhash is required for the 'xxhash' prefix caching hash algorithms. " + "Install it via `pip install xxhash`." + ) + return _xxhash.xxh3_128_digest(input_bytes) + + +def xxhash(input: Any) -> bytes: + """Hash picklable objects using xxHash.""" + input_bytes = pickle.dumps(input, protocol=pickle.HIGHEST_PROTOCOL) + return _xxhash_digest(input_bytes) + + +def xxhash_cbor(input: Any) -> bytes: + """Hash objects serialized with CBOR using xxHash.""" + input_bytes = cbor2.dumps(input, canonical=True) + return _xxhash_digest(input_bytes) + + def get_hash_fn_by_name(hash_fn_name: str) -> Callable[[Any], bytes]: """Get a hash function by name, or raise an error if the function is not found. @@ -60,6 +92,10 @@ def get_hash_fn_by_name(hash_fn_name: str) -> Callable[[Any], bytes]: return sha256 if hash_fn_name == "sha256_cbor": return sha256_cbor + if hash_fn_name == "xxhash": + return xxhash + if hash_fn_name == "xxhash_cbor": + return xxhash_cbor raise ValueError(f"Unsupported hash function: {hash_fn_name}") diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 602eb81beb010..774200deed158 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -12,7 +12,7 @@ from typing import Any, NewType, TypeAlias, overload from vllm import envs from vllm.config import VllmConfig from vllm.logger import init_logger -from vllm.utils.hashing import sha256_cbor +from vllm.utils.hashing import sha256_cbor, xxhash_cbor from vllm.utils.math_utils import cdiv from vllm.utils.mem_constants import GiB_bytes from vllm.v1.kv_cache_interface import ( @@ -83,18 +83,19 @@ logger = init_logger(__name__) # # The function `init_none_hash` initializes this variable globally. NONE_HASH: BlockHash +_CBOR_HASH_FUNCTIONS = frozenset({sha256_cbor, xxhash_cbor}) def init_none_hash(hash_fn: Callable[[Any], bytes]): global NONE_HASH hash_seed = os.getenv("PYTHONHASHSEED") - if hash_seed is None and hash_fn is sha256_cbor: + if hash_seed is None and hash_fn in _CBOR_HASH_FUNCTIONS: logger.warning( "PYTHONHASHSEED is not set. This will lead to non-reproducible " - "block-hashes when using sha256_cbor as the hash function." - "Consider setting PYTHONHASHSEED to a fixed value for " - "reproducibility." + "block-hashes when using CBOR-based hash functions such as " + "sha256_cbor or xxhash_cbor. Consider setting PYTHONHASHSEED to a " + "fixed value for reproducibility." ) if hash_seed is None: From 9ae3c55b10318ad7b0c19becb0dc8ad41c171db2 Mon Sep 17 00:00:00 2001 From: Yu Jiaqi <54204033+piood@users.noreply.github.com> Date: Thu, 4 Dec 2025 00:12:58 +0800 Subject: [PATCH 08/47] SigLIP example add chat_template (#29902) Signed-off-by: piood <2477084691@qq.com> --- ...ai_chat_embedding_client_for_multimodal.py | 3 +- vllm/entrypoints/chat_utils.py | 35 ++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/examples/pooling/embed/openai_chat_embedding_client_for_multimodal.py b/examples/pooling/embed/openai_chat_embedding_client_for_multimodal.py index 47c2c5030078c..a7ab7e73e7d42 100644 --- a/examples/pooling/embed/openai_chat_embedding_client_for_multimodal.py +++ b/examples/pooling/embed/openai_chat_embedding_client_for_multimodal.py @@ -150,7 +150,8 @@ def run_siglip(client: OpenAI, model: str): Start the server using: vllm serve google/siglip-base-patch16-224 \ - --runner pooling + --runner pooling \ + --chat-template template_basic.jinja """ response = create_chat_embeddings( diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 2dd5b9c8f8aa0..1b3a7d2665b41 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -1139,11 +1139,19 @@ def validate_chat_template(chat_template: Path | str | None): not any(c in chat_template for c in JINJA_CHARS) and not Path(chat_template).exists() ): - raise ValueError( - f"The supplied chat template string ({chat_template}) " - f"appears path-like, but doesn't exist!" + # Try to find the template in the built-in templates directory + from vllm.transformers_utils.chat_templates.registry import ( + CHAT_TEMPLATES_DIR, ) + builtin_template_path = CHAT_TEMPLATES_DIR / chat_template + if not builtin_template_path.exists(): + raise ValueError( + f"The supplied chat template string ({chat_template}) " + f"appears path-like, but doesn't exist! " + f"Tried: {chat_template} and {builtin_template_path}" + ) + else: raise TypeError(f"{type(chat_template)} is not a valid chat template type") @@ -1173,12 +1181,23 @@ def _load_chat_template( JINJA_CHARS = "{}\n" if not any(c in chat_template for c in JINJA_CHARS): - msg = ( - f"The supplied chat template ({chat_template}) " - f"looks like a file path, but it failed to be " - f"opened. Reason: {e}" + # Try to load from the built-in templates directory + from vllm.transformers_utils.chat_templates.registry import ( + CHAT_TEMPLATES_DIR, ) - raise ValueError(msg) from e + + builtin_template_path = CHAT_TEMPLATES_DIR / chat_template + try: + with open(builtin_template_path) as f: + return f.read() + except OSError: + msg = ( + f"The supplied chat template ({chat_template}) " + f"looks like a file path, but it failed to be opened. " + f"Tried: {chat_template} and {builtin_template_path}. " + f"Reason: {e}" + ) + raise ValueError(msg) from e # If opening a file fails, set chat template to be args to # ensure we decode so our escape are interpreted correctly From d1f7392c5f774245d0a0776d141a64e72ca3e8ca Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Wed, 3 Dec 2025 11:17:07 -0600 Subject: [PATCH 09/47] [ROCm][CI] Fix v1/logits_processors failure on ROCm (#29927) Signed-off-by: Micah Williamson --- tests/v1/logits_processors/test_custom_offline.py | 5 ----- tests/v1/logits_processors/test_custom_online.py | 12 ++---------- tests/v1/logits_processors/utils.py | 2 +- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/tests/v1/logits_processors/test_custom_offline.py b/tests/v1/logits_processors/test_custom_offline.py index 1899737737f4b..e3ddb6138cfdd 100644 --- a/tests/v1/logits_processors/test_custom_offline.py +++ b/tests/v1/logits_processors/test_custom_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import random -import sys from typing import Any import pytest @@ -10,7 +9,6 @@ from tests.utils import create_new_process_for_each_test from tests.v1.logits_processors.utils import ( DUMMY_LOGITPROC_ARG, DUMMY_LOGITPROC_FQCN, - DUMMY_LOGITPROC_MODULE, MAX_TOKENS, MODEL_NAME, POOLING_MODEL_NAME, @@ -18,7 +16,6 @@ from tests.v1.logits_processors.utils import ( CustomLogitprocSource, DummyLogitsProcessor, WrappedPerReqLogitsProcessor, - dummy_module, prompts, ) from tests.v1.logits_processors.utils import entry_points as fake_entry_points @@ -162,8 +159,6 @@ def test_custom_logitsprocs(monkeypatch, logitproc_source: CustomLogitprocSource kwargs: dict[str, list[str | type[LogitsProcessor]]] = {} if logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_FQCN: # Scenario: load logitproc based on fully-qualified class name (FQCN) - # Inject dummy module which defines logitproc - sys.modules[DUMMY_LOGITPROC_MODULE] = dummy_module kwargs["logits_processors"] = [DUMMY_LOGITPROC_FQCN] elif logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_CLASS: # Scenario: load logitproc from provided class object diff --git a/tests/v1/logits_processors/test_custom_online.py b/tests/v1/logits_processors/test_custom_online.py index 3e0bb02ed68be..3dc6b89790157 100644 --- a/tests/v1/logits_processors/test_custom_online.py +++ b/tests/v1/logits_processors/test_custom_online.py @@ -14,11 +14,9 @@ from tests.utils import RemoteOpenAIServerCustom, create_new_process_for_each_te from tests.v1.logits_processors.utils import ( DUMMY_LOGITPROC_ARG, DUMMY_LOGITPROC_FQCN, - DUMMY_LOGITPROC_MODULE, MAX_TOKENS, MODEL_NAME, TEMP_GREEDY, - dummy_module, prompts, ) from tests.v1.logits_processors.utils import entry_points as fake_entry_points @@ -47,20 +45,14 @@ def _server_with_logitproc_entrypoint( main.main() -def _server_with_logitproc_module( +def _server_with_logitproc_fqcn( env_dict: dict[str, str] | None, model: str, vllm_serve_args: list[str], ) -> None: """Start vLLM server, inject module with dummy logitproc""" - - # Patch `modules` to inject dummy logitproc module from vllm.entrypoints.cli import main - sys.modules[DUMMY_LOGITPROC_MODULE] = dummy_module - - # fork is required for workers to see entrypoint patch - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "fork" if env_dict is not None: os.environ.update(env_dict) @@ -99,7 +91,7 @@ def server(default_server_args, request, monkeypatch): if request.param: # Launch server, append FQCN argument, inject dummy logitproc module args = default_server_args + request.param - _server_fxn = _server_with_logitproc_module + _server_fxn = _server_with_logitproc_fqcn else: # Launch server, inject dummy logitproc entrypoint args = default_server_args diff --git a/tests/v1/logits_processors/utils.py b/tests/v1/logits_processors/utils.py index b8548bc319554..e54da72e5e2ed 100644 --- a/tests/v1/logits_processors/utils.py +++ b/tests/v1/logits_processors/utils.py @@ -27,7 +27,7 @@ DUMMY_LOGITPROC_ARG = "target_token" TEMP_GREEDY = 0.0 MAX_TOKENS = 20 DUMMY_LOGITPROC_ENTRYPOINT = "dummy_logitproc" -DUMMY_LOGITPROC_MODULE = "DummyModule" +DUMMY_LOGITPROC_MODULE = "tests.v1.logits_processors.utils" DUMMY_LOGITPROC_FQCN = f"{DUMMY_LOGITPROC_MODULE}:DummyLogitsProcessor" From dd5d1ef780b5b73b9817e5dc8fe9b3e98a399e20 Mon Sep 17 00:00:00 2001 From: avigny <47987522+avigny@users.noreply.github.com> Date: Wed, 3 Dec 2025 18:45:31 +0100 Subject: [PATCH 10/47] [Bugfix] Mistral tool parser streaming update (#19425) Signed-off-by: avigny <47987522+avigny@users.noreply.github.com> Signed-off-by: Chauncey Signed-off-by: chaunceyjiang Co-authored-by: Jeff Cook Co-authored-by: sfbemerk Co-authored-by: Chauncey Co-authored-by: Cyrus Leung --- requirements/common.txt | 1 + tests/tool_use/test_mistral_tool_parser.py | 847 ++++++++++++++++++ tests/tool_use/utils.py | 28 +- .../tool_parsers/mistral_tool_parser.py | 608 ++++++++----- 4 files changed, 1277 insertions(+), 207 deletions(-) create mode 100644 tests/tool_use/test_mistral_tool_parser.py diff --git a/requirements/common.txt b/requirements/common.txt index 8b9e6b935bd20..f18560b98d16c 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -46,6 +46,7 @@ scipy # Required for phi-4-multimodal-instruct ninja # Required for xgrammar, rocm, tpu, xpu pybase64 # fast base64 implementation cbor2 # Required for cross-language serialization of hashable objects +ijson # Required for mistral streaming tool parser setproctitle # Used to set process names for better debugging and monitoring openai-harmony >= 0.0.3 # Required for gpt-oss anthropic == 0.71.0 diff --git a/tests/tool_use/test_mistral_tool_parser.py b/tests/tool_use/test_mistral_tool_parser.py new file mode 100644 index 0000000000000..e5deb7f40eb35 --- /dev/null +++ b/tests/tool_use/test_mistral_tool_parser.py @@ -0,0 +1,847 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from collections.abc import Generator + +import partial_json_parser +import pytest +from mistral_common.protocol.instruct.messages import AssistantMessage +from mistral_common.protocol.instruct.request import InstructRequest +from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall +from partial_json_parser.core.options import Allow + +from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall +from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolParser +from vllm.tokenizers import ( + MistralTokenizer, + TokenizerLike, + get_tokenizer, +) +from vllm.tokenizers.detokenizer_utils import detokenize_incrementally + + +@pytest.fixture(scope="module") +def mistral_pre_v11_tokenizer(): + MODEL = "mistralai/Mistral-7B-Instruct-v0.3" + return get_tokenizer(tokenizer_name=MODEL) + + +@pytest.fixture(scope="module") +def mistral_tokenizer(): + MODEL = "mistralai/Mistral-Small-3.2-24B-Instruct-2506" + return get_tokenizer(tokenizer_name=MODEL, tokenizer_mode="mistral") + + +@pytest.fixture +def mistral_pre_v11_tool_parser(mistral_pre_v11_tokenizer): + return MistralToolParser(mistral_pre_v11_tokenizer) + + +@pytest.fixture +def mistral_tool_parser(mistral_tokenizer): + return MistralToolParser(mistral_tokenizer) + + +def assert_tool_calls( + actual_tool_calls: list[ToolCall] | list[DeltaToolCall], + expected_tool_calls: list[ToolCall], +): + assert len(actual_tool_calls) == len(expected_tool_calls) + + for actual_tool_call, expected_tool_call in zip( + actual_tool_calls, expected_tool_calls + ): + assert isinstance(actual_tool_call.id, str) + assert len(actual_tool_call.id) == 9 + + if isinstance(actual_tool_call, ToolCall): + assert actual_tool_call.type == "function" + elif isinstance(actual_tool_call, DeltaToolCall): + assert actual_tool_call.function is not None + assert actual_tool_call.function.name is not None + assert actual_tool_call.function.arguments is not None + assert actual_tool_call.function is not None + assert actual_tool_call.function.name == expected_tool_call.function.name, ( + f"got wrong function name:${actual_tool_call.function.name}" + ) + assert ( + actual_tool_call.function.arguments == expected_tool_call.function.arguments + ), f"got wrong function argument:${actual_tool_call.function.arguments}" + + +def fix_tool_call_tokenization( + tokens: list[int], + mistral_tool_parser: MistralToolParser, + mistral_tokenizer: TokenizerLike, +): + """ + Replaces the textual token sequence for [TOOL_CALLS] + with its single special token ID. + """ + textual_tool_call_token_ids = mistral_tokenizer.encode( + text=mistral_tool_parser.bot_token, + add_special_tokens=False, + ) + # textual_tool_call_token_ids must not contain special tokens like bos, eos etc + special_tool_call_token_ids = [mistral_tool_parser.bot_token_id] + + # If the input is too short to contain the sequence, no replacement is possible + if not tokens or len(tokens) < len(textual_tool_call_token_ids): + return tokens + + result_tokens = [] + i = 0 + target_len = len(textual_tool_call_token_ids) + + while i < len(tokens): + # Check if the slice from the current position matches the target sequence + if tokens[i : i + target_len] == textual_tool_call_token_ids: + # If it matches, add the replacement and jump the index forward + result_tokens.extend(special_tool_call_token_ids) + i += target_len + else: + # Otherwise, just add the current token and move to the next one + result_tokens.append(tokens[i]) + i += 1 + + return result_tokens + + +def stream_delta_message_generator( + mistral_tool_parser: MistralToolParser, + mistral_tokenizer: TokenizerLike, + model_output: str | None, + tools: list[tuple[str, str]] | None, +) -> Generator[DeltaMessage, None, None]: + if ( + isinstance(mistral_tokenizer, MistralTokenizer) + and mistral_tokenizer.version >= 11 + ): + # With the newer versions of the tokenizer, + # we cannot tokenize free text + # so we need to create a list of messages to get tokenized + assert tools is not None + assistant_msg = AssistantMessage( + tool_calls=[ + ToolCall( + function=FunctionCall( + name=name, + arguments=arg, + ) + ) + for (name, arg) in tools + ], + ) + request = InstructRequest( + messages=[assistant_msg], + ) + all_token_ids = mistral_tokenizer.instruct.encode_instruct(request).tokens + else: + # Older versions of the tokenizer are + # able to encode directly the model's output (free text) into tokens + assert model_output is not None + all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False) + + all_token_ids = fix_tool_call_tokenization( + all_token_ids, mistral_tool_parser, mistral_tokenizer + ) + + previous_text = "" + previous_tokens = None + prefix_offset = 0 + read_offset = 0 + for i, delta_token in enumerate(all_token_ids): + delta_token_ids = [delta_token] + previous_token_ids = all_token_ids[:i] + current_token_ids = all_token_ids[: i + 1] + + (new_tokens, delta_text, new_prefix_offset, new_read_offset) = ( + detokenize_incrementally( + tokenizer=mistral_tokenizer, + all_input_ids=current_token_ids, + prev_tokens=previous_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + skip_special_tokens=isinstance(mistral_tokenizer, MistralTokenizer), + spaces_between_special_tokens=True, + ) + ) + + current_text = previous_text + delta_text + + delta_message = mistral_tool_parser.extract_tool_calls_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + request=None, # type: ignore[arg-type] + ) + if delta_message: + yield delta_message + + previous_text = current_text + previous_tokens = ( + previous_tokens + new_tokens if previous_tokens else new_tokens + ) + prefix_offset = new_prefix_offset + read_offset = new_read_offset + + +def test_extract_tool_calls_no_tools(mistral_pre_v11_tool_parser): + model_output = "This is a test" + extracted_tool_calls = mistral_pre_v11_tool_parser.extract_tool_calls( + model_output, request=None + ) # type: ignore[arg-type] + assert not extracted_tool_calls.tools_called + assert extracted_tool_calls.tool_calls == [] + assert extracted_tool_calls.content == model_output + + +@pytest.mark.parametrize( + ids=[ + "single_tool_add", + "single_tool_weather", + "argument_before_name", + "argument_before_name_and_name_in_argument", + ], + argnames=["model_output", "expected_tool_calls", "expected_content"], + argvalues=[ + ( + """[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5, "b": 4}}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ) + ], + None, + ), + ( + """[TOOL_CALLS] [{"name": "get_current_weather", "arguments":{"city": "San Francisco", "state": "CA", "unit": "celsius"}}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + None, + ), + ( + """[TOOL_CALLS] [{"arguments":{"city": "San Francisco", "state": "CA", "unit": "celsius"}, "name": "get_current_weather"}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + None, + ), + ( + """[TOOL_CALLS] [{"arguments":{"name": "John Doe"}, "name": "get_age"}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_age", + arguments=json.dumps( + { + "name": "John Doe", + } + ), + ) + ) + ], + None, + ), + ], +) +def test_extract_tool_calls_pre_v11_tokenizer( + mistral_pre_v11_tool_parser, model_output, expected_tool_calls, expected_content +): + extracted_tool_calls = mistral_pre_v11_tool_parser.extract_tool_calls( + model_output, request=None + ) # type: ignore[arg-type] + assert extracted_tool_calls.tools_called + + assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) + + assert extracted_tool_calls.content == expected_content + + +@pytest.mark.parametrize( + ids=[ + "single_tool_add", + "single_tool_weather", + "multiple_tool_calls", + ], + argnames=["model_output", "expected_tool_calls", "expected_content"], + argvalues=[ + ( + """[TOOL_CALLS]add_this_and_that{"a": 3.5, "b": 4}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add_this_and_that", + arguments=json.dumps({"a": 3.5, "b": 4}), + ) + ) + ], + None, + ), + ( + """[TOOL_CALLS]get_current_weather{"city": "San Francisco", "state": "CA", "unit": "celsius"}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + None, + ), + ( + """[TOOL_CALLS]add{"a": 3.5, "b": 4}[TOOL_CALLS]multiply{"a": 3, "b": 6}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="multiply", arguments=json.dumps({"a": 3, "b": 6}) + ) + ), + ], + None, + ), + ], +) +def test_extract_tool_calls( + mistral_tool_parser, model_output, expected_tool_calls, expected_content +): + extracted_tool_calls = mistral_tool_parser.extract_tool_calls( + model_output, request=None + ) # type: ignore[arg-type] + assert extracted_tool_calls.tools_called + + assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) + + assert extracted_tool_calls.content == expected_content + + +def _test_extract_tool_calls_streaming( + tool_parser, tokenizer, model_output, tools, expected_tool_calls, expected_content +): + other_content: str = "" + function_names: list[str] = [] + function_args_strs: list[str] = [] + tool_call_idx: int = -1 + tool_call_ids: list[str | None] = [] + + for delta_message in stream_delta_message_generator( + tool_parser, tokenizer, model_output, tools + ): + # role should never be streamed from tool parser + assert not delta_message.role + + if delta_message.content: + other_content += delta_message.content + + streamed_tool_calls = delta_message.tool_calls + + if streamed_tool_calls and len(streamed_tool_calls) > 0: + # make sure only one diff is present - correct even for parallel + assert len(streamed_tool_calls) == 1 + tool_call = streamed_tool_calls[0] + + assert len(tool_parser.prev_tool_call_arr) > 0 + + # if a new tool is being called, set up empty arguments + if tool_call.index != tool_call_idx: + tool_call_idx = tool_call.index + function_args_strs.append("") + tool_call_ids.append(None) + + # if a tool call ID is streamed, make sure one hasn't been already + if tool_call.id and not tool_call_ids[tool_call.index]: + tool_call_ids[tool_call.index] = tool_call.id + + # if parts of the function start being streamed + if tool_call.function: + # if the function name is defined, set it. it should be streamed + # IN ENTIRETY, exactly one time. + if tool_call.function.name: + assert isinstance(tool_call.function.name, str) + function_names.append(tool_call.function.name) + + if tool_call.function.arguments: + # make sure they're a string and then add them to the list + assert isinstance(tool_call.function.arguments, str) + + function_args_strs[tool_call.index] += tool_call.function.arguments + + assert other_content == expected_content + + actual_tool_calls = [ + ToolCall( + id=tool_call_id, + function=FunctionCall( + name=function_name, + arguments=partial_json_parser.ensure_json( + function_args_str, Allow.OBJ | Allow.STR + ), + ), + ) + for tool_call_id, function_name, function_args_str in zip( + tool_call_ids, function_names, function_args_strs + ) + ] + assert_tool_calls(actual_tool_calls, expected_tool_calls) + + +@pytest.mark.parametrize( + ids=[ + "no_tools", + "single_tool_add", + "single_tool_add_strings", + "single_tool_weather", + "argument_before_name", + "argument_before_name_and_name_in_argument", + "multiple_tools", + ], + argnames=["model_output", "expected_tool_calls", "expected_content"], + argvalues=[ + ("""This is a test""", [], """This is a test"""), + ( + """[TOOL_CALLS] [ {"name":"add" , "arguments" : {"a": 3, "b": 4} } ]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3, "b": 4}) + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"name": "add", "arguments":{"a": "3", "b": "4"}}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": "3", "b": "4"}) + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"city": "San Francisco", "state": "CA", "unit": "celsius"}}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"arguments": {"city": "San Francisco", "state": "CA", "unit": "celsius"}, "name": "get_current_weather"}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"arguments": {"name": "John Doe"}, "name": "get_age"}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_age", + arguments=json.dumps( + { + "name": "John Doe", + } + ), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"name": "add", "arguments": {"a": 3.5, "b": 4}}, {"name": "get_current_weather", "arguments":{"city": "San Francisco", "state": "CA", "unit": "celsius"}}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ), + ], + "", + ), + ], +) +def test_extract_tool_calls_streaming_pre_v11_tokenizer( + mistral_pre_v11_tool_parser, + mistral_pre_v11_tokenizer, + model_output, + expected_tool_calls, + expected_content, +): + _test_extract_tool_calls_streaming( + mistral_pre_v11_tool_parser, + mistral_pre_v11_tokenizer, + model_output, + None, + expected_tool_calls, + expected_content, + ) + + +@pytest.mark.parametrize( + ids=[ + "single_tool_add", + "single_tool_add_strings", + "multiple_tools", + ], + argnames=["tools", "expected_tool_calls", "expected_content"], + argvalues=[ + ( + [("add", '{"a": 3, "b": 4}')], + # [TOOL_CALLS]add{"a": 3, "b": 4} + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3, "b": 4}) + ) + ) + ], + "", + ), + ( + [("add_two_strings", '{"a": "3", "b": "4"}')], + # [TOOL_CALLS]add_two_strings{"a": "3", "b": "4"} + [ + ToolCall( + function=FunctionCall( + name="add_two_strings", + arguments=json.dumps({"a": "3", "b": "4"}), + ) + ) + ], + "", + ), + ( + [ + ("add", '{"a": 3.5, "b": 4}'), + ( + "get_current_weather", + '{"city": "San Francisco", "state": "CA", "unit": "celsius"}', # noqa: E501 + ), + ], + # [TOOL_CALLS]add{"a": 3.5, "b": 4}[TOOL_CALLS]get_current_weather{"city": "San Francisco", "state": "CA", "unit": "celsius"} # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ), + ], + "", + ), + ], +) +def test_extract_tool_calls_streaming( + mistral_tool_parser, + mistral_tokenizer, + tools, + expected_tool_calls, + expected_content, +): + _test_extract_tool_calls_streaming( + mistral_tool_parser, + mistral_tokenizer, + None, + tools, + expected_tool_calls, + expected_content, + ) + + +@pytest.mark.parametrize( + ids=[ + "single_tool_add", + "single_tool_weather", + "multiple_tool_calls", + "content_before_tool", + ], + argnames=["model_output", "expected_tool_calls", "expected_content"], + argvalues=[ + ( + """[TOOL_CALLS]add_this_and_that{"a": 3.5, "b": 4}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add_this_and_that", + arguments=json.dumps({"a": 3.5, "b": 4}), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS]get_current_weather{"city": "San Francisco", "state": "CA", "unit": "celsius"}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS]add{"a": 3.5, "b": 4}[TOOL_CALLS]multiply{"a": 3, "b": 6}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="multiply", arguments=json.dumps({"a": 3, "b": 6}) + ) + ), + ], + "", + ), + ( + # Additional content should not be after the tool calls + """bla[TOOL_CALLS]add_this_and_that{"a": 3.5, "b": 4}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add_this_and_that", + arguments=json.dumps({"a": 3.5, "b": 4}), + ) + ) + ], + "bla", + ), + ], +) +def test_extract_tool_calls_streaming_one_chunk( + mistral_tool_parser, + mistral_tokenizer, + model_output, + expected_tool_calls, + expected_content, +): + if isinstance(mistral_tokenizer, MistralTokenizer): + all_token_ids = mistral_tokenizer.encode(model_output) + else: + all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False) + all_token_ids = fix_tool_call_tokenization( + all_token_ids, mistral_tool_parser, mistral_tokenizer + ) + + delta_message = mistral_tool_parser.extract_tool_calls_streaming( + previous_text="", + current_text=model_output, + delta_text=model_output, + previous_token_ids=[], + current_token_ids=all_token_ids, + delta_token_ids=all_token_ids, + request=None, + ) # type: ignore[arg-type] + assert isinstance(delta_message, DeltaMessage) + assert len(delta_message.tool_calls) == len(expected_tool_calls) + + assert_tool_calls(delta_message.tool_calls, expected_tool_calls) + + if delta_message.content is None: + assert expected_content == "" + else: + assert delta_message.content == expected_content + + +@pytest.mark.parametrize( + ids=[ + "no_tools", + "single_tool_add", + "single_tool_add_strings", + "single_tool_weather", + "argument_before_name", + "argument_before_name_and_name_in_argument", + "multiple_tools", + ], + argnames=["model_output", "expected_tool_calls", "expected_content"], + argvalues=[ + ("""This is a test""", [], """This is a test"""), + ( + """[TOOL_CALLS] [ {"name":"add" , "arguments" : {"a": 3, "b": 4} } ]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3, "b": 4}) + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"name": "add", "arguments":{"a": "3", "b": "4"}}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": "3", "b": "4"}) + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"city": "San Francisco", "state": "CA", "unit": "celsius"}}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"arguments": {"city": "San Francisco", "state": "CA", "unit": "celsius"}, "name": "get_current_weather"}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"arguments": {"name": "John Doe"}, "name": "get_age"}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_age", + arguments=json.dumps( + { + "name": "John Doe", + } + ), + ) + ) + ], + "", + ), + ( + """[TOOL_CALLS] [{"arguments": {"a": 3.5, "b": 4}, "name": "add"}, {"arguments":{"city": "San Francisco", "state": "CA", "unit": "celsius"}, "name": "get_current_weather"}]""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ), + ], + "", + ), + ], +) +def test_extract_tool_calls_streaming_pre_v11_tokenizer_one_chunk( + mistral_pre_v11_tool_parser, + mistral_pre_v11_tokenizer, + model_output, + expected_tool_calls, + expected_content, +): + if isinstance(mistral_pre_v11_tokenizer, MistralTokenizer): + all_token_ids = mistral_pre_v11_tokenizer.encode(model_output) + else: + all_token_ids = mistral_pre_v11_tokenizer.encode( + model_output, add_special_tokens=False + ) + all_token_ids = fix_tool_call_tokenization( + all_token_ids, mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer + ) + + delta_message = mistral_pre_v11_tool_parser.extract_tool_calls_streaming( + previous_text="", + current_text=model_output, + delta_text=model_output, + previous_token_ids=[], + current_token_ids=all_token_ids, + delta_token_ids=all_token_ids, + request=None, + ) # type: ignore[arg-type] + assert isinstance(delta_message, DeltaMessage) + assert len(delta_message.tool_calls) == len(expected_tool_calls) + + assert_tool_calls(delta_message.tool_calls, expected_tool_calls) + + if delta_message.content is None: + assert expected_content == "" + else: + assert delta_message.content == expected_content diff --git a/tests/tool_use/utils.py b/tests/tool_use/utils.py index 7584b903156b7..de7284a309c53 100644 --- a/tests/tool_use/utils.py +++ b/tests/tool_use/utils.py @@ -123,7 +123,7 @@ CONFIGS: dict[str, ServerConfig] = { "supports_parallel": True, "extended": True, }, - "mistral": { + "mistral-7b": { "model": "mistralai/Mistral-7B-Instruct-v0.3", "arguments": [ "--enforce-eager", @@ -145,6 +145,32 @@ CONFIGS: dict[str, ServerConfig] = { "call the tool. Otherwise, answer the user's query directly " "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT " "to the user's question - just respond to it normally.", + "supports_parallel": True, + }, + "mistral-small-3.2": { + "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "arguments": [ + "--enforce-eager", + "--no-enable-prefix-caching", + "--tool-call-parser", + "mistral", + "--tokenizer-mode", + "mistral", + "--config-format", + "mistral", + "--load-format", + "mistral", + "--tensor-parallel-size", + "4", + '--ignore-patterns="consolidated.safetensors"', + ], + "system_prompt": "You are a helpful assistant with access to tools. If a tool" + " that you have would be helpful to answer a user query, " + "call the tool. Otherwise, answer the user's query directly " + "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT " + "to the user's question - just respond to it normally.", + "supports_parallel": True, + "extended": True, }, # FIXME: This test currently fails, need to debug why. # "granite20b": { diff --git a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py index b89db60545abd..aa5089ffe84d7 100644 --- a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py @@ -3,12 +3,12 @@ import json from collections.abc import Sequence +from enum import Enum, auto from random import choices from string import ascii_letters, digits -import partial_json_parser +import ijson import regex as re -from partial_json_parser.core.options import Allow from pydantic import Field from vllm.entrypoints.openai.protocol import ( @@ -23,7 +23,6 @@ from vllm.entrypoints.openai.protocol import ( from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( ToolParser, ) -from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger from vllm.tokenizers import MistralTokenizer, TokenizerLike @@ -32,6 +31,22 @@ logger = init_logger(__name__) ALPHANUMERIC = ascii_letters + digits +class StreamingState(Enum): + """Enum for tracking the current streaming parsing state.""" + + WAITING_FOR_TOOL_START = auto() + WAITING_FOR_TOOL_KEY = ( + auto() + ) # waiting for the "name" or "arguments" key to be complete + PARSING_NAME = auto() + PARSING_NAME_COMPLETED = auto() + WAITING_FOR_ARGUMENTS_START = auto() + PARSING_ARGUMENTS = auto() + PARSING_ARGUMENTS_COMPLETED = auto() + TOOL_COMPLETE = auto() + ALL_TOOLS_COMPLETE = auto() + + class MistralToolCall(ToolCall): id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id()) @@ -46,8 +61,8 @@ class MistralToolCall(ToolCall): return id.isalnum() and len(id) == 9 -def _is_fn_name_regex_support(model_tokenizer: TokenizerLike) -> bool: - return ( +def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool: + return not ( isinstance(model_tokenizer, MistralTokenizer) and model_tokenizer.version >= 11 ) @@ -69,16 +84,22 @@ class MistralToolParser(ToolParser): # initialize properties used for state when parsing tool calls in # streaming mode - self.prev_tool_call_arr: list[dict] = [] self.current_tool_id: int = -1 - self.current_tool_name_sent: bool = False - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list + self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START + + # For streaming pre v11 tokenizer tool calls + self.current_tool_name: str | None = None + self.current_tool_mistral_id: str | None = None + self.starting_new_tool = False + if _is_pre_v11_tokeniser(self.model_tokenizer): + self.parse_coro = ijson.parse_coro( + self.update_stream_state_pre_v11_tokenizer() + ) + self.bot_token = "[TOOL_CALLS]" self.bot_token_id = self.vocab.get(self.bot_token) self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL) - if _is_fn_name_regex_support(self.model_tokenizer): + if not _is_pre_v11_tokeniser(self.model_tokenizer): self.fn_name_regex = re.compile( r"([a-zA-Z0-9_-]+)(\{[\s\S]*?\}+)", re.DOTALL ) @@ -131,18 +152,19 @@ class MistralToolParser(ToolParser): # jsons is difficult try: if self.fn_name_regex: - matches = self.fn_name_regex.findall(tool_content) - function_call_arr = [] - for match in matches: - fn_name = match[0] - args = match[1] + for single_tool_content in model_output.split(self.bot_token): + matches = self.fn_name_regex.findall(single_tool_content) - # fn_name is encoded outside serialized json dump - # only arguments are serialized - function_call_arr.append( - {"name": fn_name, "arguments": json.loads(args)} - ) + for match in matches: + fn_name = match[0] + args = match[1] + + # fn_name is encoded outside serialized json dump + # only arguments are serialized + function_call_arr.append( + {"name": fn_name, "arguments": json.loads(args)} + ) else: function_call_arr = json.loads(tool_content) except json.JSONDecodeError: @@ -193,198 +215,372 @@ class MistralToolParser(ToolParser): delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> DeltaMessage | None: - # if the tool call token is not in the tokens generated so far, append - # output to contents since it's not a tool - if self.bot_token not in current_text: + if self.bot_token_id not in current_token_ids: + # if the tool call token is not in the tokens generated so far, + # append output to contents since it's not a tool return DeltaMessage(content=delta_text) - # if the tool call token ID IS in the tokens generated so far, that + # if the tool call token IS in the tokens generated so far, that # means we're parsing as tool calls now - - # handle if we detected the BOT token which means the start of tool - # calling - if self.bot_token_id in delta_token_ids and len(delta_token_ids) == 1: - # if it's the only token, return None, so we don't send a chat - # completion any don't send a control token - return None - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR try: - # replace BOT token with empty string, and convert single quotes - # to double to allow parsing as JSON since mistral uses single - # quotes instead of double for tool calls - parsable_arr = current_text.split(self.bot_token)[-1] - - # tool calls are generated in an array, so do partial JSON - # parsing on the entire array - try: - tool_call_arr: list[dict] = partial_json_parser.loads( - parsable_arr, flags + if _is_pre_v11_tokeniser(self.model_tokenizer): + return self._extract_tool_calls_streaming_pre_v11_tokenizer( + delta_text=delta_text, + delta_token_ids=delta_token_ids, ) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug("not enough tokens to parse into JSON yet") - return None - - # select as the current tool call the one we're on the state at - - current_tool_call: dict = ( - tool_call_arr[self.current_tool_id] if len(tool_call_arr) > 0 else {} - ) - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if len(tool_call_arr) == 0: - return None - - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - elif ( - len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1 - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - diff: str | None = current_tool_call.get("arguments") - - if diff: - diff = json.dumps(diff, ensure_ascii=False).replace( - self.streamed_args_for_tool[self.current_tool_id], "" - ) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += diff - else: - delta = None - else: - delta = None - # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("starting on new tool %d", self.current_tool_id) - return delta - - # case: update an existing tool - this is handled below - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - if not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=MistralToolCall.generate_random_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tool_name_sent = True - else: - delta = None - - # now we know we're on the same tool call and we're streaming - # arguments else: - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" + return self._extract_tool_calls_streaming( + delta_text=delta_text, delta_token_ids=delta_token_ids ) - cur_arguments = current_tool_call.get("arguments") - - new_text = delta_text.replace("'", '"') - if '"}' in new_text: - new_text = new_text[: new_text.rindex('"}')] - - if not cur_arguments and not prev_arguments: - delta = None - elif not cur_arguments and prev_arguments: - logger.error( - "INVARIANT - impossible to have arguments reset mid-arguments" - ) - delta = None - elif cur_arguments and not prev_arguments: - cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False)[ - :-2 - ] - logger.debug("finding %s in %s", new_text, cur_arguments_json) - - if new_text not in cur_arguments_json: - return None - arguments_delta = cur_arguments_json[ - : cur_arguments_json.rindex(new_text) + len(new_text) - ] - logger.debug( - "First tokens in arguments received: %s", arguments_delta - ) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=arguments_delta - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += arguments_delta - - elif cur_arguments and prev_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - logger.debug( - "Searching for diff between \n%s\n%s", - cur_args_json, - prev_args_json, - ) - - argument_diff = extract_intermediate_diff( - cur_args_json, prev_args_json - ) - logger.debug("got arguments diff: %s", argument_diff) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - else: - # try parsing it with regular JSON - if it works we're - # at the end, and we need to send the difference between - # tokens streamed so far and the valid JSON - delta = None - - # check to see if the name is defined and has been sent. if so, - # stream the name - otherwise keep waiting - # finish by setting old and returning None as base case - self.prev_tool_call_arr = tool_call_arr - return delta - except Exception: logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) return None + + def _extract_tool_calls_streaming( + self, + delta_text: str, + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + """ + Extracts tool calls for Mistral models + doing tool calls of the following format: + `[TOOL_CALLS]add{"a": 3.5, "b": 4}` + """ + additional_content: str = "" + if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START: + # this is the first tool call + assert self.bot_token_id in delta_token_ids + if not delta_text.startswith(self.bot_token): + additional_content += delta_text.split(self.bot_token)[0] + delta_text = self.bot_token + "".join( + delta_text.split(self.bot_token)[1:] + ) + + delta_tool_calls = self._generate_delta_tool_call(delta_text) + if not additional_content and len(delta_tool_calls) == 0: + if self.streaming_state in [ + StreamingState.PARSING_ARGUMENTS, + StreamingState.PARSING_ARGUMENTS_COMPLETED, + StreamingState.TOOL_COMPLETE, + StreamingState.ALL_TOOLS_COMPLETE, + ]: + # Return an empty DeltaMessage once the tool calls are all done + # so that finish_reason gets set. + return DeltaMessage() + else: + # return None when the tool is not likely to be finished + # This can occur when the name is being parsed for example + # and we wait for the name to be complete + # before sending the function name + return None + + delta = DeltaMessage() + if additional_content: + delta.content = additional_content + if len(delta_tool_calls) > 0: + delta.tool_calls = delta_tool_calls + + # HACK: serving_chat.py inspects the internal state of tool parsers + # when determining its final streaming delta, automatically + # adding autocompleted JSON. + # These two lines avoid that nonsense while ensuring finish_reason + # is set to tool_calls when at least one tool is called. + if delta_tool_calls and not self.prev_tool_call_arr: + self.prev_tool_call_arr = [{"arguments": {}}] + return delta + + def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]: + if delta_text == "" or delta_text is None: + return [] + delta_function_name = None + tool_id = None + if self.streaming_state not in [ + StreamingState.PARSING_NAME, + StreamingState.PARSING_ARGUMENTS, + ] and delta_text.startswith(self.bot_token): + self.current_tool_id += 1 + self.streaming_state = StreamingState.PARSING_NAME + delta_text = delta_text.replace(self.bot_token, "", 1) + if self.streaming_state == StreamingState.PARSING_NAME: + if self.current_tool_name is None: + self.current_tool_name = "" + # The name stops where the arguments start + # And the arguments start with the `{` char + if "{" in delta_text: + tool_id = MistralToolCall.generate_random_id() + delta_function_name = delta_text.split("{")[0] + self.current_tool_name += delta_function_name + delta_text = delta_text[len(delta_function_name) :] + self.streaming_state = StreamingState.PARSING_ARGUMENTS + else: + # we want to send the tool name once it's complete + self.current_tool_name += delta_text + return [] + if self.streaming_state == StreamingState.PARSING_ARGUMENTS: + next_function_text = None + if self.bot_token in delta_text: + # current tool call is over + delta_arguments = "" + delta_arguments += delta_text.split(self.bot_token)[0] + next_function_text = delta_text[len(delta_arguments) :] + self.streaming_state = StreamingState.TOOL_COMPLETE + else: + delta_arguments = delta_text + ret = [] + if self.current_tool_name or delta_arguments: + ret += [ + DeltaToolCall( + index=self.current_tool_id, + type="function", + id=tool_id, + function=DeltaFunctionCall( + name=self.current_tool_name, arguments=delta_arguments + ).model_dump(exclude_none=True), + ) + ] + self.current_tool_name = None + if next_function_text: + ret += self._generate_delta_tool_call(next_function_text) + return ret + # Should not happen + return [] + + @ijson.coroutine + def update_stream_state_pre_v11_tokenizer(self): + while True: + (prefix, event, value) = yield + + if prefix == "item" and event == "start_map": + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY + if prefix == "item" and event == "map_key" and value == "name": + self.streaming_state = StreamingState.PARSING_NAME + if prefix == "item.name" and event == "string": + self.current_tool_name = value + self.streaming_state = StreamingState.PARSING_NAME_COMPLETED + if prefix == "item" and event == "map_key" and value == "arguments": + self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START + if prefix == "item.arguments" and event == "start_map": + self.streaming_state = StreamingState.PARSING_ARGUMENTS + if prefix == "item.arguments" and event == "end_map": + self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED + if prefix == "item" and event == "end_map": + self.streaming_state = StreamingState.TOOL_COMPLETE + if prefix == "" and event == "end_array": + self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE + + def _extract_tool_calls_streaming_pre_v11_tokenizer( + self, + delta_text: str, + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + """ + Extracts tool calls for Mistral models + doing tool calls of the following format: + `[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5, "b": 4}}` + """ + assert self.parse_coro is not None + content = None + delta_tool_calls: list[DeltaToolCall] = [] + current_tool_call: DeltaToolCall = DeltaToolCall( + index=self.current_tool_id, type="function" + ) + current_tool_call_modified = False + if self.bot_token_id in delta_token_ids: + # this is the first tool call + if not delta_text.startswith(self.bot_token): + content = delta_text.split(self.bot_token)[0] + delta_text = "".join(delta_text.split(self.bot_token)[1:]) + + # Cut smartly the delta text to catch the ijson events + # as ijson does not give us the index in the text at each event. + # We need to cut so that we know + # where in the text the events are emitted from. + while len(delta_text) > 0: + streaming_state_before_parse = self.streaming_state + + if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_opening_curly_braces=1, + ) + elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY: + # Wait until another key is sent + # or the current tool is completed + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_colon=1, + stop_after_opening_curly_braces=1, + # if the tool ends, we want to separate + # at the start of the next tool + ) + elif self.streaming_state == StreamingState.PARSING_NAME: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_comma=1, + stop_after_closing_brackets=1, + ) + elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_opening_curly_braces=1, + ) + elif self.streaming_state == StreamingState.PARSING_ARGUMENTS: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_closing_curly_braces=1, + # we could be more clever + # by listening to item.arguments.* start_map events + # and know how many curly braces we can allow + ) + elif self.streaming_state in [ + StreamingState.PARSING_ARGUMENTS_COMPLETED, + StreamingState.PARSING_NAME_COMPLETED, + ]: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_closing_curly_braces=1, + stop_after_closing_brackets=1, + ) + elif self.streaming_state == StreamingState.TOOL_COMPLETE: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_opening_curly_braces=1, + stop_after_closing_brackets=1, + ) + elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: + content = delta_text + delta_text = "" + else: + delta_to_be_parsed = delta_text + delta_text = "" + + if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE: + self.parse_coro.send(delta_to_be_parsed.encode("utf-8")) + + # Given the parsed text and the possible streaming state change, + # let's add to the tool delta + if ( + (streaming_state_before_parse != self.streaming_state) + and streaming_state_before_parse + in [StreamingState.WAITING_FOR_TOOL_START, StreamingState.TOOL_COMPLETE] + and self.streaming_state + not in [ + StreamingState.ALL_TOOLS_COMPLETE, + StreamingState.TOOL_COMPLETE, + StreamingState.WAITING_FOR_TOOL_START, + ] + ): + # starting a new tool call + if current_tool_call_modified: + if self.current_tool_mistral_id is not None: + current_tool_call.id = self.current_tool_mistral_id + self.current_tool_mistral_id = None + delta_tool_calls.append(current_tool_call) + current_tool_call_modified = False + self.current_tool_id += 1 + self.current_tool_mistral_id = MistralToolCall.generate_random_id() + current_tool_call = DeltaToolCall( + index=self.current_tool_id, + type="function", + ) + if current_tool_call.function is None: + current_tool_call.function = DeltaFunctionCall() + + if self.current_tool_name is not None: + # we have the complete tool name + current_tool_call_modified = True + current_tool_call.function.name = self.current_tool_name + self.current_tool_name = None + if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED: + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY + if self.streaming_state in [ + StreamingState.PARSING_ARGUMENTS, + StreamingState.PARSING_ARGUMENTS_COMPLETED, + ]: + if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED: + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY + # the delta_to_be_parsed is part of arguments. + current_tool_call_modified = True + if current_tool_call.function.arguments is None: + current_tool_call.function.arguments = delta_to_be_parsed + else: + current_tool_call.function.arguments += delta_to_be_parsed + if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS: + # It's the first chunk of arg. let's lstrip it + current_tool_call.function.arguments = ( + current_tool_call.function.arguments.lstrip() + ) + + if current_tool_call_modified: + if self.current_tool_mistral_id is not None: + current_tool_call.id = self.current_tool_mistral_id + self.current_tool_mistral_id = None + delta_tool_calls.append(current_tool_call) + + # HACK: serving_chat.py inspects the internal state of tool parsers + # when determining it's final streaming delta, automatically + # adding autocompleted JSON. + # These two lines avoid that nonsense while ensuring finish_reason + # is set to tool_calls when at least one tool is called. + if delta_tool_calls and not self.prev_tool_call_arr: + self.prev_tool_call_arr = [{"arguments": {}}] + + if content or len(delta_tool_calls) > 0: + delta_message = DeltaMessage() + if content: + delta_message.content = content + if len(delta_tool_calls) > 0: + delta_message.tool_calls = delta_tool_calls + return delta_message + else: + if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: + return DeltaMessage() + else: + return None + + def _split_delta( + self, + delta_text: str, + stop_after_quotes: int = -1, + stop_after_opening_curly_braces: int = -1, + stop_after_closing_curly_braces: int = -1, + stop_after_closing_brackets: int = -1, + stop_after_colon: int = -1, + stop_after_comma=-1, + ) -> tuple[str, str]: + delta_to_be_parsed = "" + for i, c in enumerate(delta_text): + if c in ['"', "'"]: + delta_to_be_parsed += c + stop_after_quotes -= 1 + if stop_after_quotes == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == "{": + delta_to_be_parsed += c + stop_after_opening_curly_braces -= 1 + if stop_after_opening_curly_braces == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == "}": + delta_to_be_parsed += c + stop_after_closing_curly_braces -= 1 + if stop_after_closing_curly_braces == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == "]": + delta_to_be_parsed += c + stop_after_closing_brackets -= 1 + if stop_after_closing_brackets == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == ":": + delta_to_be_parsed += c + stop_after_colon -= 1 + if stop_after_colon == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == ",": + delta_to_be_parsed += c + stop_after_comma -= 1 + if stop_after_comma == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + else: + delta_to_be_parsed += c + + return (delta_to_be_parsed, "") From 19bee6d12d985c231b16374c99836376fc0c5706 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Wed, 3 Dec 2025 13:04:59 -0500 Subject: [PATCH 11/47] [Performance][DP/EP] Add silu_mul_per_token_group_quant_fp8_colmajor kernel (#29470) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath Co-authored-by: Tyler Michael Smith --- .../benchmark_2d_silu_mul_fp8_quant.py | 244 ++++++++++++++++++ ..._mul_per_token_group_quant_fp8_colmajor.py | 86 ++++++ .../layers/fused_moe/deep_gemm_moe.py | 114 +++----- .../layers/quantization/utils/fp8_utils.py | 133 ++++++++++ 4 files changed, 496 insertions(+), 81 deletions(-) create mode 100644 benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py create mode 100644 tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py diff --git a/benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py b/benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py new file mode 100644 index 0000000000000..04921dafbdbea --- /dev/null +++ b/benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py @@ -0,0 +1,244 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass +from enum import Enum +from itertools import product +from typing import Any + +import torch +import torch.utils.benchmark as TBenchmark +from torch.utils.benchmark import Measurement as TMeasurement + +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + _per_token_group_quant_fp8_colmajor, + silu_mul_per_token_group_quant_fp8_colmajor, +) +from vllm.triton_utils import triton +from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used + +from .utils import ArgPool, Bench, CudaGraphBenchParams + +GROUP_SIZE = 128 +FLOAT8_T = torch.float8_e4m3fn + + +def print_timers(timers: list[TMeasurement], cuda_graph_nops: int): + print( + f"Note : The timings reported above is for {cuda_graph_nops} " + "consecutive invocations of the benchmarking functions. " + f"Please divide by {cuda_graph_nops} for single invocation " + "timings." + ) + compare = TBenchmark.Compare(timers) + compare.print() + + +class ImplType(Enum): + SILU_MUL_PER_TOKEN_GROUP_QUANT_FP8_COLMAJOR = 1 + REFERENCE = 2 + + def get_impl(self): + if self == ImplType.SILU_MUL_PER_TOKEN_GROUP_QUANT_FP8_COLMAJOR: + return silu_mul_per_token_group_quant_fp8_colmajor + elif self == ImplType.REFERENCE: + return reference + raise ValueError(f"Unrecognized ImplType {self}") + + +@dataclass +class BenchmarkTensors: + input: torch.Tensor + output: torch.Tensor + + # Reference act output tensor + ref_act_out: torch.Tensor + ref_quant_out: torch.Tensor + + @staticmethod + def make(T: int, N: int) -> "BenchmarkTensors": + assert T % GROUP_SIZE == 0 + assert N % (GROUP_SIZE * 2) == 0 + + input = torch.rand((T, N), dtype=torch.bfloat16, device="cuda") + + # silu_mul_per_token_group_quant_fp8_colmajor output. + output = torch.rand((T, N // 2), dtype=torch.bfloat16, device="cuda").to( + FLOAT8_T + ) + + # reference output. + ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device="cuda") + ref_quant_out = torch.empty( + (T, N // 2), dtype=torch.bfloat16, device="cuda" + ).to(FLOAT8_T) + + return BenchmarkTensors( + input=input, + output=output, + ref_act_out=ref_act_out, + ref_quant_out=ref_quant_out, + ) + + @property + def T(self): + return self.input.size(0) + + @property + def N(self): + return self.input.size(1) + + def make_impl_kwargs(self, impl_type: ImplType) -> dict[str, Any]: + if impl_type == ImplType.SILU_MUL_PER_TOKEN_GROUP_QUANT_FP8_COLMAJOR: + return { + "input": self.input, + "output": self.output, + "use_ue8m0": is_deep_gemm_e8m0_used(), + } + elif impl_type == ImplType.REFERENCE: + return { + "input": self.input, + "act_out": self.ref_act_out, + "quant_out": self.ref_quant_out, + "use_ue8m0": is_deep_gemm_e8m0_used(), + } + raise ValueError(f"Unrecognized impl_type {impl_type}") + + +def reference_quant(x: torch.Tensor, quant_out: torch.Tensor, use_ue8m0: bool): + """ + Reference triton quant kernel from, + vllm.model_executor.layers.quantization.utils.fp8_utils + """ + assert quant_out.size() == x.size() + # Allocate the scale tensor column-major format. + shape = (x.shape[-1] // GROUP_SIZE,) + x.shape[:-1] + x_q = quant_out + x_s = torch.empty(shape, device=x.device, dtype=torch.float32).permute(-1, -2) + + M = x.numel() // GROUP_SIZE + N = GROUP_SIZE + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + num_stages = 1 + + finfo = torch.finfo(FLOAT8_T) + fp8_min = finfo.min + fp8_max = finfo.max + + _per_token_group_quant_fp8_colmajor[(M,)]( + x, + x_q, + x_s, + GROUP_SIZE, + x.shape[1], + x.stride(0), + x_s.stride(1), + eps=1e-10, + fp8_min=fp8_min, + fp8_max=fp8_max, + use_ue8m0=use_ue8m0, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=num_stages, + ) + return x_q, x_s + + +def reference( + input: torch.Tensor, + act_out: torch.Tensor, + quant_out: torch.Tensor, + use_ue8m0: bool, +) -> tuple[torch.Tensor, torch.Tensor]: + torch.ops._C.silu_and_mul(act_out, input) + return reference_quant(act_out, quant_out, use_ue8m0) + + +def bench_impl( + bench_tensors: list[BenchmarkTensors], impl_type: ImplType +) -> TMeasurement: + T = bench_tensors[0].T + N = bench_tensors[0].N + + arg_pool_size = len(bench_tensors) + kwargs_list = [bt.make_impl_kwargs(impl_type) for bt in bench_tensors] + + # warmup + for kwargs in kwargs_list: + impl_type.get_impl()(**kwargs) + torch.cuda.synchronize() + + # Merge into a single kwargs and qualify arguments as ArgPool + kwargs = {k: ArgPool([]) for k in kwargs_list[0]} + for _kwargs in kwargs_list: + for k, v in _kwargs.items(): + kwargs[k].values.append(v) + + cuda_graph_params = None + cuda_graph_params = CudaGraphBenchParams(arg_pool_size) + timer = None + with Bench( + cuda_graph_params, + "silu-mul-quant", + f"num_tokens={T}, N={N}", + impl_type.name, + impl_type.get_impl(), + **kwargs, + ) as bench: + timer = bench.run() + return timer + + +def test_correctness(T: int, N: int): + print(f"Testing num_tokens={T}, N={N} ...") + + bench_tensor = BenchmarkTensors.make(T, N) + + def output_from_impl(impl: ImplType) -> tuple[torch.Tensor, torch.Tensor]: + return impl.get_impl()(**bench_tensor.make_impl_kwargs(impl)) + + # reference output + ref_out_q, ref_out_s = output_from_impl(ImplType.REFERENCE) + + # test ouptut + out_q, out_s = output_from_impl( + ImplType.SILU_MUL_PER_TOKEN_GROUP_QUANT_FP8_COLMAJOR + ) + + torch.testing.assert_close(ref_out_q.to(torch.float32), out_q.to(torch.float32)) + torch.testing.assert_close(ref_out_s, out_s) + + +def run(Ts: list[int], Ns: list[int], arg_pool_size: int) -> list[TMeasurement]: + timers = [] + for N, T in product(Ns, Ts): + test_correctness(T, N) + + bench_tensors: list[BenchmarkTensors] = [ + BenchmarkTensors.make(T, N) for _ in range(arg_pool_size) + ] + + silu_mul_quant_timer = bench_impl( + bench_tensors, ImplType.SILU_MUL_PER_TOKEN_GROUP_QUANT_FP8_COLMAJOR + ) + timers.append(silu_mul_quant_timer) + reference_timer = bench_impl(bench_tensors, ImplType.REFERENCE) + timers.append(reference_timer) + + print_timers( + [silu_mul_quant_timer, reference_timer], cuda_graph_nops=arg_pool_size + ) + + print_timers(timers, cuda_graph_nops=arg_pool_size) + + return timers + + +if __name__ == "__main__": + T = [128 * i for i in range(1, 16)] + [2048 * i for i in range(1, 65)] + N = [2048, 4096, 8192] + + print(f"T = {T}, N = {N}") + run(T, N, arg_pool_size=8) diff --git a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py new file mode 100644 index 0000000000000..e4617072cd52c --- /dev/null +++ b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + _per_token_group_quant_fp8_colmajor, + silu_mul_per_token_group_quant_fp8_colmajor, +) +from vllm.platforms import current_platform +from vllm.triton_utils import triton +from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used + +FLOAT8_DTYPE = torch.float8_e4m3fn +GROUP_SIZE = 128 + + +def reference_quant(x: torch.Tensor, use_ue8m0: bool): + """ + Reference triton quant kernel from, + vllm.model_executor.layers.quantization.utils.fp8_utils + """ + + x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE) + + # Allocate the scale tensor in column-major format. + shape = (x.shape[-1] // GROUP_SIZE,) + x.shape[:-1] + x_s = torch.empty(shape, device=x.device, dtype=torch.float32).permute(-1, -2) + + M = x.numel() // GROUP_SIZE + N = GROUP_SIZE + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + num_stages = 1 + + finfo = torch.finfo(FLOAT8_DTYPE) + fp8_min = finfo.min + fp8_max = finfo.max + + _per_token_group_quant_fp8_colmajor[(M,)]( + x, + x_q, + x_s, + GROUP_SIZE, + x.shape[1], + x.stride(0), + x_s.stride(1), + eps=1e-10, + fp8_min=fp8_min, + fp8_max=fp8_max, + use_ue8m0=use_ue8m0, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=num_stages, + ) + return x_q, x_s + + +def reference(x: torch.Tensor, use_ue8m0: bool) -> tuple[torch.Tensor, torch.Tensor]: + T, N = x.size() + ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device="cuda") + torch.ops._C.silu_and_mul(ref_act_out, x) + return reference_quant(ref_act_out, use_ue8m0) + + +@pytest.mark.parametrize("T", [128, 256, 512]) +@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2]) +def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int): + current_platform.seed_everything(42) + + input = torch.rand((T, N), dtype=torch.bfloat16, device="cuda") + + use_ue8m0 = is_deep_gemm_e8m0_used() + + # Test + output, output_scales = silu_mul_per_token_group_quant_fp8_colmajor( + input, use_ue8m0=use_ue8m0 + ) + + # Reference + ref_output, ref_output_scales = reference(input, use_ue8m0) + + torch.testing.assert_close(output.to(torch.float32), ref_output.to(torch.float32)) + torch.testing.assert_close(output_scales, ref_output_scales) diff --git a/vllm/model_executor/layers/fused_moe/deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/deep_gemm_moe.py index 86cdd25f2c873..9f47e692d5ae2 100644 --- a/vllm/model_executor/layers/fused_moe/deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/deep_gemm_moe.py @@ -2,9 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from tqdm import tqdm -import vllm.envs as env import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.config import ( @@ -25,12 +23,12 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( from vllm.model_executor.layers.fused_moe.utils import _resize_cache from vllm.model_executor.layers.quantization.utils.fp8_utils import ( per_token_group_quant_fp8, + silu_mul_per_token_group_quant_fp8_colmajor, ) from vllm.utils.deep_gemm import ( get_mk_alignment_for_contiguous_layout, m_grouped_fp8_gemm_nt_contiguous, ) -from vllm.utils.func_utils import run_once from vllm.utils.import_utils import has_deep_gemm logger = init_logger(__name__) @@ -108,70 +106,6 @@ def _valid_deep_gemm( return True -@run_once -def warmup_deepgemm_gg_contiguous_kernels( - w1: torch.Tensor, - w2: torch.Tensor, - w1_scale: torch.Tensor, - w2_scale: torch.Tensor, - num_topk: int, -): - """ - DeepGemm JITs the grouped-gemm kernels. The JIT'ing happens based on the - input tensor shapes. In this function, we construct all possible input - tensor shapes so all the kernels are JIT'ed and cached. - Note that this warmup is expected to happen during the model profile - call and not during actual model inference. - """ - - assert w1.size(0) == w2.size(0), "w1 and w2 must have the same number of experts" - - block_m = get_mk_alignment_for_contiguous_layout()[0] - num_experts = w1.size(0) - device = w1.device - - # This is the maximum GroupedGemm M size that we expect to run - # the grouped_gemm with. - MAX_M = compute_aligned_M( - env.VLLM_FUSED_MOE_CHUNK_SIZE, - num_topk, - num_experts, - block_m, - expert_tokens_meta=None, - ) - # Distribute expert-ids evenly. - MAX_BLOCKS = MAX_M // block_m - expert_ids_block = torch.randint( - low=0, high=num_experts, size=(MAX_BLOCKS,), device=device, dtype=torch.int32 - ) - expert_ids = torch.repeat_interleave(expert_ids_block, block_m, dim=0) - - def _warmup(w: torch.Tensor, w_scale: torch.Tensor): - _, n, k = w.size() - a1q = torch.empty((MAX_M, k), device=device).to(torch.float8_e4m3fn) - a1q_scales = torch.empty( - (MAX_M, k // block_m), device=device, dtype=torch.float32 - ) - out = torch.empty((MAX_M, n), device=device, dtype=torch.bfloat16) - - pbar = tqdm( - total=MAX_BLOCKS, desc=f"DeepGemmExperts GEMM warmup (MAX_M={MAX_M})" - ) - num_tokens = MAX_M - while num_tokens > 0: - m_grouped_fp8_gemm_nt_contiguous( - (a1q[:num_tokens], a1q_scales[:num_tokens]), - (w, w_scale), - out[:num_tokens], - expert_ids[:num_tokens], - ) - pbar.update(1) - num_tokens = num_tokens - block_m - - _warmup(w1, w1_scale) - _warmup(w2, w2_scale) - - class DeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute): def __init__(self, quant_config: FusedMoEQuantConfig): super().__init__(quant_config) @@ -215,11 +149,32 @@ class DeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute): ) assert M_sum % block_m == 0 - workspace1 = (M_sum, N) - workspace2 = (M_sum, max(N // 2, K)) + workspace1 = (M_sum, max(N // 2, K)) + workspace2 = (M_sum, max(N, K)) output = (M, K) return (workspace1, workspace2, output) + def _act_mul_quant( + self, input: torch.Tensor, output: torch.Tensor, activation: str + ) -> tuple[torch.Tensor, torch.Tensor]: + if activation == "silu": + return silu_mul_per_token_group_quant_fp8_colmajor( + input=input, output=output + ) + else: + # This is a fallback path. If we find ourselves using any activation other + # than silu, we should add that activation to + # silu_mul_per_token_group_quant_fp8_colmajor kernel as it is much faster. + M_sum, N = input.size() + act_out = torch.empty( + (M_sum, N // 2), dtype=input.dtype, device=input.device + ) + self.activation(activation, act_out, input) + assert self.block_shape is not None + return per_token_group_quant_fp8( + act_out, self.block_shape[1], column_major_scales=True, out_q=output + ) + def apply( self, output: torch.Tensor, @@ -261,14 +216,9 @@ class DeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute): expert_tokens_meta=expert_tokens_meta, ) - a1q_perm = _resize_cache(workspace2.view(dtype=torch.float8_e4m3fn), (M_sum, K)) - mm1_out = _resize_cache(workspace13, (M_sum, N)) - act_out = _resize_cache(workspace2, (M_sum, N // 2)) - quant_out = _resize_cache( - workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, N // 2) + a1q_perm = _resize_cache( + workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K) ) - mm2_out = _resize_cache(workspace2, (M_sum, K)) - a1q, a1q_scale, expert_ids, inv_perm = deepgemm_moe_permute( aq=a1q, aq_scale=a1q_scale, @@ -280,17 +230,19 @@ class DeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute): ) assert a1q.size(0) == M_sum + mm1_out = _resize_cache(workspace2, (M_sum, N)) m_grouped_fp8_gemm_nt_contiguous( (a1q, a1q_scale), (w1, self.w1_scale), mm1_out, expert_ids ) - self.activation(activation, act_out, mm1_out.view(-1, N)) - - a2q_scale: torch.Tensor | None = None - a2q, a2q_scale = per_token_group_quant_fp8( - act_out, self.block_shape[1], column_major_scales=True, out_q=quant_out + quant_out = _resize_cache( + workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, N // 2) + ) + a2q, a2q_scale = self._act_mul_quant( + input=mm1_out.view(-1, N), output=quant_out, activation=activation ) + mm2_out = _resize_cache(workspace2, (M_sum, K)) m_grouped_fp8_gemm_nt_contiguous( (a2q, a2q_scale), (w2, self.w2_scale), mm2_out, expert_ids ) diff --git a/vllm/model_executor/layers/quantization/utils/fp8_utils.py b/vllm/model_executor/layers/quantization/utils/fp8_utils.py index ae63b4a767268..6e73833d1ae1c 100644 --- a/vllm/model_executor/layers/quantization/utils/fp8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/fp8_utils.py @@ -492,6 +492,139 @@ def _per_token_group_quant_fp8( tl.store(y_s_ptr, y_s) +@triton.jit +def _silu_mul_per_token_group_quant_fp8_colmajor( + y_ptr, # [M, N] + y_q_ptr, # [M, N // 2] + y_s_ptr, # [M, (N // 2) // GROUP_SIZE] + M, # num tokens + N, # intermediate size + # Stride + y_s_col_stride: tl.int64, + # Information for float8 + eps, + fp8_min, + fp8_max, + use_ue8m0: tl.constexpr, + # Meta-parameters + GROUP_SIZE: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + # TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks. + """ + Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then + the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills + the outputs tensors at the right positions. + """ + + pid_m = tl.program_id(0) + pid_n = tl.program_id(1) + N_2 = N // 2 + + m_offset = pid_m * BLOCK_M + n_offset = pid_n * BLOCK_N + if m_offset >= M: + return + + offs_n = tl.arange(0, BLOCK_N).to(tl.int64) + offs_m = tl.arange(0, BLOCK_M).to(tl.int64) + + base_y_ptr = y_ptr + m_offset * N + n_offset + + act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :] + + act_in = tl.load(act_in_ptrs) + mul_in = tl.load(act_in_ptrs + N_2) + + # silu & mul + act_in = act_in.to(tl.float32) + one_f32 = tl.cast(1, tl.float32) + silu_out = (act_in / (one_f32 + tl.exp(-act_in))).to(y_ptr.dtype.element_ty) + y = (silu_out * mul_in).to(tl.float32) + + # quant + _absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps) + scale_raw = _absmax / fp8_max + y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw + y_s = tl.reshape(y_s, (BLOCK_M, 1)) + y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty) + + # store y_q + base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset + y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :] + tl.store(y_q_ptrs, y_q) + + # store y_s + group_id = n_offset // GROUP_SIZE + base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset + y_s_ptrs = base_y_s_ptr + offs_m + y_s = tl.reshape(y_s, (BLOCK_M,)) + tl.store(y_s_ptrs, y_s) + + +def silu_mul_per_token_group_quant_fp8_colmajor( + input: torch.Tensor, # [M, N] + output: torch.Tensor | None = None, # [M, N // 2] + use_ue8m0: bool | None = None, + eps: float = 1e-10, +): + """ + silu+mul + block-fp8 quant with group size 128. + """ + GROUP_SIZE = 128 + assert input.ndim == 2 + if output is not None: + assert output.ndim == 2 + assert input.size(0) % GROUP_SIZE == 0 + assert input.size(1) % (GROUP_SIZE * 2) == 0 + + if use_ue8m0 is None: + use_ue8m0 = is_deep_gemm_e8m0_used() + + M, N = input.size() + N_2 = N // 2 + + if output is None: + output = torch.empty((M, N_2), dtype=torch.float8_e4m3fn, device=input.device) + + output_scales = torch.empty( + ((N_2 // GROUP_SIZE), M), dtype=torch.float32, device=input.device + ).transpose(0, 1) + + BLOCK_M = 8 + BLOCK_N = GROUP_SIZE + assert M % BLOCK_M == 0 + assert N_2 % BLOCK_N == 0 + + finfo = torch.finfo(torch.float8_e4m3fn) + fp8_min = finfo.min + fp8_max = finfo.max + + # Force even division so we can avoid edgecases within the kernel. + assert M % BLOCK_M == 0 + assert N_2 % BLOCK_N == 0 + grid = (M // BLOCK_M, N_2 // BLOCK_N) + + _silu_mul_per_token_group_quant_fp8_colmajor[grid]( + input, + output, + output_scales, + M, + N, + output_scales.stride(-1), + eps, + fp8_min, + fp8_max, + use_ue8m0, + GROUP_SIZE, + BLOCK_M, + BLOCK_N, + ) + + return output, output_scales + + @triton.jit def _per_token_group_quant_fp8_colmajor( # Pointers to inputs and output From afe9eb408ee1191cd57a68d46b6ce2860b1b41e1 Mon Sep 17 00:00:00 2001 From: elvischenv <219235043+elvischenv@users.noreply.github.com> Date: Thu, 4 Dec 2025 02:50:53 +0800 Subject: [PATCH 12/47] [Bugfix] Fix flashinfer ar+norm kernel not available issue (#29960) Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com> --- vllm/compilation/fix_functionalization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/compilation/fix_functionalization.py b/vllm/compilation/fix_functionalization.py index 76068f86ebfb3..2625562aadd36 100644 --- a/vllm/compilation/fix_functionalization.py +++ b/vllm/compilation/fix_functionalization.py @@ -104,7 +104,8 @@ class FixFunctionalizationPass(VllmInductorPass): mutated_args = {1: "result"} self.defunctionalize(graph, node, mutated_args) elif ( - at_target + hasattr(torch.ops.vllm, "flashinfer_trtllm_fused_allreduce_norm") + and at_target == torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default ): mutated_args = { From 2fc5d6e0d7596dd93dbf4e1ca776f17449bb2143 Mon Sep 17 00:00:00 2001 From: Yongtao Huang Date: Thu, 4 Dec 2025 04:14:44 +0800 Subject: [PATCH 13/47] Fix LLMEngine.del dp_group cleanup condition (#29954) Signed-off-by: Yongtao Huang --- vllm/v1/engine/llm_engine.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index d21cdf04ead26..8772f2e488dc0 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -409,8 +409,6 @@ class LLMEngine: return self.collective_rpc("apply_model", args=(func,)) def __del__(self): - if ( - dp_group := getattr(self, "dp_group", None) - and not self.external_launcher_dp - ): + dp_group = getattr(self, "dp_group", None) + if dp_group is not None and not self.external_launcher_dp: stateless_destroy_torch_distributed_process_group(dp_group) From ac1886588fd8799ff874b860b6c266a84d5a2b2b Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 3 Dec 2025 15:16:54 -0500 Subject: [PATCH 14/47] [CI] Fix re import error (#29973) Signed-off-by: yewentao256 --- .buildkite/scripts/generate-nightly-index.py | 3 ++- vllm/entrypoints/serve/instrumentator/metrics.py | 3 +-- vllm/tokenizers/deepseek_v32_encoding.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/scripts/generate-nightly-index.py index 8d09ba178db7b..4d28ec9619de9 100644 --- a/.buildkite/scripts/generate-nightly-index.py +++ b/.buildkite/scripts/generate-nightly-index.py @@ -7,13 +7,14 @@ import argparse import json -import re import sys from dataclasses import asdict, dataclass from pathlib import Path from typing import Any from urllib.parse import quote +import regex as re + if not sys.version_info >= (3, 12): raise RuntimeError("This script requires Python 3.12 or higher.") diff --git a/vllm/entrypoints/serve/instrumentator/metrics.py b/vllm/entrypoints/serve/instrumentator/metrics.py index efe0c63a90714..5231451383a2b 100644 --- a/vllm/entrypoints/serve/instrumentator/metrics.py +++ b/vllm/entrypoints/serve/instrumentator/metrics.py @@ -2,9 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import re - import prometheus_client +import regex as re from fastapi import FastAPI, Response from prometheus_client import make_asgi_app from prometheus_fastapi_instrumentator import Instrumentator diff --git a/vllm/tokenizers/deepseek_v32_encoding.py b/vllm/tokenizers/deepseek_v32_encoding.py index 72f43395b192e..fb8989e651e3a 100644 --- a/vllm/tokenizers/deepseek_v32_encoding.py +++ b/vllm/tokenizers/deepseek_v32_encoding.py @@ -5,9 +5,10 @@ # copy from https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py import copy import json -import re from typing import Any +import regex as re + # flake8: noqa: E501 TOOLS_SYSTEM_TEMPLATE = """## Tools You have access to a set of tools you can use to answer the user's question. From 2902c348265639de300c95cbcae1c26486f57ac7 Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Wed, 3 Dec 2025 15:49:00 -0500 Subject: [PATCH 15/47] [Kernels] Remove BatchedTritonOrDeepGemmExperts and default fallback to Triton (#29929) Signed-off-by: Bill Nell Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Tyler Michael Smith --- docs/design/moe_kernel_features.md | 3 +- .../moe/modular_kernel_tools/mk_objects.py | 17 -- .../layers/fused_moe/__init__.py | 4 - .../batched_triton_or_deep_gemm_moe.py | 180 ------------------ .../compressed_tensors_moe.py | 59 ++++-- 5 files changed, 46 insertions(+), 217 deletions(-) delete mode 100644 vllm/model_executor/layers/fused_moe/batched_triton_or_deep_gemm_moe.py diff --git a/docs/design/moe_kernel_features.md b/docs/design/moe_kernel_features.md index 44aaa65218cc4..48341d199cb80 100644 --- a/docs/design/moe_kernel_features.md +++ b/docs/design/moe_kernel_features.md @@ -90,7 +90,6 @@ To be used with a particular `FusedMoEPrepareAndFinalize` subclass, MoE kernels | cutlass_fp8 | standard,
batched | fp8 | A,T | silu, gelu | Y | Y | [`cutlass_moe_fp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.cutlass_moe_fp8],
[`CutlassExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassExpertsFp8],
[`CutlasBatchedExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassBatchedExpertsFp8] | | flashinfer | standard | nvfp4,
fp8 | T | 5 | N | Y | [`flashinfer_cutlass_moe_fp4`][vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.flashinfer_cutlass_moe_fp4],
[`FlashInferExperts`][vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.FlashInferExperts] | | gpt oss triton | standard | N/A | N/A | 5 | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],
[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moe.OAITritonExperts] | -| deep gemm+triton2 | standard,
batched | all1 | G(128),A,T | silu, gelu | 6 | Y | [`TritonOrDeepGemmExperts`][vllm.model_executor.layers.fused_moe.triton_deep_gemm_moe.TritonOrDeepGemmExperts],
[`BatchedTritonOrDeepGemmExperts`][vllm.model_executor.layers.fused_moe.batched_triton_or_deep_gemm_moe.BatchedTritonOrDeepGemmExperts] | | marlin | standard,
batched | 3 / N/A | 3 / N/A | silu,
swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.fused_marlin_moe],
[`MarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.MarlinExperts],
[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.BatchedMarlinExperts] | | trtllm | standard | mxfp4,
nvfp4 | G(16),G(32) | 5 | N | Y | [`TrtLlmGenExperts`][vllm.model_executor.layers.fused_moe.trtllm_moe.TrtLlmGenExperts] | | pallas | standard | N/A | N/A | silu | N | N | [`fused_moe`][vllm.model_executor.layers.fused_moe.moe_pallas.fused_moe] | @@ -114,5 +113,5 @@ The following table shows "families" of modular kernels that are intended to wor | backend | `FusedMoEPrepareAndFinalize` subclasses | `FusedMoEPermuteExpertsUnpermute` subclasses | |---------|-----------------------------------------|----------------------------------------------| | deepep_high_throughput | `DeepEPHTPrepareAndFinalize` | `DeepGemmExperts`,
`TritonExperts`,
`TritonOrDeepGemmExperts`,
`CutlassExpertsFp8`,
`MarlinExperts` | -| deepep_low_latency,
pplx | `DeepEPLLPrepareAndFinalize`,
`PplxPrepareAndFinalize` | `BatchedDeepGemmExperts`,
`BatchedTritonExperts`,
`BatchedTritonOrDeepGemmExperts`,
`CutlassBatchedExpertsFp8`,
`BatchedMarlinExperts` | +| deepep_low_latency,
pplx | `DeepEPLLPrepareAndFinalize`,
`PplxPrepareAndFinalize` | `BatchedDeepGemmExperts`,
`BatchedTritonExperts`,
`CutlassBatchedExpertsFp8`,
`BatchedMarlinExperts` | | flashinfer | `FlashInferCutlassMoEPrepareAndFinalize` | `FlashInferExperts` | diff --git a/tests/kernels/moe/modular_kernel_tools/mk_objects.py b/tests/kernels/moe/modular_kernel_tools/mk_objects.py index d79fdfbe07af3..99b168dc75548 100644 --- a/tests/kernels/moe/modular_kernel_tools/mk_objects.py +++ b/tests/kernels/moe/modular_kernel_tools/mk_objects.py @@ -13,9 +13,6 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import ( BatchedDeepGemmExperts, ) -from vllm.model_executor.layers.fused_moe.batched_triton_or_deep_gemm_moe import ( - BatchedTritonOrDeepGemmExperts, -) from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, @@ -286,16 +283,6 @@ if has_deep_gemm() and is_deep_gemm_supported(): needs_matching_quant=False, needs_deep_gemm=True, ) - register_experts( - BatchedTritonOrDeepGemmExperts, - batched_format, - common_float_and_int_types, - blocked_quantization_support=True, - supports_chunking=False, - supports_expert_map=False, - needs_matching_quant=True, - needs_deep_gemm=True, - ) register_experts( TritonOrDeepGemmExperts, standard_format, @@ -457,10 +444,6 @@ def make_fused_experts( kwargs = batch_kwargs | quant_kwargs print(f"Making BatchedTritonExperts {kwargs} ...") experts = BatchedTritonExperts(**kwargs) - elif fused_experts_type == BatchedTritonOrDeepGemmExperts: - kwargs = batch_kwargs | quant_kwargs | deepgemm_kwargs - print(f"Making BatchedTritonOrDeepGemmExperts {kwargs} ...") - experts = BatchedTritonOrDeepGemmExperts(**kwargs) elif fused_experts_type == DeepGemmExperts: print(f"Making DeepGemmExperts {quant_config} ...") experts = DeepGemmExperts(quant_config) diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index 669abcb3d6ff1..9103e84aa7057 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -60,9 +60,6 @@ if HAS_TRITON: from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import ( BatchedDeepGemmExperts, ) - from vllm.model_executor.layers.fused_moe.batched_triton_or_deep_gemm_moe import ( # noqa: E501 - BatchedTritonOrDeepGemmExperts, - ) from vllm.model_executor.layers.fused_moe.cutlass_moe import ( CutlassBatchedExpertsFp8, CutlassExpertsFp8, @@ -98,7 +95,6 @@ if HAS_TRITON: "DeepGemmExperts", "BatchedDeepGemmExperts", "TritonOrDeepGemmExperts", - "BatchedTritonOrDeepGemmExperts", ] else: # Some model classes directly use the custom ops. Add placeholders diff --git a/vllm/model_executor/layers/fused_moe/batched_triton_or_deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/batched_triton_or_deep_gemm_moe.py deleted file mode 100644 index e69e9fd307aeb..0000000000000 --- a/vllm/model_executor/layers/fused_moe/batched_triton_or_deep_gemm_moe.py +++ /dev/null @@ -1,180 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import ( - BatchedDeepGemmExperts, -) -from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig -from vllm.model_executor.layers.fused_moe.fused_batched_moe import BatchedTritonExperts -from vllm.utils.deep_gemm import get_mk_alignment_for_contiguous_layout - - -class BatchedTritonOrDeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute): - def __init__( - self, - max_num_tokens: int, - num_dispatchers: int, - quant_config: FusedMoEQuantConfig, - allow_deep_gemm: bool = False, - ): - super().__init__(quant_config) - - self.batched_triton_experts = BatchedTritonExperts( - max_num_tokens=max_num_tokens, - num_dispatchers=num_dispatchers, - quant_config=self.quant_config, - ) - - self.allow_deep_gemm = ( - allow_deep_gemm - and self.quant_config.use_fp8_w8a8 - and self.block_shape == get_mk_alignment_for_contiguous_layout() - ) - - self.batched_deep_gemm_experts = ( - BatchedDeepGemmExperts( - max_num_tokens=max_num_tokens, - num_dispatchers=num_dispatchers, - quant_config=self.quant_config, - ) - if self.allow_deep_gemm - else None - ) - - assert ( - self.batched_deep_gemm_experts is not None - or self.batched_triton_experts is not None - ) - - @property - def activation_formats( - self, - ) -> tuple[mk.FusedMoEActivationFormat, mk.FusedMoEActivationFormat]: - if self.batched_triton_experts is not None: - assert ( - self.batched_deep_gemm_experts is None - or self.batched_deep_gemm_experts.activation_formats - == self.batched_triton_experts.activation_formats - ) - return self.batched_triton_experts.activation_formats - else: - assert self.batched_deep_gemm_experts is not None - return self.batched_deep_gemm_experts.activation_formats - - def supports_chunking(self) -> bool: - bdge = self.batched_deep_gemm_experts - bte = self.batched_triton_experts - return (bdge is None or bdge.supports_chunking()) and ( - bte is None or bte.supports_chunking() - ) - - def supports_expert_map(self) -> bool: - bdge = self.batched_deep_gemm_experts - bte = self.batched_triton_experts - return (bdge is None or bdge.supports_expert_map()) and ( - bte is None or bte.supports_expert_map() - ) - - def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: - bdge = self.batched_deep_gemm_experts - bte = self.batched_triton_experts - bdge_war = bdge.finalize_weight_and_reduce_impl() if bdge else None - bte_war = bte.finalize_weight_and_reduce_impl() if bte else None - is_bdge_war = bdge_war is not None - is_bte_war = bte_war is not None - - if is_bdge_war and is_bte_war: - assert bdge_war == bte_war, ( - "Both implementations should agree on WeightAndReduce impls. " - f"Got bdge_war: {bdge_war}, and bte_war: {bte_war}" - ) - - if bdge_war is not None: - return bdge_war - - assert bte_war is not None - return bte_war - - def workspace_dtype(self, act_dtype: torch.dtype) -> torch.dtype: - return act_dtype - - def workspace_shapes( - self, - M: int, - N: int, - K: int, - topk: int, - global_num_experts: int, - local_num_experts: int, - expert_tokens_metadata: mk.ExpertTokensMetadata | None, - ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: - # Note: the deep gemm workspaces are strictly larger than the triton - # workspaces so we can be pessimistic here and allocate for DeepGemm - # even if we fall back to triton later, e.g. if expert maps are set. - if self.allow_deep_gemm: - assert self.batched_deep_gemm_experts is not None - return self.batched_deep_gemm_experts.workspace_shapes( - M, - N, - K, - topk, - global_num_experts, - local_num_experts, - expert_tokens_metadata, - ) - else: - assert self.batched_triton_experts is not None - return self.batched_triton_experts.workspace_shapes( - M, - N, - K, - topk, - global_num_experts, - local_num_experts, - expert_tokens_metadata, - ) - - def apply( - self, - output: torch.Tensor, - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - activation: str, - global_num_experts: int, - expert_map: torch.Tensor | None, - a1q_scale: torch.Tensor | None, - a2_scale: torch.Tensor | None, - workspace13: torch.Tensor, - workspace2: torch.Tensor, - expert_tokens_meta: mk.ExpertTokensMetadata | None, - apply_router_weight_on_input: bool, - ): - experts = ( - self.batched_deep_gemm_experts - if self.allow_deep_gemm - else self.batched_triton_experts - ) - assert experts is not None - experts.apply( - output, - hidden_states, - w1, - w2, - topk_weights, - topk_ids, - activation, - global_num_experts, - expert_map, - a1q_scale, - a2_scale, - workspace13, - workspace2, - expert_tokens_meta, - apply_router_weight_on_input, - ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py index c7368bf427fe1..d7fb6d2ca367d 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py @@ -90,8 +90,10 @@ from vllm.platforms import CpuArchEnum, current_platform from vllm.scalar_type import scalar_types from vllm.utils.deep_gemm import ( get_col_major_tma_aligned_tensor, + get_mk_alignment_for_contiguous_layout, is_deep_gemm_e8m0_used, ) +from vllm.utils.import_utils import has_deep_gemm logger = init_logger(__name__) @@ -1088,9 +1090,11 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): return experts - # triton path - from vllm.model_executor.layers.fused_moe.batched_triton_or_deep_gemm_moe import ( # noqa: E501 - BatchedTritonOrDeepGemmExperts, + from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import ( + BatchedDeepGemmExperts, + ) + from vllm.model_executor.layers.fused_moe.fused_batched_moe import ( + BatchedTritonExperts, ) from vllm.model_executor.layers.fused_moe.triton_deep_gemm_moe import ( TritonOrDeepGemmExperts, @@ -1098,6 +1102,8 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): assert not self.rocm_aiter_moe_enabled and not self.use_marlin + use_deep_gemm = envs.VLLM_USE_DEEP_GEMM and envs.VLLM_MOE_USE_DEEP_GEMM + if ( prepare_finalize.activation_format == FusedMoEActivationFormat.BatchedExperts @@ -1105,22 +1111,47 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): max_num_tokens_per_rank = prepare_finalize.max_num_tokens_per_rank() assert max_num_tokens_per_rank is not None - logger.debug("BatchedTritonExperts(%s)", self.__class__.__name__) - return BatchedTritonOrDeepGemmExperts( - max_num_tokens=max_num_tokens_per_rank, - num_dispatchers=prepare_finalize.num_dispatchers(), - quant_config=self.moe_quant_config, - allow_deep_gemm=( - envs.VLLM_USE_DEEP_GEMM and envs.VLLM_MOE_USE_DEEP_GEMM - ), + if use_deep_gemm and not has_deep_gemm(): + raise RuntimeError( + "DeepGEMM requested for MoE layer but not installed." + ) + + compatible_with_deep_gemm = ( + self.moe_quant_config.use_fp8_w8a8 + and self.moe_quant_config.block_shape + == get_mk_alignment_for_contiguous_layout() ) + + # If this MoE layer is compatible with DeepGEMM, the proper env + # vars are set and DeepGEMM is not installed, throw an error. + if use_deep_gemm and compatible_with_deep_gemm and not has_deep_gemm(): + raise RuntimeError( + f"MoE layer incompatible with DeepGEMM, expected " + f"fp8==True, got {self.moe_quant_config.use_fp8_w8a8}" + f"or block_shape {self.moe_quant_config.block_shape}" + f"=={get_mk_alignment_for_contiguous_layout()}." + ) + + if use_deep_gemm and compatible_with_deep_gemm and has_deep_gemm(): + logger.debug("BatchedDeepGemmExperts(%s)", self.__class__.__name__) + return BatchedDeepGemmExperts( + max_num_tokens=max_num_tokens_per_rank, + num_dispatchers=prepare_finalize.num_dispatchers(), + quant_config=self.moe_quant_config, + ) + else: + logger.debug("BatchedTritonExperts(%s)", self.__class__.__name__) + return BatchedTritonExperts( + max_num_tokens=max_num_tokens_per_rank, + num_dispatchers=prepare_finalize.num_dispatchers(), + quant_config=self.moe_quant_config, + ) + else: logger.debug("TritonOrDeepGemmExperts(%s)", self.__class__.__name__) return TritonOrDeepGemmExperts( self.moe_quant_config, - allow_deep_gemm=( - envs.VLLM_USE_DEEP_GEMM and envs.VLLM_MOE_USE_DEEP_GEMM - ), + allow_deep_gemm=use_deep_gemm, ) def get_fused_moe_quant_config( From b5407869c8594d8e3c4ee3c09ff7cfe454be0798 Mon Sep 17 00:00:00 2001 From: Elizabeth Thomas Date: Wed, 3 Dec 2025 16:00:52 -0600 Subject: [PATCH 16/47] [Bugfix] Respect VLLM_CONFIGURE_LOGGING value (#28671) Signed-off-by: Elizabeth Thomas Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Roger Wang Signed-off-by: Jane Xu Signed-off-by: Nick Hill Signed-off-by: Johnny Yang Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: bruceszchen Co-authored-by: Roger Wang Co-authored-by: Jane (Yuan) Xu <31798555+janeyx99@users.noreply.github.com> Co-authored-by: Nick Hill Co-authored-by: Johnny Yang <24908445+jcyang43@users.noreply.github.com> --- tests/test_envs.py | 51 ++++++++++++++++++++++++++++++++++++++ vllm/envs.py | 6 +++-- vllm/utils/system_utils.py | 4 +++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/tests/test_envs.py b/tests/test_envs.py index 6a9835a68e7e2..11bbec38202bf 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -365,3 +365,54 @@ class TestEnvSetWithChoices: with patch.dict(os.environ, {"TEST_ENV": "option1,option1,option2"}): env_func = env_set_with_choices("TEST_ENV", [], ["option1", "option2"]) assert env_func() == {"option1", "option2"} + + +class TestVllmConfigureLogging: + """Test cases for VLLM_CONFIGURE_LOGGING environment variable.""" + + def test_configure_logging_defaults_to_true(self): + """Test that VLLM_CONFIGURE_LOGGING defaults to True when not set.""" + # Ensure the env var is not set + with patch.dict(os.environ, {}, clear=False): + if "VLLM_CONFIGURE_LOGGING" in os.environ: + del os.environ["VLLM_CONFIGURE_LOGGING"] + + # Clear cache if it exists + if hasattr(envs.__getattr__, "cache_clear"): + envs.__getattr__.cache_clear() + + result = envs.VLLM_CONFIGURE_LOGGING + assert result is True + assert isinstance(result, bool) + + def test_configure_logging_with_zero_string(self): + """Test that VLLM_CONFIGURE_LOGGING='0' evaluates to False.""" + with patch.dict(os.environ, {"VLLM_CONFIGURE_LOGGING": "0"}): + # Clear cache if it exists + if hasattr(envs.__getattr__, "cache_clear"): + envs.__getattr__.cache_clear() + + result = envs.VLLM_CONFIGURE_LOGGING + assert result is False + assert isinstance(result, bool) + + def test_configure_logging_with_one_string(self): + """Test that VLLM_CONFIGURE_LOGGING='1' evaluates to True.""" + with patch.dict(os.environ, {"VLLM_CONFIGURE_LOGGING": "1"}): + # Clear cache if it exists + if hasattr(envs.__getattr__, "cache_clear"): + envs.__getattr__.cache_clear() + + result = envs.VLLM_CONFIGURE_LOGGING + assert result is True + assert isinstance(result, bool) + + def test_configure_logging_with_invalid_value_raises_error(self): + """Test that invalid VLLM_CONFIGURE_LOGGING value raises ValueError.""" + with patch.dict(os.environ, {"VLLM_CONFIGURE_LOGGING": "invalid"}): + # Clear cache if it exists + if hasattr(envs.__getattr__, "cache_clear"): + envs.__getattr__.cache_clear() + + with pytest.raises(ValueError, match="invalid literal for int"): + _ = envs.VLLM_CONFIGURE_LOGGING diff --git a/vllm/envs.py b/vllm/envs.py index 4b594e54f7219..60d91e9850970 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -37,7 +37,7 @@ if TYPE_CHECKING: VLLM_DISABLE_FLASHINFER_PREFILL: bool = False VLLM_DO_NOT_TRACK: bool = False VLLM_USAGE_SOURCE: str = "" - VLLM_CONFIGURE_LOGGING: int = 1 + VLLM_CONFIGURE_LOGGING: bool = True VLLM_LOGGING_LEVEL: str = "INFO" VLLM_LOGGING_PREFIX: str = "" VLLM_LOGGING_STREAM: str = "ext://sys.stdout" @@ -623,7 +623,9 @@ environment_variables: dict[str, Callable[[], Any]] = { # If set to 0, vllm will not configure logging # If set to 1, vllm will configure logging using the default configuration # or the configuration file specified by VLLM_LOGGING_CONFIG_PATH - "VLLM_CONFIGURE_LOGGING": lambda: int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")), + "VLLM_CONFIGURE_LOGGING": lambda: bool( + int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")) + ), "VLLM_LOGGING_CONFIG_PATH": lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"), # this is used for configuring the default logging level "VLLM_LOGGING_LEVEL": lambda: os.getenv("VLLM_LOGGING_LEVEL", "INFO").upper(), diff --git a/vllm/utils/system_utils.py b/vllm/utils/system_utils.py index a4eb8f4d4fd7d..76cac59c18098 100644 --- a/vllm/utils/system_utils.py +++ b/vllm/utils/system_utils.py @@ -204,6 +204,10 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None: def decorate_logs(process_name: str | None = None) -> None: """Decorate stdout/stderr with process name and PID prefix.""" + # Respect VLLM_CONFIGURE_LOGGING environment variable + if not envs.VLLM_CONFIGURE_LOGGING: + return + if process_name is None: process_name = get_mp_context().current_process().name From 1109f98288b4a77f10e6f3b520b07005a0143b13 Mon Sep 17 00:00:00 2001 From: Shengqi Chen Date: Thu, 4 Dec 2025 06:08:19 +0800 Subject: [PATCH 17/47] [CI] fix docker image build by specifying merge-base commit id when downloading pre-compiled wheels (#29930) Signed-off-by: Shengqi Chen --- .buildkite/generate_index.py | 46 ------------------- docker/Dockerfile | 3 ++ setup.py | 25 +++++----- tests/standalone_tests/python_only_compile.sh | 6 ++- vllm/envs.py | 6 --- 5 files changed, 22 insertions(+), 64 deletions(-) delete mode 100644 .buildkite/generate_index.py diff --git a/.buildkite/generate_index.py b/.buildkite/generate_index.py deleted file mode 100644 index bbed80ebe8476..0000000000000 --- a/.buildkite/generate_index.py +++ /dev/null @@ -1,46 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import argparse -import os - -template = """ - - -

Links for vLLM

- {x86_wheel}
- {arm_wheel}
- - -""" - -parser = argparse.ArgumentParser() -parser.add_argument("--wheel", help="The wheel path.", required=True) -args = parser.parse_args() - -filename = os.path.basename(args.wheel) - -with open("index.html", "w") as f: - print(f"Generated index.html for {args.wheel}") - # sync the abi tag with .buildkite/scripts/upload-wheels.sh - if "x86_64" in filename: - x86_wheel = filename - arm_wheel = filename.replace("x86_64", "aarch64").replace( - "manylinux1", "manylinux2014" - ) - elif "aarch64" in filename: - x86_wheel = filename.replace("aarch64", "x86_64").replace( - "manylinux2014", "manylinux1" - ) - arm_wheel = filename - else: - raise ValueError(f"Unsupported wheel: {filename}") - # cloudfront requires escaping the '+' character - f.write( - template.format( - x86_wheel=x86_wheel, - x86_wheel_html_escaped=x86_wheel.replace("+", "%2B"), - arm_wheel=arm_wheel, - arm_wheel_html_escaped=arm_wheel.replace("+", "%2B"), - ) - ) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8bcd7f118f1ef..73cb4d7e0dc10 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -196,6 +196,7 @@ ARG SCCACHE_S3_NO_CREDENTIALS=0 # Flag to control whether to use pre-built vLLM wheels ARG VLLM_USE_PRECOMPILED="" +ARG VLLM_MERGE_BASE_COMMIT="" ARG VLLM_MAIN_CUDA_VERSION="" # Use dummy version for csrc-build wheel (only .so files are extracted, version doesn't matter) @@ -216,6 +217,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ && export SCCACHE_IDLE_TIMEOUT=0 \ && export CMAKE_BUILD_TYPE=Release \ && export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \ + && export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" \ && export VLLM_MAIN_CUDA_VERSION="${VLLM_MAIN_CUDA_VERSION}" \ && export VLLM_DOCKER_BUILD_CONTEXT=1 \ && sccache --show-stats \ @@ -233,6 +235,7 @@ RUN --mount=type=cache,target=/root/.cache/ccache \ rm -rf .deps && \ mkdir -p .deps && \ export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" && \ + export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" && \ export VLLM_DOCKER_BUILD_CONTEXT=1 && \ python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \ fi diff --git a/setup.py b/setup.py index 8b2b4f7e5def7..af7282d4f747b 100644 --- a/setup.py +++ b/setup.py @@ -346,10 +346,13 @@ class precompiled_wheel_utils: The order of preference is: 1. user-specified wheel location (can be either local or remote, via VLLM_PRECOMPILED_WHEEL_LOCATION) - 2. user-specified variant from nightly repo (current main commit via - VLLM_PRECOMPILED_WHEEL_VARIANT) + 2. user-specified variant (VLLM_PRECOMPILED_WHEEL_VARIANT) from nightly repo 3. the variant corresponding to VLLM_MAIN_CUDA_VERSION from nightly repo - 4. the default variant from nightly repo (current main commit) + 4. the default variant from nightly repo + + If downloading from the nightly repo, the commit can be specified via + VLLM_PRECOMPILED_WHEEL_COMMIT; otherwise, the head commit in the main branch + is used. """ wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None) if wheel_location is not None: @@ -362,10 +365,13 @@ class precompiled_wheel_utils: # try to fetch the wheel metadata from the nightly wheel repo main_variant = "cu" + envs.VLLM_MAIN_CUDA_VERSION.replace(".", "") variant = os.getenv("VLLM_PRECOMPILED_WHEEL_VARIANT", main_variant) - commit = os.getenv( - "VLLM_PRECOMPILED_WHEEL_COMMIT", - precompiled_wheel_utils.get_base_commit_in_main_branch(), - ) + commit = os.getenv("VLLM_PRECOMPILED_WHEEL_COMMIT", "").lower() + if not commit or len(commit) != 40: + print( + f"VLLM_PRECOMPILED_WHEEL_COMMIT not valid: {commit}" + ", trying to fetch base commit in main branch" + ) + commit = precompiled_wheel_utils.get_base_commit_in_main_branch() print(f"Using precompiled wheel commit {commit} with variant {variant}") try_default = False wheels, repo_url, download_filename = None, None, None @@ -502,10 +508,6 @@ class precompiled_wheel_utils: @staticmethod def get_base_commit_in_main_branch() -> str: - # Force to use the nightly wheel. This is mainly used for CI testing. - if envs.VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL: - return "nightly" - try: # Get the latest commit hash of the upstream main branch. resp_json = subprocess.check_output( @@ -516,6 +518,7 @@ class precompiled_wheel_utils: ] ).decode("utf-8") upstream_main_commit = json.loads(resp_json)["sha"] + print(f"Upstream main branch latest commit: {upstream_main_commit}") # In Docker build context, .git may be immutable or missing. if envs.VLLM_DOCKER_BUILD_CONTEXT: diff --git a/tests/standalone_tests/python_only_compile.sh b/tests/standalone_tests/python_only_compile.sh index 7cc5ef6596490..d29b9afcc6fbf 100644 --- a/tests/standalone_tests/python_only_compile.sh +++ b/tests/standalone_tests/python_only_compile.sh @@ -5,6 +5,10 @@ set -e set -x +merge_base_commit=$(git merge-base HEAD origin/main) +echo "Current merge base commit with main: $merge_base_commit" +git show --oneline -s $merge_base_commit + cd /vllm-workspace/ # uninstall vllm @@ -18,7 +22,7 @@ apt autoremove -y echo 'import os; os.system("touch /tmp/changed.file")' >> vllm/__init__.py -VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL=1 VLLM_USE_PRECOMPILED=1 pip3 install -vvv -e . +VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 pip3 install -vvv -e . # Run the script python3 -c 'import vllm' diff --git a/vllm/envs.py b/vllm/envs.py index 60d91e9850970..2ed5816b350b7 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -80,7 +80,6 @@ if TYPE_CHECKING: VLLM_USE_PRECOMPILED: bool = False VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False VLLM_DOCKER_BUILD_CONTEXT: bool = False - VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL: bool = False VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] | None = None VERBOSE: bool = False @@ -473,11 +472,6 @@ environment_variables: dict[str, Callable[[], Any]] = { .strip() .lower() in ("1", "true"), - # Whether to force using nightly wheel in python build. - # This is used for testing the nightly wheel in python build. - "VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL": lambda: bool( - int(os.getenv("VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL", "0")) - ), # CMake build type # If not set, defaults to "Debug" or "RelWithDebInfo" # Available options: "Debug", "Release", "RelWithDebInfo" From ad32e3e19ccf0526cb6744a5fed09a138a5fb2f9 Mon Sep 17 00:00:00 2001 From: Xieyang Xu Date: Wed, 3 Dec 2025 17:02:02 -0800 Subject: [PATCH 18/47] enable multi-node in external launcher mode (#29833) --- vllm/config/parallel.py | 8 ++++-- vllm/distributed/parallel_state.py | 39 ++++++++++++++++-------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 4a8c8bc17cfc3..20de672257107 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -593,10 +593,14 @@ class ParallelConfig: "max_parallel_loading_workers is currently " "not supported and will be ignored." ) - if self.distributed_executor_backend not in ("mp", "uni") and self.nnodes > 1: + allowed_backends = ("mp", "uni", "external_launcher") + if ( + self.distributed_executor_backend not in allowed_backends + and self.nnodes > 1 + ): raise ValueError( "nnodes > 1 can only be set when distributed executor " - "backend is mp or uni." + "backend is mp, uni or external_launcher." ) @property diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index c82a77c216af2..f910f10407d44 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1169,17 +1169,13 @@ def init_distributed_environment( from vllm.config import get_current_vllm_config config = get_current_vllm_config() - if config is not None and config.parallel_config.nnodes > 1: - parallel_config = config.parallel_config - ip = parallel_config.master_addr - rank = parallel_config.data_parallel_rank * world_size + rank - world_size = parallel_config.world_size_across_dp - port = parallel_config.master_port - distributed_init_method = get_distributed_init_method(ip, port) - elif ( + if ( config is not None - and config.parallel_config.data_parallel_size > 1 and config.parallel_config.distributed_executor_backend != "external_launcher" + and ( + config.parallel_config.nnodes > 1 + or config.parallel_config.data_parallel_size > 1 + ) ): parallel_config = config.parallel_config # adjust to take into account data parallelism @@ -1187,15 +1183,22 @@ def init_distributed_environment( rank = parallel_config.data_parallel_rank * world_size + rank # adjust the world size to take into account data parallelism world_size = parallel_config.world_size_across_dp - ip = parallel_config.data_parallel_master_ip - port = parallel_config.get_next_dp_init_port() - distributed_init_method = get_distributed_init_method(ip, port) - logger.debug( - "Adjusting world_size=%d rank=%d distributed_init_method=%s for DP", - world_size, - rank, - distributed_init_method, - ) + + # Use appropriate IP and port based on configuration + if parallel_config.nnodes > 1: + ip = parallel_config.master_addr + port = parallel_config.master_port + distributed_init_method = get_distributed_init_method(ip, port) + else: + ip = parallel_config.data_parallel_master_ip + port = parallel_config.get_next_dp_init_port() + distributed_init_method = get_distributed_init_method(ip, port) + logger.debug( + "Adjusting world_size=%d rank=%d distributed_init_method=%s for DP", + world_size, + rank, + distributed_init_method, + ) if not torch.distributed.is_initialized(): logger.info( "world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s", From c493b9d0924b3810439fd3fcd17995f3bb93bb75 Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Wed, 3 Dec 2025 19:21:45 -0800 Subject: [PATCH 19/47] [CI/Build] Add MM code path to Examples Test (#29986) Signed-off-by: zhewenli --- .buildkite/test-pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index f79e9266559f6..a79f0b0c6bbdf 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -387,6 +387,7 @@ steps: working_dir: "/vllm-workspace/examples" source_file_dependencies: - vllm/entrypoints + - vllm/multimodal - examples/ commands: - pip install tensorizer # for tensorizer test From 33a3d6c79826aa7938db45a4e24a213664859cc0 Mon Sep 17 00:00:00 2001 From: Iceber Gu Date: Thu, 4 Dec 2025 11:48:30 +0800 Subject: [PATCH 20/47] fix LoRA-related examples (#29956) Signed-off-by: Iceber Gu --- .../lora_with_quantization_inference.py | 16 ++++------------ .../offline_inference/multilora_inference.py | 18 +++--------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/examples/offline_inference/lora_with_quantization_inference.py b/examples/offline_inference/lora_with_quantization_inference.py index dc5c6202fa57b..2f3564b597556 100644 --- a/examples/offline_inference/lora_with_quantization_inference.py +++ b/examples/offline_inference/lora_with_quantization_inference.py @@ -23,31 +23,23 @@ def create_test_prompts( # this is an example of using quantization without LoRA ( "My name is", - SamplingParams( - temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128 - ), + SamplingParams(temperature=0.0, logprobs=1, max_tokens=128), None, ), # the next three examples use quantization with LoRA ( "my name is", - SamplingParams( - temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128 - ), + SamplingParams(temperature=0.0, logprobs=1, max_tokens=128), LoRARequest("lora-test-1", 1, lora_path), ), ( "The capital of USA is", - SamplingParams( - temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128 - ), + SamplingParams(temperature=0.0, logprobs=1, max_tokens=128), LoRARequest("lora-test-2", 1, lora_path), ), ( "The capital of France is", - SamplingParams( - temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128 - ), + SamplingParams(temperature=0.0, logprobs=1, max_tokens=128), LoRARequest("lora-test-3", 1, lora_path), ), ] diff --git a/examples/offline_inference/multilora_inference.py b/examples/offline_inference/multilora_inference.py index 5e5da2c0144c9..92021f9fb226c 100644 --- a/examples/offline_inference/multilora_inference.py +++ b/examples/offline_inference/multilora_inference.py @@ -27,9 +27,7 @@ def create_test_prompts( return [ ( "A robot may not injure a human being", - SamplingParams( - temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128 - ), + SamplingParams(temperature=0.0, logprobs=1, max_tokens=128), None, ), ( @@ -41,22 +39,12 @@ def create_test_prompts( ), ( "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501 - SamplingParams( - temperature=0.0, - logprobs=1, - prompt_logprobs=1, - max_tokens=128, - ), + SamplingParams(temperature=0.0, logprobs=1, max_tokens=128), LoRARequest("sql-lora", 1, lora_path), ), ( "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501 - SamplingParams( - temperature=0.0, - logprobs=1, - prompt_logprobs=1, - max_tokens=128, - ), + SamplingParams(temperature=0.0, logprobs=1, max_tokens=128), LoRARequest("sql-lora2", 2, lora_path), ), ] From 5f91cdda75b24a3d9cdda8c82897db07b288b5c9 Mon Sep 17 00:00:00 2001 From: Li Wang Date: Thu, 4 Dec 2025 11:53:00 +0800 Subject: [PATCH 21/47] [Misc] Add docker build env for Ascend NPU (#30015) Signed-off-by: wangli --- .buildkite/scripts/hardware_ci/run-npu-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/hardware_ci/run-npu-test.sh b/.buildkite/scripts/hardware_ci/run-npu-test.sh index 29c8f5ed5a91a..0db1abe37ba11 100644 --- a/.buildkite/scripts/hardware_ci/run-npu-test.sh +++ b/.buildkite/scripts/hardware_ci/run-npu-test.sh @@ -74,6 +74,7 @@ FROM ${BASE_IMAGE_NAME} # Define environments ENV DEBIAN_FRONTEND=noninteractive +ENV SOC_VERSION="ascend910b1" RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.svc.cluster.local:${PYPI_CACHE_PORT}/pypi/simple && \ pip config set global.trusted-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local && \ From dd38ba3a2682d6f73b02bc983a5b0157ed3e5498 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Thu, 4 Dec 2025 12:51:15 +0800 Subject: [PATCH 22/47] [Bugfix] Fix adapter_enabled IMA (#29977) Signed-off-by: Jee Jee Li --- vllm/lora/ops/triton_ops/fused_moe_lora_op.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py index 413ee8ecbbf96..34383cdf1767c 100644 --- a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py +++ b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py @@ -96,10 +96,14 @@ def _fused_moe_lora_kernel( slice_id = tl.program_id(axis=1) lora_idx = tl.program_id(axis=2) lora_id = tl.load(lora_ids + lora_idx) - moe_enabled = tl.load(adapter_enabled + lora_id) - if lora_id == -1 or moe_enabled == 0: + + if lora_id == -1: # Early exit for the no-lora case. return + moe_enabled = tl.load(adapter_enabled + lora_id) + if moe_enabled == 0: + # Early exit for the no moe lora case. + return max_loras = tl.num_programs(axis=2) grid_k = tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K) From 28097d5638cc695f4644c411edac8eb05a03b39b Mon Sep 17 00:00:00 2001 From: gausah01 <141038176+gausah01@users.noreply.github.com> Date: Thu, 4 Dec 2025 05:01:15 +0000 Subject: [PATCH 23/47] [Bugfix][CPU] Fix CPU KV cache fallback memory allocation (#29604) Signed-off-by: Gauri Sahnan Co-authored-by: Li, Jiang --- vllm/platforms/cpu.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 2b2c2f9cdc571..a2518d5fd3dc4 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -10,6 +10,7 @@ import sys from dataclasses import dataclass from typing import TYPE_CHECKING +import psutil import regex as re import torch @@ -147,11 +148,21 @@ class CpuPlatform(Platform): from vllm.utils.mem_constants import GiB_bytes kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE + node_dir = "/sys/devices/system/node" if kv_cache_space is None: - kv_cache_space = 4 * GiB_bytes # type: ignore + nodes = ( + [d for d in os.listdir(node_dir) if d.startswith("node")] + if os.path.exists(node_dir) + else [] + ) + num_numa_nodes = len(nodes) or 1 + free_cpu_memory = psutil.virtual_memory().total // num_numa_nodes + DEFAULT_CPU_MEM_UTILIZATION = 0.5 + kv_cache_space = int(free_cpu_memory * DEFAULT_CPU_MEM_UTILIZATION) + kv_cache_space_gib = kv_cache_space / GiB_bytes logger.warning_once( - "Environment variable VLLM_CPU_KVCACHE_SPACE (GiB) " - "for CPU backend is not set, using 4 by default." + "VLLM_CPU_KVCACHE_SPACE not set. Using " + f"{kv_cache_space_gib:.2f} GiB for KV cache." ) else: kv_cache_space *= GiB_bytes From fca3f4665838605e268a8408bc7ca359f5d5c14b Mon Sep 17 00:00:00 2001 From: Benjamin Bartels Date: Thu, 4 Dec 2025 05:50:27 +0000 Subject: [PATCH 24/47] [Frontend] Fixes anthropic /v1/messages streaming not containing input_tokens on first chunk (#29971) Signed-off-by: bbartels --- tests/entrypoints/openai/test_messages.py | 11 +++++++++++ vllm/entrypoints/anthropic/serving_messages.py | 10 +++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_messages.py b/tests/entrypoints/openai/test_messages.py index 3e390ad496428..b804a1a7a841a 100644 --- a/tests/entrypoints/openai/test_messages.py +++ b/tests/entrypoints/openai/test_messages.py @@ -69,9 +69,20 @@ async def test_anthropic_streaming(client: anthropic.AsyncAnthropic): stream=True, ) + first_chunk = None + chunk_count = 0 async for chunk in resp: + chunk_count += 1 + if first_chunk is None and chunk.type == "message_start": + first_chunk = chunk print(chunk.model_dump_json()) + assert chunk_count > 0 + assert first_chunk is not None, "message_start chunk was never observed" + assert first_chunk.usage is not None, "first chunk should include usage stats" + assert first_chunk.usage["output_tokens"] == 0 + assert first_chunk.usage["input_tokens"] > 5 + @pytest.mark.asyncio async def test_anthropic_tool_call(client: anthropic.AsyncAnthropic): diff --git a/vllm/entrypoints/anthropic/serving_messages.py b/vllm/entrypoints/anthropic/serving_messages.py index 340dabf0e7117..e7ea3bb59ca70 100644 --- a/vllm/entrypoints/anthropic/serving_messages.py +++ b/vllm/entrypoints/anthropic/serving_messages.py @@ -183,7 +183,9 @@ class AnthropicServingMessages(OpenAIServingChat): if anthropic_request.stream: req.stream = anthropic_request.stream - req.stream_options = StreamOptions.validate({"include_usage": True}) + req.stream_options = StreamOptions.validate( + {"include_usage": True, "continuous_usage_stats": True} + ) if anthropic_request.tool_choice is None: req.tool_choice = None @@ -323,6 +325,12 @@ class AnthropicServingMessages(OpenAIServingChat): content=[], model=origin_chunk.model, ), + usage=AnthropicUsage( + input_tokens=origin_chunk.usage.prompt_tokens + if origin_chunk.usage + else 0, + output_tokens=0, + ), ) first_item = False data = chunk.model_dump_json(exclude_unset=True) From 8aaa81b35f96a3b5c56d3dccf58f48129ee34126 Mon Sep 17 00:00:00 2001 From: Kuntai Du Date: Thu, 4 Dec 2025 14:00:52 +0800 Subject: [PATCH 25/47] [KVConnector] remove unused code (the model aware kv ops class) (#29709) Signed-off-by: KuntaiDu --- .../kv_transfer/kv_connector/utils.py | 87 +------------------ 1 file changed, 1 insertion(+), 86 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index b8eb5ea3b4939..b2c2c0e6b596d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -8,9 +8,7 @@ from typing import TYPE_CHECKING, Literal import torch -import vllm.envs as envs -from vllm import _custom_ops as ops -from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config import get_current_vllm_config from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.logger import init_logger from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput @@ -21,89 +19,6 @@ if TYPE_CHECKING: logger = init_logger(__name__) -class model_aware_kv_ops_helper: - def __init__(self, config: VllmConfig): - self.is_deepseek_mla = config.model_config.is_deepseek_mla - self.use_mla_opt = not envs.VLLM_MLA_DISABLE - self.tp_size = config.parallel_config.tensor_parallel_size - - def get_model_args(self, model_executable: torch.nn.Module): - model_config = model_executable.model.config - self.model_executable = model_executable - num_heads = int(model_config.num_key_value_heads / self.tp_size) - hidden_size = model_config.hidden_size - num_attention_heads = model_config.num_attention_heads - - # Deepseek's MLA (Multi-head Latent Attention) uses two different - # kv_cache shapes based on whether VLLM_MLA_DISABLE is set to 0. - # When VLLM_MLA_DISABLE=0 (default), forward absorb is applied, - # resulting in a kv_cache shape of [num_blks, blk_size, 1, - # kv_lora_rank + qk_rope_head_dim]. - # When VLLM_MLA_DISABLE=1, standard FA is used instead, leading - # to a kv_cache shape of [2, num_blks, blk_size, - # num_key_value_heads / tp, qk_nope_head_dim + qk_rope_head_dim]. - # For more details, see vllm/v1/attention/backends/mla/common.py. - if self.is_deepseek_mla and self.use_mla_opt: - head_size = model_config.kv_lora_rank + model_config.qk_rope_head_dim - num_heads = 1 - elif self.is_deepseek_mla and not self.use_mla_opt: - head_size = model_config.qk_nope_head_dim + model_config.qk_rope_head_dim - else: - head_size = getattr(model_config, "head_dim", None) - if head_size is None: - head_size = int(hidden_size // num_attention_heads) - - return num_heads, head_size - - def get_kv_from_cache(self, kv_cache, num_heads, head_size): - if self.is_deepseek_mla and self.use_mla_opt: - key_cache = kv_cache.reshape(-1, num_heads, head_size) - value_cache = kv_cache.reshape(-1, num_heads, head_size) - else: - key_cache = kv_cache[0].reshape(-1, num_heads, head_size) - value_cache = kv_cache[1].reshape(-1, num_heads, head_size) - return key_cache, value_cache - - def put_kv_to_cache( - self, - model_executable: torch.nn.Module, - keys, - values, - layer, - kv_cache, - slot_mapping, - start_pos, - end_pos, - ): - model_config = model_executable.model.config - - if self.is_deepseek_mla and self.use_mla_opt: - layer.self_attn.attn = layer.self_attn.mla_attn - k_c_normed_k_pe = keys.squeeze(1) - k_c_normed = k_c_normed_k_pe[:, : model_config.kv_lora_rank] - k_pe = k_c_normed_k_pe[:, model_config.kv_lora_rank :] - ops.concat_and_cache_mla( - k_c_normed.to(kv_cache.device), - k_pe.to(kv_cache.device), - kv_cache, - slot_mapping[start_pos:end_pos], - layer.self_attn.attn.kv_cache_dtype, - layer.self_attn.attn._k_scale, - ) - else: - key_cache, value_cache = kv_cache[0], kv_cache[1] - ops.reshape_and_cache_flash( - keys.to(key_cache.device), - values.to(value_cache.device), - key_cache, - value_cache, - slot_mapping[start_pos:end_pos], - layer.self_attn.attn.kv_cache_dtype, - layer.self_attn.attn._k_scale, - layer.self_attn.attn._v_scale, - ) - - def get_kv_connector_cache_layout(): # NOTE (NickLucche) When running disaggregated PD with NIXL, HND layout is # used for faster transfer. From 80f8af4b2fadf85403290a38c8ae77f01b6b5378 Mon Sep 17 00:00:00 2001 From: Jianwei Mao Date: Thu, 4 Dec 2025 14:04:44 +0800 Subject: [PATCH 26/47] Fix error while downloading dependencies for CPU backend (#29797) Signed-off-by: Jianwei Mao --- requirements/cpu-build.txt | 1 - requirements/cpu.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements/cpu-build.txt b/requirements/cpu-build.txt index e18e0825fc428..1ea401a04a12c 100644 --- a/requirements/cpu-build.txt +++ b/requirements/cpu-build.txt @@ -3,7 +3,6 @@ ninja packaging>=24.2 setuptools>=77.0.3,<81.0.0 setuptools-scm>=8 ---extra-index-url https://download.pytorch.org/whl/cpu torch==2.9.1+cpu; platform_machine == "x86_64" or platform_machine == "s390x" torch==2.9.1; platform_system == "Darwin" or platform_machine == "ppc64le" or platform_machine == "aarch64" scons; platform_machine == "aarch64" # needed to build Arm Compute Library (ACL) diff --git a/requirements/cpu.txt b/requirements/cpu.txt index 21571be479c83..7a670812e8943 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -4,7 +4,6 @@ numba == 0.61.2; platform_machine != "s390x" # Required for N-gram speculative decoding # Dependencies for CPUs ---extra-index-url https://download.pytorch.org/whl/cpu torch==2.9.1+cpu; platform_machine == "x86_64" or platform_machine == "s390x" torch==2.9.1; platform_system == "Darwin" or platform_machine == "ppc64le" or platform_machine == "aarch64" From 9ae2f603748446317c90fe40f1eb269e9a027815 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 4 Dec 2025 14:22:20 +0800 Subject: [PATCH 27/47] [Misc] Various cleanups for MM input processing (#29970) Signed-off-by: DarkLight1337 --- docs/features/multimodal_inputs.md | 8 +- ...ompt_embed_inference_with_openai_client.py | 12 +- .../entrypoints/openai/test_vision_embeds.py | 75 +++++------ tests/entrypoints/test_chat_utils.py | 127 +----------------- .../test_completion_with_image_embeds.py | 17 +-- vllm/entrypoints/chat_utils.py | 13 +- vllm/entrypoints/llm.py | 1 - vllm/entrypoints/openai/serving_engine.py | 6 - vllm/entrypoints/score_utils.py | 5 +- vllm/model_executor/models/hunyuan_vision.py | 3 +- vllm/model_executor/models/keye.py | 4 +- vllm/model_executor/models/keye_vl1_5.py | 4 +- vllm/multimodal/audio.py | 7 +- vllm/utils/serial_utils.py | 10 ++ 14 files changed, 67 insertions(+), 225 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 4656ee43ea251..2b25dc7666c37 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -795,14 +795,12 @@ The following example demonstrates how to pass image embeddings to the OpenAI se ??? code ```python + from vllm.utils.serial_utils import tensor2base64 + image_embedding = torch.load(...) grid_thw = torch.load(...) # Required by Qwen/Qwen2-VL-2B-Instruct - buffer = io.BytesIO() - torch.save(image_embedding, buffer) - buffer.seek(0) - binary_data = buffer.read() - base64_image_embedding = base64.b64encode(binary_data).decode('utf-8') + base64_image_embedding = tensor2base64(image_embedding) client = OpenAI( # defaults to os.environ.get("OPENAI_API_KEY") diff --git a/examples/online_serving/prompt_embed_inference_with_openai_client.py b/examples/online_serving/prompt_embed_inference_with_openai_client.py index 0bbe4b8f5ee9b..889be6820e70a 100644 --- a/examples/online_serving/prompt_embed_inference_with_openai_client.py +++ b/examples/online_serving/prompt_embed_inference_with_openai_client.py @@ -28,13 +28,11 @@ Dependencies: - openai """ -import base64 -import io - -import torch import transformers from openai import OpenAI +from vllm.utils.serial_utils import tensor2base64 + def main(): client = OpenAI( @@ -58,11 +56,7 @@ def main(): prompt_embeds = embedding_layer(token_ids).squeeze(0) # Prompt embeddings - buffer = io.BytesIO() - torch.save(prompt_embeds, buffer) - buffer.seek(0) - binary_data = buffer.read() - encoded_embeds = base64.b64encode(binary_data).decode("utf-8") + encoded_embeds = tensor2base64(prompt_embeds) completion = client.completions.create( model=model_name, diff --git a/tests/entrypoints/openai/test_vision_embeds.py b/tests/entrypoints/openai/test_vision_embeds.py index a6593c5b05e2e..42d9fe4840bbe 100644 --- a/tests/entrypoints/openai/test_vision_embeds.py +++ b/tests/entrypoints/openai/test_vision_embeds.py @@ -2,64 +2,47 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import base64 -import io import numpy as np import pytest import requests import torch +from vllm.utils.serial_utils import tensor2base64 + from ...utils import RemoteOpenAIServer -MODEL_NAME = "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11" -DTYPE = "float16" - -def _terratorch_dummy_inputs(model_name: str): +def _terratorch_dummy_messages(): pixel_values = torch.full((6, 512, 512), 1.0, dtype=torch.float16) location_coords = torch.full((1, 2), 1.0, dtype=torch.float16) - buffer_tiff = io.BytesIO() - torch.save(pixel_values, buffer_tiff) - buffer_tiff.seek(0) - binary_data = buffer_tiff.read() - base64_tensor_embedding = base64.b64encode(binary_data).decode("utf-8") - - buffer_coord = io.BytesIO() - torch.save(location_coords, buffer_coord) - buffer_coord.seek(0) - binary_data = buffer_coord.read() - base64_coord_embedding = base64.b64encode(binary_data).decode("utf-8") - - return { - "model": model_name, - "additional_data": {"prompt_token_ids": [1]}, - "encoding_format": "base64", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "image_embeds", - "image_embeds": { - "pixel_values": base64_tensor_embedding, - "location_coords": base64_coord_embedding, - }, - } - ], - } - ], - } + return [ + { + "role": "user", + "content": [ + { + "type": "image_embeds", + "image_embeds": { + "pixel_values": tensor2base64(pixel_values), + "location_coords": tensor2base64(location_coords), + }, + } + ], + } + ] -@pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_single_request(model_name: str): +@pytest.mark.parametrize( + "model_name", ["ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"] +) +def test_single_request(model_name: str): args = [ "--runner", "pooling", # use half precision for speed and memory savings in CI environment "--dtype", - DTYPE, + "float16", "--enforce-eager", "--trust-remote-code", "--max-num-seqs", @@ -70,11 +53,15 @@ async def test_single_request(model_name: str): "--enable-mm-embeds", ] - with RemoteOpenAIServer(MODEL_NAME, args) as server: - prompt = _terratorch_dummy_inputs(model_name) - - # test single pooling - response = requests.post(server.url_for("pooling"), json=prompt) + with RemoteOpenAIServer(model_name, args) as server: + response = requests.post( + server.url_for("pooling"), + json={ + "model": model_name, + "messages": _terratorch_dummy_messages(), + "encoding_format": "base64", + }, + ) response.raise_for_status() output = response.json()["data"][0]["data"] diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 03a0c058ea690..75be34820bcd7 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -29,6 +29,7 @@ from vllm.multimodal.utils import ( encode_video_base64, ) from vllm.tokenizers import MistralTokenizer, get_tokenizer +from vllm.utils.serial_utils import tensor2base64 from ..models.registry import HF_EXAMPLE_MODELS from ..utils import VLLM_PATH @@ -85,11 +86,6 @@ def phi3v_model_config_image_embeds(): ) -@pytest.fixture(scope="module") -def phi3v_tokenizer(): - return get_tokenizer(PHI3V_MODEL_ID) - - @pytest.fixture(scope="function") def qwen2_audio_model_config(): return ModelConfig( @@ -115,11 +111,6 @@ def audio_embeds_model_config(): ) -@pytest.fixture(scope="module") -def qwen2_audio_tokenizer(): - return get_tokenizer(QWEN2AUDIO_MODEL_ID) - - @pytest.fixture(scope="function") def qwen25omni_model_config_mm_interleaved(): return ModelConfig( @@ -134,11 +125,6 @@ def qwen25omni_model_config_mm_interleaved(): ) -@pytest.fixture(scope="module") -def qwen25omni_tokenizer(): - return get_tokenizer(QWEN25OMNI_MODEL_ID) - - @pytest.fixture(scope="function") def mistral_model_config(): return ModelConfig( @@ -150,11 +136,6 @@ def mistral_model_config(): ) -@pytest.fixture(scope="module") -def mistral_tokenizer(): - return get_tokenizer(MISTRAL_MODEL_ID) - - @pytest.fixture(scope="module") def image_url(): image = ImageAsset("cherry_blossom") @@ -239,7 +220,6 @@ def _assert_mm_data_inputs( def test_parse_chat_messages_single_image( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -253,7 +233,6 @@ def test_parse_chat_messages_single_image( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -266,7 +245,6 @@ def test_parse_chat_messages_single_image( def test_parse_chat_messages_single_image_with_uuid( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -287,7 +265,6 @@ def test_parse_chat_messages_single_image_with_uuid( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -300,7 +277,6 @@ def test_parse_chat_messages_single_image_with_uuid( def test_parse_chat_messages_single_empty_image_with_uuid( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -319,7 +295,6 @@ def test_parse_chat_messages_single_empty_image_with_uuid( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -332,7 +307,6 @@ def test_parse_chat_messages_single_empty_image_with_uuid( def test_parse_chat_messages_single_image_with_bad_uuid_format( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -354,7 +328,6 @@ def test_parse_chat_messages_single_image_with_bad_uuid_format( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -367,7 +340,6 @@ def test_parse_chat_messages_single_image_with_bad_uuid_format( def test_parse_chat_messages_multiple_images_with_uuids( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid1 = "my_uuid_1" @@ -397,7 +369,6 @@ def test_parse_chat_messages_multiple_images_with_uuids( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -413,7 +384,6 @@ def test_parse_chat_messages_multiple_images_with_uuids( def test_parse_chat_messages_multiple_empty_images_with_uuids( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid1 = "my_uuid_1" @@ -439,7 +409,6 @@ def test_parse_chat_messages_multiple_empty_images_with_uuids( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -455,7 +424,6 @@ def test_parse_chat_messages_multiple_empty_images_with_uuids( def test_parse_chat_messages_mixed_empty_images_with_uuids( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid1 = "my_uuid_1" @@ -483,7 +451,6 @@ def test_parse_chat_messages_mixed_empty_images_with_uuids( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -500,7 +467,6 @@ def test_parse_chat_messages_mixed_empty_images_with_uuids( @pytest.mark.asyncio async def test_parse_chat_messages_single_image_with_uuid_async( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -519,7 +485,6 @@ async def test_parse_chat_messages_single_image_with_uuid_async( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -533,7 +498,6 @@ async def test_parse_chat_messages_single_image_with_uuid_async( @pytest.mark.asyncio async def test_parse_chat_messages_empty_image_with_uuid_async( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -552,7 +516,6 @@ async def test_parse_chat_messages_empty_image_with_uuid_async( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -566,7 +529,6 @@ async def test_parse_chat_messages_empty_image_with_uuid_async( @pytest.mark.asyncio async def test_parse_chat_messages_multiple_images_with_uuids_async( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid1 = "my_uuid_1" @@ -592,7 +554,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -609,7 +570,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async( @pytest.mark.asyncio async def test_parse_chat_messages_multiple_empty_images_with_uuids_async( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid1 = "my_uuid_1" @@ -635,7 +595,6 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -652,7 +611,6 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async( @pytest.mark.asyncio async def test_parse_chat_messages_multiple_images_with_partial_uuids_async( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid2 = "my_uuid_2" @@ -676,7 +634,6 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -692,7 +649,6 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async( def test_parse_chat_messages_empty_system( mistral_model_config, - mistral_tokenizer, ): # Test string format conversation, _, _ = parse_chat_messages( @@ -704,7 +660,6 @@ def test_parse_chat_messages_empty_system( }, ], mistral_model_config, - mistral_tokenizer, content_format="string", ) assert conversation == [ @@ -722,7 +677,6 @@ def test_parse_chat_messages_empty_system( }, ], mistral_model_config, - mistral_tokenizer, content_format="openai", ) assert conversation == [ @@ -734,7 +688,6 @@ def test_parse_chat_messages_empty_system( @pytest.mark.asyncio async def test_parse_chat_messages_single_image_async( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_future, mm_uuids = parse_chat_messages_futures( @@ -748,7 +701,6 @@ async def test_parse_chat_messages_single_image_async( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -761,7 +713,6 @@ async def test_parse_chat_messages_single_image_async( def test_parse_chat_messages_multiple_images( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -779,7 +730,6 @@ def test_parse_chat_messages_multiple_images( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -795,7 +745,6 @@ def test_parse_chat_messages_multiple_images( def test_parse_chat_messages_empty_pil_image_with_uuid( phi3v_model_config, - phi3v_tokenizer, ): uuid = "abcd" conversation, mm_data, mm_uuids = parse_chat_messages( @@ -809,7 +758,6 @@ def test_parse_chat_messages_empty_pil_image_with_uuid( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -825,7 +773,6 @@ def test_parse_chat_messages_empty_pil_image_with_uuid( def test_parse_chat_messages_empty_image_embeds_with_uuid( phi3v_model_config_image_embeds, - phi3v_tokenizer, ): uuid = "abcd" conversation, mm_data, mm_uuids = parse_chat_messages( @@ -839,7 +786,6 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid( } ], phi3v_model_config_image_embeds, - phi3v_tokenizer, content_format="string", ) @@ -857,7 +803,6 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid( def test_parse_chat_messages_empty_audio_embeds_with_uuid( audio_embeds_model_config, - qwen2_audio_tokenizer, ): """Test audio_embeds with UUID (no actual embeds data).""" uuid = "test-audio-uuid-123" @@ -873,7 +818,6 @@ def test_parse_chat_messages_empty_audio_embeds_with_uuid( } ], audio_embeds_model_config, - qwen2_audio_tokenizer, content_format="string", ) @@ -889,11 +833,8 @@ def test_parse_chat_messages_empty_audio_embeds_with_uuid( def test_parse_chat_messages_audio_embeds_with_string( audio_embeds_model_config, - qwen2_audio_tokenizer, ): """Test audio_embeds with base64 string embedding data.""" - import base64 - import io import torch @@ -901,11 +842,7 @@ def test_parse_chat_messages_audio_embeds_with_string( audio_embedding = torch.randn(1, 128, 768) # Encode it as base64 - buffer = io.BytesIO() - torch.save(audio_embedding, buffer) - buffer.seek(0) - binary_data = buffer.read() - base64_audio_embedding = base64.b64encode(binary_data).decode("utf-8") + base64_audio_embedding = tensor2base64(audio_embedding) conversation, mm_data, mm_uuids = parse_chat_messages( [ @@ -921,7 +858,6 @@ def test_parse_chat_messages_audio_embeds_with_string( } ], audio_embeds_model_config, - qwen2_audio_tokenizer, content_format="string", ) @@ -939,11 +875,8 @@ def test_parse_chat_messages_audio_embeds_with_string( @pytest.mark.asyncio async def test_parse_chat_messages_audio_embeds_async( audio_embeds_model_config, - qwen2_audio_tokenizer, ): """Test audio_embeds with async futures.""" - import base64 - import io import torch @@ -951,11 +884,7 @@ async def test_parse_chat_messages_audio_embeds_async( audio_embedding = torch.randn(1, 128, 768) # Encode it as base64 - buffer = io.BytesIO() - torch.save(audio_embedding, buffer) - buffer.seek(0) - binary_data = buffer.read() - base64_audio_embedding = base64.b64encode(binary_data).decode("utf-8") + base64_audio_embedding = tensor2base64(audio_embedding) conversation, mm_future, mm_uuids = parse_chat_messages_futures( [ @@ -971,7 +900,6 @@ async def test_parse_chat_messages_audio_embeds_async( } ], audio_embeds_model_config, - qwen2_audio_tokenizer, content_format="string", ) @@ -990,7 +918,6 @@ async def test_parse_chat_messages_audio_embeds_async( @pytest.mark.asyncio async def test_parse_chat_messages_empty_image_embeds_with_uuid_async( phi3v_model_config_image_embeds, - phi3v_tokenizer, ): uuid = "abcd" conversation, mm_future, mm_uuids = parse_chat_messages_futures( @@ -1004,7 +931,6 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async( } ], phi3v_model_config_image_embeds, - phi3v_tokenizer, content_format="string", ) @@ -1024,7 +950,6 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async( @pytest.mark.asyncio async def test_parse_chat_messages_multiple_images_async( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_future, mm_uuids = parse_chat_messages_futures( @@ -1042,7 +967,6 @@ async def test_parse_chat_messages_multiple_images_async( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -1058,7 +982,6 @@ async def test_parse_chat_messages_multiple_images_async( def test_parse_chat_messages_placeholder_already_in_prompt( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -1076,7 +999,6 @@ def test_parse_chat_messages_placeholder_already_in_prompt( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) assert conversation == [ @@ -1091,7 +1013,6 @@ def test_parse_chat_messages_placeholder_already_in_prompt( def test_parse_chat_messages_placeholder_one_already_in_prompt( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -1110,7 +1031,6 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -1127,7 +1047,6 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt( def test_parse_chat_messages_multiple_images_across_messages( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -1149,7 +1068,6 @@ def test_parse_chat_messages_multiple_images_across_messages( }, ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -1164,7 +1082,6 @@ def test_parse_chat_messages_multiple_images_across_messages( def test_parse_chat_messages_multiple_images_with_uuids_across_messages( phi3v_model_config, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -1195,7 +1112,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_across_messages( }, ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -1210,7 +1126,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_across_messages( def test_parse_chat_messages_context_text_format( phi3v_model_config, - phi3v_tokenizer, ): conversation, mm_data, mm_uuids = parse_chat_messages( [ @@ -1222,7 +1137,6 @@ def test_parse_chat_messages_context_text_format( {"role": "user", "content": "What about this one?"}, ], phi3v_model_config, - phi3v_tokenizer, content_format="openai", ) @@ -1246,7 +1160,6 @@ def test_parse_chat_messages_context_text_format( def test_parse_chat_messages_rejects_too_many_images_in_one_message( phi3v_model_config, - phi3v_tokenizer, image_url, ): with warnings.catch_warnings(): @@ -1277,14 +1190,12 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) def test_parse_chat_messages_rejects_too_many_images_across_messages( phi3v_model_config, - phi3v_tokenizer, image_url, ): with warnings.catch_warnings(): @@ -1322,14 +1233,12 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages( }, ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) def test_parse_chat_messages_multiple_images_uncommon_input( phi3v_model_config, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -1344,7 +1253,6 @@ def test_parse_chat_messages_multiple_images_uncommon_input( } ], phi3v_model_config, - phi3v_tokenizer, content_format="string", ) @@ -1360,7 +1268,6 @@ def test_parse_chat_messages_multiple_images_uncommon_input( def test_parse_chat_messages_multiple_images_interleave( phi3v_model_config_mm_interleaved, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -1380,7 +1287,6 @@ def test_parse_chat_messages_multiple_images_interleave( } ], phi3v_model_config_mm_interleaved, - phi3v_tokenizer, content_format="string", ) @@ -1398,7 +1304,6 @@ def test_parse_chat_messages_multiple_images_interleave( @pytest.mark.asyncio async def test_parse_chat_messages_multiple_images_interleave_async( phi3v_model_config_mm_interleaved, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages_futures( @@ -1418,7 +1323,6 @@ async def test_parse_chat_messages_multiple_images_interleave_async( } ], phi3v_model_config_mm_interleaved, - phi3v_tokenizer, content_format="string", ) @@ -1436,7 +1340,6 @@ async def test_parse_chat_messages_multiple_images_interleave_async( @pytest.mark.asyncio async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async( phi3v_model_config_mm_interleaved, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -1465,7 +1368,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async( } ], phi3v_model_config_mm_interleaved, - phi3v_tokenizer, content_format="string", ) @@ -1482,7 +1384,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async( def test_parse_chat_messages_multiple_images_multiple_messages_interleave( phi3v_model_config_mm_interleaved, - phi3v_tokenizer, image_url, ): conversation, mm_data, mm_uuids = parse_chat_messages( @@ -1505,7 +1406,6 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave( }, ], phi3v_model_config_mm_interleaved, - phi3v_tokenizer, content_format="string", ) @@ -1523,7 +1423,6 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave( def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave( phi3v_model_config_mm_interleaved, - phi3v_tokenizer, image_url, ): image_uuid = str(hash(image_url)) @@ -1555,7 +1454,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interl }, ], phi3v_model_config_mm_interleaved, - phi3v_tokenizer, content_format="string", ) @@ -1573,7 +1471,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interl def test_parse_chat_messages_multiple_modals_multiple_messages_interleave( qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, image_url, video_url, audio_url, @@ -1601,7 +1498,6 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave( }, ], qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, content_format="string", ) @@ -1627,7 +1523,6 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave( def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave( qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, image_url, video_url, audio_url, @@ -1671,7 +1566,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl }, ], qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, content_format="string", ) @@ -1699,7 +1593,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave( # noqa: E501 qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, image_url, video_url, audio_url, @@ -1743,7 +1636,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes }, ], qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, content_format="string", ) @@ -1775,7 +1667,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave( # noqa: E501 qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, image_url, video_url, audio_url, @@ -1811,7 +1702,6 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message }, ], qwen25omni_model_config_mm_interleaved, - qwen25omni_tokenizer, content_format="string", ) @@ -1837,7 +1727,6 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message def test_parse_chat_messages_multiple_images_interleave_with_placeholders( phi3v_model_config_mm_interleaved, - phi3v_tokenizer, image_url, ): with pytest.raises( @@ -1861,7 +1750,6 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders( } ], phi3v_model_config_mm_interleaved, - phi3v_tokenizer, content_format="string", ) @@ -2237,9 +2125,7 @@ def test_resolve_content_format_examples(template_path, expected_format): assert resolved_format == expected_format -def test_parse_chat_messages_include_thinking_chunk( - mistral_model_config, mistral_tokenizer -): +def test_parse_chat_messages_include_thinking_chunk(mistral_model_config): messages = [ { "role": "system", @@ -2269,7 +2155,6 @@ def test_parse_chat_messages_include_thinking_chunk( conversation_with_thinking, _, _ = parse_chat_messages( messages, mistral_model_config, - mistral_tokenizer, content_format="openai", ) @@ -2353,7 +2238,6 @@ def test_apply_mistral_chat_template_thinking_chunk(): def test_parse_chat_messages_single_empty_audio_with_uuid( qwen2_audio_model_config, - qwen2_audio_tokenizer, ): audio_uuid = "abcd" conversation, mm_data, mm_uuids = parse_chat_messages( @@ -2371,7 +2255,6 @@ def test_parse_chat_messages_single_empty_audio_with_uuid( } ], qwen2_audio_model_config, - qwen2_audio_tokenizer, content_format="string", ) @@ -2389,7 +2272,6 @@ def test_parse_chat_messages_single_empty_audio_with_uuid( @pytest.mark.asyncio async def test_parse_chat_messages_single_empty_audio_with_uuid_async( qwen2_audio_model_config, - qwen2_audio_tokenizer, ): audio_uuid = "abcd" conversation, mm_future, mm_uuids = parse_chat_messages_futures( @@ -2407,7 +2289,6 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async( } ], qwen2_audio_model_config, - qwen2_audio_tokenizer, content_format="string", ) diff --git a/tests/v1/entrypoints/openai/test_completion_with_image_embeds.py b/tests/v1/entrypoints/openai/test_completion_with_image_embeds.py index 276de2ff8e2cd..b30556fbc81fb 100644 --- a/tests/v1/entrypoints/openai/test_completion_with_image_embeds.py +++ b/tests/v1/entrypoints/openai/test_completion_with_image_embeds.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import base64 -import io import json import openai # use the official client for correctness check @@ -13,6 +11,7 @@ from transformers import AutoConfig from tests.conftest import ImageTestAssets from tests.utils import RemoteOpenAIServer +from vllm.utils.serial_utils import tensor2base64 # any model with a chat template should work here MODEL_NAME = "llava-hf/llava-1.5-7b-hf" @@ -50,18 +49,6 @@ async def client_with_image_embeds(server_with_image_embeds): yield async_client -def encode_image_embedding_to_base64(image_embedding) -> str: - """ - Encode image embedding to base64 string - """ - buffer = io.BytesIO() - torch.save(image_embedding, buffer) - buffer.seek(0) - binary_data = buffer.read() - base64_image_embedding = base64.b64encode(binary_data).decode("utf-8") - return base64_image_embedding - - @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("dtype", [torch.half, torch.float16, torch.float32]) @@ -73,7 +60,7 @@ async def test_completions_with_image_embeds( ): # Test case: Single image embeds input image_embeds = image_assets[0].image_embeds.to(dtype=dtype) - base64_image_embedding = encode_image_embedding_to_base64(image_embeds) + base64_image_embedding = tensor2base64(image_embeds) chat_completion = await client_with_image_embeds.chat.completions.create( messages=[ {"role": "system", "content": "You are a helpful assistant."}, diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 1b3a7d2665b41..077fe681bc5b8 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -536,7 +536,7 @@ def resolve_hf_chat_template( def _resolve_chat_template_content_format( chat_template: str | None, tools: list[dict[str, Any]] | None, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, *, model_config: ModelConfig, ) -> _ChatTemplateContentFormat: @@ -593,7 +593,7 @@ def resolve_chat_template_content_format( chat_template: str | None, tools: list[dict[str, Any]] | None, given_format: ChatTemplateContentFormatOption, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, *, model_config: ModelConfig, ) -> _ChatTemplateContentFormat: @@ -627,11 +627,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]): maximum per prompt. """ - def __init__(self, model_config: ModelConfig, tokenizer: TokenizerLike): + def __init__(self, model_config: ModelConfig): super().__init__() self._model_config = model_config - self._tokenizer = tokenizer self._items_by_modality = defaultdict[str, list[_T | None]](list) self._uuids_by_modality = defaultdict[str, list[str | None]](list) @@ -1612,7 +1611,6 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None: def parse_chat_messages( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, - tokenizer: TokenizerLike, content_format: _ChatTemplateContentFormat, ) -> tuple[ list[ConversationMessage], @@ -1620,7 +1618,7 @@ def parse_chat_messages( MultiModalUUIDDict | None, ]: conversation: list[ConversationMessage] = [] - mm_tracker = MultiModalItemTracker(model_config, tokenizer) + mm_tracker = MultiModalItemTracker(model_config) for msg in messages: sub_messages = _parse_chat_message_content( @@ -1644,7 +1642,6 @@ def parse_chat_messages( def parse_chat_messages_futures( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, - tokenizer: TokenizerLike, content_format: _ChatTemplateContentFormat, ) -> tuple[ list[ConversationMessage], @@ -1652,7 +1649,7 @@ def parse_chat_messages_futures( MultiModalUUIDDict | None, ]: conversation: list[ConversationMessage] = [] - mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer) + mm_tracker = AsyncMultiModalItemTracker(model_config) for msg in messages: sub_messages = _parse_chat_message_content( diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index c121fa71f0196..481a47a97f7d4 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -834,7 +834,6 @@ class LLM: conversation, mm_data, mm_uuids = parse_chat_messages( msgs, model_config, - tokenizer, content_format=resolved_content_format, ) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 9642024dd1e9e..bfa98f29a064b 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1088,11 +1088,6 @@ class OpenAIServing: Sequence[RequestPrompt], list[EngineTokensPrompt], ]: - if tokenizer is None: - raise ValueError( - "Unable to get tokenizer because `skip_tokenizer_init=True`" - ) - model_config = self.model_config resolved_content_format = resolve_chat_template_content_format( @@ -1105,7 +1100,6 @@ class OpenAIServing: conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( messages, model_config, - tokenizer, content_format=resolved_content_format, ) diff --git a/vllm/entrypoints/score_utils.py b/vllm/entrypoints/score_utils.py index 8819c85af9a26..072ddd4c90b16 100644 --- a/vllm/entrypoints/score_utils.py +++ b/vllm/entrypoints/score_utils.py @@ -89,12 +89,10 @@ def parse_score_data( data_1: str | ScoreContentPartParam, data_2: str | ScoreContentPartParam, model_config: ModelConfig, - tokenizer: TokenizerLike, ) -> tuple[str, str, MultiModalDataDict | None]: - mm_tracker = MultiModalItemTracker(model_config, tokenizer) + mm_tracker = MultiModalItemTracker(model_config) content_1 = _parse_score_content(data_1, mm_tracker) - content_2 = _parse_score_content(data_2, mm_tracker) def ensure_str(content: _ContentPart | None) -> str: @@ -188,7 +186,6 @@ def get_score_prompt( data_1, data_2, model_config, - tokenizer, ) from vllm.model_executor.model_loader import get_model_cls diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index 6537b6df876a9..5aef09ca9c256 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -62,6 +62,7 @@ from vllm.multimodal.inputs import ( from vllm.multimodal.parse import ( DictEmbeddingItems, ImageSize, + ModalityDataItems, MultiModalDataItems, MultiModalDataParser, ) @@ -570,7 +571,7 @@ class HunYuanVLMultiModalDataParser(MultiModalDataParser): def _parse_image_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], - ): + ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): return DictEmbeddingItems( data, diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 8817601558148..09acf8372e168 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -1000,7 +1000,7 @@ class KeyeMultiModalDataParser(MultiModalDataParser): def _parse_image_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], - ) -> ModalityDataItems[Any, Any]: + ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): return DictEmbeddingItems( data, @@ -1017,7 +1017,7 @@ class KeyeMultiModalDataParser(MultiModalDataParser): def _parse_video_data( self, data: dict[str, torch.Tensor] | ModalityData[VideoItem], - ) -> ModalityDataItems[Any, Any]: + ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): return DictEmbeddingItems( data, diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index 124e9c2afa217..2b04e3bd4b75b 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -333,7 +333,7 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser): def _parse_image_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], - ) -> ModalityDataItems[Any, Any]: + ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): return DictEmbeddingItems( data, @@ -350,7 +350,7 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser): def _parse_video_data( self, data: dict[str, torch.Tensor] | ModalityData[VideoItem], - ) -> ModalityDataItems[Any, Any]: + ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): return DictEmbeddingItems( data, diff --git a/vllm/multimodal/audio.py b/vllm/multimodal/audio.py index b93a42ffd24c1..062547401c3cf 100644 --- a/vllm/multimodal/audio.py +++ b/vllm/multimodal/audio.py @@ -11,6 +11,7 @@ import pybase64 import torch from vllm.utils.import_utils import PlaceholderModule +from vllm.utils.serial_utils import tensor2base64 from .base import MediaIO @@ -135,8 +136,4 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]): return torch.load(filepath, weights_only=True) def encode_base64(self, media: torch.Tensor) -> str: - buffer = BytesIO() - torch.save(media, buffer) - buffer.seek(0) - binary_data = buffer.read() - return pybase64.b64encode(binary_data).decode("utf-8") + return tensor2base64(media) diff --git a/vllm/utils/serial_utils.py b/vllm/utils/serial_utils.py index b89fa6ce4db66..a6d717e03d37d 100644 --- a/vllm/utils/serial_utils.py +++ b/vllm/utils/serial_utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import base64 +import io import sys from dataclasses import dataclass from typing import Literal @@ -52,6 +53,15 @@ Endianness = Literal["native", "big", "little"] EncodingFormat = Literal["float", "base64", "bytes"] +def tensor2base64(x: torch.Tensor) -> str: + with io.BytesIO() as buf: + torch.save(x, buf) + buf.seek(0) + binary_data = buf.read() + + return base64.b64encode(binary_data).decode("utf-8") + + def tensor2binary( tensor: torch.Tensor, embed_dtype: EmbedDType, endianness: Endianness ) -> bytes: From 82a64b3d8f93521d39569078d4ac56992a50a640 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Thu, 4 Dec 2025 15:12:12 +0800 Subject: [PATCH 28/47] [Bugfix] fixed deepseekv32 tool calling error (#30025) Signed-off-by: chaunceyjiang Co-authored-by: Cyrus Leung --- vllm/tokenizers/deepseek_v32_encoding.py | 6 ++++-- vllm/tokenizers/deepseekv32.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vllm/tokenizers/deepseek_v32_encoding.py b/vllm/tokenizers/deepseek_v32_encoding.py index fb8989e651e3a..521bd92959312 100644 --- a/vllm/tokenizers/deepseek_v32_encoding.py +++ b/vllm/tokenizers/deepseek_v32_encoding.py @@ -95,8 +95,10 @@ def tool_calls_to_openai_format(tool_calls): def encode_arguments_to_dsml(tool_call: dict[str, str]) -> str: p_dsml_template = """<{dsml_token}parameter name="{key}" string="{is_str}">{value}""" P_dsml_strs = [] - - arguments = json.loads(tool_call["arguments"]) + if isinstance(tool_call["arguments"], str): + arguments = json.loads(tool_call["arguments"]) + else: + arguments = tool_call["arguments"] for k, v in arguments.items(): p_dsml_str = p_dsml_template.format( diff --git a/vllm/tokenizers/deepseekv32.py b/vllm/tokenizers/deepseekv32.py index 7466ad4076bd8..1140357cf861d 100644 --- a/vllm/tokenizers/deepseekv32.py +++ b/vllm/tokenizers/deepseekv32.py @@ -43,7 +43,8 @@ class DeepseekV32Tokenizer(HfTokenizer): thinking_mode = "thinking" if not thinking: thinking_mode = "chat" - messages = messages.copy() + conversation = kwargs.get("conversation", messages) + messages = conversation.copy() drop_thinking = True if tools is not None and len(tools) > 0: messages.insert(0, {"role": "system"}) From 404fc4bfc049fc86cc1ddd1d975ecfc72609db4f Mon Sep 17 00:00:00 2001 From: daniel-salib Date: Wed, 3 Dec 2025 23:36:57 -0800 Subject: [PATCH 29/47] [Frontend] refactor harmony utils output message parsing (#29820) Signed-off-by: Daniel Salib --- vllm/entrypoints/harmony_utils.py | 216 ++++++++++++++++-------------- 1 file changed, 117 insertions(+), 99 deletions(-) diff --git a/vllm/entrypoints/harmony_utils.py b/vllm/entrypoints/harmony_utils.py index 47a252348c102..bb932e39e0472 100644 --- a/vllm/entrypoints/harmony_utils.py +++ b/vllm/entrypoints/harmony_utils.py @@ -328,6 +328,105 @@ def render_for_completion(messages: list[Message]) -> list[int]: return token_ids +def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutputItem: + """Parse browser tool calls (search, open, find) into web search items.""" + if len(message.content) != 1: + raise ValueError("Invalid number of contents in browser message") + content = message.content[0] + + # Parse JSON args (with retry detection) + try: + browser_call = json.loads(content.text) + except json.JSONDecodeError: + json_retry_output_message = ( + f"Invalid JSON args, caught and retried: {content.text}" + ) + browser_call = { + "query": json_retry_output_message, + "url": json_retry_output_message, + "pattern": json_retry_output_message, + } + + # Create appropriate action based on recipient + if recipient == "browser.search": + action = ActionSearch( + query=f"cursor:{browser_call.get('query', '')}", type="search" + ) + elif recipient == "browser.open": + action = ActionOpenPage( + url=f"cursor:{browser_call.get('url', '')}", type="open_page" + ) + elif recipient == "browser.find": + action = ActionFind( + pattern=browser_call.get("pattern", ""), + url=f"cursor:{browser_call.get('url', '')}", + type="find", + ) + else: + raise ValueError(f"Unknown browser action: {recipient}") + + return ResponseFunctionWebSearch( + id=f"ws_{random_uuid()}", + action=action, + status="completed", + type="web_search_call", + ) + + +def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutputItem]: + """Parse function calls into function tool call items.""" + function_name = recipient.split(".")[-1] + output_items = [] + for content in message.content: + random_id = random_uuid() + response_item = ResponseFunctionToolCall( + arguments=content.text, + call_id=f"call_{random_id}", + type="function_call", + name=function_name, + id=f"fc_{random_id}", + ) + output_items.append(response_item) + return output_items + + +def _parse_reasoning_content(message: Message) -> list[ResponseOutputItem]: + """Parse reasoning/analysis content into reasoning items.""" + output_items = [] + for content in message.content: + reasoning_item = ResponseReasoningItem( + id=f"rs_{random_uuid()}", + summary=[], + type="reasoning", + content=[ + ResponseReasoningTextContent(text=content.text, type="reasoning_text") + ], + status=None, + ) + output_items.append(reasoning_item) + return output_items + + +def _parse_final_message(message: Message) -> ResponseOutputItem: + """Parse final channel messages into output message items.""" + contents = [] + for content in message.content: + output_text = ResponseOutputText( + text=content.text, + annotations=[], # TODO + type="output_text", + logprobs=None, # TODO + ) + contents.append(output_text) + return ResponseOutputMessage( + id=f"msg_{random_uuid()}", + content=contents, + role=message.author.role, + status="completed", + type="message", + ) + + def parse_output_message(message: Message) -> list[ResponseOutputItem]: """ Parse a Harmony message into a list of output response items. @@ -340,119 +439,38 @@ def parse_output_message(message: Message) -> list[ResponseOutputItem]: output_items: list[ResponseOutputItem] = [] recipient = message.recipient + + # Browser tool calls if recipient is not None and recipient.startswith("browser."): - if len(message.content) != 1: - raise ValueError("Invalid number of contents in browser message") - content = message.content[0] - # We do not need to check the VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY - # env variable since if it is not set, we are certain the json is valid - # The use of Actions for web search will be removed entirely in - # the future, so this is only necessary temporarily - try: - browser_call = json.loads(content.text) - except json.JSONDecodeError: - # If the content is not valid JSON, then it was - # caught and retried by vLLM, which means we - # need to make note of that so the user is aware - json_retry_output_message = ( - f"Invalid JSON args, caught and retried: {content.text}" - ) - browser_call = { - "query": json_retry_output_message, - "url": json_retry_output_message, - "pattern": json_retry_output_message, - } - # TODO: translate to url properly! - if recipient == "browser.search": - action = ActionSearch( - query=f"cursor:{browser_call.get('query', '')}", type="search" - ) - elif recipient == "browser.open": - action = ActionOpenPage( - url=f"cursor:{browser_call.get('url', '')}", type="open_page" - ) - elif recipient == "browser.find": - action = ActionFind( - pattern=browser_call["pattern"], - url=f"cursor:{browser_call.get('url', '')}", - type="find", - ) - else: - raise ValueError(f"Unknown browser action: {recipient}") - web_search_item = ResponseFunctionWebSearch( - id=f"ws_{random_uuid()}", - action=action, - status="completed", - type="web_search_call", - ) - output_items.append(web_search_item) + output_items.append(_parse_browser_tool_call(message, recipient)) + + # Analysis channel (reasoning/chain-of-thought) elif message.channel == "analysis": - for content in message.content: - reasoning_item = ResponseReasoningItem( - id=f"rs_{random_uuid()}", - summary=[], - type="reasoning", - content=[ - ResponseReasoningTextContent( - text=content.text, type="reasoning_text" - ) - ], - status=None, - ) - output_items.append(reasoning_item) + output_items.extend(_parse_reasoning_content(message)) + + # Commentary channel elif message.channel == "commentary": + # Function calls if recipient is not None and recipient.startswith("functions."): - function_name = recipient.split(".")[-1] - for content in message.content: - random_id = random_uuid() - response_item = ResponseFunctionToolCall( - arguments=content.text, - call_id=f"call_{random_id}", - type="function_call", - name=function_name, - id=f"fc_{random_id}", - ) - output_items.append(response_item) + output_items.extend(_parse_function_call(message, recipient)) + + # Built-in tools on commentary channel are treated as reasoning for now elif recipient is not None and ( recipient.startswith("python") or recipient.startswith("browser") or recipient.startswith("container") ): - for content in message.content: - reasoning_item = ResponseReasoningItem( - id=f"rs_{random_uuid()}", - summary=[], - type="reasoning", - content=[ - ResponseReasoningTextContent( - text=content.text, type="reasoning_text" - ) - ], - status=None, - ) - output_items.append(reasoning_item) + output_items.extend(_parse_reasoning_content(message)) else: raise ValueError(f"Unknown recipient: {recipient}") + + # Final output message elif message.channel == "final": - contents = [] - for content in message.content: - output_text = ResponseOutputText( - text=content.text, - annotations=[], # TODO - type="output_text", - logprobs=None, # TODO - ) - contents.append(output_text) - text_item = ResponseOutputMessage( - id=f"msg_{random_uuid()}", - content=contents, - role=message.author.role, - status="completed", - type="message", - ) - output_items.append(text_item) + output_items.append(_parse_final_message(message)) + else: raise ValueError(f"Unknown channel: {message.channel}") + return output_items From fd68e909db1804f211707bb027a49b82bb5c2d8f Mon Sep 17 00:00:00 2001 From: CYJiang <86391540+googs1025@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:46:15 +0800 Subject: [PATCH 30/47] [docs] Remove _total from counter metrics names (#30028) In Prometheus Counters always expose their actual numeric value with a metric name that ends in _total. We should document the base name, as this what appears in the get_metrics() API. Signed-off-by: CYJiang <86391540+googs1025@users.noreply.github.com> --- docs/design/metrics.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/design/metrics.md b/docs/design/metrics.md index 59cb6ba46fe17..13264f6861b0c 100644 --- a/docs/design/metrics.md +++ b/docs/design/metrics.md @@ -57,15 +57,15 @@ vLLM also provides [a reference example](../../examples/online_serving/prometheu The subset of metrics exposed in the Grafana dashboard gives us an indication of which metrics are especially important: - `vllm:e2e_request_latency_seconds_bucket` - End to end request latency measured in seconds. -- `vllm:prompt_tokens_total` - Prompt tokens. -- `vllm:generation_tokens_total` - Generation tokens. +- `vllm:prompt_tokens` - Prompt tokens. +- `vllm:generation_tokens` - Generation tokens. - `vllm:time_per_output_token_seconds` - Inter-token latency (Time Per Output Token, TPOT) in seconds. - `vllm:time_to_first_token_seconds` - Time to First Token (TTFT) latency in seconds. - `vllm:num_requests_running` (also, `_swapped` and `_waiting`) - Number of requests in the RUNNING, WAITING, and SWAPPED states. - `vllm:gpu_cache_usage_perc` - Percentage of used cache blocks by vLLM. - `vllm:request_prompt_tokens` - Request prompt length. - `vllm:request_generation_tokens` - Request generation length. -- `vllm:request_success_total` - Number of finished requests by their finish reason: either an EOS token was generated or the max sequence length was reached. +- `vllm:request_success` - Number of finished requests by their finish reason: either an EOS token was generated or the max sequence length was reached. - `vllm:request_queue_time_seconds` - Queue time. - `vllm:request_prefill_time_seconds` - Requests prefill time. - `vllm:request_decode_time_seconds` - Requests decode time. @@ -571,9 +571,9 @@ model and then validate those tokens with the larger model. - `vllm:spec_decode_draft_acceptance_rate` (Gauge) - `vllm:spec_decode_efficiency` (Gauge) -- `vllm:spec_decode_num_accepted_tokens_total` (Counter) -- `vllm:spec_decode_num_draft_tokens_total` (Counter) -- `vllm:spec_decode_num_emitted_tokens_total` (Counter) +- `vllm:spec_decode_num_accepted_tokens` (Counter) +- `vllm:spec_decode_num_draft_tokens` (Counter) +- `vllm:spec_decode_num_emitted_tokens` (Counter) There is a PR under review () to add "prompt lookup (ngram)" speculative decoding to v1. Other techniques will follow. We should From 9aa33a74b00b2db7d7da22a59ed64b44ebbabe14 Mon Sep 17 00:00:00 2001 From: Charlie Fu Date: Thu, 4 Dec 2025 01:52:28 -0600 Subject: [PATCH 31/47] [Rocm][CI] Fix test_speculator_eagle3 by skipping the CompressedTensorw4a16 Model (#30001) Signed-off-by: charlifu Co-authored-by: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com> --- tests/v1/spec_decode/test_speculators_eagle3.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/v1/spec_decode/test_speculators_eagle3.py b/tests/v1/spec_decode/test_speculators_eagle3.py index 5ce6e1593b5c1..9a252cfffc8f0 100644 --- a/tests/v1/spec_decode/test_speculators_eagle3.py +++ b/tests/v1/spec_decode/test_speculators_eagle3.py @@ -5,6 +5,7 @@ import torch from vllm.config import SpeculativeConfig from vllm.model_executor.models.interfaces import supports_eagle3 +from vllm.platforms import current_platform @pytest.mark.parametrize( @@ -21,6 +22,10 @@ from vllm.model_executor.models.interfaces import supports_eagle3 pytest.param( "nm-testing/Speculator-Qwen3-8B-Eagle3-converted-071-quantized-w4a16", id="qwen3-eagle3-speculator-w4a16-verifier", + marks=pytest.mark.skipif( + current_platform.is_rocm(), + reason="The tests are skipped on rocm platform.", + ), ), ], ) From 3f1b03739ae1422361446d3d23bed970bd549ebc Mon Sep 17 00:00:00 2001 From: TJian Date: Thu, 4 Dec 2025 16:20:24 +0800 Subject: [PATCH 32/47] [ROCm] [Bugfix] `compute_attn_mask_seqlen` for qwen3 omni (#29974) Signed-off-by: tjtanaa --- vllm/model_executor/models/qwen3_omni_moe_thinker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index fe825198dcaa4..e6979211b707f 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -494,7 +494,10 @@ class Qwen3Omni_VisionTransformer(nn.Module): cu_seqlens: torch.Tensor, ) -> torch.Tensor: max_seqlen = torch.zeros([], device=cu_seqlens.device) - if self.attn_backend == AttentionBackendEnum.FLASH_ATTN: + if self.attn_backend in { + AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.ROCM_AITER_FA, + }: max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() return max_seqlen From 5430e110c099fdc6c8c80f443bf5adffe67aa30b Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Thu, 4 Dec 2025 02:20:54 -0600 Subject: [PATCH 33/47] [CI][AMD] Match Main CI Behavior By Skipping test_eplb_spec_decode In AMD CI (#30006) Signed-off-by: Micah Williamson --- tests/distributed/test_eplb_spec_decode.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/distributed/test_eplb_spec_decode.py b/tests/distributed/test_eplb_spec_decode.py index 868cc702866e2..22977ce94404b 100644 --- a/tests/distributed/test_eplb_spec_decode.py +++ b/tests/distributed/test_eplb_spec_decode.py @@ -6,6 +6,7 @@ import lm_eval import pytest from tests.utils import large_gpu_mark +from vllm.platforms import current_platform def get_model_args( @@ -45,6 +46,12 @@ def get_model_args( return model_args +pytestmark = pytest.mark.skipif( + current_platform.is_rocm(), + reason="EPLB with Spec Decode is a work in progress on ROCm.", +) + + @pytest.mark.parametrize( "model_setup", [ From 68eb5c8d970a453a440776211f8dbff215fb40c3 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 4 Dec 2025 16:21:19 +0800 Subject: [PATCH 34/47] [Misc] Move functions into `PoolingMetadata` (#30027) Signed-off-by: DarkLight1337 --- vllm/model_executor/layers/pooler.py | 50 ++++------------------------ vllm/model_executor/models/gritlm.py | 6 ++-- vllm/v1/pool/metadata.py | 21 ++++++++++++ 3 files changed, 30 insertions(+), 47 deletions(-) diff --git a/vllm/model_executor/layers/pooler.py b/vllm/model_executor/layers/pooler.py index 7dd02e32ff211..185e03e5f3bd7 100644 --- a/vllm/model_executor/layers/pooler.py +++ b/vllm/model_executor/layers/pooler.py @@ -64,42 +64,6 @@ class PoolingParamsUpdate: params.requires_token_ids = self.requires_token_ids -def get_prompt_lens( - hidden_states: torch.Tensor | list[torch.Tensor], - pooling_metadata: PoolingMetadata, -) -> torch.Tensor: - return pooling_metadata.prompt_lens - - -def get_prompt_token_ids(pooling_metadata: PoolingMetadata) -> list[torch.Tensor]: - assert pooling_metadata.prompt_token_ids is not None, ( - "Please set `requires_token_ids=True` in `get_pooling_updates`" - ) - - return [ - pooling_metadata.prompt_token_ids[i, :num] - for i, num in enumerate(pooling_metadata.prompt_lens) - ] - - -def get_pooling_params(pooling_metadata: PoolingMetadata) -> list[PoolingParams]: - pooling_params = pooling_metadata.pooling_params - return pooling_params - - -def get_tasks(pooling_metadata: PoolingMetadata) -> list[PoolingTask]: - pooling_params = get_pooling_params(pooling_metadata) - - tasks: list[PoolingTask] = [ - task - for pooling_param in pooling_params - if (task := pooling_param.task) is not None - ] - assert len(pooling_params) == len(tasks) - - return tasks - - def get_classification_activation_function(config: PretrainedConfig): # Implement alignment with transformers ForSequenceClassificationLoss # https://github.com/huggingface/transformers/blob/57bb6db6ee4cfaccc45b8d474dfad5a17811ca60/src/transformers/loss/loss_utils.py#L92 @@ -466,7 +430,7 @@ class EmbeddingPoolerHead(PoolerHead): pooled_data = self.projector(pooled_data) # pooled_data shape: [batchsize, embedding_dimension] - pooling_params = get_pooling_params(pooling_metadata) + pooling_params = pooling_metadata.pooling_params # for matryoshka representation dimensions_list = [pooling_param.dimensions for pooling_param in pooling_params] @@ -606,7 +570,7 @@ class ClassifierPooler(Pooler): if self.logit_bias is not None: pooled_data -= self.logit_bias - pooling_params = get_pooling_params(pooling_metadata) + pooling_params = pooling_metadata.pooling_params flags = [p.use_activation for p in pooling_params] if len(set(flags)) == 1: @@ -704,7 +668,7 @@ class AllPooler(Pooler): pooling_metadata: PoolingMetadata, ) -> PoolerOutput: pooled_data = self.pooling(hidden_states, pooling_metadata) - pooling_params = get_pooling_params(pooling_metadata) + pooling_params = pooling_metadata.pooling_params assert len(pooled_data) == len(pooling_params) pooled_data = [self.head(d, p) for d, p in zip(pooled_data, pooling_params)] @@ -724,11 +688,11 @@ class StepPooler(Pooler): pooling_metadata: PoolingMetadata, ) -> torch.Tensor | list[torch.Tensor]: pooled_data_lst = self.pooling(hidden_states, pooling_metadata) - prompt_token_ids = get_prompt_token_ids(pooling_metadata) + prompt_token_ids = pooling_metadata.get_prompt_token_ids() pooled_data = list[torch.Tensor]() - pooling_params = get_pooling_params(pooling_metadata) + pooling_params = pooling_metadata.pooling_params for data, token_id, pooling_param in zip( pooled_data_lst, prompt_token_ids, pooling_params @@ -757,7 +721,7 @@ class StepPooler(Pooler): pooling_metadata: PoolingMetadata, ) -> PoolerOutput: pooled_data = self.extract_states(hidden_states, pooling_metadata) - pooling_params = get_pooling_params(pooling_metadata) + pooling_params = pooling_metadata.pooling_params assert len(pooled_data) == len(pooling_params) pooled_data = [self.head(d, p) for d, p in zip(pooled_data, pooling_params)] @@ -794,7 +758,7 @@ class DispatchPooler(Pooler): outputs = list[torch.Tensor]() offset = 0 - for task, group in groupby(get_tasks(pooling_metadata)): + for task, group in groupby(pooling_metadata.tasks): if not (pooler := poolers_by_task.get(task)): raise ValueError( f"Unsupported task: {task} " diff --git a/vllm/model_executor/models/gritlm.py b/vllm/model_executor/models/gritlm.py index 550e8b014d5e7..2aba626a7c737 100644 --- a/vllm/model_executor/models/gritlm.py +++ b/vllm/model_executor/models/gritlm.py @@ -14,8 +14,6 @@ from vllm.model_executor.layers.pooler import ( PoolerHead, PoolerNormalize, PoolingParamsUpdate, - get_prompt_lens, - get_prompt_token_ids, ) from vllm.model_executor.models.llama import LlamaForCausalLM from vllm.tasks import PoolingTask @@ -153,11 +151,11 @@ class GritLMMeanPool(nn.Module): hidden_states: torch.Tensor | list[torch.Tensor], pooling_metadata: PoolingMetadata, ) -> list[torch.Tensor] | torch.Tensor: - prompt_lens = get_prompt_lens(hidden_states, pooling_metadata) + prompt_lens = pooling_metadata.prompt_lens instr_lens = torch.tensor( [ self._get_instruction_len(token_ids.cpu().numpy()) - for token_ids in get_prompt_token_ids(pooling_metadata) + for token_ids in pooling_metadata.get_prompt_token_ids() ], device="cpu", ) diff --git a/vllm/v1/pool/metadata.py b/vllm/v1/pool/metadata.py index 7bd2c7415dafe..9ee588ea44ca4 100644 --- a/vllm/v1/pool/metadata.py +++ b/vllm/v1/pool/metadata.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import torch from vllm.pooling_params import PoolingParams +from vllm.tasks import PoolingTask from vllm.utils.platform_utils import is_pin_memory_available pin_memory = is_pin_memory_available() @@ -40,6 +41,18 @@ class PoolingMetadata: pooling_params: list[PoolingParams] pooling_cursor: PoolingCursor | None = None + def __post_init__(self) -> None: + pooling_params = self.pooling_params + + tasks: list[PoolingTask] = [ + task + for pooling_param in pooling_params + if (task := pooling_param.task) is not None + ] + assert len(pooling_params) == len(tasks) + + self.tasks = tasks + def __getitem__(self, indices: slice): return PoolingMetadata( prompt_lens=self.prompt_lens[indices], @@ -52,6 +65,14 @@ class PoolingMetadata: else self.pooling_cursor[indices], ) + def get_prompt_token_ids(self) -> list[torch.Tensor]: + prompt_token_ids = self.prompt_token_ids + assert prompt_token_ids is not None, ( + "Please set `requires_token_ids=True` in `get_pooling_updates`" + ) + + return [prompt_token_ids[i, :num] for i, num in enumerate(self.prompt_lens)] + def build_pooling_cursor( self, num_scheduled_tokens: list[int], device: torch.device ): From 899e2ef558e7345b99bc0d53c2e1c60ffdca7470 Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Thu, 4 Dec 2025 08:22:03 +0000 Subject: [PATCH 35/47] [Core] Fix standalone runs of test_reset_prefix_cache_e2e (#29899) Signed-off-by: Mark McLoughlin --- tests/v1/core/test_reset_prefix_cache_e2e.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/v1/core/test_reset_prefix_cache_e2e.py b/tests/v1/core/test_reset_prefix_cache_e2e.py index e543c30a156ec..083fc3f34f545 100644 --- a/tests/v1/core/test_reset_prefix_cache_e2e.py +++ b/tests/v1/core/test_reset_prefix_cache_e2e.py @@ -11,7 +11,9 @@ PROMPTS = [ ] -def test_reset_prefix_cache_e2e(): +def test_reset_prefix_cache_e2e(monkeypatch): + # "spawn" is required for test to be deterministic + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") engine_args = EngineArgs( model="Qwen/Qwen3-0.6B", gpu_memory_utilization=0.2, From b8a6ae415859bf8c9a3509cbd714695866e35d66 Mon Sep 17 00:00:00 2001 From: "Ye (Charlotte) Qi" Date: Thu, 4 Dec 2025 00:45:57 -0800 Subject: [PATCH 36/47] [ROCm] add fallback for aiter fp8 decode mla (#30005) Signed-off-by: Ye (Charlotte) Qi --- vllm/_aiter_ops.py | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index a8f472d147a0d..35920d826578e 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -283,6 +283,28 @@ def _rocm_aiter_grouped_topk_fake( pass +# Cache whether aiter supports FP8 MLA parameters +_AITER_MLA_SUPPORTS_FP8: bool | None = None + + +def _check_aiter_mla_fp8_support() -> bool: + """Check if aiter.mla.mla_decode_fwd supports q_scale and kv_scale parameters.""" + global _AITER_MLA_SUPPORTS_FP8 + if _AITER_MLA_SUPPORTS_FP8 is None: + try: + import inspect + + from aiter.mla import mla_decode_fwd + + sig = inspect.signature(mla_decode_fwd) + _AITER_MLA_SUPPORTS_FP8 = ( + "q_scale" in sig.parameters and "kv_scale" in sig.parameters + ) + except Exception: + _AITER_MLA_SUPPORTS_FP8 = False + return _AITER_MLA_SUPPORTS_FP8 + + def _rocm_aiter_mla_decode_fwd_impl( q: torch.Tensor, kv_buffer: torch.Tensor, @@ -299,6 +321,16 @@ def _rocm_aiter_mla_decode_fwd_impl( ) -> None: from aiter.mla import mla_decode_fwd + kwargs = { + "sm_scale": sm_scale, + "logit_cap": logit_cap, + } + + # Only pass q_scale and kv_scale if the aiter library supports them + if _check_aiter_mla_fp8_support(): + kwargs["q_scale"] = q_scale + kwargs["kv_scale"] = kv_scale + mla_decode_fwd( q, kv_buffer.view(-1, 1, 1, q.shape[-1]), @@ -308,10 +340,7 @@ def _rocm_aiter_mla_decode_fwd_impl( kv_indices, kv_last_page_lens, max_seqlen_qo, - sm_scale=sm_scale, - logit_cap=logit_cap, - q_scale=q_scale, - kv_scale=kv_scale, + **kwargs, ) From ffdd18111b767d271786b982378d51b51ce151fe Mon Sep 17 00:00:00 2001 From: Xu Wenqing <121550081+Xu-Wenqing@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:46:34 +0800 Subject: [PATCH 37/47] Add DeepSeek-V3.2 tool parser. (#29848) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 许文卿 --- .../openai/tool_parsers/__init__.py | 4 + .../tool_parsers/deepseekv32_tool_parser.py | 591 ++++++++++++++++++ 2 files changed, 595 insertions(+) create mode 100644 vllm/entrypoints/openai/tool_parsers/deepseekv32_tool_parser.py diff --git a/vllm/entrypoints/openai/tool_parsers/__init__.py b/vllm/entrypoints/openai/tool_parsers/__init__.py index 89e439dd53f5f..ed43ea7eec82f 100644 --- a/vllm/entrypoints/openai/tool_parsers/__init__.py +++ b/vllm/entrypoints/openai/tool_parsers/__init__.py @@ -30,6 +30,10 @@ _TOOL_PARSERS_TO_REGISTER = { "deepseekv31_tool_parser", "DeepSeekV31ToolParser", ), + "deepseek_v32": ( + "deepseekv32_tool_parser", + "DeepSeekV32ToolParser", + ), "ernie45": ( "ernie45_tool_parser", "Ernie45ToolParser", diff --git a/vllm/entrypoints/openai/tool_parsers/deepseekv32_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/deepseekv32_tool_parser.py new file mode 100644 index 0000000000000..4973deb7cefa8 --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/deepseekv32_tool_parser.py @@ -0,0 +1,591 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +import uuid +from collections.abc import Sequence +from typing import Any + +import regex as re + +from vllm.entrypoints.openai.protocol import ( + ChatCompletionRequest, + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, + ToolCall, +) +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( + ToolParser, +) +from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike + +logger = init_logger(__name__) + + +class DeepSeekV32ToolParser(ToolParser): + """ + example tool call content: + <｜DSML｜function_calls> + <｜DSML｜invoke name="get_weather"> + <｜DSML｜parameter name="location" string="true">杭州 + <｜DSML｜parameter name="date" string="true">2024-01-16 + + <｜DSML｜invoke name="get_weather"> + <｜DSML｜parameter name="location" string="true">北京 + <｜DSML｜parameter name="date" string="true">2024-01-16 + + + """ + + def __init__(self, tokenizer: TokenizerLike): + super().__init__(tokenizer) + + self.prev_tool_call_arr: list[dict] = [] + + # Sentinel tokens + self.dsml_token: str = "｜DSML｜" + self.dsml_start_check: str = "<" + self.dsml_token + self.tool_call_start_token: str = "<｜DSML｜function_calls>" + self.tool_call_end_token: str = "" + self.invoke_start_prefix: str = "<｜DSML｜invoke name=" + self.invoke_end_token: str = "" + self.parameter_prefix: str = "<｜DSML｜parameter name=" + self.parameter_end_token: str = "" + + # Streaming state variables + self.current_tool_name_sent: bool = False + # Override base class type - we use string IDs for tool calls + self.current_tool_id: str | None = None # type: ignore + self.streamed_args_for_tool: list[str] = [] + self.is_tool_call_started: bool = False + self.failed_count: int = 0 + + # Initialize streaming state variables + self.current_tool_index: int = 0 + self.invoke_index: int = 0 + self.header_sent: bool = False + self.current_function_name: str | None = None + self.current_param_name: str | None = None + self.current_param_value: str = "" + self.param_count: int = 0 + self.in_param: bool = False + self.in_function: bool = False + self.json_started: bool = False + self.json_closed: bool = False + self.accumulated_params: dict = {} + self.streaming_request: ChatCompletionRequest | None = None + + # Enhanced streaming state - reset for each new message + self._reset_streaming_state() + + # Regex patterns for complete parsing + self.tool_call_complete_regex = re.compile( + r"<｜DSML｜function_calls>(.*?)", re.DOTALL + ) + self.invoke_complete_regex = re.compile( + r'<｜DSML｜invoke\s+name="([^"]+)"\s*>(.*?)', re.DOTALL + ) + self.parameter_complete_regex = re.compile( + r'<｜DSML｜parameter\s+name="([^"]+)"\s+string="(?:true|false)"\s*>(.*?)', + re.DOTALL, + ) + + if not self.model_tokenizer: + raise ValueError( + "The model tokenizer must be passed to the ToolParser " + "constructor during construction." + ) + + logger.debug( + "vLLM Successfully import tool parser %s !", self.__class__.__name__ + ) + + def _generate_tool_call_id(self) -> str: + """Generate a unique tool call ID.""" + return f"call_{uuid.uuid4().hex[:24]}" + + def _reset_streaming_state(self): + """Reset all streaming state.""" + self.current_tool_index = 0 + self.invoke_index = 0 + self.is_tool_call_started = False + self.header_sent = False + self.current_tool_id = None + self.current_function_name = None + self.current_param_name = None + self.current_param_value = "" + self.param_count = 0 + self.in_param = False + self.in_function = False + self.json_started = False + self.json_closed = False + # Store accumulated parameters for type conversion + self.accumulated_params = {} + self.streaming_request = None + # Clear previous tool call history to avoid state pollution + self.prev_tool_call_arr.clear() + + def _parse_invoke_params(self, invoke_str: str) -> dict | None: + param_dict = dict() + for param_name, param_val in self.parameter_complete_regex.findall(invoke_str): + param_dict[param_name] = param_val + return param_dict + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + """Extract tool calls from complete model output (non-streaming).""" + # Quick check + if self.tool_call_start_token not in model_output: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + try: + tool_calls = [] + + # Find all complete tool_call blocks + for tool_call_match in self.tool_call_complete_regex.findall(model_output): + # Find all invokes within this tool_call + for invoke_name, invoke_content in self.invoke_complete_regex.findall( + tool_call_match + ): + param_dict = self._parse_invoke_params(invoke_content) + tool_calls.append( + ToolCall( + type="function", + function=FunctionCall( + name=invoke_name, + arguments=json.dumps(param_dict, ensure_ascii=False), + ), + ) + ) + + if not tool_calls: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + # Extract content before first tool call + first_tool_idx = model_output.find(self.tool_call_start_token) + content = model_output[:first_tool_idx] if first_tool_idx > 0 else None + + return ExtractedToolCallInformation( + tools_called=True, tool_calls=tool_calls, content=content + ) + + except Exception: + logger.exception("Error extracting tool calls") + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + def _extract_name(self, name_str: str) -> str: + """Extract name from quoted string.""" + name_str = name_str.strip() + if ( + name_str.startswith('"') + and name_str.endswith('"') + or name_str.startswith("'") + and name_str.endswith("'") + ): + return name_str[1:-1] + return name_str + + def _extract_param_name(self, input_str: str) -> str: + """Extract param name""" + start = input_str.find('"') + 1 + end = input_str.find('"', start) + return input_str[start:end] if start > 0 and end > start else input_str + + def _convert_param_value(self, value: str, param_type: str) -> Any: + """Convert parameter value to the correct type.""" + if value.lower() == "null": + return None + + param_type = param_type.lower() + if param_type in ["string", "str", "text"]: + return value + elif param_type in ["integer", "int"]: + try: + return int(value) + except (ValueError, TypeError): + return value + elif param_type in ["number", "float"]: + try: + val = float(value) + return val if val != int(val) else int(val) + except (ValueError, TypeError): + return value + elif param_type in ["boolean", "bool"]: + return value.lower() in ["true", "1"] + elif param_type in ["object", "array"]: + try: + return json.loads(value) + except json.JSONDecodeError: + return value + else: + # Try JSON parse first, fallback to string + try: + return json.loads(value) + except json.JSONDecodeError: + return value + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], # pylint: disable=unused-argument + current_token_ids: Sequence[int], # pylint: disable=unused-argument + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> DeltaMessage | None: + """Extract tool calls from streaming model output.""" + + # Store request for type conversion + if not previous_text: + self._reset_streaming_state() + self.streaming_request = request + + # If no delta text, return None unless it's an EOS token after tools + if not delta_text: + # Check if this is an EOS token after all tool calls are complete + if delta_token_ids: + # Count complete tool calls + complete_calls = len( + self.tool_call_complete_regex.findall(current_text) + ) + + # If we have completed tool calls and populated prev_tool_call_arr + if complete_calls > 0 and len(self.prev_tool_call_arr) > 0: + # Check if all tool calls are closed + open_calls = current_text.count( + self.tool_call_start_token + ) - current_text.count(self.tool_call_end_token) + if open_calls == 0: + # Return empty delta for finish_reason processing + return DeltaMessage(content="") + elif not self.is_tool_call_started and current_text: + # This is a regular content response that's now complete + return DeltaMessage(content="") + return None + + # Check if we need to advance to next tool + if self.json_closed and not self.in_function: + # Check if this tool call has ended + invoke_ends = current_text.count(self.invoke_end_token) + if invoke_ends > self.current_tool_index: + # This tool has ended, advance to next + self.current_tool_index += 1 + self.header_sent = False + self.param_count = 0 + self.json_started = False + self.json_closed = False + self.in_function = False # Now we can safely set this to False + self.accumulated_params = {} + # Continue processing next tool + return None + + # Handle normal content before tool calls + if not self.is_tool_call_started: + # Check if tool call is starting + if self.dsml_token in current_text: + self.is_tool_call_started = True + # Return any content before the tool call + if self.dsml_start_check in delta_text: + content_before = delta_text[ + : delta_text.index(self.dsml_start_check) + ] + if content_before: + return DeltaMessage(content=content_before) + return None + else: + # Check if we're between tool calls - skip whitespace + if ( + current_text.rstrip().endswith(self.tool_call_end_token) + and delta_text.strip() == "" + ): + # We just ended a tool call, skip whitespace + return None + # Normal content, no tool call + if delta_text.endswith("<"): + return DeltaMessage(content=delta_text[:-1]) + if previous_text and previous_text.endswith("<"): + return DeltaMessage(content="<" + delta_text) + return DeltaMessage(content=delta_text) + + # Check if we're between tool calls (waiting for next one) + invoke_starts_count = current_text.count(self.invoke_start_prefix) + if self.current_tool_index >= invoke_starts_count: + # We're past all tool calls, shouldn't be here + return None + + # Find the current tool call portion + invoke_start_positions: list[int] = [] + idx = 0 + while True: + idx = current_text.find(self.invoke_start_prefix, idx) + if idx == -1: + break + invoke_start_positions.append(idx) + idx += len(self.invoke_start_prefix) + + if self.current_tool_index >= len(invoke_start_positions): + # No more tool calls to process yet + return None + + invoke_start_idx = invoke_start_positions[self.current_tool_index] + # Find where this tool call ends (or current position if not ended yet) + invoke_end_idx = current_text.find(self.invoke_end_token, invoke_start_idx) + if invoke_end_idx == -1: + tool_text = current_text[invoke_start_idx:] + else: + tool_text = current_text[ + invoke_start_idx : invoke_end_idx + len(self.invoke_end_token) + ] + + # Looking for function header + if not self.header_sent: + if self.invoke_start_prefix in tool_text: + func_start = tool_text.find(self.invoke_start_prefix) + len( + self.invoke_start_prefix + ) + # Find the end quote for the function name + func_end = tool_text.find(">", func_start) + + if func_end != -1: + # Found complete function name + function_name_raw = tool_text[func_start:func_end] + self.current_function_name = self._extract_name(function_name_raw) + self.current_tool_id = self._generate_tool_call_id() + self.header_sent = True + self.in_function = True + + # Add to prev_tool_call_arr immediately when we detect a tool call + # Each tool call should be recorded regardless of function name + # Ensure we don't add the same tool call index multiple times + if len(self.prev_tool_call_arr) <= self.current_tool_index: + self.prev_tool_call_arr.append( + { + "name": self.current_function_name, + "arguments": "{}", # Placeholder, will be updated later + } + ) + + # Send header with function info + return DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + id=self.current_tool_id, + function=DeltaFunctionCall( + name=self.current_function_name, arguments="" + ), + type="function", + ) + ] + ) + return None + + # We've sent header, now handle function body + if self.in_function: + # Send opening brace if not sent yet + if self.in_function and not self.json_started: + self.json_started = True + return DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments="{"), + ) + ] + ) + + # Make sure json_started is set if we're processing parameters + if not self.json_started: + self.json_started = True + + # Check for function end in accumulated text + if not self.json_closed and self.invoke_end_token in tool_text: + # Count total parameters in the tool text + total_param_count = tool_text.count(self.parameter_prefix) + + # Only close JSON if all parameters have been processed + if self.param_count >= total_param_count: + # Close JSON + self.json_closed = True + + # Extract complete tool call + # Find the invoke content + invoke_start = tool_text.find(self.invoke_start_prefix) + len( + self.invoke_start_prefix + ) + invoke_content_end = tool_text.find( + self.invoke_end_token, invoke_start + ) + if invoke_content_end != -1: + invoke_content = tool_text[invoke_start:invoke_content_end] + # Parse to get the complete arguments + try: + invoke_params = self._parse_invoke_params(invoke_content) + if invoke_params and self.current_tool_index < len( + self.prev_tool_call_arr + ): + # Update existing entry in prev_tool_call_arr + self.prev_tool_call_arr[self.current_tool_index][ + "arguments" + ] = json.dumps(invoke_params, ensure_ascii=False) + except Exception: + pass # Ignore parsing errors during streaming + + result = DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments="}"), + ) + ] + ) + + # Reset state for next tool + self.json_closed = True + self.in_function = False + self.accumulated_params = {} + + logger.debug("[M2_STREAMING] Tool call completed") + + return result + else: + # Don't close JSON yet, continue processing parameters + return None + + # Look for parameters + # Find all parameter starts + param_starts = [] + idx = 0 + while True: + idx = tool_text.find(self.parameter_prefix, idx) + if idx == -1: + break + param_starts.append(idx) + idx += len(self.parameter_prefix) + + # Check if we should start a new parameter + if ( + not self.in_param + and self.param_count < len(param_starts) + and len(param_starts) > self.param_count + ): + # Process the next parameter + param_idx = param_starts[self.param_count] + param_start = param_idx + len(self.parameter_prefix) + remaining = tool_text[param_start:] + + if ">" in remaining: + # We have the complete parameter name + name_end = remaining.find(">") + param_name_raw = remaining[:name_end] + self.current_param_name = self._extract_param_name(param_name_raw) + + # Find the parameter value + value_start = param_start + name_end + 1 + value_text = tool_text[value_start:] + if value_text.startswith("\n"): + value_text = value_text[1:] + + # Find where this parameter ends + param_end_idx = value_text.find(self.parameter_end_token) + if param_end_idx == -1: + # No closing tag, look for next parameter or function end + next_param_idx = value_text.find(self.parameter_prefix) + func_end_idx = value_text.find(self.invoke_end_token) + + if next_param_idx != -1 and ( + func_end_idx == -1 or next_param_idx < func_end_idx + ): + param_end_idx = next_param_idx + elif func_end_idx != -1: + param_end_idx = func_end_idx + else: + # Neither found, check if tool call is complete + if self.invoke_end_token in tool_text: + # Tool call and parameter is complete + param_end_idx = len(value_text) + else: + # Still streaming, wait for more content + return None + + if param_end_idx != -1: + # Complete parameter found + param_value = value_text[:param_end_idx] + if param_value.endswith("\n"): + param_value = param_value[:-1] + + # Store raw value for later processing + self.accumulated_params[self.current_param_name] = param_value + + # Get parameter configuration for type conversion + param_config = {} + if self.streaming_request and self.streaming_request.tools: + for tool in self.streaming_request.tools: + if ( + hasattr(tool, "function") + and tool.function.name == self.current_function_name + and hasattr(tool.function, "parameters") + ): + params = tool.function.parameters + if ( + isinstance(params, dict) + and "properties" in params + ): + param_config = params["properties"] + break + + # Get parameter type + param_type = "string" + if ( + self.current_param_name in param_config + and isinstance(param_config[self.current_param_name], dict) + and "type" in param_config[self.current_param_name] + ): + param_type = param_config[self.current_param_name]["type"] + + # Convert param value to appropriate type + converted_value = self._convert_param_value( + param_value, param_type + ) + + # Build JSON fragment based on the converted type + # Use json.dumps to properly serialize the value + serialized_value = json.dumps( + converted_value, ensure_ascii=False + ) + + if self.param_count == 0: + json_fragment = ( + f'"{self.current_param_name}": {serialized_value}' + ) + else: + json_fragment = ( + f', "{self.current_param_name}": {serialized_value}' + ) + + self.param_count += 1 + + return DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments=json_fragment), + ) + ] + ) + + return None From dfdda96747c4d06e96355ea96c7207c9f7dd3816 Mon Sep 17 00:00:00 2001 From: Arpit Khandelwal <60464796+arpitkh101@users.noreply.github.com> Date: Thu, 4 Dec 2025 04:15:04 -0500 Subject: [PATCH 38/47] [Core] Remove forced None assignment for deprecated PassConfig flags (#29994) Signed-off-by: arpitkh101 Co-authored-by: Cyrus Leung --- tests/compile/test_config.py | 21 +++++++++++++++------ vllm/config/compilation.py | 23 ++++++++++++----------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 9e912c6d810d2..8dd6959a01d09 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -392,39 +392,48 @@ def test_pass_config_deprecation(caplog_vllm): assert "enable_fusion is deprecated" in caplog_vllm.text assert config.fuse_norm_quant is True assert config.fuse_act_quant is True - assert config.enable_fusion is None + assert config.enable_fusion is True # Test enable_attn_fusion -> fuse_attn_quant caplog_vllm.clear() config = PassConfig(enable_attn_fusion=True) assert "enable_attn_fusion is deprecated" in caplog_vllm.text assert config.fuse_attn_quant is True - assert config.enable_attn_fusion is None + assert config.enable_attn_fusion is True # Test enable_noop -> eliminate_noops caplog_vllm.clear() config = PassConfig(enable_noop=True) assert "enable_noop is deprecated" in caplog_vllm.text assert config.eliminate_noops is True - assert config.enable_noop is None + assert config.enable_noop is True # Test enable_sequence_parallelism -> enable_sp caplog_vllm.clear() config = PassConfig(enable_sequence_parallelism=True) assert "enable_sequence_parallelism is deprecated" in caplog_vllm.text assert config.enable_sp is True - assert config.enable_sequence_parallelism is None + assert config.enable_sequence_parallelism is True # Test enable_async_tp -> fuse_gemm_comms caplog_vllm.clear() config = PassConfig(enable_async_tp=True) assert "enable_async_tp is deprecated" in caplog_vllm.text assert config.fuse_gemm_comms is True - assert config.enable_async_tp is None + assert config.enable_async_tp is True # Test enable_fi_allreduce_fusion -> fuse_allreduce_rms caplog_vllm.clear() config = PassConfig(enable_fi_allreduce_fusion=True) assert "enable_fi_allreduce_fusion is deprecated" in caplog_vllm.text assert config.fuse_allreduce_rms is True - assert config.enable_fi_allreduce_fusion is None + assert config.enable_fi_allreduce_fusion is True + + # Test hash consistency + config_old = PassConfig(enable_fusion=True) + config_new = PassConfig(fuse_norm_quant=True, fuse_act_quant=True) + assert config_old.compute_hash() == config_new.compute_hash() + + config_old = PassConfig(enable_async_tp=True) + config_new = PassConfig(fuse_gemm_comms=True) + assert config_old.compute_hash() == config_new.compute_hash() diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 963b091939e0e..d3d50e6ae7b2e 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -4,7 +4,7 @@ import enum from collections import Counter from collections.abc import Callable -from dataclasses import asdict, field +from dataclasses import field from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal @@ -13,7 +13,7 @@ from pydantic.dataclasses import dataclass import vllm.envs as envs from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass -from vllm.config.utils import config, handle_deprecated +from vllm.config.utils import config, get_hash_factors, handle_deprecated, hash_factors from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.import_utils import resolve_obj_by_qualname @@ -196,7 +196,16 @@ class PassConfig: Any new fields that affect compilation should be added to the hash. Any future fields that don't affect compilation should be excluded. """ - return InductorPass.hash_dict(asdict(self)) + + ignored_fields = [ + "enable_fusion", + "enable_attn_fusion", + "enable_noop", + "enable_sequence_parallelism", + "enable_async_tp", + "enable_fi_allreduce_fusion", + ] + return hash_factors(get_hash_factors(self, ignored_factors=ignored_fields)) @field_validator( "fuse_norm_quant", @@ -267,14 +276,6 @@ class PassConfig: "v0.13.0 or v1.0.0, whichever is sooner", ) - # Force old flags to None to ensure they are not used - self.enable_fusion = None - self.enable_attn_fusion = None - self.enable_noop = None - self.enable_sequence_parallelism = None - self.enable_async_tp = None - self.enable_fi_allreduce_fusion = None - if not self.eliminate_noops: if self.fuse_norm_quant or self.fuse_act_quant: logger.warning_once( From f2f4cea6ccaad20becb6f02e253ae673f8a249ae Mon Sep 17 00:00:00 2001 From: rasmith Date: Thu, 4 Dec 2025 03:30:22 -0600 Subject: [PATCH 39/47] [CI/Build][AMD] Skip test on test_hybrid_attention_mamba_tensor_shapes on ROCm, requires FLASHINFER (#29995) Signed-off-by: Randall Smith Co-authored-by: Randall Smith --- tests/v1/worker/test_gpu_model_runner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 0439bef1226e3..459abcfdd53cf 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -761,6 +761,10 @@ def test_init_kv_cache_with_kv_sharing_valid(): assert kv_cache_config_after_init.kv_cache_groups[0].layer_names[1] == layer_1 +@pytest.mark.skipif( + current_platform.is_rocm(), + reason="Attention backend FLASHINFER is not supported on ROCm.", +) def test_hybrid_attention_mamba_tensor_shapes(monkeypatch): """ The GPU model runner creates different views into the From 842aba501d92ac77874d45612b7e3c6fed2ca243 Mon Sep 17 00:00:00 2001 From: dtc <790567447@qq.com> Date: Thu, 4 Dec 2025 17:51:36 +0800 Subject: [PATCH 40/47] [P/D] Introduce Mooncake Transfer Engine as kv_connector (#24718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tianchen Ding Signed-off-by: dtc Co-authored-by: Nicolò Lucchesi --- docs/features/mooncake_connector_usage.md | 58 ++ .../kv_transfer/kv_connector/factory.py | 5 + .../kv_transfer/kv_connector/utils.py | 124 +++ .../kv_connector/v1/mooncake_connector.py | 914 ++++++++++++++++++ .../kv_connector/v1/nixl_connector.py | 128 +-- vllm/envs.py | 10 + 6 files changed, 1114 insertions(+), 125 deletions(-) create mode 100644 docs/features/mooncake_connector_usage.md create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/mooncake_connector.py diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md new file mode 100644 index 0000000000000..653ea29ad9433 --- /dev/null +++ b/docs/features/mooncake_connector_usage.md @@ -0,0 +1,58 @@ +# MooncakeConnector Usage Guide + +## About Mooncake + +Mooncake aims to enhance the inference efficiency of large language models (LLMs), especially in slow object storage environments, by constructing a multi-level caching pool on high-speed interconnected DRAM/SSD resources. Compared to traditional caching systems, Mooncake utilizes (GPUDirect) RDMA technology to transfer data directly in a zero-copy manner, while maximizing the use of multi-NIC resources on a single machine. + +For more details about Mooncake, please refer to [Mooncake project](https://github.com/kvcache-ai/Mooncake) and [Mooncake documents](https://kvcache-ai.github.io/Mooncake/). + +## Prerequisites + +### Installation + +Install mooncake through pip: `uv pip install mooncake-transfer-engine`. + +Refer to [Mooncake official repository](https://github.com/kvcache-ai/Mooncake) for more installation instructions + +## Usage + +### Prefiller Node (192.168.0.2) + +```bash +vllm serve Qwen/Qwen2.5-7B-Instruct --port 8010 --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_producer"}' +``` + +### Decoder Node (192.168.0.3) + +```bash +vllm serve Qwen/Qwen2.5-7B-Instruct --port 8020 --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_consumer"}' +``` + +### Proxy + +```bash +python tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --prefiller-host 192.168.0.2 --prefiller-port 8010 --decoder-host 192.168.0.3 --decoder-port 8020 +``` + +> NOTE: The Mooncake Connector currently uses the proxy from nixl_integration. This will be replaced with a self-developed proxy in the future. + +Now you can send requests to the proxy server through port 8000. + +## Environment Variables + +- `VLLM_MOONCAKE_BOOTSTRAP_PORT`: Port for Mooncake bootstrap server + - Default: 8998 + - Required only for prefiller instances + - Each vLLM worker needs a unique port on its host; using the same port number across different hosts is fine + - For TP/DP deployments, each worker's port on a node is computed as: base_port + dp_rank * tp_size + tp_rank + - Used for the decoder notifying the prefiller + +- `VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT`: Timeout (in seconds) for automatically releasing the prefiller’s KV cache for a particular request. (Optional) + - Default: 480 + - If a request is aborted and the decoder has not yet notified the prefiller, the prefill instance will release its KV-cache blocks after this timeout to avoid holding them indefinitely. + +## KV Role Options + +- **kv_producer**: For prefiller instances that generate KV caches +- **kv_consumer**: For decoder instances that consume KV caches from prefiller +- **kv_both**: Enables symmetric functionality where the connector can act as both producer and consumer. This provides flexibility for experimental setups and scenarios where the role distinction is not predetermined. diff --git a/vllm/distributed/kv_transfer/kv_connector/factory.py b/vllm/distributed/kv_transfer/kv_connector/factory.py index df871dd7cbe4f..02f51a1dce112 100644 --- a/vllm/distributed/kv_transfer/kv_connector/factory.py +++ b/vllm/distributed/kv_transfer/kv_connector/factory.py @@ -190,3 +190,8 @@ KVConnectorFactory.register_connector( "vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector", "DecodeBenchConnector", ) +KVConnectorFactory.register_connector( + "MooncakeConnector", + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake_connector", + "MooncakeConnector", +) diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index b2c2c0e6b596d..99d3be57c1381 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -4,10 +4,13 @@ KV cache helper for store. """ +from dataclasses import dataclass from typing import TYPE_CHECKING, Literal import torch +from vllm.attention.backends.abstract import AttentionBackend +from vllm.attention.backends.registry import AttentionBackendEnum from vllm.config import get_current_vllm_config from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.logger import init_logger @@ -181,3 +184,124 @@ def copy_kv_blocks( src_tensor = src_kv_caches[layer_name] dst_tensor = dst_kv_caches[layer_name] copy_fn(src_tensor, dst_tensor, src_indices, dst_indices) + + +@dataclass +class TpKVTopology: + """ + Helper class for tensor parallel and KV topology information for + mapping between local and remote TP workers. + """ + + tp_rank: int + remote_tp_size: dict[str, int] + is_mla: bool + total_num_kv_heads: int + attn_backend: type[AttentionBackend] + engine_id: str + remote_block_size: dict[str, int] + + def __post_init__(self): + # Figure out whether the first dimension of the cache is K/V + # or num_blocks. This is used to register the memory regions correctly. + kv_cache_shape = self.attn_backend.get_kv_cache_shape( + num_blocks=1, block_size=16, num_kv_heads=1, head_size=1 + ) + # Non-MLA backends caches have 5 dims [2, num_blocks, H,N,D], + # we just mock num_blocks to 1 for the dimension check below. + self._is_kv_layout_blocks_first = ( + len(kv_cache_shape) == 5 and kv_cache_shape[0] == 1 + ) + + attn_backend = AttentionBackendEnum[self.attn_backend.get_name()] + self._use_pallas = attn_backend == AttentionBackendEnum.PALLAS + + @property + def is_kv_layout_blocks_first(self) -> bool: + return self._is_kv_layout_blocks_first + + @property + def split_k_and_v(self) -> bool: + # Whether to register regions for K and V separately (when present). + return not (self.is_mla or self._use_pallas or self.is_kv_layout_blocks_first) + + @property + def tp_size(self) -> int: + return self.remote_tp_size[self.engine_id] + + @property + def block_size(self) -> int: + return self.remote_block_size[self.engine_id] + + def tp_ratio( + self, + remote_tp_size: int, + ) -> int: + """ + Calculate the tensor parallel ratio between local and remote TP. + We can think of it as the number of local TP workers-per-remote TP + workers. Local workers will read from the same remote TP worker in + groups of size `tp_ratio`. + """ + assert self.tp_size % remote_tp_size == 0, ( + f"Local tensor parallel size {self.tp_size} is not divisible " + f"by remote tensor parallel size {remote_tp_size}." + ) + return self.tp_size // remote_tp_size + + def block_size_ratio( + self, + remote_block_size: int, + ) -> float: + """ + Calculate the block size ratio between local and remote TP. + """ + assert self.block_size % remote_block_size == 0, ( + f"Local block size {self.block_size} is not divisible " + f"by remote block size {remote_block_size} or vice versa." + ) + return self.block_size // remote_block_size + + def tp_ratio_from_engine_id( + self, + remote_engine_id: str, + ) -> int: + remote_tp_size = self.remote_tp_size[remote_engine_id] + return self.tp_ratio(remote_tp_size) + + def block_size_ratio_from_engine_id( + self, + remote_engine_id: str, + ) -> float: + remote_block_size = self.remote_block_size[remote_engine_id] + return self.block_size_ratio(remote_block_size) + + def is_kv_replicated(self, engine_id: str) -> bool: + """ + Whether the KV cache is replicated across TP workers due to the + number of TP workers being greater than the number of KV heads. + """ + tp_size = self.remote_tp_size[engine_id] + return tp_size // self.total_num_kv_heads >= 1 + + def replicates_kv_cache(self, remote_engine_id: str) -> bool: + # MLA is always replicated as the hidden dim can't be split. + return self.is_mla or self.is_kv_replicated(remote_engine_id) + + def get_target_remote_rank( + self, + remote_tp_size: int, + ) -> int: + """ + Get the remote TP rank (on P) that the current local TP rank + (on D) will read from. + """ + tp_ratio = self.tp_ratio(remote_tp_size) + return self.tp_rank // tp_ratio + + def get_target_remote_rank_from_engine_id( + self, + remote_engine_id: str, + ) -> int: + remote_tp_size = self.remote_tp_size[remote_engine_id] + return self.get_target_remote_rank(remote_tp_size) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake_connector.py new file mode 100644 index 0000000000000..705960aebe2da --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake_connector.py @@ -0,0 +1,914 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +import threading +import time +import uuid +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional + +import msgspec +import numpy as np +import torch +import zmq +import zmq.asyncio + +from vllm import envs +from vllm.attention.backends.abstract import AttentionMetadata +from vllm.attention.selector import get_attn_backend +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.utils import TpKVTopology +from vllm.distributed.kv_transfer.kv_connector.v1.base import ( + KVConnectorBase_V1, + KVConnectorMetadata, + KVConnectorRole, +) +from vllm.distributed.parallel_state import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + get_tp_group, +) +from vllm.forward_context import ForwardContext +from vllm.logger import init_logger +from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket +from vllm.v1.attention.backends.utils import get_kv_cache_layout +from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.request import RequestStatus + +try: + from mooncake.engine import TransferEngine +except ImportError as e: + raise ImportError( + "Please install mooncake by following the instructions at " + "https://github.com/kvcache-ai/Mooncake/blob/main/doc/en/build.md " # noqa: E501 + "to run VLLM with MooncakeTransferEngine." + ) from e + +if TYPE_CHECKING: + from vllm.v1.core.kv_cache_manager import KVCacheBlocks + from vllm.v1.kv_cache_interface import KVCacheConfig + from vllm.v1.request import Request + +EngineId = str +ReqId = str + +TRANS_DONE = b"trans_done" +TRANS_ERROR = b"trans_error" + +logger = init_logger(__name__) + + +class MooncakeAgentMetadata( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + # required for @cached_property. + dict=True, +): + remote_hostname: str + remote_port: int + request_ids: list[ReqId] + kv_caches_base_addr: list[int] + block_ids: list[list[int]] + + +@dataclass +class RecvReqMeta: + local_block_ids: list[int] + remote_host: str + remote_port: int + + +@dataclass +class SendBlockMeta: + local_block_ids: list[int] + ready: threading.Event + expire_time: float = float("inf") + + +@dataclass +class SendReqMeta: + reqs: dict[ReqId, SendBlockMeta] + lock: threading.Lock + + +@dataclass +class FinishedSendReqSet: + set: set[ReqId] + lock: threading.Lock + + +@dataclass +class FinishedReceiveReqSet: + set: set[ReqId] + lock: asyncio.Lock + + +class MooncakeConnectorMetadata(KVConnectorMetadata): + def __init__(self): + self.reqs_to_recv: dict[ReqId, RecvReqMeta] = {} + self.reqs_to_send: dict[ReqId, list[int]] = {} + + def add_new_req( + self, + request_id: ReqId, + local_block_ids: list[int], + kv_transfer_params: dict[str, Any], + load_remote_cache: bool = True, + ): + if load_remote_cache: + self.reqs_to_recv[request_id] = RecvReqMeta( + local_block_ids=local_block_ids, + remote_host=kv_transfer_params["remote_host"], + remote_port=kv_transfer_params["remote_port"], + ) + else: + self.reqs_to_send[request_id] = local_block_ids + + +class MooncakeConnector(KVConnectorBase_V1): + def __init__( + self, + vllm_config: VllmConfig, + role: KVConnectorRole, + kv_cache_config: Optional["KVCacheConfig"] = None, + ): + super().__init__(vllm_config, role, kv_cache_config) + + assert vllm_config.kv_transfer_config is not None + assert vllm_config.kv_transfer_config.engine_id is not None + self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id + + if role == KVConnectorRole.SCHEDULER: + self.connector_scheduler: MooncakeConnectorScheduler | None = ( + MooncakeConnectorScheduler(vllm_config, self.engine_id) + ) + self.connector_worker: MooncakeConnectorWorker | None = None + elif role == KVConnectorRole.WORKER: + self.connector_scheduler = None + self.connector_worker = MooncakeConnectorWorker(vllm_config, self.engine_id) + + ############################################################ + # Scheduler Side Methods + ############################################################ + + def get_num_new_matched_tokens( + self, request: "Request", num_computed_tokens: int + ) -> tuple[int, bool]: + assert self.connector_scheduler is not None + return self.connector_scheduler.get_num_new_matched_tokens( + request, num_computed_tokens + ) + + def update_state_after_alloc( + self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int + ): + assert self.connector_scheduler is not None + return self.connector_scheduler.update_state_after_alloc( + request, blocks, num_external_tokens + ) + + def build_connector_meta( + self, + scheduler_output: SchedulerOutput, + ) -> KVConnectorMetadata: + assert self.connector_scheduler is not None + return self.connector_scheduler.build_connector_meta(scheduler_output) + + def request_finished( + self, + request: "Request", + block_ids: list[int], + ) -> tuple[bool, dict[str, Any] | None]: + assert self.connector_scheduler is not None + return self.connector_scheduler.request_finished(request, block_ids) + + ############################################################ + # Worker Side Methods + ############################################################ + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): + assert self.connector_worker is not None + self.connector_worker.register_kv_caches(kv_caches) + + def get_finished( + self, finished_req_ids: set[str] + ) -> tuple[set[str] | None, set[str] | None]: + """Get the finished recving and sending requests.""" + assert self.connector_worker is not None + return self.connector_worker.get_finished() + + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: + assert self.connector_worker is not None + assert isinstance(self._connector_metadata, MooncakeConnectorMetadata) + self.connector_worker.start_load_kv(self._connector_metadata) + + def wait_for_layer_load(self, layer_name: str) -> None: + """MooncakeConnector does not do layerwise saving.""" + pass + + def save_kv_layer( + self, + layer_name: str, + kv_layer: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> None: + """MooncakeConnector does not save explicitly.""" + pass + + def wait_for_save(self): + pass + + +class MooncakeConnectorScheduler: + """Implementation of Scheduler side methods""" + + def __init__(self, vllm_config: VllmConfig, engine_id: str): + self.vllm_config = vllm_config + self.engine_id: EngineId = engine_id + self.side_channel_host = get_ip() + self.side_channel_port = get_mooncake_side_channel_port(vllm_config) + + assert vllm_config.kv_transfer_config + self.kv_role = vllm_config.kv_transfer_config.kv_role + logger.info("Initializing Mooncake Transfer Engine Scheduler %s", engine_id) + + # Requests that need to start recv/send. + # New requests are added by update_state_after_alloc in + # the scheduler. Used to make metadata passed to Worker. + self._reqs_need_recv: dict[ReqId, tuple[Request, list[int]]] = {} + self._reqs_need_send: dict[ReqId, list[int]] = {} + + def get_num_new_matched_tokens( + self, request: "Request", num_computed_tokens: int + ) -> tuple[int, bool]: + """ + For remote prefill, pull all prompt blocks from remote + asynchronously relative to engine execution. + + Args: + request (Request): the request object. + num_computed_tokens (int): the number of locally + computed tokens for this request + Returns: + * the number of tokens that can be loaded from the + external KV cache beyond what is already computed. + * true if the external KV cache tokens will be loaded + asynchronously (between scheduler steps). + """ + + params = request.kv_transfer_params + logger.debug( + "MooncakeConnector get_num_new_matched_tokens: " + "num_computed_tokens=%s, kv_transfer_params=%s", + num_computed_tokens, + params, + ) + + if params is not None and params.get("do_remote_prefill"): + # Remote prefill: get all prompt blocks from remote. + token_ids = request.prompt_token_ids or [] + count = len(token_ids) - num_computed_tokens + if count > 0: + return count, True + + # No remote prefill for this request. + return 0, False + + def update_state_after_alloc( + self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int + ): + params = request.kv_transfer_params + logger.debug( + "MooncakeConnector update_state_after_alloc: " + "num_external_tokens=%s, kv_transfer_params=%s", + num_external_tokens, + params, + ) + + if not params: + return + + if params.get("do_remote_prefill"): + assert self.kv_role != "kv_producer" + if all(p in params for p in ("remote_host", "remote_port")): + # If remote_blocks and num_external_tokens = 0, we have + # a full prefix cache hit on the D worker. We need to call + # send_notif in _read_blocks to free the memory on the P. + local_block_ids = ( + blocks.get_unhashed_block_ids() if num_external_tokens > 0 else [] + ) + # Get unhashed blocks to pull from remote. + self._reqs_need_recv[request.request_id] = (request, local_block_ids) + else: + logger.warning( + "Got invalid KVTransferParams: %s. This " + "request will not utilize KVTransfer", + params, + ) + # Only trigger 1 KV transfer per request. + params["do_remote_prefill"] = False + + elif params.get("do_remote_decode"): + # Add an empty list to worker to create event. + self._reqs_need_send[request.request_id] = [] + + def build_connector_meta( + self, + scheduler_output: SchedulerOutput, + ) -> KVConnectorMetadata: + meta = MooncakeConnectorMetadata() + + # Loop through scheduled reqs and convert to RecvReqMeta. + if self.kv_role != "kv_producer": + for req_id, (req, block_ids) in self._reqs_need_recv.items(): + assert req.kv_transfer_params is not None + meta.add_new_req( + request_id=req_id, + local_block_ids=block_ids, + kv_transfer_params=req.kv_transfer_params, + ) + self._reqs_need_recv.clear() + + if self.kv_role != "kv_consumer": + for req_id, block_ids in self._reqs_need_send.items(): + meta.add_new_req( + request_id=req_id, + local_block_ids=block_ids, + kv_transfer_params={}, + load_remote_cache=False, + ) + self._reqs_need_send.clear() + + return meta + + def request_finished( + self, + request: "Request", + block_ids: list[int], + ) -> tuple[bool, dict[str, Any] | None]: + """ + Once a request is finished, determine whether request blocks + should be freed now or will be sent asynchronously and freed later. + """ + + params = request.kv_transfer_params + logger.debug( + "MooncakeConnector request_finished, request_status=%s, " + "kv_transfer_params=%s", + request.status, + params, + ) + if not params: + return False, None + + if params.get("do_remote_prefill"): + # If do_remote_prefill is still True when the request is finished, + # update_state_after_alloc must not have been called (the request + # must have been aborted before it was scheduled). + # To avoid stranding the prefill blocks in the prefill instance, + # we must add empty block_ids to _reqs_need_recv so that our + # worker side will notify and free blocks in the prefill instance. + assert self.kv_role != "kv_producer" + self._reqs_need_recv[request.request_id] = (request, []) + params["do_remote_prefill"] = False + return False, None + + if ( + not params.get("do_remote_decode") + or request.status != RequestStatus.FINISHED_LENGTH_CAPPED + ): + return False, None + + assert self.kv_role != "kv_consumer" + + # TODO: check whether block_ids actually ever be 0. If not we could + # remove the conditional below + delay_free_blocks = len(block_ids) > 0 + + if delay_free_blocks: + self._reqs_need_send[request.request_id] = block_ids + + return delay_free_blocks, dict( + do_remote_prefill=True, + do_remote_decode=False, + remote_host=self.side_channel_host, + remote_port=self.side_channel_port, + ) + + +class MooncakeConnectorWorker: + """Implementation of Worker side methods""" + + def __init__(self, vllm_config: VllmConfig, engine_id: str): + logger.info("Initializing Mooncake Transfer Engine worker %s", engine_id) + + self.vllm_config = vllm_config + + self.engine = TransferEngine() + self.hostname = get_ip() + ret_value = self.engine.initialize(self.hostname, "P2PHANDSHAKE", "rdma", "") + if ret_value != 0: + raise RuntimeError("Mooncake Transfer Engine initialization failed.") + + self.rpc_port = self.engine.get_rpc_port() + + logger.debug( + "Mooncake Transfer Engine initialized at %s:%d", + self.hostname, + self.rpc_port, + ) + + # Mooncake handshake port. + self.side_channel_port: int = get_mooncake_side_channel_port(vllm_config) + + self.engine_id: EngineId = engine_id + self.tp_rank = get_tensor_model_parallel_rank() + self.world_size = get_tensor_model_parallel_world_size() + self.tp_group = get_tp_group() + self.num_blocks = 0 + + assert vllm_config.kv_transfer_config + self.kv_role = vllm_config.kv_transfer_config.kv_role + self.num_workers = vllm_config.kv_transfer_config.kv_connector_extra_config.get( + "num_workers", 10 + ) + + self.kv_caches_base_addr: list[int] = [] + self.device_kv_caches: dict[str, torch.Tensor] = {} + self.reqs_need_send: SendReqMeta = SendReqMeta(reqs={}, lock=threading.Lock()) + + # For kv_both, we will act both prefiller and decoder. + if self.kv_role != "kv_consumer": + # Background thread for sending kvcaches to D. + self._mooncake_sender_t: threading.Thread | None = None + # Background thread for processing new sending requests. + self._sender_executor = ThreadPoolExecutor( + max_workers=self.num_workers, thread_name_prefix="vllm-mooncake-sender" + ) + logger.debug( + "Mooncake Prefiller: use %d workers to send kvcaches", self.num_workers + ) + if self.kv_role != "kv_producer": + self.receiver_loop = asyncio.new_event_loop() + self._mooncake_receiver_t = threading.Thread( + target=self._receiver_loop, args=(self.receiver_loop,), daemon=True + ) + self._mooncake_receiver_t.start() + logger.debug("Mooncake Decoder: start receiver thread") + + self.finished_sending_reqs: FinishedSendReqSet = FinishedSendReqSet( + set(), threading.Lock() + ) + self.finished_recving_reqs: FinishedReceiveReqSet = FinishedReceiveReqSet( + set(), asyncio.Lock() + ) + + self.block_size = vllm_config.cache_config.block_size + self.model_config = vllm_config.model_config + self.cache_config = vllm_config.cache_config + self.use_mla = self.model_config.use_mla + + backend = get_attn_backend( + self.model_config.get_head_size(), + self.model_config.dtype, + self.cache_config.cache_dtype, + self.block_size, + use_mla=self.use_mla, + ) + self.backend_name = backend.get_name() + self.kv_cache_layout = get_kv_cache_layout() + logger.debug("Detected attention backend %s", self.backend_name) + logger.debug("Detected kv cache layout %s", self.kv_cache_layout) + + self._tp_size: dict[EngineId, int] = {self.engine_id: self.world_size} + self._block_size: dict[EngineId, int] = {self.engine_id: self.block_size} + self.kv_topo = TpKVTopology( + tp_rank=self.tp_rank, + engine_id=self.engine_id, + remote_tp_size=self._tp_size, # shared state + remote_block_size=self._block_size, # shared state + is_mla=self.use_mla, + total_num_kv_heads=self.model_config.get_total_num_kv_heads(), + attn_backend=backend, + ) + self._use_pallas = self.kv_topo._use_pallas + + self.zmq_ctx = zmq.Context() + self.async_zmq_ctx = zmq.asyncio.Context() + self._encoder = msgspec.msgpack.Encoder() + self._decoder = msgspec.msgpack.Decoder(MooncakeAgentMetadata) + + def __del__(self): + self.shutdown() + + def shutdown(self): + """Cleanup background threads on destruction.""" + self.zmq_ctx.term() + self.async_zmq_ctx.term() + if self.kv_role != "kv_consumer": + self._sender_executor.shutdown(wait=False) + if self._mooncake_sender_t: + self._mooncake_sender_t.join() + if self.kv_role != "kv_producer" and self.receiver_loop.is_running(): + self.receiver_loop.call_soon_threadsafe(self.receiver_loop.stop) + self._mooncake_receiver_t.join() + + def _receiver_loop(self, loop: asyncio.AbstractEventLoop): + asyncio.set_event_loop(loop) + loop.run_forever() + + def _mooncake_sender( + self, ready_event: threading.Event, base_port: int, tp_rank: int + ): + """ + Background thread that listens for Mooncake requests, dispatches them + to a thread pool, and sends acknowledgments upon completion. + """ + + frontend_path = make_zmq_path("tcp", self.hostname, base_port + tp_rank) + frontend = make_zmq_socket(self.zmq_ctx, frontend_path, zmq.ROUTER) + logger.debug("Mooncake sender starting listening on path: %s", frontend_path) + + backend_path = make_zmq_path("inproc", str(uuid.uuid4())) + backend = make_zmq_socket(self.zmq_ctx, backend_path, zmq.PULL) + + poller = zmq.Poller() + poller.register(frontend, zmq.POLLIN) + poller.register(backend, zmq.POLLIN) + + ready_event.set() + + try: + while True: + sockets = dict(poller.poll()) + + if frontend in sockets: + identity, _, metadata_bytes = frontend.recv_multipart() + self._sender_executor.submit( + self._sender_worker, + identity, + metadata_bytes, + backend_path, + ) + + if backend in sockets: + identity, status = backend.recv_multipart() + frontend.send_multipart((identity, b"", status)) + + except zmq.ContextTerminated: + logger.debug("ZMQ context terminated, exiting Mooncake sender thread.") + except Exception as e: + logger.error("Error in Mooncake sender thread: %s. Exiting thread.", str(e)) + finally: + frontend.close() + backend.close() + + def _sender_worker( + self, identity: bytes, metadata_bytes: bytes, worker_channel_path: str + ): + status = TRANS_ERROR + + try: + metadata = self._decoder.decode(metadata_bytes) + self.send_kv_to_decode(metadata) + status = TRANS_DONE + except Exception as e: + logger.error("Error processing Mooncake handshake: %s", e) + finally: + pusher = make_zmq_socket(self.zmq_ctx, worker_channel_path, zmq.PUSH) + try: + pusher.send_multipart((identity, status)) + except zmq.ZMQError as e: + logger.warning( + "Internal error, maybe the server is shutting down. Error: %s", + e, + ) + finally: + pusher.close() + + def send_kv_to_decode(self, meta: MooncakeAgentMetadata): + send_reqs: list[tuple[ReqId, SendBlockMeta]] = [] + with self.reqs_need_send.lock: + for req_id in meta.request_ids: + send_meta = self.reqs_need_send.reqs.get(req_id) + if send_meta is None: + logger.warning("Request %s not found in reqs_need_send", req_id) + return + # Mark it as not expired. We will send it now. + send_meta.expire_time = float("inf") + send_reqs.append((req_id, send_meta)) + + self._send_blocks(send_reqs, meta) + + with self.reqs_need_send.lock: + for req_id in meta.request_ids: + del self.reqs_need_send.reqs[req_id] + + with self.finished_sending_reqs.lock: + self.finished_sending_reqs.set.update(meta.request_ids) + + def _send_blocks( + self, + send_reqs: list[tuple[ReqId, SendBlockMeta]], + agent_meta: MooncakeAgentMetadata, + ): + src_ptrs = [] + dst_ptrs = [] + lengths = [] + local_base_addr = self.kv_caches_base_addr + remote_base_addr = agent_meta.kv_caches_base_addr + block_len = self.block_len + remote_session = f"{agent_meta.remote_hostname}:{agent_meta.remote_port}" + + assert len(send_reqs) == len(agent_meta.block_ids) + for (req_id, send_meta), remote_block_ids in zip( + send_reqs, agent_meta.block_ids + ): + send_meta.ready.wait() + + num_remote_blocks = len(remote_block_ids) + if num_remote_blocks == 0: + continue + + local_block_ids = send_meta.local_block_ids + # Partial prefix cache hit: just read uncomputed blocks. + num_local_blocks = len(local_block_ids) + assert num_local_blocks >= num_remote_blocks + if num_local_blocks > num_remote_blocks: + local_block_ids = local_block_ids[-num_remote_blocks:] + + # Group by indices + group_local_block_ids, group_remote_block_ids = group_concurrent_contiguous( + local_block_ids, remote_block_ids + ) + + for local_layer_addr, remote_layer_addr in zip( + local_base_addr, remote_base_addr + ): + for group_local_block_id, group_remote_block_id in zip( + group_local_block_ids, group_remote_block_ids + ): + src_ptrs.append( + local_layer_addr + group_local_block_id[0] * block_len + ) + dst_ptrs.append( + remote_layer_addr + group_remote_block_id[0] * block_len + ) + lengths.append(block_len * len(group_local_block_id)) + + logger.debug( + "Sending kv_caches for request %s (%d blocks) to %s", + req_id, + num_remote_blocks, + remote_session, + ) + + start_time = time.perf_counter() + ret_value = self.engine.batch_transfer_sync_write( + remote_session, src_ptrs, dst_ptrs, lengths + ) + if ret_value != 0: + raise RuntimeError(f"Error in batch_transfer_sync_write: {ret_value}") + + logger.debug( + "Sending to %s done, took %s", + remote_session, + time.perf_counter() - start_time, + ) + + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): + """Register the KV Cache data in mooncake.""" + + logger.info("Registering KV_Caches. use_mla: %s", self.use_mla) + + kv_data_ptrs = [] + kv_data_lens = [] + seen_base_addresses = [] + + split_k_and_v = self.kv_topo.split_k_and_v + tensor_size_bytes = None + for layer_name, cache_or_caches in kv_caches.items(): + logger.debug( + "registering layer %s with shape %s", layer_name, cache_or_caches.shape + ) + cache_list = cache_or_caches if split_k_and_v else [cache_or_caches] + + for cache in cache_list: + base_addr = cache.data_ptr() + if base_addr in seen_base_addresses: + continue + + seen_base_addresses.append(base_addr) + curr_tensor_size_bytes = cache.nbytes + + if tensor_size_bytes is None: + tensor_size_bytes = curr_tensor_size_bytes + self.num_blocks = cache.shape[0] + + assert tensor_size_bytes == curr_tensor_size_bytes, ( + "All kv cache tensors must have the same size" + ) + kernel_block_size = cache.shape[-2 if self.use_mla else -3] + assert self.block_size == kernel_block_size + kv_data_ptrs.append(base_addr) + kv_data_lens.append(tensor_size_bytes) + + self.kv_caches_base_addr = seen_base_addresses + + ret_value = self.engine.batch_register_memory(kv_data_ptrs, kv_data_lens) + if ret_value != 0: + raise RuntimeError("Mooncake batch memory registration failed.") + + assert tensor_size_bytes is not None + assert self.num_blocks != 0 + assert tensor_size_bytes % self.num_blocks == 0 + self.block_len = tensor_size_bytes // self.num_blocks + self.device_kv_caches = kv_caches + logger.debug( + "registered num_blocks=%d block_len=%d", self.num_blocks, self.block_len + ) + + # No need to launch server for D node. + if self.kv_role == "kv_consumer": + return + + ready_event = threading.Event() + self._mooncake_sender_t = threading.Thread( + target=self._mooncake_sender, + args=(ready_event, self.side_channel_port, self.tp_rank), + daemon=True, + name="mooncake_sender", + ) + self._mooncake_sender_t.start() + ready_event.wait() # Wait for listener ZMQ socket to be ready. + + async def fetch_finished_recving_reqs(self) -> set[ReqId]: + async with self.finished_recving_reqs.lock: + finished_recving_reqs = self.finished_recving_reqs.set + self.finished_recving_reqs.set = set() + return finished_recving_reqs + + def get_finished(self) -> tuple[set[str] | None, set[str] | None]: + """ + Get requests that are done sending or recving on this specific worker. + The scheduler process (via the MultiprocExecutor) will use this output + to track which workers are done. + """ + fut = None + if self.kv_role != "kv_producer": + fut = asyncio.run_coroutine_threadsafe( + self.fetch_finished_recving_reqs(), self.receiver_loop + ) + + if self.kv_role != "kv_consumer": + with self.finished_sending_reqs.lock: + finished_sending_reqs = self.finished_sending_reqs.set + self.finished_sending_reqs.set = set() + else: + finished_sending_reqs = set() + + finished_recving_reqs = fut.result() if fut else set() + + if finished_sending_reqs or finished_recving_reqs: + logger.debug( + "Rank %s, get_finished: %s requests done sending " + "and %s requests done recving", + self.tp_rank, + len(finished_sending_reqs), + len(finished_recving_reqs), + ) + + # Handle timeout to avoid stranding blocks on remote. + now = time.perf_counter() + with self.reqs_need_send.lock: + expired_reqs = [ + req_id + for req_id, send_meta in self.reqs_need_send.reqs.items() + if send_meta.expire_time < now + ] + for req_id in expired_reqs: + logger.warning( + "Request %s timed out after %d seconds without " + "being sent. Freeing its blocks on the producer side.", + req_id, + envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT, + ) + del self.reqs_need_send.reqs[req_id] + if expired_reqs: + finished_sending_reqs.update(expired_reqs) + + return finished_sending_reqs or None, finished_recving_reqs or None + + async def receive_kv(self, path: str, req_blocks: list[tuple[str, list[int]]]): + req_ids, block_ids = map(list, zip(*req_blocks)) + metadata = MooncakeAgentMetadata( + remote_hostname=self.hostname, + remote_port=self.rpc_port, + request_ids=req_ids, + kv_caches_base_addr=self.kv_caches_base_addr, + block_ids=block_ids, + ) + + encoded_data = self._encoder.encode(metadata) + logger.debug( + "Size of encoded MooncakeAgentMetadata: %d bytes", len(encoded_data) + ) + logger.debug("Sending kv transfer request for %s on path: %s", req_ids, path) + + # Send query for the request. + sock: zmq.asyncio.Socket = make_zmq_socket( + self.async_zmq_ctx, path, zmq.REQ, bind=False, linger=0 + ) + sock.setsockopt(zmq.RCVTIMEO, 60000) + try: + await sock.send(encoded_data) + ret_msg = await sock.recv() + if ret_msg != TRANS_DONE: + logger.error( + "Error happens during tranfering kvcache for %s, see logs in prefiller.", # noqa: E501 + req_ids, + ) + return + except zmq.ContextTerminated: + logger.debug("ZMQ context terminated, exiting Mooncake receiver thread.") + except Exception as e: + logger.error("MooncakeAgentMetadata transfer failed for %s: %s", req_ids, e) + return + finally: + sock.close() + + async with self.finished_recving_reqs.lock: + self.finished_recving_reqs.set.update(req_ids) + + logger.debug("pulling kv_caches for %s finished", req_ids) + + def group_kv_pull(self, metadata: MooncakeConnectorMetadata): + kv_pulls = defaultdict(list) + for req_id, meta in metadata.reqs_to_recv.items(): + logger.debug( + "start_load_kv for request %s from remote engine. " + "Num local_block_ids: %s.", + req_id, + len(meta.local_block_ids), + ) + path = make_zmq_path( + "tcp", meta.remote_host, meta.remote_port + self.tp_rank + ) + kv_pulls[path].append((req_id, meta.local_block_ids)) + + return kv_pulls + + def start_load_kv(self, metadata: MooncakeConnectorMetadata): + if self.kv_role != "kv_producer": + kv_pulls = self.group_kv_pull(metadata) + for path, req_blocks in kv_pulls.items(): + asyncio.run_coroutine_threadsafe( + self.receive_kv(path, req_blocks), self.receiver_loop + ) + + if self.kv_role != "kv_consumer": + with self.reqs_need_send.lock: + for req_id, block_ids in metadata.reqs_to_send.items(): + if block_ids: + # Already gone through request_finished() + send_meta = self.reqs_need_send.reqs[req_id] + send_meta.local_block_ids = block_ids + send_meta.ready.set() + send_meta.expire_time = ( + time.perf_counter() + + envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT + ) + else: + # From update_state_after_alloc(), + # but not reach request_finished() yet + self.reqs_need_send.reqs[req_id] = SendBlockMeta( + local_block_ids=[], ready=threading.Event() + ) + + +def group_concurrent_contiguous( + src_indices: list[int], dst_indices: list[int] +) -> tuple[list[list[int]], list[list[int]]]: + """Vectorised NumPy implementation.""" + if len(src_indices) == 0: + return [], [] + + brk = np.where((np.diff(src_indices) != 1) | (np.diff(dst_indices) != 1))[0] + 1 + src_groups = np.split(src_indices, brk) + dst_groups = np.split(dst_indices, brk) + + src_groups = [g.tolist() for g in src_groups] + dst_groups = [g.tolist() for g in dst_groups] + + return src_groups, dst_groups + + +def get_mooncake_side_channel_port(vllm_config: VllmConfig) -> int: + # This logic is now centralized + return ( + envs.VLLM_MOONCAKE_BOOTSTRAP_PORT + + vllm_config.parallel_config.data_parallel_rank + * vllm_config.parallel_config.tensor_parallel_size + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py index 41e32bb73d40b..24b7599a4fe0c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py @@ -20,10 +20,10 @@ import torch import zmq from vllm import envs -from vllm.attention.backends.abstract import AttentionBackend, AttentionMetadata -from vllm.attention.backends.registry import AttentionBackendEnum +from vllm.attention.backends.abstract import AttentionMetadata from vllm.attention.selector import get_attn_backend from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.utils import TpKVTopology from vllm.distributed.kv_transfer.kv_connector.v1.base import ( CopyBlocksOp, KVConnectorBase_V1, @@ -668,128 +668,6 @@ class NixlConnectorScheduler: class NixlConnectorWorker: """Implementation of Worker side methods""" - @dataclass - class TpKVTopology: - """ - Helper class for tensor parallel and KV topology information for - mapping between local and remote TP workers. - """ - - tp_rank: int - remote_tp_size: dict[EngineId, int] - is_mla: bool - total_num_kv_heads: int - attn_backend: type[AttentionBackend] - engine_id: EngineId - remote_block_size: dict[EngineId, int] - - def __post_init__(self): - # Figure out whether the first dimension of the cache is K/V - # or num_blocks. This is used to register the memory regions correctly. - kv_cache_shape = self.attn_backend.get_kv_cache_shape( - num_blocks=1, block_size=16, num_kv_heads=1, head_size=1 - ) - # Non-MLA backends caches have 5 dims [2, num_blocks, H,N,D], - # we just mock num_blocks to 1 for the dimension check below. - self._is_kv_layout_blocks_first = ( - len(kv_cache_shape) == 5 and kv_cache_shape[0] == 1 - ) - - attn_backend = AttentionBackendEnum[self.attn_backend.get_name()] - self._use_pallas = attn_backend == AttentionBackendEnum.PALLAS - - @property - def is_kv_layout_blocks_first(self) -> bool: - return self._is_kv_layout_blocks_first - - @property - def split_k_and_v(self) -> bool: - # Whether to register regions for K and V separately (when present). - return not ( - self.is_mla or self._use_pallas or self.is_kv_layout_blocks_first - ) - - @property - def tp_size(self) -> int: - return self.remote_tp_size[self.engine_id] - - @property - def block_size(self) -> int: - return self.remote_block_size[self.engine_id] - - def tp_ratio( - self, - remote_tp_size: int, - ) -> int: - """ - Calculate the tensor parallel ratio between local and remote TP. - We can think of it as the number of local TP workers-per-remote TP - workers. Local workers will read from the same remote TP worker in - groups of size `tp_ratio`. - """ - assert self.tp_size % remote_tp_size == 0, ( - f"Local tensor parallel size {self.tp_size} is not divisible " - f"by remote tensor parallel size {remote_tp_size}." - ) - return self.tp_size // remote_tp_size - - def block_size_ratio( - self, - remote_block_size: int, - ) -> float: - """ - Calculate the block size ratio between local and remote TP. - """ - assert self.block_size % remote_block_size == 0, ( - f"Local block size {self.block_size} is not divisible " - f"by remote block size {remote_block_size} or vice versa." - ) - return self.block_size // remote_block_size - - def tp_ratio_from_engine_id( - self, - remote_engine_id: EngineId, - ) -> int: - remote_tp_size = self.remote_tp_size[remote_engine_id] - return self.tp_ratio(remote_tp_size) - - def block_size_ratio_from_engine_id( - self, - remote_engine_id: EngineId, - ) -> float: - remote_block_size = self.remote_block_size[remote_engine_id] - return self.block_size_ratio(remote_block_size) - - def is_kv_replicated(self, engine_id: EngineId) -> bool: - """ - Whether the KV cache is replicated across TP workers due to the - number of TP workers being greater than the number of KV heads. - """ - tp_size = self.remote_tp_size[engine_id] - return tp_size // self.total_num_kv_heads >= 1 - - def replicates_kv_cache(self, remote_engine_id: EngineId) -> bool: - # MLA is always replicated as the hidden dim can't be split. - return self.is_mla or self.is_kv_replicated(remote_engine_id) - - def get_target_remote_rank( - self, - remote_tp_size: int, - ) -> int: - """ - Get the remote TP rank (on P) that the current local TP rank - (on D) will read from. - """ - tp_ratio = self.tp_ratio(remote_tp_size) - return self.tp_rank // tp_ratio - - def get_target_remote_rank_from_engine_id( - self, - remote_engine_id: EngineId, - ) -> int: - remote_tp_size = self.remote_tp_size[remote_engine_id] - return self.get_target_remote_rank(remote_tp_size) - def __init__(self, vllm_config: VllmConfig, engine_id: str): if NixlWrapper is None: logger.error("NIXL is not available") @@ -958,7 +836,7 @@ class NixlConnectorWorker: self.consumer_notification_counts_by_req = defaultdict[ReqId, int](int) self.xfer_stats = NixlKVConnectorStats() - self.kv_topo = self.TpKVTopology( + self.kv_topo = TpKVTopology( tp_rank=self.tp_rank, engine_id=self.engine_id, remote_tp_size=self._tp_size, # shared state diff --git a/vllm/envs.py b/vllm/envs.py index 2ed5816b350b7..37711dece9abc 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -175,6 +175,7 @@ if TYPE_CHECKING: VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost" VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600 + VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998 VLLM_ALL2ALL_BACKEND: Literal[ "naive", "pplx", @@ -197,6 +198,7 @@ if TYPE_CHECKING: VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16: bool = True VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int | None = None VLLM_NIXL_ABORT_REQUEST_TIMEOUT: int = 480 + VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480 VLLM_USE_CUDNN_PREFILL: bool = False VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: bool = False VLLM_ENABLE_CUDAGRAPH_GC: bool = False @@ -1260,6 +1262,10 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int( os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600") ), + # Port used for Mooncake handshake between remote agents. + "VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int( + os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998") + ), # all2all backend for vllm's expert parallel communication # Available options: # - "naive": naive all2all implementation using broadcasts @@ -1369,6 +1375,10 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_NIXL_ABORT_REQUEST_TIMEOUT": lambda: int( os.getenv("VLLM_NIXL_ABORT_REQUEST_TIMEOUT", "480") ), + # Timeout (in seconds) for MooncakeConnector in PD disaggregated setup. + "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int( + os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480") + ), # Controls whether or not to use cudnn prefill "VLLM_USE_CUDNN_PREFILL": lambda: bool( int(os.getenv("VLLM_USE_CUDNN_PREFILL", "0")) From 6366c098d7c76120b6a55a6829a2649c727a2862 Mon Sep 17 00:00:00 2001 From: Noa Neria Date: Thu, 4 Dec 2025 12:04:43 +0200 Subject: [PATCH 41/47] Validating Runai Model Streamer Integration with S3 Object Storage (#29320) Signed-off-by: Noa Neria --- docker/Dockerfile | 2 +- requirements/nightly_torch_test.txt | 2 +- requirements/rocm.txt | 2 +- requirements/test.in | 2 +- requirements/test.txt | 6 +-- setup.py | 2 +- .../__init__.py | 0 .../runai_streamer_loader/conftest.py | 39 ++++++++++++++ .../test_runai_model_streamer_loader.py | 0 .../test_runai_model_streamer_s3.py | 52 +++++++++++++++++++ .../test_runai_utils.py | 0 .../test_weight_utils.py | 0 vllm/transformers_utils/runai_utils.py | 4 +- 13 files changed, 100 insertions(+), 11 deletions(-) rename tests/model_executor/model_loader/{runai_model_streamer => runai_streamer_loader}/__init__.py (100%) create mode 100644 tests/model_executor/model_loader/runai_streamer_loader/conftest.py rename tests/model_executor/model_loader/{runai_model_streamer => runai_streamer_loader}/test_runai_model_streamer_loader.py (100%) create mode 100644 tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py rename tests/model_executor/model_loader/{runai_model_streamer => runai_streamer_loader}/test_runai_utils.py (100%) rename tests/model_executor/model_loader/{runai_model_streamer => runai_streamer_loader}/test_weight_utils.py (100%) diff --git a/docker/Dockerfile b/docker/Dockerfile index 73cb4d7e0dc10..0d50d97e54c6c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -580,7 +580,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ else \ BITSANDBYTES_VERSION="0.46.1"; \ fi; \ - uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.15.0' + uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.15.3' ENV VLLM_USAGE_SOURCE production-docker-image diff --git a/requirements/nightly_torch_test.txt b/requirements/nightly_torch_test.txt index 53b012372be8e..7b2c665448a3b 100644 --- a/requirements/nightly_torch_test.txt +++ b/requirements/nightly_torch_test.txt @@ -42,6 +42,6 @@ tritonclient==2.51.0 numba == 0.61.2 # Required for N-gram speculative decoding numpy -runai-model-streamer[s3,gcs]==0.15.0 +runai-model-streamer[s3,gcs]==0.15.3 fastsafetensors>=0.1.10 pydantic>=2.12 # 2.11 leads to error on python 3.13 diff --git a/requirements/rocm.txt b/requirements/rocm.txt index abbd33d6e1240..05b9a21791c92 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -12,7 +12,7 @@ tensorizer==2.10.1 packaging>=24.2 setuptools>=77.0.3,<80.0.0 setuptools-scm>=8 -runai-model-streamer[s3,gcs]==0.15.0 +runai-model-streamer[s3,gcs]==0.15.3 conch-triton-kernels==1.2.1 timm>=1.0.17 fastsafetensors @ git+https://github.com/foundation-model-stack/fastsafetensors.git@d6f998a03432b2452f8de2bb5cefb5af9795d459 diff --git a/requirements/test.in b/requirements/test.in index da7a7db1f00c9..dfae5b75821f8 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -51,7 +51,7 @@ tritonclient==2.51.0 arctic-inference == 0.1.1 # Required for suffix decoding test numba == 0.61.2 # Required for N-gram speculative decoding numpy -runai-model-streamer[s3,gcs]==0.15.0 +runai-model-streamer[s3,gcs]==0.15.3 fastsafetensors>=0.1.10 pydantic>=2.12 # 2.11 leads to error on python 3.13 decord==0.6.0 diff --git a/requirements/test.txt b/requirements/test.txt index c5f103b8b0d78..571194e05c1ba 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -965,11 +965,11 @@ rsa==4.9.1 # via google-auth rtree==1.4.0 # via torchgeo -runai-model-streamer==0.15.0 +runai-model-streamer==0.15.3 # via -r requirements/test.in -runai-model-streamer-gcs==0.15.0 +runai-model-streamer-gcs==0.15.3 # via runai-model-streamer -runai-model-streamer-s3==0.15.0 +runai-model-streamer-s3==0.15.3 # via runai-model-streamer s3transfer==0.10.3 # via boto3 diff --git a/setup.py b/setup.py index af7282d4f747b..6fcb6653bc4a3 100644 --- a/setup.py +++ b/setup.py @@ -797,7 +797,7 @@ setup( "bench": ["pandas", "matplotlib", "seaborn", "datasets"], "tensorizer": ["tensorizer==2.10.1"], "fastsafetensors": ["fastsafetensors >= 0.1.10"], - "runai": ["runai-model-streamer[s3,gcs] >= 0.15.0"], + "runai": ["runai-model-streamer[s3,gcs] >= 0.15.3"], "audio": [ "librosa", "soundfile", diff --git a/tests/model_executor/model_loader/runai_model_streamer/__init__.py b/tests/model_executor/model_loader/runai_streamer_loader/__init__.py similarity index 100% rename from tests/model_executor/model_loader/runai_model_streamer/__init__.py rename to tests/model_executor/model_loader/runai_streamer_loader/__init__.py diff --git a/tests/model_executor/model_loader/runai_streamer_loader/conftest.py b/tests/model_executor/model_loader/runai_streamer_loader/conftest.py new file mode 100644 index 0000000000000..9a022f6bbd9d1 --- /dev/null +++ b/tests/model_executor/model_loader/runai_streamer_loader/conftest.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port +from vllm.v1.executor import UniProcExecutor +from vllm.v1.worker.worker_base import WorkerWrapperBase + + +# This is a dummy executor for patching in test_runai_model_streamer_s3.py. +# We cannot use vllm_runner fixture here, because it spawns worker process. +# The worker process reimports the patched entities, and the patch is not applied. +class RunaiDummyExecutor(UniProcExecutor): + def _init_executor(self) -> None: + distributed_init_method = get_distributed_init_method(get_ip(), get_open_port()) + + local_rank = 0 + rank = 0 + is_driver_worker = True + + device_info = self.vllm_config.device_config.device.__str__().split(":") + if len(device_info) > 1: + local_rank = int(device_info[1]) + + worker_rpc_kwargs = dict( + vllm_config=self.vllm_config, + local_rank=local_rank, + rank=rank, + distributed_init_method=distributed_init_method, + is_driver_worker=is_driver_worker, + ) + + wrapper_kwargs = { + "vllm_config": self.vllm_config, + } + + self.driver_worker = WorkerWrapperBase(**wrapper_kwargs) + + self.collective_rpc("init_worker", args=([worker_rpc_kwargs],)) + self.collective_rpc("init_device") diff --git a/tests/model_executor/model_loader/runai_model_streamer/test_runai_model_streamer_loader.py b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py similarity index 100% rename from tests/model_executor/model_loader/runai_model_streamer/test_runai_model_streamer_loader.py rename to tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py diff --git a/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py new file mode 100644 index 0000000000000..d60c9ba64cbdb --- /dev/null +++ b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from pathlib import Path + +from huggingface_hub import snapshot_download +from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher + +from vllm.engine.arg_utils import EngineArgs + +from .conftest import RunaiDummyExecutor + +load_format = "runai_streamer" +test_model = "openai-community/gpt2" + + +def test_runai_model_loader_download_files_s3_mocked_with_patch( + vllm_runner, + tmp_path: Path, + monkeypatch, +): + patcher = StreamerPatcher(str(tmp_path)) + + test_mock_s3_model = "s3://my-mock-bucket/gpt2/" + + # Download model from HF + mock_model_dir = f"{tmp_path}/gpt2" + snapshot_download(repo_id=test_model, local_dir=mock_model_dir) + + monkeypatch.setattr( + "vllm.transformers_utils.runai_utils.runai_list_safetensors", + patcher.shim_list_safetensors, + ) + monkeypatch.setattr( + "vllm.transformers_utils.runai_utils.runai_pull_files", + patcher.shim_pull_files, + ) + monkeypatch.setattr( + "vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer", + patcher.create_mock_streamer, + ) + + engine_args = EngineArgs( + model=test_mock_s3_model, + load_format=load_format, + tensor_parallel_size=1, + ) + + vllm_config = engine_args.create_engine_config() + + executor = RunaiDummyExecutor(vllm_config) + executor.driver_worker.load_model() diff --git a/tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_utils.py similarity index 100% rename from tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py rename to tests/model_executor/model_loader/runai_streamer_loader/test_runai_utils.py diff --git a/tests/model_executor/model_loader/runai_model_streamer/test_weight_utils.py b/tests/model_executor/model_loader/runai_streamer_loader/test_weight_utils.py similarity index 100% rename from tests/model_executor/model_loader/runai_model_streamer/test_weight_utils.py rename to tests/model_executor/model_loader/runai_streamer_loader/test_weight_utils.py diff --git a/vllm/transformers_utils/runai_utils.py b/vllm/transformers_utils/runai_utils.py index eac4294bb59cd..041056720a96b 100644 --- a/vllm/transformers_utils/runai_utils.py +++ b/vllm/transformers_utils/runai_utils.py @@ -18,9 +18,7 @@ SUPPORTED_SCHEMES = ["s3://", "gs://"] try: from runai_model_streamer import list_safetensors as runai_list_safetensors from runai_model_streamer import pull_files as runai_pull_files -except (ImportError, OSError): - # see https://github.com/run-ai/runai-model-streamer/issues/26 - # OSError will be raised on arm64 platform +except ImportError: runai_model_streamer = PlaceholderModule("runai_model_streamer") # type: ignore[assignment] runai_pull_files = runai_model_streamer.placeholder_attr("pull_files") runai_list_safetensors = runai_model_streamer.placeholder_attr("list_safetensors") From e96a6a6dca930d00902852ea6937a214a584b89b Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 4 Dec 2025 05:00:16 -0600 Subject: [PATCH 42/47] [ROCm][CI][Bugfix] Fixing the `Multi-Modal Models Test (Extended) 1` group (#30013) Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 6 ++- .../models/multimodal/generation/conftest.py | 16 +++++++ .../multimodal/generation/test_common.py | 12 ++++- .../generation/test_granite_speech.py | 15 ++++++- .../multimodal/generation/test_pixtral.py | 10 +++++ .../generation/vlm_utils/custom_inputs.py | 2 +- .../generation/vlm_utils/model_utils.py | 45 ++++++++++++++++++- tests/models/multimodal/pooling/conftest.py | 24 ++++++++++ tests/models/registry.py | 4 ++ vllm/v1/attention/backends/flex_attention.py | 14 +++++- 10 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 tests/models/multimodal/pooling/conftest.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index ee4fdebae5675..022b6ea236d54 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -987,7 +987,8 @@ steps: commands: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 -- label: Multi-Modal Models Test (Extended) 1 +- label: Multi-Modal Models Test (Extended) 1 # 60min + timeout_in_minutes: 120 mirror_hardwares: [amdexperimental] agent_pool: mi325_1 # grade: Blocking @@ -1011,7 +1012,8 @@ steps: - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' -- label: Multi-Modal Models Test (Extended) 3 +- label: Multi-Modal Models Test (Extended) 3 # 75min + timeout_in_minutes: 150 mirror_hardwares: [amdexperimental] agent_pool: mi325_1 # grade: Blocking diff --git a/tests/models/multimodal/generation/conftest.py b/tests/models/multimodal/generation/conftest.py index ee3ecdb10fdb8..26f8586742cea 100644 --- a/tests/models/multimodal/generation/conftest.py +++ b/tests/models/multimodal/generation/conftest.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Pytest configuration for vLLM tests.""" +import warnings + import torch from vllm.platforms import current_platform @@ -14,6 +16,20 @@ def pytest_configure(config): if not current_platform.is_rocm(): return + skip_patterns = ["test_granite_speech.py"] + if any(pattern in str(arg) for arg in config.args for pattern in skip_patterns): + # Skip disabling SDP for Granite Speech tests on ROCm + return + + # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers + # accuracy issues + # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace torch.backends.cuda.enable_flash_sdp(False) torch.backends.cuda.enable_mem_efficient_sdp(False) torch.backends.cuda.enable_math_sdp(True) + warnings.warn( + "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp " + "to avoid HuggingFace Transformers accuracy issues", + UserWarning, + stacklevel=1, + ) diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index 0eaf7198f91b7..f896126a49089 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -403,12 +403,13 @@ VLM_TEST_SETTINGS = { # So, we need to reduce the number of tokens for the test to pass. max_tokens=8, num_logprobs=10, + auto_cls=AutoModelForCausalLM, marks=[large_gpu_mark(min_gb=32)], ), "glm4_1v": VLMTestInfo( models=["zai-org/GLM-4.1V-9B-Thinking"], test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), - prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", + prompt_formatter=lambda img_prompt: f"[gMASK]<|user|>\n{img_prompt}<|assistant|>\n", # noqa: E501 img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>", video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>", max_model_len=2048, @@ -423,6 +424,7 @@ VLM_TEST_SETTINGS = { models=["zai-org/GLM-4.1V-9B-Thinking"], # GLM4.1V require include video metadata for input test_type=VLMTestType.CUSTOM_INPUTS, + prompt_formatter=lambda vid_prompt: f"[gMASK]<|user|>\n{vid_prompt}<|assistant|>\n", # noqa: E501 max_model_len=4096, max_num_seqs=2, auto_cls=AutoModelForImageTextToText, @@ -737,7 +739,13 @@ VLM_TEST_SETTINGS = { max_model_len=8192, max_num_seqs=2, auto_cls=AutoModelForImageTextToText, - marks=[large_gpu_mark(min_gb=48)], + marks=[ + large_gpu_mark(min_gb=48), + pytest.mark.skipif( + current_platform.is_rocm(), + reason="Model produces a vector of output in HF on ROCm", + ), + ], ), "qwen_vl": VLMTestInfo( models=["Qwen/Qwen-VL"], diff --git a/tests/models/multimodal/generation/test_granite_speech.py b/tests/models/multimodal/generation/test_granite_speech.py index e39dfc888779e..f528a993f8551 100644 --- a/tests/models/multimodal/generation/test_granite_speech.py +++ b/tests/models/multimodal/generation/test_granite_speech.py @@ -8,6 +8,7 @@ from transformers import AutoModelForSpeechSeq2Seq from vllm.logprobs import SampleLogprobs from vllm.lora.request import LoRARequest +from vllm.platforms import current_platform from ....conftest import AudioTestAssets, HfRunner, PromptAudioInput, VllmRunner from ...registry import HF_EXAMPLE_MODELS @@ -34,6 +35,12 @@ audio_lora_path = MODEL_NAME models = [MODEL_NAME] +@pytest.fixture(autouse=True) +def set_attention_backend_for_rocm(monkeypatch): + if current_platform.is_rocm(): + monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "TRITON_ATTN") + + def run_test( hf_runner: type[HfRunner], vllm_runner: type[VllmRunner], @@ -111,8 +118,12 @@ def run_test( @pytest.mark.parametrize("model", models) -@pytest.mark.parametrize("dtype", ["bfloat16"]) -@pytest.mark.parametrize("max_model_len", [2048]) +@pytest.mark.parametrize( + "dtype", ["float16"] if current_platform.is_rocm() else ["bfloat16"] +) +@pytest.mark.parametrize( + "max_model_len", [512] if current_platform.is_rocm() else [2048] +) @pytest.mark.parametrize("max_tokens", [128]) @pytest.mark.parametrize("num_logprobs", [10]) def test_models( diff --git a/tests/models/multimodal/generation/test_pixtral.py b/tests/models/multimodal/generation/test_pixtral.py index 3cad2c43d5623..375099f4365ac 100644 --- a/tests/models/multimodal/generation/test_pixtral.py +++ b/tests/models/multimodal/generation/test_pixtral.py @@ -15,6 +15,7 @@ from transformers import AutoProcessor from vllm import SamplingParams, TextPrompt, TokensPrompt from vllm.logprobs import Logprob, SampleLogprobs from vllm.multimodal import MultiModalDataBuiltins +from vllm.platforms import current_platform from ....utils import VLLM_PATH, large_gpu_test from ...utils import check_logprobs_close @@ -165,6 +166,15 @@ def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs: def test_chat( vllm_runner, max_model_len: int, model: str, dtype: str, local_asset_server ) -> None: + if ( + model == MISTRAL_SMALL_3_1_ID + and max_model_len == 65536 + and current_platform.is_rocm() + ): + pytest.skip( + "OOM on ROCm: 24B model with 65536 context length exceeds GPU memory" + ) + EXPECTED_CHAT_LOGPROBS = load_outputs_w_logprobs(FIXTURE_LOGPROBS_CHAT[model]) with vllm_runner( model, diff --git a/tests/models/multimodal/generation/vlm_utils/custom_inputs.py b/tests/models/multimodal/generation/vlm_utils/custom_inputs.py index 8c9c390911bdc..84109233685bb 100644 --- a/tests/models/multimodal/generation/vlm_utils/custom_inputs.py +++ b/tests/models/multimodal/generation/vlm_utils/custom_inputs.py @@ -140,7 +140,7 @@ def video_with_metadata_glm4_1v(): metadata = VIDEO_ASSETS[0].metadata question = "Describe the video." video_prompt = "<|begin_of_video|><|video|><|end_of_video|>" - formatted_prompt = f"<|user|>\n{video_prompt}{question}<|assistant|>\n" + formatted_prompt = f"[gMASK]<|user|>\n{video_prompt}{question}<|assistant|>\n" scales = [0.1, 0.2, 0.25] video_input = [ diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 87cd5c3cd3554..b2c62fbd119cc 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -25,6 +25,7 @@ from transformers import ( from transformers.video_utils import VideoMetadata from vllm.logprobs import SampleLogprobs +from vllm.platforms import current_platform from vllm.utils.collection_utils import is_list_of from .....conftest import HfRunner, ImageAsset, ImageTestAssets @@ -366,6 +367,40 @@ def gemma3_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOut def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: """Patches and returns an instance of the HfRunner to use for GLM4V.""" + if current_platform.is_rocm(): + import types + + config = hf_model.model.config + if hasattr(config, "num_layers") and not hasattr(config, "num_hidden_layers"): + config.num_hidden_layers = config.num_layers + config.output_hidden_states = True + + def patched_prepare_cache( + self, generation_config, model_kwargs, *args, **kwargs + ): + model_kwargs["past_key_values"] = None + model_kwargs["use_cache"] = False + return model_kwargs + + hf_model.model._prepare_cache_for_generation = types.MethodType( + patched_prepare_cache, hf_model.model + ) + original_generate = hf_model.model.generate + + def patched_generate(*args, **kwargs): + kwargs["output_hidden_states"] = True + kwargs["return_dict_in_generate"] = True + return original_generate(*args, **kwargs) + + hf_model.model.generate = patched_generate + original_forward = hf_model.model.forward + + def patched_forward(*args, **kwargs): + kwargs["output_hidden_states"] = True + return original_forward(*args, **kwargs) + + hf_model.model.forward = patched_forward + hf_processor = hf_model.processor def processor(*args, text="", images=None, **kwargs): @@ -406,7 +441,15 @@ def glm4_1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: if videos is not None and is_list_of(videos, tuple): # If videos is a list of tuples, we assume each tuple contains # (video_array, metadata) as in the case of GLM4.1V. - video_metadata = [[VideoMetadata(**video[1])] for video in videos] + # Filter out 'do_sample_frames' as it's not a valid VideoMetadata arg + video_metadata = [ + [ + VideoMetadata( + **{k: v for k, v in video[1].items() if k != "do_sample_frames"} + ) + ] + for video in videos + ] videos = [[video[0]] for video in videos] else: video_metadata = None diff --git a/tests/models/multimodal/pooling/conftest.py b/tests/models/multimodal/pooling/conftest.py new file mode 100644 index 0000000000000..c5f40cb42ca2a --- /dev/null +++ b/tests/models/multimodal/pooling/conftest.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Pytest configuration for vLLM pooling tests.""" + +import os +import warnings + +from vllm.platforms import current_platform + + +def pytest_collection_modifyitems(config, items): + """Set FLEX_ATTENTION backend for SigLIP tests on ROCm.""" + if not current_platform.is_rocm(): + return + + siglip_tests = [item for item in items if "test_siglip" in item.nodeid] + + if siglip_tests: + os.environ["VLLM_ATTENTION_BACKEND"] = "FLEX_ATTENTION" + warnings.warn( + "ROCm: Set VLLM_ATTENTION_BACKEND=FLEX_ATTENTION for SigLIP tests", + UserWarning, + stacklevel=1, + ) diff --git a/tests/models/registry.py b/tests/models/registry.py index 6b1d24b1c99b5..bf88bac209808 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -667,6 +667,10 @@ _MULTIMODAL_EXAMPLE_MODELS = { "moonshotai/Kimi-VL-A3B-Instruct", extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"}, trust_remote_code=True, + max_transformers_version="4.53.3", + transformers_version_reason="HF model uses deprecated transformers API " + "(PytorchGELUTanh, DynamicCache.seen_tokens, and more). See: " + "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/discussions/31", ), "LightOnOCRForConditionalGeneration": _HfExamplesInfo( "lightonai/LightOnOCR-1B", diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index fe92f6570501c..a2a6eeeb16b24 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -31,6 +31,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.batch_invariant import ( vllm_is_batch_invariant, ) +from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import is_torch_equal_or_newer from vllm.v1.attention.backends.utils import ( @@ -927,7 +928,18 @@ def get_kernel_options( if torch.cuda.is_available(): device_props = torch.cuda.get_device_properties() - max_shared_memory = device_props.shared_memory_per_block_optin + # ROCm doesn't expose shared_memory_per_block_optin attribute + # AMD GPUs typically have 64KB LDS (Local Data Share) per workgroup + if hasattr(device_props, "shared_memory_per_block_optin"): + max_shared_memory = device_props.shared_memory_per_block_optin + elif current_platform.is_rocm(): + # ROCm fallback: use 64KB + max_shared_memory = 65536 + else: + raise RuntimeError( + "Unable to determine shared memory size on this hardware." + ) + if max_shared_memory < 144 * 1024: block_m_candidate = ensure_divisible( max(1, block_m_candidate // 2), block_m From 6796ce8bdbf29f5624fcdc03792626574c919b41 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Thu, 4 Dec 2025 19:11:59 +0800 Subject: [PATCH 43/47] [Bugfix] Fix the issue with interleaved thinking when using streaming (#30033) Signed-off-by: chaunceyjiang Signed-off-by: Chauncey Co-authored-by: Cyrus Leung --- .../reasoning/test_base_thinking_reasoning_parser.py | 12 +++++++++++- vllm/reasoning/basic_parsers.py | 9 ++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/reasoning/test_base_thinking_reasoning_parser.py b/tests/reasoning/test_base_thinking_reasoning_parser.py index d31b1c7d169b7..34e9483de54b3 100644 --- a/tests/reasoning/test_base_thinking_reasoning_parser.py +++ b/tests/reasoning/test_base_thinking_reasoning_parser.py @@ -112,7 +112,7 @@ class TestBaseThinkingReasoningParserMethods: """Test the is_reasoning_end method.""" parser = TestThinkingReasoningParser(test_tokenizer) end_token_id = parser.end_token_id - + start_token_id = parser.start_token_id # Test with end token present assert parser.is_reasoning_end([1, 2, end_token_id, 4]) is True @@ -122,6 +122,16 @@ class TestBaseThinkingReasoningParserMethods: # Test with empty list assert parser.is_reasoning_end([]) is False + # Test with interleaved thinking + assert parser.is_reasoning_end([1, start_token_id, 2, end_token_id]) is True + assert parser.is_reasoning_end([1, start_token_id, 2, 3]) is False + assert ( + parser.is_reasoning_end( + [1, start_token_id, 2, end_token_id, 2, 2, start_token_id] + ) + is False + ) + def test_extract_content_ids(self, test_tokenizer): """Test the extract_content_ids method.""" parser = TestThinkingReasoningParser(test_tokenizer) diff --git a/vllm/reasoning/basic_parsers.py b/vllm/reasoning/basic_parsers.py index 35084c0e7cc86..e78ac4a5ebb37 100644 --- a/vllm/reasoning/basic_parsers.py +++ b/vllm/reasoning/basic_parsers.py @@ -64,8 +64,15 @@ class BaseThinkingReasoningParser(ReasoningParser): ) def is_reasoning_end(self, input_ids: list[int]) -> bool: + start_token_id = self.start_token_id end_token_id = self.end_token_id - return any(input_id == end_token_id for input_id in reversed(input_ids)) + + for i in range(len(input_ids) - 1, -1, -1): + if input_ids[i] == start_token_id: + return False + if input_ids[i] == end_token_id: + return True + return False def extract_content_ids(self, input_ids: list[int]) -> list[int]: """ From 1b7c7f5159484063af28cb47809d79e83d3301ec Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 4 Dec 2025 03:18:29 -0800 Subject: [PATCH 44/47] [release] install regex (#30008) Signed-off-by: Kevin H. Luu Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .buildkite/scripts/upload-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/upload-wheels.sh b/.buildkite/scripts/upload-wheels.sh index 2eaa91c04086c..0ac8fdd45bd0a 100644 --- a/.buildkite/scripts/upload-wheels.sh +++ b/.buildkite/scripts/upload-wheels.sh @@ -81,7 +81,7 @@ else alias_arg="" fi -$PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" $alias_arg +$PYTHON pip install regex && .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" $alias_arg # copy indices to // unconditionally echo "Uploading indices to $S3_COMMIT_PREFIX" From 74c4d80c6ca8160578b6e812079cb11dfd8b3d22 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Thu, 4 Dec 2025 21:44:15 +0800 Subject: [PATCH 45/47] [Model][6/N] Improve all pooling task | Support chunked prefill with ALL pooling (#27145) Signed-off-by: wang.yuqi Signed-off-by: wang.yuqi Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- docs/features/README.md | 2 +- .../pooling/classify/test_offline.py | 7 +- .../pooling/classify/test_online.py | 12 +-- .../entrypoints/pooling/embed/test_offline.py | 2 +- .../pooling/reward/test_offline.py | 7 ++ .../test_all_pooling_plus_chunked_prefill.py | 53 +++++++++++ .../pooling/test_extract_hidden_states.py | 1 - tests/test_config.py | 8 +- vllm/config/model.py | 24 +++-- vllm/model_executor/layers/pooler.py | 93 ++++++++++++++----- vllm/model_executor/models/terratorch.py | 4 +- vllm/v1/outputs.py | 2 +- vllm/v1/pool/metadata.py | 29 +++++- vllm/v1/worker/gpu_input_batch.py | 23 ++++- vllm/v1/worker/gpu_model_runner.py | 50 +++------- 15 files changed, 224 insertions(+), 93 deletions(-) create mode 100644 tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py diff --git a/docs/features/README.md b/docs/features/README.md index 5faf3768f3214..684802301a44f 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -54,7 +54,7 @@ th:not(:first-child) { | beam-search | ✅ | ✅ | ✅ | [❌](https://github.com/vllm-project/vllm/issues/6137) | ✅ | ❌ | ✅ | ✅ | ✅ | ❔ | [❌](https://github.com/vllm-project/vllm/issues/7968) | ❔ | ✅ | ✅ | | | [prompt-embeds](prompt_embeds.md) | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❔ | ❔ | ❌ | ❔ | ❔ | ✅ | -\* Chunked prefill and prefix caching are only applicable to last-token pooling. +\* Chunked prefill and prefix caching are only applicable to last-token or all pooling with causal attention. ^ LoRA is only applicable to the language backbone of multimodal models. ### Feature x Hardware diff --git a/tests/entrypoints/pooling/classify/test_offline.py b/tests/entrypoints/pooling/classify/test_offline.py index 1063c3b6b755c..a07fcd372721a 100644 --- a/tests/entrypoints/pooling/classify/test_offline.py +++ b/tests/entrypoints/pooling/classify/test_offline.py @@ -61,11 +61,8 @@ def test_pooling_params(llm: LLM): @pytest.mark.skip_global_cleanup -def test_encode_api(llm: LLM): - # chunked prefill does not support all pooling - err_msg = "pooling_task must be one of.+" - with pytest.raises(ValueError, match=err_msg): - llm.encode(prompts, pooling_task="token_classify", use_tqdm=False) +def test_token_classify(llm: LLM): + llm.encode(prompts, pooling_task="token_classify", use_tqdm=False) def test_score_api(llm: LLM): diff --git a/tests/entrypoints/pooling/classify/test_online.py b/tests/entrypoints/pooling/classify/test_online.py index 6fef688586955..1a6c33b455e65 100644 --- a/tests/entrypoints/pooling/classify/test_online.py +++ b/tests/entrypoints/pooling/classify/test_online.py @@ -255,21 +255,21 @@ async def test_pooling_classify(server: RemoteOpenAIServer, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: str): - # token_classify uses ALL pooling, which does not support chunked prefill. task = "token_classify" + input_text = ["This product was excellent and exceeded my expectations"] response = requests.post( server.url_for("pooling"), json={ "model": model_name, - "input": "test", + "input": input_text, "encoding_format": "float", "task": task, }, ) - assert response.json()["error"]["type"] == "BadRequestError" - assert response.json()["error"]["message"].startswith( - f"Task {task} is not supported" - ) + poolings = PoolingResponse.model_validate(response.json()) + assert len(poolings.data) == 1 + assert len(poolings.data[0].data) == 8 + assert len(poolings.data[0].data[0]) == 2 @pytest.mark.asyncio diff --git a/tests/entrypoints/pooling/embed/test_offline.py b/tests/entrypoints/pooling/embed/test_offline.py index f5eab4c29ae18..12b47b1a08a8b 100644 --- a/tests/entrypoints/pooling/embed/test_offline.py +++ b/tests/entrypoints/pooling/embed/test_offline.py @@ -42,7 +42,7 @@ def llm(): @pytest.mark.skip_global_cleanup -def test_encode_api(llm: LLM): +def test_token_embed(llm: LLM): outputs = llm.encode(prompts, pooling_task="token_embed", use_tqdm=False) multi_vector = outputs[0].outputs.data assert multi_vector.shape == (11, 384) diff --git a/tests/entrypoints/pooling/reward/test_offline.py b/tests/entrypoints/pooling/reward/test_offline.py index 0255704cecd94..b061b55145155 100644 --- a/tests/entrypoints/pooling/reward/test_offline.py +++ b/tests/entrypoints/pooling/reward/test_offline.py @@ -36,6 +36,13 @@ def llm(): cleanup_dist_env_and_memory() +@pytest.mark.skip_global_cleanup +def test_config(llm: LLM): + vllm_config = llm.llm_engine.vllm_config + assert vllm_config.cache_config.enable_prefix_caching + assert vllm_config.scheduler_config.enable_chunked_prefill + + def test_pooling_params(llm: LLM): def get_outputs(use_activation): outputs = llm.reward( diff --git a/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py b/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py new file mode 100644 index 0000000000000..c259c532220b2 --- /dev/null +++ b/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest +import torch +from transformers import AutoModel + +from tests.models.utils import check_embeddings_close +from vllm import TokensPrompt + + +@pytest.mark.parametrize( + "model", + ["Qwen/Qwen3-Embedding-0.6B"], +) +@torch.inference_mode +def test_embed_models(hf_runner, vllm_runner, model: str): + chunk_size = 10 + n_prompt_tokens = [55, 56, 57] + token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens] + + with vllm_runner( + model, + runner="pooling", + max_model_len=128, + max_num_batched_tokens=chunk_size, + enforce_eager=True, + # `enable_chunked_prefill`: Set to `False` instead of `None` in VllmRunner + enable_chunked_prefill=True, + enable_prefix_caching=True, + ) as vllm_model: + vllm_outputs = vllm_model.token_embed( + [TokensPrompt(prompt_token_ids=t) for t in token_prompts], + ) + + with hf_runner( + model, + auto_cls=AutoModel, + ) as hf_model: + hf_outputs = [] + for token_prompt in token_prompts: + inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])}) + input_ids = inputs["input_ids"] + output = hf_model.model(input_ids) + hf_outputs.append(output.last_hidden_state.cpu().float()[0]) + + for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): + check_embeddings_close( + embeddings_0_lst=hf_output, + embeddings_1_lst=vllm_output, + name_0="hf", + name_1="vllm", + tol=1e-2, + ) diff --git a/tests/models/language/pooling/test_extract_hidden_states.py b/tests/models/language/pooling/test_extract_hidden_states.py index 0d41b93233d5a..488b27e2da0f1 100644 --- a/tests/models/language/pooling/test_extract_hidden_states.py +++ b/tests/models/language/pooling/test_extract_hidden_states.py @@ -20,7 +20,6 @@ def test_extract_hidden_states(hf_runner, vllm_runner, model: str): max_model_len=128, enforce_eager=True, runner="pooling", - enable_chunked_prefill=False, enable_prefix_caching=True, ) as vllm_model: pooling_outputs = vllm_model.llm.encode( diff --git a/tests/test_config.py b/tests/test_config.py index 019c0d6d8733f..203447cd531fb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -629,8 +629,8 @@ def test_s3_url_different_models_create_different_directories(mock_pull_files): ( "internlm/internlm2-1_8b-reward", "decoder", - False, - "Pooling models with all pooling does not support chunked prefill.", + True, + "Pooling models with causal attn and all pooling support chunked prefill.", ), ( "BAAI/bge-base-en", @@ -748,8 +748,8 @@ def test_is_chunked_prefill_supported( ( "internlm/internlm2-1_8b-reward", "decoder", - False, - "Pooling models with all pooling does not support prefix caching.", + True, + "Pooling models with causal attn and all pooling support prefix caching.", ), ( "BAAI/bge-base-en", diff --git a/vllm/config/model.py b/vllm/config/model.py index 655b7c995f6d2..ae5189ce68d9a 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1780,20 +1780,22 @@ class ModelConfig: return False elif attn_type == "decoder": pooling_type = self.pooler_config.pooling_type.lower() - if pooling_type in ["all", "mean", "step", "cls"]: + if pooling_type in ["mean", "step", "cls"]: logger.debug( "Pooling models with %s pooling does not " "support chunked prefill.", pooling_type, ) return False - else: - # pooling_type == "last" + elif pooling_type in ["all", "last"]: logger.debug( - "Pooling models with causal attn and last pooling support " - "chunked prefill." + "Pooling models with causal attn and %s pooling support " + "chunked prefill.", + pooling_type, ) return True + else: + raise ValueError(f"{pooling_type=} not supported.") # vllm currently does not have pooling models using hybrid, # attention_free or encoder_decoder attn types. return attn_type != "encoder_decoder" @@ -1817,20 +1819,22 @@ class ModelConfig: return False elif attn_type == "decoder": pooling_type = self.pooler_config.pooling_type.lower() - if pooling_type in ["all", "mean", "step", "cls"]: + if pooling_type in ["mean", "step", "cls"]: logger.debug( "Pooling models with %s pooling does not " "support prefix caching.", pooling_type, ) return False - else: - # pooling_type == "last" + elif pooling_type in ["all", "last"]: logger.debug( - "Pooling models with causal attn and last pooling support " - "prefix caching." + "Pooling models with causal attn and %s pooling support " + "prefix caching.", + pooling_type, ) return True + else: + raise ValueError(f"{pooling_type=} not supported.") # vllm currently does not have pooling models using hybrid, # attention_free or encoder_decoder attn types. return False diff --git a/vllm/model_executor/layers/pooler.py b/vllm/model_executor/layers/pooler.py index 185e03e5f3bd7..d1942689d7f5c 100644 --- a/vllm/model_executor/layers/pooler.py +++ b/vllm/model_executor/layers/pooler.py @@ -127,14 +127,14 @@ class PoolingMethod(nn.Module, ABC): self, hidden_states: torch.Tensor, pooling_cursor: PoolingCursor, - ) -> list[torch.Tensor] | torch.Tensor: + ) -> PoolerOutput: raise NotImplementedError def forward( self, hidden_states: torch.Tensor, pooling_metadata: PoolingMetadata, - ) -> list[torch.Tensor] | torch.Tensor: + ) -> PoolerOutput: pooling_cursor = pooling_metadata.pooling_cursor return self.forward_all(hidden_states, pooling_cursor) @@ -147,7 +147,7 @@ class CLSPool(PoolingMethod): self, hidden_states: torch.Tensor, pooling_cursor: PoolingCursor, - ) -> list[torch.Tensor] | torch.Tensor: + ) -> PoolerOutput: assert not pooling_cursor.is_partial_prefill(), ( "partial prefill not supported with CLS pooling" ) @@ -163,27 +163,65 @@ class LastPool(PoolingMethod): self, hidden_states: torch.Tensor, pooling_cursor: PoolingCursor, - ) -> list[torch.Tensor] | torch.Tensor: + ) -> PoolerOutput: return hidden_states[pooling_cursor.last_token_indices_gpu] class AllPool(PoolingMethod): + def __init__(self): + super().__init__() + + vllm_config = get_current_vllm_config() + self.enable_chunked_prefill = ( + vllm_config.scheduler_config.enable_chunked_prefill + ) + def get_supported_tasks(self) -> Set[PoolingTask]: return {"token_embed", "token_classify"} def forward_all( - self, - hidden_states: torch.Tensor, - pooling_cursor: PoolingCursor, - ) -> list[torch.Tensor] | torch.Tensor: - assert not pooling_cursor.is_partial_prefill(), ( - "partial prefill not supported with ALL pooling" + self, hidden_states: torch.Tensor, pooling_cursor: PoolingCursor + ) -> PoolerOutput: + raise NotImplementedError( + "forward_all is not implemented for AllPool. Use forward instead." ) + def forward( + self, + hidden_states: torch.Tensor, + pooling_metadata: PoolingMetadata, + ) -> PoolerOutput: + pooling_cursor = pooling_metadata.pooling_cursor + is_finished = pooling_cursor.is_finished() hidden_states_lst = list( hidden_states.split(pooling_cursor.num_scheduled_tokens_cpu.tolist()) ) - return [hidden_states_lst[i] for i in pooling_cursor.index] + hidden_states_lst = [hidden_states_lst[i] for i in pooling_cursor.index] + + if not self.enable_chunked_prefill: + return hidden_states_lst + + pooling_states = pooling_metadata.pooling_states + + # If chunked_prefill is enabled + # 1. first store the chunked hidden_states in pooling_states.hidden_states_cache + for p, hs_chunk in zip(pooling_states, hidden_states_lst): + p.hidden_states_cache.append(hs_chunk) + + # 2. Once prefill is finished, send hidden_states_cache to PoolerHead + output_list: PoolerOutput = [] + for p, finished in zip(pooling_states, is_finished): + if finished: + hidden_states_cache = p.hidden_states_cache + if len(hidden_states_cache) == 1: + output_list.append(hidden_states_cache[0]) + else: + output_list.append(torch.concat(hidden_states_cache, dim=0)) + p.clean() + else: + output_list.append(None) + + return output_list class MeanPool(PoolingMethod): @@ -194,7 +232,7 @@ class MeanPool(PoolingMethod): self, hidden_states: torch.Tensor, pooling_cursor: PoolingCursor, - ) -> list[torch.Tensor] | torch.Tensor: + ) -> PoolerOutput: assert not pooling_cursor.is_partial_prefill(), ( "partial prefill not supported with MEAN pooling" ) @@ -399,7 +437,7 @@ class PoolerHead(nn.Module): self, pooled_data: list[torch.Tensor] | torch.Tensor, pooling_metadata: PoolingMetadata, - ): + ) -> PoolerOutput: return self.activation(pooled_data) @@ -418,7 +456,7 @@ class EmbeddingPoolerHead(PoolerHead): self, pooled_data: list[torch.Tensor] | torch.Tensor, pooling_metadata: PoolingMetadata, - ): + ) -> PoolerOutput: if isinstance(pooled_data, list): pooled_data = torch.stack(pooled_data) # pooled_data shape: [batchsize, hidden_dimension] @@ -586,8 +624,12 @@ class ClassifierPooler(Pooler): class TokenEmbeddingPoolerHead(EmbeddingPoolerHead): def forward( - self, pooled_data: torch.Tensor, pooling_param: PoolingParams - ) -> torch.Tensor: + self, pooled_data: torch.Tensor | None, pooling_param: PoolingParams + ) -> PoolerOutput: + # for unfinished chunked prefill + if pooled_data is None: + return None + pooled_data = pooled_data.to(self.head_dtype) # pooled_data shape: [n_tokens, hidden_dimension] @@ -630,9 +672,13 @@ class TokenClassifierPoolerHead(nn.Module): def forward( self, - hidden_states: torch.Tensor, + hidden_states: torch.Tensor | None, pooling_param: PoolingParams, - ) -> torch.Tensor: + ) -> PoolerOutput: + # for unfinished chunked prefill + if hidden_states is None: + return None + hidden_states = hidden_states.to(self.head_dtype) # hidden_states shape: [n_token, hidden_size] @@ -686,17 +732,20 @@ class StepPooler(Pooler): self, hidden_states: torch.Tensor | list[torch.Tensor], pooling_metadata: PoolingMetadata, - ) -> torch.Tensor | list[torch.Tensor]: + ) -> PoolerOutput: pooled_data_lst = self.pooling(hidden_states, pooling_metadata) prompt_token_ids = pooling_metadata.get_prompt_token_ids() - - pooled_data = list[torch.Tensor]() - pooling_params = pooling_metadata.pooling_params + pooled_data: PoolerOutput = [] for data, token_id, pooling_param in zip( pooled_data_lst, prompt_token_ids, pooling_params ): + # for unfinished chunked prefill + if data is None: + pooled_data.append(data) + continue + step_tag_id = pooling_param.step_tag_id returned_token_ids = pooling_param.returned_token_ids diff --git a/vllm/model_executor/models/terratorch.py b/vllm/model_executor/models/terratorch.py index 19052c8d49e44..9f34090e31071 100644 --- a/vllm/model_executor/models/terratorch.py +++ b/vllm/model_executor/models/terratorch.py @@ -64,7 +64,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from .interfaces import IsAttentionFree, MultiModalEmbeddings, SupportsMultiModal -from .interfaces_base import default_pooling_type +from .interfaces_base import attn_type logger = init_logger(__name__) @@ -220,7 +220,7 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor): ) -@default_pooling_type("All") +@attn_type("attention_free") @MULTIMODAL_REGISTRY.register_processor( TerratorchMultiModalProcessor, info=TerratorchProcessingInfo, diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 88ac6b4aeb4bb..546eacebf83e5 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -89,7 +89,7 @@ class LogprobsTensors(NamedTuple): # [num_reqs, ] # The shape of each element depends on the pooler used -PoolerOutput = torch.Tensor | list[torch.Tensor] +PoolerOutput = list[torch.Tensor | None] | torch.Tensor | None @dataclass diff --git a/vllm/v1/pool/metadata.py b/vllm/v1/pool/metadata.py index 9ee588ea44ca4..acd1a00e87553 100644 --- a/vllm/v1/pool/metadata.py +++ b/vllm/v1/pool/metadata.py @@ -17,6 +17,7 @@ class PoolingCursor: first_token_indices_gpu: torch.Tensor last_token_indices_gpu: torch.Tensor prompt_lens_cpu: torch.Tensor + seq_lens_cpu: torch.Tensor num_scheduled_tokens_cpu: torch.Tensor def __getitem__(self, indices: slice): @@ -25,12 +26,25 @@ class PoolingCursor: first_token_indices_gpu=self.first_token_indices_gpu[indices], last_token_indices_gpu=self.last_token_indices_gpu[indices], prompt_lens_cpu=self.prompt_lens_cpu[indices], + seq_lens_cpu=self.seq_lens_cpu[indices], num_scheduled_tokens_cpu=self.num_scheduled_tokens_cpu[indices], ) def is_partial_prefill(self): return not torch.all(self.prompt_lens_cpu == self.num_scheduled_tokens_cpu) + def is_finished(self): + return self.prompt_lens_cpu == self.seq_lens_cpu + + +class PoolingStates: + def __init__(self): + # for chunked prefill with ALL pooling + self.hidden_states_cache: list[torch.Tensor] = [] + + def clean(self): + self.hidden_states_cache.clear() + @dataclass class PoolingMetadata: @@ -39,6 +53,7 @@ class PoolingMetadata: prompt_lens: torch.Tensor # CPU Tensor prompt_token_ids: torch.Tensor | None pooling_params: list[PoolingParams] + pooling_states: list[PoolingStates] pooling_cursor: PoolingCursor | None = None def __post_init__(self) -> None: @@ -60,6 +75,7 @@ class PoolingMetadata: if self.prompt_token_ids is None else self.prompt_token_ids[indices], pooling_params=self.pooling_params[indices], + pooling_states=self.pooling_states[indices], pooling_cursor=None if self.pooling_cursor is None else self.pooling_cursor[indices], @@ -74,15 +90,21 @@ class PoolingMetadata: return [prompt_token_ids[i, :num] for i, num in enumerate(self.prompt_lens)] def build_pooling_cursor( - self, num_scheduled_tokens: list[int], device: torch.device + self, + num_scheduled_tokens: list[int], + seq_lens_cpu: torch.Tensor, + device: torch.device, ): self.pooling_cursor = build_pooling_cursor( - num_scheduled_tokens, self.prompt_lens, device + num_scheduled_tokens, seq_lens_cpu, self.prompt_lens, device ) def build_pooling_cursor( - num_scheduled_tokens: list[int], prompt_lens: torch.Tensor, device: torch.device + num_scheduled_tokens: list[int], + seq_lens_cpu: torch.Tensor, + prompt_lens: torch.Tensor, + device: torch.device, ): assert len(prompt_lens) == len(num_scheduled_tokens) @@ -99,5 +121,6 @@ def build_pooling_cursor( first_token_indices_gpu=cumsum[:n_seq], last_token_indices_gpu=cumsum[1:] - 1, prompt_lens_cpu=prompt_lens, + seq_lens_cpu=seq_lens_cpu, num_scheduled_tokens_cpu=num_scheduled_tokens_cpu, ) diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 516c76a5e4b15..ead7a3619dea5 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -15,7 +15,7 @@ from vllm.sampling_params import SamplingParams, SamplingType from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.utils.collection_utils import swap_dict_values from vllm.v1.outputs import LogprobsTensors -from vllm.v1.pool.metadata import PoolingMetadata +from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates from vllm.v1.sample.logits_processor import ( BatchUpdateBuilder, LogitsProcessors, @@ -33,7 +33,6 @@ class CachedRequestState: prompt_token_ids: list[int] | None mm_features: list[MultiModalFeatureSpec] sampling_params: SamplingParams | None - pooling_params: PoolingParams | None generator: torch.Generator | None block_ids: tuple[list[int], ...] @@ -51,11 +50,18 @@ class CachedRequestState: # Used when both async_scheduling and spec_decode are enabled. prev_num_draft_len: int = 0 + # for pooling models + pooling_params: PoolingParams | None = None + pooling_states: PoolingStates | None = None + def __post_init__(self): self.num_prompt_tokens = length_from_prompt_token_ids_or_embeds( self.prompt_token_ids, self.prompt_embeds ) + if self.pooling_params is not None: + self.pooling_states = PoolingStates() + @property def num_tokens(self) -> int: return self.num_prompt_tokens + len(self.output_token_ids) @@ -255,7 +261,9 @@ class InputBatch: # This is updated each time the batch constituents change. self.sampling_metadata = self._make_sampling_metadata() + # for pooling models self.pooling_params: dict[str, PoolingParams] = {} + self.pooling_states: dict[str, PoolingStates] = {} # Cached reference to the GPU tensor of previously sampled tokens self.prev_sampled_token_ids: torch.Tensor | None = None @@ -413,7 +421,11 @@ class InputBatch: sampling_params.bad_words_token_ids ) elif pooling_params := request.pooling_params: + pooling_states = request.pooling_states + assert pooling_states is not None + self.pooling_params[req_id] = pooling_params + self.pooling_states[req_id] = pooling_states self.logits_processing_needs_token_ids[req_index] = ( pooling_params.requires_token_ids ) @@ -469,6 +481,7 @@ class InputBatch: if self.is_pooling_model: self.pooling_params.pop(req_id, None) + self.pooling_states.pop(req_id, None) return req_index self.greedy_reqs.discard(req_id) @@ -837,13 +850,19 @@ class InputBatch: assert len(self.req_ids) == len(self.pooling_params) return [self.pooling_params[req_id] for req_id in self.req_ids] + def get_pooling_states(self) -> list[PoolingStates]: + assert len(self.req_ids) == len(self.pooling_states) + return [self.pooling_states[req_id] for req_id in self.req_ids] + def get_pooling_metadata(self) -> PoolingMetadata: pooling_params = self.get_pooling_params() + pooling_states = self.get_pooling_states() return PoolingMetadata( prompt_lens=torch.from_numpy(self.num_prompt_tokens[: self.num_reqs]), prompt_token_ids=self.sampling_metadata.prompt_token_ids, pooling_params=pooling_params, + pooling_states=pooling_states, ) def _make_prompt_token_ids_tensor(self) -> torch.Tensor: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 3f043e3b2648b..a7eb9cdae8b10 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -131,7 +131,7 @@ from vllm.v1.outputs import ( SamplerOutput, make_empty_encoder_model_runner_output, ) -from vllm.v1.pool.metadata import PoolingMetadata +from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates from vllm.v1.sample.logits_processor import LogitsProcessors, build_logitsprocs from vllm.v1.sample.logits_processor.interface import LogitsProcessor from vllm.v1.sample.metadata import SamplingMetadata @@ -2291,20 +2291,6 @@ class GPUModelRunner( supported_tasks = list(model.pooler.get_supported_tasks()) - if self.scheduler_config.enable_chunked_prefill: - if "token_embed" in supported_tasks: - supported_tasks.remove("token_embed") - if "token_classify" in supported_tasks: - supported_tasks.remove("token_classify") - - logger.debug_once( - "Chunked prefill is not supported with " - "token_embed and token_classify tasks " - "which using ALL pooling. " - "Please turn off chunked prefill by " - "`--no-enable-chunked-prefill` before using it." - ) - if "score" in supported_tasks: num_labels = getattr(self.model_config.hf_config, "num_labels", 0) if num_labels != 1: @@ -2381,11 +2367,12 @@ class GPUModelRunner( ) hidden_states = hidden_states[:num_scheduled_tokens] + seq_lens_cpu = self.seq_lens.cpu[: self.input_batch.num_reqs] + pooling_metadata = self.input_batch.get_pooling_metadata() pooling_metadata.build_pooling_cursor( - num_scheduled_tokens_np.tolist(), device=hidden_states.device + num_scheduled_tokens_np.tolist(), seq_lens_cpu, device=hidden_states.device ) - seq_lens_cpu = self.seq_lens.cpu[: self.input_batch.num_reqs] model = cast(VllmModelForPooling, self.model) raw_pooler_output: PoolerOutput = model.pooler( @@ -2393,7 +2380,7 @@ class GPUModelRunner( pooling_metadata=pooling_metadata, ) raw_pooler_output = json_map_leaves( - lambda x: x.to("cpu", non_blocking=True), + lambda x: x.to("cpu", non_blocking=True) if x is not None else x, raw_pooler_output, ) self._sync_device() @@ -4248,10 +4235,13 @@ class GPUModelRunner( prompt_lens=dummy_prompt_lens, prompt_token_ids=dummy_token_ids, pooling_params=[dummy_pooling_params] * num_reqs, + pooling_states=[PoolingStates() for i in range(num_reqs)], ) dummy_metadata.build_pooling_cursor( - num_scheduled_tokens_list, device=hidden_states.device + num_scheduled_tokens_list, + seq_lens_cpu=dummy_prompt_lens, + device=hidden_states.device, ) try: @@ -4278,22 +4268,12 @@ class GPUModelRunner( supported_pooling_tasks = self.get_supported_pooling_tasks() if not supported_pooling_tasks: - if self.scheduler_config.enable_chunked_prefill: - raise RuntimeError( - f"Model {self.model_config.model} does not support " - "any pooling tasks with chunked prefill enabled. " - "Please add --no-enable-chunked-prefill to your " - "config or CLI args. See " - "https://docs.vllm.ai/en/latest/models/pooling_models.html " - "to learn more." - ) - else: - raise RuntimeError( - f"Model {self.model_config.model} does not support " - "any pooling tasks. See " - "https://docs.vllm.ai/en/latest/models/pooling_models.html " - "to learn more." - ) + raise RuntimeError( + f"Model {self.model_config.model} does not support " + "any pooling tasks. See " + "https://docs.vllm.ai/en/latest/models/pooling_models.html " + "to learn more." + ) output_size = dict[PoolingTask, float]() for task in supported_pooling_tasks: From 9998ea5b576972a508c854227f57829dd4bca940 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 4 Dec 2025 13:44:50 +0000 Subject: [PATCH 46/47] Delete HF version of Phi 4 MM (#30049) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- docs/models/supported_models.md | 1 - .../generation/test_phi4_multimodal.py | 281 ---- .../multimodal/processing/test_common.py | 22 - tests/models/registry.py | 4 - vllm/model_executor/models/phi4_multimodal.py | 1447 ----------------- vllm/model_executor/models/registry.py | 2 +- 6 files changed, 1 insertion(+), 1756 deletions(-) delete mode 100644 tests/models/multimodal/generation/test_phi4_multimodal.py delete mode 100644 vllm/model_executor/models/phi4_multimodal.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 040107c11efcf..96d5ec25c0064 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -711,7 +711,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + IE | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | | ✅︎ | | `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + IE+ | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ | | `Phi4MMForCausalLM` | Phi-4-multimodal | T + I+ / T + A+ / I+ + A+ | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ | -| `Phi4MultimodalForCausalLM` | Phi-4-multimodal (HF Transformers) | T + I+ / T + A+ / I+ + A+ | `microsoft/Phi-4-multimodal-instruct` (with revision `refs/pr/70`), etc. | ✅︎ | ✅︎ | | `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I+ | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | | ✅︎ | | `QwenVLForConditionalGeneration`^ | Qwen-VL | T + IE+ | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ | | `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A+ | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ | diff --git a/tests/models/multimodal/generation/test_phi4_multimodal.py b/tests/models/multimodal/generation/test_phi4_multimodal.py deleted file mode 100644 index 62456221711ed..0000000000000 --- a/tests/models/multimodal/generation/test_phi4_multimodal.py +++ /dev/null @@ -1,281 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import os -from collections.abc import Sequence - -import librosa -import pytest -from huggingface_hub import snapshot_download - -from vllm.assets.image import ImageAsset -from vllm.lora.request import LoRARequest -from vllm.multimodal.image import rescale_image_size - -from ....conftest import ( - IMAGE_ASSETS, - HfRunner, - PromptAudioInput, - PromptImageInput, - VllmRunner, -) -from ....utils import large_gpu_test -from ...utils import check_logprobs_close - -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts( - { - "stop_sign": "<|user|>\n<|image|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501 - "cherry_blossom": "<|user|>\n<|image|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501 - } -) -HF_MULTIIMAGE_IMAGE_PROMPT = ( - "<|user|>\n<|image|>\n<|image|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501 -) - -model_path = snapshot_download( - "microsoft/Phi-4-multimodal-instruct", revision="refs/pr/70" -) -# Since the vision-lora and speech-lora co-exist with the base model, -# we have to manually specify the path of the lora weights. -vision_lora_path = os.path.join(model_path, "vision-lora") -speech_question = os.path.join( - model_path, "examples", "what_is_shown_in_this_image.wav" -) -models = [model_path] - -target_dtype = "half" - - -def run_test( - hf_runner: type[HfRunner], - vllm_runner: type[VllmRunner], - inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]], - model: str, - *, - max_model_len: int, - dtype: str, - max_tokens: int, - num_logprobs: int, - mm_limit: int, - tensor_parallel_size: int, - distributed_executor_backend: str | None = None, -): - """Inference result should be the same between hf and vllm. - - All the image fixtures for the test are from IMAGE_ASSETS. - For huggingface runner, we provide the PIL images as input. - For vllm runner, we provide MultiModalDataDict objects - and corresponding MultiModalConfig as input. - Note, the text input is also adjusted to abide by vllm contract. - The text output is sanitized to be able to compare with hf. - """ - # NOTE: take care of the order. run vLLM first, and then run HF. - # vLLM needs a fresh new process without cuda initialization. - # if we run HF first, the cuda initialization will be done and it - # will hurt multiprocessing backend with fork method (the default method). - # max_model_len should be greater than image_feature_size - with vllm_runner( - model, - task="generate", - max_model_len=max_model_len, - max_num_seqs=2, - dtype=dtype, - limit_mm_per_prompt={"image": mm_limit}, - tensor_parallel_size=tensor_parallel_size, - distributed_executor_backend=distributed_executor_backend, - enable_lora=True, - max_lora_rank=320, - gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI - enforce_eager=True, - trust_remote_code=False, - ) as vllm_model: - lora_request = LoRARequest("vision", 1, vision_lora_path) - vllm_outputs_per_case = [ - vllm_model.generate_greedy_logprobs( - prompts, - max_tokens, - num_logprobs=num_logprobs, - images=images, - audios=audios, - lora_request=lora_request, - ) - for prompts, images, audios in inputs - ] - - with hf_runner(model, dtype=dtype) as hf_model: - hf_model.model.load_adapter( - vision_lora_path, - adapter_name="vision", - ) - hf_processor = hf_model.processor - eos_token_id = hf_processor.tokenizer.eos_token_id - hf_outputs_per_case = [ - hf_model.generate_greedy_logprobs_limit( - prompts, - max_tokens, - num_logprobs=num_logprobs, - images=images, - audios=audios, - eos_token_id=eos_token_id, - ) - for prompts, images, audios in inputs - ] - - for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case): - check_logprobs_close( - outputs_0_lst=hf_outputs, - outputs_1_lst=vllm_outputs, - name_0="hf", - name_1="vllm", - ) - - -@pytest.mark.parametrize("model", models) -@pytest.mark.parametrize( - "size_factors", - [ - # No image - [], - # Single-scale - [1.0], - # Single-scale, batched - [1.0, 1.0, 1.0], - # Multi-scale - [0.25, 0.5, 1.0], - ], -) -@pytest.mark.parametrize("dtype", [target_dtype]) -@pytest.mark.parametrize("max_model_len", [12800]) -@pytest.mark.parametrize("max_tokens", [128]) -@pytest.mark.parametrize("num_logprobs", [10]) -def test_models( - hf_runner, - vllm_runner, - image_assets, - model, - size_factors, - dtype: str, - max_model_len: int, - max_tokens: int, - num_logprobs: int, -) -> None: - images = [asset.pil_image for asset in image_assets] - - inputs_per_image = [ - ( - [prompt for _ in size_factors], - [rescale_image_size(image, factor) for factor in size_factors], - None, - ) - for image, prompt in zip(images, HF_IMAGE_PROMPTS) - ] - - run_test( - hf_runner, - vllm_runner, - inputs_per_image, - model, - dtype=dtype, - max_model_len=max_model_len, - max_tokens=max_tokens, - num_logprobs=num_logprobs, - mm_limit=1, - tensor_parallel_size=1, - ) - - -@large_gpu_test(min_gb=48) -@pytest.mark.parametrize("model", models) -@pytest.mark.parametrize( - "size_factors", - [ - # No image - # [], - # Single-scale - [1.0], - # Single-scale, batched - [1.0, 1.0, 1.0], - # Multi-scale - [0.25, 0.5, 1.0], - ], -) -@pytest.mark.parametrize("dtype", [target_dtype]) -@pytest.mark.parametrize("max_model_len", [25600]) -@pytest.mark.parametrize("max_tokens", [128]) -@pytest.mark.parametrize("num_logprobs", [10]) -def test_multi_images_models( - hf_runner, - vllm_runner, - image_assets, - model, - size_factors, - dtype: str, - max_model_len: int, - max_tokens: int, - num_logprobs: int, -) -> None: - images = [asset.pil_image for asset in image_assets] - - inputs_per_case = [ - ( - [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors], - [ - [rescale_image_size(image, factor) for image in images] - for factor in size_factors - ], - None, - ), - ] - - run_test( - hf_runner, - vllm_runner, - inputs_per_case, - model, - dtype=dtype, - max_model_len=max_model_len, - max_tokens=max_tokens, - num_logprobs=num_logprobs, - mm_limit=2, - tensor_parallel_size=1, - ) - - -@pytest.mark.parametrize("model", models) -@pytest.mark.parametrize("dtype", [target_dtype]) -@pytest.mark.parametrize("max_model_len", [12800]) -@pytest.mark.parametrize("max_tokens", [128]) -@pytest.mark.parametrize("num_logprobs", [10]) -def test_vision_speech_models( - hf_runner, - vllm_runner, - model, - dtype: str, - max_model_len: int, - max_tokens: int, - num_logprobs: int, -) -> None: - # use the example speech question so that the model outputs are reasonable - audio = librosa.load(speech_question, sr=16000) - image = ImageAsset("cherry_blossom").pil_image.convert("RGB") - - inputs_vision_speech = [ - ( - ["<|user|><|image|><|audio|><|end|><|assistant|>"], - [image], - [audio], - ), - ] - - run_test( - hf_runner, - vllm_runner, - inputs_vision_speech, - model, - dtype=dtype, - max_model_len=max_model_len, - max_tokens=max_tokens, - num_logprobs=num_logprobs, - mm_limit=1, - tensor_parallel_size=1, - ) diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 8ef1fba8df3e3..6b9d388f2b9b4 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -396,28 +396,6 @@ def test_processing_correctness( ) -# Phi4MultimodalForCausalLM share same model repo with original format -# Phi4MMForCausalLM, so we add it as a separate test case -# Remove this test after conversion PR merged: -# https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/70 -@pytest.mark.parametrize("model_arch", ["Phi4MultimodalForCausalLM"]) -@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0]) -@pytest.mark.parametrize("num_batches", [32]) -@pytest.mark.parametrize("simplify_rate", [1.0]) -def test_processing_correctness_phi4_multimodal( - model_arch: str, - hit_rate: float, - num_batches: int, - simplify_rate: float, -): - _test_processing_correctness( - model_arch, - hit_rate=hit_rate, - num_batches=num_batches, - simplify_rate=simplify_rate, - ) - - def _assert_inputs_equal( a: MultiModalInputs, b: MultiModalInputs, diff --git a/tests/models/registry.py b/tests/models/registry.py index bf88bac209808..b9f9945eb5fb8 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -771,10 +771,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Phi4MMForCausalLM": _HfExamplesInfo( "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True ), - "Phi4MultimodalForCausalLM": _HfExamplesInfo( - "microsoft/Phi-4-multimodal-instruct", - revision="refs/pr/70", - ), "PixtralForConditionalGeneration": _HfExamplesInfo( "mistralai/Pixtral-12B-2409", extras={ diff --git a/vllm/model_executor/models/phi4_multimodal.py b/vllm/model_executor/models/phi4_multimodal.py deleted file mode 100644 index 0f1230a55bae6..0000000000000 --- a/vllm/model_executor/models/phi4_multimodal.py +++ /dev/null @@ -1,1447 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import math -from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Any, Literal, TypeAlias - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from transformers import ( - BatchFeature, - Phi4MultimodalAudioConfig, - Phi4MultimodalConfig, - Phi4MultimodalFeatureExtractor, - Phi4MultimodalImageProcessorFast, -) -from transformers import Phi4MultimodalProcessor as Phi4MMProcessor -from transformers.models.phi4_multimodal.modeling_phi4_multimodal import ( - Phi4MultimodalAudioConvModule, - Phi4MultimodalAudioNemoConvSubsampling, - Phi4MultimodalAudioRelativeAttentionBias, - adaptive_enc_mask, - unfold_tensor, -) - -from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.distributed import ( - divide, - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, -) -from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - MergedColumnParallelLinear, - QKVParallelLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm.model_executor.models.module_mapping import MultiModelKeys -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import ( - MultiModalDataDict, - MultiModalFieldConfig, - MultiModalKwargsItems, - NestedTensors, -) -from vllm.multimodal.parse import ( - AudioProcessorItems, - ImageEmbeddingItems, - ImageProcessorItems, - ImageSize, - MultiModalDataItems, - MultiModalDataParser, -) -from vllm.multimodal.processing import ( - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, -) -from vllm.multimodal.profiling import BaseDummyInputsBuilder -from vllm.sequence import IntermediateTensors -from vllm.utils.tensor_schema import TensorSchema, TensorShape - -from .idefics2_vision_model import Idefics2VisionTransformer -from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal -from .utils import ( - AutoWeightsLoader, - WeightsMapper, - init_vllm_registered_model, - maybe_prefix, -) - -_AUDIO_MAX_SOUNDFILE_SIZE = 241_000 - - -def _get_padding_size( - orig_width: int, orig_height: int, target_height: int, target_width: int -): - ratio_width = target_width / orig_width - ratio_height = target_height / orig_height - - if ratio_width < ratio_height: - padding_width = 0 - padding_height = target_height - int(orig_height * ratio_width) - else: - padding_width = target_width - int(orig_width * ratio_height) - padding_height = 0 - return padding_height, padding_width - - -class Phi4MMProjector(nn.Module): - def __init__(self, input_size: int, hidden_size: int): - super().__init__() - self.up = ColumnParallelLinear(input_size, hidden_size) - self.down = RowParallelLinear(hidden_size, hidden_size) - self.act = get_act_fn("gelu") - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x, _ = self.up(x) - x = self.act(x) - x, _ = self.down(x) - return x - - -class Phi4MMImageEmbedding(nn.Module): - """Image embedding.""" - - def __init__(self, config: Phi4MultimodalConfig): - super().__init__() - self.config = config - self.layer_idx = config.vision_config.feature_layer - self.crop_size = config.vision_config.crop_size - self.image_dim_out = config.vision_config.hidden_size - - n_patches = config.vision_config.image_size // config.vision_config.patch_size - if n_patches % 2 != 0: - self.img_processor_padding = nn.ReflectionPad2d((0, 1, 0, 1)) - n_patches += 1 - self.num_img_tokens = (n_patches // 2) ** 2 - - num_hidden_layers = ( - config.vision_config.num_hidden_layers + self.layer_idx + 1 - if self.layer_idx < 0 - else self.layer_idx + 1 - ) - self.img_processor = Idefics2VisionTransformer( - config.vision_config, - require_post_norm=False, - num_hidden_layers_override=num_hidden_layers, - ) - self.image_token_compression = nn.AvgPool2d(kernel_size=2, stride=2) - self.img_projection = Phi4MMProjector(self.image_dim_out, config.hidden_size) - self.global_img_feature_extensor = nn.Parameter( - torch.zeros([1, 1, self.image_dim_out]) - ) - self.sub_img_feature_extensor = nn.Parameter( - torch.zeros([1, 1, 1, self.image_dim_out]) - ) - - def get_img_features( - self, - img_embeds: torch.FloatTensor, - attention_mask: torch.Tensor | None = None, - ) -> torch.FloatTensor: - img_feature = self.img_processor( - img_embeds, patch_attention_mask=attention_mask - ) - - patch_feature = img_feature - # reshape to 2D tensor - width = int(math.sqrt(patch_feature.size(1))) - patch_feature = patch_feature.view(-1, width, width, patch_feature.size(-1)) - # convert to NCHW - patch_feature = patch_feature.permute(0, 3, 1, 2) - if getattr(self, "img_processor_padding", None) is not None: - patch_feature = self.img_processor_padding(patch_feature) - patch_feature = self.image_token_compression(patch_feature) - # convert to NHWC - patch_feature = patch_feature.permute(0, 2, 3, 1) - patch_feature = patch_feature.view( - -1, patch_feature.size(1) * patch_feature.size(2), patch_feature.size(-1) - ) - return patch_feature - - def forward( - self, - image_pixel_values: torch.FloatTensor, - image_sizes: torch.Tensor | None = None, - image_attention_mask: torch.Tensor | None = None, - ) -> torch.FloatTensor: - image_pixel_values = image_pixel_values.to( - self.img_processor.embeddings.patch_embedding.weight.dtype - ) - - target_device = self.img_projection.up.bias.device - target_dtype = self.img_projection.up.bias.dtype - - batch_size = image_pixel_values.shape[0] - - img_features = self.get_img_features( - image_pixel_values.flatten(0, 1), - attention_mask=image_attention_mask.flatten(0, 1).to( - dtype=bool, device=target_device - ), - ) - base_feat_size = int(np.sqrt(img_features.shape[1])) - img_features = img_features.view( - batch_size, -1, base_feat_size**2, self.image_dim_out - ) - image_sizes = image_sizes.view(-1, 2) - - output_imgs = [] - for idx in range(batch_size): - height, width = image_sizes[idx] - height_ratio = height // self.crop_size - width_ratio = width // self.crop_size - area_ratio = height_ratio * width_ratio - - global_img = img_features[idx, :1] - global_img = global_img.reshape( - 1, base_feat_size, base_feat_size, self.image_dim_out - ).contiguous() - temporary_extensor = self.sub_img_feature_extensor.repeat( - 1, base_feat_size, 1, 1 - ) - global_img = torch.cat([global_img, temporary_extensor], dim=2).reshape( - 1, -1, self.image_dim_out - ) - - sub_img = img_features[idx, 1:] - sub_img = sub_img[:area_ratio] - sub_img = ( - sub_img.reshape( - height_ratio, - width_ratio, - base_feat_size, - base_feat_size, - self.image_dim_out, - ) - .transpose(1, 2) - .reshape( - 1, - height_ratio * base_feat_size, - width_ratio * base_feat_size, - self.image_dim_out, - ) - .contiguous() - ) - - if image_attention_mask is not None: - reshaped_image_attention_mask = ( - image_attention_mask[idx, 1 : area_ratio + 1, 0::2, 0::2] - .reshape(height_ratio, width_ratio, base_feat_size, base_feat_size) - .transpose(1, 2) - .reshape( - 1, height_ratio * base_feat_size, width_ratio * base_feat_size - ) - ) - useful_height = int(reshaped_image_attention_mask[0, :, 0].sum().item()) - useful_width = int(reshaped_image_attention_mask[0, 0, :].sum().item()) - sub_img = sub_img[:, :useful_height, :useful_width] - temporary_extensor = self.sub_img_feature_extensor.repeat( - 1, useful_height, 1, 1 - ) - else: - temporary_extensor = self.sub_img_feature_extensor.repeat( - 1, height_ratio * base_feat_size, 1, 1 - ) - - sub_img = torch.cat([sub_img, temporary_extensor], dim=2).reshape( - 1, -1, self.image_dim_out - ) - - # Merge global and sub - output_imgs.append( - torch.cat( - [sub_img, self.global_img_feature_extensor, global_img], dim=1 - ) - ) - - img_set_tensor = [] - for output_img in output_imgs: - output_img = output_img.to(device=target_device, dtype=target_dtype) - img_feature_proj = self.img_projection(output_img) - img_set_tensor.append(img_feature_proj.flatten(0, 1)) - - return img_set_tensor - - -class Phi4MultimodalAudioMLP(nn.Module): - def __init__( - self, - config: Phi4MultimodalAudioConfig, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.layer_norm = nn.LayerNorm(config.hidden_size) - self.act_fn = MulAndSilu() - self.gate_up_proj = MergedColumnParallelLinear( - config.hidden_size, - [config.intermediate_size] * 2, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.gate_up_proj", - ) - self.down_proj = RowParallelLinear( - config.intermediate_size, - config.hidden_size, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.down_proj", - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - hidden_states = self.layer_norm(hidden_states) - hidden_states, _ = self.gate_up_proj(hidden_states) - hidden_states = self.act_fn(hidden_states) - hidden_states, _ = self.down_proj(hidden_states) - return hidden_states - - -class Phi4MultimodalAudioAttention(nn.Module): - def __init__( - self, - config: Phi4MultimodalAudioConfig, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.total_num_heads = config.num_attention_heads - self.head_dim = self.embed_dim // self.total_num_heads - if self.head_dim * self.total_num_heads != self.embed_dim: - raise ValueError( - "embed_dim must be divisible by num_heads " - f"(got `embed_dim`: {self.embed_dim} and `num_heads`:" - f" {self.num_heads})." - ) - self.scale = self.head_dim**-0.5 - - self.qkv_proj = QKVParallelLinear( - hidden_size=self.embed_dim, - head_size=self.head_dim, - total_num_heads=self.total_num_heads, - quant_config=quant_config, - prefix=f"{prefix}.qkv_proj", - ) - - self.o_proj = RowParallelLinear( - input_size=self.embed_dim, - output_size=self.embed_dim, - quant_config=quant_config, - prefix=f"{prefix}.out_proj", - ) - - self.tp_size = get_tensor_model_parallel_world_size() - self.tp_rank = get_tensor_model_parallel_rank() - self.num_heads = divide(self.total_num_heads, self.tp_size) - - def split_attn_mask(self, attention_mask: torch.Tensor) -> torch.Tensor: - start_idx = self.num_heads * self.tp_rank - end_idx = self.num_heads * (self.tp_rank + 1) - return attention_mask[:, start_idx:end_idx] - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor, - ) -> torch.Tensor: - qkv_states, _ = self.qkv_proj(hidden_states) - query, key, value = qkv_states.chunk(3, dim=-1) - - bsz, seq_len, _ = query.size() - query = query.view(bsz, seq_len, self.num_heads, self.head_dim) - key = key.view(bsz, seq_len, self.num_heads, self.head_dim) - value = value.view(bsz, seq_len, self.num_heads, self.head_dim) - query, key, value = (x.transpose(1, 2) for x in (query, key, value)) - - attention_mask = self.split_attn_mask(attention_mask) - out = F.scaled_dot_product_attention( - query, - key, - value, - scale=self.scale, - attn_mask=attention_mask, - ) - out = out.transpose(1, 2).reshape(bsz, seq_len, -1) - - attn_output, _ = self.o_proj(out) - - return attn_output - - -class Phi4MultimodalAudioConformerEncoderLayer(nn.Module): - def __init__(self, config: Phi4MultimodalAudioConfig): - super().__init__() - - self.feed_forward_in = Phi4MultimodalAudioMLP(config) - self.self_attn = Phi4MultimodalAudioAttention(config) - self.conv = Phi4MultimodalAudioConvModule(config) - self.feed_forward_out = Phi4MultimodalAudioMLP(config) - self.layer_norm_att = nn.LayerNorm(config.hidden_size) - self.layer_norm = nn.LayerNorm(config.hidden_size) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor, - ) -> torch.Tensor: - residual = hidden_states + 0.5 * self.feed_forward_in(hidden_states) - hidden_states = self.layer_norm_att(residual) - - hidden_states = residual + self.self_attn(hidden_states, attention_mask) - hidden_states = hidden_states + self.conv(hidden_states) - hidden_states = hidden_states + 0.5 * self.feed_forward_out(hidden_states) - - out = self.layer_norm(hidden_states) - - return out - - -class Phi4MMAudioMeanVarianceNormLayer(nn.Module): - """Mean/variance normalization layer. - - Will subtract mean and multiply input by inverted standard deviation. - Typically used as a very first layer in a model. - - Args: - config: [Phi4MultimodalAudioConfig](https://huggingface.co/docs/transformers/model_doc/phi4_multimodal#transformers.Phi4MultimodalAudioConfig) - object containing model parameters. - """ - - def __init__(self, config: Phi4MultimodalAudioConfig): - super().__init__() - self.global_mean = nn.Parameter(torch.zeros(config.input_size)) - self.global_invstd = nn.Parameter(torch.ones(config.input_size)) - - def forward(self, input_: torch.Tensor) -> torch.Tensor: - """MeanVarianceNormLayer Forward - - Args: - input_: torch.Tensor - input tensor. - """ - return (input_ - self.global_mean) * self.global_invstd - - -class Phi4MultimodalAudioModel(nn.Module): - def __init__(self, config: Phi4MultimodalAudioConfig): - super().__init__() - self.config = config - - self.encoder_embedding = Phi4MMAudioMeanVarianceNormLayer(config) - self.embed = Phi4MultimodalAudioNemoConvSubsampling(config) - self.relative_attention_bias_layer = Phi4MultimodalAudioRelativeAttentionBias( - config - ) - self.encoders = nn.ModuleList( - [ - Phi4MultimodalAudioConformerEncoderLayer(config) - for _ in range(config.num_blocks) - ] - ) - - def _streaming_mask( - self, - seq_len: int, - batch_size: int, - chunk_size: int, - left_chunk: int, - ): - # Create mask matrix for streaming - # S stores start index. if chunksize is 18, s is [0,18,36,....] - chunk_start_idx = np.arange(0, seq_len, chunk_size) - - enc_streaming_mask = ( - adaptive_enc_mask(seq_len, chunk_start_idx, left_window=left_chunk) - .unsqueeze(0) - .expand([batch_size, -1, -1]) - ) - return enc_streaming_mask - - def forward_embeddings( - self, - hidden_states: torch.Tensor, - masks: torch.Tensor, - ): - """Forwarding the inputs through the top embedding layers""" - seq_len = math.ceil(hidden_states.shape[1] / self.config.time_reduction) - if seq_len <= 0: - raise ValueError( - f"Sequence length after time reduction is invalid: {seq_len}." - "Your input feature is too short." - ) - - batch_size = hidden_states.shape[0] - - enc_streaming_mask = self._streaming_mask( - seq_len, batch_size, self.config.chunk_size, self.config.left_chunk - ) - enc_streaming_mask = enc_streaming_mask.to(hidden_states.device) - - hidden_states, masks = self.embed(hidden_states, masks) - - streaming_mask = enc_streaming_mask - if streaming_mask is not None and masks is not None: - hs_mask = masks & streaming_mask - elif masks is not None: - hs_mask = masks - else: - hs_mask = streaming_mask - - return hidden_states, hs_mask, masks - - def calculate_hs_mask( - self, hidden_states: torch.Tensor, device: torch.device, mask: torch.Tensor - ): - max_audio_length = hidden_states.shape[1] - batch_size = hidden_states.shape[0] - enc_streaming_mask = self._streaming_mask( - max_audio_length, batch_size, self.config.chunk_size, self.config.left_chunk - ) - enc_streaming_mask = enc_streaming_mask.to(device) - if mask is None: - return enc_streaming_mask - - feature_lens = mask.sum(1) - padding_length = feature_lens - pad_mask = torch.arange(0, max_audio_length, device=device).expand( - padding_length.size(0), -1 - ) < padding_length.unsqueeze(1) - pad_mask = pad_mask.unsqueeze(1) - pad_mask = pad_mask & enc_streaming_mask - return pad_mask - - def forward(self, hidden_states: torch.Tensor, mask: torch.Tensor | None = None): - hidden_states = self.encoder_embedding(hidden_states) - hidden_states, hs_mask, mask = self.forward_embeddings(hidden_states, mask) - - unfolded = False - bs, seq_len, _ = hidden_states.shape - max_seq_len = 500 # maximum position for absolute positional encoding - if seq_len > max_seq_len: - # audio sequence is longer than max_seq_len, - # unfold it into chunks of max_seq_len - unfolded = True - # the unfold op will drop residual frames, - # pad it to the multiple of max_seq_len - if seq_len % max_seq_len > 0: - chunk_pad_size = max_seq_len - (seq_len % max_seq_len) - else: - chunk_pad_size = 0 - if chunk_pad_size > 0: - hidden_states_pad = F.pad( - hidden_states, (0, 0, 0, chunk_pad_size), "constant", 0 - ) - hidden_states = hidden_states_pad.to(hidden_states.device) - - hidden_states = unfold_tensor(hidden_states, max_seq_len) - masks_unfold = None - if mask is not None: - # revise hs_mask here because the previous calculated hs_mask - # did not consider extra pad - subsampled_pad_mask = mask.squeeze(1) # [bz, subsampled_unmask_seq_len] - extra_padded_subsamlped_pad_mask = F.pad( - subsampled_pad_mask, (0, chunk_pad_size), "constant", False - ) # extra padding to the pad mask - extra_padded_subsamlped_pad_mask = ( - extra_padded_subsamlped_pad_mask.unsqueeze(-1).float() - ) - masks_unfold = unfold_tensor( - extra_padded_subsamlped_pad_mask, max_seq_len - ) # unfold the pad mask like we did to the input tensor - masks_unfold = masks_unfold.squeeze( - -1 - ).bool() # unfold op does not support bool tensor - hs_mask = self.calculate_hs_mask( - hidden_states, hidden_states.device, masks_unfold - ) # calculate hs_mask based on the unfolded pad mask - - relative_attention_bias = self.relative_attention_bias_layer(hidden_states) - attention_mask = hs_mask.unsqueeze(1) + relative_attention_bias - - for layer in self.encoders: - hidden_states = layer(hidden_states, attention_mask) - - if unfolded: - embed_dim = hidden_states.shape[-1] - hidden_states = hidden_states.reshape(bs, -1, embed_dim) - # if we ever padded before unfolding, we need to remove the padding - if chunk_pad_size > 0: - hidden_states = hidden_states[:, :-chunk_pad_size, :] - - return hidden_states - - -class Phi4MMAudioEmbedding(nn.Module): - def __init__(self, config: Phi4MultimodalConfig): - super().__init__() - self.config = config - self.layer_idx = config.audio_config.feature_layer - - self.encoder = Phi4MultimodalAudioModel(config.audio_config) - - audio_config = config.audio_config - proj_input_size = audio_config.hidden_size * audio_config.downsample_rate - self.vision_speech_projection = Phi4MMProjector( - proj_input_size, config.hidden_size - ) - self.speech_projection = Phi4MMProjector(proj_input_size, config.hidden_size) - - def get_projection( - self, - audio_projection_mode: Literal["speech", "vision"], - ) -> Phi4MMProjector: - if audio_projection_mode == "speech": - return self.speech_projection - elif audio_projection_mode == "vision": - return self.vision_speech_projection - - def forward( - self, - audio_input_features: torch.FloatTensor, - audio_embed_sizes=None, - audio_attention_mask=None, - audio_projection_mode="speech", - ) -> torch.FloatTensor: - audio_projection = self.get_projection(audio_projection_mode) - - target_device = audio_projection.up.bias.device - target_dtype = audio_projection.up.bias.dtype - - audio_input_features = audio_input_features.to( - device=target_device, dtype=target_dtype - ) - - audio_encoder_hidden_states = self.encoder( - audio_input_features, audio_attention_mask - ) - audio_embeds = audio_projection(audio_encoder_hidden_states) - - return audio_embeds.flatten(0, 1) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - -class Phi4MMImagePixelInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of images - - p: Number of patches (1 + num_patches) - - c: Number of channels (3) - - h: Height of each image patch - - w: Width of each image patch - - nc: Number of crops - - H_mask: Height of attention mask - - W_mask: Width of attention mask - """ - - type: Literal["pixel_values"] - - pixel_values: Annotated[ - torch.Tensor | list[torch.Tensor], - TensorShape( - "bn", "p", 3, "h", "w", dynamic_dims={"p"} - ), # may be different per batch and image - ] - - image_sizes: Annotated[ - torch.Tensor, - TensorShape("bn", 2), # (height, width) - ] - - num_img_tokens: Annotated[ - list[int], - TensorShape("bn"), - ] - - image_attention_mask: Annotated[ - torch.Tensor, - TensorShape("bn", "nc", 32, 32), # H_mask, W_mask - ] - - -class Phi4MMImageEmbeddingInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of images - - f: Image feature size - - h: Hidden size (must match language model backbone) - """ - - type: Literal["image_embeds"] - - data: Annotated[ - torch.Tensor | list[torch.Tensor], - TensorShape("bn", "f", "h"), - ] - - -class Phi4MMAudioFeatureInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of audios - - f: Number of Mel filterbank bins (80) - - t: Time frames (M) - """ - - type: Literal["audio_features"] - - audio_features: Annotated[ - torch.Tensor | list[torch.Tensor], - TensorShape("bn", "t", 80, dynamic_dims={"t"}), - ] - - -class Phi4MMAudioEmbeddingInputs(TensorSchema): - """ - Dimensions: - - b: Batch size - - n: Number of audios - - f: Audio feature size - - h: Hidden size (must match language model backbone) - """ - - type: Literal["audio_embeds"] - - data: Annotated[ - NestedTensors, - TensorShape("b", "n", "f", "h"), - ] - - -Phi4MMImageInput: TypeAlias = Phi4MMImagePixelInputs | Phi4MMImageEmbeddingInputs -Phi4MMAudioInputs: TypeAlias = Phi4MMAudioFeatureInputs | Phi4MMAudioEmbeddingInputs - - -def cat_with_pad(tensors, dim, padding_value=0): - """ - cat along dim, while pad to max for all other dims - """ - ndim = tensors[0].dim() - assert all(t.dim() == ndim for t in tensors[1:]), ( - "All tensors must have the same number of dimensions" - ) - - out_size = [max(t.shape[i] for t in tensors) for i in range(ndim)] - out_size[dim] = sum(t.shape[dim] for t in tensors) - output = tensors[0].new_full(out_size, padding_value) - - index = 0 - for t in tensors: - # Create a slice list where every dimension except dim is full slice - slices = [slice(0, t.shape[d]) for d in range(ndim)] - # Update only the concat dimension slice - slices[dim] = slice(index, index + t.shape[dim]) - - output[slices] = t - index += t.shape[dim] - - return output - - -class Phi4MMProcessingInfo(BaseProcessingInfo): - def get_hf_config(self) -> Phi4MultimodalConfig: - return self.ctx.get_hf_config(Phi4MultimodalConfig) - - def get_hf_processor(self, **kwargs: object) -> Phi4MMProcessor: - return self.ctx.get_hf_processor(Phi4MMProcessor, **kwargs) - - def get_feature_extractor(self, **kwargs: object) -> Phi4MultimodalFeatureExtractor: - return self.get_hf_processor(**kwargs).audio_processor - - def get_image_processor( - self, - processor: Phi4MMProcessor | None = None, - ) -> Phi4MultimodalImageProcessorFast: - if processor is None: - processor = self.get_hf_processor() - return processor.image_processor - - def get_dynamic_hd( - self, - processor: Phi4MMProcessor | None = None, - ) -> int: - return self.get_image_processor(processor).dynamic_hd - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"audio": None, "image": None} - - def _find_target_aspect_ratio( - self, - orig_width: int, - orig_height: int, - image_size: int, - max_num: int, - min_num: int, - ): - w_crop_num = math.ceil(orig_width / float(image_size)) - h_crop_num = math.ceil(orig_height / float(image_size)) - if w_crop_num * h_crop_num > max_num: - aspect_ratio = orig_width / orig_height - - # calculate the existing image aspect ratio - target_ratios = set( - (i, j) - for i in range(1, max_num + 1) - for j in range(1, max_num + 1) - if i * j <= max_num and i * j >= min_num - ) - target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) - - # find the closest aspect ratio to the target - image_processor = self.get_image_processor() - target_aspect_ratio = image_processor.find_closest_aspect_ratio( - aspect_ratio, - target_ratios, - orig_width, - orig_height, - image_size, - ) - - # calculate the target width and height - target_width = image_size * target_aspect_ratio[0] - target_height = image_size * target_aspect_ratio[1] - else: - target_width = image_size * w_crop_num - target_height = image_size * h_crop_num - target_aspect_ratio = (w_crop_num, h_crop_num) - return target_aspect_ratio, target_height, target_width - - def _compute_num_image_tokens( - self, - orig_width: int, - orig_height: int, - dynamic_hd_size: int, - vit_image_size: int, - vit_patch_size: int, - token_compression_factor: int = 2, - ): - """ - compute the number of tokens an image is expected to take up considering - the image encoder architecture and exclude output features containing - only padding pixels - - for siglip, vit_image_size=448, vit_patch_size=14, so output will be - 32x32 feature map - NOTE right now, Phi4MM uses hard-coded token_compression_factor=2 - """ - assert vit_image_size % vit_patch_size == 0, ( - "vit_image_size must be divisible by vit_patch_size" - ) - assert vit_image_size // vit_patch_size % token_compression_factor == 0, ( - "vit_image_size // vit_patch_size must be divisible by " - "token_compression_factor" - ) - - target_aspect_ratio, target_height, target_width = ( - self._find_target_aspect_ratio( - orig_width, orig_height, vit_image_size, dynamic_hd_size, min_num=1 - ) - ) - assert target_aspect_ratio[0] * vit_image_size == target_width, ( - f"{target_aspect_ratio[0]} * {vit_image_size} != {target_width}" - ) - assert target_aspect_ratio[1] * vit_image_size == target_height, ( - f"{target_aspect_ratio[1]} * {vit_image_size} != {target_height}" - ) - assert ( - target_height % vit_image_size == 0 and target_width % vit_image_size == 0 - ) - - padding_height, padding_width = _get_padding_size( - orig_width, orig_height, target_height, target_width - ) - assert padding_width == 0 or padding_height == 0, ( - "padding_width or padding_height must be 0" - ) - - target_feat_width = target_width // vit_patch_size - target_feat_height = target_height // vit_patch_size - if padding_width >= vit_patch_size: - assert padding_height == 0, "padding_height not 0" - non_pad_feat_width = target_feat_width - math.floor( - padding_width / vit_patch_size - ) - non_pad_feat_height = target_feat_height - elif padding_height >= vit_patch_size: - assert padding_width == 0, "padding_width not 0" - non_pad_feat_height = target_feat_height - math.floor( - padding_height / vit_patch_size - ) - non_pad_feat_width = target_feat_width - else: - # small padding shorter than a vit patch - non_pad_feat_width = target_feat_width - non_pad_feat_height = target_feat_height - - feat_width = non_pad_feat_width // token_compression_factor - feat_height = non_pad_feat_height // token_compression_factor - # NOTE it's possible that the non-padding feature is not divisible - if non_pad_feat_width % token_compression_factor != 0: - feat_width += 1 - if non_pad_feat_height % token_compression_factor != 0: - feat_height += 1 - num_hd_patch_tokens = feat_width * feat_height - num_hd_newline_tokens = feat_height - vit_feature_size = vit_image_size // vit_patch_size - num_global_image_tokens = (vit_feature_size // token_compression_factor) ** 2 - num_sep_tokens = 1 - num_global_image_newline_tokens = vit_feature_size // token_compression_factor - - return ( - num_global_image_tokens - + num_sep_tokens - + num_hd_patch_tokens - + num_hd_newline_tokens - + num_global_image_newline_tokens - ) - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - processor: Phi4MMProcessor | None = None, - ) -> int: - hf_config = self.get_hf_config() - vision_config = hf_config.vision_config - vit_image_size = vision_config.image_size - vit_patch_size = vision_config.patch_size - - dynamic_hd_size = self.get_dynamic_hd(processor=processor) - - # we use default `token_compression_factor=2`, - # since it's not in HF vision config. - image_num_tokens = self._compute_num_image_tokens( - image_width, - image_height, - dynamic_hd_size=dynamic_hd_size, - vit_image_size=vit_image_size, - vit_patch_size=vit_patch_size, - ) - - return image_num_tokens - - def get_image_size_with_most_features( - self, - processor: Phi4MMProcessor | None = None, - ) -> ImageSize: - vit_image_size = self.get_hf_config().vision_config.image_size - - max_side = vit_image_size * self.get_dynamic_hd(processor=processor) - return ImageSize(height=max_side, width=vit_image_size) - - def get_audio_num_frames(self, audio_len: int, sr: float) -> int: - """ - Compute the output size of the `extract_features` method. - - Args: - audio_len (int): Length of the input waveform in samples. - sr (float): Sampling rate of the waveform, either 16000 or 8000. - - Returns: - tuple (int, int): Output size as (T, D), where: - T: Number of time frames. - D: Number of Mel filterbank bins (80). - """ - - # Resample to 16000 or 8000 if needed - if sr > 16000: - audio_len //= sr // 16000 - elif 8000 <= sr < 16000: - # We'll resample to 16K from 8K - audio_len *= 2 - elif sr < 8000: - raise RuntimeError(f"Unsupported sample rate {sr}") - - # Spectrogram parameters for 16 kHz - win_length = 400 # Frame length in samples - hop_length = 160 # Frame shift in samples - - # Calculate number of frames (T) - num_frames = (audio_len - win_length) // hop_length + 1 - if num_frames < 1: - raise ValueError("Waveform too short for given parameters.") - - # Return time frames (T) - return num_frames - - def _compute_audio_embed_size(self, audio_frames: int) -> int: - """ - Compute the size of audio embeddings from the number of audio frames. - """ - # `_compute_audio_embed_size` in audio_processor use torch for - # computation, therefore we re-implement it to use pythonic - # numeric computation to avoid extra tensor conversion. - audio_processor = self.get_feature_extractor() - audio_compression_rate = audio_processor.audio_compression_rate - audio_downsample_rate = audio_processor.audio_downsample_rate - - integer = audio_frames // audio_compression_rate - remainder = audio_frames % audio_compression_rate - result = integer + int(remainder > 0) - - integer = result // audio_downsample_rate - remainder = result % audio_downsample_rate - result = integer + int(remainder > 0) # qformer compression - - return result - - -class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_audios = mm_counts.get("audio", 0) - num_images = mm_counts.get("image", 0) - - tokenizer = self.info.get_tokenizer() - image_tokens: str = tokenizer.image_token * num_images - audio_tokens: str = tokenizer.audio_token * num_audios - - return image_tokens + audio_tokens - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, - ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - num_images = mm_counts.get("image", 0) - - target_width, target_height = self.info.get_image_size_with_most_features() - - image_overrides = mm_options.get("image") if mm_options else None - audio_overrides = mm_options.get("audio") if mm_options else None - - mm_data = { - "image": self._get_dummy_images( - width=target_width, - height=target_height, - num_images=num_images, - overrides=image_overrides, - ), - "audio": self._get_dummy_audios( - length=_AUDIO_MAX_SOUNDFILE_SIZE, - num_audios=num_audios, - overrides=audio_overrides, - ), - } - - return mm_data - - -class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]): - def _get_data_parser(self) -> MultiModalDataParser: - feature_extractor = self.info.get_feature_extractor() - return MultiModalDataParser(target_sr=feature_extractor.sampling_rate) - - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], - ) -> BatchFeature: - if not mm_data: - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - audio_data = mm_data.pop("audios", []) - if audio_data: - mm_data["audio"] = audio_data - - processed_outputs = super()._call_hf_processor( - prompt, mm_data, mm_kwargs, tok_kwargs - ) - - if "image_pixel_values" in processed_outputs: - num_img_tokens = [ - self.info.get_num_image_tokens( - image_width=img_size[0], image_height=img_size[1] - ) - for img_size in processed_outputs["image_sizes"] - ] - processed_outputs["num_img_tokens"] = num_img_tokens - - if audio_data: - audio_features = processed_outputs["audio_input_features"] - sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate - feature_sizes = [ - self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data - ] - processed_outputs["audio_input_features"] = [ - audio_features[idx, :size] for idx, size in enumerate(feature_sizes) - ] - - return processed_outputs - - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - return dict( - image_pixel_values=MultiModalFieldConfig.batched("image"), - image_attention_mask=MultiModalFieldConfig.batched("image"), - image_sizes=MultiModalFieldConfig.batched("image"), - num_img_tokens=MultiModalFieldConfig.batched("image"), - audio_input_features=MultiModalFieldConfig.batched("audio"), - ) - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, Any], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - tokenizer = self.info.get_tokenizer() - image_token_id: int = tokenizer.vocab[tokenizer.image_token] - audio_token_id: int = tokenizer.vocab[tokenizer.audio_token] - - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - audio_processor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) - - def get_image_replacement_phi4mm(item_idx: int): - images = mm_items.get_items( - "image", (ImageEmbeddingItems, ImageProcessorItems) - ) - - if isinstance(images, ImageEmbeddingItems): - num_image_tokens = images.get_feature_size(item_idx) - else: - image_size = images.get_image_size(item_idx) - num_image_tokens = self.info.get_num_image_tokens( - image_width=image_size.width, - image_height=image_size.height, - processor=hf_processor, - ) - - return [image_token_id] * num_image_tokens - - def get_audio_replacement_phi4mm(item_idx: int): - audios = mm_items.get_items("audio", AudioProcessorItems) - # TODO(Isotr0py): support embedding inputs - audio_len = audios.get_audio_length(item_idx) - audio_frames = self.info.get_audio_num_frames( - audio_len, audio_processor.sampling_rate - ) - audio_embed_size = self.info._compute_audio_embed_size(audio_frames) - - return [audio_token_id] * audio_embed_size - - return [ - PromptReplacement( - modality="audio", - target=[audio_token_id], - replacement=get_audio_replacement_phi4mm, - ), - PromptReplacement( - modality="image", - target=[image_token_id], - replacement=get_image_replacement_phi4mm, - ), - ] - - -@MULTIMODAL_REGISTRY.register_processor( - Phi4MMMultiModalProcessor, - info=Phi4MMProcessingInfo, - dummy_inputs=Phi4MMDummyInputsBuilder, -) -class Phi4MultimodalForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal): - """ - Implements the Phi-4-multimodal-instruct model in vLLM. - """ - - merge_by_field_config = True - - packed_modules_mapping = { - "qkv_proj": [ - "qkv_proj", - ], - "gate_up_proj": [ - "gate_up_proj", - ], - } - - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_prefix={ - # Multimodal embedding - "model.embed_tokens_extend.": "", - # LLM backbone - "model.": "language_model.model.", - }, - orig_to_new_substr={ - # projection - ".img_projection_": ".img_projection.", - ".up_proj_for_speech.": ".speech_projection.up.", - ".up_proj_for_vision_speech.": ".vision_speech_projection.up.", - ".down_proj_for_speech.": ".speech_projection.down.", - ".down_proj_for_vision_speech.": ".vision_speech_projection.down.", - }, - ) - - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality.startswith("image"): - return "<|image|>" - if modality.startswith("audio"): - return "<|audio|>" - - raise ValueError("Only image or audio modality is supported") - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config = vllm_config.model_config.hf_config - multimodal_config = vllm_config.model_config.multimodal_config - self.config = config - self.multimodal_config = multimodal_config - - # TODO: Optionally initializes these for supporting input embeddings. - self.image_embed = Phi4MMImageEmbedding( - config, - # prefix=maybe_prefix(prefix, "image_embed"), - ) - self.audio_embed = Phi4MMAudioEmbedding( - config, - # prefix=maybe_prefix(prefix, "audio_embed"), - ) - - self.language_model = init_vllm_registered_model( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "language_model"), - architectures=["Phi3ForCausalLM"], - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - - def _parse_and_validate_audio_input( - self, **kwargs: object - ) -> Phi4MMAudioInputs | None: - """ - Parse and validate the audio input to the model. This handles both - audio features and audio embeddings, but only the former is used for - now. - - Args: - kwargs (object): Keyword arguments. - - Returns: - Optional[Phi4MMAudioInputs]: Parsed and validated audio inputs. - """ - audio_features = kwargs.pop("audio_input_features", None) - audio_embeds = kwargs.pop("audio_embeds", None) - - if audio_features is None and audio_embeds is None: - return None - - if audio_features is not None: - return Phi4MMAudioFeatureInputs( - type="audio_features", - audio_features=audio_features, - ) - - if audio_embeds is not None: - return Phi4MMAudioEmbeddingInputs(type="audio_embeds", data=audio_embeds) - - raise AssertionError("This line should be unreachable.") - - def _process_audio_input( - self, audio_input: Phi4MMAudioInputs, audio_projection_mode: str - ) -> NestedTensors: - """ - Create the audio embeddings from the audio input, where the audio input - is pairs of audio features and audio embed lengths. The audio input is - created by `input_mapper_for_phi4mm_audio`. - - Args: - audio_input (Phi4MMAudioInputs): Audio input. - - Returns: - NestedTensors: Audio embeddings - """ - if audio_input["type"] == "audio_embeds": - return audio_input["data"] - - audio_features = audio_input["audio_features"] - # (e.g. multiple examples) and the second dim is the multi-audio dim - # (e.g. multiple audios in the same example) - - dtype = next(self.audio_embed.parameters()).dtype - audio_embeds = [ - self.audio_embed( - features.unsqueeze(0).to(dtype), - audio_projection_mode=audio_projection_mode, - ) - for features in audio_features - ] - return audio_embeds - - def _parse_and_validate_image_input( - self, **kwargs: object - ) -> Phi4MMImagePixelInputs | None: - pixel_values = kwargs.get("image_pixel_values") - if pixel_values is None: - return None - - image_sizes = kwargs.get("image_sizes") - image_attention_mask = kwargs.get("image_attention_mask") - num_img_tokens = kwargs.get("num_img_tokens") - assert ( - image_sizes is not None - and image_attention_mask is not None - and num_img_tokens is not None - ), "Missing image inputs" - - return Phi4MMImagePixelInputs( - type="pixel_values", - pixel_values=pixel_values, - image_sizes=image_sizes, - image_attention_mask=image_attention_mask, - num_img_tokens=num_img_tokens, - ) - - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} - - # Preserve the order of modalities if there are multiple of them - # from the order of kwargs. - for input_key in kwargs: - if ( - input_key in ("image_pixel_values", "image_embeds") - and "images" not in modalities - ): - modalities["images"] = self._parse_and_validate_image_input(**kwargs) - if ( - input_key in ("audio_input_features", "audio_embeds") - and "audios" not in modalities - ): - modalities["audios"] = self._parse_and_validate_audio_input(**kwargs) - - return modalities - - def _process_image_input( - self, image_input: Phi4MMImagePixelInputs - ) -> list[torch.Tensor]: - if image_input["type"] == "image_embeds": - image_embeds = image_input["image_embeds"].type(self.visual.dtype) - else: - dtype = next(self.image_embed.parameters()).dtype - pixel_values = image_input["pixel_values"].to(dtype) - image_sizes = image_input["image_sizes"] - image_attention_mask = image_input["image_attention_mask"] - image_embeds = self.image_embed( - pixel_values, image_sizes, image_attention_mask - ) - return image_embeds - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - modalities = self._parse_and_validate_multimodal_inputs(**kwargs) - if not modalities: - return [] - - # The result multimodal_embeddings is tuple of tensors, with each - # tensor corresponding to a multimodal data item (image or video). - multimodal_embeddings: tuple[torch.Tensor, ...] = () - - # NOTE: It is important to iterate over the keys in this dictionary - # to preserve the order of the modalities. - audio_projection_mode = "speech" - for modality in modalities: - # make sure process images first - if modality == "images": - audio_projection_mode = "vision" - image_input = modalities["images"] - image_embeddings = self._process_image_input(image_input) - multimodal_embeddings += tuple(image_embeddings) - if modality == "audios": - audio_input = modalities["audios"] - audio_embeddings = self._process_audio_input( - audio_input, audio_projection_mode=audio_projection_mode - ) - multimodal_embeddings += tuple(audio_embeddings) - - return multimodal_embeddings - - def forward( - self, - input_ids: torch.Tensor, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ) -> torch.Tensor: - if intermediate_tensors is not None: - inputs_embeds = None - - hidden_states = self.language_model( - input_ids, - positions, - intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - return self.language_model.compute_logits(hidden_states) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - def get_mm_mapping(self) -> MultiModelKeys: - """ - Get the module prefix in multimodal models - """ - return MultiModelKeys.from_string_field( - language_model="language_model.", - connector=[ - "img_projection", - "vision_speech_projection", - "speech_projection", - ], - tower_model=["image_embed", "audio_embed"], - ) - - def get_language_model(self) -> torch.nn.Module: - return self.language_model diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index d3b6268e7647b..a4a964bc7c1a6 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -374,7 +374,6 @@ _MULTIMODAL_MODELS = { ), "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"), "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"), - "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"), # noqa: E501 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501 "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), # noqa: E501 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501 @@ -507,6 +506,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = { "MotifForCausalLM": "0.10.2", "Phi3SmallForCausalLM": "0.9.2", "Phi4FlashForCausalLM": "0.10.2", + "Phi4MultimodalForCausalLM": "0.12.0", # encoder-decoder models except whisper # have been removed for V0 deprecation. "BartModel": "0.10.2", From dd97e047e03f5218bddf05fa71f0592df0b8e30e Mon Sep 17 00:00:00 2001 From: Yongtao Huang Date: Thu, 4 Dec 2025 22:04:42 +0800 Subject: [PATCH 47/47] Fix broken multiline assert in `LoRAModelManager.register_module` (#30032) Signed-off-by: Yongtao Huang --- vllm/lora/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/lora/models.py b/vllm/lora/models.py index f568b8b9ba595..ada30da600440 100644 --- a/vllm/lora/models.py +++ b/vllm/lora/models.py @@ -574,9 +574,9 @@ class LoRAModelManager: def register_module(self, module_name: str, module: "BaseLayerWithLoRA"): assert isinstance(module, BaseLayerWithLoRA), ( - f"Module {module_name} must be a BaseLayerWithLoRA instance," + f"Module {module_name} must be a BaseLayerWithLoRA instance, " + f"got {type(module)}" ) - f" got {type(module)}" self.modules[module_name] = module def create_dummy_lora(