From e1efa3707b4d72798fdfff32382d1d1f456d0ea8 Mon Sep 17 00:00:00 2001 From: wzy Date: Sun, 21 Jan 2024 20:15:51 +0800 Subject: [PATCH 1/2] updata lora --- __pycache__/function.cpython-37.pyc | Bin 8509 -> 8990 bytes .../global_settings.cpython-37.pyc | Bin 726 -> 726 bytes conf/global_settings.py | 2 +- function.py | 21 +- guidance/lora.ipynb | 30 ++ models/ImageEncoder/__init__.py | 2 +- .../__pycache__/__init__.cpython-37.pyc | Bin 257 -> 300 bytes .../__pycache__/adalora_block.cpython-37.pyc | Bin 0 -> 7105 bytes .../__pycache__/adapter_block.cpython-37.pyc | Bin 6066 -> 6066 bytes .../__pycache__/lora_block.cpython-37.pyc | Bin 0 -> 7087 bytes .../__pycache__/tiny_vit.cpython-37.pyc | Bin 13813 -> 13994 bytes models/ImageEncoder/tinyvit/adalora_block.py | 224 +++++++++++ models/ImageEncoder/tinyvit/adapter_block.py | 2 +- models/ImageEncoder/tinyvit/lora_block.py | 224 +++++++++++ models/ImageEncoder/tinyvit/tiny_vit.py | 6 + models/ImageEncoder/vit/__init__.py | 4 +- .../vit/__pycache__/__init__.cpython-37.pyc | Bin 242 -> 336 bytes .../__pycache__/adalora_block.cpython-37.pyc | Bin 0 -> 9244 bytes .../vit/__pycache__/lora_block.cpython-37.pyc | Bin 0 -> 9229 bytes models/ImageEncoder/vit/adalora_block.py | 268 +++++++++++++ models/ImageEncoder/vit/lora_block.py | 268 +++++++++++++ models/common/__init__.py | 6 +- .../__pycache__/__init__.cpython-37.pyc | Bin 329 -> 321 bytes models/common/loralib/__init__.py | 5 + .../__pycache__/__init__.cpython-37.pyc | Bin 0 -> 232 bytes .../__pycache__/adalora.cpython-37.pyc | Bin 0 -> 10157 bytes .../loralib/__pycache__/layers.cpython-37.pyc | Bin 0 -> 10083 bytes .../loralib/__pycache__/utils.cpython-37.pyc | Bin 0 -> 1496 bytes models/common/loralib/adalora.py | 356 ++++++++++++++++++ models/common/loralib/layers.py | 323 ++++++++++++++++ models/common/loralib/utils.py | 49 +++ models/common/two_way_transformer.py | 263 ------------- .../efficient_sam_encoder.cpython-37.pyc | Bin 6647 -> 4584 bytes models/efficient_sam/efficient_sam_encoder.py | 97 +---- .../__pycache__/image_encoder.cpython-37.pyc | Bin 4667 -> 4722 bytes models/sam/modeling/image_encoder.py | 4 +- train.py | 27 +- 37 files changed, 1803 insertions(+), 378 deletions(-) create mode 100644 guidance/lora.ipynb create mode 100644 models/ImageEncoder/tinyvit/__pycache__/adalora_block.cpython-37.pyc create mode 100644 models/ImageEncoder/tinyvit/__pycache__/lora_block.cpython-37.pyc create mode 100644 models/ImageEncoder/tinyvit/adalora_block.py create mode 100644 models/ImageEncoder/tinyvit/lora_block.py create mode 100644 models/ImageEncoder/vit/__pycache__/adalora_block.cpython-37.pyc create mode 100644 models/ImageEncoder/vit/__pycache__/lora_block.cpython-37.pyc create mode 100644 models/ImageEncoder/vit/adalora_block.py create mode 100644 models/ImageEncoder/vit/lora_block.py create mode 100644 models/common/loralib/__init__.py create mode 100644 models/common/loralib/__pycache__/__init__.cpython-37.pyc create mode 100644 models/common/loralib/__pycache__/adalora.cpython-37.pyc create mode 100644 models/common/loralib/__pycache__/layers.cpython-37.pyc create mode 100644 models/common/loralib/__pycache__/utils.cpython-37.pyc create mode 100644 models/common/loralib/adalora.py create mode 100644 models/common/loralib/layers.py create mode 100644 models/common/loralib/utils.py delete mode 100644 models/common/two_way_transformer.py diff --git a/__pycache__/function.cpython-37.pyc b/__pycache__/function.cpython-37.pyc index b32aef7efdabc26f9b628fc1748ab766f9988770..2ed370dc08a7a34353457fe2766d183a9492228a 100644 GIT binary patch delta 2649 zcmZ9OU2q#$6@d4yR_NnuE#1CQ_o4?OMkp?xF6FvCNK2jGEoR&AQe zYV_T6&pG$pd(WPGr4P>>JQY6^i$w%@KKj!y_5CMq$M*w7;qL|}@^T%az!kbnUVmm=pG-rTIH|rJIDj$DavJe9d)* z^~1DlL);VX3%-09W%d>&G+F2|)0GSn+)TCCCF{u=0U7=4KJUAk^$hB1H%-wVnx?%p zL;K*p?=b`(0oLaXy4`Nd?Q%PBNMP@-r2H|Uqr<+BUlynL7kS`tT zf-#^O=@M#EW#Hjx5NmW7mjM74Dub!aAUnUB~%(=R6W}g%eupm@0?3$j*jh$9YS@iCV;nC9rxS5>V!+&NKGVy z)dCTG>&|_%SS7w&!=A)Ga3gL*yFdw~6nb>LB45k7kIV^6*~*A3yCc;}I#DAu(ItR1 z2@>qSL!iexgo?Z;UA?j>ILF*(+=O#{{n6@Uu*@lX9P%R5M>_;Za-|yt{2%LB6lx;v zbQ5%HHw0^qRi|+gSJ|TQzQT~pS!a%QJR>ZFRNR+ zkzaGT5}69kfS-JV--^6R&hg%+*Pl~jW|8UzJGW9a&7#F7!9huy8rx8dR$<#ugR0r8 z!!%guazR&OmtYU43x(o}<`kLNU=`Io(qn&d?-RYJ2T{x-h5%l$plh2t8wObxnT+~2#0UcKS#|_*6p=%W z^Hkf^&KQV!_`>t?ESrXgodC!IhVPU;j_}j;1ZqY0Br0Z64zg1ye^DoRiL*IGJ3y8m zwxdZV>>MKt_jw%h4B}b-Nn0ZH9EvmizipEveyWHUu*)XQT?K3#TID`aw(vx)bn1?R=GU5e*7ldla z>-9W7kJ<}}i--{75~2m*HR@K`VVRieLyE;ULP0O~e*J;eYIC&s_sa9%I|6h`@H>lxb9F z>=ksw1E4gnK$Tc3H5mIAmpZfSJD~L%oi(Q0YsCU(Uq@HJZmy$r1MwQ-8vrF}n0b3k z3E}C2Hc)c5*z0J01Fh0>xwx%_>btI&+&9tam(QF0pmXTO?_&RZi0>nQfVhSDA;Lcm zKSt>%03{5wsg`Eyp5*9e$$x!NKkuT$0lklyMEk2y=?!?4;E&h$E+bdcDZ)nG61%4%&Z1PXaE=K=0zn&Z+`}|kQ#}oH! z%Ab>A%vJusEl{t^K;WgFu{+;Fp{f2!N z4eh>R)36~iIk;ghd$q_+n<-#~_hVSalFjh9M&TEG5(y;=H?(Ht3Ps!2ZNIV=zTI;R z{-d>~Q)HTtrDxi(sOp8`p8>y4F7sE@8FGnN(-&Kmke^TH1GBYe$`C zC*p*i#z`6M&4q|N2XtCw3j7bk+&#ix;gFl7_$G6KBC&Gw8B2A>G$@dc`5s=<^tKo^0nt`c$Jw;j!;#j9c*LWu^{&yQBy~c zt5ky+ww`By)~#q%xUA9C%VNEQFbfTwy_&Uyh*eNz&#mV9oK0u$cji8 zpne`=AtHfTgjkGNf>_G_3H2BIKpcSwe!s1!%V40(0gw+XNLL`-jjuwjNLQm`4ay!m z<3u)~36F(_&@RukS5Ay>0wb3Pb<3s$0&IeA!6{o2+W-?@J)cXZdne??Xnq7TZbwT$ z`=z0;$E_UUk~NkE!)-8dqRyxv{Q%TE*owxL#TSKr=1e~V6ZDM7OpdFW|4Qyo*$2&M ziQwQBls+!&=FVq%5N2sHldGCNnuz(0JA?c?+;THived}J(la>%Q@)u{C5 zBpTg5JBQK*#M_9Ah)V)HJ->O7q5Ll5J;eJ6_Xb@<={jJ@2Z@uWn$@{%>)E_}#tNYy zqXXV99)Rr{nra(4Gij;W$L#6+AIaT&-!~=5tP(gpy9m2b`4B^XiTDaJBjA0|onqnENCYKC+)3MUB!kLf?t>3l zUZs{!kFkBNsX1b2oHPQ8ZBbi@S2L={m^lbJJ17Y2)TxF!-6^`Z0LJy zI7hPnk%D-=6MFs!> diff --git a/conf/__pycache__/global_settings.cpython-37.pyc b/conf/__pycache__/global_settings.cpython-37.pyc index 07e163d094db0faee96f258f706f54cf21c12299..6284220de6fee6dde5a8889f2e193cbffb2c6ae1 100644 GIT binary patch delta 29 jcmcb{dX1IWiI 0.5).float() #true_mask_ave = cons_tensor(true_mask_ave) # imgs = imgs.to(dtype = mask_type,device = GPUdevice) + '''Train''' if args.mod == 'sam_adpt': @@ -130,6 +131,16 @@ def train_sam(args, net: nn.Module, optimizer, train_loader, value.requires_grad = False else: value.requires_grad = True + elif args.mod == 'sam_lora' or args.mod == 'sam_adalora': + from models.common import loralib as lora + lora.mark_only_lora_as_trainable(net.image_encoder) + if args.mod == 'sam_adalora': + # Initialize the RankAllocator + rankallocator = lora.RankAllocator( + net.image_encoder, lora_r=4, target_rank=8, + init_warmup=500, final_warmup=1500, mask_interval=10, + total_step=3000, beta1=0.85, beta2=0.85, + ) else: for n, value in net.image_encoder.named_parameters(): value.requires_grad = True @@ -180,10 +191,16 @@ def train_sam(args, net: nn.Module, optimizer, train_loader, pbar.set_postfix(**{'loss (batch)': loss.item()}) epoch_loss += loss.item() - loss.backward() # nn.utils.clip_grad_value_(net.parameters(), 0.1) - optimizer.step() + if args.mod == 'sam_adalora': + (loss+lora.compute_orth_regu(net, regu_weight=0.1)).backward() + optimizer.step() + rankallocator.update_and_mask(net, ind) + else: + loss.backward() + optimizer.step() + optimizer.zero_grad() '''vis images''' diff --git a/guidance/lora.ipynb b/guidance/lora.ipynb new file mode 100644 index 00000000..a7c672ec --- /dev/null +++ b/guidance/lora.ipynb @@ -0,0 +1,30 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use Lora for Adaption" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.16 ('general')", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.7.16" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "e7f99538a81e8449c1b1a4a7141984025c678b5d9c33981aa2a3c129d8e1c90d" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/models/ImageEncoder/__init__.py b/models/ImageEncoder/__init__.py index 621435d4..c4678781 100644 --- a/models/ImageEncoder/__init__.py +++ b/models/ImageEncoder/__init__.py @@ -1,2 +1,2 @@ from .tinyvit.tiny_vit import TinyViT -from .vit import AdapterBlock, Block +from .vit import AdaloraBlock, AdapterBlock, Block, LoraBlock diff --git a/models/ImageEncoder/__pycache__/__init__.cpython-37.pyc b/models/ImageEncoder/__pycache__/__init__.cpython-37.pyc index a70a670882b9ead234475947f2ea53c9eb05a345..74f0a98334b71be8bd7a5d4276b9e3b7ce4f9223 100644 GIT binary patch delta 136 zcmZo=Vn>go{{##uTvu PiIogR93VEw#HBF+=5{7I delta 71 zcmZ3()X2o^#LLUY00g1Fi&MQP@=7u)OjMVh=p!k~bc@F^C9$9+wa6(aKRNprE12@r VWSux&jdLYK5gSlb5!=KIF#wG&6Wss+ diff --git a/models/ImageEncoder/tinyvit/__pycache__/adalora_block.cpython-37.pyc b/models/ImageEncoder/tinyvit/__pycache__/adalora_block.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9b68ac3f4cf72753aa086ee53e3ac774814548d GIT binary patch literal 7105 zcmaJ`Nt4{jbw&axY_)XtGR@{PBZwo}ESb~faO|*yo{@&jEOd}9TBL?@OEQG6M0KOO z3MeK}-DDYc@HEG=9l;1We3N8HwQmah=8JE-Dtzz{$b&)=@}+(75AebE_Y$>qo3v1o z51B6)I!L6^!nSqJIK$8;*E{A9wGN{tyl}TGn7;Kw+qQnFncB9E zPukLW&>L*+sv$ty)-*@&3KD%k+X2b7BW=Qtu|tJ1Xcla26#8HTZ#~y1M!}9*p#^4M z=~@LVXaf360~}j%^um~PoWeZPP~)PXS5#ELD%UU>W9x)ggXJ>Os>w<=97IAoK@cZ# z9t2WP66xL#pGRUV6@6(Yiqg(V_t9r}@5f0L3TbrO?-IEojj)|do3=6?=F;TIv@B_6 zQLiJbaT0W*FdvF2lS_Ls=g}!f4ILKMF2)IZ^j)!5&d&{kb}!7bAo!W~U*EjO!#sTN z+TrN=wMVxeTze4lxE=Q1LH?avJRIbaxYkd3)XT2j>xaA1M@btcaV?LN=Ld0qE#x7s zA=v4q?fuQcb3qbaN6MNYn(i?NzZzS?&tan)4s3I_CvBaY;Gs92)g7elLy$t7Xml!L zoXdpGbNW1|&P>&sukc73*;cC}Xc~g1Ay$ac^2IVpOBZk8CD}7^k$8$+B97fj#bGG; z1`1gNL}MP7F#CjeRJY1pHDEeM_t!(Le*}A8AXEbOg^4TUD%{9nAr8h=oPAbQw z{Lr2wc1nL7>E20`gBv`!y(LIrgi=r!IeKO%skA5og-dy*QP!4Z?T?{Q=1v@D zYHm{BNm^AwGgP}Ny^!Lf- zGCjY!M5$T6jha%LAWJp2F;$1r%6#`1DnTbe51lJ5Q>nx+kSVmTGGX7-$1q!k-lYUH z)-}(;Ijm|IH5fGpuEDW=U+6@+)-lg(YgD7-pmdlw^jp(F&-C2T#%6AJ9UP7^@i>kQ zU#oU&Xh~%QWusf?CXIWZW>W1o6kSrZS+?etm-e_)RJwY1d9qTV7Y?=3t*DY&mY|x} z%wYfO8w;U3ArsIt#SOfqjSc5E#@#ODvMJo<9(*(G4t%qNQ?5(*R-Q)*smzIk z(S!~1GyR0Ds<;O-_tr}&P^E#CeGUa!*qK}}D!Fl_3qOZ_IRY@)*R!wWmCqrN@WK#h zuCGEM0~t+)GQFvfgaj3llC5pF}zmcy`&gB~g0LpoMOXV!e9qUJI>|o{&-fpao zyDH`n7EEn zJ*>+s8t-dGb#j4n=QUEYr~mmxd;0HBwESYXF}ajqoV-~WW$R<5*uh&^@t3%b8o#I> zv&okWm#i+Kzn2P6;`qER@%uQAT_ile`nfti$dfurvQbJzY1e`bURz-x#F6I>Oz|4e$L6q=P&D@8ffTu?fS%K|HGub@5bOK@`+NCv% zU}xCrprv;|O?DqgNrn}f2XS=RS}J{vfrW@~(Ig)dc|!B^faTKOkoG`4A_m)&)*!_) zGs(8(^5S!8r;|nCnzY+eAH0inLs}(!rt@b#&>v#J>?Vk2din)s0nu2>6s^!(CwR7| zTQE=#Gw~z5a|Nje(`CVMS**!ME4OB6J@;0RUqNt6hv`G54wELq2{7m{^pi8vAA!uEoJkn< z)wMrGlOG^uB)1yY*<2e%%Hv7<9S2gm}ex37Z4WcA6T-uz2NIpERd)T|PDxc7}CR z7@H8Yvvu-6cm$mwHnM}4f?UcDC~ZIaZ`8L|#W$!*Ve9WuE+Rs53{@TU;=B~>Zxh3Y z8R!r5NV?ePBpakq@;{~uN9{Yg`1whygOzrccJhAsOxjQf<&{69;r|3=3ed!xGyuCT zy6PPWnuKIeCEJ$_8V(^z7A~+v(uBFGc}AX4V>JVyUSDCC^dEsw>7^kp=Ue1QXf>$w%h+12Hc$Ns3suOJ z7>hwDHlOyt>Zb#meb~cD)8O5QXNorcyHO_`_VSzl_08+^ouRJ50Cy#?>;EWEv6opM zw|yQB(B?M`gtH5K0YaODGmFmmB7Z*;I1aUNS{AXW-PGCmJ5hcZMag_hbh@v3=8EBm z3HKk|e|Ty|@69{ovLQZDaneoop+(v63=>r1U>7@)B%2f%tKKIB0jGGQ`QbDJEh2l= zChSg66NyYYF|&UISwKYv0F@~YC>X_c4geY3H?ylbuDy;JAR~-3AS28(;G)BgeN%jz zI|R-WD|aW}F@eG=#xd97ssJ=Q=xM5+;9UT8!w7Iy<62%rt7-wQBarrz#;xZK@GJtG z<9gwZ8~GB-bpSS-+b?N8&BEmlMp??2QKO?&DbNO>i)OTfuiNmFaP>{henr47iW!ld;pBK(~JC_=l)!Cl>LUUB%~pzCqk`t%AX^0o`|{~ zA5%bI;Bt97?}e$74VwM*iVYjRt;&nef_CUt;6X^@Z;`SqAXqf4(b$aAnYM=1nn=T`&2guKA>Yq%BwN&#MBKpW<`0(60ZpQ0fM z^4D_Q)+j+{V-+cNjeT7_RJf(gX_M(a+K*Ury*?%?C)QJuA2Fez_Pm5Sr$!JPD>XMoQ5-a!14=1vAx zW23iTxBNv@T$!G|K$3P4@U)GqXLT!fnZ88(QQl4)xcC%*PsO*1%%?CjbA+PB=O9N& z>SEJrn7WQL)7i?K&hMzY)|&V(wQGR@@6*-nW|EY*v#Yb><_r{sJ?Sm-;%}(aN*?$7 zo8_Mmo9Z8oOgRu)dyEPKSM(>7wAyLEkD%G4uTi`&D~O^I)msnLjeO}wyhqW~A$gpz sr@Tq&YiJRR^_4kVopjzWZyUc(lA7?a(T=5vToPwo`&b-KmILx-Pg4LqQdMkk$Dd(`&SUIan{qe%QxF*s;u{oZG)9O zvv24c)B2TdOL6p`-FLQK~I;#qH-Bw zjvKMA(QSga44%iU;8him+O2?B<8|=r)1IrGJ=9u_6UmZRv3)|K?;|xQaT<@4a)H;lo@Z8<5VEWc`ZQJ^xW@_6u zK50wies8d~tA+q+Thko9D@gSHYzHLQj^PA_fm zZw{UblI1#5)&$XXk2(0&*b06Q8`W@Bo3q_$>(m4fz3G(hAZ71?6xu|i^BChqCTyP5 z=Q(v+s@8moM{>xvS`|Ul5Ht<3LWI^XmO)y&cm*%Xo{EdaQ{)nH>`p2UL%}yt$QmFT z^RR^3Z&an}PusK{I@cuZM>tn62(U%N9_4F6@MIYF${JCnam|lDxc3dUHR3fY*Qfj6 zj(WWySj?iu#0As~nv5X%ts!NUahT55J;ghtJ2PR=HqmIVJN7A!F(PwPIVQD--W;)0 zn&U|KPMRFt;KA)JL7F0zYP!hLGdoG8KM5G8^hcXOKXRyNw?ev77=>9>3ahZW&W#t& zvCcHAomhpAQ*>2Jd*T#s*DdC?-IC*RliK5?bzUi4$}3&6wj^tR2vst7;xJQlllo55 zstTH++D++&oKJUNR`OJ|_sR{o%7w}5?J#fek*?j~(j+C6ZaYn~JWTRHT8B}*yO)bE zV7!($6^c-@9jWhU(#moX^GF(lkV}Why)ci{L|VHd9S$g){N@s+ zX88tcN@;>D)!4>V9Y!nj-Cw8#od7*_uCz?062C;I(7MWaeMcX|U=@0o5)4?^JO{V1 zs$JAz&KS4`$M$`y6XjaRJg==$jgEuTVc5`bO#?mCb3+@Ox!rYeIL5@|I5K>#+O44_ zl?{}QZk?Mn?s=L?wcAj1NzrE6npa-f<4RHK>fPnZN`YQD)JnIaN@iJtYFaZ>{hO~Y zgzkh)K+6<2@RBw*oX5$o_%cf3&!}bhO}&gCHN4Wmqu#LKD}CJEXG2V^>fi${_86Drf-u+n=W%rt#l`~?yEF6=F=qO7#j!Ly)cDd8yl z&_PRfD9Bq$)?a9^Qd?M}Aon%-dR>;h}BHPB7G8*J2A96U9~=t}$MDtM|$ zyp|(AhnMh)yiSBvQ>{wcNgC{mkc;<;MUu9d&rBMXHTgWME+b8?iN+ePAYL13{?OOx z$=(5(Ta&7?CMK`I|IVyQP4R5*Ja@^aaF=`V%CI}|$_`GsF5O#s9wnqQCk{pvHptKP z6SAt}F38+ZFQGt{22%D36kuUza=obJ#*r@k9QNf1kYHcWKA%@Ufk49lLY%q23V{r4 z*dQ%%$P%q#3gnx|R_?(nLEMkXuUAi%&<30uYPFya;jI-GXKUIyZEWX_$r5=K)HL(u z?h34%P0&LjfmOw!_6m+D-dg@jzCJmZZxnzg=kYC-vnY40AF;86nLBv9u{QR0wQ==n zr*PiazWcp#?di8v?!ma8pD$|EyQmZ0pizi}QNRHuPDdX%ibmn>+dpFBI!5)dF0W|3 zqZQT31gdzaMM)#cpGADZe;*wJ^%o$4aq-*RbNxaT_&$Q9Wjp*9(`d zF5$ix3Q6Mlye;wPIF4N;JiqycIz7mfI!UroN<`&qI6LEjY;+Lwrx{sn59*Y~8J&OY z@@aD&XR$KA?Fi<2IOxH;_M${KZe>{{NLD@&A{Ek2hW%hK3V8GA?z#2Pcr{Ng&8Pms(C*w&C3~jxXFbqgV!-Ssh-P~F1!e)!SjrTw&|4>Xwx(M!P!2Qk zBfN73sRq+!!Ejlu$wn);W@kP3R*zmna7u^iL!}OrCcz0X=r8n>Gt%Tj`@WH^Lk8ha z$e|!t=_t1nz;!|P^ew4}J1~mm{~#tJj1y_^g&DR$hyj>lhsa-n%%Ge}0QKdyKS7i4 zBV{DF8rIohmOffqtV9hyUj>;}a=dqtvfqNhp=#eT#`@S8!&bb6MB(fZHxT-v)v#ki zLmHfHne1Idp&=b3SYvza6xPIn69o{1i#5kUOJ~Y<;hZU>JzBWj#>jdT7OU%#-Rf4w zzk}B1yH}^(OEAA~0~QRr++9MXV!VLOfo3~R4P#ima;#69)447m8wxwax+#oJh}qdX z`9C~@P7oW}!An6dWe1eDpZqWCTdU%$RHd->w~oS0QYiV?sKQbEPA-0alImckou!?;A3l{f)IoXWZ_@C812P3@;#C@e-4`}}8gAQ4IK#fW66w?1=9{EEwB2~%ATp<|8uki*{%@tJBMw&k)l=Ng& zGxy7ETL~CM3PKe@_&SJ+5h+th@4_dmKz`ef9Pa%y<5gaJZf?8ELwm|YHyXvuRO(~&taVK z^e`ZlH|zCSv3z1J+Vy*b(rfrvI=wWc<$Rr-2(1Qnei>D()#j-mVWA3^5@Rt4Mdj1} zSN(KAvk!Y1X&Scsa7)pq|9;d7hrRr!e|_`%d}pX@Fu)bbXZ3%Wr`XFZkJ~Dc;;H+hY9!Z-+ORs zMQ_hL&&K*K#iIuw( z@0bu_72}xe@KZpU9rQF+Pw*}Pw_ybMsc|i@p;fhj$`L|)LF3l*26z@B&2hc(#*KUl ze&QGO4Yi;&gm@@e_)53?MnHmgJU#%x+UZ69&NF|mIm&p$Q4-7$)Dv;mI_1w1IZs4gjS%2N)my|+ zvC!L;E5ok^hEtwFHh*8MOCL3{LW2Ec~h*lx2MZYFde}zFcji3pcxl4tpgD?6e?2lLC%_Do7uf!xgyd<>i=0|RPn`fwH_iX zf%#4^Pzs%{Q3~TzyiE*;8fZ&Q1fyNqCVGW@ODM^`{ zy2&zZsx;boNo{7pjY&&HxFAp3WLD{+H!0^Gr0gmP zCkfYnszEZfF2kE7M8})Zw1GF_dGn{*zYt);X_znq?~Yapw*c&%0yXcBYF8H)_uWzB zs(;D~s@l0Bd>2>K9elhzr?XZI#^e>MQsLV?hLc|O6o?t$8;D=i+{u(`Y;^IZwJ+M? zn)K|Yk+g$=r)^v}tDCUP^cmWW@@eLyL@6W6oA!qRk$Ptpd%yb&2uH&?H zw(_d;B~{m26W^wGEf8RQx{lpUlJZ7&bynP*VS%tGy+vO96Lnh2<9>g${I_9K{bi9U zzaeXnP(hfA{!)@wJMH%oD4X;(it%LykuxHB>%O{$FTIENFnTg1M-%px6DeH`En>00 jGDoYE&O7A|<5x&b)a4!7X^M?&u+%kUWqoa9?e6~rw%npm literal 0 HcmV?d00001 diff --git a/models/ImageEncoder/tinyvit/__pycache__/tiny_vit.cpython-37.pyc b/models/ImageEncoder/tinyvit/__pycache__/tiny_vit.cpython-37.pyc index 9c2e625baaefbf9f7106adfe12d79247e8736713..d3ceaebc7d769de21ce11bc7a12e709efec528c4 100644 GIT binary patch delta 4988 zcmb_gYiv}<72esG*Sp^J+SvI0z)!Bh#x@3$Vkk8vG=QCkM+i&H<+5{)7w;?QUSmTD z3x#A&qO_&cq-mO_kVci-H2qOSNFPf0(>Bng5`9S3&5!)3+6sw2l(te^MQYEPyWU+} z1gQ~N`|Zq`b7sz*^UaxA{yF*GL}(-!^gHn1SFXRRf9X@_L$&Pl=esXt>Js&ldgkCh zUKVdiG)5X_I~eyTnj%e!U?iAmjx;mJRtFF9^0N+Ju6vHOoN+~3Q4;1AD5=OvmZ7AQ zFG0zYoTLpURlFJ{)j3H!N|y2(l+@UgOr!%vwY&~RbvdO@ULWb=4ZM*z@n+t_TVM1< zy7@BRcGeMD&IfrH?>@^S3SZ3?-h-_kKEzk@UTm%4Yj_`Dg{_r*E$`<8*y`nMmqS^1 zllDHQFw3)hDjnCc;oAbKw0Mo}D~+c>8;Pf)lj2`&d5Brz-Ld4chhw{UpnR_0`2?#C zp|RyM^hcEv%WdfKai~EF)QT&vvz{t!Hj7PewYvdZiW3@1o6SHQ=}2fNNy&H$rZwQb z;%DyrykrHPqQbL8$?GW$mXVm$A&k(3AH$8{V`b$Ko*VVlFt$SI-j22vD2;>;CDmy? zc6h>6W67AQRKOAxgPglYyy|UZA@NIZKdTe}^)_rEzih|*v1OI%Gifc!RnW{>3Yux~ z9)x)(@{Y^t2F2cz?~D$R(jdWVf+2!61Y`(n5fqo)K6GH)@);8;Fr!oEqEJ95j&2pB zzExYh$grDWIe{G&YAFP_k4z5^ktg$!sfHdg>e(VnO0Td=@m6Vlun$KSe=Zo}kEP0k zltGa0OV7z^#cR7dOhX$8HWE-EB0j@}7}wS4?W3_@9Ps~XZ6#_d0n44_u>{a8mMg9& zE%(uwK5dnxbx2H^y1nQpX=Zz6@mQd~Tc$$Bh<(~(do4Cn*sGm>3QN*skyj|>{6Bo-@ zMz*8+*!^R$gUr0}aoD;IO^@Tp@FDnI4(4E?8rU}19p2++-xrTp4P`oJ*%2>yPC8-4 z^qq4it7PAW{hekh_T8rcoYM?kD9bvsEbGj=vhJ)W>&=#Aec95ie>}k5+;i4v22Z%p zuq}==>>@kiP(vsD+%Z`SublADdS_iHLbDF;ea4L=0XuanwynS+D!OVkt{H}%$0+H^WDiJGVH%^w5UM@z`v z6xC>;DsemhzmNQ{5u0mI4lF#O=)63)G7tG=SblxR1Wg4!4L8LHwH=v-N5v{pRi28f zstmE^-g@7TJ%wVASE~syL7v>9?C;0%i}*3Q6eu_Fatw&_vKIZZ!_}lI!>iaK}tNG0#vc$ctmK6J=#$d((hQ zybQ6(D@%)xOADFH6i^jJqL4=_UYQ0wMneZlS~eX^^3=3S1qAmaw3ZgOTbypGYoOp* zA?cqQ2bJql?U-Fds>S7&_A2>c%JWci>d$kxTK>&e%M2IS9;fA(=Jevx?Q&2sq&$y_ z+P1GZOId!v3aW-~($WT&IsGAVqb)rwr)KhzgWUNBxmY@{VH8S~K^#$*}rEfnLKleOo|r1u=@ zdE^xW2s|2!G(;Z~*Sk};|vHO5H-1T9TyshLB%94r)!SdQq zYnTzIx~n^%KoQ1FIv|5UOBD7GPhodb{HQy+CYK_qxlqZfO{V4_>^2oA^<&d1;D#x7 zF8@o0N{mtpGDV=@IST3g>>1sN=e3PJwQ)8F5t^@B9G{&hM0FU1TR;@3UfiPpi7H$lC>-Vk(0H zCutGak(gCMo;zme=DdF$sQ5Qn~=Kmr-_!e4w8;4!j7Rcm$KJIZQ5?M~y}Rpiy5h#U-x1;d8g@mLu3j!S3=S2!^ikfxZ^&yvP)h6*cUQ9Z z22S`pCjK^9%ich5K3n9;9fM=L^BP*;Y`vR4(NhL~i0}vT*TK52)KXmc!*vq>qPTOT zxby4c&Vmuy=@X+vbzyn@ee%)KmrUYszN151**@{+kd~pJ6sldi>2O4X8rRZT=5sRn zH2*dZ$X9v9U!)!+DKQ?(5iIX$iciIL`Q(8f60axNMzEiNjuKoYc#Gf~!R2v6qA6S< zpq$7U6dkcbmGli`9lSgg{Te6<5UE5InRQzsSM-iL&eCD1cN_=(hWJ#_P zCb6124HDQ{yKxdDwcGx1^20zZ2MLn4aM9+`A4QA&33@==A1xZVK=Y@~j~WHinLSdZ z9M?bti@5K0XLe?1XJ%*j^hVEQN2z~{if4WIu+e{uL+u$E1o>%5%tCF&w|i9jTf zsE^b$#~ugw^T48m2lS$;hSRP{BT|CA3@K$OQaql5Sq=C)QBt&PJ4rzs z(F3X)jcbMx)#Gth%}Xf^nrRRTLlB_>AH$8{W&SbYb6SWj04Hc}XmoMu#q4<qn=UZ90F zia!QwLUQRcs>GDD-}Y^Z#|$%?N~E`)5mmv#mX-cdN(*vQvNnsUU<0cW&jdG&$;H`J zQus}R<(z|usrwLtboU`_Wu2_J?O{Z+10t^^pfr$-&(RVK2?@fv=L9BPcqDkKsk|x*W{Gf;BL*G*y1o&0Y{U zs|HHkruSK=S#sGYnyTwYoZPkOHA@%Vr`b-&X?BGzI8^_Fk2|JIU~$2B#&gEC5Ip1H z?&sae2_!w-L2vHBLRNGTcbJw?HJ&!bh3YXsxz}=QFk!%Eal3j$hFoQp#Zyr&u0~VI zld2IrrAvQeDcgrk*pGni1Jpi9?NZy_-^R98dICkK?Fwf5UZrs|0CIyIr|vO=M+qJy z2ocEe9Hx5qp?G8UP(}t`hL&Qk1izFiBqVc!lE88$QXJ?gf>Hu1YH)<$TLd!Llrc(~ zU2Mo$+dgTP)RQ*|?aMeIZ#hZYxptgdWRT3dT*4K205oDs(TWUcAZQeGO??|e*i@M1j+uHQmy1*e@ASiA zqbD zbPlUXqR31^bkg$1V@V^eMSFA%#Y{q`z2(VjqV%`iJQg*9iaG9+SQ4n_z?c|n?qY*N zYaV!{mRO!0xZnFYP$9z82y3FUx?s1E`SxAlBn|LMLsUGo445SrImhzP#*#cWE2{=B zL;Rz;nLQ)|p}L0S#12ZU)HtYIk81PwPFf`%2(?u;la3I<@}~H7X)g3{)=$J-kY&Mf zT0SXG_YmE#0eL|R`H1*w>#J)dZJB`=D?>MFY6EMVzE3>XmhK}i?g>gAoZ)4BStYp< zM&26ByCtqAPH=78A@N1qL~o&(ONz-#2(O@mycE2O-6IT1m&8l$TL=CRsZl9~;^I=< z`rd6hwJS}%+jJ3oI=y3`_X0}cQFvaw)3L7Whos_SPEX!Ot4x)r-?Cp+5m>u3vp^$d zRAu>`r|$O%9w(3$N$!=`sQWa*B7#+()Dk+!%Ol_$Lt_AKdE9KW=)6GVUZinF@{|BF z%3{;DU6+X)LQsmJjUoYpKvrgoy1iPN8{fz74)M>m{~47BlH4b!X-E-*<*_fla{pn4 zz^kuSf+v|_YDrUtr*H^{S{fvs5+XqfQuj01)y3{`w1#MRsh8qrxQQjjpTeJIsHP|- zAiX0UBb_yjRQxQUWAg!Qg4{RTsr4*E&KZ<)Rxug{cs;kXhg<`Dh|W}t?XZms5s@sj z%CjE~>bS0%GoTv`#Qv`Fj%&!wt%$VVAt%2oo)({Uh18W*-LBx|OQ1!=DFn+kh3@ji zCXy-8;VO|yz0`59>OP5s{w5w?r?APTC)a)KY@(^GYBde1lX(B-c9O{}mI1GcRChZ& zCC+zm-*S$$ze|ICc!2A1l`;t4Ch8~x)_CW5G9MnxH;x}Qpr!Fvd{I>P9BGql^3YHx z{EEbWO%Ngivk5KYOiyFQd4^{XjdQqz6<=$fUrns^3 zSmq;=`3(Ue$WIZV{3@`{OI@-F$h!Mw*mczPE)(Z^SF`KlYVWbk6wNS=5PK5swZDh* zmJ|~=fpDvmTKC;gOKw#AfjHY+!^ikmLw6BVtcpUmx$d!g&#Aq z&|k}Lpe14P^ZtPXYwqbB+oab~!>!Q$w2SU8@D{?K#drJbGSpIByWs;Oe^}W0ZDHqk zg`E`~+3B)M@E!$bDiy~oi8-I9+l!M_%97xfw~X#&(!Twz%5}TPMaAIP*{Ik%sAbZq zK((v7+&@1iad{J$U(5mQTAl+bJ`>mFZ3e5t=Sss-GWsF=h#L@7<6vpKS!MSuuA;-SvmIn`1Z3ZS7lYDQW>i3tlT5c H4%Pn`jHcNF diff --git a/models/ImageEncoder/tinyvit/adalora_block.py b/models/ImageEncoder/tinyvit/adalora_block.py new file mode 100644 index 00000000..8ec46ba7 --- /dev/null +++ b/models/ImageEncoder/tinyvit/adalora_block.py @@ -0,0 +1,224 @@ +import itertools + +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import DropPath as TimmDropPath + +from ...common import loralib as lora +from .utils import DropPath + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, + out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.norm = nn.LayerNorm(in_features) + self.fc1 = lora.SVDLinear(in_features, hidden_features,r=4) + self.fc2 = lora.SVDLinear(hidden_features, out_features,r=4) + self.act = act_layer() + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.norm(x) + + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class Conv2d_BN(torch.nn.Sequential): + def __init__(self, a, b, ks=1, stride=1, pad=0, dilation=1, + groups=1, bn_weight_init=1): + super().__init__() + self.add_module('c', torch.nn.Conv2d( + a, b, ks, stride, pad, dilation, groups, bias=False)) + bn = torch.nn.BatchNorm2d(b) + torch.nn.init.constant_(bn.weight, bn_weight_init) + torch.nn.init.constant_(bn.bias, 0) + self.add_module('bn', bn) + + @torch.no_grad() + def fuse(self): + c, bn = self._modules.values() + w = bn.weight / (bn.running_var + bn.eps)**0.5 + w = c.weight * w[:, None, None, None] + b = bn.bias - bn.running_mean * bn.weight / \ + (bn.running_var + bn.eps)**0.5 + m = torch.nn.Conv2d(w.size(1) * self.c.groups, w.size( + 0), w.shape[2:], stride=self.c.stride, padding=self.c.padding, dilation=self.c.dilation, groups=self.c.groups) + m.weight.data.copy_(w) + m.bias.data.copy_(b) + return m + +class Attention(torch.nn.Module): + def __init__(self, dim, key_dim, num_heads=8, + attn_ratio=4, + resolution=(14, 14), + ): + super().__init__() + # (h, w) + assert isinstance(resolution, tuple) and len(resolution) == 2 + self.num_heads = num_heads + self.scale = key_dim ** -0.5 + self.key_dim = key_dim + self.nh_kd = nh_kd = key_dim * num_heads + self.d = int(attn_ratio * key_dim) + self.dh = int(attn_ratio * key_dim) * num_heads + self.attn_ratio = attn_ratio + h = self.dh + nh_kd * 2 + + self.norm = nn.LayerNorm(dim) + self.qkv = lora.SVDLinear(dim, h, r=4) + self.proj = lora.SVDLinear(self.dh, dim,r=4) + + points = list(itertools.product( + range(resolution[0]), range(resolution[1]))) + N = len(points) + attention_offsets = {} + idxs = [] + for p1 in points: + for p2 in points: + offset = (abs(p1[0] - p2[0]), abs(p1[1] - p2[1])) + if offset not in attention_offsets: + attention_offsets[offset] = len(attention_offsets) + idxs.append(attention_offsets[offset]) + self.attention_biases = torch.nn.Parameter( + torch.zeros(num_heads, len(attention_offsets))) + self.register_buffer('attention_bias_idxs', + torch.LongTensor(idxs).view(N, N), + persistent=False) + + @torch.no_grad() + def train(self, mode=True): + super().train(mode) + if mode and hasattr(self, 'ab'): + del self.ab + else: + self.ab = self.attention_biases[:, self.attention_bias_idxs] + # self.register_buffer('ab', + # self.attention_biases[:, self.attention_bias_idxs], + # persistent=False) + def forward(self, x): # x (B,N,C) + B, N, _ = x.shape + + # Normalization + x = self.norm(x) + + qkv = self.qkv(x) + # (B, N, num_heads, d) + q, k, v = qkv.view(B, N, self.num_heads, - + 1).split([self.key_dim, self.key_dim, self.d], dim=3) + # (B, num_heads, N, d) + q = q.permute(0, 2, 1, 3) + k = k.permute(0, 2, 1, 3) + v = v.permute(0, 2, 1, 3) + + attn = ( + (q @ k.transpose(-2, -1)) * self.scale + + + (self.attention_biases[:, self.attention_bias_idxs] if self.training else self.ab) + ) + attn = attn.softmax(dim=-1) + x = (attn @ v).transpose(1, 2).reshape(B, N, self.dh) + x = self.proj(x) + return x + +class TinyViTAdaloraBlock(nn.Module): + r""" TinyViT Block. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int, int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + drop (float, optional): Dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + local_conv_size (int): the kernel size of the convolution between + Attention and MLP. Default: 3 + activation: the activation function. Default: nn.GELU + """ + + def __init__(self, args, dim, input_resolution, num_heads, window_size=7, + mlp_ratio=4., drop=0., drop_path=0., + local_conv_size=3, + activation=nn.GELU, + ): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + assert window_size > 0, 'window_size must be greater than 0' + self.window_size = window_size + self.mlp_ratio = mlp_ratio + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + assert dim % num_heads == 0, 'dim must be divisible by num_heads' + head_dim = dim // num_heads + + window_resolution = (window_size, window_size) + self.attn = Attention(dim, head_dim, num_heads, + attn_ratio=1, resolution=window_resolution) + + mlp_hidden_dim = int(dim * mlp_ratio) + mlp_activation = activation + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, + act_layer=mlp_activation, drop=drop) + + pad = local_conv_size // 2 + self.local_conv = Conv2d_BN( + dim, dim, ks=local_conv_size, stride=1, pad=pad, groups=dim) + + def forward(self, x): + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + res_x = x + if H == self.window_size and W == self.window_size: + x = self.attn(x) + else: + x = x.view(B, H, W, C) + pad_b = (self.window_size - H % + self.window_size) % self.window_size + pad_r = (self.window_size - W % + self.window_size) % self.window_size + padding = pad_b > 0 or pad_r > 0 + + if padding: + x = F.pad(x, (0, 0, 0, pad_r, 0, pad_b)) + + pH, pW = H + pad_b, W + pad_r + nH = pH // self.window_size + nW = pW // self.window_size + # window partition + x = x.view(B, nH, self.window_size, nW, self.window_size, C).transpose(2, 3).reshape( + B * nH * nW, self.window_size * self.window_size, C) + x = self.attn(x) + # window reverse + x = x.view(B, nH, nW, self.window_size, self.window_size, + C).transpose(2, 3).reshape(B, pH, pW, C) + + if padding: + x = x[:, :H, :W].contiguous() + + x = x.view(B, L, C) + + x = res_x + self.drop_path(x) + + x = x.transpose(1, 2).reshape(B, C, H, W) + x = self.local_conv(x) + x = x.view(B, C, L).transpose(1, 2) + + x = x + self.drop_path(self.mlp(x)) + return x + + def extra_repr(self) -> str: + return f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " \ + f"window_size={self.window_size}, mlp_ratio={self.mlp_ratio}" \ No newline at end of file diff --git a/models/ImageEncoder/tinyvit/adapter_block.py b/models/ImageEncoder/tinyvit/adapter_block.py index c776b068..eabfb9bb 100644 --- a/models/ImageEncoder/tinyvit/adapter_block.py +++ b/models/ImageEncoder/tinyvit/adapter_block.py @@ -194,7 +194,7 @@ def forward(self, x): x = self.local_conv(x) x = x.view(B, C, L).transpose(1, 2) - x = x + self.drop_path(self.mlp(x)) + 0.5 * self.MLP_Adapter(x) + x = x + self.drop_path(self.mlp(x)) + 0.5 * self.MLP_Adapter(x) return x def extra_repr(self) -> str: diff --git a/models/ImageEncoder/tinyvit/lora_block.py b/models/ImageEncoder/tinyvit/lora_block.py new file mode 100644 index 00000000..0fe5e3d0 --- /dev/null +++ b/models/ImageEncoder/tinyvit/lora_block.py @@ -0,0 +1,224 @@ +import itertools + +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import DropPath as TimmDropPath + +from ...common import loralib as lora +from .utils import DropPath + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, + out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.norm = nn.LayerNorm(in_features) + self.fc1 = lora.Linear(in_features, hidden_features,r=4) + self.fc2 = lora.Linear(hidden_features, out_features,r=4) + self.act = act_layer() + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.norm(x) + + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class Conv2d_BN(torch.nn.Sequential): + def __init__(self, a, b, ks=1, stride=1, pad=0, dilation=1, + groups=1, bn_weight_init=1): + super().__init__() + self.add_module('c', torch.nn.Conv2d( + a, b, ks, stride, pad, dilation, groups, bias=False)) + bn = torch.nn.BatchNorm2d(b) + torch.nn.init.constant_(bn.weight, bn_weight_init) + torch.nn.init.constant_(bn.bias, 0) + self.add_module('bn', bn) + + @torch.no_grad() + def fuse(self): + c, bn = self._modules.values() + w = bn.weight / (bn.running_var + bn.eps)**0.5 + w = c.weight * w[:, None, None, None] + b = bn.bias - bn.running_mean * bn.weight / \ + (bn.running_var + bn.eps)**0.5 + m = torch.nn.Conv2d(w.size(1) * self.c.groups, w.size( + 0), w.shape[2:], stride=self.c.stride, padding=self.c.padding, dilation=self.c.dilation, groups=self.c.groups) + m.weight.data.copy_(w) + m.bias.data.copy_(b) + return m + +class Attention(torch.nn.Module): + def __init__(self, dim, key_dim, num_heads=8, + attn_ratio=4, + resolution=(14, 14), + ): + super().__init__() + # (h, w) + assert isinstance(resolution, tuple) and len(resolution) == 2 + self.num_heads = num_heads + self.scale = key_dim ** -0.5 + self.key_dim = key_dim + self.nh_kd = nh_kd = key_dim * num_heads + self.d = int(attn_ratio * key_dim) + self.dh = int(attn_ratio * key_dim) * num_heads + self.attn_ratio = attn_ratio + h = self.dh + nh_kd * 2 + + self.norm = nn.LayerNorm(dim) + self.qkv = lora.Linear(dim, h, r=4) + self.proj = lora.Linear(self.dh, dim,r=4) + + points = list(itertools.product( + range(resolution[0]), range(resolution[1]))) + N = len(points) + attention_offsets = {} + idxs = [] + for p1 in points: + for p2 in points: + offset = (abs(p1[0] - p2[0]), abs(p1[1] - p2[1])) + if offset not in attention_offsets: + attention_offsets[offset] = len(attention_offsets) + idxs.append(attention_offsets[offset]) + self.attention_biases = torch.nn.Parameter( + torch.zeros(num_heads, len(attention_offsets))) + self.register_buffer('attention_bias_idxs', + torch.LongTensor(idxs).view(N, N), + persistent=False) + + @torch.no_grad() + def train(self, mode=True): + super().train(mode) + if mode and hasattr(self, 'ab'): + del self.ab + else: + self.ab = self.attention_biases[:, self.attention_bias_idxs] + # self.register_buffer('ab', + # self.attention_biases[:, self.attention_bias_idxs], + # persistent=False) + def forward(self, x): # x (B,N,C) + B, N, _ = x.shape + + # Normalization + x = self.norm(x) + + qkv = self.qkv(x) + # (B, N, num_heads, d) + q, k, v = qkv.view(B, N, self.num_heads, - + 1).split([self.key_dim, self.key_dim, self.d], dim=3) + # (B, num_heads, N, d) + q = q.permute(0, 2, 1, 3) + k = k.permute(0, 2, 1, 3) + v = v.permute(0, 2, 1, 3) + + attn = ( + (q @ k.transpose(-2, -1)) * self.scale + + + (self.attention_biases[:, self.attention_bias_idxs] if self.training else self.ab) + ) + attn = attn.softmax(dim=-1) + x = (attn @ v).transpose(1, 2).reshape(B, N, self.dh) + x = self.proj(x) + return x + +class TinyViTLoraBlock(nn.Module): + r""" TinyViT Block. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int, int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + drop (float, optional): Dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + local_conv_size (int): the kernel size of the convolution between + Attention and MLP. Default: 3 + activation: the activation function. Default: nn.GELU + """ + + def __init__(self, args, dim, input_resolution, num_heads, window_size=7, + mlp_ratio=4., drop=0., drop_path=0., + local_conv_size=3, + activation=nn.GELU, + ): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + assert window_size > 0, 'window_size must be greater than 0' + self.window_size = window_size + self.mlp_ratio = mlp_ratio + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + assert dim % num_heads == 0, 'dim must be divisible by num_heads' + head_dim = dim // num_heads + + window_resolution = (window_size, window_size) + self.attn = Attention(dim, head_dim, num_heads, + attn_ratio=1, resolution=window_resolution) + + mlp_hidden_dim = int(dim * mlp_ratio) + mlp_activation = activation + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, + act_layer=mlp_activation, drop=drop) + + pad = local_conv_size // 2 + self.local_conv = Conv2d_BN( + dim, dim, ks=local_conv_size, stride=1, pad=pad, groups=dim) + + def forward(self, x): + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + res_x = x + if H == self.window_size and W == self.window_size: + x = self.attn(x) + else: + x = x.view(B, H, W, C) + pad_b = (self.window_size - H % + self.window_size) % self.window_size + pad_r = (self.window_size - W % + self.window_size) % self.window_size + padding = pad_b > 0 or pad_r > 0 + + if padding: + x = F.pad(x, (0, 0, 0, pad_r, 0, pad_b)) + + pH, pW = H + pad_b, W + pad_r + nH = pH // self.window_size + nW = pW // self.window_size + # window partition + x = x.view(B, nH, self.window_size, nW, self.window_size, C).transpose(2, 3).reshape( + B * nH * nW, self.window_size * self.window_size, C) + x = self.attn(x) + # window reverse + x = x.view(B, nH, nW, self.window_size, self.window_size, + C).transpose(2, 3).reshape(B, pH, pW, C) + + if padding: + x = x[:, :H, :W].contiguous() + + x = x.view(B, L, C) + + x = res_x + self.drop_path(x) + + x = x.transpose(1, 2).reshape(B, C, H, W) + x = self.local_conv(x) + x = x.view(B, C, L).transpose(1, 2) + + x = x + self.drop_path(self.mlp(x)) + return x + + def extra_repr(self) -> str: + return f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " \ + f"window_size={self.window_size}, mlp_ratio={self.mlp_ratio}" \ No newline at end of file diff --git a/models/ImageEncoder/tinyvit/tiny_vit.py b/models/ImageEncoder/tinyvit/tiny_vit.py index 3760f09d..288dcf5e 100644 --- a/models/ImageEncoder/tinyvit/tiny_vit.py +++ b/models/ImageEncoder/tinyvit/tiny_vit.py @@ -19,8 +19,10 @@ from timm.models.registry import register_model from ...common import LayerNorm2d +from .adalora_block import TinyViTAdaloraBlock from .adapter_block import TinyViTAdapterBlock from .block import TinyViTBlock +from .lora_block import TinyViTLoraBlock from .utils import Conv2d_BN, DropPath, Mlp @@ -192,6 +194,10 @@ def __init__(self, args, dim, input_resolution, depth, num_heads, window_size, # build blocks if args.mod == 'sam_adpt': block_class = TinyViTAdapterBlock + elif args.mod == 'sam_lora': + block_class = TinyViTLoraBlock + elif args.mod == 'sam_adalora': + block_class = TinyViTAdaloraBlock else: block_class = TinyViTBlock diff --git a/models/ImageEncoder/vit/__init__.py b/models/ImageEncoder/vit/__init__.py index 807b9725..6442e7eb 100644 --- a/models/ImageEncoder/vit/__init__.py +++ b/models/ImageEncoder/vit/__init__.py @@ -1,2 +1,4 @@ +from .adalora_block import AdaloraBlock from .adapter_block import AdapterBlock -from .block import Block \ No newline at end of file +from .block import Block +from .lora_block import LoraBlock diff --git a/models/ImageEncoder/vit/__pycache__/__init__.cpython-37.pyc b/models/ImageEncoder/vit/__pycache__/__init__.cpython-37.pyc index 87a1f3d5eef40b874b5ebed98f66154c77986497..6d594fddc461b1a4c333acd751ae86bf7f4222ba 100644 GIT binary patch delta 213 zcmeywc!7!6iIS27ML#_0nS@YCdo z;!T98h))73DPjWZy9E*}C`m1XiZO%4SRwK(AT}4sNT?($P+%oP5hsu;;+!}sii-`% I;$Z|q09yex*8l(j delta 159 zcmcb>^ofzziIav7r-89{8O9Hw06C}tp=Iha9{ ziQ-L!$iydsq>7k;>Tj_^*vvroN`@j< gAO$9Vx#_1QmLwYMmseHl2Rr&s{O`^NQqRK(0Il*TQ~&?~ diff --git a/models/ImageEncoder/vit/__pycache__/adalora_block.cpython-37.pyc b/models/ImageEncoder/vit/__pycache__/adalora_block.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee7af73bcbd9b63a549ad882454d88ce162b735e GIT binary patch literal 9244 zcmd5?%X1t_TCc2Uzx1%I@k8q!ikV?hgDlzd%#8_h!T0?EAvs&F! zcU3E^TCx?@7d)^hUpat-P;-KS;KY##f)f}1f;tgF1c7gyIRL*ev#KAG$IByv*j7|l zR#s+KK7QZ#{k|`CduFDh;b;B*Uj~2sp{D&aJrutZGN0j!FELHy8fP7?%eGio-_pCr zmcgh@@0eSryjxv+%g)Q3uB~fK>z204C^5L%sdTGbHKu*8af{pEXxt9$-I*80)+|aK zUP4JJFPTG0nO9I!$xG$~bN5WJ5a{x*Be&4hFKTr0>MMiS_{=xv*4a_5{5|RyIke}q zuPi>t=TUQ!%4KaTKl94u3;Zm~-zmyQ_i+#79JN_|yoM>9`MG9lTML-ROjf(aeH!EKPSn^tdbBD0Fm6Vo8whusiemS$owVGz z-|Iypaib=lLmnNvev$+sP2KfF?usC8^S5H&Y%a@<-gm~JjxQQguUTDW`2pp$& zs=r{V=IM!%>Zy@hX(=tI70&i_v65CpJFTRZrq1=3Hfoup`c{ILiGw!gqJ|zEZoD*5 zqlEgJSLP-b-SXS`WW~*Yx5Q3-WAsFYSR8k`9VYb~?nb}6jjh14%Za#+mLG;eCtjUu zFj~md&Bog#R~p^8f!}DkkK3DSgQt5M?Ss4A>_k2d;Vaq}8Ufz9tv2UDC`Xt?Zh*mY z-VS%DV7jAX7v1IUDC(e_FPiSA=m#sV&$;Wnoxm5NzugI3+G`h`?e_!me8t@hkR14( zelXqP7!2-m7_L5yc)t_S3IQaLupyY6S;^}vZB;vQgMG+z>|R-MF_Ul#Zv;}Tkh{}~wlVJ!#-?XD7Cm=4A>Bxr zUcrQtx{J?mL?P|p7bKvhKcY@wY7H7%!|ycuG+984*0GrRsZmU<{%9p!n&EgTQt8w! zQHF$zDkY?U#T+GPkU)vgjCRf=#pI9_P{sguHtLnkg8G9JDtS`FcwS}^8n0&MTZIzL zOe{jE%Ga~f!v|kVg_;?dZDyc9F;AbWWo8_7ny3w-7o>dDu*f853pG4MjZNL?TfCz z6Wk3OcoJ&|?PSeYdf$_qzS?^(&f_ce;rId)?Yk9LVfw`1l2Kt7^rLqs?Zs-bAay-s z&%^@SHFRo?pPgTKaF72GNy-L#svYA#WCQ5dT?TDxikm6hg&w_#a@>uJnyw9ufeF1U z>%7pP>H{{g(9%57vYA+_4z#n8noa$MiGKBzo!4+j>tpEVV~1J|^D<~(b3+2+HJPH< zG}wAc(2V5rDJAHKd6O2MK{( zPO`*IhoEKRPeJSoTX1Ea2I`D{1R?sx_mLDlDKK`;CJQY5*~mNKJdEqFOv!mbJi&Ru zm3Aw1aSNOWj0l_uoCut!1^|@AC3Lq@H^o`p#Udr|P$J<=v!@^3EnGDBn6_t34ibHr zz^T)BrExc934k{$0U%#!F!(mt@I_}Y;8ngYKwVhwM^6-usB za*dLYDIr)&B#KWdN4iN|r-Zgb;!gY)<>V4jQmlzY8*T6dTrnZ2WMMIEx#?5rl5P-H z+C);IOS149;`T_646VZW2KFBAWA zM+QTYFGa2k>o(#N6HH?DHErqUy$t|oY%ktm#?+qDKGKh>QeMLMuaFtT7a9`0mH@n@ z4ID63c9x)OByc=X3Q#Xtcuknl^!Dt}2R-3|ba-#`7GAQdgXj|2?FBu_&8fF78{3$R zZa+@kZ4h=59khAC-F6csmKf$6SDmO5UapD{=@Y~j#II4;^JD!jxf>`0C|4kEQjH%X z$qaxdGx3>S$>FluTX7tKM?~RWAz(F0%Vp&+3*s*IT4oX1ha;TB!)uMRhcH>v3cOYR zvl1y7qOFB0l)6#sLsFj}E=BFmR+!C7pcxm~f^IV#G{e?icC;{3PH(ptqzINXCd?!h>ChV3seKGHN*Za`Agv>7 z>qGO{;^vS&H3rV0G$^MI-rIOzI(85{(uexIHh@13{qx5OymI-To~w!n=4{YLoyu+v zL6y==M)|ZnuN}{*uTh2|3iEomfgtziIAzTyX#2=NVcGDi}1H!I-UPD{wJb5sLTJ4x( zN2Qc4p>ITw)ZH4|s10uhJ;2|AG>|OV$JP+_IlLA8SwJ7*6)~r#-lTYA$^RNENLoZF z4&{@OCmp$A6kd)F0@3k%JyK@MMajc<@H%982C6O$0w_IY-KGudGq~rH&%+y+3Lepz zJrpQ%^3&TZ?)??_$%=dDcgMCdcZ8&3%|lhHD0jZ)NWYTtwu#_YKDHa9LHbR&sP4#_ zD!%CchD#x>y!uLBT}JUvJWP&h`F;=M>|qu(Z7BFw(B5e!GCq0O=7?f$2S6<>K(T^y zlw;kltcle3WKv#B<3h9=t{q8p#_nYX`W9c~L3~Lil}3ah=uSWC#}K|T1Rqn43Gus= zS<(Ykx5G@o-^=tTz0Aqycv2LX(f*Hc#e`+8I;+n!`Xh~fiG9EXy+=%gOaFnO71i%X#;l~^bf+o8Jur3wV+@nM3*!WL(n@o03{Lzz!$_&f*6nm ze9LK>)OG=0?f||9;H&CeQhAoq(uUw1p{)bK`O~zFh;-=>po~*nrVgc4(ljpu`a_NS zB{l2+NdfgIDq=O^ZY#`3qP^3mh+B*xRS-6UI1fU1VcH=BIqcsA_I$m60p$O8091WT zm=-ZNRdgaEC6RFGF3SUhx4^Xk^jPd#6Rk#za0=$jB(L=`+NBOyt-M(#pWSdL))!JU zUQ-$QKP~@CU`Q?`BRU~9;-`2)-nF!UBS{gzhvItW1Y$)%y=+s`pkyR5LJE+Sr}yZE zOx~wl9m!wglD0sbErH1_F~|fY1ad)d>t<%}=OO<+b%zC(=LJeS3{;{YNrsI}nuZLS z1hSC8?5FGnoFJ}WKnQ;TC9VQfsj+7Wo*@2^>L4hfH>w;|#W=PliO`1zTqh{>m!Lwa zq%o;6)bSRo9%jM}(H6li=NR%eWFRvn1@i0$-^G`|gJUuJ_P~;LK%=M*)glj6jcruN zhHTknxM-x)?+^kbH_bUN4{<6arRr%RRi6hGZ#^AK9tA-mO^rq*q~(BxnGC>=T+|pR z0uoTtOQO)d6hTDqCD`+5^t7Q-n^K5|v>iBBka2%^e++W9XD6#s&wJxa`G!J>3e$CJ zN)ukRyZiw5tNv{=aQ7M5S0e%VC*sN9wj8Ivlh0LALh=#D0+QJX=pe8|-U-b&GbtRq zk^3+I_CpAS48R%SaY|r_=44e^_CWL^>`@?oLOqhPQI3KQzaiW~Pn{^-foIU9H`JkM zA<1g!Ns>k0j+1OIUnZ2$=Q&Q-yhenrjkD5Tk)ulTEF&*CVT-crPLLG*^B?dH9e{C$x>_#NaU)CMyH@O1F98tOUeEZEarVi3bh=hDj17|hEj9Zyv} zoq=hy^15JHLo1z!@EQ=rH>6GLq6S`f=1uQZnW|CrNM1@>Uexd%usjH!=~I+hiprh* zR3mW5#NPd?ki;*?2T6B}bG6uQim2<}=#lqu<7Zt2?hn>~rlg>iB)#~?+S(4zXZzc$ z@D$dXej`8(cMrP_&^20X)L;z}{_VA^*T47aC!buq8eF?}g*UG^uYIz<9(?@KmG$k9 z{Ex0)_t!tU=C3zCy`q}rhXy#)P-nb2AAp>{53;%MySJ}@d_~S@TDnm=oDj!RTX*W{ zYk#6jF8Q$OQpb#2$WuH;2BqWVE`x;bdo7sdeeY07GU*-q6$o!lbX)*>B+k>SphZ2% zBe24BK5I`LpA|ilt^IE*7dfW

qPDKhL-_aJT)>_uu!I$k^@OqgbU z2(_iA>biw${DnP9ZK+mpd5`+pll{n!PSNtW?VRdE@##~`Iu)H#qdc13ZXGsOT6>Ml zH<8=RpAU=Y4RC2v5lY)h-9~ZU%nsUN-0w9?}TiQ~$lV>pE5vu(wE_MELjtE7K)!^~YvspH4JO<^Kb?WCcJ9$n% z)`TN#(ud>(O7duAH4IgJ=WOw~+U$ooYL@>ZQW@ukRP;8awi|_lRH2F>$#AzrkC!MR z&sW8~Wkh_L9*N_qHKd_WXzLM4%1Yl?he0=C^R<|06dfehFqN9IaC>2X;l_s-m)QLI O3-8ex4E@n@-+uv1*)MPa literal 0 HcmV?d00001 diff --git a/models/ImageEncoder/vit/__pycache__/lora_block.cpython-37.pyc b/models/ImageEncoder/vit/__pycache__/lora_block.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fb8da26fa28b285ea05265890e9c421892da2be GIT binary patch literal 9229 zcmd5?%X8dDdPf5o0Q2BevLte~4q`hFC!t7*vb~nFtF@#^mJ^BVvS};DRe}YlF`OZ1 zfMEkeN-WGJa5FLkYLD=)LVmaZ|a>ui@%VsNul=~lOEO#4*hC2oDAaVxO)W}X|{vna8- zgAyk%nL|mLS5Q*POXdS}Zy{I=ba~g2TWsn}8eP2l(%>~d^NqQEZd5COkNQOp?fLA> z5})JqsCk~sWo;^7cxm!Qeh%et6=kFQxQB6$+MIuB^SAf~w7J0bFSYu`SF~nzBXfS# zOWINBce2t}zt;&ebL&|z*nDNsYnIuaNcf%hPJ^l!zu7z!r^p1_Hsjj1&e^uXb#7oN zC0rIav5Z){rW4rsbAobEL3wpa8{Nw&vx8bNLp5k|usjyGUsivrH5qq!`J0*T*A zRu21JuNC+_&dS|R&lA)ta}M?oy`8onXO$4+@H+mpKxAdVk>uHGKMp(*bi7^^XA618 zI}SwTwZnFjRn-8!xP2UCm3G+cCn|4=AnA*+QOnn!?gV$*r?`nE(VlB3YzPpiR;oW| zspjd4k?N_DmQp7zrxnijb+MXOLo2PMm8Q=17Zz%nqxz);EfX7U%<~$0u(|QVKn(}= zHLuJ~ta!<9 z&o&!xlU!+Z;|6}C<$m4XQX4$o(`X;um1ZaMX$W7?w$KRh&TX|h4?;P@Bys}`j`Mc7 zO9j&%6}#xJ>_kxq-F((`w?sc!b$!lV-|Ymx2>qQ-;L={Z=DWH(Wl7*TC|SE%+HKsV)aKW;nEDpLy=0SUJ_+U zxTsP>%2&)$vVa5{d}g$B9w{c9q<}I8u(MgOWF@FSC?Us_8piXo5~1;0R=!;*!OX-W zgsOZaa~?hXTq@Mez-%)E{fT+{R4p^(pwmQc2)!WXqlQKHSdER8l9-`nJW6V+hRU+C z=QTQh9DCkhX#f6To%@M@ef{Y8+4`5aAFV$McpFQ11^Fwtx!(gU*1HjhzF7aP>+c5l z!Umqi`e8d+C#&Si9bfA`6BqFb`f7X$iT2$Jt1x}yZ`r7@OZxHR$Wp8oOHtP|_EapQ zQA4Mu_*waN7x(y&k)&*(r`iebLpFd`-DA+Brnr@|J!sG+l;dtJX}UHr1|~GFtn*xd zq7T@hgqG&1md&K3>Oe0WsoB(@o9I_h*+mU^v_64WKC!9QFfW7NH8&*`UXc-cMT4z7 zf~G6C27BD{dqFmz3l7kfNVyivsFE$_#r<&n;%#dC9qPvtze4%~?Bx4UU z_BgY8KKEKj;vK3XlXt0_4VWMKg5ShLd;y8ZOjgk=>;jwD>3&{!a2+pJdZ>+e`9+dj~Xb}!w@E(tSG6<%>4A;!>`15X$&Q);xm&pv@of} zQfgJu>IfJqO#&k;C6Q>fGHWXc<4A}#sydScu@Q7So;RLeorzzihBQlIBOz?d89L0g z2~;Nj6u7RkMOW5opw8$=_@Q6?07=1(0%Ol?vY>=N3wawnhjIO-DR~aaCU_3`(O!iv zUINbn3j)sp4+77rVfP(z8QpExO>qu)ah{U5D3REu+0&2iC0sQ3nD%DO2@-vekg3ym zrA;?w2~amG0UDocu=5tz@I_bW6M=D9&poI28f=>Jv<>V4jQml!D8tv{wTrt7NDPb`zx#LsNl70_W+Cox5OVaLI zVB(M9MF6wuDW}sQ;4f_nGxAv`VO9prD!e)cvl+33u8)84sNYH2SBP`DBO{>*mm<`K zMH}&l3HGq|ines?{w7c}wh^x}U20EhAL&O`DKD}6SICU93*87VOXywJ1~ynJJ4ZM* z5_lRY1)|SusXT* zsT-v}B=zaxQjG3gg;~r2!MMa0b&FXb7?$p`@MXr=O|gxjnC#P1X;Bm`gm2VP+DX>ICr;@5E# z_mE^ZJQ^aD3NJP?-F^~eRzKW<8zz36>XZ|Tw!no66lg9$9e6xwL|rQ5MUX@M4)yxG zve!6jCSCt&W*)YKBXJ*P_3GFhXY7vnO{%;~m6@l8!VWUFpRq&9WXINXY_XrCc1(mw zW3&3I{=R+z5P4g#u?+y`cwr2Y!ibl|9aP@BzqMIEW6wA6Mw#;OQ4TtNf=T`l>^Ygy zC%9tren1^d2+PR7Wwv-(8d)%t_o#R29!+^5gjRC*BwGI#8Q^3Rtqe$D7c|%?q(CHt zbsIn1uww8};#*`V;ir6^5Uz=D;*ZKwt7#4O0b$p=sG+TWk$e|It#(53A}3|b=o=9u zbuSGq)P^5}9^mOf8b}tblhP3NIs6p-m4H6NE8g2C6O$0w_IY-KGudQ@G@k%)i|qu(Z7BFw(B5q&GA4P{=7?PG1VAk;K(T^ylw;kltcle3 zWKv#9<3h9=P8~^e#_neZ`W9c|L3~anl}3cX=Wajh#}K|T1iz*l6XJI_D@or^-HtN- zK`+xE_cA-5<7rV`LHj?!6%&@V>a0G`=#Mn^W%eEu^d4~xj+?^#734A!l1Ti2B_?FW$sqF&1 z+y;CNz*p5TN#$8aOACT?jJ7re=g-qJLeS11K^do(Ol?Z3q-ibz`a_NSB{l2+SpoIO zDoQosXe-P|qP^RuC|iucR1h|TI1fB`VcH=BIqY8t_I$m60p$O8091WTm=UPFuC=s(BS{f|fZ}@P6kUaxP z4>MtgXp69xeFFI!GLRXP0(n@2@8Zkf!8sUxdr*>fK%=M*)glj6jcruNf^1o2xM-x) z?-2YVH_bjN4{;16rRqr`Ri6eFV?7%_9tA!jO^rq*q~(BxnGCv)T+|q+01{BrOQLYS z6hTDnCD`)_^t7Q-n^K5|v>iA~kg#_`G!J>3e$CFN)ukRyYdkB ztNv{=aPKMDS0e%VCt}Fowj5`^lh0LALh=#D0+Lw>=pe8|-U-b&GbseSnfovQ_CpAS zjJ_G*aY|r_=44e^_CWL^>`@?oL_LzRQI3KIzaiX3Pn{^-g=f&DH`JkMA<1g!Ns>k0 zj+1OIUnZ2$=Q&Q)yhenrjWcJz$WbMEevucPvPD^SH%JQp`49Mp@*-*%^jY16c3Ab)S`qK_JgRV>qpDEu>KtP?|(Mv6p_e)KRhQvZmD)Xa@j33o85gwEg`oWdD{ z3i39%l!1H|eh2vowZY5)JRQ8OhI)273-&aZ7{svBxwJAg2J`Ys$5Rzg3ovb^ye=5l zu$0b2cnt{RYtm+ENdvE2c-=cyrfL*Dl9!T}&ujP&SRMq=^eMh9MdfaOk`cILV()%Y zNaE+?!=rn}Sz7EiMbvd~_Q-p<`I9b!^@kfjQBu%Kl3sjseSH^avi+Skcna%HzY!pS zyN}%l=o+ncYOs#L{?7Wf8{hl*qmQm%3$9c-9o{s-4?_!}Qx z_ct0JUsX-=g94mmsB>K$3qa1_1=&3C-8(lvyej82E#0UbPKe{EtvhwxbudvSmwecC zsbj`1NIxYY`66a}E(4rpXu~%U_pSCBC z%!(e#*8Vq@iyTvakVZt)pX#vCdxgD^dl0t?_98VVovIcKA`R~fCQP$Fg4$A3b=^WW z{=%N5wp1&)vQPc&%YI}>XK4AGc24!7`1F}&or+GWQ6A6kw2m69t^LN8TgdI_&qu}c zCb%@I2&L_$ZlkzvW(Dmq?sp}%t}nh>eIUYu_z04t%)sO*0i*t+h3iNWhsX2<(kYoC zyo0PnTiQ~$)2A=uOH}(;T None: + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. If it equals 0, then + use global attention. + input_size (tuple(int, int) or None): Input resolution for calculating the relative + positional parameter size. + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size if window_size == 0 else (window_size, window_size), + ) + + self.norm2 = norm_layer(dim) + self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer) + + self.window_size = window_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + shortcut = x + x = self.norm1(x) + # Window partition + if self.window_size > 0: + H, W = x.shape[1], x.shape[2] + x, pad_hw = window_partition(x, self.window_size) + + x = self.attn(x) + # Reverse window partition + if self.window_size > 0: + x = window_unpartition(x, self.window_size, pad_hw, (H, W)) + + x = shortcut + x + x = x + self.mlp(self.norm2(x)) + + return x + +class MLPBlock(nn.Module): + def __init__( + self, + embedding_dim: int, + mlp_dim: int, + act: Type[nn.Module] = nn.GELU, + ) -> None: + super().__init__() + self.lin1 = lora.SVDLinear(embedding_dim, mlp_dim, r=4) + self.lin2 = lora.SVDLinear(mlp_dim, embedding_dim, r=4) + self.act = act() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.lin2(self.act(self.lin1(x))) + + +class Attention(nn.Module): + """Multi-head Attention block with relative position embeddings.""" + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = True, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + input_size: Optional[Tuple[int, int]] = None, + ) -> None: + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (tuple(int, int) or None): Input resolution for calculating the relative + positional parameter size. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = lora.SVDLinear(dim, dim * 3, bias=qkv_bias, r=4) + self.proj = lora.SVDLinear(dim, dim, r=4) + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos: + assert ( + input_size is not None + ), "Input size must be provided if using relative positional encoding." + # initialize relative positional embeddings + self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H * W, C) + qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + # q, k, v with shape (B * nHead, H * W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W)) + + attn = attn.softmax(dim=-1) + x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + x = self.proj(x) + + return x + + +def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]: + """ + Partition into non-overlapping windows with padding if needed. + Args: + x (tensor): input tokens with [B, H, W, C]. + window_size (int): window size. + + Returns: + windows: windows after partition with [B * num_windows, window_size, window_size, C]. + (Hp, Wp): padded height and width before partition + """ + B, H, W, C = x.shape + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + if pad_h > 0 or pad_w > 0: + x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) + Hp, Wp = H + pad_h, W + pad_w + + x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows, (Hp, Wp) + + +def window_unpartition( + windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int] +) -> torch.Tensor: + """ + Window unpartition into original sequences and removing padding. + Args: + windows (tensor): input tokens with [B * num_windows, window_size, window_size, C]. + window_size (int): window size. + pad_hw (Tuple): padded height and width (Hp, Wp). + hw (Tuple): original height and width (H, W) before padding. + + Returns: + x: unpartitioned sequences with [B, H, W, C]. + """ + Hp, Wp = pad_hw + H, W = hw + B = windows.shape[0] // (Hp * Wp // window_size // window_size) + x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) + + if Hp > H or Wp > W: + x = x[:, :H, :W, :].contiguous() + return x + + +def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor: + """ + Get relative positional embeddings according to the relative positions of + query and key sizes. + Args: + q_size (int): size of query q. + k_size (int): size of key k. + rel_pos (Tensor): relative position embeddings (L, C). + + Returns: + Extracted positional embeddings according to relative positions. + """ + max_rel_dist = int(2 * max(q_size, k_size) - 1) + # Interpolate rel pos if needed. + if rel_pos.shape[0] != max_rel_dist: + # Interpolate rel pos. + rel_pos_resized = F.interpolate( + rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), + size=max_rel_dist, + mode="linear", + ) + rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) + else: + rel_pos_resized = rel_pos + + # Scale the coords with short length if shapes for q and k are different. + q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) + k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) + relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) + + return rel_pos_resized[relative_coords.long()] + + +def add_decomposed_rel_pos( + attn: torch.Tensor, + q: torch.Tensor, + rel_pos_h: torch.Tensor, + rel_pos_w: torch.Tensor, + q_size: Tuple[int, int], + k_size: Tuple[int, int], +) -> torch.Tensor: + """ + Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. + https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950 + Args: + attn (Tensor): attention map. + q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). + rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. + rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. + q_size (Tuple): spatial sequence size of query q with (q_h, q_w). + k_size (Tuple): spatial sequence size of key k with (k_h, k_w). + + Returns: + attn (Tensor): attention map with added relative positional embeddings. + """ + q_h, q_w = q_size + k_h, k_w = k_size + Rh = get_rel_pos(q_h, k_h, rel_pos_h) + Rw = get_rel_pos(q_w, k_w, rel_pos_w) + + B, _, dim = q.shape + r_q = q.reshape(B, q_h, q_w, dim) + rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh) + rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw) + + attn = ( + attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :] + ).view(B, q_h * q_w, k_h * k_w) + + return attn \ No newline at end of file diff --git a/models/ImageEncoder/vit/lora_block.py b/models/ImageEncoder/vit/lora_block.py new file mode 100644 index 00000000..dfc8b1c7 --- /dev/null +++ b/models/ImageEncoder/vit/lora_block.py @@ -0,0 +1,268 @@ +from typing import Optional, Tuple, Type + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ...common import loralib as lora + + +class LoraBlock(nn.Module): + """Transformer blocks with support of window attention and residual propagation blocks""" + + def __init__( + self, + args, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + norm_layer: Type[nn.Module] = nn.LayerNorm, + act_layer: Type[nn.Module] = nn.GELU, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + window_size: int = 0, + input_size: Optional[Tuple[int, int]] = None, + ) -> None: + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. If it equals 0, then + use global attention. + input_size (tuple(int, int) or None): Input resolution for calculating the relative + positional parameter size. + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size if window_size == 0 else (window_size, window_size), + ) + + self.norm2 = norm_layer(dim) + self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer) + + self.window_size = window_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + shortcut = x + x = self.norm1(x) + # Window partition + if self.window_size > 0: + H, W = x.shape[1], x.shape[2] + x, pad_hw = window_partition(x, self.window_size) + + x = self.attn(x) + # Reverse window partition + if self.window_size > 0: + x = window_unpartition(x, self.window_size, pad_hw, (H, W)) + + x = shortcut + x + x = x + self.mlp(self.norm2(x)) + + return x + +class MLPBlock(nn.Module): + def __init__( + self, + embedding_dim: int, + mlp_dim: int, + act: Type[nn.Module] = nn.GELU, + ) -> None: + super().__init__() + self.lin1 = lora.Linear(embedding_dim, mlp_dim, r=4) + self.lin2 = lora.Linear(mlp_dim, embedding_dim, r=4) + self.act = act() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.lin2(self.act(self.lin1(x))) + + +class Attention(nn.Module): + """Multi-head Attention block with relative position embeddings.""" + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = True, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + input_size: Optional[Tuple[int, int]] = None, + ) -> None: + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (tuple(int, int) or None): Input resolution for calculating the relative + positional parameter size. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = lora.Linear(dim, dim * 3, bias=qkv_bias, r=4) + self.proj = lora.Linear(dim, dim, r=4) + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos: + assert ( + input_size is not None + ), "Input size must be provided if using relative positional encoding." + # initialize relative positional embeddings + self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H * W, C) + qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + # q, k, v with shape (B * nHead, H * W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W)) + + attn = attn.softmax(dim=-1) + x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + x = self.proj(x) + + return x + + +def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]: + """ + Partition into non-overlapping windows with padding if needed. + Args: + x (tensor): input tokens with [B, H, W, C]. + window_size (int): window size. + + Returns: + windows: windows after partition with [B * num_windows, window_size, window_size, C]. + (Hp, Wp): padded height and width before partition + """ + B, H, W, C = x.shape + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + if pad_h > 0 or pad_w > 0: + x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) + Hp, Wp = H + pad_h, W + pad_w + + x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows, (Hp, Wp) + + +def window_unpartition( + windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int] +) -> torch.Tensor: + """ + Window unpartition into original sequences and removing padding. + Args: + windows (tensor): input tokens with [B * num_windows, window_size, window_size, C]. + window_size (int): window size. + pad_hw (Tuple): padded height and width (Hp, Wp). + hw (Tuple): original height and width (H, W) before padding. + + Returns: + x: unpartitioned sequences with [B, H, W, C]. + """ + Hp, Wp = pad_hw + H, W = hw + B = windows.shape[0] // (Hp * Wp // window_size // window_size) + x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) + + if Hp > H or Wp > W: + x = x[:, :H, :W, :].contiguous() + return x + + +def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor: + """ + Get relative positional embeddings according to the relative positions of + query and key sizes. + Args: + q_size (int): size of query q. + k_size (int): size of key k. + rel_pos (Tensor): relative position embeddings (L, C). + + Returns: + Extracted positional embeddings according to relative positions. + """ + max_rel_dist = int(2 * max(q_size, k_size) - 1) + # Interpolate rel pos if needed. + if rel_pos.shape[0] != max_rel_dist: + # Interpolate rel pos. + rel_pos_resized = F.interpolate( + rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), + size=max_rel_dist, + mode="linear", + ) + rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) + else: + rel_pos_resized = rel_pos + + # Scale the coords with short length if shapes for q and k are different. + q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) + k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) + relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) + + return rel_pos_resized[relative_coords.long()] + + +def add_decomposed_rel_pos( + attn: torch.Tensor, + q: torch.Tensor, + rel_pos_h: torch.Tensor, + rel_pos_w: torch.Tensor, + q_size: Tuple[int, int], + k_size: Tuple[int, int], +) -> torch.Tensor: + """ + Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. + https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950 + Args: + attn (Tensor): attention map. + q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). + rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. + rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. + q_size (Tuple): spatial sequence size of query q with (q_h, q_w). + k_size (Tuple): spatial sequence size of key k with (k_h, k_w). + + Returns: + attn (Tensor): attention map with added relative positional embeddings. + """ + q_h, q_w = q_size + k_h, k_w = k_size + Rh = get_rel_pos(q_h, k_h, rel_pos_h) + Rw = get_rel_pos(q_w, k_w, rel_pos_w) + + B, _, dim = q.shape + r_q = q.reshape(B, q_h, q_w, dim) + rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh) + rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw) + + attn = ( + attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :] + ).view(B, q_h * q_w, k_h * k_w) + + return attn \ No newline at end of file diff --git a/models/common/__init__.py b/models/common/__init__.py index 5ae450b6..a5093def 100644 --- a/models/common/__init__.py +++ b/models/common/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. -from .mlp import MLPBlock -from .two_way_transformer import TwoWayTransformer from .adapter import Adapter -from .layer_norm import LayerNorm2d \ No newline at end of file +from .layer_norm import LayerNorm2d +from .MaskDecoder import TwoWayTransformer +from .mlp import MLPBlock diff --git a/models/common/__pycache__/__init__.cpython-37.pyc b/models/common/__pycache__/__init__.cpython-37.pyc index 349c893da6071e4050886924fcbf193aa78d0236..8b22a033724594e9c184b4f9017ad522fb372d35 100644 GIT binary patch delta 132 zcmX@fbdZVHiIO{MNB~5x43dZO5^i@YKxdb0^Gid#n~>Y$@wX%MMW$i5$4>S Tf+AKRYb8Sw$jTzFiSMHToGBPewrM&m~(RqikN^pql8P!^W)1CE8|Ow67!1FfGSdpikN{ix7ZUQ+KX5~ yJgyv&3GsPA1x2htft3tJoInap{IbzcNi0b;)-SKB)DL#_)%Q(J$xNQa=m-FVR<_{0e63dK`IN1|Uxu@vvtKLt0sJ)EDzV%gA!-HZtrk0L3 zZd8I6`v5q&ogFCXL~U%pZKN_6QU@6R&}+>&G%#bQbH}nIWvSq@05w5BNjMxEQ8Qg} np$sc3t*er`i%K3?p371;c|JXMHk@OoRbt`3Faspt3u#E-c<4NR literal 0 HcmV?d00001 diff --git a/models/common/loralib/__pycache__/adalora.cpython-37.pyc b/models/common/loralib/__pycache__/adalora.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62583a9b6dea4b580344bddc20855515c65ce44c GIT binary patch literal 10157 zcmb_iTZ|;vS*}}GS6`;5XQp>{_TqJ8$5GnEJ7fXdtTwiHZI6lB9Xq>=*XcN_t?5%U z(>>kQy{D>oXF4?ovYW(Qd{Gn-L>8K{ghT{FJOl}NKmB;d<{c|G@w2pVPD-@MUymPT{^e3I}zLU%G47=S5NEMd6-aFAM#eR-5|(E6~RE9N^@VqJtYhw8rtXN6m`8oi&$kMqjQRz!c+aXA#S&JxoaC;(^4#TS;CoVXLDB;F zA#Qc8U9V-6LNjnTeJ|=uKTL|-nN$%RawYx551DorWd#ziwEE5 zI=Y1ThbG?V~G(2iJcNJ zaa`cK8M&?`@5?!WB=@o>y^bIGGO<&%C%M4C;|_e;4HH{Y>*DbF{P1}x$x1KS3HqJH z5?MeedH@a~KSX8>MRV7Xmr-k_+^FjD*2;p~jxeZ9 z-h}F54^@|&)G1ELtjlH0yMt9U3}r{H0}a;7J7hf1*mU3s&lSMeAeylg{mv9(6i(Q5iBqTzPT?YwUNbm*q1^>%hEJWnuB3JA-?o47 z#-IG?{ioJya}y#Z8P$2n>q-%dR2V9lTb?8!VNw#2YVrYs696?cCDx;?;BpkjMk`f| z{48oO;tr`aG`(n)aOc78kLrW@3Gvh##t7nu#*}K|o;n9Lpc5s%S?lCi)u8Tk7Q~TK*I& zQximOtJGvFT1p5fqGzR4U1cYdHnE`q`|ao>BA8@{>-rLUDRo0JEbA+JUN?+$x@8O& zKC*}PT4rM5C7mERT(FySwB`XUx9cW(*M)xVw^=T_?ybJp9=4dxAd;uKC~KqJZ7Xx9 zRJf!%kmNb2ovv(bC-!we2)k1L3>%chOn{Od>H*sEkT%n?K}7XC{h*U87oDPHdjt3Q zqKonj?(nw(#_p+JH^mv>8vo#;`nGSK(1y?w*1LJ-LTBn(-x684qmIbI9d(q`3^!G| zty(>g))HFhMH#IH<+N&WS|1b@aSYD&oLCme;ntSKia3Giyf`UN;aL`^#RGUQ2m`EH zP3Eq`nYq}ejs_~cc?eX#itGBeU!7{If__!F7WBreSLj+)qirv$-hua1-SDf>@oisJ z{oB6Wt13kb-=`Wze(!8m&G@p{^X2*Kb`p4fc4DL&u6 z_4v8|jjJ!VnqSJRQ5WI#oL42TO1U)W@nJtL4wv9py8>0e{^0$lNJ+q@Nh_2*X%V z=*w8tS4Q{GDMLLpzJNp>;QKfnwD&R=4TvzN6@O<#)>{zzIxP{jl)y8!bH*7X(Z#A zxE1qMQBZ8ad=< z-Bk^%9h{zC>2%kRxQl9`bTPQZG%(X8UHK^|Uwxn0OUff#H6rlup*D3Djlr4ePtSH8 zf1GROH`T}g7==i?OF==f3TP-&ZLh$ogKw`H201Q-RSq~Du zm!w~(mod#X@+I{7F7A+ytA?x#8NGtc3nC&sk)hGc_~rG%PkfY)GZIXfp9XKt_)Ia8 ztcs-Xp&&pmAdgWD5WCN$oUzXYpSh1H2#5qJVrxg2kE1TL58bwJ1Ojz-oLPa0$aU0f zs5el5Sunb?R+xB1D@&)e6>Z<4ZbsVBM4cn7C9R?z$C${Dvv>9VEWgvrE+cciW60kU zMqsE_A-}On7u8;0yRP)+2PPJrDo-cH?L-!p1}to~yBk9rFJED2=4zC(tmRTUZJ;7? zhIVO4p_y57jlPu1(W~b={kGfa`kR|GR7w`Z#B&LRx*+e@j z-GoTTHf?i7Uk3S>^})1fefFc1jVVTIF-lVu`)3q(--lRfO5FY);s()zI2sU_Jgr_! z-^t4Bk#T^il!;s@GvflH)A74C^PU;L+lSiQzl`)N#tPYNmK9rr9udZxWjfGhZg;W)L18;F&^s9n`w)o$fk2IbB5 zdM5SWiG^BGzKVxr8eEeF8SEt*5dzM8%W8s?P{LJW-+q>@hK9;{4@<7B8>af4gn zmBCF$Rs^MTKz?2aRLYgTymyz^? z+hGA+*1GyqVs^Z(!SP=V{M`o9zzt}k>J=Z+Oyhj@k+5b@P$k>=yzBrZnMS)C_(}d1 zuif`ko1RG}<|g|PIf;!#Ic7<6K!+Q`LH1|yHIEpu(QgxFs{NlZEMy?8A=5_IUo?vP z;DKpG&5UUfIlPE4-U?BlMgisk`ichSP4GRW0LKsDaU*>~qo_ZT@k9Fvw>)ejsJ>IgtU&Do@Rij zH+8h@ct^}yQUj|U{ez)tO{^KWg*)?(z6)JDlqq~KF9z=Ss2_R{8S6Npcfamz#eh&f);m;CwEcPP)Ja#5-IlQTGQ9F!JRg^_qJ_99ip$#mV!SX~CG^FY4aCD+NgcKqs#9Bf2D(Ye9?yRF6|&C!`%J&-8v{kXgTvTaIgS}C8tJdEgX8a zmSFdCJI1hP;h-YQvAwA;YcYE-9-5emrDL&;{Z!&*WE?B;F;%xb#39ZKPQYoR%0TO} znN~-&Vs~jPaPYkwR5IyG4EcS22lr?tX$@DhfbWggfc~gijF)C=mhiPY#=78~$&md1>!vWmtBjI4- zl1fUfPGZNwv0;@m^JDuwC=aVC-YUwe9ZxOFYy4eP1~lI8ciGj%;z%Nj4YxUATb7!;pHE8Dn5)kzmxy?iKzuzn#-KWWnbng^>FDm! z*!Dj$eYgpr;RF#soG8+WmS9Hd(egYdeP229&bFdyp&u;A(DPk3t7cKnv55L*oK2ol zDP%`KqkjtJMf^@1v%c|tRy5YiRN`YhLqaJ(N||AJxS%OC4XjNXxWJFX0XZ~)318Wg z=g~$FnITRlzzG|DXB=#%y^(uR9xQ@KJB<)8VtN@6Y!iF~AUWoCyuA%%0>;5M6CM~* zWf#Q_igU;kk|>m4Czx?n=7;`<7YNK3?mVFb;bnBx^s>QIO3Nt0FGbieV}57GjU7)K zs>3XG6aU#*OUzB#g@U304i;XL(Y7o7tv*N?hEz`#l?h)(KzO8~ zR+-TVsp3&8MESVXBg!$8J(kqLis|DYFki-NlJ`5kXwOxCFV7?ri|5z!b$0y~g1ZDt z@wh0BVjr?nk{hp1-efPXn;v6S&jP@0xt-`=sOQG(o#8Y2P4r(QZ}BKx(@%n<@J9iN zF}z^B*(jF)SD52?c*ZBXFP~fM>+ACKn5$-_{%dVf{w!YQ;{;CtBw5!L-3I=+@g%Dl z7fU*c>3PmImaY>pa!%z+NtjaT)YnSPzkfsckunRz5x6dGur{ z<56EjO&pR3Ck~6RR{_s1$!|TtU9?Oe^(OKNex%)N>f@%VaV)u2Zgfw82sIe5s zTabrGArE-UMMwaj7nB6(LkZA_62K!%oALz-FrZqdCBTji96e@_$^cvd$eVERG;b|} z%i$s;Ct{vrNR`8hKKH3EWpgM0d62(~+BNxWfJ25>_PXYzR+3DZ^69U$L{5~yK|mIi zZvvpF;<19_vGo5m@+~yrPc6SVHxCaq85w(79R`v!-W^&OfTB&!BR>Cl_|r}yeE!2@jDv?>dyzs(&+Wr?J literal 0 HcmV?d00001 diff --git a/models/common/loralib/__pycache__/layers.cpython-37.pyc b/models/common/loralib/__pycache__/layers.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d9a8c5004a38bbd86a341158a01b2e55fd03072 GIT binary patch literal 10083 zcmc&)-IE;GRqxx=)6?_0v#ZhSBZ{+eWSjw8a!m0RgjvbP5;1zvzB{te!Eq+fXmdgcL&N&&y$>7JgCUCDMCLQmb( zw{PF>zUSO?&-tBm=SIC=v3UH$--qA*nq~cqTx6t(^Eyg&1fngi`c{W$yRQaz#~u_q z1(mit9Y^jL2X4pB+DZf0wv;vSI%Q?OW$A);9$DIXXm={ODry&3Zs_)^VdbIIsiLi< zJ+ygft4Ui8ZDn0STP1I+qphlIXse;EA^jR!-Ll&CCm5x**|rnsW;cox@0uwt_&H%wtB)OR>%t?a<7x-*EGFffqxY`g4DN@f?Ig-1mHx;C ze$e0F3zC}DwHfV?4&tOf2+eNj-wV6DdvUbQ5Aoo+fbu#@^cu)mjV<3A+n{P((CUt3 zF2{DSfV%w=?>WOtDlFD@5wtjVKwWKNNbcq?ULr?ZC2D_LWmi12+(L;QkTJWf&A$an z)CY$1EaX3LT-8Aw{NmMnhxf1Ey8h19cS7CW4*D;n{_=Gl?8l+GIvDA&A6?xZ4F;p( zRWjP|-oDzWAfhY#_Y?10KN#HB!E0Z@YySWc4ohuzrsuUzYo@aE_>RMPGn z?UJELj0=+3!(mc-BNd=2vBecBYUAMo;&-C3zcYP6`pCU0JEq3xyvZjUxQa-ulB-KyXtV|bUh zzz-+)qNJYJecc_*$eJu{bDDT?#4Nv!5`72~TRqi-5<{Ksox<4tfl^jnke0&O(P~_T zIv16dsrcR4I)W1Sibw9aFjhPEu{x%*muA}SaUpN_(6@|I`2zIVuj&Gy^=o76sD2DJ zm!;rr3TJI+X3!j?D9A>;VU~DtkwhkplarL+4Gbn6I-a=cT20Es@ScAdno*RvvZSwP z=NqD>^7f<@L6ITMq;A6Z54t9d{9P01q={M>`}>o(Mo{n-e)1B@7fEt`mw_ejoqK`V zjb`*mREI=%8BV!9eYRgg&$m!AMr*?8bLy}$Wt3lmu2j)0pJK9FMv*E002K@!nv03tK}|(F zbd$6qobm^ly6o=lm{tYWq*vU7+$}tKS;bm@PW@I0y_ih z$M^TcWQEow6bs_tzVGkArX+3zBNBzFh((v2RF^2E#Kj6Z=*LMZ*n#cxuP-VWz?=0= zl+=8vCe*8<9Gsi>s&eeZ<*Dqm;cvDJVm}RC_*PqOLG@dxnzJOUBrP%m71RCiFMw_z5Jj8tfNHV z0m)rsYw8-qB7mCvaY@`_5pGc+BHZCwv{&H<=@yF}PdD@u++#Vc^s2gv_GP^S_gK?s zu-aG6RlI)6{!efv$Ho>wLA5*dcfue(fXtH`z+HYdW6SAjlEzLj1lI9+LO)qI+jN{f z$O?Oj&-_bNxO``PrZ?|1$Hj3;EAg2w%@P42bbkyFDK;Cnde(D#GwrjUQ@kcLyegiv zCZ5yv>+qZndd{(HeuTSjZ>iVh*FI7%M6}UfPK}}nE%&8n9Xy%mNN5tx8VM&l zRbfJhR9;kObD2amNDK)V3jBl7q9+Qx7 z;xj!gfWxKKu$}HSi+E7X%{nTx)-{H)5T|(!txiuSATAww8o)&tK7z+#`;iDccb@;44-tVhF&Q*w*_2qq&!F81^HuZM`@+)N8STFmBnK*waR)-U-KPMM`%$O zpqTIBI`uU+Y90#w>qCVz{7vRxoyQaZTHKs-Yc#K%zo-dhn>NK?7Li&`po%6dWW znVu`16k?owQwBCZ*UEUt4E>1G$=1e&OeUG@Uz{YYFz0qRh+wZM9V4saRn!m& z2`o$v{9C+7k2b|RVxCegID=MX*_vNtPioCXgE&=e5 zyEt{x<~Q&V+%s@W{ZyPnFYr{bVt$=>uaP_zr_j6oK8ir76sHiYTsi}+Qt>LH!P~f> zALMNvC5l0EtO5(l{LQY?$lmO@bI8TLV1-C}0l#<|eko=6#Y@v#ow8oV{W|ap@k_B& z(M`P!98(Qzy}DjO`x)H=j%nz#z%l2N+B-};=~Qr2Y3*V}azJ99d76Vws^KuW4Is*} zH#NAlirYjkJjfvDTM*18*lZf783a>4fndB0!Pv1gb|0!$WDD=u(RvKi^-z7N#zZ#m z7y+3NG$Z)tr?JPL=BL0j;!cmM<3e0xTMO( zP)!yLv47&+?S}UfPf{b@p1`*Z5TQ5{Ok3B&Svs5_m}AHG$$T zphr5SoXVw2_i1EyI3>0?l22Zol;srFA7fFcn%;4fP=V9F!=Qv#icXOzY$eSOI?t+Zf5lRn!zHMMqh7KXRASN-b2W#4 znQWkGEAS0}4l;`bCKxQuEy1T3%nv~wp(zUce(cic!*G2+_IhR1VP1|9ZV48sOWiqE zaU<;ogHt$K;$6WHO=P92zz)kyOQ6R}3&{Cc32LrtygAECl|Bz@eg-y3$aRJd<`wHy z^qfwmO59yDI@pgk+Gl1A5!dV@reh`|ozxUXY>$R_zoyfL!0<0A9}J`S55f@B^E~@< z?vZ27nk1b39kzc31Xfklf%7NGA}L{WA`bj}C(#85^Uo*|^A{5+-hyHP9zsD}=G#{0 zR?<8)r#j?n@n^W0QHzt%{0EpiCn8Oi;&HANk3XYQ5M)l3A_bk9QV68~St!L5usczT zOLHw+W)bq3d zijq$~BZ7sgUp~RsVuV23nMVkJhl?2nkO;y2d`t+GP7y#2RR9?QqWn|aWxXSEKwR-x zF_QAL5d*BQgQvVPG6~%7Ip!BCN0@9_jvDIqhf0@VIkLMCiV|@&#znC*X%bT+j0}K$ zju`tH*5W@X)y!IaX-5Y>L!$9EQQKE|=v=y#8Z!;F5 z(p2yx#%QU-)@dwnG7wpMdLTk^Ek+|UUNM)lL_S4G@>_VzF^cH4=cfCK4)zo;Kj)P^ zueszc8TQ#m$&WH>>nQUn)H8V?lJ0dEkwSHK2_c57JzYkKQPM4>P%8*AxWmpAYF#(- z6zUS%8HzB4x&jA(HgR8vRex1agN~m}hA>H@Ztn%dA@-5eBx-(Dy%QQV`%xF>U*?^w zZo0X9dIt4x&}TOOz;2;4p3R{;$e<#&NFxwndo0elc*l#r9=nJWtAkjrbva6ntc;l$a*x)ndQW`OX?`Dwy^lXpII zT6L04Wk%HeAxS`Tig=%zOsy&?4Z~=wy2|ZG>>V(LddX&yO~|Em%Kx8%wutTpa`z(1 z=K|U znK3?(h|q1yCe7}I&C@m^pUtEonRzC;>9oWl3zG}T;ZrdA^bN>gn({{=0RE~Of2Zv| zax!T*Wipuz$F!t-kDI8eXfu{z2X*-u1I{ick(R{a{yEO&mUWI#$p$mdvh#>@^L$wqFCE2C(>=(sYw;ipEk4NV6j~o8zc;pQnuNcc)HsUm;ew+HX~K~ zxyin?gj5wYOrt4Dh+U(!YG}8xWk?6J248sC$Rw8U7&=p9Z~g$C|1Z;V)Cl}_EZrjT zN=(He8P`HGuH=Xa{~MN6 zrgg(mMgFUk{%1rsyXag~-JA+dYLpfJ1(p6~=r!7bG;Jt0v!+%GjaKVg>y6gl{{YDF BVZ#6b literal 0 HcmV?d00001 diff --git a/models/common/loralib/__pycache__/utils.cpython-37.pyc b/models/common/loralib/__pycache__/utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e98d9a1d12f69a808c70cbba3b41f7b6b0742db2 GIT binary patch literal 1496 zcmZuxOK%%D5GJ|Yht^V_trI7J+a^T;1&9=&SHY;=)Ifn)hdMw2q3&WW(#cx;GD#y= zu(~t>>Ym$U4$iGbPyIW*_LRSnQ)jqxT_oXR$k~}8XUOj(w|c#pKwJO&FL^5@S6~T5%zxNh$kv_HjVKz16x3vB1JgR=!e3Z_mvb?NH+4rnp zR6^#K?`NsDfs*D}l~abZq4^*jeLwJmbKT% zmHED?b6LpJNb#LgmFoKn!EXa-X=!a~@h4HAg7MwvP^2dPdiease7L*0Gu)9Po2L1F z(C=@Gw1#4a4jVn3Rz*>jL##W`_J_wN%k`k1+uKE|W(n>jaeGR2VpN)y>3%MIkVAhC z9ii7~L}L&Rf_fL^5?JuN#MBLl`2cYbjQTZnEpERek`%DH^9wnkKuGUZbP6nZN&mxm zzc!2!)lr-KNX@EB)(9Qt+H&jGG=FSbW;0-?|%P8)m z_=p^hfn7jv!5UC}i`ZNT(d*Dac;E=f$!oG8UT)Z<4a6|`2zCdt~>pU8G}(?0hD@U za1pQYf%tf#UxCLJx_(}$z6sarV+FIeU?lRgYIPI!HRxbBx6bXZTy-0Maq(rHY6abH z$9Q0lmhb|^7lm72*Lh~tRdD~GypxWplJ;$IscdH-o{I$UPFuz@H6D_^{N8DN0Y(P~ z0p7I?L>6*jA94}(E0iuI=`NNgKCmZh$gTfIdRZ{G!`HX5}zd;{MJjVN^))Wy2q>L>pK-sn~| literal 0 HcmV?d00001 diff --git a/models/common/loralib/adalora.py b/models/common/loralib/adalora.py new file mode 100644 index 00000000..2c711cac --- /dev/null +++ b/models/common/loralib/adalora.py @@ -0,0 +1,356 @@ +# ------------------------------------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +# ------------------------------------------------------------------------------------------ +import math +from typing import List, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .layers import LoRALayer + + +class SVDLinear(nn.Linear, LoRALayer): + # SVD-based adaptation implemented in a dense layer + def __init__( + self, + in_features: int, + out_features: int, + r: int = 0, + lora_alpha: int = 1, + lora_dropout: float = 0., + fan_in_fan_out: bool = False, + merge_weights: bool = True, + **kwargs + ): + nn.Linear.__init__(self, in_features, out_features, **kwargs) + LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, + merge_weights=merge_weights) + + self.fan_in_fan_out = fan_in_fan_out + # Actual trainable parameters + if r > 0: + self.lora_A = nn.Parameter( + self.weight.new_zeros((r, in_features)) + ) + self.lora_E = nn.Parameter( + self.weight.new_zeros(r, 1) + ) + self.lora_B = nn.Parameter( + self.weight.new_zeros((out_features, r)) + ) + self.ranknum = nn.Parameter( + self.weight.new_zeros(1), requires_grad=False + ) + self.ranknum.data.fill_(float(self.r)) + self.scaling = self.lora_alpha if self.lora_alpha>0 else float(self.r) + # Freezing the pre-trained weight matrix + self.weight.requires_grad = False + self.ranknum.requires_grad = False + self.reset_parameters() + if fan_in_fan_out: + self.weight.data = self.weight.data.T + + def reset_parameters(self): + nn.Linear.reset_parameters(self) + if hasattr(self, 'lora_A'): + # initialize A,B the same way as the default for nn.Linear + # and E (singular values) for zero + nn.init.zeros_(self.lora_E) + nn.init.normal_(self.lora_A, mean=0.0, std=0.02) + nn.init.normal_(self.lora_B, mean=0.0, std=0.02) + + def train(self, mode: bool = True): + def T(w): + return w.T if self.fan_in_fan_out else w + nn.Linear.train(self, mode) + if self.merge_weights and self.merged: + # Make sure that the weights are not merged + if self.r > 0: + self.weight.data -= T( + self.lora_B @ (self.lora_A*self.lora_E) + ) * self.scaling / (self.ranknum+1e-5) + self.merged = False + + def eval(self): + def T(w): + return w.T if self.fan_in_fan_out else w + nn.Linear.eval(self) + if self.merge_weights and not self.merged: + # Merge the weights and mark it + if self.r > 0: + self.weight.data += T( + self.lora_B @ (self.lora_A * self.lora_E) + ) * self.scaling / (self.ranknum+1e-5) + self.merged = True + + def forward(self, x: torch.Tensor): + def T(w): + return w.T if self.fan_in_fan_out else w + if self.r > 0 and not self.merged: + result = F.linear(x, T(self.weight), bias=self.bias) + if self.r > 0: + result += ( + self.lora_dropout(x) @ (self.lora_A * self.lora_E).T @ self.lora_B.T + ) * self.scaling / (self.ranknum+1e-5) + return result + else: + return F.linear(x, T(self.weight), bias=self.bias) + + +class RankAllocator(object): + """ + The RankAllocator for AdaLoRA Model that will be called every training step. + Paper: https://openreview.net/pdf?id=lq62uWRJjiY + + Args: + model: the model that we apply AdaLoRA to. + lora_r (`int`): The initial rank for each incremental matrix. + target_rank (`int`): The target average rank of incremental matrix. + init_warmup (`int`): The steps of initial fine-tuning warmup. + final_warmup (`int`): The step of final fine-tuning. + mask_interval (`int`): The time internval between two budget allocations. + beta1 (`float`): The hyperparameter of EMA for sensitivity smoothing. + beta2 (`float`): The hyperparameter of EMA for undertainty quantification. + total_step (`int`): The total training steps, correctly configured before training. + target_total_rank (`Optinal[int]`): The speficified final total rank. + tb_writter (`SummaryWriter`): Tensorboard SummaryWriter. + tb_writter_loginterval (`int`): The logging interval of SummaryWriter. + """ + def __init__( + self, model, + lora_r:int, + target_rank:int, + init_warmup:int, + final_warmup:int, + mask_interval:int, + beta1:float, + beta2:float, + total_step:Optional[int]=None, + target_total_rank:Optional[int]=None, + tb_writter=None, + tb_writter_loginterval:int=500, + ): + self.ave_target_rank = target_rank + self.target_rank = target_total_rank + self.lora_init_rank = lora_r + self.initial_warmup = init_warmup + self.final_warmup = final_warmup + self.mask_interval = mask_interval + self.beta1 = beta1 + self.beta2 = beta2 + self.total_step = total_step + + self.model = model + self.ipt = {} + self.exp_avg_ipt = {} + self.exp_avg_unc = {} + self.cat_ipt = {} + self.rank_pattern = {} + self.get_lora_param_name() + + self.tb_writter = tb_writter + self.log_interval = tb_writter_loginterval + + assert (self.beta1<1 and self.beta1>0) + assert (self.beta2<1 and self.beta2>0) + + def set_total_step(self, total_step:int): + # Set total step number + self.total_step = total_step + assert self.total_step>self.initial_warmup+self.final_warmup + + def get_rank_pattern(self): + # Return rank pattern + return self.rank_pattern + + def get_lora_param_name(self): + # Prepare the budget scheduler + self.name_set = set() + self.total_rank = 0 + self.shape_dict = {} + for n,p in self.model.named_parameters(): + if "lora_A" in n: + name_mat = n.replace("lora_A", "%s") + self.name_set.add(name_mat) + self.total_rank += p.size(0) + self.shape_dict[n] = p.shape + if "lora_B" in n: + self.shape_dict[n] = p.shape + self.name_set = list(sorted(self.name_set)) + if self.target_rank is None: + self.target_rank = self.ave_target_rank * len(self.name_set) + + def schedule_threshold(self, step:int): + # Global budget schedule + mask_ind = False + target_rank = self.target_rank + initial_warmup = self.initial_warmup + final_warmup = self.final_warmup + total_step = self.total_step + self.global_step = step + if step <= initial_warmup: + # Initial warmup + curr_rank = self.total_rank + mask_ind = False + elif step > total_step - final_warmup: + # Final fine-tuning + curr_rank = self.target_rank + # Fix the rank pattern by + # always masking the same unimportant singluar values + mask_ind = True + else: + # Budget decreasing + mul_coeff = 1-(step-initial_warmup)/(total_step-final_warmup-initial_warmup) + curr_rank = target_rank + (self.total_rank-target_rank)*(mul_coeff**3) + curr_rank = int(curr_rank) + mask_ind = True if step % self.mask_interval == 0 else False + return curr_rank, mask_ind + + + def update_ipt(self, model): + for n,p in model.named_parameters(): + if "lora_" in n: + if n not in self.ipt: + self.ipt[n] = torch.zeros_like(p) + self.exp_avg_ipt[n] = torch.zeros_like(p) + self.exp_avg_unc[n] = torch.zeros_like(p) + with torch.no_grad(): + # Calculate sensitivity + self.ipt[n] = (p * p.grad).abs().detach() + # Update sensitivity + self.exp_avg_ipt[n] = self.beta1 * self.exp_avg_ipt[n] + \ + (1-self.beta1)*self.ipt[n] + # Update uncertainty + self.exp_avg_unc[n] = self.beta2 * self.exp_avg_unc[n] + \ + (1-self.beta2)*(self.ipt[n]-self.exp_avg_ipt[n]).abs() + + def calculate_score(self, n, p=None, metric="ipt"): + if metric == "ipt": + # Combine the senstivity and uncertainty + ipt_score = self.exp_avg_ipt[n] * self.exp_avg_unc[n] + elif metric == "mag": + ipt_score = p.abs().detach().clone() + else: + raise ValueError("Unexcptected Metric: %s"%metric) + return ipt_score + + def _combine_ipt(self, ipt_E, ipt_AB): + ipt_AB = ipt_AB.sum(dim=1, keepdim=False) + sum_ipt = ipt_E.view(-1) + ipt_AB.view(-1) + return sum_ipt + + def mask_to_target_rank(self, model, curr_rank): + is_dict = {} + combine_dict = {} + singular_dict = {} + # Calculate the importance score for each sub matrix + for n,p in model.named_parameters(): + if "lora_A" in n: + rdim, hdim_a = p.shape + ipt_score = self.calculate_score(n, metric="ipt") + comb_ipt = torch.mean(ipt_score, dim=1, keepdim=True) + name_mat = n.replace("lora_A", "%s") + if name_mat not in combine_dict: + combine_dict[name_mat] = [comb_ipt] + else: + combine_dict[name_mat].append(comb_ipt) + if "lora_B" in n: + hdim_b, rdim = p.shape + ipt_score = self.calculate_score(n, metric="ipt") + comb_ipt = torch.mean(ipt_score, dim=0, keepdim=False).view(-1, 1) + name_mat = n.replace("lora_B", "%s") + if name_mat not in combine_dict: + combine_dict[name_mat] = [comb_ipt] + else: + combine_dict[name_mat].append(comb_ipt) + if "lora_E" in n: + ipt_score = self.calculate_score(n, p=p, metric="ipt") + name_mat = n.replace("lora_E", "%s") + singular_dict[name_mat] = ipt_score + + # Combine the importance scores + all_is = [] + for name_mat in combine_dict: + ipt_E = singular_dict[name_mat] + ipt_AB = torch.cat(combine_dict[name_mat], dim=1) + sum_ipt = self._combine_ipt(ipt_E, ipt_AB) + name_E = name_mat%"lora_E" + is_dict[name_E] = sum_ipt.view(-1, 1) + all_is.append(sum_ipt.view(-1)) + + # Calculate the masking threshold + mask_threshold = torch.kthvalue(torch.cat(all_is), (self.total_rank-curr_rank))[0].item() + + # Mask out unimportant singular values + with torch.no_grad(): + curr_sum_rank = 0 + sum_param = 0 + for n,p in model.named_parameters(): + if "lora_E" in n: + p.data.masked_fill_(is_dict[n]<=mask_threshold, 0.0) + ranknum = (is_dict[n]>mask_threshold).sum().item() + + if self.tb_writter is not None and self.global_step%self.log_interval==0: + self.tb_writter.add_scalar("Ranknum/%s"%(n,), ranknum, self.global_step) + self.rank_pattern[n] = ranknum + curr_sum_rank += ranknum + sum_param += ranknum*self.shape_dict[n.replace("lora_E", "lora_A")][1] + sum_param += ranknum*self.shape_dict[n.replace("lora_E", "lora_B")][0] + + if self.tb_writter is not None and self.global_step%self.log_interval==0: + self.tb_writter.add_scalar("Budget/total_rank", curr_sum_rank, self.global_step) + self.tb_writter.add_scalar("Budget/mask_threshold", mask_threshold, self.global_step) + self.tb_writter.add_scalar("Budget/sum_param", sum_param, self.global_step) + + return mask_threshold + + + def update_and_mask(self, model, global_step): + if global_step 0.: + self.lora_dropout = nn.Dropout(p=lora_dropout) + else: + self.lora_dropout = lambda x: x + # Mark the weight as unmerged + self.merged = False + self.merge_weights = merge_weights + + +class Embedding(nn.Embedding, LoRALayer): + # LoRA implemented in a dense layer + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + r: int = 0, + lora_alpha: int = 1, + merge_weights: bool = True, + **kwargs + ): + nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs) + LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=0, + merge_weights=merge_weights) + # Actual trainable parameters + if r > 0: + self.lora_A = nn.Parameter(self.weight.new_zeros((r, num_embeddings))) + self.lora_B = nn.Parameter(self.weight.new_zeros((embedding_dim, r))) + self.scaling = self.lora_alpha / self.r + # Freezing the pre-trained weight matrix + self.weight.requires_grad = False + self.reset_parameters() + + def reset_parameters(self): + nn.Embedding.reset_parameters(self) + if hasattr(self, 'lora_A'): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.zeros_(self.lora_A) + nn.init.normal_(self.lora_B) + + def train(self, mode: bool = True): + nn.Embedding.train(self, mode) + if self.merge_weights and self.merged: + # Make sure that the weights are not merged + if self.r > 0: + self.weight.data -= (self.lora_B @ self.lora_A).T * self.scaling + self.merged = False + + def eval(self): + nn.Linear.eval(self) + if self.merge_weights and not self.merged: + # Merge the weights and mark it + if self.r > 0: + self.weight.data += (self.lora_B @ self.lora_A) * self.scaling + self.merged = True + + def forward(self, x: torch.Tensor): + if self.r > 0 and not self.merged: + result = nn.Embedding.forward(self, x) + if self.r > 0: + after_A = F.embedding( + x, self.lora_A.T, self.padding_idx, self.max_norm, + self.norm_type, self.scale_grad_by_freq, self.sparse + ) + result += (after_A @ self.lora_B.T) * self.scaling + return result + else: + return nn.Embedding.forward(self, x) + + +class Linear(nn.Linear, LoRALayer): + # LoRA implemented in a dense layer + def __init__( + self, + in_features: int, + out_features: int, + r: int = 0, + lora_alpha: int = 1, + lora_dropout: float = 0., + fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out) + merge_weights: bool = True, + **kwargs + ): + nn.Linear.__init__(self, in_features, out_features, **kwargs) + LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, + merge_weights=merge_weights) + + self.fan_in_fan_out = fan_in_fan_out + # Actual trainable parameters + if r > 0: + self.lora_A = nn.Parameter(self.weight.new_zeros((r, in_features))) + self.lora_B = nn.Parameter(self.weight.new_zeros((out_features, r))) + self.scaling = self.lora_alpha / self.r + # Freezing the pre-trained weight matrix + self.weight.requires_grad = False + self.reset_parameters() + if fan_in_fan_out: + self.weight.data = self.weight.data.T + + def reset_parameters(self): + nn.Linear.reset_parameters(self) + if hasattr(self, 'lora_A'): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5)) + nn.init.zeros_(self.lora_B) + + def train(self, mode: bool = True): + def T(w): + return w.T if self.fan_in_fan_out else w + nn.Linear.train(self, mode) + if self.merge_weights and self.merged: + # Make sure that the weights are not merged + if self.r > 0: + self.weight.data -= T(self.lora_B @ self.lora_A) * self.scaling + self.merged = False + + def eval(self): + def T(w): + return w.T if self.fan_in_fan_out else w + nn.Linear.eval(self) + if self.merge_weights and not self.merged: + # Merge the weights and mark it + if self.r > 0: + self.weight.data += T(self.lora_B @ self.lora_A) * self.scaling + self.merged = True + + def forward(self, x: torch.Tensor): + def T(w): + return w.T if self.fan_in_fan_out else w + if self.r > 0 and not self.merged: + result = F.linear(x, T(self.weight), bias=self.bias) + if self.r > 0: + result += (self.lora_dropout(x) @ self.lora_A.T @ self.lora_B.T) * self.scaling + return result + else: + return F.linear(x, T(self.weight), bias=self.bias) + + +class MergedLinear(nn.Linear, LoRALayer): + # LoRA implemented in a dense layer + def __init__( + self, + in_features: int, + out_features: int, + r: int = 0, + lora_alpha: int = 1, + lora_dropout: float = 0., + enable_lora: List[bool] = [False], + fan_in_fan_out: bool = False, + merge_weights: bool = True, + **kwargs + ): + nn.Linear.__init__(self, in_features, out_features, **kwargs) + LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, + merge_weights=merge_weights) + assert out_features % len(enable_lora) == 0, \ + 'The length of enable_lora must divide out_features' + self.enable_lora = enable_lora + self.fan_in_fan_out = fan_in_fan_out + # Actual trainable parameters + if r > 0 and any(enable_lora): + self.lora_A = nn.Parameter( + self.weight.new_zeros((r * sum(enable_lora), in_features))) + self.lora_B = nn.Parameter( + self.weight.new_zeros((out_features // len(enable_lora) * sum(enable_lora), r)) + ) # weights for Conv1D with groups=sum(enable_lora) + self.scaling = self.lora_alpha / self.r + # Freezing the pre-trained weight matrix + self.weight.requires_grad = False + # Compute the indices + self.lora_ind = self.weight.new_zeros( + (out_features, ), dtype=torch.bool + ).view(len(enable_lora), -1) + self.lora_ind[enable_lora, :] = True + self.lora_ind = self.lora_ind.view(-1) + self.reset_parameters() + if fan_in_fan_out: + self.weight.data = self.weight.data.T + + def reset_parameters(self): + nn.Linear.reset_parameters(self) + if hasattr(self, 'lora_A'): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5)) + nn.init.zeros_(self.lora_B) + + def zero_pad(self, x): + result = x.new_zeros((*x.shape[:-1], self.out_features)) + result = result.view(-1, self.out_features) + result[:, self.lora_ind] = x.reshape( + -1, self.out_features // len(self.enable_lora) * sum(self.enable_lora) + ) + return result.view((*x.shape[:-1], self.out_features)) + + def train(self, mode: bool = True): + def T(w): + return w.T if self.fan_in_fan_out else w + nn.Linear.train(self, mode) + if self.merge_weights and self.merged: + # Make sure that the weights are not merged + if self.r > 0 and any(self.enable_lora): + delta_w = F.conv1d( + self.lora_A.data.unsqueeze(0), + self.lora_B.data.unsqueeze(-1), + groups=sum(self.enable_lora) + ).squeeze(0) + self.weight.data -= self.zero_pad(T(delta_w * self.scaling)) + self.merged = False + + def eval(self): + def T(w): + return w.T if self.fan_in_fan_out else w + nn.Linear.eval(self) + if self.merge_weights and not self.merged: + # Merge the weights and mark it + if self.r > 0 and any(self.enable_lora): + delta_w = F.conv1d( + self.lora_A.data.unsqueeze(0), + self.lora_B.data.unsqueeze(-1), + groups=sum(self.enable_lora) + ).squeeze(0) + self.weight.data += self.zero_pad(T(delta_w * self.scaling)) + self.merged = True + + def forward(self, x: torch.Tensor): + def T(w): + return w.T if self.fan_in_fan_out else w + if self.merged: + return F.linear(x, T(self.weight), bias=self.bias) + else: + result = F.linear(x, T(self.weight), bias=self.bias) + if self.r > 0: + after_A = F.linear(self.lora_dropout(x), self.lora_A) + after_B = F.conv1d( + after_A.transpose(-2, -1), #B, 12, M + self.lora_B.unsqueeze(-1), #3072, 4, 1 + groups=sum(self.enable_lora) + ).transpose(-2, -1) + result += self.zero_pad(after_B) * self.scaling + return result + + +class Conv2d(nn.Conv2d, LoRALayer): + # LoRA implemented in a dense layer + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + r: int = 0, + lora_alpha: int = 1, + lora_dropout: float = 0., + merge_weights: bool = True, + **kwargs + ): + nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, **kwargs) + LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, + merge_weights=merge_weights) + assert type(kernel_size) is int + # Actual trainable parameters + if r > 0: + self.lora_A = nn.Parameter( + self.weight.new_zeros((r*kernel_size, in_channels*kernel_size)) + ) + self.lora_B = nn.Parameter( + self.weight.new_zeros((out_channels*kernel_size, r*kernel_size)) + ) + self.scaling = self.lora_alpha / self.r + # Freezing the pre-trained weight matrix + self.weight.requires_grad = False + self.reset_parameters() + + def reset_parameters(self): + nn.Conv2d.reset_parameters(self) + if hasattr(self, 'lora_A'): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5)) + nn.init.zeros_(self.lora_B) + + def train(self, mode: bool = True): + nn.Conv2d.train(self, mode) + if self.merge_weights and self.merged: + # Make sure that the weights are not merged + self.weight.data -= (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling + self.merged = False + + def eval(self): + nn.Conv2d.eval(self) + if self.merge_weights and not self.merged: + # Merge the weights and mark it + self.weight.data += (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling + self.merged = True + + def forward(self, x: torch.Tensor): + if self.r > 0 and not self.merged: + return F.conv2d( + x, + self.weight + (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling, + self.bias, self.stride, self.padding, self.dilation, self.groups + ) + return nn.Conv2d.forward(self, x) \ No newline at end of file diff --git a/models/common/loralib/utils.py b/models/common/loralib/utils.py new file mode 100644 index 00000000..3e0f55e7 --- /dev/null +++ b/models/common/loralib/utils.py @@ -0,0 +1,49 @@ +# ------------------------------------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +# ------------------------------------------------------------------------------------------ +from typing import Dict + +import torch +import torch.nn as nn + +from .layers import LoRALayer + + +def mark_only_lora_as_trainable(model: nn.Module, bias: str = 'none') -> None: + for n, p in model.named_parameters(): + if 'lora_' not in n: + p.requires_grad = False + if bias == 'none': + return + elif bias == 'all': + for n, p in model.named_parameters(): + if 'bias' in n: + p.requires_grad = True + elif bias == 'lora_only': + for m in model.modules(): + if isinstance(m, LoRALayer) and \ + hasattr(m, 'bias') and \ + m.bias is not None: + m.bias.requires_grad = True + else: + raise NotImplementedError + + +def lora_state_dict(model: nn.Module, bias: str = 'none') -> Dict[str, torch.Tensor]: + my_state_dict = model.state_dict() + if bias == 'none': + return {k: my_state_dict[k] for k in my_state_dict if 'lora_' in k} + elif bias == 'all': + return {k: my_state_dict[k] for k in my_state_dict if 'lora_' in k or 'bias' in k} + elif bias == 'lora_only': + to_return = {} + for k in my_state_dict: + if 'lora_' in k: + to_return[k] = my_state_dict[k] + bias_name = k.split('lora_')[0]+'bias' + if bias_name in my_state_dict: + to_return[bias_name] = my_state_dict[bias_name] + return to_return + else: + raise NotImplementedError \ No newline at end of file diff --git a/models/common/two_way_transformer.py b/models/common/two_way_transformer.py deleted file mode 100644 index ae5056e5..00000000 --- a/models/common/two_way_transformer.py +++ /dev/null @@ -1,263 +0,0 @@ -import math -from typing import Tuple, Type -import torch -from torch import nn, Tensor -from .mlp import MLPBlock - -class TwoWayTransformer(nn.Module): - def __init__( - self, - depth: int, - embedding_dim: int, - num_heads: int, - mlp_dim: int, - activation: Type[nn.Module] = nn.ReLU, - normalize_before_activation: bool = False, - attention_downsample_rate: int = 2, - ) -> None: - """ - A transformer decoder that attends to an input image using - queries whose positional embedding is supplied. - - Args: - depth (int): number of layers in the transformer - embedding_dim (int): the channel dimension for the input embeddings - num_heads (int): the number of heads for multihead attention. Must - divide embedding_dim - mlp_dim (int): the channel dimension internal to the MLP block - activation (nn.Module): the activation to use in the MLP block - """ - super().__init__() - self.depth = depth - self.embedding_dim = embedding_dim - self.num_heads = num_heads - self.mlp_dim = mlp_dim - self.layers = nn.ModuleList() - - for i in range(depth): - curr_layer = TwoWayAttentionBlock( - embedding_dim=embedding_dim, - num_heads=num_heads, - mlp_dim=mlp_dim, - activation=activation, - normalize_before_activation=normalize_before_activation, - attention_downsample_rate=attention_downsample_rate, - skip_first_layer_pe=(i == 0), - ) - self.layers.append(curr_layer) - - self.final_attn_token_to_image = AttentionForTwoWayAttentionBlock( - embedding_dim, - num_heads, - downsample_rate=attention_downsample_rate, - ) - self.norm_final_attn = nn.LayerNorm(embedding_dim) - - def forward( - self, - image_embedding: Tensor, - image_pe: Tensor, - point_embedding: Tensor, - ) -> Tuple[Tensor, Tensor]: - """ - Args: - image_embedding (torch.Tensor): image to attend to. Should be shape - B x embedding_dim x h x w for any h and w. - image_pe (torch.Tensor): the positional encoding to add to the image. Must - have the same shape as image_embedding. - point_embedding (torch.Tensor): the embedding to add to the query points. - Must have shape B x N_points x embedding_dim for any N_points. - - Returns: - torch.Tensor: the processed point_embedding - torch.Tensor: the processed image_embedding - """ - - # BxCxHxW -> BxHWxC == B x N_image_tokens x C - bs, c, h, w = image_embedding.shape - image_embedding = image_embedding.flatten(2).permute(0, 2, 1) - image_pe = image_pe.flatten(2).permute(0, 2, 1) - - # Prepare queries - queries = point_embedding - keys = image_embedding - - # Apply transformer blocks and final layernorm - for idx, layer in enumerate(self.layers): - queries, keys = layer( - queries=queries, - keys=keys, - query_pe=point_embedding, - key_pe=image_pe, - ) - - # Apply the final attention layer from the points to the image - q = queries + point_embedding - k = keys + image_pe - attn_out = self.final_attn_token_to_image(q=q, k=k, v=keys) - queries = queries + attn_out - queries = self.norm_final_attn(queries) - return queries, keys - - -class TwoWayAttentionBlock(nn.Module): - def __init__( - self, - embedding_dim: int, - num_heads: int, - mlp_dim: int, - activation: Type[nn.Module], - normalize_before_activation: bool, - attention_downsample_rate: int = 2, - skip_first_layer_pe: bool = False, - ) -> None: - """ - A transformer block with four layers: (1) self-attention of sparse - inputs, (2) cross attention of sparse inputs to dense inputs, (3) mlp - block on sparse inputs, and (4) cross attention of dense inputs to sparse - inputs. - - Arguments: - embedding_dim (int): the channel dimension of the embeddings - num_heads (int): the number of heads in the attention layers - mlp_dim (int): the hidden dimension of the mlp block - activation (nn.Module): the activation of the mlp block - skip_first_layer_pe (bool): skip the PE on the first layer - """ - super().__init__() - self.self_attn = AttentionForTwoWayAttentionBlock(embedding_dim, num_heads) - self.norm1 = nn.LayerNorm(embedding_dim) - - self.cross_attn_token_to_image = AttentionForTwoWayAttentionBlock( - embedding_dim, - num_heads, - downsample_rate=attention_downsample_rate, - ) - self.norm2 = nn.LayerNorm(embedding_dim) - - self.mlp = MLPBlock( - embedding_dim, - mlp_dim, - embedding_dim, - 1, - activation, - ) - - self.norm3 = nn.LayerNorm(embedding_dim) - - self.norm4 = nn.LayerNorm(embedding_dim) - self.cross_attn_image_to_token = AttentionForTwoWayAttentionBlock( - embedding_dim, - num_heads, - downsample_rate=attention_downsample_rate, - ) - - self.skip_first_layer_pe = skip_first_layer_pe - - def forward( - self, queries: Tensor, keys: Tensor, query_pe: Tensor, key_pe: Tensor - ) -> Tuple[Tensor, Tensor]: - # Self attention block - if not self.skip_first_layer_pe: - queries = queries + query_pe - attn_out = self.self_attn(q=queries, k=queries, v=queries) - queries = queries + attn_out - queries = self.norm1(queries) - - # Cross attention block, tokens attending to image embedding - q = queries + query_pe - k = keys + key_pe - attn_out = self.cross_attn_token_to_image(q=q, k=k, v=keys) - queries = queries + attn_out - queries = self.norm2(queries) - - # MLP block - mlp_out = self.mlp(queries) - queries = queries + mlp_out - queries = self.norm3(queries) - - # Cross attention block, image embedding attending to tokens - q = queries + query_pe - k = keys + key_pe - attn_out = self.cross_attn_image_to_token(q=k, k=q, v=queries) - keys = keys + attn_out - keys = self.norm4(keys) - - return queries, keys - - -class AttentionForTwoWayAttentionBlock(nn.Module): - """ - An attention layer that allows for downscaling the size of the embedding - after projection to queries, keys, and values. - """ - - def __init__( - self, - embedding_dim: int, - num_heads: int, - downsample_rate: int = 1, - ) -> None: - super().__init__() - self.embedding_dim = embedding_dim - self.internal_dim = embedding_dim // downsample_rate - self.num_heads = num_heads - assert ( - self.internal_dim % num_heads == 0 - ), "num_heads must divide embedding_dim." - self.c_per_head = self.internal_dim / num_heads - self.inv_sqrt_c_per_head = 1.0 / math.sqrt(self.c_per_head) - - self.q_proj = nn.Linear(embedding_dim, self.internal_dim) - self.k_proj = nn.Linear(embedding_dim, self.internal_dim) - self.v_proj = nn.Linear(embedding_dim, self.internal_dim) - self.out_proj = nn.Linear(self.internal_dim, embedding_dim) - self._reset_parameters() - - def _reset_parameters(self) -> None: - # The fan_out is incorrect, but matches pytorch's initialization - # for which qkv is a single 3*embedding_dim x embedding_dim matrix - fan_in = self.embedding_dim - fan_out = 3 * self.internal_dim - # Xavier uniform with our custom fan_out - bnd = math.sqrt(6 / (fan_in + fan_out)) - nn.init.uniform_(self.q_proj.weight, -bnd, bnd) - nn.init.uniform_(self.k_proj.weight, -bnd, bnd) - nn.init.uniform_(self.v_proj.weight, -bnd, bnd) - # out_proj.weight is left with default initialization, like pytorch attention - nn.init.zeros_(self.q_proj.bias) - nn.init.zeros_(self.k_proj.bias) - nn.init.zeros_(self.v_proj.bias) - nn.init.zeros_(self.out_proj.bias) - - def _separate_heads(self, x: Tensor, num_heads: int) -> Tensor: - b, n, c = x.shape - x = x.reshape(b, n, num_heads, c // num_heads) - return x.transpose(1, 2) # B x N_heads x N_tokens x C_per_head - - def _recombine_heads(self, x: Tensor) -> Tensor: - b, n_heads, n_tokens, c_per_head = x.shape - x = x.transpose(1, 2) - return x.reshape(b, n_tokens, n_heads * c_per_head) # B x N_tokens x C - - def forward(self, q: Tensor, k: Tensor, v: Tensor) -> Tensor: - # Input projections - q = self.q_proj(q) - k = self.k_proj(k) - v = self.v_proj(v) - - # Separate into heads - q = self._separate_heads(q, self.num_heads) - k = self._separate_heads(k, self.num_heads) - v = self._separate_heads(v, self.num_heads) - - # Attention - _, _, _, c_per_head = q.shape - attn = q @ k.permute(0, 1, 3, 2) # B x N_heads x N_tokens x N_tokens - attn = attn * self.inv_sqrt_c_per_head - attn = torch.softmax(attn, dim=-1) - # Get output - out = attn @ v - out = self._recombine_heads(out) - out = self.out_proj(out) - return out diff --git a/models/efficient_sam/__pycache__/efficient_sam_encoder.cpython-37.pyc b/models/efficient_sam/__pycache__/efficient_sam_encoder.cpython-37.pyc index b89839e04ba9deb0bf2419002df686a48721bab6..250b68fc17e2b12d55021d3c83e6b77e9305e9a3 100644 GIT binary patch delta 1451 zcmZvb&2Jk;6u>>Z>-E}R$Mz<1oDWbSN!v|ABcKRWBATRMQB^`AK@mQ%Y`oLNvAw&U zov0~t6p8e}C5LFP6-v)YTq0HK1&K2i2To|XP>BQi3pj9q_hw5#h*`~V=6%fU%zJNs zfAhDAzNJ#Bkb&0z?rZvV@Tjj&o*bR`#=4b8rCV)OyH3L)nVXrw3~~=ML5^D6{YTG6 zr>sXtkPiwEjRVpc0dq01!EB@XKyQo&WHsZKgiS6_8`543{7%IDrB2jrD=FrbrBXH> z^0&c$a)$^)s^V9Tog087xh5NgL6TibmL$0<$+9HtY!q}U#zar|%qq$g;H?F zV=8koW!)W4U5mNk!ln^3=uEaGx!*8%A9&$Tm%WavVo2|M{zl@(QS$HeD%&RIf7_>+gs4!ee{uj9HxD~xxzm$W{oH(ZUC&`?Iq0bE@vAiI7Lw8Bkg zS#d{MMY*Wbx{b~?@x#EaevGgz#y)Rud9=Gh1LlaC!NvVNx?sqeg}?_Tv&;`SsWkmK zrePoptMuLu4SCD&uzqyPp_75xbRAXaEd3GM(lWr=B4P<~6>)Ygb_2zmfJQYI&j)V} zt9n-tKovb6{R8O9nz&YbwL}WU#I6}y9}(@^b@OH1U{*Y-NirvnhAuTR;BJB817NBc z-^ST0;$6fQ#5&?5#3zVP0j`lAUv21tLh2ix_J7KrO@QGmaiKoHKZy&c5a$q=08&pl zli9nS$mdK$Z5DxrEY0_$V5dWw%EiY@vqr-tVrolv0UiJU-7kP?1$t5kScXQ_S5F6^ zNi5nrZ`gtp>MaiH0f>} zG0yISS{fTs)KND}qqcy)XNOE@m?de=ch&9Q!)iz{>K3VM7O{z=*`%&JbrVMCmA?Qc C^h$>S delta 3520 zcmai0OOG5^6~4FL)vxL48QU|HfFHpz4aRZoz``<4#<87{*sXvQC2As7YPxD>x~E^h zRkp{YY8o__lFckkZCP}S4M>TEWWfd`b|Bynpu4hQgBRI>vcr7mR=ay_g6USBI(6>3 zuXDcd-20CgzBpUDSS;oken0u?&%@kT@0FJLgL@~uA!l!}zzXcUEU-hTRo>O>^RPI9 z3yV9kRA9*kd06rj%K|Kgpa@HGVp$BW)`8YR$V1^$sD;bRjP#)NGb5M_%6E-jUSC14 z`JjScl}RuAxqx41)rEV1JD20Kup0RNI28A<@RajXDmC%qJl0&bL6VafqCjNw_ID7KSd=2A=89m}t?9 zP(oJc0jq0)^)XxLffnd@-MStafq9qJjgOhZ>L%82OLsMn!*1N{b!!cCGJ=IS={laM z1`)GeHsWK(S)wJ((-NKxwG}on6Qh4DF$2E9VAf-!WughwEwV%p^gSI`D>g>TI;O66 zz%Z>*+rX&!J^1}YngrV`(901;yadHVy4{-{Z!`3RNSaZ@Z--)z%+}RrH}r)Z>NmD- ziTy#=gJws3oqQZLd23BSBF$5sr9(q`aCsa`l>`+}n(ic?h}OXD9Jjc`OZ*_;sZ4`& z%JZ7tW{mV`cqM$yM;aog2RvadJ%P2QuduN> zGAJVfr+kb}gwYK2Wd>;|sVyV)8XH&xd*CEiVvkIM`dBL?%q?q?4cuhtw(P(ToIQ>Y zH*v}gP8fEqWm6``SRY{8kweRXMsRg&;sh2dD8Yh5JXC^^^YGt~9zA-9=Me%#(Wts& zh1|XgA#D*xn|?pc;%h@0c5cR@bYtOnqkb<6MOFDlz4f@`Z)b_gDfElyXqe}znb*Wu z=tGUkZ%Do+`7LSqaon9@Jp=g>_?^QOQJxqtXgMULq+3W%Sv$xJL%zaud}n@^q4l1) z?TetwYoOgxbP~r%yg*{gFmaqdrv!Wq289FS4XrGqcsN$c6=Z4Pnf|66J2ixDN-`es zP;VLQ`aZHzHi&E>AjoDT`j*tMwfi%g5yfagVnB2eVjs=)emUU-J<?lrb1WwZR+m0_)+c7@UCBp^mB9MOJfIEpot9Swy>+f&GiO)5u;nOLaKC`YKy zPkPv~0}o@7?ec*WyU?A)4h&Vva|&Q)ZDa5AUyp`g{NYy@s$A+}KSEZh?uu`bI0XU2 zD(M7jafuB1r!Xnqs|2K4PjsZ&g?w3&O`L~wW;i4DW;a%WlzOM#M?F$!so{%_NSq}8 zW!1<t%keU@m;G~EnHMJdPDus-2@uo#Ore-U zuTV@?6C@=ZPY^ujhZvP!d?Wls;W7T14Fv&sHq?&A_s~^bCGi%-|HJf;;4_S9Ud9*k z-zjG6dnTyGWw<{Z(`Z+az5-7IZ={YmD+@@aBFJ7fr0uUo-u)xSAGM+&G}pvgNO2zHRcRZZ*;yp6uyWwq*M#qUV zE7EFFGQSbVkK691zqL>BkJ7)}pX3*4z(o=V(p%0;-^rjzr3hDuu(3sFex&dquEW?+ zhddMlQy!0E&FS=WXXF}jWiYpWK3+(F;~pQ{d*)ys=k45RF_|B8&Tf4AK8xMaf!J*w z1ch<|UuHPRapl}I;zQZ@E?2FpZQ_wrKKVU!<})VhEd>Q&py>Y$$)1A;njP_uYpNph_q5JDwl(V-#?zI_jw2x8+>3+egignmgeYz=#s4u2$;Y2QA4U z?YS4h`F>}T(zkFh%2dI zJp5LfRGa(^ig~q?73Jw@lLV318VN#|Am+`ME<*ael>WK+&hnR>Ex$x-bTvI&I%aBy zR?^b7(q&8CEUu@Yl_Worelqt;gThtKWfvC(j5-b4AweJ%ACb60qD>+qu?+zp5+1Uv z${lD@Uu$nY1wnBH4gX3Hm(Q1DA}O2daO3r%%bE6H^dc1zj41NkUEF#68j2Jk{+YFURJO^B<;f z&VOyF3M8d2E39-NDFREq)r_STZufg47C(kr8f(2?TM%ip0MYFG_C7u2rE|Wcc6WhN h9nqD};Y*r>LRrxYyr5T>E5|AaD@QG@p!3Sn{{V5|#CQMz diff --git a/models/efficient_sam/efficient_sam_encoder.py b/models/efficient_sam/efficient_sam_encoder.py index 0371a270..9708143b 100644 --- a/models/efficient_sam/efficient_sam_encoder.py +++ b/models/efficient_sam/efficient_sam_encoder.py @@ -12,7 +12,7 @@ import torch.nn.functional as F from ..common import LayerNorm2d -from ..ImageEncoder import AdapterBlock +from ..ImageEncoder import AdaloraBlock, AdapterBlock, Block, LoraBlock class PatchEmbed(nn.Module): @@ -39,96 +39,6 @@ def forward(self, x): x = self.proj(x) return x - -class Attention(nn.Module): - def __init__( - self, - dim, - num_heads, - qkv_bias, - qk_scale=None, - ): - super().__init__() - self.num_heads = num_heads - head_dim = dim // num_heads - self.scale = qk_scale or head_dim**-0.5 - self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) - self.proj = nn.Linear(dim, dim) - - def forward(self, x): - B, N, C = x.shape - qkv = ( - self.qkv(x) - .reshape(B, N, 3, self.num_heads, C // self.num_heads) - .permute(2, 0, 3, 1, 4) - ) - q, k, v = ( - qkv[0], - qkv[1], - qkv[2], - ) - attn = (q @ k.transpose(-2, -1)) * self.scale - attn = attn.softmax(dim=-1) - x = (attn @ v).transpose(1, 2).reshape(B, N, C) - x = self.proj(x) - return x - - -class Mlp(nn.Module): - def __init__( - self, - in_features, - hidden_features=None, - out_features=None, - act_layer=nn.GELU, - ): - super().__init__() - out_features = out_features or in_features - hidden_features = hidden_features or in_features - self.fc1 = nn.Linear(in_features, hidden_features) - self.act = act_layer() - self.fc2 = nn.Linear(hidden_features, out_features) - - def forward(self, x): - x = self.fc1(x) - x = self.act(x) - x = self.fc2(x) - return x - - -class Block(nn.Module): - def __init__( - self, - args, - dim, - num_heads, - mlp_ratio=4.0, - qkv_bias=False, - qk_scale=None, - act_layer=nn.GELU, - ): - super().__init__() - self.norm1 = nn.LayerNorm(dim, eps=1e-6) - self.attn = Attention( - dim, - num_heads=num_heads, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - ) - self.norm2 = nn.LayerNorm(dim, eps=1e-6) - mlp_hidden_dim = int(dim * mlp_ratio) - self.mlp = Mlp( - in_features=dim, - hidden_features=mlp_hidden_dim, - act_layer=act_layer, - ) - - def forward(self, x): - x = x + self.attn(self.norm1(x)) - x = x + self.mlp(self.norm2(x)) - return x - - @torch.jit.export def get_abs_pos( abs_pos: torch.Tensor, has_cls_token: bool, hw: List[int] @@ -210,11 +120,14 @@ def __init__( self.blocks = nn.ModuleList() if args.mod == 'sam_adpt': block_class = AdapterBlock + elif args.mod == 'sam_lora': + block_class = LoraBlock + elif args.mod == 'sam_adalora': + block_class = AdaloraBlock else: block_class = Block for i in range(depth): - # vit_block = Block(patch_embed_dim, num_heads, mlp_ratio, True) vit_block = block_class( args = self.args, dim=patch_embed_dim, diff --git a/models/sam/modeling/__pycache__/image_encoder.cpython-37.pyc b/models/sam/modeling/__pycache__/image_encoder.cpython-37.pyc index 03c2d532dd3463bf72e03becc7c0666b31e60fcd..a484990dd39ec99125a51e1c52c4177c64e82315 100644 GIT binary patch delta 1128 zcmZvbOH9;I6oz}-&P*Rum=_RGRImsG3ZjY8AmO1=Vr0ZfL`F>0nHHI0Xvf}FV+!%?8Mwc#3T)8so)|D%xiF@y95klfj`t_gx+rZG}b;&9OeA@h@n?q z6OUHx;=CUTZ{kfG>$|P!8$vDQ5Bs<|DI5MEZ$A5rIsh%ByqXU$~!M% zu|~{eJ+6fbIm;Ttmb{knm~wnO6KkHU`mL_z!I7hw<(?}*4#Ua#5*2El(Pms@I7fO&{ z;<$8%E^MHeGegFtb}}nHKyBoc^cJaPRX!K(gJfsP8(AH`0P-?LmdAsjM=A2MWiDxY z!O^O=<8y_|JWfC?zz9`-Y}Uq0IyRO`W4wU+$x8gvAUqVGhiG)tv>Y4barmKVTCt)# zPSLDXXe0ZHvIq~u4#Ip} zI6kc633nq=H0FV{m1N?uuy)^WcUJir>j^`OP#31;pC{>RkOkX*EL3ii;sJl{y_mHJ# x_(b|A*gICiwQ+H-*RpUwA;}IjM&2g}R2kxT0dW9iJE$X6BP5+t`3#?0{tJ6o@=yQ( delta 1059 zcmZva&1(}u7{+IIH_3h^%_dEXE%rNUU0WL}g0!I@exOBdP}T2%Qau zWW+vwPhRG0JE3;`bZ4+KfYC){22AlGGR1tL*t*59#F!(QGIQiVPJHL{Rm{-{j_;4+ z1B_)V@HI)W!<*4N^R`MUKUjL>sAbOdJDkIJpgLDMGq9~}#vD`(?na#09>(a_gGFQq zcB3|1%kepq$(y1nZ3}kHk+!hsO3al#U)lE6{XM}!dfepZkQOwe0=?hTxqnUw3ky+p zAkxZ^ImCmS^Q5X2M3SC0djF=nHun_t2m6N8C@}D<}OYV9p5rrX;3MfgESh1d;|l z&9I`B^BcO6we)I*9V=V9Ny1R`FvXM`Sgnu^gP1is6U^cieHR>?fJMnDRQ>b$vQ;7E zGCYKIJy$X;E0-^o*!CNZG)yL82J|Be?G2^)4Evu8-NnQ7AT%@)gaY6#=jB`nWOaym zb|ISION{ivI1SLaEYh<--tru0A<bendGllx`A*fgMtD3yG2*W)rf$};n z!)%3I0b>@BVQ>SwZdP(EQI-WF*8q(hAPevaUCDL8|3e>UrAO>%!7C;5f`D)YVfZF+ zG)$nZwb5shF4enL(XNo(jgfMhr0ADO5?`R-BkAsWHi{?!7~fo}uBPavXhKkhXze%q Cz}5W# diff --git a/models/sam/modeling/image_encoder.py b/models/sam/modeling/image_encoder.py index 3a6f2893..b60b760e 100644 --- a/models/sam/modeling/image_encoder.py +++ b/models/sam/modeling/image_encoder.py @@ -13,7 +13,7 @@ from einops import rearrange from ...common import Adapter, LayerNorm2d -from ...ImageEncoder import AdapterBlock, Block +from ...ImageEncoder import AdapterBlock, Block, LoraBlock # This class and its supporting functions below lightly adapted from the ViTDet backbone available at: https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py # noqa @@ -78,6 +78,8 @@ def __init__( self.blocks = nn.ModuleList() if args.mod == 'sam_adpt': block_class = AdapterBlock + elif args.mod == 'sam_lora': + block_class = LoraBlock else: block_class = Block diff --git a/train.py b/train.py index 5674c192..7cbb591f 100644 --- a/train.py +++ b/train.py @@ -5,33 +5,34 @@ Junde Wu """ +import argparse import os import sys -import argparse -from datetime import datetime +import time from collections import OrderedDict +from datetime import datetime + import numpy as np import torch import torch.nn as nn import torch.optim as optim -from sklearn.metrics import roc_auc_score, accuracy_score,confusion_matrix import torchvision import torchvision.transforms as transforms +from PIL import Image from skimage import io -from torch.utils.data import DataLoader +from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score +from tensorboardX import SummaryWriter #from dataset import * from torch.autograd import Variable -from PIL import Image -from tensorboardX import SummaryWriter +from torch.utils.data import DataLoader, random_split +from tqdm import tqdm + +import cfg +import function +from conf import settings #from models.discriminatorlayer import discriminator from dataset import * -from conf import settings -import time -import cfg -from tqdm import tqdm -from torch.utils.data import DataLoader, random_split from utils import * -import function args = cfg.parse_args() @@ -137,7 +138,7 @@ for epoch in range(settings.EPOCH): if epoch and epoch < 5: tol, (eiou, edice) = function.validation_sam(args, nice_test_loader, epoch, net, writer) - logger.info(f'Total score: {tol}, IOU: {eiou}, DICE: {edice} || @ epoch {epoch+1}.') + logger.info(f'Total score: {tol}, IOU: {eiou}, DICE: {edice} || @ epoch {epoch}.') net.train() time_start = time.time() From f845a4bde886525de02aa03ae0eeb452e7ffe715 Mon Sep 17 00:00:00 2001 From: wzy Date: Sun, 21 Jan 2024 20:59:40 +0800 Subject: [PATCH 2/2] update guidance for LoRa --- figs/lora/MobileSAM-Ti (AdaLora)_loss.png | Bin 0 -> 37974 bytes .../MobileSAM-Ti (AdaLora)_performance.png | Bin 0 -> 24110 bytes figs/lora/MobileSAM-Ti (Adapter)_loss.png | Bin 0 -> 33939 bytes .../MobileSAM-Ti (Adapter)_performance.png | Bin 0 -> 23635 bytes figs/lora/MobileSAM-Ti (Lora)_loss.png | Bin 0 -> 29273 bytes figs/lora/MobileSAM-Ti (Lora)_performance.png | Bin 0 -> 27737 bytes figs/lora/lora.png | Bin 0 -> 7477 bytes guidance/lora.ipynb | 80 +++++++++++++++++- guidance/mobile_sam.ipynb | 8 +- 9 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 figs/lora/MobileSAM-Ti (AdaLora)_loss.png create mode 100644 figs/lora/MobileSAM-Ti (AdaLora)_performance.png create mode 100644 figs/lora/MobileSAM-Ti (Adapter)_loss.png create mode 100644 figs/lora/MobileSAM-Ti (Adapter)_performance.png create mode 100644 figs/lora/MobileSAM-Ti (Lora)_loss.png create mode 100644 figs/lora/MobileSAM-Ti (Lora)_performance.png create mode 100644 figs/lora/lora.png diff --git a/figs/lora/MobileSAM-Ti (AdaLora)_loss.png b/figs/lora/MobileSAM-Ti (AdaLora)_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..8d1bb7119798c1cc2a48f05c7476dffe8a1858ab GIT binary patch literal 37974 zcmdqJc{r8-_dfdUz1gOnOqr()rOX*Z<|L^^DI)AB$}F=C8xbLMWL89FOvb`iQBsC7 zX3P*W%NTLiK3Yy%1VQ$l(b2ky zAXpNDU`D9%@Ef^~fpPdt(e0G6+a)J!H_z*LZX)NeyE)rCx!K>gbDnOebsY`kFzzw!r*{^fkYtNU}4)ZntIdnT-_DsLf`GsNk)r;ScPxq6Y zeOg^x4Rqp!0>j`xrF}O`oJiiam zjxfRJ?#Bp7IC>)oo&+Cjr4w-K@bMG9(Ep#@p3i(ygeCcer(;AKb zRSw<96r-pp;Wlc?LUgmfB!f! zdE9Y8q&{zJt(Zc02?3QaVYmvXZ(Dc5`SW~t|z5myYKXAE#iVDwx0|&fd_={V^yF(LpGj2-U zDn2azl};b0I-aJhu6LU{Y2GReO6o5~clpbbb4?XfgT5vFlBd91?2v)?Ybc);W23V}+GN@0c$rcZ}MF zKH-qD9{K&UIJD2c>F4|VOQR8zqj6^?8uqJglNy4lwzhqG%shEUM@JXYPf9=X+TA~K z==T7eg^>K0#dVx^-31NH6D`?S`ge`Y%54v5N3sVlFE4x4QzhzBChB$_DqelRUyV3e z&&nsBSNwJeML%vrxZV*T2=$(sI(=JY#$KgvTryrXG`&bU)Rx=6T<$QhHIY69=*g zQyKkXSL}ZGU#e4GI6A_?H=Z9~Tz;u!k0|(VIOnZTKZ%fdciG4+B+1-|MzU_>Nn3HA z;Zc|O`;L!99Ume;jaH=%(d>9sdA4)9l;RxA6YDR@fv%IS*TVViiZThpeO z72Vz4H@;`?+$$&$yH#$oGS|H^x>{b_8ztP7 zu9cM_tE744Q0?85FS`Wb1ngH{L^!#)ChQ*aoWLVhi|^=CRV;EZ9#mAU?i(BSTj$YB zQwW^?o@e!UWqMhCT+n>p3ZapumRp0Oq_rJ&km75$K0k;t4%Ulr9`CZN2Wy+^dda38#Tn8Ze^~J7(a+QWZrT3&@>s$3 zSL7vPg>Bc-YaV?!r@rMzTo^VoWoQ^!{Q2I@e+LXx+s#e3sQMRYB(Dl5dg4r-YP)7@ zU}x5iE9^Onszzy*H^<{4M38gN#HX08qxL`3U6`!ur}RkD;^B$;DqQMurzX^&{{8Ed zIbZwwH47Nm`^n6JhbI=^G6!t+4`=nvxXhLJjKaP4OPc~y=gysT^V!Z#GS`GzloA`T zVrDfgSFc{J`qO%%)Uxq`b+NSN2Wq67k0LBT>Wmi4MRUK^%QU1FxL-%|?P3rM7nf*P zPtVynp)+7Ui4xW0OlRVRp5I-c{|18`pDZ+ca|O$3V-xivtE{&B2%?)a0+fqqzO<9?f#V?t1-hVuEh&%Gcw=3Vwr$ zqDHz%&O+Um9=!0CvDmkJG|WQCMH7=0S#XB88y_6_2CHsXw;vy@p{u9&h}VDP`ex~4 z85`Z|>gumW=~CB&WSvLUVTiZKKgE7?s2n^5KVLR9)ZvWM_W8HklUrQ(R~O4)c(tr) zqQ~y{eLnvfLAq zG~bf_96i@pb{fk+2v)Dxmagdfj6&Vytyk%G@`m4NepZ$`mcMqXiRHs!7jTKW6e zb(Yy&NLtVCxRnFe8SS_bAGZCTz;8hP1|Hwh-cE9#>CS%k3=c-)YPw@q@2_Hb^uT=e zByZlw-IY8wjW{7+I4OG3lM@ro9IVKv*48HX9<%!pEV-@A`eyE4E3@ABcYPrmVsQ4` zx7T*6R=%CZAQpbB2auUhap+|il$R)&n3`kls#-pr#$B<=9ywY3eRwlyo0cEYpu zxwj=miQYG!l4}dY3v2%1;llT;Zm3Hny)-;V2x#M|2D>= zH>UG4Q~UxZCMGtv|Flhf>@K$0k9_qvgVh?cF6Q9iII{bupHeh#Wd6K7^^YNVqO17-JI$oQ$jMcxq(cFqb|F+fz<>b1i z$Aai25F{ZnvAM^#+(dbM@jW_@2N4nI0$QP{2Ha=rS7F~jUypY@J-r{iKN_!3NAsR4 zHm_wkaA4xqxufSYRe3I7zKkM(MTp-&Q|yXvuVo zislXCgycyKM*$g(8r}&I7vap0|Nf|`1tEZp1YEYiX4aHY2@VO^fdB?&UkIjRT>x> z-N)eYvUkm|Ugfa0voo`%SE=5bf3t)J_o2HBr_9ZHN5{qhfJ*uOU6JS095l>S^&7WN zvMZ{_A5EkMyX8kcTqL(56RG^*_^TF)llA`p7GWpQfXf+$pXn+7=-Q?-xy&mnDjFLX z_pNAMuag6fPcHR4tHn$A)wVIi0lO+!ip}TNYUT&uSbcwX{%g=aY2sDc>(r+oa^GI# zyuY7M+t82&cB8hg9$?G$g`wK2*;g&wpF1HfAbyHDsBmYT`g2Q5Y(~aI7!u-B%PU#C za|lc|n6@Mb|_AyIaG82M=P5Ed0t_?C!X_rYjX-_*GN}C%4wM zzQ`&2Z!3fnI0V!rtQA$)dadqgpU=duL6wjEw@V=$Tl%Fycezdm)gkOJNH7_uH)M! z?E;H>UI?4@(tu#|+T>(q$(E)3a2{ZR1FyVv{~N(qV(mpsOB<7%EJBq5DCUsQ+H5!k z(5HPpcJ}rk;xDL%1I%x6K)fJkW##8n`)#ecpD0k2mhPDIdQ>6si*~-Z^!*WpWny@1 zp?-M6mu!>ajd!YDWp%8XK6K5!>#}hP5d%?qzVpZ%`@UD!ZI_-Ke%0RD{*$@8^76GG zKpU&RS2smOL}KcrYig9b`@GT!O6aX;l?_%Omk%-CRu<7wI|(mDcC?2VHr23c{4K z&S@JFn&CVA(+bZcKPS z$46}tB3a9zn|yjfFsm3DZHY4?py-oJqK0(PCdPN`EnwF0z(s% zqbh&`QQ(sk8A&)(c;#$(c=$I!a7Q6F-d(m6H>)wk@;eN@?VDTQ0pNND$$|aXsXHu< zZ%QXd2F41V@qwtH3*pUsyBQW<5)-onW?ALe+G7zQj)}<#*zra&Tz0lb4V<`~@$nl+ zl$1e z@ptar$;r#`(e;@~G)noWDwF#4qPBMEYv-TuSck0ymeV!k1`4S81)f}a`Qn9UTZYQg z%J&On75$EMYjbYwc0EOnXy!NDD}Cd}4T>#~uy^!8q&)Fh>%4T~C-{^`6b@XTPAHf> zELk%pp!{#;dqdrUCnX>Li=Fuy{p7zzzc@4Nh(b7HVPg}Nm+vm-$%ngGyyApVzy+8f z-9wfcN4_E=A)$Tc%9RqQV8HF$`rxV@2Wu#|D(nXyOZokG%)5{jm!)rRjA)@?$LmlxXbl5k@Wb%Ee zuxO>+x`Ogm!Pk*q4F^RxR?#}8&^$FYmRZw-aGdEwF+{4%UUj^P{veUCoDSnRIQlE_ihzCM$)!MDX&k*3CM~3QQbY> zYn%^H;)&_7*SLCK5Fy`)CCZi&&cU3AtdL$zqT%$xGi$q!L{^kZ^CcrCIHgOuo`0Pl z1)|SkgXi$O>|AIsV@^gtXE=r#d2m29CO#eq7)aWxi3q5Nx7Y%OwW~8!@H=24tp5F7 z)yv!kQ0Lruz=$7E1B95nYqKt!N)TdD$cw@pAda?w8`W!~_{Z*K$P~hAj6DR?v$&^) zRQashe#unhi(r#%Dl&T$lbRZ_z3mGqullV&nx9Eq;T_-1$&bxVs^YUkhF~oX&NZ(sisu9?1X)GifNhuh`8PE+oZzJM;apw2mS$1>GkUmo z4OgdpYyZRH$1?N*~dgsy<4E)bMH?$o65Dr1j zc{4322%9&9;a5`|o?9(~wYc72;Q&c59e~OEuV2faOP0b6`)?KYPs+;6KdGttF)hfR zVvSt=&!SSOAYzh|_xtR%A+!5FcKh~ikt0W3Hq+wbm@c3V{V?PqyXj$sfy7O~? z02xv%Y5QI&G@C~m3Ulv?14(V(Q`z_RFJBf8U^#Gra`_PkDUUj2ICCB$WjvQgk}>Gn z8_o~*!yP$tME|*=h`2bwe@M2cr>7+{<1;emzpoa7^Fa_ifqi7fYj@|Lfw3_#jV6Vx zCWYNI#T~q@y{`gFP2?_G6|R3HOG|-$j|XSjAab1hP@d~+>hYH7t75EZu=DozhG!-v zO}eQd_5)QNu$lmH`DJ97@6+?#@!1CU01(=n{lgl>1O1`uHixtXj%iYoI8g3*`Fy(! z36Toe@%@q_M<;mt;Ro=+rS^TtI=i~Aug&%$BvZ#~O0zc}jd8+f5#-OWukRtOl(8i+*50&TJ*rcHCQS};81AEo^yN#*jAk|t$PUFbodgk`9tKCI*NPLm%jW#HmM+<5&%L2;DaO?u&wh?ogaw#@ zM`mPn!`Z;meR`~Gw0_$uqh4Tt4uZ{CPaByO&-3d2doCIc7`J|4#KNw(&r+Zc)d}Xo-)i_HRQ*H(33K=j~en_6fKK}D^J&X!PF33i^>EiL>PY+5=ciWSl_s{lg?hrGZ{lj9#=%P&X1L z{?EHY70$%XnyQbcQjtu`SCe9d_Y8*>M`rLWYDI#Zu_0KUh#vW^x1W$uO}53gu}SH_ zPN|F2(KbPTqUc(HjWKvO$>XH4-E6t;u})EvT^9E6m8#%Xw=RPM63|@(edTt zh(7_OS~_DsEC+e{hc==fKKDfc>P;bs@T7x!1%WMY9T>vBZix}s3S@Dw=JX_{36c76 z;BG305Qvn#?M4uh!tVE$u|dAa(bv<%>ko&jze&MR_j+RPU&OcHh#kKpg+7uOedGJ+lmHvD2bqr*;(7`9t7G>uFE)8xg5{%`(VfLI8$e zGnWJjYA`$UpvWR|&U)`e(3-)ElHXDa4Y}i}5fPRLG3ir>K~trI3}Gp8fDUaDq($GSXFd8>slultTT^ontJtl;C#9 z*2;y>pp^p35Y51HyPk>$&+qbDR3&c?wg<6OGSdo=Xqb&*oY`g3>ClBE&OpZ}s|Bc{ z^gcN2gWzg6H#bE@MP0rB0rLS+fH2_E@6cO&_xQ7OkAsd}dPW?s^IMjKG6K7rzcN&* zp0n2Pyt>(vrC+6L#dgjTPYk`!B!%4SvSiI_9CM#LlR%=n@j)XSiL|Df{7n#7j(}qj zbWjLOw#oEkg{s;K>$Z$2NV13kBElX#cn3gbbonnNg}jh-3(wc@`mZbu^Fd)iA#^|A zLpF0Sjk;fGy$UTNxr}N>TigYhj;J8DcWl@oUSd>cr zs&n?NtBVMIPr90_2VgkGk1b zQ+#oEhEtw>WD1i7NpKjWV2Sr1>rLs1#vp(tf|8PL6RuE)ODm!xMR6UksU1p0Q*5=J z4MDJM!2pB|N%nmpo4VGWa-!0gUg6lCd#PqmvpENM5TpxpbAHrE4l3SXF4oLe4;8<4 z6A!UOMD{sgssHmhg#>^EQ)0cJM=j-ZYpZrNugZ@tQ^(i=Wb`DyJyV8M?i0Vgkn4}P zf4Gky+&2GSObB@vLyhE}PYo@zX|wEjateTFLwoi~6lGR-pa@7eSLLcAFcAeN<+OBc z5+8u~Z_m5HS77#r708vi;!U8hClDs&*@aAc#Q3e3G_Ut~EZNtpkYAhZwt5J$Z)9h8 z@z}qph7-Y(6WI45{3@-yh;AH9oD3@@F-m3dehhJbpc+c$MJ;W9P?Y$}o2uk~^qF4z z@3mDlWghUEDO4Q{77A(oMe9xN=G!7nbP%l}&2Y}O9a%krum}^uYQ*)ZLaRs|6KJ7 zuHr8}*^~Mw0LijD#paxTl1*yF0s%N)NogkHr-%oc1B{PvjdJdTE;jEKw%eOpTtW& zJkO7s`q3M|khnn}W}lxgG^TaJzPGk{hUdn{Q** z3m@}73pPpXXQC^Z%C)`B}TSxFR|oj?=#+fxVqaB zgz$(|mfdBx5;uX7*1_eQ#4RVu&yD2>fSJ+4l)2s>;A_Lw+c;KdpF}T zZe0`t#ld4}rt%~PrgUyYhEUT~f!E7Mo(LG~{3)dnI}-;d6&B5pCrvWF5nkR|2ZN-M zrfo_>^dm{(-TB#uN5@wkhU5bwJ(Ef<6rQ}n7MyWP{rPbd{ppi7(f~@zCGn)|nrWDo zo2*&v80NxqeDv)*%@rLlCnJ`f3a*GJW${c47~Ua%az%k=b})vR9CKe|%NegvL6SC^ zfkVr8mYngqKIk7e|LF!`IYMaB2dd(xPJ)T1B8$H1m?-lv%&(7=s)qlFRs^!R?g5cc zM1=j27R{A~(|D;LtjHm)Q|eZt!b#o+cvei-OL6`nKX%#wAd&`{dl(^0%?3Gz*U#4H z|9bx_$#P#BdW|w6We>~kJGhZ{ZB=vcNzF{Z^)9h<3`*Y23R_Sn)Sk|H4mb-!RC%**_@_)^rb zJtFxfQyS01K;D`McSQci$M>vjqXv8*U6(hcsG;b<+dn9HJ3h(J`B)a^5P^RtCqsvV(bu= zT~|iiZ1{4qcA-|mNn~0V|DSbcfpvaYEJ0L6B4hZ@mf!qCI7D}JCdx5$rWbjlFQV+v zqxkX@6Y8~e(D{vDcJ>G4$f0g_k2c((n^f|;$*-yC?Xw9y!$Fyb3i^ltu{3maQ?d>+ z)SQcstRvMmkM66}DdYaf0i;Q{Y&E#nzwTj7u^xH)v20%YrVHB|R}X z$ra%`Z_WO-`%v`MArAGsO|AdL)SavmT8;`PsrxAM55c2@?}7AWA2*_9H;_Q;7XgR= z*(aEz%ZEM~au{0qy(=U}l=?ZVM51Vq=DM;KL!MyDtn&iRGNNc{Dw>!wa6mTDs-l;= z&}SXLdBx~t7Nt{)yzgzSi|_gW^{J*C##D$*bgX97q97H8DtLyM;oWa$G?BxYoXxR%%HJ{B73v zX({V3?DoTTgJY{8<KC7CJZ8KRP-S))BgcL zW=bgA5zA&WfoZ?yJ&{g+#%-9-!WcxzEA!L;--ER_>r8(H<^?oq3U=05o~W8a)*HCM2nOv zxLf}nqHYM*8koOp_X#`5q=A$;V{le#Q6RGxIANNNYlTkx_(>`dxo>Gk4Na~7L8HqY zrOLM>|97emv}jo^lR)fLy{*g1$+^6~1c=D0HBHg6__n>hmetc*ArmvRx%D2q{gyn} zo*p+N<4uwLC2pkVb8q?q4)U!999blab19Y)koM@gMBdVxUpD(8;)+NGWnc)DU0q$t zvbULlTO*z3_5LvD{=e~k2dYw6Acq<|k1x?)sHT`b&#a?cur5p_O`_C<=BaiZ88jOC zhDQ?~JyKFJH!(r=07-3UE?iij_PJndYwHX(2LX9`R-j)2Lk!qFLo1JlmoGQ-ojP?& zHK2A)C{*LFD_-9byOc*UdzzVg#EU-WDM}jz{kIWwIuy3AxilVVza_f?OOyS5TG_m~ z%a)OiZE}TCSHSN#w!)lw><}(_z8#!sdD<=PnWbzrq>d`mhN5nW;veiZ$~c!Rgzt{gqIl&%5YW(;&2hpA39W5`kOp$EBZtnX-18jrra~odP-_#b{{=?` zD7UcDQA>mj6ft64b{~knJBvn;U_ph78CH7vab0JpVZLF}$EWv>SpdcxmzHv@AvtiU z*LDQ`#xsvToqcQ&kfM&(&WVX|<^+sAOJ(9+Rm z8~%t-L}f{!ohxzvsVQb$fSg{v8Q<(U_vDh z{ql%9R3OvmCpVVHpj>*p*r{?bOoHenALwO5No5?|d@rBcu$#-^Vqp>vL_Xt)R9_p@ zA_v>en<#^j_TNL+j%oo*jcmt79X#BW6p5>Mk>tIX!&C+N<70k2Y`w2Yk`tE=MAnWuKY#zY_Kqn|r$A3UK*cGyj1G+U3%bdGPz<43XPmedJ>VQRwECSex)rl&ks|rC%S`ThK z@F=5WN4{PAf>ky;%&G1xx{K|;<60m-_Gu5hFQSiDl1x?qz;Mg#9+Pe2)-cwRufz&5 zp8v?oaQeSswj22Muu3c9xSTUDr#gvMm#pfDiM5FTj}bova2LMvims|6M9x_=awcR5K+^LO(MsG zmn*#iH7~k~q1xK{7+nODa!(Gji6r>tT7sBTps5TM1H$yQ;PLwxKEW1kY*0s)2@p7O z2B63ge>#B-PTdvu$wIX+QsCv|6yd4^~Z} zhP2N*B8|uO6a}e+x0_5eO}|B&+sAJL;@SFn=_`4mUbc2A$GJ0G&`KoALcS0CgkUq; zFh9TgB|+5N2I=l>5a(KAr8VL_@^8nTV^n?RdMWWeSZmAR*S|qQY|+f z;Dp(7z%&J0G4kvKp1lu<&vP}}N#&(NL#!opk`?sV);#f9F~n5L-V_6=%jbhm%Lf6P z_D1mtV>;O)hxX_aN$T7qHh z;@{!j;*d?t$2@DWI<<+3NcQNd*fFgk;eM7Xtb0oPwUDedl~04B}|IBPZJSQ zBFWhb%i1pZZ+_=KM$)B>Vek4ZL#mjp{6!#^&sF<*zow z1%Is-)8usA(5@9H*YbZ%A#H@QGrNN93z?1XEyc`wqhVx5R`d|sE&rS@Y^aYEN89$6*-`|SaqcB!RL zDXnl{RHP7)-J&p$b7#%Sa;k7Dpd?W9!9*s&Ar{klDj!ntm$z#=-g7;{DI7e8b&0jD zek{zMPF}?`&->p!p0Pd4FBVsR29G>w?V;K>hq1kF1f%;DoMrk2@!>M3*FL{kb9 z62oxYl&a?eBBgo^3tG}4L#8S1m?wW$t9Ej+1zSsmPF+gg%wyD6K0>ByMh)mm(qu!e zasew}13~*ZGMmXrg-ltUaP2X9!fZ@%TMJRZu{&`c{@!>6Pk%%3Q}@$}s zjt#OnN&kGyGp4$l7Db%xZ zI6%diz6Qg8_6D*SgEZOEi6)T->4Pd%=k4$%h2_@lue~06MM+dh%jI^>F(aH`S?yKz z9|ZV&vus7Y%)WSZak1;4^wv9W5ti1IrKb?D2WYw(B30jp)b0G8*#Kq5#I8TITEu`V z37QLjf+)t}j_de|Y6|;1m$0m^h-u<-WYcU4=eaR*dvIAP4(>HQf-(J$t zz+5RX4g{>s2?f?F(D?nB#Wx?XVU;26B$E0~UbZ6L$7v7;cojR&szRsNH;bhb>eLcG zzt3*}X?ZI0^0t<4BtGD66`N}6x@hD~;^^9Da|frEc0T_Zb(foq_b)-BP(A9&g6c6q zFpopJi_9XRbjgd#9YOjOhW4Wr&PV&+;G>v(M%O#C#Xq&y{z8UyhsM|qQ@KMRLLuU1 zfXKTO{d5M`ZtfpwTGeg(BHdSIsa~V()XogNenE~7w?yHopPN1_I^ksB8D+U zkhy_;>;CMly)U2)l-5Vo^S)^#??ihs6hgEYRV`LkCyEXK;6 z9_Bsrv9Q?jese{g_`##tOQ+r5M|Sj0J^sDbn>w2#0&km;f`TFKRevfHTGB_Af2c+Y zx)Y>|K*5ZVcm8d~#>aoZvVL0i+jEuuK}CsY$qiqT2O8}=4p`}4^Y@&{A?;IWRis1g zr@ez<%@6HYUNbGhH>Iu$txPiopOBmmd5OQN;J)_vh3S*k|L`FO`O8~`J(&gcOaN&* z3KikgcV4;U-oZSfHW4ZRMq@duRUZGW;Z~^7nXjrAGn3rpieFA_C5-}oR2uzQby%)W z_FWrHhJq0lQg!nhy}wF<>B}HB@(}eql;sFHTK|*;y!Kn@Vfe}$u+nkb)cs_q(kLNI z2!k)9G$~g}E;SbF7Fj8Fgl9KLHx?g(9a&3bqRr~4@rzQ zA2yA!s+V%_cZg{`y(l0!R9O~0?{@Zeo!;f(j$&!F7B}~GL1YB5Fxne|jKIQwl`=i)47GIataR=UKUjR@nifQO0uy$*rrFN z4!>s#1FAtSPJ?P*5dI@a&a@JI?EYUDyN*;^tb3SPC;V>f}kqhO20|lXX0Xu*A zl(%NfP|XqAb@3KNl)W|XqIJ7FcQfbJI<=JCuqT*g5Ub#~c)gPS!E1GlIgLyu(Kg71 zg#OmuasymgOyk}Hzk>y~@cKy!RiL*Z3iN_5n)Y%#BLuDIg+lS}+f0u%JE6DQHSbc^ zGsC0OP@`jEVX5-f)6mc;BTM{8v@<(U&8+>jW7-|cn5e(XG(XHzEsE(0{luxzkO}NS zBI&H37^a1TEI-s6n|?!u>XRN3{Ld->)7ZYgKI2M9v$Btwcjpx|&GSH9Z(YU#Iv%~~ zyxiQMV08bfw=MsL^suXw=Y?EJOa^|u@_9#qE4LrAES9JG%ix(5ctndfdo zD$HZqnoW&xr9=iYa*%r&Xb|S@UMjKAjXP>(?V1>^QVAN@;4pr9YaB#m6@m&k3O( zsChvLJz*QrO$7~(LTZnozY9(t6R=p>7mDTnPiv0~8hD6bw{!FI1XWdeD+fKe(1V$i zBlMxM@#p-T1#M99gWzpup`VG#Bn8KQRG1i0Lz<8w#im$iGCw80b)dLijMkT~9Ep9L zb+Tg<&vLm_P^E4n4lTB4ZSc^5D!Q|?Q}*uLiV9ZF^MC5oR36JaaJFWsM4XCX&Cbc8 zKvhoAN;<9`{qCKnNaEU!i7J<|5@_oZyH>?O$10`{Ed!`_p1>)qQ}+ur?~iag2->5C z5P^>L8jn4V7;{}o&=oGK-|+$Qn}C`cAKEQ=VV(DZfE<{6m<{r83hQwkk9X##9J`6v z=f!-lnB2Wr@Iizn-AVmtwNK>dGq#&Fu=*~*EZS$c7ma41cy{O^oOF& zU_It{3)_m1d(L0`l76CRViHAk*3swFl@36wmiv|uDBMNSa$>Z_rM&OgAxuEf2$8Jh zTz+k9ifjx6|(4)A%Q5z9jHEb0S>a4DMwE~A#3Q?BQDcX zvkc5iAFMatRz&m^0$8i;`x+ZC0K${* zqY4I1h231&D|E!=Ee-pzr*g4jE=ww(wn()inA9(5_0h%DgN1aKnJBm>ddz9vE9y#lR%d4@-%@G#$soje(@KUEe548D$cFD~i?~FLQLplpYZsw8t<4BjW8KD8|}+ zGe+c-ATjQVvUv}uCA-_1a^9#zWxn#z`e$;*U}3Y(*On(!0P~T+7b20~k4U|)Az@~A zd)-6v#jThJ*0Bm~JY*z>>1l{2CRvQBwTElGp}pw!BsKfNBPPL6X;{U}=o!go+1Q?Y z<0mfg#92`Lp0M_t9hpp%j|+zwXw8Z|FlXcL{a~=eVrCnz<+5S2{P0;bY6Ji+LZ?jw zE{YYd`O+^|6=HHupGe=u6+Ct>Wlk3T2}HpqgO3E_T__NT&I%Jj16SKXXs(DAOTdyT z_(C-z?5;(T7~k?xx-O^5@y`rFL57^om=lx^WyQJ}8BnX@t{lOj>Z6jO8kL4vJhxJyN@S1=H|Be7dIgEP4=p*5%Y3XZ7QOy*J+Tix*8WIYaH9dX%tN|MR}E?OlBFCI+q@f%8z|A{{QIgiArG}D7DwL$#6-ZF>B^Z(D$nz?MhMv;gAP~>${yIbZo`Tdb-uq*`jFII5*2yd369TCZ&VMk96CVx1t}SXIb2%vBUi|f z4*sCzE;P_^;A*@$oy~0Q+j?p30u;k*Zc)&TQ?qNNVU!!8xEO2HOYv<7*82IgGUTK? zeU@f+%-M(zd6)fC8 z;F>Pg9D;L1h@4!$E#)Oct4(n~Lx~h3!K>YnrodM9pTKGhfwiphrvC5G@5;9pQ`ss< zqk`h+d_95exz9mf`v{fGa099HZD8J-qBv@hzg^%^cLH-BC5&8UJI^nGW6#GkAo?#8 zKAkJcz=hH*9r@KKFO5BPm;`mhQpTJKR}$a6>b_tH(mFhCXCZZL&c_MH^kG(NttIuv z{!O$t9M^_~Zm_eH-&}7`T=r&=JVtUU>S>^~qKJ%{gS_-i+~WNHQwS;IZw`XMBP{gI zXZ*`(7!4GjJ4$`Fs(-tgow*G>QV6xU1&J#C*TnDsF% z!b(+YEnvwirxFzmO>l)K;cV7~k0V7(`7+6+qc5KZR-G4klEpkR%wEXJDfd2M?VYFm zxxQ3&u^6E<^oZ3-X_zFytmAeg6de>#Kqg?dRfie5SGdjfiBT%3@_m+3|Eqv_ybZtV zQEWi2dmuiRDb>o>KC_ zQa#0Jl`P;qhAA%g+=@$8dlvN>i$pHO2hrU<{GM1r?0Y6S7w}X|OT*d%_i@;F^1h&=<)y=#TZ-y;f$qe_$8&>~00e$yI5Y9e_fqDL7apRA zPmY$ATeQ&T64i3yRq}q1%y-#wDQoq8^f^Tx_k4n%Ptb+uzbE3Qq`-oYq|Xg#1)oIN zpBkLZrnaB{&Sm};vvM73SU#*o(AkC?Foso74(ExQ*E znD~)M5~hIbf)+<_mq_yaT)E!M#xwwTXHL60tZk~9b+GADmMl$PoK?QsF~8o=nD+eJ z(<%Q90X|nKtgMc^dA6kD-06o^Q@is776NEfa`qv6}8FtnMw+ojc*|5 zts@4s7$XV%Bi@HgZtYYRSgvfom%Bct5M z;=BI-qWa6zQa_Y&{hWI;B{hX7oR9QzAD)3{nJvzBt)$lq;)kxq{9fC9OcR@&*u#S? zu7sW}F;?7owvX6oZi`jmN*0lfRc97JBE% z+?m}**Fae(?t>x;gQ|wYHQIfGubSUy{fx=VyNcH1hjyh}f`i&UboEk6En5%iC@zh^b*p524P1 zk1v1KAThGGXCJWCZugiY1a1dTtJb!n6x}2C@G`k5z=iR1E-w!jYC9#gRXn)4lxK`T^m|cjO;R&A=KkcR zEfBFh6G(3&8_NhDOvkLa;q~n?OH0aOwZ?S*oDAnxH zxBSG7e;d3A$tgO+u0M4a(b_x?%(#k!h6==PrMd@79hd@|=C7B3-UBCnXBJ!3Z5sSS zA2?g@o-X;A7x?(^azji}{9wgYM3r5(0`M{LzDvg8QEExp#lr2GTh5hUP#%u^T~S?4 zuO&;m@nL4XEU^f0&us9~asm0ZS-%Ey@pIwevWE-~d=@4e3>B`mQ_ju=5@RteFTdAN zEp2D=hiU}uxYB^WxO#B3Lrt)?xTl6+oE6bG8Jv3M#@hJCUCOwr`d79p z+A5_qo`pXS%$jEF6$PEl!X5hWYr-YZFrlS!=-<@R)<%1#prbqNyBu&zdtbl+5(UU- zuWxTGZ9ro3WBV>0Yx<8AO4&aD7yG*|JCji56kTuqA%=s=kbOjM=;S+0G92F5tX@#$ zt~RmOQi3O?OWmw5GFsQq`UK&~LY;U3> zRgbwJA<#B6J~{ccpy12Is%fQTVJQPBnouEy)>A34CD%mJd+WW}c4<>L0^gFgNxWpY6*Mv6Dk!sBNj zLEDwD0i>))iM<}yMx9(8w?>su(ela%Qs_Zb5qzhBm}C7vk1rW2TsS;|{5esS@xXx) zP}?+4tfCFYMQ@jN2Y{8G8m?DE+uhKnF|@ZFv0HP3_Fv;F=Yy!`o4|35!s+c2+Lc)f zX*{Ct?X3uXWkIOS`D7?T1Je7&@%Rf*>gtA$#LRvYNR4C-pQq*^`v?j(`_n(yc8Y6j z*;J)!&%kuPNYvTCOwIl?H&uJ&{Rznri?biD)ISoz@JST9`wYK(+$hj&MLag@9@hOL z_68TbsawWX^?-yc{Q**}Hlg7`fp#OKgy z2^rI%$PkG#&oVBVWh`SdD`ck3taCrT`+W93=j?s<+55czyq{0+8*4q!Z@lm8`d*>h zN-6n!uFgi3Q;Up;h+8($z*RCMzv3CJeWL4>1Ws`ZbgLXh<6A|+xw{E7FX}W$dj%Z# zZMCrbpHEhN8mCx_%O0YTL#P=qMLPhC>J4CqHo&xiR>-fsI!yr(f(CFI7|Bn)3F1@& z_w|krP#H$c*TN?RUTQBOxXhZ^3Yt2&IdEbotfaZy^wwws=|tyM7RZI?*)jOM3zWIs z?HJb=g4TwovYc9x)5zOVGb$+ue@^7x-;&{*!ou#EWhkh>O)DRcj0NA|KTiamJy1R6 z7=AFtVFV6OW_d3u6R%|zSS!Y%1n;)#QuHr!ex+B?@os_QVctwiiy2$+^!_!M1Hs(A z4{2Jd=S?$u4~28br!tiGzrpHHA6F-~B3r3-dF0?>Im~;!KxKHj?ReB0({DsOr3+sd zOxIuWpkU`^{jHjxZE;V)akwH4=0nXsED=1LKW06l&Z)PVwUcIf`Ua%%K%^1mhWifsF&c&Yzt7)2Gpz+%Bcss}7pd$4`EeS%p^8dv%R~z1 zZ&td4k3p1X|7%Q8ku(!-nP$$^isnvMg{LjD|U{ z@Q0?qrX56!r6`A^skOBZY`BPiKvP|vGHQ+tWfW4fu;4|TkQ2~Q0fCvW_Ha-!<9kP0 zEHiy|H)jZG%osN1bw!6;g}6#|;?G!at>g`pSK$q z5EOK7=>}s3ElMy0&=87b0_qzjYHLqAA24ad={|}sm=p8u0s2iWV^4?t8X@JZ*V%DskH@heIVu2K6S+Yj8jJ zS->n@0XNiEYU6ki!x6a2ajaoRUw!z=hX?vyp+1NDhd%=6OgoTH%44IKBYH=T;{D9= z0@iapcxD7gc0bds^$ztv1MRF|F!VrdM$fV7;Aq!X4+S*)pEeBnA_n*4bwQ>)hJOBI z2>W*-PJ+a2wxko=J_^nf0y((tM1Q` zfunu1MvL;T!_+c<-dA;R)1+^|t5>pNdu`Gn$W4K{Nc`uAbW?oCoF*i<8qS=-HR#df zsrq?5`5oLT2~Ly_71=jag1|lYIG!jEU57%ccv;DU!belciKUYU1HUg;K?SJ6IW_{c z$hnEz>CRmg?2Slg%7HJL@GMSNUZs-zei#}%(yqvXWQLcuN#1ZT{4(L97Fv)HZ-I;Q ze-q6@B*5*l!I`%FzKk+MFo;Vwk007op1$d2L4Nw4r(^8{o?uz>b2TbRlZ%57ojLx? zxbyU}(}X$lcig!#gv}{$ZYYSDN8Cqbl#qUBTByjXrZk^>7H4cGG&@LzQz~aA6ADI_ zoNr@W8BZjw_vqXdq;I(k@|Qe8%+)1Of$yx!AA)O6UK4HM0p?t*&0o}(;B)=BlK~~N zEUI1cu$o>Ly-y)WJF`y{W*pBcrymxL<+IGKxTa%I=(44Gan-zV`V+k<1>I$$ADpwp zTneu{fEgcb8tW6 zIuBFZl2Hl5#pTLi4eor#v;B47779MpaFfn7!G+)nsMRH+-YyCHJw|dfLg9iay8lr) zw~6B%g2Rho#F4kiQ{GFQxD?4lyE+k3$XkL?>*Eq66&<`Bc2<^$(uQSt#V|Q{J$D z*7(W+m*jTlE+70nWM7ABlz8!+>1)mTHOg2!CtMUJNm7|gs!dOqiElJkmEjl7yQCo| z7y0ql64fkcbrbDvbhNyVRXnH05I7-fy}gh51-9Zy`>8Qa@WNyl_#&MeAPnrF&y`fa z_v+RYOjy?ky4%|Y{x=o(^FlS}gnF&`2zp}*ORc}-c&A#p)X_pR{W`})i~%je7Pi6& zpT4qBGK}rKtP65rMG&^`%3}xpiYW;D13y*KJcLn8=ttA+dZ7pXBv3SnWd^+PEb85J|5+PEg@{2w?HD}vjPDLp-d33 z?znd@Y|&TxAmaX~oB<(BTGN2=2Cwf*2LHOHrYAe@Tt$j~EX17+0=pApsOF;+e zp-R*@HT1I!c_O8&7Yf=WYDBJ+t<7mb102oVLN5wn{+CklE-@pzY9XN(RHYKI5(=wD z*0(=BBBig((TO;?nelW}krU?iG;V!hJ&;khO$a-ahh1^QiWmLp;sktii~|4`x44K-LCLaRUz z);Wb#lYX!xc-sz(&DKO=r9iqN2C-z%WUnFk7@(%g>UYT}vkWDFp8sRjFPgWxfJRQj z4^ezEP5x8VI#Qjzr8>gu%nuib@03?Cyr&7UO-ga3NIW}7lXkwdIC0-ivNerr?r?uc z`0^~V&k6eRtN>A9mzo%=`1C)$@a0e&)Gxs2If(94Q;_u6mSUksTqb$i{2bV#pMqvU5 z{GsyV6rEyF;d3pL7a<-rp(F^oAqQ@8d_Q4{$VJ1(u1Z9Dx!Dj{;JVME`v^{);P>~z ze#MKZX3%a3JdF^RKdH3vOO`!JFo}70+nGithDD{t9oeNs0}%+%(D-s4S7KGF{`)%a(XFve3QIV&bz0P z-;pA0=LH$5wCtsVkC<}&nYEE%ZtcH%jVcM8G3QhCZdvQW9f&cPFPZ$iE}U6gh-1V1 zR({mz8m)HNI}~R2PEDOP#!d+c^=|bKhz{VK-WE2@KH}Lgo40oLO>QJv3KXWJlp@SK z8)@;RjeG|8`WqLQ$NL(WTBm|`_Vc>J4wl+D(>Vk_^iRxO?HI-zMHRSq9lcvq{Zu=P zB#y?auRZz&1x;yFKppDXv@swnmS>bl1?FLFdW@rwVv3WGD_Y%ZmOSFJ_9|yMEHJqk zbbU&i6&E@isgr!?_(tzWTFq6-=|)&da=ajIPgK^+`(}|1Zp2iW49AlgKrJd6Ph68D z4qGDi0xy?@FmXckYY897M?D+ezz+yEI>IU^{^;vR=NC_WP`(~O2X$527@XBb*NYCI z0#udHqq0Bzdf!H0_9*9Q5*|@I%J0WQ)SCq#AGE@IJf0f;via~YY%~bRtYWhK1}^*5MfqCQ&>Wj-#E02o=$E$(I$*Aq zor^_y+I<2_c8LD-9q&}=R?RhelVIkB({42;Cx~D7Nb9;Jbi(B;fEnl=q^JY-#*_nh z7K2!jGCrSRsRva*-endy2yxf^eGz1riT6Hzl_sW1ve!jCJrdmWVR_s>h(}sA^z)V* z2Jsrdz~VmDyq~#^{`7=Pljh3aPNL|!v>&IJUY!)fYE!1hOa;I%KSCpKbw=i~ao`b6 z(){HIUx%m_--KpSz3q;4akwqv;pJAy?vXMuHB~H8pzDu~?8Vl&6gBAGIGuW_u zHaBXh@!F!u+o%bteb0QkH*%`uV8yNy-&mJy#2_74+EgNhuDai|JC4{O)_LwsXFh%- z%^tAFDSe(IRR}`Uj~ah3YRbd_C!oFDaVob8S3QSoqrq#C_Aq zyquMG6k_YB@rq*TN9?9(rA`{z`gYb|3WwuwnROt=a$%4X({AhyTJ;zOH^&j-;I0ok zMyd8MjGY<>cdHk|@^2%N&w9qa;%k0LKzFY8GcBHYPV00}aaVHceF7?^HLj#I-7$Jx z?Ni8=vR6grR$a+PlJR1+6v2)uW_*xrS!Kv!iN+vO*T%hPs#585o4}aIn1+1dQ!P;0 ztGt>bIW|8Zi0WvKRw8rqyjGtOVkX`CTdHgG84!J}sj2#p_1MiSd63Z11Q&{kZZpgXZtQKBb z|J9M6;^3CseDGi(hPJz&MN!WSg}4W@Ctz&Zj2Vv6W$D!5i$|3oa4`B7a#6NlK9 zV6voj`*=(1nO`5t8F##(eqt6DJoulxgt|&bTXinq9u=3L!?;>-QB3TK6o_A^f&H5q z!FNq!Hf)DgHY^t0QJpbZ@S*)N5ttJ(KJM!k&PKXUkdS#ixqPCA7kv@*osw+{Gk=P^ z4USnN9{ri)9Rkc^wjX^}*&F_7)_h^Rzg%0(usj$3u{75WXcMy!Iw*%pbmpY0o9)~s zl&B-d!9tUM>+Mb7PN`2iqHSk($BuTg5Y;36sKC{F*oj@z&q1t|1mbPs+}oH$kY(#F z3ivBw1QW$t=Ld2Qc&2fSoh18FgnLkxD3c{e;ty6m1Pz|A&#P$s7%NzCyWLilZzK&w zPx8}24^0uAeNa7?mA`$cSV5MK^jeI%x{UkCeu~`PjR)UJ!;u@?TUhy_i=Kx#h_&VX z$v0xG)1U2!yQmZB$S3I?G9@ud>nud9Ri>YEb(sEU9ypS#S%zuscYOq_wm+wqDpZR= z11zN--{61H$3HKS>&%YJhh$7@PdF5Tq6UR9-evlWGyB8|ZNcWpE|k?{_j#?$#wq&$ z9j}cFOq(>D6P{(wtIO z7vc7}kfeq|0Cj*Rd6s@D`_ul+p{vw^Wcto+M9cG;Fut2VzH*;%OIaQ=6|TQG;fWws z{igKY?YE%bd2x~Yv-(55EF~>No{pm;`G%;hWBAL^K{CRGJf3u#kZo~Xc|7~F$3)6U zreiRsOj~AQ6?$q?T_sX?@Nhs`1CMUXCyHWf?x6FM{A9$>+lG+@@-lLxW--z8f0yDI zDIs0+qUl%kLT*ygzFY4G#KoA_HR_FHe%mceKfQN*lxF*Kr?rzDZ8Ee6xwNihmZ~wz z)3Jv_p!(kXghL}%3jh}@Z+58<);vB1j@@Y9u{dp+~8@h1@r@U-1r>`R$a&E zoQ>@`kx=IyBaRQOVQ}hnde;`y<9n+(5&mm(P*|w!{su60fR{G-tIBA{rZ)!pe3s)y z9P%ytC5iuZ>l!^Y9V6O%&L&s%Zh77sM?8_~d0s+qG%wY202l@Mp;HdZ0V1R`n8e}x zIvMJr7INF<$gAaZvThUFIX|bv+b<~@yk?Z}U?^rkH@gxW&wmP^uLk{=6NyDO;azs^ zn1nnRr14jGaL?uo8pd@P9B&3pExG1TG$bUegKEH?$QWG6tD6!X5F!-x*}E*JnD^&5 zni)os#EH473yAe1((AE9y3C22{~<_(9*#hcgyMULy_(v1wu_c;Og#jFVk3B8|6;$M ztJG@}s49x-9YC(r^PNhNq1vhp)+M%wtQw49k!U@%bhq--La`N$K17oHKyky~xZ5zc+lL2~PN!L#oxSJ{-5(eY{RY2tF5WFZK7jet_R zWKa=0P5gY{GDvRM)hQS?I1$ACzM)y%S*vT^ERzL3JY|L>X&X)Zq|SFC&?vnq4Le<$BN!|l8w3r zvVQpbx##g6{oqg&B9RdcQlBMGxu@^`Anp1`-<2ynuG})URq-d|fG!3Zj7U82?cISc zZZPZMW8CtFgNcuM^Dj!=TxdJ1a1jI5iYPg;je3SLa-mtrS~NBfddz7Z z(upJ+V{CBeDG65a1e}Q#>~||u_c0nK8^4GeOCFk+ciDA-tAJqH9}Qje1YnF zVsCs6?dDU;%UB|D5j)FoyFXJW?NZ}YZfJdeQ5 z;FZPn*UD*)D*U2uWP==qw@p+SrDdAGX&){{w4IUF9n&NrIVM}e{qOiex{R@Q$S??E zn1UWIOxD@K0 ziT5YEShmdKmeeT!|K-bIAQTn;C{gJ)GnVOQMeL-Feim}AZ<}WNevFv! zI{U-){Lbz9{(Z+{6#=av(8NMq{Ua3Nz5GG|OA^@^&Rrgwnc6ls5Yi?;b&p5wy#S^} zTd(iKb*G!zsXC&tqJC$Yw?XzH_eJKzp*j!|pH*u(Qer0&u# zb|I%E(dq$dcij=ydjlf}MWKT{U8eQ5biq)yX>!D)eZZ#k}nlz+`SJ%Q7NEu|Lq|XTm;h@1vLK_sU$kIQN$L z6Q`(JrbxEs_c)qU@CN(r>Rwg7P$v@BQ}7WldOle^Ih1yF+r>m&lgV%Mh7nAEyRUQ~ z#fV%`RWLhqSuOPI*+FfdbABDypdplN6}s&l;`gmrhjUsSZK}5-ID`MMF54_uE@(k( zp{DIPb@gC?YxRZk|+iFFA@o zpO8!qZt)hY-wlPJbnjH)VeDc(a`*RfcER`HD-p&2^;kRwD%9}A4z>rA3=wL(Zq@sXB+YxZighs%?THSX=mxfi~> zZJC+{;eqe(iryM{_p^u9UOK|}j)zD@Jn=gfHM{JD(H_N|!kOoNtVXn!_BKb1uuNwF zDM^FFx-NW2n{zFZC!VVV5R(p=RiJVQd=;P@VI+L+<5KWp2h@~iUtMFPIe?8P89DkPBipIv7`=-tXt#RJe+XoBeIED-_MYm1({9}-`5q0by6!ma$U(x817`AiD&*O7Zf+pKiK3 z9d;_9$3G!-_?)a+FaBfy;NI(9hK69jPX3b)4ZTP-Yf7N%{0P_eh>ky2bD=Wum`X0T z{BSCgy!%_#Yb&Mlb(Pr&%j;FA(d_={)7(7mx7VHEg7H4DH>rK7OdLwJVkbXk&|0JW z3ucs{M@}HPF#W09gRktnYAzTyaI85Xzy5K}yHdJ3O;|uJoFfIWl(!hU!79b@8xaNM z@2nP@Vk3&`x5Jpl$RHi!BrM>-L!bCx*rP|943F>MFUGJ01&Osy-^y>6JD0DHckZn* zOgS37CfTUG(}AVBTBv-iqo2tQ%9q@FX{u0tP)PXjd+R~soLm4%P%3p(o(u|csunz4 zcTHIN^>J+_XSS8G(}^b-m7sXyI=!P#Uebm2IxvX;X5%L>mubPOHAlhLn&`XTXJSzq z3`^I$IzeJ>dD(%KiXWM3A zIyHV+@+qe1j@xSR?2owF(dY7~0IR<-6*ji9crQilj+(0B_hX37$CXbew@wAGPa!uJ zbN60}-MhwU8PSYGDk>{GGb%IIXt=5Da%KiU0aJ=qYH|1~lxe{*0hq&U-E42J^%vM= zbwNA2V8sbRw7fisX?dbDwm(M3h8W&B6|CiIF}adqd&Q#|n`0j^yfQy3z@jK3c|>QB z$^%L7vPL`Mz*xQM(|Rip5A;*WkGa43x}riB$hCMt6s-IKR<0sgI=)IktGvJXUYu7O z0MU|q?mA9)9^GV3=?VrqM-qgc2v=tV>KvH4UnzA;FNc0@PQ4jQKFND=bLjaXJ@9BQ z*m9K3+NOT#^~su>6fm-g8|r&8=_qQFfR(+b8&Ik8T>I3KjkYDbQcJbtw+ipV{ZzzF zuRKLEcKLpu#D=Q`nQCcLJ#~`A(7XKh|kS@4TW`jKmOVigh8clfv03H{_oFv~18 zD(uk#KH{aE=UHU*g`r)|g@baurV94MvS=$FTI2PAN?vZ_iNpie8^~nB$ZT#eid{t$ zKu~x$YI7_7@xm|)sOKG@tveM_uru@5a5=@6`EF1c$ls zj9H_g>v)7RRX+$xo;2Is!6k7oVzax-Ic|P^@x$~eD4zge~SAJmNemj*3hG{QhNgLOTwS9Rt;!5we|3e0_}C*c_O;jbJKJ)p(BDhD zS~A#ov`^_6LQa4vmT@o#TSIyR9DY#v&*pto-(3k4&v}Wo*)+}g$M7iMfanAe+_ivV zpa1;;T!xLDp46}-Lc~XZ;T;Ba(@(PjbN*7K?+z9ahhU&=0P%sC!zj^pn=|CxhdOJT zg|fGj4Qp3E-QF!Pob6-nlB0_d(@eKSFpHmwY`(%Ow=3zO&$ua#h;ob5UVyD6-`x(k z3^M{zzI(0dQt^V5%XJ*1F>z*DD3SnFXoC=o!2vZG`L2YlZ@+b_7;hEIRK z-?rb>U&LUalk%FqcAvZQUD=pq&%t&~+2?1jyz5!R0SY^A{AVQw28N<9aFP*0hxJc) zb94QQXVf}!QFTL3A$o`DG+3$wi?@ZVk;T^L3z^y89lz>kj=lB|3DZ04SIAG-3T*Y> z3B7|ld-3P;4GVsf+sB@F9aN1gJdBV&iNYkn>S~Lh&ijbmANAbS%#>FR@8`jhw{k`0q3xK6Qd# z%%Cr|FB1v#f1f$E?#aMPU()bbsB_W*#D{m2&+ zK3Q0qedA6uBTF7NRGYGIQy!lSc-^+zjT{5o#W?LSRYjxg83g&Qdf+U_QZ}_9z9xOv_l!iX z9&UK!{(JUxs$pt^Yj@uphW=%wdf#YS-7l~GA5<(mdIX7ygKx0?&zHS*m*UKfk6t2P z;M(c7Jv&Nk7*YZ9W!?v3TJ3oZr96w|dtG{aNBFw6))+ni$kHJmcfV7?_O2d5pgIX8 z-NR0BMnv^C43uw+yd0xK?i1jO9WZ?Mi_*q|wY27qv)r&Yf_&v*F=b|o)FUgyxbwYp z4t6*?%&3D1FsBvQC#|7nFb##_x$ypIQ^CtC2{HZ=C%c3tkZ@Rm0is zZV;iMa@W%RhEcx^OZf2voO=#jyqEmJ=XT?;2ii^Xl>dHSoN#QrPn?f+S%>cBTyDYO z)2l~lpaJwo=M)^zvf*f!KRn%&bl*3ZinsLdBa4xtJ zA$nv$GH)sp%@AkTWUZ;Sxp3|^9nq?G|Q z5&a0KP~$hbw7j%*de}~d^9(Hb9w>y^qb*Y&z#9d6#4Bc{Bm?#sdTck;b%#^iMFYMf2?o3GCtEPSEEa zE%3JbF0W+~SSelds;3)0mY(NXBtDR--fbg@2A9U(=$u7{!@`^hXnVWkk8d#Td) zzb}Q>Lw)9@P%olufmE^}^J+Uke3C0P?XE1BkZ06rjbu@upb{e#cL)O}_ds>}xbY6#<5f zT(w^MG~R(59_StbjW}0kiu*~tqC9w`5bYCrlUp^Kr?8c;NPW*_50wQx_tUy@HmRZ) zDbgMLy=N-U%K*pmQ!O(sxR{PVg=E?9*Vrk5m$5RS^nwroHE=o|wnHETfX=Dbn!#m< zs%O4|8Mf!mA7uSuzmaoo_NcEZh%?hpJAl}wvg@2oN1vD|0SpoSxG#9oiH8+r!nu5S zGk4cxb}TmYuOqrX?6RuxxRJT#QYS$-9z@>n%Vp(E_$5DiZJDnMg0!f?5ppgRL|i4U z24j`KO;F=q0_bje9)p_|x(}t(P`9ZoW@?6i!0~{Hrs+~dv}Ejt64RSce~K*+5x18` zG(xIJ561L1r4DX11v55wm40z%9DVbJRiAfulPH{5GuX2;wNv%-^cjYOCOsYI7XoCw z&spQNF%M_0}z8JAV)mqb=1?ML2W!(Xkq$TReP*^y}(I z{ijs+7TGS|-mK)j_mkQ4XJ_B4um7}@oWAbFIzX;k$qo-H&S8_Vrh6`jGeU{^Gqny+ zg&3w!EpxgQ0;7n+C_8kh^Tt8-AW$f1rDaUoCXV(a<86x?bVx$Bt#z??YSBq3bJF_J zvZ5={J>_S9P?EKCXd2)A;1;8ABqzloW*{NFFj$|{cZhHF0$(#Gq77g#uTHh4+tnb8 zHb}MY8Au1$))0MGO-jG^_$(M!HRQ3@@s7-`w4=qx^9ezQU`=?CPj*0lhe~CNZ7F8m z*S2_FT6^tVOEok{&i}BK+Bky!`bOu3V_~LV`xd6xR&>;uGPlzt1rHhvplB*9o2ftB zQhpjxjlnY+;&LmB>7W#%lOlOkDHU#pEa?$-aX^(U3NM|_G8T=^8Yo1aCpkjZHM^r&>teOisrexRpx@lMdnRG04UZ5*O}SLeWLg_-LYfd-mo8*{ z+91dd4}0B;eMy}fJX;fAxsME`;njWn?J!hj5hPy`0N8Z@&WuqIM=&;5FMqhl*3=i2 z`C}4eroBgDut09IRe#i0&Xrb&^@!f^8M3dcx6O=lETG!${`={!U&Zz5a7*c^)n^!T z0`%z5TRhwkk0EdU+HRvd(m)*7ivhYle#Iff}!8 zs^KBnFNpqUBBeJ>i6&6b^=p4IIO4*NWw1pvD?VL|d}ghjo*7OCX2|BOjn8`2UfGiR zaa^DHkSk%mP2~|laZeD)soil~M-WTXmpljcahbO#1{>DNArhu}k09}I?)em{-czDc z%Nd)qoyV+@2>h0+-LepI0`#J*{HD-S*S+9Z(-%}UUD`B?kJ57ltk2L*HiZr~uumDX_gN9IkbL$^l(e-_0{b)avd7EW1j>#TEc{d9)G z%|FPEWOlJF!?_kA#xt$2 zcpB0cfv)aYvgq+xi5(6BAt_6wHg)IJ?9&SOP2~@M?Q%5c{BVOE`>Vha<9@80@t5-JITNJ=Z&PvzXqN~r$MSF)%6KkzECxB! zmd6{TqQc+b{5sR0ZESVk^s1`fWmWQ#y=_eD<>C95)^%dSgO8`#b?s{(E&lu&|JSm8 zWrAt2srXjO(IOiY^_7M$ynY6M-zW2BDEvZaJ*)j&YW+EKg8W(fhv&errE2eA1J*_C z7l+1&2_|wSR(Gj??zDK!Pt5wLR=)9e-7j;Nu%VQtd+XP}S(9HC3C%v<^Tn(b1(p}- zT5Q<5B#zSe(X`oaQgvse=Q5qpvWJdT{do*h{L=d`+qeFivHb>}R@J_TYOeu+b-=V( z>q4+K!ZLmv8=SJex;R7R2rQFK8j@2e!~AC`etQU!i&h4a^abu(UUn(S}`>awu8&{SQ$NzvF|W_oJs z;HBu?v?c?xwPH-&6NA*c+&Sgy%h|`U`bIqj)u+BZ_Fw}JgOjFONEiNFx?r_Qz&Az| z*m?Jf(i`HoTUc${dRWjo>H^7w#ywmkdq3!_3*v52tJU&(Uxibr`{YFNy?rOLzofvD z2~!g*j!pE|i0B1anLjTk?UYvsnSmg^Nj^>Y-p~eTr%L*_Eym`9W9z+S*o#qv-yVbq!zh$5GJ$a{Kn|_M4b-ND7=MEg9HPu53?HD^*?Z+?)KuF~H zwSVEBx9223x^yewOY(F&v+%O>0FrC_b?oeCvaNGSnViM8^nTT7-Ur%iLK9`%MkZ-% zwXkkMT=C9cGG}may2IN{MmX=(>^3!KLz|V&Ur_h z|M^RJp+EATCrg7ir{8k82nsJ_X;=3o357?x_Kn+6xd#euOPV{o>fei`qC}@~@VCxP zM`>*VLJnMppw3c-6c5;rpY^dCRO~t2p*g!}1gCx{bLv3G ze;%P=&W+O`s7bD|ZurF!=u_Q3tVd0-y9|A>SIfPE36iJA>b_T>qI^eL{DgZkXECK31x1B5 z#_P8h-=ExZ|3DArbHEexns&^YTK^NBr)kGJ$RTo1`Vj~JsoawEFKGo{P6+cPoaven zyPc2ZDI_08D;w50;}jSRqT;mGoWnE_bw&XrnDXdVGL!#Dq&0vM0q4}z)T9VTlH;1M zgCTIM)GG^cMkxRGpHR6w8TK$@&V3YiJa>gNnhHeBGK-WK3o3!sYX8F3_z?q~JK#>R>i#wI^{zyzLr)FfTzGvm0?*li?OW{C`w z%sJQOq9Z$g8b9RE;qrfFo-dYeBYW)HQ98XSro+01_cH7Pfd}_rkFA5v5I)gghMEFt zvjZV~_;3=eWMf5rCbi&=`7z}U;1;x^jplw*vr8QUFIfX*c2t%v0%;j~Z7C21@95~5 z&EY(vjI`ckcm~Xn)}i&SDxk634tR-R75pvHbmy07LO(E(fU#U zcl>t^GZL3CUv`Ad;R&Fzi6kc{e_I)Z!e;cYofj=21@7^~EHXBM%UG#zn@Q%t1HiwN zb!f1)j7EOb_&@w|j{uSt(*Z2R<$wAykRFs(GW-~2{JCVHIS*wp#BiKX0(PS9nW<)N(mHGP>n^6vHkoft8I12M9W)wM^}0)X||41$M66+(IF+T z5u$iSohH@7IOSNO;eEl_m;=(2Xb`kHfnuBlUst=S+ud@4t-LL zzMClID_6RYhw!7pGQZ(t`|i$~K6&*j1cVa^l3!2|zi{+;!?}@k;M`C5h6mY&0EoSf zH720%faWOR8cR1726N{(9>6TE@LJtx^u6SAu6^XWC$M7t2gx>nTO0pbvdshFfNBFG zU;6&fr2Kq(u$(pm2n$$GWP7k!&|wte=x)@T1I1E-+Tw|+H`h-s`xdB&HDf3aQ~aAuPk{^nDn6C)!RnBLT#Do`~uT6~5d(M_wZuD-yFMlp2XoCKhzuVuJ0 z+WpXoq*AEA3C zoeuxstu{FQDS94-AKi>zK4ExS{N17GTu%Qde?5hEjmFs`niqpWi#ouIAAZK9(DHim zhG&I1B--tVfN_M7nVHEwLIuBW^8>`c!UU?R3Qy8jii+esVXtKL+yd@c z=iQ>3s^^-1|N58e|Nh<*k+Xh4VTQ->H`Va}_Bil_Ed4w(X=&-s8D~_?11Y7PbB6m5 z0A`dV`e4NZ{lFmjoc~s&nfUMNfOw@O*>EoeZ~E(`s2&BxzHyl|nuY2Bt<*G|-yU)c zWwXK%2f=gvI1Yk{<Z{0jtF3#3|1qp9HU!OHiHq z4l`xN+^c%9mnTw|EKq6ecUO+7!saUc2_=?@7*u*lpgvOQLz?$x8$%#An5^-KyUoljw`aZF_AxWg2~?hsEK7v-26qHGb1Ki}gwyGlPj-3c1ZofY>yL z#;_=BIEu=I$-r-Vf5dxz3?4b7wbXb5Gc^wVYS09K#&z}N zb3j=h15>)cyVckt7*o;AR*%Jri3u_223D=#dkfnz67}pEIfAkQ!c@(U<|GcikwZi~H!&ni+$tslDx) zw8W^G7)s!sLN2z~##j)FR0XwG?Rr^wPkW)2H_lUNoBXntl46F=RX6}RK=;c<={hdp`fRDAZ7=Ym4G zY5;6_=3w?lte>Rx9nyW!x5&Z{B2q_&(Yq0~*+#;p5>!%%+{u11WhE+e7UD z=X9u{U>b!8*`3{?2(aM>|GT@cNU$5@9%{5k&Eo4-kSGk(X~J`n|5CIlpa{`ty8#CQ zAQc`=>xcyX46Htyi<|$@D4W(R!a-pJSi}|hP-vvD&(;h-x;gJLOFQwRFTRBY^TfNS zw|Bfh&m2zMql=5Kb@UEM+rOKK0fGkCT>GrrjI6AB_?16M@;*CK@LmIe83_Y#0(RPZ z1}#}XdGDyW&zDy zL{FXYIu0>01(h#Wv4LNnS;o~iYO!hpRmV8=T)%NtMurWx2=s>lf6>NkZ*$5kXHehP zHq-0H_27sIRVSzX9jTGU;y+UgBo=@Ur~+<72UKyeWL-$rHERfCfybTTKlJGPzz)Q; zKZEU`FI<%;oKwzV3!s|;-j~hB04*GioY{UUVehKU>IYY$m z=&&>8&tH`|?n=PQu1L_((0l=>{0Tmo`8ql}UvhkRZDt@iy{3GIcFwJOJHETp)9q49 zm)iUH@4wuRF+T1MxayUuO_LH42)d81TQASjh&!v%($o}nhBdq!4mNin^y3Tqj5}6$ z?YQ<|ihj#op8&h1@?rA7`AYxwtry+7(`nEIJZ*u)ZLTr&bb<8$&L$st1h!bd%P-Iu z7#tR+Y-cA7uT>6NTbV=ic3>g_VcTr3Kd-;UKJlwf1*Gku^j5&0LQ)Z$cAVBBdG_qM zi0|fs{QP`S_jHxJ*~6)r7S^*CH+o+kPN*I;zMWTau`$7jb;BIYl$iUQ%L$;-!Q2J#hHL7#E~1==@qYs&*(DrJn36BUY+xDt1k z265QidEh+TOM@mF*;Op7?yQwm*FzpNykOI&4qrIsEx~el3u3ELbU3D3+na$T6_|mE`-K!Lq#YRFs%ssag%|NpqcpfZjt8sq-xL^SC$FG zR=4K{5KN+n{1Zsc5rvbbnK_YwfwK^B7L?H>8F_f&ld~vbd2{5{$&(HnOJ-=2DbiW( z>+1z^(waHjLvABu{s)8!>ZzXqTMV3w*X@Vt>0Hjhc}b@~I6S;Qk+frlhKA&0s~XjrdM!h!L7O$#=;Ybl!v7uIWv!6A|6(N4ZzkUby*`wZP39+Sguf_?HB}VUxclN z({oM?jg?&KLt)XmK%MyhU@Bh9ZGj!u8-P&wZMCn{ulP{DcCzDZ80YsJK wSaQcR1E>_&<(6Z+nI$BeNZu*}@${RAleH_uk`{(Lz*KQX-L^y=PWtGRkcgqGUxO z;d?%Ly+7a2aeT+`_s8#_Uq^KxJfHV9&g&eH>%7j#bx-}K5;X-21%e>d*OV1B5QG4S zAed<~68MeChw)kXknp;y=cVatVVe;PkQf@U(Yzxx^>Jcj+Rh zotKxJrzAhW^Z$B*&(*`0zn7zOQ`YxH5E^szA0|gO+a5uTQ?Dsp(eit?G~pYh z`Fl@xRo6#y;q~Le5;5y1w+8q#w4-otY!`{$r<-_M>bY;Xcyq>5X54oBo&B|*AYxI~46 zrwT4nm^gUa*(>n>_g@x!rSWEUNK?(&w~>KI2b-nAb>>W4#Zsc7OEYif_@G~mCjq|F z^^Qtb%bz9uV%ff-Ykg!an^cRFX!qEW1eM6p-9^)qhSwS!D^nc((womRQ?JaHfbfpB zv(#vDGlcB#ZaVZ+a?vs~lf&C|rxeB0BgG2y^Ql%EcK8DR{3Pe#;Mm&UeyN(omn`Yc z4NdFJ2wuB(P5q6Q0830~Z*PJ73qzM`V(tWrrLpq7we14K%CWxhW95#HOR8)B0;<#|( zg3r8CfzByTPNX%8THeCqD|O@YdcW+$EY+lw($e+K&Zjeel*PIrbi+M%!WP51+Ew3Q zr~a(i|JBVe9=dm#ibOuM@=F*gdsCJBj8$M!?MlOmR8i-at+ko8h0KQ;b#<&Av23k> ze=iD}H6dO?`Po^ko-`>&&Fq_Py<$^N_{rBdPHZfW(zLa;4J`g1 z-R*f1^oyLEn_D2{-xkY@{U3Qn=^AfnSy=8T@aTRmA2q36cI5s5V-pP8a!bB`_L;al z0Y}g(hJ=Pa+PK!|rD3JZ?~wZ1gz}ARkJF#Mp0k$R(jxb7hGQRVWp$d0giz9J0T;Tz zGes*mRJH#;f#<_Zc~Yhr#f{~O6ODU+^Cs3sVlesF?Cmdn>rK5=;f!xF6W|mQItu-= zs;H_u?>v(UWdmS91oM+Au&$P$1f-bEt7BznTx4m9oVfk-s zO|R4@W*rMCe$<$nLb)Y!S|ev+!R}pKoB8h_!xhf3&T=Lu?Cmipbe%)Fh!kU3 zB7G0X zbskGsdm+Ol;c@!1wDeM6$MD(f$+!oM{A0wW+Ic!(D)27yZfHtau>jRgo zl|bp;e+*r~i$Zk&R3BLBZ0O*S3trZuCs}ZK?9sDlPIytf0Zu{qhwleH-;XdTINrGX zCv{?n&$Ph^p$+}Fc86Autn~9m5^9#y$lbU&TEUQies8_LYs5c&YB1w}o`;7%EG*2T z@0mn}Gnf#qprGLJA2wmTxP`&22efi`KYh|$TWzJGY@v%HwpUl+13JI;Y@(r z0>9-D)R+fuUKOEz5%Sl*d_KRZ+!;od z8RFA8diyrPhg@wQIyN@m>t`Rs;K0vfuyy}_b)V$nQGJ!2{a`9&=f=kOkJ;^;Id5X* zqp0X0_!PcngJEA@dq-bh@%XL_6%S5Tiks?!5$ITVc`AcB6r@tjFE zX1?^g?}gIMBtGNYI(dWkN7ygl1$)%q z3Hto`^QRK)2hMmHHYR;+<#&qh^EZESbj;1oRe60sI#%IKDJv`cbE3A=#CfX9 z{nXz-OO9gXU3mNnh)q{jR48KEr5!5w>-^UT>ij(;qN2WjOLbM$&^UUW&*&~_VD^aU zx4AhkDXA0TB-AY_mu%ybl5`8$_V)H}2u&t*P5_*`pa<(l$HDOswNe;*RF_rmpGb#< zr%&&%uDS_3j`X+$X5UP6>eto&_~^nCzEbJdExMAD5)j|xb825Z<4@U5)>Tgw1n>Mo za6LWO@eL&57|(A#S6~!;MpZZnI+>ySds`66^9oeAAb?~-PXbI-;@&@wp|Kd22)tlv zyNgDi`%F6*KR*o`*}>ZjTD-gKFi#ZMuK7AU8$a=bKIp^`zzlbik{AstU7UA&qPu0dU>N7#Tnw03t>hJ=Km^K#Zuzu|mL&W(WV)~V@> zW+x+4A+WLsZ`=ZHwft0UzPi5d>Gk+$a<7fo;t2JFVEG6rqWCF#AjATBfD{k6Wn)=DFr{_54M_VgNuVeXz6vW@L#w8xJ}p}32Zk?$Qx z=)YTwJyUE_K4z=a%|3XDonfx2k{IMRwPzEg_PIK|B7eEm`Z z@MQ1W+X>>%MvVL5ofkg6J$0PdAfmHVdEs|}(|8r#O5OU+80Vhe-mB{B>X{X3oooQ( zFN`M(TCRSweE0aVz*0ua(J( ziiw)U8J?m#i@Y5Dx!GB>A46}p{@`8R0V1D*aoUtJXRE!a$m?X24XGR30ihfRjc#v= zq;$rssHiv?V|NU|;vIq>j_o}!6t(SR4chpi zT@(n*%gV}nnV+9OCUG(&3XRUD=V&)&r$FTMiK*P2d0*gnD`X}X%iTgBQ-`C z#hoK;l0!9mrHzfD1>L2cZ8e3{%M-Or1nK2Fo1bl86u1XBRZfMR0F8RLyE*mHyfq4q zmwqetn?G&d^O@9Djk&YLydHZbyVAJ7NA2xQ15<-|uV`VBvE+H|`0=lx?P1h8efg*+Td*bqO%oza0%q8HyqdyjJEA;GbXM(o z&_u9eEbG1Ia8gT%aLU)NUAcapD&*gqQV>YQ?^}v(ZVcDrSUFD=74z-v7a=Sp;ms9# zE8YB!Mnx6Ut4)Nc*IC<|d1wg%k4fB(^ZHqO*ziR0H~u?nSmAWV$>}2My1{9ifh|{U ze@msKrNxz%38O)>%Iyomaefo37{zd~Ni%3+_^h-ZGZhIHv(UQ=<%*lxc@+%9xXQMM`#+fM!Z}#a|FydnpNWHL+L+E zg+|r4Q9WvIZm#lMvnZdbbeXshAu$7h?PxIn$=q{y`mUVKaLDB&^yJm$Cp;BVcD(~a znS!5R)xdYJ4DcOa0q7jJDgEpsi&y|``wCu}5l{p@N%)nGpPx&|O3r%glPHqecO}C1 zP}A%C;8i9n5>QP3fX!uIbT<&6lyokF7I2q*-CFwzi-GhjCof*SZdY!AzT{H>Tj#CM zk^$b4*QiPzp(Q6LFAw=zh_Pn6ZKbV)-v zWdITyJp9}cT#Im`G6#4Do3^j8fjJJIv!%OR6~v(NOv<-#Oh`aNLc-$xlM5%$pGVtx zz$G7b*Rmt_UQW(=LH}Q~4ogK#HpYMF0FQiq9=J>g5dk*0_ZxzD?*CpK#g`A)Q1#$l z8;xgd{a2ewBNa2>^2%?3r91#DqRjkoJ|i=;E!DN*f($?;hfv$IkFP~UM&4tD6{(p| zzWYP%i~?^G8Ry9ilJ4$qd04x(5kuE65RQDjJfF3LL1awYDN$Kh3dDrL&WJYeC z3WTlO8cgTw!iP-nAChP6|IGrljjo&w=>Bkn%j4SrRQ|t7tLPHgBc-3CLBZ4Y zOKrrwsI?&L;t(K5cI?#Mw*Kex`J z$sNo88SMWrlRE{OC$I_%M@|!|>ySN}C1|VGUS9uMnP7qmGDNtHadCxMEDyH8-K_Oq z4i}npGP-(AChl?25~uD*&u{ntHYSY_k}Z=F`M!;i>+a=IE>x5E=xRiseIj_pK~ScM zUD2NV7QECUNrTPu*f^K2*aYdTzPvI-yixV9#-+bHlN*!EYgyLZq=SEu>`IJn1!mrt zu*J)=C~Jg3QMRd7tw!vxj?AA`La*l8igZS7HpHK?czZBbMP=ywvuNL;|Sn{&=6&+(oku6o2ZadY3Ox>1FYsJt|yg?m_TO4<6k zt*zEG%4av@2&Ue=z-6dvKi9(WT7>0&?mGD!nb=9_D2H0=?x%@l zsN0r-9pUZ8_(X2mP!@Kpp*wKLn-iZ`{ zTy@=-6>I7^nJ?J&BUra{>Hg=wPs-NVcQi2A5ZYE4%1!xj?$_P9E~k%V#HS)8g72h85 zX4FPtzyl}z=ly5xC)XxEGb_DPPxdyF>_~cy8TwUjwNXkFF@jrQ`&!d1z<+}x@LUMd zqtN(V4;>b1+ThY-TVqpPyPH?MTUK1bg?!l$s!H4R_eZ-SgVL}uIt+f4Po)@J0(=O=a)mFY$MoCy2=5gfC|b9}sL zL=jj(syb0S*&MU~`AJ3{Q>TnRGg)b@>b&GR{U0{@_dZm9yKi2qK2I5C^LV4uoH|%n zZQAZQNm;4ZwgJr}*of4x?WAIguWXSjoDRRQ!TFukUPI}}N&-IkRM_O>u;(<&rn4$G zxp8N)dU=s4N?C5n!6h!1M}JRh=vI^)isjU=6DPInm=G9A9`t*yvfI{yCW^$gX_{BM zB6#@T)6jm-y{vD{PP z?R6Mtc@>DINdD$w-|lQLu=+f->@o0+QqsV>dE=V2Pt_)Xah?^Im*v?VM!9_5Cx z?0G5(4YIrvOj6J~wf8{vbY(_2_Qw(Xt_M8KiP$7-{~*GbH|OoiwYd_)-VgN1RM}JO zN;mv4(fqy5-9LP+CL#_nVQ%cmAH!wmhrC?n8G3)OOGFNeS1qnlkOWNf$E?icUp;y9 zju~>=A%{mF`-5YUcaFyX3x4rKfU>sGO5A%FZTOFGIQL=d-x3?{gt8;4cVq|ZcFQ*z z-yZ2JUJwCJV7N_ zk>x2S%$aq9<>blSQzTf2Dll>PH*+e~{x{a2PjU@vFpnH~k}5$yt^{8bihwtbs+%G) z_b8VM6lk6lmNSbKyC1&*f%*|SA?HLD^WmKm zxi^D}-mINcy8g}EPo{oO^|zYlHs}+Yd#WmL+{-Z(I3{PXAl5dPwNtxM%9c=eTzBI} zVv`uMN>vp`52rt~OjI~6Sve$|9w$3V{3x_weDa%)S<_V3;v)n3RK-#8iDy~Fg9(f! zH?<0P6M~Hle!UbY72nME%2Qb}mTxN#7v@{?@UZsg_a==Co!rw)AR&A3R%^@UQDN(k zFX#5>QAl$Vx)6KqF}*5j*0D4AQwK|-XuOhm+f;A!wc~@A%(Nd2YqYgZE*cvOcWYcI zi)!u8&bmjz_(!EOSL^AIVb$^@cNflRZ)T9HFlvhlJC64QxEhLJLcWiw{0ggofeGa! z{Pppbvm}4|W;k~L9MY(KN3=h-PyWYK>Lr`Ti2#wPd`n$zYTi-TVWwBAy!wQu4;pZj z9|d28!I2ob0HV?Ri=QBrH>t4SA??!_HiC`7>jqV$@uhByc@=qr$u~4|obPu(hm)}j zWQ3fbyDy*hUS~Jb^zSN8|Dz%1y7_kSG_~%_5hR3{ZGr+2*x3$5dGKBA5Zhq%Bz-DR zWo0xW$z|#?4vL)NSVsyoPC4!ajlf@( z%CZUUhcou3G@wmUww+`~EkL2erJ$HOC-j z1S!~s(IO6?<#GSsT}Id{!B)7bwUsbFJ{~d}B(V8^no}+`wkhFuhHy3J$#F*%uGWww zk>eak@|$~7E(y9$UWTk3BV=2dgsjMfE?sgQ4@Hk%L>!N!XA!Snoeui9)&b`U)HeOk zC*JY8rr#Hu-fSD3Nqvqf3s9^wkH)zkE>QXnS%& z6G6I@_|a2=)vj|d%trF{hyP5ywIn4u!g?3B1qcG03p5e@@As(bjj?4CsjW!inNg`@ z3+|1XhVlwGU+1?(NB=u&HJdbTVKNsS5JEn<{>sqi3i3`=yU;Uf|`uYH$B$Y5w28plz9VF#_61BkvP-nxLnTW$t$NB@9v+-hE79>~39L zI&N!vA#Z)rnEPUD;wk1&9;XSB>8#Oo-6ygz80?D$XH8Br(FJoCp&51yh=h{-O*!D zbcVj`)hou(ni94IU8eo&sE}Ml$0hS;bMJWoYkl%V;|ob|FKr9?WEm-5xkel!l@c>` z(*|#H?(86Aa0cN(wl)%=wB6c~?zA+b1PP^n8EFv?;qaH>K2bw}mn-Ah$cNn|)Ni=979e!IU%*Wi>k-Lx?T zO|ZUJdl9Q^8qC07zo8M+QMcMc38z1=4^~P+QBhR;@lr}}lk0vWd&i9#i#`RHoT{FP zm^*x5yC>#Xb$`vVY}(rl=umpUY*38&t2IgL_x+`Yabw0usrMZ9ng7Q9OC2LIw;NSo7;-q4pq*Yd9{sh1;#g9ux|WTcdo_7m;!Z}%`;G)Wx4MQ)MZNB| zYUVR@-bL(U5h<2xCn8~0;V{BZn8nJ<>XUPSXD8Oq&JHHIF{tV{9Y%^K2@#~mdocyKZ0jn9BjiCMHcb;u(NiF(b%{I}@F3!}FBV!eRaYv-K@5>!RMz=PNIn6xWzT*_D zd~HNVWMlTR<)D~VQQb(V69wASDOiE+}$qzdw6VMD4{KcyHkll7bMwZv0yi+gZ! z@c6t!5Cc;Gmql6KRjzqR+3Kd-kEuIHpY@z?8mpgQTWS!tn$2Ts_Ol@F!Q}S~< zvfs{Iv|Oe7>96D{BDLi$Z>5mSk>w0?=V)`=4Hb{)71k6r7-#BS$s!OQS#n5Navxqg zEek3VP|uE@Bg@GVC_ZGTMg&rO(d?6^nzZ(eARKEA%RN^-6(i_&KgFd(RC!aH1BsT2 zb*8*UFXbgeQS=h!!#hXlLf9ARzGvTEQ%Wj)@-;Qse#>K1 z;#p^v1%6VfiGH)KqG_Vukh5pB)qF|wjn>^GzvR5)7?&4M{f-9b1Zp#A9dxlCu`Qu= zKU+&{`csk=AzK&80Ou4?z9C<9KGch1JuH@9wU8&fm}-in3g;czKQq038Sf*j#-`X7 zF|%yKJ5V24l>fz`K&M%=C^FmR0rs;=0&CkD0#2monFKxL%oHKd2PY>F+8!`w78ehQ z^7swDx(4MWO}B|D5d;oy-l3l*VqSTaaT;@1K=+2~4%b%MC>c zt*fq%B+lqnZeEUvYv+;{E@rr$${c5^5%%w^$FV0`oQ-uE z!EHz_O}5a~8~nw~an;70v1YJwl6`&;mDRjuN<|}SfkC99B+<+2^=`SJ~Z{D`JyohIu(iwe#{7SD&F5t3DOPzS=_uyt~9a1cFWLX(K= z^ZmXNNxqKm1RJg)u?tk| zm(HXHKf~9&iTSra{r<1!b@^^i)kpF#KZtqPne4U|AU-uqA*&|+Rq76BL+Qqf$1Q#0 zKWV*f%THK;N~eZHo3K3Qc`OE3i37$6%=;7f7MOm>boY~Sxy{PSj4Vq$T%j>)nrnIQ z?rYo{$H_klEXw*!BlbSD!ix2ARE^loKn~VK?-jCyWPGZoGp>cMp z**xQ5G2$LlczTEA|+wYENFe!ZV;obklHo92bm7|#UBR~2Woo0 z-E=zg=he4qt8WPoJN`dJ?%IF3V{%I)tJqdGlt}oNcEua3Z%NbmU37zwe9Iw;S(&+@uVpL6;;jpb}fvGx5H#1(qejn>bLz)9K}^7qzy&Yxr@& z_y)4oM>UGtaktC8if+HJp~F$+a`$WRQFZrM{qV>B&8~WPeD-=Gr|z6>?R?Mh-no~2 z5b$SQ>rV<~7>Wl;rxkUZb}=E$$ccAcq1eyo!p^+KwIe-}-FcyL-Nmu%J7)L`#|+xH zW0vV+=c9bQas{+pgl1sX08z9Tf~McBj(+BNUHpUNFw=<0F-yx+9~#R&&Uu~Jce%y< zC#}1!VTkqTTP^Z}P~O z`-!ebmCg8I=Dp4{WL(AHi-r4Ef?|dHij-b)e2e`VZE{1PPt8R#hDDlpX7Kgi3$3@3 zY2IxbTs_Twv)#(wyo!}%rsIrwf=r8(dK31XV4fttWbM;OnvqwZKb{k!a9a)Hqfot3 ztRs>C`v@0;F|WR}#C>eUc_cVbhF|@+-WwLXGNOw20@+c`$7f%?h%t!a9cT7(U@CJc z3{<|~ge*8|TyjL)jkBUgT-D3^_22MGDK;#@Qr);-|m~ ze~~A5q^#kNF^IfrF2g5lABCoU)9%q?->(eL=)kTpw9WdgmFb16e-bo|YBZu$BeOai zG;9Km^c>^>$;D&X?`~`=SmDkYYy~!r%Aa!_K3SShv_I*=YP$4>b$;{HetENH_!9{b z3RS4WtX<(ge*sUIB*`%wi~$X2MkOE%3Nsgson3wkw6PnM3_`2HsASidRS_wa7#XF)`gesNgV z;16M$(ZNiO#CZvFTw`2!4C=-5*BfJH`-SL{jx*4I5t{jT_wF-l!t_#s z)V%2ZnXyonByIKk(r+YS{_So5jvDop6V-tTW)m`cv?|>gXA6{ais)X-umhqvIW&4# z!1NSEZ09TeQ<|eWl&t{^B6cUnshq=8hqr=4uK8P^FC%)lE|I|S?yCnqA7f&3y7tW5 zE6%G`snP4FPHG74@%+3f^bl#2!1Ym*t$be?QK!EDo_lpBo+_x6pStnI`-X2VMcfPn z<9PKmL;hNUtUoQ??(<-`n7kK`EywNYtM=HBNvpjL!cG#8T*SIEG?Qrzi2Y9vbr60DNf@qGH>DvGyM&!5 zN%GWqqm{T8MFmTt(t<2pG;-Q)ETOpM@t`|SS1We>tHSq~wa>nKozwUR(8*2H;C)rE7$3)Ay2!(U{xhdeKr?Sr`T!j^BQ=u(m! z;KnJTHQnMQ;vbn{^J)Uc6FAB%@CsBU=fkL zvopjr`wuo!pVo&|WdnbF=*cM<72@>$eHTGq%_mIvUrJh?Pf=CA>`DG6>Au@^5c`p5 z{jH%P5`^Gg}<-|{hK36V*(Pko!Gf!Q@AwzytIMzzH9 zW5C4@@)%L-5wm%&2s!DVqbd+Wg`7W zQg`cZJuygd#_ro`*NG3JpK81C9$b~5_-G9nStYMd{W(}$r zqdsxO1RHWKa;f?VSFgZpT_OO~>kbSG3Vi8Z4UG)Zm}1P+pENmTh~9u)z~ysn0}^}@ zPqdTQn*V5mcRB{Sgh>+K=Nuz+^g9(=JsA!2p7%ij-O87FBU{RhcCa}gb7()d(6~Dj zd??N0`1AVPR{$#FFEp!7?>@x%>Yw#oz5NTM9*K!FZt80`(vP`4=H6HW^AK>HxbmzM z^6TRxf=orR%ehs#dX%AR;~(+#n;VI8>($*?Q7MN$PvnkZTvsWU5+SYC^!ypSKIkZr zZ^+cX5slp+2^%lFB z*d>!wgs&V2CmBzWa}o8K@Rewj(md;z_eww4zTpstmx|O{WPv1*$5kp^Uy}hw>kY>H zW1r~moWgON(BRIIn7@@KbbjCa`c^jw#q6bKGoT!pVIeXH%OAanhhtMX74;w^!P=Tp z<8FM~P1V$k(jDb!8{L5leb_+x?Yd3~> z;5$%pTBRs4#BY6h*^cq+R@-YR&d+fNUlp!MhJ)?Jzxkw(e*5#9=P#vv$r_!me#y3m z-9O)Uap5|28H3tUCo4kpth;S}@yJ)b01Psnc%)Y_^tzbBd8aG3XhN11m82Cl&fE0W zmWMfW!_#)LdE}U!*Gn8pWz}ocddN#Hl3;Ry_Wa1>!|AN{%yKNvH{3`N4dIn$aD! zEn^XqrX(YRxTilb0+l&b_*h#4qn+b<_-8d8n?^!O=ZIYm{TQWYVKBCJ*4UDW3}HXJW5|0`DP zOnW;x`o;t|wBwtM|PwV%?|EerH9w}6&txQef^Xmw*Yq0j7176pXMVG&Zwk(U_d_+_8 zdNqPu<_%&SI*;}YsH|@~mv0)9C7t|la3-0GJ8?EgoS0w;5^$1X>!yudPCo0s??bT} zX3}lS1-~Dr`51nwiW^}s@!3}_-bn(9|9A*9GZGI^jOm{A>w8US9ao)U#isyY+V0nARp6MtVhusi2 zsp!F4GoZ>PL{jXrD=T|PTq@K`FKdamQRK}axXd>Gzb4)!L?XajM1>pL&HF|~i+`UMTl_1**mvq~~q zQqd^1FE;3_n5e0=It#%u-9*s33lemC70CGl(oZo5B^4|<7`!(DiTUTU}#kb!h2#6h%XCc& z^O!KyvoxCtq(kNBrz&?#D9>f@8T=$mFKmQ7g%Pq`UtA#Rw1$#tNI9NO2w0 zFh3FYB?-C6k7mLFcU+#m)%PoF(I;@7h| z75Weu5!X{iskykg7NEe)By`t1`J%b%wFqyj^Z{(%mxHBoU`jfF&f3dNn61_U#cC@#57nhqiD_Aqw!KkhAbb-++wvjCO$SzE_4n_qP*$1g zLz-$z8CjFTR5%sgGf1w8B&lcwv};5R zU(qJV{UiXte-eG)co@mCAx2qmEU_5%@AwK z$)>Y*;v{I7R?SF`R|Q2ni-Aa}F?2cPcKw)&EgRu^;eodV=t-TB8KPB_%YZ!~@g2qwj(nAf5xusO~nai z{ir0*Qe@=`k;Ika&}T=`NfaOyB3{pk(U}XGWvEMbGhuZOX3d8_Eis`?K75%ShN$b! zhV45LAv1`O%@#ustJ{-&{z;!GiFz+j7yzC4iy*z&(Sw%`rtM6X{`9bg<7k71r0T7Q zt9Y;^#-spF|7T<~S<^5-WgGOpRHBTi*zP8@MJni^(1AFP@(S7=6|`UYUPvGxr7Fg= z!n6Ng>p?p{eCJ?D2AE@Y$5=uCy24?Bb9E4jL)kZhh$q`3!qPN81*j^_YQ)` zBS!kjmXzK(#P19m8XnOQrnl(x;9!+I^#5m+k$Ei0f#?s^$Y_oUE5c8VDqRVznYcL= zX66X1zd4{==6qp782Vrw4h7+ca_~ywZ5YYLQrt_61HJVJAsSf|tItd)qVvpivqPXv zTPS4rx2}@tm2J?b=0j24lP5F-;t?Ddeke4Ss@wa!sI02$Y32E(>-c|T@fR0HLUxua z;rFfdYL~-A97i}MC0W44TY*{#jFRQf*B7FSjVPHM(kxG)W8rK4Vu_@RxyAx*;VYDj zZvi=Bpu*YkGSu~w(a@kJscye!uPKRrc&T+L>mF8iqyQ?D?*eNCE$$z%6K0YL5{JXx z2&i|Sfl6z4uXWar|9wb_;E*S3yqN?oh@jMaby(N*3(zo{BPkits@gA5Q{0kd8gft9 z#J?@R*XyDxqGc~mjvDOjmjNxh1 z0iCt0_2rwd(0z0L<|LGc>MhDZF{$#+n=UOn7oy7vIu9tYLhg3C=(!c^_Me+qtW${d z2i=aMak)!g@G&;WhTEKtu=5%xs){ru*htN{hql&^C*bsj;vR&reEy;JLl4{{q3xS zy|f-bAt|<1@W7Vo;k+Ce5iOYyW$s_le4P$GLHv>cde|BqY7YfOcUiRnFJKL7v@^XI zM{ax?lgvGeXd4{*lsj;|2h>0~h#*~CTdQyef(LvmU%#IJy9$c7;gl=&N5iuSA_I6gb;ArYkubVMLopS)D+JvBve9YNNz|xa8YZ( z;n4N6^Tl%mODg??U0i&;c_qp~d-?L^@E_UWzrq?C8UjF`!;6;cKXuO=c5wf<K)14Pti4wV;2zyaMl z<#rDtLoV6&y#po&-H8(qpn%x|P#Dlg-W?PgJU)8^%#8@0Wafvvg~qjGY%BBgq{wiE z)RO{&oUp`0&vUPvs0mOAFtsRQA+H7mNCs>yg!QJ1;m<}rlMX!YV4IhblFxwkIqe2{yhn-9^ktEYP`~Z%S@lOB$B3~aaZ@UmnP5lR5&2;Ug5u+mGo{U zkCIabOAi9fF9;h#wSu^R+OV|omy@}l^pwz+UCVi?{oiwNW=v%;h8qL#Q)qU?48b( zrV_Q7GN6VR{zk3~X7ut45P#rZNIgE%u0O_l7*(#LLt55wJG4C1((>P5G=m~K)N0IO zaE8R{y@w%A^g4rzjUn07!$87;4xr`;XHpno_LC_v>|_D$`r`tJ0T67P_)^My48m~x zFjNF3b06z*`g%A7*z?^^;kn50KS54V`#jUZ_lB6j^sfWdf2z%rqjgYMjk*cqOSbiqbZ;qJLaAPGzW4(t3 zq9xG$pN*jc(n6XK9T9BCi1YH1k^@(C*oYv`jPW39-NmSPIbq@sl?UR~qeI{j1JSIj zXE1B}AOI=fgivZDdTsM?#L(WEQZG#2;WI@l9L4Le2kYcOgueHkK*#>@kqkVt$wM5x zVx3VJS$zx!qDihw{9mj_+o1=P!E)%te!+4Glsi*#)*he=^vkbHw2BO3nMDH`a-%{WP!~+Oc_@)0{b|x| z0GlTd8a7xxR6AJ`Y}Qc%mShaV!h0AKK*(<{0;)TlkqJ!DTAzSq`t(6yXCzG5066Gy zs82$$0=a;;4s8o|+6lSunXtTv8iaO}kVRm3l-us}AxeU`xjiM2bOIJT5pGg3)Eo0=yZ7!j$8pKWrKIT3?7>wC9YCai>x>tLt=UN+ zcl(aJlfq&2)2g|0vqN}%Plb$&b%IO+W#|Y0-DrMvMioJT90l-y3plm0Kx{DH^n2Vz z1Y@|UUqS>9FZW~MxBtBc-Clhs@7f;2mec9jRKuouj#|2Ji(D&<0OzXTDDLk09Ob)Z zz|F(c`dm7YmV?6;|5_&xrA)qi_fE225%`erySsroG@#d;&uE?t9_SEH-sE$S|NMU~ zJo>T0g5H?W(%QNtH#-cW=$3(j0~!hT+gXfA&uU7P#T~!>P4Xayo=Z$bP>ReJFyK%Q zbCaF$NX-<~Ele*iI@Vr=5*~xp>vMzUxEll5?!&NqDoOeHl`B^O`~VTL82>yzK5SrM zz>$3ixJ4&i>()q6{uI#TcAxGvKs^xTl}jf89*s^{jJ8`nf<+%X1luM5kNdxonaO}f z-nzd(@;=r1U$U^YEZUI)(kD20pUQTN!km1>;}E%aooPo1n$UVV|6f+XFGTNL(5v$k zq@bh}w0?gKX?k_-F#yXjV#?zPG8wdf1HE-2Pp3f6%S#yQ5irQ)jc8H|0hxP`sZ&ZULA(Zv)dnwZ53{|JBtLtyetTebKCR@gJJQQRXt@ zHop9ONYn4fn{0<57bu|ekN_3FO`~RVNcnC&$ zC~pslc<0Rv{_~~>J+$;3z-u&DN?~1q+&&B-&`1_EGJGhEI&e|B9{&S0(UIf>EI&k) z&?2`X@?WS!U5fjGqY^m^EeNF`2vl&QfsGlSkRQ0TjCR=gX(jTR9qzDPpx-PZ)lq`~ zydTV3%f_CHQ~1zb^TTX3^lNd4NCjE7zYE^?tdIK>*2)z8#C;L7~YQ% z_Z%uLDijU<S@;-^6Mbio{E`^z?Ux$Ff2n17#1eLjk*XNJIsO-fd>UHw!x0*ug z3NS6R*^g^vVHp`Gf#yemK)!`_{G_!-jrw`O0j&@`<7)*{u2-xwGBGVId=uxTp`k&r zHa2H(hlhV=Ga|f6aO-?b0cA3C+TBiLy^eqWfUl~~^z{cEk%%ohkZhQK%{&SMDqCHq zqGlxOL?Olrr5=br=K}Hv0g_q8n7J!eZ%z=>Du3Zfm&KkNfC-A^pFL!`-3IRpvMdSM?%3-}QV zsw{wZjHa~Eq+Ub>5!^rV<=0$ytV;%d=9uOgA?-fB!5?T?>#RmoXi#+~ySJ6;d@b=m zmZ9Cp*VNy7Qo^mf6D6t=mSZsLVG-2?Bh{(~Xt^5%R^Ni3xOM4mETF6ZUU*91V&c~#>PeV&!# zv-J-fgovL1{o4@%C;uS7Vq8`T0{R#pin%&!H z1JJonCnsJ*?nRxCaI0Uvn&L%6?Zsl_b@HU|+{=&C3q10ye zuJns1PtY)f-UNa`CM>+JuFhd+4z9%+9o>u?!;W?!IoQ`8ZwTyevnYz7in^?lr7@u{ zmVb(vgybtELOVz~f^f{2Y(B*hURP7=Kxy#ygQxLZjj)R~gVNk#6I+Op3vidm<8u5& zaosB{!|V4H=?C{CJ?nwcrw*A#*q%)TiMp-Ya~@5Jjl;$pyj>IAbK<;sTo$6>F5uHNgL zY4?GYdrDA{4uio|1d~!R;cldgntyzK1EtxzRL>oUJ9r3zxe3YBJVMz>N6*5Vl2ie0 z4jQCLypxc3534;^xyiNZ?g znsJlOff?t}fvVFU;J6$?06uY_OOB-Lw;i}Ubj!e5dMy_bLjA}sOV}ZBY9?5lK{Iz} z!F6Frypdk+!a)?FAHJJ%;82JBz~khU{I?B1cw(+ks(x_Q)U8u!E2a0){b6VIWw@*d zRvZ2T5ta5d^k` zPDR1LmjD1by?^*!2zu_Up+fJpP?)_Qb%gCcFsM($^)%=e2-D5sg28{iqL~B<7!tqS z&^2rfk^Z~6@(}XI`EYF!+}X7CZ@nL_P)jli=yI8Y8$8INn7yp_H>}nLnWx?6_pXIG z88jG4S(i6WN|gX&w51{n?*Zt<3rhV$e(TGSyaa8DHjl*4cRT`bBY zZsIkp;DlQz;3giUn2x{O+ag=LZ*2G&5V#8CKFTWp{{1_O^Wd7Hhajzyntdozj)ZBD z15@5^w2%!215_L`b>xOdmu?ckMhVf_3X~ofT#?=(NZ))kmn8|i;BZnJ9efDnfbi?y z=)F%l!+$bIf!;j0N>am>#7y}An7Aly-}U|ZDGU-fGeEd)hY<(_Y}iG!OS9ae;c)qy zsCy1#=mD7K!;)dNsfQJ~6zWZE;V9!%V`$?FEFWYJHTlFX*+Pj$XKf;Ls9m zxb>+G?6Wv%``20Lo#A}rCNoH~dALfdy*WCcvzd4eF3d0}chGhY1Q>!r;7$^GBcpL| z({liN!OJ8R7%%sn;l@GQ{!^(f4W!zf|Bq76J)G*pkK^BR9f?S-qcm-(CM-nmB=uXg zS|v$BtArz$b=^m(94qD4LTOHhxg18Qqrz}P<#P7^oc*5d+4k)B z$M4tk^z^6>-|Oe|{=7e*&+AVU`Y9ov2|3PK_Jdy@ zpU#}qD`vVe$6uYjE-4U4xHEF-xW$kyItEa5`KX0*L|X3Xk|Tfu^cz z0@k9N!<-@_BGM*va&lT;TPrTmtH7Fl$FS_)B*V;0k!hQz=pl3dG9v>#uO&KPys>D} zB2tI272*IP^Tfn$^Z3N*Xq9*}R;&yXk^#|)XlEbiww}XTQ#LiaiF^d?M}NW=gRQb- zk~=OE9c}ROf5Cs^>waN&>voT@M>1HOZwr9k=J(~qNcy5rzvqY(Lm!-W43CD_Tuspq zT`4U%i?B%U0tqz?P|zc6jURQ~7INjMJ)xbSfU04e)98iSf5Kc;r!P&%fak68c0bN! zbxqCNdU_9r#_?XSsniXz(%prDZ8c+t+$@zc)nW5vV0*dO?|l-@sHmvLor|H6gSEA_VQD(z^Jay`@dZcpUri4#llW@xZ}=H0)r5qx zC9+W=yYVT*VBCCeK#Hi4yYBf~?MQtivc7BE4jHq+rfVzM_si3#yLwGN&f{PcU0q$v z$Vd&2X>ox~|B$3yp8x$#&3qH-AJaxeqS7brWkbhFmT$BJ01wJbW%1%;5_MNY>ds{b zb3OF0@#u=Jn`6R~#FM|H1q0`JO1UmP^MKC}l~^RX1>bfFhHkkUOLxR?9UGnkxJZZ3e4!;2g(waE8d`CE?tT8KG&X>wgYTe-htUo@NR&CB7a(h)6 z%x}e%l0zS48I|y0C7!njkK#e5sWQM94bDM_{`d_2tV&VGcfYySwLe^yxN83ev#3z? z*9t!8VrE|xy}#DcuZa#IKX-05=|Z@erP0`tGDKevOtmV|h#(70(QsVc_g4s@Df(=BWVd#)MwUF_*aCX5lxg@oFAPR^MkBFz* zMZV1nKp~OX@Q5DUQR-JSYT0#qZ|t->=JkQ1FvG1bn;zd?DtsR7VX(K1I;~YTHF9`N z$QBNIeE+1*)cpGNSzElYJ#`e?)fTyo0|w)+1JjQfiL>E>bZZX0@M;OT2d|}=8^r4-iQHF&uTx@0-G$zv+i6S zC=n^p!d~}2*CruJUxMeL2(C&Qdns3_vCY*r8IT*jJvWQ0>2!5BpH0S9@WkP|LK9Fs z2Kz0=dDaPjDwmd{9j^`@Ryggy`V28UT(5e@OYMH_h9xx_W&}Z7UMmrpM|Jaao1?~> zck-gqjg@Aw=GC8jz&syMCbe*`0nJBPG7=zK()>bqZtP5WP@}VH^Xpq}{tIAn69a_D zuERG1_2XNQ+c#Hyg*iuHf<_*$Q*L8V3c6-=bT*$|p4c6pp~#HDyJ(Fms0+7>c#+Gz z#TjhhFF1&!1S>M7sHjNCue5NV%oe-*_wQdl$k&8ZB;e&W`{GYX%!c??A*;X&VB+H9 z7^VUq%oYI1LVqSB92Ldrt*$tj>W>J)W|76h;^g(hLa)_lR8&>HOROOGkhDX#Ty{@IKrrH49*vihyWrPS zz{I_H@i7Q26xs$gkWTTXT+xr=(J=7oiNw{|F}Rj{F#PP=5LpyDcgQh`9QPk+FIFVR z%ZmsruyFh>PMJ(wREgX}3lIEY9}{B@2_&E9D1}n?l}NY`B@t60>e%=!*rTzr5$r(! ziCZq0*W21M{AkSWe7^e(0A_&u9qWn}cOq7-MGn0K$xnA0OvMeRHT%)wr^|+&LOBK; z7d42XYSeXP-pG~IACWMuZUk$)V4DO}A)exyX+rR*XlmArob==OR4Ay}Ue36GulAI9BJXe@+N0sDDVo?1QEN|E7(+>717dV&_+U7}Y7beIu- zOWAUmp=LkXNr<3&N7i1UUW|?-FooSX5zC1&h{U&k--1eDXSVs%HdIef&*IkALY+j~ zh5F=h*(+Ha@stt7*#3zqR9i?`1ND86>@6%b0{`p3As+F#EIH$mjMD@YlTWDg+_2ym#c;_D4dYE}rHk zoyIw17^aX%hw57NBqwO3XT2alH@B9@?@hPScjNi5h?#kH@LJ?V0KlHQl2i3<*>YsT zRTWEqFpS!qhrY!`O6);@$Qxm^sPwDCL zmg9BYaN=6-LOx0c4ks1Jtr0vpga;hy2;sMlKMF5pR zqqDQRebeAA6tvs%lx?FO)*ejUNC-k8gJQQR_BD!u{({aBUp= zMt_^A_+QY)tJ}+>Ew-=RQS7IwHg5&C@SzF+L`LrW5RVXLXxtP^qo}`JOF66J?hmyo zX{@2lM4QfCT+yZ1iO?c%%^7#mOt3XiNR~k)Y zQ9wtAMMQXmQDufziDy&Sv8h;cEmpX$PfYUA0A*1o-b z<;tq2ylsT%j6#Tg(}JVN(EIPxaDk+GJeYjSsKy;!To#ZQE*K$T>>B@8VmNl=U52|N z#RC)zH`SNjihDrdUaGFCUS>Qo&qeUut?uq~E;h8*9*j@!e@#8eQ^fj&TBUimq>_@7 z!SOEf%tAuU4em5qEGsR29yJLP{A}hOZEY`v3eFXaz;4^K0N+lO zPQEW=%gWm5a7h~W@QJ$qjvGb*=^|*Rq8MuLgob5LP6NWos{#GW41vo0$TMlCAAuF* zEalRcX#|b`Oo~xRCCHbP*1QviUa(x<9g%s4adXrlj7aoiaXJyj6E@wK0_0;yOxW&| zJ3Z_?J>Np)HUu$GT1JM33YfT9V14@=bCgNUfgrm8<*MuTkKP~$kFkYwf&_OgNi#y8 z0zVYmXTVPaCP{wN@z?zh4r5iYc=R7*K4U$`5U$|?q+AKH*WY}3Ay(|!&l941(@d!h zqU#BjLBfr>?V-%j|DfWMdJ7P8D^Va2Z@HUT*0*b}@2@jNswIvo5jPPoQ&9O>^z^FE z>r|mpq7tuj*_mH_Aq5WS2GgYETCcyp&yuDbc;*s+>i-0oKM>2+uRgS-B5U9{gJ1tz LzuBR{-ZSA>`i6+4 literal 0 HcmV?d00001 diff --git a/figs/lora/MobileSAM-Ti (Adapter)_loss.png b/figs/lora/MobileSAM-Ti (Adapter)_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..2e8d1bf4c0ac75782f3ed9ab1bb2e2baf1743511 GIT binary patch literal 33939 zcmd?RXH*qi*ELv&oP!`DSy5Dif|7GkKm>^bB1lw_AW@P62Lu6;B%laLP{|p|@qi#f z6eMQ>i6TkKu~*^q_V{{?emj2stM9mXTs@~w)vjH8uQlgfb5*d$O(jY)CNcy;D6cDD z)j|+FEP`OhNr>PRq3RX?uzz^v zqR_=Fyc||8E)LG(moC};uLoSTce1?nm8YT$PD1LSeA^j8s7=xTFc~uG4-mv3xqekn z$30!{qDB7zXeDXqWc+9?eV@VVp zZ@Dc^CPza+LrY7m`98(Pl{g6g$?z6Hj=?8FmnpDU;h#_nd=B&g&QFLk{I#cr|NjSH zhbG9U;G!>RXW4CT&n79PXpDEI-smdv#V0l!$N4_$7dtsnVsnC?jL>y)=+^s<<=Wkg z;;z3*qFSdqz0cE=wa1Heq=es2y7^X|u*4vg?%Yk$DxRyMc=#NV|z-{)onky+PnzclnioIfa^R1*b z?QJcID*UTDQg3et_Gat7_c=UZ=yzKl3+2(vlg;k!^I%VOpCo<3B-Ccwu?{DhY-aaq zocsP>;6c}!=;-M2_BU4sitdr`Y%GR^u}bKcceKUu&>lPHcc<*3>8EFu?!PkAbEHde zrhf_`KT+Uw=ndm(o-DnUtqbEz$;im)s+HPb>kbMDiIP@VQE5mJbLRK>V=>|}`?hPn z)Nc4R7Z*`LKtQLJU~lbAhx9H5(NoNOrSMYE-JSJ?6piyVa49(1L~hww)`q}Bw-URT ze{wQM{8M{1slV-47$x2LFW#*Vf;08CI@3g+gr`G6!6wPL(4XJ@^6ZUd-PijcjcPoF z-o1Yx`R2{-Suc%D9r%~ZBL|<#*U|B}uiIx(ct%nGIh2`a(b(3O*Ua2JwS$^X zGDysMhC^7GAzbFbJzzupZC>ST3PiuqjF8Q9E=6TH->~A?Q(|hxY~9=f&von9Um)jGw`U(m2F z^>_o_GmJvk^opTO0YAKVDfRLUGmF{b34N}7w7}ap-1}(p*C=i!dRdMq3(|ih~&Ri`|Mjc-m$|QuP>ro1SX~V-L;U;QoB1= zB?bxNkI0bmnVDx02qvqRYqS5*e2V3S{@T`tH5CG^$_rkoB(n z1R-4Q$=Y`%MnNl@jpZNV7mX@=%9FUFNa3z&4z_+$t0qdGd@KH_0XF&SL?HX*NQLu2 z@dG*}8CE>fp7PPS&YR(VT+Hnf>*GVEb|El<@8OnKR#tFvLN@&7pDD5zn_rxK>hX8R zv_H?Ne0Qk|6RV`UTrtZgXg5^+Iloo2k-z_=rFwQ*{_T%MiHV8sOG{y}UCes3 zZp{ss!HFU6kBC?+*{MT;3SA{|6gjvf2J)pad9b5?q5hQ z{S0U5H#aw@Wo6~_8Y{YYgMw9@US@xd?uDXx=x(|9zNgEt9#U2b*Q@vMr>&g|!!3?f z&|xbqT45pimyLOnL+sT5c%L%y@se1vbuSsGQdmv_(F&T)LYYN)h1FlMNiivfvo*Cw zo#R>TfW^M@Rpf_q_cV758MVxwL#S~$=V-YjzI6%H-Me>>Ka_C&%^1cmb9Z9>-u?TN z=}F!%^Xx_|k0abqjMCNObteQyM@LiZUPFe2Bmj^Lo0PV zPpl(ZtTI=e-~+$%a?zZ-HB5wrgmCoM?vKNVC5zKvUVqQIeZ~Fp<2#1SJ(*E1k3ITS z(u(bWu)$$4N#gSI^42}ge`h;>LL!biaOh}hi8#pXwO)k+v48H?eN6L4eQxg0XIreNTp_`}Xwf)t})q zXy`yA33v55tcTR}*LV@WJfR0&ji&K;vp&MfV7a_gix=)%=$>~Wp{8!mx|RLm!q}JB zmlQH}bO%lirzixHh2PfK4?zx`TG}Q6?_iwUA`$M*sKY!${tgK3Gq%nW{ z_Xi5ig?1|OWf%H#dAxWFlgI+z4b7fCy$b-hwFRCcKj4%Psi_l?W&)Fvl0H^h)LUFm zvFXp-8h=WiGseciaDUy?%IZ9tzyLhZqhPGx*eA3@=8y%RwV0UL7W`CGN5?`aBM+wg zaBo&7m*`n%rv@75$WL5sOq6=eJ-S`xwtRQwQNIyGKdgEhHnzK8dGac7I9WytqN$k~ z1Ah%U8WbN)&CYUOzFgF>fBEv|h}W+zA59f-!=)s3bM$>7J)MH|Qd?WAah@KMYV@ZN zjd)eAk;lt5?$a@PIb&nBUXMdMzJ%W#yE)>1jGlg~oa&e+Awc-ce^x$G|C~=x`kdr* z5d3JN@77Qr6T3W{d0dN_euo0`>o{8@w$(`%Vy%P(IU)XNQ zAGRF}L|%xD4NV~@_!RfQB}(0y*cP@Oz&if?Y&y{p&{gv8-8&U>t0WnpvC{;z)6)}x zff<%>-!tw;Vd14q`CY8vN|jYq{IoK2XqbXq3K@}Li^eZs$N_c>Kq~boW`9!9 zGwjaisW)Dud8BI{HZu)bfc(&PUKM3!Su?YD#R@`a&l3Ld+GHi7WU}ZMoCB;<@Mu97 z9`s>7;Bvl)JiNUAtrmXi>F2&B$xuVqdBu0fpt!ZI4R#YA?Ag@u3Ru>~54wpuB>$=g z%%UJ>*uE|l9>Fap8&MF#+b zKY^!Qm6b(cFFn6hyM>pTnW+z0K!^LdmbP{wsMl-=)zibAZGsoVsvEZ0Cdeh(DB@9IAQdUz+A=Yla&+U0NO@i|HSR_ck_Ng&&41;2tENrw0XD)NaL zvji{+SYdUt3B@O-rYS28~fKVG6PU zJm4|8+PiW+j0IF2w3+4NC`PF{t1tugMr)raHs>C^8NpV=V6Fnp$yVgHS~5+!R5$?24k(DAXc zG1s?`RX~ebSEnI>1o93jXsD}_G`$;DC^9PQX+Z)1{H(iqBce|}S)3mpPK~NPk?n#r(|1ChC*x5;h}IlA z1E;gLfGB!2Mk3jNW5z_FUq_n$Km59raNm88DJ@Ux#X7Ne!sv1I8dyfesB>`5@QK@* z<4RgZ|HG#dRqLsDX8%6OzPMZD zz)1puL5vB`q;;OYvm)k|--IF!oY9G&NcKdNSttBlM-T3rKKv;s+(%^tGWX7fsSgjm zsmglvFTc7jm@MIO!EcDo5f{xR7R=ENPhQ423>Mu3loSY5T~(C=c%t#W-Svh3_rP>J z=UwK8N_a{ij#@kJiu`!;6c`>g>_dPLqE1u5(3Ef7_(Z2J=^GJ2gMu#0j>Lg-$M>AZ zuSUyB{n1%^euDr9aKUtEZ9Z1Q?IHkF+T+Iqwzk|YEG+cx>HyQAuo$$h#)(gX0=5JE z=oUG5Ga+IKfDEb?p-h_FYpl?`dGhz*$AM~(D*%eal-N870NLZ%N)(*3GUTy`YSooh~;{TNVA`XMM@7M@m1eS$RJoN9jEfjDS`XzCgtuYVdV z?Io`Gg3zL9j@=(*PdIT#FU4AG_-0bQ!<| z27&cXXu>8%K}sqILxe&50up3`Ta$)qFMQC=W&5Lc)Xc5$qr-o0=`{%!cwr-KF9FMM zEbzEe9&auUl-k7?f9Dhv8#McnXaA!XXspovj>J%i$SFouuBakY=`w%1WU$)fBRvPk z1eg$lt#F<-0|*8g{S&}mbZJjQGQE2DE*q>6l)#&;jMuNPwed8oaU4Yl!*^84hQ0S# zQp|?6ViE{|n}Un9cy9Vl|H2gr3!v ztBh24E9W7*LXE=O+R_cno!%{&BZQSnQf(f>zfc~Tuav)nSh@v5r2Y;Gf!z6oh_7IT zIJTmR2$A)kPeNyc1p;;sFCzScR@RZbDy15o`U*%XrjuL|lSId28#8SaZRO!WLpV^i zANLK9!B|OQum?Umqlf-B#SZT%?>SMTf2+^M^f*(ZnUlu^PGW}Gp69%O6sM2AS3O6F zdsg`19WN3GlFw20?Qut^!3aMd?_%An!hueHc|Q0x3;N+EQ7P$%y&ii74|u#+ogi^F zF#TeZMb2Lqe9;e|oKZR&03i-fN(s%#h8UQ+?h2tC`A!CH9OaY{tmal)_{^pyUY4ot z+kLn+;CLnF`|U?CVSVXZP9Li?81eaYiz$a;` zvwc2hCBIGa1zo#Sl4SLS+RM&BNAF$9YWzFi-#L*Qce-1r^*c&UPTwj-&`GYJG0Tz|jb4hcDV z_Uhlk?3CXGwKa!MK2qo!oLNKWc)6lH2!5eLf32WF;XZ^HWKc_w=l6S_kd{`2}tpQjOYTE-57? zA#znw(E!z@=w3!e<&G(3_sTsgy{{9#jUJ30B@pzIA2ok4ax=xdu`A3G|F5a-b*!^`n`dC zc|Q)}GNE~%6N~VB?YO*P7NM%~T$fLh_A0RYei{WDzyJ^|;FotWYx)Hy7{ugPPbMS3 z3BqvEFzA%RGZSE5qV6jk()(-L^8)r`)xl7;01S?YrC$U74+SVZDDIuSa^D1Y0HjWN z5cpRiz#mn{3QQHG>hNQ1&Nn67v#;{>e39scxoUj_uq%>NX%coOl;No033dXau^lc| z8=8wuNU+t6&;P%fXb>!@M=2Q@B*3XpKZHSMWSocW#%ecQnj5HRbY+S1EaHR$-j;ji zS6O8%vckpX)xEuuy2MsoRk-V?ps`1F3*~uFy!Z+8ZAH1f4wS=ZwoEMCW*@=)G4b_b zTzrMFIzkWVg4&hZA12|w+FgL){_+Zh@N>DC=^DI{%HJSW7%B`BoPOm?dS0qSe<>dV_tB>b?#GlP+tXE>aI>?g!jQIIG!?a;g!E=} zJ|Ma2!Zgv?E%A5TW-`^?i9Y%_`Y!NA1uVPMYoCihGvORTMQD1W-p{U@h5nw_$$2s~SO zxWp9WQqB7BXkzp>tm0)yMs+Cf8vxHz&O;|+TaCkyqr>PU(R)(Q_a0=!qiSbEtr?ur zWF845u2R&oQirBJReLHU!?12thoy7BNQTo-B9#>2BPajXNWqr#ETU(3>3@ZftJ%Q2}l1?yr9ER zrkIakv8l|x@riwE#AZtbF(xe;ncNc|9yP268l%@~3hx0*)2N3HITnjXgLJ$E%hU>Oxn9;;ULP6mVMSa+| z++MtLG^myZo&gh3#;Dwd!p?3AIUoOpSVuGTqwLYU;woYYRlYR!q%eSjrv(IPH8nLM z^h5FeS?>m9qb9V5?l^gsMfwRNNHL!vIXX->l4h!TdrR+ZE{8)oAbn`o7^E?JvVc!_ z!{1kY**inyn5Df2eoaZow2*Q2;hA&dpRc+m(<2_+LyDQ~yl1!>j1 zGg%IlHzM&zzx+WIc@LoyGG=Da43H)q$aFL(PUuZLvx_+q0GiHOt1y%scNfxwKiH?J!zXVCnHe9vQN{Kn4@4}?r2 zbXtKtl`Mw*R~BRcVD#}atH96eDALVqn#b^8xk9J-o)&l!L;usKPa}a5txG?6@IbgE zza1_9qLLjX9U<%gkh+S8td%;!zU>}yHbxr~s^7bCo7q=kw6;X=(k+dph0XO{W}SguC~{HGv4AYL?GUYOI``9F+N$)z!<&q3(Hi zzS#C(n%ai5I6|p=YkONRS63ld7aw8sTGIrrzw?(Dl*IoLm!~8B?W1>zHS}Py?b*Ry z>G;4fgqSG7MgXaVYCaBxZOC0T%*;0H(uIYE42u|(-HpYV(y|$-6E?vy4p4lX$48|1 zRuppd3mZ00Xa!mr6R(saz|xX6Khje`F!-49b`*o41P_$Xz`He`De6hms9O5x?_UAa z1_A^J)sOH0PiYbFp<@*Y+3x@iJ=%Y1cX~ETmO$=2IX12+QpQc>_!<%2zbNMWf1Io+ zQj2U0ivDk=tM$JPEFbo`Xyj-NEALM%KBwd&N!^ufBbI#|e{3+|4{I;*kC#_~jv@T_ z+UQl^TyFYhSLM2x+Yy>@s6%(jBGu%AS4e zjbSh8$^0T;J(6Am+i|LW-Ui9^G}FhM(vwh6IOxrY*2r$F9i6KE_@C@_52u#uZ}1>- zBW5`0_H9bpFZM!5w8n{zNcK&HV!K5tKrklk>53C&u!7LE_aseC_^~E|C@FugAJHGAFHkh> zZaLzB>!BD8t~O!c79Rdvl|&jW7{L}E6fFo5KnOe4Z}f4JzF=v?2*;y@mAE7^abmT8cC929c$oe0$8(mDe;c!1~9-d;9d+tEmp>rA$)HD-JX*`c)h~mNS zvUpr80GkU^X`4R_x!mjJ1-NuuSLn&0@Sdv0f5Zl}fID7J(Y%nt`&HxG<=a>EG|Uha zubmU;%Rb)bLKAgH3OXwR@|q7hgCx^bd9pF*k0}XF7w1L(=i;=EXVflg<6?Rd`qp)K ziT`9@+FXOXzL`a^3Ll&(Q2n>$$RO!ODkAra?%y04y)#J9e?%4&KHb9`FS_>y%8*oG z>xzwK{QaY@u5AO{0!ko+q;$`Bme}aoj#l+Ds-*g$TFgnz;CG?I6xjH)|by&3!|k+HEs06iO< zn>jf-KUwdh_6;b;#ELqerec?V0aXW6@F#)PgH6?{m_DD4s3j~^hV5z0IkSH!SNP2V#0+r)tsOn)+dsu$$ zj&;BWC|aMB<({Gz7zDSs{kPC-*Ok|(k{*sEMkH4o=@7I)bKC11)CoVEJFpnx5M`!9hJ~cIE0#y?f3RHXi z<+c-Xo>8^1w7kI%C7Jd9>n-+;IT~?zr3ZGO|1qOa^W>v`etyYXnGuPJ4;`hXq*za# zI;Fb|jND9nHw!8cW`H18Ha0$&{qQCi5h+s{h9VNU0OFHAX-<{ize$nB#8FiN>!m5a{x_ki!ffPVI7aA;_W6*V$kW}jc|bcc@xf zja7>=@aiK-toS2tC~{%JWQckj^{ZU@z_Akq-VYs5FlySL785%Ol|}>$Rct)|aJFau z{>l^i$;k*9n~Dc|zJIt*m!Z?A5OQVS!u41X$dcew>q1&kJ@=jG^f3(Vi%8Ty0m$Mh z6sBC3$5^3y9wqE~=G-}jOkU=k-@|2>bo1|qg2~CeFXy&Ze;zAZ=Z5vAY$t3##!IM- zpf))KC5fO>vVqcGX#YbRk-d7=|C-N#ECYI%`X5djh*uDBJ)l!QYlEcre-c79z4f*w zWD6)I$NBRlU@OER)qmP}w${4UX-}RE7P9F>{b0gUW8h^)vt@P}FxKpb641%bAzQ{s zk8Q2~6!6~LQn6tJX4m2sxH5?#me)km<1P2?!$O-?N0tXS-pdDUj8pR94 z|HQ*8CFC^>9#))Toi{67?W&0h2QuE7qQEHSS^P;{`QDVPlVXVtawFXDN-RM%g}&@Dgs%xvG2rJy6`cJgh4>1EQc;IlDmnAgqfEiqSaV?(_P~sw4g8jNy8LM1b!d{YwW#Ehz5UzgM&ks>ExN_BK<3t4vQDG9pAoF zI=gUqb%azEesTzp8Zw9s%4WRxcT5it_E8HXG7bSB5O)ANt&Kv?ekN4Pwhd;z zvsS?cwcQkPTtum2PXx>^*aNjPdaZ2-3NWZcBTmGg>c~JT0K#A3qlmVa2VV{4gl3Tt zmw#E*&6WC8iS6LAJEgXn4*Q>|WpZu>W>_yV9Vff-;v#!&7G%wr-JcoI$oEdyISBX! zI0PC%iNqkNhX+dc1Tk5^*mo>sDthpEb7mMtBV8&qZ)K-e% z{0PaxDEo}vdkX`};F12hOjuOy*40EBTt_5`GI-PmF$ULWvE48OL{n7kG&4I5N*3yi zs)ibP*l*4l&evw0LWy{NE$095PvK(%2M~m|{&vXF($H`ikyB7KqSg?YMzIghI+5rl z#0hz86XlQelPnQesH&70K-|?WPkjBFI8;$P`=;jflrY}hlJ0t0Y?UJ0e~kvS0+w5hU-+5>jr_dJG*q&=H8YXzdVU7GMih z2qSilY=aGM!s>a@xXceWRAAHopTw1pw@%L$%QLIvIj*CE z3p`tTBwan}?(`y95~B|epv*DQ&zbeounjlBgcy0LfbNlTs_+8B2ChrRMPRc=Z&QA? zrrJI)CMu{j!lnAYI+6(1o6`_|Z~yQF1r|-7%c5NT_fABQNBy`s8JWmO^S$TGsc_P0 z9(?VuB*|-@|6pI}XuNET7$9cR`)w}02Vb_V`ma4HUhq?3Q`gb|Nh^z%u;1-`M$TUe z8xwlWdzKlAc$4nd@-x_$9W#Z>3-5Ls;LCgGEJe8xwmNGn>|nrIDi2B8f0J+#kf!4l zSHzudPs6X=1jtm6CX$l}RT@_zm?NPxfbxC^RIK){K{U^=aM z(C62df3PoC|Gk;ou?x2L$=&6;W{^Y+!G8z>>tv)7o8#7k^9Rs>N|^a1ZdYPH1k_2y zZCBv)2g)naq^I9fAso@U#ZI1~dErzdOG_xKFRXw1pCXD_t_y(By$U{HYW6297}PO? zRyx6rI8fzw>0s|~hq%YvFb*Qwn))+pABPl1Fe9mNF71tsBBBli1%|Y9zrF?No~OT; z(kGW556YotF9-zdo>E}HvY_pWRyhl$=7@+0lXYoQQqtpoMxZOCF6N3s4O&G-g_kWI zYN}RH_)iVjHZ**ZkiZfWzN71{W@M+>9`Z*f7LDM=^pV5+YjkNTKCYf&#?4Ul*Iz_rb5SWbg> zet?b3ab;ZY$P8_J5%oC+p(U=8Eozfyu; zk)`sVVV>T^w%WsHo#$>SAys1xh%-wKQNgo#*$+lINh(0v8F$OCB2cwElV@BPhW3>3 z@lm4i1MtX)P&T-h2KN`mD6}aD@|!u>He-cs7+_3+;JZx$>Z278CWafBV+hz@2@%x# zG4nN^7})|FX^IZ_D#U`@CqofSo3}i!>f(#G;N##8zE>iI{9!|%NXs)?gwK0 zq_PpOiRz_aor@C{M218Od2874kEHKjVqon>ZgefzGY^t3K|*Y@)8fO)d*$xxSuAkT zP#$r3RMemtyd!*#8yRgcYp|=O$|1B`V3KvDQ!eyHb@~ixP|G8#-d5dLkA{fQHKKy# zJiTB-&76+?2he5Eu_Q*xK391m`ugD*a+(dJm&l2qoU2s-AXNzaZ;pL28qa>e!IOTS zNdELXwl0u4Znr|O+=vWIB}!c`Yj1qY>WFstQlmq*q-$B_7RDlBQq%(*Lf9k4KSLUw z-P@<;;)v6-`VKK+dH7bACmD~NCs`+8PR|f!n=#{c2Xnh3`1fWp{lW;78r~YwFqz!n z^P*@DZh{9`%0`$j66+xRb%&JWbS9MvUqAAS+EF+k!z&<9<8L4(1Lf{p;V@R)Q39un zQYSp$lS+#VHQ;eSL@-EQ7ZR)eg%<-zD8MDH1NL!#Es z9z*b=Ly;K+oL5-&wjb#g-Bo}^k|V$9ToRi(hqg{ z@Jv!zF&Zfv1J$Uq6Z9KBv-qR0>8g~9bT^H!5aY5G5e%+F0YBMQ!Z-E8IMXXE?ck9( zMLQS<_ZqDdP+SlG2^VQj>Lwz(>SF6+vXzXYTuznxfh%|<1&o4d_J>+XW?UlC>Au5- zk@oZ$7g z)w5R!z|KoN?ME2?lKTsX2(Dw4)&{YYfJ5?Zcqfmf#kWUiv5n)8BS=s=-hpgnZU}fu zS=SL2$l4>gBkPkMfGzDMA^j_x{p*tSuNWs%G*cc$=#_-eFJRR;Hu5k%SxrKrM}--k zepsGam9(-^*Kwa4vypaYSsfwKm`}}^Sxm_O_d!O)Mm!ZQ#g(MHx)QfYcRk?HZA<$w z!i(c!4m&lG&x~NpZ`KQWs>&!u`_lF!Lk7eJ&+gg5JZEgcnN@g;4W;=>{jfs2+AAIG zH%}Sc`-8dg3b9+`Pk~%(bbI5KmQf`$GJ4Rnz%-7cH#Z!9v+9*mkdLio*pF{U#K;=e zX7gj8aC!F#%TRW03hcTUgTwa5KYH>QmiAeqG`t#GW@@AQ>SguQWi3dkL9pJWgo%7z zpOC?(ULxGXBZh*vu-`n6L2*yB#_^3B$N$B$6SR_{GB-E3#-^qTkg*u-z+xQ; z+S6$e0;Hv-=YW&BE{)8u^y{8zjJl%x`+CETg>V=MHiMydU`O6Z@{uKT{3mu~E9RB& zg-&Kov*JrZgHnX;|8$L?plgg)K0@mYGvD5ZG&Wv^=A%>6(rn;O&>dbLtvZ4B@c~wI z+*&b3jelsJ3R;mq2&28QuXu0!LA#0hk6N#lwY95Ho=84_{``Y5wd4vyL4@zUzpjUx zAfW1nwt_^wdZjy?Us2kKi@sxljaTtKxyo9wdbo4?U1r&6_c6WrIyBhxE{q1RSz%`m_d6L+U91N{zMC!q2E!pFbR}oB-F{RcmW*5N8OGmiG2WD2k!(tao6G zK}DM}?_E23dU}0=>w3Lu!4-MN78%sS>hs~VOV6Wu52`A-2$w*XjyivKIv^9T3Ge4JM1RYOA-@SB5C)pM??+So&95&m|>8Wgol{u4`Xh%2P(;CtlCv zuZ%`}&(Ry~UtkxSzVpQZ+r=asuY^oj5v`Q{?bp;dYqX!e{1!FazGHIS;fiQ)R|Qi( z0r8!_EOIKU&)~uSXBe@5?D$J}?_-uQn4atQ{*VJaCxR}y^D!#WA4m@Z*HdU#=$+-e zcu~>Ov2e}f=FKq5{GLMSM?M{=>r)w+yC*r~Fs++1D&e~bmVjj@TvTfFV(K(~N=BoV z^A>1$S@z)NaA5#y3SA~Ca3g4YTp_dpqss|ITn^|=gW4ZC11~l|KYxvufkC&UChu`c zF5&r`6P{MKyQA{oY!LO-`!{^*oU%KErBFu2acLzTbySpR-QL1;Dd0c*PL}ziBQZ37hEv-rayKGgDI#1)M5)~ zW9Fiv91OkE@`w&u`JjH3M#ANbynGos8Ux-hgRcZ}V%^-xK5r^%M?Q(t#(}?J2Od*| z;f>o!&+rrD$2Q2Tx^ND}QzASi+uo03TxXtSeBY=&KIEl6!^%}KK|6U+U@hutIkBaA zhgJL1*E~H4a6DyXk=ma0KX6oP$vUy*w~ae^@(#X1YsL+-p~xa5 zdgg&~g)ISXy3a82<4+3j6*KZBkR<{@u>H+&w6zePzPOWR2Z_n{ZtFeHpmNLy=PS-) z$*(YM2<{OU2KwRhkoPhO7}H#bg$!*WSYfIYT~X zAk;`%UYba37%zmCz3mX)L6Yo2rq6<*fOcpogc0C#+}7Ye?y8P|Vt&yWpCp=4R_q0G zf95!+R5~LWVdsT4s5R!HPWBTpw&Tjp^}T;dlB-^XJ*7+cxkK#Pplx$8y%iOx&Hz%> z@fF0g`sslSC>bE6XtAM87Tsqa%=s;8Bqd!S2UG2Z zPh#Tzqx#XzcStNpuGRh~xY7VCVk3q(ck#UtHvJ(F20u2OeM%ezpO^SVvRSw#A#8~# zS>E3MbHgW;CU*-l*WYX-y!aeAEq)jYH98VTWT6T~Wiq*>+WTz#9bn z|0X{Sp(GK{_2cre$_T;aFU-o+tuH%1(~jr(VNx*o9JfrCJ#Zy?`1+nQdOzeI@6tL< zqwm}J_e-cy;?<~aX0K~Hpr(I?eL{!@;22fgq{TO}s9?pt)5Rnr*ywmd{x`Od4N-*L z0JpT&<4HE~O$ggXVzO_GR}m2zc+uIXf%!*L)~Xkj>W>(SG#=xx^rtGBvW4~j3m#2W z3p~JFz>2Fyc+Uo*6duhlDAmUO>Jq}P3*v2!MrOD4GZ+5p_|npHA?v%>x768pjpuRw zqm+DmqdX%^XPj8!cV%j zW1;M@PI8Cqcq+r2FKL)`5-Q5jQ=^s&3sGT?RDS9ICGmTjlWuAA=?MEtm ze=y-uEGUA*BD&O~v3)&v8euM?#l&BXbW#C zIqq()qjBNu`#YsM_&MwJ1$Xur@ecN@J=XG#v@dOWluuaeo^pNrHDkkj@+7XW?=ZOS zCVs1%toqwo)p@J*F{_5)?7`k**fnCN$Okf1lq^A-=p?Ou7#v3J@dmy6SdisViq9)^ z3<&nt?#t}z@YXZY5_ox&R3@4V7YLov9I;#fr7U3p9AHj@M6yL~>J(T)A#&`It4plh zqvsFrcxB>4!8ZhZuF`Y}f3g?di>2;kNWouQqP&qT$zr40+t?>HwBX^gi+9%@%KI%^ zE>FBsd2pKm2aPCDCIGSe_Ov#IX2o(x`F(Fn=&+FX7>>NpzM=tC1 zNYkH7_~L?*c}>@d-UpX4yzyun@a7)w};-9wv7(Os-EobX1YEu||G zGIf_s5!5(=M&|FicnA_$P8AW9Q4irPyX_l(DIYRabCbQ2A`NhrW!>7K?U1gcY6TH!($2-lf-AYUNa=; zg*?LgaYW}H+WhFTj&D{Gd1@=_(sbHCvD)D8x)0K+@>v@P#i=tf54YxE1^B3k>TEn) zK#YdJK9SP;IA!Dwehp%DIaw}{B;(^ja_GM%Fn8raMN13cGvGxvcXsdC{Vm&2>m2D| z1r>a2VnAqqrK#w7BY2WRR+G7e$OHNa!W5&|93+gZ z*YPH0_9)@Gep{lVtq?``!`KBABfc~&M)po%bA9l%@lGkFzphK}+*E17q#;fX(3f92}3c}&F5fi?My3TzeflO8R6!WDi|7?UatGDVfAd+=cdmv!-^e6uK zlJm?*Z({x1hk9c{cTF{Tj*BKF1Vm*Hwp`?uQ<=|Xixi^3TE`sY+xr@?IwefW`w*F@ zndCDHp%Mv4*lJ#ePKs|0X)Ll{Z+&@CD-$gDHYjW6e>(MQQ#2QTI1+R&(H#%^WpE(6 z7g~0(fgvyLn{hr%G&?5e`t?^o^@oh(R>qGI+sWK(PhQg? zGhaC-x9h&{k4(&_WUe~5mC(Ns#T&>Rc{nR9mq@bGBWurpy?bKHfa;v(bsrgrV9;35 zd-?x{o`DYot?Q3HUY3%CcllWTsFga+$thsg7_{R#c=2n;D2vMrF3h$NiFt6U%oESz z;@Z-@v=0VYDzzqOdEn)Wu{udo0G0?w4EAKIx@?kB+-}_FQQYDjUz02#I<1$B#oijlYy2 zhta=yaTC8H@y-k8?HSL&#W;honGG^aDOtT+w)`rby&!}UV7h`tXd{JS)NkIrf!9Hh zP*IhZv$3$e0J8$vG;iFVFY1ZT$m zG_u>fCtC)12AavuDZIAt9lXwGk<{Jqlp+#GGoD$$ZzPX#V2XwH@}E8f|9SL%6Ys;N zpF`I&cp&kBg`)2>`QNrFbX@G0Fm@Bd{=zpgoBwVtWh1svjx437`-&D(jW;c9m&e^A z0og@jh@fxFJAA{ID!1}?lCqizHmJcul+e}=Fhpv_CCF5l|E?7=_1aQ|MISK#W}69= z&iyJ;x(U4tn#X~yKmFhLz-ZljyxNg9b^8z6U6Ai z6xF2!H5)bKG~QrlHm59dY--7bXYpG0x^rI>$#~w!nXgv`a`A1&GaEY;o)W*0!ytQ7 zBQ(FVKhArAznyv8b1Ek6bfx$_i6~F===_MzT-V5xx9f;eNclnj6GYPcEHj~}$CDql zn)^`R`9JTAxUmRQMksWAh6qD53;MQ|9xmujhn9Nq{>BWU@9cucORL_jnBLi5>ynk# z)xs^S7HcQAf_kflLY^5W52-t1W(1w9n1bJH;c;q*uS5!cY}~DM7<23QcyS*>^#nCS0Xyv7*xAM{wLH>koJYLG`09= zXb(QPTNHq7ufltX-a`+b)>|2Nc6KyCpk)j&M|Ec1;z2v;5iB@h`VZsB{j9Ypc2o^6 zf^lG_FO&9rV|HIhrt=C|#63ATVwNULTo3FYi3jX(tgb)iOY^g%Wf=F3-}XH7gSWtg ztfKbh3ob}|_$IgU(j@1v(mdpMI9c{*yM9mJ5GVk+Fs`^*8Kd9EEr0C5A9h&!NK)F=gE8yhr%s{2!u$nfnRc^w7BhLdwCX61#q8EX0?@`j1Q;#%INUNw_iwF zoQPH?{M72v_`Ob8O)}NzV@(s%;@8OlGy|rvWGqLqhvvU2_~bi%xCw96f;ZsaFkBm)*m`0(51e*O8n=jrqC zPVmCvP@uaW)6CQAVk_3E;1BIjP!3?kYNM+g`0Lw*Q+M6F*se#|z^IQJW`gx<(`&ub z7bjvUl~#WS3);T*OdqYIIVMEEXY9*;HW!mqPhz4@Y>Qyp`Vetzf&$@%Lza|V*orw9 zr+H1Myqw<@<5}rhsSix@@LfnJNle0>F#PX=Pk^)qjp&U)6TZ#Ih6pgKZOvPVgHQRl zr1^gIg?FE!Ymxgb5w6V@^P`35RUg|riE|H0vKYc3VL~}R&AhGL)*fh(Df^>T=+fb& zvJ*EU=PT^IMJj=+M@>0Z=RFcX?0cYcgLQJLTE2I4Rld|aFfn4cUFcU&eEBqeQ|**> z4kMDO1wP9V!eq`kPcd7*SpBrK0^1B>F^QLKFFCH|>y^t?Kl*n2Z+B5K<66ueLjE|z zDtD(_XHDAZ)Ct5qn5vqZ6e*=c z1*N-7LJ(Xw{&+Hba!`my};c2{J)v`=9~Fu&CFVJmTR4LTu0gOe)oQ!=f3Xy z`d#bIZ$~?aX?T7qU)W3t7ZcObz!EASf|>iGa)Z}KS@m-6%h=PwTV*RGb^MYGWCri$ z6$N8QY*ZS&%|VI}Z+gPNOgiWmm`!L_9iTLYQhxv$0} z)Ld?7-t_X?wx;N+xeTw_#K|BgEtjxu91YY6M4nhHm^gBt;{7X4H9~qLao&RCOvA8Q>^@j0V9!5 z>ysb1@Fbr4xR;~a4D{1&?PG;~=G+{t*p|j3ZnJUUO3QEh=8}0dY9HZuMM8^tNbYSX z%U(;|8LP|cTeYER2QokMv%qK4!jj2PjCZ@$*Q#9_jr|@ZZJ|UduD?I63}k3i&@uJQ z%Fs-qlH{!1vbA|ZCGnaZmE3@aw%L@M6?g6!TZF8TUI0B_@s)EhACujmB=S?O^KRX@ z8)5#I`B|}DGDsB~r>Q*+vnQD z9jismy~p^N`CNa^ASC+@ssE)2JrxI=v)W8>wT1+}3OXl|ba6dXU8{8<%ky+^_@-1! zUgMIlbMNXh`TLh%1Kt$E!C5L8_;2WQFxJ!?Sq&57uO!4D&pebTC)BT5CdGi>R-M-6 z=b$f|*>hunx&^L1Fq6=`yFl)EdGgmAy|Ac-FNuk2 zArbKbJ-K@=sUA5^B}!FOIU0`UGs49)eZ$5wfrJ`X=ZhImC-5({nKp}U=hhHb&U7Vla*zh>i zLYuGN9BUg)e&K_gSeASpWjNe{@ARR9^JJdd?)}xJYb$?&{^Z8i2sJ9b@o0gZ;Y+!gNGO>xH3_pHo+Ix;F`-FU*xgjsKh&`wR*QnJ+rpK z`w-RxCUcC(==$E&AQrSJukv#AXP4Lgcso3`DnB{P2p^B3dNKlG3bH%p(pRVrG+p(J zlCMX;tG8xBUQjqt;^Eu0R(fv+(r^*f#Y>AYmMGD~7Ex6`AE6ZHd5e*VUBPc+Ra$tu zKK-T3^27nkM`$vkZUR4+&|B=FEZj7)N8|RN-*lNhX^zXvQ?hP$!DYHkJbiN7r1YV2 z?=PamSD#gPik=)k>?C*Mn*Wk%OF*yjb8wtm_s5m}9zGZz@cUiM``ieF7DNih`Ii_K zyv{j%a;81aq<2d$)um1d7hTV;9`jr=vwA=0G!$`YTyp z4$(N|oFs!9m3b=kKyvB^me~0gSa?{5O2t>V>Za%JMC;-{)x>N$NN}pwK4v_A zi9x8Bdx~<|>QSotF-?K`Qf}~Ut2dop`FRokLCdEzo!Q>W^9H14TViQgoQzkSR2yp0 z`CKAL-Ed$kw8s&OLTB1_O-hHtt-c$$+v|mAD|dMN`~@AKjqh<8o(q7sgqxN(KPAz_ z0n3PXX@WiXz-uNPm)FF!OTG#MS8V3wH50A{UkncPI^{@fuwOT0z>%V(iE}gWzSTs< z=>AqR%Zi%OXq70n&WyB6WQ>kPDM6>T_}1*TyBd$!a6k_Fkou!>(Y<8H{z=|kn#Qtl z-ET)~ltKuzg_swCil;5{Z)bz-5>M42Rqf7KtJm&6<|oh{Tt!`ytg9UDz(fT)SqCO? z5jCaX=g0{Z@5VnYlFkn!kG8(1e(E=5$UB|GvG9S`#H5iuWDClTqrD(~Gtj7Nd3LwG zvj~Ouspod$DPMn6DH9|ZD;vNdrwS2t?RU$Ov6ac^T)}t=Q<=Nf(Wn>!Qdcu|kOaPz zCy9*kZ_6ycyuZ}{i_WNtRnV6`z_Z}tOj=kg z)_wc6Rh%N`0PS6si_neUo}bmPh6FJmYV0@)cn3q9WwFOTJWrrOAxl=|{H3OeacqQv za!PWOp%j@l$D67myqy8qYy6n?7dp>G$JR3YqY~Bp`A$C-y!7VjvH#PXT>DvB4n8%} zp_%IP8Rzr4`oCi2Wa3;#_3ss^y!txr7i~TLvPS>p)`D4OhhFQsg1}c*_U)B3nyjl{ z3|u*sc?3aC_~wKru=co=$?eMj?)UKAy3&!l*Z@Zat}!yBVkNkcbCKK|=jYq{un%I{ z3nVp(zje$w9C1svtM%`Hj!k^m8vB@G+Af(L&+|l^b5E8G3cv4epTE&^oNcX7XF2{L znHz1?`XUg6+%0kC>cW;uM%WWUMFUw*R#uq=UZQZjEY!vEI#;&1!zECB1&$u?odzhv zK&pS3@`rjoszEbHPZQL4wxD>?rd^-r<;nEB2GLo;3%{Biy+OWw>b)eNvfd|u3BDpJ z^%$;i4QP;9iD;y&#)2fZLDjD4!E z{ZK>n(KqS*QawrLZy7f#cn|A?vV@*G(_gv}$j@mHQ>B3c4tF??%^K_6>MLDH@uFQL zf6sUNclg#I0jz}A&hM3I1@MMFcZaqg>r;&|U}*k^k_=zv zz}j+_Ju?9w!;d-^KA=g`yfHsec)Hp1f!?&o?xUxU>S5s8L%U(+<UuFD!fm|WU^hQ*jA*l_&*bUQO`|W*6 z`^SBmkx-i7P~|`qUtb7rdrbEi+7ZiuNw)<`re&cMf843p#X$qk>>zP~VVyt*H#nIq$5bop)48QagGH;#alCja$A!H)odYlpC6Ga)D3y}_4ui_Z{@)umoJMLUSVgm z_FU5B%XAjvxTqKhQ>sd@C5_@+f5BdN|CI4EYHMUlnJ?E*aJhrO3_4iK}Vrc$9{iBjxjmC`S$H~QN4j8JP^zdR#iX`tZ-F6Ved_}rbh z)dR(+?aAJLm4U(*$seV@)}1nbyX&w_``3#Fla-OiBM+%090B~o>8|S(ikKkqNVjUE zlVtcBmFU$|b{KttTZ4AgyMt;ck&x@n)G(Xg^mS+S+qUz-_2A>iky{*lYhXa>LnJ3_ z<)!3`((?R^}Bk$zu z9P>@E-aTAQZAkBNd}P3^`{UF$d)Q{_%HKS@Bug3fO(N?_uX8M<+eSw_B&>PAddm=R z(A+n$P0wkaP+3@oOA7B~DSc!?Q%&vpu|HwEZr0m`5391T zE0JSM>f*`t8d`Gbz9*_>pWv+uAKw%#FwNq08&^)_v2DSf`M{Pi(v-MlO+ zn{581Bl$rinu3A&$ym09ceLD=7Ac3rOh zBJtvE^%DAtu-*tU`go8)w?-8OTM3bj?XH}sA*|7n26t*;X+>!yiOD;54t&`;Tq1Y5 zlgfmR5mT{{??4$OSkWuoMYa()!cjsl$B8lZ+y8g5+nsJGK*FQc{9o1xi;}CIlKA3Q z2FSDcS8boM{^1g1dj5R#ZWd!YAIZvUD3Pp8?CI&^3kz17vkZ3V6+%62wvzK8Je?-jHZ-HH&M8K!Z;JBjCTP-uZXs{Q z#w04==qu>fwC}xhZ?XO%Jn+)W;TWCSqW)6Vs&VJV&RA{hEzN;gD1K)BjQ!YFWP>n^ zcKzk1t?T}F4M+X2Nr8YZq3r^)%nI*LuSp|$eYco68iS`{4 zl~VST5A;LK_dQtD3T@Zq`Jr489qHwc7iYU?Zo?u|x+7;*?%iqU#F$4Rp2QpWl7Ej! zAQxnxOr}sw{qh&Im!}8~kGnH9j*hky@%H+r`iiNmN|ptcA2AtH7Y+%RSkjO1?MwMi zy!)ULQIQdJfll<-xx{2P+C&z%XR-0|9AhD4_R<}1dRsclI?zj%4yv9-2!&*zEJInm);lR=cA6*f$=8d@y(^+jKf zHsTpR#99Vv;{34Eb$wXj&@tUM<5OH5)ckjtQ)3WS+sPwu(p+5I! zx>TqA8ShEDzziIfF!~IR|0Xt|ITx#70STl=$m2TuA=BXRU14BuJ!P~ypIS1gHb#Aa z9ZBS)YUD+byC-4NQ))@9Up?v3VOdb`5%J7YzHV$IoVI-$>T%Y4bhQR;MXTIV>s9Zd z6|Wb6I&;n$%FfET)00_n)?90}+9yJzZB~*yi1l)g9GNMIxbj5@|5WzTh`@f7tmtV8 zK@%ceu{xI)y8U<$T@hIft+hv)rFtEqk83oI9aB7iSz{MCU$b1@HgDtL`2zlxlVXA& ztK4fOqx{4d6wmb=wiXUqK+;)HEl4o(?omQJiqRn(=E?*RK3}Q-^Sxlv4_^G7{4O$T zXrpDHztflA)lj>@Vy@n*men%p9y=)Rsp>|a$&vf=y77F{LU$?*SFTvx^3)Tv(#yrY zHh57?bP$-9Ao48|Zdu$AE<5vHOJo|J*v3jBQ z`X0+D2#du4a>%{-B#snP7qkI#Zv3s{8=Fg8bWc?x5>pFc0D^Nep996g$suVc7mpen z&OMiXLYu-OuvHcF^II2!BSt_mP z1@S?IAI!l7V?|p@W}pHOGcS7CL5*J^TE~PTFaV3@a~C+RTCb{U`qJvCe8-sD%ex1o zJH=?xUtuU76hF4UHs5`UZW=MCW}@yz;&_=a7t`zs|C>IEa|<%Mu99LMZJ)rtyD ztyK(~>nKaUOlJy@@$rwKs=~{;vR}Tg3J>O=;vVfFzRv~3gX$xNgpWm!XE#EXNQ_Nh z(aODANp6Jt3kwB@QNA`4g(mW~ayIaBKGW%IOAu16GhH7>-)ZcrRGILzdN?9mz86 zkRQGs@~8HvgM8@9WZi*MH%}dw&;uL`VO__gpMTg}kmGAB*(>m_nqN@9*;&Of%@z;L zY0Vqn7mOz~Xr9X>1gwJL<3(bJ^!W^q1ByWbYFKRK)PCfPXY3@;>H3NllQzeGnAIx$ z$~8`{L5AS_a^8?)pPF&!OOwT)t~`YQ-JWjkYF*X&fGW$*m{sFf_OjD6y`!8xLzFuc zsQ%@9qM%Juj5OM&B&IJQuY+HOzagCPxb1CNVA}S%RED~tMYe0-(r-}6{N>kGrP@+494E6k4Wx{TdXev(OA`uS(Tv}4=Q0|pV1$8o(Vk?X zbFtL7YO?ZiF6P^ZX1fcCDqi;Mv(vu@E{^+;?oa#DZtdFmO&I=IMw{4To6F?geH+dG zqt)j?$;SHm-K)$PyEWp=Tiqj6uFWncyN}S3v0-Vo!v50H>+A`i^d7$f;E&ZYb8A%f(Vn6mY#-S4wxqXEN<=jv}PG<%`RH1o2CmPWd2d))qvyI#`qST{gB zrOCVXpkb~yR^SPkCmEQ%Y`=^&{DPT@ZLURb=fdN+b4NAPF5WKc#Y{>oJ@E2Nd_{SQ zeY%yvF#EpA(W4|g687qzpj&+7cz=5&que>ie58yBeR5~^Z-sZIM-JIfntZ&6sB&^`Tt54+y#80F(k17 z>Q4yZqqsmig+c4&^z<423#6mg9#?Qq9L{*`R%Y~cxvv9uvhS-RoiC?q9#n3RhpWSR zF`|+o-&NjQhTmRSOiJsUCM!3b1)x-=5f)%_N~jL~`un`DTt0OPCaSEhgaY6*L198Z zNN7(Nl6|d_sV!q$ zh2KJVE$GvD_*?nQ6s^K4i z-}((M{SZk%93ya_F9C)RflEMq0YZO8{izk>(k{Y7hIGL#-<_i)-~o9K;8|;i#_Jv0 zRU|0`!NDUF4Y>x)s>zI2ILFWvgEq}f|A*GQM$H)C7q21Jss*~X#zm;cuV1Q(puKCO zyX6RNHXmyw$zxBB;;k2mCa1n8>Yn-2#R zdsTfSWu!fG;X0`;Bx{V`Y>qO?xXnG_K`^o2q(r?Q?>4Hd))DoA9_H;RyP78RhoIz=V|Tc_AVU%<<|Va8Q>=mi%m=n1sd0MR$dL{->jl>j{qCj@l= z^(DOteJl{91^)f5>9*S;&IPR7pc|3&3Zs~0C9h-@7r3;#er8RO617<-PtCp`a=wXv z&m_{JMRhy7!1OHXcBaFXBN!$hYh%y-v7Tqj^D$f}L_&pNM)=YwTGz2%FS)^7ngIia zNQv8cdM{vDvTnnEa60vgq2{u-jW`iO&F3LpBaDqQc&TUai}s({Kgp&H&@J!D7 ztuJ zly8RkeQ)!}Yz+{lt$9)Vb^q1>gZc(l=$2htEB5KqR{tk%yq%tzy<%$7{bt`QY8_Br zE1vAopQBfq!L8+S{G86=MZ0+PRNQ3w{?z2fxqzL$JvWTm1EcJ}(_C~-r@6wjD!6zk zR1VBPq~R9Tm%Q?pwiuN#b^f{B$Ht`ZO;1V}8uCUnr>UcG5RuiEai8mcU;V%qgluWZ z0}X>*<_#7w4m2~rFK0yR12z#Ohoq<&FKhQT&wVRo{SwnTLH6q2aCOn0ZvCrzCAbE3 ziWxjMFrOQ0z?~FeZk^Ngi~r5@8&OEVW$B_r84>S5fb52ez}eG)(S6Z$ds7l?{NFqLoAPdF~&0ZotR{0Xxgf%jAQ-y}cx)#%J_L;EQG(4ffXTUw|N%2N1CV z#^Ao!d<+bKM8XjtF8Cpyc}{?fg+W9_=j7p|M+Mvcq1A|qu^5@5>i(XQ(}`0LhSs2U6Oz`}NKy27_;sQ+qRxK8)C zmD%CK(ZaCTk(a#pEB|%qm?1p`x9VaV1eZxgsF#ud>BK{pb7x-`Mv$Q>t0{hAEC>xd z>$&T*)kz&+E=Fr-;NQqj%Yh}jmG`!CW-uC;zHiWW(QGi_pnl?sP^xRZ%82?b@!id# zGn>k?n;BvrV%gE=K^6;soh^rf9ZEsajxcOFNrsl}Ou0tTkmrUlFeIOH*9f=;;P!1? z3YTWZzHxh%mW<2~Ov(Q|StwxOeuaU^7@ikb{yPH~n0y{EdeKu1NIk^jglvr1k`je4 z+qWP`V>TsP9}@-0AjD#H1fF*8*L}BvXrrSzotm1;08TI}ehUep&TAK?dz~fMQK! z?x5p|HsOW=S_EFFtDF6p<_9dv=}pveP+2$FJ3pB6wff4bx|^4Z^!pD_D{UAJ!Ul1i z8@~5!>677?B(|{Bo&}UBi56InOlERO7wszIFnhWZ!+a79!yuDmIod(jXWf&n{mp6q zs!N*WB>e|x3=i1!Czbn>|3-=R09HEy+^@pI#{KU+H-?;8_rUR8b#^ccd1!b8xq89g zkcEVBL83YJP$&Q!5F&Om4_>T9M$OrLW8hQGnr27(uBm_$ey*kEj@`Z-x3*oiVk`3UWX~X zyYm}(dtrEXu{@#&C+-gT*&|6I$*j*Ps^*+Uy|u6tX}d~(kt~*$CLp%}+F1Y8p6=Fyx>zNwqV#}NGzUXcX;4!>j3r&4Ky?wDEZB>8)KLIY(+Qp15unzee z9b#Qq4wGBmjBN%p)4VKgx7}z<5*|u}JtO0=$S;$e(GhkG7zap>O)}5hEY#3;BGDgq zQ50@01s4N)H9U98@`c{S`S*Baz&hfJA;=Nk+~23qJZ3wsaq6_E1;9{Ij{Y@tu`UNc zy}%N4d-!O=W-uc1xSLAR#cY`lxe>SYqzl%#M^n9ia$`#(gK93jr)p*4*0kI-<7rI# zG<~WgQvf-lIOwJY55|XXPi^CRM!kgLdErKQBk~d8{9KNlg15){b2?b^9v-dYB%bV` z-pl_Q=;j!8#Q;%>1m-)bq%D(9m|BtOs*!3Trz&0)qe3T_K1fuD9{E(0bV@MuG2}4E z25M^{G&+_KOPuY(GB~05B|oCVCQ*&`i?%WzGd+c_-%N*-BI)q4yf2D<6vK++pKsO=A`dmQ{-cBFYdhN z@=C(*e$HF*TA@pT)FA8=5k;}XnhgD@+JwqbwL!b7&FMgaw@yY!_L2~y**i000^bU9 zEQ4xu!X@3_kI?&h3rU^!lr^!8%m9;amG9-GO*z$|%fWJvt#BGn7-!1#{YGk%jOUF$ zUT1IwmSVb55FGzzLJU;LQ= z&xySu0NML^Pm&lSkcEVlH!g$wzCYr~UI#5SLBWk0kg~GkMA1VA%}>OS$)Gd8u+T6# zI5<3Rb0wfFX4aY`R*@M1E)5Xic_pCXYgod+GH4*GrVb2{kZADY=>8y4P3OMfrv2H} zsjm3Sjwd-b21Fs54P@Vp_|;NQ8OrEW{6(D&%)7zDo`f*qd5)__^dJA!ilQ0Tc3Wpyv!uyZ=q%@zNWF zh6bc@;O93rH5mf@s$=8tY*_WJT_7L{iQ9u&BQP}noPn(0yO0|JNlOjm<21m3hcLA& zdtr7HuxLtbPE=2?uYZ7I$`~Oax*udei2wGDLG*CU9zhF7N8dm|vk=fBlF_=e2e~H6 zuwOs19RrUklJC{C?h&?%q>2$hp%2^jqroG6krrV5eR+KQ&V2|bzCGS?SZ)RQ=$QX3 z&_%G){6{2k_kquNN2lb^%KCq5<6bFG+bIDG4{~t`x(eYZB7?)=zLW;|Uf%-1=$fDq z41=T$6JY!xRU%`U=uDITZxsOaZNa^ng(4EcrX%DWB&v}`B@w&su#A&h&<2&{v2z*1 z;a=J%Y!=TtY>eFm*zzZc4xJ@h=CA2=l%bFANmHRiphTbU3s8QLf{nu;(Di}^S_HVL zX}AZG_XT1VrGVJG1=FyP24O-LJ4cfpyrnId#ViCNS`Qo50Q0t&B=6SdSy@>%08}W2 zA0SoaC_I?q?^t>1H?_IRg+8pkn0MF3g+qA&WISI6G~o>-DGR)`8pPOM;vzl3Hw%Ef zl~ua!nJYq#;c)AJ=?PF?d)62UR5qBk4?Z0GmzUe4)0|aAgbGDW3@;H9ErVcvCRV|k z@(m7Y9(n7}c_|AZxCF@qOu}vf zzz{XLSn(YOs?!x;zGM!OWrXtxM=mZd?kw!S$}1X;QBhGiD9uUxZT=q#z#oP@x-N)g zb6D=VngVod`5JTZpx} zQo4OQYzuH~Bse-*`Yj<6!aCtHOMs*^BWW>{i^AYgl_7+uq!VE}k*~V6*$O)>At5UC z@af_7SO@4vBu~=RIZG1U67_(|K^`V}mrR}0Br9PwIHlHkb$|d2JTQ`ez2^MH99-HW z1*QbRG0}VY_%X3Mn?}I65>yEA)!9DDqvYTAoOj<4R7+K?hZi&7eo?u|tXryr7C8LX z`eE7GcY+T*@+{MULJ4^<;4DBQV}L^Nw917WJ|0{hF+3k#00;mW9sF9GsHRegy5p;z zBMVsyldHlrKmi2pXcnB*=)_!Uz{$%%_%ck;Lk7XeTv*wFJBbAhHRN0G2l5*da!|GS zuTgutNqZ__P5=mJ3ZBwtn9F_-=`T8Cv@|pXFbN%ec=9g*(Sp040;C!M8q_0j9Btix8oc!RYO>Rsf@YyVbc=uCA_SF53(s{vC_yy%oH)X8fP}!Rv?hKIK$Zi9$N3 z;j%;63)E(^Gs}TG|Hid)?8kbZU1^i2BNs;ApIP3xqkMm*au4gV4 zl4Zs6LIpMv20P4uOwRvL>1l1gu=D_>Uv+xz2?K)`c>n(6cGUMJMv)nLiXr?e5Z(>o ztASY!2lb-2P#nF51q&$-U^J1aECA=`yM#bg-#0+9b-gY|!d2lh=CI;|KuYoRZ6d&Z zNoMuK6=PmNUhJI_gwO$s3`kYD2rCM-zkzjp7KR_-VJ6|b4ZBudw1HO;P(EhCpO270 zw_ztj7W^miNZ0jS;zTtd!qZ+LN# z1YQIj0-(!TsIYvaqoW^)>$|V>A>1qg7WixDw7~B_mHB~bvcrb(FbSEt+wkh6d=MBV zfVxV{fE}UjFX3KZSvi=V5Fa1E=M1?H0CormM?8{gM;QW0P@wWc@eT!4!wln{JBqIl z;Zw_d*E+6}BiI<_1Yrym5(5CK?N3EFK=$o|I9cFfBcVXdpMDxE-2y1~KY!U(fPhp% z&?fSYL(&=IJ^_ZX38YzrTSyYZ=}{=aMl%C4nglFngZl0S0E(sHx1c~xfubMk literal 0 HcmV?d00001 diff --git a/figs/lora/MobileSAM-Ti (Adapter)_performance.png b/figs/lora/MobileSAM-Ti (Adapter)_performance.png new file mode 100644 index 0000000000000000000000000000000000000000..252123c864c5fc1d81d8f7ddad9f85b7a9489224 GIT binary patch literal 23635 zcmdS>cQ}`C_&<){PkUr#CR;-?3t90@M#)M<_K31FvbPXr7g16u8Ie89Dl>Z%*{keL z*6+M|y+7~I@jHHh{Qmj=@$JyT^LgLbI!WjLJNs~^sKoH%ko3ht5T;DGIc5!_< zH7>O*A@Q5n#)35Z)zuUuDy~_INDhwC>JOJ_YpQQo|FGjAc{G&#WVz?Vcoh2&CnjGt z458dBF76N5TNm43sh153P3-KioMTVA;P_E_c=A9`>&uxX&(%+F;vbZgum>Pi@Nb7K zC6yE_3;$X|FpQ(4;YYV@ObY8E*m_JdT)%Jfv2mN zPxzvZy`TQSzVVJ&pp4(^%MT2kztUd3i%n8vyLhpw!nz@no#E2`8(p+21Vy$Z5BiDe)Z+qL7i-shW*_<%ENy%4ZF=SXKLpr z-wgj|1$Mf*T1a;KqLX;z7Q~Ty}-8U@tyf%7Nnq21l{VER+PKzgeX>79F zT_`sF9zgs^qOax3NNCvUD<s-PUiHuaZRu+4Z@m^*J6}kJjh<+AMx&Xns!Yo4c*4scAKEu#xSy*KP-?^b=akJ(=5BK=9%3FDN`W!}rLyMWx))ErxDldey@{bBh} z>ib@uis)#Xj~_qwk1YNEN)Ic3||l)^szs!v9jUK{0VC-zuIy?OIIIGD(Dy+gru`t<`zeIbtbb?1yYPQ{7Z zbF?N%1o7$?^~Y*zX_@|bN){(%@uJD=XH(P4s6PdVloX420=a&fWA*3k3eW~k3tCw^ zfqeR9*z7(|F|kw9d&~Y^?pj`mz97R1?Gih7ap!rW1eG9QUdh!4P637!n>OLB(&Pw( z&_lJbeNAp!4o*%hoo#UiDoKw1|Kiv$3%uQBhGP8@;+#o<2oO8$i6O%7YDlS$TO5At9PO zpOmV1)@Bw4&z?Is-X?um6V5C#uRI#QFCFsyxq!!}_3nsneK_+92#lEiVP+-|aM8Da ze?Mz~a}@+OL15LD2meTs#Eypo_ZuiZccx-vzrB*}cFn|`dEr-6BMVz;)bd7{LCoO{ zVl+_~#>K~17eGwC+^cIVX2Ty)6?SDf*k-V3?910hnWf?KT(>oI$-U(tqE3HlhFvGf zQWsG1DzxewA2~d@;wa9H%BG&r{ziYf#})U@`Fk@CIy`hkt`lL~z&!$9{~R6XddUbW zSZaVn{5~i|jEC@7Ajp|BXOKvlnZ#@UZ!Lwb@-2I~fyFUe`cAaa$Fr&`EHEPhh4fxd z)53XR)zo+HB!k7a^@V)~q}U8WW)W)!og(XdGi{T6MFK8UF8+$&CyO0Dzdh$-WJ%4_piN8|z=1pQytaC0=&(aq0sNAxE1ZoJS$L2iM ztyer$?Ck8x$;mQA>@``%!`x5FnYjZfIoR2+fkymUF+y+X$&LhrW^4W9T0GCc<4gVR}eN66O)>nXVkTwDYt4HU!K=D2m*URb?VfG>9+T_h9VZ7 zXQHB`HB1MA03l!(h}Z*W2aB^y=DvIlppYNFW^j8^FC07CvHpN@F^uJV&KfhQr+4yo{iPdM%o{Eg{ z4S&UM%y#_vvs>i0Iq*PEO)XN?eLbyfwVuU8DNTbrDj|UvsCxPG9fUF|DTzZ`dSuE8 zRb~jA_s$f3?A)`4tNnONXQ)ATu7hr`IQVB3{p!{DTz2J`!kw)_+sZ)2l<$>?`?jf5 z6jIx40hIkE_8+=dL5rR_e|~VJ*k+LKaJybQ2LvmwWm-Je9_TRJn{^rt(DNrxaCU#( z#f5i7kk06{@mFl?9=BZ$0=0?C!)5bktc(m!T?DJ)2a3_Uu+#R7Lt?4b)YRuyYHTa_ zpGqI@<@c^Cy%8dQ@3r~5yi_kmee6^Gql~o~`_#uXt=F>da*B$U)Rm3+95RDMX0Ggz zkdUDB*v*6Hbcm&AcK(C5FZRv8zR0fChmK%qlw?{TPDDXbf3Ux6Wi1Wj{7{H-cXxO3 zD}`m@FYne{_rMZ;DV=5nze?0=DLp-1AihPojC#m(ttB`-+}SM?=8imk<((ya5UN}y zdx?sfIjp-|Bh#Rqk(QPgUM`t>rK|VIV|DUofn^T~aJWj-SEo61!=;9v=nDxFF7k46 z_^t1*hV=LMi^m9Za^iwyXKd78g?%~DN$oN|*xNzH50x%v3FlET@-<~nbD;im9Hta^ zi$LsUzdA1@cKxxN?-xWmu@R_p(J3V1|F-<3X-zp!07G$gol@k{P_X9byQuaM{zA;Q zk;=ovN`o|6!m+i0oJj6`j$-j zd3h&8=y<=Q7V%E1irB_J49iAt7I_->YK(4f_D&ihC^~?O*i2>O`1b zOqEG0nRk1ZHU^IC{rjCCZ=w!LWPE%GXdDhn$)PSe2a{ZFyINYUQLyPQrBYH-!&9pz zC8A0&|0^S&m)+NA{ha1{jccFNf1i%!*e5w~-s_F93onVz>ijZ6l zZrXrNbar3TmT(gsta_Q2#$33ROi`HK2?^Nmk%>#y6R5j=jT`0`?sVMe3iY;L0bk9Yi)l^J0V6g_l>`Sp`j;e zXlU5La|6ylm64GFYC4ZGX|y4V({=pW`DJiu`r6T zbeHz#f3aR;G&eU-*2*^6oGL11dha%U%7ZMc1+1|k@V)0sEq%#C(O_g`Bob2lWTHMo zqURs8n8U}q9TDq(-}F1F<6xcg`9H(xPB;rNqf|=0yLX>ne$XnapZ}|(3!LM`%*@PQ zs@9FD*RN5o;S@R<VUHzaaLHBXxG!Eb z&ef)*p$UN5%sPq}*bMes#Di-NU2t-6d}onF{k6@7!3k)RWiRDl?F%-9>hs;K3}Iy0 zp^3MlvGE$p*9nQ&?eHV~lZ+uw2w~b@ha4S}SUeqqMqy?W)B$P?) zWQ6pADC(Q?=$Ca@wA6mE##o3m+jm4ugN|{|@)95M+R|5w7ik%$umf8w@LEa5=i;?% z*Z6QJOYA3>yYHwPuT8fw2$}?RmBCc;2nqWvK3+lQ2a74Ly02rkwY3w9&L=;GknY86 zAi9Bbyr5~F(QJDXss-}$@*<<71C}aw$ql`C&YHKq)BiwmukGDcL6aJyR0|_y6r7ST zS_6+@Dw(>) z{IDY#UnN02Fhc4dQL5_uy7AhOB=vN=f9n?3gGE_&iG-)Ynp1Jg_n&WcW~wowrlXst z-DC1 z$e)LV2!f;XF(qYSmrR`Ts=JWX!M15YU|^ER*5VCS)%xBnUH#Tvq8s~cL1d?&+E3Jt zp{zcW;9|GYqcs7{QegY??b}eZrdMO7Gw-g0$Jmi)c1zD=q0klF+!ZiS6yS^1F}n=; zO-|mAj*Xodt@iKFF~$W7{_Xx~CBz635BU)2DW}qpvfT_U%>GKUv6G@?%_(}4WTBKY zVDg@eJ)RD}o25e>CeJDE^!JQ`+Y~ME+uI^sHuL5<_Pby7E}&mKiYGw4s2XOW{Pz;H z?2P8+FJFH#+wmc&{f{Gr}Auq%LsP$*KY#s`Xo-Rn%45D6JV9_P3p%G0)PEN-u zA4(ZS!Ab|wjLF~b^vwn9kvVY3Yaqmvla&oXUjdJb4*a{v+_xS*`T{;>G!Zc|74o)X zx3g1S&~{j|BiB?RsmD?ddW0Y|(G_-Qp{uF?lb;(nt$NHGKZ%MF|nqcK_r_PaOqN+ld;DN!Fqa(6!e9Un=QA!uF>OL<^oE^=0> z6Dcnacb&Skbg#rIeRBWp9vRFSen^PsQdTd|Lw{&8Mf<>q*wMp(MPy-fi z(e9-RIhV8@Vs{1Xc2)UjKx~Cnm1ypMXv+ec!%i0t(l4)h-jkLuiO$ z+s6bxed=B!1cPbNpFMjvI3l9>FnpI2p63YfK7IQ15}M>`43%~nS#5i2i%w3u?*D;A z&uxz~1YPh<0K(mdTk9hs>F{yp;v@XB;n6=Q|1lkoWF;U8N9wq-={Pis*z{Kh7khkV%)GMlD()Iimxt{0xiJ;4 ziSavyheTM6sNkjvkvDa>E-hpW`@SgG2>$kt5~~?SGydE`kMLXPWu#^*f|iCXpfFcz zJY}W`_su`*?XxfwTO+|+*Z2>Y19Mz91?)Dtj55|8|47~#VZ1^al z?A1hxHa#VKkCd!49*(Z-Ka3L z^QpMR7AR-)<_tV$ZlpL7Ij@jkYb(kubW~2iw>@ge9YvoX=eKsRRoMLj1up)nLc*&t z!1?`IB*Ej(}@ zZmnpobylg*<-2@3D?5yQDXde_iqf}K$KCaflpcvcuehkRf^%dAg{6V{%k_~^-bk^l z5>k9RF7}a+e|^cAz7&${J*95R7jVrck>=riHg1+*ZjqTEJx=Y|bD+{KTC6Az-yKgtm5$pJXt!yq( zq3IFU^@(jSw_YxR@{MnZOxi}0TG?WdR`xFmEn(Np@|T>_ z+nVf_GIW>K;w-W{@(IfKeqgI_S6SPYN9a(W-6*|e>;?n+n$_YyKnSH z8S7R-^OxyI!h(|ppPIW49e~qB!P2lq*lsKPNg&| zHc9P*YS9)kbn3vEj8r2a-(%#?<{l1fTKraV&4Gca+w~H=K;&QXIPR(##u=hJ$D*$Nv*eE zYr;v`3JM>GC1ueGa!{#x+@z$#N#lzC9&Z|Vv2@nU&6~Gnn=fqxdxMAa#~g?7?@O~g#Xplt{g%h=T+`3V1gd8^*=GtC^l+4Y zzU51)?poN>TzbN#RQ<7y{;mGv{X-Xp1$%`AQ>l^Lz24R&WCw#-cOS8Sc z9F6l@8pq8C7CLju@-VjxPlwRdF^0gj|KFd$^Enm)f9vmlHn&xAJ`r@D@Avsz>8}o^ z=Y^jx@OLvkEi7%n(n2?C8SwQoy}{|GIfLoMhcf~XC|t>x{FApMSYF^Cc;{2F?ZhWr zk2aheot0;sJKtxlO8w9#&xt9tL(`JmxSi8QjmtOj#-EO&$~AFz-#kzHvE=|7a^;xL z-R~3FI=OnyKOaAJWF(ix%F&x=C44QZYaR2l3*8dBNgbstfV{BHk4w27uONb>a0PeA zM?BlMgC&@CRPO^lU0b5UyANszyI?1$edL;Wh+IYov|b*vPoH}-V3+jyiH;9@4G^mL zla7trnvXokswe?=_3G10quX0m9m?vnpSenMRFd$1eL@nl3ClvG+zdnqmDuL%k``d?x#_cR#WqF=&c0;wSK!js{rFZJNZs@U99 zT+$xK2{IQprsf>VY4(!Eb9X|(zR{iVo&5dW;bVoT^{=xW91f>p=6rOdcJ7r<-HYce z!le|b7Cu(>?YOfUv&B1uMP&5FL`r7!p~Gn_EI6BY@ljFT=7gah^uKYAc=TeiZp<}a zSU;`1V9@PMyM^dcqoGht=vn>7V$Q64(9(%Ld*4itE;g;owLySNO$=|(dcA$2vYjFcxKi27`hFa(W z7~6wp3Yl9%R2G~JT0iM?aAXj&LRV$ zVBdwT`gAjix%?-EWwMhpud4$>zmW>NDP%=_J9#~8x2Bi~8yiRI_?-CKOq`KpIPtZh zS)*Cu40E2+Lu2G)-GvrhZ~Y7973Nx0`(D_xiMjQ1#d))6d&g9+5BWn@U*j0;z zq(uw=NadBxh$eZ+M;_U`#h9}C-72P_Tt20MKrO{fdQiMjQRmZL?gjRZakI_7UMI_* zz_tF~bf#8Q-3K074tjNVw=~Yj^q4bAU+kG=ZsD)(7*c0yx%eYJ`KQj71??w=RG8#5 zvTLcY)Z6F!FELybm#0!4)l9A1F?%i4S=^$D{m!NuE8hz<{IHX9L)x4CPkl<&Og{wy zBC~F}Zrg_=nB*SEi*qi-&b6rO{;T2B{O*RP5`!N3zKtfOIj!?0S@?8gWtrx4B41A1 zPPW{|&A81?#hiS7sc3ALnl)U?hO_;w2p(fhXP4X=3c)wLT%Ru`H5)ytzJD!JCD15= z@tTfswBpEKkbm|4x@Y-Qn{U2-a5EEqqxF3LJ4vw4!67Aca(?gZP4?{k4X3ns#f$6b zPWX1)Y1US~+e+!vYMU~Oa6c(Cx;=2|!g7LWkp6j++uDmv=}(+2^ns#bu7Aeft%xnH=C@7sz2#tP;QKx{^n$Tv5){Jk zahdk>QaI%kDMtN#+`-b(cVSx{TD92Lti|$LV>t5@ncU85l8-IQxPBJ-dZh&4QggG4 zCbRFxwV&({1OM7&3lsI;x^_|4$Ji@a#h)$V=Rw3JcDxhhg&{?&vA`MzUhFxqPQNuA zlrk;-v1!)xNhe<8wBiT#erdzVyBSN3JY3m@=Puv)ETa$<`M{$l;IHT6QeU=DCD^9Z z)!B!>&Xt6oyOfl}bgrXX*xH}>k?mbieCGtNg5KqX&mHVKpVgC92kkt_3_%Y04r5eS z)gB3qji#G0HAGnw`?T=Mcw4kECN5*i-0OpMagFiZi-{Gq6PGrFF1_Xz)zo;_{KNrR zkaJECUq##S(5oW3w<4v`-qN7Q>SN+GeJ`JaUD~$$SP)U;yBSq-iT(Te zc=ekc0adTZxjph%{9x-OHRto{QoI}~|J5iYs_I@~a1J9|`RnT-7$P^SbdxrZC8G&u{%DRWc|2 z`ME{dlC_17^p^rYXN6qih~b^e{e9;uq@T!pmvDVhXLRikR!vA?mo1Ta+40#MWp9g^ zJ7k9D*GjwGW7Pc&Y`)Jcmo*Yo5q0M|cfZ$+=MjCY`@}FmWL`M%-VC>;HGSgkvF9Sj z>V`5_%En4*4+!J>A2m9CXCE@7z0p#1zD-llK`r2JmaNHrwv(Lx?R<*vXLR1{KHWBW z$y$EpWeQxe|2V#^n3t7x-+2qb!)A#euse2Uhe-#lG5Pq#mV=D-Ww%pJO z+);hZ5yg~CkVCb%YDfv*oYgs`OwFj3)pxjfu}xd!b{Ht>+$M4Lts4gEUG9N6vn=$M z*<)Wb^hsN7KD*ZCEoLiR*P|=(B(CL zCvrk}Z6W0xgBG{12vu(+y&L;{VF-!L+%j@cSeJTWdunM*4y5H8oex&CR$cqC!s8Ii zs@*<}&k_Ma3&B(A-`l3{+V&|_UoVp`cx{ujSt9u>P z*xo_$59H-rj<-&NRcF47SJm9T)6a3kio#`+vHUTu@O0ca?au;NPi7UmE_`?r$#9`O z+?J}!PjT|`{gvI{HK@SO9r_;pJE?sIpFy0JVeiI7MC=B2jtNa-=S{Ce852eyB??Lz zBh_9v&u?>QME31J$v@ao;q{N|yvA67x=ykLbCMeB$)o|4Aet6r0qR_P6OK2_r%OV} zJ8l-bL=_#l`aPtl4Gn0(lB;l3j&D?swD=~(c*amxZN`4=ldd4w6DwTtTd40#nUN}P zytEf{N@;t%?~Zoq+!^v2gNMItSZM<=y%zTf3L<13iEvrnSOMuFLgn|6nku-XTn7_L z5XU5b$Ar~>&q{^rXNJ;^9N`{$f+YR*z+aM^s%JS)zyDWR4TpFuKQCspUAaOJNpM$y zNpxfzR_YWR!a*{ci=LmimX(tmd!_rRI$16xv2V93D)x52Q=zVqOoo-DCnQDst;B$C@o@C31GWc;SgeOA&Qn<(v@Cqy* z^Mans)r2Tg128AgB^FC?0a}t>c~C|uTh^Mz(u0|f^O&Wuh^iV%^Gj*>yTOI50iFb9S&`J3@M z=YHc!#JASuny05Ei|Z&xaj+MV>2bfXmsSS5;L`T_k`8ZDESMJCTY zi9v4#0?<8UzfJ5s6mbZggD3Y)0io1t&no-S)?$=%u2 zrN4L?vcs8rr6<3-EXf0w#BRKX*b}y}WPfwe>S{!_k#Dt~F^$2^o6mNo0wfl^DSQ#} z6S&C|xYwsKtP%K|-b5$f;0F|C?B)g_74J9$D_i?K%(kgzjAm9IZ;S4qDPsERKOpWR zuaQIk@?Wi~$zf42>Ds~$3B(V7f1JlLx^@lIY<}6qqO)Ct<8cCE8^o8zxgfZMUDprWF3`Bh0cc_!w> zS?qmhvei$_4LTt*qH)JetwGF+%JSz zC3hxYqB~*_Fd|&!OUXD1B-%~kXa+s1i2nIz>{@EqwbZro3o<-(qx5s55eHj1;qQ}v zAL5?;LZEVRrUxgyq8Upcg1_x>&WI;=^XuPtpDwyEGbou`OU8vnxFpn6yqsBRlBma< z*o>AP>P{6-ErH~52Iu?j+HT9j8!9UDIy#JX;mo91M!XlVUOffGSlLi2KnUa;?xcTw zre-F=H{E|XaalBx6!*osUOj2YcS2EX2ZuOXZJ6&9R?@wTE-5)_#5f1JHDXF4A1d3l zjC%M7w2RLZ&Sbh=QoBf7Qt^Q2)5kO888t-Yf6<}OzbEoV1Y8#NloG^g;kZ-|m@~(9 z#ROgWuL29i_rje|iEzND4IF_OHOC3M!r@g%sw(UH(g-V@Qv7(SD)kHwbo%`h=lAlI z+$&a{vGjV6y~cagm5rt;sv@#+Ootu4-a;}fOZ8@Uwgz)SBAt*E-Rgs(x2gu@HoKEm?M&NQfI6gl+6Pg z5@S!}3{_GS*$&>9E85{rB|21=ewOVmZBvZO4X*f&lz>%lYeLVKF0TDC`QQ*#~q zZtfkpooHZL#^>T!J*4@}6Gbve3Hy1>~TJpv0VSxi7XV-K&#mT+p^>U)1SBvfSwIkDYB- zn4R5vrk7iMFru{h-q+r1knK``NNOKDXm7z2z78(iR*93gYyu>+Znu7KS#{CRIxe0H)Kkv`kQ&Teb<(P)LvgE{l zJn(1#VjHhTKeu%drXaP?j3oAPtwen6EwRDke0fOSJ8B9kpxa$WNc87C&!aTV{*u+U zJ%BZMpoIsSTpjov@~IJ4|c8s8-G87ZmMSB z(%u_=JTOoL=?M%6u%>TJxQTi(^f%hF{<#)0AyS7MG|1?o)ARAG?qD14`(o6-C~0;+ z_EI<~dtJ-1Jm3wRy7p6qg_jgnlw{EO^@+~>U-=k2m%Hphe-7Eyy5&W||NHsJXE8~e z^UOk0gNIiTyEpW3+;hqpiN466hRMtnnHcmhMf&${$;zHN4|{GlhdW!1Z2vYa<+>b4 zIl_T>QFBP%C5JW{tOVH7OYTz0F9IJ>jbzc^QuzT3Y%PHNj0v(r)^24io7UY!s-$-d zHgBAQrKK4&AX7hk3EQuCRe8FN3i06FPzlDGXhbUf!~%WDejr?0o42dp5&OQ^Q8+wU zQ|vu}H8rAZ8c@JoITRA`N#Ls<8&ZF$sTf+3teOA*{eMjGW!?+HeY4+PRC@T-$*OVp z06os4{I8uYSz@DgH9esU?&y#YjN?!X@pMX%&!L6|KQ6Q24>msH{zjlKW(%kjdLKZB)G>5k2#&%iGfCV1FhiD@kp+ z*Y>|@DHI*}s3F`oS@HWb_0-!7OD4^`^K2QR)A9+IQgd*?Gfz@OOg-{HG?Q>>$2->bM8-?|RbicprS$*u9c-{ftiFmBS%_bZJsBSB;>rzVi}p zEP+!!n_EpqtO^z}xz|#uW%|$i96Vrc-B*A$xu9#B!%a&sKi4XvvsL%;?kpv9z#6Um zx5GQ;{wl>wlWzADK4X351Lr*IL}nS#zOSNlas2bumxz7&w_#^hMATJNK^Iqgq{_#; zxyCI>r@AanN7~G6$+UMKURiS$IC>Cepv8hM0*^RYfrvTuZHw3teOk&|U&c-25 z!I zAo3Y$*ciuWz?&VF;VqTn)?OIt!wp-eCjPA4ie{tl*XMlvDXRiLj|&!9-MT{i2302b zu*#lGBqb%C!Rr$<2=c><*!Skj7()jq0UhRB#ni*FjW(}+`U5G$XChP>R})8Nu-#yF{qKFJ=aX&mjYp5*{*+|lb;O6(?fus@Q|;Ff5cgo_5FM!eD%%U>f{r# z8CKX*RTCtd^TQnCS~_hQ-zvz?F!)s%8SPmLN08^-87s4?mAJ?vQ#U#8FNL!F8kBn= zES+kQW5xP6hB7O(;-r=D^mWX4tAP8{Z@f0cJD$_*C{Nh)D;U*<<)HT#lAftZ5yDL4msY&YG%l}G-+pRQ|Z(E`}a1!P3 z_XN{2$dHN6f${UTK;S3ia4|0d#45T^>NNSs{Bub0lqN|7zZN3xE_ZN42H{E11mXR{^4rj{seF9ra zp=8pNAD-ChM=(6>+1r9bizsQ}dQJ~k>>m}jk^R@C2*c1>MscV8e@lOaxrfZN@X=z5 z!zyl8u}gOq+j<$R%2Wz$;%&Hqtp+Y?070qFvdJ7wgH$?vN&jO&LCinYE~dB zheq~_Q4(z84L+1a#x!=< z5}Pez985~OZ1=zIxq6TTyW|r{ybll~%^U6V7vA35%?k+m63bj;CnfaU3=<2Z^u1?^ z!*@A7lJZn=t6((?9}TL#iE$H$`sESav?p}9zvN_+)Le5=Wq;-h?7{>kLe|bE+xB$_ z-n_k!cH)GWgRdBuDC1k?y>$S`KODeJr{}zU!xWhj{B_17h`^EI$?ZiLX*+F3aE?F| zY42cOb-xJPiFlD5;y83!EvY!85P}1ekO;~p09|Pc0K3?sinKH$6y~X?#{}4FahJtV zKus0BRvg6kGGhI|Ga~nnz!)2om#^RG`=TFG`gq1bG=#ifU*QE^A(@5xgTdXFhbf^$N(W)Nb|Sp0nR8OSi?OQ zO9{I^DdQqMh80XBo@-=M+oQP0;qM_J@zu1og8|Mb^C9YA7XZ{n9z>n4yNk;O`!j$C zTrayZNv&Mf)6>&mY|E;C=gvY?wBlL#9rfl6&Lk&1!(SQBqU;LxIA0AiaU_ zfc2v57f(VMPLHlj#3 zdq8xe5G4V|FaZ50HF=#Nz&%0WzwdPv_sZ&@NnCn&(*wno z0Et64?_R$77(lPKr{isxzmxJC)rFpV_m9Zw$L%WEz3v-Qf<$--;EN*w2W(RP1izuB zMGh?ju;@CsTNMc@A+8wu)klyOK_PMg|4Ia~%Cp& zE__sc)f3T%hehHbbzs1kjII`1s>l zBC1oO;Jv3_XhGT0pCHCLB-S|RP^JRKcKU=g{$Gpk}uj-hj zcF+;%ar}?E)wvW$bAnmC)h6^snD9}FA*)BY&i5h@pOBcR0j&JL6~Z^#WW31#F7a81 z2w@@(vT-~x0Vrtdpqq)9UKAO1`v=hAna#plwg1Vc`RXN!3UN4H#0Se2aFGi!?`7a~eLdknwiH z$bD2TrzHVVY8;I@W)a>>anOU744;U^atW%qlg1EA^s@js684#x`$y780BVwY6NB1~ zE9lhcxJJ2?ru4CBPt|BoNd*6GgBm)D-FeDmuoV|{k{gEel+OKlcX7d6eU?QK*;NIz zyv+&ACf-FU5Ow;90yF}mB&c#_42Ly|9#Oj#11rh--#>vF$lF&xqh{%;@$*RPVN$fg z5fT2QN7V<69f1ANVj#(Tcy7k5SeUf~ZUjNq__)c(PoI(j*X=qMNDnVvP^GK?`dZny z77zOummy8lqvXhTXt9v_8hWlAVMS4(yj9;9ITI5O`EaIZWT!={9tyeVJshsr*(KCI z|74o&h+1qBsll_e4^cs`+}}Kn78*Ft_fr7?RYIrGG7uzFw_=Y8bc?L3A!7^aC#Vd&G73P>+2L{tv{*-#zHLpZQy+Umh75D|xRt+~5W*NH z?+Iy327r~Kcxxy{Ykqh2Gzw`284duTs>$y%r=uFJdrC{A0${^z@xk?$C7bSu!n3!x z2i1huY&kCthYgo|^!1wq`W9*gY&Y}Vn0J|c)sgPcr;g;sSdtMLfq4wM&t*YHU*RKa z0A4pYH=Fas#9lIp!_!Ho#iT(Jes~YJ3sFP=L-1xppcf+p3a6Bn4TG?cp8SLFf08G^oQ{+@{!u@3 z?wm|^ADXU)x+j-}&E3hDs*pDlaGvk~#A{?Kx{0}etaF~U;Z;GklnDUs(ZaONrICn_ zAJ2p`2;%JQ?8FZ7U%W_))*nC##T#>O3*1x3c64#QtG~K6qcUuF2b1_5YLE-2u?_^1<%XM3o<|lhjdm*(j%y z|H1_eHMi3>XMPiZeEy$)^h_qmserN_d&oskly41Rhtj!(EnP_V0W5VsVtaf09M6;- zlO=M@AA<0{05>-YcMJ*GQlb5$eDkcQd_mKSrt8>qE7mUp~$c0MN zJJ)eyv|0zsL^6fD|2T>REWS9lQiVw#19&M^S3Xrm?UjTJ-zSxX=K!=#HmvkUk0ijL z3flY3O*DE46zwWa#yhunOOHr@WXJKCapZz)!NGao5M(Lobu8x*b01lc{NT|R9IlZhA0T*Oi4GP*$EA`3K$}7fxwC+o3nk!b~O{snJ}J1uRjjl&?E)r3lId zqUVUk1WqSWb_xm02NLMe{_nRry$2Xy8EPa@W~}-<>K_k*TJ`($ePKp}k^BI%xXE=O z61d>jrX2W4U4BdfWN{;ch^t=#d3^FA+PNes=A02i?WUVYCb9ztuX3iROd9h#61Jny zTpO6lGwrC@l7nE5y0RJ_EA=yI+bvA@u~rslgpNgr2*QtQvKd1#YLKJ-sXiN}TeDbIz` zwNDb9@EvXy2OX^;eoSAOkL?3yj5QU>!~e77*##x?msqPF7QPcek{ulMmGL7tsG*zz zDNhha?p_N)c#k|bVHVuP;`~jF5Ydr$5#@mnILBp#E7X4Ze~hiWgf1DI(2+C30XdR& z=Wwt8A`_tR?1J?N-{ZJqC;-zwIFRbsA=7q5EC*b4V113wZGfiH02GuKeY`)Q_6Xhs>HMF43Op!@0RNbL{@;}Jk!b~37QU?n3 z&luq_O4kpH$35;n1Sz@XnfcFfU*H(h8^ zr_ND7zLQ|F8kwDnp0@n+`iMDDQ)ZxIwt$<~2?anPgC5%&h+diFg_G3ONquu2l|DZ6 z!{uLlZALtnuA;>!Fb8lh@F1AtFD+Nl=20W&M6fAlM;9AxEx$%epnM&5A4uu>Y9N|~ zLd*aN2HDW+#ZB~#4C*#>p{fwA{DErfK7AKvC}rtsK3j=rbtsLf9q(*BR{-Vru0XpW z=wTw`^Ox^0&fY>0x)A)8l@-@fKf>jH(+GWMpZtMPsOsanmxl+{%IRHVP{2<~NB0!` zqkyMRMKbpIG~W>#DB>aCdJi8+isq&6wp{~8eeh;)OP2t(5M^bIP}^|b(lX=!I@6qU z41NBbgdt460P1XdZ6K6WR8)M+qWshSv}nE>Hnm4&p%mR3+ih8dhl>lsMtLQr(5}_E zIEKmT>E1sKXhj1!OH(u){6!BtIG|W&fEDH)#=T{BZwASHLg$`UX3++P8?xT$5vFPR z_ISuPnC)e#0>7S}?Nw0@ky27gN&f*^S4>J85ppbn40pJE7SWP1Ha6b4bt@k=Z1DR!EHi7jd~dbM70OZ3NNXJ~Z4i5|BVELZO*)ERIf@~A z{r}r2A?Rfa#t^)rb^gFk3tssC{{B=Rg%{|Q?VI3_C-^%GFOr=4Sjad`X~ zEkLsVRnRCUa7IGnwBl4CNgJHMC&6x^)x=%)bbNZjP>7u&7GiLu5016V;>2YPB?4yl z8G5>?SM0UxoVs=I-aTc*#JWqzyuG_4%Zhyu%&vX5;5h1e^aGh~ng8H>oXr4rK3f}b zofUiaSPwu_hW@gqMDu?d)`1!%+We9>Zelw&ds++K!NqEgv07_kNb=HI(pGY#T7cpc z3BP#}0rq9aDO+jOTa}8yw1*7fB+4}&g+)h$fMUY8I?_B~C(7e$8Icyt@}OI&?ydld z4H>XRl8%GLQVQI}E2zM8A8SEKv$+cf3M1S?#~NS@G=K?7{xx?{%+}XX#!N-u&?QS$ z|8cD9KzZ|jD(FSHj~5BHVYXJ|mF1s%P5(O|4Z8II%-hg27(P8q*w^w{^>dE3Hr`+P{m0S_`OSRMG+lC!z_VN zEoq9CpDKzqH?R@{9S#A3$V$w{nE))521DT-27#D_g)hp*1X)*?w_?SyX(L9{E3a?aWh#f1E{w2SeC{6duUUC7B$P zoF42z9zzQ9@HV<9QCnZ(juVIqw0C!*@@dFzCgEXE25o^3F9co!R=qrLC0tr9Dr;(F zc?`al)UBWwv%rz`R}0dGdSKA#oMDo1)Xbl`a$e;v7JwkI8b4vH-cd*pq4B_R(&)+e z?=q_N9ZU9%47O%jn1DAZ@oyMF5gsP%ZU|(4TBXk38Z5Fw&)Lyy7S?9k>hsK+B=r;z z(G}6a^pCuc216*`vJmo^9KtYa`pb9H(b1VeEhbt#yfD^w4)z7g-Rgc7TCG4Kqus|Z zgo40t`9dTNs9!W$%pGR#XqYp=Idd@g8bEZt(&z5qzmFyId-S`kRBr@I%-{+DmIrO` znd5YZDQ8d6<^XIv~PTytlm`3a#am`dLvly z+6tDiBjr}C`NiT&W9pn(^?M(YwC+{!NZ1p8xRSxI+Ndg+igOtXq1n!#Cj=Oq{|JsEeJF`yhold-ae@2_?(X7{Gn=Yb9UDaV`4=P7bEH>Z|LIPr z-dy}00(ml1DCmXA8W-+~2v52m#Bud1Gp|lTj;#b>f2@Z}s7_ybR09`ipx1oNLm4+z zM9vQ6rxHpB+P>5(&ttutN3U(&dV$zf!CsIdo^#RNB$$MJnrE96DE3t0GwQP-6NrgNXd7cFAs za6>N;f~Xt)>3E3dl*fJqP54tt0dywk&T4#vQf@Tm3oj3Q@7lm^ zN7=m#ec6T)=ihst1e-n!%xU2M=PY^!QH!=oUvrN-S>lmHd(fhd`PTmP@{ISMP$HQ_ zQxbZc1>7@HUbhOh=ez&PP~cv);>@s>nVDG_eWCa)^ zlzroYM%RB-`FKVJErxda?L_arzo8Y%EI|kLuC;InNJsP8>_&Bn?^=^hP|MT!o)U+I zzVMk$usibuAdRTqsBf1)W8DKw#L#EoX$UURK*@&~!8yrM-Ww7qqq+4gpL6%)Z7ZI^ z7P7!CD+0*qz=QX-U|vx0ADn)CyVv9`n7;wK9<>mMpNOL^$0eN}T@eF~11eC<)D7x> zF$f$?$SW%TG>f2SWsLysRFYie3-W)w+-nReU_&~uG_HW=K1V~6eyW$6;>!9VuJ2OiAq#qa4rDC!On;-Y3s)$7 z19y?{|I^92M?<;war{AY4V7}Fkcv<_l*@_A$+e417uD-zLdT^PrO2c~ky0UbIx5Af zNlLjTQb(E?R1QYUrD&4eGBQI+jb_ew^PY9ycfG5%&U*bbtTpTLJkQ?yw|~Fy@4G*r z?uV(!-c5u{aH^^*Ylqn^UEPXpbWC!b>*WQf!QGVnvJ0v4{#3i;Q`?~71RgkOROX!} zp*k9B)WJ%^U>`;zjZ_xmL=3)002pUYd{)NyMISoEPZN*G+u9-*?(+&*8F@%7GJRiQOq^x{|Rx5w{%K#cJohCT?3>o9477zta!Kp{DXyo3g zkCuJMSa*6lToKpwk!Q*wRy(d<&4$dCNPt5?+LQuRCU@q^$NPO^L1LsQUNFah5l~Q* z$ZSIM=Wi2aGGE$WAiyYF*tfm`0mMvc9BbynQMPRhe?Kt!PIhC)DZjx!Mn;06t}j zJ{X_BOUU3!NlAIUvPs^q_g6XwxCX6vBw948=sY^!)X6M3Y`y&~BF(CrzTW=PhG_B8 z<(Ej06H25Uv#z`6&7)KHOs{IL&uGVkKJ(1Vaupk=*k~ec{V&FkC)FfdxI!0Yr`6_3 zWaBV=*;MWjTH|P8R0d=#KaUaF4vqo zHHlqWIZIEkGN^xD)AX>5DKa^G249B+Y8Ka2Gxc-wRU~malCb@xXrSlh?E7Q6<>()0mZwV0X7+E-kqJH9M5$7i`090C>q6=~aT0Q{sCrfM%(pd=w7;ppUKXKx?tCvUiwFlyX3dK^A^QiZgK zNn{0S=0f51%*;~Ko>f7<7&6*oKaB_wyUcjNlf4jF5fQm83QnZ7W6% z4b0TpCEPgO=+Uu?sV3`K=AB2bG-0K3mRdl zt_thTx|0k16n-_5m6tC?q;z$2gHmzk4){K#VcAp=poYaK2|y9j7^RMmj-j1M;-Yg( zzQ|+8LX5H`gFG^rCjQ`gy(7aZ#VibKgM?!q5O&~4%oicNyaruu94nwjl@Jxg!Utj- z6z>;)9d!TQB~f$9F6{ck_KdWMTRH6^fjWnZSHj84#=GJ5SNrB;PMm>-HA!Ywhb46s z7itl`^3FTj)U=6s>YI|WWy@W3)@!)=ul9qJz9fu{9FVZ)HRajJ&h*#_T1G?8mLP5? zl32+oz&t*!H)fy$jTx?u6qN-Q;f<;cxc991+`;n9whf- z)+ixIb_@wIMV@#BJSCw?`r0^rLf;9`{kg;tFhZ!#&W%<$cLfMxmub)y$TO5R2A-lI zzC_pg@4eRmCZrwe0(zFlBhXJ^z5S6+|6%RgwPEooEIjJ>+n>$N2$1LZH@BLYm~b-P z3~vW|$X@LH2`a8D*V0bEH921W=;_G>ZfjK%~`B zToL%;v0j=d=6F-SpG$lQ-a#7j@vVs5-A~gM2=WO7G_7&-o(ZDicC0^gFxEEEd~1aw zVxpGw*RV$nlF3WSDE)?=111O-JiP>L;-XfS@%HUoPYh7gEgyvs`|oEQZ?8v(Zd+vQ zjg3acj7$)QV8KXAN?L6);a2E+8ciB)|GRD7yKYSSAy4W1dv`-Nb{5sqY2;Z6;> zoVRY#)X*R<0M#i|e%M(X;1LPc2b0$p?S9bSVlak;qKW6j`&|0AtBd4)G45Hh?pZKv zTM`r>VRh;1#x~xIgI|M*E!HBG_nr7^3S5I~b$?mA&oj=AH90V3pyHrCb@HnCk$Expi3BUgeGLI41jHYSeLct!rMo{p`ris=Mu zNd5=lD+Dk`bWhNe=<%&{XU}H8ihQm+cdjfd=%J3{hW7%Fjj|UE=#Gw#te*M|l7xc( zhMY4gvfk-&ec8$DSJ9393CyGoz}Dex%(6UCsz{&(Fi4kr76ymAo9s*{=|Sw^$Dl%9 zs_$XvdX=#eNUeVIR}T(}mpF>Vl2hUY;7pNNBphYNC@QL&x9&Y{v8VGym;8Ji99zi1 z9jLfxZLoE4IPO&z8%OGO5?p&MoTX-s%PIK%gw`?|yS!X3Yl9&iCi7v>Fv2YH-BR1z zvPgBgY}+Ls@$cGdvX+vD!2g4{4Q4`ew7KN^tZ1M9#3YE@q#XM%T<(EmSi{I)$dHX2 zH%?n$Uk=NO+I5LOge3Y$L{1Ppb#);5D4wvNMd=p-tD`hocQp$nLIH?UZ3rXmV^a)yLXwIF zG~9gnQ_%gxXePFaMg3wGNTcf$!2W5+eU_o4eQCpu-9RasRdwm*EPl_#4I8*d2Of*Md|(#U5`pX5S2C0BxwT#lJUQQQdOeEaxmho3xI+5Pd0 zZi&3DZ89m#2h_32*+Bh@Rn|(tMlEcAApU9u!f7D4#&|l^Z%M*Rg=Roepyibk%4UKk zqA&Cq)CnZEf)NYA5`RwBQ-nKVFTik=bwor2bJ?tkNtV|DP}I@cr*CK|MIi++hr_3! zelZdkRY?s4ZSCKjUv2{gFaV@q>vw5TekKq-aw!u%Y7m_s)zpwO;i4wA_o2Sfz85}` z4ABUkAl%?8kfI0ftg&Fwp>T&F3}S|pFnD2^?_q^ zcA=jVf>U#G)~Hv(QIIypg7?hF?dMLtK1beyukw zUIdqgJ_a9}6-7Z0Cn1XnaU{_ourQ1dAdgxo?+2ebV8DS+&>7Ij`Js82nan)QzY7+- zCEMECiclO2fJ%Ou;t6+$-5=*o71Y($Nn@8VVi6-K7MM$Mgu(~$orw7p)zpsorP9am zYuSC|>vX7IWdQLeVAgA)va!L&OD-4wrhU>xQ&|Uz$w}PYc+QeYkrlSaB_7jtbP~tR nCx2tD#|%RMAVvHqaX_?eRcW%b*+v-7@w39t**1H*=b=9VmbyQW|*X}-)NTW283IZx2jiiK%Nl6MwDoCf~0Tc--6%j#Fq@D828WX81H!ZIs5Fj=U!{BIiLAFa}#*y))i7B1|kGONUvR0P(=_N zEP`Oh2npaD;m)Bc_@B6=qL!nYjhUm%eS1^n=6y%oM>dX+9vZPao7y`(w6PZC73LK@ z&-%d8(bhqNkI(AgKj5{oH|P6yuA~bFIc|GZ+W|qy@1y@=(qvK}B8XkXH3d0!*Vu*O z$F6F2op5DJ1r>DyY6YqamRn-!IsG*;@prh$!xz8yq;4}0PEDCi=fCJJ*wE3X zP*lOAgn#0s(OJi272sc1FoqSrh%;qE^?7k+C$8roW;7LMjX2^SR_w@_h_`T;#ynFWj0 z=VwL__C-&eIPpk|ggxUJu56M>1BK&pUu@O>c4pgT5*Pc$izj?p$9~1w6xI;NBSoVDsc--o_F?^P(yJMkpXHjrYD(cm%v8sbTzBtcy?Ish=gRj;W6eCof zot;X?tE;Q^337hRy_yEXJF&MVSMbPKq@TxoZGUAqKEvK(pSOktG)W8%4dohrAtGTE zX`R_;y5u}o`;_<&Pvl*ex{8gF;A<~AwdV^sZ@j^VFp3)enfmIwTysoCP0jD**;|Ht z+iP?eO$d#Qjprn%V{9trw6y55^-6q&E&IlNjxqB;wiXo?H8C+UTpFpM6M7KdY1|UU z_beucF(V^mmT6F)TIidp6#~_hI)3xfIp$nqw?NJ0AD%-!ES4 z(_QJ)wHbe{Yn``h8<_SK9J@fpxZ!!tY;Weui2G8_m!Go-v*+%9l=bkCBqAdEC{0kJ zM^>U|9pM10O+MG3(>Rw?6*?W~-ZqoRasBn1*2Tc20P1sh4VzzaH?_9b|M}T6M~WC5 z8717!Tv;d@Wx&U3XlQJXKV#u{p4aj?*xOhOJH~h_;0Qbd` z)V39CYwM5-Q>1!2-aAy+x+LXmqo}5W!jn$1rLrYy$Jw6aOXYuWea4YRis!P+Y^UG3 zOteM|&h(_eOz~G%psD6IOcJqxwJ7%3x+r2Za(-Cz-n|f*bksP`cr|_fP?(hy5fKrv zjx&8(j3aJy7Xp$V{Mr235|yl5WR1EQJooGX|?iC*>v-$}DBkCsv?y!MfxpO*uFXov8Jx5$^-f=PWYJ zumohVF}av%sj2a0X&p0qXF1=wZ~gfh5Exif?Sm^|JtXGp>PmMn%<^XRMSSqZi$tw= zKWh0zoW8<-=1l#^4b$Ug&I>rm>eA@w{Pf$MzvZuWvKpQe(~d1I**G46C#tBZ_;~Nl z8MuPGM7((63)91*mU=78fy}_bpuWD|aJucCSy+6)#~WvoM1KBlm!ZW3?5r=Gj5@E) z?&j_uOifLl#1%9FbH^P@CvcfcQRR71(22%SrgKs@!=+B_$Vx%KzM#Y?lgkhrkuR&6 ziOJOs3`>he{k$xq^~lAAWiN{tP9Sj)8&aR-B}^Ua4cc>*A33%l^dDQXLW!5~_Ib z{l~v6xz#=d*xA`Rh2B(FKH9FV_;YBYFAx`EABXz4m9b-W@#$uQ-^% zi8Phu`rTa*0T~$sN&Be)*QJV$w;rT*OR84HH0LSs<@mQg(+OL=8mMsJAQ(MMSs-nO z1pde^TP!7xjEvN(@{&@%em&1?Z`&=^TlCQYQlb~y$m|ZG5ZyvaTt<@(Axax(IXT&- zrS-Qf<)4uq1{VkpmyE zzp5&k%Dr{n*0;jMVDTF)t~b)25+fde zZzfL;4G*W}w6wIS+_{5gllPaUQamN;n%gF}ySJxnTER^j{S3pZ!bo%K)N=^FAGPvu zEc>&cDkax|m)!p|;4t0BCMkJpWMo8X!)0mYo?E$y+6A>z=Y^o4Afl=1Y0<~eQ*XVi z%``3&@zTsd#PeKIj&^a%DeB#6duqN94#Z;{fqSK{L0 zN}>)p29|;w)6wDK*l#hS#@!!oy3S=4jZIHKJExi5ldj&@+*}Je@fjA2xITW&CMHIA z?bC3A1CyO!yKbPoDT1d=AQzB-MWx1)Tsj&|+*1qxa@mbGI6f%30 z;8|UF7K(04Gt<*Q{Il=8@Z)~h`)hCb@8h_w&j&) z7gEC5TEBrP{hNHtell3a#w5kyT$n_+bKWisKb}&aR-zk@ijEGPnVC@2$cv1&zRK%n-DmUeGP1teDT(XeK z`?Bu^XJqhrKvHw^4O?AxL^t6;iG%q&=0F0q4>xLX$t4jP8JXwwf(=swu+^2Q`>@#? z?>OkU8O+Sgl5WRKn@RN##ZMPBgfay~%-EflIj8|E@owL~eM;OO@9gck`X~dhhw7_# z3r=Yl1O-pRAZ8)&0zQVRRV!Gdzz{-k{q=c)(gTQk zKCFbrzdyu0U%Y+urlO9{+tt0Tztby|VKPQNX)0!G`(n}JQGZ!l-uS7 zAyVF-=}7E&AS7ltfd}aD(k?_GI(|b0U1bQzsK?g=z(RM5Ub!nJgkFwMpFiiht{64_ zv^Oy~Hy$jsatdN00Sut3s`{Sudv(^r1v z41Bb|Nb=8V7o3Nfth>SU^N(EJ-9;_Q)!?j!XA)OZem>Rs_;~o&SV7tbBo`l?g%YCU z+ ze`9BRbMx@l_WW(-9xdW$D-W%EGmMA59>uE&@D6H`a0v5|szqFP0U$wIC594Wb#a)H zQRGoLb7p#a4H^;QIqSN+Pl9c4B%FA_tdKTJCMHrjis#mVMDv?>PQw#?qIIT3vULiG z9DcRd>~8&yc>UVHxLEA{`}dHr+NV|koYhW!eJNnvKmrMpVzM!;adTzTZgoZlkyTX0 z@9OGWo(Se+zi@#R5~QM=n*^Z5*AmliKO;45e|K^GUY|)0(^;!PE9UQSS%Z=u%*mZO zbH;G2+DGe?$#LYKdt_77Rmts{<+U|MaEpF)RmBOtyawe6a-o(y zMs`Zp-u{zg{0&Zdd;5#3YHBy7&k76o&bT61uU$*ua7DOydHq=lm;e41EcG6*eG2eh zp~`D-8FmmPp?-I=Nad$5UYtl~L`i_;yvNkGa&j0U2GYy&@*#`lq@-WK3vDL9<{taXcHsZ?u$>n{HKL4 z=I=VVT)=$jH(XgubouVN4CAX%tw5o~ASGhncIx*WWF6VWJ7bB@m}}u8h;VbUmWx9r zO6>xxl(3BK_Gd)M5!gGdQ7sWMF_wa` zX)WYt6)3xYcceN0hFi578XK!demFvq!>4b9LNE}Svjzw z6RP8L4-X49xPq6L^sjF*&thXwtu74G#)w$GAd-UeiS^~Zn|I0RFXBOsTTRSYj)l^P z>|Uv3O{vpdN2VqpQvD^EcBT8a3q_Oni#b*{{fBS=s5fH}>Wpm{4)`Iv1up^Cxk)}T^BZ?g-+=2GnA{#30VH2ul_2*m zJsPt@rQexW7xU!eN=%X~4 z$psQ*f$E$O#HuD87@B(>2Q|g=KJp;-$g0D`_rlnZIGk{T7;H?HjNB$k2s(O%^I*Y0 zhcy5Bp+@z;@3n*O^N0d4@+goW9imZRL7}e}>A00g^aiZ{_!N`y&WHegfWyNV7qLgb zp=p5MT%(u0i#ZA7JcS8Wi`z!RkE{^_6Po4U;`GU8Qd>zffWa2m(WjCppg?#m#Yk_! zO3pK4P7P{T~G$}DURIc@Q@o< zE6ax~NQ9XboS=XjeF!$(w9cSA=>`AsyJwX|bcP)KsH2drj z4!vlom&fqP%8T$z@pPg$M~itl#D8Dcwp=DXaYnhrRY%8&+vXl4vc|sx!_a`=vjMV6(Fi1#f+57i+ zm6erfx(y4v?J(k&>}*P9*nHN#L%4^y>f5bP&6WU}#9#Jp?iRX;v?F&lH79{Wx_ax@ z>&2H}P2m4Z78V&_r=cEUU?N8V?5{DCBTzN~9qr}i)g=xy{C?G5GoOM_ulZj^D;Vgc}j2-zAa!!nbc3 zhM;l)u+HMOc|W*K*kw_Vik9|z(ckHI-3PDMUjBRH?SNH=+VuaJtVQ2WlM@qXs+lC+ z%~a9S(psA1iB#r_Z04*Iz0PlfWcKRxJjl0=EZTbW4h?XvzdD~>PDI(-LhE6k;nZFW zG>f0%AhOE@l@pAF427uzSwl ziVXmA3uVV)DQ5{JdHI?Hmd;9Se(C;YcNUQIPX_wkKRqFWL*J-W~tz*7xxeS!J_5o?kSNU<@ ziDrhoIA{n_98hGDz0#}W`NaSHK+yaI&+cl|oz;g+<^}zbyzL3*aUUGBMAGf%@zKQe z8WIF1m-cW*PvFWPV_i7bdX#|?0xVgzT6+6GNj&MbF*1W3MeyrJ|LNgTFh3ggD43(8 zRv!iP!^5LsevJfSLXLv@3pzvsISP}9hkyZ&euIJEU?J{OA4Y5h#u-Z|N1&F1WxZ|) zCP=>LHYCm0>1;?wVab(7coe#eSmcg&-9r>aSb!i#R{c$bK(52{Ufa=EZ19yV#goiA zM4Rw3JpToFemU2fB0uB9jI0xcN$^`%XW;QUyiL;U`^XIIdlJ?nKPXL-XyCb7v)6G< zj(+P8zvW;3*C+H!!^0EjA(QwS9$8HUR7GvB0QR*!^7;ODL`%eXTBj#$nhVg3H0zPeYFLtf=d3%12pGH2q2CW{v-)X=Psx8 z9bu`dCUi)0Pip>Z<$q7rW!{rxzcP00@($gh01qszJ;GAQVI>0c83_;JR2n{9O988N zc&-Nr&Jns}Q0&9Ok2tbwx??V)g=*3-a2Q#byHqq~A6^umsZo6h8C3p)Nvx{WN2udy zh$8@Uc&Gc9itR;w0hJ}uz%t89uBla14-tjJv*=(k3Ap!!wLO>u)>wIv8un}^id$G6~AGv98%(XHX=P`#y-zgeE6ykKSUJbGO*c1)n;@`tEb}cX7byz4` zl81oVr_i}HzQtkdc8j}N-D2vM`gm=m^8av-uUhS~X;=SQhwE7)C?2&x4>x}M2%uG9fEIDY*208q|;jXC-i40(BZsFnoqA@6=2L2_e1P=+eWN^Jc66d-#->U3%o z@Yu8^^fh#J2O9ChR^j>Vbo%NS84OcB76ROSSR+}$K35r)SLIeDIDy?k?rqvzaraRWh$ZeqnL|>dk1R{4wNi>*WF| zNl6MC8qQ%450CTPBj>Z=AxZXL*%a~HI38caL1J6SB@PjhAM#J(&QH{1a*-bLdBoed z7JtOk)6<=B%3i&CpP~xN>9tW%e%`FP zz3%9d#`+uPTn`X!Ru|shISSI32obA?0I<6H0TwC&Yr=0obrsbGf!1>hHOMu@02%qS z@01bK&2I=pAL4v`?Q*nN1?wT{gfnkQB*w)>XkT&)7&oPj>0iIt&!2Cda)1iHRiZK- z#B&4zJ%E!X{*nuCP;hX{AaJ<4t8U*qUkPJPd4zdseiwSpotSuPomk2}*6M4oo~eBt zaQ@W%BW8q$im1M^(Ri*etIJa-iw2}kMF0a|rvVqBI0_JA_QvLii}^th`?KGDZH!dr zfCYMPot4F{QJuQ>y~@yrV^@;u!Soz~O$u?34rMg{&#a5Lz#l&uI1AuQ{y$@aGw(j< z=2B>DYd5vE5g_kVQ*p8C3+~&i4^i0#F$40=UuLteNop?F)?1 zJZ6YPL38gMAx`NNH?|9zguhEtW@vs{?fIY)a`@9@DA+)s3;_0&`0uZ8@}rfWb~7Dp zkACD|<5_J2O$4}@_mj^^=zUR*A~{U@GGOn%kJ^6!;Das!nZ4DMfK;A=mV<+!8Vsml z_rHHKMI56Wf&MTA5*=C{zP!9G9eDgC=NO39)?Zw#rMYnjsu^VE@pIV_JXU?g{J{N1eydx} zh{zrMM*}$74l}(sMtkq_9}<$W@9vo(>Axk{FovH6u}5D)Ieu)C?Iy-Dp*P5hIVbJz zWNl<7bv1$iuxuOk-Gqz}Lk~KD=2^wPc=J+J=kjZ9$Q=_lPwY+ zglY-mi47|_@@Mn*!U5##o0v=z--Mmgp?V?Pg$rXA&_r@i{-{*Gz=b33y)P-@vPgnx z6*e=? zXxg*4_|}HFJ$^h6;Q+yodhL(`C43sGh8PUO3JVD3saE_s4Mm6zD#KW89x;v!Ery`}Z=kOh-1^xO zgv}{C=WD5fFmAf@P|Rm3c%>7i+F$R?@$=4n9tE*^{yMBYa9Onm-g_LLRbUQEZlRhJ z=mzI6ie3EHLA?BfBq0%}*fNb4Pd8F_LKyH7D5kKe` z1pGPs%vgeZC1(M8Ms!;`6W9Xf4~CHWlQ9KBJT(Hf4#7flO9B#@hLN?`lm|CW)zWtH z6ECZ1FK$P5I8*BdmBh3p1}|o!bD^)$$x(sHfr{2AIG7mS&!AaBaY>0BcIZI5`ezi^ z!UP&ID=iA>+df?#k7Z=*fhHj7qMN; z)uieOBjl}GSku_n3ttjKF@20&aEtFLurH8=&Ih^H!xCswjC6c9zddB%av{9sCk_3n zXtNBWf$(p0KdM)Aw2%Pf1}rca9hpM2y3v z%)SbA3Ewb9a@G-{v&a$eVBRxNG6rgMuKR{H9-iJkc^W#p0O-3wcOYt|^8!OPLkJB7 zdj!jy7;+oqLbl?|iah<<%f+R})>L0;hlUboba~ZzRCIJ)i1I#uBq74r4c6)|(5Rl~ zjgrrz9#0QHJ=CAoIbC*cYKCGAgNQ{R6$|Fw^9ahYHvpK)gGxOhiBN?q4@{BVU9Kzs z`C9k9xKe3D0e$t4?y3;j0*8cS#9s?%WD}||Hcn0g=otc3w~g`2#21=V z?Dt^f-L(XzA^&v_cXt(m|Aq0c5;Ant{uXj4O^AFQ^SsWiuLMHP1Yimn61CiVSkv~{c0+{>`dU6g|*5QLUsTNv) zO~nE@=aT1Gjv)Smw4Or*4AGXxf4=3bw%_xrzVH=^Yd+_9_neOPAt~8^cm+X6bu8@A zP9NqwO{QsSePc!{DJlL+V7GH9R9}BS_7dM2XP1BPWnO_A;^aJjNi?LR;o8+X~)9Rs=@ z#nJQO7{^HB01GAjT%Q}9thEM3E%sjiz_b>s#!AR3N z)5FOj(PNN#GT#~gMpn|%^XwTXV(XQX_eicn?}P(t+8&f$39oqVTba6t*{6wC>~N8( z$is@vo*)+IgN|DDQEtZI%3jVD!f!pSwcD^5R|6jAWqhwO|Z`wG8+ef?=(>{u{?d^Pcg;LSLx9Ep}k~@Z`pX&?`>D6-Y57I@##O zNcLSE2bB9xTQG)C7Hc?D=)fv5jYv4sV~t&+`tdH`%l#kqQi?b9oLUXBM%hs zQJ`Fl)k$307fq%(I2Fa{_D{?D9D{%)nGz5h>JL!^Fm)~vwgh1Y6>h@NSu+0Xmp>?` z#&!OrN?|$o%4m|6LM{XH#FMK|lz%oGo1BqB^x5ni`c-*(Isg4yys2NmbpBY0$7X&@ zI0V~0ccE7Le=pFoHk?423qiYQpl=GgLN0|vs}M8_3{NbzN9b7*d_#RkMVAAb7610` zL#at8X3GlY77o&3Q2<(UJ;2x??}MFDr_gvYJ9eo3pmWRZcV&&=v1I59)q+N~PKdd2 ziqHdR2bChA2F=o{;$k{zc+tu;!be*>kZMphJD{8cvejsLtmc1ZZ%c-Bwnw|*^J8S@>aVtU!O%VR+j)QYu`c=73E%G|Z^0?PIsBW6ol_tE^*`2Kl@98f zn#pNtO}jaNa7MAs0yF|up1@NNf?9(ft*x=Zk!Ld{QWCn zGa?DSUpWySOb z#3|Fd@=PJv@)Ngk2Wy)vwA1cd=1ED7qUtUx`LagWKQtUH(fd%FUVahe<#UmTidlgo z3G(&5p@?5$w5K_7N)8_gyinbMl6#f&T4Z%Bj8Fjk|Y6y)lNxmigjmA83 zau*B{#C{}zuzDT+^;6VrnxJ!*wvwp|+xjVeNeLf+_@i`c;o;rQItud+9qpa4F6CD2C?@l=T>RiD zIWgNsA|V~hg}q_Q@#)a@SRAna9HcHzcq_n=?-p*+haYnX$l><=2(MT4LQ>(6kF3rz zy8R3K!N)><5LU!wezb`=98rEBxz#t#lo5Xw=!-yy9) z_zoL!Kv%uGX2e(B_j%)^^ zCsqQidX6EDPkzv8+F%upWJPQ$4p39inI}=X*97#{bbWvBv6i1aX~H^;VILMQ=6lmq zF&XCXlN%lmK!y$=dz-@qhc&4B=N}7Q^3-pog&0Sx!}$X3_v+e<7Cbz9XWWOi64myR z4Dm~?CcU}zJq#Dt<9o9X1nH4kL9953M%sOv*P!1&!LSO?Sa8@*$_?vQrA+>eKd2dI zS(3Y^5mNB1kMmTOdJuq)5o_L=k+#&f$r^oSuBnR;6j_U&_y|K>C%~1p2vMd^}(XmMHd%5*wT`R-qtU94q2^WJt9-bp}& zxAR-{GKmZ29 z-w~3=-!)glv`t0TUH|QSd}Se`OOWJ%nTZ&2b=s2|aAwcZXs~Fm4LQqi-$}OXC^l$j zoYwxeT$M_rr&^xvN)F+uTbMClb>Zope4?08{TG$y03HhgSTRs_Ju&Mrm~X{HhA2mw z^_6Ln>ejhS_b0GdA1ID>+K^8u+zMPfW?5^~XpbN)k6Loeem{`kiQ44yE^eMXFj%ob zn%F6>6xlC5?s=e4Tr%r50)VpVE;y_o3C7S%iaHHSw1L-gmzQ=zuu1}(!4T5h_{dPpzoWjGu zyzL{@RG-S;#{WPXsO0YL0#kA4Zba5IC52lMrG&Y(MhQ!mPERO5wX>-Lr?gznGO#c| zu=xA;&RF*jL2dsq^|LnG@$}wjiM@rlC_am-iB^bYT;amQ48+BSJumON`*@)tmYeH} zTr?38E&X(kS_%9~#+X(X1mNDt zSa(%hVY8)ZL+*Z#!F!e=y`QAabH8}5ZP)X^fqXclfT`|9_|f;u z3Z!9vxzgz!v-Gh#Spj@&7xX`HD#@881=?8!3i!D06^Z1{@bPLre3!zu6E!5I&NC~3 zhdtJQ?_t45G0F(2L@auG7sj#)u->LrV~$yUd^hd-u~#Kuy(g|88Fp*_gPUa<@8vzr z%cj4)p>AlHFSDJ$@l-kM%b($ChR20Y@Q#m^xg$RJ@plaR2j(3k&6ot`EFa}zw>iOu zZ^s628|%7BOK%BwPfYV>Wz_u0FTKuD`qZsXjqOUha>z^PPeMKcvG5w0g?etFS%*JC zyGwo1v!WQwm+fmg@9B{0=w16vvk8*!D%>@C5jhHeW34MlwPsoSccFn2+msXEzi8M- z9G~g0$+^~^XC^q4!9c21@%Nau+eZ7{3tf3<4@~BMKvP46<;vw47BYQ>#{~oeHF0z& z9E8VQB3e2$5B{QTi-i@0UzPq_nD`4^jfIZ)9H!F)Rm|SCw9EUnv4JWqbk1w$+NDPd z2JeiRy33eqJ%+*!GHTnO@%3(~X9}txtUp{LCVzVUW!%YZKD@7mmkDeK10`e2^n8H1 z+tiOwx90?@ii@jz&pMv(WByE=KoXa7chv@2s4puWUu3{i^<@h7Po#%yPn+OHw3*rC zK%r>pT4d-*w0?2N19#OjeqHyse2!j9gMg<}>N<^Owtch^{>E>EXpx8MNqrCUCQT~6 z9^H)D$Hx+SlOui6IQv^mnPu-i{+M$L-6~{W7~q)X{HYtJYd!t8M=&ZPLX&{}>8YX^ zEuKz;g9BgT1HeibpTb_7on@%zfyAvhU>a{nKGkSCpe_rFuupI#2Wi^ZanDHv6+hk@%yG* z_y^eR1S5#w`=7_fR7bXO1BtyFKaaB1_@5T%N>{CGucLlRpk$;KmB1PQxgwqyS4&Gj zq}ie+t9#I1l9s1{=AEz9O0`p|@QZ_;$>(aqu_DkmcuWDO<7Oj`M!tvVY|4c4?)q(B z#L&Vh|Esab@V!pqdPeLbIW2oqUspc<)v2EMd)Vvho2v1h50vMkwjD|SC_uUgvP|st zz2{!rPIq1#X&H<1|8VrT+U^M5blVVrBo*;RPLKm834NwM8si*PrR^Upvdu0Gf9jgHqZZ4-`pe>9E%KN7k78 zWtv$)Fu8ik1)p20PG|M4g2K%&B&@>C{P=p>$#P2i^pe!}L!I?~d>;sXvHN#*cC&$X zo$;VV4DVd&_A!N!`22aVu>Zb?dCO-seo?!B3NL?ftdFT#$uxvOPG9-P%qa8H&BbT= zwAS^d+VU$3_L{KbHqOh(X0oI2GYqIcWmTl^-IT9lw$szQgy()vAhC*fW@|LGo8~d8 zyXFVg$$8UzOI6I{(#3KI?kn!ZsXoy3@^K(jm#0%Jk;(;+ja?UjP{^MOXtG;=~A)zk_@X)b$Yu(}Gdi}?Jkw@Zj=AwhYw7cy* zF;@Y*;JNaI4@u!+tp{ZbK5Vs|6k%$SgS5jo(ya8^&HsRQzfx7yFuTR*g1(WXLyN@ z-NFgt!!_%9N+P}#VrItg-`S@f93qp=i}{e2o4A$*?CYWe2p%?O=gH}M-N15HYA^8@ z!!s+9y$t6ETu&A84hI^4=U8@f`J(I6FrK+{jO}7|@|Q`Q=l6*=eerRsNw4tAQTcc= z?an@z0$6~Xw5yWUIpMGu%4gLp18suL+Xw|>9ng!9trGprGS*+#j)SPWJ!PwJ%(~W& ziq{raA8|f#J{~K*RqXxXO7JZ!vVcF0iGLLbyt64LcM3jEoItu9uB)h~_@`dxz4=>M zVLZu_Wbaqb@6D<98_$;*F>(~FLF|8&H6V0GS)2c9v70iYv|lSr92!bx`V}_EZK|Xz z{{%l3>%G5Yv6jOvn6va`&w!;`w5sNXEpjqK2F^|zgZ=Mc5YrokLiIs2bcj_tS$HLOxqO{(ROlUuU za<$I3Uk&Qu3vA+f^yhCtz}sMY*-z0FETReo#giWQImq7e)jE9}(#As+R9-nF0!2P6 zg4}t_e{G+FYkp9Y~s!M`1#vCwY~Ei2XEu{ zh3Q85aDqkR>w(JN^$=q$+OY9jy5fFD19{eekJnaygQC20Tx>m-TOnF`_&xV>*uEsN z7YxILTZWE;3=)kl%Z-pVPKmkT7p+<9_P-67Cl|z;sI6@KckHwmK3Z|VdmkTs5jH=- zt@e-8+it}Zzh7y!mP-?et`jOwQ}C4$?~~~KAYOL#q7OBVs#fG2`=I|J)teN+Z}IJ_ z(Z5%mOQrOnUyr(X)=X-{0zcR0U84m4V|Iu+P>SzMXmre!5H+iB8ck~>)v=6aB9R)M}_P4)q5X=f7&D;T3w7c6TL&DsFHy!K4iz`BJ|Ba*VH!|Qsk#y zxF^J8DHLG2bWZrCNK{Ic0tKZo)X3jo0o>rvBxJDqD|o{nC%mxI+v>U-CMYQ&p3DI6 zj`!M9frpfyfc9uh+X#dFQY%;WR;@BXKejzghdOPxdQ!eJ=~KPCOO?&1*L;QwS0x`X zZVtCzNU!ZLb8ZhQJ*!ppXLJ{-2>R1Ui{L(`~wKh4IH!w3J4 zp^Z09g+SDQyT;;X9@D>X6z_E~YMnT;>}2C+I2mAZ8xwPKIUtEk<6z{3xmWi>bpKr8 zvq9Ub6J&!mt5Y=}&v9{O`WM9zLzM=*UO`D;`(?bh)1JXFPWTV&2k)6$UyR;P)Jaw} zN|hxEe5k!nj?L%LyB?)5P!_f?7BWC9I=*3MU35j^0!Sf`ve$VDzjgNPc@N^$QQ1P7 z@Fnsl9~HccK?sXjE6)~HC?R+y%$8-S=NyR+W${6Z-YVFMPs@@XYjoL+&_9aNcfJ^hdy z-Xe;TMo#sfzQNP?V0NSeAZhV?@?WK*pG*zIqfU69-gBI;;;i;?nd43?V9#D+c#fgj z9(I6ADJLWcSFdD4QaRHx%iTGiTcPhK5o1`|!cM#Y+_c5CqjF=qwLyK-T4oq{*&nK( zHBbYm-}Km)&EIfoENP*Bv+||VY-7wa5P&gcz-O0vNGD$To7w79aLh*wm+5bU0r*9;Rx!z4lle7Bym zEN$JBiHK`LczYuMFZURZ1V>QSEeTOk9+-afE&FMS-jVd4$GbB`_|XGB;OcVK8?^i*_71>)Uj zexS5L^tp-bEC9t##~-p#qHXReFzai%Wr*&(QUwXicp~ZRNN=g~%SX^sq^I4;o z+b?C>g)*yWCS{!Fge%sf2d!>y8VAqE{NKL%$nT~{rHFQcFZ?L*D=WJM(v#h64|lRL z^z*P0`gU5u+kG_NHa%kUeUnARSm4C%v+KdwaElQal zSJ$U6kK}#xq5}h<;i1f|@V8cF?Ph3jdU$j0z&TdVTCk6OFd5xv(2NyFBraWHgP@+b zXpir8>9uj+WBZ`Y@CC$|npI*(zqPaG&ep1jW6{CI%dI)}eGnfOQ#^yBa-9KTE2>kJ{Y^!@<3@DZ)@pHzVUKFw_1{GPPb z*q`Iaa@peM)<($c6{ws!G-e4tHvw+vzBBv%D(0${Kk~Ox{`-LA`D4C_WW=Mb`FJ^x zd@rVi%8U9`io32e>`J^pe2Tm-1qNA=ZYyn%?|75eK!}EFw|q^vw70?d_`zUe3+qk# zA-P%k$5%Yuk!t_ss7IxH?-CVHAnJV&a+}(6Ox5T=c^9pP$-)j_WI6>#n*ACXF_@PK zW(W!J>D;I5E_;_hH4&!PiX(9mUOa$~5G%>eF+cRrO1Q1g_`r**HSmW6;yu@+zfM9U z-@WI|QVh3c_Jq(z6HNn8YIqO6@dd|qWM)4P-YpkEb+aM2xvZJ)RPxI0*>mXIa}zx|+W}$Z--+F0Eia{ox)$9?y5VKq;7&k0S|J>Z7=1S=Y!NJk=iC z*GZtl5R1-!ATTGH!;5lrE`Jc{PH<&Bg7O^(Y0tT2u~upAKP+QCf4}@?pdKg6G$Brz zPL`J<)Nr!WG8ah&jK9r1@&S90-VMHoY;v^| zbKuo0rttCcnd#4ALH`g)TJIX)2l%TThc5|5{H%}B-g7NDi1!Y5F|<7c>WuG24ZrT6 zOOK=jeLn#gyk7tOKGU33c1ql-Uh0$EMbDJZ8TI|~yGwjoInyHCO{eJY$MtO)zld$4 zkPB<`KQMY$h}B|GR!rt8`L&QZQ=H|^CSEhCQN6AMVr@;^onvLT05qA2#N*2I&J{|F zpDJ1p3HO>duJf>Um$}DjHQq3JS368wN5d~wlcnKZi>lr9fok5j=W)T<>=i*|sE@BU zP3t-Z-Vi0raYXg@m%epPC27$9pj)uM?wSDIU@DV>O25RH2;FxcWun9kSpjmLn1Egv zZNl}t%G&)O)(!PaM2S;SmV5mb&%~9PgMlA*7prcHp0}BU#$g&ND%s>tR-v9vzU_{V zJ5y6{^i#uMzU&@b@jno8ZiBx7Ljr$Xq#l~1mOJFh$7K$-$m1UWigs%(1&w-hdt14v zsj2Bhm5PP>f>kcV&NEFa#rf6_eB3sWxX>6jzL;_DS6U9m)XWznkYs3(<=L!?1lzok zgIQlz!|^90#g&%)0MXyn7d`K~c%jscH08VfvZ(|P{kv>(8Mm<#qT4)+-p`P&2O;YgIdv;CTidd7#94_|>2 z1f#ai^?~&sTSF0~?g2Z;+W!}oY$w`Eg>kBoUqPWuU z3^+}b%=Aao+a`-P!mu|wG^7TNnEqRB=N%MPyJh_blqQ3sWSXR+1VIE;q5`5Kf*^uq z5MCt*$*~C%B#MZVgNT46k(_f-L?sB=&;*erG&xD?*~j<$=1#b`rfRCjzpSGB(DXUy zInUmEt>2nd4{`2mF!QPiG&nUrmT8vpv};*M*}vsMrIomzcb@SxNBjHFd(7yKppBF+ z7%%nhddUhD>uGbtn=`vi-~Sj;#60P!&)h?S)|1@`DY>xUS_U{CfKJ`;aZ{>IqVRQ(R-J1U1%UOlh@ZD#v6@8~0T8UYShAXx?2X(~{b3SzVj2L7Y zJ>9rP#{Oo54M$cfcVUaNV}L()c~!XXlW$=aNNKt%)|*pS2rFxREz8p>jz}`=J-v*=SMjJ}oQS_D<%MNyHCV zjdH1=C;56^{PPPpn$_0R`$wr1|FZXyeG z_IOXJZ%~P{b8toPVg5%dE0Yu9!k&cnWX%ZUYU(c6Pq_Qf7W$CBBu_5#c?!}{F4>gWuHK*(bD>Ze0ySQd^&&j&m z*N@9aI#Re}?cJhDmv7n!akverf=Gc+GF0u7=bWaU*9J2A)vLO(p067wxaA$9}GvQRPv|>)$E(zBe!X z`JJ-mmjs3926yfz>fR29$Dz!zM@s1Oy89wn^}`rU;jRgGelYfSZnwQf{nKd<9o-bPQaWPSNyf*Rox7xk2w#8m?!{@K zm=jDM&7sPYfn5Q-hh1=Bu#r> zT}2o?MxJvg5!?fKA}CP!^$(2dGX8@`UbKkuB8uZu0Aef!))sh~(CLCFbYa_ska_@v zXabnIcv&yW#i|+1_U<5&L~%f(kpX2CyV?Nyn&;7OlSL7v?x%llFVy>M(w}?SU!O`C zAKfDyqUy@)IoV3x7ou0RoO#J$O6>7&RVEgV-E&jmq6jF&t$sOG`fQuPg~#FHhNFEj z{wh$W=gbhJRY9n{Ive_E@CFuoXxFV1aEZGiu1cX5M`0wh(rvefTQv}R+lq`AHfTGb zb$s94hC))-kyCl=t+aKoU^BD*=n?c~9aWPdrlNAOI6W)H)!Ter<0{OT?7pctNxEfI zImKyzyRJLs=tux8hyMKBlP-7?nAA*)he+;?#MXqUoIm^q2DQ6proU62PT%<^?7Mee zURK7@j;XpiQcR{VNyg({ZE}RdDc$wY-dGCEeMN~oQ@;({d+O+(NNmiRY!zs z<|5#onCH(Qr^MP9a;#LpK@W1?)7P+BzqyP0aFO6WM&__Q)?^2g*Z zvJB1*rd`Z)kjF1M63&eCF^3g_krF{iu+*pcIY|#mZ(>=Mb#(ejEBYy52Xx$s zIS==ZZTYMvYzqrc&J#I;Y80Zr@8EB|*z##AaNkEssrS=#f5XZS?MQ?n)la;uV1 ztQ5)z%A7sRQKCE>5c;q|KA>e#3yZC}rX$M+d()qbk;IY3eswaUr%?8Xn`C4LyJuTQlB&s2a{Gde%!U1icBb|p?^<~m(We&)A5;!>4l%SI@`Bc%VRp#O z%FUD8ljd^fq-+b^COPDz4<;tH?RaH9PX?veYg)%DqXu#|(5NJVIeU|!e3 zS$8ksX4~E6+nwi5yjclgY8kB??u*pdDt%2l6f?p)hJjJ4W7aEjt_!`tGC_foyhMi- z4)FNSG)BwYxGzYlV)`>~6vV+Jjs00oNIH}PpNh8l)EiRs;lJ5Czj{Inqb)Vt%XA*o zLk~rxU_eEmpc^Z`Kg?e{x|Il{He^r+FHLH|VV!1OLe9Q|IUYn)r@oe&chZ-Sv1 z$D%SjVRZEyH8QTcdl5A+vVKI85AY;D%Bp3b;4`U1|55}vR>qH7qX#LvuPU}^x{AOnHIXQV&iZ_~6+Zhx~liQ8=gcNY``um3t; zwm>_ZB9h-2y;HoGCfA}m(1d!r@ENr)d2hIP4aQf>&>^Y!I$ivYnr%Thr{XBg?w~?m zAJMerIapw?Ni1s#Dxh6r3K-tKcKMk9!@)h<_YO^86&G=M`CpR)FL`b>+-cJGxig#m zOyJlH9hgw%R|kgFU7uqOgW=n4-OSt)^%5wpJ}hNBbiCIxKd#N){Dm_QqtXfom$a^6 zvA>|>E#rdn9x*@)s4qkZEvr3m;=8EZ)#8{^_J}2B5-0&$X2RJB8Wo;I57li4=FL>$ zRuHF5>YU%Kz(X4W7IO^2TXpk!eu2*8qJGk;vJK*ptL^xCX3W=h1n;Yj}PL5H+Q3< z%^(>M9%Ar5&qv~mabf0>KF?f5W4d#BX-**;7i4_i-UMt8G< zRiyk&wShm3F`jlE8ete-|K;5xb-5+vnd!dqj7ia{zm9OGaL=-_ku`qwnaLvW`hGPo zes9IDc4PPk%&gkmg3MYb2Te_=lX9=2J??*ae(;pC+Q^sRscSsTx#{_#zPmXal&hL5 z!x06!{M2BfS{|Xlw!EI$kfCYaqecvwpq`IXnGa^aQ?pZXCRJ^Y95xu6X*B`2t|ehC z%1AB4U0qxJ$H2wm1X^%?K9tKU-B#U}?>_BtbD0d-OA4Cb?pK#f6ED;{ zg@3t53nEZg{!>HL^e0`K8-hE=v0Lp)BftF$nF2WKa;WW$s>)v8-P|pc#Ht7N8~hHm zWz{T_8@Jk6N}olQ98iD~f+hi#+SOu9?3&c)K>W3>5kq_cdD68|9Z23f1Hs zbrZhX7AC99aL6z(b1aP`D&9o>=gYUtA9vb^sixwzZru85zj&u&XYjde{bGEsu}FnO zAEj0vA@7p&l>`;s&8p%b(bm5`EARJSHcUVGX@%GO3K}Ao(K1V{gW(t;l&v=is{LAN z2XRlV%9!@0w@n@RXK)iAm^haD0&p@QaoT#b=aMgr>9iFL=Q#fQ`aU3B{R?Ouq6zIajJD-pE!0LyYoI7lUm_OCub?D3J~R=9)fI2V zu{jcaR=UgvO%=So!k02+K|=MZww@7WTy1275=EYCaAAV06OlA=mFXQKzzhp%;kGb` zP0TNbeOY7=<$qSwY{=8W?$!!bN|`0|ElH9h!b`rF++mj+=zM9JE*i{dyk-Ht2GWPb zY+GT&137Q%l`Y9wiSt7HO`^)t-FAyUJlNn=`dek@q0jkc1bwqT+2ga>#ufNy1#4k~ zR?CsTw3GQqQc|q2Zu%41f-!+&#{%314PUz+k5c(t#aN3saDWcG+*rbs=2fc&X?=?H zxp|^hO-9Yxz~U))Oq8Jj`M5WUiA_Ac-BNZ}4kf9mwGz$1o^_~h5M~Plk-$= zG(tUe(|P?Rr;7$)NQBUB3J8tRC>DsyRmFSDUiAcdZye zQR-V;e1;PyQ+)NZ^~6Dj*n}FGHYqx%qaUffT~0HON|ITE{T8g|R~Vf22?^HDMWQzD zHY__X%Zi}jg&oy+s`}QrcLu@pkezf)2#iBu0&P3sCFcG}>%s|Fs@A-UP4PSjv|p*6 z(?jJeC9dhYU@0z1sV=`l>{u|)D<)zoLpxuCHfP&qb3=IN+Surb1ou(J`=p?DZu4rG zcO?~d+ZjxE3T1|2wnl;RHKvJ~#pi8G^_W6;sgna4FDRVtX98hn1*PneCoer!6%Dbf z;KFEarh45~nMJBg2hthzD8q^HT+Ro+Jw>tXBj*@0y7ONwar}sspr>!5HmgyDM#txf zuIsvc=z0}dYFj~qnZe!6CJ@su|8xsEH&dgC@_RzYtFJYFjCI`K{vloatkE>ltM~i} z8k*KxTGsDG`kbY+gS~6E0E5@hE%24hOm#s<5qaQ+E&vwrp1T)UWtGu>b}m@df}J8G z>UInsCLOpq2m9Ex_~75|Oem?|u&BM{qL6(#_vpdc7*}pdTSC>hhK-o6LoyDA{5GCH zJZZw{E?+Gphh9tY4X&m3?*Xe!DK&B@?@Da1e7(9ipF}ph_3^={y&Y7&dX3*R0SV~s z+Ej03a#OI3ezG)))8ms6!d^Vn&O`m3<9Lzyk?(PI{up;FtqZ8j=6+t`Z)~r*HdMdp zOc3xJ4mcAiH6A`kR=lz@-;LX!0jEzWjY9``RE0*|T?DrOeNZ1iGFEvBqz3n=J~g@j zevt|ry5b8Z44Gs5WNCXI?H0_5(*kdE+AF+Dkl9`+*5y3L*n@&SdOH2Rav+7g+lYKA zpLRD#M1eY8g-4oJOQNb1S)&VDgg5&&Rfe3($f$fXn_{MC<&Y;zNolYe)G#2pe|4== zJ$!(S_7@8Z7HQ}bWE&7?I?ctd=;=Rzn~a)ibQJA|eUBbn_}n}+T3v92_Irb=VF1Ip zWmOhREWyuG@s*>Z4Mpnyf&GG%4{Oo>BaYqAOvq!d+sMg7pyF7p{Agsr`@A|@=@55Q z=P#lWi^qdgh4-*6y}_=O0pY5MxD-b9Q8|t6)RE$P8aQa^%}XWJZ9dft20YYzY4{~I z=I6r(B0C~y&>>3Rg(lMr`KNxc2fRxK zWdn1G4yTp&sb!lhmlhAq>8U=x1)VgbdbV@Ri%U0##P})Ivz!%+D2!{U4p5C9-FI`4}JD`2TM2IJ-bay!W5x#)(F)vUy)W2Y%=oq2N^!G^;Go; z&oB62>T9gtl&fAC^HG#)zVm_d!vOdvz+%M#8>+pTP0$*1^1HC+vR4N+*09#Z##@_j z-L93d;`%8@V%>N4$8VQwiTxs?*_Bjxq@D{FR*ybaC2Tv`Yw z<(MxmPy{V&b)K^gowPOm&V(2#UP)TIS1^DI9uVPK-x{V_@Ohl02}8_cR|cnlf*Zk7 zl;V@fUDJ>JH8FSZ`O}eL?j1Cm9P+rNjr{^rWT;p|KwoCJ@9ISkU5?!^Oh3R@;JCJn z{o!A9-9))Oq>p<}FQ4MI$zck<{b!wny1IK-GP z-YRw(STa0Q)Lxgi*aVjaR5@mCfs~R(zic5;LX=mM1-S}AzOf#$+!VF$Z6lu$-Jo_n zy|ULmCl1%@eSngB?ZeieGb^WUP_kruhWu4N)je zO~(1@*H}XlY%TiQStdlMwuDhB#cBH#n9i>lAfhU8R$7U-Pat9-Wa~0v80cs? z$o<%T_-yCY%moCe)8AfZ9zt^ynHma|k&6)b=tp;D$v}RAsmh>+x)cQ9Tu}x>%IVpp zf*4hYS)igFH6xwKa*j!K*s@#-heO8n3m<~%KljV;IvelL!{!V;DS!1rI|H-`huPJ~ z3&zwG7!K`u2ymf48~VdXsZWjZ`^+<1j(HI7aoS*+5EvNaQN;nPJ*ORK=J|cznYovrNB;x~(OrdYnL`fmS%`Th1k?Pz(NPeEkXj8latfN@T_=NoHbe+Xq(4ZyVvQzlqjk=9R|m| z3gAFh4VbE=Kr7H=U!J3ew7>Cv5;HyhllsCdB0wLrUSd~WW^^XJF3?%pk1y*?Za%mfo4MnDw81&9ra0H|LCn*C-vSiqHb6kfMz474gCMZr^*^O3+g^NME&?=qAnqv-jt&kogAuuQlS$LSfCk}v zpimqa9Z(Vy63T-ze_MOUCG2@3l>>oIf&iYOXJ0#@Ee3yV1gG@CY&Gc~zWfEmdLXe1 zDN-)Y!1#~M&lf_~jt>01KTmQI368K3XL}qR%!cP;V+#YS?~sMoXC9z_qg0K&TFm;r zfo+7u_aMv=FrborHFOG6AOZA#rkO`~(o$y(-*r`0CAjdwzP$`XwDfDy7^DMQR2EbK zfIEXsQ&4`=~=iCjfPp7(4&BdmSE8=C=q2yhi88xG0Mw=UpM_~5hExd1B4 zEg(PzLIF;Qqd>mU$UBV%C;ot-YSpGR|3*05hzrb^RMyvSzS!#Ii$>yvhRSW(aTWF= zcrGTUU@#RQJARy%^NaPUHw;Kpq}`N(|HveC&kxRF?e1bt?EsT2xU7FpO=Vz-2M&&o z2oeS0LVs=ngltJmJGD_R249pj0e2iIQ+xa6yySibO2Al_lfwWmDsHjQstm?1fc)15cgV0l$v`OXsYO?B}6}*#d~@oPZFpD2iHI$AH0p4twAY6GTX9N@y4x zFrODQf4FwAyA%%IcZTCsK^HB)ayvp)N;yU|=^YaIayexf{NPF&{bd0jzy!HcFGF4gbdOND%Z{BF6MNm^ zQCFY{(P*U_3pbF5hh*a)j>Z2f{7m;NEQau4OsTsG`y|T$PThYcc{T6b{icA!2v{3V z$q{De5XfypRZ7Y3!el8OA~WDtY5^84f}ziJLUN>LznAcXK_7%(#TGrhnH_fpjw4xL z@Q{Wyl~;g1-v&2&WYi^Cp2=xgQ1tZlG=Tk{o0}W(nN(7G02UG@+m8s19b&)`x)(x$ z0;YE)BvGMZr1&(-18_;mp%LgUa1xN`A7Fn`v!oSBNDh+l1Q+yj?@f^jNI__2HumO` z#lpl3LFXY+S`)AgZ;ufV&=5|O=_mt{LH9~6j=+1WVj6SUq?Wny7$$#~a)r z#RuRV4>vaoY4QWC&6x->B7Tr9^C|ffBcF0aBMe*4K{NvDvZ*N#ECtbUb(6Z`U6_%( z2=ZqpAs!<(m53xCKz2wIu*)1)f8KDOAE1ThUKtcIDd!-mjT1T23t@c(RR?k?T6P51 z3VC7xH}3wZ;~$#i2=EP!B#6lJhhL3^tt_vuKC?6S**LD2CMV3PBYlyEg5okTXeS2YZrTDTRu&le z2<8CzJ?wbM%gLH_T^bek@qxsoWEqd6knw^>IWLWJLV}aH)Dpa^r_5Rh?mQ$W0fI3s zZt(!kA7NYKXyE>r+33H6%o0oWgyDM#22#YrNEIV&*FwX>NKv(`9|e(E01J!LsSw+A z=Ie19A#TiQXs_U`E>E&UJu`a}qM*5T-MHh!8+E zB8=d2&w04)!14~49rA9-`9e_m4?idS1EC1X3jh?oT4I19Yh%|2t@0PaC2KZr~RIkNLdt=0XuCvDV5pY7rt7cM5 z+J;Z^S}_6gJf=@nYu2a=G=#psKy aVi_>mqTc;cf{@u8$rc|h467X-Qsgxc7 literal 0 HcmV?d00001 diff --git a/figs/lora/MobileSAM-Ti (Lora)_performance.png b/figs/lora/MobileSAM-Ti (Lora)_performance.png new file mode 100644 index 0000000000000000000000000000000000000000..9214609320d8b78863af6388e1ef6eb3ff001d5c GIT binary patch literal 27737 zcmdSBWmuG7^fvkoLo;-DD4=vmcPI)-cb9-Pf^-Zijf#}CNViCLt0*NU9U>qt-Dl75 zf8O_;>-}`T9=*Iq=BfRxz4pDwm>E`6>>Ga%+$?Mq*kLRu~f;>Vzf?Q1Yo}O+V;=H`h z|N8_U*B5rYKRBDlz)f)69vgZ<5TPaNAFNoW=s5&A_CHaS)A7mNnfEr(w!6jLm%t%X zJYo2xNM=%8Oje=G=}=?H=7S&bg&E70b64WKQY!&Xdv14EL|8(1Sy)&;@gb0?!xWLQ*Wl=vZa#bDR>&dHSY^>t=uxs=27HY+DbYL36d>m7ObcFxuuCrSQ6ZZb;=__z$qx)*yNcUt}X=(R* zPH;kd;AQvdDDlwH(8g#!R<_RyPVc~g$VFOXqv#hKf$5o<57pJxqaKm?w3z9tks~7` zkMc$jwbugeOv+gl0^-8-)ZMmxBW=w@P`f*2j+&si0fG{>TE;p*fZCLOt z8#*v;@ml_t%&EbVjOp5QCs8u=NfO)UPTg+DZIIKf~PSe$} zFid=T0|SGwfvc;-;r)}7M{;to$tv^20mk6g(e?Vey1u?xO82vUO9Y*$#(H})r+)Y3 zBn3G+`P2HHuk1OBs;Y4(haW%Qs@LW_&F@tjxOd*N z*RJsNDL$KecEC@M*$?VD(mprm>o6!iJ^ifX_EJVQLl{3fIXO+t!!|Z&cy^Z7dofT% z`usQP+{-D`3l|>ZI$}7}n{*lpFRsZ7<5#@xXJ~qQFK;LGD~uw(e0ltPZH>AWA2$LG zH$t9w_~q{}#Hd%>YddB5Gn`Z9vG|r%g{EQ`Clsk$rf2!>-TlYP%H6<983-gZGjnlC zN!QDH?=ekZkN)CC9~>o13GdTWoM;e4CWS|9tGO>iY*cE#7vQ?=5BgoNP{5znhy z`m?3{`7FE9!5{k$J&*OFhrYhPVe2EGFNbAropdjwzqkRI`L3g(K?JN><9F<^Q}Z-K zIIp-EJBjV7gIhZjcH>rM%laJGb_87;T}`J1Nmc1bk#ydBYl7Kaz_jPwMo>oLYPh$zp{)E^K05vdZ0DVu^eZb;nxpSv32MZm2X}4vsKZV;k zSF7axeE-X>NpNF2OH0cygM&X8Tsp43pw0w(2`H-8tikD5C;%f$9v&WpHa{WTAMYbN zJ7rZfMQggKI)M`}1LzP>Vh6|;hXUuhCXc7}cEE0DI}MAU=hpxxfyr^LT}+fV1x~A{ zr&rz3;Cf(fd~rP2{E|~rQgUf|ImE2ZZyC5zcO13g{=op_R_0Wm0@ls|zyUCzo?N*w zCTujdOwrL?dN7O~frWQbMj)(ziSzaGVPa#$EYqv#cI~*vOjeG-V(`fw_0ZPRTH4ye z!@|PKT3udVp1Is|bo%c+H z;#ty8`h({rF)=aK&wsL&n>3NE_QyNAJ%!vQpXuqP3{|jpO^p~`9LL4Pbk{r2R{I{> z2K@WV?&jqM!i`*5`nQsjk~HZ+aT8NhOIKHbD})GoaWY6lM`t{jK9ot8Eot+xg7^)C z8wKB+^bCP#xVA%?(dFe_z+RXj3irFbC8#rs2UfHk{qQh;VQ+75al@UFkr4%MVA09= z9c_ewx3KDqA+P@QN#5O^|K{e>RYynX0gVnSgn%ik8nswiT@ADBiCq4vF4G6r?fcC0 z7O?ygjM66S23lGqp`oEJ{}M8WfBe7~5)!(1|Nip_;eV4_T3Wou`avE*OhFOpzS_Th zI4m=jK&?}vLnZd&?!UvsgMWj(ceuC^^b%f*etzO0_%Ce8Pk@!<=It$FKO0@VhQh|V zw$p`J_M89`EJB9f_d<^EC3*Syh-MpHU5SJr;Bh&!S=!%aV-xmC2oFb}{`IS?rzii{ z4jurQ673>BtKKL*-I(%#?e2uOOtNYlmq{r$O& z>k+Q)=WP9njEld0K}r*$T+j^yvS&i@$uGOtUu~!aZR3Ux;8+ApX9ue@!#~Hy_JONe zjTdWj8(r?9mN+n5`}N5z=4o$C~zGLjcFx=Z_e#dfmdDX zn2f<^36?dB$J+||`(M&}!DaS4;XbSJBKs5i=0Z&dwXv~yR#sM?E*EBH>a~?iqxnk4 z|KU6BA(*UHu%c(G;{R-Mj2l0CM1&pyLUFuwI|x=Fl6Hf|_#dEi=|Rz4ou8lobnzPk zM~J2Og6{ zOBD?Z4w|zJzQ2jfOC4oT}^tAT`nv9Gofs5*IaNY5dlF zHeZMuo10mrr08X?_Td5o0@XeTR`6$ZuJs#%RPyUXS);Xf>Q0O8^oi0J_F6V$cw?w zxi80qo!FsF@7}$`1{nc)Kwuz-qoX79g9kciXRC=Abw(8)3=XMS=jZD=5hEiSawR&r zftM%F)#Xn-QbLH)-M^*)$I<``>s77}2L{NJnFR#2e6}hY!asaq0aXyQgoH83tq8+` zTjEMfONWB|acO0RWiIG)r~Be~t8VysvnU4_3ZI^yW?Q@jr2#tBofCXh;+$klomNO? zO-%(N@55LMoRgCi{1kJ!9WBZD`32u~dO4!^x5rM~{&ET)6%Y1Sug(wUL6&Rb>dM>L z(t-!V6Pa3;_{4^PmZb92_VsAB-w7F4qh&2zq(E z=jP+0${TT3-JfZct|lmNTS@|KUCoxSyFOa)~R+N@`sx&TujRhhL>-9+7uA^*S2 zG~EABMcpp?7^XuV4K+12$SUk77z#nu9=}1ClYR082Uwmt)!3Fh4pxm1oMmZm&vn}t z)R3B-9}5cdK+x)UiPYqS>O;c9;Lri>m(kJDFBQxzEFb>d34~FqIHi9!0x9FcN%c| zML%ROtE;Q~<@Y--Ob%=*y*WIYx@JFPxvUX0HXy|1sd8&gR41e zvDHnP^?Z6Z7&oWvQ0v$vJC`FnY1k=6w!e$-bBZP|O55@bybFXt?mQ+!f2ZGYo4`b0 zPF`UY#wR1gXk72q<-gZfgda=;qJYmv-m7Ym(+3<+=$ovLkR`9d-GgCv!SJ`IP-yeA z7>^uMJ)8i*eLQ zxZzH?G-G(`{_I?K@;N`kHk(9dNCmy}87$gR)~Zd$)+uWXqZ|9cyn#rz(mv6 z>gDHB1a{-YR<*(lGp=+a?iJV`p0va#T;Pl+XP3A`iP)i)AvCH68gD^dm_12WqTi$8 z9JnDL=U1Ng%n0wa@vuR3h7h zAnLMZ*P}^jX+akep1u5Xc|?rJek*}p_nGX9Ud>o)wXR5KV+5Mi9%)#P-Ky!uj}M=* z&_48LJbFt0nwDJ;M^HU>nQsfBVTx%Y#OYgjC)8hnCm6SvZGSH=sjbaaTP|Cl3`<94~&gsSF zlv<87eMCeAfLWQ>X=zb%VVJ0Nf3N+LCp0X~eRoa_RcRKNm-m9uyZrYrE;QW{ECYy) z(1yhz+S7S&MjZ}vy(+VK5HGRW)iPy=v!#qD7`~dR)ZsQZnj!)zg(u>RYTY3}b*Q-) z!niLxUh;uz5^vzH_H~|ydqc-snpTYk8&10ZJCbzJJx(@66&b17>LwQ zJgM+0)?MLU{jM!Ld7tST{k2#1ZmNLA9A9xMjK_tv!wbjwZ{W^$pW~z}%@XB(Gr>CC zFVc7)x3S)y#@wcuH(b4$quJlm*JnOTM0292ZReqWq$XLtuD!H<(fn1ls{y8vjpdYu zj7zEu&dns@oId7Up_!!btGv0$8_74m)Y*Qj`;gdegLY|2A6cqQr?gFp%VzI1IU2rZ(Ti2PLcFsXr$0?32~$Y)hcXiI8H@Db zefqt8#W)dS>mhUD{lUzzm1%L*GHY|&py2T8$iBvJtkx`7doGFrN$rs z?0^>O5ez+Pi6h0VG9&45Vb_ngrWU%&N46(MRaJfNIPkzmd+jf`&7<~<_r~~biGPAB zPbALM2tEs;41ZYn zUY|_%-&gWrKy~Or1k~EVUT$J;ZY5{#+H?USYHzHlj$+B+D?e_6>qs$WF*}9%r=WX6 zZyQsDbgibvZ{Tjvu|9V=i8rDVCxrxQ_r0E^DW$2IZ4A2iJ({#9W+d)e7+pZ$(%4~p zb7}H;ogGy<57y8vtKOy7$s~P)2Nnxkw7Ep0FhY|E-M!HB+fB2qK`<3EQpv|TqWa{| z#_Yw^XV2$K$a%&sHAP%TUQC%qY3^WY;Glk*B4oDNW7n(73NEZS%zx%v>Kzz)f^J2< z1a5go{>d+myr@|z!U@`)N9OTHPcdIpxvdqSxelF2s;GSZy>bih2zse2KkSz8*(L}H zK=n9qi%Y)0b>E*tYFF5qCiw5xCg}GhRS^W1s}N^_rCCceTI2iaYAb^@+-ZN<{2sA_ zc~MB=b?o{h%aNAUGihT|OhL7nyuI&!xmwx0CuAm&qDtE(I#%)Wov#ce^T)hGlH|b?dqM*9a_RUoF z?mbL7u+)O3dU{J+@88~{_pap=@Q{`Ty}wmqu(aCW3Q}K%RsMVL(>9hk8PUCR6uEBb zO{UONza>iIb|iRqi}ByQSwc>XJ3-uK-+B#m2ZA9j!g2bBQ_Kr#t_71Y{KbSq9eoIw z_UCQESXzB#*FQA=efnFu%%~5`WK5Cm(%QYz^g9&~)UV9O(7Ve8@yqVeOZBGr!@`iA z_BAA>d|6MdG4Ein{Cn6ULox6M&5E4I>MhmPjK@_m6ZBBup(TD8@}=xOc%{$#m#0c6 zM;EIz-?7cYz?k0?=k3kMq)!1gGu|6p%?`y%K5a`=-8pi7=~hIlRrukX?8wr$k?X(w zry{$8tdd6>sul*u1^(BB**7?*`zn`HWtr)p>Q|}~geOpx1?J_S;WbH35Qa_h8kx!< zn$f3t+76Q%z099R_#|{cDjzw?z(pra*0ZR*N7BNteG0#i)oUI(l?bnMC$gi#8uy~U zd}uT|Em&tn=T&oQ?f#vUDdFqZDK?)^LgQJZPH}0-)!mx0vNlubA4%_5;&{z_j{4}u z`us=Oq#Zf&*B*sjjH1;#h@kh#KfmayE(C~zQg3c}OLSL+#&ocMCdLffnaUg$+*)@w zKD_Y#VT3>;kj~xEv^2txn=>XvaI1Hz=$sWEHTdbsAARGHXu#n~SA4ex7G=%9qUB!- zh`=^RvMGTF7!%~O{m}q`4gCtGBk-4It6ZeyZ|@z>l^Ir$*wyWw>(KoJM9=R_8C=ET zl%wxsm^wl$XCoJRd?IQl8m5P@z8#YZuk_ zxxz~EB@JGfFoA5HjJ5>`sBDauH~dN+ zmX3@p_E>6E$cZ{CHPxX8Nw}w{h|A^ClL|Ck?Oxe5Qhi#)YM+CFb?qDa0HhkccVtgS zBQw5yxMrZhxATTKq-SxGn{{norTpw=%g-|zJPf(L+al`!Jtm#|70zwn=oSqXs3PbA)GM*7IDY{()U~ zKm@nuz?EsKPH;z@%QtVCU&8;NnSQ~oeucOaQ!zR7J|Xrxe7kx7>ll8xpL?*o`j~5s}CG zcSqcCi3Z{>l%gl(_spL1S_N`bPTz{T{SJOfu9SmqvB4?Zz&~~sG%AQG>w=+_Qrrqn z*UMh{OmiZFZQj$hI-Yt~&KI?Oz)u(Wy@Hp7QhVUj0W z&$A77#IBy2Hsp=rY{)1e%pxm;L?(ZfX`r>SMI}iDupllkQj{wwDBn!Q-BFb=X`pH{ z53X6ySBZOwDT9wGi~LDf5&}A23T#PoY)M_9GBYzXLs4Yym%9?h6E9<@F#o5l5>J$J zUqNS6O@nxOc|k9Y<-r3oP;)EF%cDU|Vq%7ub5n~143hTKB87Uf6EfH?X2I7lvpgDH7ZXs;1!EhV zkCuz$(K)>?B(zu-za=6AxmoVl>@w+H+tS0>DkxIpH1**{dayxEn9Mx;uKRonLEd|1 z_rYG*NZ4XL$-sva$KI*;`*6PX2YkPM4~>TU-g&yg=9=c2CYJ9KIlR*<32$f+fjWnW z39m1Xdr;gh@tsE&fc5kNEyR~YyqugpfHLLt+BUelx!jvs1(ZZoOpLsxW!})ai;K(5 zx+~?g^Uf2IzQk7@4l`Q?#C0hq1mW@IuKEcutadVXYm*pqSF^TWoI6r`jyGwoD~Otm z(~Z-Y^WSk64%kdE&NHs`zg;KJaCj~yPEL6+-bPO!h+UuePSF2 zH9i}p06n|qf>bkyqZ+{AItN_~M+X|u_0HEy@a^NQ)<-nOetAm$NAy9olrKEOPpr-s9jf(!m(n54I<(NZ?psz$GNU@ zl(|{3(QM*%NR7qQ1Ae?0!X_k~(yx7As5kNSYD22Tg`1mY?Aba;!f0$Z_ikd_t&01f zjSyC=2tLn`jUr`*8B^cty@dCYRzCD7+GtThvq^PX2JwRZTYsOJn9zYTg_dq_rj8S0 z0<<6?XhZ@Tf`fxW1^du-$6@S%v9G}VT|(!V2K-w;HjcJxI+dHYs5@WJH+I&;^X4V_ z?vBGN>7C=|A2_(`Dd+C>$mYC4#?JOdge7^!#wx>+*6bFY*4uwr1>6Cn@Ptip^IG}# z`qUgS9+-f;b9P%S*34hp-Aw=m&|^E2s7Ihva+wI{YQybqXAP3P66)+K3p8Iux1rRx z*i48$TXUl+)mb5tq4cO+ZXn%!{S~>8mC66P3@g&>Se7x^?%wZAIeUH^fttUR;v`>0 zaZIMMeU72#`L8Z!EP|c4edEQTacT4HnH-=&B_wDwL|ps3Bk=fL=0sJq#Ou1w0hQi+ zd%bvT*&UAaU*mAQcK8G6e4?5qGj)z{-oK9l4Y`N@H=w=xCONs_kPf)mh3Nx{6K?le zk*e&KPS3G#Eqc*w61>YrVfTy6iNhATvrPF}If&K;Vyd^~c~)wb-8d?UipFpc9y+mP ziW{s>YIq~c@WZncOqS-X&Q6G36>Ws6pb||mxX8pr!t(NRKt6^>MPUaV^iie>I~#?4 zrKLf2u}}6E6C>~_tw2xcfA};RRDbEew%&SwZ|?)2E5H+>IBOjb9qs-OZvqc9&`*5Celo6~ueCa!%;L->8CalU_rt=! zfE+WT(b8UopUCUQ=whR`wMGeslSEF0V=~7a!=JA;ui5O?t$5nw+OQ=G&=lmQ_&%3O z$V+a!UbLyDgcx_#pgsPxF@0)vrY#P;vnb%@7+3oF6DH>g`)bB~s-}(^K7@Vf)!F*S zl>bwUer+(d(f!}x_2q1G@&3eb$COQR)SOx|c$t3b+wC>^ZCchndH8GEPns4YS{Wr) zXz{bM7a>%{Xz89bvaCKWc5NYjad6Ww+H02#L1ZteT{gF4YT5$$nqP^`?Ib^PVV-|% zsEaQ>O_s<=_|ZObr}cHLH*_ZY%j$8lL}a+%(j?EWn`JdSUO%n{Z8h4ow2>#b${~{X z5=Q>8GcBE_uftmeZ@1021jg;Dy`-9u=@t}TO|Kf${@&8 za))1}khA~jDAFV`No?E!wk{ak_jf76gieJUscd>`-8X5JU@YEZ>7>p$H$O$G%=tXB z>yWz7pz%I|KS|#+H9F^tk*|FO;Ig{)FV8x;v*!5AH7azjDQqMV8T@-2Jnxp(S1F@b^R@6|BS{H7b(7AF4h>GM){g@aU@@8ML zG}VY?a#Z1pDuB3~Q-X)k)ULQHRfIk4S`w}Sp-Ve#W4T-?YI6VHRMYSyW~DtfH7cU* zm=L+jD8!49{E8dOkCo+`mdANIT=Tv-O{~cfl1W73do)g;DRIz_Kx{*CE6pYEpXV*H zg0UQ*PkqJj9c)x|`P}1x2rzwaE8ZCDB&avAElsWJGhrj^LnARNb z3uk}r?iWlKO)~o!m}!EEKu~aoc>_xp!EbYDRdnNNat7EeehsN;;2|6Gxuvs5Xo25- zUf0nna}&MK=uhHNBf(5a`ANkyJ3l`jZIpSpv7*d?8S$c4FrCkuYfYCUp!z<6%($L$ z8(lKh!Vl7(X5SUC=%+agX1{)czChD#02}jvO$qGq?RUIc8Ax=7Ut!g&!(o~2T2bD@ znFDWrqd_Z#-A~|XZv3Y-1ng^$J!z4unBo4h6J0R5N?@w7LP)bh%X)s4)8L;gi+}?> zHOB+%J2i34<;8iIvSsGCNG9yZ-h1BP9&55+^<0_Yd=4a zNKc8FVUGSep&}xE;$ze4+KPCA1octnQjfeiTHCDdD z-14{_O>p(&zM6Mvt<}3^Z~rO7AZ5-buZJ+%D+LGH*PJedq;Sh9C8Z}A4*$D${?VM` zK)(M&gJ~E_wWqT?PeRS@#GPDtNr%D&g(X;YI+Y8c;HhN{_27#jFIj-ux z6Eo%)@K6)JU?Ft(3VgWsQXploIsBi4yI8XS<;8M=M03}`<{$ID?G;>L-GMj2$jaXaBDP6P zZe>OXHB0crfGtLU49ZD(saH_!pKc-o+NFq79dFUR8}wJwEt%6u$eA{uP&(8hr_boP z7GL5FX!XXFsA9%#ZpO2_;>m?~)XJIZ1C51LPCTJVpJSDw-D!f4uPW7}ItzJK>oL5+ znv!|}AIUnP9{V!GCt9ECUE40Q@kgsNW#zoC$nEpVB|F<3>E7a$itGG-L9&qOJ5TZ! z(q7!i_gum3B-e?MB3|Xde3gXj zCc4Qn^pSE?hY51mlD}orKl+RHR47vC?7+|~Ly__;m9gQAGYwh4yn|-)9If^;l7j8Ay2^nlPy-m_INO)>jTfM1T zD4c75gy#mJt}S3`{X5^ZRw$Fj#IjE7sXE=aQk`mjBg?MF-L#R8B)JyKpTN`&UfW9x zvA#3Z$258`sJjE{MnyUeThCjVM&kO3CX2_<{7#>7Eo37dlA(qKVV}K)?8bb%!_o?1 z_vzqG4lk20T|;1?eB|*&i3wIzh(pC&rc|%=d;R|*pvWcWH_*1Tv}Bf)k^+r3X&D5K zMj7%wZf+q-^)9x-mYwZKChU8hoC<1c1esY`%d0S(6kPn^8 z*MV8u6}Qs*mf_(hxEVERq zJoS!ZlwrX0{9BbeRlcVzM~zB*@avdAfRH$h*b5zS!FOq>oODR0Sr?3PzbnRAR7u9% zn}5d^f!n?*>#dz+W{_~Z^Q4RM3w+(R)H z2$}~#)`x==8#%LDNH1~oE<5)wZv^6?Atal6MiWJPC%+L5lP zJak*V|1CYGrWW(A&kJLH!d7uM#Ip3*IdhJ&Ry0T~YSv6BPRzgk+}u5N|I}B!YeqHS zLkkZx<@WjnW5PV2x|J-uUM)+_A*GO_3en)jy3?@T&*7*OHON(h7x5fp=84g<*u4s( z<-PJ9@!i!umS|JEO0sv$eJ_jLm;UKOov$^#1AKyA&uDz6Lu~~TY`2%9AGiD$0Hmp>5hB>BFmLq%6hFy7qGWXJywf_FWB;HJA7D%~Qj@{=+ zd#0Z7R>}R)HY@0EwtjwQd-`l=CdNw`SCxhcIq(YIF_^hz0_{dTIV7>cMAk3Q$1JS@ z{wEQCz}ZyleL&FtdzHog31@UpQmOIulrKD+FHE<~`B_X5VG8o|tWr%%izs*<+)}?& z^pOO&331!hhG&bv_u-U3V~4fqBO1Flo;%Caa+2(Jt#JzjET+m?YWkH(VoNisxxO}2 z_EhXpWS!E_{h>%v;30BYju_z4Jv|NRGzA=4=0*`x(M=naOCL@vgK96Y=W%U`A)r^c zu(Nx0yfw9&?Y}$9kR$9&gNuiE0BkDk^SxBJZ2!C9umxxVqg3W7fie)Dk^<=^Hc(A3 zZfxo3>3#V0sZ!F^!~_A@hM(7%9A8Nk7LxCDCg;ocNYbhFZtJ@4Au z`N;@_?w0uM)HGLhP8Vis)yuRo`c&O{7&GMviz7C~JqLEpI5GH>q0y6k&q=gu{+fB@ z2f^V$WP@+Yy}GCI%JIQNHdg1AgEPkq<-Y~yCI%=QUiq$UnWomoKJb^ASvl7ISh&6}Wp$E=AqwZTcV{^pbI13~{mRa= zj;v9&FMk;~Le<$6nToSg>>OMXNLvy}5U6O@*I!*r(SS@}r8(r;|MURDXap6xi<5%ATpJ3}u zFJ5CTV@^qb|9gc5xbL`2!vx<#T;}T=$`O& zrg#`m`|+sQ{rL#WEL6P-Vqx6h-Dkph>Wm;sJaklN7Bat5II1xYJ6PFpq0m0(;wbAs z7)RS~d>yv4UQ7EuWoLCSF3nB7*Yk92tKN}T&2Fh9IP;;Up^y{V3rU$$P6o!YU%zjV z4mg;ycmznW1Ecrjlcaqh6l3Awz@_9hL&<6QJpQP*-dt?OwZbWd29?TO6 z)Vwd+{I>Hzw)|xC`M=1?36;*tTQo}Hm>h36SmY1NGOvnXrQE)Z)mB^(HJ>2Geyr2W zpCE+P{9aTnpXODg(D?E!O@I#qDOGnxa_wAkV5Tx>FDRUWaT858o|B(>2;d62!TEu`~suV!jrJ3OmyVO5r;7O1V=zUH*9gc2_Z*hHytdQ zCxRJ48#TXsqxc|PFY!YkbBYFG&kPfaDPjSYR58)K;C;5-Ju;r%X`|? z5R#!i!QjY((ep6ngi0mQ0(y`HO~2<^VpfP;N-=in0^2RhrA~l$bTyhiyAxLqT;)ds_}1Z(wXR-YKR^t z7K2lDG*0|@GQ`;jA0)K5%bK5@fZw(W(wVYKcV>Fxgj%@IAEv&jnuzRj+WDkN1Im;A zZ)t9duhRN2@&S}DN4zZh?0={hwH=PJ+M;4xWnc1zr&89*gurU`6QEtPhw2>(lsV@n zf57kbwDFZ^C?zF*{5wmW6P6kz6+bt_pHdqcVR5pP_jX3=%WKuYrwt)#Co7h9PEJll z)YQ=c)14umvWckp_&xyhw1KLIhK6CUi_)DPFw-0J;(4Q%hwnp;d&aq89+wTwHlq8# z8J5m)mljWvb3RM!mmwpgRz#kJZRZ|zH+qO*dXo@hyqWRLQ{jH*JtNl8c?qkovt6o% zie)GFx6&$j!GIF7Oay_aOR%SzskYceBL8gZnR|Nf*F}cjwCsg68R6FXsjrTeaosNh zJ{_Y4$-E(ugXVUetM}U%sD8~4PCre$%d2Fp8jaiet{8f!J7KZdS2i6aOr0DX zM6jCOO3u%8;BXrNC*I7r@v+R6by37laX9YpMjoE6J*I)&@o*ta>gGESG77qQpfpUHjRXn7&POlcc8pIwp2Ib96VAYf5f(`aY7{%d^r? zW{0MN3F)$Sp7A2TJg*2k?|E|Wv%dv@J&3IEa8;mG*Sr35v}Rh?yFN==^G^eJ9L)k` zV&9p&d~6LUJ>Pw?T4p7mFsMHyxoHr2V%*o4x*S&^%U2bhNH`Z7_amZuqy z01Z%6*4RQ>>NxJRIrXzqOM@UC;p61SXI7odpK_f3ea{sxkO5&sDk)r}Jm~X9>WnC^ zG%ALqXQS%^@jtyd2PuNcqr{ z&aRy;Y{{Y7jNct0U}1Xaha6eUb~0I9V1_v-P1;g6#LDrBm|e})wlq4aesMcE{3z8) zxC(tivU@@8fyUQbfNEPwPad$e%kLbbj7O1hv>-hN(wAvHUKs%OKE7m*8sDb7L znAHOMIpK4n=8vr~joZ#bRD zowB?4yOztdm1KTP%teYl9V6z2S*lmS10iTCK{$<;H0B8$uIeRcr>&c>mH*+9^Ff!o z!oQA=Sr||oJ-(r}|2f#1x7L|^3*|2ec{#qX;$oaW&J$f$C~nYd&!XkKw9l9%GO6Lk zalHv-*ga9+4sfRmrh41=<$g(Lkugd2i2)6|oQciZW}fMOuXEOMyNeZ|$)tbz!Gg2N zQYkf!z8*{i3KEo%bYd;3cV+*3^ewueT|^z8R0}$F9W}S@I&R!Kki_~Gdo)-s{Ur$?dWLr=9i*YFJxbdo@WLj_*{%fD|Fx4p{-F`|}(^JdY$ zcUo4uZn7-;WU8a?Cc;Kco|gvGOTip_hK`b&&&sXJWr8rE{lC+0daO~EtE=O%oTuAi zN*~|{lZ)%_ppnDqk}v?0K%boXmHHt%eH9z| zvyKUqbd}R82GoiQh2nPB;)7s24L8(pN;WB%LKkxw!e!J7S9^XOJpRGklO#jmlS?E5 zoS!^n*cOy#|5N<-2KukZZ)3W$R~9_6aiD^pKGaQbyc_c=-8h9GLW53BzxNE`HC4YI z+#lQEvle;v_}}jr)Vo_#{}B_w-eq~8-zhbWQVr+Sk4(^1a06p5ab4>Iu3~4mB>7k zQqblr)&yrQT|hpy>75itFiDRB?BMM05Y#FjY1|I>SqnwLM#vsuBSlIeBMXQuy!m0y z-sWm-HsCK^P`k7}!3sgsy+bZ5XMYiZhd@e@zChK_)AxS+SbB;8j{xHIG4N&~g=qF5 zFjR!Z$miaxIPlICF44El+Mm6sHo^~M+o%I`X(rjit@kboF0R|cNO!?{{; zby8y(B`#^(ok{k$YB(ETo1zsFOr{CN{Wrq@j4^^P+eE%JuX5E`0u>v=N4l`81#WkS zzm@5nd1f@O&2BH7F4@>Q|E2(Ae!>W>H&PBakb#8j$l@a6Z!cZUNj}ALj(#nG7ZQAm z<8cL>2I{S#2N-}5?8e>SqH_MNqp7fE!~Rw;?^) ztRVo15KuyW?c54z(#zfNv=Sd#M2jg4cZRcLqk&F<`vX=+C2d?cE^sR`R{WGZa}23r zJMLl|fHf3+<=GFC#%isNj*+W_6rb^848-1_5INJityIqea2^f)sk( zk-)?2E6@x;CIJZRguCmj2FkF6LNca?kDt1HyLRb`sW(DzLU^zbJ`XUmkJa2vlY?eb z_-(l~OlFz=Lcf@@xM&)w3Kkv(-`5{58B97pEGD^ z6UbtUzH}H>v2XTBWTjlPSC&>QAkS+xD)2}EWgc)nREm0o>ehLNw6yJN34&eg?jm%1lE1{y7yol>7JsC>i z!3C&^eu~jK=aXCb$)!!^>G2+$BLvbcP7EvW-8a7rTRgJF4PqzSm?)13Js*N*VRc>m zxvH^rZi=^&ve7{7ZDY8Xxbmp_?{zWtX4htcd`dXGv6gHCKRoLVStusLb`&u z_2XjyTDCZyvhg#wB;fw54PGwVtZ%@Ul(%5LhHec&-)vU=kWX6rT$Z-rxhkuqvLO4h z8m&i%kjbq<-j#SPaMR^wi?l2Sfaj0APP@PCnu(_CMY*1u^cM=IPba0ll8W|4VbXWz zEPm;3N@2aH>QsgvckxpXm6vBmF#%)k^tv9vkRck%NlP0LagpB6&|C|)8$M>JOWP*; zYu#6UmssBJJ-1Kkkp#j!9d0jxq=GAJj1c6dXJAE6(mhcK(8I|#_r_&(jvhFY>jEtt zsEDB1xY-miiz_=;K3Re+WCZnd|Kb6V)in}*zTeo6M+IHTpcK33sUhUvXu(66pN3;k zg-9SyM;!z2BV5SB$n&*kw|U@ArI(L&03n^vGjy}&d|FBRcc4D(eU-7wZ*R%9aC0dM z7SZSRV9<#ASEQG^5% z$@w84^qf2()5+aGttfg(cd1gL?=rE{c`K{Q7*?snFL zIymba5?ZROS;m|2Vhc?jKm9Cs_syL#6`b9 zJn!QtOlF3Yeiz^hZwygT86OXGD*RMRbLD4}d3B)iF6b1?g*Q8){`FAtL$LekCwp6$ z#rVjNW~8e{Pl@U~opGkl34#VM_bHDa)D9;ylmiQ+cP78;?aM1t#Q6MNpAcLoz=(== z$*_GVKVmi0`=J0RaNcAgM)*+29{$fUxAh1*NLICxa&8;H_6IEp1?dbZ=QPEtewHLV zhQ|q%4Q+Hs^WYK|0ZUXIGKM+*x@ky%xCTMyc0Ua-KvN?M0n#?I&!6x_YwOWAR>$QQ zxl!kng56-aIuk1$+z2FfL^)y^e&-;0w=l$b-&*%A^^K5(H*WTB( z*1FcU*7tg4w>P}+7m9jgfO@jhBJwq8QRQv{CO{gJtvQnw`BZGMX%L)<5vcJ z1xbcr(KNQ0$fo#wk{<3c;EMT944ub<@m7>=(Y^)9S$1%9iv){9diA04qXXh%VmqE> z*^It_A7YWNkHqg>L^1lidVF*#V!5T17e>9_Oce@B9%nkY(IR5pk+`M^Q2(}vO9#2fSn_|YLy*N@S zX8TqytoBSUTu8TL#@LmkWm;pNah7BA#1Vm%NH^uC9KL>dH4eY@==1&EFX>$X1W!f$ z_;F~s`>xEBCr>CA5>7w6`+RK6q&V=Qa|C!l?fGGu&mxJqXg2PKOZ){H8GJ~9T}zlG z@wO_4#>h&LUiRpX6MT%S`i6#1r0O3&tU>Umj>q%jLBsU}H&-TWJ z+4}iLEL-jXZN<*C1NhdRTN~z3uV1}FG+IS{{f2>pbLYg7%{cRL!>jspq}takmE5}u zrh&_{eD%+o|338T@X~ZHhT3I+LEVj0wryF^&|AL&B?mg^ z^6F(SW=s?`514!9J8d_>8KmhrhoHT~06Wg!*YQ^(r zB`_M+rJ&l^7C5VOKYIRr7B~aXo;_P$o0pe|jGX6hoDLm2mBH<=DR}pgpBv@#Lxv21CS@g-rlux1 zBl_;{S)b#Pkm>HO41kB!;f9`yFp}B`h|WkZGSbJ#$Cu8Wh|2I6%h^;rb=1pjjWDaA z;dg_LZPpY$e$P9IF@&ktjfoKe6^N0%<=)=jQ6MUi&uzJ4VZdN(dH`yhKcMq-tflVZ z!&p64TkQA0t4|c`Zjxe7y6+3n!oD?bZtH1~@@VIx;%C z2Ki!wT^X9w2hXO(F37_65wC8`l8`0*w8WBHlBnX4XSIUCN%dPldGl7@pRv78NiW|h z+t(F)9ZAz0=L)k2?=^&7T3pR|5~R3kiyWjjeD;>R=i3) zL;-TJMx8RU`^lHmHPfficpKNsU^}fGXYQPHmlyT>*nH22v2x59%cs`#p&IIeTf6eG z{R)xsZ{93hCXrON;54MOyYQw|GemY zLOG(${guAIq75FO%;8Y<`&2`@DzPZFo5fl?7pE93O1Wu}Eg1YF`u3aQ zy`HD95GCp2z=fLSvE(B1FJ0$YcC%RAv}E=4G%F>ZP^!^ZSZK;T_iWLcJ;-jImcYZU zo8A(|m@y;g_aQO&zV`DMR+fnhLb6{)F$GXpYrTE3Xbk=FFVMh<7TP8egMrIGASo^rYrvwTEC_ej%n zHj^v+9;N7qHB$L+vXBC)P{9_Pi-q?I7PBy=*KdqhOpRts$<*ECPVn?&N!wLOv#n6z zOW(Aa*<b$rd>peI)?8|@WXt}o}eoL?`GpXiT{Xnc(; z(;+{Fr7=GQOYGR$YN$x5I^DIgt4Y`5%61C7q!2u92|xAZQ%5)*Z<@~t9T2LEOulSr&RNHBAy5-Yk{p(u_oA^2N?$>a5wbX9HU_8<2n_Ya^y!L|}42f2<*ZERlu2 z19Q~XA3rr&s$$B4rR|(mO9H7@WWrYyDhfbQpf~`=A-20INe|!o9t<1F;sZ686F4t9 zu&Hi8)m*8p>(^_bYfeCO%YhMs?b|AX28)Uk_yG{l)3U=Wo?N$!MCxj1eB;vV9h9Pz zI5B1ZJZ`?}2qKk{B~M^AzztmnhD)$X66p*G)#)tt_+9Q=b+9DSrt3FGomX1@eRzp+ zycz?nDU%C*!^2Ujsj2?%XKvM5_U@FBoE)ooj_@m-%1T_WWm7lVV{L5R;yB%F-isR- ztFh%q>;<;m;4*8X+o%#<)d2z4Q6Na#Ag=<~i`jy3{X+?Ao~g>7Cybr+9D=m!U20cu z%YBrL?<2VCKd<7jkj=1b7UI@juqjh6jA9KDNfA4N*CaS|t5JobY|#-L^zW>o0-ICc zfnfkA&I@!7iQ6HItaKED7nWJw=ihE*qXQFg0xwlsT7_ z6eVsSz*u`!z~$=M+Z3C)C&~KnwoK9fchRs6s+yXnb|3y?7kA1$-I65Xk3QMvc@g3C zBTqY=GsG#;Bu>Is3fv*a$ocnnOMf3>T?GUN_JShecFOs-?N92OnMAG}ezw!J;Zhd( zYE#Y_aUoA8xewVi$4;H{98(9n?%uD7FFHvp58QlqW0Q@|aY6wbxOeYfwyS4e8+W;@ z9%HfrMX(!n+Y*Qf(uIeRp#da~=jT2#6n-O*hf{nqVWFyol!D6_6Lo#t(g=;;=Nv0a)D;BG@uLF z=qC<*eC?v6s-mLl&XXm`(l9GGa*VY}M#l=2D|4g(ga89*nv_(g-Gg=6*7BN~VtKnC z?)3^n`bJZd_GRDUyT5*_8W|alf*7JDIS3BXX`{3ko!xWu3;5RyLX_vRCBZkxzbp;R z_KQF zBd60d8D8^+(P7Q1D6_?e=lpghawSBHh3Z>IPt^#9*@MAF%qkmVm8Ys2i(1t95eZO7 z!5D~gqW^EaQmXM){&@Nm*|V~$FreZMJTqZGUq2L$%IQ7L_W06svXqnn;m{s#V!Z5@p|NN&WmE}fSo*0w$@VA zMF(0gq1h@q&lvsKyw~HOFIsHE6*H+3p{r~>sM8B*{ZC#(ZqllYlxnQ7heMVgXHetQ z6y{wmiNZHy0KNFC{HOTJQcXMCr5DC8<`6CAEu|va|8h_$wTW zyV;t8R3)c~PnK5pL^Q_RUYo_NqkCZ*DaUOgi;ot&>`$s;-;$(o3#-9b)^w@tUuse} zX)Rlg#`9yu{5@EKtk`nVe>xAX4Ny#)k(ry$r0_n@Fw1Dw=nxW}oFb%~tUht1oH=uH zniRX6j^c?yF5SY(DMu`tG;0?fOT~ZMEdwKAYfMFlkz;?&UtOQNQoqY!FBvik7>DT@M955<2lsZ$S>?y+R#TG=fQGbmS;+F)QvdazG6J0@ zN5x?@UIwnRX>G`^*Cxh{{+u1ODvh<77n!!<+#E$k9ztS7Fx$(jh{Tm&Iz8{+i;-K4 zOdt-BM5?Q+yGDQbUtRLo)RD>KWyczp9QZg2SR>(GVVISJaTFUD*Yf5oiIO1%MyFaV zTfL}>i3te}*BLN7ItyQ!Jo;CQ>prKRaMW|2!{pWfA&#!_&ps<9QCnME5@bj6rfhE5 zAdm}!eTJxL#pB1hCL?z*c)j0V7gQ!b&=xWp_$N=-@R`8FL1@tpyXv~TuT&}OvdK1Arp~Tlxm9yBNIVK1G7(%Y0 zl9tz&mXrv~$aH571R$vO<mtrZTEJ{{j#ch@{`{huVoY*!w(m;hiy_)o z>(DS@YN0}_$Ej_1_O5&PqSQ0sEzv<>M~y66v?#mL!}CN@5g$d*!C(Z-dHfxqGNKNc z90FC~G0^&2K%gJ*NP8i+rV85tP}4;Ff} z@*%NYw7L)Ftx9~Mn4jT4OIMO@r~G;D87IDQ)j>_HWS_q|jM3Kmb)~lUvr$1;CWNU5 z{ZxWY6l1C3V24e)Yl6+LedyHRDcI7@`xyL_)*R%_4EBVtGlM8Ix+XZOq{?vPbjk?F zYqkO-dgpj^6vMKfo#FaYat{CmELa$KMs=q z0=vv~>cOPA#>7<0U&b3wOCT-G@@8&bx4zq9m`Dv%nJN0bt}4v6Pyt6_RhlpM;EG}E zu=RL5LOxA7Vv%}=yMZ-$3vGaL!yGk-QJD}*tD{KBr0|Zv%G92yr9HO&+5^kqKe1Xyfs>wTm~I`-~_%&<6icR6r4XiZtJgZX-+T)ue$N z#W1_n-@zwK_k=hKb>T<}jU%gbwrOl6LhYK=R8A;I+jSx9UoO{;%oXOM7fbAByTVaN z&k;l|PEQkVr6zU$DZFI{yq=R*AQIh=H*@OJnxGr)v|iZZ9<2*_m?<;bqJJ^}`QZu6 zHX_h94cytQEU2di7VUd=tuIKkFjlgUnq=qu4g{4{k!ye>+P^+omb~@ zk6sL*dGy*etSHt@MTfK)$wCeE&d$AP<@HvJq8XNY3^(n&_>+F>c4P7BsTohuSoAP_ zfstY6XM$z5qQg8|=aI&hwHlclGgDD^a#{xAshEtS>+D`nB@L09K?Aa@&`+{wh|4N! z`)rKzW*r3qUW>Nt%xiRA1h9S&ZOykUcb%L**#6{nab{$)mkPu_LRBt5;|*tfTw^E5 zi;7^P(RL0|qnuEq+U%9N0M^r?;Y5?w&`W3?Xzu0)4rjJHk>tR$u&C&H@{1Ily@PFM z2*Az=g}N`ypR{X^;hIz9TnJe=RjFrE1^fQA04o|0STVR~mmL}(6?_Q7)2zzMO5*n= zL`;x4M_QYuL<-pUnj~?&YPdadcu{hI5D~oqfkNv#0Pb3M|GvxE4+OP3edxa*Fq~WK z)mzQmg{38dp?Lu#>%aHF+EDnylV~GnBEf{RC5S+kn7CNV?Uog zd)7Io-TSyi81tfPiqjw81wN2H-JQ=GcEltU7q0+`?q*(|Uo`M?Rh^y1k81-C9=wC% zfm_D^JUmBD!avC{#$+5v+ez=#@5DL@L_2*0gE^X-)?`UUmNdxSKDk-}QNsg=z?JaW z7B@g#M+y8SF0dSSftp9h(y@2lGQe6^_uh*%-@5e#+LofOm?rb-=;$ch76?;nyPKp5 zAN7x*Zwua*d^9Dha#`S*A_)|9Dig%Z=Y%c!k4i)T6j2(l{J0*1gXP$kVKx3P_*dlD zj|1`mEf6d#lP$Kz?iXhNmn2D_(04J4VBN$>^6m2~lhZ*QM7^Y_a}- zgwCrcX7@ztK~_Xnjcnn6m-Mj@>@*%fzK8H{7$Xa?{@2hKaTFIu_rCd zPcJzLIDei|x@GT4trskd^rilr8fH$uq@-yitiAB?=yEABw($+-K3n?zFNA)x(6cIb zCy?Vuco&F|ju4X3^p7*{AX)DY<v$GFd z8HWD;w_~@u&nA`=5cDhYXopit)3?|Q7osC%f+PTUKyW&fz_{SdaD6Av$pFrr1$WT$ zeQ#Lt@e-iM&dj}c#W2m{CCKN7;QW#sF)_j=ADdPzKUn)=`JE=v#?=vtIsWb28ilx* zr*tMB72efu*8p))Xw{f7Orn=?64lN=D1dz>x7z)V*s+_3&(t7B(TjjSL1~>Jva=+y zDsZWfj}K@BafowP0SB6&KyTpzLk7)B^UDSou_bV@3GBM}VD~ybFYYio zDn(^KU5uDr2`?V;)e-CmA*YLMT>04EvtlLB+y$7mGf?BSpSEchyclDZ=)jXtl|F*C zE>8*jk8}WaC_6(;j1L=HmB5GJ8UQSWmkP)4Br+2eGE-Cac?1PT(G&`p;Fj2)h?eC+ zH-5kj=Lbd2E$&NZn`=d9=QffpmS2lx4z4td1zGl`MXS#55hKEY4Xo7mM^o{lS7O*d z=g=jVbp9V&14S2fg6%0;X>A|A0wWcvkNSPWSEG8G3)Bp`QJW| zL}Js&`d~14p0ITHUC#p=<%)xFLeHH$*W?W^7*1i3O1F9q?@?dj zz=bMoQW4TErE6_1hVb*A;o&Ub>-6Sb)FY#oPv~T_1Y73Qzdr1HK=P-)53lOCz>+4r zR2V~j7-j)L6azjQ23L~Hja^PUh5^Zm@31|o2PY47Y>Js82;%H$Wef}s%6>P~rOB`H z!GqANSFbkx1nt|)&+k>fCxct}rXpycmDoSZ-T%(sN$cJ);H!@CYe5M}xqEn=d@h}r zW}W#Ms9)lQrdR+__aZ=^XjJr*bN`csQ-oj*j^mbmS5X4A#`yugsAkxW8>%;!ByjmN zHXLKCsdZsCvDgd5Sq_>OnGLllB-gQu8^}jy@^ZiC*FAJFv90RLVFiD?ME=C}{PK!~8Jy(cOPJ`c_@B&V<7+^39t) z^0ymq0q=_T>BFS2z58n-vZu!(`MbQlyrYLlw(oOQ+X5C8m;Dzp3H2SEXt*%c-_;Z5 zO6^*qRjEJY+Y3jyg=Tj-NM6Q`2dHS_&JKm2;+50rtsl#^y+!JhTA`dYCV4}&{+}#&rt`nFDbbKWk!r>nW z9`fUyi*Sr4Wf&Ne#V!g%5H@14Fv62p?B~OY79HEbwf&rQ^+KQ#4-bzpDe)!N_-pp< zyCpnpmPfU9V#K#R7FS5Sjm%V;)yi9JYGt$Or0wBpuYa(Q=gW_FcvBvKHT^;$1yDC^y#@jK6iSC0!TjEyg}9F z;Y+{0$)MuDtUfMMd0b>Z=*Ku$AzWm>DG!*qp|VTg6}|ATowcUB>@eA;;6(?Eu35YG z$o;taQbUUp=E6e#2C;L5#iA-|Y7F2-P4OD-(R0$epUT)X_89`j?coAdRq+c4UQ1R$ z)RNDBTl&RbZpo6CfRmac?NfB^cmEv!{_gG(91p=~lxD%h&o4r3l;rPiuKA-7;Qa6q zgj%G}Fo=?nvn}wdSf8z70zY*n_I|_3YjL!}o36sV(GIs8yx6Ai@P)+KjSc>j6UG-b zJWdkk(rF{d$`sKKJ$zrGI0kgaAoi3(nw-3BAt zCV7AyEdt`W5VqlAx)a)bTjAZCH>cz5h{fO+U$j(uoVG#y2duId?E#EQ9JrtZqgrjx zBU#c&Ii5+!u5n=zQkO-nL7WmHe^zySG&U9--*y!SjK7S?#RMMFCRR5IM}Vy7#Euj> z>;euO#<7YKAk|4Sme=bB$&#1{; zPC9RUAfCfGOmn!aPz0M+wVy{i)pu0$P~|lzrM#hZE|ru>qkXx{`1JL)YO!ujIEk!j za?pXiVRSeI(C5$im|ke`j*o*8Yw?YwSSWjnh4XrFF3OgH1F%1$s;l)8$bnsx>4>Gk zgeY~~7Wn0D_RpuawNt1V_~%}KS+AceMD)(I*fQHP-s?387eoTMLs9u`ZQvH%hs&T|?G!6bmb|*-UPMa^!i2K_v8+ zhcK}k<1PC(xRBC?ABwoHU@*tBEAB#UU9C1_PI6>MvBLnP(CQi;X~|!1K^hj%9G}6>ID38&t8w qtNuCLie3oTRnpGh|L37r6Eaiow_N4j+@wz7pWoJ*>R(^u5c*$14F$mf literal 0 HcmV?d00001 diff --git a/figs/lora/lora.png b/figs/lora/lora.png new file mode 100644 index 0000000000000000000000000000000000000000..4d8a3d0fec691016a148890114b5223bca708dcb GIT binary patch literal 7477 zcmbtZcQ{;a(^sMdtCM8)zC=w{iQcUeEm5L(HVC4V)mD#XiQd~HQG*}|q9^+5l8_*} zXi=klN1o?>zw7$`{r))DId_|xd(J&`|7IpeS6dB40w%%1!2zkml=X3Na6!OBKuieq zoGR4X120@peKkd#s$s@;pmD=d;h_Q!&f5gib87;iebXKG#1jXHDeC&cP58M7tYG}C zuB`Cb&tf}g`z6yfqVJHXJ}4Sv4#zxYn%3TuuwUX)Pch1R760# z<7%U+oyvYsM+!SB9sr;}k@ZcrOIY8t$h6r&MgshZ3_{ z=z_y3g&-is=v&EzY~7?9NXkV5i~{jq*#CE$YF?e=$eSpZAY&%EAZZ3g;!jBIT3u69 zSxHIB#QFIPvF^Py+DV0bu|7#}mHR=n5Gkx(WRc;ti&i)Vb0^icG+2FIl? z*czA>4b?rt&KX-+7!r{)j&_N(5&w5+IVYD= z%OVO3<49J$4ME8!tAY!lo$~PC1yC$8hKf+5NCyJ{eR)~;SKusO6pk)9G(9Wp(Fco! z7->>c(phE>gM$C7XT6aRew}v|&2+&r2Xn7|CnnO`N@cH5EaudI`#7+|3U`rMR)1hz z2~KGVL1oH|u)qD!=a{cu3tXGU6POG6WECT7; z?8U4b#bpr0O#zTg^}S7=QW09U7C>`}L2nuY^mjQwe@UzKyq>R*w93) zN^Q-X4OswTrk%cB&y4gj;>cyzzx#AjOc%#MNG1?Qm|f%HpPn=kHQgkftWA&KTOOuzsogWynwfZG9aexfnnZTe!YhAEORF5~HR{ zri-c&pI6&JVt3_ZzpC9v&LeHmscNR(>>%3a@);FIdsE62>2zjlzPk*Iz(2*~WzWN_ z-p>__RsjpXCaIOBggv1;$(={<>w>w^ZY{M$`CcpU+aA~|6^G)z`d%j{fMT!7HRfTV zI{788Pp(nKDSV9Np|qb4|QvoZH`+%@}7c#dL>DnQVrq zgmJf3OEdg2aI`|8f&P+MS?pO?P=_ZKp$U62GXxdeFU}Z)zs?#Vy_-Ejgi%l>gk^V@ zvHaKVEKAPc6JBCMKVc0z}eaOH0!h)xBbfY;{)WK5@IiZ$pDj zrjwbX&J%5>Jn2;AM|+pmTSLH9ZTR_QBG07JcovUE*@2sRz-%3_3ykiK?pWb{u{yJmZv{r_WT!UjQJc%$w!e@RT zet03IB}rjXa=I2jg4(Q;p{2hbO(q;|FN+I~V=AJ$(MdeJ2Uh}iGK$4HyOrXd_J~n{ z`?nC6AOxlSC<-d?c7(pXZ$FJYG?B+Nt+QtNZX+B}i?=#{Des@-`PWC+>`U$=whf+6 z$4k}9=&=ABdaVhTKX%()ltGnz?(uq~2vC>A;Qkik6Az9JemXA?kf&LiO|51DgqaWk zg$y@w3(b2v{My_=`fA&2iZ{7i2nd~zp62R4xcXdO7r}e@<9*%6)(FLeQqr8}U|x@F z&2tyy{_Y8!67}6!P^tai$f=Ub^Sc>b$Vry^FdJHo!X1azn0Fa6~@a5cf8 z57qI1w;c$)b925Qd=i1zKoDvupMwoy)l#=&%5?g+b+UXm_m{eIZXbP6^>C}Ng3MlC zwN#(c!&g}OtKPwF(+T{RjVl&9V!XuZ=8!+%$Fc<Gfm;epz|H zazMeVX|i4-;pk%7-l9yi@dCp6us%QLGNJ@i5X^hiOC;wFln-=c#J-941_?GhD1E}f z&ecA-7OZ0_{2flomKuQ!vO&YDFgrGDpMO1qA~ffl1Ne#Mju777_X z96NmycT$gs_{Z@)1IW~VWQ~889$er3$_28jDlhLB8GIpRQ(an-_a0>jwi*^vo=c6q zRN?yG6PR5YK^SzDEKjYtHeP@bM_n6G-TOuzXDpiCgO1%MXkb;HNI{{QWJvIyk=}~R zR9CV>E_4u1`6Qm0$4&B7>)zMgcFvsxvaALSE^B;FZiTMdQjvTb=yt~m0Oz8B{@2Z~$ zDZQ*uGSr`d^Th*=UE5+CmC)a9B+>9VFGcWvkdfZ(UP@%SvH7N+0eHW8t7d_ z>PHv182mfb{^}cW8n6?9tzvJ&S=sRenIU4H0f|VN1Zgbl*P{9q(^;y_aYEPqv)wAK>{I0cY}Rw*9}hKvz+;l`+4*Bg|6>;Vc?m94C*S zoi0#Lp9n@^Sq)-T!TZ6ts;=Hb&hlvgTq(w~4?MqM`J8_u$g`pgF6JA9DP0x}auM?G zx4QYa_LUN_9t^zFwfL-|B>S>DIg<>d@P--%CA{F_kJYI$Wml<0qXBptQ=x+&;dTqt zf5XO|u8_8Dv!2@vhzZzVMP`eonxC&s(!jd)2<+c3KtDLbZ)OTUdSN!ss=vD;+3%&Y z!vt*wKz1>oo;vVx3T9CFD)LFX&yQHn<#z%%TYV|@D}46m_C=yn5(y*I5+h$QTQz9S5o0_=N_FxK z1CO@GJ|3K&`d_ThtHm>ZDq&Uh!U~gWlT5ojNMG3Sd292Pb>9TM5;*j8F07-;x|r>w z`^OiSuL_{Dcu5lO-$NvQw$kTXe5>a^1U9BdhMQ@5VbN~3%W@V$QjP9;0Y6YLP1sf> zTFSO(R0NW0+{j&TdCs4Ud{qNe!+A6E3SR4i(`x^XdB%#=>e#wfys+}cqHO!ZS$q2E z(&2MGxy(vR-p)dscQ3&^o}W6jo#3n|6-SOP?8&_td3q-gejy=8rRbLs3|kMymYa2* zWrv`ZXErYl?Lo$|V1>5xt#|cMm#x#Zl*qx6RNSdgjNF?aQ&k!Bufh>); zvfPWUB)t!ugKMMt+sQGe3x1Y+ImFA9MDe#zT+V;P;J*zDnIN;MWdJFFHM&6I9#<3= z{T5W$@MZlC)O{nQuRg7|J>PnMkLtV~CJ_=qRC$A>*ve)>jX=Djfs|#${Mb|>c17I9Cbf6D z+fTm}UEvkiyD*{*zd=m$mH%lyqgkUV)Re4gn&g*I7Ej#00|xH58nzC9=)K*iRUP#( zXk=cC>h{91`0P~lD-%rUqVU_SAJ1R0)5bht9M1@MNo}z?Vj%1EW-&S?TamQ6*dF0a zTsJ?s*O!)KIoSp=JuMS_IS5Cc2e#3i)G&EjcKoX{%qzh8%WObNab9 zlH1Min}k`fG^gb()WoQO@B6xp2h*+dfmgV_-H*MWUdi}6n%*ThNhFNDQLRF`A|ajT zW^zXiQsw{nApkhM6*nPvdkrbKi$jNH&qa&Njnh-s;>Q@oR3}B|nxMO2f|$;FQs$-I zxl)Kdvr2MeEHB72mp!g^O7Yfs1JqDbw%VPZ#M9$9OAsoUKQn%;7sFObCf$e$^@wh5 z)Y|eVqj?0zV&w<$x|G@3QfZXtk_n|`FN&m8d3p=iZdaOv9rUs);|e|OISzJHPG%nrm$l4cHPuHC42sZf_4Zk98(Zq6&s^Yf$GCGOhP(mniSFcXJ|h>CljGNy2>*kP~Zdrs5uUY)3%G zCTAr`y01u}qY49u1Ch_(-9v>PKEje0^%7H-lO}%;myH))1}zT>meE2KvLfwbossD6 z#kLp{$IA6eFs}s43-WmpnQ)mf6SHS1sp_0YNaL89+&kD{s(O}$aQzu-&Ttq^AbuU{ zWdd7a@-17JYya#&dLQ|37$e*4yZwGa)?>^syX1R&SYX}Iywcwg1g5l5Zr3e1>wB~s zH4(KsY@y%by$pmd6@shhciX4%LSf3>w@dzrgWIC6J@ZB?q2&7KBinr8iHL)z{y(CI z_t=n%v}O?riW3r3MB_@gDWe{|IFvz{Eh*g?=zTtD&7O4S|J#k>!#~TDBX#;#pVGAA z!t?u;Y0**U-*59Qtmvct;yu#It6kaB*%L>?{IkZ%7zP;*lD$QchuuxwI|cuQY(G4! zsTqH--n#nmmRk<*0pjG>7n8@vZDeYD8P!8DkGU?9LB{wIO8pT3+IxM$C%;Ss{Lh&A ze5ZpcuATe@7bq_^R`n5<68F2F2~Bg#?#fNax0Wti`B9S&FF`RZC*Gi z-!>MTN2=AyZ?6xn5B{jtUI}Q4d>^(aU-HV;S{r!?{dhKuMXyszPcNSQo_u%Yb`2Zr zp)Vw5sc4Bw2u4)EE8;%pV*{T}Xjh)=wd(u%)I-loaT;{rXmp99o}9FOJj>ddY%iPe z`zuO#YUIcVjbdZhOmw6G4nNIiSV{2wJ^=08D8}h?a;>tzJtgJ^6%PlNUGpi1?i$6M zY$_k*Gtttb5a?TUDB3kv;dwz9DW8wAl>2A*4NKQYzAy{z-=AhU5Hg`?={MisQ=e}r z-(=^sM3vR94Rza(86 zAReK6Qlcr*99$In5BqSaS6^x02Mt|gh_DrXl3**Dku^EPDL0W1E{Ac@*mDuo<_F?P z(h@Se>`er6e(9C;O-mej=I2vQH&O0A$wU9MP#6n4wcvUlYv8z90KtawKDC~i_5D)9`gio90ensK zXwQNxC+^p6ko#YqK2Ge_=;PTj!2Vp5I|uLo-T7XM>Qkn?BF1kblHD?of%|vqE@-~o zuKwQOKP`^y7WC2K-^sjpLErX11*XK=d0nYS1ePspEWNxs*L3K9#0febF<>;SLdLHL z)k{e46f7q9&YYoG?2(5vhd*Q!e46ilrIgKMkv?enj;4{Zb!3E9HQ{!Oc+ z6PMpT+OZpfIYCf(L3`zCUO+Y%O3@v^`}HNy#A^E<$Kku#y!DHt8J)bzuBIy><$U{l z2ercY{H!zVu^|#a!HZ~1XiG_2k-1iGkFfyu=2bn^EDY_&9oL1+MZ~hs%nqW7?&PA@ z&$YvZBGpVN*@+p*x}fQ)UKF!^NE4cw<8%F8j`dVR^3;1ok8>ZxTCMWb2RtZGur;O} zcEX=D@rkBccU>daN5e4^i$s55+A^Q(DKWvwWF9#DyJ^{47OW$*$58ES|2k|ILE?9| zuCvcnAe&?5;@$mHXH&65HXItwqOPgbmhmFdkchd3KcPr1VdF~b8dRHDSPVz{kj_TQ z?%!J03^_@xhvJZD@4LEfZ^}Ao;0FeQ8nIUZn0=Phc3R)P!DqY(oKgcjxZ=URh<3<_YSh%!|D(sahHFI z)tZ`|iTrBb6hw4@ngb;MzrUGQG52bfWG*|{r`7yj~Q}!6?f8y Vl({tkls9p3)K#>Vs}!xm{s-`Y6&e5l literal 0 HcmV?d00001 diff --git a/guidance/lora.ipynb b/guidance/lora.ipynb index a7c672ec..f4bbdcd7 100644 --- a/guidance/lora.ipynb +++ b/guidance/lora.ipynb @@ -4,7 +4,85 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Use Lora for Adaption" + "## Use Lora for Adaption" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### A Quick Overview\n", + "\n", + "#### LoRa(ICLR2021)\n", + "\n", + "\n", + "Low-Rank Adaptation, or [LoRA](https://github.com/microsoft/LoRA), freezes the pretrained model weights and injects trainable rank decomposition matrices into each layer of the Transformer architecture, greatly reducing the number of trainable parameters for downstream tasks. Compared to GPT-3 175B fine-tuned with Adam, LoRA can reduce the number of trainable parameters by 10,000 times and the GPU memory requirement by 3 times.\n", + "\n", + "#### AdaLoRa(ICLR2023)\n", + "[AdaLoRA](https://github.com/QingruZhang/AdaLoRA) adaptively allocates the parameter budget among weight matrices according to their importance score. In particular, AdaLoRA parameterizes the incremental updates in the form of singular value decomposition. Such a novel approach allows for the effective pruning of the singular values of unimportant updates, which is essential to reduce the parameter budget but circumvent intensive exact SVD computations.\n", + "\n", + "### Application in our framework\n", + "For each AttentionBlock in ImageEncoder, we replace the two Linear Layers in Attention and Mlp with LoRa Linear Layer and AdaLoRa Layer, i.e., one example [here](../models/ImageEncoder/vit/lora_block.py)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Training\n", + "In SAM, EfficientSAM, and MobileSAM, the adjustment of models using Lora is supported. The -mod option can be used to specify the fine-tuning method:\n", + "``python train.py -net mobile_sam -dataset REFUGE -data_path data/REFUGE -sam_ckpt checkpoint/mobile_sam/mobile_sam.pt -image_size 256 -vis 100 -exp_name tiny-mobile-isic-256 -encoder tiny_vit -mod sam_lora``\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance VS Adapter\n", + "#### REFUGE\n", + "| Baseline | Backbone | mode | DICE | mIou | Memory |\n", + "| ------------ | --------- | ------ | ---- | ------- | ------------ |\n", + "| EfficientSAM | VIT-Small | Adapter | 0.8691 | 0.7915 | 21275 M |\n", + "| EfficientSAM | VIT-Small | Lora | 0.8573 | 0.7703 | 22777 M |\n", + "| EfficientSAM | VIT-Small | AdaLora | 0.8558 | 0.7596 | 22779 M |\n", + "| MobileSAM | Tiny-Vit | Adapter | 0.9330 | 0.8812 | 10255M |\n", + "| MobileSAM | Tiny-Vit | Lora | 0.9107 | 0.8436 | 10401M |\n", + "| MobileSAM | Tiny-Vit | AdaLora | 0.8863 | 0.8031 | 10401M |" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Curve of loss and performance\n", + "**based on MobileSAM(Tiny-Vit) model and REFUGE dataset**\n", + "\n", + "#### Adapter\n", + "

\n", + " \n", + "         \n", + " \n", + "         \n", + "

\n", + "\n", + "#### LoRa\n", + "

\n", + " \n", + "         \n", + " \n", + "         \n", + "

\n", + "\n", + "#### AdaLoRa\n", + "

\n", + " \n", + "         \n", + " \n", + "         \n", + "

\n", + "\n", + "It can be seen that the training method using Adapter is more stable." ] } ], diff --git a/guidance/mobile_sam.ipynb b/guidance/mobile_sam.ipynb index ea843b71..e0f1c282 100644 --- a/guidance/mobile_sam.ipynb +++ b/guidance/mobile_sam.ipynb @@ -90,17 +90,17 @@ "\n", "### ISIC\n", "

\n", - " \n", + " \n", "         \n", - " \n", + " \n", "         \n", "

\n", "\n", "### REFUGE\n", "

\n", - " \n", + " \n", "         \n", - " \n", + " \n", "         \n", "

" ]