From b671ca7b51f9dd82b1c0ae469b61bcaedac4ce4b Mon Sep 17 00:00:00 2001 From: lhb8125 Date: Thu, 23 Oct 2025 20:12:01 -0700 Subject: [PATCH 01/47] support fine-grained activation offloading Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offloading.md | 29 + docs/source/api-guide/index.rst | 1 + .../offloading_and_recomputing.png | Bin 0 -> 332427 bytes .../core/extensions/transformer_engine.py | 10 + .../common/model_chunk_schedule_plan.py | 7 + .../core/models/gpt/fine_grained_callables.py | 21 +- megatron/core/models/gpt/gpt_model.py | 25 + .../fine_grained_activation_offload.py | 603 ++++++++++++++++++ megatron/core/pipeline_parallel/schedules.py | 12 + megatron/core/tensor_parallel/random.py | 7 +- megatron/core/transformer/attention.py | 67 +- megatron/core/transformer/moe/README.md | 14 + megatron/core/transformer/moe/experts.py | 63 +- .../transformer/multi_latent_attention.py | 38 +- .../transformer/multi_token_prediction.py | 5 + .../core/transformer/transformer_block.py | 8 + .../core/transformer/transformer_config.py | 41 ++ .../core/transformer/transformer_layer.py | 54 +- megatron/training/arguments.py | 11 +- .../golden_values_dev_coreweave.json | 110 ++++ .../golden_values_dev_eos.json | 110 ++++ .../model_config.yaml | 139 ++++ .../golden_values_dev_coreweave.json | 92 +++ .../golden_values_dev_eos.json | 92 +++ .../model_config.yaml | 134 ++++ tests/test_utils/recipes/moe.yaml | 10 + ...test_fine_grained_activation_offloading.py | 185 ++++++ 27 files changed, 1840 insertions(+), 48 deletions(-) create mode 100644 docs/source/api-guide/fine_grained_activation_offloading.md create mode 100644 docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png create mode 100644 megatron/core/pipeline_parallel/fine_grained_activation_offload.py create mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json create mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json create mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml create mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json create mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json create mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml create mode 100644 tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py diff --git a/docs/source/api-guide/fine_grained_activation_offloading.md b/docs/source/api-guide/fine_grained_activation_offloading.md new file mode 100644 index 00000000000..b4c2ea753fa --- /dev/null +++ b/docs/source/api-guide/fine_grained_activation_offloading.md @@ -0,0 +1,29 @@ +# Fine-grained Activation Offloading (collaborated with rednote) + +Memory capacity is more and more important with the rising of extreme sparse MoE models like DeepSeek-V3 and Qwen3-235B. Fine-grained recomputing reduces the memory footprint at the cost of extra recomputation, while offloading could utilize the host-device bandwidth to achieve nearly zero-overhead. Fine-grained Activation Offloading targets at offloading the activation at the granularity of specific modules, so that we can calibrate the amount of offloading activation to maximize the training throughput. + +**Features** +* Support PP=1/PP/Interleaved PP +* Compatible with fine-grained recomputation +* Support FP8 +* Support MTP +* Support mixed dense & moe layer +* Support A2A Overlap +* Support CUDA Graph + * (Temporary) cuda graph scope cannot contains the offloading modules + +**Usage** +```bash +# Enable fine-grained activation offloading +--fine-grained-activation-offloading + +# Specify which modules are going to offload its input +# Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". +--offload-modules expert_fc1 +``` +**Compatible with Fine-grained Recomputation** +- For modules with minor perf overhead like layernorm or moe_act, use recomputing to reduce memory footprint; +- For other modules, use offloading to reduce memory footprint; +- Make sure the offloading/reloading could be overlapped with computing; + +![Fine-grained Activation Offloading and Fine-grained Recomputation](../images/fine_grained_activation_offloading/offloading_and_recomputing.png) diff --git a/docs/source/api-guide/index.rst b/docs/source/api-guide/index.rst index 710a7caf4de..ac6d7cb0b2d 100644 --- a/docs/source/api-guide/index.rst +++ b/docs/source/api-guide/index.rst @@ -22,3 +22,4 @@ API Guide optimizer_cpu_offload multi_token_prediction tokenizers + fine_grained_activation_offloading diff --git a/docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png b/docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png new file mode 100644 index 0000000000000000000000000000000000000000..6c8afa78bb180a0815aff02693690b864e9b01f8 GIT binary patch literal 332427 zcmeFZXH-*bw+4z>P!Lfeq7L@*-C?8~5C;#x)#z@6hLxX~se14JQ{F&<%XMbHn z{-QX;L~;Jla|#O8GtB>->zv{K>l#W5iZBO?bAMf9Og{cPRsHqy&+%;bng8A~oAU3g zFBfK?{rmjFujimh(_@$-A1=L8e(p&@A>8)s`wX|W@iYa+BMOzLkM;b{Y|c{E^DAT! zcOzcEet(CTdP)Ck)tyGC_Sct|nI?BX{XH#RoPUn2RrZz8KNTrPak z{?_uwgRA#0UI#ore?V%T8BLe-Y$_=ZEJk?^1fl&QcD9YhB_$x%g2uqXRMC6qu0EnT z^Y6>QGWZt<|C+(SRPZko{>y~_GU2~W_%9Rw%Y^?j;lE7y|6?W$b~q=NX=~l3@V{{y zTuPZ%SnI?ssCy#nG*DHsshXO*>~>Z?pu3UEUQnOyH3j9xJO7_A`$AtvPKK6d26K`! z+`4Z$?a?a3(u!t?!%{z)E0r(axeuq@JiF%mV=c967$&$X@v($e9!WoaLk0}g4XbbP z{pruwLY%BzG|fBUU~vWYfi~s0y(p*c4_X*}wN34K%F7H9qJVLHZo~nRveQ32i?6jn zLesr;Nxd`q`=Db#J%hNusTQooQaS{BbJ;h1l%YVg8KRRl^z7Sr5pM(}`3=}#9ZKBy z%=GlwOeSGNB%iiXO5&O2BSNR74ytuGw2oYdT16G&gfI42P!L`pnbkI|@q3l3Ne-?! zDyhoCu#$PvW5Neq1**@k{8ICsf=8Q1N*2zVE*nq&he$76l`!&dwDK-pWd0EJP=_0= zmXd=bgmPY|{O!)06#8R6={x6Zsn$}5d6E0GZ9xY6q*#5|T4kFfg#dls9k zpl>snd{v9w!^R~Yg_;=5{>E}P+COxuF??JdKk^M!Q0NL-7X~#cF=c*swK#Bg8od>} z{;}mIrR0#Lqz`|WREH_s2T%BeYEE#sJdG}h^5SKIwxjXk_En_CclV>4?sZ@gtL0&q zNXtF|u79GstKUfgR2ax;x`WSO`V-E<>6GbBt7Ex>zfnB(jh#{Sy>p>gCk+z1ZfoDN zY#e+43npJ~QW9;N(}up5ZWsH%n@vamIH3xGMrDp^yr2PC6x2J_jd06yQSy@{pMUVY z&kV`lQ=%rJJ|^*P=e|2Nqyt>y1X7s|p#cTFayiP9nT19Z2QuTh=OMs*?rDwpK*JtE zFW%loUr|rwKWA{UzfTesp0ae7yd>T>$Y*9DaQAF8chs-v7}j_(64RIV-bpQlaE#l}DBJJHx)9K66_;g)-*ZwZEc|V4ia)G(H5Mna3>+g~ zTD^SolekVFsBA}LN+wxn({gk6RU*_)O0l2&OUuUbm!-7u=OijFi>ERMyVji_dPome zE%MXbVobV#mD`Cb!Z-gHodt(5`n^4O@&+WN{fh2ud&L!sJNJ>4rbf zCcHo=sN9lWe7t(Tk^zz>ee|*iX;WCeekS2AdT`kKzc>6#7B9-rmlrn#hP0C&B)lcd zvp*D8dmo#SAF#s4){Wk$az0V8~>w+nrVLw9l$D1Cc ziD5NE&&w~fl=%JpUSgiFS^khCBlihV{}5Y84XUr2)iG78m^V<*zl+XHzmfGfeEv~( zw%Ni!?{VHz-cr z*y+ho_a0Ebipj=X=x>%C!l!mx-Q2S7FRAIRjfaCc;>#Gx;y(8Bi}_1deUt}xg-mw- zBx8!;afZW8PU25>+DTKasTGfFp&_3Aq-7~Fe5|^Nnpm-PAv{7?=@ZNER{oDulgSo_ zqua7Xomzvv``53FZVL&}dMOpSO|g;9?}YjbUGlt5UwX*-lMg-1XBFg%nhD%N>>j#% z^;CR8rChXpZ4gJJc^GnV4@kJqq(s;*vORGM%QwxfbkjRrjPjI@M17jW09zRXH^%9T zWgaaqtMVUp#mfWgbkd8HIky;Y9VxHE`E~LI0=pZh^Lt*M^PqmU*xWX-v#e=P+<)h?U!T5ayr7iY6v(5gmfUB;pP<35dhLH- z@~7m#L)QU(|GX2te!f-@Ku4qJS-TfhwadkjbiQ|1#lP;#%EypD?V~`^v(8ntYkcx0j#STyd@)@5ZhrqK$oqBO9nLnW_WMh*e;d%n75bg?Z~@A3u!28S_e67d^^v5V z#y-tqY;lj`FYsKvxLMeYI=@#^x{vLmg9Vi^-CscG6%!~rY3TFaMH3{Sjue1id{jQ= zfj3VXbbj!z)A+dff5@=%HQ}>GD#fe5x55m+=;SZn{_oj>N3fiD?xTM=@XoN-!ll>| zIfFh>Gp==gtCi_zTBORK_pJi>MRPI+Cb!cu%PH=b{19^);1g}ZbEuD!##e7$T5Y^> zB#*o%6;JkGR|c}V|F^Y!a_nTKrok20?92AV!{hHV|GzxVFMRnu3IUw?`@Q6-wla~F z>GPae##L}-BI#NSydLMjw>2)>MpyLV&(+v+eW~H}e$>8lg8r;@FP`~Rca{FS|0Mui z*+xu_Tj~EICD|gFM%0PxsqGL65Y(i6tMdPl@GtSyo8G~dRNTLNA43UIb~5a|626tR zBo%HwFYRq{8U!!MAR8 z$g?A?Rg{@rj-TF(<^pbvK=8?PTmK*CS}=pfQ&=j*f z_(sX17HT3PS8x;342c4UJ?6gemmK&T3_kxiow`l}4rWAxCs@U*{B6iC5q{vhxMk>%Jb( zBvle&^Myj9a|P7#m5@NlPo*nD|?d8O&)KS1V6__cx4Fmpu$& zFMFGYaQU&tZn;4g4+k<-WnB}7f##0$Pr zHH;U1)VlY~=&Snqex&Nq^YG<^4fJ!XpAjulQ!s;Pv(pQ;nPoeoJG?=((N6cD+Ub++24%9o$uXvtY;=qR2 zm(g|yg#cAmRN#g=Lws^&uhNF}`>pomoZZNNnD}X1XcM+F*n#Jh%a#Ct+G#mBa;NVo zjv@X*I@pk3!S?jdf6nXY6PcmQ&|4P_*w^XI9Df~Z=qr4Z)GnSA_`^qIySXJ$z%>ug z%tH>_)0bQpE_N=4R{=&jacA7WmJ}&F5GX(*C~!%g8t5Xx1UZv~yViL({4)box8KU6 z;HMSaL));Pm9$WxsN@&i4??Rn@65FZrXIpfG9Jch5)bPV>n zE+SyB@rEiDuV+OfCBoqlZM=89Sy29|k!S|fSIfA45)>koO&hzNuS$1OFQ|CW{A#yb zA5$O9db1BcAS4zf4VbiW*Xqn*FptrR8`5gcK8#$Vrv|mf@%dZGRiGxpQDVq<2W+!ig}))>)`>$Xfg?LEQA&Tq7*6jC&>;|Tu{xy$mvXo&jf&b#O~A z1^!co^QMRr28$@b_6L$_0v3Oi?-hJOu7&3_>PwSTGuMT#)8sw5sW?y{`l5hFz2(l$ zL744&EP7^(x~)6yJI+6E9?tNb#hU%Mh_X%QB1>t?j)s=YZULq3o=Gq+u(ehzJq$>- z;lJ{fReZ$GSG%Sm@3L>f={^9KR;8Eg>w#*Pel@6*Tg{`_mGM|e+c_sWX~m)plDK58 zU_U4C-^w%BH^RtOvch~Y)pg{e&%HQqGrjSnN{k2zfIiROv}Q$>Z1MrjK{|2L+$0nC zSj)d`sRwW_gn){YkG6Hhw@#toP&V6x`U!=f^CHKDwo>@ta8?~*!_3CT-L7?_3e9+ z58V1(s>Etf15Kp4!eUO-tA-jnR%DL%o44K|h^(y+w=T;%_mh!V2DO@ZyWCRFT6*bi z^sF2mYhZ7U8#ZWg`gqHq7e?2!m8s7}cwm3|wA)IK(_glyhVBJ{4oGL~&FVnTwh{Ij zS6Gw4RanP7q*HrNtt2M%7L(zIyOSuPL-z+_M*P|o73hWKg_g`y(`zQc={+rEeN1Yh z=iYJ`(~fWmL3euNJuRuU3mY&Z9-^LUTr%czYN${M<(rtOa5|ylIawI0<{67NJ$2S+ zwnAP+>YryexASMsdAXNFH>Cnacw|5lZ#N9iPMMZ0=56n*wm^$KP=LR^_^%l5#-rB} z5q&h0W5K$lKTw0|5ezWR)UgwKeWXiLTI|;B8~srshN3NH0pIYA0}b!If)iDw3$b}xj%QZ`dC*fx0NjZ~Z&J^D5pmiu-!>q(Fe zARy|~P{SkpBdH<6GW3~x;8*v#=nA$JsTKv%Zzox@SiL=io$jL=z&hP4MQVWULSzNcajlz2vz2}l3YyAbEH{d)(x~P0iF!4= za#^ZjE zLSMw5tFNzbu9jz&ApU{6QWnE~42h+QLNQbL8m_1kYZ8YKJXTtCMw4-&ou>QKHMc=8 z=$m$HV@7~_7*;@4z@t^4RD=02r7Ot#94|f_qTUE1XGmgu1^lw`ad5LKtKR0>MghzW zfx;~bYH;uz`eeD;z`IK-i;lX~!m4-banvR6sp*$lkKVKlU3@M%B>tH$Lu~xqiIWzr zR{bM1X9KQet*H7_8Lx5wL70QkPq7CL52Z!SKon_lT|xXjFOLwBZl5UUb>AlHM>oef zG8smRa()15A&_m(KE2@?R!5JQoV<$N58LJBzD*Q-W=>t)51I4>?}wEJCejtgR|+5* zG0Aug)<>*GXf^#KMuIQNoM(Q~ur{DxE6@4OzQ+~qUWiq_s4IBj;BEy&W)gPVQvtNs zYa@MB6mO-yc_v}np)2&KlHh9c%4+Y?_{x;o8g9tl|I~v8eU_IPGt9_qIlo^~wYZTi zllyIX!sxg|zB}(gBxnu z&kRA?6}D>wNYvL~ig)WW8v2iSDcd$~$qECTJK%-% zam~LMuiGduy>fT=e;FqA>g2rJ&oqaj5fRrfGwJ4-nVuupP7&4@W?kTl{8fu2Zl0sX zSnUGCq$4+8H1s9V_(H2ua_SSt8<;ZHXkC$-yEe%PH?=z5P++Y}5tna@Qne`WeHlY4 z0O9p^pA1rN;g)eLM41PFtT&GDQ=ZL}$=>_a9dt<9bt7E+I98Z1DIZB64_fI6BkGq` zC`=8HmCJPqPIzsX*sF1P2Gv6uhXKDbrdw^{&Mp{c+4u6%cSVd(CxXa^ zpOH)ly$lYM_?#2en;loX>Ims-g;q@cI57S$$u;kunDjzrelf{JcBUz=_N%`H6}JQY z!HjWTtL^azL+YW9@p0ThWK367u{mSKX6VxmxwK!Pc7<>jcXu(!up~6KIsxR{$f%seBLdrx==r6KNT>2j^}-nUB(cvCHh5HkOjPP>-s_z zHembF=uz~>R=}*R^^)h|GL>Tq>nza6Ng+$*Dagb`S~c*IO+U&g&%$t5N?3@{tg~(5 zI4yW9`tQQoui%*GPFrW^Y6#0lTgv!AHO9hK6Xo0b9(z$CE4s#^>nJy0a%G@D<1Qt; z3=|N==>mq`7Yz~Q*BtHHNXuwx72zPYkEew9_zZ3AJEY=7Q<-ku6pM(6*gQLheD8oi z!^2n>YEDx1d(=tzG5JQ!jnJ38aNwosgpWmJmRy<1N zl6(exeABUN<<*#-@nf+uCx_`5>nMu6E%8ueOmV=veEGczcw#GLG^W|ZUkuEExiy3c@;1EOfkE<0H+NUNw(Lg^bNb)B&>XclmLz#9uM37c&B`XpwtF=61 zFe19TZAYUqpN|#OYPDdz>7R(aHWLZUvKsi#j%+Xi?+%X2RlE&k+`u&}{kJNRUDw9h zg0OyiuJ>xaXKo6yW+97j=F$Ou=MQQUZ?xOq303Df1JC+1-h-@gw*rPt?&T}TJgjyr2hozW=Dx6Hu`mc8!blM_(WPv~N)_ z_}T8`PC5Yb{7sT&6jbu0oLZ>B3{aASOSN1Ib224OY3Uvt(jJ7ojrWS9K1#X)E&DRE z9_|U4<_(fRoXSH6u4Bn8<|PXVC|=U%39&N@1wWF;i4b0!6i#_>=~r~`LM8~MbUct^ z30-&n5WFrV+Y;Dm+}WJpW$qP)APbsxvt z`qCRMQ#j`G#X%9#O3umcjU6wm0d5kGw~qW^OOW>uy>fbwq(g>wPgsxnJF@@>>@<(p zbwc1=P;Hpo^^TGx|FbVCwtDIbmgO$AK3%?Ulk$BVAwFN?@3eQ z44|C-bPu<&}nl)|gL&CXuf}%L$PlltX(P4U=6<^g> z51tRL*v)_Q4D3uEO&YK7@ev0Tn|m8{g%fA49eFC~l7r&b2O^>k&V}Xq&1xcpuy8z2 z(946S@CXKMrT921-^+V{}pw>E=RHCms^t}LA@Z@?aU zr{dqKS2@NENt@@?8$ao6Eh2J-JI)NFKhke_#!mW=kQrJ1Mw$-@3(&ur3is92{DQJ` zQ-b`wFH#0(^G*uj-*t}iJ~CX9%RCPKGQ3x`M=bOZGB~{Z+Nd#~x>?^nP<~)2^#%vzEchhCMdy-RZm)EWy>0 zlovkSki1VU^~?AJvf@*su%#rpo1Clf&q;cm=hCz!_5FCO5aWF>y3hQLVZPCPJHp@w z@Zcl-4KUB%goN&hF>Dw^tLlHo$y5;))Wp zdD7!$V4=MUd=>)%DU@PbQny+4#6+-x2FD+F%4L@~oXRqe9O3(#I@Pj|3QD_HPDjE3utqKI zlz1zr&E`T(NQeVom!Pz^8?;&=^Df)+Yjv21d?L}NVbV)gHu0uyY2UnyXFxSubx?8h zvfdygT@$Cpp^Wac@lo6ccz;$(&wOLIri0&y_n0riwF zCbROvSZj18w3tx&C~e5zQMvDnPTi^w9+Q{psTsJfT?Wb9euKZ{>^9!of%psqhgHmk z`3bl)^LwB2dBY!k3oxvB=u_UT4DR;f@jlH5HnbXu!-cTFe6yx^WMs?nO(_kK$9r;u zwWR=p^ZxSAn>D^2BB4}z<;AwApiT?`cB+|x!E~7E6rs0nr1rq@VPw`DS!L;`6O%3K zpvIp|z&@xnw$QbR`7R-@o*qpoJ<0lLv8y6AXA!K%bQ>d@F-y!XrxdEC^r%6>^knlw57_a^t6uDNv=i@I+B4AS zcN&NcUn?~t05boR(Xk1+kH9CYE$dXQ&$!mI9p!l zz1LEAZJrzCBBcsYpO8KJAwFF2tF|}<+d3;Y+J2f~>Cu1$5uUVx^2*QAuQq2QCL)fj zyNIuQKZ=5|IL%hPoO9TvH^T@%zK6JZSo6Fyp{7E6R>QwyxUEM_3qnlk^-2jl_G-@3 zk_-h^%jxtH-iQWOx?qP0wULa>EGEFZ7`+9WRwAG_z=U%iB%`CGiZfv!_4CY)i;LmN ztbqwa>jtKmFNOE4${ih*@Q(VF8XAbq8gXbEs82{A9np-WCeq|OXO)w5I+F)sNvJ!R zrhUEHT)WeK!#zzwCl|S-NV7sMStRSm-Rna;+TH{|6AJ?@zKEQ$H3}p(LWPdfJ*PaW z+-Nj8w%@3c>)htDqqOn6IUv*}qD zTA3ruigt04gIq0ixrXzP@NnU9p~++Ttpt8CQjv@}_dS~9;ViA6cR!;092}_zs{*TG za=rA}yK8V>~Mx23JoE$xKw0X#nvPgOF-&Ex{b>OalH7$By z*KNu(P)CTrwL7FhJqCG5Zr8E>W|p6>D0-}Mur?Gvd_ZjAP2cI2yW{MDA_GhZ_r^`t z2$#bKxpUu}nTc?g7v_DkfA?Ktb37GV=k`zgB4YU0))Yw}*dANLuEy?YZicAGWQO{nndT-~_-v_cqdv~B5Aju9> zD_gry6zsvCb6eNB^p+%jmP)+^*CJlRsAS}RG)(-6-jUW2tad%53{+(zLq#jFuu({N z)@2`}*sETmUNr4dI%{=WuDvH|7wt(^k@z35WYzQiG%Y1ZVBfG6E2oMmK*{n&nCEJ( z)rxk9VHMpEFOKs019pVraa_#fBJhh*9D7oL`rdbomHm=G^SdoVnN>VMy6YG&qD~JQ zJLKXUG<@4ihnQMC(FG98=Qt7?=Msm3x2KZ=QxfKe4rc`UOv*N8v2r#UJ}E-HjdEW4 z8XGmS>nDx8$vde5=2NRVN%=jl#lfv~cUC;#@71|5HM}&khsR9~O=!C;XP*@j8Qb=b zvZ|qr1dDHYvhviLMb}^kROS8Z=g6g=2PPHFd6@k!FH&3*>;Gu__;U{}p-@a_66SKg;viSNC~njQ|Y&W`4z z3;&p;4K%;q0MwRL@owx0oMH%UTkCexaO=v zQ};Yi>N4YTjjU9n{b-{LvrlL*J(t9c$TBsVJZT~NE{*2P>OU?$9rZ~!a0$}4Hb!xS ze7oFB6H^mFrQZJQq-k&*-_$8>0Ck%0enz*7_3_MKBUn=4T(p8W4AG-6vKx@LKV ziD?N?^<4F&&AQ?pLpFAqr$=+;9=YPyh4)aBKA~$O28&dBmPLK%e;S#|uMpD36U%S& zeaSu|N4$C4^I+4J^oK51%%gIC*IGLY1QJ$t@?78ad0vHx=VN0|75$#aPMf_@N7tEX zfjsYlMoTisBTb&YBPHIKEv7c3e6 z%DV~l+(^@zKn@k}X=?yqY&2B)FNE9yiJyPE9ns@v!^IZUnoqKWx^2?OAG& zTJ-S7%PjHQ{lOweCcr;?o8*=ko)7ypd)Bi-iZA#niQd;h*|F?kS)CqzwxdBDt?TwA zZC(=hK0b$%bw6l13w=4SDOA`l9i{nYnAijVv>y`b5>T?E$C02*pj%@o3{~CEJ8w)i!odBJsIv+YMW;#NS{Wl39p;%znTktc}2=?HYFHq zt+@m{M9KC|e{GFboZ47YPO06LpAt?CC4eADaBlUFmW(S`^jD!7wv;vQ&|a)%Tad`k?lZ;MI9wGSjc}@`*J!Omfl+1 z@2di0Kw-WX;FAa9Voak-LqNsx&&H8I+jWfW zgEejhY^NM4Tsy3_6+u9!z98RU6CpWT@z1K9pLq;WwYX%1>nBupc~Z?M7Txrfs2Ve zuK=MYFQjg;dfX(u#x{;_pS^wcxpJ%DhyAq~k}epk9e?PwO{de;qD7l0FP{?IjOUqW zq?Tu-!5I=G0ASpix-ma-ooWtDpojU*T`w6Bo&Mp?1JBM~HH^?0S|%57&q>rD!DSC* zB-9zW$9+-@h^G#lI7?zP-DI4QNc6gfF~Hr9U9x!Aip0+J5QZKyAZhi2#8(?;L>!zE zNQX5JG1uN!ug&=~-Bvm?o(r(Qtx^)DK0Vx3OP7hq>^P{|*c6eKncz8c;^2%15~9{t zzYRkutxVwSO3^|Hf6#^1pGui1E1e)(6``*<_*}T$%7M3SrfTC*8#47op4TG7vg^MM z3L6^-R)FR^HLuhfMvX&}RM0A74l2;1YNx}M{sisrT^e@?2>q_uvrxtW&`Y0V}MYg0;wyi_Tz%wH@n;VopYNJ=k}&6 zaZY_kOc3RJp!fmUyPg;Kvha`c~k(*R{`5IQzt(wxHoZY%BHiE0z z(+ZHUtaIDzV<7oZa1YyLPV3$AR>!mt0EZ zJCTQ)l4qTN#@iX9i*UCV;;vbv^N>)o&Z?!)$G5c?IrY&g45Yzmm$g>i1}MFt+Qm z{|JxmqJ_BemKRv`MZ_a^;wPDTmJI3D&5Q49T)J&o(tNUlNK_F7-nl|Qjod#T+%~jC zgRK({tc5n@+A@NrD?Uu_bkK>jh+scFcTLh2_bK`Vy)NqW7(IK7)5rb^D-~zF18=3( zXxr46Wk-yEn&3{k@<=&d;pV4fS7713^D3D40|;IEJ%+=q#WnX@ZB*nzhB8s)$9#BX zbK0JGMErZ3n5{!5CydAu51nh|DG0LrS`{w(;#%!0M7Ymtbq4M^*s#eID4q}AVG9b2 z5r`S@A8Me@A4HH9fnzRGi}EO6M) z#rSdO-Im`v%(Cfp^|=fwr;xGe9j_hk-5%OCkWgoq%SP^<3Dff^@YGvW39kvy8NEvS zanCRf{BTPEV6FuYUb>=&smQf{HfHko{{lej&=X#-av*(yU!saHW;{GZzJ@f)q&Q9;( zmzf$TbP-doyp=pxQ4}}z1;256711|!&v<@t8J6p9Ya(1eF5JwtX*23QUD~DLA7wU% zb)W6)AgWn{x?%}9t1!}+irJQp4v%nrySNy0Rk8Dl}3@@AVK`K0tYFa z%U<-!2+vYiN~a0_b~z!`0|}okHm`w3bY+grZ&_)^xhIT7Y5k{*BbN>@7*RP4q)sMP z3M#^)kBrQ=4`+pSe#-UwmMS_nxo<5x_E%kS9F`gXH2?M7S;*%8vW)AXT zl<4KO@$4v(99D)_dw2QHs6@+xJVW200TWB$ht{XlxHB-#b$ZeF5u;omP4~ijI1su! zmtyl-@=kQwY;-EYGE%0y>B@^|6EHvHEEz?h-}8if&F?Ox!B2;~K^70JJH!c&+P!Ca z=aQP7xmmCGrVG1WBU@ol`K?=DQ7rOuj}B>^ZK2f zQ|uhN84{{lmrHnc*I>L&$`!CxDY|J7uzxv@nBh5i{QV+)sA1r^*(0@@tjmJkb>VGC zw>L#DtfRkz6z`&%B(kFN29hT**9oz9Kf*Tuj8(=rer+IEd(dQV(@#SiEvE5m(S7To zE*zlpNUmC&+ROOerC@4{j_ImAb61PX*^%#@!=9m^x(6wVu!NMC| zuPdsO70cHOOxb@vESr2&0rM=88?{bPpp5W~uF<(ebRZk5l@umY=$h(pPHkBCHI-gK z`vc==l#=iT4e4c*B1&JGNKu#3?_ZQ04>L^m@>yk$9z8AHcj~_KZNDHR<=gHl&2g2g zIqG(C{bx(xP7Od#ntp<;?|bm0V>Xv*s&9hBmuoZ$ptpzl#DVX2FZI>t^mXwa7CBy8 zHbeovJDzh^dh?INL0*O8J`em0S7&qHcizCtv*%Fh;S7lT48)43In`mp$P3hmNyCt< z-I&x)SZ>K=swgai8;&-&DR|JFwF?cSVcoZKiPU(djzAij6b#^nQcqPDEx26NKBB5! zRt>BUZ=~8g&lAz(_C{w$BVD-i-1@Ah$<&$B`ixx4{bg^XN$PcRR?`=BO+bHU0~@?P&7)Aw5-Gmo^yPb^6$FD~5P4ATI? z7@F3x-r=%m^v)Os9Zd)T%KK1ZO3InFkoPWOgX10#Z@XIGI zsXzerfEAg>olP!|YnWG&BaZ>WoSe=5CsHfFjT>lxpr9*csQzN_=oSn*xc!jy&V?0! zR4`W2RYr(P^e!E5{&EwY+$Z|E>TtD*b$rr}=ZdzetYHUDSr}Oze>nY8hu{^5%|*|%EJa8kQF%ReY-Ir^>!23xENmft2geZ19aGQYX|u+xRIY}coyXesVd%6M<% ztj_e4QIZ3h*I$=TK@XCBOtHwj; z(&-l)GEokvR`A%yG43SZ`}x>3$jYIP$y=-8;nVIs;KwB=m4HjNB(IV(lW-4H*Wi@J z1&ot1{#mp6?pU`MnEsTHfsw})GNrdi?z}?IGT@yQsXLEp**Vp~4ZT3eHS0v92|_T7j-EF5rLbyuobePB1?b*-_)YP%#lN1kFRr-L4yo6 z%!=bn&UIBo5QFl?ViglkCcy9f(+@F-I~6hVVBhp4E1f}X6dCPRE`%7Dk1R=}Q3X8y zMT(HrrJDB%Ck4H-axmvYBl5Q6_2Uk!T6tE!jlRI2kgn(b@?c08k@!b2(@)aO?ZCqo z6^7~Iqyzs0xZ7W!7}c2utVPE3dpM8%JE)* z+>4K*a9RJ@D^VcuP&nO6&nDYaBI^cI@z0y2QH1t>>gPEihn@%Wt#HY+d#g9e zapi!Tt$FRL4i1y@sRNwe4MsQc1CHIUCLCh{`?E3ef%xeed$>7h!&31R*R_3tfmDh0`gbk^YTUSqa z-{|Ms1UvD=2Z(|`r^#~5kd<^Rtfv!`6Pe95_6P3&HdwHx*=7;{Yv@!un7q5Z6=7Zi zUUuI+^X{(nYG`R-cTGR*PoRMe-~Q8P!61uWP&gHNymUv%E#&)EMosxqK7ooca2tkg zm{Do3peNcpQA^i%&^_xzRpprt)l(r}=vcMYG>kMLT?B8FWE!W7t>z%e)eGo$;AvJQ z(I1o{CSr&~`bk2B_E1jJix~ z&Mkd>CRhK~23Ysfd(YBlHJ<9|098|y9<+wbf4j>sUgp&9)yR5tM*9Kf!yE6+L_9rl zDeh?-ADKJg^s;qx*`at?ngFw+%54$r*`BDM&vs~;P&u+)B{56T^`Pcp`(MTFP$iXve%KB1-3!X`aG#0`6-?UsHuhtTt;0pC629@bjn3 zAk(+Ye}v8w;KGzo^kg5u;x9+;k&6AHIw64 z{x%nR0WyZCo|2r{x-@KZo-2tZ$*v{;TLGR6@WW&g$QH3x7xV^8#Q-hX@iWscLc3f# zU3@St&?H!Td(~?69aiHYdF4sW^=Wg$(zgv+6M)wKyz>b##UtM+&{NYXwNV0*s_53? z{sZM=-+uKsNQeI#No8YXjuJFln0Z?ZRk7(hK;OHhF=%|xwpxCosH}9ekq>lZcm1y8 z>KRs(W3>cQ=8{r4P_QL47dZ*l(2iZuKo1Vh9{rR;@pMkO4WV!1&%h!gU=j4A{2k;e zt$Z`AJAyc`J2No=h%r0~13itChLY#7{t&Ujp+t!5;pm#Ns-#dt=w6RSW>9>n%am|5 z6)=AX`s8OSxdsA=NQ&9Q$WI$JGZ#Ztdp|aW@OOs1fOuwG5gY%i4G^ zLk-$RVwAb73Kdl66V-kNBG1#;@8gAZB&5wr&R2{PO9svzG?|aTAjg{Y`7xg8|MP# z$UE(~t7aWNUS8fwfpneGKC5re{y|*Bz*({jw+wTz;4#RyBJxjcMQcoDOQ4LHKd8&T z2G5PX)%(3AUE-YIBO8{!U<3Z>wxd-2`I#Gy?@1SWfzxMshn}TP@PsNCXzDSg;m!2t zAU^#mGY&rJbiiPC`G81N9j7J=$%_YtYjHzjB{X$mMkpKmS=VB>kyd=t*J zvI7Dd~-6fKojJZ4DPJGSSyuq9lY>d^pkbloc_!H(#6q} zP-!WF4zC+!;W3V(Kwt{F7o7xWfWD7UQ@{zPcAsA7JXn7#9ATWh^4AQFjjKaYn47sP zhwGaGPvPwxG99%9+0S}Wa^UU`RMP!uB-7EvgJ=~8O)FmgC7c^5y3fiu!k~8b&9R?b zC%xen*OLPVT3MY%xetOxW7vs(2^?`Jj=#b60}#Z!=NJ4X@oG?Br?8&2vHW#nKM);zyYL6HI@(ii&P((7SsM zxtSr+QIV)L0kG#m1>0()?Yt5Dp<#uZD%~@yij3g}!=ipQ0GKF1u*OcSUr9a;%6mT) zWXhwxA-CW>(3KZnyMMvAea>|EtioV*2`@82UW-nf*=l-mk#`@NexKHx_0vlsowv04!HpjQP=kI?DZPhC-0N@@At(I!EU!5Piiu1-5TGSJ zS2MRK$B!R0NzL;LirC$Q&~SS8)nnUvG2u`6^@69pA6jzSt}(5g>i%Sdksf~k3<%>j zke1b@y~xTq(~9AN{1nes434W?y^jCoW6MK71G{|>*A(=Y#3p69mT!}rb7%U(CN#LT zErH*QMh84lFv$yfvF_@so=mz$*Xy7JchlRW#?>!PGV>newd+B=iE}xW7KZ7!_>B65 zyj!sRk>pDLO6&V6-uCG$luh2;(|Pu#a}Gf5-J14vhm!OJT5Y7fp55kgHdmkbvFqXH zexv_0rr+Yi5$ad$yY`H}A?7Rx``m|1M#n$pe! zZb&q~JY2~>%S9y;M1*vFInKU&Tf3o>a|h6A#{e=kB~LdASm-yKz%*kC$U42%V$LDd zgWwx8X8u^3`-F47QhjthSQQqv}|5-<5xko80vJ-#Di+9gnC-Bp`QA%%K4l4 zy&UgRn)t~ukyFzGWZMJofp9CG)X_=Rgg@zN+PJz>mT}`Vp2}PkR$W8g{J@~_F8mvI zqUyna%z?>({sPCv!^f3(iGkg&u?_?&UH%ZF(}BJvyKg@ng4nAP9F&~6zqGj@>Yiut zn@6w}BnMv)x$OS9SbAjqZ6UK4xWPG-R=f?>kx^b`j;zSJX+=Hz1&x!leXWR*ev~0^ zuoqXPE=-9?xcH$`ZTA0R??1zu+P1%8cw4rJ4G|TkDhNmyq*s-u z^xlgS0qFvU4iO8zNbf{IdXpMzQ0cwb&>@5t0tr2kJPY0DZ1+CpKJWi}Kl;ThxYn9; zuF>Y0WBkTgB%VCH*1LfVyKQv%x1fgrw5WDF#M716dCA^=^5wUzpuTUcaq8i4NTeSmVt&wSx%<^%)B+y0J=~2{03x%$qe56jz*VF+P?OlyPf_Wy7 zoP@~R_mBG|e#Ic~Y=&;)qj@fQ;~e~N7|BP=&)V~eFhA9?D6wfn(MI?^ zv+*1=y1X~*?=EFDNX5tFBytaGO~{z+)4f%BqTuB1np6DOc0;0;6VONQcdL{j=da!6 z!$?`}(x0-0EUBkJ#?~@@93^~PV!Q^HXcQ7Ou@~{~pK|M>A=}46`x@&{2x(a{wg?sH znlEGRSYd6B{Tn;^x~V?c=}+!1PqGn6&#z%uuiter;Bq@YRsrWcYc8NrfGa-YIB%ie z%(6pm#=)XL1{7jSeDZce*mHzFd?pzB6gQ8NtThH8qEn|?cn7aHgFwh zs$^{=JfLwX*p%}mzWf44bkr+4)!MaQjZjP)?MJ7&xiq4j&6v?jtJzAH6U^@r%D)x? zqz?1B%YTY--=jEghe3Lu;^EWxuSG?;4S*;Y8TZ7?-a#o(+#V;D)tG*R=lrVFRZ+o( zqUMieFVZM~XRa;ArceNJm2JZi)p0^E*-&G@3+Q)|Mag@qhtU-69PZ)jyC>wVz8>BF zyfL9xt8%N}ItG$j(9u*>xQ)6?eD|0s_U<&kRF8>B83I%xul*Li`IgOg8P+aT%;{w5 zx*o*18>)!04nG`iU{Bo-(G-7C0m}(Ho%uNyTbQ=7b6MW+9q%bOoFm80Rcu%R%=f+%kt!V-&IOP*>kV6vB(_FYZ;g)#U3$&y~H9+%vuPgTise; zSxx+wG>!~DDJ4o{SI2NKL!$@M?IP#J@wGI8gFb&|;lHq*k8+mUVRPbfJ6&>i&wOyS zUbw!Xea|P{(2`MWYwoex`n$*b{7aAd^-7M|H;NPUp#{8<*k@KQTC^%1sy+F6kC&si z-ilR4Ku8z&Nuz<-KTXb{*K5eNWsUGL4<7(yfnQhB3WII(WQh+}8Zlfv%*XbGMztj$ z6}~$M(}^dMY4_rdz?dMa9X=^q?I>x!+GwE9E<;69ydayo1ozA*T zR1mj#Y8=W0G9t)XSM5m-z3>0pJ|Uv{5j0hxNgxe(P}U%jF_^VpI)7+9R_tUwGBZDB zaaEXp=2e~TGLf~R-Yy{`CF7Cz=kTkPLi#fDh#z+ArU^@n?%YZKK+HymLgs?)OE<1| zw-)d8Br_R>n)6ZPeuG}h_^_`$cWqjDSl8|^PJD;gfQ<8E5R2APsS<6-2*%0RqkBKYa*+#H`X-~@e8*<-=$KGBz0+uCpt=LayI+NFHFc zGio1In*YMm#V5-9I&cmw44Me*1hPnW!Q!xE+zWPeB!3(2kz(8}{~{L;nIxlGwa?83 zEE}x{MCd4Zh^j)jhR(~OhG+|U{jzJ%_@`K*H zFQ95e?~$Q&g~i=DLTB!-;ogWn|0$6KoJ<&b?o(!g(1h@I z!z{T5`;QLZ;i?dRFJcg8aG_+#HL|GKgz-so^TksG*_r#BgY68+l*D@lRtd%y8bEq! zXZzE-a3-LM#chK9I%sQ;NkK5vi@PWj5RMyNK4e5MvCsv-s8nOxJG8a_?J;$a4!lmA zQCx$fKon@!(6w9)DPK>FO>8b8OLV?jD>lHFLYIR=w__q5(OISg-o@tzjWnh`gx9a; ziG7eHYq`=h6&Fi%wlZm#KtoS;urhU!x$S?A9IeJtMmr>`tLLCbS#dFWo#k46Ie~E= z(ZqVT8tw4==)(5H%O?CGqpPfJtIoShbb@>#McW0s^B1*t&3pu);iFUi70CWw3=`C8 zQN-mgnBH*3Wv|ibp<{9XKVXz??AkqVfTFi|><&jeG`&Fij4)6>^BtnAh}Z{*dN1?? z0I1<0Z*SlP)Z!gvk~%MMXfN(@jJGJTPlLC2 z&e7PYD1C-^E5jx8r#`YywH(Z<7wNSq=I(fSjF(t=Oj;b;$!O^y@`_YGWya<_H4q5(;YH43qGr0#FHe-C$1ckSrm;NDZb~cElGiDlY?Y$)%w_#hjog~}gbnwbjFtPR za|h;Sx^>&0CLD@NQYN~uL=UaCAgY6i(Jg`l*Tnt(N9yfeh1<)7vWBwn^{y}E=uPQA z$1>}fEGl8V7?(hZ#GwQy9ptC{ks&@Zs{+f>V3yp2+>5qFm>U(OU&oI+)L8`^>zQ>s z68JsX>TL-t4R|g;j~wr58?_>9>`_zuv^Kc{KKIK)<#x{$_pWfv;k=NM|k9t#@CQV)A5waXamt%xl#-ueS1spOH%Zynk<)9bRL zC;4$s`=CDGcU%)$Rljdecl-7agru=L&`LrpzDZcuWENuhJ%KQqp@6^pHqGk-L~0>)rl@} zNl#vfK>f7LTPZ|`T30&H)jy3q@aqNT%-mc_7l%lBZVvMB4JtIIhy9VsjlA0Xn%&8# zsab(Cc$rI@dnQCS8G{t=>zSFU z+#K5Iz0Xr+N0I+c;4GUuP?q!QkYXyiJ$E%PaSLr z=Gl$bUi7uezulv5`026Hh+!e-Ejbl9cMBXsYmJ)BO!&6N2)UOUVv*Om82-fTZYrHY zt-)+!j79|ar$ZP6(kGbmlV&#Wu1;f20C9d;{Mxn&HLm#CeHnxHa;Q~ay@c3y(nkp> zeuRB#AHD@ZYCYCErt7zmPE`jb)*eKMXoI}RN-(Xjqwe}jt1qIr_bsg0w)YZ5Ai^E! zf_5%7&0$uc8VHg#cRMieu6hLhg9(+*`g73v48<(Ad9QWGs{_oK)jW|zW7PhMXuyiN zjn>DycHcy>*h4GHV`|B`Hbp(>OqVtD7yhU!k&%~srX9+ZlpfbA zADZ6RDAb7YJjRK*0*-*b8J9}wCn`;)PCB6H6eJlflKQqG|@KIpQ&dgJn&$T6*JPrmeSheMVq?ils`dt&85&uaHK_| zNZl4PDWjK)MrQA*wf(huPr~m&&GZ^?kaB^QRaO0rk-{}-=2=4-4zKoz<8# zD{~R@p*W9->1oo~>c9&L) z{o_|HtNJo`jr8IR2a;c}N98wTuq&1sg(_7d*`p+9y;!@f=w}i&H0WQ8J~4x4ykwbs zjxB1DRNgd_iQUd;tDz#xp5RL@V98Nze=l;4nggAYz@F8+>K%T4?^!4X#j1!3lAu+b zXv({v)}$PG>T0Uj^SXkmu&i|zyVR0x2PsS)1lMqqk|SiEBy?Q8@JQ3(XnWRURVsTj z(hdGH3HzI_7nKr5K{CcQMP8Qxf3Ond7}-(0jFVy8m3_~B$sG}|C}Ax5z@hS6%Veeq zv!5HZ?rj=St+Y38CgBwA^_lIU)U;m$iEEDcEyqN)0SxRUb)h zvpf(JYH482UBm))imOEsXHk4+DxEi}EsF%*t0zph9k|aB?#-n8BUMsu_!*@#H7^ad zX~D*+q8*n#F1*iR$?++<2*R{3s6LVdM_wBCPqpnuk%CmJb_!B@B1QmY(!8}*K+8u} z`hXrHP_q#BB!&3Bn+USp(x7e|oEdp76(lWQ)N@?b)Uh~J17cXr+k72}*y(&+v$Fb4 zL;TSrw(ZLDj77Axj8_U`3Yz7Z*GGSB&dPH;P3di7E6k9yep(^%*zLUVWkZ%?Lfn&q z+P(2+kE<>^eB0F1vJEDAm5=-Q2n(NGatxtc%prVwItYv1I&l>TlJ~pkmA&ig=2!#g zuiDNx`xXsU8mX_Zp3Q)QkUQBiGKEgCT!|gGXldLk7ZuI8cBmCyNa|0=^INYU)ci(C z8EXq{ZU&r`$p@2{O*Yokw709q)=r1Gl+95Dg07(e!zYo54ILIrWN%KXP&;+>;mLql zr`6~(?FVy*XMJ4`C{QfXaW0X0(qW~$BbF^C(=}i&`P}qXw+aS7RF{^DeEV@7(^K%y ziLjwZ>m_hk$z6)~^i-mcW|dJv*Nj%;I6OWyL&5Il%tB+KZ9PPRRat(;9=xB%m2r2) zhRVpdp0hI%@2XcWdGGauA@Z{%AJn=1HNl424xT(r{I;2Ij1am?t6@|w=`p0pag}n- zdehHgE%0o;a`v6^-MPNNfEB4_z&}!9?I1Q;S=&@8FXF#OVD8p(YT>)5ce7Eqsj{ru zj`qgE-cC{VVQf#1hWgsf#0M=Pa+MyT@dml`-}m$z;0o`aBV zK}LJ9?Z*1TH6S)-oAF*)?>R(xg%%qA09m18qxT~v0+j!2=CyNlf4vT1x~)XFR`t$d zG$c~@0_U#Eh)eDeUPaeurm0}1gi4oAZG)S7tNN=$L;h)>hUS2JB<&%T&W;;6BpTz= z*V4P=J=i|y`Db&-V+iWpNkcZT|99_zXIobdU19lnI>^YTx%6arX_d*VT9z zwxVYaJT-7Q1~4@HaPaC1hm(bV_aN+TebIc&&>k7CAOUtv&g=Re8;s@ zmwMl<<5eFR)T}f>at?@T*<^Y3&~2dE0aJJHj!`)}lTFZfiN@ z;qr@+UYLvck(F*V~W-6XtRo&&b6=L%J$(`+0`=YWP33Nl; z{(Y*5xpKa<-kCRO%SmYgevWw3g>_kxe^8JE!sPVU>oy_$oa8HVdH7eS;5H(86YVQ0 z7RfssitiylM+)Hqlk5>HJH97@RQXh@p4;bdL>~qhY!P?Wx|5@hn2gu%NF}`OcGMw# zK`7a^xM#*g4)lo&8m7NT5slx6Q4^R5w>lLY9q$riR5uPZWFdHl0RBelrIKk~T9-@_>f@BKd3vcg8_ipR z!IW6sw~Hv7V&ACaoG=$i{KzL^pFTk8h$PyH-cJ6 z6*EH?4env$OG0~>iZn7Iz3eWIHTcZT)yLoM?2lyVEUJ@xg92r zm1w)CLyW=s|p|e0s0MWbhN`&_ph*uV?0xiWuHE9L4h0ytD zhfv-7gM4`=@H`mGWNTJk<^|osqL<0`x^dztnqgqGK@2g}4y!Yx|D4*Xf050Be#6BI z#!c8YYGG~>H2`aoi6oO}@7_Xo@$SdYG9yIx$H*K*SocjEo>Q|(;-59;u?URQZg{sk zl!Veiy~OIZOhv~7_usfcE5u+K-wg3NF)P$0#JkVv4p*N`eYz6H1on%FAsSn`b{{P+ zxD(Ze>a}RITJUZnE$cjZJ=skOTti?z8*^;mWvCa|p2T|yzioCSL{*Q~;9JGo^;n>L z#-M@_suR3^PX#}71>T^uI;vWfVfz+WMu`4AeZ#6QZzY4QDngeiqO!WH-=!Yl^_V@j z_49G_oUUs6sxkmcb-BApU_yD=N;TlvbRHiVO3rVS08yMK@qyb0FRbeQ%Zjb)@#vZ@ z@dB;=D6p1_6IuLV+EHR6QlInWhkvrJ#5&233$&9NM&yq@Pgsb{aO{%$c{VTw#^q~WSD z3ya3u88COE?)M$_mTf#)9a&|{^*sn1m*y+I9_Ee%pK~KxOeRYS92#CJ$Wv_a?3YhI za@s}W>g=gBu<%0cxjMSh)IRv@O}?i10I*-?YHiT9A}yQ3I04(VbH+fGsS|HKS~7NF zTm>9oWFa>WiN_XAXM77aQxJ_Av?wY=LiqU!O@X*;k*m+AdyFmU^IF3f(R&FEpX_OC ztfeIvZit3%8Ldx+!Sx&%n|c8&D{z$#Utw5Yxp3scPv%XF|Gsg$O^}2EiEJ-g>J_gi z|Lmz%-mSIW?`QZVACe3^26RPtz|aG6VGr{Fr@x=4ts@GTx5!^Jh=zRcqvhZW#~1AY zDMZ9Nxh1IX2%b}=aID2n5Lcw+4m?0F=`ETy_~1ZzA6`bHnTw~U`?xQ{&ubrv6WQNS zq$1K^lVT$3WSHD#&O&UL7EMn-PYd@uy<#M&Ca?_xyr`j93ZgB!)pI}>1c2U}?+ z#L~^GR_f0V9KZRzzw8>Z$h(z6SU7q0k%=<5I=(1Hh%8mP;JjvwdBFCks|fA)gtQl> zl580fjl9%(6W)P6cMq*^+YVf~xLd>zc~}f%$aUIVUC+#mXI8|z8o_SUVt@rISNwP-tzK_<1p}&$TogC zvrBfaEZwRni^NU}x^mRp;znvqd=S z*xse8Ul)cvIE)5tNl%AvkFZm+C{Rx0YBN2%x;p#;qcC;`1wNu)^#-+qyTJo~wd$_M zBN)K^YjY)!c~T*GUX%kpT!fk33T6)3YHfinfP2MZjO=5Xm*n;a>>cH9rz2TFS!sfw zf~H*uUBMq>RDPow=#CNO^w~gq7}3D?iPdD1UdQ&7Qqjnhe97l??y%gUdGy+~b0#3W z1oMG=yW$5VPU~rLGSwx_*b7a?ftfpH7oF;pi;!Znvm(p`d3+yjC%}>KMZLShw5bwf z5#7Zr=pDS_aXU6<#K@Lg_^w-YONBc{b*8JP)(4%R_7ii?%hmg?!?w@dfv)XbpW{n* zR3{W>NjMH=1_UdGCs@JZ-vn7wvV;_76W`YZx%XC?!1EI-GNEU$dc8B4>HOn{uQee+;IVm$#QO1u=FA zf1?@mx~HQSGH&Ex&=hiYN=0kd={`NsLp`ZTNN$s&X*c#8UKQFC-ps1{?KqWF2Okth zL^l8#} z*G&vV%F>w+RGVn88k6PqB07rB6J(3B*g|z`O=f%Y;(e<_RabnV`;`hJD=BtGwQ%g3v%?SI_& z0La0sk|wLz3<)qa`~>~@UQ2fO;fy>JUxWYy5Y0TdH)3oae`rGYXgOGw+3j!hoaOlZ z+w?4@>+Jni;0lm~dU-(~`TLhY8k2t?>~H^MpSMvbe(?Zg^68zEh+JApH2_agPoEMT z$-l-c)BOti*o?qL{d51-A5-REoe@3!M@#fh<=3Fzwfy*hz3s1LXgX)8!bx^yFi&Qd zQYy|cfX~AgrMM;h*#Q2J+ksA)8?@eZF$}n4>PE4C;mPhq%I5u`i|(rv#CxVo`g__z zml_X$%7OlgTl!$|)Jf~cN>t5*{Y3MmB|pXiQHlAo>~nvK1p9x~b{d`hnITP#}iR;OOADJ7IQOq)<~t7nGsfQk2m&+!T&J3fWBrF{83XUpw)*5}kmH|hxkD-@m+^=! zTL^;-%)2vG@hsJde~AOYK${?5YW{rm_(9zVO0O{+{1>La(+Ge8fO-|H;&1#R?$0j+ zD7_2*nm?sM|I~yBIzGw#-^>C>Pu;24h^J()(EjvEG`Cmo8Y`Ii5IJ$_Utg;1_@yR& zX^!os&$Hi!&RkqoK>tCi4<q zv(){$rhhzlHaO_Z6CNwVqfXa5sby3+qN!I=B$gLr}`NZAI^{s@zwN&0;PpC_!JO>Bm$$U)J; z{bJoAJUfhxSO0d`(^n@6WRfD13IOR}e`y71A2QJ$@q@+>EKc9ae$_mt|8K}r#; z2qR))tMnJ1-aB#9h~l$y-W}k6-Lq@dZ=ajCEt9iIa|FG}vhi2ek@FN`z_dClynFOl z9-Iw^0z3|b7K9%Y`svFPn1^riaSA_fdSyIgR@`2vpYLp2R$tU4*7cVuA3T*m`&c#p zieH=Tm0P{OU$9HK*Zwl(L3lbG|9Ysu0ed+83hQ5=c>Sf3bWifIam?U8z^%-h*SjB= zz0Zt7{zl)wE%8sf{EI03y7Y^5zog}t7W~qJUs~`>3w~+A|4|FFve+2^(F@>juGlY^ z<8Ln9FZ%oviC-e|OACH!!7nZNr3JsV;FlKs(t=-F@JkDRX~8co_@xEEwBVN({L+G7 zTJTE?{+kvw%YU@aKf`Uq7ZPvxW>(?;9d&nlC;i(162OOWax%kkD|f!$ zRgI|4|F@9$(=;Iat{7FjedS*U`#C-EQFi7?QhYwCu+Y)Di{pDHz4uPF2pQ_|+r+kBs_17ZD75FoD)Q zexd(D1E02AGu^C_vv)_&0NL_{2P&*to-3RMCMDxoZ zKF-nnO?v-FvAYosuuE-IFYf;$;}0IZ33U3Ogz8^Lf2s+nhisF@v;PyDjv4&l>G_|y z@c#z}`!e2h5=zD&gg%ca&u?b<9|`?4d%j4MKBkMiyPs1$l@7XZLg!%B=uIBPC?Gv? z#lfi&#VBz@jT6m*3(=hkUU|fGl;a<*WSg7OdG_6iz(Y@l6_`MIj&jnM=OVIlXWpWQ z8B{ip{LJ0Po{6@z{jW@z@7oKrx;MYx}w-?MT;SQ1JnQig^@l! z0Qb)%ykJF=`fSV!+$zl8y4l7X-K2C$>ZA1yFL3gW6D==HCkmqfMQ;A9wE2);SxWi1 zt&d)~?kbm^R4N;U9ZU*iXx?(qUr&gg`QjujDdF3htefn41lNEpu@dYjHWVV2vG_zzQ| z$m)~X7_~lC|IR&gkZV-;e`dtJ%;PO8$|?4S&BHF^B!dBMTdFqRA*9kVpzoOSneQGj zXLcs(O$};QcL!c1D4DiLjXF|4`5#vQV}NEIz>x19C^z+?25o?ESK1C`hI42bP4WF9 ziBHwfCmqCqTckzWsXZt(s!iqpAHKjJ&wXFO0br_^TT=TUJ@pFO`I;bC()n~gUha>z zEmgj5j0%l3xc&1J|Bc&XzrcAn0W`|uSw z%nwr&kq|X?zBHxN9n^xLihf8}1DZOYxJYDJiRCX3sW@o8aVu;)fE+KqcC%QR!9=AH z_>1!@+k5bV;+%Nz~LnsvD7K!)M^}Il%NVP>gu`_ zd9*De;AV9_fK>xmIcD|0N_Cu&Pi?DOe8S0=7V~;Z28!Q&+2U2Pv;Hud38MjFZtpJX zVCRmNC0CEem$75%_P<*+gPcqs8S8qgO&@BLfvS4ItGbs%?EPJpr1=9r7TTwCQgC6@ z=0;>E`d-j1I6Uk0@MsKTaHjrbV*HWV@2?_jW3{sDqDDkyR`hAp)1F#rLWD$+o^nfd zH@=ljDT|taimc(m1^as(h?w!VXEE^hJ^PFeiQ6ynSM+|76}&6_0ddc`<*paJw=5E! zv|H7h_1HwxmV;SIov2hDJLd0H{lOrIKy5wO{CfsBu6LJgcLW=gHc;I#7u}($e1`GAqEp% zPhd9tJ|X!`2Set!Rq&*QVnD@IW7JKgQuz5-+9UQ?-i%MLr=$?8W-gvj=_8zU|Eq8Q zT@jbKH+-Ne4)Q8GRi7c&M14=YA^dsZE6_ukrC;9Yb+q@pkwj0La|P-en)i&>*^I%9 zZ6-=GCtzSDy?W%jBdYmOq|id-{@AYNt;@M;`ishB2xPnB*!3L z|71x_VVnmje=omsvy4uELQhER z`9pe&aLSu7HjkFzBH;FJspwkQ!*p^vvLT1yMU*#RZ0)Eh_eZ;yI}{nbPnaSvhX%Et zKZ=#UphZVd3#WIGmNvGT%6&CQ(+Unl@#NmOE3Z$sA$I7Fi85mG`OflFo42_)BpK&m zfaaI6#S49CyBL2sWG5^nVjl6>*dN-a0G^Jjy)_P{;Tc8_C!AOfMLc)68??c`&aC!w z1#P)^yiVD^sN%n}oFM~GGg)+4-xA^>$QoZX=_$9r zptY~Hy3`?2BmObTVKmX2q}q9Vw_Q-j`=hh5(R|t%J_-~g0jH^*Nv&&R1$*V%Qo{;4 z*@url8P+e7AglTJw+-<~hYl0HP~9Yx_v;+g75QvX_(6I2*?~i9Yf*gJ{q1k*lIn}n zkCe&}5!WE96}t-<<-en?0lCmPJ{D?lx9G(^_t}3oYEkt01&gu1Eu-t_ zA)*Yhn#qVZzId_d={seSN&PnjN1`b^$SR8Fy)G?OD3*hw5Mtww&KF*GpCu7yi>cvb zpoq{O3U+u}W|r3fGD0J_q4o}&j&vDONfErfnV4eJ6B52=3Ev%I%32TPsdtNw zE|M1GPjg3wWh?9IU9~1iON+L#4uy0+PmGnT%^TOfKN&ER+-$WQ7BC{f#z`khji$C* zM4v-F@3I3IJ~5R?awchR6x<`?j*T9uJV|_Sa99GMm$nXAMNC#V8!xRH3pU%cX$##aILamH1SQ5gA$F+Nzm7+e9!ky=VP;zVEOv7o)$IFRy2(9T$okMxDm6~GP69sz zJ&QuT&e(?<(}N`c`wYC{i$x6GVM_roxc1$LCnt1sTvotYCG96LQh6YA{@cj;p>Cyt znsMQ=GLLzUuEr>hY54m6&8`*hYWb1-)#LS%gL^g~)Q-ztoKNl(;sTQ74LvB%awL)d zY$oD-PoB5HXr*|zPDc&I)>|p=5d3BK%iCTk=uEtS{%U?rz{x=vx8V}y$KlcbIYJ`= z<_VpoS50wftZ9F{>nV@Re>N0maL`!dNqFm99^C!&*gqW5nu-S=k_{!+)QZt{qGAGN z!o_!BMgRmxm=>A19K#U_ln>#)N?$H9gF@WTgvD| zlpoLjQ1TG12BV&N%QsuaV3dTQHWXSdQK`1u1Y&6sk2I;AoVhYrQLaSW*bmoeR zTug{@mrF{k(}5fLh_-AiN-8B58d7;{?k)My>GxtVToTbux;#m}pz#%)842TxKh)yf z7l}YbppMMQhd#yBrKR*V*~@BnyjQD&jo*_UFri-*qS&9fv6xx0=55^B)>sJ???klR z+2w>zyBWqv^R?gR*&O$G;iF=1o3C$iuH|Bg6ro}=NuRnFUS!gR@OW8vq%6H-$(8Ir zX=!YqPS}rY-Hr21?u=*?s4AH5Zoh1%BgEo2+SJ!0flLCoIQMYauPm<|9Jxj&U%-^6 zPqQ)tVUnj=U*dVmgA5p zHgw0c$Vxoms3NYnL`_;~)$7wty>vrGWzp|cl(bA7`N5tBO&J+YA{UN~H3e*qbPDw+ zyXAVfPl?;@vv*YK)Kvo>Lk|3{_Rmv&58L=yGTn31N23R)#!5jrNkZG`JSAJ;KP-x^+@|`ckqMhcUPE-ePb__#MD06^&yY<)Z%23&ER^Np;06{ zWNWEt!7x(F$g5YT)?dGWgKtrb-sQbhNwp+M!bnLm;A>4Y<9(*7FJThfwyRT`lY5n- z9sHEF&j?w$wgs6gZushv7H9BC7iMKS_HK20$4-AKHcgP@a4*#NA+xd<;y@C^CSEfH z9;3UzG7j=suin@`pDztHz4&DYB>a1GZ$@>aAJzv4|76y4>!^&-OqAJi-*dq@~h0cQ8qA|@| zr>M5Q!#F)ldNON94sjV6BY3e1F`?bk-stFEKmYwmJ3p-o`$;U3OT;4VHc{o}IAU1b z*+XTdVf9QHiza$M(hIZ7h9b;t<1pC<%80mgH00?>r{w^U$jTs1aMPbvbjR}bC5l35 zxog$9x-JWah)pTo7}2nsXW2XhctI9bhlSkbL1&X>DS)vfR3ZqmdKBHf$6h&l#yPYzxmbEM7S* zIUpEx9B}5fdl`&GjZHc=nCsau&_3NsglxSAgPTZ1v*`D79d|RSJ93rS&ve>n-5%MD zJ2KYgs&%U-YRii1AN)%s)7k6z@Qb=3Sz)e06d2cM%F?WX++`~Y7OL%!b76K>mhMi> z&!E72$`5p-jBaAN=+RNT!gNL+tgOgFC40&XMqA#jhH!g|x?S<&)SV$iV z>+h$j9ar>(Mb;D6FQ68Q05$Cs1T@>uJ3~hXb9NYZmj(bIDN|{=U=h9`U{{zZER861 zRK8T4$e1`8&=}Pm(-d3P(zumKz3sWS%(`Daru|0#PH7S?cS>n)2PV$WTML~?{)8g7 zaiPU!i`phFk0nC6CT0kn(-pXk9(6JkJ2aT&JvCn44s9ypT6{l?j$L=Oe$|ay`jAoc z%EHm!_D$n!mt=UKY5jUoTg4U4>YDZ-M#-g3PQg!2Jg!RxDsGstj$TqW2Tv$i)|qAfz|g6}Wd zA!0C;S2R~%HWzYE*I+TF!XUwgDB6IgDDhVUN&j>Jzg)e#wCtwuvV73{1eK3}b?Dz} zz?2)@#Z&8c#NFPrUWm6%{n`Q@5-*~uU$d;^@sK4>zi7J?JRd7w?rAGyKu=L_*he0u zLgnyA-sLc6@>vkwkmFja`E%ZG9)&aBufW8Li%5hZ(WAU2=1M^JiP(i)cY+a(*aq(PxrT45J>2wnWRqOc13kuse` z$126=IWgjS(Q)g8!uO)X)F||T55?8)RA#2cpT%ZmR(#C`OZtE{&3qEs2143UEscJR zD-igaU>%lW&QE#)$HV=o+dj}&BsNW<<%e=S>EL{r9OaR<+r8idu5rwm1UbR z6sCu>R-Tlp1@_z=#KJY(b%n{2ns35-FcZ+o_4pGY!MJcBa&>R{H1BfBx|eOg?F$kC ze;JYw1#kYz)!%%QPloUI(gI)Z)7&?^lbeKJavD}%#7sHc?Qsc}o8g-s$7rFf(7GM_ zj$!mE2+7iCnJ})JuKfCa`x+WQZjs{2!czyFCL1%n_KV+6pwR-r{yfNQ-8|tv3EQ<{ zH>um4MCJ}n7b+<)j|j8HAL8mq>Z=wewG%pKgH$*GKX70|EdO9WratX?+K0ba*8+>L z1HRg8W<^7+#*0H7;}QvDOV9weQnn7IMY1;0@*Yb^TNBL}XOaysJ2~N%^OpJ3a|+h8 zKEV42Ek3WXm@X!gtlkU7aan@as!bZYPe-T98{bW7JNuT2{nZLhTzd=9LAZJ!F9*WG zPPv5>U&_xbvksZ@_T{MwGy>5#OMn%5eK>OO z%)YkLwA^5slJr1`Z6P1#Uc?i~{=wwkF4Fg7Q#rulNp(lQOZY=^dArZ7TyOp)7vC?{PI=Zvta z_HbW`IBAfdgQ`P#$P=Mq!6}o);hfj+7Eg&9bXnVlJ(Hk;1?!DZOc`JT6!@*=bw+l| z`Mb)M5U^cOu^Vtb!P00Io#mUZ{EsGo>$^S=Vdi3P+buegF|;?f*@E|SL%jaoimJjM zb5LKn?kLrjub7&*-3GU=uxYK2_vLU{Po_upB-|Di0544qN4s36DQV<>>CGZgLO7&( zj`ygBO-U@wi2POUd|0_Y1h5sK?^#}7c8?LwRb2aOSZ*m(wKp6Vt#QfqN&UsPns*$D zE|2w!3k$*?9dIJDITA`$f*o`xWRTUe8i+7Cu0s4?6}qV-qKf3=l4nzHQ46htx4=iQ z&%R2C;b@$lr;hNde|^|mZnBB>;+7ANUbC>$Vu2pX5m#p+>{d@1ecp~R^EB}(-M<8L zzi@E@NsOaYhjU9J6VQ;hnhO&Hu)sbJVq8L3ZE>dU=giudT-;B^^81%hL@&1q2;bZh zhIk2bSN90B)Stq-T6*tj$F4+{^{7CrL z;X{ji)AtpqI0s*PHFx9U1U|sJ4F{__e8|UPZ%1QZ=QaWHM@s75mt{ViG(;&Z%1vW` zC*M;vbG18XO)$Rw*g5-_Hi@zuSI_B<{G`nAclBSh zUZFzRAg*<{3Alh4CPgB$`!&4XW~r|yiT4-O&V-!rzPVh2=5U!^r6d@M9?6cQz6o|h zdLGBxT2uv1RX#PWAA>K2oR{qHb`CDYp)oRpZwA(s+U z|H|bPCpk~w$t`=m{1R7$sr!65On?|xCwSjCn_ZjrD)Z`E!9+7&hYs4WhWU$zTo8O{ zCVS}Eof>pNTpuu^wqH_;TNd2A72V{7weBDwW43j6>|c~RqnfQd(A0%&_%6xn=_?A0 zi%QLMI-z>*?p4lROvp?ydg9RY)hdH9hV-Npa%^7MFJtG5tnJ=e>GT-5pSp38cDI)w3| z8D9szYUvcU^|-H9`}v4GQZ~1T?u0;RgZg%{!5NSkgkeV*O4bwXYAj8+YDS*U35LG1 z=zmhf6e)y_?O1KL4CYC2%Hxrpoc|2*sj>f1vvP*|p+Q!+j|iezLY8s#$id)FWd2IK z{Sg19^ka8iqu3^NzyEf#kieqFp^P-aW-OLLg3jTgbY280*S;a`s-y9Ry*`06$xA3% z>O`UEv9oua7oNt?=mO~~yKjphsQJEKY8VSc+vcYOo;f(t(bfo*XbWrzV~d5Ma09t3 zTC(-=vVWv#eoAS?eLHzYi+N`TDMMJTn9#c3J(V}@+X>Zz;HJHk*`lp=O!O}qY-M}9 zPzihX1t233guH(1rM9<}5@nq!;U;bC93%N~S6x!J_}X5>xlmY_lI@6nplA^vOZFNM zY@HE1EM*M1pN?z&EEkqGDo+_3ch{|`01inq{+v5>l1?>-+f`MM*TSvsXcW{?lU7Zu`ht zGM9$J8^7lKd~2+fJJs9?a|k%mCf*LaBki2jg`>qbAO;HcOL}`7j<4>VLJRNCl;HEl z-Fy1B4c1wA?W(ILN-fRymyL)BAY#KYAO7dQ`orS%J^}pW+%O(yVjB$p&N*{8*nB^rDKr6gXtMnFC+>Ux#)3ck|)RHgK-x#b--3x44 z)!WmmBctYs!k5YwRe7M1g79qYC@f#nZZSATa678-{`@Dpl{(8cfiI6wpH(#XY1^O^ zjQ4a^%uIh~@Uu**AIsGG%PxOWdA?XYtiaxmsw^eo0xX|sW^W6w-YgRA9@}4(&Xg6HDyR~>WfNJ@L%tlg4vvl1Z;nd zlEKJDk3{YKBL~lqmw3$(TZ!Xo!&iRqLW*KcKPg) za*iIn04qOF!6j?(wb*n~3DM^#yZS5BJ99dSpLjGe^Gep-+vdyD8KS zdwJ|rb@R&iJUWFx9(I2%VpmwabSN)M@r;*O4Yl+*c%MKEWs|H~?GTMHOBl&e{!0b| zh-Tg3_VIyshKwWxi&FWlC%ng5IInO`6fRe%Th4~&BvXLA95CJCa)Lu`Ufy0dD+u4> zs`#glPk?Bn#C0Cs98p=fwQ-_lih9L`iZ>2PFk6}yJBuadDL5`o_5 zoRVhA(uCRPXRaEGYnD7>8!(=9DS~!_NHiG zMS_=uG~7k}Q~y+4r&q#AcH?{zV|U&lly(5Gy#ftere-vk5#yXAV+;)#l{9s3Z1-Pf z!1CkXl8?xiX=J{U5yhwe1^wnl%eHc=ZuX2(47uVq3f<`*g(nz=i!<-sMMPN`npwW-eW z1R?Q$D*_9LLH0*sjx%rkqJ9tmO`S9TmIHc4p*;{RnIjOk{A%j!oydpg&OMeM6KnY1 zI`1Ny>J4(Ff?#bOFf`&;2Ou5j*T1(!%V8o-+{(FD8RC}&&1SHrrbjB`3)^+(1#+Xp zEEu>`b1EvT#|VEQL`F~3#tPAL3{Gv&zvYJs%XWN1Fj*0x1gBmXxM=8N3)m)E25bwpW@{$ zYp2`LD`SnC_s_wauTF=#>P$~C%;A&uiZpS}vdkIokQa2J*R})&%^C&9nluKalCm)q z1r5FdLw!jB9GAW2B$_U4e{?Ote~Vjf_iwA3PlL5kooJ2FJ!3Yv2)X0dn>fFdCx!w< zQQmyHvSYPuv@z-1m?)JS6;Zf-ob7J-^+<^XWs{E)tIa^F`-v%~o(;{&H;CzHmrbS6 zso$YojCQBdIv3PL(K8mwrbwkHq~V|E;;Je*EL($v<%zB;Nl4xH%*Y+a0t4Sm(G7Kz z{2=P0p09~gBFy}ES7(MmsX?B`Eg_&6(d`a=BUOoiRtD@TPCq`nH@SCbGX;Nkk5*V( zOG+SU=-3;nk4x#9q2FSoem2pltG8JE>b{ju(|MzM7Z*x;MDv=<#}jchFePPaCSiy! z7Utd?c4dr57qz{Fe`5}2DrwF;(ZF@#5wi(*n7H{nBT9K3hCi*>Q^*eO3#t7wOC5ue zTwgp=AZkzS?5sujZoKi}xLopx@hmlZ?e?887sjAi$=PKH5FHoq1jeZ&op5o6^j$UU>d|`m?^=v-rILjU1$JcwJX*OXdqV99k<4S6Ip_`u@Q%s5)aDXD6IIH+W z4TY;x8Y(D5yizg}Dv6+B6t<;jjH9#FjXW=3?t?lubE0ianyHoW1oBEA$w{>5O4grpB@bWtkT8Yai{ak?ZZ#Pamjy8*54_oTz^$ z9ecUER=c{Vy?;2s+wltX(1sUGi#33k*q27d?T2PZVQ2Otrf1OOlEcOkMFOddL|paDtZE- zhM2bH&8Y~FWGik!UWjt{vygR9Csq}5aw2ydu2g*6V!F?rQ0_5mi+()QBeb0EjGfv^ zFte;R{}9clMEl7NRYnfn%Zz%Q8y1eiXm2Xadmj#5nAx2G=_@9Sn)xkWcwy)`<^+jnq!IDQ_&T5#mP{%K(@M;Z{?G} zK1ez=ms2{dHP}NA%j1k&Fj)I7JibLk^&u2I7~Oa3ics*!HNgU6W#O_K+Xs5*e)lDS zqB4(_JQhEeAHcz1=iS3?Yzz)#|8>pfF^(!5$VPHd@r0g*QEmwd(OB{XdAha)st5Cr z(spAL>-uRz3Qe}}wiCrcIkH%c%P$AK)iauD-0E3Tq|+;}Mb|6lo1XOG(JdiBRIQLJ zArcZm;E|M+bn+v=-WEQW5#u3|cQY58{+dV18VK}>^1Jh#;QkbL!9Av6Wp}x!gEyxK zF}K_3u{#1Aju^Xv>o(E~HSzLteDc~`7^UGb`mNJ>fy6nz8d&DrnLjqun!Z|JO;%W3 zST%pYD7k&sV?AP^!>xelws4W{hM+oP`M!TVNxbot00QW5ThsxItpyH5!`yC7vLX zINNd%eg010Lnil0VzS7Ih+@sQ+~u5De0r06N8XaFO*t5y;{$8iQKAf(k(&JzIqYAk z&Jv71Bdnm|y$=|?x(Jk46R6U3N4Yw0pI$~2vW z@tWqEMc8`TlLxe#2DLES`GB)Os@;(N(Rcku{tnp;Q$`jE;24;CGv01&_$7Cw5pGi& zgb5;0Lbe{uniM)2wXm__S5{H1`8OyTng0q%b5)F|F?UquA5}J&iGD5{tLuzfl}PVWC#Il|ZJ`p_471hQX8B41xQd zoeioX*=QcvG>lGy(6_>Mnhv{iUlrlJ9?`ae-Y_kZTnWTz#EnYUccqjX#*3SgpNX3Z z@OaNiVz{d1(_0lgH*A3-412gfBGdl^U9!j%jSp{u_^@NV%99$0?kqM`up)15z0cgM z>e!qsc#`6;xnPaFb4@^M&zt6Ms4?j!dbS~`E5sJA<)G)S?^&_!X64p0JOD%hf-?iC zWBAXBd&}f##v&S$wik4xyT`|gHm=Q6y9{c21aI?IAU&21CyoWe{l&wL`VplXowCur zMr(94o9Dcht<^?E9Dw)f!&Y{TYT<_Ukos6W%23(>-(V93h|#(v_`p!ed|KGJcU3fx z(PnFzjV21|EIHFZF0Ia7gN=$frPgiUy;~hkX(V@>dnYM(&omWo#_(XidCbHsG>H2; zTH{tiuqf@rV-laquphH|66fjGf)Qjm4(+Ociv9Shdj_7s zcvHQRW_zbOVb^{$7QTjqn`o5pEM`|V=x)h@;(z*l%lo73P5K$RofpwY3e3}U24YU< zGVOCh`{omVhB{Zk(y|-JDnvMX3x$H;ubcC=T1PwK(>r%=tYQi(&I-jpxE>OL+j$Iv z9x=;!GS&SyL`h;m?{K`qHZDl1Hkf6@N!KM z3uom@2`&yI+fR-~Lhet+iu0|F^vk8PQwJ-SRceTenug*-clzecK`e!?=!*jsy6_bW zOQ`2#*-9)13a4L99cn$N78Waa!sC7ZUi@2)3YRnhGexF;wEP7%M9Byn=s5F7LrHe3 zjP6`}FSxDZNqPs2F*b!RtVzt!slM7`t(kCfK4`#i+&_ zH8b-@jvB+)l`iTX+c?X_?4e%ku3)&<5)L@B#nv-*snLLox?#)cFjEwQ+`Q?%7ZjUF zY~f9siS3fSx=imZBl@_QU+JdU&edfh=b?E=YD@on+eJojXsziGF)RH1z}KX9fiM&$ z45)dkBB9>M_r7JJj)N~gQv<99pU>E+Rw{tBIq2GwO?o7Dm_oAAZKM3N(I=LjY>1;M z$TAY0?>s%mvZ<_QJyfi{4LB@lR?!(@WvshNHL0J;anHbJX3-1 z2BVg(ye(UH5}*Tl)(yRWuguv)`_6_$Ycu!y&z+Fpsq5EC?m~X`ArB|p4*7maCU@$6 zc@2W52r4-!o9JypLSE5^)h_0#uHk=erPdz9v-Z^dlnrSovdh^mBF7N@mxe4i9p?)x zHzPF#PunnPUFB)%c1&5e>*M74vMbXPoK1uTlVAXuYJ4Zth|!8MxS_3}97H`?tI(J{!y@OG(gczH67 zJ>~NEKxy;!3nHZpy{Q=Ex>uwm+Y9!ks()D7nyOc&`Ot2WEwKcY-*W<4z)As+`G&9y4m9)`F9P0XFzBKCxK|L`au5e5 zd)R2&aRxfS-nLw*x>hAus1o_^GG{?Z^jwCi7|*Y8=d40LVl|)o?r&e@x1V@jS%HzQ zyL+0hkEQ4gpshL}AL>M=Zil?eZj)2pcsKi-0Z^dK6GF-YLGKy4=wCP+-w;6uR|OuS z$H{!CrfK^csyF&u$R&bgPhi^`8}ry;%Fxd6R3fOiqk}#+wZ)TS!fzq=WE-G9=(VW@ zHncYXN_n5VZI&e5*xNoUC6bcbxcw}0frI*(uZ3;_+PuArt5)f_QOVhnFAQ{7UC+Es zk-2s4l;2O2N8X-RVgE+;O6K<$pZShnF!+gaqQv~g?JB1uhRYEJTm>FWJ=;BAx^?{G z_2T^XlG7$J5%UhI$$dHEgG+z-Oa?vEs7t z@E#yy_V~Q>Ljf#?2W}#0c7G6KFjSSJtuPO>1iOqcgtD{e8|@TX!}~_#LvP1+VmNay zQh<8KU{k+xn?*x%qInuN3N7bM)gis1^Gh*tg%C?SZw9+dSzh50&GNKUPVd*Z^Pk%h zHkmON)@IWBKFt<;OlhXV3R`Ux^p6TFKNR~zZjY>O;XBH7JQO3+R7f#>6*kI8wz{yM zW9e&0ufvEoZLb7WH_OJRFPKKtFb`(A5L#5|c7txwC@nNVBBA5A&R$hW&E(A7>*{2h zQ?!&s=COXg^!6i~VT!$wop1q<{>N^1PT#o4>xJ5+U{w}v{B%38gwIl0vkhJhYoElH z?1=@H#xp5rh22xvjPJYv8}G_s!d}dRMV-GL^Y(g&(%DgZxBRhAeAd=#Cfo3YvDJsr z*Q9m*dO^6kkY_ti<_8~lQO(b(-etN9Rnp988eq#7nA#8$=H>hQt}Pp$O%ty=X~>E{ z+eSQvAZFv|@yS;7oM5os3?D`4ijExiNjN%!#Uolex+_eRWFkwxC>$2&{Iq2a@DBTE z`ylPYZo08aC!=T}ctW7Gi|DDVn32?8!=}r2&a&v(0k&DWThym3@C;uCne$xM9ZTQy zdkK%N)$y9FRXRP8Wmad0u(io*Nxv0OcvXpg5{Xj)m_2A_NF6HQG6QrMU>BqjVGUPJ zeP!Kh#Tm%8`A)Wj(#%e3+S>yeWUPPEPv6@78|>67I)w#hbKL0Qv>9W@z$-$jn1QI_ zlU)8ovo5=x{yt2zYvIN11FYF{TCpAq9|GrFB}=3(tcZJG9Y;eI8`<)c>YjYa7#Htg zUrhcbG@&_{j-Jk2Z?{@ojiVDzJ!I&6iD?Z1>M1h#(v?e5&>q(-Y^v^ILWq)J;!`)Y zch@j)klK}bdo|t7w?MaI2jsE_-9~SXGS%PvZj=+w_-WiG8*3R%B)pg#D^O6=Zz5A) zdY7ePXu}httSM%ZGdI0W`DF>$DV^+)Z{`{p7>(3#7v9Cxk~il{=nFHev9M8({wY_= zta+oVIE1cO!VTw|@+rgJZpEe>$z(jkgxZeTBn!e(^pgvdN=UynnusSdjQOdWNw`6 z6)d>S1D1diZ?i3y;Vkw;+hdX8A_~Op)^WB#qzc-dJ7-firHETJoCCAy;k6NbZE#Ht z0&+5G$;{taq}xDz4anEOG}I8vE;pgwM|wK*V%1qoR!!n|OlYoF^(7o#4uzc~l|v}b zAwb3#>}3i*Q{mU!(gmg-d`z7ntO1bF`S|;_cI|^Zh0t@q{p8oj{)Qq|>??u0(MR}Z z%($_#f{;bd!`XK9cZ0&_Q;53@zaJ-j;9@`Ap6zTliSX z3R|0f7ej48fDDk8MX8wh`)wie3j)v|y zxh~!1ktd0`FPHaE0^!*}nulCqsc#|HnD(Fno)t3u6@0|XrS1X+Ze7oT&>`#5gwA@lmFdBe)EC*3LWGS|){}!~+ch&%k*%s=_7w_# zeOmk#4&0ix=_6VNHA;P%uFxYUi7J~7R2(6yqV4XR6Iylz?acKNOMV@%#B4^{*`0!I zIepC;!M2RTckIWjteXYj1ucCHn|lU3o={up@@0d%Vv)4Oc_N|F#bopmM*NP-;m}FF zLYn=i-LTH!4{L}M{G)K-b;%g@#7JsYt$QEU@cr!M# zP6Gz81w*?= zkw{;eu0F0`5;{sGNlT`O@6H+9GaJv`1`*}S1~(vxV*jlNh4C*cX3P=vtLgrB*=G>V z5qjpnB&g&KgmlyGsSQv_lkRXixo^xe6YU|tusY5IWMdZ}P(_Dv!L~k;2V0H|8{A|k z=&)}KmCmD@K82iJe$OOGqTUf1W;kudy1&NT@AWZfQ>7N+dJWp5tF+PPNl;RnhVi*nsI%OKPs2SS zbD#~{cF!utW1*x5lvs=QI?n^p;fF1b$W z#15y&y0QGU-?dX^9rUx8!uU%mnDg(7{8_`!8;b)8_M1ZGwaJ}lZrsZ%kl6afDK7kE zU){|gF`g1eAR4hKzjMt_45Q_1C9C$X#`4X^yHFxd{tC@1+W)-+cTx0Kgu(;*T!C_x z4U$b^#z;t5fZu9EAx`c%*`m!o|+pAC-pI<1@CIKD9IzznGyRoK;n=)SNojz-G#Rk!H(M#X`ZSX(>`8z6u~h+|!BX;+w5$M2DFsUbNENzCDJ9g^>9 zP{?l)#aS$}SVtEVrqVm60x|DZ)}+F)z4CpzDX`AnwX;=WVOwglaEuk!Z0pw+5QxIS z$DrKUmblp%-qhG81_%tOcQ-j0XTBg>g>qA!-PHu#x03o}c+8!lnb7%HV)MjCFIb3g zxot&AWwty`Oe|8Q8D?8yeULJa91RS(tsJZ3Jr}K%JfYbHg>{xq#M7`bd1F=4Ux6>Z`Ii2Ns?AOQo zMq$#^11uqC5*8Fk7w#Bp0Q`ryW3}G@4rn6=qtGTVJ!Iz^zam%~(m*K`n{8OOWlNzi z1*|@QWwH|4atL~H~u?uIrVI1xFJyE%py8~ zw#kHf!C}1thiY$=z5y0cFLTMr0_egCVgL&jYHfA?T~AG;J#`)TGJ2P7j14dN)Vt*) zi1-SR#uMT`@jbLASaQM{d>z17=!>DL__MFJWwDtN4bs&cErBUjo_X~~43glWPhlTA z+Pt({-h-xrex@N9h=^M#^G_PQo6uoJ9I+CfQt9`4SZF&%AVcwZq2kM%H?|2 z`U@?EwL7-4*%*nZkHj8h=+@_Uw_uQ{<y?mnDV zR>>PDu}9w)BL~z|mob&z<12lAxcp^mWrFf0JJCejS;VehgdTT`%zjT;`X9v@Q zZ)%mBgWB{s`;yTlppmn)HhpXBeLc1DN@`wuAhrMj!`A5596A0qmyhnz@iYW5&3MjP zVvU2I+r8AI(R0&lYn`u(?N-JjxQ)p1u7tNM%FvfFrlMx=F*)82B(qZM=D9n`MBD6+ zyM-L9S`xObHVeq(Hrg2L!dlC(x{A3vSFp=LVAkABE!32_n`L#%mMdU zfa?v7CTr7Ww(KjluI&}E41!sYfA^*P^rFw`DUPzF#*nuxP-*5a278au5vI;V>!V{F zAe)JSZrHwiv#MU6gHa#(!Fh4jcHNiwv|!AL;k>8I{?X1q@V?Aq;$O=Jnw8?&1%)kK;OAjhDGOgQt*ZE>wz*HEPek+%=m zOa0`ReT<>;RMy;C{Wtp~zkP@Rv+G-I5c5C&!B2{cm{=UH=(-fVv~xWjZfeH&)07}iL(Ud|zxJ%+3bDtqDuNgGIkhA_7!N6zS_un-$^qshw3 zcsU2%O3`k%{J364t`mL2ewKTfDx;Y!w6jAo?8y}rNS5*nIm|fZe%w6Aieh!)uAA53BELtGdF#vFf$2@}z8`p`a2 zychY~7w~`?(9Pvm0qz<~o{1tBD<+rnAPY%zJyl*eot-=MPeWqyQgq5{%(ZSy#eu3H z#2QUMxRn-52fd8Q5voKl87B_xY5fyw;#H=^wjOVXI^Q9pwIMP)kYErEEX}FPAl%sd z&TyD(X(0O#(Cjzc(?o%)u*`zmiqB((R*w`TXhcFjl`?jEGr69YC zL7b4^6$GUQHGhEekbGWGfM=4iw)k3XOOe}-?J3Q_uFP*cJgOjHtGe( zm~;CGaaQ8Drg|9G0#!w=v(Zmt&@)x*Y8OosAF#r}ToeeX@`A}P1WV! zxP;bK!nA`JUZ)>uj)fh&p1m$Oin^m)yz^MhXV}`>;kxNXWwlum+ezt6z?T5u4tsWY zu*_$-(z-Kr!+NU5T2OZ=dA)>4NLWj--dWnksJ>Q#8zqRoU1_5^p(4rJ`1Kq70kyq; z_@1P9#k5Lig@_MlE+r=ptgo*1F9w`uJE!8pI;$ooaKReC#>b%y#2;LTb~hxd)17PO zZO7}YGEc|CZ%X1yHjxuy?}t5*B-?%zqk{@DF6ajPM8noMaQ5kC5;97`25{)B0_FyZ z#VQItP>s@B?>eJpB`B8ZUzinJz^tM(sab-1Piy_;SIYXZH>BnTld_2dzcAulmC4$7 z%_zfpR0_mdsP=G_XRPVcYMDQI5AdUWlt$3d49TkIZ5`iy0gyudw6JoA2+C&(@pO`f zpSlXgMuo`<7})HZNSF0R%TgLt(ByH}I#!!#OZjO-o35>2D~1d#-Wd#v$R2pUEBC^! zXx|ZSZUnL8#sEc2P{B?3m#FE~9J)IcJ1_E7Say|$_s0~hY)D(Tx5;%=pZfI1cqQF= zmFl8h?e5Ii4Na2_l35;@D%T>gbj=cMju<4BaMPn}(Z0wtS<>m=^>nUWtAd<}NC9Wl zwvO)Vp#j(@*fQD^R_8DT9>~~)8fY$vQF_3cbkb9ifSK2Uvs9UjLnUuT(7XoRQRU$5 zGN~p0)ieQ4mHgu0&}S*Lek-DBIpgN5FsdIqsqzf_36~rPVX^%=MhkG(5G?hSG2qkcoY7@B#N>@Q)=MO==izQmZH-Igym^f-6_u!b~^M|0je zyWZwOV^$!~M{9+1W6{!T0My-6jXO?pbPjW=7E~8~Z3Zz2&-zg*HZB{}3B(>OM^hFvIbh>9^|{Ci7JwC@Hu{z1UUOc*#3V(Fxw*Y`7)gh8SKYzQfOCs)~~$j7F#g%LD@F~bRc z12K*o9RatZ!)#!lN276a+m#?rHbU5^afS{=x9yFTDAVMXr2x-4sUjM@EYr;GKK8Jm zA`8jqogTH&uv*9taukQOc2`=gSA>`c9XZ#@b$}M=yQvU$6vMd^!1|q=#%V8FcpEB+ z?Magr!0!qqSa5I(8L?!zzFUgCUg|txZ{0mS?DLZRWp;c(*ue|7smL3ayg3mSmNV+S z`HHdsBd3%!SG!oq@&_OyKll4gY5)k_^B_xuOrKHeOvyH%8y800klgrEoe5*tmCmsJ z;%2`vkpyVl(Ph~zKktOYBlyl={T$r<(BZkYg?G%MpnCoNtL(-S{6Jsx&;t%AoR%P) z!knAL$iMc9;xDL8e{~vT8Ly)gc!uS+yol zz}%lJ-yv%<#}kq(L{JDQlAlFr>1#bI$>;9F+ify=)$eLqnHqaFIqOgjvt@x^g&ngu ztC006J$M434SSV^rCj%fc;4Kbxwf3$n=H%$%=V4OBp7O}mJiiL5Vjs2%bE*yLo8K8 zHi8XsS;NEv6NfYsiqTOKuD22C+9d@p{#pfXs~gnrd4W7^GZD zPM%dY5O`G9nNSF{Tk7me>^g18p#tQ7E^Z`D7$6@krBoW)iVn}~J0cP~b&L&1on3Gw z1LMl?LU+IAXu$e;Yg*Buj8W&%! zOs$6rPicdI0WLSwt9`EDJnmJoz-H>(w0(oUYF!ua#1r-@Kt$9!Q;#gpQM(lKvGqvy znz1W-rfOsP!aG@@uGCly)v3^m_0gtcjS;96Kc z=o8!9^_UlCtn9gi9OCJ#sz4PoHZgWHELVFus8!;57mig2IsX|vu2=RIcV6JnIaz-k z#;=dF3(e!B56Ph2b@t^l54Jx)t91s>Gn#b_D*F*^c zB9c?j{GlLdMVL41a&V2{RD%E8D2XYh&=0I?L2)d6?5ymb@BC}T(!Chr%=RHMm^Lct zEz^Z0V$3g@Yy6%i|;&t%oy+xzCrZVG)1x@MQjAF~SupLs0!Gq;!y z*KvuH{QDj3{bAqz#LsbTjHAHj!0LbX)gO-s_*`lIak1$C-Osgw0ciO$-_xWAq|x76 z?fb@?H8cAoym z-bBdufD*=o(0_FF=rHq|Fr6=3)j)FEg<_=K^9~|giE-kOOZ@oB$cJNNgm2R>o(I?b zyU4yn3s9Hoqk_M4{vS49oC7XISm$5d`jKq^#wY*X`|o0DA8!8(^S?n^W_oD?uQpM8 zPiF4}v0vi@U`Q6)wTg*zzjXd#Ju`zeiR?{u&t9lr^vDYAp8=46hDL%bfC{s4+G@!I zs_npLf4xI9^G#q9F17gEbFUAECI7knzx{)^e@}`SOb~uJ_dY85%N4xinrkE%J+J(y zk=#HvVlY+biqth@;yBWy*y;TaBquVI(#2bZ%0GD~3>YYFEGA)FMtcYVY5IWqBEV@W zdg$|gSl9#1nT+Ozux;{<4>e*12U22xLao{V7r=Hr5S&zb=ii$Ik7WFOUQ}L>J`&IcOx%IH()rMYrJ?WTO`XjSMkQ81fd_(hQ!7^JiPHtG~d0g~A{!`9{chJDasF!H(|o$|YIu$KSu-hX*ZzuCPQ zfTQJue}AaISkG^7uJhf$$v212O&{&_s-#WZO@#(@V_72U32^(3IBPa|Fr$9>wvgY{C^nS&G26Y{~eV77s3CD5)aTX zFq!?o9Q^mB?teM>@4@LG^mU5k{~HnAdhI_8N&>SV4W^OYJ2uwYTSW2ie4V2;KW8FE zz}{Gp(qAc?(Q29wyumMJ>-|(E?{|P zJT?Zt8GVzs;^YtCZl=G;`@Dl*!rDFlpt@TBC8^@cu634{RDHG7mM!sH)D$rnmF2V8 zt~?BL{$}pwm4j!}<%(aO^V(k6-x$5OMX-q<)3^}ab>d|^9651|V}nUoG>ba>hUH>z zaka^hwvL%GV0r0DzaXR@I0V7Qug5;o!G>%R(LbeeyD{%bv##)G-NT8JM;A|H)wQPH zUrGgMX|Sc96?iHu54QeE`SACiH8WrPT%~a3w-<-Vz&lT-DL&JPn_{>?jZwW(C|q^y zAU*SzQ9&<8)6YC{;jFS*kJ8CCU_T7WN3N#<>j8ypLmxDM{q7wTtF|Y57C79TL)9cI zdEIE(pC92IagDRxnaLGeMs(=2a!uR4O z;pj7-RD0I+e&_)%+NF$K&(9y}o31X3{czlqdq&nXOZ9_W@fsX85Dg4WRFd!1uVOif zC?G`o3V!htbsasq{7J~cf7?p{UIVNcyCin+L$m#lU$jEq;&9-&d>*ZGaI=yVC(qz+ zaWK1;IO{a0asZY`TT7as2JbCwwnbI4@Wb67 z17`N!GyXb%)Up3lF31Np6YLYEX@y_dF^BE~M}B(C;KV^p&@)oqXOa8(g1fafzAE;I zg@*x?)H=CP@Q|MkqN8t8*P76}JH_66{Lp0_Xz4q?12qwx(tYrEz}E7908rJRURb0$ zbPhGYNqV`Dsy9AB<=-9hV1m#3%_^v`rUjKcy;{0)@OOX*UkOl>!Gi6fYLQ1R$6iT& zr=*?C(W}SS#)AHw1#oZyAZFM79kLl%?C9Bp1OgDg0}wtdO9VeVbQ3?FlV{BL2)|+~ zSnm)o0w4_u`;M#Wt}*4qE6OYY{!|5eg!Z3U`u<0=|7ii~KaTrk^y;A_np1`Ral}RJ zDAVBun;k9!M`T>JPuS0WM^9F0q{Z*ysK0~X6m<}>kL*qk0-K#M4i4eB;W>yuU=cPp zfRmmi+5eH&{ioLyTq84`kz14Zqy-#t_K*xw=Xlt=hjczAa0==Nhkx-V5P+{gg|;}j zRkPwrfxF-DJQX~0sQeTF;){O(QRPP0cl^*L3@YT1{ z^~*#B*qZU&sO6>*^hSx{(W%=Uu?^&!6-b$(kVD1+Wb&rT_q1tHM~|P66WvXz*m`rizq?wh zvK|@LPMv8MDzT!UrQtRFQ&7CzBiW-%CE{C5TJ7~=aTz{jWK1VOT zT5a8aS)9S(q2(b20y?Tu{wlLt-ak2N?+>6r#o1YDVhfX;)E~HE$nIxxl)3h6TVA8R z;bwZ6*v{*`!aJ=5^^?ID{B}~H%Yv2=2_^S7hl6X}yBe16OpLgU;Sz}d5e!!!0;)0V z`TYa@{DJ+dsQDIGq4n0$n`%a}=fttUM=5|sJtxgSRGZqe@~TkrMRd%4N8N`Cv4TT+ z^#kbmL+{HgKAJdb-)6C&-b-{(VG|3cZ`%IWoO|(8Ms8OEw7ySMerB{(6GcTGCgpbO z2Z?)_k7OP#o}|BLzL>=6eJHW~N^bAgpPzHSHaFJ!abD*^YvlWZuL35L6Vn0E{jl9Z zz573xIkOj9ezV1ONHPD<7yj+CSMPlk{1*rMtIhVU#d{r)o%g58c@N8jzwuj{XU~pa zLLRuHd!&rDus!mSBtGz!{a50h*fY}%2TA)s{M#4tjt7)|7GUF zp#FdNB-o#(^`bd&azYu+@*lM|e|f{dxXgz;fNZRhD-8R;{ox*=0fjw^4@W!IYChcf zi{<|X|NIYx&r~aerP#+9XxaxkcN zqRlRq%0hL2B*2P*wa5VuNcOPQLzEb};p-A<7 zE`cVP(ipl@)E?a=WIt9aWIsZFm74vqzZ`D?$BEYxxRXMHJqSnVyO)m-}NtA&u$% zu`;v!UQaujtUtb5=-*{@R5Q*iUyhbsd*iX!si|E~sj~tl0?m&?6vX&OMR+?+2L2lan+Gz@_$Up>A_M8wl`T8F1ZR32*s) zf|Tv#LjP2Jj@CM=2Bs~xGApcbw<{))AkSGWG8JU~>-C|Fop2*cY+gY_#>kUHX5F6< zAej2`OP7$18HHk`J%(QA6psa6z5=*eA*Us-(GtB{9*6MQoce?99$nwF95Soy_Y>43 zY8orf%AC-uQ}@O4SC=9=V<1s#!K9#pI(zEO&SkP`oxL&h5$LymlliY6kXaQ8UZBSP z%{oxibpq0NRnqyn0=#WPCx0b%R5X10rQ))vS87fsK`getUeJ1Cef^Vujz=4e(NQTD zz55%z_RR1-f+upZp>Q6SD*D)03`&ONDC_w6>bBj?2D&FU^*v|`N)PJOKQDT6@jVrv z{MLB3%I(IbU>BjJdM8aJ`}O?A_w3_uLG&hm>}~VcbIug!{ZW(T7jv?r@=4+0Dx_Dp zcwsvY>+AQ?DJeBiJ>_vG64Tgg%jzZ#7unvNv6qZ&>sIMCTz~7jIAJLem88+Qwyg@4 z^?E-QmSskO%5S`TwV=>TT&s9GpTTI-^H<_5P2Eo;PoQ!z2F)D6;USye@mkGuyG}Hu z=stYkPvmm|vDgPSS-fin)O{Gk=R|{FBPt2%uhnj9z2-xpPf!6vb9=y&>YF00h@Ws$& z)el1=mocs`g`;kj-Ek#wyXnsr&TIB=8^uF&-}G~aZHnrbUcw-87OE0y7L<)~o$Li& z<2S>0jqLU#O&oiS9uoF4+^LO}7X@C*INP zKXH@uW{Z~NS5wlR*sheP9Ub+UtT5}(bQL9VJ249~mGQ`SL8g8K*ZI{C!rwaI%^(ho=TtUYiO7;S_vso2K#n!pyRk3Ubl_ zUKrh$wF~mR%1UicDfegD7@W?^bGGB!==cA?b}lfEB^+_RE{nY=6zM3BT@o_r2|k7= zt$p4I3;fiE5M*7UEP?1`u{otfYgx*-NsU)cZDhGe@2u!@kBW$!QyM~!fyUWl6u5%6 zoLg>yDm(J}wvZqw3w7|<;F{Z0VBlRHA7RrHjYdb}18d4?`9mp7;0-qr6frA@4G(e% z`LsNJ=wBNV?LPLiwk3K+K^@wXXGaCbXQw??z=o?c#pxOi zY6zwcA0}!o`Kd7`tL=}_A)KXvvG~W`#boB5)I_u!Ed(b!45QG>XRf?`- zRw{C`=_cOWo-8@og*h^`qDk~vMtTv%i#LP)8!7vhvrG~+8oufujM!P?TsCnJ8hy_3Wi3#sf|4Mz@x}f z>fTWGsFAb=%v_>jCfxq!`+B+rrrrbc%VO#b7 zWeYd^8X#+HlyoeH znrOKb>c1)HBbttgz#t(bn67ITxR+HsO#*{=@94ZK4i}aWUa>SMdWiwAF%A?5ehvP# zf^iJ3kxti3xfDIWnX{rW?h=@MJU->pn#?oZW~o(P40|0Fb&k`ju)3a*3r7UQ@H(AL>X`*qG($vzJjM|hjZRC4}+0r02ABVQ_`rgghAs~QYw>|Ma* z@%j3)x;T+P(8EJu*}yv_xkLDcG(z6nQIp)fjRyKC77V90%KT{x#;4G?L6k*3d5pRw=x);d9$fKOoIm132E%M|9-aeuOdXQ7D@{9bsZKvAej zm6yd^lTKkwNn+tS&h-VyyDQ-3M8__;k1X0m5@I2Fo3zVeDlZGq8g6P77lR9?i-~3p zi=|u%fns{{1_!5IlEPrq#qh}1`P-E~3;GV%Nfr16iJRxHvfmrrmP!cQ{NkT7d)hUNIQkG(IF71l4gVK%-Ujs7it>;Ib=!%p3}O-XiBmMmFJHg&!2snGS5@L zw(VHh-(j0R-1{pTw!LE@-ajzB?IBZWUcRXfBJSFh+~pSAx!=;udVITapkp)1piRB_ zo>Z$rCvFa zyG%mUr!KLDa3wft)J>s_wcmM^bO3`J5?iZveQ(wQeWNJ0x-(UFD?0}5r)LI-#Oo)K z^+HC8vtY?|(^`ERI<3KlJR`*XbJxM7o!wu}MQz{P?ln@}9~;)*tu-`9OukzZCW5Rd zD-?1@X=t;3xXN!{Ie&^{T61si3$$&l!nu=auE!1VI)JvKu?jOc<3uSNLgYYZs4=E{ zqGq1X+KkKlb46R`^d`(VfsSpyqS`G}bsU-bx{4dT*Kkk=-7x2?+W3H*RWI*V(iFE? znHXkask;SOZu#|6K$&}g_~U8}_CrRm6bcxXYHPT;18|5kQgW*tXi8u3;2swb2Zq(h z`}&BnUhpHtdp_N>Vz(JkCA)7cygCEdKgZ0>%|}arC13t(-1=j(anFt+xF2E5KQn&m zjpOg#;2@S}-S6`tFJjPX8Nu1$BRzWTqGepKA(O2*ILsI)vgmf#q(tJpG5 znj?-{!J*P?$8CEcO-9U4zqW+Da0?~6uyKc2VU-Jae(Pmt>j@p%$dc7Mej92iI*Fy6 z=`Z$hZ-T7vHb(z$3(Q=q^EVjp5xz)qY~g3~W#$|1P&BIlV19S3D7m-QuM&^DNeSi( z$;Hhsr#m$wy`AFw1W+Beccr_(v4aK@B)%Y9oQa*?E{R6I9$fB9cI)DxrTZgW?@JdA z+0YO5+DLI^lNQLOF=u?|X;x=euw+(^-Eu0%l2YILys+V>ja`xWWgO6tb8#%6a_(z# zG{TeOCCe5d107BBB#!xq%_vD{4XucN>=s(9e&|H(BlsFmRIBqPQ@%Xh@A3jZaxRS( z&&DqM!f?;ovrCq0rZwIl&Y~VrH=gc^D}h{(xe}w}&{SD%{Tca@e+_3)-?5t*lO^Spcb&UJm6pAX7odS<@|rNCID_OVHQ zRGi7_AU+srQ39AX5CqIrek?jLB#bW#sGRLPg&*j2A$EM=K~W%AC~$?hXz^Onn^ISt z9=EuGh zrqSKkA6w(do&E;Gqs5_wFa<0-4#LPMJ+1|GVG}<%ejQblLcnz@yBmt32=C6_MjFXt zHm4;Wd*@F>ea=h4ExoF~V3BvPMrQhAig!8L++~yK!#8o4L1v8=q=YW%USi`rJ%>Og z-!KvBsV4uO|HIyUhc%gg?V@8J#z90BLDsqJp55(7S+w^b+Zv48uq-(mP1+ zAiWcmUPBK(N>3nQLJ295{W9~Lv&EUS=ljmSuJhOTZ^~PqwVw5?df#zvq^IVf;DP79 zlRKO{p>_Onb|E*>c~WloApkxvFbMja59bmC;M_`HuY}0?MGZhRjTzN_qQY#vv;it| zQgc>4$x@l54bEGCXc9&?p42I6x_3*pI0chl2rjod!2zyqOrNx=_CLNd-KYkWJ9c*l z>(kR@M)X3L#R2TuwQ81(>*}HR5};B2P`MS`4^6M((Z@zo*8^I}IjcuavrbwQC6IRc zF3`+>GAI9uyQZ5joD8?h$8QkdhoG6 zKPmzNoePSjn;SXy`(Em$W7`kgYu^+{fOMt5#f1#I%g2adDg~eNnfR?c1C`UVpxh21 zbGJhcc*{KN;xc_lCaehurRZb~p(a?#8--^hm#H1)?s4a!&8h3<_7e659CbV0S@}YH zMM`O9u0m8Mc2(9PfPy2{`$TT)Ci+y{f*$GVJX|KKlN~$Ba$1NF8TIPQYJDOMs7DoSfv^)bX|e>P(b;PrX9i<;yRwjeC+I3<6@= zI(lVuk8vfM(UU}Yo{_yp|)Gxbk zAV2iDJZtCt0i zrh*^7ts@c9X4H6hll`5rfE=_nb)#x!c&9hW(R*@SR-kutHhl*^+4H~#wJblBF-?6c z`}4~PubQIj^bXrYULE_kY763{2+!YR#)0Ab+ITmqAIwe$2XElkcIYnh2Pw<6v3OzGeK?ju&Ln}hqHSTGDpk5vc~V}7Y+xWq}-tEO&QvQDISkbl5qLtrqxWdy(Djx z{@wH%NEz%bH{H=ViiEd8YYY^+!Hmj?J+)3_$NWkPL!KH&eVok+%F)5Q^DfJ>&}R&0 zLcjb~`%;T)4^f(u&eO8-_AzX<<$>s%z_L)XKx+<{8egp|pYKkM^T9|5gGx}XsN33n zune<`L~ZJUc~qt7^0`eDL#_fJS^b5)d7VWMj6?yXCF8>I;pZmZgeJvHQ{(mZs*Sv^ z#-w3_Up~x(9GDoVxZ2^JScY>&V^gIMo5>z8FznqaVHu9Yg!OLbXLyU=!2YRyL+Q#r zGw0n1hjR&n!}`RUn?;AepOKhMYV_44Y?(7>)KztthUN>x^dS~PX*#2N3g!ydb94}V zx!Ay^O|P{}Q|WR2c^p(z6<)`Y6Z;V+ZQk8gr~?$Ci222CQe^porFK1)4niGxx5S-f z1K)sGnOz{Gs^j3ugKdTx;vhp|R`vWoG_lRH8+;KJyPjV4dMImoZq7tl4~oL4uVUW< zUwO4rJGDDuW8!x?YE*rOeoeO-+}h&eZQX0tgczIYJk=ygygJA|sDqSRmFA4;bCI{l z+*QbX0RB#7;RD@jUPSeP8fVt$FV zv-k08pKBk}maP!u#%HMEWyfMk2tvVOf@Hw{SA7{j*xpnOaN zg((T|9~6Uv(!Y&w+8238bZ#YrQpv?}3h9M>0Qv*92$E{gNMZjP6}%q6Cm4W}?=T7- zHKa3IC{K$FrMl<=MVm5-IYO$a&sx!qY5?maGu&mhd>@{BGK0)W|NMH_gIY6?`2MfDd6yG24egk) zd-Gpq#A1YP;)%kGaVHr~7jnL#hBU;Po#{Ho)K#XQ|Yu&~^1t>%Drg&-?s4d@?u~@9;o@#7$Rw9_) z{Uq6dKM(@S7(!+wKgS4ZVfia5+N1nYxlW(__!a3Yda1%S{&KU=T~0gPp6sbjob{Vs zD47s`W9ba!818#rz()MG9=^CR$9k|;BN}>{g?)=!L9ixZI)_IwuOYb>$jUs9@!3W9 zY#QXh@Y0sNz>ZLxcveKVY~pp;s*pHF@h@=ju%!yeYWl%;AnW z-_q8d)}OC2H*Q8YF`Us;uz!M=80F3kI-}GWH_R2UxO4M^Cg7*{Gve2MJtOpQs8k-u znjPWSJ_IUy>*6%)6DjM|6m;jzmI>BAN9o8lJ)CPmIpl@-0}_KV#IOkfR8cM1rj6#p55IW| z9K{4fxN9a;21Q1ptyAGo=E$8cuD>aY&0>d^&c$a=CV5&#)cL1iiTdl07)=+cSj$61 z>qhtyKR`4(-|yJIHFtxr#N1+F+P1ClW)GYfx>cU8L!Mtr#bs)+#aA5pI~?wykC90p zz)q*_G0@jA3kupznHQPLv=3WNGR@0Grakkp@ARdv2beiV_(N`9UfnBBf5EuFTRclB zF@9mzJ#ow4B&3d%p|?+*0EOlU58Nh)KLC_V}UMJ9kf^7L)a(oAJN^X`v*q+bIYr z@5J^kip$M3A}7TQA+t|@zE|2_@bypKeVNJ26k;%k8 zqX=zHZWeV;4pM2D{f&VY$Lg`mRAi6-=a0Lpc2>H&imm_S?mU8?MK5>X0H+}9GwJ`R z+ruxfFg&?iAn8N@VR{pC`v;(uukcfxCTC1}s7TCqbdXqQK$Bns)a#uqfJ0nDSYb%K zq+-Cu6Pww#xAH_vRhlaspq126mf;kk?OzZ$+IdTO6l)Td?0HoGbpAS~Ab7Drjq&EV zSmF0N| zDQy^u^JVgDc$!t2*@sggLUf0`pm%Zn5Zz95GyfGrair&Jn@m96|@OBVpOYql;vceKCp&(i=W=DJV-#nSmc+a3f$)=D^9%5TSLG`j&kkXT$c9< zJ5_-LV(OZ9>S`E@31j*`U=VS+i{{JN;X$9ZDRDwN$5e~o`^1sM$z{t|uhzLjLFq;i z-RP^8R^877*oAJ31L?Mb6}dn|_!@vJDdUYh!``RP^mBtn$7*h@%eDS5cVo`S{K#VX zB+2SQ(#v+?kD^Nm(SS4lA8(%Pmb;`ZG#Y%HP3`_qzfjz{e#RUbr~1WdT8OM$k~*%e z3fmscpPTXC^lVElN1?V+?q2I|mW0$4y%`}7UBbcsjv~}`BbAjROY(W1A54szwMX51 z5jUff@pLVaQbq|5Zumv=77Z(^K6L~8xp(SW+M$)#;uo;(Y*)yoV@Cq1;+Tgrd~!e*Wg+uD_utO8^vCP%tRd=QlNMyzn;;w}=IEdfobqpJ4; z*E%RvdK{ExD&w1q<0a}|#DkiV#Rjf}J<+H)1VSzNxjUHKJF`B;&xyD`7NC>y&gnPJ z%2I@%Az{rGtjg-hdt)FW%X{3t9OCZPN4*ol@7+T7ZIWyb{i|xXq4(aP)hSn_57B)x zuK2#>{8B|$bos3hAFm}nJ&RdjW-iky_I7`et5{q|wouI@gdCo+$K@}MzH-?Ul!}p- z88d8sAB%}PXx*Q3Yr7x=ZSrx=EifQlRD(V1P|=K#osJO}7OpAOR%lBtma;W0wO?rQ zCCzAJyW?%drL;@X>9z}_fm{K$sA66^GxB=(F=DYwHtYswpQu}}H#s6XE$F09erA4_ zSn*=&Lk?~H$1Inp5b*jkb5Z1rY12f?Abas8yT+$&RTh?K(5XgQ;Vv$x zm8wI=-z>K2Wat?ddXt_kI7x&R01Sh(b0+Y{^z-pk%x?409Eoj{`9_z8;k>EKcnbuH z?7kaxwXD9#WLIWC-Z5d2F}f*x1-?!q>r0ReCP7x#JS&?m zTja<^gMv@X1&RbH%WS`&s=-w>Kc5{$ej<#S2e_8{O?lZ`n(9Ex94Fbztf#*BW}P+k zg)8Ehc77Gaq-aXGVH_V^^fYNE>^FHxR~G|BRytCQ7iGUKSvTJK0Lz}#bYp>+L0<|b zzkDDu`24bnJ?W{q3qHqf+X7GW9V>oYVn!Q3wIgx6afw2Lxc>Gb1OM~6z&WC16F|+` zKU)TmVD}<*A6GD_m=+p7++OOxa4Gk8Pnnl-f5?^f9%WXjF%Giq0Dm6zFx4*_V4}g0;@oV@V>^*Mjwye9{IdLV z8lFl?MmT-lV=Xn&*U11!o?#egs0OfS~(FG>AqF|FTl)+EX$ox0nDMy9&NHVQ3OJG}ajh@9c9UsK|M) zMUz@oEaoc1E0}CoBVG4H?$&Ns1iw9J8&4rUwNoz|_GEE2g<`wDL4P?&5?9umHEo?n zcvA`!xxs7<-|=@HD5ISNvH*pzmp7GW`%^AvJ7!aGp_y2j-DNwQf^Rfd<-|!@*1A@& zRvzCfiLfrNSU?hH<|pC9%B1-?BUur6Mxij1Ga{`}+d5~*4*7rv{#qIi6c@8ra-DyB z1WGAn1HE7xsQo>j^vZc=Q(eg2wGWYywBUT3KH8*t*Q^}()J3C>HeKdcWvN3>VmH?0 z=?a+xHQ=06)zWq+K}NH19i?nU!xqG3Z?lg3RcCH-+UDtVZ-K{o1tXU1#?C(l&FiO! zKVIzPmB3ASWo=oOERcgZb|Jo-)I%dFAV{x;mEM(a^&IFTZ*Kqmwa(=uqX|gHgc!k@ z)!?7rALkxBl04jGJok@LH#yCLdO?K0czV`^qc|+;5~^WrQ(YQ);T2bb0i=fM5M>zYUiVw?qM&d2+VyB+^hZYPs&%q=(xVK49V;WG*Etr)F<#S17N3xB*V_Hk)3 z7_6{zH?%S)%h_o_i6y_uhu`h6k8|3(jafQs2dMm!=ewzza$a3ygDyhEmRwzly%ldc zo~G%*6X;Cz3)jI=l1&z`t|cnvp4gt1rUdj-&1_vPSwY?a{aS&j+_F}4>(lF>vC{7Q zo316hDL2s9v@E*DUtaysI{jHhcz23d-5|BcZTz`qS5vMPs#lIh_{ld{YYEswBZ&!V%%Se zDdCE-+x*6I{{?(?#6kbD5>t$^Pp3>lg%NQo&JvLn`arbmVfX&%UQ36n!|XyV)nU#a z-UNDQRo|Lj1G^>ODs@Dey4Zy5cj-wOZhOD7Bmtd?dtZfE3KHI`*ETnsgM%$U)Bxyr$_LE^5{*k94 zoYnZo{^xYd@r2xP48g7H_+8VmxyaS}-G~*}W~)ZM0}G;Dj7jKnA9Q^MyvDrQ@tcn< zR!*@Qs=6kiTr)rYCD8pkVTXEu=D2wgIO{GXaF%SNY%SUS91gffQF|TD4=ba56%Par zQ!TJ5q~%8ql1rH^*g?+ac#>dxthkfU0~Rn;DqGJ_hVIXU9o z=I4d!16q#C{aT03l4^n2tJK%#=kRZL$O88Vi{(7ZfC?=y4R9Vm@3u*D!jR$Qhw?UT z$3tdCsDqhXD8YW#Xc%A%X~3)_zb2LC>R|N_^gZO7%xs;VI;puc^frYFNtRnl{l6y@ z(xbG30b`FX>sQB|9 zrt&>cT=2CK_kwlfMzGF@xko0cejekOQoBb%9e!cVV-x7bB!S#<4Y!QTgoIpAsW6R< ztbA>HCVFLtOWQWJUh@b0=jsch^OI9jMFuETx~g8iKcoj$_W16nozMBJJK>MSM|&x) zhhF%0ZZJt^Y{ z^5wU}Oig$KSlLTOPPJ3O6dC^IH4SXX`p z9MubcJs74(6Ahkk;aSR|M*B!u*Xwm6@11P8`{QS?Y36JB6h^eJk`oJZRuBp{d0VOd zCLQ(F{+i&`WtMiHm%}XmMVHq(*`_RJ3+Q&dL zRbpfAIUKM#67;GZzV7?Anr^&2mqT=Oc&CUzt|1FdH<^AGX@owC{--f9q8y4(K2{X+H{0Vp2muni2-e2a5cLtREiq-P%II zH+9w@a?ggz{8(+OcJcJA-valWRjGoWEe5<)1-8oO%NJx)Yi4X@`1yB=c^Kcs0;WCk zm~>d=Gnuwk=zvJZ>&?l{VSAu{)oj?h@b%AmQNfkc>=t@r`F*@N{porhspB)y)Hdf1nN?I=oHKAL-NsGl=Vo6* zJ3eW$Snxy=DCyjM(&8^)v6;14ZckUSUw-&HXDM9NGxdV6(s=tLv{d_At6i{}Hfz`G zZ1a~5`)xgQ>)jt~mtEZ1$a==hWcfLz_;79H4>4E&)$VDs4}|v;3{EO{>so zb`us}Nh15~452$Eo7@r!Cd^R?UhAFQqGJ_8+Mof?rRxmrdeHpW&9d2Lv8kVT`K}C; zC*pl%QBtn^4u*rw6E7A&z7J#Gsu48uX~yLxu#sKv*CZHO0x4wAuL3&Cld!d$K}mbX z$%l=c&4F0h3@>B2Uv>peGirAJ27IOI>mc*&y50>;s%xbQPd!&u=nXs=%2A8DA2th- z>BbJzF4Kn}aI^3&U_4eMS5ub+64TCOkS!yR@mhOB5LC!M=cOLm=UkJ+(I z`SA{xFxkA6QYdq5r=Jl5Pn&TnZ=MvL^{reeQ%mDHRT5XuY)?0=P$^j0^LIfLGII+U z_qr$M#u0Bm1numXUS}50s|Fv*$r81+oSt;O9JCun#O|awRIHqjcLo!c%ftyDE1^9i zcm3^{7tLchxR-ooBtf`?zD6U66bKV&p&?fMTHSvz1UXw3Un*^S>IgQi;=_QUD6^+ z`ZFW1d%eTwx`fSHeSu`4+0Ev7Q1RO=_YeMjH+m^(cY^g6KOQD4U+j$AR!%tdpg4vPpT9Oct*GNZujOWDv&=wV z-g&^GRvO+Qo$Y54Ta^l-3X!H$aM1__PAP*yn0kG}IkV4YestU!n#YfL5!1#d{V|tO z@by*Cu0Y!Tr7a%bCf9jM8u)fOX=dwYZUCjl(t7^Zt$rEH^;)rEAH=R61+5v{Lf0r& z&`~AQbrIh&4ciHx-+4u?8r2KQk$M31;N7J3TpKc(C=+E(C}0)anP@U|sY)W3dqesz zGGz0TkB)1DBiJ@=Rq*5pQ~Din7deD-@y4iO{i}?pt6|jm+I?zQv?r#Tdh&^3SyM?x z$gHT_{xaY&ZK0g!d87Cro7elyrk`TMb!%;}-Gtl<;bK-+l{w^3jbVjIF;1;BuuK2V=nlGg8Up@|Bm{i2`z3X z+Ruy(gqI8p2=U{wsjM~J86xUhU^k-UNUA?07A5-pm$@L}1!w>T52f}c_7CwYeX6n65DMpu;qyyZM5M5(%;s!}X9vr{G%S1(fA;4* zifzBGs9cn8I5RX|>$)rjvY5jhNG%eoQZ1tx!8tMeLGp%v&Qg#7ErH7AK&t`f_6bCf zhu(w)r=`MrT5gO|nj16$9d| zp;wQ2Zqtu4JF1seeoZ3N4q*7HIi9Y-yYc{o`bxb>RSgd(#8Z`|x)dh587Il!T? zG9%r?eJRo9Ufbf0=J{Kc1y3b&9G)5v_xZqzYzM;dJ4-OySyYLjr^*DgdmS&V({|WS zVtv6>QmVO2U;O#08gx?pS$L_8h>?e-=QTx8`!ocjt|`4;0#6^WflX3Ua1--=X&<&R z`%Sh2o^VKfhN6jA_eV&&aSK5TsZB+Q+6z*EaJFI}GpA1vqw2)Tk^Fgv!mG)_3hXj(_Zf48YW4cNSkY+RJx;@3K-`=a0Sh~b^t{0JSXqd@N zF3E*?sF3LytGcp-TUjy~q9a1t5OQ%inZ+TQE_8meG!3ohLmeVuo7x@N00NF;R^&pe zh%S}bF0ay}%g5%#g;v8egKe=xB1^Q;eI)0oj?zVI%TBy|z$S}BeBUD1MOXv>HTGW`h8ak@kBsV3Mw5ik^5*#lF{>m5QvplRkhK`UXbIU z8-$8D5QFh;9;s_pgH0YO0?|5lUm$r5C26rVn?BfJxmyQ>U(N&UQH+E|K7B`j7|yOEzXS9IXO0@X?!aksYz zMyEhdii>;JY;(Bynj5Lo+E#kk_S050mDR+!XB~pAv@MoWtX#KdCZuU9#Hx`#?x~f) zD4l(V4ac|BpZ+tPZ$6?;K=lp|ewl?D$mgn3hx}u|EiNV2DsM|VOn==Kx}PRBPHCtKM&(dzt#1%NCWR02YCK6LY8Z;j8fbdik3Wqkqi8BHf+ z-T8C`)ShQx4Lv@6|3dA@XnW+|L+0n0FE7UzL6V=N#KjZsJGx+1OPe01w5cE^2<*3- z_Z9frY4MiIaU-PaHwiG{XfU8y0cQ+}G+YPA-3qCTWzDz_aB5Yv@Pz^NQ<(|?QI(Zn zHEW)Ny!VVsUw@yni@!nl>7I?^s9f z_+WO4QgSpdGRni;c>035IIr7Br08AXhD|1qO#4gyuJGRr=OY{MeS9*fH(5VXmPR)a zq)=K0>_<9URHF&#tV2OdcCMyve-CYQAueDP04eir@o;b!B;}7GUh=TH>eOO{3%Hl5~xPs|IWE z;Wmq6Sb*Y{iR~Z)UYrmRgWe^}0MVVhLtEhHKuNaK6JJ>#NWL0>2?+O4-EDAYO87)q z&}r^rm$_1d&&mBRpUzZdjyZ4S#6>tqx^;vS1V0OP_!S-x0)@EioR#>ia<&6~hhd*bTI7kBZ$umMB7P7n3? zU-Ku^*()U0K!2HCWEw?L2BiIfDGbg8YJ_dcHjb5*1DpII7)Vu@*w1pr64o}os^gUy z;XeomJ}^)`Q$~hz5G%~$x)7cjboS`GDQotsJZa{j2ZwrDXnS=SRB$2@{&aPk5E!#s zwSrI?-BED$9kBSJuaDnzu2S*X7mZ+DqZ41C-e;zpZK@9L=6 z#oHjE(q@~58Or?-t?d~%=Wz=lpX{e;SU~ALh=5why3T=ZZBmN^Nw@89HdI@?Q_U|w z7f5~Xy>kIkxsS?6;|xjf;w)2;xwotK8uGVKf@quZzcM8M8O3A2^G2hu4nN1xraxwE zH+!2B%p)uh%;@aI)wtVgW_acu@UEJrcl1>{HH>^2DNJPh;YQP+6s0d{3iDCjsApcd zwV_GbVV{8*5u>l$=H`fbX*j6a5VpQ9@HHy&XK6@b?A(C#ZM~~#aO7bkFb*QJwP;|~ zB90QH&u&}sO{2r45W1UI3wp3{7Q*H%c4?gjJJeAdLnM9BnZY!$FzxQ#hZtB@Xfxlw zj87fT(6%k~_R-{_qj5v&xC51Rvs}Q>@cqI4O^sz{!PALEpAJ{hFZU88vcC*IG4nG9 zg?Y$jN$KwHK*T~e@*2k;&uEO{&W*YO^qlu*xwECSEC7!~bR~@kI6-1fWJ9YJw&?9; zLV8VpRO7+#6z)82X=*SHCjg3K_H)*&Nf(n?Jb03lQlW)^l%Jz~1=`x*KW0Tz>lD1q z4UY2Ky@YxTYHsjy?8A_Mna$7~Gqx3Hp~ZS0>6B%CZD>&gYY$QInZ=VJMt-Uwk@nFr{6mj)C}7AOG@;hR-;rd@%ieFi>gkOsm6v<`BVc~ z4O6F7a=&y(&=p9|bps)V-@dOokf+Q2_@(=3{r8|bVRF@!`KPFQyS5f&%Ym_nmAFNP zWx+`@mjYdXty)mVr(z8zKV{GFS1y>fkmw*JTE&~*w#JI|3~H6q=E4uSi1P#JOs>V< z98vL#u;GYZ9v-SNXFPt^ZQC+LHPwqYx9WW6(x9jW)ywYyid^{K$tXeWURB#n><%TM z6Efv{NW9~ChTP48*c`0MKBZ9$sZO{j1L}(#pyub_gNa-;(y`J!nMn5+O^g|U_490-!G#9nsl_T+>RAuM?j;yzf7uC*kU)`d zMpcAPH#_()!XXMI>rOn7utB8deoYE4kh?y>Hu|;O)NFz*$x<=!oTl~xKdtm_Ob@ju z0-wKaPYq1sLC>w3(G8I0WlQvzLw%dG{2u-MK*n7r@=#w6G%t34={dkJy>IHNBsBlR zZeE?NCz+wAmxznUdFuc~!@eg$`f((qihVqTlGl-Z&7@!AVBbiipMw$f&r_#>wt%?K z&f&HC`Zu45726!0gw9U@E$I3KnZz)bgK*J1rrL4^=IrG|pl+db_-oLxH(1V3s*E3h zZnZ#u^o5Q;kB5GbW*e2(cv;Ye3U<~7ZL<{XbQMo zP~^K{I(bR9rDLk19F2RI=ulvk?&as6fT8nl&%`70m zqT3$vX=zSl{*V-G(xy_afeO5wZ0$Q9$B$@1R^8m8V#!*erun)u3vsb?(_794_U-;zJ667^A0SB z%w1vd&d{Cy1tBUzr`UF9eeVirMR2q}>AjOiO>1+2@5Ucqoe|%gZW$h5%f!r2+oB|& zb6zi-ZaD1Qj1aKVq~-9u$O*{VCIsO&6;P} zr<1eety#BeyF-kiW)8o%m_+()H6UOt#wxy&id;8HrsIeMPY(;|;GdO}Ms6~;QE!$< ziPq%8C`J=?_QfcdF@9okvGaKcMDNLH&3I^u1R+yq2Vhz9O}K6FMq0tslRrRfqD zX~yV^E`y@ZPiAg%W%k)of{ylX59$DC25}=mjBAPQ?c>$VR8zyOfngNfj1y`+d{A}g z;vp&+obdCW=V*diX_d6jB8@t^$yIG4DxyKuv;|wV@zp;Dgs=VD!-g(P$tMCH@chiw zA099|ZGF$fYi(~v>@?!?2A4l2kgMGV!!p4_i8(kQuUYrebE&$@A68K@uLBACQdoxq zuE*xYulX)`se*?QLa--JpUyzu8NWGf9$UOj1vW65`k*`5f}QVJr0sjiAzaqx*Kq;4 zh!zp4RdwwEiW0(Q2)~}+-MulhYd8wJJ>KUq$rUgw<~2Tv=sUv1k*gs8K&f)R8J*w) zuVV-uLY>m)M=!I_s`AWy)nx;+e)0CP;1!<5d9g)5i)8}0#O(=6wj&(B^|;)hPV%6% zKe+21D7eA#tn5H_iKG+@!be=EuctTUT7AT1(QBh)+9O1GL-}i?GENRXNf{PI`22JF zkl~%+ULs6H5cbx|DXml6KI54*85(?89o#Zs{Bu{R-9+DhbY?}t(-(}RAbq-Y;~N4~ zqE5dlRt|R78cMFcD1vpuhUm_1(o>_4eYA6S41tj#s{-K5nPE{BQF;Agvag@alK1{4 zJ7+O4F(f!q{&l^>y%}seFyO!!m)=8ux-39l!LN@Gjj5~3hbfqWQn(Xf^0IYXxxk-C z`$XMb5y>7yGn4BLbGJGw$S(~ht@Xh4`Rv7x#!4t_q7Nqz>)K6 zs9R~QE-Qjpgi*!g9=*ZgWZz&4of=e ziT|1mNVDnHSt1M(XmD_`v!7ziw$6U6k22VE%T^KeGV?!fD-hrAU0y+`i}TWTUR028 zxw8bfxAXDb^L#FM+{(ofU86Z10s@#m7)HSKmH~5SN46|XjEzPouNL>jK=)JTw3#3r!g)NWdTBWZf z3x^NM@Rd;sso zK}YlyGx)&(S~5}46Fms13;me{6T`(D(KOKp@uZZP*o9+(Y{r6FKOjslWh^-mT|CSO zS_E$ec<6LeP>V%dwotk3ZjOkYkebW}6Ecemhf~J-_jm<9Ewg}_Epzd+;=D^m)Eq#n zIoh{M*VprjuX*??N!;Dym2cg#a#q;_3OGH?%{EqQ>l=Ja*0ZPi;S$JHB9J=)NrU)e z1$yV|ogD=6UghCR13QzN+OxXGnzIM2;mk1-`%GA0=f#ICu9PeJvINe!Rg}} zPK5$C4y!b`ap+T1m}|O&_?KhA{gE_s!{Np|4!qWWKX61v?5Og4e})PVB~>**%)O4k z?A%mgYI5FuXl;9!8eRMObB#j7^@^ZbfBu%;IG4>e(gnNkRqBC%7n?Uq=uS>u*rwT+ zSGE0-J@FXPMU}+2Di%ijfmambz@$cQ(~2haLGY?4b@e`bVN;gDpHYAj_Zyq=mIqIB zU}Ven&WPKWo=WnWW(JeU-faRR#$j47fFew+z`X=sNuVpfx0lRK(dv{ z@7}3`nMNM(@=yZmcH0xtBR_G_dM9Cb$6L&4sk1KFtzT@^nx)N?Yz}p#mfp&l^ct!|j<>c5Y=P*A6TbN^6q>iqL2$Gllg$43C5rP|n_ap_ zqF(Q)176u>&{0ngWYZg>xXX&2-gPPgdL0}}%<64+5^OES{TxEs-Q_!E$qxot_~xkbFT&WMSNOSv-PHK~K9HBEg`vYL zHOlaZjy6AAvB0gQqe9w1lRsn0wdn~+kD=u7$4q27Mz>Y+etMItHMxy*AvOKG_?Gi( zgY#xBGNu;syrvSbV;t5jd0>TlN}66G`}|mgc)7Qph(Q~rx+#)hxJDmgJO*yaEt^YW zcM4FN@x-~zg)s2{86Di`dPj5CN!hYd$SuQ6yx(QpTfAF>kvJ$)2ywXOmIhfL5NTac zD%Vi@J*D>;sNbabHZ;9TMIJZKT>o+W#9Y*hMXx)GSM&kQf00SO(@oZ?X}`y?a(q&8 z_td)m`Oez_l6#hP^2rX5rH3nnB^D~g1bwBOhcI4lse-QAr_R$6?Xxni%V9Ybj9rg$ z2YUsvS4gp@yT7AEz)g}C8@(p&qk2;gjeKQuezs=I*C)4Uy?lY@Lb9Duk*UXB2olyg zz?)kB%!IUW0(f*{_Dd4Uoe8q6bK^jFK}_)^>h}Yuo`jO|#0tV@9J2)Pxemi|DhD4E z@y!HsXoUg;(#84cyUBTvv*vyGF89n8b`h59i`N`%4fo0(y2p?NlEFS@j9`|i^hNMH z6&&AfE7^%%k=~6E-0b|maqn^3jwaQe!8c^GH9EedxxhK#t zlpg~3paGj!yH0a4~ zk?4-p@%>GHW!(xHk%bRmENhMG_Vo_tqzmJ88nFBHbA&nIS}oUq0NvaKhGWdyuPOq-r(|h zBaWd~gZeo7gnfs46J*_6TW$#l#$iDd{&D7g!p$_*2o!Okp<>`}Pxi z@%gnAoajd4ytv@ES$E~;b~blkj@6`cr}6Hv@gROrKf`7jbUsz??eql6a|)55fwEM{ z%)zt)T7tv!h08iz-t6(tSdG(1e`}y`D5tUz5;b9nR0Vo0tp`S=tex*3+l3z2gWI^q zY`RrR_axTD>V3smlTg@Y!()lnB;`hnBTrzqchUT$Cn~S&hke*q)PRi!iW)b=O&pbh1+xzxw-Z& z`DvZ%B=*3xw3t{en>;(4u`2o(Hb-Gm++mc0T026ISKDJ8Z_sEpO=ho52%R!pj}kBQ zgkY$)f=L&ytA_V7ukZyI#*=VImuznU{d~E1A9rbgm*cVUZ?8+bvphAVJN#Sp*6bZS zhQ<3Sg;ll+Up7w3^U*IxYNlU*w;HJSTuIfOvBe^sO4w3z8^$z54cTokQ)tv_P)YDT_1yJ3V5O!Yq)+Pm9#%f<777&~Q1RA7blj zFPN6|^j%M9qW|pQI6AxDvovxvIhrSF*2{c-w!4@wJRZ!2{EB`ucko0?cP>5sE^HXP z5`*rs;Y!VW&4RmfoBERHsNSVw!+OaCa!a|fA<4`BRgYP{!F7;q{I|_8=huF1uM-Vg z3_fD*kI@vCCz>b#Pc}iJyrPodG{k!vI`-6S{n2CMFUIkKZFplMrrOK7hL_8l-9N}+!3n^(>A^Z;Jl;I_uH@txtVtczm=i|4)Z`IMwqRK;t5gTY&{wRi#HHnG zm~034tD+c88&i3ID#-q}esJb25OR*+T!xt(zawumm%whO8a{ow4N%e~H^a&V|IeH> zKlq1SZh^iY_sv0vQFpx#Glj1|?U)OVP=`t`g+3v+szX~x5@ZG-s-1d$%&L+5WR2F zAjs*-vm&)O+XDeWgU)q>Qn>QVdfJV-GT36@>ABs1SV$*-!5n;eaN3U=CK)vGnF=j; zmn`?WO+Ae}n^}&FM>0w}s3Ce|8L86rnEX#oE#$OzOg*(NY6{If0J#+%B@hiGiNMh% zb%)KBZ_GONsl{gQGVh>*-4vO6)T!}nX=yJtxIJYc%$UgFmKP=f!ja6R$ONx@_0t7w zy}$CYTTcF@W)$^X^%o|EB`2bPWb023*M`gArg3~Dmq8%}L3XaQI>i7j=Z`|#^Hn!S z6Tn`5A@^F}?(>HcF0=B5=rLe^4faXv^O-V9@QAUr@tlhc4jOg#axu25mXso_>c6a- z%%cX}MRrq9Iek2!b_R7UvrP-xUDnn>gWDf>A8Y`Wd+DK|pVlSd9oy%rdm-z74hn%ncbnbt89+1i;O^)Q5B zO+(k0son$&z&${9D2Mt|xl{$X{KYT99@RAQK=0-EXZ!0EN8KtIe) zCQBl`lK_qwFK8gdS$X)$fk+#(ma5tApSUg!+C+Jg?t2)H1v*0JR z`J_YjJ&8;9-Ty)xSu3L+uwB3Nsy?Htp!w`K(iKwO9`?LJgEc#C3X_ zT<6U2jX*578q;)8)uoh!(Wd5)?0tUUQzs78UyAi0J< zW?90qI@peKh#;6SYr!2XSUY}noI|zMdfMKp&XAK;8uavU6N3bTzN7F2*6iifk6Na8 zC-ntyo%{A*j{pkGsdu-%pQc@{1hQv9`m|nt{-&cn&=<#U;PgZ(vI8jmRho2WP~+t1 zZ#z25Ig(Yw>Occ9GXR3%26a?;0|!ecJS8~oUA0(tIu^OHCqu0o>G4dC37j#yiehwB zmt6ZjHUxOyq+YSvz*9!|q!rE*z`YzyE*J~>MywnsO7D6S^`bf&^rGt25* zWc|Uj{*q>Woa2&_X&W(SO#@A$?NQz0y_Jx?SWDhTT;V96yAfu8uY^1lTQ3y%rSRyD zf>3Evc7JJQQub~o!}kJgPrJU69Ba9ndnyY3ha>3whyQbXSttXV5w0%r&&Wvsa?O_x zCl)*vR#N@nLN@)6pZVip(j!DTW1s(bckwsk`>>XvP>@O5g30$dw)?FY^hlvtn|+io zK6-!ez6Jo5W~ce@^okbsztU@2;C9VizS#~gMeUi4#E9x9myJM`|Jow?M*?V`F3J|? zzFqkm4z;rU`g*t~|H-F?6j~bMo3N;(?!MY4g>QfNjb8k3ALhLTpf)Z&yRYNEeX^`a zZ66I{6zQZL1C@Vm=cDQE<9zOf;M;Y7`EM7zPw(H>XJ==4>UTE{U-t0)zybNXw4N^M zrfN$u7~W2ZAj;g0+*|Ug8A{bPqJFMbFgO^^_!c-J4ll-6V`pU~4j6KxJoUfbVn6uL zu#P~jR;OmI)+$6dbSp_^!GbpSOsB(>Ka}e=fSg=@YMBOqZI6_Trx%Tpy?WE|GX{ zT<}wZra1wmbd)gsH@E1!C;N~1eg_bq6VLqfcmHzl_FO>h%Av6CZ>0V&fBAOLUekJU z?*H3AW3K^k%qh+AZ~n_azZC%CP5Iw=%x@l1SrM=;xrcWDze9L)0*V;BR4wq&FZ3@j z^EU|Z@66Nx`$hjp!2jFEo!WBW=)^aR|9^qe{yzf#uiLxxKmN}S{zmKmk1G0qxr2Rw z{8wAyzo78B59 z`%wOLKI`ufasEdamN&;TXlI3D9db^rJ^Pf1g=&%Uo(#?0l+NFOz5uZE=FMdYEdMJw zf9&;}x26dhNeizx@45@Ft+8?*nTg{&vjimczayCf*7+Zu?|+o8|H1iu2-ZL@z03{qzj+XAfOilth7l4!4QKWbdvzqa zeQzjRqxCq=Wztp%=ok1-mM!z~^1q6j=6X_sKG2Zx;G<=S#rNy)zkNbTz5vS{ z{kP-apW?Fe5f7JzYVz0ESEm8Wfj0oMbkEd~SdNje|KRzZ+F$Lj&*!_GEVuGn(TDFH zJYb~##GT$BxBUA3lYW1&zH~aPDH0Ysqsfj_p8gvu|HuL$wk9H8(Cd`j`wwe<-`bxu z`Dxb{XRII7*5A&hk)f08;AaHg{`EQ+&c0g5#7l;13X_N|GLgv6v{q-tn;MvOPTLi)Bl7?YitOXQTeZO=e(XY@`v0%S+<0O^znl2>y@jjB&IN|Lrte1F;dyl3r)vP}=-tSjp^ zzyI;0dLRNuPFW5&{HwHtD8gm_kb20X!{2^vbYj*9=a+@Ip%nv7eF2T40x z5h5L6e}DJ?y2TazWyFg%9<{%&!%rCe`R9*!{wB8bP0!v}gmv2#{{1%JelpE_e%sew z{MDYEf}7+vttI-dWlp=wqyi=M)nP$EM^Ob-xNUvUXRiv3BOS?f-sbwr--vr8c6fGq zXI#J8?P?_GyPY`!1WOZlCI4n)Pd(sE+8^hMhJX8=8c;wprWm3pjS^`Cj8j)Vci6n;B$2mf-VpjCk*K`*jNRBK6~eCMY6!= zL^98*rv@4VjZUb*;4+uw|M9^3e{73C@%-I_hHDmC?M(5tNP4NvXPyUfv0Ry%Z&`(M z7mVMhfP(MF)3|wG`r-6jyrjFlhhAEi0I*X}fAQTS|K7d<06bqmZ&C0lN;Tj z(^aF?7dXDwd5B*Ajw9|DfC&-c_Y(J3_$xPi%I<%=Kt^}oS*nX={V&qqJD{m_+Z$Fy zR1_VhN*|TpRGM@U>C$@_PfWK;KlA3-WBX`!6-cHHAu5Z<*Mt6&(4T`5QUM^{=W4^}pLz1v z_hs6Is7km?ys`@Pmi*F@S|LK zTjJjw9rM>Q{7n4cZutCKj$MBMp8vX)&u{;qwjX-zthIhI>5U>S(mQqIktxV?>_4GO+aUCUyRB9yXXI!UH_1Ury(G*36N|0f9*s3 ztgS%ZU6FL(c8>G%Kldm4)jfZ5FW2bNv|BJ|ci^s*IBK;`tMKxbY$2F%jI(iSVXCX< zXWyf{6y`^tg}gp5bLmHlqb#>edBzffhVTK_*E_(}|^y7LzaiTsN(2dFYWFN?X=IT2qlEy8aoDl6+ zJN>b8h2DxYN=f^MGJ-j?M|WeK5rPyJdo3_vWlQ!-b>KMKEoVUJl`8fCS2J2EAF7&y zFGH(c)Ckf9CW}i9e(aSVn#So4HO~2KLhxM8RVPdHnG8TW<=*%8zm~spO)~8^3Wbie zLm-L7PYDG_p#mu@lPEOR`R!71c0T`kF^B$2>Er;KHL0_Ku!}#be`vvv0-t^??!?-; zpSa|CA9#_p!^TBK>;lV?q32A{LWjyW(>@F~hYv~Izw9?*CdT%&V|%Ad0`gpOPP_CU z6%FWxkB|(9syi%22zF2e5OkkR4a6_-sU+<2==5%e>E`HgVHrv;Y8)g9RqXW$`R&=6 z;>xWTNV)IWsOIGPE?izx{#kZRz`<_If2jR@wBvD7-gT3p1c<$`wU;2?+FMuL=h zw*+p-Vd{)!pO=Z4{}X_LZ$;c1#3&|_H zpv=|o$s_sk;B}exVv}E3Qkew#Hs%$_TG~(W@WeZ18$b35qcu_yemaG&rj}?)_(VNk zyWr?|0T;*7Um=Q*hky%NuPOV-{NI1;rt8Z;wn-W0bfbfe3k_23RE@uXsmaq@PCuQ# zOV<>l!kBmWu(fbIXQ<}s{_D72whUeD!S#D1q?hmLULJ$zg^-+qT{tcnR#~9gF*C`2 zlD|s^p{;2_*x#iJu$exsEH$H&**F*KVaIWTJBM44PCY7j`fbVL;q2x`)H?L$@nyEVE;+EC@jKq(=88mMZ5swVxIhar6(eq1PMbl%~c z9gE6mU!wJ5bvnNu&3eNA}eZy>fJ9kR|irdO)2y#%DZGWhv&AjPhSd(;)0Uros?Eg3g|9$k; zE2!Px?^y;3$fLHzs;tX5@fL9ru97z?%y47zg`aZ>_Ic&LXr(V*5&~+6)4+7djs~{3 z2YTwme7>)wp?eeO1dcLM8pM9!SHI~}inWV0QuJ}DrZ9RH&>y24*HDo7O$ZJQ` zX)=Y`^Lgh7JJ5&spxqud9X8wB#@h`7Ia3Khi?lNVq7WFPh$gn??Oh2`r*FWWb68z= zdwA3LsG`8FGveL#$HbfHH#CydC2wz66hEoy-!ci-N}+wrd@uw1){a#VGYsZ(Ih?%IOVjrL~Bso^0Dr?z$$F~=w03@u%fGY9H>-5oxuU^NfVWoWHVeMeT>W1k`= zat>MI78I`a9<&~qkNe5(YUH_WCAtt?dNPMAe4hb&VS|C82~bTl;OW zZ&4h1YqSSW<31y3rS<+t<@Gx8jSZR z1z{ zDc+sZd$9CU3ef)L{TQL6@;lMM6r2;dd&xQ!!OvX({$$DKM>geu(9VFMMc3|iEuX#< z*s)zZe*7GlF5A#)34RB9i<}x@Q2w&?#=ho*a36*R7JO@gFE)!FZm1M7_)QP>o? zc3mEb13+9_z^0hgeoT%EU1i;YMv8Bj zlaifJ!A|Gviy51t{hrqm^9}n8?n#lO77#N2Lp^!~!SIe;$3g!_m(2PVT}{R!BN|B@ zb^y19P96A;o8k68iK}D{fzHV?IIo@ccs%}yB=M(7m=Xfqnyuz_2DHhse|#55ax>LU zKnmK?+^CcC;JGSPV&gomCWTHu{A(vZ^=n+!004ZG!UBJ&thncw;=9e`TiSWC zvFMe?)+-Q*es`~xe%v$14r_$Zq!p>UVz!?Qo3C$TSsRY<&a!FTcq#OBBcx@ofsN1a zKt`w-Yg~I5ho2F8I+o{K{kpi3m*T;kPRy4W>*uN(71wLGDJ2M^{CWc)?1Ck)$u9_t zD7m1{vM8f^4p8xmWLV1@;y%o~`?V6K>L%t+0#QL?RR6{&+u(c74uWzy;`8i{~uge2_j+J>{uC~Xsf_Ic8!!G!*FPYA$ zFWwaoQ`L0}q)1@9qMYEbMlZfty`d&dVMg5RcO71180cH01wtK{!{gcRSvPoAYhMX% z_9x3={|O?hC)l*35skY_>ZXT`FdF zc}^z82Gdnshu?VR2_%2tXhprSmWQSeyVV!z$EzVk+5%r_ zWG+PNvZfTz4H_u;OY%Z?Ttv(Y_vh8pqM|h;%(9zn>LWFPDK6t4ggUIVU?_V|Xs2{4 zjpSZq0em`FCP_S<9Hlroa9cbIrl6`FF|%yb}_P%mWR}RBZWriI}7S!%|md2;{2Ww~AgNSd9Ky z@N`p)GY^Z3gCD_Yl71uxIGL6L?d~pbTGXN3wV=bBu%{~;_Rt-DVqqgtEnNm$8Fwd} zaVHv+wpx!?Y_$tqrz#pZr>--?<&T%NLdV#C6U@;ISQL&a01j8yHRYf?tjy#1lSZD$%l|6zt8DT;%Y zc%ceh#l(>Ib>k2lyF;ZoUV71G0FBznJCtzjFyV%V>d_koz8N-(cFN1f?#d1yG?y22 zkw(U_LI*cI+iM|VO$T|q$_ZFf8?Z?f?AStvA8Pthh1AXc;u7Pn=`bv3cHT|4SaN}wK z0EHw47%)!Zs_SOkB6_ar4SbByn?$^yrUN1n*N%XLJ<2$0%yxl^5b3bI8j_IMFh_8h zrql!iWRS(Je8xJpSl!Lqj%$vEY*C^Ty;g-Z9C^-FwtqHW~G z3E;5hr*Y|%-H?j7W|CtA!F(lYJSSPbj8Uqdic=ghA03)4}JiSlH6?5hTEn**yd; zl2|B@8ebCr#4q+fSF140l$=l3lFvJJ|1Sr%!x$0f2*g!}}BfFFhXLjf%{nV384bIov=E06`(w=Ne-G&H7uK zBb@J#4A^n&`>+UMZE2+G-VU!(e_6NvU6rol0j&>~%t{hbTsKXz>hhu)sntiA&kkFn zmXMK}>=UJbf`s799*1w+g1$d2>zhGD$=!QA zmg!P*A>e+8gWz}#bN03C+u4uLTpdR3ym-eE2)P!QPVmX_ndS!JDVobP>piX|FAMC9 z@d-D(jphA*;a{Zvg~$rhcbPgg3=+Q|A04aK5ebeefl0!4@>+xnN5S{)`27(5<=_9x z0oG@-e54W11bwVS0DG8U&9>#;`bPnMd(9M5cPt~-`I;PV+ecG|eKJjBd=aL!Ew1hO z#ZfMxYAtMCbbUR+m&dArQcn?=o*A2t3Vg2WxGgS6XqBFYaJO~&OS}vG94d!LUD+91 zV0XUz%G7yAJtV%X%P;_eV}!jL75^e5fD!f=LUkY?ng?}FW)i>MxcXNKo+bsK&o)Heu}5#` zOP9cNCutwsHXu1HRc|oD&EA-}X0eti^VL|rp|3KyjfJ+7$&EHNM3(0j@z%~d$t_kI zp8fnfZNfLUW2q`yL(9R`Kb7^oup}!g0WY~hf!n63@oGAJ3e`g)rMAoBrZgqmZ3RDJ zI%}>}R9dVKi&(NXg7@bWgHtlEc?oFA_bvCR< zJZH4J)R+0d+CYrHl+_2X%>3`u~;v|9l0fEz8S)TsdY6_STEj zUFbHYBW4;&RNp+I4}D!E(|y#>pInCXpWpd%sfIUbj@GqsZoll1|B51`r{-P+<5rhnk=kFx=ORNht_o^ zp|V}rbvbx-`b(UybMr$2oB)sP8&n-9ERa8*(NuO%_3rg6FzjF-x->;ZHF|_|1J53_ z^BnC|I1*IL04bj^%O95|H$IXk82IPZX&l%1&_rrrtW-_A7=_m%zF!C}sD6@@?69Re zFC03%`{+x%=GlU)MLTsLOC9j%(t9mY9F~qB2$QV@%w7B0n}oSdwr07T`_3I*0a=AxMrSfjfSYg4;rI zIhvxz1N>6{dTF z$s&p7SX9eyEvCFQx%%O`CZ_{>F8|T@)}{$Nyn^PTMWrS@aK0WL2sfPMb}7Qry|`(B zjq{~kV_;&wG`1|^3eRJ%7MSfey^9it_EvAJnr^o%TnJoI$9pYuai^RTbUxyF{&dMh zTM+B9G}YAqI7dIwr88*Tr0;OurqSaUdVZQTk$fA(zcJuaeA1silZ`u%)YTi1t9^as zfm!Cd=^p@QiR3-*C}hJfTpFLbh1Jyx;J-PXn-_NJO7o|iVNa1@Ima%z99_g&=+h~_ zjMW;I3(5G5%W#62-}dUYBwbs@jPbZ7oV{{btNDubIB)(8ejjyCv*+_xJ)$_SWv4=Oe^?d=F(rsXQ~DN%!qQG6r( zf`?76Taf)XYY2K$8tzG~6m=Kza3HVHt8q6eb89%{?3xpHs#0n{V<=xZ>}u;2%j1Ox zCcww%vy*i%CEtp-@aIVn)*5nd_vrT=JsZuuHx?_-^Rq3VA_1L!mpL%{;?hY0sB?V_ z0juIZ5F`wOp0|AzPyDKC3FcN=quY zUN030$p==IUtkUsk%Alc7%m48-m(fihqDShmawJj$|q5be_dzxY7AJ7HG{>b7q#CS zw7fLXS;~^L&#T)RT--~Vw0CrmO_|9(6=w={3si@wlr&jp1WEQE5-UG7m}-FYjFe6c z<;Ugs^Sp;DG`)&CKQXCK@&`I`770}IQjnBli+vPzT+5z)v=03L#>v zz1?b#45s3(J*p2-Rke#(ZRp}pq3o((>-^Mwt|`ecqlfm0hBs*Y!v4PE#ndP68EdK}>z1Y0-$6BU8h!tE<++$4CO_C5<9eCQ#X~Q{EdCuKFYI|& z!<8(@j2x*8?7{)*7JFSv&=oV(xpZADCP7ojcpEePDa{G@U0Q*?od}crdB`Rz&p7g7 z14Z3cCtNkGK^6;yX@P}hF1`@8Q^)RN zb;y_m9US#KY}*lqPZL0iHI3?;VWN1GuM6PDMJ0~H9<2-4pLJPXW4Sao{gSnD#Yc{A z;Yhu=w1$OZ0ZmNEByyxlLC!i{cHWK>{uskh8lKtWgs)*_9BV1Bat6ch_ba+Wr_^_U%gZ@CX*q?j?U=#Bf=0Yr7{9A1V zjsrhH0rEXV&xtEaZQFjI9`l}3hew_u=HFfR$^OV2GHIhrPgo>tpCyT|#E3>|ovsZ!ImmzV{jYo_zc7(1#))k5^wA80u7dBYk41d^ zc;WH$db0I7VD0;@&qOohi0^sN>uGEe*a*#qi^_Ym(7&!{cQUW05(V6GP=ii9~uJTwP;1IdfCTr*pAZ)D&m%t@&py z4}DHj@Z59^;C(mycNOzKLGa~pb&dD`FdNzEi!WDqpYCi>E;5eITr!jXHv|8t1db6W z8%`oAmvdj%M=A3YDNZkB)TK;%(EK0Tng9GF|6)k3qBx^hz&i@ffoV5=GxPAlr^+X_ z_k_rfU53(J1-wgiL!TM@UkGC_({0@rK37$h!Av?GRd|`anYE2#7wOF^VhTg{oMqEzAFf9j8fugtQ0Oc>f}1`x~L@;$D)Pht3^E2*z zPy?jD8XqMrj48yXMUlLo7)}o8I2c70qT7(Ath1dp^~nl6UH8@24peE~WBUXenr`Q< zkT*lo5)JO8(xpOFRK~XQ6lXYb!>I*0iCML3?qMIa+zAQVvud6aNGDqafwNeXhg({& zp|~vA{Jq~p?PBB($lwm=$ak8OCZ?iA)2hJ1^-q5OCJB)q7q)~9>tB_4*6q&F>CfPo zy$ewwv;3i%DLyaat_XgoAc?>})w&xH(&RSNTD>z*m7$Ata=|7DQ&mz8Qu%W88#d%B zn-u!Hv9HO;`V2R4bPq^}jC8gb@l}!sLYts|MjHYx=Hvu3$B2qp{^s>zcQ6HYCpTuQ zg767(gbDm2jbhfDcwg3g5Y&Q`Y8$gBNUhF+_3?&h2rpy6lcOOox&0Fl{VkDpC7^Q& zTfWZ-Vo%U@A#FQjuOuw2TC1@G!&LHA3pQVA@H`fV@w)8xdBlU#befC{VskEY7%J`afQ*}G(!J-*LCRXBW@;@V`>PFy*Y7Eqyv=sbQS{~caCniAHh)r+ zO5|;p(U=9rOlyEZO&CQ*%?05N7lij}9L0-QV0Fbh&1olw-mwQM-XGE>)oLvJPkE&x z@0ebdNpw3js4D;TDZC?ed}&>xwC{4cOPCaac)R-039`0cH?vb-4w@V`nW$eEiyodF z&9hi%s6h`0pdTz=FG$GXao&_Y*zp>5d311r91XI`5OytCXuK{n`l+xM#6;`%#;!31 zKgZ$Slbw+@f;XgY?ypbV=k_|B7Ek$LJ$iY z4#FPX*X}@*4DWX>1Clyypz*dzbZ25I5uH8|OL`>}qOtfw#j}E~Wv|lBxvMC5^nVR? z??MCu`mBxT{Jua<6FaiOWW^?m{`8Ghyl}{`cQPcVCKX&!VV{=1^RT zXya8pXSKd_ZW+g$FC4}BD|~^RPE89Y~yMlD-QagC-uRb$eK)uKoOI~?NJ+TmtI%R<}zI0M=s5&>A;KQ+vw^bIUgnV3m zh4&>MJ@E}&SpdaYzp+bAt!Nr0Dp_GS8mT;~_Ef86Joiynl;&-+A-#*0&hHNR7eTOK zaper^+<>X*J^)fl(j<7Q3ZlOZcCn{D?LC{2hNRIV3z@FI((w#hVNE@ zmpLNbFoM(43bE~6?z7NV9*SCv?!W;E;xXlfYI52&4%jF`ZR5e2V$_$|RqC9eDR`2w zL6jR@W!R3#Iaaj+sK8Cf=Y_@0~nX3%f*c zdE4b-Vwbz7Wt4m)2-p(O0Kc=b*u9T@p;AfgOH`GCwsW->enwn)5LJ4UA%J6;Z9GZw zf34&1{Pj*&jFRGwDfhR_Bxwceo=L8k`Cpenq28(YW)k;Q(@K!qcXY}VXS7;|$!m}w zr?(~^7YDx?dHr%YOP_(!d67AWcxS&$WNI)(E&89P7PMbwpDkZ~1z7-e72XPyd3 z$2EhyWV92c8;y?vu@|qNuDoF76O6LW;8Efv=GsT-tOX883mAB3T4+tOL*C^yMxv;3 z7Dp+~pk7Yuw4Aa$JE8k3MR849$h#KK*Zu%mxE2{{Tg7dSXkk9=CNv+u)aAaXa_6f- znn!_dP2psQN=JYrENrlkeGrMaYtSPg{>b1l@$Du*WcqE5k}pz`#{>ag(qhaqW0eo41sMZ^Z=r=I2 zUE_7rW?b!sN^G>8+*D&jd{}zvQr1gd&$6Z>#{}Lb=nO}24lFcpF<{WR)s zCCp);vBjh(0Bb_kTe^&<5GVw+3@e2vSQmQ$xJIg|-%7SWK06Pi zXfsY-%@%pOR|!@q?}Qt#(rAyF^57nW{rr`v8K|5o#WxesPGRbayMe`_hxe-~$RWRV zj;;I--UT+-A47*G7gUKxZ^Us&mNa)`4+d>adfdK}HRo35y|7*aQt zs}>g*?VxOS?iXEgnmtWkZ(vy`-%d%j5%_65J+~~hb`Mu6vZLeZjx|^{Ik%4}Xdx^( zHCOh&@T)UP5co=2zEeEH!HsE@;YxJN9(F{>iM)5#JvVo4c(R>qgWaug^N4D+HO_=3 zZ#OB9@KzTZxe^M|IDmR+25xQ)T`WW{HmGO6Oe)_{b)8&q?4)L@gY(^Q(FWhgf2a~u z1P;_F@UWD8zvGCe?O6`ck2_%MbBZ*Y4=y3FCo8BsKi!A`+hjd(tqV&G@${(L?Q+)` zD>8}tEE{5hhrbCZewlGEkWyW#Jc=vQz|C5<76R$moY&ISX@607;ggBx*~z1ZEr~2k z{)M}xvspXOaZke+#tRmacO!v?5Syg;j{BdPk&r*cBNb{;S(2A76xQqhLK!8 z)E}>u_vx@l-yUgt`N~d0u@y5$!(_QOFxKVSzAt0r#m(e8n(nYVd~~~NcXzVRsocrb zgTKTM4!6h{xMS=1smEy+X9==ddUNNdD0V=)fUPxF!L^a65qS&KW2>t+5{7xMhd*~E zt*QakOT{}Y&R!F~>@rfNm_8W|9AUC*RKBC2HsXo$=0CiC$4<4iZOM&zfAM^)cSB9A z@H@10ajWEHa(6B*zqCQXQxOS)YS*=aixp$W<`-$Cc0MO8nxyM^hwgYsWbk0n9wioK z_B#^KR)5vC`KbgKdiqB>?aB7_HP6hAcm#jwl66_zoKbvrc0aS2R?gQexYfmN+!^zF=vy z3eui^MZao^G$E7k@WL7%l_TK1Ini%F4?xVMugaMh0Wd_g%({yF^8{z;sZJf6AwdiH4=tP}(O=58jRN?FHvF z6v=;mX~FR@@=7&oq=m_Jg1=@e4)4rx4Q8J0)!z#acS$V~5_)*pdihR%udvZbyXpP4 ziZP77iNttec*lZeh@DX(2#Ff8P0qKfcTV(@yc8i|+$*dFs>3`b1WyQzhd&u6#YlV; zY57)|{@(raONH46gKeO-@;CCn6fIWipn?km(3Pa^l%DT?Z!3}c)!k_#B5hcxMyb|L z5+sJ9#Y4Udp;YWzc$I8rcxGy+ zZ(GeV^UnxZ{Z_A6Ai4Dmv3O_EdMBvJ6E>ZMXeuU$*bSlJnBk zA5^Ny<473-byUlHS%nLtD?$Ux5zYIqn&}l$Oc$wDI*K5QVUl8M>iS4u^#-JG@?_^; zM|lq!p$pO6K0JeXs!^8>oKgWkWGc%puiTVv2CZ#%l}cDloNfa=T~r3z9Qws7QwBQe zR8SPaKD;IQ#>?Qf#`XMHJKL=7?SfBmckrZdt@HY9%scLuage z;Al@=)t#&DU-)D`zlWrB`Cvn3!Y|PFh`9G3y3)OaTskl=r`sKU#{8d) z*#E56UAzdZ`d)=^&7nQ4vP*v~9Inn9MavLxqU!%;5J^U<)WRNkDVn=(p2>ur>5(&~ zF#A1%(r|iK0&;T&84x?nFx19g2y3P1NYC^Lan1-T!Ul*SNfWGF$mYf}H}8k_Y%$GV z*%R^G#F^ASkr366+yRj1FdP$#aC6*1>}8+p4bG8tsMDe7RUfa_7$2U2q!l@V9+p>m zc4rtKDu+w?x6c?Tl(&024*b?4ov}vXZTB_tlDi4B#@Au%ZxWrK<%3`iLzTV#9eeCs zm$Zp}_N^0}8hn!~52|Vf(p-5960NHr=S_6ziN4L7%4tyN7Tn6OjeF*}kxZG6sww_c z9-vWO(YpFxW+F`7v#p28Q^d`w7m}6d&zNmY^+Y|JGeCO`6TSa=iA5Z~z{@Kgss6G( z!d!rMa+E$BZaKjF8tq`XYB{7-v|YT4jP~@wOuR_FJyaa${BDVOp)$E9%z0M{WvI(d z!=kpTZJA|*bB=zMQ8j26yVJyzmh5uTweoVyYv?kc)Z)O~fW(biymO~*-uOH%Pk|Ty zfE9()(v;{e_ZThTWqg~Em~vztlE=Y0X&yk|adG7KO3{v6`y6|T(w_6{XkkoUE}oK6 z@|`qCoVlx4uWs|D$=rF~b*FWcVrC|pV2gQM4l!w$zhTM$xFu7Jl-}Wahsq7hx(7ai zn{8eSYy=wd`RH(wh3XcL*azLOM>^=WbGn`!)UNANVcxjK`a3Dj9>gf`xPd_Dt`yBq zK@)b;u~GDYkjq!!D{PEk#nYebX+yrgEu31X66SAUxGPc z!F9SdjwS0CdkeyDg`C^PRZckl(A9O)g!sW+nhdcXu0VYrcfZQfw+GbdK%Qm%(>nAP z5pqcj2YEHExjc04b;doLXx7P*_jRW-Mlh!iEsvYyj?3RWHdXMK5(a5dJtbYvVlG5u}|gPigIKMQEdfdmyiqwtGk+qPv`5zt3e!E ze_u?eLK{ftFY$K=mT|l7vM_}dUF&sk97h3_XdItcM-sV9mrG@61EQix!1zADG^?*d zZ}7T6S76F5)sl4w;o&%*#EQ`DjSJi&a;=bkYt7qSdeYpm@{iRIn#jx_9a*QUUN>@X ztr)#2EtfOYZqTx?8?#nb>sz!l#kfSsrNO4Ea^xd3*YH$`S-sidmDe^SGZ0v?MOInw zQm<)_?3f;k*V&`xQi~;Z5)X!pWe=?GDkh2kO1@cRk5=PG%8U;a`OAn*MP=9tYN|tH zjNp-Y%cAwUs@~6&R5dLg)d4ScYb>Xa;()yWPg$u2th}rb;X>cM?rWE45#7tU7~fc^ zLS$a4k}tm7moZviI&`!$SJ^6AV8IOQ{BqqiM_60o3iMtkIv{;58HRL=oR$4M$^O3_ zXbDH3GNsg0yX^+SIO8=G@m6s#JX(R1bUbLQ#TF`b?{+8P;L5gNzSg>xDR z2U95X=RKxee|*T97k{@5`Rb_DGxkm)sxA&UIM&Giqu4kMfAc?X(0D2gpU7dzxf3B$ zs+TX9?ou=_wWnZSf0#l%OCIQsxeRBmW-hrLW58|pwU?5q0}?dD;K$^G z=Ti!ZRUupS3B(me#`orEHO&CiEqy$x#_mhRm+_h3N9m-D0Cn{onH(6?65*?F$Tptv z70(RwM8-+RCiY}gbUKfs@wR+el|btj0V+do=gluxU?ytBsKK(u{DWZ;-5|v~C8LS^ zC20kv>6Z8L&Uc3n!A)HJy74;9TC=oNEYWLTZ${RLG3=Q^2CM66&Lplyo~j0ci7k#j ziI0EzgzH?v=;808FpTe zv#bt>RxdBN&YBn?`qVTPOzBF#-_4s50B48TjrqnMO5Ikyn$TKxSFDiAxU_T)SEJM* zncdM?{z5%dO2rDuroV3X7n&N1X}FT!A;{(j4H#ds=6%cZUKXy<>>I z(5N^cd6TBdQOa`6fT7d?6AP7Nn%C5_?CBlM(3sEAmrO8~PBINKSK|@6Fmu(kIq;D# z6(P{escJ9%hDC6$>c9o=)xx#cGZIdd2+>(|cx#a>sNL}L2w8N#Sy6(iN5U`+)}%W# z@2^kBQ>yEH75&DPr6?y{J!r3DMicDDoy$$05KzN8-zI`%#PLw=N{DsdcynjA=7hqx#Wm`*`L;K5%|ATVBu^T|@2gcA>6~BK4AT{eVjn1dc8X72}bH+86+!{I4WoKJ?y=HO=@xf~gjGXw=>Sy$h%EXPvR?2_%lU5J@q zZYhXYe}a6lrqsi1d8Zmp89=ojr<>^kwtZuzRhHjrBU$jxXlorM`GhwvKXTX$o!)z7 zUti7YqN!Kj%L()$x2empwNYSgkm!835o(J%k`rh6P}T`Y(&{@_|_ z+T9y+=bS?b1Usb`3-J9T{>^l?`faCz7P0-R&y1J;R$lX)y+NpAO6Cm1%k*vUg2(9W z#kSEN(WgE|K`|HSMn|d@z%|tCUtSe-cyrA_7sU&cbk?@ytwR^U;Tb?DqZg}G{ZCk9woW6vpL73J~}hS)P}dskq*Z} zfzpkz<~fXonu+r@*&q*D#No>FRB?;cb+JqR9b{mimMrZ7W9b}t3!b{33**_(G*nmT z0Yq{cgGNY)?MfG2o;(hbmRV-0BVaJm9yu~~*Z=@>J~lU93=_(JRpdSPnNg2{i**OJ z2ERqgJm{*7n^ei@j9?z5GzmdkfSyL3}-;KC@{BImL zR-rBPDAozFo+uwfsgz4I&YK3UJOr?HiZ!B{UG%`@Dm8kU{f2GoZmyUF6mmn7a55F_fd>QM)42->zpUJtRAeCh zfVa10BJV(y8dp=Lep^b_%4Lyjq5^zPl#{HU!O`d_spm;Zukpd4r$yk3DQq&aX#>U; zUlHP#5`O4X*Dy0kF6a@h?p0;Ou@~4rS<|#KT)hMh_F()w$viMMbNR;{rPL&3u$8EV z^BYaBrf~+AZAoQ9X_22yuxyD8W~d~cqD)oi6;22Yd9ztvt?pe7&E8K+H$$w@k3R}q znA}tn?SHhRK4cl#{II+NLC?4>RH34{&RSuAU#nrcXG0aT)8B#mR+pVKI9S&A5e}dt z2)}SpApk0uby8-OOY-6BtULzJZ>$FJPfOOjtwFD<4G9o4eGU_)t00;2#Yv72akR$4 zOJnO>TS5$FLlF=`j>EHF^{+KHFm7lWl@80Hw)I&*F@ps6jTa*VzAFgFD(pqH*eaJ{ zbZ=S!++ttn!$)(w*FoQU1cCFH9}7y@RvI(3smoakM@D= z9fb|pjAM%-6^aXDSUW$fy30OOzgDOpVsMTK;^*H=`@$ag0OcY!HuyEEB18bGYl}fY z)up;fg%f=Ra?{kRrF@vMvS+$XXL>zThcEgxoKYkCzs@Q7GY|-s$_kyTaffE!2_RB< z50Xi@bcPn*y8gw6FfK6qHJXAReh<>OF9uf7SobV+o1dE~bbB<+C=9-0u`PA_3wsk1 z5#Dikd{-@8;`EI9{5n6(Cd^qv^AOkuXb(&gOsKoDCbV0*vXwSbq%m3O^Lm%;wURmt zW{u!ch$8*6e?{z{djWLTK}1AYWh@I?8dI?oF_0e~iu+1zBd`{CE-N0iZkb<}r+C6fNZNn>WvneOtQmpMbjBOustY;p=LpnWcpZ7oTo0X&h?BQC_|>D_=C;Z zpuh8Xw+6=2j!dIw#A^)gN^}BHMQ-V_yn+Jaf$o)26&CK_YPX8t;x(n6zAz^{+uEt? z)d>dY-|n)V^&Yuue?Rmn&uP5N(>z0a-7K7dt;%1nG#NA4kd6O?&!ri9n3|Ui2GLr^ zEOPMR9(eP)Cl@h0%Cjl@V5rc1{el5 z>rO!!#O;((vGF7{_@Pvgp4FL=iHgK)3zbQgpU6y6q$D*BMPD9sI6ymHz%FSG`Yun? z)o+M5bF;o$GZ@R$JVc%ikeR}t7_+%* zBR@80=NRB^2Z(@JV0jGP9^y8Xhs1m%RJ+nJkQgUhXK=nXMP$u4YLreDz77=k0(+eE z+thXhl_0&>XxY-{IrA*?9plW%sR@Ww(|kEs(<*EzHuC4p&Giu-v9_ObC>N>mG*1qAU5WQy6-4&ky z$JuuWG?i_8kEkFh;sg+tW}#P+-cbZ8(wlT?(m{F+h=2@LX+c^50qF_7Lo5^l=_T|i zy+=wCNC?(va20$!BEREZ- z6q0quFoT;Y{+&i=`kdBA348d=Ow$`sY#pG56-G-Z-1i z%s0M@gKx+0S+p)DA*CcYi>HShnkj{<8Y$JIRe*D6#iX~k6iZ;cbvymxW85Dz^jcS8 z$@DbX)@r2t>kn+O7@g8Wjv4?`#FMW90EC>~XQ^nge%JU#CR{Ya)ku8JEVA0ML3Qt6 zV^!yu)*;MX7Fm%Pla1K}&EL_%g^iSB#45n`6s5t=2SOYI>78{uXl}z41DnlzE?>b-g5i4{ z%-#?zJC(;n*itfL(v>H;NDWL_JYPe&xyKqwvs-i?mv|#N<2)zP1^8k0xL@YsdGadsa)53V$bYj#I zQKJD=~>q}#^! zaBMlg_S}i3R40Uefa(utE;oaZ)lrw688@N7@x>vvp0?w0v!N zmnS^-K0UxIVle)0o|aa`cg?hYx53v`RnnHT9Xj+lG?jN_+?ex?RsT>UrAwu+ zZiQTY^>oRL#Vb?d5M_T*!HmCpOHwiAh>VE^BCWZ}lCoVXJav*$S8#&_l9dmrMczGlpZA8oSnm^Ge2x)cG7s8D=xmXWBpp0hFi)SiEy7i2EZ6-ZG^Yh z1%vSbH=XuJ?bpM>vlE1NQoGsQ*YLsu?c&wMwZJ{YCt0!UTpe6a*SCfn${oF;!K73k ziL7G=K9i`jM(|j&aj~2Eba7vj$ID6KB!W1tDFTdy7!_ku1utDZjgdAl7T*Lwfk9^j zkM(`aK5tFp5(y1X#~qc3nrjUYc`J6Yz*I&Gf~b>jhh6)TVkO$*S24kI=P9adz6c|y zSf|2%i*|q%cs`AGNOE9mx<)MP5O98mPu3HAS{G*@I?G$E_a;(vkpTe+wK&ls3=9;7 zQ*VVi%u3eOi}v_2miTF;ts*?M)KS;$nbX58EvUabXw{A?i+HQ=NfwwX*QDVTSc%1p ztpFITXtCxgA7B=<8BPV@vyCTb?tj4F(~u-g6%MM3C*B4lGaWq7;a`5|gv*tj+ZYfC zNV)w9^3CdrA#ufYFY8jJ@fOfu+o@budDnZ5$e0@hsg~aSSz7Iy0ni{2OPCz=_{$!r z-Th*{d~;(}S5!>A_^&Wn_vV-Mm^;;}i3>X(wm%3uR6X8$%dqGq#)Lzra(8n5(huUj1H@k!fNWV*|guI~J z$DQSFN$-8a4?7CCHgDl4tSpYoQrZ3hQ~>@HfNuH7DNFA~%KW21xl4Wl$*OQ#+xZ!K zE2;^=D{EEUZIzAX`Wvsl!XAQ?>b$xRnEDBpi?~&05%Km`#2hOZ3sr{H znCsFw=sflj`70XJ)O-j1;#E@6p4v^hy8z}SK#fML-b0TTAvmD8Xv{sfvoMpSmZmI` z$~uW;3u$a;8%Z+i8@niS_wA#KE>00;RC>)*R!5cy665Wj4X7YL=UVX7v1vax)&neu>Q3q>-ob^ zRh`}4rnyhqjqBB@bmtgEi|~R`w$cYzyIbZLHc_*5Toh`*rDkuUe6Xt87~+(5B8#Mi zvlFvqaL%!MIii*T^~D-53R2Vw6^92fj(jh0b`Bv9QYSMcgfP4x+kXz3x!>)-vW^*K9@B+=`e zq%^yjTtg=fGk_w=o2H@>Dt9deD@;2~Z= z$chm#3IOz^3bBERURdQM+IGeNXiY;pKz5F$TSLZYNbf%m>BA2_%ScT;Y31s@^%%k8 zs-gT75@#$m94ZQ0A3dsApMu$z?H>oKsBHvHl|r>k zi+~6*>4SmOht)vT3lms8FNMkF^zXN8#~0G4u!}&6_>LFhUsA|(w^TY^dPpJB;iKoz zfc+U(%TDSPfmZj2Bea+AaUvhqV0_46ql+y|E91?$nZW(8+_DDl39j-yxW+Mf?C5+6 z)*9kWOW<$<#Yl=TsxLzIS$IWqmL_h-it<#>%s7D$!{$7qT4(?$S7vusVZEMKp-`hj zj-wU*zw5$pBx<(?g8$T7H4mMqH~vjg{^|#~deXsnI@a{N7rv(b<#)^^o0-%jMYOGb zmOPEQz9;+mF-y&X>T~aHHqk=&S0C7dle9OAc(C^&fi__PG$bQ%QPFu{xiuDg_;67s zzkZ~uH^qVFy!mn0_kukD0%i=*N)z)261AfxfB{Hv?hF0X0RPWU(u@F8tVY~%cl&PR zanGa&KSEzwCQ2NYed3@JgA=vlmX_zDw@H$*Vsyn|l&w7>P|xl8oO3^x0~4p;rwe#=>@6(3vDWc>X?EW? zGRqB<>mJe2LZL&RqG##CzB4?)$0UvceanEf@3E)@7f7%pm!cN|kGzCaU-yE3bH14&7;ziafOOvOL3r%F4R6xfnx~ZG;r{HC^^3UfLjLk8dmNYAa9ZAzQh%k z+%xipDvCTRmUhe_Agw35!B!TSd?Rwum z`1-SF8jlauE>d_)_;_nL`Hs`JvMqf7esbC2CyxQ(GKbpdfG2PoRV#eA5#VFmiO<$c zm~^-KPKti|F^T1|h8B6%ggF~3vY>#a?jL8tzy9&SG@vy0Tga3W`5zzm<0e`Do`R+F zCSTxt{RF0GjhrmDmXZ?vON$AGEc~g@`CUWb&zm}7q{|5-aqB%lG}Znr-qNg;ODHG= z)Gxts;?z&1!r^2wOi$0x4yJ#l$DeC!Hp*@Px!922#Oa?b_TTLcBA4^$VwFl08$*R%C?=|kv%zf+cN+5D1bpy zlVp^BEGt(~i!Qlw|Guxk{^c2vwh1TvJI8)S+`5DuG?K`>l6NJ+1 zSEv6DH-ApF|9!18ULaN&3nDi7|3vYAv`asI<7`WSx4I8Mo%-$_|BH_yV<#f!h&VEl z{xv=N8M`$r0N!%3E&TSszw-a~TRF!Q5vwr|hW*5e`0304L?;R#0lcN9cK_|aBq@hh zc3JnPs+3gkU$SXu6rzprNWc92d`atXqQ;n5@zehuamx70D}HM&yaIouCtxICu>ICD zZB7wr{VHU!4{o}~d}Z^Mv03gSy?Wf5Wk~hq0`V&b#Yd3DT#4Hxm_5AHb&^@*(;m!E z_0QntuXp*_9ZJ^8_8D#oY*e9}qT+!jH#{ch`B3)B;XXhY=knj_^}uzKECX5RcSV%< zVpD8%D?+|qe%T?y;*3beHr%ewDwE0c1;$^P@;tUuG=*z*XOe50o*Nln{}@oh=Rz(q zc%5XuFpDutht`Akv}r<6#6?fp+}&V3Zqy%KMlv}kwmol9Unh3V0{@FO>VK*b<*o7p ze>``%qsUQikB}oLksrY_v{yx0(O|xP`9qO# zv8LpxqLpu^atj<0^w6%3d#wv@X4d&)>w)y=5eGXtiSJW_|Epop zgvzne#@MSyAk~zzuqI1~!))rS_(5i2Lsp&H!SRxdXBA5{j05n(h7GDTXzmUUeNCwX zjvC_RWW?Vc*JWonlv&ru@OcV-25UZc@gMcPcKs#rj8VzO{+BPxTn8lN>LqXWsQ+~v z-zjvt?)7OgT2iFCo#bZ5mqU1#k1di4ZuO>HJCoFmzN9rm{g_z`421c`^bYw+*ukK` zQ)+fzAr&NcVdw$kXnzl_$lc_jFcPWD73Ai+d_Qrs{-Z|Qoe9yE_=H`vz2cIWlWg$P zRdTc8WF*K{27hoP(wi%m?e>S^uO(6q9C{7}J6XtIHl?~zt|d5PGE|sPce?bky9za` z3{Z|ax)jx4@qGVbNv>&Kzwq&8`0AHpa^o%HbbX{p+>VF#ec7ulX1Rjt?nb`L>2iO1_A_x^qgaQJi5&V zzItxybFNW7G~wOvC&eU?@8T9A(#hqgO0+6G?j!KfYGU$WA1xnYJ#E*^iq*N?BSvTZ z-LwsD52w?{k_ixkT1Ao3`fz-1$gu1@%&>F}Z*g(tc*6OXb4%9*=BuU^ZHb5per@Ty zWxZR|EK%ozY{{BC7mo4xaTLb}lcCOFneW|g=62I4dYVOu1B#4D#J`@mr46;D3ydB^ z#f|Qky{-Ow#&&+VSax9npNl7FY%@bM(r|M_40yDGmr>Z(XgBxp;#J}DhEG)>|D2x@ zMMaC|X3{@YwZOfYvBH?)KX#iL2e?vt@r}utL%rNvi)RYRfL0Y-M9YJ+dNex5Jq7Dz zc!+1p$AD?Dvu0Wr@Xj8JfM9cAx<-?`YVZ|>W$&_$-F>6*mbwk1TgC^OTqpO=-6&() z6pkSZLZBd76WNS=3~1M?YrWcRll&&xIzxWYvf0)3T-VW~%jHOSV2%j|p|2KQlH$Hy z4*`HL=w-|HR)Tl$@QU{$zvjcfs!Y#NigEg|AAN1!o8`j|RaB$-W?4~uHI{e#LVtNr_me8yS){FFu5wc1Wh=lGE%g zla%4w;wr+gIfJrlnJn|AS=aq!Z4AB|rIMP(^0lz7%DyO9!Gdv~^$ zM|W$m<23ECN37GGZs90KcTC`U`jx}80FV}#mI7whUB?JyrDsGxbVp6`h0<OyaoKaGPhT0Ur}qVeXZ;TCy)%udlnWlQH+x|yYW_^`tHdF_~W@tjG!O;q6GC1(EI6%eM=(+Wyh@Y@090| z#dW258?rPaU2h&$oLa!HAJfx0j#Eu-N=#Qqf<&%JMRxT$*e2>aJ==RiLBD{$SZ`3Q zF(IEyM!U}K^vxqjJDQN427lRKKY$r?pAz+~JWp%!A#rk8M%`dMFTI5p&bdRIZf!x$ zpt#`prR$|$A6ujoy!3r$ES7r=K6hkh zSUNlJOr_TXokRisY(nGPiAmC4E1429AD23MZ zLlaH@>v6D6Vu6c3OKQfOC(`YGHld3rpgFSGJv>bnmkmMO;Sj7abZTqc$~CY1G)2`e zZ9qbtnm(6}yUL*$))1HPYxxXWVuN8fMcP0e3B~SHemjs!O$)SW-JuD20M%tKZ1x${ zLK5F%!SaM#{Z6pKM!ao65)7i{dARQ`tfwB$F15bZ#;lQeN*UAXv&B5JD@e20?BlGx zgrhKNP`iJiX+ixMC`W^07CF>-H2at#YMqR9nmR;`jGwKC~(G;m`0$7b+ z8g{NN{rYz3J-3Q<-5Ju7$>DQ6L2sT$Q%c0uPVJOGlC}@Py;fcuZGIw&AXc_Yi?>6z z@2Ju?!JKKi0XPBfT9hdbLVH}!r7k0{?TYBtk|{Zqmleitl#O!2YqGY6=44vNl@(By z=eHOMk7CP!cHeeUS`QkRgJuwxCZVu7FB#5_)e6+Nc#Gzm-mzh&P7mc~j z{#I;NQeDC03!X)_$t`4NZhUvj@T^gH)o0g`92=;6+dE-gs6W_2FxfKi08T~j4X`V` zvyRpjFeu1%B!w|vr6x9TSt!>xwWxdg(VTnP)S2uR!a8ic)=u~c{A4o_XE9QXfGh?s zcl3i_aXsjuZoo_!O$PX9d^k!E0S6mSEz`JE^lzV@wuT)gLAbd$JRbY=b-w93yFtbH zLoMUy)(QIVedEHhPN2bd&sfu;nL&&K|3xm~Pvzz*!m#VpMI)|?u!gUx^X$Dx#~U+b z8T9R#ciV=_Nc8m6)1l7qURw@Jz8#n2$O#KGBgr|Na0!$UOp(JF2D%=5=^Sf&*sDtp z_gpK~>NrWQ8Y!=wCUr7-+Q4Ohj9#r?9(5SqV#!p;_bwj~^lfZx^!a$%njR5^Quwz_ z#4xc-Ntr-m~Ga;}uWW3d05nnH&f{agkP~w5&HONDHlUdbt-aZng zD#ia*UCI7cyyyt~ZZxXVN--04VlWL$V<8F%AK@y%Y)Rsj3xn-uQFm}Ql5 zm|*{_O)?g-1wl;h(%))N!OIkB2d3dVTw+%wbh%p0`<`Knk(qgqrhMjU2)en z6z%K+8Q46FLjWb5cJy$Cu)`N=4?-%BD2XYcrtq!Eh{+k7%BGuB_yFZZGkyO_jL%E6 z=vK7(t4A6iI6vCR2Qy_oy*39QUi4}x;AI|tpmRanj$3#P)uW+YpUB6iL<}TXqj}oB zSXmLP6NzxpesOeUkE^yVmSJIxiyzpPR6s`@O|SO@E$TNw>JyqGK|DFgFTCL*JJ5bp z!~-sTfMIZ*o%sE^5|D<<3L1uEt=_m^Vc_Jr8O)T}%a7ELuFUVaX5>y3^?b&#+c_Q| zdF-cG{6P7$s*}iZeB)7TtHIpkei`CKOaJha?#Dh-c=C-)#WD7G{`Twt50m}Jd8eCj zq{_z2Ki)qVy=#8XQ<`CV+9`Eu$xejsc7vaPA+g5||E_HH%9R*KRD;B&0LGy!2#yW*Ka$N^pI{|ce1fa|LUrBizT~4M{~9M zb>VAw^shI&cf~AGP7TGD5xiZ@B(nR$_Hs0-DLXo&J!5rPy(B@d=e=vhF_5u74YO8v zrH+V0Hc@_N8p_v{oBP{2`@Y_C9dR9SgIdgCjvcaO%dzG;^JfXTz`Uq2`e>Inv}7$5 z|E{8|mu;X~Z8_be?`0e{szrV{e#MZ8X*k|1{qXHuy=Hlq5c^!WlV!Dl^n(Hr4y+ z*|LfJfyqFp?f_FjR{M*&tDiTLRs+OZrwt8@OeVFZQOeBD5Xh zc7OEs?hDh#wJ6rF;lZq0`NrRFPB^T=yqueP_yd&c6X~Z-_k1E{#+&coW1jX(F;`s- zilp7@08R0AWKiv*wJu*pUKPn$li&i58$Ml22R>(%FO`dX9SS!a>~9&Rc8j z+6OsdYzp-~)`7I?w#ZgY2@CJ{`c{`k{g~PIu9`yQnmjS6h1ChpzSpx6lU7J( zS#z9Bs>9UKBE9r+6QvQD%u6_`U(LM+{~II-?UtfExsG;^_O6XdACxj-E`Ml@ZS@?j zxwDblhml-<$~2Th?QQ6%xaxGY9afN@KD#Hxu2=RYE@gx=7Chfv@*&=|HxEwY|JZ*k>L$huFOaa zf4d@B4^(yG^D#sdp^bSeE~m3@o`2@0 z=X$IE)EL=qI9i6UUC}FaV(?7q11}$3>_+Q&4H7+LI~B6Ka0x6XWX)lHwA}P|Qs^xKC1zz#d7Vv&Gxg$GO>k5&-(;8eF z5}HPGYW>C5`B$gFot~!8wx}jUBo`Mg8jkVb0&n!JJhLI%``UfxS3Gndr9DpBwLYV5>1%YwUg? zV$-NmpqM2>+!lW~43m{l@tZs>4t}KwzGA|R<@X*WOY84_s!dcp5|jpex};gZf=SI) zdECkpapu}1WFuy?XiN-6MG1R{wbl8m>TVbPltlK>iNp+2zs#Ezp>3+$wZHEa*KSqT zG$mLXQ=2a-t=Kl;SZ5Po-xy5EpIBZvsCK6cj?@*hf6Q)>8&o+QTtyZCn(M3AD1y@^ z{!@(!3t4^Q!^1d)#x>z03woqF5nP?#HVmtMf~>7F0}Q{$%;XP!=W6d{65D;vEPN)0~RDl5qV}vl9Nf&6rHYyofUR z5jH?*@l)(8V*CUqX(M+-0$Z9wo4k%Tn$EX9Z7o(jKz=da?h|BGn?m2Z1|(_E8zu$i zy)=T;_^MKF3NvT+*|Z?zL9}MVRe20o9DKj={May|#9thGq(y5zFCbJkkS(_pPeuOeTx|J+CL&;fdph^%1 z(G`rFe|iGuASPa5-ygR?uUXB#Zt(#w=uk?3~~mgdDbX?fUn47x2(@0WPmFNUnB>}^r_In zm8?P?J6R^ivO2e#m-k!!cJP_|-hBL3dAlDCnX;7yalUva9xwL0`QwUPst1}}4tksl z=wkz^#8F&08`N6o=tkz=vVO1z;?{O;SSuaJx%#8qq(3P)|1^xtc0V)69ZBQ#A)Lva zui3-vR27r;p&bXO-dm)}&Y3cq``kU#o-I$rA3wV2peDu#)xB9_Qs_DNy05h`b2Yux zs-P}cKf1KAYW!+v?c!Qv6fcSC)b@I^o<-w2gnT75=W28eJF3#D9~n2Z>kZTDjCBpb zr9Re;>oIv#TVt~GU@zpucnzyb?z+EMUJb!gnLf*w$rL7j#f>wvRe<}zuT)<#;zfM<(zio}%gSntng$z!1?KntnsMG`!A^&k(q|f#-1A`j`@8)v zPZpEL8D~B&w|ck8272e&HL6YuYvp&>Ad;wcql9gR2T@)XC?8-FWV%|V4&2o>OFgK_ zxM_cHSnp;fFJ5lDeZsrX3gHx#wNF$ZBg+y5=4rl-Jhw&mjTpz-abiQ5W#LX!ipmeb z12d*jAA`zqZ;;lssi>Qt&?%eRHgA~7jFkMGSmcAb?ZW1@)7X5YD2v5Rf|UJ!e3!ZWsV;3LZ^m^~h{=t(-{b3h- z_iPlx)Sk-H*5+IomsQ$l(Eoj8o}_j;%tM~C1m5GgR2sdaazVRP4taV@znSTxXU)0_ zr6X=4b0X@I_d?rKAzkKI1+|7nS8fvz-$q-f zDBkUkIiChg3=yW#errDN8F8Uh>(!n4=N;_UkJ0iaev-(TU9#2Cs>EH9Y}@PtzAJ@P zFPY}-`;5KD{9^F~xNSfB0l|TorLXwg>#5Ks8F;tGZoy72j$87uT}?1e)Mvl_{oau;@4x6_QX;-2Lz{ke9c$FPB-KPTR=G$CqgCKdMp z2qw<$DPB*Z5@`{q>ipH*8;#PJqVr^Qo~+O9=`Q%g+V_X`A3$5$OI{&%S5n%Em&mNE|Qvc&x|VZ z(}V4RY_XIxv}y|j!5+j8HyB(L&2M`F?WoUf=|`1-2W8blMQG>2UzXF*gyG=>x-~Ji z>K*QY1L38CB{kjs6owKjXuL&CWTpB1F$+X>n%^zDO(;@Vf!btmSs1l=~6Hp zqJ#}@$-Xk58-CF803!ik-AW~NM2IuJA&&#*!1;W5Vt%c+-;IgbLU@`*zS5F&eZcV) z@zVXDT)*>UycPmIzoJv)-A~2ebZeWV$+$ZV_?wU+x`8&D1LS^jtxb=s?~xCEHROmE zH$jV^*jor0(U1}x2tM{oe)XJ9$iFYuxn|sB?99t7y5b0{CxolUsNmES2_s^TtSeRPP??1gm^qBlfoJ$;YClY?&6hHetaB&%{m|XS zHX$Lc`!8KnzCjZ6zFv2=Vp^L;CAL@Zn66C@Jqmz(LlWV6YSAHYpH*yPc&g9#$9dL`0;y|yZa?U zgpwdhco03N1oTpAnmxxkHA?0WkEk{|gH8kMfW{K2iw=Su&!G`PmvFeofm05I#FGNEPh` znh2zK4z75$`d2|}hxKt6h^vLZ=V-dNtYi|xcT640X{fxme9E;bg%lA{+d`bioZ3QV2n#^YU`zTbdbX0{othih&FN%-e-V?9ML+Ip{X7}2?3#A?TcAxY;yrL z?^DTa4i=RF3l(i$B*&p%^|d;gRc-scoHvtJm@-fm%Q}l6;@D{YL^1BNo0>}qru#Ob zMolt79#*-o_~R`#Kk*2-j2_p2KDt8=F7HC0%D6EEt4M7V0O6l4je-z~cJWGjwTXwZCDoz=CQn3_R`4B%uLygRPCCXwJjcC1! zGUI?733XW)yI6B%9c&;2wj(tHir->%Hdhpyc8}KDFP)XG2GT~Y&9W0RR2wNKWAul9 zGGtjdoT>2D%R_lx97Wv}6FHGw*FLhg&P8FYCG{q>yjM`d0dLndzhGO-hd5X1E>CmC zbBQY-w-TGef>PhBM4I&2iGRT`TpT*unmcM(<-Uc(=yulz<%HGI%%RB8;)Mp!-~0C=O^&EA56hJ1379QzW6IEO-O_mVeNF(10{x=+ZAEvgApMVrv3pJ5%a>S_Y-yY zED9bfyp+|pOlUg-6)~CZNWS#!cjW94-|>Hc;p8&Bq$WArw((K%L+{$ymiw)C#I@wF zqe#qrTN1n*T-N<{b}e*r>rB`z0~+@Y{pjj(G6U>ElRvdCL?zwKmJ?$6*$~x^*iLPh z*(l3mMaMr{tU}pen{%o+BL>}>rgE1w#h9&d@l4Qwp9syIK9o>&oi(2@kvGSjcso~c z?8)X{#f(sI^cpR6M%(~}5xSuWs+FY2azeQpqM3D{`=Zt3=4M#*@$rK%R>@va3-LsVy{*~b7 zLuN2?t~95-;~?;Lt>ThGy>22;kzYc#wsu`;^e-4;4uKFI;C`#scHz~J8U)57kAKVp z;8)t<+P2{D;H;4sKInjt57`}Tz7hdzZgNI;f#Ax2*RVesy{p~i7ZR{pje@M35ip;G z4VfRMG0srUV;3j_JepDRcS(5!E827;#rlYNn0U>e+%8R?o$T zhwbU#a*0G4odYm*nrHY6jEKMPSFA^q;nLz4EN+!F=lGX%;6RT(1>3dXb-cEW9FG=3 zggti-xIO#A{b58~0jMZp#=!>VORn}Uzz_&LP*<-RC~w@#K}@BwOe-WlnkMH$c%gZ| zcng$e*ssvBWi05uhl6un0`CJA*L}?@6 ztBDmf2p8};m%(^_MO|DLNyP!WgVnv1@35T+~KG z4t}@lOTIaE^DFPVN~p|2F6#0W-6Fa8s{OWa+cL=>eMBY^@1LlV^-xVzg{-D9 zS&5$K5lHk6V%0)5rFaH@Sq6-`F{;hm{(9Fst83lCV}hk%C8x1FqpqI$nm1OOjUF#o z(UieSOS}M48|?Q_HUg|S#~9Hb*ATgWP<+r946z4ucO#vSp!->SO>!TM$8aJvMP?Vcak{{;u+5c?CS4Ji zcufP>!6SbYK80X18xL8}&&-4ifryf_aYdNUvW=i|1kmrF61sw5JuhX&z=ClKmJu@i zO!Sbszs4;!uT`db3Z%=m*CFWqe^>BAs!8yh&3MfmoCTt=T3B_1)2*iMLSzHzIB7qm zO&t_Ob#v?bW)PK!ipTpNdiNKIHml>UMr9D=`~;@+1h6|@@pfHZca`}cuyXj4pOjGncZrFsh7VK zoIF>QwP~-aHzVi0f*{Pg7u4m1tpTifBC`wbE$D?SCk7xkt8ym`)DSsAFABi*Bf?!A zI)l5`0a=2vqs%zEd9Y?FVIFsBpUH4K>F&_-J`a9za`9~cDZJ>2FtpZl^t_2A1`OwR z=W(mC--(57e39BMMO5v%Mma?tInVJ11 z;&e2Oi`U3yB;5^TdzYW4bt}9(urcM^wP5;bb?7_iXF>i;#GOwNnbTnTNj}KHeOkmD z@=KB`Qs^oj=5tQp?9u&d4|>{(|$e9GXn9aC!~i6Mow4Z^h?Nv7@d>8T}=VV9j8eY!qNbxP-I+~)dBICM8K zi+pQgc*$8?6?`Qy%qL=GJ#eymRV2&1C72pKnaKv$-&2WHEXcWMex4|!Ic7PXMHRS( zvKU%Q;aK#f?0Ru@h;Kn=$Vh{Ukqj$Df)boPJU+DTff1bB z8)S*L%8Xjam?FI^M9(`fetv+EvM?_I1?wSQl~=m28cnjQL;kR9*K55wat1rb42>%rj37Bzf!_siy3ZKP{1Z=2|4LPjx% zT7S1;<-v=QC#%cg6?=L6UDe+bp8Qz|lw&17^W2-~Zo3H9-D--i8tz0dNjyB2A{#JT zu9v)^xzI3sYdK)rqwlUZGI&a-tgWJ9W7yS5R_TLr&<)Djh~XA6zR_$&95E;O-c@xf zM`lxfn5}X+jPiC_}3~e)zmIR6eN7EQR&g$eo$2V>8KsMjh>fY_@b#wyC zf3n&?JS;BJc|upBm8T-N|CCjp+A%ORuu6maTF{#Pu!BHMpnAGTqG*L8*?wkNqp;G2 z{Dcy-5e6cecVpDSVYU@)%D<3dp3udc{HRk{*|gkNn)s5!{D#VlK&wVNv_L0aK)5=fScGED!KpKo!czv;Q+WhTo)58Ve!_)7a~|QA{l$?< zFix*ypVJdVxb_Q?NY*`KMNJsamEOKPU=WcUmvm+B>VZ%ZGY=pZ^dP)oHbONn0jOUZ z%f!FP3b=;J!IiKEt=c2Q8i*`=76)}Kdymv=hx3wiS*K-RYR4v<_XJ~bM&^)vMn!06 z5PpphuCnOv6Y5B~HzA{Isbi2ndyDZw@Uc(*{<%s~>LJ)X&A zYhPRxPHj7OfBlJf2$`N;`LfzH{^J$TKl`0Hmht0~Et9(3(fMz*!ZRBNBTIYo7 z&6Ypa%4kaIy)pX032|CE-FA&8IOJX{Rfgc@`J=-J>e!Glg7kbwaHW&oKCN2b5ZMYl zVRJ(vJuaZKZ4=}LbyHL!{^9HE&8q7a16SgCXt!Bx!p2X{Dzx2ZTAVlM?1A=iMR6xP zPE&%bO_jL4Cc6dU>P5k-_soeZ=|XF0cmXR3?4m>*Tg_os3GeXRBblKyqzD#LTt#qF zH}fdfI;>^J+MsRCY|M3eyjDFhc%_HaIF?nu<5x$vwE z@xej$;6M8R+NlS~!Vr8pk-s-26?(SbpW_DMA3Fwfg=g%7N!*IU@C(||q(1y_->%A# zWqui65+bUKTul0eWf~vC-6IE|YZ+3#fg+1@&OvKDQqPkOWtL$7Jcj|VRw3G&dGOf0 z=%Wh{A)gl0GIm$ixX6_3NaD>0ad5VGQVVs0^65+jx=1UQkun;U8>LuCB=M82dy4WD zLzcJCu!r_RXw!|>vU#;_dmm9@)kRYu`7~!D(KoIQ$9jhd;|bGBQ|<9XQi#vBUDzEs zlYFE(Vw4yq)4OfymCrzw&O;b;obx>YDtk!nJ?XCquRsVdnBnq%wT;7&E7){#*!~X< z<6y^bOitCyKsK=t1+i>@lb!?iOeU!@K8#JDr~9b`5whpVS_`RHx;gx@eW+dZKtc9a zK37oJomD&`c3$*FllhJ1z>o|($;gM?y{n}i!r0<`5g|c7_NoSNyZW|4 z_}?1iQj44UB=^1^o2yvwC<`mPPF9KAHf3bzN3d_VKByEo2PoYqL;(h_8=zKuRe#?y z(nA{Vtvsc5zzMY4f)H>DD>F!X^H1ulWyy;i_U42aSRd2`u zt=P@k%QVk zI(K?vyt}B)JmQn%TQV7`h$IuQyKx42RIdYX2#k15Kc0X=p31(PpX~j|J%%*OUsT0k zvgzBcEJK`GU#PCwLXtsvaV2@?d(MfU@;^iR@XtJIS&>IRJ?|P~&?XOVEK5C5uY|Xe zg63I7AXm6^B6~dMx8s}rqbJ7#=&K6(Au`LVhbs5bnKF-?bM6;}1uf|g8?a=!zpm2C zHUNKWV8`8GM&rb#(tC85p-Cr}XPtA8Xn2c#C`>?2d+>e?IcQzw4C~&}I_lOitI*wd zoXg!W6dQ+zvv?VGWKoWH(mU+PTVTI~@O964bC7#MQZjLMU8-Y?%Bu|p_jAMgeBQJ@ zA}3DW3$|VTxQiFWR^Km@U4d$pm|C~CaHb99B4quwh3;;vz>{^ zSgrepoi$6b?|?ZO=yHa-$N3;;Z_>ubvUI4f5a5dUmSI!vJPk+mYb_!~@J^4f6Dxd5 zSuBf8CS|j3F2`dyYcY^UO}hu#@~IPZTaoF^J-;iVH-5;6`~&L#FTdbmKSg|D;aapK zVr}Rndv1vVvocfDip(FkYKzPuM>z9DelUaz#l?p}v1g>R*{hSlcT`NN;QrvOdB}csL%O^2(MngHQ&a!w z@TN(YsOGQ*G_fqHF-XU|5lD*-naUFW5c52!$>lBZ)H}T)#2?>APT`#I zac5T4YoWwnNnKr!eP@{kwgOMy5w-_K_?*3@ras6*TQJ`DVtv&0EFE$ytoH_a&77V) zkU)I5Jk(6iMI8NL>mc+Km}D~hrQ&8zL91Umo`xuzzf}47*bk3NNd#i^9VYv2JMU?v zqkKc9z(aaI59_L)5o%K^FscGGT_UUMbuat-=%k&2M+;K^J*MWLffy~{(?32VuAEoC zzaT}WY)z>13rAgkYkmO;t)p^2i~q=%A4+QY#CJRmi$LSKm!OFxiPgRH zS#h|4$w_IivL{Uvk+8nXy8i9ev_pN))h>C#^iRrhuj+5jbj=Y+yVt1#@iXi8>+s7^ zd(H1!xC@_#1os?bg6TLE#TC`xEazU=1d&J|FsG`_eP}rME5xE;8MrwwvnGi38`4J) zZz14;2!1gp}J& zWm39#w@4?p`Y%I^zTco+HU~-LB=^~ngx`UjzqQL*x*uWwKggNpJrcxr10;S3%TMlF z8D6(;G4wRPPHPRW+WghtoDb74<%~b>L1^DJ-RYdIJm%_mo6rR^amH-E^p&OSZvK?3 zP-(II*yA?a=Jb5tH6T#_ac5lj-l1JTN4@rap*)vx>W`U4?k{F>^?|lspMQa=mp-XI zZuHDz%Za?c{^)6Fn>D#kMzD1}UeyGyuM4pMuQVv|0 z6jiD1PZ;pnTR2`ubO=ML{1Ti3IRxbIC93TO?y^*J`ZH3q#w8A%p)ObvTUIf-nKQMu z)2hqoBU)ces0w$y=JfC7I2-ot;M1;isiS02iBZ0gvW5Isr7Mm1I8Ug((sleu?+#sm z_&;U&2QvRp+mxnU$h^&l0gLA%jn9M<7r%<#&4|{Z8x{F&wV)bMkOH+Z6f-*8l)Q~g&`aik=CKRYle{SP#WoOq`Mh9zJrRdqVIFP zzwaNenPKLfz3)}`T6^zx%&fbU8@3)|NvSm?88SDM;Nf>zkI*-lE0k<<+4+hg_#(3> zE_sN_+q~bh{P={0*Y+>YXo~LY{n1n=mj7j74h*Ys?-&n47U;tcan9+B9yyeEZ~F3s zw?>0w{p_~YJ63J5qz>v6XA$(pcO?)V6x^)4RO&@ywnu#Gn62dma{L107E!hp#x6Ol z;pfc$6rw;47EsZ6Q4oRkKU6d(@5dk=7HMNWNOLIS@a7szvNw_eBC;&q+n$%Ho?k%~ zi^X82w&Wb137YW06B9SY^Um;uOW2EXnPk^@;*IUVNY@}UdTBWy#=(BvV&l3VGV+`4 z)2$RO{(VmbA<%}TO%5;9Y2m*-i8*HRyr3Sx5Y@K_m;Xog{`yM58N<5!s!uGQKcy(> zQgF%#C5pA!YA9Z&_-3x5pjT}z9#lMyI?K1((!-BZbj^gGefJyxsDLB=c7IOGOMg;#S$1@alz`@R z-MfE+{l|WN5xh16iDlZ=BGz`1_%=Pah_4wFG(tp2Yn-I*bV&mZP<$Kh%zDMIe(B?tAh`y>1kN(!jJtpEj`VK3zkd3+N&olH1jf!kk)0`WOxWda2PU44Xh5bXSCn#XDz^)=^7qG10^7J4Giy*vPq!@aE+YiD(p zNPZTqpv3ED_v1A8+5i7x=mJzQ*`zCre>C4Z*+T+WaIM`lewGo?Ess6Xm~wh80SDOd z12vmMf?Ny9QIjQ&!M^UGEbYs1-Ge+3VdjDM(#R#%eEZe@X*6oR*&6fz0?oJl>~`Wm zbSVi5&oIvde<&6si%j|GAbxC*`#kZ-on@WTTgyxU=9xc?YUU4-&DxMrSamqSnZbilVIVBCJ5^{)4W z_4`W*hZixU_opfep-lg{|Eu1<#UL2LIK}Ez>H!`C<&dY(Pyx*LIu9@ZxWjBOdTaSn zJ-_K-lx96hxXSH`^@B+Q7WT7X{Bp5?I6(4=FCm4eRiK}q#^L_^(G$@px);EzPG<-^ zz`E{8K%gkoFxsq5aXPVY(Es*_R`q0ieED(HnU)kWO&IHWp4iCbfPRwKb3c?vpyI~; zNm|;psE*80Vl|%sK3Ymjz$=4>uRQpp=j8~_q`}Fs<_;sZes&1DErWhuRm81Ze%;!g z=i1wgIGYaNVBH{n_UQiu)*JPD!0j=K9E9 z^k<(qY>M|A-#(1t{`B$BiT_lAtAw{7yA`0H>dk4WB)I+F_yp9{SK+PKe@H6E^t|Qw zk#3n)R{z^bVX_TsSZ9*?fYi!$x~9p#uDG>c=1A&JS|?cYqC8&_*LuMloZ$wC$!;7W zG3Q>jpf<_#uyGuWCht!(VK-~|nGC*HlVZqmW@%~hAsVzSAz{up(xTua?q&ZnvG=%u z7OuzE6aDrb0c_Y*aHzgcxR-WG`t?Xd(U>RbP!+pSqjsI9FIz`hEIlDNtZZkwOKyHy zR7{Zg%6c*3#{1F0b?%MvPcscGQl}?5bs1uiDUxB&D|dtRd?akHB9jx~v487XcIt80 zX|C4KXnCJ8#gB@&gCewR{N;PtbKC@?v2PI;#_bNpsZ=#+bEONi(?Spe@!LQ7yB;bv@5S;KI;Jtq=0C}2a1})d*P?TAdkG=e> zXFj6|*mr;ca7W=L{;%Jm)p+HOq&s4e?f9qIv+*%39Ee#7D{?iOLm~u>|NPL#qKgS? zCzfFOEP?T?LjEGfP3hZRB~_e%Hu2jFf0Yr(+~ZtUq0Zj>y+4lt2-}*PN=*4MmBK>y zyt+aknRX;mXoYE@vcWr+rDzOk4eMjQr2fol#x{c)XKB5x#7&L zuin25Ge(xNK6r!tAtiDg`d-Xqv(CJE|876(&xyfq0`=J8^39A*EB82Rr*mGT-u1J* z+@8~8;-qS?I6t|aa8O?Qa@C$f0<}WY&>b9JnL(H~KzB8_BIvMmajo_d`O8H!Dz?l6 zv;4Y&T8&k;jY@vq!D#1SkoltyfL7>iz! z5NE1YfkA(xwhffWO@K!GAXpSoyWvoQW1goF*!;7y>pi z%uMEDhsK@C#ayR;4}tdcrYy>E1uDYB(uk{Q7>?&POv#QyvzAC4?WXhEkG?AJPpPhR zQM~?L$-e^=kaiN}p(ce7|0@)7*T14I$d%Hm`QHPqI^qUTZ|b_IKq{7+f-zw=>b7hH z`j~GIotcgbj?wK2P5rRsH#qiJYJa-gQ%C`QzfXD+`ernqpS0fn>0@P$&=nJiy&(u9W#>H=L`+^02(tob-^VK8;U|iB_)_=SCAK``v zgG+J{?56T9x=iqXrvGK6KfVl8!vOq)h*$ZqS~S%62SS929VUDLM_QdwL} zg8iK1&)@u?M(Bv~nKT_T4gD6sqqi|x&W2$?Z7qS~FeM?w-mmEWdE4(pl@&$Lc|g8q z@BEz;IzS&1{3-R@)PRsI$K!slGya^w$(58ZDk*Qq;+#4m*}Lew-#nA}WRm60KcNyJ z2WSManji6RoZu9EnB2Ac(QgRSZU-!!6&f(%Jwd>D%VlRG{=Ctj^A30~z^8bUW8f9X zs+~rsU*y(?d6NI2aV7sRYceqb7!DQRn19FNq{``l|KrDDCPm%czwKL6T zP(U)^8>Z|vLw|1^FI$ctq?d-+d$X(jDeMhbVBsr*C)v!Oj9tjo4%H3MR$5{{pWIDW z>M%5j$);aAvRghoFpm6BfMiF|Ve3(3Isa3|3uX$q2Lhcz4&A{~A|R6+@r*iyF7Q^s z9!NCgMz(|d|K|99QG#z$dhZWJ$Ggb5TzxBmuIR@9wGPXSAsEOcrT=e-FQ#aY!niKmH_{#qKz}`cETw70Wqmjg0 zFVf#xOHl^2Ql;^yWyWL!-Y7L~zuEMxPOK7q9E61RBtMYsBi!BTf5Y|Dt^u7ze_aO%;WE$H^E!z5?8InrWmgJTE7I6rCBPe1sW9# zU|rPxmXC;kWq*Nn_SS%cfUq#c=Zy>hEm;+~?e?nsVFzo}YHn)(+F}G;zO>3s6tJ`k zHi_EZSUWpAm<&YM`1om;`72BRw>@C7RN}pTj?|;KH+BM>I+4P4y4@EDw6rWg=3H$mH%O35c zF4$+l$=L)!74bfJjp%!}470or)U3SCM&I)Q+!Vwf`?sz{0#R4u&f0_QKQP1v`LgbY zBx8ReDXq3zs)x^Wv(WG86gU~hX+C&Jx-w$Ir4eqt7)V|#tx+~?=L0T~-gXn_3pM(T zLk9Hau&n}m1R@G}JBcLlF7YJlY@uoq>9#k$$(IbQ{Q4rv*h>e$ zSOnsz#WlJkCa6lIh{)$@z_{SOT%J3dZJ)G}K3KBQB8pN>CPEqo$g@QPHI|a(fvdNQ zUZ+OAbj9)+?!^e};ZR(vRH`ssu*6sA8?`Rqj!`<0&!4d&k6sXcUY(z0eDI7aw;gT0 zoS1IgJY}foRo1=8T61O7yIz_sksIU>BP~V=kTYE6otmR8u|Bx0vF-j+hGmgH-4E5K zSsrci_3+uUnl6I9Bwx0t_3bBEjT|bsA|1k6v?n-c=0i3Jbuu+5X7&-O!%I9bhDMZP ziSgC%ZS0O;-c{ZWFFcfxCbqpY_TwQbp7<%D*0ER_EF*R!?TmzBTM!H8cDfZ3{;KlKIeI zXU4BqbAMg?k(~$*eXgB%1ZimhO{Q2{%<;BL8>7Xp3Z$H25}S&*wT!7E4UBDh>M-UnP+v~M!ud3iE;alved_UnW=Q@V zKgSZ|(MZ?g^L(D?g+_MA`N|ays;-`3?rK)D{Wn;RoMnj{W$se@O-RY+7JjC@k|X+O zb~7o%O<~+uBn`hP0A>zNvP~wZvNnl6OojuGA}1CLefv6@O3iM@5k$2{k%Do-+;ARu zPeQPwzOI$H#3?bAdsS4vtp6I;euQT^A-WB>cwC)tl5NLslFruUB%3`MAFKi{b>{L5 zZ%vKolhkM$Xf!vLOwJ> z_r-Z4=PlYDLPM3kP^918tV*$%{G(52;zh%|R?Y0m)KU=tXjL+9x-E;37I$aHAv$%| zrH<)XKKjCcwpbIh6?1tdktQlf;UT&aZ5}6tMOOD`N}eor6yjF0f*<}Sf= zv@fTiZ3re%1s%y#K!@|7{|K-bc~s{D3$xE|zLRAD;-Be5K1a_PQi&U8kV zw?}TcZ&ATZ#@09N2jZrBfeECk#A)wSquw8A#b-D_U9U0FyE$sIHAhJFsFpR7i&_u3 z<}Ig-x%%!ol2sqJzrgA8Muy8#ZG}RToisV}RLMJ}x?qlwV%{*Ri8Yo zfy2cgb&-hkh=PNlw09lvABw0qn>Lq;*;1k#WHl=7R2k_vFYSM6v}ia^QZ%#GHSMl= zgD-o6*4~pJb%(uK;ztbV8nCK!6>Qx}3ldbm*cxW8iq(T|TM(z?9d`^ z{$WhF02X}$1qj=HhDW-&C%JK+X2Oc^K_{AG^X<;(O*&MpmV*fy zq!z0|2CUjRcf%aIm;dMp4UuR zC0BS@+iaQS(lQ|xr4+-rk;Hi`aiDe8E8 z5E-6N)}_``f~p8{R1h`Q(tHRvTWb0c$>->1_OV^uuy+)PN_j$(Peur1+vO!=oFfxb zdrgmdJJ%X7gO;R)wCeSfjw-`__*mx|J}{0jFXd!%d}y5^f;K0wj|>bMGK&LIDny{u z0c-fR5c`R%QycLjq{=O!v)MnWwd)S32jbb`6u)>}yF{(VVJdwf-os*JNv+u%N&3m; zpGH|qsUmenZt}XPT~I2#J8E?UPO$Qs zftTNnGJ2vb!OEt!u73K7a&sF7a#E$Fe3(C6X!f-L4R$qIM!v*aBe1IC-40f$RnAf7 zb>k4C!2ynlaKr~NzBf6`)MJKDR(iL&n+|@HA?)r&C7UYAC0%=>QTaGJes+^(z9wNl zXc1Iq(~CpWT{xNIAuE}E1=f|(Y^df4J0e^%o+|lCwrfFd>n_sBY1Z1v*TF8-x8TTZ ze(SI-t?LeMTa6c0AqL>u_g4Vi1arN61W;*A%tuGG1$X#Q&+G=K9&C^A`3A!CIQKczt+jO zFt0T3zYlAgUEhThSydRh%=smak=Qi~B|{5+nUnN`qeHehU6{5Saq$Q8@QCsgSLXRi zGl#WRfF7w16nuAp{X9`jl94iORmkF23|C}} zfO_$$^_#oiD_~*cb{rM12r!v`_C?b1``Y7u9W*3@$=-Hrpp~S7yR*RBdG)27t0z^- zRsx2YIU{nJ)C9rKC#x7p!Vs-a>|-`~!e;=4nIkurIRKJGdmQD`=0D~G=IDB(yVzz3 z+EceM&|$k2O!4&-oMjMoJl@_Kv4s$!Dm4u9pbxvrt!cemn$`j?YrdJa7STewr)-3} z%Q*^>S~8z$F!-?KLx5mvXY-O0=C6|ww3{J0hWm-aU-`#jh?$xoYRUBlNl0&l%5;3d z{0fSmI8V%98Z=R4@38Hvensot-BQnT@O%a=NkilG2Q9+e-tLD3)X(xk79oDgk_7&T zR^!NQ?KtP84&fiifPj501c41bk-Im&pZ=y$Kio#698d?yE=`yG{ZU-^GXzjQ+1hSA z?^hn|{WCD{vKYGlm`4gv$!39~)4GEm|JFc`JKFHf!blKiH?g1dBO}V|!=rrr7drQ5 z?eyrF`eXL{wiu;mxa~;P*=p1g)WDYasmn{MW&LAV9t;CFoL8RN&n{aH0};jYm_x4x zcekb6T=QAa(2MsJsZ9Za3U4VlQvnxj0SGGNjp$ND8-&hGM%{2Ax`HQl*hqGxyI{#| z9=C+|&RjLz;y43{YDrTg_aZZfGrdb_4tK4!vS=Z zoRmyPgL8%r^HhqR_m$_&73f3jqAUZA{1Ya&LLpN@@95~VyQGUN6;^i6;ir&JUK%|# z?!Ll$2ejVZhB3uvNA1$Gu-x=-X9y<20zLZ4XOo*YZnTxgJ6tMt3obID5gktXzTl*# zeob2lwCrj3?FJfLP+Vm~LhOKMsVH2&{N9og{IYJQGhJ?h0?1W7NsetT9Hf3wD?YD4 zbvbf_>`7?ExFl2YxKk(7zKnFi@BF`DjO zeQdH zpx!FLL&|j~3>F_|N*LR52tyuNLv#jXU$}=P$?GReN`trrh^+%lxj;ikJkD>^vYCf6 zB`j4xDk?UXsNy+4SeKGs1G+A48b>wH*so;$@p!tQDaTcT+jb}AMvjTIJ8pwtf%!jU z@FLavY!ws>>+gC9)B-$wlq!3ZwD%qF(53G1xA({&#$MifLL4KNk%;mjr^?%Bd&3`L zewXeB#Ue2GC(|K?TyINWvuWjG`9v+6Gq(p(CRrVw!!nyvGlK4YPD>)xV5Liq27{x! zs761LLIpZX;>~J{;5ygW*!O#uz~rE~u5{3T>1cTdsmigPuaW(%b?4SWrevL}UD)bp zYAJV_4p4{I+S_*&qZ9reGb9DOEd@uf_(R{uQ#`h>I@EPp zJJKZkG^TnI>zy)b)O8FuHIhUXZ-XFdXgU;gd)M&Q3_7icnU`^{+%z!O4g8=c4p@pd zvLb;yNoSb>s|1<~`pbSfmAI?uc~KT6pUdTgkT*>FCo~zwH~`CAc_c_ek_iFi&BHy>~3hahAv_0Wx~MxGytB;T@(D%wN1JIQE`D z^Yh}l)cbt$Fv+-SGU3iZR8e&!wKJeRc!8Xk%rOM3n!`kks&Cu}lTQ zpgNaEuscp&e(Py3fcrTrPE}3{$HTt#PTWYZf?mg{AsI5_&7jJUSfEi)GqEbFg9b^9 z-dt3Don>o2CvU9G^Hr+ak_Y0G{xd$JGM>&X_Rk|3;JD39k131k2cttL<=cPcTtzql z!=MQgKwy!L$mOlUvWW*^>R6wFxHyJO=7q}Zb(~&O`RJhAu#gqFv&r&-AdLxT!_L7r z7-@ulj{OOj!BPg0)#hZdQ*}h;%8<-&?fR^#vQgfRCK%8>iuZ8bgJ(fpXhR9n@PX6@ zH3iFmL>!zV<%hlK>D!z6r+v91lHK?_Fe9rE8$*|hG)IokR3hDA(7Gi1N zoNw?>NB*;Y%2}88OQ~7l*$EL#V~qcOSF0YQ7MrhRn2z<7JIHK~_^xJYsJUIO6IdrH zVXYJaiE=M6kEMyUjz$84Gt$C&cpTsR2)@pL3O!%ZD15W(3tla;>9%9la^#) zmnJ@Mpxl7VI$r>XdJ$XGy)Y2yuBg7uaR4XCZs85x9 zM1fDl&NNom;h(ki#N8u9t)pSq4vqh2wKs$%MS~3uP_6uMeYW(P|DCr70^gwgM!5i$UbKi{_un zj~HXs>0r-P!{(c>$QZA@T)NS@{E_#r!B3?KQa4n$tu~Y8hOcTAK9=$)08)aXWMMiQzqJn5}#sUjz`!02OLVMdtd$7Gn4+!`1c=YjJGf(iE z%d+0YzP)!u*T6Ud_l-w`IWuZCT*7l9-Q1^P`XfF1&VufrtlT$F@G;~3Rw0gQ#X!I8 zj73X60~{7dBX8<;pH>{rM)H*6x9o)h1-tj|LrI~CtbJ>8{1nZ)H|$)uw`A4r;!T0s zS(JjVj#OSptN;O+ESE%tGcX)}a;C5d-MiuHw^p@Au21u`&vB zuzdTd@&(1U+LU%hHz&I_nEe8a#H?#D*U5Zzm!IH%9~!Na06$Ac(ZtujM*9(hM+Zu- z3#PQkZ!L_C$FVYsi$!Njf%X_IAN-;}2svz97nq4P&kt^U`Au1!hld~==O2T+}Yg-cMoE{R<`#%Q<8Xt!{1*@F4)V7iG`AsEnB%pDS@?h>@o z=1do(Aj5OV=+<$!d=eFkZsr2CJFZlwb)Dx`NGm=@XL<}Cf~9*f$|$Vmw(-g_eYmLeoP(NyAb=sim zw4UaXXS=hT6FKufx7#;3QzkNE{1y#}c8T$zCDo=v1=SE8Yh%z|tGTp1VG3l;fL701 zBaAf605%^9+D{*NeBwvAX7~f%N5Byl&AtZIOW2GKpK@DEnNxXFAH(a|uxhUqP%a`y z7B4D`P}x(KAzoloZ+42;4OIXn;sw{B*%v9*Fx{KU{9z0WO(7w$w8QQ-v4 zVAWgUZDJJ-MH@bYcms`P+JstDxJX2~mZ_r8@qQaOaVS7Fq_|9E@hZJbbaBQ3zHL9%j}$yQ~jxm?sW* z7AjA-H%?>7Z9k&=s{A;~YQzcNCz$=1Lj{*34he^hc{uOJ@+9mk;EJh1Lo5WPz^{!S z&3{V6laGT-5~%Mbc{s8UzTvkYdBg88fPrZNtY}8(Z&8^3Siuwcbl#dRum@9yJsmgt znIOqBkN|VFy{TQ+aTVfd-5Ubv;6o{DJ_E&LD~sKN!>KesFL6tw$qh*MJUI&F`_cFS z!sbzLfa4Q&T`K=_MvlmOEGJcpaB#B(Syd+<5&I_$9=e>q_K%M2VRLh#<;1%?oH{4O z!{0f>GLsOWn73akD6mZo-|pb(3TSpxVRFNzZgD|!cm%@DPoyRvD;r zM|?r?{U4-3o}te^lk=G01wu~~8^ApL_mmf?$mwL{s8I2z`!TU^UtM}QwoT^g=4;ND zW+E8B4JY8E`m%5xG(Z%+l~7!)Qq}Gk(p|9p3@E-x5BB!oR4ekq>Uq2Ff zSi~@R1GL=JItaTHF76$?Yo;%_Aj1rMhVPHZnN8X;myATSZEiMNbGdwFl-9`IIVy9D zBAmizoppy2|NL81cpF_CD{X8>;A}9?Hy0VCZ>QPm^NH_w|N5J|t*39>NOocNlll>O zPCYRU`|7&~GDmNxDqpp8HuLnpxv}zfY@2*Rnz-F*i6xU6;5bHt_wqrrDkb~pMInzo z9@sO)@`5R|fr5zRT9PBq_JBp)6`#<%$g)d8Zb}1E-GxGUS9}0t{=C8;)+U;0fDSC& zP>~YdDl(H(`R4{XlIe@wprsb{^j8gat9Jcjglb9@vA2{iM)@5+e?Jif*qr_@FS^`j*Ap)~dcjZ8{udoyX{`KMSEx4SG7>u|8ke!7s7 z6^D*a`|?BFCeBc{lfjX}1-t(Ke(1{b(iBH@tNsj+RZA$#XuqGSR+JNVwU>l0%|lr? z>ouPv1QEB)4I!t3Ji3HgCs9xX0S=DLvPYOIB&9SwmpY34{^oGqGCtoPMsRM=-3=#ae!6pRhaglk_fbyE2Uj-#$a?&a`TVO+W6sn@p+L{4%F3 zZ+DRK^Wvd%wju%9_beLUE^5Br>i3{&cxMP(Hys)QIqqb}k^SGKbXhI^4?oZQaSfTo;@I)#=3AI!XQx0Olew#Ecj zo^d54sl3Wj-WBsm$-C6^Zn{HMX+smkmyNq_;VPjVoCVKYJK}eHk#3k!DkcfZM@-!Y zul{XI;ef`g=yg-5PRs8=TU(&X^X_Dl?X(UrR{MN?%-?*Ot<>TT#iOG7^2@E`oNx^e z&2fw4cKq#B+FPGqyNr2oTz^8p7c5@+f;wl%j$7)W#HE>qB5RdQ##?~pyh6v*qnXC2 z)Wy{WiLi&PC>1QybqUn%c^f(BjxxDjpCY*z)0cX0xzX+m};G@6tpmAs81@S3L916$9 zbV}-ErBo%aAUzSJ7jfopl?cj2nwp{S3!CJIo0?fk?0GugtXgzMYpd{)oZ6OVrr(t4 zH%8_F|A#=qp<{Sj<+r#8W5;_}v2Tavg0jfyf4%Q4Gj4Gdjfl%`Cup_6*eX)nY>eEf zO7HM3<5EB5+M1jHRL+#KyZFX!GCR3?=wl@TW7o&K3zT^W%51F-{Z+6R$q3A$G}FNm zP`bTP)4ZP=6EOq+6X*2+B6(XoDQ$?52VO}b7;8NGx$lguc02E~k~Av1 z3DEgkZ4T}+&DU9w-wkBNtt9C$&4%GZs+?j@Y&g zXB zxluG=Tg9<>wVl>I)m^sdrSH^*gNatf=rI#Zp?)VRUaiK{1A+|KWxH9{kzsuebt=j( zDAJsrHywh=cNH$GFpR)(~cE^A_OD@&^P#^G5Dy3kZ?6Gej!uH3M0#0nVTH@IK&4a9VstBVX_ z2zHf+yC2zF<=551FuMIhF}9CmXUC2%`({9AiBJY>ix>O~wqb2!6(8CHqPdgvkR{zC z?2iv$Hr6ebU1~Y|{CU7D7h+gLy#6SpHedTFhvzis!h3H?Ga;jNR-K6lw{Z?nmOzoV z{3vc#?~rFgg++_MK6vM=ZEHaJC#b~iS0wM6AZ#+3r6z4ucJ@7f4*V&I-ah5i#4W=l zfo$nG#^WxQzFBS+r;L#gUv+JoHWvEB-!XQ&5_i*ReeUsOD>G-Q{CJyBgjUpzjU_{5W~6d>>3vjsoDL_ zBcWjwM+hM1IYG{Q5j%FgEqS(O+ZR(ySIwAAdY%|BamVde%|l{w<#1b3wkVS?qML!y z9xhE-2)sJkUL#k`V?lT{PpUI)?;K>PQx>dCw9EEpGhH#?8+TuNZ;?%qiO0M4U z0Nvx_ltZu4+8B?}jFu9MS>4pq10B!jqQQBo{V^T+-7$GG)Abd~0Ud^jzdR72p3v~n zC9t0V+?igHFtU4BLtn%u^pt6?I-oSFC_WAvjgvb`mQW=>@_JfPN*apdZ7pW4VjRO? zc`8%8%Uid8&cn@a#yxwf)>n@^tJ4&1->SPUGk^4jywp}~kPbI~7%NHCl*{(eujg}020D+O0yGd@^Qg?Sp_zk4<;5G?qTP!pp8eR6|d9o0yhA9@DTMW0dU9NGiFY3AB=4-R&Ez*Ee#EEvp51;GI zzBgH~ipnpV69S`6t6Qj1QYI*tFM8k`@}gM{VyBi^cv1xOcBX;+!8wV`pWtQ@uX*oy6%Lb;>PIhesWf zPmF687uTA6nX30=B&L;?p;{|HdR=RLU|&&1{%Oed%VE`dl|ePcRyiO!i>7 z)v8pslBm=#Aeu#DQP}f9cIM#bhsXP&WXp_G z$vi1=1&1l=rGWxuzgv>poM-#@D&uXst7+GpHG|9sxVJ`?yZBKyC_9yv{>w9Z-A;ED zhi5XJmxfn;_!Mawoi)85;}7n4Md1D}qZ6aK52}R7S#X^#7s)0Ftou_}JI@`~dRX)+ zptOc%+XbD#1YXUwccxfCVBVEQLQeJkaBzue{*lI)e2M-ny3($LL>C$%H7ER)QQ9pe z(sZu%STz@%l`-9eh?}9G)TItHEum4Em3^AEId4xprW^Inp{;}CK(>9~`$oFyt36ZH z)97SlrUGY#23XrXRc^^gq&al3!80v*)+!ie+5&DdfUM*o_iRv1Etyt>xOFNhk}e_I z6$yt(2U(G(p;-9H>f@*E)3t6QC@U%H@4gyF_TcLA3Tm9(Dh>!{4it+k7=eUkbaTz* zsh%b>5)REIwC%Q$h+;V;t*1AhP9>T4r6D0osYM!FSI@{kctt;Me67a5jdP*;;9!Mj zp-ifqgT$_pA80bwDq^K=IewfX^Nqq(06r=k%QA6!b3IQD!M zwwRTl(aXNTULTeW=Y{Ts$lGIj~4~?)5Sq zhu|lye=bn4Jh*A!AX1D$eJm^}cF^LFF%kCa8%Y!JyU~WNFfMMP66ZXrEj9Ck$yItv zvqVlMcI9EDL2TM-w)IO=-SQ(S8^%uX5Xt}HjYHO*HUxTu19d|xc%~_| zNjgE?<>>8(_;hvuj}84W-)kR_O*33(iGYNg1V}+16kd!!Ue>MixOiQn=8;Ky_!T}AQa(BDGFw<3kW4BqOIkHm}$Ir5JUl)u`M!ubyb z2>8GYJifyhsF>*o>36jg{qZ6Xoj}f;hRCMgIFLf(hdt`n7l==WN6ftnwx7kcS}h8- z-2hg#;z>p}D?Z-#WzWN+3qp~ntr&ZU!B<3I?f#Sje=?qk?M74LZk0B5UteZz{^qn_ zcNwo@plqsHVYs+#hCE2Tr3ib-rNq+fkTzF2%M3BDu_EaH4%qi@K==`V(Q&@Q17R#* z(8r2u-FU01HzhK-WjW9+1=$t@gQP?Twj2gt*dp1Mr6g8z_L?9O9Ab{#u{R5J=o51mdxl491wqN7xnTn2pKl&VMMPkVepTc?-jK?!8-tPE`s*bhmPM&d; zaMW>r2%=cbpueeEf?|EKfWmQzKhpDFO~w~-YKo&4O-lCdV1SCaD4^gZYC`9r5xx3A z8aFOG>?BWdl1Gt4;R~fTtZ;Bnq)C2Uc-W#jBs8^TMT>^t>Ma7Et?7TbbVc9VOBh(7 zX>1Mc*3Aozqe{R>znGi)is>c!njbgVG2Rc{U>h%tgDZ|FwD~NHuF;c`9=2xSp-Fj@_}tdjbjSBry!yKS#K)-!Z&WPBNSU&FFJ^yMbH5R| zAmTj6R{4;!6;*)J^JFC(&36{P6W!4@5U;FG=a%Ri{jy??@Ea1^W5aN@tz3-l!>D)W zj{`X)S<2V+V&A1F;f53j)N8ek19HL_4i$((VleT9X`Gf52LDcfTM zr90FRd@-9ntOaRx6etKbOfE&MqLkaZP~jhZTAcFv=~4q3k*&i{B4E}|1y^yQ2AZvC zllGYhV3+BryD*QLnFp_({;a6P$e*0%qA@?`_3LuiuOlS?O?WsaPzM!uTBvsei5roEM|G^3G*_D3Cp0_vkqBV@^Aw@$wx;Eo+K$S03Wb^OWRNN@hJCdzAO z>-86hsfX>fZ&NTlVw6%-_7b=Av=Nw`&1T4aYN3z@c#oNRIy}9iSDbiaKEks;pI@<9 zP>L)-{$Wz=)(hc)MRQbV~v?!BxFrqBWqo0`5G_gi{ zGK-Fg=rxrYhaB!J6TL%H5_!sn1jW)A1l{5j3Pe$nQ-wcAw!;;`4@bSSXadP);Bm6L zJX(2DCB-27K+jzo;&G);;xL|?AT3E-ghN7ino{mX5ci#@ovbw6M^{qIgeVJBk3|Vb ztZDe?G@~Z3a~DyrH|=H|m#16;wO%iO<}UvHc!f}X?n}i&mi8r1fq8OHjSiF1ttQC* zHbryp7Vy_%Q8Jp?d(H*%5-?tQ%Jb|&F>+c@f-miEmuf+4STs{+Q6z_ToYAPlJr-uJ zeb2Z$N=W(LwO;3RxbF0I-nYJ|68evC7Z3wFrFA@Ak0AWr&|?fP&Z5m79Asef)K;*zWMG{@uSygU5C%kVY8+91jNE>-q{L5iIhylC_qb;}YEQH*-R zEc+@5@lTj7$WZ);K!(nxRLMWKfT!qjTjgGN9I{wUs*w@Z-4@Z<_akqm# ztu*1qn11Hj#R{q=Y`HDjMimybvg}cNiy~`&d{l$W*4&g-h#M*{&(~WM7VAl{Noi?s z46Q)n8Arl+D@W<=M-7O6Jm?_@Tw_Be{_xRh0qPgftf_znYp%{=Erwq{{1!BGU0;nr zb?KTuU;pDjk*`K9?7*<`E!PH`IVlOSb^`-Kr3)Q=(MFN z{Fo*IPh30msF9$RHIDI6*zsPnN4v|mWW7CZ3O2PzA}%-RRS~B59Zd1rj|-MUEza{B z1r$JVje5xh+rhMZV!3t>E(dZCL%f|Lv3-Y;tVUBFi&^y61ct3=8QwLL=l2eA18iu* zSRgoKPOD#ILZQqi0d5!=w??hJ$k-7hog)rXT7NITZqG!LLl>7fqj)^YIiJJY&zU#^ z4R~{Alaf7GFeRY#jyB z=Ej%o1vyQ%B2-#R<~O!AAw%X$Yi4pAt;5v{Ao{cnVnj)A@lwnVJF%mCYehG)d$blDp7 zM@|WEWSpL$mqdTyP1+Tdv+X(tO?t58WDP}b7Vpx|O{vguf95Sv2UA+tv*Cpolrf9< zx4RI;kA@7zwm44~eNkAi!ImjaeHBUMuK-rTHm9|}$2@_4Eim^zl)-ki?M}r(wx|0P z@xX|AhW`v;CPz>3aF**uJ2XpE;jc!T@=F$cWAWk_=Q(!6m#(?YFz$|-4zAg6smeJw zp6d@On;^Ffd$%%3+dw_<+4&c)Q=AQyKGb)Yvuu9@GN$Arg>n9Iqpn?1RO!OIiYe*x z25w1wSQKSU>K7Y+o$j2BXyk9Fq=6`X&d>F5f?$oqVzrtVO#)2$6btQ$iTYh+2)rM? z#6?Q`+_oG6&kcRSjO`r~j(0A-6W9uxyP*^}_-{K@WZ^|vXPI~pzjJwBLc#A)jc-A# zr782efrj(-QuWPa^JK52{$m59{UpexYdZC4aE*Ois>xEjH{m8|h!1K^>F$1m;7 z_v@k-i%K_pS}FLB$$`4jZGh*3WOj=X?{gqq?{B0jL{IpKp>`XkKVSL1!ggYCZ4DTk z()naTzn^2@yoE;Aw#>O5Mkn$Qj1>K#z{36fuc_AN6Et{^ZY-MT#arAI+zk_OjXc{XplXih$^~H*fq#pI)Qryi@ifOIUN~S(0)G zbg_F*1jWI@pooR-XTP^s7ZF0ZaGztFd3@!ln{@Uu!*QQH4}`?!u;8wK2R}7YOgw{_JZzaAdnp10EwutX%ho( zpAw8xV^WXg^T1u%*hKk6OncfSs4fdO&Fd8bt|=3|NdNy>d+WHUw(oyf5CajBR*;Zz zfB{B8L>iP6vtKi}8$`|mL4aL(Rq zuYRw!w{SI6&U|4PI-n`~@J+LjjQf5ug*m-cKct7($p`AP=avz)E~zW<;0H)`LMpA< zQMAIL$e909)C>LzQb!yWyWK?qth1)Pz90tk*^5wc3!)^X%D2@|?*kNa2iO6{Saajo z1@^+3aSUyt&7k}%=l+k^j+_)J)mJLbg3e8o51K;l3uMmfFP#PCa*bD%nJ(QM(UM?O zB5P*WFiVYJ+?hUT`)VMRS!udJi_Q6%#v(r`u;TKgQ{(^^lkE51{>-Sf8*LIA3(f9v zb|Z_RM&$T{*_~Z)cb8AsJySTrq2$`VE=&A(i?c&|GVI}Y=_t)mLe|>i`Bj|Z<#Nm| z%%y!2AXdMfE){F`PHOy8Rz21<*=D#m>D;)NYI;O}r~8ya>SQrOVuIUZejv4z4{AqJ2kX|_ii{m-*WDx7VYuMP-?6+_CU`z>>m@J!%Wdz~|==mrN;c2KEfw5Px9C*P5R3Jdb$aEnRP zsm{OM@eMUIdlk2bqOE}Ny(UbBK8&mPo3{?iEReNOU|)a`J-&@v~*MQFHC z;1B9mbX!@q_0Z%gX`Ef;`8IA5zSkgS_TIwfQg#nosOBgdekKnuf3oh#T5euyVQaR6 zj!KGA6ui>9vc|cYl)fPGrYx+5fuc;RB^%t62?Yuss4LA~;zw(LVX*HJgBv3)FqiNO zGx}dPrYei73HPMaFdN7!CcYw3xkSgwUMD84V!l}E-fqBF(Ws8|i%RbBywGA6>lrHP zVSx5Nvj_afD;ijCc#L4j=UZ-K;a;x8jqr5J{2b}| zRq+9(qXmvBpe(0f7YD=Kwl3Oa2R10B(4%mSXOSE=H+O5>m#4lPQn^K5;ueAgv8iA<>GB!`i>R0JIWy&ZHue@3jm?G0d9FQILvs) zxxo%cdO^1_)QV~qo?&u3YR8sW-04$_-s=a;ur8>{7qE`G%90mbZ-)ZT%NVX;WjiI^ zk+-_41AOR!@JLky08@!B$;6bVV6$2&zNjjX+J0MrVl2x3sKNb5-Bb@Ir*WK*h9adC zm#Z|Y=GftT8!A%Y!S55O+IO2DdI8oGyV#`kVel*g`;7<6+yO`-m218Lm&51V8~d?W zs3!|x;1RgeUnD+)Mj)zj!#2av9`TIV2hGUi-cAZVqs_@EJ zpd1L7YX$^>;W>=}pd*$IpY|H?h1srW#ntk`lUW9*t9o$^7dn>Gr3@X~$ol6gfSiKE z*e(h~kJJmt^32qJXD(0C-SfsLh5zqG(U1T#8H}(`%8bA8-fy%2{%1$%1)bbu#g;(` zSNL_FZ|0qsmL@%?FD~dkU&}Y}N+B+NoHtg8qa|IUBW5(mI!4Sf@C9KmIDIgjJ{82S zt>{XXXWwTKH2?%f+#>q6T@X-!)6IeOAB+l=4!M+Dz;ST-ft@b z{OOj1)GSKXFSGu8*WMW@Y9ohd)qz~Q*J6@illS+s4GAkyT-tL0F=xb|ZpBV)r)gJC zb$}++PjJ*SBRHt z(U@M~M8SeXVzV;Jb{^)?E+)O|H?4-%z4wJ#H^cKUaKc_i9WdNIqDLp$1g-kA%T!9@ zIM={;H8?DWB+}X!j~#la$!wDLhO0pJ9kg&=2SlE%%N9NQg3Ie^<%LDwv-N5YA!57D^O8EvFC~2xNoKbj${gz7y?=4)-ERyRO?o#kA*Zv)Ou2)^u9jdg)8# zl%K8ZEB8gJfCCdS-7^a2b)7HmgPm;osdV9j2AWi%gw?5*dZ>|UEN{4#7GoT}(*y%` z*bpn}UGrY7I_9brv%cPf%d8IpbXLxj_QMM~K?AM$sqH&3J&s~;wJALYC$>yp+6MwN zHDsRskiuj(difNM)C{P}KzES(YQm1~uGjqF#wcad+KXc^m14!NxpkwuB9Bpvw`G0% zB|IXL{GQZU;$tJ#`cV~UNWh5j^DA^CQ;L?iPnwRi1mHPeAGSwJi8OVblls#SQM~j3 z%;= zeB|CETp&k@6Yo+`o3Ge`-_Tk zs*;HU90%GPrOiDy43Ls*fN#0e0H*=+)||13VXD-Y=Nt8PbDCN~#%sU0V1G@?8Vsap z%Dya4=$w|0lS=%%u-w)*SG#Sq{U~tayPta9>i#f2`{sheHf2fW>}O*$4)KPbndj5 z5QmG*%z2t$6#pYkNP)az;X9i+7Jh5M1CxdY_$6DN3^h)Org) z16rS~)0^GQ4V=<3&n9X;nwvD$+Zm+FG-Q*NF059{3e(JYan^l2pVs^bYhrW9-T0gzrx@mE6a9aC}%J8@wC|XQ0GrxwTry@AMp({o}R_!7BqjG+v z9!IA4%%mC?fOs|?4Jgx0cMrCwII>*Dm-N%v3C&zEyvxPt$hW z<>*jZcoaYI^>Hl`hy6DZ+pJl_OR#))VUf#u3FbdL5I>X{9e85OEXoad6h{6 z(JieOw_aHD+wS2KX(WbzPi+IPx$oCA;+j?#?p)Vz?5ZH6{}PB$>atW8E2vcZ5_Y3G z`z6-~U~*}tpJJY+PqvH3I2h}nuVY)g+zEc;smLQL$Wnxuek5KuTv4DM8XNZZ0gw*K z(>g1AW{Aw5<&R+Z{*gi(cTsdZMgbp>x0Y?;D=BPJ9WaS@XhEAn84jbkOYcz>JnM4& zWKCX$1mzZElnkKXPH)$I%9y?CKuN9T{08e?a7qvv_u#jA?ccYWLHax*$P?f?ZJ)27 zp@IIGm*m1sVf8SJU@wbqhMu-oR z%;NkfljtT1in~d1POrxmwpq7) z@5JpeLj%lEd28Y6-L3%+FvN>{0X3`uXMI0-_pHbs_>Bnf?!{%wqi3mJ78`j9+oaU0 zcF&EX7!^5a74=>M7+T20+RdPY8K%^cuYy}Bp-I^N%U3#IRH~0;MSTfUw)O}2N`GS0 ziMsJA)!`Z_oQnp?`3~4==Xwie-ik5}@x* zoQ_ato%}X)p@|;=Eq$#JY#IKx@CP~J10$^fG(h4{@@xh^31`|-w61WXlb1nJSAT^w zg=9NF>OORqr>Ii^$~<5s#q{DWW?%)KPv9Io7lC=!haU|OM66k*mJC3807EelzYo3x z6f;Gd4ROw5VhGBG@_bC?p%Sjn%h81idyRUq2;{L+EVKuYbSYiV= zI$ar56rIfV4d5%hot<*%vyKhs(v9t7)8h?nJkAW#3lp_&gHQq0eM(wHNL6EXA*)}} zbHKH2;t8Ok+Klst;+{ta%rq-y2zDP~!i*Qsrz=>gIYOs?;#McTg3rhuRxiCgo%Z=h zf@a>L0(~B0rO=6vgKr6hY3Xz%{{r;&UhS&@`W+&nE@_pz2~cy-62C>G8K8xtnyZl( zW+^@qx&CFhcX=jbsu|(*>00R5BZO^+^8^{VXKUFZ>xRI!fIShv@QuM@44c*3!6vhP z&nxQPpDC%WYnQp^KVt%kHu*`S<^ExwM~kMRpOVJrAs#V4@=XSS6~fnYl;ysD$l{qf zL46-^B;t}$6|+=za@7aAj2#wscGQ+8;G`HIUa>J~yV|^qS`z-PnF>%pGs-BHx>D@} zf7BB77Y=Z%fCdCE8Cgr{l226(R*2WS7Vav(q`80)oEamI5ZmF7@29@E}OTL%b6JLUZUUrVS|3=qF6KN4;wO zA$EaSnTmc_!>Jfd3 zTuZis$&yKD)1J2m6ZM1L2ESy}=O+E3nsbURO<3K(rV>vG1`tOYAkT7<&t_J%6aP3?Mq!vcwJhamcbRVkJH3(5HGk>M*b>UCLy zF@n?T)RKiE^6Eg;6&eTKVk)NHmzQtx^`?r~+%iaajEIfX0nJhojLzO_KPE$XeA2kx z{5}NWwu2dLTdp!0ApIpj*5pcMJ!69tH76;#md-ls3{ya7h%Kw{{6|-ux^kOBpJ^qo zr#bE0|6>v3%q{>~lgzo>!Q3Fv*8R|0`85G^OQ<>MZy*gQr{Q9wkAhI?iyX#+!-hnW zMR1xqt(fflFon>L+kJZD`Xa{)J}$cN7h5Ee*cocQmqh-IEhp0*HugGJX=)2W1NE<^ z1m#jQ*UM$Sr{>Jr28UYn&*6Ji?N9=NjS7`aCJqcI=xFRw=s9b62IIQ-!x3+PV;(71 zyz;U~E|$}+yWJah<6(=9V#-CLm0Zy32V2C=PMCkC+O#vRDa3H8H zq$jm~$~Ms#sLt7_gicw{pg#Dw1I9a0ff!o#d4u#HIM;|6#rZ2~!kfL4JH`JuOiVFZ zIw8l7$#)|q83;FWB61{SYu}7&Wo-kX7!e}SJn|Kgd}l7tJ~qEon{`ynV)1lYJh?B~ zo7#G3Q|BtTvLFkWbFG9I{?M2&E`MA;y0WE)p&HISGD_yCe16AjPm`gW^N>S@{P!&1 zh?y%*9tAj`{)zGoUVF%9*m^EFa4?RjnDe z7dR{j;&AK`wl({hh4vUiGSK2wg0dVmvj$n=M}Ps&LftywtCpjb**!7XRwmqwJTb$X z4hc~73Rr5TmeJDxc1loODpEzuEpzq#=)gH`-S|KR9S;w$lH#QEftpLBbnbEs0sKKG=vJ!$8%^Jicw5M5kSSgp~DRjaT ze~-I5J^}s1v){{NJw5fuU)sUNK}wIUH0uk#Pg1B91`8%{B1c zJ)jdxr~09sD|I@ng(%><)eY|%%jnL!^NCz>icISY6#qJW`2nh2Zh4cWBlda8q`BnUacm9Ve9oN}IZZ8BKvtSVdXY*$~F; z^QV&yI^2Pt*{E}4cYl|V6C6}f{aj_gbUFzrR0&dLZO4VMYOBm9yxxIvYrfcbF8)yU z!MU0e=)9rYoXyAFGCp|zW#`*L;*n*ifU~*`7$3L%J=9?Ss(EB#!C^TyppJm<=^8k) zQ*YIuUB>73zGql)ft(SEgNha{uW1Tkqo^9Omcc!*&VcHF2I3(!Kyuqt`^*6PhX5B5 z-pvfRmqaFUsqxgmdKL@ufr)8=${rI-lX9uWMO>i2=Lr%CtL*bQkWMtKv@Dq}Q7LZ2 zpEqLb(;3WrY`!Nwwl%-04*s~!Qu2wUhqJOSvJ0;FnF&` z_TmJ=e&=B6iWPUr1KF)m-g3X)h8T*ih}sujI0{Zc5S@_j znvo})vrA;skI%nWyfJ929`9#u#gODn1oYT;?)wu8RBjxVdjp9>LAZ>9jcnk>b|WPv z&&my;7xAcA{_Vb((FdrKR%JV5kS2FKe&QL&aQvK5GV+XovYV8@xV67#XyURjWEDiv z1=dQ7T`pA{_|_+`SI=f8;JfrDMHB4C-EOrhma9cxXuG)RrBCF5V1lFJ?JHYO-V&>@ z^&af5t=?A|Bc94Bdko4>Qmmu^(hrg8QBHl1{!fDpaWIxm-sdXbmj;pq4c(C4Roo@z zOQofRooit!vll?Gh%q=0;)8nt&kh1Ov($HwB>En6_w;=^>y(k7 z(`&`0IVj$5jrNxLv0gv(U?ItDy&jY&2iB>UV7UnG260BU7Zq~c zIeiIm+Py7$WOg3|{V#K8)*<|0Jhv|7P8cIT*ne244$>b`T;#(4ODN7;F)!aS0z0Im zAUf$lPvJ`)p(P0mP=WEkKyi|UR^t?67DqJIN;_9P>0qPb zOG91EHe&a_AgROQr4u*z+_XIrrkrg{Q$n^01d@x%on4gG7TB8V1v zhB{89eW1MN0UYXZSudgO)|Psq?Gi%*j*!=g)2CEuF1XjD)Yvmg0q6CosXLJ4OU03U zaUs>CrRrhp@}q^%*5-u8Esd%>=8CSePhWSq@#5rZA$tJRB#M<3JgYXUWnHFN5{x-r z-n3R-il)d&kEA2LUibV$=23 zXO(vSlz9?HWaDv2?BUxRKwEW|(E6J?AOW|&cB=(FiOrRM6N?NSl-BBLj$q*2gXnCG ze8Kf^D}_>X$lH%aJ}))SldDTwSV0&(Y%Avu=Qi9JuqYjS#QGOL*OnPy6&4vZvyKSu zS-Xh|bW;uuGB_p>EGciAQfpLZeW;~ybf}Ci+lCe13cV2mR1SDn9ngB0Dp}b7{`0MTJjoNHIK?;_|B>ZNp7TkBT8OuT!aH(I&<*r({V=5cS zCgkS^e9jLC@L$I508>O3g1pndIw(p_L>%a#>B~zYGMnt}uy5Vl#3V7fXRef(?mF>6 z4Cn-=XIb1R43g=)QMoOu#Cd<4P+(QZw)|QrLc{8hX^k+9 z9aWwSEI;}3EV?XG2553iQNL}$T@ClP*ol`^zyw-Bjn2T?2o?SXXG0Iw`$lY&bmROT z4?vB1Kx=^JtZ?g1W-%ZIGMIO~ruj2xDV7oDYry@X%i2!f!hNmMq={RmxUC9`~k%L#`$eL=Nh|C)Jl0Rar)3yoksr14<~qJ;~tP*a2O<+cKB1a`7FBs^P8I@@*wq4{>W?p z{PusnnlVNm5mvz2w)yMkZ6}8M-&4w_2SO6?OJw5xoRXGLLy@PUS5|Kl4 zjG^;qbj^Jyn?nLp;#PC4TK|=&S_zN^yvAGF7yp9^J02kuA}MTP&HqYtBLNig9Mew& zk;qES85h*3mRZL_iDehUdi;M)MK@?p%qmbrruH_3vZ4>CFO2)Jx3%QW9+XE zDW@k8Cj~nb2$Oesp_T5*j-3b=q1K|PM0CyBG?8l~Ur|8{bgH|OW zKg)q(kUXNOJ!hjd&`2x!qRG;9ncC@o`C|6eM_{yW1Q@rtDuJCQv*f&EKNEoxoCih* z_XB2GUFl$1)V+qsF{Aqybz7-tY?lGlOW^}!m>=a-{C8UZWZUO_ji|GN6S`V=56nU> zWfj-7JL^rN4$|D8A?V^E8fWRnvUkr%U-h+hoptXNq2mPIie1?3D)m*0Lmp(^9LdJd zRa;*+tLJ-t){sH2_Y$SjpF;H?U*+pWHPaG$HAgFixXN*cIE(;ip@*7VQkGu*7p?S*(t%rk!v>H5WN;FX2?m)ApzX6RF@{vxSQPtAabJwDH_}(-IzThRD{= zo>4Pn&5LAd1ox!ZDcLxKI5c8$lM3=_a9NxOF{kCr%^aC_WUEO3t!DiU*+7yo`AR|3 z_r?8^!h6On50Pz0F54mC|7n(bZc|D~wrlLadSdcFrcj2o?jtKHmSX=4qudjTV@J-i z>y~l-iM9SQjS(q8x|VFEN-6%EgHh0;sHK6!>oZdCVMOtt{x}~SQXvzM$}|3{0O-fk zxJ_LNkwwaVtAfF;|KB7VK&|^uvi!iCXLaxjAQ|J^qsgEjU;h*58hP?DeJ2^t_k&jDlw{B^(SeWqake`fnf}AKUc~L9!%M=3@5n1Y=A0 zZ^7(B{)Gj^3R*bwUKbNj37&>pO-{lX#_Q@t&yYzi^e=L2LPjTI_m$%TPAoeyFH7h4 zxB23l2@?gi;L8+rbV0E2IKrN0eE!;VFA@@z4!v(GUK6`&hv_k^iH(nqP1&SyJ&YOy z6)KHQWk|L7zIKG=c$=HEvZuGVWuql>iN&gI{cvj}XOGVMOZOLUZo`bS+viX&^PNMz z{+w^CLtgaz)j2fsA3p+@9dFPW$y8^XRI_1{tB-P9qqikCYsa=fk)C|$@8@UmQC+;D zg^K%AL5trOX~mg8od5gnzrM2u6~F%1Uyv_|Ye&DTWFkU1`bu?j!c;0^P5RsZ<4f0{ zpL!Cgx2+ZuvJkhYdxB>ng>HQP^{GdY6Zw4k9UnF22LfPnzuxaiMe}m?zG5LY+a1rJ z|C0hgA44A;Pk)~Tbv@R3Zhb&a-p&zf8zGDF3&sB?<_&)|&$Ezke8AB(CK9`_b7V zX6~N)XL5>_gYU0)e@%1h=J&PcdwziqT^z54ro2|d)2@3fEnJnmT~fE>*)vwH||P!qKP zIEHlZ^69(Zd5)Y4czM9Z$EOyBFBW%ZN)1@JCbk^z$4vsxe(pd8+V5XDgKz&HLI{#9 z)O-enzm19foZI`$*2@fZh4xZBa)&x9iw3->9yvv;f97@g2zc+I^MMif@6_PaMn@g7 zL$aa@D)`!|3H?HuU&ifr2Vjmav}k6m&Oji-fdr^bI$mb;GChFe`sAw)bq|gicwck zE>D2o`Mf8k9-*n!&|!)q*QP|vczS}A5z1%>1fdn&Q;W3rVu4uyJ8$rj-vMMU;_|c0 z8@z5x=@@LH0zrfs+P^G!4H{By_Lvk}sow4w3jxJCrR>!y?ux$ocP>87Nqk-aX=UYC z_~WY47viTZx)2(1o4mxW;47T5UY_J4c~_YLs=J8gL>sVDxcQt0b5 z*6A)$wrxdRV@x#iyPB5^9FRhJ6Cx7j^6!K{LD}z@Zh4e$P)Bl1)Bp&#kZid@|(G^XRiX2hOIY;`SHjz0D%U6ao$uq4q3K7;#h)_8mcyK~oO7}ut$|Bni~&`Uk=v7n6b%NW_a_>a zCL->MYq{RfHq9tvZJgc{=u+q+OBx8u6|dAegqcrF7Ns!naYfy!lvY((E7?}wG|cB6 zcN(dD$f311-Kng0b6Uo!!h0u+&Rh4U^QvL$GHsc?R^Q@$IeUq3vsmP=s+9=AW2C~5 z8$xOhlGjsj<+2qI4^zg9Q)dvKBcoWG{Qu6Y-*xnT^a_#P${oGCbj1~8(S0^HAz{TW zy>#2VX<5@6_L@XR4^Y}&Jyf!(5ZK;^l=okwIn3U8X72lqPrL6rO<+y^U|+xnPviRb zU0@KHsO7Y|AS%N$wa2azu{U?#7d)BwY>tNJEs{6r%ZB9+Y>7?oXWbJrY0%7Vsgf~Q zv<-truaVHyv8`BAo2l7Dg5IyS7D<5-N>q>uBmE(;fO6lQYO4o?gPG=37 zzHJR0vt6lOfPUvtU*Yt-%&aOc*li3)CD+ku(OVnJ|7Co2h57n+#$V1KrZ!~VUbumJ zohI~o+rD$Ym6n?8rTX8^_c_!^-!$I2aS+3XGwzzj9Y^yrmQMz7| zEGFyLmVSA8-u(`LoHtsSjE0DqOIihQbfq@-%X4UhR4YW2SQh5oft|Se0$R7=nH!VI z9Xu%>u_mO`X86aMO;2tp_K8PMCv-DH$r zjlj5Bcty&r<8wlT`OA?a>xPC@rTRy`D+%qy%MHQHhxHzc4fiFEbtqg-+@OKFa*vet zVir2yB%ludL~F7bIf zrctrj{HemSdc)nZagAC-g`(VqV*QO|;%yrtEkr7YHCLEQob1L-q6^>-xj(iyosf%K#hCuLNuAjM+tkO=Es!baSbsu=iL}=Z=QQzAzAQ zs^keIOV-yr(AjzxB!~1VD%B5k4ccSS#LgdE%qn$d$ZD=vRm@QN@1oB)P%*cKF9*6K za`2N7&~nbMH;VQBiv(es#B88UT_HGI)t#M3{6j_PD~V=P%Y_RW0v3Iw&l22Pwf`^CG+VwvED|ND-{|J z*&772FN4U=nawA)d<;6k`rC8iBSm$j>Aqvn9CQhl$xV{rcCC%i_KtSXhX%Y@!ksEI zcQ(0x3=y66OkW<#Me=MKGnz$k3}f9?V`)Qu8~IYI3EB19{IBX4Dwd6dl+t!-OWLA= z(BjEm{4_ACYMeO}`O;=FtzF7*Jbmupp0 zgPLpM(Be&W+M?~x-`JpfFmF`2Pj%`9gi0%rG(cya5TC`oFGtC(PxQ!ge9N5PS$6G3 zXh#Ld?od#%eiXtdUr8e$mva;%YFicY2vA?0Su&-7@(Ou2x)T+!7q|W5?lQu-DrW(5 zC`^Q}+e6H|6Vd#YbP? z7s1wQE)MPusCfR^+1PMaP1fl!G}F?f!0SA2kHcYpM$9K!d9AT$y$e-2KJ$*85}&`@ zo6Ntm4vutK`;K%MPdkeL_<1wVsOP_!T?8pWB3y=t2k7Nz4?le|z(}QYWu+T0PHYK~ z40eIU+>W8*W*A6;E`~j0|HtD2RAMsD3aY#6d3%%^$u2&PB+*m^Y}HTn7!9?ddMxN7#$W2_Ghh! z7Iu6lOK8U{RtszT+;<`zZbuMG*G^e|)+M*mEc0uxzkmlZeNvPko6s;}GiUpG!=cXE z*G$i4{o&kRxmrTax3$A}=j>55pHw=S7g%K-+7pptyg9Tluo)hpIkNELrKr_OiT&6y+;?s(hx9t5rTA89v6eaH0OjNx>3+r2QtIy0tHmBe6!TJ$oYf84b>Di8hW#{wN^747#d}6yJLE4Hh{Xk$el}Zrv2Z=& z1{%>Ly$a0{t&9sVT$LQ05}jh5GK*-Y^`tobnl%GxY@bpfD%X+;4BhK0AABoHTd?w~ zCjUIq*z=6I0*9Lwg@@)o&Vh777UKTbV&B?cqla@?V?Htm(OwOURtIA=TDP2S6Vy%C zEE8cG>1!{aCCeOOdOF*6Led?)9Nk=e;)L^&%vY1ZQk_2LJ~jQZsC~b z62BCeXEU$~$VcVg{+4oJZIW`ES5e`|;V);a&{H&%*ur`Ie9aJ|vw<{P5^Q{-lw zFSGr}v1ZCx!iTj}o+VQCZ8S$EV=hDDM!|Tia`P-h>=%J}Kya*+>q-x}A`z28X6 z^=u;OMyL)o_7e4m;&K8-534Z>eFRR&Q4 z9mF?Ja>et$^*MTRS0MP^?txbD#@*Q)LN25ncrG#4&nD&z(@OEUVcv7=%Kp*JZPDTA za6&ovEpeeEA`T+=!{q(Uu1?G@$1fW75fv<*yXPkzRXNn7!bOXly-~Ef$g{(Tb)C%% znsu!3`=tkyaxN7*3peQ&Q!_J!uJ{`E+3jV~B|>bC?iQ5gg2(rG;0;vHfD?GytNbOL zYmsd7Xk0W zI??QvStnXnt%pk~Q+uNfmGWy$YFbg0_AVnAy0)p0Q#8Zu{W@G^x5#D55Z&{fw$dKR%ljV{zl0z1$rhx-GLtT%x$o1E$4Z%)fJ)Zd77l%{K$tbOyyz$zS z*+o|OlN9gAU0wEJ*2vun4YVleRLVdlGv&SN#L#HP85`=vU8=AbV-4fZd89U#w^YXV zJ*=C3JbQbVZWb%3K?r`S6s$(uYcc&gKIEAT9=jrx{oSD0&Az=X$kAJu%!1N__=$FT z*RWyP@M)>tsQGn<0(`vH!h!LQY3U~x;h*~ETU-*JPiN>{SN3_%`yAjuC+4E}HldFP zXu2n5Dik)PJg-Jy;6d0i64sJ%X|1)|l9sDQ9^4LKMugN&VMuYp!kG6394w_jo3-hN zPk!Fuqwu%jQ;oEs;YxSfRlJEi%PX-%RO&>p<0!N+^TF<>ZgHBq|0dE%*(;#J5o3XU zl5+Kl>jq1KPWnD#uFtl^w{|Yhaj%h(?|S=gI?j@&Bc5cn`@-yZ^C`<$Th(A*%zL*a zoO3Xzvi06h5KF}>=x;8j<@WQKFH5X~FJ$}O%OyrHDtwoRA6xT_eR z0yS^G-Gw@Pb@nMOYpUc@-4mfYzwm2eW4mv19C3{a8y8oGBfFECThbgd|D1%zZmm7k*b*)NAlCVUl#g;?YTDawlu$M-NwsRo*LL`ZO z#)iBeum>8e`xIJqbM|q^#MDk=tAIHf;J0y>8RQuhA<&RQX zkE_ORj&ELR0YXtk-vOY421%(g#g4Kz480en8>by8IccE}w4M>92}*k~sy`kQzjaMU zx7Q?}A0h98(DO-olCLp3OrHlYQlV%cnppR3HKNb5Z50%3uD={_vRY}*EF~5igYxD9E=qk* z%;fqgN*$Z(_l9Tg_!KHcHn>kl>$>LH&F@g5n=Yr*R351))K8nkzuQY6jZ+z})7mJG z4;Anw>D1fH_bg^Ly7=0n!WFBq#P&tu^!tYCu~|Mm)iQiljP~Y&ocGgOEJC@AhpH{u zqCsP&BF7+m9D8td*40!|wW6l}Wb19pBErUJ@Q^o+l1;XG6Jq^p;_#X3jGiW#(rYkP zJGXs>*c5a9b6c$mCMc76*JM`^PYSoQW)o*>E0+IpNI~(&au588lt)8RNj9rNH@2C;MGp0Cx>i9c6b{mTWQvM;^({|J}r+-AJaR7OS zdv`|0p=RC-n*JU^tS3V0{#!4Qciw0loxZrm9T(?u+f)=s8bQZ@IGC-Um)l!IW_H(bMY0||fZ1wbd06I=&*=9rPu zml=5+nZ|3~ib-kaZpyx1MiQf3j;l}R0ekN?HflQx6Rs2Y3}XVkfX{z<0fbavN1|6) ztwxe-S)=u5@(P$6CLYg~SIavoHm%pyP8SeLDyNDv9{b=8xmhmYbCzHC&&~7AxHWH; zC{wPwTN%;HR;oGdF$ua02NF~>;(tlsuK!*3lNs{AVT`}C>3C8#QdFAIxsBV(b;a*0 zxN8!)dKb`v95r7$nsQnf(^7GI(lmeV)Eg%knYCz|BZ_QFGY|>YBK@nP3JOF($^}WO z*PD7)LrYz6L>{&u+lIVwb>LC8m;9!d1Zzf9n7BspWjYJx1qOU$b($iC_Q$3BnH(VxB zrl_jAHdO2vUQq)n@CRF_Z?MePJG^i%@A))fzzXNKE0=LtD!($ zCwxV8DZ@8dIF$K%h@|t^U4Pw?cTag$oSSJ^BnTQyHJAdISKT2n1`iT$9I0oCT;% zdyQ;uRrYI`%5(iql@TOa{k=iQ+Lqfow7O(B0mi{a6-n!OW`?)%kOA@FSeQd$&X%~< zKE$x%wUgID)|KsUu@axEiPBv0!o;DzZOI+!i$C!TAjbUFfy74Jl1bJI<1(I%7Fvk& zpK})?BcVs)WPEXQfb<)C52rL8`U-mTHfve%QYIAF z3SB$>4B{xn^Hp{89}>>?WVUE^CDoE4Eq#hi(> z#7kfvx0o%`+2Y43*FM6zVg`8g`QB+jF8klP;ZkC_DWMQIg4KhW|8X-LB(~oyQayOl zq{KpYEo%ME6n=3w{WALuRv<-|Pu`GNdAW@_?EP|uJ`hIB>VBRc;I=1DcZ$&j)tfx= ztBY`wcg_LVZYCzU+baS z(aTTi*EZ5ZpOJ>HswfbQYR-xS{^a|J_hkR>75H9$lHwkm)OWAxuG_)82);|vImloe zC^3!O@8U2c->qvz4A3 z;uDUpA1(T{l8!;n4S_ZHWp*mamjIxlW`A~CcVes9^f(9AQRo#HTU+qA=vJn~oUY|y z7qI?R-r{_-PfPq_MwcT0@CUgh_CScwXyWHL)DU@evnUGJ%OqD^X}$b~th3rNw@U%T zGXof&Jis|zuBs68ntrhNc7WgH9#&u9Ua^>KSPC?9clkUd;3$;6{~;hj_8>`^s%!b= zmb_b|86J_MrmJYbupI0$X3X)izo1O)ipmg9qglDRScozN=?&IOxn4)yc?Iq_m&A*r zLlmOBvh=-{8T;h5e`7NE`q!ldHR2W$UUB&~^+=u^npw%?hUc_Ol-Nyi8I%to^) z6%TZ}fr_cwhIaN64)qaosZn|^+-Z4ncez2 zki@FsQp)}D{!*6LmxS-UVrwM?UF(B$PbN?*m4mk$R&O@6+HXEBEOT72z$YUmF(@%{rRl6ohx~st^lHG2Y&Txar zaLZ7!Y|dVaqhP#4msl!_YiG;aMAyQMfAPgYtjg+GJv%|l!rhGyj$jaEeMxb5Y-8|X zPCwsY^;<^N4-|r;c8&4gtyL%jM^1g`1|jrXIheNudJZ)&)QRI>0%MALifhKRYD|75*Ccp7_)) z7Ou1x`YBJj#{0x1Q~x}S_qL0C{ucD14At&koI8+%I3mOSm|D+Fj#!5;&>H6yF%y$*!VTos^=v}G?^JkFm*Im3`uPCWZc~=FR=!azSPY!OZ zrREGVT?5~5abLnsm>@ryz>Y#VXWR!gCmgm&;Sz|?^=#bOd78<}itL9nrEm;%;`Lt}fzK9ttYuCvaUkia&p?}M+ zX=>g74dQ3rCUPz|pw8!2k}fmSEB1Htt!c1L&F&;+MZ7E4DD_vf%{pE+^UFjnsP}cr zF}ae{6*TE@99+CTS>-jd?d~@N@}MtR&(7St<&eP@-!RoZII_&HL+zt~WuLA<@0RJ* zxK!o=X4PkUv<@ldr?aRmg;4w%q$YYc`%@MMUUC7w?^C>JtF;P9j;9K&Jl#(~d!Q}b zc5mE6Uk3BJCEoNF0Hx-)UaORs%Yx~pJg_8%Nu(q6{{aR7~2RKQJTF z^E*v8H9=FsjRf@U@ffp0S*c@#*E?`jjh{FZz|Yqmr0mWbYC$ zz1#p%eyw`Yp$w*g^}d8wD)dZNNX0dr8S4wzc6g8S1aZwDBaFHt{@H&1FQh==KprYY zofVn%wWl=za50+sb$+m~I>Cf$yb)@d)72);HleGO*}p)&a)6e&@8M({{BAV^{VXE- zRXkQ-drBJ>)8m}K*l(nO=zTtPL_pt`&1;1Pv>f+TLMl5IfB9xP#GHaaGzL(90!isg89g^W9v~nkf>ne}CEN)x?rQnr+N8uH~| zIl|>GRY*cye0qfx{gW1yQ5id@S+5^H86swnX}HYYebub+1!$-*5FH@44Y7-*o=kpt ztaB~EYxhnF>T_|9=EcmNAhY8q-<>G~O*C`QV^a3{cL$$nHdYqIw|M90g@MM2y# z8~GL4s5t?hfU~erEN^Vp0qB4z^l-Nx!zqL z8*MZM7!o5?K8I6~a0N%Wd?F_fgx*cX7(u&aI&W#(?Ap&889Cjhd=?XTGO~dx`h*vr z|Gn-%x3&1&IPv%M6nuDf$O#TDu1SG_`rI#yftGshhbMe^+DkD*AB}7;!Y@U2q$-E; z$QS838;bQNXfz(o47o1=?HI~VHEwCO`1QP|&fwtjFkLWuXGV{A%@xOq$C{1P*!t(~ zP3orAnG~m46vi~gSZfc+gd7boZz7?fJ&U5_xUW!&zU+%>Q7ax7+YLdCW!{e=d&f^9 zF>R>+@~H=YAJ6u0jfQ!|)L*JOI=_3D(WUXMy=KH6^el7fmV5x}iC6Oe8dDi~;*I(b zh}*-(S#|i$L&7Hv-xIwiU%xx$$8!VZ*m&j-TZoBSy?$m`zLR=Mg!^0^Xy3N`4Kw|8 z!^(U*nu>7t37+AB2^C9|nW>=AqI01WfYk}B#9a$;36LI$}!#pTCnW_O-= z$^YpjX%N`dOopa^!c17eodG31I=J)1s!`foG(y__G%xnIUZpN3`fw6zI?$ zc&Jlmzi1{0e|!KJyNrCGqS;{VpU~y*;tsVw8ymNiDLYVt%gbtq)A(+fCPCfb%G3>} z@a1;GreuF&M6@b|@iUxyAro>#UjC0GR_f>3U-;+^^aY<7TRDj>g5=w}jY^ zfNKhGZQVl&!=4}^j`-;zAvmVT2r^qaz;;m1Q`|-))p>?z=}@`Rw0>p_AZ}s`q zc5K~ewbUW7tX?>rZ>)m(cofx-PR>T`0T!cJEaX+<^s|E~a8F)!3UHjOqo2U=z>du5 zz(RViDi0X6TA~BFD_vD2B*@N5M{epQ;GxOLOM7gKe6GJo%?-7GF6D zRD57nny^KY{GbN2;=?aj$O~ z$`RGwuWWvV&?U!zTSVQ3?B|xnY*Wy*Jy#Cv!pW>I+3Uwg23JU8{(SGdiNvccHSPxutnj%+1K1^IYG*=Nqz)+W0(Iu^3}&bCz_gbXP5Y z&qB8PD^z*J;tm~M83W{y_soF_RgQL3b(XCcWLq0Lp&Ap9TA0cx43wK8#F~ZsAF&ol zcAkh(e>+Tpmc<~g^G02GkJsxh-4kc-9T}DL$BgKBaBaF}>j=zknbqsb8jEG$F*Rl= z+?eOGzll+l#4YsbjL@o8Z0rjm2Qig6mzgK3jiOu2UR_alf=gbO?a1`d={RY&qTPl_T)T6y(lu7SqMTj=j!kLE73M1xvIXAAZkVZVW?^zbed_P_ChDP!Ak`zDmCQ16k!BiGGjC&bG_tlx`n+@_C zn8$KEewwhQ4tb`yz*zr~?vifo{PVKi%5LnB5rQc2jKQ&8$KpGp1cxXz0sDuDa@rE~ zH~sr!Gqr?hLxh>#2{F~&?#-1@305_P2hRI{WR)x}sYtGpWQD+ZasI2rSdR~#hd?|i zH--g5y$3ihR}Zj@J>Cy?6^IB_+M!V6E0fXdz?btV<<4*Ib1&mROk4XgdhJvd zIcSxxV&Mqbvmp15m!AJMXlJ!tb@<9lefx##FRz$?1#x#-E(mB8ZuoyD)Aio0sLR{E zR+3MCqh4gzd9i^5`8|MGUJ^l5x-QpZaoY_p6X$-bkUmORufKar&|NWV`Cgb~;g&g< zJqec=e{jH zr`_UNNkXG?8t+5gwBY;k-q6cADz%o`ugOn;4+mmBO`icm{y9FLp(Kj((NZoS5CM|~XH#+fyi1M}70 zp{KHrSGeGYn;q*evbhx)Wr> zbX$%59c*`Q9rl7_9HA?v7zx7FEClG9Pl$?+Ry2Crzq*Tf2ubqHqbwiUWg^#wXutyE zk5mSkvI>WzGv1z&trY8=TF%Lrn3S#)**>V&&y&+QV0!FRTx`4((Ts4BYvY>*@H`sB`Rww?x@XLf(LoRrnD8cE+gW zqp)>-FI$8b0ix{po*;6jhV_7 zC(D+7QO(&(X$YB{1mI`OTuyxx77xB z(BtFe-DfM_h4FhAqf}hGeiH)9YCdV>)o}Di#V=pAUDPE7<`=zG-O(doh+q<4Q6Isl zx6qW9$hGLE%a>jF1$ES*-B5-&bCwOvuN4PfmLCpb_lB_acg<$JZi#npRw2pWvbT1o z)Tabjmk0?ab~)&1MHY70N=EU=Z!EFNz>xt9C$4Rn*xo2@x^Zj3mYJ6G4aWn8!f+&y z-;$U6N5R=rrf z8~XgDeMaJ|DzIPfP}eqKSRyBkf!Q@9J7jZb7Hw#0d!ch~85F^98Pd%39;bx-M=U+z zz3Lm%A*7Rmaje{zEI+@DXJ6SvPwOBjj!t}w0vRz52l3qJIvZnAx~Bm^9*?7-T!x%c zH3>B?;06cWT@)d_FO;&c{oDFsgIbnBvAJHUek|h25{$IL$kkZiPp*AI+*`b)>45l1 zRaZ_--M47f?V#@P#;g4YhaVTb_dmb~>@neiTmwgvxTD~~otI1OXpzw(0uS4HARa?4 zZ$=Gsq)yBE;+10ic!JPBM-_F6)<+9{znSxk?yVPfII z+!e>a?q>#3oiF z!!#9Knr^5qFj}4G`615Iw;g5=DO!6x6bC{ri*F7{a5b&I5+|I+NVt&Hx< zJy0(Oa*3FZ%Ie~1;=DWO?jT#Lce%LeAt>&&=_gwH4{U9tr-y>2w-+U2<>YK*2=XR+ z%#Y&<+{2#H#HVIF4c&qK{-1~H(Y@4urdJ5%?ir5>Ll;Zeg052^7|owh{#hE3az6D< zCh;p&j>CLFaCD5L+G>?L(a+6BDW#@&w7%-T1<U54g%=+^L}VPv?cBe-YNgl{y6 z_xgX3qmPM06!lgNRkgQ)dLt0b47kCXa&OmUV4hC2yG32%xWA)C*5bYQNJb5g%Z;-8f(v`D15f^^}|J7pD9> zI>q58jbmYWfyzS!k5|i4TLHn>`WT?m4<+r5wsQ$3{Kij&7?VqS7TuJ2M0nZLU`<%H zk>_BCRd&=Vt9Ibz>vFg=tL@g^?vor9(&7 zNfnM7WPhZKTRy1r_yjV2g9ba%t4?uclnlrC!tK2abb@{koghYq%?Oz+AB$ zeKt(rN#1aSE>3PzsQmTLYs+4*lQ8DufY>&KdgbXod^K#8U%OL_Hh`&jd0HLjKFB$v zH3WT+;XNW9bW3n=HH7sIF?|{Z+yZW-#7%3Xg%Y~LT3So_j+SP4UdhVYsb?r+3l|vg=agZ0~rfXL-&5t=B`^5ZJtP^gL-HwIV+UTnUzy zkdhpMkd3WNH5Rs|s_Ev|qxZay9(-9sg)dqy({@G4#OavxtX6T?lz#}CUhl;Yk7<=j z7Rph31z``Ne2+A&uVGagDEQQnzZ>&$JJC-) zL^so8Qv~mk#J0?Sj}kko`G-DFHfoSO{X8NtV?L=14DA->!lf4<26FAT7;dA0!4I5y zdN=QwF4D)X5U5fQo;BKBgii@{r&SY`EC8wvfUl7BcvJ~bYKGjnkFxEvc0*m-i$*!E zNh&P^v4tm1V>V2VMWW1p`1J{$Vk~`GyN68+OK2^a_MXh6ECN3kU7^!Co0d4HURS<1 z6I#Ak2TTPdZ{)t{!cHe=_l)dB1~J+#)h}l>4)?uV8;ZW^7nCFno;$~z zOb*sNNnIrN(m_s}tI(ukMsV@Sj76zIyF7r;>b?7?QvB0uPJzEGu>WMwJtJC$i@Km{ za~Kg!V2&W5e?Daj4X(-j_~8#;F0=bl2@S1ozhVxAwbC}&yY7Pd zgTk=M+(I2YXYQ?sE3f+oY#I^hK*;S>Y`p)^>_keDe&GJiZ~YR|(}87PCmu`IAwYot zPJ;z&XlVV&AtF0+fzcdAaG}e6wI$ZD6bA%!AEFa=v8fV`nP34m_81nmMrXZku@2Hd zMzxOeJ`hpceTPw!V%_qa?eBazHOd27{=~8+cJ0{mVnyy)habC16G1~oU_(2f{U(*; zz9inyGI4ym^@;^&){CJgAi=@ekL~bfOq1veLPMoPd|5{;@Na{R!Yr`?VEx`+c$`x0 z_N@QnHmF2${!o%b3ykn@%&^2os6^>bnU-oG(W5*{kX4WZA}WPyJ*4`> zNg(h;T)EnzRg!Tq2$yp*mKehjd9vYKaWv*iCH?BtazZc|Aop0XTVm#wa$HEX#htDn zwJ!Fbz1>4~q03dgP|B^NR61wl46Rxw-S>wl(8)RFwbDM@eAQ%`XL$M$ftthqmBJlX zPJ=>jT;ezFTj#8^?fQj3MXj=gxhK}vd~G@@`#uDFb7MNvg5|Vgwf$qKR`)($p zt|sQROU<;`Unu~Icpp1Pyt)3@$IkisMJ65PWQ$doZ#4EN5h)UzdSWX}t_-vq(29y6ssr`0|sF8hji~eCSe5;Eev~Z}lt{13ZI0<6aTF(9v0YHChvh+XzLMAXawnx$8@yJC=*)n+h`ER0qo~n9lf0 zzJ=YvsRO|MU=sxRyubX1oJH#Q=LLc8YKJ1DcGt{Un)yAHCWl^;L9%3mNgp~CbIn~E z0=qDzVzp86H)AQ-pEUgtdV7y=o>n$1y<@bi2ba#%%wQosZXzaqE=+XcaLS^mABoQ$`-{h;&RKcyid>T773Z}Asj(VNvi=G zmw;&yBd|M0FJBl+TmAJQUMSQPU#@N(Dt2Pxue-Cj={oVPeeH&Z$tpz+V9(qHRvFEt zj1C)~=mr}62OJcg_yPjN$Q6BoWaEeno4iAX@j5tYh+^vdZl6c0OyP0J@qGv<29$I0 z!)bUPiYS!jX9{blIn)h=!F@<(6_bgDCBakowI~OjgoYxpq96wcuqf^!$Gun z0ya}dNW4~L@d2CnzC%;D+_`$tzI$8orJw^;9{87ylg>c{Q0%poI~t`Z5b(x$Yqsg)P5+=^__I=!$;Pwun~r;Z z?@Lkd&*5^+M=A0k+)JpTdZT*9Q5BvwekYc5^9MhnOK4$wH2X;&BzToqrIm^I&&3^D zcVck)O|g@Unsvu(^cx=c&u6nTOI^TceNLjT2e)c;bER|Bb2f6mx#@qgc-)n~@nE<6 zGL!>^x2Z9ZRU*|*Eb{)=Zc<~gmPM=lA9(UNt`JxKZy@3u8;_JvMMu5+ps910xHDRX z7}uq|=1#?>a6yYPGzIJ27!uc^hrdO|y8_}cXNa+&S@P#L82ZW!4$pwLRm0pvi9^1H zDwFX`$nPEGXI9GlezDAVdHo&&ToCrrSo(RHxZdqI)!J`AhO{Akn#H#y4{25y(1mCn z9Xove2nCFfS2L?y==NqswDm4eg0(U81`i=@Tle;a3!C4px&xFdBFNI879YxDaf9J?67%m@=!34gH_Wfp zXOJKm{ni`xu(_3|I*HRttV$JrsIpSXIjA2g&vA4YFrJlW3P{Q$-_~>?WU}P3uFRR= z{YpA{LNW`ix0A*{95=LZ@8gfhkxO16`F1+x$y?gvd#SKC zDPvB{bFGm%SgfXUxHP}U!~eKc)u7f}$q^ga3RYDa{Z454sl>W3PBnJbpS#y$(xrbR z;&hEY@YC-n2)zC7C*UG5`$d~Nq}*9G<2qi}aai1Y6>|^^jZHk>s6()vtXH`3XC_|G zX%aWRb1OlduulIdAeCjCh${cw$X@|B$!d^blFUTs3#)E~H88p92g~HrNM3%VrOcFl zG?5&aPm}0QyZK4|&>;6{gAE&>r#>wjckYm3@Fn-?e?M4x2+#g^!8oWfyFd>|XUcUc6oSH4o*H0^mNx4_| zQEG*0*lBOt#7x+4D5zNf^vXTL5}|`;E7TNs+WM^_50APKg_jT8{j-Pql?`q+A|I2g z*qNk281Tyf-?bEf%w0BY!fHyg0CHp=9NRz#OPZR2v#Gg19`c)b^ z+RgvCZ3(@du;y$d!8u!luF&BTk6EQHl}80<4rpbpKULlS;Z=7gpMArO1sqcr zJaUI+F2?0rsFx?su*vf5eW{09JQxJdw zJ5q`&0)HPaMeME$9WVjtlF#4G$-gD6DJr%c`rJ*aI$HP~!diVDUupsq9_#lWE#f05Xc%%c@+7%G{12l3T`O@SXt#O#(j8-}I7^T)zk_3`J7VW+%5;Gc_%8ZZTn)_?fbO~7J zB{hU$C1I7?k*nN1JNnqKg3r0G4<$7gPOqnzX?|dlGo7=StC_2rQR~r>mdhokT(gG!W#JF-n*%(l1?`@@MgC?3(U>cCLG{s^x6EG*w0= zdAY5YGAMe<(H<)EKUDQjGlbA3MPvN&aI^64=pQ)6#3@=Io)^2&7h>$d@UthUG z%+7wHyksxN#{tpv`PY1sB~NmAxp7*)+jBw})0AYX9BcsZ?}aR+IAk^&7NM@F<16is zfHPhR=ihunhek;~z2&?&sb38v$^1=`_|@L5A+O?*qP6}9n-O&cr?7VYGed@gA!6-& z(oQuY%Ci91EE9T0^+LV#!qH~TP?6B&c}ruWxf2`?)?p|ZHaqyZifkjLWS||wQyM)^ zmXQ%Gk(oFMT%<3N+3OBfbgsr%%8H7999%dkKXS1hpZ&b-RP))MTkMRU(adBv-?>J; zcgP8(9~RCbD`}S6WL7F(Ck*i|?4lXUNIz&jCCK)kX9~~~n~-gZ4MSRkG>fAtm>H-k zzB3Oh|05T(k!=mWvQE7#{k*eX9c&zb$v7*nZusUz#%dkDDf|yS@|!*JscF7gd`W>E zhV9Kr3HcMb_d#Z6h9r&jK8>7C*I#Ev%Bdijur)$Y^CxVmESHYD8X^lar7}5wmZ1eB zKLkg|Yh=@iz3h*Dp&sfO+r6q#1;w8&^P8~OBr2^u#qP;Zg7O-FHf^0EApu>M%LIir z9VwV%!{dM~MqOU>iP`wB^%6qO9hhufs&_D4$Cs@L4f7tl&JGv1$xA18(CgF|}8+h45e(y1_RSE1VgD&qe9eZs}11jgv5gU+5By88WO z;+=O*$JQnly1F&~;~L7n4wR;pH_sj#m0!>F`U0z~EyAo2y8ShIGmYozGjRtMf+e|% zH%Ys?P2E-=+~$A18HPWBbl=p2l-T!bUOuL})yC|34$D!vH=J{Mw0YUU9|At!Oew!BEQM_q=m^*SX zWA*#X{M{K?uj-8C2Q7zZ9j$F!-NBdjGDl50-`9S0+b~T|!y;t%ZfP+3t#t=+%YQ#e z`mw-|- zOFc(km@co=sdrdH))mee(rDMMH66YgZhmW<{$aIc^GnT=>#>D|K-G*BvNvV4;Bg+8@L=XTTn za>v=|8F3Ny-j}EQX%ipYQCuaLWM2)h-j}Q)w!iP1fL;jxP#~xKug+A2brE<(&^l)( zW@+1nr8M3-PN^+DUO?Ysvkp^Kw(;Q@8Hv!4#?foHb%^)xH#@Uz7j=)8^L}u<9r||( z6~VxbWy`;YEjZhrtbdR)5-F)OgY8kS4w1Sixb)Nfa4G!7<-Zo|CxN#=9ln8=YlFaq zFR_kumk<>E5z1Y@VS@6dSJj2prB#oM)QR>BHGDA^G3?tDPZ`(wOD<~;)l_itE=jJv+7Kx*_@jLjgU%pRCQ1I z>QbYaTYf)?GagG5Uv6QnG|l|`#QDb?6I`p5zH>N5wPYt*o;Y*NFkKEI0cJtvg*AjFp;dnqs^3w(7e-VS z^FRC&ej%cWYK5+DVkyk!9w8RV@vt>P$x-$DX;epy(Q~l~y$QkDICHQd)JF(tu)jhO zA4{hf?A#fSU&dYmr_CvIzn9JGUVN`O8*E4B75YE8uZ99q@D4ZO(`zyZPV~OM{BZei zP)W!p`Ma}Zs^38QG0q)?tNl9a^DIaZ+I=Nw!qzRWM8`n zdhZYBo^j`O^sC&bZ5=g{D5S2D-w1R+3I{GE(6H{2ZQ*LcKfVBoAk?0jZR%_g!>Mwz zuj_o*O4609&+0uJMUDMm?Ru{$qZ<;=eZ7-*K`Z_%He6WJfa8$RhF5=7&Oh%n9rk5w`ODM?vif%|#h1{w{@aa~0`711Z_CAE;_JkvP-jX=bx@z zRSW%JS+?K>NM?%y?$O(Cr!T^tb0mO}YZR3G?Tn{B;c*0U5RDpE^Ke5-`PdV-blm)+ zJxc!cmx#X8GyKJbgMl7*{7z-$%Z-q0n0w}&M_9HQOur7JhjeYi&9_ah_eDl`xK?O; zW(gp|Le+cLs|O-cS@gtzi>#l^S*ll?j5dtQa%yEBql3q85c?-8a6n2{$G?92o!R<5 zk7W6f^d0xX#+|152jhaRMv*xa1H%BX@Tt;L@j!shCPA$lB6UIF<5tca_yqG@Q4DLNfpVz z9^ub?M9V}xQ{-mG6Y4JU3?-gemeBOJeG;N>3Y-M=s&PrHdP}006+a zrAlp&gz2A)tkDdFKFw`({SFbKyiiE~PkEu$C8~-wnLRaD)flf3mN^%zG1-Vri`j;< z2wja%Y`s_Xy;vtZf~<5E5+Q8H-SFsr;Q`zf73bjxGR|kQhq=h(3RK;%{i)0$e{(eb}%Q#1WsL+za9oi(1T~Q9Hkv ztnE_VJ=<{hdbpzUgK@Ne%h>;v!d>bR`9evs?=$5q{VCbP!92y7)rKr(KY6qHQ9j(4 z9=vo11OPr9h&?Lh4Q2(h?zHnZ9p)vUnMGJQCf)-CXl#!HFj1$44lNaGdl&xaq<_ZQ zyQjoPV}kqPfk3fY!5%2&lz0N3>xY&_PbZug_*7DwWC-_$U!ufUcp5ISHoR@k5)W4u z)Y45e^jCh|$vNTNW$*E9Oxo)65!Hoze?X(*t?R4?$MTEkE#|SQJI9}qtph2moB0Zz zoj(mex4p0OX7|a*H+^RjZAo6HT`}Sc1jc1P@NZb?ppl0+ePXLl;=!q}{P<>UH3oXn zpYiYV!it zi5maFJpO2xLgem!uk$~=!Q~D+&5GK{vAgy9G#q_yD5dXIdt0d|YW+^=Y1UN%#$+y6 zR?wLmW{Z6NMog%m9z!ooa0RSxTLxk zR(Y~0qkf#X>%gtpHtNnxZr*z7-&tQ^RnQYs-8) z6$Ml9NQskqOIXk&5U3iHV(I+{6 zI*e;M8X5jQ1h~mFgz(%AWlV?DWH-hn0}P8VDh@jxUJT?jlld0ftldG(x@EaXaJ8uL zd(=~IWq-Rxdv+sd=RRoYYClMJ@BZ7S%!iia;2SN*H%^pab^5(TentzUO+UyeMxHa{ z$o|<_UAP);NEFv;_{@1a z_F7aB-WzmOPtSOO!OUH(2`7sfI2MM(PtWTup1l`U#I;jTgXOQzFEx~njF4(cRRmiP zs55c>1xNd{=62q=06OjL3~ObaD9x0cM7muS_S9kL)cSaXXjmMVP!mXWHxrj!rFwOj z`~oQY?DejC&+K`Ei7|(*$*{T@5C5uc7-EtqeKjwOK zKwRiOsw97=^42a&w~{5}%{JKzjXx%L zg5C>>`%O;zw{EhDVv8R5mHQJE{KJaMHO}!0gmzJjy5Q8$E2gr@W;@s_e*G7%f9C$5 zot_}s)$1Cswe}sDhV%7WOY4hB>|%j zC8MRILwpp5rChfd<1h4`A{#T1HIZ8#4vn;+I5JTN>4i0CYcwit2giW(!!j`{^DX5~ zSCo#<5>qHxn0}|q{xJ>w+npr%_>m|PIw@<9kNHk@!w#bmou=_YpGmGNkeL&^DB~Y~>mdG7{d*AD)e*!j;@^y+ohAWq2Rg%l-gmWMOD` z$~ot`BL%3A-u&Be`|pqV!c`4s0xYzW{_0HWisnj=#+AnRSO4C5gdo_q96l^UDG)Mc z_TvIVZU)wQ=OXnK-%oD+d33l1wnQ#p#9{ zLS@*O3F;f~7|&2o1k7KjVoI}rui4bUaKXA3A*$M7+MsZZ7+Qfs^{ZWXa>~zgs8-p6 z0zu{B)7Vk<6QJ9gKiUIrj_xOrL!mlI?}Q!Zo_VK>Jro^o33&eg#IFfb6(71D*2Tr$ zK`39#oCu7aJ6Qa2`U2M^g7j&gGgLviDOnT(0!u9TO4*tf{0(q>3x7BeyC!!vtj$!H z6;6!ggy)9m6z9mRG14$vG`k@-5&9Em6OVCX=+Q9K)dmC8OvRdafjRa4(?WH4vIY|ZeopM`vJLYE)G z;fZzO2=V6`J@a4xIw~n(Z>#aC z^vKmsy5?dBt#^V9cy^Y7U{ikgPX47GdoyV}0@8Kq)O>BFJ1KHsDoQVP`+rBQ|JMx~ zBCgfHL8G4WeHxJQW{0W=+ONAWA!#~HmKQ+o&0*Vw4I)2DS$a)9(vKIP7^Y+k1mlR( zKDP9??Z6L2dnnR(#v+lP2@C3T{i}O9^F9C|2QJ6};WoKnB$F!ukjWNJ+*pOnVw-GT_6WCyA`LI zy#+-s3;oz(i)#iXH@(Z#4yrg8UzAmIvUAs&1ChQ()f?>l+jdVf9V{w1e$>z1+hyrI za4V9w39Op~r#-h@xV7WUvX-ZVBv>jHo>9%N?bqO0oou?x^?i$QK02MebIyC#-pJZ` zx3?x6Cz6Xt4X}e#eS_cJC$F(@u?;g>w8UYbWFqYE#;)S#wtSy;d22IbPfT^dlNNX8 z_cOlbY^72hVMgztA4iVttXUqxnDwdA5lCj+?6vuxdB#>snVuHgpS9wz z^ybU+Ks z@q+rKG~hrjrZ36}55nEwH6SclAM;)Q)dXpGi1?VMaC0X|M#EbN5yb`~EGtW?UZW}D zOsSq;+)H&3hcbeejDGN1-=*{m)Co-WtQ*RSb9BeE2F>Iv3lukHCN%w6!sJTls4S`P zfL{-NwK*(Ap36E0tTZQZ&~$pgy&KT+mH!LYRVTc>t_!dk05;HtALy;zNT3Ng&=}!AO0*?6x7<=rQHIK@2u0dQAiFkr~X(w|Jee@4x_? zbSk;Pw3PAox4Dznl;{?_x4al(`2gs}dU`0VIF-clU zFeA5Do4qW&Q7Mu}_V5PJh6#u5#B}%$%>d>T1K4OTzh=?+4OvqEoG#U&FSWXhArz~_ zg_?tXJXCF0iJS#I7d5 z6~n5HN+-cokzkM{7ObP*G33{FTQt1tcFuf8l}yV3hsW{CAL##Jr2jou|M64YZ3lZI zK*n*WVn$sd#T9lbTtQ&Q16f6!&9j!zbH6$07M|0`4?^<~iq?3vQ?fJOrR4@RW)7*} zp>pnxTy-D2DjzfBt`(69kXMzf+9ECyyD#rG1DcyQK28+1h}^g>X%Io!v#FSHOHkW+ zdGBD*k)E(<&=T+exh?dHM8QKW^wc0~g`iI6$Qheq4aX=YjPqR!)SJ-tkFv?TrB)I2 zI94Uf5=ndI%s0>~Y@<^`_kx9lRRACsr54QdEi$2N5rJOOagnpLOA=-}@cQsNl~P{s zc{-4wf$I+;PMA_dbZCQMWlS3#BWw*wm{fX$(D+h`mw`SicfUw z^j39);cz0k;gj`~Ho?}9*BjTFudTi)D2Vv}lD+ds6uS`W%VA}qp9+K3xBDpzcgU-{ z*rEcyC`G7TqI^hp!PtiWiBb6YJInptB>!bmUB$gp)9GQLzM&6fIZ1K)q-8lRVABUX zwEOM^tN7xV3q-_MZ!-%9hFDuA?XtM~f6i~h3vM!zIfK;&=J47jK^2Jy>XR+B&ft2^ z%T=xx8KWP0#na@<`|m*X$BlCPRm`z$k~`{Oxq9qBJ~xy3#5PhgT)yZj#ny=i+K<;} zzNy%K8C;t~)1vcIVI~M@9r$3<^7=k1&e~?#a00|5pCf78OCt0Z>?Bar8;tjo5tp6h z+RUU|`)X9RVUaLcONM&Lx(O;p1>JlIVNo4fwf4JV?ebvx$iyX+;h>WixHOEMoC~S; z*YRuFb>V1tsb{^6oSeRMU~f+k-^ho{X1u9s0!?UP1jcdFLR$h}1zhpmrs%YtB`?$H z@;-67j~HtCbDoj)n)^FA^qb`q+wIrmez1MrgfFC4ngS7dDTrI={3Gj`2mbOr+b}PI zn8PvR1!Mf6IK6+sb(@pCvp5_2#Elm2d=xg30p*rw zaqX>7Y@?OTi3p(9eFAk7;1v|T%+?{=s64Y5LB_1`1bsEF&i{mJ*#eRM&S~l^usTh2 zboJ5I+r9?O_~aIXrJ;Oq@a#BiE#Y_-UbOxoK4lP*l5ZyY`M=*!{kMPQw06lldafP(qwI_8h z*=c9H=V_>Y;7N?e<0Fl)O-MzEu5IjSk_2)b4^IpB>yeG|y(tG5Nvc4|m-kC{710*w zdUmReFF&a!;a)*`diU*;FPze`!xA1;29@{h0_S&aC(aD=G-upQ3fZLhnHI%do~)Nj zdYI5yD>e(JB#@8T9ea13g;pd2STnFm9Z*}%L410JH;i4l+rBvy{!mLcFI?`9YOhpN zZl}o5v3X`#Zva;<840(<&{Mw5Sl7zSdxs}EVTBHlem;jgA*6G``_fVy5V4W>ct-VN zyIkTEm-Ahppqhl>&EDE`v*;U%zdW~;LGa~U&@Ho881VR=BTj86Dwqk_Z z2hfRsJBgnc(d*LXfNdM@q9(!R54uE({atG(cx?6|X@*&@kmsr<$AP9r|lS}mgs7LD2PJry|jFJGvR5oYvd;IJ6_IHOug9SzO5V6ng=@)P7mhclCC#xIJnC3FiOC7OJcqBOK3V9ghxyO$K!%$O%DVHADp>g z!A9*B8C9J#25@#>26m?vP;XaK{+b5=aWj6sIe(r|$n4b|8b7!_@y$4hAHv5Fz!X6`i`NjxY`X^G+~IT}$rzUp@TD<~!L?fZiZ-#1%2OBY~xjVYS7=873gXYLqZM5d(fB7`a8 z{UmogRlJG}4;4qLZq^K8Z#KnSvmMV|TeKLQhuvL2aI@F6|MuqRGoib;MY?@|)r%FgZ9;T6Uz&Xa~vE`Td69m78DCC}C_wjo9GQ@;u(zx>9?2mi=h+2&nu{OqTgyne%>JY5QnE1vc3%*D5&K_)q2`KHcVVOb)E zI*wV}ETy|;8dX`-NPVyz_j@Bl@Eot5yBW!x?Pu#~!&at(toW<}qRV{etVe#A$epEg z5-wKaNtd`g6udt93TMZ>P@A$eA61r@mV?4kVP!gFKHf#v^EvkPdDCI(>@#n56_61r zX^9i=vzP|OqOtY4E$J!5y(iNz7Hh}qo}e4R&E;yg5oQdf2zRs&%w2d+!oX}UZR8e< zWDc}RKKpIOO!P7LTR~S@Ku0!3?A9LEJvO8-AsWFP+O6lbFZ3<-c^o-IFn5(}cio#t zMYhzIlHNmZ48^=xQ;;JRkP6?+x^smLF`aE40h3TIBuU1st4Mk0qze&nQcRk@13QQP z3HZNc?J@f^6f;IIX-rWq#v$IQj>d2b4se{&H)T+o=3hEomP*`qu<*~DoEo`W@8j&E z+ceivm26$ETY`A?BFeBvWp6|$S{tc|s+3GuFdes%<1QZ7QyX8iuY>I~z;|L9N-MDU ztj%7*b{n0min`AgtZhVYeF8q-y+zMIi_!RS=ODaE4*_@aL>;6Kmu5|?&s}zqNCUow zP1hhsHN8iY?lMZq#6HgWgfBUx|F#A}+p(L>$i!nOJoo_dlN@pm_FRBN(E4U*r*L#X z^jbg6ytDw~`(!^`M;!_t=;4vqi=9X6sGBMn%I!RqdZ~pHS(TcF?AniFDZpIw%Of{Z z`B(#p78 zLIO^Ui6D!GX@#W{<9H`DiCE zo*5*4JR9z%bvw5hWhc?Fx+=MRN0}$(Y;41wR>$uqBuVBfx@fCT$B1IJ)=WZdC{Cv^ z7Bv^(AGW6sAzfVD^o5{VAUXFU5N%#hiUvCvwm!C*qT|ny2x^0MSp1B_jObUfF9ksQ z$Xa2vSnl;9HC=L{0+zhpW-}<|D&Vn%hh$8Is>rS%D9lnk5&b&W&h;! z-=h5$`9tpF8?f_h?Ey6H3jHdNTiKh@yH2oYi3c7V5p8=p%#n!Zq$s!6vYTDBHXy*l zsCqpDw;PkJxY)FvZ{r#o;g;}!7W+V)fZ`By!9r*RvdvuiX^aFh5}HD~@d8o{gqN3c zro$vO5~=C3&O$WQ-#lDNH>Bx`t8V_tS^kN41m-8Zz>;r19=2NnSm?rTTiePri5}O} zW4rQKtMp4ZkTpv0g92oe6m^<kHSP*nlfriQl%xKd%Zg9+cuU(%hMF>cX-ia z>uivSoGeOhH6{NID9?~N|SK;`d-66L}0_Rjiw}Dk@;oPt%ukDiTkv~XU-D` z_ie8&d@5dc1i$B=?vflqjfPwJKq~u&Q4OEBWMNSkKExWHujL&oZ9(Si5O8`TZcNtB z96gxc?H}}rnc5Pz9u=Y6qE`$=4d_|irb?D!ITyZ5?};enCf(PDV8BK8p?iCfG-|Nn zI1QsJ5G18hfRXT-<@wKBH@uydMY-4nqg2!GylogBu)kKC&A+XiZ!Y=PRvos1?CVkT z|4xRg($!+E?d&H6Z9HjFAhlw5Ptmlm<8OuKkU|~Rq+cFg;IkSIG38ylFZ}Qy;D|@n z_%?Ii8l#6?(-`LlN?I94$t#$`9L(QLzAR8NVy5 z3YN!i10^SBE4-f%i4|gMjksnvc3Pl zkDE%Ibj{M)(#72N+WSQ+-zXhN_DwFN8V=oJxfYgVFKRxe(8y_eRORA8!;+k_t+6IH zqA=e?I=&eFH{&Rl+QLqcEk(?8+F8EUNm90Ev%$ z5p|oFZjoh65*PQ2fB{pq+-LxJc0aBR4H-MKu*j%mj+$Z3qi1YWoTdbuaT!4h$YJga zPC@lzQ?QPAYzEmf*Be#~Q}h!4)^Om2BH#Tb?b#$CS=hM0ub28@Kq&S4#{Gq8(<9DG z<*QZtS5}`YqDT_a7NfRb_9TJ5u)41Mx!$fOMTp00feQSsy&Kkb*O&QxsB( z-ej~YcH5Npselb%Y>FJ80W3`Tdnv+qY_g7e$~U=P6b&6NL+Y}g4ao9Ly@tRrI_Y7R zdT;c##od5#;loCp&W^lZ=8Zuh5@>(9EV8DTTBt43KAj$bQY3X7r-4;ly`7%B0Ul9DadO@QSuqJ7nGb$z>J2%+g^Emch-e34oEc%QeI+x!Um&KqG}p$=K3GLTwC76HYX`5 zA5_vLr;!e+`h5rzjINXeeAa`>bYBr6Yu#h2?;KO@3fbsU#J-&MV-NY&`G&ht_j0BV zQ>3d(5m__R9Y`4E1wDC&K#(cM)nPNPW>JkpWt&Y*%7ad4)9~}zQnh53%DWx!jaGfs zoRmL+_2lN4&&&zT9!feCko)t`>e6m~3+i*!R33+5N?;-_pH#VEm`|+)f)D!H#DaYP zfd%}J+YdN{cS$47_i{fGKABYQIJA-(gAn3_JgK#&jEK4Xn#-u`T+(CcqhW8Kp1qt{ z`X}tfW^awUtw&ymWK&X$c>3n^i1>UXG}^gknPs`$X?8x7)^X7uNTaUSUTba8o_g0_Mdw&g z_RU~NL)pu)@~}0jM^v8`{1n=DFg`J^ zt48MYi=Af(sy1LD0vx-aJl>!aqtWscQW6xM>p2?>YaN|M-@A1t)Wl2WTZ)alzZez*ap#nUY`&WpBn)Pf2 zA}D|WavWA!6Nv4t%qI@~ilo$u9!oAh-XB=E zH0>_A`|fJ1-^Eq1X?NTquY+rmeK%vmQxxDQt~mM&Hha6ET z=S^qpT+;a=;RUI0VaYeu#JPD3I9sQwSJPXQxQs#rT+Kh(H^brPC~H0zm#pU4>jTQl ziIiskw5y!A+lJcEyOwr!9Z4RuoP2C5R6?!DD#f}AZ^wy;FuR(z`FW$co`y1Mz;u)i zgjr;&2Lj@Rvqg1^(Izq=jRG|KHgV0d9nXh5d^8FQ?!@C!boCQY$QF{$jOl@F4 zj-i;FY8=nsZw=~{hVFGjB&oo9yyo|dO2hZw=;)gkYejq(d37m3ZB{|l=+Ie$Z{44L z$Yt8@g{xPQ^{qlv5Pjixm~?&mu^Ccn?rbj+chU*VUYxDb*$5SoV9ISx!mJI|W<8GX zG*u#eG(R6d*8^)tqILYQfW0sSh|jBL-TH*HAvCa(%TZB0l2=xPO}V()b=R{YIn80- zrb@o*Z$R>To-#%A{#0{z{)ak8_DE$qSX9YeH{=#a^1a)KZ94wfhjuMJw2RpLzE0?H zhYINTyCdu^pE*K-B1{iEptFmB&bEhT$RPQKUTh%wD`lM)5N~K|ccklt{BqE{sYV>v zm(^ERzvq(sUC7a;XH%iPdvGd zzZlt;q!cB5tyPVoz<`1->v2ek>WXhYd*Wg-B!@bqY`&l7q3gRcF2;#VV6P7i8l;hQ zxouwh><(Xd_74&b9k3cj41mSHW|u`hrmne-zk;UEDi@r^WM$=6LgJ*jmnZUzL{$ge zfYS2nKqtJgjNrkC^j37XjuiE3MVF4k9A=gDOIPMEwbo(r&B&$SmiWZ9newCGSPmkioBh!01lu>1pe+W zpzA>biRAot-`vsNj((GnoJH}{tU-0AR(~myH;}@?kW}%P+&RJVWu#NrqKvH?pUO zQR(Efk)}$LP3+U*;R6cG_ORvMoD>}*U%{dHHoZcB-YT?o(QupSpaKBTZ~7ak0See_ z%#lQtvX+_WjVf>5BJdL%sR@haLnF#nhXoB$Sv}DFtd9EA`-XBbd->l!h;Dqn2>F6Y zH${KYiPqMm#0? zAj;1hW6Z*>Ymp~8KR3U}EoBH>Ra*4)FS2Y#rdIGhl;bAHsuXdD1GPW0wxlT8vcg}1 z6QKq>1hzgxAco}NE2VlLI2j*uWBqRoW%?9^^Nx9NElyIalz9iTbx2;9NiCg%-Kh!l znocK}6HyM`BQu-446cmAO!$cp4Vcyg8yuP!1`UFo(2wZZC8D9UtG)tZNon0e&BzoW z}owi~lZC>VwB9YS&_753Z46z5Tnw}Et2x=7^$FJy1)VBA!K zZODZz__zqC;7qf;qXDzb=CAhh6I`j4ESBE|ilA8WSgB4vJSm9 z1Cg&r=xm6qGQCVU)3s>*)bFdnUB^TV$eL%{-YMKx8uoT7;xq;7 z<-SJQk0W*$SCKm9ghsqyDp(|*SvKn-*C#JlZ-}Teg{QaMWo1vn5DynMr%Kac@`s)) zkVyb)Z)d-$V=$~Z`{&*O_@&2Y^J2k06H#HRdNw+ke zUVDM#dn9h~wQg{{^u}c(tWZm6%Ph)HM*`I}-v;;`9ufSbA|c=-x`n?2g=oI>2Vyz2 z>rX{ag*8b=8O4XWBm8`I*L#=;^VL>7>z7#$HA-#&wb0`cv3u9Apkot%<&ZvW^a~2J z+v?0ehgRY4&DD#HQnVLUUFG$A+g7y5rvrnub>yePI&``rW5Ubcl7n9U!||7D3%aj! zOGZh0&b>IUhyPV*x^8>-?gSWZBQDOfBp8j}P`c=m4wf|+Me@;yk zD?Og)wW|30viS=?!dgQFekHFIN?TmfK_yW(*8S4}>iY;Q+*j{pV_JKD=US`G*PX4b zfh#Vv=`R=M7S;c!p>pzofQ;R14kY-UOd6Sh2wwV5)xTqpbSi!t0>i-l#G1=c4P_nL z7NjcNTjw$6Sy+wrEcSi*e0GdZN_eqDZybEO^2Fl^ltmOLye${L4{yv9Fu7&?R!f!E zhB4xAA)v(CJ6wyJimO`2GN>y**u*N_D7tyM(SFcNuI1vC9#|iKnqJl_fT$2(Y**TM zRB2EMvoIzY!oSkA359KW1YQ3-w5{B57zmEHc~O{gWP9WC6J>tdQJ2{j_7;7N`D5Qb zKv!@;nqURsiFJ&D&*(dzRFgl}j3rs$$9?P%%qZRuNJD#LvWlINrQtS|u~%9`XCfO{ zh`pZJ6x05Jvj0r+@hr7f;@2IapQl^K33%4Pk#09JaGqL{x1Nr0L=#s!?E?a((>tK7 z?*Yyb`khfq|Ia*s^H12ltHp2BE||a`B1uQvR@h}OS)Zog=9vC*23Lu{!^G`9H16A9y->C18dqtet~HONI7U?LO|D0dg!? z@0hDiDpUWb5#p3I|2=FFHxE)%qbTjRkE_Hf)rwY-e%kS^CcPvGhoz_16!y?qqcBfL z!u;NX=xH8DAM@Bn2tmPJN{lj zTb09|BjMh$sQS8H*m2p0yE?#uG#TA*Sz&*>H58A_0iMLOfO*TQUDi9rHk@%k-oOv4 zJeK%_{9>_FbNsoHeu}9$5`RSerQ!Q9;kN(6<-kYgORq9+huGyPoYJ?Gqx^zrc1^XD zgQMkQ$@xO=CEAxew9jx!lQy0u24HxA=%vK{F%D_}3gD%r@uG|c{6J{6kC&p#6CT`^ z0Uh6502uF1OGdN(9SHy6;2t@R4JBn^?w7WSyi+rU+ysXCby~vtr^C2j%KOf!m3wwN zxXbw%NM)7WTQ1~@d*dV%6z>Lb+Sj^h))X~dL`n@*-8B|JTLt&9->7at&8(XmfxVZH zI3tm*sqv{t8~zt>eWz`+6!jpuAxO3WE~lqtAPid%s*=eusRnIlXA)EXhTKWI_NOrOaQaP5w8((F&7HO zt1P@PC8MBVFfMzomlHRLTk1j ziI0-j9^QA@eLYQRy%$3n3^F1@G57N0IkLoRYEQW>7vvYJ4eTFjEOdjwdoek6VyuJQn_VmF!+p3K*Q)Z7h-1!jqYD>&r?mF& zd0@Z@9n3RAUhQb+Q@EyK0wg^*w_fzKfpCq}@rnZEt^-K#pOB~iJ?=lr-;+2!$(ZlU z{Qtq+G*!|jBN8sS?Eg-vv>9Mm`CCVI|3|WaA1qo6@K{;(mAC%MZvQFf_tF4j3wLy* z!cm@651tY@F#E?WN&xuQ_Qk}fJl9`H=@jk!4}c*H8^Ond!CG?tb>^&Ni&XGOhZx4b`8bK3Ods?A^U@&7Y%Te$^9#4TqQx@wiH3>=+e zlMK;ARobYBXMQgEk0Zn>JR6Oh3lx&%5y^-~2-Cz1T)qUHBd!oQ@KHXU6`ev(LO&jH ztyZ9B=QJZ}qT^AFo5L2)Xdk~F`}Pm(xrINMPm<2pP2Jq*=@Q#ycNH=#LoBVI;EeXK zfEnV73(%L0xYz$mSl7uDhD2qZIv#%A#hp9J5Wwre^sf}Bt>0e&6qjuNY9J(BXppk_ z2OK&*oRt>im(1J%F;Ul1b|5>OajASk^BQ&!ZM}36^36t;0%oG)mx%@G(9Ft zc~@V-(pE`!lmpcHkl}&nN{ze9@SvJiA8bw_D1d_1La&cV<-oY$Ef0&PqtDQYQ3vS!D}{i z&j%td$in>WgnjmxP25nkJjJ$dH!kblwTI+l4U!S!^o&!Oxy_GNOuHMFD4{t&?bRZO z<~6>?ZCbWq7QP6Qf$@dFo}olE;zoCM=?I-$i+80T0*!WsS0y)9p;aOd)E3-s5wrC` z^ROaGlMrD^vKA#}&3LXj-@rjI8S)mUv=_H05cTJc8Q!nk@|&;58Lsm8TRSUm_}~Yak9GF5K}X4 ziOnpDERqVxxv52UO!UYgS35o|wtNfIZ}gYi?n`)IRtkmJ^~}#ua!xTUxHD+=39&S0 zB}FMHhlTJ;O>B-jpmIv*Sy?n(A_8KVoiSzAfw9gom>9jD{#a92#B=oC=TXbO>U!Uu z!I?I?hRgd1cn@5iD#h&I>@9R`;dDf&sNFFrD+sCi23YCuWJD8%t((LcAe-VUbP3es3 zyjZ)`pql5`Wu}L!>?Q}eXl4xqbehn)P%-8wTFHS=M47oC*RxqJ+rUjPxmpow<{1z} zn!)Gw%&hOFwVOI<$xI3^k2j#|(K{MTxXQOX zlKn|-oR1nRN-uy)*)#%|=6O=SqU8NBKYQJ;I%oApDkli$9FH-;b}$?1qh@mSl~{^J zo%>8S!Z>^A^^%;r)2&+)<`(LCZYDO|_Hpd?HjC=mzlH+74NDw{-M|FqT%%*MdbJQY z5#KG$>w^gKzp<)ym=ecOaXPEKjK>u{%VBW#C+NB;eAS`-5mS+fNOqvx=dih^ow+C1 zxtT1~WLMI)x)b^YOZhd{7xRhzSyO6v!<*!l*_m#zNY?qyr#EXF#l;3YS4XCzo^Ezf zZZK#$$sbA|+XO@Ru3#Dc%(vp=CFA8|)XxoSiaNI62BMc+rMH5FCx6s+IolzEIINK(=a{igCnEW@NIQE9+jc z*@C3h&2&#E{0`ce+_V>0*C*-LN4;!E>MU&&h6KgBmUXUZQ;YC*SurJqMQJgLumL-Z z@Hv0J41a6OzKXX^p1g-FrA5=OMjj61WKBVnWX+lqZh`n9*}!bTda!jv^L|atF0|G1 zpKV_pJ~CWd9*-9?NEr6Cdhc*3cI@tE58~=0pNk3kU$>d66!A=`ts-wr(BDNNc0%T}p|j>ud@F|oT%L(R0kJ8a0Kj-_i|=kl>( zqouBOkcKIQLT2ruZkId&td^o^i6)s*P!Y*mJm` zF@AI0UVp>AL;RF@e(+B~SP4{eJ`B4fd-XR^F^|8i+US zlq9G+n15y^QjFxxuA3Q&Y;6w2KhtV(+jssqbicBMIibdBLEehbFw+u<)sQQfaD zS#&5EL4+o1wh9t#wmpwmAr+3TvkFa<`V9BYcbQ0aDp^^IQtG~% zUt`G=uSE1=l0^6S_cWOEvIMO!oC>sgUgMv`AR|J2SQeB|d2yTohV*<^R#>U{0VecY zj3p^`6WI_J41AoBa~M3{z}BSAR8#k>2#bJnp^J|#T=uR!4?DNDYiylU>l^Lh52?W# z^vKrhOvFOP*tOrj#cV%de>A@X_7f`k2+0wjfTn8wYuP6C5xJ4zToz_fyNsaUvwts+ zEjOlmltMdBPK!(&{cPL8;8XT>*cKDlA7=riK-c_9UWB*xJaXfyR!*B?(DQKC;HB&H3X8T+K2I0IZ{8s)Tod?RA#rYFhQUlCH+B+E7Ovq zJ@u8nkI1P{MP7;zj${3;k{Z^Z*apXd+BD?CCrZxk8#So?AOb!@+HALJBgAq+JAHn4 zjs#gyF)Hnbzy@~BDwV26&ywUtcIrHzO34d}s<{XtgrMV`9hFWplMnqI=Tb7a2zo^l ztWK>}a=u)WRe7Rpi<8o}JfI@9b8i*HzPk%&a#%|E03#!=iJ-X%-t7t1wbV-4j_5Wj zO}ZInTeq-GSC%<(HvP-e=g~!t3Z=rVi4TWRV1kcm6(c6W4Nnhg;mN}5mgy2#ErM}f z7mx07W;G&_Y~vm2H$f$uYE^Oq|6um=Kxm|EXDfeHpZsHs;L?(QZ-kTr=dmD)^A~J{|wI zQ|qVD1&RXq8oYn1)|7bT`=zDUEicm=d?|gwAo#B=pDt*|bqhn@v=61kYI9sM$J0kh zybcNM9Tem{4na{UHw2S@Z{?deO7e}u?9n!AeK#l)hAYJ>2r1?0C`pp8s-EXE(Q13c zQ@YbDJZ+9n5AG7ac5_Ko{4&w=Bn#ns;Ixj~iULJmNu|oTFI754^}fY(PfjI83FnV% z){UUsvj1K0F6QYP`)}~c(FwTU0Kb3htv+hF*KMk(PgBqp*-k!MW`=FGvHFajl+c%v ze_;(?rs=H${OJf3n+R4S9^AmiEncJ}e_&Ojc0L#;%5^i<&qhZW{+S2r?3pkL zd2d>oG}GhhHJ%cBORPpnyZM9?{B9(-0gd@q`{L3m>v5tC^6@MsjnY$jBZzxfNlkrr zxoHvn`a@N@XIXwbe1*v_RXJHPB=_kQV$ z5sUfAnG1c_gzlL8+Y&0sT!8bOa-{Odj`ULX=>7k`9OISqyO$9{EiLQIqmOtat5ULJ zLm!9)G?=Q~uh5B8??`bR4}?{ggFh7MlFy z1E@73E|m@Mve^T5Z$`ptueCYKtq>}Yc9RVvg`Co&1UGvF&4rLu_vdYH6O4jzPhr3j zzkXM13BF&;G7ypf;<;+xi+S1wmUEgemrpNni5l33zjOLJ&K1{-BLnC)BDU-qOZS@` zwnImW@n6bH-L;oq5?;wJbDgf~U-$&OKM*PUUgD9Eh5JoHS(r$*DYCYvIDy==P_Apm#^ttH z$#+@L1PI6jt_Z$rKiq+y#dQ*4gpV z)O9W&4+Yo#czpc2H%MQMp3bxXIz&jgjki_pbjyxCa3kW|)45uP!r4^@0R&eSl_VAH z`Wm#1+FGLV)NqNZ5b35 z3_JvN5>)t3^&6d^_FLm70A3;VT<;eO1X4RI>1ZYIp&-~KcKnQK_!zqi6?>|4(>lQu zWg0ooq@o00t^CzbumfT#QoDG2@%oK+Ai`mPJ>SJrN-`{gsh;mHumg$kO75)>{T<>}ls1UVXqopG#;o!r7oincag zi8wim;*(<^Is-{{*D#n%d}%}>s+XhPyn~m^<)sNEP_?tYXH=$D$@Oc|vXl^t1*41o z1zeReo~fNpKrup_zuvM_fD-ZmpXm8es5%@z`G~xbN*LU}dX70IW~{R68(ZEtmgp4W zjYgigwto5GK^RbrnaS!}Vb9-^8MnLA%Vxiq0%YaC*sh(B#PJ$g-vFt7h{teO;>i`q z5bi#ooj0#_9lPl_+d8NqG-U12bkfoW$ZpwwP)}yrG_ycrbV01PTBDg;bS>57&M-OU*0!|#M~}fr8)RIGfv?c@-i<)w zpgskh9<2iR%$oH9+^J-jm60kr?3`AQ1b0gLx4lsZtY2l|D=|SMhJFr?`RvMUiMnr_ z?$1kxbmwP6;l%OK>s>ZX37^Y+v90f{VKQgN_IKfQ@6z4A$66;8_B*wT{Ku6ectj#j zBhlycU%ZcZD<6M^el2U^{ zq2jbCt*5l5Wsb0Cl)39RspN)^&9zNM`}Zl^)UE>gv*@mNQ;Y18-Tg+Td>K7BbLlRn zcOzohw`^9m#z@M2E&X}+KxCn?VD8+j<4VN&wk}`876=eS!9sQ1m7mJ%eONd$hhEm| zk}QPwY>3vqIo&*xF^a!ETx7eio6LmcaCxMGK9c?`M5Qzr>B|`kbHiw}15U(a#_~iy zJFlgPrTiz-wTlvNUoWZJT6v4*0h}1Q;JWA4wUBh;IeE67nmBExqW7M2@r^Ygsp1t6 z?7(m^V0>=Ui|%CoV%yJgb&<M_BM^cFUh($fow1%_Pd>A!j{cVW_E8#5m{$SHBea zWxq@qfK*87=F%YLGoPh{tEcH1Cmp?o#Us=WEnIIB#TOsVz2huVi?-R{TVJWx=ZAlO z)w#Rr0(ra7kYLhDj3{ck>Ky%>T-i)N__1NnEoKw4b2|ReQ&@1MRij((n z()@rZ>H6=TFLEPnCpk>>;bTb}*9>&XkeoubYM!)zP>_>VCWY~MC*s{zR?~{hdd}mM zKz`E{Ipi;i(DB?Hk3^d*j(NNhcX6ndDES-}C8nI>Z(^`Dcc>Yyu7)j@bn(O%+$0c- zYGrmDaVi9+d z^mcNE&$5`qSJMtquc~Kd-Rfe(wps|LD2D86MZEKJ(6a6mR3%4yBW-;$`QCmN z(WpsQ(xX(s9=^3#=}%7)R~7HZDXHGyxfa1f>3C9JK%gSc2W$rIeCrB7X*)0GfdE}` znp%NtQxiSc{xVhQrBIlwQSErrw=wc|L=$(|SZSnA(r<-MiZsv92^j z44z_$NZdJcI4-HoTDJurY#h|hpv;DV9{wc?$IDbNg<%wPC8gb)UA ztu}5avIW=0N_L>6%FtB~13|DB<|I^SlM=-DSk>CxrpS9h$kzAB&X=58a_s4k<&SiB zc1eKhG?d=WRcg?GsgFU5aPfI{j1ck5TqX%)soLmEE^2RjLPss1m%p6>Qr$Up0mfTe z=r~r`2=Tw5vfdyhrqO}{l=V5*XGvxTztfMsq|wO$yO*VDIyOrrlzqF_?9oV-ls{#> z8eRIy00`I?x44vtfw~9`-0a3J-Q>xA1zHkQ2lT8ym|RH z*4uOy8`m9Vl`} zdbA|EaI2N|r={6CPJA%2L@Z#&Cac{qROIwr+(K>DxTm=d;;S!)g4j>u(Ms|c&OH8U z30|BxTX{?aAbQ8WPS+doMRD9&6298q&u7N%iawyNuH+1g0OoMqD#$9S!P-1}uXba# zG(1J4eqCD+z5q$B7TQ)zd5MGctE$uiD`42ryvsQjoI=_PS1)yFd?bdxMP=f>*K%uL zq}f@V0P3VY_}!ZX+yciBQGRX1D#i#^k`Fl?HYV$k**4=u(pXcPVGHCIE=&Tafxx!wlZ+&V% z1t-Bf{;le*UabbRWE^Y(otHp6?2eW_v_|zto#{Kya<2|!(Ni79zAb4EO*JaPB~_y& zO&+F;Ds*nQ*iNVAH1z-!`sl6%oScQfQd;kRw!nq7j7v`k)sv8{N%S~Uzzfh90u7!m z=XsugL-hqUC?%A7J?SJ8KAxqpN;KXP0O2t2_l90mh=*&X9l%}I>pxbX0~RPYpStZ# zQti0*X@4P~r-XB?9v?3htMxh-*e2OjfCm3X$@zG2rg~D%h&V8RCp1YP^9J>a+GV+lz zfc4@MXV8Y=Cw@g77${l;nSeTSzt-ai5w38(6R1Bhy1=w1rII;i$tM|*i9a>C9kD4^ zKfEX8quI$e7Ee8NG}T*Qakckf0S6#jC?9t^LXuFOq}Pq;kAVVMDLodKwLyq@sSg!$ zmX>;d$!HotKR25qPT1#<5Al)F>O=%l>JLO;_?>Ol`8pD(-`U^2(VH>X2B{ z|3T_6coa%k>y5eO%->83?6621i{q;~$s4rjIZb;^on6FvwTj^E*uxWm1p04wp#~s& zlw&Te)*4V+IQ+HsPdC%MhDV_Fiw=5g{1#|(%~=4Nf9;)Z zTP9m8@SRN%yae>p|4MXT1HVAYgr;D*^ig=xaQ0#S&TirNu1_48hLr-=VQbIb?N3sU z#l&+9kKwuRoRcBjpVI$0)~ z^NtNrsEXxDw%~aBza5T{BkjbrVe!%?b;@gXf2jdDGy~7j=mSb=wrc$kJoE{Izpv+G z3pk^ zUBOK&PIKrluJFtscdviYUw`_4LW>zE5PjK5gg^Liu=s;c+-DM;04tI+!R7wxpE3C) zF9IOPZI)GO5B^VGmWm?B7JNOf*7zh=dF1d#7XuqBo;TILf4xgPMO*O?@SsR|JlL#Y z?L(Zw67h*L=VFYA%KdGXtfZF11q_!SBEJY%Sp1ye0QMWbg7~GQEV!76zL(*;Wof^7rf3=8uvTAS=1{bL1I^0=ud}g_LsP2jU$bV$O6-3O9)XL zM~dfLrpT2l_IekCt0=iii-}b(6ca0-=OT_EeqL3gx;9E3x+PvuXHPashH+nN*pRyG zI2*}rvdU(Q62D9DUIqL!ND^)&%{1!uF}+di~H1gFSrm=5%o8W0L-l&2ZL zdJ&JU3BRYOm6(qvRzlh0d5Py1qtuR+++~#4UB|YX-tzUF-rDuxOz&?07Vm}}B#TQA zVXEbkm*T&yNs3>p6k825P7*2V z@g{U_u>y>Ufep$oX-54#PPcPzPr@0~Afb;+pI#?`GH#%uWPYIR0N)>D99#?bZ-dHu z{5B<_*PP;soP+|D;o44{x!U4SH5RE8>&?zJK>b{!DRXfjdNl31?%g!CDs{1D0*>~Hs80;70KFSLxfO>=1n)eHFv2l|dXp|XrS z-z)|$b7RMUY4c`8T3CTRWUo8Em;0GBnm=#`E@nY2lz2!Hjk&Nat4nvCK5 zUy4@0#s2)950&0Hih^0s6i>xl@!*ExX0FMGl4FC^(ZEHo{R>v(m&Lf?nL9<(7eI^t zS6RclN<_0`^3)QXJ(gYwq&Vm#*k%)5vbsi5-pm(4K@Ye%nDx$}P34qgi?! zs13A};;xA4h;L2jB@Mj=dfFW5cDlUGzW=_YpUjQ*amJkbqm8?5TcvakMHK_LZ4Ipw z-P{zrM8sTD5dp+tV-TjAVIGKf!EprFsjQ?9yG<67?emgS zqJ$h)pPDs=DM(T0)bDoGH6`j?a{=CI;XvvFs0%VnxUW2zoEj%HeqBnkUP=p+ZIW{$$7MG`S-(hBnvy*&0{Kgw@05y~h!+d=;6Mm1 zex+jV*Xx8O17U;s1T&d_kFNG+rRE-7HFRgM~l4f;$-(hLI-f_g2lvH)@0Su`x13P0xR|5F$OOr0 zdiw!bwxtMxS>kS-0#8%iHA0lPRTYzXGolPf{M*=?x`g~JKN4MMH_6`Ftqz_XJsTUL zYl37+FBFyq8VgZ&b-%JY6p?0*cyH4CCa<5L#HQDyev8iLpa=#{@(DH0Ut-Qx;Q!50 z>HM$8hD`;W7LMJu>jmjGHHA+j&Rc+n|5dwgMEym;l$5ii@xI2rvl?gby}EkmX;yNR zA_9??xSW|&lap(??lQfcxOn8p4O*1QEl`#KNeoD*io|17s#1rL6@_1d)~5 z;SCAmkb@WQts_&nZ%r4dO;@n$L$`}ZH=nRSqbqNiWQ^kSS^}4kO%AtQNZgxq10lZ* zlyx-Pw$86qX4=dhGSSnn_O?Q*#!|Ig&c4`qOy|EBVv}s?E3i6^>NB&oHtV}Sonm{` zoonW#l48jjyV8SzQB5t{C-*cgSLa%;HX!;x54Tkgt|QIjK<-+KLkAkd>94pXj}9E+ z)=|BUD0;fM-0*pDH7Lq9^kFumWD6n3GodtUL|1MbQb;hTT*Yi!tF&^@{V`oX3cZS@ zo_t~k5(4eEC*+m;7BxE?PG}=&J9EPuQpviJ`+kaoE@+XtZH}LZIst}MG*~t(??d^p z)DxJosFXHR%v+jX-OS-F!S>OWkx2Q?9kjE5Fq9EO|EiBbYk#|aJ(-x)&mwM*n>OC) z=!KA_uNb57t$2>rhQkS)CKY8GArY2ob|np6-!Yl;Psz7F=LS~;-#*wdyC24ov`xX% zzh@H~d$&xfa4t#EylD=*Ph3v2SZ6zN%lheEY+58uyh1FAlxN#!VDQc3k@6eX<7~RI z^wG9R^zim_Q9o2ze|v;1ywkPz_EFkpaCT?Di=Yz%B*Q z5##Qh+C#pJExaQ0>b#-ethxL_8q*%yRr{POA(RGo;H*|d>LQP|kt&jt@%>*VwjT{H z^4u}MRrj*wM9oOz>btKQxkBMiOC{j)aB9CYb|XcbhP9WYy~R+5_>!A?aa_Wi)^kp1 z6r)5*8b@-e1n2k>s#I`3`4VsDV|3*7ml#G#SZ15ReCMU)QlwwivXC`PczfrY;QEsM zfv-x?Ef$X54~ z_y(2Omhs39!Iw{xOONaX585`W=^w#M3d@zKQ^&sKmf65^QhwUR4xAwykFTh@Jns%0 z`O#4H%HR=`B9n9a?2bm|Y?TdP;LF`Qw9zUZKGq`h}Y=wch%_~(U+d{zSs z>n@88Ya6B8A}1F^my7I&2?XbBVs)@{Ld{QvfTQnI5Dhj1^G4n0urA6)yZx=-GWvcY zZFwA#qXz1YN#Y72S!{S)oOxl&#ZMjD@ibdWgz)F$mF6biMQnc!KdBF1G8tuXM&0QM zf(SYrUn_M*Q4wE*5>4|te5UY0Ey>sBGIR>H+3oDHnEP{M=Ss2Gf*Q9)j16+{ngiA& zB33u;_O0=1#Ot{g3e6|(1|p9J-ut*N%9K4ZF@NP{%R^VUMLma9(xR=IS=8On(2iIg zs&2Y7y5N|&t{G$}&aJ?ks-|1!JXAe^PI6Ceq0k7j!}{)cDBRwwtmd~wQ3)EYB8?CNyymUKriLQWr~$pbtuK%NOn;L zmC3|$=`ku~O-RhFTQ@4%4yLo(z=U?;+rRm^deo}HVN_JC$^5Py%*b`u>AS3*1!9a} zVv;pOQ{)6~Suv=`Xm5rzO?OHDLX9hAHC+Yd@uWFb>s2@D42;GABPvvU zD`!>@mE?5kq@l~u$CF7O0`)4^b5l(^@Xd~2w2vu)N8h3HYKzF~hML8$zimYJ@2iM?yP zvTkw9+1N4AaC2aC2wU$rGA-RYvj)+s`rR!ihViuZ2Lf8Ei4xgKdmGrmBKt_$$G)|a zRvyn|R`xl|I_PyIvfL1^aK5&0EI4Uj7i+-IbonY*kfVG)m5204Qo2k6Rs?GWL7@8X!3RlhDOVrqzEe_HfUkp-oub)R zbjP@h*IskSwm?GVU#iva6i^kdwqEor&Ttr&qr5ZScHusO~`X0yz zjIyhyrZ(N?IB_RI9gjFp5Txw8Q}=LVK&4w2rD;f%z)tN6Y#X zb>7pB3kGzF9cA@NUV}rEV=>8gZ>o^3IUumRQ9KD|nN8_1pS5bGIYg($B6-oLv^azv zaOxJ7Cs&?SKECu&J(rz%g>Kzu1q5n%!!BfPI#X6u^SCX?ZqK!w zvx49C3E!jIxsIGPnQFtRV$`~$(xIXJP2ZJeT`VIR_==OSm9t`lZX$`{j{mV|3wh66 zSc-NA%5xiFWXx$acjy$}r`cV~Iht@uYk zjmYCw)|T|F?CEt0nyzLS(tT3{+L7ENaktd99e7+@HECEoN2cV9a`SdjFNwc22>*ZVy?0boX%{y- zqK;jRsGu|j=}i>r0UQEImo6}5c`%p@Kk$uBns+1`0@w*Ts`ZJvTJ!o zHP`aE(H1sSQ5m&nmd_)AlQLvl&lb0|`qifQnbGaSjxSx*=GLU0q8Comz756(`nh|y zb*mK#n7qNyf@&u352@N&K&o(Pls;hWp%zxOFQ`Js$Y^N4M)68g-m`> z2WvGVIx$ip)o*@Cbk5==zw)`>4D8lafVq6n3%CeqPqNAJEeE5Q4D4&$c^_6_#3%HJi5{F13?b|>-iEp=ZH@pW;I zETba%SUH2-Ixm4A?E;uf#0*!1gX*AiUdelFUvE{Iv$gw&w$y3m?UE+Ixx1FfJC>7U zxjt=V+iq)Vvd-Ah?+h=yqJSQk69inr{-os?o+|Wue7vBcqn<_~SAdmKpXL05y?wiK z`3f()pnO9vl- z0nxXcJN~`4?FEQ_;}>t-$#6r9F@K-*l~O%nv)j*B@rILCEg z2gT&woctP_P;bt>F_@vCbBD~NBQ1jbwaPPBGiLFX?RtKZ1t9YR<KrqoB&$CMNG;DG59hm_JQCtDwNKW+}n4@N6M|nBWRC*5FNv z?UvZyKakb0n)3$Lw<$)2)B6mIILogNeSDY7hP77V>iDB`yh)m+3%u@ z_C{cdcv@F+|Deu*h)9i4Oso2q)z^mnC_ArVe($*&lWZGi~kX=J0V0R~{Zv=|8)Z9cd8 z`o5@}fERHbE$+L`Co*#wg^{Cdn&*iDWX%YmxE<{z%y^)3^3Ehdkn`C@1u3yoDA&-p zU@S~KzS?X^z;}%VW(?!;+PH|C>1Yf;L!4fV)hyvuT#-Q+QnnHBUb}y82Acx*)z~am z1(qFl#j9X!)xUUYw*}+hc&$Sna=B)s5;d=iF3oLs-fUwW`D^^%KY5qfjy((uNk)~T6BwN%}EdGXAyhYPfWRCmr~N+5~R{4 zn7C6|l6kocIO5UGoc!d3jV0%m%t`G_++D~cp%)6;_w>`caW_4-y0k{Cvb&=BNI1Y8 zXg+BzFEi>9^3MdIgXj@*owB9rjX-gBjW@cu0ZW`WTCL(P2+`o)DCd8J4dG%-EQ=uM zxSbv>JKr^TBnG>vAzmcNht*)OB1Ls%G70x zIs^C7Ri5st&4gPwq` zy%3kL2Omy2UT~cVrsi$kKwd%4yx-i*Jj&L!tpo@)fiTBrjxWXnaK8}`O()3$Z}4=_ zTibP70Zx1Wa*Zf$&GcaD>>1!>8EC|(`RkCfxvO=6@D4^Z3Yy%V{CeNo4NAW;@{q`L zPq#!vdu;VfRmkBIz=!YLGxvbOGYRi4n1DG5IIds$p zns5e6e0EuEbU^eTjJSmA0lc`@8Q#-O1&~|rOXM>tIb)W50H;sIU@rWymXvYc-iWRc zn)@zR+BtQ_8&dhn$YHqyvpS4@PH4^!PS@z|KYaeH{eK%0zUxu6A1eEgdH(0aBgH)&WX_;!QdMcc25Uv{zVq?;x!4p6 z*6dCMeoX6tsrp`h_qY&EQSc4ouPC}NR0h8D;OLR$D;k#haS=Ze?t3U#L?P4=))j0B zPB{SJ9Q(&9&sX=D+`0uA%{u>s*c7SrH|{jUzPFeF&1S--Tpm>HmJ2y=1P&0t+(u;v zsV>cnB<%;H&V}qjA$@z1zS(A|@RrL!eEjO`fq9r$Ur}eza2;$j*0`9>i+0Pco!gH9 z#KHG0VcsBlGI>G|Jk4-Q|3~@QGw&XNC0x65aMAt0^mCwimN3vLsjaM=5)MqrluJ9n zr>n21<`A|cKZN}N5j+iFS8(Y24m6rYWr(>fDeR>X2fL^D!vm!7d)QTf%6vj-emHp2 zV&jzR3h8)a*a0DYO??E2yq~+Z(R?3y1GB;|91%<=`*Gj~qNHMuaym3@In|#0;RM7* z1N1O||Dn0D{r3=t#ey)}Z|_(9A-s~R+*>`mY<=&qKwo&*e`W#vXCJTGzZ_ahg#%~n z`ki-(hV8GswX)yI|IF(>n)5E4a{7ui0Fdze==kpR2~CSR;j_P-A-Yep|61sVDv)%D z%d)TQIFLK};UWIB*YD2ny#+cW0cZOb3s!jFrvAsze_YbfgA-6`aoB(69REjwJ5~Ks z29o2zw(Ii~?&vQTvuscEl8F@Mi-{=7yUhEA;oxJf0m?mf+va<%G=6c!)!n+n@x|Hc zeE{cu5A@$1#%}D{2twtj-@X%yIbA=`&BA2*T0dZQW9t)|B~fB^VA>(QEH~(W#vGu5 zu_v&i5~2(GJ+WdnjkgY2M`*ITAoofVrWE`lEEUSjw1ha56T_7W)#uXx_Jw#c@O|b%wS(s5&2#w8e{G7i@vjW2 z+x$EPykzu%WbOM4%_YDd1%JN>Ki=anRWz|5;u4=qS^F`MZwv^Cn7co1*V;H{I5Tdu zlo7RWiT`l;zf=a&USK~fuQA*YuHmn?^51pt@d818brAQzzv8z)$An_cHNowznSr zQWw6Xu%F=3%}}4iAo96l!9U`RD6k$_JT9^xc^K7R9Cb7b%>Y}Q=VOFn z=B4#E1_4;7U*bC|laZ-NI=J#4$>*xx)se?cpDMJj@U#6}?)q7tXgVELX0)4jJ^_)V zX`0$DpV#O-RjY$`(`zlI`i=nfR6uUYwCwEPBlCYPKessVwdF=BOji@5+E+|c+o?l< zN=V9OaKp3jP=q&Z0jbGUw)oZ1jDMbIE?uf#q68RgC?d<6E8aM4RF9jB&>Jq#Joce>wyoESkBZG;+V5WRR5$=KinZ3^#w zQNBvZfaB^W6q%vt9i*D#;T_D~FSnGM2ypFbQ18}fxSY)5C5IxoGN@A4fPVEEeC6f9 zq`BVoo#Me$%t%NEjCm7YbqkZ*TO|x_JQvn2*6*ma_$tJ=5xygP{aG-`a(582djnGo z5|UHWM#6}Ra9cSs)>?r=&2T42g_S#^tx!+`b7AORJkAot#oXSW#=PwYvV6G%IW1!V zTHS#=F?GWqX$4N^%FL1(#;(;HwQUwLBV%0ys=$~~kflLNa20!MQh4F&Ht3X7QX}Jt zZ+Ac2uoB*)hH`y6JbTwsAWSL3wkF#D<;)r%Ux)$;N_70(S1+!HZGnmPRtIfkc5dk} zCt!rvH{y0jr$R@D#yJF`m&3*va2_Zz6RoB*APbp91_wh(ZAJwCVXbLwzERdnn^~OC z@ASe~xG?1mVG=8qG6v^X(}2Aq5t%4;&!zQVEJD`PNEBA&yikw6trA%m5?H0$6avDQ zrbrZF#S{qO6d&>~6Q&h}bz5_8kM{PD#iGG|mm6UR4CNTLFvc)1G&no7MW;rR!s1)F zW_O$psL)QeAa;g)^o6>ybVrWczz(4|Do9?&N4~T2optZ{*$N~!VhiGf0yZ1g6Kt0B`rnyo}aeAs8)1if24Ak&D4a0kUI zc<4E+P+eGSjeoQgKZg&8H^5!npQZJwGO>z6B^s|0U;<3aGmo^+2DR+^RED^!P!46a z32axCED0pzp4g?Ox3NnGu`eVbGUNqzgj_j2ctY}4zk0-fE*53-%`aKHm8Rr#lk^MS=P%Ef`Uc(3|vrTTj^@4CJ71#PKxLD4q?js@Kel_qe~<@<;g}? z%nFlUl!Y8n-Z+N^^F%|py`2fAVFm!hE-Z;@siKdLGZyowZpFoAOVGe18N*o1%k&>& z#F~_d%a(4Pr8PzbWcIHu(%+2m(`CC?WFDnZ+jqPrw4q47Dvk@r19Yzhy-@_k0Dg{$V+GB;3%v3|sv>{%ua-qO{EZ8HM{fV5bP4~V@h!`(=; z-WjbGVldUri?PV-l{rc(ZB&`~os_iz-4gJK1Y;7(=K7`znrqC{y00m`2%?G+*Ijg5D6~vsch*Hc>DT1w4t@ zUKAViWvmPWXWpu}5tyMhuN%pMvDs@_W;ATjM>*-b-iZt3n_Gsx9=3M=eNH2m>#yMg z4(ay?b+LU@F~|tc9*r}euMKK5AjTCNZh{}WO!ZZV2HvG|a0|E#;|l`5 z{yN_m)Rx`Ij$vWh7GsVHfhj>e?@1Iu*Gk|ikPC_g>+P%y5j^u)_u#eKp)G&-N2ZXF zyORb~YTQL35IyBowZgz)+*Oj1oVX+=4O9ua(6PW2=AEU-8$h2-Qw)P#Ss{VQSHf8H zrxf+%XC#@C)|DNDm_IF>UtF+6suqj0oe$o`#p3~n=+hyh2Q2zveZdH2Lfm7i6MFII zqPtUuk%*{S8E>J~K}DhEHry@KP>RnN)l{ie!_5hFeDJ#|cp=#-$y;jlysfR0eF}P| z1)b3dIUxz!u_~RKRnlg&yaXr=x^AweiCii3msva?#(Hm=-odQN7jjzmLPrO4u{6Jk zUJ7>Qoc>y^#nWq>cB1tnKV}8?b8XM)-0(eue8$BXdY6*A8-LS+J4C{A!y407H~5uS zuWcS%uFmLD5Zu4;aohc~*Hk%F*K2dNLqf0)ah)tY-_(!D)8wojY~7>>%b8EbHap2! zi)$qJcZ)til&5jO7Jj$Gne%TPT z^BHDY>2gw7@oeGJX1}%1>ImghwC}sxskzlZH{K<2m5>_h)lD-V@4hJ}%z5$aF}?)m z57^b&Ouyr`R-lYml&~`1;0?-sR!cc>=_Rdcxu9ofFF&PVcfXK{HocxNOsrLMm#2~^ zkK6FAlZ#Db*rfK3Tu7y&xE} zI8ym&>OPEC7P>k+0USpUjzy4j;z4n29ZAD;d4MBv~F9%C16ww7PD(<<}CkVNPtN zcS6qhSv`BkRzx>6G0Y=8qz$-Sii@C(fQd7ImCX}@mR|yN42Nv`tFdsU6V^Nq;4FvS_?%;Ca zUz~?EgkFvB=;npTcynn+2By&V`W;WXmlS0G&%ac?D~s7Z0h* zeWjX$7f7BWbN~d1Hca(_G=(y)O)qAT#;JDpY{qGbcMt4(J5QE|3D-CjDTTrG#`;#~ zj$PVRYIH*qUF5I%N`KZ!qz!9GNME_UeS!mvhZ5?O8sw%F*&3v6+E$Wxhh~~w$%$1b zsehR)IbO#6*vwgha(bF^*T$l;ddNaWx9RX8nFcX7W#8*GZo5dzE@r+U6yJS&SJT$>?hWt_VUHYqeW6~!vBcKj#cDbp!MMS$<6)ax zsCopv;C-745IY#@WAo~+qWQXo8eR_UTtV>3!X|qvykxTyTHLsEv`PnY=CGp6c9&@v zA2s%R8p-=?SB%h|ujl0w!BlBxG?JtE@66eLZ2Rzx0DJB^d546Xau{e71{-XxPZ0HX zO?#gb6gg2(DY+ks4!EbAc-7aigQYg>FZnar>6O{^u9sMCXqXaDUOMG}Ol&}OO{gZV z#@%0r?_~_%l%9eR9_3P*)tdhd_tvap=}4>9VwLz{LsM+ZPdMX`nc+MGm5dnH_;{@Y|i zi)Q@9pltn6a{oMIu)J}rs`IEU3GP?BX*7g(yE7>qL0`!rN%sPgVxD5;v0H(CfRCPj z>sRY$Q1^P}!-R?{Rii8`MY*U>8SyG>TYbayd$A$mn`#+rf$dL^@&(BIWNDSWk3fj} z_%DdMw^u)&!}YAKGYd}h&gd5>F&QP(MP*G<6~mk$gt#zPXC+Mu`ZJavAf#CZLR%z* zD3}%a34IiJEnrAxCo{!s`^1%PUM#*jh{=b(m?>2)nT8f0xYh9ple8)-jQ7q?#er z<8S)a@*B60zC4;rw&B_Bg-o;sn`}CQ)YjEiD$y70byJyBtnBJ)TsAdZ)>Q2%O%$2W zWUE%Ea3Xt%q*jJ$7{E@yW9dq7Q8>+ms84VH9Rz@r{hFX|IpW9V9 zFgExrA^9avEKI|$mw^w~Wh+-^M{R{m2y60`4Dy z5;*5|)H2bPHQAe+P`BP6FmV$v@Eg1{ID=*N?(`yOO1AejkH6gW)#qbxlijGOUXVn~Qwqr_w z>X{e9@W9K=4kVv$v>vf4UyM}9NM?AVtQd1z(o8!v>%B&~Ube-CySKPY-b3p?YcZAq zjV<+hb+N{qHTHBKQ#5I|E&-5S*UZBKnuCi@>X&rozUjp zGlP@F%hfMNx93qax0!2I*ANP>hV@XgrD^yzh#EWp&PYZm0RiT7!6ODmpFaS)$P0*Cu@n{l74fuUeYwN z&M}RtE^GzI;(ceI@0}hSRBPl=e~!Y$V4z8HgK& zB!qk~KB>KQuJQ6l$I0pvD?-EC3TTDXIx(Z#ZY;u=vzinGv6F5}U}7cUpX2D4?tn_` zZI=FWpKEk#HlU1jY{NpBC%sj-p(BKD24Z0Vd-U_?5Vl6?3o5sL^FER>U|&9T)W67n zdKwu|pXiW4wdDyf{Atd(GPR13@X;l74A66>O|A^xZrV}}c%L53dOt&5_9}2JXhVy*Z8Ffx{>#s zcJuD3j3z4gN8i(&}W5nq@$P;;j4P-`_%orPVX!*=ky46?Wt@ij%jN>?}<~ zw@0UQ1^Ijsb1@90&Y1PJD8UL^Dz!C3%62WJS!hPJ=kTrr5Y@iTY`fB$o}|#8rs!zI z-d3)V1PaEkS6?0E2=*vhTW=GEc%fkNK=nx*ITeYM+7^s(;kG0j$lIlAdNKx1Pc%1i z=etcdEPXadyx7U4O-Yc8{w-5We3&OCG-So9a-Dl_umCuGq^j95*PdJZxO_G`7p85b z;P9}?jA|~mBcDruXT3KVAzQf|WmjK~#aj@%dzbMc0)^(FwwgsZuT_0BKm^i@*{>Nt z4IDXOa=xp{e4m%qSAFd>k;M?t&_b(|xu!W@MIRL9=Qr$udPm=kd)Ys^#Z~HOZL#P+ zCekyk2NrYbGbTQhWE|opnlGcYp$vzIZ3CYvIw94!9mXL*3jm z9^5WqwBE{7nEL9h2T>qYe{>0M!Zbe^J*AiNB9*^D;jvUrHdLOA+N5(94&U&TJJNSN~`h&cg$oeZ?833Ac6N49~es zB~BKD@Ih58bJm`}@qu4QFt`(3AcVXUbjaQ6Hih`k)e`u0bqICqCaT&quM79C!|5cw z&*hDj8AmK1I?932T=+FaP9 z>36QasBqJ+Nm$>XL9XqGmYm~dQ!ZA++8OTE^A9$#BrGVpCU;N8VBC?mi4BSwD{@p( z{ZzM=!i!XqbG&taO)@q za!84F-r3UZf)V_bHDyu?I3E)+V}Me2SHGzS$T&);PLMBNk-W4~da61g7yBTRlKKjo zdSmxURAZ3ml~%~KecCmdG;a>UH9*SJBlrZa$Qkj44$6RxLfpSR;w6_badI%wuGYYd zWa?Nj!|(tFzWYi7uHW7|8T*{V(nJBA=gmZ$@4L0=XU>{wgYps>npy5kI>L{dSVuPs zMml$^R`6KbiA71Onr3Abt8IR9NvmIUsSP(H_zdNqF1H#Dv^8ffcoAU)X4%lE;*W^X zAm}A|$J+t}Dzik&l+hMVuV#d$0~pphv~Gd6kaf&3GUZpw(?S zwTw6$<}17eFNmCM??`ne!!2Y+Zxos@048EKtW z(A|Mj!n`{jY0`RdKys}E&s$sX{z*j7{`R$sB*k$gk#@GKq&UnA=3?W^r<2nC(9^v} zjWjg$seU#bTfRuu0*Jq2;sh(TW^DVQPixLVcrLH>2yVPQ1DOCECd|K<^L>lrbd;Y~q!=_V&?T-_Kt@1&DU`c>{?-{RJrYvpxYS{Fq?RVQ%hpoiBK6 zX>P$hZNxWlP!l(p(I5OH9wvy(b&03ys4gn29 zrgQ{3yn5VZaMi*loEgG*Sd=3bRy6pc>rH!>RPM=hEoS%+uV>hp+-ayUu76w1-)8Ob zJX*e{vVSyqNAQf9wU6M))7Xbue;sZ(5yd>#G7aQ95^K@x+&)kxRO_Nzu~v2a3fV$H zz3~`CoP&wD>gNw6PScpfNJRW#^@~!ITB^oSL}!wzpuOv4DL?IURGcRzk}-1Yaoc-s znPnG{QRsa`}Tb|&MKi& z@~485QUDuSQn2db9J3@0y{Tb{H4)?-uY3M%7)h+0!kJUHOSoS-Hp;h&y0AJ6$25gl zGYip#`H~#S*{T_dv`P5OW7ojqNe!nmR^-hosTj|1+e6CqU4ufzE{dYnv+?R1PdH9* zO7vC=>u%?U6Z>{_xD^8fPRt)ud4&#l1}bp^4F%h%(eo*7n@cUTGd1CN(sv5R!ddl{ z*E-C$?v+lDFAjiFU-ju|kI8YvtKNoj*O3J0IV8@l4#9Cw4!V#A@vEG`QO?*V)ctYE z&RehssFkkSDv=FYh(gc2@sXU+tO0GHM@RIU-Xft&yTAuU9H)s6YW7OMa|lFfhlVWn zgM=XYp?9h1kY+=&@|4lgk?fZZ5-dXF_!8Q`4O*39G7Coj;W#os@!?9VJw^xz!;tkZ2fe`p~9 zFzQ%IF8&2mq_U2X8UfR`kkq`Ef47b+RojN9AnU|A${^vQbfQaGLi@*H7XMh6oY@w* z#I5()f-)p?kP){fBYT_tHx!2-tDk4a_8nAgq@#{@USnQ`$Kz&HE1y?Z0Ee=TzSwHV zhu@Hgr^wJx<+(GAo=_TkN_bVl+mA<|d(Ovb+goSVWv6SHK97`?8!3OjX_%JW&|0ps z8KQ!Hvu(0%#osHvA#IXx=@Qr0ey4Zh=yMgzr@fRXt2J$vj<*YeMUCQ&y>>n|K7LwJ zf}07SN)0zqaG7jK`tli^vJ5Ce$UQyXSHmpxmJ}$nX=v}8F~sA_<*v4t#Nk>8)y#kg zJ8}ZaY^stg44_`1_80s5e*119Y(}G=HYIop1?10YS4Ln8D&g8omRaeD?K}n1%#~x! z$efjP!RtUFPt`OC?`9Tnd-DN1v=27m$8*8mrj3N1sIiZl3d+F^TA=SM$%2V4 z%Q@orbR6NjQ3_l|0t_=Yx8M3bqT_TP;+r?8dUA41^BepSkY>OeX%f>Z7_Y@EIlXJ&y=t8rk2M+&H#x|07>*-GzoAM723#>r|oS+Mept{5HJ>G#`^woPef z9fJuoHM>9hkE5W%OYXQYn`0g*z9WJdau zZ5Wrz5C>K;PXiy)Ye=*DHe79)w%TmH=@0$*tq4}i$S8d#57#%>>iHyJZsq+Uq1D$k zBM>m>k#qBTNn!d*qs(#-T;R}$%Td2&g%bV4iY;xHv z4f^f+J}jE!TU8VaHE;|s<81F!Tnqkur#Ni6HcRmQ^TFxop1H1TVh~Z1rVl=yBX`zACel2T?C7xQ|3ntxk?(w;2XLYm}wNyg3X0JB6| zWr_9A6FI+en}+RP*9+~!F#~-E(mR#rV~<)lG~!$+>_b8Hw8f?m%!=Tio^S!MlBZ=; zDqNs=N(AHul$V3U*bCi}bL-1P!n`K4vP{svVvtJCA$hksHXmOO*j<0v&E9O z{l;B+BPDpC)&U)oY^BoROmK=|^~~_^?*oS~2g{-a1-pSdbq5a08+?~aCnRu)#UwX_ zb`R+^j|0cC&cbdm5yiV=vLMqZ0AHy}oQcS-lCmveIb98EtGq99Apmzv&GOP5&Vo6q zxDT(euGQtu3ahXW{e(7(%J${LB!G;9sw9|TZudc}m7vg|Y*{7E&7{|Ffb%1h@OMp{ ze7MG~=)DpH6h6BwQ*jB5YV(U;!q_Sy5H4`y zIygU`k1|jpP8s)eR!juJMd#4d;!|u z*iu(b3q&FMoB!(6 z*C{2Z^^~!#(ewb*R++?0Vau(uIZZ9ha%o`+%hp26U9PPI!Cq@qG@Wli2%giNFuTC& zKVpjKg-p1Jb(-x*YgSQnE%oX?F1KQosU(dqtrmn!OE5cf=tb40t%ksNSE694q%ZIw z<~5d$rO$5g9MW=IsI`g>b)#@sK#(KoXCd(Ss$_iX$g)w%N;yk)n>zMTSU*dWD|g$5 z-h2(mSz%&Sp9!iw_TJwuzcZ2VDkrT%0%Pa#FaCU`t>d8%@yxE&lApaB!9p6%)sWXG za4m&~R@I~zgegRDCtnU@a7|Lkk(s4pMy~c9ep?Uy9YaCtnUP7i_LXk3W=6W^-J}U_ zd&G&C^K49}aWIF7O&88N8$j}k(@$`21H=>bZK$po3r2KNA6{w$E%`Le_u}b6v$ACx zH4gR_w?br7RL~J^vl$??ed6PjMV4q)IUtg+=Ab&YSEs^ynPc)2s-vCHu*sAWstsK$ zoS3#>!*yZ=8nUm)sACP1w$Ab=p}|zAi-r1FvvD&!4?q8cTs-P=eHAI4QUyiC389#Ev8Box5~P?Krq^sJwV{A>P@ z^b_Dp#+CbO`Ujq~fapA1B^d2^c`8-C zD4f%rJ}5NMT?RtBb+KPpgVjm-@Y2j|8sPQpA(h;nRN)cg*D=4Df*Zw#PJR^j5TYyxitw|2wMsf8=&WnO-qG;fg-m3`y9)y;Uqe zeG%YCk&z%b!qGS4$pBgoC{;H8DlYNh3hl%G6qXFFPKq6 z#X8Nd@t+czloN-}0gfJJlS->FeWT4y7qMCKw@E5g04m~;q>xD^a1<=k7aKy>l=9*X z==h`LnR{XR>GY%hN^31q1*9iLss z&J-+Z8xsXobj@dNqZTrDGO1D$Qo9Pmge5RA5Z1}#UXu@iu0_^6sX=OlXs0g+2MZdB zx!C*gbW0{7d;#=?lZlO-tPO3LM3BF1KKwpSn9q0yqGJQlrVDI`AXa(bkdc2?cl+!& z?J-A!(Lw-Z$)K>sQ(>GbikqID`!ryGlI{Dh4?qmOz zO_v%x($1R39XEr*bD~x%!VM&}+H_b^$@9;sBj^g3W~@fQa-yt#NvC$wk0r8t!Aq%u z*X?Bd-CS07?k*LDx^u~{;MwFVYD+Q|^kz9-h~e^hn9YhEU1Z13-7L$Y-rYZ5FwDg8QyV zUa(9x5mI0qkHC_au`L^}Q*`|ybp88Z>2SbB1g0ZJ7Q8Moxe^mW;&dW#^=+#MwVU}G ziV|IhT|3rX#kwCi!ZO|K1k|#&UVec17Kr-JLEnA~9o*Ll4q*AmPhKETBI=$$p7@0& zk}}r(GH^cN;-1~MJTK>P^p$`Ji?X_9q^Bt}Ua#V*VxhD#yz&OKM~#h*e5{9joY~8WQ{RkL7_sz*>yw z0TiT54l*_U#H}Tjhn92?r2!trv-T-v)iJvgtRSfLnB7&NfF>Z%Vq_Q4?y(KA!I#KP zsQ(qB{X-)u)l=RgvXWzD=#BnUzR<;2-C4lo4xlGF{;(474|C^!B0Dp2@B9@D-6}u9 z_)~>ZsY4r=!`v#sFK;BAs>YGm(2QZ)cjL0@sLn9PW*5se*f@4w;~`X9 zDePqMmw9epB{ug9#n4SPGU@CtWY^oMIR>cjyO$I##i^0GG{w;4WrR=&*zv@R16|tt zx%|-J%c3wwmqiZdImH|3{zzWm^EKuf2o1ll{fIG;fTnf~p0~I{TRk}2vPp(4CUsOU{tC}4P=SwxB~)Egdrj5prJQ!r9Bz06$r0)!EI03>c|nn z&kfRsJ1zxV|I@D;t4Lu^l{kyssJ^t<<-$(=b-i&^#`U}(Nq+eDFVY*o`E7H4{ZREa zh#A?=v0dV;$qTRY!ENxZdXnKl9xQ~Ni%oXT!u(9w4g&5QFvnFUDLL;;yP5zWVR8+< zJBtb@5Bi!f0{M=N=rW9=X@88fEhV3;m71@5S%#Kzc)vaKW`Mg{(;B>FE&gsHpEcZ| zNMGr(F7j`3`z4>-(t23}r0n98X{zt+$@M#>Q{q?8Ryk)hDhp=_;VWE zmRiyN7bB_(y2bTQkoYP9+8klY8!VhIGrLpvC?T68cpa3+ua?r)`*j|Ei*5egcC~RY zW=-HfeK`-y-@Tw2(<=4+^Y?GBxg9%Sm7~w*m|V?rFG)XU(rRCF4 z$T9Xd^u=m6nO!$|eg1yyOUQ|QA`T%5U(z*>Q z2byRod$*$DhnX7&a!m_uO>M}^kocap1cL^f^~x}DUd#{>cCZ!%-i&gTaaM_t=?JIh4rX=@xPQSxqsav*4sN+LIV;BZgkzon>C#R35C^6k zPRqm_!p=Yr2urZ4W_P{Yik85<0l7ia7iuJ!lt1ax_Ns{Rh{AX=vG5(4^wrFZhJ|#& zrzUSJ3&-v46Kjxmu&2dI7U+}`S=X*PxBn6Tv1@;Qme=BR>Ar8%-?QY0uSEHP{?Xasq+glkpRRr* zYp;2_&%P*V-*D@nJkw7WaE1aT@1_+yyq{G6!-xK131gL006DYJQ0fD(q5hu_``rR_ zb%6FH3Az4$fS~_kGw%{F&876|%woP<@aH%6<2ir-t#j)}@L}#@pA>ZKW+ijC)zrc9 zamUZy1v*-*D1W`+iKCb7DqCx0CriNO`S2l9jO^Hf@e@E_kW~2Q7)xGx1jIk1MHUCN4e?_kYckcrMGkQg>HV=$d`d5!a z12n0czx?p~{yqJK(FdRH|0~x2>`DHAcK<6D?PJ>iTXqw&N+b3wx*zJ=*H3<8I+~>< z*f6~^Psj4$mi5IA&x-Z5-?bT0%MGY|wnPNFe)nm`&ccsxHvGB7->~1aC_?t-r+x^r z{Hoc3{^(jwE|f@Lb%g?pSZ=uT19U_CF^o6>)jKvK@$=Qb*Iqvn-kz6>=|ydIRKUQnJp@8a9wI?_ZRhLu=u&9{_)~D8NhgWt{UEY_~ZHTiYnWD!)-b@ z36*b=Qnyv#VYkrvp8nt1+1G=6(@i(rdbWP>gp9{eIs5F5MD5&3IK7XxKijDBd0=C# zhmiZw>YjIAr|pdd%%HYAWcup~A;)dBfMqH;8zX*<9{l1!bEBM`+NyDJZlBZ+7^Z{l zaMA}1T{V$ZBl-iKV&y34b@yJ=xjuK+G3VfA`~TvH-8;--VqW{=UR{)$3FYS+L2 z{a*}tiERc>(N2s|YtN5+f2Vo$q2t9UPq5fD27un!=9=~#d|mWG?K38w0JcRQciPt+ z;B?=32AI5CWLR0e_FhD38u{bhe$K{+_W)8|ZrUZIYAJ<9+6kePh()Nf>Ks@TTd#0R z=1xuO{d*Vgy<5{EU%GJU1O-8EkwQE0?lCn_)ju@!q>jI#I`64`x3;4^iHvuvu1`ox z8cP}*85^nJ5$E$+6i0}!civiwOBh}1E~Qky_T)E;<5&Ot&%IL=)3GwoQ5segO2}hJ z&Pn}t=$HSz_BnE8J}VD*;lNKZ$A`E%YgG)LgJwOBHRYrj`lB>Tl-cj>Thw^#_@d)H z223SS?%9%?vY$#E)EPtTt%&V%E>|?nHtw$UwU8z3VDtNTySMMpMZX=^oX3y$I_=-6 z<4NG|@;bx6U3{q8*4BP<|MxGROXX%Qp&%c#^>0t5mv0?j6U2$*8yzawz1?g^Yd%Zw z+veVXv+eyLFt?O(=KqrHuTN5uPr6buJ0u>pg|!%dF9jdQ0j7Pwxksj;Bsxb`CasW* z*W@sb-i-+!^3(W_3_qi?)tx7YILjYjnmhahe}E+~o_njS8E6}g(g?hmCb9q94UV2M zluyNqftQTeO6{$&&-FE2HI!H6e)cSv5B;9~xijZA^Y;?4x9Lco*F(CR0>T*u1s}z- z>gr$L-cP%8*)o3Zh|~{@9OdrgSgQTIIDYk>Jg~T?dQ1gqp9s;I_+cKZts=a3Elkn$ z7~WE^gKLLN$ylg8*o@>%l5u_Zqk!&h?{f&ia;sT`9;*G9w)38Iy9BVCLkxW80QuJ` zq7uG|HfgHw&i*UUNxi0^IDIeYH%{dP7b;S$KHS@mo~Q-x=sqF$vpS(^ws*^%0j#Oy zenMV-BBgxdhEztT%CV&VQvZ6F;`z5_*An`e_7iY}=^jZ}GZvfy?FJJe2btH-^X|7O z#~>ub{tIujzw-UIo%b5DG5ZB8J6=ln8)0*SfacGi5H@~~uuCs{826Kh^Xe<>OBWNs zGGt;ZMQytL{_oFaQ~9lR?a70SV>_00|E>Ue0PV@cpHZ^=az)yG{cCAIjb{ ztjTBF162`F6cHt$BA_Tr5m1oO1*IsU^xk_5y+a^~NHHKFy;p(ILhl`oNbdv)y+ud> zDM{$u=-&Ie|Baq=?)}D_gqe5NthIit&bW@aTNhXFTsTGHv*+{K6k~rABQZDB=Px>d z?;(*GiNY){r~lQj^TPYz>+KX6Vn6*(IM&Czzei>46ZVn*3yIl&k=S~W_VhpUaYYD* zKz*N;=ML{)%mSa@b5$in#d6qVuq>dDrvAmB<65iyl59`?AAr*X1E^p1We)O#I|Tvb z7eAh6!?)sk6w%K@D2UT8UCbRiJuQ*d5=d{PE1N4Oo>pAxqolEYne%DwL}`WO{9rmg z?+6{M)o-3T{$L#bmvJd6h%8K|K%sYfHR+M(fFYfj^&=(?C#H&ze=rn|P4Vy%dv~gG znnj}iA1d?Axg|Qox3YV0tf)_IxnCk(ys+z(8Q+YIQ1Bn4N-!&SKVolkwy4bmyHd%# z9A`VdAmSSSXSw@xN3Mz+T9VEaDb#%UR&3i|0;Piw`(suj3v>KZ5HY;Z#ZJrBDHZqo z5k~DN@S6=jCO@k4`*XPA;9s(~DA`Z>hb;YNIBTi&pT8-|frdi0e0TlKSJ)pIbpB z8)fAn_5NRIaC#vnx`wTNlhM=JsxusqFI@iA!f|{>bO!5LQqa!#G^gI-)qH~%{{wj; zL`n?T9Ki4HwO2JauPZq523l@lY5!R13*vTN^ox-C+lK$wXMxIwty*^Z+f!DJK0gD? z9%AXwqDS$m&g6T(F38-bfv!?$5++P0*(rE?^6u`P$BXBPXUC>`?thJPow@&qK5B`$ zbU(0x+~vQf_=9S{zfIRjcXxmy`2|R?pSPBr{rR<6Hwv_|)g>^wj-DKNAm6 zYH?S2`pT>`*!|4uclNtWYk6b-=r0)kCudyTd8WH2;U{kU%j}}}2qzjrwyBI>mAbP} z8@tc^$(MohL_H{B(U*D}5&j&SXm?#Yn2z}@X;5v;Q zpqN8nMK@PKf%>&$)lG&;@h%Nkeae_mdR!61UG5S$;@D55VWHUm3&Rt8o4Q_Ye`QF4_A z&2pNPX;=!b181mg2_;)ZdCAx@TeDci_UCVQrrl=0C;aqWr5mK`s>H6>92#T^|JaIk z6}1&n+f)mnqzH|Y9e*HSKb)uMha0K<*;h{~Z}e*tUa!Fa{cZmE7C`e#wZt}&OK&!7 zVkaE>b2i!`M1hooIDCmk%)A!y3j@ zU<()ORE^4wzrM+~xn6C%diU|8^KL2o>#J`%VpDR{gz$Anj^ek^{~@0LB>x`^zOYC` z+zFAq}gX$ z(uQLcV=d9k-(s)(%m`0AAAb~5B!gW>mG}7lp^E;T_8%`*qWjAZac6n09@lo=SuDS- zaoeRb-u6o0(3OIJbM>9nmKSQ^JF2(UrSJrb=G291kB7mB5WT zC)59B!2clNKYw!dE)fkcMBnAI4jVhcP@A=8Dpy^!>*M*)9w}CwAyI)oF!{d~=YJ{0 zz@lIJVPTkyEh8!0My1Ve?EJskN}6amt4L4WM0@=I(?6__h%eeE9bk15C;6ynxeI^!h?APUF%5CP)lOhn%n^0IqJw@~95{$L2RQS9gq8tjQL9)xr z#59q0`arsWmP2z|TBkFY_(}G3LoOBx2vs=6Zcznwc7B?Y`8Za4Tw<(-fI{U1rWBT0 z?Nlbm;*fGm9dK6=*uN>hP{UxeiOA~rp!GCvdSf`Xt~^~r@Drg|2esvCV%ryUxnV63 z+M$*FrU2vW(zE_m;j?`;Dad9*r3G?KpW@(VweQPO#R^t)4y?Y#ragiKs{LxtL?%y# z4(HkpAHN46XT|DQmbJl4<7nKP>m-fZRag@~cvj%mD!?Z(6bD+*v^S*!rl@-GHxX;l zOMq5l50CnU4vYU0zwi`HP?PCp_N5(bF^TPSdq&mYTcVBJWd-m2&l=`$0MPlskV+#` z_WGD~(vi9m$r%b3dd01mhy*BsnH}||5_t8V-M%dq(?>8R}qdxI(3 zp6}Yo#J>L8Pj**G_i|bWNho-VA5{O7@26ah5BG4YncUIdgc>R}s|1s-SVsQSm-vT+ zV@9WTbNReGl)-qolHD1Q%fY3-D4RA6RX}SC25;$9*m|W)sPZl~z4U%5T!lhsDHlU!WlC$=}p zpuICrYa##O)zH-|5 zYzs{vhCv)~BJO>XXq6i(%6v>V{7>K6&*=hI$aO#~6^&~OS~@uHpnAtRqbC~o zsB`rxO=Hu-wROdD$VuN@zEMd0%`Eo|rk5V=CM0DR$0epVK2V&u8mC&+`aP&7*K94( z`dmFS7Lfdd2@HeJW0StiATEZg;|$oE3QO!g!lSTf3HEH*Ih;RWkFj%PHBoLvz#I%C z=pwA87=4s+;i_XK$MtO3yUV&(fT9{TBPw<}S$s*Q5B*6_0DG)eBg0F9aS{yvt=sME z^sHw7sy0{KXeq$`zLA-V`@!rYurWm=s&2IGNDUdgCnVF`nE_PU)m+SwT2vAqHGe-z zY5qQf*kH)E%;v;Ge`x4`_SBlePou%**jCFzYiI*tW#&} z(q-G21OX~j7={4Q2L+0E#+#7_8e>Nwv4Ec2JgRJ~n(}HEkZa`Q-W0= zCfc>~lRyqozM)KyA=-0!kpy^6Gs*cn72l9nKejr#N=pZdi+{&sA|3ta*dlBkDWeFE zA2u;HUvKzGVp#O7JLr224(sGNAw9STVN~sURfZT|GHP462 zFqLIC#D;%>M1}50uw*5bZG)Rf9&_((wBd}6VLOsWjEeTkX4w%JHJ?DX&Qb`C65(Ew z*^)V;6*~Kxty4OOPj!FlJ0D2^Pkib++buDpO5W0rQJJ*LKc^JvS%0bp^jMe1%}XM#NeM6%pG*31KSh7IjX*=r z`mEsbP&Hs~=MCTo=q&f~T^@S(c=)8@>IzmE-@6>E{ zuVX5~wkMuvnDBQ0<ks&`L-vXTNJBJ5N>vNi&@1*ivGnuO*n_|7h_9pAa^5Lw3$^DY6R@7LJ|trR%DpKlVH-o@dpjq+4B*QT@mPDC>iI^<o*r$UMg%*?xN$c z0p8q}eMk>0sh#g}c|HT;gZp`jiQO$58#gF0_rPDS&k0v-GEn!riCc3S#wLS_lN ze4}^*KY*W{Vr`=8pljNXhyF~7=NRj ziy|!$%kH)Sq!9fA?n!gLbv-|>;SJ-BmgsMzO#8j%n34t(l^2hMQz}hkMx&W|j)AjANxP~Jr**O( zzPVwGbWmcOBZ!Et4PSJpnXv2}UZK?gg0F?}uS`uMFJw9hr^Y1;A4NOPrTIBjP9N%9 za2@CLx-vHosyt{+#*7e-_$>}S3Mm~pD5Y%(_ge@FDnSUPD=S@CWol> zYVVl2dKmXotN?DdFL5S7I>)w$3bq>8T}RupDpsspg+7E_~^&^;Tbc~vzL?s@KfSB1kjv+Z!@vRIa6|J`an}1sY^_JF>Z@>Z~Gqce<52`I& z8C{Mo>%@z}Y*iW}6B<=e{T^U^6@|IeBUDS=LmGRwWmbLVZpF)HiPt7PYd5GfmJ1T! zSP@$+Br;5?k{AK`(VJ&nEvFRC6o$esQS#J2cRel`6Y7_#Z-;0)8l*q69%$&!;I%e` z*@P=jmx(D~M*EiEyCUeWBxUisqZAdJI_&kqbUs{A9sLNeEJF`db9gIj?Bb*r_tVR_ zm}2IPGY}4F+O`e!dq<-BP2qKzoIoFhF+{dWPCtVZ+myoZ`e-t})KzMudXL?VNzT``HD0AhbyKX_PVcS)_Ib5&y3AJ874@h^^3DYq9Rxvco1jm;q8 zsNRB*u)M{mRn2LnRGk|Zb}FQD`Q=#!5|)PdK1l7NocD5YYo=1OoOO*2_ceD&7p`I3 zbc4cvr_+mbM_$vMK6{x6(ev8)U75vYy}p@Qp|qw<_ma>Ve5%}QU4x^U-b(M?`*AlK zWxoOSB|_+Zg30o93#&y`0%r8P$Ko&lX)Zp#;CF1V$S6sF&uFO9HG;PFsYJp%Dtf%F zA=TFj8>YDm$C~uOtq~z}@b~_vIXB;co?RoVMpxGP&vEzI`2sqtRGw{5380U&%W@@f znPqaS@r36Cu#cUZ=2VW&`ss~G=FvW-@$TR)!%w!k==;2b(3`bONbl_81FvUdcj04N zV1ieOqAmReCllBs13VQq2mi(B#%r4L+z_Qpc*ETvF3-Ofh$dT;kYmvUO^A~?=nG_I zzUou8eMY5$UOLN3#(hYGID?KY;oZ+ZP?6}{{fK>!0#~3kbYFET{RXN>B}G7gWS_tx z9S5qBEps)|Qx#qf9M=BQ_!;yY*EY}F%vu*6*iIFjKI&kK z!EObY<|=A0mZM;?Ruf^~RUKRry_pw$N{KIM^rny*1d1l68i>a&i zYg6~?d_5(#nleF^gJk(n$=FA>&cs+VlqWQd(Vb0f2r;P~-LH0+Ec&VO6MI{!-TXsv z5|PdSQa#8`}jpuD+SSyT-`gQ;pwe}P!@gn&IRvP525p! z)Q&ABGbndtbF;Pk>zD0Nzl@emucPGos-_yfVl@wbJusW-FXDUkku@((#=jY}kp&v| zuKTReFowo)G36th=}H&s@2#$4>M0%^R5Q!xe{1BB7qhPG4OMCHRU~(x+TLMdlDU7B zmk5~q60`20z>afcUO8ezLPXw}U90WgW`aJV0zB}Vpxga0xdifxj#?`6$eR(K!JO!I z5S})_%2)(DUs97P)-dti(Pf|6pk2v7Dqi&+-rzaH_+KhJERgf$Anjs~Bas*SCiTgc znL?$>SEh}SL#~m1&X%-msEfJ6H`kb@PeE@*n0pj=)y4sj9T#o&hMT4IU`%(s;keeV z**;Vxi`*I^@EzOIVbV3o#v{;8SAScx!XI5OZp4&v^d+-Xrqu683GP`wQZX;B>9kZr zs@t*g0lazjVG+ z9I|GG{zbMGW&FHur8^bgQRlq&g;6=V_(peCRo4{OH{&UPGK zlGCEFTI$B#L2F%-5dks5E56^MYx|VBeFcU^4o08$9O|3<@o_IJj5aEK^SNzsD$t#` zG2n{Zu5Yv&wO=pQyu7^R*u!Cro;%C_fw`(kdZdQHfC&+ObnIvR&TLnHvLOA`$$uz@YH(qE&4oh(;V^S(~t=-T&!t8_)b+vFOpu_cgpx;}H) zn{H(zq5nKNNFVOBAn(!Y*^zTR|MkAl`!NA&gKR4%zE6KKd^<0F_4(esCCb(iR9w&r z3@9)guF)2xPv?oZEo$Xk`@Lo1QI+H#{v@hcza1rtq>jwdLB-qUdf{p6x3f!}^>*pY z)bhlkUO#H%&$0u97Z{{eg;(^b@cXjW*CCbUMshM6v1LkMfkeU2)a~+tFki8K6=D;i zraHl!UlLsQ^Gpm!MNy?ydCsC-5-2-Bw4y=Wh$?mB_UsT%Z6u{bY1)WWuFmjmfak5E zucFL->+DVW#EJ5t1ak*r)@4l_1)w3XfH+I2quem$xxw=m*a z0JOngLH&$yPGCT@9YTc!NYWyQ0ICM;k32J{U)&@?7+QR~FjRPhOY!()aGAGMvGo%B z9=odL)ciFR&cx0FY0#wl5tN?Vr|_)a8}rV8cx!~iZE7)?(&m9*E{&L)X`fPRW2+=! z{LGU}^sL4xch0Zo-raxe+8p$875Y%|3sg2skfI{U;YuJ-t|$+E~S(MtIBjD$Ap;Sc(L##a7l~RycI6TU!Y1e72sU+uz^=2E~|!0$uCl?OP2` zI{9p!-h(C&x4CNP&3ow^oi;WodZDiC!ohYPA=Oc@PY;z=yz?Szf4NeAb?PJ1m`TAjXAzoIx2&ho!;1C9L6AqC zJE3KU51v@vLspE$>3vxh=4`fe<@6w)&4H?KM32K`);)(@rt~ruUUG z>NO4aN|z7Kp2l!J7DoU}O^n~lUKYv!M?US}AxEwAH`1?#D;XR`#@H-J!N{x2*~ut) zW{z~5GS!(B;mc3b9zl6Y6-IKzZ)tn?T_!IBjUh!jB>D4SNj{BL* zlO^j28J0fdS9fhpxpC@cX`XRZlg5F2=dr%WOCyD#$()+!)CBtr2R?m}xLY2BZME$& zeMdC@>STk7+Ep{CcPmC_>r^yjvkN8w{KoXC-_}khLsXM~m6&Kjw$8qDuaA0H0qGXf z$`k&=U~ct_lK1n{1Y=66sChMn98?RsO+%tE%Nq4?#9u0Py`18|Bg5(1=RU|VITe9Z zidxx%q}K!JzHNCD1itRfR`y#CTkp4Twbv*G>8(UY0lPIV z65f+dYd$S3jRH-tY9euiw(z6%jhf+uKGTsN4aJMqJIB{JZ0DqXu;D8<8z4e5O3&$w z#1!nPua8y1a`vOtf)Lm$y_5IWRU`Xy$%n$ev@64VSh0K|@odi^d#ThY7vD$@+huKy zeHkItK^@kwoVhq|ksVbR>jTMKNR3nyOOXlHmO*>6G=3>72MylIUQY4-5j#aNLE^ec z@~`b{N|Kbuih6{>H%Ii0yH^{L^||BzhX{Pt>%+y57n_#Ky@i{>T8Ct?4}WHVFDs`T zMWB%T%eMYM3>{FgAo~XeYc*7?+4ZIJUKtyeFN?P&%qk;`MxQVhkv!vVe~xO27>7PQ z&`(>}fk+j{O0#iQ1&FK^1-rU2>W8!TshLs*#|V=a7dut!!SVTyl$V$GK7dc?HMshl z-1LJjH#}mdZY$}({zNI=5%a8{f*xjES~h>rPgAd}cJPX}W36$xE+Ah~e+4WFxMC!U zHMV6~QjxlyH$s&I13!20dD1mf;|>@~tAh(K3_o#SIkRq@E~}R&Efjv{+`u`!@PHNt z{VQXCk+zN|=^;e!!q%k?*5hQN6;5co50tAm~b1@*9M`?@Z;CcOkv= z*EcsmdZnVZsdfu=4mErfG~D50+t=-OT7CO&B^&$eOxPX1!xQl0N!HxSw?-$Y`0P)_ z8p}i8Bc+z;kux(a8mLyz#{DzM?d2~@YGs}5OTdz>;00|9dnC_NE{Pg*ta6Bchp;${ zaa7$S>&Zh_%GtfBUPn3X@k5O<7(si3cz-hz($3FQE$3PrM$SY2%~U=3Q@NI=dS$f< zPgIYW#OdNbVq_wp4CG3G=QnS_3k=q1ZRgJWRvS={w5Z}Ut)*~OC$qB0 z+s=cD(S5Cuyj$%yRPO?q8hApaD8N5fgb>D3@TY9XQoD0y)y%77p1bano~oaYLKner zXR2-zt;Nb2ZPtQse2O2pqq@r2kWfX>Wu$R{dThjq_qtXGymaS?9b}epX<7*!hYR3q z%r^E#I}U|P9;N5TRQi*P|?FWCrdFU z!aK#7v0s2`>~3D8YMS2ZAQ*L<{M?0|i(@ml@Kz_g%#wxc2~Y+0E9wn7-(%W*qZae| zP}30iNwv~6=9!H%N(X}v%-)QQz0?wZZF0(BAvyOk2{QU| zQ2Z+Qr`xQLjPs0e3{&yklN+=K>ES%gWJxZ~F69^xEY+`Ir+GzUa!o2|L_7pv-;|=1 z+WT~4O9EMfXI!y40R@@cu0Zy}j=kXlu_?`QAhq7DuQ0%5eBabstnMZ?a{HUVAZ8pY zTfa2niP0D9s?2Y00I%!89Yw)i>C#bT?=A!RHe516QwP|{B&+Fl4BX;)8@@d-6Ei72 z9QY>jV>^g3uC$Y&0!ObsEZeqmFPSC1)h|5m?o8+tfuB3 zpm@}xfq6fA*{AAOdqcn#XX77=Yr1Pi!Mwh%9xdT~fMfPFPxTwJ^g&yDW%z#A4EL|( zV4o$_i6Z=8+iem!MIBK@ZKfnePZa*lN7ipUfh-l(@TwKnhP>~5{Y6q3pz_#kQvwvW z^O=GZ0xz-uT6Lhk+bT)_l~|>-#j#ZIDc><1_RTT$y-%UR*XI+W0^Y@8iMY?f4|wy{ z7p7xMzkG`BxB;8r%Uww#bElVS?<-|~+fs5>K>g(=>iO*tscw!`9BxXIM_j_eIYBNe z^xoHFya9nHQ}W32=c5Gn99Aqr=Gy6ih}6aV4{Qr>vL0U(6v#$5T0AXTJe(%S9A(?r zeWgeepbs6aSbDbM%`Z`)lRxP)gaB>O~xixoRVnDn+P7WTiM{N{kF0kUg z%#PgN_dQpMYn`GIKPLI~^2t|) zL&%lULmJi*eR@Zvc>&G`aRsF%I|gY3l@%%*@M{YX>tpp4ddF-DNxXWwWPGG5eZnM6 zQR9_kW$2r&BS&3Tw#Xr&J}(?!R4Bh4C$eJv6Wj{JE!-!!gDeji_30NmIqCNXGs>5B~aRET^P|_$VH5e%0!&Y`_I$K6+e9K~L=g2r;P1DM0uV z>_w1@9+6ez3-gry+Nz&jMoh2T48e-0FB%NF!}P0}H6ib|@G-#JW$bAF@y^n5_NQt^ zqN${f&b>FkV+Qe}R>~;0ES@Zo)!q)Q($2hbeay>%v$)8EMKRWXVE<`R@rzu21GUK_ zfu`bavZUMa_`(wRdfUg7))e5D#|C9Ud9}j_s#ZOUP1@sZ)WYlpQD$ShBPsn8<--8w zst+?>YoiqhtDeL#Q#?bfJJIoSF<}*pw;}qU{wp&jV?a?wv*MvR>_X(RDp~&JkfF}! zcOG9m4;*;DyszC!Ua{^)EPWvWfQqB3swK_j(roWKAjNg+rAf@Igl@wrSx%J7?UiOV z8}+y&3E{Gv+d1T&%jaVk9v#cXz*R+HH;Ts89F1oR^!JA(IghkTLzsIv&K}XnI%W*g znqk>bcl4Ymm1b}$B?FD6rI2(e-~^JgO~-vyQZP4n%Y`dvrI1JSAK?oF&Qm`WbQf-o zPWTw!D2;W9&ov$B$QAF2(gVMNVsE zkp_wQGIVbPcx zEa z&8SW$IFpY3yFsRNUGSq3@W`}OWBG!|M-qG9aU8O6ds02;EPD|+b?I7`M(=7|&mGCX zp&(nt9v{QVq_fW&=8dbn9jvQ>^`u8p3y#&+Hlo@H9O9dMcdJulA)Y@>V7g|ep+2)p zl*{$^>Wf#7gpq@Yw(*_(f+~LBlbF^BuN2p6mQ>bv{uZ< z7ol2BH>Byd-%RI{0jod|A4EMdcjR53%Ip(feJ~>LhufoF^XqniZe2p=NjBXg1iLh6 zJyWulFYeyd?gR=Z<{uE#lIQUbHW?Ke?A~0ed3t*&VqH)NQ62KdA5#|C=B5Fc*H>~w zr=~XC4)&XTPfCOYu8AuVn#R~;~Z)Unw^bK51YYP z={Su~!?(wM&xfQS`{%U@ds5O`kB7ELUsHMsa%v9O+mz>5V7;t;JuxpK7^3<9qc4|k z!cJ7)pIcwv!lZekC7w7^C?%gq)>=+hSr}rK_-p8K+EI$;h+EeW`#VDrYJkk1y~gGkmy)$a@kmJpxut6 zPmeK3-LsSwSgyE@A7}D)2L;A*^=SUpkYbB~c+@5y6e4|m_%jzC=65r`?u?TLEiuu| zk*Cb4k2mfC3;kSUo-Nbe3mafO!x7y*$QY{4l4(3F>=7q5T<%=$Dd}1F9RloC50#+{ z?MKQQkbKkrxP2Y>q19-@efUi9PM%%=yCbZ$p2dn#b`!*ae&S0+tLf2)ZNoI%dJP8& zPL%(9G+mtKvH6Au~o_lpa6ywzn~De_&zOR}_XIlcn+KQtf~CwQ-o z9Mti;oZ&zVUI5?OB|uGKz0J|~mtrV^Iv>GE!vlfBDvR{a5}}mam(P7sn?(sdP6Dtb zKg)PG1elxY8N```jK~qbgQR{M^xby88_Rt((%f>&TFOYN-iU`;W;%uwdjp))6n zryCJR431OwB3L2Dif}Z=eIJuB$_qd1*T4WW<)Lpp1=1t`ISU}oZPp8%u7_wF zv~AwhGWJHcrTeOVm<0oYx51?&gfJg4{BY=*BZffu4Brvs zt@a?XA&J}qr9EL9R6r*uh>CkUdpFdKJ%Ujs$@haL@=>9PwPCamhR^{`<7iva1~gE! zHoWNyIp?V}DQ!feFN1%{ zN;0Euhr3JOFb#g{Q8o27UY-?o4TlWBzL^!=FhHq<4$i7?sC_ z6Y;L1__ym^6kwQ0GF<&R+&-HU$*}6p627OzFnK^h|0q%(=kCelVQn}&cJ9c6rmBdtqMTU#fGYDB3+cq?82mVKQ=o^MqH}H$ zE(lk7Qj^!@cADd2P!Ek`tHDx2#1e2X)Euz@`ARo@3Jh>N#_pG;m5b7t0OnlsDfBPV z#j+W!fHe~81Vxfc{n&-d>IuXeg6ZDvDu&dT9ZGlKiZfh%3M-zQ+3=1T-!O3;^3S*Q z&ALrH(UO|8=L{>8Cj1# zmSakDskSKMQ0?EI=7ocMivsd-#T(wM*N_z-d~BXZ+!`Hd*CtRd1`XP#)9oADA75(4 z25aHJ6|NKE>pSrp^4uZU-hCZut)|!Q_nHT6IMAyFV%V&P*6bY4@PTeN-_hA%MlN0UJ`?oPuT`LhT3|4K5;GZ zsia3D#k~}WdgWxWS6}^naoo;vu4OAT`}moQ;C?&V;m+sLTJ__9i)+^OVwX58!+l-sRSBm9~o+))qPtB-q ziW{1-q0R%DszThtr>eQP7(PGqTS=*WXpqucSn#b8^!SLL8YM66T6i>V^8A-r1uC~3 zKfA%O+H!d*HLA$1*Yv7GXriEFBH48o z?D868PNLbejN6p&9>+1UU6V?m%&Id`9;@$mXMMM_-T9szHnz_}W6xoO58i~n=+Fol zqG3|P>ch4k8yIwe?!3`h**4j@f~u_`-x!tIsA$35E~-wD3)6V56fHcC6O%G3XGg3> z<>Mb5NG{4zfG>!#kK;fjhBl*6l1%nRwsKo;Rqr=SU+Y&rwIBJYRjyMr!9s|fqOw>= zH-09@kZ3u~;=fpV=3n{7s655`?fH1KAHL(ez40)EO$;#zLxy@;^}8D#)*wE-vDFt_ zf`ZxF7?x53?YJ`E^O%U2!?DfvGm+V?94_LnMWWA|sx=(8gS zia$keN*5)&!j#aQ1R{~W3I2^gUc9iqV-+BKQ-zIz|l6B)rxRBR=Uur+^-ZEa+`cmBi- zu^E7Tiv@gRk}S(x^@-FI4aYR*!^dNG>|HTARso{oKCx{RT`{SH&?A5a>-_GbwZ-El zM?7d-tU9J9>ZhxkrEg3F*LLdh z4qZY-cJCu|hF@`#mVRQ5>6^Ky{JK}~*`LsIhQavRG7G1nP_Xs4T8(>G9Abibpr;kq z_iI13IS9EfJl`&}kKXkiKZneD&R(m0kTTO*$*%5ct@O#)+MUY@(w-Sw)%Z;*k>K;% zSBtYIo3>$WChb$zRmR7r?Hd}7z?<~xPnXNx7jGvILN>zr1k4pjfdW6)b}qn9x7%S`nY8nG>AEs3OH}x76f?u-Oj3Hlg62 ztK4q0CYCX#Q5tJZg4l<45X+Bk)D0v>&zn_wxmUq*MfBw%^3 zhB@Vm9JlIE9HdeY>sh@%wi#X(a4k}hE{WZf^7Dc6%67S`a9OJ{@D+m0cK9!@_|W&lyhJ z-08*6BxoxDe{2jv)c^)#^4 zLLqt7BvSmf3QH*a+1!QaPd{SR>a()@lu|u&b4F!sqhRGJu`=ht%;BxQ7enzWZF^zo z($1BJAAK?It)Qyv?1k)axM#8D#$9%~b$C>Btg}xBKtHooF{TjLl-3C`nIs073P|~z zeIUNR(v$BfZ&0u_7YuJuU*=Q2lN4W)O7HWPldHBZhEh5*W|Fcob!Ngq|%3>Iq->VMV z<7EHTy_3-dlROx)99pNMjcFp@Q+6KU{F>MR_WA&)j3mS9YJW-LL^g7)8ll&3J+rI( z)%+N0>PJdVZg8l=+(m!!im^ODs99qa%x*SoHta|x!l$yqyz{%e@Bc0TOE*53hZvvP zj?fn9b8U9oNGGfuDEY$8`BA^Jp`P`VB#MdENBJ19%?HS%LC$gdcJ@t^k=5d1fe9Av z2RjzFtsA3+VJq_(mweJ{7X#^eko6`=D`@Rw=>|66MiyR`MSqk>uV-B0qhW5oB}`kN zjbvD_Lr*8vzPtU0gxq=jp4dEn^M$wDf13JTIUYaDL_1mpe;AAY?X>w(P)JfrTX8!2 zf~0K-c9L2(u=ulg#tqtU+7%t{8I#AxugGTDja}yjelf{V#n1-q_j&!dL*)0I;`1`< zGb$kuouUONdgwftIHI?;HJq}pG?W-()Gam*svIETWU(XOB)L!>;qr8Gt|x9qg}3rp z@IHzpRBz%EKn zJ@1_o`Jr05QlWB*X94f(+vb|=6M;g*DF@Qxv9s(8Wiw9rIP>E?(P6teliOB%!{pA% z8@b77$_AFmU;%qiy@t=AsSWSOg27p3riQ}!1?BBDXHol}?ukO96raf-Uy7aeCFDly z)~gxa<%$Kr@5xPn*M52SFMB-vYmZlXTaN$JeK~y|JCqy#(nYA3q?0;H={NQ&8u__1 zPo&*eQipvfb9`@cQUY$KRKEB?EPh2NED^zysck>-S(k%dQy+UK21}$ym$$XKrv+vp z^IiAT|8UR0NpEa8Cq#rY;nJF{~LTxq(^#=8^Q@xHl7cJTZLo|t83UNjoNM8 zc_(7~)D{Oo)(qU$I)kUbq5hKE zn%+#wG(-#MHSK*r%Z7D|M~Fw6SF|g+`3E2RBa;U`g1)j$E;MZte1tHPfj4D=rk86% z^QY`Zmt%E5V^|_0LF>!XzFYg>TiNuuCl_P&Pi~^?V#%ruPoGcm%Q|EDb$Cv1ZTiE1 ztu30_p#*qqw+^nX9bEk;Jb&t1@oGt-Yhsvn**pNT4rQQeefshy8SK22=C5L6>zA{0 z!EkI2511l{*H{mp87^H|;!D@Ymfv%6PS$g4&evp1l@rAdrf7qtU7)J!sPy+?RyTdd zSKjbKR9o`)EvZx7YzFuV_9BlYIwJyEc0)Y5A!j;mH~`3!)*e}EV&hoeWpm)5yT z+j8KJD&Q_5Uf13_f$}>kLpBext~3pMW&|A{^=8A5Xu$`_;|9DQs;9lCwEbD?6XyV1 z6BO58ebWZ>awuyRYMCkxD`cUk?2xpa(u{8Src@>4D_hyPQFrfl{`%-l)-}4iI5YdP z2+zWCfOYlfgX|=|g~je_yI;LKob@M{(cB#;n;(!o9enn-_AkwHy8T{V)C0xUe1*mE zaIhgaf7HT{dhM;*R*TXLS7}bEsb7y04-~Ny91^2SZlV9KACTbcf4V4oBk@XM#?&%> z@>b5M!Q-cTCjY|qC+83zde(eTRSp)8 zXKA$k8pX&$see>@tKuwAKqRGn>2vnhohqJ_i?lnO$O5Ua8K-T1MR`9ns^k^rkeTn+ zPQ!k5wO?g*%wxzeH9P^;47{AgHc0Xi8%e?tw`QG;q_u>G%20W5qmLrhB?oxuV(-nfvc{{6PKLxYc)4#r{@mW^l#Z*_t9e3Z)=V-YPfXKl<6ir4Ij!CP|RFxi` zh<{R+%=TB?JqiZ+DrpS^an{t1!8E75ooKam&<9W&-iLp`zq=#PRI`bzU3-VmL@KU&?8d90Q|0UCpfE~66zZfQ8sRz;N%tUr(juu z@$>*Q!pVKz1`z|P>K|8VG^}}t->VEgcxmY{h=GHrpB8OB75`xt=hZiDp64IEI_!F& z$G;qfv~U?USGx(`G5J-C?I>Do0SS8f3t@0s=sKdknOOSKEOwkPRX%a49bWolu~sP~ zE#`;UKsEz0VT5bgbJ&rU{jch|-`=?&&n4oqINj+_gMLefcnoRP@UgU-RT_TnhaQwt z;s7id&19!_FuW`W#qGB-K7uG48Gy{wB&9`f6dM0dB#TpCe63-C%_FBjf9-V;s|jIS zfrF6}-tQDO|40fS#oUX79iEqhyJV;zvQ`Y@jr;{{&U`QMa|K!M(Nmxl*nG1r#8&1e z=|)CH5SOK17#)D0E3s-Z0X4kkjr`>XJ0=8VHm?V8k`LWBgWP&_;g?+fc>Mj;c`4C4 z>6n!+Jp!=3KCqJT-EzYGv*pwQ8g53&ga5L`&_@=@$fw^}G^RUQ9tO;$>Ni>a$_gKd zQd1H5!yL1&)IJ4Q0VtC#iIv8-Hui+TNOLZ$`aMg2e0+N{jwQwB8!ZU5NveZM*?}QSCgkBRuhuniY^9ne3=9}-=eeNHgkdw3b+N-a%_g-s* zfagPA^m0qwEd3pCYo7H?pNhUP9+L` zD$#w|zM6l#)P0uadpWiy#>a041oO^ySQ$nxhT|l!azpF%fZ%6AbA&Wy3Y#bQCZ)Rfa^PrE5$@! zY|_y#cyCxq%j6~Zd{L2kyJc>ew)0`Z(PMFxcz}U#4%%PK{TBViVaO%RfI|UuL2n~{ zpC)Z2JILhsB)?A#I0|uXJ%B2KRf1ulRtc zVOU31MyUel?MywTo!rKM1Zk6bx8i;7PSnp>tv&j zl)>7BE)QvZm*KyT$lZS6+xIDYEKJ7P4siW6lw@=rmLALGZ9*|4i=C6eUO;!FE=Dc( z*>7(pHkda)0DFceYQat4ww-cg!JnGpR--HL9@|M&&p(gXX>7yXZ9yCF28-Lw_Ma{9 z0Plz8YvIbJ%dyLEWC>(&t9H>A9wMfe+~Kl@pWOKd2Pgc!ab@e=5W6ggbyU@$m1|T2 zaAuWw@N)4lv=BM*gFdsGnPX+?P$aG@7vZ%5XJyP>Lrb>aOvL;1>d%_X*N&zz+|tcy zp-~_D5@Oi%HQS1#5Gid`{OQaWr1YTqXIuN+Kn9$s@eVVazXqwQiad$pUdEAnIVn%^ z_6twj%oe}=@Hu*EwtQ+d&ySOzV5xnMb$^5{j?Y+Rn%$eL&>NGN@hEaf*E(;}WQU`_ zQ!85ai%ZT@_niEC@@#uWVw|dw#+Qt8ogFGf7YmubUDi$8)9S`78Bx`f@a4S%?X{O! zosCwPM{k)UvLdvr_tK-P6K$Q4{P961v;8y&kHX5aL9_7V);&)_1Rxhr`v!cYFx0{45w^AexZr3mYr*|n%rS_9G zC)x0-JbWg(kQJE=6m1FDS|#|5*kP)S&M^!|PpjCRl(2f`M-{)9Yb%c#l4Y3X)FkMZ zUzlk%Dqiv-mS^zJ){p~5F8IYEnDlBTpR3x}h0B0?R9Y4x;%k+E+$h725Qs4*`$g(o`@r*G%u1U) zLWgHS?bG`#YAUjdZ(n!yp3Ku+ePNh1ar3he@iU-yYy-(W^SZ)})YVd;x`}ILx~mRo zywRJ9nOi9R;@drZ|3zX9v9lF71B&>wlK5-K0;ZwsWFEAJ%FnU=)wkB4S9d9MbexHg zh?{Fp?K65La3ovt@w_9e)0l$|g%xx0`zjN|f?-Af#2c`glbI)ZC@%L1ZFG#^Zxum2 z=#~^gQVgYJ0m)M`w0GC>)rlHp0yU8C_L)A}J9^O%_M~0S-kO--;F&H9Q=)L!r6uxu zhs2t;*Rsz-76~@;mhrD10BXyqhqgaiO5G zlUfE<^Xw!q4d;ZAH%PX=KH>%b?6&@;Ex+BBkh!jyOI2{JQs^(i_;tlKpwvh01*SSo zrM=1#I{)(DzM>~QCPCTjo%(2Z)s4oE<1_h&-U8|=g*rFz=^ZBCk>&sR_l4H=Ze#G9cXA6$75_~sc(1^{^PCq3PfmXL_2d5~6%RgTzIjr!aI$paW3uT&s>f1MpkOQ| zUKV6fc=2opMe*;!=TYoS-l!xrk95HCh5v@P2PBDRGYn1~L80$M_IH_1&oAZ;DpFfW z!UZ2Lnquu=vxXmoD8==J&#-5y)i$9&;uWj=X@2S!%F&YqczmY?b5|YmjE~nf?1PG? zN^IZbIX)B`k6>_kx-lB&!r>gWXkxy#?s)W+g)jf*FDDlHd8HhIq)iK3ZB_ol8_uIO zM33hEj;!rp{VVYb*q)QAn-b%D+5!) z+g~4X^N;-Q`@afrYxlB3DxOu4Y&77U^pzH-`Xp2M@fcm?ot*3Sj$!ivwQxz1Agz(9n|mp#Gvn`jcrBS;1jB zs@Sugibpi&bp-){KeG}Y?#chnw0{xP;17Z|VT1PTksxp4$D2UKr-&zKGBSmq?fCzm z_E%-N@0WOhlg}WtOxgEE{)bfFuQBQ`ieULonKa?E)o-xP0=cczdRU%em0a%@E$@+B zfsw5xIC}Xx35Y@Idbhys&vJt1Tb7~P%2a>B^(SXAT?h2S`IH{|jO#b8sp}%KXnEe?rN%C_KN1ZYq>l|8evqll}Cs zX#YXFoXg@@rK5j|o1fnO6Ok9>0d~F6iD?0jZT$J&zkTDJ-#S{B8he7{+F}DQ| zY_MP_v*h#t48}PXIiS9jYMP1UQEB)K(tmtVy8;w~Y*`V^r#w~;`XAr@3vdN40I4cL zaa8{oIITM~OS8{$Rm#^6H7*n;fx zl2!Rg;@0{7_`JEfxxD=8bqSX}`9HhO{i1)w0_O>QCV?Qh-!Gp1s(X&;lb_p8j4@un z2NPJ%(fogz;qOOrj(c({GErht>}a2k!%b3t)t}_aZ@x=cU#0XUZuGzXPml}9i9T(C zq&d?bYAEvSsg8jo>T>>p!feNmh1!3DJn`1`d&U`x0-$39|BTze89GJaG<~Myv=
  • ;lrGM=y2xykGsXq$naP8D(Udsv z)sSvj+UCOGks`Vk+k1;1MaQY_7!T0arXp?X{y)+7*5zx|L!O$531jx4h;SH@BZaB@ z?1~a&l}SRPcso0bMW4j6A-cTqP)qS)WdBV1N&Ma2MVD>mT(zAp*B#URs`Z!2eI0z( z>kS-Nj!9dsC&{anyUEMJmjBoh(c$`S!FMR#;)s&Ak6j}#16SpxEJLW^;|9-;$x%wq zfoj1na`F9>YUyITz45INl&&tpQQ%T44o2D7;}ZC%Q4|<%M}HgTl@8spQT{b73M{u> zzKx=xbF4$*zlY^^UaijCR6}fpUD@I>E8tiR@JL>D)vbT*0og#>l#>}@QA`+uFv|9L zn{-$8&fcL$Tn8L7MhjR??WH$v$GP$FH@@cQE?lzOD%HR~YUR9SEl@wlXiZQ4KfUzh z*Fto_c&0YReuvvy9YrZ%Go5{hwGaAbEtYE?8>Q-C6qlvq*nb@5ci&-+gHfIrksn*z zzlX)pH#YwO;f9*IM-{&BI0&%q0vmAIW8)wBY3&C*j&B{nCJW*Bj!NGQ8DKJm9CG9$ zjSn!#e#am`7PJh0YhErdZ%EEg8Xo zOclCI02Z!u)Pm?x)4roa98fF~U$)xczh9X@yf5G=NxK99E9*Y37ti>sH-83jM#CWW z0KYwV(t>~f{y^+~lMXnKqZBUoPn;(@jW_v?e>;e=$2#Gy1p^9Wyll`!`Us;PmTv&k zbb5uNgC(VlWpW$^JjE1nTGrZfxfPFM{^;UwbNN>KOK0~B7fv$dWGuXUcid6wVgbZk zvyS`dB*y@}En#40&?;cmAHiDIJ;!KdrgBVGoQeYg<9s-DY~9C}*KmH}o2oD}?)$|k zr?=ynOa2Mbz*=+h+=wIh#bvxB2dD-5%H2l5R;VzB;W@6wsCIDj@H~ewt7^;sk!!JNtD>)F9YMhDu zF+TifT3(JR0Ny4eBL6S%Uhf*g^Qsc^dJjH45rOuXkCwLD$`G%!F8l!&(i>Ku+TyuU z>)tc)iy;eZL!3He=-zNOT;b$C|36vE0}QSO0x%|(_1`+I3dR@|q)}ciu(i6`wp?xCjHmhrG6ixqV zLIC`5Tn1KkQt;(rPV(KNzdiD;A*t|~?%?mg0^qQ4#vp)IO>OMiIUNSwKpO9T#W(Zv zNB%MMa(vj7mjI>KR@!--4ymviVc%h`1BY*HWBfMVp+~UZc2l7IwFYz9w*@nLVnclJC|RfQUe3`=ET zvp2IegpAA4!G9M+39+v{z3e*5!k}~2$dhg2JxE>dVdb;bWdZGNeHtfxCRPSaQCzaQ zb(K#}+9E4v&0Uw|J6XDD6>PF{4%74mp6>Nm>WZ=ZoPe9@B_Mh9>3713<`W1`M8u<+ zs9V}jEfK%05t{IahSQRWGgL7u%(a`xbA6lPZ-w;t13v%=A=m#%JnyU}1p7^#=rE`J zMX;EsqWad&9w9<*lP)ya5&`Pl&+8mnNt&1~C~53${-{qjr2Eq^_jLn81LN`0^Q1>u z(L~+A{TrJ)FF)-*BnW@;R?6ThX_$(is})RX>vI9vb`xG;Td^oyWukB!(cV2kuAQ2z z)>WE03pLxdsh>+tg#R_n)Js5dZrRc3oIbpQ7$@JXyo10&v*z@fUwa_FdRH@H(nfHY z(k#0&3|IL;Wo!42D3HgA(ih!u>HyHzM4R~@oqrOiz8_mjGTeRHxyKli46Wt3V}c|B_f=yBw9&2 zG(*W%QWExil^0TO-ubsoEs#EZ8hAn zV+D%0c*p-G*$xZ<96{cBC{YXi!JQ88y0wPh(W!KSer z8+%hj%G{pFP%y$?q1C$|4P!Q0FpeQ8+q zS|o|S_d+wpxJw(HVk0rrhj4jEylwC1X$*oHuBY3T=l7@gWk3q}Im4fMYG)6dl3E=C z*^&c2H?@7~+L4yaZQ8yy!ORsb6PlRy^Dg7h3x?bVI)D_UFIncmr$1`}P+!iJi%ID@ z_3ncTQ&sk8>4HmRQsrtQp(kcrD1LpI62!}@C#?Tu_0!W5iQZnr$8t@jnIS{0mbI!#Y%KLDP4zO)1b3>eJp5r@71~I-%&J!=ez5B@5T|uTds3GKUGNz z2EHoaJIDB$P3tiBL|x#J8xL*Z)G9iSToD*o6!w zYgZ)K#p!h|zMwuMpX`AbGAMYIQ)7M$VKYRF+O3D`<=_iunVpt95K8S_u?vGQNU@*i z6}US11UW0MXIHu==2mK>zJ86ANpo*3`T8E=(G?|icF-4qTlba%6AVJ6B4MH@-D;>B z$-|mt0|<20*JB<{D)@26oWC$K>vb%|Y>`Ycdl6j-v6h}OTOz@-RLY@SluunvdH zOV%gn6GcT9Y-~a+URm9R8iU$z>W@4Gh4?pNnsQJ`haVn9Egs2?cJ1I&&tZK3 z=6mThnu&=}yJCN2RnE#N>>wDukz(_ggn#>x;&F zb^*4xRkk2*KC9P?zWITs#?eZ6GO`7J7 znHuSg{DF!orU-tERVU0ALql@mq(+}D167-)?v=65h2FlJCAf!m*Qeh^;>IHOU9mSU zL$1XrL@s>^cZqKIwAW9%xCv+?c%pIv|UzB;uBdfNqIzcaO=ySe1WBli$$ zv$&lWqoIe7?25t{zBQMN4^o!8R4z}MEJBJB3xTx8&1~v}_{b(E@20z{Eq)>a@F8WQ;4Bvi}8#FeyuW zY0N#DRC_On`&QGDy46IX;YNUX_SS>xmiRO!cfM2AcZ;>FTOY4&neoG+?(}W8z9ZwI z!CCqboz|SZ_49J(4Si|)lYAB{USMis7Z?axyf$`;c9PbC)u}Iye3%O;``h+I{MgQ>@aGKCE$ zvzEFg-AYBewGn2Jaw&5I?1S|7deSTM8&NVLessyjldE64?F`$7vPHNB?|N312yK~Y zGoft1CUf3{^+oHcK_a84-y}XDAt!TKdjxxx6GC+|^1b0+*rZ^$ZSvP=A(iy7OMI9=P)l%gc)o(xuE9`lajag57E5jj#y`wM4BWf*48glJ|IhT886-6h#s_ zaC=1pPu?rA+ir5gXyHLdRV@?Sfyk&JLPZ8KLCDgK)0~b>2I!&V9}aP+Qfkao=0H zIlK0{$l2Pd&~5>II{;4^7dz^R-o!8P*i8~K?i`D*;lHxEU(D^;`?5pFV`d~^sV1X#N}0astb=r*P5C0WL41TncJIHJS6tOpqFxv3LmQnNsG*X zcn>76Y6FGOv8T9|tNd(97q0k@K%^`uX-*)tMBFCzue&ry@-q5HP-|{0@uzK<&L+T( z<|zYur`EEoqlGy4eYakFo?Ec@-P9x#;lLShl-763{V)Xo^Yh9@S-p7Ga6@3F8{65T zvPw*<%%MdJelBEONA9gp37p*tvD>0s_ zK2L!K>O$(#)!b5LE~cq(eS^roBSk zN48%^Frx?q=~n{jJL^wRd6d%eFBD5)dL~^@dy>n}{oJh^ft8kUZ9^HPHC70D( z(TF%%(8lX|^0gen@@#n_bA~G}E&U1foMBDjJzs9lLq_+Weq&vls5jMPM$cjEs-6?n zm((YH>bzDh&6~S=KF+Ol4a}{43H$n0+osHEgqVicbL7$Fi2ikjmMn7)_Emw0%dYZO z8GSVy^T9%Hb&NchboW-N^_;R8V%fp5vZY9x^E;)Ko|=6KqSY!W1+Bf|$UTs}Uy~oL zoA|3KdruW%62%C?HT7=2h=jr$a0FR?idqJ7QHr)Js|cG~riU|7%6#uKZ`JghWAD@9Q! zApYAAS*^>qC!xF3ABRErL4AqV9 zx%s&Q6)8qNx)T}f)Je+>_~~k_UMy-S_&C7JV(9z&^mBpqt727GrMR(glI9+u<;D>Y zR)+0&(t{G)1j5d;vBB07B~m+@HUUc)qUodHg-pm&lV2e-?;O0Vv!>uM!e}&_kZ>2J z)Y^Q3Dx$kS^^$w=Vwk!B*@%)(05UvgzE36tUvSH-=j#j|zFL_>2XuC=oT+#$bwjaR zgoZcFAe_g+y!;NGIhO#Oc6E9;lS@P+Da4F9AUohvu*-zD?I#~z;`R>-JOcdLFP z*Hmhbyn$`Z-1+cZnQw}wm-9VZTf%Q$ zE3clU_ZL-C}5a`3)qx|{4yw+7yBeSBZ)jq1~Jc;`+JrL65pYj!-JAHMCOoK z`C!CZJ1Hae)vWwYqMDHGP|7}ad4|@VofmYcZ}+&L;n{Otq*f;l772T+{=qUXXeU}4 zh8xPjy~4fkWSp1Ra@lv0m&p*RN<;Yc3L-oO7vxWq7OUUW&X8*?a;)T?<5Q#uyK_d1 zh@q-LZlA;S#W=ymVuoh!uJt$x&S+J|W-0pIn>bll_j(koKs5>0R>G4jl}nK>zprQ4 zfquD0Ud*9cihYZp&B7*dQ6#uQb9LK?&R(f$2^16-E|UaOl)rAp2|>sC$fFAECuG^= zgI#~GCv-3_8WqsPIF&{bO4f;pU*KMBs(D$sh!3))lkh&*@Zv{2%pwcKMFg1YGK>Fl z{V9^CPmmGMA4j_{kuPfrgQ-;a^vXZ1L)ZH3_cu;HuN5T9gc;FFp<*5=@i`>v#2nffx+6h%1?(O7*^`m^)6zq{7rCDs`JgCkO_jtp%W9rkoi+e`#eKpjv`ZPMj~&S2FMHC&z+0u1 zqo*U^*UY?(j0C!5q^XeSflsQ^F)~oE=jT_3hih73>628lRL=N}G;fB!!pys_;EYyB z?SvFF3jF0ENWVAFJgXC4MW|!_kSTC8LwTVr*o^O5iSmF&vGtwxi!D<@09XZyo# z(b*>o@Z{!wslc_!KQc?wv?VR`qb*UvSI!yEmpLxoPXwyJC6j>GjwL?g{wG3wVpWSQa0k9y^r@0?zNk`B^h6K zvC|axnu}b*9#P}i^r~(4nB^qgso%v%*qa-vD(wCmNi3(;s+W6_YG4EEEm^YGo)c1Z zD}&RejGHMxxT*PSJ1Qo#XnzaM(>;*T6t<&J6G~*jJuQ8&xdxF$WanKU2bE|>@JCG# zZ#EQUa5{5VRo^Yq4p)wI9^_#U&<_dO42hz$)S4qwWe-m547_GGw}vQ4Oe?G8X2K%4 z<${uf7NdFk<9Q|(O&;$akl zzz-b`hpEZOhVV3Uah)0`r!eJa()&3e0e(N>rszY_=a}qzu&Hy~5^weLHec5Mb$1L; zXJ^Zb2>(MY>W)G%AERl~J9OJM-o}K8Zq>WDg;(|AC~4G zSjARa4e>e-WTQ}N@#?G1%$vwaPifTamdB7OGPy5)>wt?0X~?>y+8OV;rXiie?-P77ZW*sjlJ5@4FNWgQ;Xm&$2Gh>Ry~)@7!x zVpg$Y>AM+&SEV(AM7&3nuQ{DhlPj>ya&)C{@thiJnIG2?*y3UbgsfjAe&QTKfd{}z z-lj;?>kfu>DlLhr4)Yhw<z;qv_p}w2A%D2rh&k9xE7mM*e~6;m z(w7sE5_I!f^UxK8Tvz&@3iCb91mt^s$9bcVBz)4DfqbH-a;v^Q?5_;Xj}N_8n5H-PWm>GeQ7Js!E}B}YaAAuoTACtz^%r7j9wlr2UJaZb57}n zej>Ls%qR(3`N~U@cOmr1(Z1$pTYcxrlB(W&>=th)h#Kpq1?l`m`yU|5q}ZCJe$NV1 zKarixli#77w_hH<>X59J?L`Ns`fNhKzwn?h3ykek-)f#ue?!%L&#gaMX|ctOnT4<0 zbm#Xw&xK6JuMK+IWvRLEl)v7X64~&0ef6dUvh&gwrd>s5UuaM(iS(j2o&D4L^nt`Q zY=VK(nr3#H$G$AqwEbd%m$eu~aQ2NSKmVSa801MA3#|{^j+Qr7KVBKk>OX^z&Y){|i z$&)y;RSqQC6tjALa$*L1AKU%qczyd|V0wOm2;7R;-Q9(zp1pzL5t{zsQIC*7zKlf> zuZ}}^Z%?|S_6sH_Sp^U@SRf07uJcl;U#Q&?%Xt z7fvyC{!3V~B(naG0TE_XtmAyJ{#Jx0)w?7HjvTtBYj2}Ea{DbZ>RayV2Q|QhsdQ_K z%MD)ey?(=B2TPfcNl$oUZzHz9(5<6)CDK&f(}lB|G@q~R5z|c{5}g72TEf)vS&_Eg zJKNNQ_MQ$-x0(26StK)Vf|*5VQ_V&bJo}|)*E|W?uCe3`_L?{2^IuWn@9zou=3}q<^6Q$$0;jh5qS7j5;3Q#|dkYIH5$a%j>e`W*~>KiiGaId#K1L?aVmG6J8RV zls2=uD{2aXG!qI!)X=N)0j?VLMsy9@@ZuT4ZlIM#2XEtBy>}lEh6R*&;M=*o6@ye# zj3RQy7WCz z74+S$j=|w^>~-D2na-U5*%BN{8-81R?b>O*%P*^Kw_j%LG!)c8T_4n|a$Ga5rgBH~ zl5%#Re7@6oN5r4^*2JeDIY`Hh+P8TrO4`xcvq+6QsKBNAIajr_q-zOMPg|x9D<#E| zs>-*RBpW&?^FuzcPEdd{!~{Vlg7?!%&x84Icb>8+R)TT%VreX91?u|eq(lVm z`iC)A?iF+P)H;tR5F|x%uvt_$U;ZofIuxSzyfxX#{_cZdc?smCw#W$aO+OuqJ(G%E zqYIl2&}SwdU+fxo>UbA(+dYZ_BUApg#4ir*&v8ZN)|J8^Ol)+(#G6weIDcYdAnl8* ztIyj6Z?c(k3H|ioJ7HPyS^Vs(FL%dTDFv03iYHtREu}!E4sBYAL8^I$Xu+kN(P?Su zbf}NH&8XB!biJEBvhYqS*S(wI5l`CX1PQW0BR%p^njWi&+=5jsbXmfaeu9QJP6mS9 zs({LK@hrVrEGrt4QhrQ+Fm=o_-m1K4uR9z2{&1#Rc7HXu2D$Lo+esGzm+DRju)aRT z#+p^c);h*e$R}#Ypgw~xiD-2Vn(#NVm#+MRUoeyFA($pgM1H#8l%I7A1N+lOSMW9Wx~!Mt;52-cLw*WX zzdcG(_Enc`-0ga3*-Ut2DpI+DneW8J2@oR0%t{p(lcn6^Ax(iL7<7Q;#Du-FI=h)%4C41>|>b?6*j zBX_@L4TD)cq<7Iy0ix1eAYY%%^?AQQNUu40%l@f7rg$$e1><|#y4SkJuV(UQ2C_bQ zcfaiM8g?*kM7`0HjZYETS9#|p>c!InLcbDA0(hfkoLv5D$Lk*lWMb$5u5dVIsd1F{ zhlA_O8eMoHY#+8OX`95F8H4WIO)Z*+Z+o8a^UP#ice=ceG`w9k^Rv@tNFCM?<8&)16Y^q(Bhj?EZ5s8i5{3A2_}Aw+?rpEv zDa+yK;hf(#lzmULD`W>rwXno>mH-~b2NeoI&#_hJBz8U+)XZaKZpaq+eg>`t!V*%7t z%jchJCe-_U)(~g5@bo-04Q!;;1T^K0>t0^Xg+i`yp-q#CXtiW8=Nb9|D)_DAjzSc~%eonlE6!4Wc0dNNBk7e)bm? zH}Oz?k|Gy)1d$#(`&Q62c=_rGA&}^+OqH@Ur+>u>*Uq9WnebdjoX=?`L@=YW9Pc%N5zB9;u>2+jA~w5~@;7DcspuWS!AFb>GG&){C0 z#k2VF<`<&do*H3_xuZ)Zj8n>Pl;~d#RiBN|q@)r+^z}S6>chDdE&4}!m2KBPEg4Av z-2E!t%U{?-j(d8bfIrJ8J*Af-b+PH`x`eWK6gCmWOn&i4MC$vT@%BjTdsf-g>PMoT zHwzMm> zX}(x3YQ5%y4{Z%h=Gz3S~ArQrILcLjTMLw z`uR9`bp5(K9a@?;+04S$N|=p%pMw@W!x=3vX*^<9=hiMDQL0awj|kp$|5!ch=vr?| zXOEUq)YR?ctt*lLyviP621y#`m@}0`?rcN?u8Ii7w56w&L)xFg|3l!Ai)PG5yXsHD z|1Vv*QT$Q0+m7>fPfzQfk0&R*%Dqf!lgHAvxGOrVBGX-oDK%#P#!M+Wr!4OCjA7e~ zU)n5YOSN0w%}*NsicNE{2Mj_rCxH>F({$ONH_P}fSYoe}&K~bG48-@^wG?`vE(sJG z&Z5;?k@k8!XJijL9^tXPS^H9MhvEH~47BLtMngBX@I)PGqQ*xkZeS zpF~%l+uB={V?&chaZS{8e2{_QW2#;5cg7Ue6}4}9TLVIyzNkb?Kt9Ff+Dl(>d<4XJ z=4@LWGM3ty{i(%#36xj+MY$1|`gH=j>T97y5^=3wX<t@iaBPgo}6m&7-h}%-b=qzuppJJCCc$Q>yWAd zPA3&TZHbE#PE^6vL3c_V+Yk(iHCyhfJwR4KY!6-CAMO;Bwzu`LAuA#wvv)i2(?zNa z?=G|O$Gl+sK2R;^DQ}9#O2&y5XujO!rr)Jz=EB|k^`+oB&W>u&1u*SCx1m{raM{*8 z$nA9ayo2%ioB4uhQ&0^%@2wCwd$E04zDRW4OhUM!p3>tkhx2fgv{{EcO_Ox?hO*$B z6-n3Z)WC_@9|_T^4AFW0{kF~=N7X~i5H*0Dk~Z(I!rjPqpG9Tipr%f)01kS;c9pXl z_$o1B*(r#!?nX|VlC1RU^8d^m<;q_go|@+#SOG?`9n#j%hK-L8w&nY2_qv{CGh%ei zd^oK)`28q*r!^BQ_V$r;AQQMS3NFGZ9^|Uy~F_JXt)|&10BS8%MR;1+C<75v_lOYV5uN9zycz z3P?@w4$qth5eD*_KCK4VSMNyf8ax$t8(ak8h*qaTjbb`vfD2T~B%{(>hhYzoXuk5c zQD8T4$Zim-OpC?_s^$$UowA^AkoaBS`mX^ch7hAO`wC4~3+bS2eUtCje$Cqf7 z1Ww`X(b|9Ncnx|6;xnHfg<@#QQH((s^M7#guU!UTvh!G-9fhT z+5FLSrENUE$&IHkigV)Pw?8WS*wvq<<7bpf>lGn&+?9|SDT#8=n{secLO*-f=+sk? zCEtBNgs=Ni_DvCuO&*Z`!k60q?&;+7+g`;m>(^REviBpuduSRdcodW6nCV(d2UWH} zUxzleW+?`~f%btX4!n?cAxqAw70wqwTpVXut>b=}xw>Ay!(r_Y90iYvS`>mIBRoF= z8ex)v>!|zpF6R_?qaTJxvNWnxtnLwlcZ_!mRZ!pqIM~~;^u@~98pZNrq;0?Oo8p3$ zBq9@jITLfrq#CY@V&tQ^*$AS^6iBtM05}6lA3252$V?a;y*xrNoNCOJeONw$*n6DF zY`-LWJidzTFpz*Wn%qk%!44E*O>x{NZ(fUd`pD`Y-nu49sP&lePad-G-;vmJ0Cfg^ zNHf-Vz%nYjSvu6stbbm0my+4BR;;-}8#2bQ)p`p-w%P_wC6>3ZskCny!d+|64UbOn z?9~c3vz~dE6w*4l(g2;~Z4~5#M!)uD>ys0=2o#ym)yLGu!dW}6~`4Uy&GUXU|XP)(o z=-yJ!DPsLMT2wJ#MKLogO1s7PVIr{Ct4TbuL6$-628#jH-Y>X24c=W#6WKi*P%Y|w zrjGyc{jc(S*TK4id_d+YSBlBa`)rWJcv7b4b?2!_)dptu#^6p#KuHaRsT;eAS{f&BRt}&t|Qe}8r*qVFK#ao9-JXs=Z+|!)A z61}4=GF8aV_2NOm(%!C$mRs%m_Lt8%cGcw%g?jl59*$a$NCzcET;m&b%Sj^}y6HTd zGS(?Oaq``K?IOa{j(oHKZ0}0Yw2SA*fTf+VZQh`hXV^ByvaKuG#wp|<$WU)bsSIyqs?2Ev zZ`lN{vP+#L-BY&TL zO&{<@z4b_y5l63))k!Xv@=gL}bMbxJkx-?o*)Yz{k$leVL;G1kV)gtk6PplN&+`Ut z{`tF+g~I$IA``7tlxt6C-rt>I=bssw3B%FS(v;?PcYU?gi$uMJCJZU5pq-G&cuRlh z;1U#WroEXyKPg2XlQ)tU))`M^0qkb98y0Q>yBSA>Y6c%jvIyD5%Z2^?fZ##OXfg@S zUUm`kD_G)FQ??zuAUHchjV6Ia{EdeoD=Xg;P-A#7Szfyum#s2lFEe5nK<8#KduP%| z(cFi=4rvzQi}2)&RHwO%GCCtkq|%Z@DA$^_>DTZWJNQ*(pT55&S6+q&*IhKTZNzNnFb-J?Y#n5sY3OXRJUJ5RV_zG3GLIh%bCw?AY>xzaCccf zMqf4*Mbr5*O5$!brhk1|$!EPeB-FUj8QGKlthdKC8A%@pc)PmNIhSNn(ESR*JN>E4 zB@V)`O7_|A_TYLv#rM6 zt9Bq94{ay2h`_68r}mr_&X%~L7>0W{eFW+)Y1yDqoGWApINMoyUQX}cR2`_4*onbR z+l=J2a@Voi1T#Vad8=GH+Yc#Ij9Q9)?S0y;$r?k|fC{-Sa+w09##FkbNTwa3+t#~% z_vxfysNoPL^A@9Yz$fLE`B=dB)p$4@k^>gyESFo03x5nShri1BJhgc*X>vr*%79<_ za)Q@s%$&dq$SyJeZT_R4X7M$e^$#E9388#KH!kYq0`M=H*O+>}q>PTD3{Q3=t5F%8 zHhmE%!^j%^DEVgS2PNPXR;AIW#Zi~$XNXtZgg(-6$L;W2n(gU(T42i2!}r}ZGV$E{ zUr~C9Qxb(nL&m^{cJ)pmb@5%cR1E5VnuAPpPA-y{Wg--`Be_5p8ZNMZ%}!5jelsMf zBrD2c@`>QS4Mf%bP&Y}rTkp~na8>Z%) z<;hN-yFEsTJLU-A00>1dxP~T57;S9?PglxXa2qU|a1+S^3#-~zSuwqwlQ|^cW56CO z%T^DiJUh4?88dJ1k?_UG)HUoY@BU8Oox0cO3H2Y4JSXZp>L6a8Ieox2kE7c|4=Dvv zgV{8Ma!kTq&v}&j2p>&gB)4~@1x3C*qndOxb7;gyxvAD}?O=L~&C8N@o9;dRsFEZq zzYEU{kgKV3%lDG^BF>cq|ce-cE;H@)pjk73)k$> zxvwcYnP((P7R_jFCzTNn(%p$${5{yXF8TKtJ^78cs z6|JJXSgOr;MVmKlWxdV{y2=jrq!ODCPvSVNH<5cv*{Ep^6FrNOfsBs`Z|fb=NUsoS znP9K!b$3idSFwo9vuH1_aqSw}e%8`AEy5epUOOVZtbn$Lt7$Q^kI}K>=JVfN%n*Lu zwv+89_#<`qCw5Y_PcLm0zEA$(iJ+VBt=J07Pe*vJmW}bNVh+W--)w7)@4>adFMNm1 zx`|{;eY2-6rvv&&5KdNx+WU7c^t*x_81m6m zaTgyg@((k8i;b$b?6Vl9HuB7MKU(OTmqB<rhPy2XcEMUE z(rfv^fV8MoO=f4>lJNlW_-@alg9A1)$ zD=6B{hlF@M9=y}2O3P@`T!)A&$@8d-aVGxR`S@Ahe&ToEh?dKTg|s%0WTs|bzgYKo zJ>5qb7rm>CM&Zg0j&Co$0fh0+#l6)Gqc{gx<&G{KJ|$B(WxK0OZ?|iz(-Dk}XejP4 z8B5~!TJ7QcyJ!R+diERSUNXZhrB@gYmsm` z*kTA>lP`6$pb$bnbUEX1t#F9v;;}=y}#eH(2mglonh5l<)p&t%tm{wQ_)?^nA|7i2VHZl&>i6 zp}bjzzpuX6^}DCfjKD`IEX-GUGD^9kqZS4RxM|1%ugrved#n^(88Nq56GAL5JUWn3 zgnm*|vek3jag4p^3rs&|&AucFI}o9M+nfzWyaQ|TCN#5&d+f2|Z=2yufS%(ZimtvN zY(0P56-D-ZIl3e`bh8k3Hs8{cja4spJH*pQyW-406A!pK23#P+Z4vvmK~!$yQA&sj380&oV|z&N<9idGF%Z^BYY`v$xtdm@#v-*#?#P((!$IAz zWkgi^ib^HHAt5n%crJn|t8)95%J`%W$R+4aL`6pRNt&n@51LqMSkKd#`%V}+#*5ao zGg_7>Dd5|YpA3YfzQ+3q!CJG&y^XbTJuN5UaDQ@(x|3Pu`*W$fM)2)R?)5S5c^44w z02Llx6vr1w!74I}T%DvxmSPqj|G0VZGp`MyNFB8Nb|0VyxVM27SlcBK)=I?bj%mU6$+L3Og2o6UFvA9 z^A?&2TuLm6J$yCuU@@|iKv6EoKFY)Q=p5*}N;z4}2QfVpXREnY*Lgi{ItmCRbFDwv zkS2L8L3DLRp4Z2FlAWAawjSA5(W|@Z#h(C9Bsjgz*=z!Xs{`i1e z)nrRaNi&57BTcmWf;k)PW`V!|k$a8qVg}vWdptbp<>)7AZ2pTp+;fK`jPH7c^Pma> zedVE>oV9Gk46I`Uxvop{>5(x?MV~&iCART`^PS$I8+l)L%ugwm`LqnG=cp=YVKbG= zieb5WBlecs+KTs`THCGLvGg$|1DDAC)!t`8s{Lio(#;R&-Xl4LUjk%&;Qe@Rv>z(e zfDtnQ9AFv1El63ku&8@xJ~PMB+eK5I{KB^K`b6qMfTvy4Dj4+WKX*dcL`3e9^=178Sj6N$pbj)H<;ioZtN4==xWH zuBR>^PXBo~?f>EJy`!4Unt<^YMMVWx5KyX0ldkj*Dxh=_q!*>P&^rXgLNC&#gOt#F zlaA7n0HFj3y%PvXhXlg+u>0=5E4tt7`F-d7{_&9H@ws#7PM^6mK8N>|CQ;Cw6Kh3K zZ8@@Sbs&pFg#Og#t^^n0XVRnaE@T3ym4K$ii^_^mEzK2qm|1D=OstxlfF0yTY#|?y zjWJl>E3y#Fs@x5Y$~_-RW$~RZ`5Bsk&;aDL!^a?sSM#6p@2i+k1CKXpP*SiiDnl(V z3m+mvH#Dq2)Q_vaCzM;Cav|C|Qov#b!FMFPseY{@Efm`4veaI=OVNOh1%bHC?~xa6 zq&~?QtYXbf>!Bi?@E(~%pw-~|6T?~G73TjajZeSnTg7QmsDw|QTl}Z1>!Zu6RG!=V z&8cyXUx1!b3h+!?_B5j~2n2&!95j$+K)~d+tVVGy;HD}Bt?!P97B-eUZj>KjIV@L& zTu0RrV%gx6p0t*)cDxYZ%4wu)zG+W*4K~cCFhXiyvhQ^uIu_LMu z(rj*NN*YJFv3Y%4Rmt(rcXHZhGhdFUk++|};X;_N_*fmAib{3=+&Mc0LP_EW(R_{CcKS^0%xlQ_R0q zrZhr8{#&q#fh~v^eRK?Jpt9&w+DXS!7CoVxih~N+;~=IR21INAUkoGXlBRF?OO?70 z%{!5Rz1eVsB|YC95ldme)dg$!y2s`XL>O|$)yGXc*WaN&mGBnlfTlir2#c_}?KDCf zYK56*J9Fyb;f83rr4J9z?}!_39_uBl39kmLYL%RfjH;1B=-V4u*iM2JAU_HQSoRfa>heXVFH z*BmfeIkSGa6ZJDG@j@c?fQef0d!Bf~h^K1P!}3fgiWxI*`Slitz*R}>Frxd^tdsk~ zr%x_TzpCQa>Xl;;!2n>Kuo9^-8=|Y!x@u`i<5>G4FN{Nx1&tt-UJ-}ewB@s|>&6Fd zR|k9**EF6R3$_nlLYC+Qs@_#=H&84K+0&@-g-u$%;QBjRBP!!My zU=-u;9?Z9^<#Z2pxZXX`M>0iO)SX`r=haK#O~THXADK^?$v*33?AVChN;>g`Ie>)r zzW`3o38%Rj)z3xfcl)VlR$3nlNZ|XaM@tR8%bbEIroUoxM|A(R75XZx-%gO#9cMkUbUWVaztavBrO#xFDk z5-&P!r4=m%O8(~|d|1MH(}=awI+>}>04;g54&{C!)&nW9cH%Luig{{`)vV()J-=j- z8FgYVlvO&_4+4ewB70&bB3kAE4KFt7)i5?4)3%Pb+zw>@LxvD)Aa)>&$hU|Jkc)(9 zOC=uWYiv>l^2OMeSdl+nI$0*NjiRkSADKll5JQu6a~}nLw{JOqKEYWP@?ykMBNvD< zjJb}zfn^nm)+c8_%NvS!sR9vBxl4|fuFIPrKOst5vA{mqaJy~%y@MN^21fC zhRq{Kj1bW8fg$gj{z-TF=0MofO!`_#SHqL`KxV$IQ+0zxTOz|RBGesMc0K@}TFTZg zx4sQvO00ysu$f!OJCc}S|LO&>k2p`}DU7et(8*nM>IKA-w@gWw{B9)+>{~%> zJ6MPjsV$WJWH;-osnLknM=kQBo|WBGyYDp2!dxVL8xXun&y1yk7&tGHW}(dCB>u_D z5x?gwUCe{I*sQ%t3WxmfjpUzt8^DD+Aal?x&yD*}CU#5^2YB85V61HQK{YW$Y~(}V z!XvL^Il}?HHnBs;&|BW+>|-hvxkaJR%0Kpa#v6mW<##2bM{BZxa1QG6bH6dfPx@f~ z06>ju>UichC)+=lK79yiA$yNaYsk5<4jJ|<#wp3Ewzh}#ME(`)`47i_;PL?`6_2$Q zS~q+ukU#$&%l-h&DenU@A^;n!NjyI!t08zhQwss0rCiXb%wK&Ky2i<1@l2bv8qL)5 z*;h$|^*~LX3-WLg(@8q9PtUZ=rcfbNs1SXgCLX=98C)q_IJuQj$&3Fz>3%1O{_~TM zH3`rrsK8~o?mOWHWQ4kcU`TbW9RzB#?V_K25sLYyAhL8MqRGT2ut=3t=eTdjQBt$% z%k8)uH@9B(W3SF1R_g=dELbPJKRk=NG7cp@aK{>9jrbGR4ZAA5NR_Sfi|m4IfG>xW zl-B(}D(3$1%5(v*Y_Ny&Y=6a_V(}>-!Z_PK1oZeR5S*!!8)^SO#{4rq3-tcL+i^m? z{raCW&A*;;tQU8>4HB3O$tkz$P2wE_f)pBJ>Y!DW`D!l~^wrI;P}oqe9>Fq%C_0t1 zoY5{~tZL6j(?utcXtAsrv83433>kU#wBdxp3$I@|8eV`df9R&!bVN3?w_PHAS7RWSx0^xY=@eD_{ZcJBfcA<6ZJk~IDwab8Xt zT^hxazAQv{C(dYiJ$EB9d0pDcX_t3kU|?dq(H8c+%6g(^H!nGPB~B3hvyl|!kD5UK z%A8OCzmFaH&8LP02=`la;Gci|lMw^rzxEyZW59cBt5ygm`e6jZ@4xtp-?O#P5vd~g zVE6Bz-AyisZQlMl0ncI~swrt~F8WgHlMA_braR?_mVHHk>HVqBIGbZsfwbTE(kFiM z0?kzFx~oaNkAFD9ZarDsxIJqWkFgOhwnQB+tgi+&uI&}( zkVPYsoqyKWKZFHnpgE@}B=y~3x?T3^?nlS9$L71Da&4yr%Lb!g(FQ@hRceqHa@Q_? zhdwjTa~wn4)f5{Cd0C0&>RuMmy--@lnA#K$d!0`p^E2bl{usT`H-ckrNSM-3``sXh zU6%XD+U{>E=yR0*Kvx%W-~0_w$Et~&R{8f+hoo3-;hG6`YX{h{C*u!Ch0wA#`y0nj zqdn$l=dGL$D(LIO`I-vriw$f3=X&DxS2H42C2G`CI<0=&^(tWKI#4?h+W5HSo{_ZC zUD}f#FR_aWuwPF!rMgEm?C-F`}}<#)^!zgCm~j((OWl#!zE!n zVw!rETHhbzpUCH~`3;Xb#0&S9*gpqIaK9C2P|m#5)^$5U`^EZa6;=WIq}0+D@3}@d zYu^3c#kI5KbC2#tS@u_y7RJynv53@ytH#|#@m3Oz$3HjtpBW7yjMKc6?{7W$=X$rl z!}~wTB3P}-M0r!v{Ru*G9@~)Eyv7vHse1Rvr{zznWMVkPGU@Aw$9RqFLq&|31^KKv4BAH|>lfi-5m>owmq z)*rhdS3DJywI;zgzvJ^C5cbEfKDk$dz5W0ViJQjt6Re#(1+6Kl?fox-R@s#O2WS@h z1n8egdU6E-G~ZgIYrg~<@@lZ>AE04}09kzYgv3)fF94v`S+@EA3TQWpeXjR>{Jh3w zXwSBM<44SxaRZ8h92NNBmj-~fJx$%xRQ(`g9L$b8@e>BaX9O7Ri!SHIzh)u--`JsP z1W%Pa=t>zRc;X$^_-dQ&FKtxTXX_8jEvX~eIWtE6R1$E!LHuB;tu1xPAXW(_6|3C8WpNQ#u zaVkAVa1E>f`R$M5>(5^YAEl14GB2;(;|YA<{7eN5){EDzKk4}GM`WBKDjz(CVzgCUtf(=^Hh(iTWuSn@$MhIn z#{pTVl#>EKFrsq3k!;CZRJ|A)8VJ{8$&uYbY^XEv5f za6v7FNq4r}d?k(F>Hcls`)Rn9_Q~(R9Mg2C)Ny1|@rE1QC)Q4bXFD(q+}9ZOX>ib5 zIYn_-A>Ok3d`v%J{{A}7dUe>VbN?#E|NBD*F07MDfk6P5_ySU5`pr5n>~8%^KP3sK zuq9D7DLDA$?Nl6ed`_9dg#VAj+kzVbx}5XIzxk8-nrBGhKApJzmEO04H$JS_F>g3~ zv-5g%y@c8zZQouq|9tg;Q!!F?QQq~^Z)XYcrm4>XKTXP)pQzch+s+n8FIv!Cxqwsw zH0tQ^r<73@TDGz9`Q60cA}3PbA9^E1$v5-ygYy0DSh?(+d#(gfGsJ>c4$_kqAhhTa2y*K|kTv zzgPM%0sQQ*|DWMdWt~>7yCn28NKU`wxemw!hOd+t{%y-=xalV!&GZ0G5x+lXZ}Q7$ z@dGsC@)u@#1g>2A!LsmqaGgIj5A&svHa2GB6UPHPUoA|7r`h`F^#=R>p*`>Fg1l|w zg-Qt3uLy4dzNVJzFD^z0I24?9J$x8$RO>=9>Ha6d$y%10GacXjp7V!}Y~Xf{^^%!O zcTEe(KkZ=pnE?D@GXQO~itf@+xTz!uLYMhQ_tSj5lcUt|pLg<^P`Q+?NlrJQP%C{udR8*@} z!rBX3;>rUKL|9rqj?FjK#Mp9%WF;=bdAe5aEBsVOAk~0Vt*Pg}cGgYs={gl)pP=fB zBzDe)R=vx^E2ApUat6|?o}HBx;>TC`sVHQ-*V~_VC%4O5Er!xxLKc7(&>P1u1&wUM zoqOCJWVWVj_mbKtV#{8B+Z21+%kyMu@VQ}YoUnG5=Xz&i;l-50gq@?!#{y3IKO??D zwWb2p3_tk~ED;>{s^{Zf?8n^Q-ZiG@=^3=&@6u=Nay6H8L?|KlVV38nfVtStcxi-+ zmCTLu{GV$`)1E4}K)7(~Y~WYQ$>bmK{)#1n<5`NmezzD}_iVoRz7!+w?@7QOvS zd=#LpD0#`p-!BKuE7u2<2~gNdfqEJzVqE@uE$sWlRkHMHfzm`n6{l#~guIjzH4G&D zT!c)_4Y~zjhjD(bCM&5w+dr`4;pvJtlCnRpn5tElecwaoSl?3WjB|aTZo2X*2H{N? z8G2B_&lZ{Yktm@uL=Jl()@nYL&3xRC_mjQ-hC}@Dw&-<*2#2!<)r-4$fH~%nRFup- zyCxj5ZL${?o?Sxt{X3dm@f_?7OBTEHgJMB_rHtpmjcnmMPI0tz~>N zC-q|QhwByISGt)75!-Z|Q%pH>thny?W-jG^#=9>dRQ!ziT%&iv&g2m)(J(W)md!MY z!=7E=FkY|2@Aeg3);xs;$?I`O06EK>dk`~P~u)3mT6?5!KM9lSWxD zFpu=O+U+u(riizZIe$8(FFsyf4l!*^tS!bgpRXNmu=%KMi~V!WjzS87!ns5)WBYba zS=G^s)#%RbQyw5b4Gjdi4hfv`mwfQYgf5cwgj}kO*U(kTNh2Lay(6K)lK?V(!Y?oL5sXgQC5q8h+oC2ed0}>| zLNbW={?zoTITt8RW9W0`VkK)c5h^S9B~qCBMy5nb3?Xyn1AqOYT^(&e%S@UZTt zM)s~8j&h;|@T-<3N`?~6+NV~F&{^oXa-wU(L8sTIhhFp`3t`vvfOm8=BlaE^O?K5O zQe~>N-EP(qN?l0dx=gfLtDXTj)TX2>@BphBBy2DOVv?&JNo<31h;Tpjx>dc=9^tvy zHUst}*ay$boUD_lfqGT*xFt)o1W4cU8(XH?W2eH z%$2{4n0N#Yl$0#@e4X8QSuIg}YQG|8G+n|%w|j&#c1amIz9|AP@I`p!B}X*3v(xDH zvcH|UNOQRR?XZP+LR-rjlcASa6^Qusjvu5lQ0rEo*Q#AFQPi(*^sT?!@i52Ag+#24 zcT|>0i1N-RU!BT$j>FEl>B^_tS;)oRPu~JX$*3cm>ERvIbB&5c>Yl$jpAu%9vbcU1 z&#HAc|LhuV$;cFs@&c)JSCzFk_d?7XFZG|S^Wi;CeD=qsV=8%70ds5k*tVMN6d?_2 z%yr^z`E)|18S6z!A1AbYZL6g2 z4!*EmG=t%Ith&B=>vZH;FVd9z<;%(py}Bn6y%WrxB&3-eWlr2LKOXmO$OJJMN{}#4 zGDD--xJwMKN?YwyZ&rX#MOXu(muN+wxI(rDnp6|wShqwbfQan2iHc(2cdR?9f|br) zvacKsq;F|4v5F0iNeYu=Y{_p#J`Hc)U8!9Es%sR%Fsnf%UtOJ)uzG02&bg#ia)B@* zI_o5PBgo|W+k_Y$7rsfAc5EZjlw(m{rGh1-MT;lkNqb*SueZ#}d9m2ZMsI1lp;$Pt z#*ADtTjZpNbg@jNFn1hXR4UOzjQcvTe6S`9Y6A`2INO;?6fI@3#yHv`_QMYrF;5Xb zSGk*)7-1fBop(DYts*1B)`O`s!B~O##4T(`r2=cX0)Od}rUAkwK?wSA8H|QN5yi_XX%JOcUZV zQEZH|OAlOvgus|=hR06m>$@|vbPSytv_+wE3V2l&PO`5*9Q8L^F3PZqBs^JyYy<#b zZ>*NfMeKcnMh+h<;@L2NqE0y`T{)U(3-4r77OHC4j8t znoMCho1>To{O)~l(#-r~9Xed?0G#JY8ip-(sTvgCSqg}w7{!a6=Y-gs(dA%8nA(qLMzfgf)RTs9Ci$=`OG9YWG|LzK4WI$9q5`@ zoMI`F-!j$7$w+ww5}I_dbHAAI2yu0Yagj2stdmmPY}*$6$aFt$CoPY8ugCfKDNp^l zqa1C?*=XIAHLUQdAJ0STA1$0bM>*3*ObA6G8?2@ZD}>M?+=*^0@c;F@OcN7ueo{ZaRwno9}`nx?@h$6v2a$4 z=GK06def`J_$Ex3Nw1eB{#8jdoinACxq>KckT8Unxtq+|m zEl-8E51N>NEAWV{7AJkst!YP8=TIYv{|QxJ#a=Au2f^b#E(ixlU!ujI$HMSd++B(! zyzD4+IFevZ%OiPb=VZFlfe)GdJ7cf(-B;o&SAi`5U~FvU*>LxZr+{>bTlL&kJrY!HZ$;goH; z=Ea&Bd>w+F6wx3L_2XQy1vZB@J7Nb{H{tdt&V7-_fVigd(RG=qSi2D2lx5Z!M`!m7pgCb$oAnv_wN?!y*EkPbBPUy@ zqGBdOhvOb*ErJvZytIb(u>`FY9S9RH&-{~PjPX{8}E;oDW-x)Mv}KiCxjw*h;tcNrMBgZRD#prk$XJ@e1CF2Nj;@WAH4%-C zC_EhF!}SSZ>0=mp;yD%PWm)nS%28fXNd-x{d6F&~?y!!E-x_Nfn*aKzmDLw^Yfyp& zyzk`8kn1@_^!ypNiEtem|A~|WVAIt83sOf?Y5fXOn`Z;2_xo`_mt|*|DoSUODmaos zX}%YvsU;T@{courC2xL$@HFfm3D-3n^WBhFxHb7QRolJjB;L^rxj4hDZL1|jpC(Oz z{sir%03PgWIu>JlpBP~-Qmt??`QT0Yx|`1AI+)K>gj0YXaeo(9KU?5_*Oc@+MZ$on z?9t>_ePr(hP{~H`!Mb@db3aQZb2~D~FVE^u!%#C(LVaUKaw9FotDNPHqOJkxFT% zF@#WdO3GuI$LpSUtD*R`Vh%@FS*dKQl(QqyT=tlmJhxkTq=uM6Puj*|$2u-8-l|^V zLindQ3^9|lGQy??bn1~WwCMUr@2Utgc4nbdSl6gUD;47B%?-Y~qu7`lOuGFecc?rD zgYujdY80vj6a*hg!8*CLQmoT=heao(5T;1qy?%6>XgLRn<1rXLYl2aLYg6#m_YBQt zK|o`DDb;avW_J}9Z}L`t7&e825|jA`_mzkgS#u&!GQwVV3MMmb`O`(%$)lu;Oe0() z&d0P2#3ar`BHn; zc@(4J&y$5w77sAU~ySgweKF$fC=4kq31aR+{j&(53C-)rZxC z)gV@avNm7GLHB$Zlj_nrLSinYx0i8TmY=nQg^e1K=TRthNLTesBw0NRW7sg--{uMs z5%iA=rJG=gY%&ZGUlC2(AxmPIx(43NBz_C)4$Uh=8bw%hw%?TL>vtFNa%LShtPW*3 zm+4Cs?3v`&9y+H@6h+JOH^XjI6eK+~p zad->hxa6jmN3J?jN@ABD>ew-LcO`WHx7WGEO2p z4nDI|1>Y5XckHzv%r`vx_eqkiWTuRMqbdx86-!2cO1b|y5I!c?zj^49#MNm+SNK5# z_QRGX20&iT^I&~#=T0Pxb+oF)v&SWyJ5x@2daq@yy59t$A|@9ZF)bznOwW&;IOchz z3-lt?$)3-wvU1EAR6%03pA`wyb()gJB6QFjCz@WbyLM3Bla1rPIniOULxcJ!_FlCb z&~ow>6c8W@q;MxdyZ5?p9SUvlb3BW5Uw00-eI1y5@**6}u$5I++Zd&8d^M?tu2A#* z15K7Py=u1IhQ?Jc)orH4YA??UQM9wZoDMOiFA|1Fpx_5}HV`1QlMnPa7g9`tQwipE zD0vi6jI~RPY0<=Yyw9OPK8a+s`be_=iD?cmdoQN_kS*I5YNkM=_?!p$G)c?nJQyS(9ucr4O-cMN?UthJi$l|Cl58#gI7d!P(P2HDOZ zIAvATxr0Z$^9Ixoop`;(PL6h${OT&vUnKQ(-H)z=c0C??A^e>y8yl#XDu(VXz$MGe zj-58@D*M5URk9m2{WgP#wbvCYy;Zf3^`iN*I-FMswh%R;*qda`;aMbPG)&6JsgWGaj*c>fLiT6S6>1C@c{?zNaa$TA>kOyPTM;^=b>(Q(; zm5OBMNzAautm*YhBxJV|4a>QTeW9pFqg<9d&jY(6HcIha*IC!X*qB;*&fauQX>YU3 zd|C6gO574jTxuj#vqR+B!J(NpRg%|7`Y5Z2T6L_J)iBg8dOaA)xI9ml@sLII_c9FR zd}}x(B!xMJU~tg7B(GbfcjRe}{%3RU>%3zc-i8iQJoqodZgHOKJP?6yx!Krih= zLXg77^?X?l!YU|=1+p(rLDiu|Hu_|O4MI~uBii=uQT9fE5%K}=uyzT@$mf*ZiqWgJ zy~`fhfS`@z(aFrll@M$nas8$4_s6wT))&`XRLyyVQrak8Vt4XfZ^5~asnUX-R~Yt^ z?x>b*RgKfuX~(2garGcq zoo-}@E#LlVVdm$LfBktsW3EUPg10m&SptV>pE0=>#m4*U%F{^ zZQEa5j0+EacN}gYKaU0m-QGXDirz5?{9|VircOxYKH|nxr8-IhH3aT;eJh z>B{Q!WSg5ipxycb^&lO!Y>gcVbPrGJ*`}(XIY1bWjP*i1>#pdV0~wT4W!fM7BFta` z4+I?)+$$*nxs{J_mR&26ppswj+!2T}OV1%)aR;SGM)LEDL`*#4N%WjEHJO&~{b1NG zzyIl_6ji;lvzhRUQCgZ&iWO0B--Ck2G$R@?k38dz%=XAh4v~wxnKwvmjjq(S>a-k}p}VT!Y4)*;7|5>+ zGyYIK*(Z#HzO-GNX=%Ql_`B)r@AF>w!!Oi1aV#cTTd8@KKGjQSR_70pb`w#HfB#_6 zhoe&8YEePpSO6p#BojWnU&K~g(UPSK5`8Pm9G5-PakNh6h~pmx6*bsTBqVV`4!f^b z7h-a(hO((qH;332JRD!k@PrZ@UW~T54{LY5vb=G3Bf9V?C$Y^zZr9#Qesa`$fsu%+ zAZc^26Hk6DpdEvDx=8)RblSW`(}i34$rpKKP%76=MdVkZ6({*w0Mp={!FE-_hwTxr z+~JQ+XK#&}baFGvca_x4#5p_0|JF|A1d|pXeM=$xv426XRB_!(NUI2ik{mHv?{9#$ zQ@IaHP}yZI+f9nbB*xQa;4wk>;j&wo2opt)cT0px!L)r}apLJ*g61&2NPJYyO=k2v z(%h0;{98yE=5y%hTQ#&%UDe2iQ>476^nVGzvF$(lurpwNN zv!Eay5007XYc3Ezm|dk_%V;Vc5X@{M&Al$3I)lsYW*XqQyXvOFtt0TFc@-fTpn>IE zut*Hx;8+ovTdHRVi@B9s&N&S&M~MFZp#!JNN&awLLev{dXA}G)M!!eD`CPvOg!UitJD&~xoNB`}@+rH%X?g&TtQSCkLZSQ1n~74k z)oyw2z+Mdb#}e040UbTO4PCv9FmCdynyeylFDbG71`gegmNDNdLXH+KUosX=TfwRh z%k|y52z9tC$tQUB;6c(}99jTdAkb6CBkLrx)7&D!*5=)sOfxfEpvm58YD1u|UMEV5 z$3sH+n(8zbd)X%gI>TAI;eI{IIClquDGQxT!X)OSTT6lVY7H}_)=<*FgOU2_8(=4f z2OF<1)jXB^GamWxag;NgJq`PkeuLNF&LKOY-|X4%6zFep-YC#|Ez@0?(y@X-C9-W6 z5cH3VT=ZK#ZW#K~J&K#)mN^T9Fn4tpk8Wvgkq^1O8z@nhq;-T?CBY}Pd3b4{ zDK4qXs;+rkrX$kzTg5v%4mO6H-5sRKoLDkPB zfGP;?HymqJnt)BUhKDbOjk!B;10m<9=ZI9uw@M$FUHJVbY4#Y!yBTWDANKL%L5jb) zhI=su?Eu)&&?kW&K%A$3YdSlW5@Dqu&@+CgC=7eayRa%Y%XhVYmxGYQlx#fb5RY^e z^o}|`g7c_>C;l3jl3A>8jthg0TE9m6Q5`&`FgtA1Y^}hZbECLKe!nX(9S2y}jNu#~ z&&^0z164={SC;Usz@(;&u=@mw{em(@LuY6%+-|~+Pz3#wyw+?j(Je76EM~|;C0!(S zlP~0CprhAus7tBDD0GfH>mqUkzCdw)wUrbUSegC^e|>mY`@B<9jc1*2-e#LP*ts7H z5vlzc7G|kKp;6TuW`3b@49zM&HAWo!l`)7J`wJl$xUN@20#i|VwUP|XL23r z9WxQ4fyE{g(AIS|Qu)OWt4;<20V1tXTCA_jJP>4K-%lQ@)3sEaDbURNBzu0gWi1(z zW>B`ypQP2j*y(ul6ByoTHDy1hRA^#YhXwc2#}%T4}Gi*0tc4#ZC72g3fM>9RsB2^u!Wuf)!aGFldiUCp=x9)Q`Xl^>(U}cw4PR4Oe;67WGU=Rs$piu>x#$k#}729ee(dz0hB!Xq3kN~=pIOnNNUzitAwd>IHfPz_u zwW3Ev9?0KHDC7Z21eyI-Wg4@Tf#B2PiPS}$&}xz?mn^>NNu(S0mP?}OZcRL)>l%R> z9x239u3-B2;k|1igqDmg=tIaHNNHVVriAsn3HD7ddR>mkU0{9NLUr+vs~-%Glfo{rbaj~ zus&$ruIEY=wSl5^RTSw3GOuyR1LXA>$YR5kVuPB0fxJGvnyP<yiol43!;N6VFpvC=e0=|H&{AzYG; z&RIxnZ11F~PCh4E@hLL;(iW1U;%-Tg_Nhz=B4}KY*Y6uu3GKEV@R);3)-~6m-Sd}g zQP(zW^gAfD+C8x9;~{}lmaC^}=1_DHzx8_kq?dQqj+m>RUW)2z4ax59X@3>RizGb~ zeHfeL6WPN_ud1Zs!#zYTFg+wSt_$kUNk+n-hA5vb)myS0;X9W=-95Hv_O{zZYttiS z_?s_z0fO5!rL$$~Zs=h`le6W1nLlr4u9byQ*>c~9%uO$Rt}C%CS}%`dp;q`Cnn}A_ z8mgulJAm7f|+FHoE~P^b>%X%vZsz#6j& zBbVMU#Q>>#80d+Gedm;sxXqSg{hrO{Fze7;zYL%jD01X$5?y7h4-B2v8x(2QJ0VaO zqhl8YsL$X+N$+VF`xPex(5yCfRGGH{G!5omupY ze#TM}3NUwVTUUo89+!A=DmN2R+RybWbaQgI3K(!?-6@qFtB(5=|?(H z23eL9In^$y)VR->7)gVwx4-VU4s7bJWM1s8GwcucTeVwzep4l3Z&L_gw)!%ukV|)$ zPR7gbQ3HZlbB(Qc@{uN;Y|W)rg&M=i&o7KaVrYiuG&D3Jo6sGYuQzM>JQ;-=i&iP? zK(G0QGB%9kL_@PS=T}8;(Sqx&n3XOFw--r%e*cz+*5L9D{%gSfx%5}`S^N#&;a-OP zC!vKoc}Tn9U_3y#WoL=EAl_OyE9yTdEnbbzFYXOrQ<=GA_T6>7vmxX^iui)xi0A4L zADbl0VZWu&Hw@izJ8w%V|J%r(?n(zj`vk{B>j?9kW-H`NjuDEej}ahY zqjuCWgI(<{=5hJGej@a|hM2_4(dhZ0eVGmUsjkAU(|W?elvj~3j|dztvBQW#3R{k$ z;J0NP>+xGg$N1H`wCKRSN)L%Jn!y^eCxfPWtmh19LcGn^QVPBbduh@%LP|tcr#kXz z`kVEiPhYrLKAELhW|R{4Zp?87?Mj#Z*0-4%a2oCl`lQ`bnZJ~LuoV7*Q&4tK3b@vB zdhMj42+l58*K7F|KB8k7?^}=OrZ*=LB4%;)lvbl_)byF{C0%SmTc8qE-ea6S1;L|) zUfBwEZ)=m5w#TWHg}nrUsot3#OrGEcTkf7K@EUV=dd8iF8j-(x0i1DC|Eov)zTq5O z8ci&0L=hx?`e^*5mcwy}zG6#S)4R2{b%W)2rV?ATWPBX$*A3KaKxLnmgjb`uTEJfr zvIMy&Kw@@^ZEmx9h2BrC;-W&`b0S1zm6>-;A`Da2D~nvf1Xbq2Lqy<|ERmvH>Pcy) z?R9qq0|)|oCpoCxD9)3t%x6!nTpA&CykwzyT1AMIZL;$#Q$e@xv+rH3cd!<7KyRVR z7e0h67W*fIq}~IO<5AtU$)X~Q@$iBUVQ62wqS**5OOKM(V)Ua&0v=OG9G3(3ug z)t;uE^A}3m4>g>NWL)=%^;elc#V#v=P8dVJ`K40lI`u@P+Q%o$3M@#2jKnU0g$GAC zwWO8s(uxBCeZw)0!H&K;wx66ZRn*ixNjH9b>JpCtmK!H;VLEC^RCI~a*m3J=YR}U> zyfvr+E{*~&$Xv_gT%jVWN8s-N&AnE-gp<58<&f4UNi?-S0Yfix$ap4H1lB-ng_77`9Tx zVUuEyki@@TSG1AYl6Twgv-U|ue&bpJ(k`M+y)~&^dhlgSgyTi*vGz)>xetR$9rORl^$Hn|wwWLwNS9F2KRV$ABTh`O1wd2U$CanWJ^Mm!qIbK-D@uP&Q9LnafC@dFRa#nk$#% zv6N+C$Zndl1(=Y7W;#pkVW~C^GgS@k>k6C&RH8^iGX_-)gZ$ME4H0S`sc~eAY!w z{hLO!sMMUvvBt;F_>D^3+I%yCidhrQliCDHqDZ=ig@>~W9&h+jWi93jSpoccJ8s_` z(VwOZyi8gG1W54x+x}2PL%;Li`xx0yiM z`(KZZT>R9Jt9?4)_|zuw=5UCXONlR7=c^b~zcfFuW}B(IpZA})KY7<`}|g-FC&{ati|)SiCN1+VMgoxly9qkKWTD@mQd zah>b5%I+r-rT>0sX6rL_eNItgFj{4ZrvDy^GTiL>451dg>5=jhDyEDU^xyvvv^WON zvxb^Ce%Sv`bbO^99Y%uEm-(+1QrB@4A44@fB;>|%1z~)+OYaA!?4^!jrR$3R=l+-F zFaG7S1bupWnC*_X#M-`vgl=(e#C%xjbSRHbA$LiO9N(1?n}WjqcgT-w6wS02LkkXUm@ek2#$EY`xn* zRLGyITD;=IM-!zuuGdi1`2aqA)Y1yNOG($Yf?h}22y77!GW=wocSWd5BFXkyx2*AA z|7B|9wNYxKTME>A%0AZ@$k zi#p~dSnT6Dd2;8T%E98Vm?duOzJts*`w%UX?nN;fL;J;7&R~viR)kxl`dm6WubPJn zZ+6L}Ii?U}u&-GE(O1~bzG8T!DK|FD&dEfVhM9R!r>dc< zyM+?G!?Q+4vA068Z+4{eC7EpRC^r^nrca@4J>&4LydUU%JH~^3+g8i7X6a#wU(29q zIRf7=;$GV=q?S1lUIWJed~I4=HEQs!R~X&@JW38C53?9KV6%q1su|KPC{37`Pg%Zc zuh<;ZC`&OgoYH287sCS;Eh*Dq7FETp9e!wYA}6yPv2++oo769I>Il)DJR0?;&$6oj z0*S)j5Q#)3PDe4hK^7UebF=@;3IFz2F5`=R+qd0`wvxA2ooT|3j|DW=yw24^ZfmbK zgxUtgH`9+Z)LQTBgghy6$jh+0)qjxbUn8~a&rkBVEc96w;-$dx+~? zMN_^%%>OK1e~yvd)T}P+I=fnlt)cLx*jx3Q-ptDFdonz|0v0!uv_6S{ zf%1KL`ny$ac;VOn;u6ZDa9Fj@oFzGjWfZeYd2<|jySd|Aa_uSS$6I&Ec|(c0*$1*E z-GZoFptRboFPlO^b0rk%s0TW0J#qA$BO!xTW|V0qGG78kPQ3D7BEGcldE~W`@nF+7 z1wu@@yG|+LX5Ok{z^%qBNz?-CL^H8j#F&ZuoxPyfMF zgI4XQ;&iWklTeQIsgQs%<`T_;G-=kO>m2LUU61629moH{mMsw%0I(U>;8HVR0m7 zs2*ZFs>(vptvcp*qB?gY#6G#U5}{GQII=y_mdMt-`gwzA%?aLMbE(_8#=crA))yQy zwjU_vT0i`f*EtSeQQi~KE2v;9Te9S>+1RuLHc>KP%Y!a|S48o*GWc6`W_$Q#Vxpl|?=; zH4FY_HuMeRY-WkpmI!>39zz|Hr}P)oWOOqXjCELZV&D5t%nMHP-CF&fnYn_0i4f`U zm0hAKNlP4HjsdK0eC(b{uQeyTDsMl&_fRoKvTXEQ-f9iQ*k|~8XK$YOsV!H;ylR^UyX^URqzn~%q;xSkZ?yB#lIv0B zt`Zg?EHr7>A*eOcsK+3`D>8#3cNZpU$b^F@{ETS4+$D=ko3>O7sxZjU3%R!M3gNJH zEz;|dha`U=;5ZKL;knXJk}@e1VdVk!X*UBVNNc7F$Y(PAL<{oDgN;&T!oopfq)Uk? z1T|Yp%jA126Bb%l2gk<9pfw?ts);OR1$&udAjizYqciz5%qdceo?)-9bUsJSGAc6h zV-Kt~)R?cEj0NC7v+`AyuelK*i20lYPnWy`-d=rK;dt6hXXRF}J1+xdqUj(PFePw- z1Y^jgOg$JB+|uqP?d0APx_32yjrHkF4{vvuKKCT)&t|Frj=VNGRS+hZF<1&jzt zR}c{Cy@Q3KbOfo9-h1zmVHBhVq}NcTcj+}Ky@VnqKnzh@=tv1IK;AeG%sAicT+jFY z{T_bh;^buSv-Z8#y4PA~?*;KN->WrOPN`BPRUar47U>A;zjnVG>PYkb?fm~!N&Z}o zfDopk>^z?r?W`b$6;X!e?7~%tg%mm4a;O49AKUx5{;iZ9rF z#IjO%g}vxlm8enYlXttk^QZl8qI=Qy%8Qb|}cNlq} z)5L9HGGHQa0+C1XN($y*GM>~SNkwflL4?$^rYYVeC2EWk14h6@m%jA&A2RrT5;RU> zx;7~?CFrxgs|L2=U(UQpN*p+- za3jGtwI?j8G=lar#Rsw-L3AThEF+8uR6a_p@#MwgUSG$yWR^yYk{U293CHq&-HF!O{H)OO|eEg2z=-v|B|~agcaI2YPVQ9sV~xll~fSo?XU=K6hH_KH3plr~FhN@2n3nP(gw zzlpwaDg9YmtNYX{lreca$Blkin{L89{Wk{(D*v1c)i(P_BrUAM(r~GqO&cOM1wNy` z3seRbfe-5C5^z-7gPc?Tn)fFyEF-s0X!~3SB0e`alv}tnZBLX$x(KZ1t{HkMYq8r8 z*Qw4DgAE_`-hEMPV?@aQ{@v@+38jNCTP7v@D&Y66K^ATYSBO&7-YPJ9B3<2pq0I&2 zGHoO@eifVs*_&KoVsO6kpq39Qg>Rc{eW^i~Q%aDRSs2s()!a?v8isA{F!Y0LgY@e< zlSLGkeetR`KFahZ6?at~3`lmHs#JRF9?r_)>pGLhgLXaxQnBCiFIP(tHqlJY+FM%X z6Dz^sjt6KaLYVy(q6HHF~Yr%Y_x*F9IRqXt-?PV~&XZFz3kU&QrY5R;o}ShFmo$f(TV%jUpvgNKcFieW*A zjq+DdwLZ|OJeJO+EJkyc!dP%T#WqLUDhO>FqFy|`;Ps-+cA0rf`6EisU)hJqUxjM( z;ZBOXb-P$4Ju;N4l>|j}Z**3AH&tSCNr2PVB zp0hO4(ldOV1Fl{C#d=~h?c=??31Rn<{W{sNlB6qL>rBEg+Tvf%Q@$WXnOy&*&lYK| zcX2QPzhU6J>B=a$B)j#}QvKykk@^u=Jqt^0bA$hzPQ!f3Mr#K;a?7SdBcaA$`}SPt zt-+sRP4Cw*<-jl(Nro9|=vy}uxlw|mv>~#}>J-dNGi67nO54iE?T;`CONZ#kZj;Xq zC{~G7pS4Zjim*7SraOXOx%Qc-1Q#beN&AB&(oWAsze( z8<-q5U8RJK7@Z05Ho)wg7 zF>Ne-@mF0MHSOY_+*Kw=iRed;XwI9A^}%K0RKnIzgQ1`KiHw1Zl9{vh1|}mzb<-Iu z?ozFx{j)ZQb&Z|+RDmaw)_bgGQqpP^F1;#lup4T|BGr#Uh?6(t68W2jo#s^Tsl%U0 zZpC?KrIRM6ZFZcbIdlxzp?02m;hNM_n%^2}l33a1Y?6ZusMI>v(aYeeHw8s;UE10= zMK}b{Vn-MwcQXBx6uHbMJ`v%pNug?c?kq_l2Bo_x1715%EjT_OX){sd&4K4*UYRkq zwg-KW-2yco!Y8;zG%OsVSHa5V`wh>OXL@9&2JA1^rW|)=YyaNQ%0%oNXO9EsgCXfm zR5#f`VH1`PZ`#c8E&#uvA2j2NJ!JT6sE@*!@=QJIH7QO?H+O9VB?sUWdb0*e_$E)o zwlB>FKg$Q0Dfm_x04KP9O&j*qHK~woj`QB1$`1-T-pUPFjpv+In{~Y(=*c zLK=CT&>~7(!lfG0q*V~*mQtC(<&S789dl`_OOR)3*5DmZ+k$)bM2u@#m$M{o{fdpY z$tzWoI9$wTt>4OjW&jfLtNfP2;eYd>F``<=MCk(9t2ijML%ZkWPfv*Y4XNi;*EvS@ z~C=8 zO^^&-13ipGg#wi$iZp2YlH z7(%DGX6uD7LMQQ;jiKfDYLB$3TO>w3EnW;V=TD?;I*Q1$9rCy2R!BDSR8D6{zLyR@ z59@69Zh{IHo^woa--7dsiM~J_^Nj8`H|UcP-uGA(UMB%_SdW$J>e#R*y3=6?39Nmnbt-jE;qR zKDv8+H^tVbv`TsGiWMA}+sEwiN!MU)716{TauD5M^nOn8MGe|=*m*GtNpaZ1r()2p zen_i93OssSh9%)x>9=W#k$0%nnV^x1o6hZHGcW)hJw_p_hHN&z@{ov_tPvQuROW3E zW9&?}{opsOJ|5TCq)o=JBP#9uoyVpyS_vb#rqVdQCm$e8FE`&(#Kb0SsdMnCbZxLm zC@nr^a`k$l$2+;4^Q*T&0gJ$!RGxaF>N*>0>}2uV!LK3Zcz-d=`t^>){kWCL$|TW0E0~SSwyuv%sLXm$39>z5GAj|3RGuXNFTNY(ks zP0iutVHs-jWAuw|@$ne}7T507=U9FIj~}lM_C%%m^c?eJ--8cM?kde( z21Rn#I}lM>N(P&k#U}Y?qQq;RXbAhC6ck%UFDB-xDkJ<>KJmks0vp|N=b^XRB`89 z2_pz1VaYTJbhs=uy}!03*rqwm3mR@9b?e7jJ>?T^aovhsf3wp;cz1KZU`Md1c8(4i zzcsfa1oGKxlM#c1x5?cGNA5sD_YN%XiLgN0W0a?&-mcg@dqWF%5xRSMse!ErI$dsf zxD|cu(fqDpwq|3ou_ZzwCQ@8!Cgh-?j(N+nH>MQ3Y?zSl&#nF~xo^vf&TO zE7QQ)${B`VTD5pjZ`ouL9yxLuX)=6SD}?8Np~5ELj(Rm2=J5M=jp<_A*c#TxS9R7D z?8hH;ayjoRVE$HB*I!X zqx26{I$qH@vw$$Rp(lj*yH?C5sy-+fNi~&zyaj7<@2=lf$!EV>lAt?Ajrf@K1@SA@ z`Yr9!+754F3m+F_Aqt(<3?uzr%awUa524|jFI&L6tnP;DYE$CImzAoNYuiPt&_Rl+ z>-n`pu3z5q1Jw+X)8ga`ss$Y`8RYQ1)}=_jY6(!=e2i{|S>3x27)R)CUr-;zVfn=X zMn0ny`r&To;t4L>>@@Z9(zISrkex}((@8p$DcDb2^@rxMs+o$dNfgt9t zO}VIg6Ytc-eSaYIrHy|$XCA3(Y8_uw>eIxhEKg&-p0$`9KL>iliKYeva>8c2LUG*y zWFXA{Ml+L!nCp6G@5iDR3a%m$YSW@Fq7w5}#I)GCd*_3xg8MLFqu=&Lg&gD#T4@ZU14lPRv_4RvWxD;rG z^>S>_=3{A6Pj)XkqmNI;cp%`eXpfJ_OM+a@sZSijY<`cl*Qu|wknRsKy^B6IUZY$o+K= z8U7s16oGc*_)c*m>@Oo=fz`b|F3Sm2QQElY1CzJ&Bn{btNRT;{6a_TN4Px?`kWk*U zd4;9#L0N2!_!om3Rg4_K&#Bz?+4$4O?j3ybn(FLjqJoYKYnV_FQJ488Rvi0H&*&2= z2vS-V1j<#j%X&xrb&@Q9>VtZr19)4A-$Q~5t-uTQj}Qow?(9i8aed%1>HWS(EAz!u zL_NE|-qEWW4I~Dq*~-vLM0ANDwl^86Q(a1M5#2YWdo~hxwIgHUf)i^?|Z^v&YqHo!1ovD-6V+KU<{`-wjK* zE#NoeYc!~)XA>bQYkzOEnXGPmSs~Ppy`lt9wV9Nic2XP;cSn7y5mW5mzqzb7%P$_8B zRVPM^Mu<|OK55=HI-GP{pkOz1UMNAJF^J`ap5|DlcvfdiQIt}a)UGMhY)vJGct<$? ztxZ?W-PC?TCwO3xuHhJHdJjUtZ*(UypA_DZc`?oPynwZ3d0wwB>6w(+}GRrhjk%Dt^V_(vq zm59E@eay+fab3lLcdba*szeO%v7n*G`3Uy36Qw1p=k%-JcHL{pwMy2l?2RoI8gj)i z8uN_LyGgW4$$4GJ>LI0gd3sB)tm|KmwAQ{UeopYLlmxq*A>^gjdwmgXX(Du0OZP*1 zx&5BwBoTkLXPfGhwvFFF{M;rHn1hYhGGsTtnVxgE;9^`X*_3CDynFsL#5!O%d+$U1 z^LMrP$*qCkxxgd)oaI`MF~e5~G`>paRpKMvCTUZ`XY5mF-Jm%EMv~*p9w%z)8cyNk zP`T_WrXwnyd>lUXyT@NDznq};Q5(@hW9lJ-5Nf#5#5BcJ@H>B~mYF6ebp$uxkeW#0~{|X)F)3^`X!~ zmqylX;Kqu*N>_7%IkbA=PwS1|$8K15k()dlFQpdFt3)0o6%mSGU==ctjIW#R{MBT@ z@lhnHg3ftm(M-7!s!y@Z`<{_;0=EYnr(KYwug$wF=4h10ovI>TA4RrH=+(TU(jwJ) z_f%-S)1$A!T=aR5Lg~Jc4sN`b`L*_=Hw$t5k3&S0Z<`)t;}iK~+4-wrn|<8Eo&vi+*zdkKV z6EWbAB>z}_s7TBH^gfYu$1Udbm%|%>W6oQ=MIM0_zV$7g!5whPeErB@2e;Msv35C$ zu(-A0%mZIe#BTzgmmqnU4xq6`X0oXX*%KgMCFH$gb8Su*+LLvankm#x+pcwuwb0;x zrodhQVwMVDo(ql3w=snen1|l0y%vus?_KM}nz!Ih4%L^#@Xwn4S%Vkz7RRF;!W;C{ z-1i=X6$B&;HgSkX3T~ak!v~*{E?g83IX5!0TIbC)q*d>9l96@fQ=>>8z!S+_#|9m; zJCSww&tI4+{hTZfTgcQIcbmQHn_&-32WEW*XtYy=T)BUq!CIB>cOYHu9*(}@5pt_v zaj)MMRhov{#E!TrLkdOFS`v>VN1=l}fXL8$81Mo|@1~lV@XRDY%b~(zMXP3auv7l> zHD{7#_~Z=ro=I03Lyfz$yq_;y#S30aG}MMb_nc8$h;o~cb6Nqo^nhM@+Gt(Rna*bl ztdg?S-^5C{FJ*e5MQxiCId9NKDSw_Ky~W1Aly=er0cKhA#UGAka+m4GD`&Eb@Tl07 z(vx+$@r`;MXRWPnWo~0z|5SA?&Ks8e1hup?lOEL?gX?RDh#AYItt;NexSB6$E*`jf zz#b5~*6!Nlc59I9PCWb75OF!?;eMt?4MfTYKAMm7T)1K1U?b?!fD;~i)reg2flRY3 zM1vN1jf|@y!d)+lR_6}ZH+!~XSG@tUej#3TY5F%dC3ehF1Y#`+FkSbsKvF{~!sA$_iisDdS&{wqV~ zd?Wo%@xZQ+czGpqeQyJZ>EJ8Q`k3Wok$>7~MeXvn!{2uAl}fCNfTV~g%!6NU8Ljy~ zVe4~_fXyn9#5x4;DmbIB=Okq~zo1E(O>O6m5JA_@Lrb_1@a5Dv06n;_pd4yFRXvs8 zEC>yN?p*hnetpC|^-%sKoPFmPP?|~oM%ly83W3A$Sqbh4EMGdZ;lQ1LDuqpp-<{;e zvHq7}-^l0CvMY^RRZA95HG3FFm7_PoEOIooPTJCy;-XZ<;C*kquKJ4OFkQbKL8;1T zEY|P6c0+lH@$Cfdc3|yU-mJ%~qkFYZJ~U95sl|7?ou3e%9}g=<<BO!wVGDQrnGEGBgeeau#{7woeA{&2X^Liq&u>i9@|i_ zk4zGPG}CSC7Ui>>o1eBBG3>oG6`S!c^|e!ii*3s#c@h}i%{6-JnPVq>w56AgQ=!H@ zEsQW3An1DKVWlX#F*eJ1Qa`L;C`ju6PDN7uIhRzS)**uAUKI=;OA=L<p_>c%ddZUAjME(LHj)0~f5g-YA@< z;T)J1&c_fD+KqpEn{k;bMthm?K1dg3<@5ABmc&WpWWf5wo~=ZXx1E*c-3lI-I)_d@y0()M zOt5Rf_wXPAkTj8!9w14(84_$u{dNvo8yPuKFn$m{F|A$1Rh0zYi?DLtcOD%qtDZ+! zQ^=YAev(;fWRo1}Py)iJ3safzXB@BX0$^FnB}G zmu&_leeF=^EfF~nZQxH<>AH4kYi5_*nKmH7SXi_~4FS1@aZ{4s6#Z-7Wa>q(+^J5BZe5QY0IGb=H!FceMaw zqXV0-CZ@$^dUX+xQbclr9d{L&3+67oGzj+8k&ags^ipmQ3ih?!%mWnk34C)2(_p(r9tnwpfJqGxNOF~6}N2P7{ks&9rb0F7A zpY_@>@Q^Oz_^nDGu8`?#@R6AxMD9tpRAFL(odv~T9pe4<&!HhJ{tU$>9Lx;{#GAS)vS|3-)ABZm9vF=}HmdfC2<*&Xuw zCPO|=6ZKPdAiO_s=)X{Y_$)`Zq->nji3Ao^UmW*&znj5)rGQIQG3x$^=6N8z~U zU`5$7Nv<(>3L%fs@eu~jU~P6Uk0BRwE`GyV=TbFMLDHu!H`L^nt%zA8r9>Gx=sta# z%ugsCTYR^=EL>W#`cU-TPnv%v3I@oz8Th$+z#6bZ_+3Qj{JIlP}5}aZ#5nUEB&`gu$755 zpDdlcj@cud+M4Fr;eLK%n@Xo3@wt7j%L(MKU8g=6Uh^zkW6aIYc5cvq z$xjobv%Het9*{y!fcDg%uw;9_J+V;8S^u%sU|cl)1)@~Y1+;|Do=Pu62BYEBhlqn; z^4@KocL-HquV}U6b6SD`Y^eMOLJm2=HC6@R6roDHq|gf}-iYQvL&|5Dgiub176`0a zok4BB^G2&;8RNs8i`ig&>!jAO?C6m9NsjMgoh@l%dpfb~^PdSNnPVKrDLUb7RnL`@ zZbCpKl5efQ{=Kd34{|NO5uh!RgGdo6;78ckit&vswNzB8P@jIMbqC)-obrec#NXgw zI_!teN7r@^)KYa_Ffs7Ix~9&MJWn({X8Z9SY zZEe{UPsi`$uwzxKU7=U?;@0MB=cW!)U`8F#WnmVVpavy~#>aZ-JSPYk>{G8aDn-5d zRn|w7Bx2vUefu5n?_2o#OygCwQkX3llPr>PN+npBF-u31h6p}N<>CfjXhV>IYZQT# zwc}+iNhEh$7-#lYz!iRW(kK0_Y+PRvPQ77_#9XPO^xiNKRJ@>KLrV>0mu;6^IJ*zw zQ_?T8=cItGyU*6Ml%^kJXj2%~m~K4vfti*}(-skTydu1QOZGNSvTsldJl9wl8CL$&5EvvRU?kLO{|n*6Dgwy8sh_s~aU ziQ8^~B=%=pbrLI59NJMN!na>@$ zNYkZ+xMO8$sMT;f;mg2-jm^O$NN%|7XMd-uCb&U(_nz@|q_2S_Geb;J@_f5xUBjfX z8Ac5^ki=U$CIY!?;c+O_f8-YFxm|W*i`CyC+639RurDKa8#GmQARCt}nYrIQ@J~Bb zmG)mKOE_ZpwwCs{?_DeOWfPpHkLsaFnoH^7z(liIEm)MRFM~}!4Ge}*>hK0&dR+88 zaE!Q?gLDrqg6Rec?-ucy{wctL&R0@Edc^)p@GQgfJV4FMsNH7;e;Zv~$F022&;H)U z?X-*Cw>E_`=f{>=k(aufa-pHUVdSf3EN3b~hOpai==I(V>W!!S+9Gg7A^Sw8A( zSuku?>TQ3-7sE95OjZP~xF90QK>93sK56((g=+mB_k|0E2abNvcO$^BWr5EGb_}3^ zGSIEfv|u;D^B$=<|62OzDR7QUvIlherTh_C7iHk;YXD|X!R?sQgfUx#H{?FOs`$Gq z7fL&?=~)KJTNN*6#rGl(U9iIw)iMHun?3+7`(babdD!OG=U@|RYz=FbK*Ny@#-swX z3o4ns75Cso+53?cc|)f``+x`gsFD0G7;nB}Rg*epSXRG|sCte?Rl=LA8{Ls15c>_y z`*`6b%6%Ri_IETwRc5cv2JRNJ#?TW-?gORyLJs1F<9nKfsTzdC_k0l;DM*t*h1W?W*Ht{GS3Bt zvPri2KG@d*B;DdsoTP!MfV*G!INVxjC0rEi6#+3Uc<4$WUQ<80BT7#dg-V=e2MyUS z!lka4;RKCUar!$FtTPILqGr4%s7^*4o_qpO^rl1UpHt9$#t;mnp8^tyhCEnbY#oD4 z+qv=kTw+KR7KF^uZAUrL8z{x!4m7OL40c|&{$Aanhll; zy|wEx|64+0o&7h%vabm)Jf6qOm>PEogr&|TRNx?h*V3sp6%%b8IqeBRCZo)E zu_nuCHd+HzIwCuDJH+YawT$l_uBr$1lDmF_Ga&#uSs);)hY=c)pTM{r*STA-1_iaJ zFmCy7$2oLG`0nGB05A9ZJ<{f@NSu*MP62xHo>2=}>xvaHUdT#DqkCwqQ|3zsW$?b2 z+wcdQlTR%Y3^5s;5th0#BdumUaIUVTEs2BP;0bT+CvxSHLEcgSPoiwGl&kYu;417TX zNJ96w3!OgbzfRib_H#+buZB9BW`591-@>MeWo28B7ExIl<~N{AOKkoWf6Aw!9l!Xy z;l%VXWy>lKjpMEfFXha(*{gPTcD+z70@fr`Z04h`Jn|`iz~9Wl4;)hcE_bcByvOF4 zdLiqcmYPULp%2q(=$q&tu)d?>#LnmY1UR3Gx5TSdPA1Z&!Ed5SYEe`?c4itb(`*@A zk##SRR_UThR2CBbha!uxcALDePe)QWUgCegIIJ6` z1(%&0geNfY7Rc!61SlpeAiR6VtO^OlHfRwl!QCj3m@JUeNk{sG4qq6QrjdMZHVSpg zz0zm;ExTcOHJ~j!{^@&6boyS95~r62jf?;B-3my0_Ex27b8P8e#NyXN@4QPj^wih^ zn^lm)r~%5dsIaTElb}c*hauZM)B@KOCZ~&RZObMySZFe=wCEhawkXO)ky9F+1p=JBvQJhQyt!LFVApg7H08ZCfYz7~}c zt+k5r#$oX}2!pErBC*s{g~(t!Z==UM@RZ(MvOlIGxi^0ym88dF?t)1+Cp7fx+~Zjo zL6@8#(nX(DM}*JlkYOkT5<>2scaqR-Qg*{w*z&^R3bg~7pF5i#Sk~s zd6%L**Ix}Ja+2LaWN{;$smP_RLKxfW4Xry@`V}`u`@iL^{^p)si8*fp)hzXjS+xZF z$mmkc;O397uGAl*xtX9E89M7@wyG?rF#IQUbER)8F!T8(#0ht%qbrvOTAIZ9f(rZ#z!>yjE)@KQ(KNNK7NCa8Aeqvf) zp|UId+YEm&@~t;7_m`x7r1i(dS}z|z*C5Vc)<5$Hxmx&cv^&#!d}nj`V?XK3;-$xj^|FtS{LYVG6?6ry%S%sJG9oQeXGgm(&HvAp(hJ_#WgOe0jl|V|+(_bcA zyuRLAZu8H5GJ>)WU+K(8P2r;i7GxH`x!q45*Kq90x2M%PTxKN${_l%Q_(V0S#m8Fq z+b@)hUX;8!0$gsZZo4AEMclo@`p55PfY0ZUX$Z0#F(5j5_UvgA2vgw#Q}SbOIh|fp z?#X$6u`nMJuPwzz=LY^ ziO)D8E+v!1R{2SGs(in@xT%$!B)ary3=;JRDA-j#TLiWV-JBw~>Af4V0P^pdi(R;H z()=MrzNqeI1598|6cqueAxX)2r)9*Rb1_a(_hxQx$d8ac14OZMK>e6{LmwXVA6Lq< zoF6>jJBx6*{m$Re$lXXw27CjfVoNIr8+Uu0aMzVPlu*kz|jjAa>tk5oSL#!OAKQEyXu_H126e2mkP(! zHQkr-4wTJD)8Y((iMIl(N%?ME_ztoW)~{oU;{o7;<8n0vdCVQBpoRulw*4QD8(mdzIb&Dx!u*PfLfk_CIW16^PSh zrDUGGI|xaCo<{AYsY%W%)^VamcIaC?G%g$Vf&XXTCSZ)^o3pW)*HkIE_>C#Ke_2&d z(6alRD#>M>{5G)D->djtSaX*@Q9Y*Yz!f;d<`jpYeE+BP)wx2G^bk*ZlUNq% znoK@2JBUgrUri~XB~~YM&1kqrr9JWrc3CNjpU;w4!42p?ZW*tbPg#?ClOUB=P2RI0 zoh?S4_i2tkZxZH%_hV3F33*_FCBvx^`G?6VdXk|l!35Q{ng0WOnkS6hIeYB_0A?Cy4!4kP>w$IVgT!V&KEiTkuT9DNoUHkIw?D#B zvS=xbWIKJjJ@)K(*u$4J6N@oja~O(49AO$c7WdW(sEMMjs2`fv4Evz*bKx@n;mPN< zz6+N|b*%Y}cqvAT+%Neb$tC@Qo}P=47bCDePFxz&%0sW{ZbQehY6c4DxfRPZ6lu4=5vt~!74z2FaC;B*?7uWL(u3sJHmPI%q6 zESp(9Q+yK&?L~xMm*g#qyFV@t5qmq!$|5J-Shgo961usz;!F8)tKacxYaGr$ERre{*1Es z9qm$&tpMra_m~PyXJlR6Vk{5ATLYF}+YZCqS+c+Ua=rYi^h1p*`^h(iw-4y;^~=q+ zfgD=qB2D<@2ZWj$N){Eu_A*v6A77ctxFjlN7~q%lLZnj;pC*t(*hhNmwBt9~PnNBb zixB)Q@mow4VDGSy22tMFdC2l<#XX${746pz#OZI0Y$h9w3aBPcenJR|%Bxa7QDFJg zK0Oe_6ZfiFNlK<4mYLn)bCFZ1V$;5=bmF?n|DA0GlP zUE&4IMgfuxoz&VFf&+W^jL|lX#KE`tMVJO_S!XB#gJA8|ln*VGeka^Bi@}PodY-EV zv7*2ODiPgxd{#xI*FsJ(ci<5r%sxW_5UB+UFvU3AM`#x}v4|d}2Y1Pd z-(%?V_Gvdet`pYhn=~!RQ6g7>WY7Kq>7V$*Kc`isiKjq?%^2p>Y|(d&-+5CM=j(ax zM$&3D&$6>0@bh+GT{pyO6@6F{KEm#4smD&9`U-NU>;LjW?@GU|&hs<>cBVrA(Fg8S zJSF_nw8Q+P1^JiZ{(i~wGQQ32Z~14o^oN!Bcd=Z&e2oRaU00Z^`HU?72g3Yoa4I~R zpt}82R{HOQx8XS&<$!|2Pq@Rs%OW5MFS1{Q@1NrE--r5Z#^24K<9VCcE2TpJxOvDp zfZAl$g1-}FpIv&!nEq`ZhL-0D?>~+a`O!LFuG0J*bn5_QwYErf79#!QD}P@kL*q;4 z%%&NBOUg{a45SbKLy@t78GxF=G6T|HAgQLm!mE zy#@ezpIld-MYvu2k!jx2xP5Z+$$Y5g@T}~i2*vZ+C1{uVS>EQK{Gs@K0L)|8K1F8) zl44bhND2YlX}zyf`kp{MrE4`=4~~w#JeJ(;p^e6W)vE-q%z;u>|Aj~Da+X_-(4sqp z;Qr0?Biz0}aB|B`{KH_C%RKsUHeZH_l1>p~c+I%N+ev1gB5=FAwtnbcyuNSKPT-cIMFp TdHBO$@PAKbl%-1_8UFr15!>LM literal 0 HcmV?d00001 diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index e95409e08e9..5f04c9ba6ff 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -299,6 +299,7 @@ def __init__( extra_kwargs["delay_wgrad_compute"] = self.config.delay_wgrad_compute else: raise RuntimeError("Only TE with version >=2.3.0 supports delay_wgrad_compute now.") + if ( self.config.tp_comm_overlap and tp_comm_buffer_name @@ -2116,3 +2117,12 @@ def set_save_original_input(module): "set_save_original_input is only needed on transformer-engine modules that save " "quantized tensors by default. It needs transformer-engine>=2.6.0dev0." ) + + +try: + # pylint: disable=unused-import + from transformer_engine.pytorch import cpu_offload + from transformer_engine.pytorch.float8_tensor import Float8Tensor +except ImportError: + Float8Tensor = None + cpu_offload = None diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py index 6a411ccdcf6..a741efc2042 100644 --- a/megatron/core/models/common/model_chunk_schedule_plan.py +++ b/megatron/core/models/common/model_chunk_schedule_plan.py @@ -8,6 +8,9 @@ from megatron.core.enums import Fp8Recipe from megatron.core.fp8_utils import get_fp8_context +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_set_last_layer, +) from megatron.core.pipeline_parallel.utils import ( AbstractSchedulePlan, NoopScheduleNode, @@ -446,6 +449,8 @@ def run( f_layer = f_schedule_plan.get_layer(i) b_layer = b_schedule_plan.get_layer(b_num_layers - 1 - i) torch.cuda.nvtx.range_push(f"layer_{i}f-layer_{b_num_layers - 1 - i}b") + if f_layer.layer.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer(i == f_num_layers - 1) f_input, b_grad = TransformerLayerSchedulePlan.run( f_layer, b_layer, @@ -468,6 +473,8 @@ def run( for i in range(overlapped_layers, f_num_layers): f_layer = f_schedule_plan.get_layer(i) torch.cuda.nvtx.range_push(f"layer_{i}f") + if f_layer.layer.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer(i == f_num_layers - 1) f_input, _ = TransformerLayerSchedulePlan.run(f_layer, None, f_input=f_input) torch.cuda.nvtx.range_pop() diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index b125ee11255..d2e4c92df3c 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -8,6 +8,11 @@ import torch from megatron.core import tensor_parallel +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.pipeline_parallel.utils import ScheduleNode, make_viewless from megatron.core.transformer.module import float16_to_fp32 from megatron.core.transformer.moe.moe_layer import MoELayer @@ -347,13 +352,17 @@ def submodule_post_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor) Run forward pass for computations between attention and dispatch: pre mlp layernorm->router->dispatch preprocess """ + if layer.offload_mlp_norm: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="mlp_norm") if layer.recompute_pre_mlp_layernorm: layer.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint( - layer.pre_mlp_layernorm, hidden_states - ) + with get_fine_grained_offloading_context(layer.offload_mlp_norm): + pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint( + layer.pre_mlp_layernorm, hidden_states + ) else: - pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states) + with get_fine_grained_offloading_context(layer.offload_mlp_norm): + pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states) local_tokens, probs, _ = layer.mlp.router_and_preprocess(pre_mlp_layernorm_output) @@ -434,6 +443,10 @@ def submodule_combine_forward( hidden_states = layer.mlp_bda(layer.training, layer.config.bias_dropout_fusion)( mlp_output_with_bias, residual, layer.hidden_dropout ) + if layer.offload_mlp_norm: + (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states, name="mlp_norm", forced_released_tensors=[residual] + ) output = make_viewless_tensor( inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True ) diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index 654827dc6fb..c77b9782c77 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -18,6 +18,9 @@ ) from megatron.core.models.common.language_module.language_module import LanguageModule from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_init_chunk_handler, +) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.quantization.utils import get_quant_config_or_none from megatron.core.tensor_parallel import gather_from_sequence_parallel_region @@ -117,6 +120,7 @@ def __init__( self.parallel_output = parallel_output self.share_embeddings_and_output_weights = share_embeddings_and_output_weights self.vp_stage = vp_stage + self.disable_param_offloading = True if hasattr(self.config, 'position_embedding_type'): self.position_embedding_type = self.config.position_embedding_type @@ -410,6 +414,22 @@ def _preprocess( return preproc_output + def preprocess_for_fine_grained_offloading(self): + """Preprocess for fine-grained activation offloading.""" + fine_grained_offloading_init_chunk_handler( + self.vp_stage, self.config.min_offloaded_tensor_size + ) + if self.disable_param_offloading: + for param in self.decoder.parameters(): + param.offloading_activation = False + if self.mtp_process: + for param in self.mtp.parameters(): + param.offloading_activation = False + if self.post_process: + for param in self.output_layer.parameters(): + param.offloading_activation = False + self.disable_param_offloading = False + def forward( self, input_ids: Tensor, @@ -435,6 +455,8 @@ def forward( runtime_gather_output (bool): Gather output at runtime. Default None means `parallel_output` arg in the constructor will be used. """ + if self.config.fine_grained_activation_offloading: + self.preprocess_for_fine_grained_offloading() inference_context = deprecate_inference_params(inference_context, inference_params) @@ -701,6 +723,9 @@ def build_schedule_plan( TransformerModelChunkSchedulePlan: The model chunk schedule plan. """ + if self.config.fine_grained_activation_offloading: + self.preprocess_for_fine_grained_offloading() + from ..common.model_chunk_schedule_plan import TransformerModelChunkSchedulePlan return TransformerModelChunkSchedulePlan( diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py new file mode 100644 index 00000000000..5b291f03300 --- /dev/null +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -0,0 +1,603 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +import warnings +from collections import deque +from contextlib import nullcontext +from typing import Any + +import torch + +# CPU offload implementation for pipeline parallelism +DEBUG = False +DEBUG_RANK = 0 + + +def debug_rank(message): + """Print debug message for a specific rank when DEBUG is enabled.""" + # pylint: disable=bad-builtin + if not DEBUG: + return + assert torch.distributed.is_initialized() + if torch.distributed.get_rank() == DEBUG_RANK: + print(message) + + +def set_ideal_affinity_for_current_gpu(): + """Set CPU affinity for the current GPU to optimize host-device transfers.""" + import uuid + + try: + import cuda.bindings.driver as cuda_driver + import cuda.bindings.runtime as cuda_runtime + except ImportError: + import cuda.cuda as cuda_driver + import cuda.cudart as cuda_runtime + try: + import pynvml + except ImportError: + warnings.warn("pynvml is not installed, skipping GPU affinity setting") + return + + # Get current CUDA device ID + err, device_id = cuda_runtime.cudaGetDevice() + assert err == cuda_runtime.cudaError_t.cudaSuccess + # Get device UUID + err, device_uuid = cuda_driver.cuDeviceGetUuid(device_id) + assert err == cuda_driver.CUresult.CUDA_SUCCESS + # Set CPU affinity based on GPU's NUMA node + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByUUID("GPU-" + str(uuid.UUID(bytes=device_uuid.bytes))) + pynvml.nvmlDeviceSetCpuAffinity(handle) + + +class PipelineOffloadManager: + """ + Singleton manager for coordinating activation offloading across pipeline stages. + Manages chunk handlers, synchronizes GPU-CPU transfers, + and handles virtual pipeline parallelism. + """ + + OFFLOAD_MGR = None + + @classmethod + def get_instance(cls): + """Get the singleton instance of PipelineOffloadManager.""" + if cls.OFFLOAD_MGR is None: + cls.OFFLOAD_MGR = PipelineOffloadManager() + return cls.OFFLOAD_MGR + + def __init__(self): + """Initialize the manager with queues and dedicated CUDA streams.""" + from megatron.core import parallel_state + + # Queue to store chunk handlers for backward pass + self._queue = deque() + if parallel_state.get_virtual_pipeline_model_parallel_world_size() is None: + self._vpp = 1 + else: + self._vpp = parallel_state.get_virtual_pipeline_model_parallel_world_size() + + # Cache chunk handlers for each virtual pipeline stage + self._stages = [[] for _ in range(self._vpp)] + # allocate streams and events for synchronization + self._d2h_stream = torch.cuda.Stream() + self._h2d_stream = torch.cuda.Stream() + self.reset() + + @property + def d2h_stream(self): + """Get the device-to-host (GPU to CPU) transfer stream.""" + return self._d2h_stream + + @property + def h2d_stream(self): + """Get the host-to-device (CPU to GPU) transfer stream.""" + return self._h2d_stream + + def reset(self): + """Reset manager state for a new training iteration.""" + set_ideal_affinity_for_current_gpu() + self._inside_context = False + self._cur_forward_chunk = None + self._cur_backward_chunk = None + # Track the first microbatch of the last virtual pipeline stage + self._is_first_last_vpp_chunk = True + + def flush(self): + """Flush all staged chunks to the backward queue in reverse order.""" + # Ensure all virtual pipeline stages have the same number of chunks + if len(self._stages[0]) == len(self._stages[-1]): + lens = [len(e) for e in self._stages] + assert min(lens) == max(lens), "All stages must have same chunk count" + # Clear the last stage and push all chunks in reverse order for backward + self._stages[-1] = [] + for chunks in reversed(self._stages): + for chunk in chunks: + self.push(chunk) + # Clear all stages after flushing + for i in range(self._vpp): + self._stages[i] = [] + + def push(self, handler): + """Add a chunk handler to the backward queue.""" + debug_rank(f"pushing handler {handler}") + self._queue.append(handler) + + def pop(self): + """Remove and set the next non-empty chunk as the current backward chunk.""" + assert self.size(), "Cannot pop from empty queue" + while self._queue: + self._cur_backward_chunk = self._queue.popleft() + if not self._cur_backward_chunk.is_empty_chunk(): + break + debug_rank(f"popping handler {self._cur_backward_chunk}") + + def front(self): + """Get the first non-empty chunk handler without removing it from the queue.""" + if not self.size(): + return None + for chunk_handler in self._queue: + if not chunk_handler.is_empty_chunk(): + return chunk_handler + return None + + def size(self): + """Return the number of chunk handlers in the queue.""" + return len(self._queue) + + def init_model_chunk_offload_handler(self, vp_stage, min_offloaded_tensor_size=1024 * 1024): + """ + Initialize a chunk offload handler for a model chunk (microbatch). + + Args: + vp_stage: Virtual pipeline stage index (None means stage 0) + min_offloaded_tensor_size: Minimum tensor size (in elements) to offload + """ + if vp_stage is None: + cur_vpp_rank = 0 + else: + cur_vpp_rank = vp_stage + + is_first_last_vpp_chunk = self._is_first_last_vpp_chunk + # Flush staged chunks when reaching the last virtual pipeline stage + if cur_vpp_rank == self._vpp - 1: + self.flush() + # Determine if this is the first microbatch of the last virtual pipeline stage + is_first_last_vpp_chunk = is_first_last_vpp_chunk and (cur_vpp_rank == self._vpp - 1) + + cur_chunk = ChunkOffloadHandler(is_first_last_vpp_chunk, min_offloaded_tensor_size) + self._stages[cur_vpp_rank].append(cur_chunk) + # For the last stage, push immediately and flush + if cur_vpp_rank == self._vpp - 1: + self._is_first_last_vpp_chunk = False + self.push(cur_chunk) + self.flush() + self._cur_forward_chunk = cur_chunk + cur_chunk.vpp_rank = cur_vpp_rank + + def set_last_layer(self, is_last_layer): + """Mark whether the current forward chunk is processing the last layer.""" + self._cur_forward_chunk.is_last_layer = is_last_layer + + def cur_forward_chunk(self): + """Get the current forward pass chunk handler.""" + return self._cur_forward_chunk + + def cur_backward_chunk(self): + """Get the current backward pass chunk handler.""" + return self._cur_backward_chunk + + def __enter__(self): + """Enter context manager to enable activation offloading hooks.""" + debug_rank("----__enter__") + from megatron.core.extensions.transformer_engine import cpu_offload + + if cpu_offload is not None: + cpu_offload.CPUOffloadEnabled = True + self.inside_context = True + + torch._C._autograd._push_saved_tensors_default_hooks( + self.on_save_for_backward, self.on_get_saved_tensor + ) + + def __exit__(self, *args: Any): + """Exit context manager and restore original tensor saving behavior.""" + debug_rank("----__exit__") + from megatron.core.extensions.transformer_engine import cpu_offload + + if cpu_offload is not None: + cpu_offload.CPUOffloadEnabled = False + self.inside_context = False + torch._C._autograd._pop_saved_tensors_default_hooks() + + def on_save_for_backward(self, tensor: torch.Tensor) -> Any: + """ + Hook called when autograd saves a tensor for backward pass. + Returns a tag to identify the tensor later. + """ + debug_rank(f"------on_save_for_backward {tensor.shape}") + assert self.inside_context, "Must be inside offload context" + return self.cur_forward_chunk().tensor_push(tensor) + + def on_get_saved_tensor(self, saved_state: Any) -> torch.Tensor: + """ + Hook called when autograd retrieves a saved tensor during backward pass. + Returns the actual tensor (potentially reloading from CPU). + """ + debug_rank(f"----on_get_saved_tensor {saved_state}") + return self.cur_backward_chunk().tensor_pop(saved_state) + + +class ChunkOffloadHandler: + """ + Handles activation offloading and reloading for a single pipeline chunk (microbatch). + Manages tensor groups, coordinates asynchronous GPU-CPU transfers, and handles synchronization. + """ + + @staticmethod + def offload(src_tensor, pin_memory=True): + """Offload.""" + debug_rank("--------offload") + from megatron.core.extensions.transformer_engine import Float8Tensor + + fp8_offload = isinstance(src_tensor, Float8Tensor) if Float8Tensor is not None else False + + if not src_tensor.is_contiguous(): + src_tensor = src_tensor.contiguous() + + cpu_backup = torch.empty( + src_tensor.size(), + dtype=torch.uint8 if fp8_offload else src_tensor.dtype, + layout=src_tensor.layout, + device="cpu", + pin_memory=pin_memory, + ) + + if fp8_offload: + cpu_backup = Float8Tensor.make_like(src_tensor, data=cpu_backup) + + cpu_backup.copy_(src_tensor, non_blocking=pin_memory) + state = (src_tensor.device, cpu_backup) + return state + + @staticmethod + def reload(state, non_blocking=None): + """Reload.""" + debug_rank("------reload") + dev, cpu_backup = state + if non_blocking is None: + non_blocking = cpu_backup.is_pinned() + return cpu_backup.to(dev, non_blocking=non_blocking) + + def __init__(self, is_first_last_vpp_chunk, min_offloaded_tensor_size): + # Data Structure to maintain reference to activation tensors + self._tensor_tag_to_state = {} + # Mark the first microbatch of the last virtual pipeline stage + self._is_first_last_vpp_chunk = is_first_last_vpp_chunk + + # Group management for batching offload/reload operations + self._offloaded_group_index = 0 + self._groups_to_offload = [] + self._groups_to_reload = [] + self._tensor_count_current_group = 0 + + # Counter for special torch tensor types (FakeTensor, FunctionalTensor) + self.torch_tensor_count = 0 + self.d2h_stream = PipelineOffloadManager.get_instance().d2h_stream + self.h2d_stream = PipelineOffloadManager.get_instance().h2d_stream + self._offload_events = {} + self._reload_events = {} + self.min_offloaded_tensor_size = min_offloaded_tensor_size + self.is_last_layer = False + + def is_empty_chunk(self): + """Check if this chunk has no tensors to manage.""" + return len(self._tensor_tag_to_state) == 0 + + def is_first_last_layer(self): + """ + Check if this is the last layer of the first microbatch of the last vp stage. + These tensors should not be offloaded to avoid unnecessary overhead. + """ + debug_rank( + f"------is_first_last_layer {self._is_first_last_vpp_chunk} {self.is_last_layer}" + ) + return self._is_first_last_vpp_chunk and self.is_last_layer + + def tensor_push(self, tensor): + """Push tensor to the offload handler.""" + torch_stray_tensor = isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ) + + if not torch_stray_tensor: + # Assign unique tag based on group index and position within group + tensor_tag = (self._offloaded_group_index, self._tensor_count_current_group) + self._tensor_count_current_group += 1 + assert tensor_tag not in self._tensor_tag_to_state, "Duplicate tensor tag" + self._tensor_tag_to_state[tensor_tag] = tensor + else: + # Use negative group ID for special tensor types + tensor_tag = (-1, self.torch_tensor_count) + self.torch_tensor_count += 1 + self._tensor_tag_to_state[tensor_tag] = tensor + debug_rank(f"--------tensor_push {tensor_tag}") + return tensor_tag + + def tensor_pop(self, tensor_tag): + """Pop tensor from the offload handler.""" + debug_rank(f"--------tensor_pop {tensor_tag}") + assert tensor_tag in self._tensor_tag_to_state, f"Tag {tensor_tag} not found" + tensor = self._tensor_tag_to_state.pop(tensor_tag) + # If tensor is offloaded (stored as tuple), reload it + if isinstance(tensor, tuple): + tensor = self.reload(tensor) + debug_rank(f"--------tensor_pop {tensor.shape}") + return tensor + + def tensor_need_offloading_checker(self, tensor): + """Check if the tensor needs to be offloaded.""" + if tensor.numel() < self.min_offloaded_tensor_size: + return False + # Respect tensor's offload preference if specified + if hasattr(tensor, "offloading_activation") and not tensor.offloading_activation: + return False + return True + + def bulk_offload_group(self, group_to_offload): + """offload a group of tensors recorded in tensor_push().""" + debug_rank("------bulk_offload_group") + assert not self.is_first_last_layer(), "Should not offload first-last layer" + group_id_to_offload, name = group_to_offload + torch.cuda.nvtx.range_push("activation offloading " + name) + with torch.cuda.stream(self.d2h_stream): + for tensor_tag, state in self._tensor_tag_to_state.items(): + group_id, _ = tensor_tag + if group_id == group_id_to_offload: + debug_rank(f"------tensor_tag {tensor_tag}") + debug_rank(f"------group_to_offload {group_to_offload}") + assert not isinstance(state, tuple), "Tensor already offloaded" + tensor_on_device = state + if self.tensor_need_offloading_checker(tensor_on_device): + state = self.offload(tensor_on_device) + event = torch.cuda.Event() + event.record(self.d2h_stream) + self._offload_events[name] = event + tensor_on_device.record_stream(self.d2h_stream) + self._tensor_tag_to_state[tensor_tag] = state + torch.cuda.nvtx.range_pop() + + def get_offload_event(self, name): + """Get the CUDA event for a named offload operation.""" + return self._offload_events.get(name, None) + + def get_reload_event(self, name): + """Get the CUDA event for a named reload operation.""" + return self._reload_events.get(name, None) + + def bulk_reload_group(self, group_to_reload): + """Bulk reload group.""" + debug_rank("----bulk_reload_group") + found_reload_group = False + group_id_to_reload, name = group_to_reload + torch.cuda.nvtx.range_push("activation reloading " + name) + with torch.cuda.stream(self.h2d_stream): + for tensor_label, state in self._tensor_tag_to_state.items(): + group_id, _ = tensor_label + if group_id == group_id_to_reload: + debug_rank(f"----tensor_label {tensor_label}") + found_reload_group = True + event = self.get_offload_event(name) + # Only reload if tensor was offloaded (stored as tuple) + if isinstance(state, tuple): + # Wait for offload to complete before reloading + torch.cuda.current_stream().wait_event(event) + recovered_tensor = self.reload(state) + event.record(self.h2d_stream) + self._reload_events[name] = event + debug_rank(f"----recovered_tensor {recovered_tensor.shape}") + self._tensor_tag_to_state[tensor_label] = recovered_tensor + torch.cuda.nvtx.range_pop() + return found_reload_group + + def pre_reload_last_layer(self): + """Pre-reload the last layer of this chunk to hide reload latency.""" + debug_rank("pre_reload_last_layer") + assert not self._is_first_last_vpp_chunk, "Should not pre-reload first chunk" + debug_rank(f"len(self._groups_to_reload) {len(self._groups_to_reload)}") + if len(self._groups_to_reload) > 0: + # Reload the last group (last layer) early + if self.bulk_reload_group(self._groups_to_reload[-1]): + self._groups_to_reload.pop() + + def should_bulk_offload(self): + """Determine if the current group should be offloaded.""" + # Don't offload the first backward chunk's last layer + if self.is_first_last_layer(): + return False + + # Check if next backward chunk is this chunk (for last pipeline stage) + next_backward_chunk = PipelineOffloadManager.get_instance().front() + if next_backward_chunk is not None and next_backward_chunk is self: + # Don't offload last layer if it's about to be used immediately + if self.is_last_layer: + return False + + return True + + def bulk_offload(self, forced_released_tensors): + """Offload a group of tensors and optionally release their GPU memory.""" + debug_rank("----bulk_offload") + if self.should_bulk_offload(): + group_to_offload = self._groups_to_offload.pop() + self._groups_to_reload.append(group_to_offload) + self.bulk_offload_group(group_to_offload) + # Manually release tensors not auto-freed by torch GC + if len(forced_released_tensors) > 0: + cur_stream = torch.cuda.current_stream() + for release_tensor in forced_released_tensors: + if self.tensor_need_offloading_checker(release_tensor): + # Ensure tensor is not in use before freeing + release_tensor.record_stream(cur_stream) + release_tensor.untyped_storage().resize_(0) + + def on_group_commit_forward(self, forced_released_tensors): + """Called at the end of a layer group's forward pass to trigger offloading.""" + debug_rank("--on_group_commit_forward") + # Wait for compute to finish before starting offload + self.d2h_stream.wait_stream(torch.cuda.current_stream()) + self.bulk_offload(forced_released_tensors) + + def bulk_reload(self): + """Reload the next group of tensors from CPU to GPU.""" + debug_rank("--bulk_reload") + if len(self._groups_to_reload) > 0: + # Reload the next layer group + if self.bulk_reload_group(self._groups_to_reload[-1]): + debug_rank(f"--bulk_reload_group {self._groups_to_reload}") + self._groups_to_reload.pop() + else: + # Pre-load the last layer of the next backward chunk to hide latency + next_backward_chunk = PipelineOffloadManager.get_instance().front() + if next_backward_chunk is not None: + next_backward_chunk.pre_reload_last_layer() + + def on_group_commit_backward(self, name): + """ + Called at the end of a layer group's backward pass. + Ensures correct chunk is active and synchronizes reloads. + """ + debug_rank("--on_group_commit_backward") + cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() + # Switch to this chunk if it's not already current + if cur_backward_chunk is not self: + PipelineOffloadManager.get_instance().pop() + cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() + assert cur_backward_chunk is self, "Chunk mismatch" + # Wait for reload to complete before using tensors + event = self.get_reload_event(name) + if event is not None: + torch.cuda.current_stream().wait_event(event) + self._offloaded_group_index = self._offloaded_group_index - 1 + + def on_group_start_forward(self, name): + """ + Called at the start of a layer group's forward pass. + Increments group index and prepares for offloading. + """ + debug_rank(f"--on_group_start_forward") + self._offloaded_group_index = self._offloaded_group_index + 1 + self._tensor_count_current_group = 0 + self._groups_to_offload.append((self._offloaded_group_index, name)) + + def on_group_start_backward(self): + """ + Called at the start of a layer group's backward pass. + Triggers reloading of tensors from CPU. + """ + debug_rank("--on_group_start_backward") + # Wait for compute to finish before starting reload + self.h2d_stream.wait_stream(torch.cuda.current_stream()) + self.bulk_reload() + + +class FineGrainedOffloadingGroupCommitFunction(torch.autograd.Function): + """ + Identity operation that marks the end of a layer group for offload synchronization. + Triggers offload during forward and synchronizes reload during backward. + """ + + @staticmethod + def forward(ctx, *args): + # pylint: disable=missing-function-docstring + debug_rank("FineGrainedOffloadingGroupCommitFunction forward") + + forced_released_tensors = args[-1] + name = args[-2] + cpu_offload_handler = args[-3] + tensor = args[:-3] + cpu_offload_handler.on_group_commit_forward(forced_released_tensors) + ctx.cpu_offload_handler = cpu_offload_handler + ctx.name = name + + # return the identical tensor + return tensor + + @staticmethod + def backward(ctx, *grad_output): + # pylint: disable=missing-function-docstring + debug_rank("FineGrainedOffloadingGroupCommitFunction backward") + + cpu_offload_handler = ctx.cpu_offload_handler + cpu_offload_handler.on_group_commit_backward(ctx.name) + return grad_output + (None, None, None) + + +def fine_grained_offloading_group_commit(*tensor, name, forced_released_tensors=[]): + """ + Specify the tensors to be released after offloading. + forced_released_tensors is a list of tensors to be released after offloading. + The tensors will be untyped_storage().resize_(0) after offloading. + Note: specify the tensors only when they are not automatically released by torch gc. + """ + cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk() + return FineGrainedOffloadingGroupCommitFunction.apply( + *tensor, cur_forward_chunk, name, forced_released_tensors + ) + + +class FineGrainedOffloadingGroupStartFunction(torch.autograd.Function): + """ + Identity operation that marks the start of a layer group for offload/reload. + Prepares for offload during forward and triggers reload during backward. + """ + + @staticmethod + def forward(ctx, tensor, cpu_offload_handler, name): + # pylint: disable=missing-function-docstring + ctx.cpu_offload_handler = cpu_offload_handler + debug_rank("FineGrainedOffloadingGroupStartFunction forward") + + cpu_offload_handler.on_group_start_forward(name) + # return the identical tensor + return tensor + + @staticmethod + def backward(ctx, grad_output): + # pylint: disable=missing-function-docstring + debug_rank("FineGrainedOffloadingGroupStartFunction backward") + cpu_offload_handler = ctx.cpu_offload_handler + cpu_offload_handler.on_group_start_backward() + return grad_output, None, None + + +def fine_grained_offloading_group_start(tensor, name=None): + """Mark the start of a layer group and prepare for offload/reload.""" + cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk() + return FineGrainedOffloadingGroupStartFunction.apply(tensor, cur_forward_chunk, name) + + +def get_fine_grained_offloading_context(flag): + """Get the fine-grained offload context""" + return PipelineOffloadManager.get_instance() if flag else nullcontext() + + +def fine_grained_offloading_set_last_layer(is_last_layer): + """Set the last layer flag.""" + PipelineOffloadManager.get_instance().set_last_layer(is_last_layer) + + +def fine_grained_offloading_init_chunk_handler(vp_stage, min_offloaded_tensor_size): + """Initialize the chunk handler, called at the start of a microbatch forward pass.""" + PipelineOffloadManager.get_instance().init_model_chunk_offload_handler( + vp_stage, min_offloaded_tensor_size + ) + + +def fine_grained_offloading_reset(): + """Reset the chunk handler, called at the start of a training iteration.""" + PipelineOffloadManager.get_instance().reset() diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index e83f8d90635..86282322def 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -9,6 +9,9 @@ from megatron.core import parallel_state from megatron.core.enums import ModelType +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_reset, +) from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator from megatron.core.pipeline_parallel.utils import ( is_pp_first_stage, @@ -562,6 +565,9 @@ def forward_backward_no_pipelining( if config.timers is not None: config.timers('forward-backward', log_level=1).start(barrier=config.barrier_with_L1_time) + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + no_sync_func = config.no_sync_func if no_sync_func is None: no_sync_func = contextlib.nullcontext @@ -898,6 +904,9 @@ def forward_backward_pipelining_with_interleaving( adjust_tensor_shapes_fn is None ), "adjust_tensor_shapes_fn is not supported for interleaved pipeline parallelism" + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + if config.overlap_p2p_comm and config.batch_p2p_comm: raise ValueError("Can not use both overlap_p2p_comm and batch_p2p_comm") @@ -2043,6 +2052,9 @@ def forward_backward_pipelining_without_interleaving( if config.timers is not None: config.timers('forward-backward', log_level=1).start(barrier=config.barrier_with_L1_time) + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + # Disable async grad reductions no_sync_func = config.no_sync_func if no_sync_func is None: diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 54cac0e41e3..2a27899781a 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -510,10 +510,11 @@ def forward(ctx, run_function, checkpoint_without_output_obj, *args): @staticmethod def backward(ctx, *args): """Backward pass.""" - inputs = ctx.saved_tensors + inputs = ctx.inputs outputs = ctx.outputs torch.autograd.backward(outputs, args) ctx.outputs = None + ctx.inputs = None grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else inp for inp in inputs) return (None, None) + grads @@ -573,8 +574,9 @@ def _recompute(self, _): recompute_ctx = contextlib.nullcontext() fp8_ctx = contextlib.nullcontext() + inputs = self.ctx.saved_tensors with torch.enable_grad(), fp8_ctx, recompute_ctx: - outputs = self.run_function(*self.ctx.saved_tensors) + outputs = self.run_function(*inputs) self.run_function = None self.rng_states = None @@ -590,6 +592,7 @@ def _recompute(self, _): output.untyped_storage().copy_(recomputation_output.untyped_storage()) self.ctx.outputs = outputs + self.ctx.inputs = inputs self.outputs = None self.ctx = None diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 659402a39b8..7ad001ad46a 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -22,6 +22,11 @@ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.module import MegatronModule @@ -188,6 +193,21 @@ def __init__( and "core_attn" in self.config.recompute_modules ) + self.offload_qkv_linear = ( + self.config.fine_grained_activation_offloading + and "qkv_linear" in self.config.offload_modules + ) + + self.offload_core_attention = ( + self.config.fine_grained_activation_offloading + and "core_attn" in self.config.offload_modules + ) + + self.offload_attn_proj = ( + self.config.fine_grained_activation_offloading + and "attn_proj" in self.config.offload_modules + ) + # Output. self.linear_proj = build_module( submodules.linear_proj, @@ -725,9 +745,16 @@ def forward( self.config.fused_single_qkv_rope and split_qkv ), "fused_single_qkv_rope requested but not available/supported for the config." - qkv_output = self.get_query_key_value_tensors( - hidden_states, key_value_states, split_qkv=split_qkv - ) + if self.offload_qkv_linear: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="qkv_linear") + with get_fine_grained_offloading_context(self.offload_qkv_linear): + qkv_output = self.get_query_key_value_tensors( + hidden_states, key_value_states, split_qkv=split_qkv + ) + if self.offload_qkv_linear: + qkv_output, _ = fine_grained_offloading_group_commit( + qkv_output, name="qkv_linear", forced_released_tensors=[hidden_states] + ) attn_mask_type = self.attn_mask_type block_table = None if split_qkv: @@ -873,17 +900,20 @@ def forward( packed_seq_params=packed_seq_params, ) else: + if self.offload_core_attention and self.training: + query = fine_grained_offloading_group_start(query, name="core_attn") if inference_context is None or inference_context.is_static_batching(): # Static batching attention kernel. - core_attn_out = self.core_attention( - query, - key, - value, - attention_mask, - attn_mask_type=attn_mask_type, - attention_bias=attention_bias, - packed_seq_params=packed_seq_params, - ) + with get_fine_grained_offloading_context(self.offload_core_attention): + core_attn_out = self.core_attention( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + attention_bias=attention_bias, + packed_seq_params=packed_seq_params, + ) else: # Dynamic batching attention kernel. @@ -904,6 +934,10 @@ def forward( ) core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)') + if self.offload_core_attention and self.training: + (core_attn_out,) = fine_grained_offloading_group_commit( + core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] + ) if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': # reshape to same output shape as unpacked case # (t, np, hn) -> (t, b=1, h=np*hn) @@ -917,7 +951,14 @@ def forward( # ================= nvtx_range_push(suffix="linear_proj") - output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") + with get_fine_grained_offloading_context(self.offload_attn_proj): + output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + output, bias = fine_grained_offloading_group_commit( + output, bias, name="attn_proj", forced_released_tensors=[core_attn_out] + ) nvtx_range_pop(suffix="linear_proj") return output, bias diff --git a/megatron/core/transformer/moe/README.md b/megatron/core/transformer/moe/README.md index 3a79cdc4cbb..cb4425ec696 100644 --- a/megatron/core/transformer/moe/README.md +++ b/megatron/core/transformer/moe/README.md @@ -203,6 +203,20 @@ Enable A2A overlap across different batches inspired by the DSv3 DualPipe implme --delay-wgrad-compute ``` +### Fine-grained Activation Offloading (collaborated with rednote) +Offload the input activation at the granularity of modules + +**Usage** +```bash +# Enable fine-grained activation offloading +--fine-grained-activation-offloading + +# Specify which modules are going to offload its input +# Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". +--offload-modules expert_fc1 +``` +For more details, please refer to the ```docs/source/api-guide/fine_grained_activation_offloading.md``` + ### MoE Related Arguments | Item | Description | | --- | --- | diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index d8dd3d03f02..8cdde0cf351 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -26,6 +26,11 @@ from megatron.core.fusions.fused_bias_swiglu import weighted_bias_swiglu_impl from megatron.core.fusions.fused_weighted_squared_relu import weighted_squared_relu_impl from megatron.core.jit import jit_fuser +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.tensor_parallel.layers import ( _initialize_affine_weight_cpu, _initialize_affine_weight_gpu, @@ -805,6 +810,16 @@ def __init__( tp_group=pg_collection.expt_tp, ) + self.offload_expert_fc1 = ( + self.config.fine_grained_activation_offloading + and "expert_fc1" in self.config.offload_modules + ) + + self.offload_moe_act = ( + self.config.fine_grained_activation_offloading + and "moe_act" in self.config.offload_modules + ) + self.activation_recompute = ( self.config.recompute_granularity == 'selective' and "moe_act" in self.config.recompute_modules @@ -814,6 +829,12 @@ def __init__( set_save_original_input(self.linear_fc2) + # This is to avoid the CPU overhead of multiple d2h copies + if self.offload_expert_fc1 and not (self.config.fp8 or self.config.fp4): + from megatron.core.extensions.transformer_engine import set_save_original_input + + set_save_original_input(self.linear_fc1) + if self.config.fp8: assert HAVE_TE, "FP8 requires TE." self.fp8_padding = Fp8Padding(self.num_local_experts) @@ -878,9 +899,21 @@ def forward( # Probs already applied, so reset to 1. permuted_probs = torch.ones_like(permuted_probs) - intermediate_parallel, bias_parallel = self.linear_fc1( - permuted_local_hidden_states, tokens_per_expert - ) + if self.offload_expert_fc1: + permuted_local_hidden_states = fine_grained_offloading_group_start( + permuted_local_hidden_states, name="expert_fc1" + ) + with get_fine_grained_offloading_context(self.offload_expert_fc1): + fc1_output, bias_parallel = self.linear_fc1( + permuted_local_hidden_states, tokens_per_expert + ) + if self.offload_expert_fc1: + fc1_output, bias_parallel = fine_grained_offloading_group_commit( + fc1_output, + bias_parallel, + name="expert_fc1", + forced_released_tensors=[permuted_local_hidden_states], + ) def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): if self.config.use_te_activation_func: @@ -940,18 +973,26 @@ def glu(x): intermediate_parallel = intermediate_parallel.to(original_dtype) return intermediate_parallel + if self.offload_moe_act: + fc1_output = fine_grained_offloading_group_start(fc1_output, name="moe_act") + if self.activation_recompute: self.activation_checkpoint = tensor_parallel.CheckpointWithoutOutput() - intermediate_parallel = self.activation_checkpoint.checkpoint( - bias_act_func, intermediate_parallel, bias_parallel, permuted_probs - ) - output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert) - self.activation_checkpoint.discard_output_and_register_recompute(output) + with get_fine_grained_offloading_context(self.offload_moe_act): + bias_act_output = self.activation_checkpoint.checkpoint( + bias_act_func, fc1_output, bias_parallel, permuted_probs + ) else: - intermediate_parallel = bias_act_func( - intermediate_parallel, bias_parallel, permuted_probs + with get_fine_grained_offloading_context(self.offload_moe_act): + bias_act_output = bias_act_func(fc1_output, bias_parallel, permuted_probs) + + output, output_bias = self.linear_fc2(bias_act_output, tokens_per_expert) + if self.activation_recompute: + self.activation_checkpoint.discard_output_and_register_recompute(output) + if self.offload_moe_act: + (output,) = fine_grained_offloading_group_commit( + output, name="moe_act", forced_released_tensors=[fc1_output] ) - output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert) # upad and concat the output if self.config.fp8: diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index a8893ebec36..a775729f007 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -22,6 +22,11 @@ _yarn_get_mscale, apply_rotary_pos_emb, ) +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.layers import ColumnParallelLinear from megatron.core.tensor_parallel.mappings import ( @@ -266,15 +271,19 @@ def forward( query, key, value, attention_mask, packed_seq_params=packed_seq_params ) else: + if self.offload_core_attention and self.training: + query = fine_grained_offloading_group_start(query, name="core_attn") + if inference_context is None or inference_context.is_static_batching(): - core_attn_out = self.core_attention( - query, - key, - value, - attention_mask, - packed_seq_params=packed_seq_params, - attn_mask_type=attn_mask_type, - ) + with get_fine_grained_offloading_context(self.offload_core_attention): + core_attn_out = self.core_attention( + query, + key, + value, + attention_mask, + packed_seq_params=packed_seq_params, + attn_mask_type=attn_mask_type, + ) elif self.cache_mla_latents: # Dynamic batching attention kernel. q, k, v = (query, key, value) @@ -295,6 +304,10 @@ def forward( # Only rearrange if not in absorption mode (Flash MLA handles format correctly) if not inference_context.is_decode_only(): core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)') + if self.offload_core_attention and self.training: + (core_attn_out,) = fine_grained_offloading_group_commit( + core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] + ) # We are doing absorption with cache mla latents and decode mode. if self.cache_mla_latents and inference_context.is_decode_only(): @@ -320,7 +333,14 @@ def forward( # ================= # Output. [sq, b, h] # ================= - output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") + with get_fine_grained_offloading_context(self.offload_attn_proj): + output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + output, bias = fine_grained_offloading_group_commit( + output, bias, name="attn_proj", forced_released_tensors=[core_attn_out] + ) return output, bias diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py index bd3aa9c8c96..001ca84c3ce 100755 --- a/megatron/core/transformer/multi_token_prediction.py +++ b/megatron/core/transformer/multi_token_prediction.py @@ -13,6 +13,9 @@ from megatron.core.fp8_utils import get_fp8_context from megatron.core.models.backends import BackendSpecProvider, LocalSpecProvider from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_set_last_layer, +) from megatron.core.pipeline_parallel.utils import is_vp_last_stage from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel import ( @@ -901,6 +904,8 @@ def forward( hidden_states_list = list(torch.chunk(hidden_states, 1 + offset, dim=0)) hidden_states = hidden_states_list[offset] for layer_number in range(len(self.layers)): + if self.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer(layer_number == len(self.layers) - 1) (hidden_states, input_ids, position_ids) = self.layers[layer_number]( input_ids=input_ids, position_ids=position_ids, diff --git a/megatron/core/transformer/transformer_block.py b/megatron/core/transformer/transformer_block.py index aead6133f22..08191c205f3 100755 --- a/megatron/core/transformer/transformer_block.py +++ b/megatron/core/transformer/transformer_block.py @@ -16,6 +16,9 @@ from megatron.core.fusions.fused_layer_norm import FusedLayerNorm from megatron.core.inference.contexts import BaseInferenceContext from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_set_last_layer, +) from megatron.core.pipeline_parallel.utils import is_vp_first_stage, is_vp_last_stage from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.enums import LayerType @@ -693,6 +696,11 @@ def forward( else: inner_quantization_context = nullcontext() + if self.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer( + l_no == self.num_layers_per_pipeline_rank - 1 + ) + with self.offload_context, inner_quantization_context: hidden_states, context = layer( hidden_states=hidden_states, diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 147c5b23b3d..2c2c398af40 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -725,6 +725,25 @@ class TransformerConfig(ModelParallelConfig): """Transformer implementation to use. Options are 'transformer_engine' for Transformer Engine and 'local' for MCore.""" + ##################################### + # Fine-grained Activation Offloading + ##################################### + fine_grained_activation_offloading: bool = False + """If True, offload the input of the specified modules to the CPU.""" + + offload_modules: Optional[list[str]] = None + """The submodules to offload its input. + choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". + "attn_norm": offload the input of the normalization in the attention part. + "core_attn": offload the input of the core attention part. + "mlp_norm": offload the input of the normalization in the mlp part. + "attn_proj": offload the input of the attn linear projection part. + "expert_fc1": offload the input of the expert fc1 part. + "moe_act": offload the input of the moe act part. + """ + min_offloaded_tensor_size: int = 1024 * 1024 + """The minimum size of the tensor to be offloaded.""" + def __post_init__(self): """Python dataclass method that is used to modify attributes after initialization. See https://docs.python.org/3/library/dataclasses.html#post-init-processing for more @@ -1015,6 +1034,28 @@ def __post_init__(self): if "moe" not in self.recompute_modules: self.recompute_modules.append("moe") + if self.fine_grained_activation_offloading: + assert self.offload_modules is not None and len(self.offload_modules) > 0 + allowed_modules = { + "core_attn", + "attn_proj", + "expert_fc1", + "moe_act", + "attn_norm", + "mlp_norm", + } + invalid_modules = set(self.offload_modules) - allowed_modules + assert not invalid_modules, ( + f'Invalid choices for offload_modules: {invalid_modules}. ' + f'Allowed modules are: {allowed_modules}' + ) + if "attn_proj" in self.offload_modules and "core_attn" not in self.offload_modules: + raise ValueError( + "attn_proj cannot be set to offload_modules alone without core_attn " + "because the input of attn_proj is the output of core_attn, " + "which is needed in core_attn.backward()." + ) + if ( self.num_layers_in_first_pipeline_stage is not None or self.num_layers_in_last_pipeline_stage is not None diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index a5babece9d0..3d61e1b186b 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -397,6 +397,16 @@ def __init__( if "mlp" in self.config.recompute_modules: if not isinstance(self.mlp, MoELayer): self.recompute_mlp = True + self.offload_attn_norm = ( + self.config.fine_grained_activation_offloading + and "attn_norm" in self.config.offload_modules + and not isinstance(self.input_layernorm, IdentityOp) + ) + self.offload_mlp_norm = ( + self.config.fine_grained_activation_offloading + and "mlp_norm" in self.config.offload_modules + and not isinstance(self.pre_mlp_layernorm, IdentityOp) + ) # @jcasper how should we handle nvfuser? # Set bias+dropout+add fusion grad_enable execution handler. @@ -479,20 +489,29 @@ def _forward_attention( context (Tensor): Updated context tensor if cross-attention is used, otherwise None. """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, + ) inference_context = deprecate_inference_params(inference_context, inference_params) # Residual connection. residual = hidden_states + if self.offload_attn_norm: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="attn_norm") # Optional Input Layer norm if self.recompute_input_layernorm: self.input_layernorm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - input_layernorm_output = self.input_layernorm_checkpoint.checkpoint( - self.input_layernorm, hidden_states - ) + with get_fine_grained_offloading_context(self.offload_attn_norm): + input_layernorm_output = self.input_layernorm_checkpoint.checkpoint( + self.input_layernorm, hidden_states + ) else: - input_layernorm_output = self.input_layernorm(hidden_states) + with get_fine_grained_offloading_context(self.offload_attn_norm): + input_layernorm_output = self.input_layernorm(hidden_states) # Self attention. nvtx_range_push(suffix="self_attention") @@ -526,6 +545,11 @@ def _forward_attention( ) nvtx_range_pop(suffix="self_attn_bda") + if self.offload_attn_norm: + (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states, name="attn_norm", forced_released_tensors=[residual] + ) + # Residual connection. residual = hidden_states @@ -563,17 +587,27 @@ def _forward_mlp(self, hidden_states, inference_context=None): output (Tensor): Transformed hidden states of shape [s, b, h]. """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, + ) + # Residual connection. residual = hidden_states + if self.offload_mlp_norm: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="mlp_norm") # Optional Layer norm post the cross-attention. if self.recompute_pre_mlp_layernorm: self.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint( - self.pre_mlp_layernorm, hidden_states - ) + with get_fine_grained_offloading_context(self.offload_mlp_norm): + pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint( + self.pre_mlp_layernorm, hidden_states + ) else: - pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) + with get_fine_grained_offloading_context(self.offload_mlp_norm): + pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) nvtx_range_push(suffix="mlp") # Potentially chunk the MLP computation during prefill to minimize the peak activation size @@ -633,6 +667,10 @@ def _forward_mlp(self, hidden_states, inference_context=None): mlp_output_with_bias, residual, self.hidden_dropout ) nvtx_range_pop(suffix="mlp_bda") + if self.offload_mlp_norm: + (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states, name="mlp_norm", forced_released_tensors=[residual] + ) # Jit compiled function creates 'view' tensor. This tensor # potentially gets saved in the MPU checkpoint function context, diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 5d1bc2e40a3..926da2ce00b 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1183,6 +1183,10 @@ def validate_args(args, defaults={}): "when enabling delay_wgrad_compute" ) + if args.fine_grained_activation_offloading: + assert args.transformer_impl == 'transformer_engine', \ + "Fine-grained activation offloading is only supported with transformer_engine implementation" + if args.mtp_num_layers: assert not args.use_legacy_models, "The legacy Megatron models does not support Multi-Token Prediction (MTP)." assert args.position_embedding_type == "rope" or args.position_embedding_type == "none", ( @@ -2277,7 +2281,12 @@ def _add_training_args(parser): help='The communicator group names to use high priority streams.') group.add_argument('--use-te-activation-func', action='store_true', help='Use activation function kernel from Transformer Engine in MLP module.') - + group.add_argument('--fine-grained-activation-offloading', action='store_true', + help='Enable fine-grained activation offloading.') + group.add_argument('--offload-modules', nargs='*', type=str, default=[], + help='The submodules to offload its input. Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".') + group.add_argument('--min-offloaded-tensor-size', type=int, default=1024*1024, + help='The minimum size of the tensor to be offloaded.') return parser diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json new file mode 100644 index 00000000000..30ea509a50b --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json @@ -0,0 +1,110 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 11.0637, + "5": 9.48263, + "10": 9.04035, + "15": 8.00837, + "20": 7.88364, + "25": 7.67597, + "30": 7.63447, + "35": 7.21393, + "40": 7.55564, + "45": 7.21045, + "50": 7.05439 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 38802064.0, + "5": 394456256.0, + "10": 571185472.0, + "15": 699100416.0, + "20": 891692160.0, + "25": 748799104.0, + "30": 794511296.0, + "35": 671593792.0, + "40": 421718816.0, + "45": 517934176.0, + "50": 472902496.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 6025468416.0, + "5": 6025470464.0, + "10": 6025470464.0, + "15": 6025470464.0, + "20": 6025470464.0, + "25": 6025470464.0, + "30": 6025470464.0, + "35": 6025470464.0, + "40": 6025470464.0, + "45": 6025470464.0, + "50": 6025470464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 45099868160.0, + "5": 49175810048.0, + "10": 49175810048.0, + "15": 49175810048.0, + "20": 49175810048.0, + "25": 49175810048.0, + "30": 49211260928.0, + "35": 49211260928.0, + "40": 49211260928.0, + "45": 49211260928.0, + "50": 49211260928.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 11.04508, + "5": 9.76285, + "10": 9.04997, + "15": 7.93865, + "20": 7.79984, + "25": 7.60324, + "30": 7.56633, + "35": 7.13802, + "40": 7.45784, + "45": 7.11892, + "50": 6.9559 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 52.8667, + "5": 2.06295, + "10": 1.09336, + "15": 1.10509, + "20": 1.08631, + "25": 1.08991, + "30": 1.10548, + "35": 1.10049, + "40": 1.11219, + "45": 1.09542, + "50": 1.09805 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json new file mode 100644 index 00000000000..30ea509a50b --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json @@ -0,0 +1,110 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 11.0637, + "5": 9.48263, + "10": 9.04035, + "15": 8.00837, + "20": 7.88364, + "25": 7.67597, + "30": 7.63447, + "35": 7.21393, + "40": 7.55564, + "45": 7.21045, + "50": 7.05439 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 38802064.0, + "5": 394456256.0, + "10": 571185472.0, + "15": 699100416.0, + "20": 891692160.0, + "25": 748799104.0, + "30": 794511296.0, + "35": 671593792.0, + "40": 421718816.0, + "45": 517934176.0, + "50": 472902496.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 6025468416.0, + "5": 6025470464.0, + "10": 6025470464.0, + "15": 6025470464.0, + "20": 6025470464.0, + "25": 6025470464.0, + "30": 6025470464.0, + "35": 6025470464.0, + "40": 6025470464.0, + "45": 6025470464.0, + "50": 6025470464.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 45099868160.0, + "5": 49175810048.0, + "10": 49175810048.0, + "15": 49175810048.0, + "20": 49175810048.0, + "25": 49175810048.0, + "30": 49211260928.0, + "35": 49211260928.0, + "40": 49211260928.0, + "45": 49211260928.0, + "50": 49211260928.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 11.04508, + "5": 9.76285, + "10": 9.04997, + "15": 7.93865, + "20": 7.79984, + "25": 7.60324, + "30": 7.56633, + "35": 7.13802, + "40": 7.45784, + "45": 7.11892, + "50": 6.9559 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 52.8667, + "5": 2.06295, + "10": 1.09336, + "15": 1.10509, + "20": 1.08631, + "25": 1.08991, + "30": 1.10548, + "35": 1.10049, + "40": 1.11219, + "45": 1.09542, + "50": 1.09805 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml new file mode 100644 index 00000000000..9a125a1cf74 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml @@ -0,0 +1,139 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 32 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: 0 + PYTHONWARNINGS: ignore + NCCL_DEBUG: VERSION +MODEL_ARGS: + # Distributed args + --distributed-timeout-minutes: 60 + --tensor-model-parallel-size: 2 + --pipeline-model-parallel-size: 2 + --expert-model-parallel-size: 4 + --context-parallel-size: 1 + --expert-tensor-parallel-size: 1 + --use-distributed-optimizer: true + # NOTE: uncomment if TE >= 2.9.0 + # --overlap-grad-reduce: true + # --overlap-param-gather: true + # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN + --attention-backend: unfused # TODO: switch back to fused attention after fix + # Training args + --use-mcore-models: true + --sequence-parallel: true + --disable-bias-linear: true + --micro-batch-size: 4 + --global-batch-size: 32 + --train-iters: 50 + --exit-duration-in-mins: 230 + --no-check-for-nan-in-loss-and-grad: true + --no-rope-fusion: true + --cross-entropy-loss-fusion: true + --cross-entropy-fusion-impl: native + --manual-gc: true + --manual-gc-interval: 100 + --recompute-granularity: selective + --recompute-modules: "[layernorm mla_up_proj mlp moe_act]" + --fine-grained-activation-offloading: true + --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm]" + # Transformer Engine args + --transformer-impl: transformer_engine + # Data args + --seq-length: 4096 + --data-cache-path: ${DATA_CACHE_PATH} + --data-path: ${DATA_PATH}/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/bpe/vocab.json + --merge-file: ${DATA_PATH}/bpe/merges.txt + --split: 949,50,1 + # Add network size args + --num-layers: 15 + --moe-layer-freq: ([0]*3+[1]*12) + --pipeline-model-parallel-layout: Et*3\\|\\(tt\\|\\)*6mL # Et*3|(tt|)*6mL + --hidden-size: 1024 + --ffn-hidden-size: 4096 + --num-attention-heads: 32 + --kv-channels: 128 + --max-position-embeddings: 4096 + --position-embedding-type: rope + --rotary-base: 10000 + --make-vocab-size-divisible-by: 3232 + --normalization: RMSNorm + --norm-epsilon: 1e-6 + --swiglu: true + --untie-embeddings-and-output-weights: true + --multi-latent-attention: true + # Comment out the following MTP args to disable MTP + --mtp-num-layers: 1 + --mtp-loss-scaling-factor: 0.1 + # Add regularization args + --attention-dropout: 0.0 + --hidden-dropout: 0.0 + --clip-grad: 1.0 + --weight-decay: 0.1 + --qk-layernorm: true + # Add learning rate args + --lr-warmup-fraction: .01 + --lr: 0.00015 + --min-lr: 1.0e-5 + --lr-decay-style: cosine + --adam-beta1: 0.9 + --adam-beta2: 0.95 + # Add MoE args + --num-experts: 32 + --moe-ffn-hidden-size: 1024 + --moe-shared-expert-intermediate-size: 1024 + --moe-router-load-balancing-type: seq_aux_loss + --moe-router-topk: 4 + --moe-token-dispatcher-type: alltoall + --moe-router-pre-softmax: true + --moe-grouped-gemm: true + --moe-aux-loss-coeff: 1e-4 + --moe-router-group-topk: 2 + --moe-router-num-groups: 4 + --moe-router-topk-scaling-factor: 2.0 + --moe-router-score-function: sigmoid + --moe-router-enable-expert-bias: true + --moe-router-bias-update-rate: 1e-3 + --moe-router-dtype: fp32 + --moe-permute-fusion: true + # Add MLA args + --q-lora-rank: 1536 + --kv-lora-rank: 512 + --qk-head-dim: 128 + --qk-pos-emb-head-dim: 64 + --v-head-dim: 128 + --rotary-scaling-factor: 40 + --mscale: 1.0 + --mscale-all-dim: 1.0 + # Add validation args + --eval-iters: 32 + --eval-interval: 200 + # Add checkpointing args + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --save-interval: 25 + # Add initialization args + --init-method-std: 0.02 + # Add logging args + --log-timers-to-tensorboard: true + --log-memory-to-tensorboard: true + --log-num-zeros-in-grad: true + --log-params-norm: true + --log-validation-ppl-to-tensorboard: true + --log-throughput: true + --log-interval: 1 + --logging-level: 40 + --tensorboard-dir: ${TENSORBOARD_PATH} + # Add mixed precision args + --bf16: true + --exit-interval: 50 + --overlap-moe-expert-parallel-comm: true +TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular +METRICS: + - "iteration-time" + - "lm loss" + - "num-zeros" + - "mem-allocated-bytes" + - "mem-max-allocated-bytes" + - "mtp_1 loss" diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json new file mode 100644 index 00000000000..3687e19e563 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json @@ -0,0 +1,92 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 11.04266, + "5": 9.38536, + "10": 8.82761, + "15": 7.86966, + "20": 7.72022, + "25": 7.53119, + "30": 7.5026, + "35": 7.10343, + "40": 7.42037, + "45": 7.07056, + "50": 6.90946 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 844114112.0, + "5": 856834688.0, + "10": 928751040.0, + "15": 952825152.0, + "20": 987111232.0, + "25": 926008384.0, + "30": 864767232.0, + "35": 855095360.0, + "40": 849505920.0, + "45": 847187584.0, + "50": 846195840.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 4419107328.0, + "5": 4419108864.0, + "10": 4419108864.0, + "15": 4419108864.0, + "20": 4419108864.0, + "25": 4419108864.0, + "30": 4419108864.0, + "35": 4419108864.0, + "40": 4419108864.0, + "45": 4419108864.0, + "50": 4419108864.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 37959917568.0, + "5": 39583289344.0, + "10": 39583289344.0, + "15": 39583289344.0, + "20": 39583289344.0, + "25": 39583289344.0, + "30": 39583289344.0, + "35": 39583289344.0, + "40": 39583289344.0, + "45": 39583289344.0, + "50": 39583289344.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 58.78709, + "5": 2.40565, + "10": 1.13046, + "15": 1.39764, + "20": 1.1273, + "25": 1.12154, + "30": 1.03587, + "35": 1.09545, + "40": 1.09901, + "45": 1.00656, + "50": 1.00794 + } + } +} diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json new file mode 100644 index 00000000000..3687e19e563 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json @@ -0,0 +1,92 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 11.04266, + "5": 9.38536, + "10": 8.82761, + "15": 7.86966, + "20": 7.72022, + "25": 7.53119, + "30": 7.5026, + "35": 7.10343, + "40": 7.42037, + "45": 7.07056, + "50": 6.90946 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 844114112.0, + "5": 856834688.0, + "10": 928751040.0, + "15": 952825152.0, + "20": 987111232.0, + "25": 926008384.0, + "30": 864767232.0, + "35": 855095360.0, + "40": 849505920.0, + "45": 847187584.0, + "50": 846195840.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 4419107328.0, + "5": 4419108864.0, + "10": 4419108864.0, + "15": 4419108864.0, + "20": 4419108864.0, + "25": 4419108864.0, + "30": 4419108864.0, + "35": 4419108864.0, + "40": 4419108864.0, + "45": 4419108864.0, + "50": 4419108864.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 37959917568.0, + "5": 39583289344.0, + "10": 39583289344.0, + "15": 39583289344.0, + "20": 39583289344.0, + "25": 39583289344.0, + "30": 39583289344.0, + "35": 39583289344.0, + "40": 39583289344.0, + "45": 39583289344.0, + "50": 39583289344.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 5, + "values": { + "1": 58.78709, + "5": 2.40565, + "10": 1.13046, + "15": 1.39764, + "20": 1.1273, + "25": 1.12154, + "30": 1.03587, + "35": 1.09545, + "40": 1.09901, + "45": 1.00656, + "50": 1.00794 + } + } +} diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml new file mode 100644 index 00000000000..8832d687004 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml @@ -0,0 +1,134 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 1 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: 0 + PYTHONWARNINGS: ignore + NCCL_DEBUG: VERSION +MODEL_ARGS: + # Distributed args + --distributed-timeout-minutes: 60 + --tensor-model-parallel-size: 2 + --pipeline-model-parallel-size: 2 + --expert-model-parallel-size: 4 + --context-parallel-size: 1 + --expert-tensor-parallel-size: 1 + --use-distributed-optimizer: true + # NOTE: uncomment if TE >= 2.9.0 + # --overlap-grad-reduce: true + # --overlap-param-gather: true + # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN + --attention-backend: unfused # TODO: switch back to fused attention after fix + # Training args + --use-mcore-models: true + --sequence-parallel: true + --disable-bias-linear: true + --micro-batch-size: 4 + --global-batch-size: 32 + --train-iters: 50 + --exit-duration-in-mins: 230 + --no-check-for-nan-in-loss-and-grad: true + --no-rope-fusion: true + --cross-entropy-loss-fusion: true + --cross-entropy-fusion-impl: native + --manual-gc: true + --manual-gc-interval: 100 + --recompute-granularity: selective + --recompute-modules: "[layernorm mla_up_proj mlp moe_act]" + --fine-grained-activation-offloading: true + --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm]" + # Transformer Engine args + --transformer-impl: transformer_engine + # Data args + --seq-length: 4096 + --data-cache-path: ${DATA_CACHE_PATH} + --data-path: ${DATA_PATH}/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/bpe/vocab.json + --merge-file: ${DATA_PATH}/bpe/merges.txt + --split: 949,50,1 + # Add network size args + --num-layers: 15 + --moe-layer-freq: ([0]*3+[1]*12) + --pipeline-model-parallel-layout: Et*3\\|\\(tt\\|\\)*6L # Et*3|(tt|)*6L + --hidden-size: 1024 + --ffn-hidden-size: 4096 + --num-attention-heads: 32 + --kv-channels: 128 + --max-position-embeddings: 4096 + --position-embedding-type: rope + --rotary-base: 10000 + --make-vocab-size-divisible-by: 3232 + --normalization: RMSNorm + --norm-epsilon: 1e-6 + --swiglu: true + --untie-embeddings-and-output-weights: true + --multi-latent-attention: true + # Add regularization args + --attention-dropout: 0.0 + --hidden-dropout: 0.0 + --clip-grad: 1.0 + --weight-decay: 0.1 + --qk-layernorm: true + # Add learning rate args + --lr-warmup-fraction: .01 + --lr: 0.00015 + --min-lr: 1.0e-5 + --lr-decay-style: cosine + --adam-beta1: 0.9 + --adam-beta2: 0.95 + # Add MoE args + --num-experts: 32 + --moe-ffn-hidden-size: 1024 + --moe-shared-expert-intermediate-size: 1024 + --moe-router-load-balancing-type: seq_aux_loss + --moe-router-topk: 4 + --moe-token-dispatcher-type: alltoall + --moe-router-pre-softmax: true + --moe-grouped-gemm: true + --moe-aux-loss-coeff: 1e-4 + --moe-router-group-topk: 2 + --moe-router-num-groups: 4 + --moe-router-topk-scaling-factor: 2.0 + --moe-router-score-function: sigmoid + --moe-router-enable-expert-bias: true + --moe-router-bias-update-rate: 1e-3 + --moe-router-dtype: fp32 + --moe-permute-fusion: true + # Add MLA args + --q-lora-rank: 1536 + --kv-lora-rank: 512 + --qk-head-dim: 128 + --qk-pos-emb-head-dim: 64 + --v-head-dim: 128 + --rotary-scaling-factor: 40 + --mscale: 1.0 + --mscale-all-dim: 1.0 + # Add validation args + --eval-iters: 32 + --eval-interval: 200 + # Add checkpointing args + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --save-interval: 25 + # Add initialization args + --init-method-std: 0.02 + # Add logging args + --log-timers-to-tensorboard: true + --log-memory-to-tensorboard: true + --log-num-zeros-in-grad: true + --log-params-norm: true + --log-validation-ppl-to-tensorboard: true + --log-throughput: true + --log-interval: 1 + --logging-level: 40 + --tensorboard-dir: ${TENSORBOARD_PATH} + # Add mixed precision args + --bf16: true + --exit-interval: 50 +TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular +METRICS: + - "iteration-time" + - "lm loss" + - "num-zeros" + - "mem-allocated-bytes" + - "mem-max-allocated-bytes" diff --git a/tests/test_utils/recipes/moe.yaml b/tests/test_utils/recipes/moe.yaml index 2bfadfa7fa1..ea116384f92 100644 --- a/tests/test_utils/recipes/moe.yaml +++ b/tests/test_utils/recipes/moe.yaml @@ -182,6 +182,16 @@ products: - environment: [dev] scope: [mr] platforms: [dgx_h100] + - test_case: [gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_h100] + - test_case: [gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_h100] ####################################################################### # Super important MR tests that run for both DEV and LTS per MR # ####################################################################### diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py new file mode 100644 index 00000000000..a3f84c2770c --- /dev/null +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -0,0 +1,185 @@ +import gc + +import pytest +import torch + +EPSILON = 0.1 + +# Skip all tests if CUDA is not available +cuda_available = torch.cuda.is_available() + + +def _reset_cuda_memory(): + gc.collect() + if cuda_available: + torch.cuda.empty_cache() + + +class ToyModel(torch.nn.Module): + def __init__(self, hidden_size: int = 2048, num_layers: int = 4, dtype=torch.bfloat16): + super().__init__() + layers = [] + for _ in range(num_layers): + layers.append( + torch.nn.Linear(hidden_size, hidden_size, bias=True, dtype=dtype, device="cuda") + ) + self.net = torch.nn.Sequential(*layers).to(device="cuda", dtype=dtype) + self.hidden_size = hidden_size + self.num_layers = num_layers + self.dtype = dtype + + # Prevent weights/bias from being considered activation tensors for offload; + # ensure we only count activation tensors (inputs x) in memory accounting. + for p in self.parameters(): + try: + setattr(p, "offloading_activation", False) + except Exception: + pass + + def forward(self, x, use_offload: bool = False): + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + + if use_offload: + # Initialize a new chunk (microbatch) and enable offload context. + with off.get_fine_grained_offloading_context(True): + off.fine_grained_offloading_init_chunk_handler( + vp_stage=None, min_offloaded_tensor_size=1 + ) + for i, layer in enumerate(self.net): + # Group by module; with this linear-only model, each group corresponds to a layer. + off.fine_grained_offloading_set_last_layer(i == len(self.net) - 1) + x = off.fine_grained_offloading_group_start(x, name=f"layer_{i}") + x = layer(x) + # Commit the group; returns a tuple of tensors + (x,) = off.fine_grained_offloading_group_commit( + x, name=f"layer_{i}", forced_released_tensors=[] + ) + return x + # Baseline path (no offload hooks) + with ( + torch.autocast(device_type="cuda", dtype=self.dtype) + if self.dtype in (torch.float16, torch.bfloat16) + else torch.cuda.amp.autocast(enabled=False) + ): + for layer in self.net: + x = layer(x) + return x + + +@pytest.fixture(autouse=True) +def _monkeypatch_offload_deps(monkeypatch): + # Avoid requiring torch.distributed initialization and NVML in tests + import megatron.core.pipeline_parallel.fine_grained_activation_offload as off + + monkeypatch.setattr(off, "debug_rank", lambda *args, **kwargs: None, raising=False) + monkeypatch.setattr(off, "set_ideal_affinity_for_current_gpu", lambda: None, raising=False) + # Ensure a clean state each test + off.fine_grained_offloading_reset() + yield + off.fine_grained_offloading_reset() + + +def test_fine_grained_activation_offload_memory_reduction(): + torch.manual_seed(1234) + # Use a linear-only stack so theoretical saved memory equals sum of per-layer input x bytes. + model = ToyModel(hidden_size=2048, num_layers=8, dtype=torch.bfloat16).eval() + + # Create input + inp = torch.randn( + (2048, model.hidden_size), device="cuda", dtype=torch.bfloat16, requires_grad=True + ) + + # Warmup to stabilize allocator behavior + _reset_cuda_memory() + out = model(inp, use_offload=False) + (out.sum()).backward() + torch.cuda.synchronize() + _reset_cuda_memory() + + # Baseline memory measurement (no offload) + _reset_cuda_memory() + inp_baseline = inp.detach().clone().requires_grad_(True) + baseline_mem_before = torch.cuda.memory_allocated() / (1024**2) + out_base = model(inp_baseline, use_offload=False) + baseline_mem_after = (torch.cuda.memory_allocated() - out_base.nbytes) / (1024**2) + (out_base.sum()).backward() + torch.cuda.synchronize() + baseline_delta = baseline_mem_after - baseline_mem_before + + # Offload memory measurement + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + + off.fine_grained_offloading_reset() + _reset_cuda_memory() + inp_off = inp.detach().clone().requires_grad_(True) + offload_mem_before = torch.cuda.memory_allocated() / (1024**2) + out_off = model(inp_off, use_offload=True) + offload_mem_after = (torch.cuda.memory_allocated() - out_off.nbytes) / (1024**2) + (out_off.sum()).backward() + torch.cuda.synchronize() + offload_delta = offload_mem_after - offload_mem_before + + # Offload should reduce peak cached memory usage after forward + assert ( + offload_delta < baseline_delta + ), f"offload did not reduce memory: off={offload_delta:.2f}MiB base={baseline_delta:.2f}MiB" + + # Theoretical savings: storing per-layer input x (same shape each layer). + bytes_per_elem = inp.element_size() # 2 for bfloat16 + input_bytes = inp.numel() * bytes_per_elem + # -2 because the first and last activations are not offloaded + expected_saved_mib = (model.num_layers - 2) * (input_bytes / (1024**2)) + + # Actual savings ≈ baseline_delta - offload_delta (both exclude output tensor memory). + actual_saved_mib = baseline_delta - offload_delta + + # Allow slack for allocator jitter and extra intermediates; magnitudes should match. + rel_err = abs(actual_saved_mib - expected_saved_mib) / max(expected_saved_mib, 1e-6) + assert ( + rel_err <= EPSILON + ), f"saved mismatch: actual={actual_saved_mib:.2f}MiB expected~={expected_saved_mib:.2f}MiB (rel_err={rel_err:.2f})" + + +def test_fine_grained_activation_offload_output_and_grad_consistency(): + torch.manual_seed(2025) + hidden = 1024 + layers = 3 + + # Create identical models by resetting seed + torch.manual_seed(2025) + model_base = ToyModel(hidden_size=hidden, num_layers=layers, dtype=torch.bfloat16).train() + torch.manual_seed(2025) + model_off = ToyModel(hidden_size=hidden, num_layers=layers, dtype=torch.bfloat16).train() + + # Same input and target + inp = torch.randn((32, hidden), device="cuda", dtype=torch.bfloat16, requires_grad=True) + target = torch.randn_like(inp) + + # Baseline forward/backward + out_base = model_base(inp, use_offload=False) + loss_base = torch.nn.functional.mse_loss(out_base, target) + loss_base.backward() + grads_base = [ + p.grad.detach().clone() if p.grad is not None else None for p in model_base.parameters() + ] + + # Offload forward/backward + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + + off.fine_grained_offloading_reset() + out_off = model_off(inp.detach().clone().requires_grad_(True), use_offload=True) + loss_off = torch.nn.functional.mse_loss(out_off, target) + loss_off.backward() + grads_off = [ + p.grad.detach().clone() if p.grad is not None else None for p in model_off.parameters() + ] + + # Compare outputs + assert torch.allclose(out_off.float(), out_base.float(), rtol=1e-3, atol=1e-3) + + # Compare gradients parameter-wise + for gb, go in zip(grads_base, grads_off): + if gb is None and go is None: + continue + assert gb is not None and go is not None + assert torch.allclose(go.float(), gb.float(), rtol=1e-3, atol=1e-3) From e8fd17f0a0cf1e932fa733e96a3860db96d0e6ed Mon Sep 17 00:00:00 2001 From: lhb8125 Date: Thu, 23 Oct 2025 20:24:46 -0700 Subject: [PATCH 02/47] update years in copyright Signed-off-by: Hongbin Liu --- megatron/core/extensions/transformer_engine.py | 2 +- megatron/core/models/gpt/gpt_model.py | 2 +- megatron/core/pipeline_parallel/schedules.py | 2 +- megatron/core/tensor_parallel/random.py | 2 +- megatron/core/transformer/moe/experts.py | 2 +- megatron/core/transformer/multi_latent_attention.py | 2 +- megatron/core/transformer/transformer_block.py | 2 +- megatron/core/transformer/transformer_config.py | 2 +- .../test_fine_grained_activation_offloading.py | 2 ++ 9 files changed, 10 insertions(+), 8 deletions(-) diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index 5f04c9ba6ff..1e63b5b354a 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. import dataclasses import inspect diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index c77b9782c77..cdc848e082c 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. from collections import OrderedDict from typing import Dict, Literal, Optional diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index 86282322def..7127f1bd178 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. import contextlib from functools import partial diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 2a27899781a..df3451a1b77 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. # Parts of the code here are adapted from PyTorch # repo: https://github.com/pytorch/pytorch diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index 8cdde0cf351..ed9fc138a89 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. import copy import itertools diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index a775729f007..f1e89980786 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. import math diff --git a/megatron/core/transformer/transformer_block.py b/megatron/core/transformer/transformer_block.py index 08191c205f3..d62ffd7e8a2 100755 --- a/megatron/core/transformer/transformer_block.py +++ b/megatron/core/transformer/transformer_block.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. import logging from contextlib import nullcontext from dataclasses import dataclass diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 2c2c398af40..e3276145078 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. import warnings from dataclasses import dataclass diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index a3f84c2770c..2f6ed622ac3 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + import gc import pytest From 85548e615f89cb75121daa333b284b754cdd8e70 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Sun, 26 Oct 2025 18:25:05 -0700 Subject: [PATCH 03/47] update copyright Signed-off-by: Hongbin Liu --- megatron/core/extensions/transformer_engine.py | 2 +- megatron/core/models/common/model_chunk_schedule_plan.py | 2 +- megatron/core/models/gpt/fine_grained_callables.py | 2 +- megatron/core/models/gpt/gpt_model.py | 2 +- .../core/pipeline_parallel/fine_grained_activation_offload.py | 2 +- megatron/core/pipeline_parallel/schedules.py | 2 +- megatron/core/tensor_parallel/random.py | 2 +- megatron/core/transformer/attention.py | 2 +- megatron/core/transformer/moe/experts.py | 2 +- megatron/core/transformer/multi_latent_attention.py | 2 +- megatron/core/transformer/multi_token_prediction.py | 2 +- megatron/core/transformer/transformer_block.py | 2 +- megatron/core/transformer/transformer_config.py | 2 +- megatron/core/transformer/transformer_layer.py | 2 +- megatron/training/arguments.py | 2 +- .../test_fine_grained_activation_offloading.py | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index 1e63b5b354a..e807ee54fbf 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import dataclasses import inspect diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py index a741efc2042..929d6214b30 100644 --- a/megatron/core/models/common/model_chunk_schedule_plan.py +++ b/megatron/core/models/common/model_chunk_schedule_plan.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from contextlib import nullcontext from typing import Optional diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index d2e4c92df3c..adddcd6be4a 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import weakref from contextlib import nullcontext diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index cdc848e082c..ae292649561 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from collections import OrderedDict from typing import Dict, Literal, Optional diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 5b291f03300..b28bbcbeddc 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import warnings from collections import deque diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index 7127f1bd178..09f95ac25d2 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import contextlib from functools import partial diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index df3451a1b77..2ae15bef0d9 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Parts of the code here are adapted from PyTorch # repo: https://github.com/pytorch/pytorch diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 7ad001ad46a..97ff68d026c 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from abc import ABC, abstractmethod from dataclasses import dataclass diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index ed9fc138a89..d5bfb98c758 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import copy import itertools diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index f1e89980786..5d3f16c1041 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import math diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py index 001ca84c3ce..a619b9ffa55 100755 --- a/megatron/core/transformer/multi_token_prediction.py +++ b/megatron/core/transformer/multi_token_prediction.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from contextlib import nullcontext from dataclasses import dataclass diff --git a/megatron/core/transformer/transformer_block.py b/megatron/core/transformer/transformer_block.py index d62ffd7e8a2..06e8f1372f4 100755 --- a/megatron/core/transformer/transformer_block.py +++ b/megatron/core/transformer/transformer_block.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging from contextlib import nullcontext from dataclasses import dataclass diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index e3276145078..c5e984d712c 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import warnings from dataclasses import dataclass diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index 3d61e1b186b..c36ff7515e4 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging import warnings diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 926da2ce00b..19dbd372da3 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. """Megatron arguments.""" diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 2f6ed622ac3..edec95288c2 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import gc From f5bf8c44b3873874a7f921b6e2705e14a66de5c5 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 27 Oct 2025 20:46:42 -0700 Subject: [PATCH 04/47] fix ft and import error Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offload.py | 9 +- .../golden_values_dev_coreweave.json | 344 ++++++++++++++++++ .../golden_values_dev_eos.json | 0 .../model_config.yaml | 6 +- .../golden_values_dev_coreweave.json | 287 +++++++++++++++ .../golden_values_dev_eos.json | 0 .../model_config.yaml | 6 +- .../golden_values_dev_coreweave.json | 110 ------ .../golden_values_dev_coreweave.json | 92 ----- 9 files changed, 644 insertions(+), 210 deletions(-) create mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json rename tests/functional_tests/test_cases/moe/{gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading => gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading}/golden_values_dev_eos.json (100%) rename tests/functional_tests/test_cases/moe/{gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading => gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading}/model_config.yaml (95%) create mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json rename tests/functional_tests/test_cases/moe/{gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading => gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading}/golden_values_dev_eos.json (100%) rename tests/functional_tests/test_cases/moe/{gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading => gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading}/model_config.yaml (94%) delete mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json delete mode 100644 tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index b28bbcbeddc..be2e5f56fdc 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -30,8 +30,13 @@ def set_ideal_affinity_for_current_gpu(): import cuda.bindings.driver as cuda_driver import cuda.bindings.runtime as cuda_runtime except ImportError: - import cuda.cuda as cuda_driver - import cuda.cudart as cuda_runtime + try: + import cuda.cuda as cuda_driver + import cuda.cudart as cuda_runtime + except ImportError: + # print("cuda-python may not be installed, skipping GPU affinity setting") + warnings.warn("cuda-python may not be installed, skipping GPU affinity setting") + return try: import pynvml except ImportError: diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json new file mode 100644 index 00000000000..b3f192ba287 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json @@ -0,0 +1,344 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.07546, + "2": 11.03837, + "3": 9.66011, + "4": 9.91381, + "5": 9.32909, + "6": 9.13922, + "7": 9.13574, + "8": 8.65508, + "9": 8.51394, + "10": 8.8409, + "11": 8.29149, + "12": 8.34581, + "13": 8.25518, + "14": 7.73711, + "15": 7.86249, + "16": 7.9371, + "17": 7.89319, + "18": 7.63123, + "19": 7.99731, + "20": 7.74538, + "21": 7.44348, + "22": 7.42249, + "23": 7.29714, + "24": 7.27462, + "25": 7.54574, + "26": 6.96838, + "27": 7.50556, + "28": 7.22743, + "29": 7.36588, + "30": 7.52622, + "31": 7.27026, + "32": 7.45521, + "33": 7.50954, + "34": 7.55686, + "35": 7.10177, + "36": 6.96431, + "37": 7.28463, + "38": 7.0808, + "39": 7.40923, + "40": 7.43338, + "41": 7.38496, + "42": 7.15749, + "43": 7.15858, + "44": 7.28852, + "45": 7.16793, + "46": 6.78468, + "47": 7.4114, + "48": 7.0027, + "49": 7.46249, + "50": 6.92151 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 911219392.0, + "2": 910960384.0, + "3": 911156352.0, + "4": 912204800.0, + "5": 920796544.0, + "6": 940387968.0, + "7": 990599872.0, + "8": 976457728.0, + "9": 998097664.0, + "10": 995852672.0, + "11": 994583680.0, + "12": 977344896.0, + "13": 1028141824.0, + "14": 1007166208.0, + "15": 987423616.0, + "16": 993054784.0, + "17": 982319168.0, + "18": 998261760.0, + "19": 984696320.0, + "20": 982914752.0, + "21": 979667456.0, + "22": 953988864.0, + "23": 972353984.0, + "24": 964792064.0, + "25": 958512192.0, + "26": 946928512.0, + "27": 948458304.0, + "28": 949643968.0, + "29": 942877440.0, + "30": 935020160.0, + "31": 935327616.0, + "32": 934281088.0, + "33": 921805568.0, + "34": 928189312.0, + "35": 922202496.0, + "36": 924246656.0, + "37": 920661248.0, + "38": 922930752.0, + "39": 922322816.0, + "40": 921856512.0, + "41": 920227968.0, + "42": 918353664.0, + "43": 918607040.0, + "44": 914948032.0, + "45": 914295232.0, + "46": 914344448.0, + "47": 911769536.0, + "48": 912013312.0, + "49": 910349440.0, + "50": 914351552.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5498353152.0, + "2": 5499147776.0, + "3": 5499940352.0, + "4": 5500732928.0, + "5": 5501525504.0, + "6": 5502318080.0, + "7": 5503110656.0, + "8": 5503903232.0, + "9": 5497958912.0, + "10": 5498751488.0, + "11": 5499544064.0, + "12": 5500336640.0, + "13": 5501129216.0, + "14": 5501921792.0, + "15": 5502714368.0, + "16": 5503506944.0, + "17": 5504299520.0, + "18": 5505092096.0, + "19": 5505884672.0, + "20": 5506677248.0, + "21": 5507469824.0, + "22": 5508262400.0, + "23": 5509054976.0, + "24": 5509847552.0, + "25": 5510640128.0, + "26": 5511432704.0, + "27": 5512225280.0, + "28": 5513017856.0, + "29": 5513810432.0, + "30": 5514603008.0, + "31": 5515395584.0, + "32": 5516188160.0, + "33": 5516980736.0, + "34": 5517773312.0, + "35": 5518565888.0, + "36": 5519358464.0, + "37": 5520151040.0, + "38": 5520943616.0, + "39": 5521736192.0, + "40": 5522528768.0, + "41": 5523321344.0, + "42": 5524113920.0, + "43": 5524906496.0, + "44": 5525699072.0, + "45": 5526491648.0, + "46": 5527284224.0, + "47": 5528076800.0, + "48": 5528869376.0, + "49": 5529661952.0, + "50": 5530454528.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 41739952128.0, + "2": 43687571456.0, + "3": 43687571456.0, + "4": 43983216640.0, + "5": 43983216640.0, + "6": 43983216640.0, + "7": 43983216640.0, + "8": 44024635392.0, + "9": 44041216000.0, + "10": 44041216000.0, + "11": 44041216000.0, + "12": 44041216000.0, + "13": 44041216000.0, + "14": 44041216000.0, + "15": 44041216000.0, + "16": 44041216000.0, + "17": 44041216000.0, + "18": 44041216000.0, + "19": 44041216000.0, + "20": 44041216000.0, + "21": 44041216000.0, + "22": 44041216000.0, + "23": 44041216000.0, + "24": 44041216000.0, + "25": 44041216000.0, + "26": 44041216000.0, + "27": 44041216000.0, + "28": 44041216000.0, + "29": 44041326592.0, + "30": 44162326528.0, + "31": 44220485632.0, + "32": 44270411776.0, + "33": 44293799936.0, + "34": 44293799936.0, + "35": 44293799936.0, + "36": 44293799936.0, + "37": 44293799936.0, + "38": 44293799936.0, + "39": 44293799936.0, + "40": 44293799936.0, + "41": 44293799936.0, + "42": 44293799936.0, + "43": 44293799936.0, + "44": 44293799936.0, + "45": 44293799936.0, + "46": 44293799936.0, + "47": 44293799936.0, + "48": 44293799936.0, + "49": 44293799936.0, + "50": 44293799936.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.08617, + "2": 11.10475, + "3": 10.48001, + "4": 10.13466, + "5": 9.79047, + "6": 9.50601, + "7": 9.5113, + "8": 8.85336, + "9": 8.66683, + "10": 8.95866, + "11": 8.29315, + "12": 8.36982, + "13": 8.25544, + "14": 7.73322, + "15": 7.86639, + "16": 7.92442, + "17": 7.86278, + "18": 7.61012, + "19": 8.00269, + "20": 7.73019, + "21": 7.4165, + "22": 7.41478, + "23": 7.28671, + "24": 7.27903, + "25": 7.54456, + "26": 6.96542, + "27": 7.50538, + "28": 7.20607, + "29": 7.377, + "30": 7.52777, + "31": 7.27094, + "32": 7.4604, + "33": 7.51419, + "34": 7.56867, + "35": 7.09252, + "36": 6.96015, + "37": 7.29846, + "38": 7.0742, + "39": 7.43347, + "40": 7.43116, + "41": 7.40919, + "42": 7.15527, + "43": 7.15652, + "44": 7.30441, + "45": 7.1893, + "46": 6.77296, + "47": 7.45045, + "48": 7.02403, + "49": 7.45719, + "50": 6.92656 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 64.40054, + "2": 2.16564, + "3": 3.72378, + "4": 1.63174, + "5": 2.30947, + "6": 1.7246, + "7": 1.5089, + "8": 1.60943, + "9": 1.48606, + "10": 1.47162, + "11": 1.05608, + "12": 1.3309, + "13": 1.06824, + "14": 1.41914, + "15": 1.10033, + "16": 1.15759, + "17": 1.23897, + "18": 1.10439, + "19": 1.11869, + "20": 1.09363, + "21": 1.23622, + "22": 1.14797, + "23": 1.23037, + "24": 1.03991, + "25": 1.07795, + "26": 1.04416, + "27": 1.03654, + "28": 1.04098, + "29": 1.03502, + "30": 1.02909, + "31": 1.17935, + "32": 1.14717, + "33": 1.05403, + "34": 1.13894, + "35": 1.04538, + "36": 1.04367, + "37": 1.0843, + "38": 1.04631, + "39": 1.06131, + "40": 1.06988, + "41": 1.09756, + "42": 1.04759, + "43": 1.09649, + "44": 1.05666, + "45": 1.05249, + "46": 1.04539, + "47": 1.04041, + "48": 1.04904, + "49": 1.04777, + "50": 1.06237 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json similarity index 100% rename from tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json rename to tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml similarity index 95% rename from tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml rename to tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml index 9a125a1cf74..d9ec0456190 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml @@ -42,9 +42,9 @@ MODEL_ARGS: # Data args --seq-length: 4096 --data-cache-path: ${DATA_CACHE_PATH} - --data-path: ${DATA_PATH}/my-gpt3_00_text_document - --vocab-file: ${DATA_PATH}/bpe/vocab.json - --merge-file: ${DATA_PATH}/bpe/merges.txt + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt --split: 949,50,1 # Add network size args --num-layers: 15 diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json new file mode 100644 index 00000000000..4e979e64295 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.04266, + "2": 11.02309, + "3": 9.43552, + "4": 10.04614, + "5": 9.38535, + "6": 9.14543, + "7": 9.21141, + "8": 8.63458, + "9": 8.48937, + "10": 8.82763, + "11": 8.29457, + "12": 8.3282, + "13": 8.23008, + "14": 7.71714, + "15": 7.86981, + "16": 7.92286, + "17": 7.8604, + "18": 7.62039, + "19": 7.98493, + "20": 7.72023, + "21": 7.39758, + "22": 7.39771, + "23": 7.28314, + "24": 7.25048, + "25": 7.53113, + "26": 6.95329, + "27": 7.49432, + "28": 7.20394, + "29": 7.37282, + "30": 7.50232, + "31": 7.25348, + "32": 7.4305, + "33": 7.48364, + "34": 7.53486, + "35": 7.10336, + "36": 6.94516, + "37": 7.26117, + "38": 7.07009, + "39": 7.40543, + "40": 7.42044, + "41": 7.34202, + "42": 7.11816, + "43": 7.11373, + "44": 7.27067, + "45": 7.07036, + "46": 6.77823, + "47": 7.1875, + "48": 6.99998, + "49": 7.45868, + "50": 6.90956 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 844114112.0, + "2": 843855104.0, + "3": 844048640.0, + "4": 842998144.0, + "5": 855786112.0, + "6": 874329728.0, + "7": 925591552.0, + "8": 915644608.0, + "9": 935187584.0, + "10": 927702400.0, + "11": 957888256.0, + "12": 923872512.0, + "13": 969427072.0, + "14": 965228416.0, + "15": 952825344.0, + "16": 943777088.0, + "17": 928845824.0, + "18": 925913856.0, + "19": 955339136.0, + "20": 989208256.0, + "21": 924095424.0, + "22": 908902272.0, + "23": 892664576.0, + "24": 900830400.0, + "25": 928105472.0, + "26": 877724352.0, + "27": 912808320.0, + "28": 904557696.0, + "29": 872625088.0, + "30": 864767104.0, + "31": 868220416.0, + "32": 861931136.0, + "33": 859941312.0, + "34": 855839104.0, + "35": 854046848.0, + "36": 852944896.0, + "37": 851456704.0, + "38": 849532096.0, + "39": 849972608.0, + "40": 849505792.0, + "41": 845780288.0, + "42": 846003328.0, + "43": 846257472.0, + "44": 852034880.0, + "45": 847187456.0, + "46": 855625856.0, + "47": 844661952.0, + "48": 851197248.0, + "49": 851630464.0, + "50": 846195904.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4419107328.0, + "2": 4419108864.0, + "3": 4419108864.0, + "4": 4419108864.0, + "5": 4419108864.0, + "6": 4419108864.0, + "7": 4419108864.0, + "8": 4419108864.0, + "9": 4419108864.0, + "10": 4419108864.0, + "11": 4419108864.0, + "12": 4419108864.0, + "13": 4419108864.0, + "14": 4419108864.0, + "15": 4419108864.0, + "16": 4419108864.0, + "17": 4419108864.0, + "18": 4419108864.0, + "19": 4419108864.0, + "20": 4419108864.0, + "21": 4419108864.0, + "22": 4419108864.0, + "23": 4419108864.0, + "24": 4419108864.0, + "25": 4419108864.0, + "26": 4419108864.0, + "27": 4419108864.0, + "28": 4419108864.0, + "29": 4419108864.0, + "30": 4419108864.0, + "31": 4419108864.0, + "32": 4419108864.0, + "33": 4419108864.0, + "34": 4419108864.0, + "35": 4419108864.0, + "36": 4419108864.0, + "37": 4419108864.0, + "38": 4419108864.0, + "39": 4419108864.0, + "40": 4419108864.0, + "41": 4419108864.0, + "42": 4419108864.0, + "43": 4419108864.0, + "44": 4419108864.0, + "45": 4419108864.0, + "46": 4419108864.0, + "47": 4419108864.0, + "48": 4419108864.0, + "49": 4419108864.0, + "50": 4419108864.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 37959917568.0, + "2": 39578677248.0, + "3": 39580196864.0, + "4": 39580196864.0, + "5": 39583309824.0, + "6": 39583309824.0, + "7": 39583309824.0, + "8": 39583309824.0, + "9": 39583309824.0, + "10": 39583309824.0, + "11": 39583309824.0, + "12": 39583309824.0, + "13": 39583309824.0, + "14": 39583309824.0, + "15": 39583309824.0, + "16": 39583309824.0, + "17": 39583309824.0, + "18": 39583309824.0, + "19": 39583309824.0, + "20": 39583309824.0, + "21": 39583309824.0, + "22": 39583309824.0, + "23": 39583309824.0, + "24": 39583309824.0, + "25": 39583309824.0, + "26": 39583309824.0, + "27": 39583309824.0, + "28": 39583309824.0, + "29": 39583309824.0, + "30": 39583309824.0, + "31": 39583309824.0, + "32": 39583309824.0, + "33": 39583309824.0, + "34": 39583309824.0, + "35": 39583309824.0, + "36": 39583309824.0, + "37": 39583309824.0, + "38": 39583309824.0, + "39": 39583309824.0, + "40": 39583309824.0, + "41": 39583309824.0, + "42": 39583309824.0, + "43": 39583309824.0, + "44": 39583309824.0, + "45": 39583309824.0, + "46": 39583309824.0, + "47": 39583309824.0, + "48": 39583309824.0, + "49": 39583309824.0, + "50": 39583309824.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 60.48727, + "2": 2.0537, + "3": 3.26481, + "4": 2.56819, + "5": 2.40218, + "6": 1.26492, + "7": 1.5836, + "8": 1.37182, + "9": 1.10133, + "10": 1.10352, + "11": 1.18687, + "12": 1.53724, + "13": 1.25166, + "14": 1.69801, + "15": 1.42166, + "16": 1.104, + "17": 1.22214, + "18": 1.34911, + "19": 1.09323, + "20": 1.08552, + "21": 1.22223, + "22": 1.19712, + "23": 1.05456, + "24": 1.03745, + "25": 1.14154, + "26": 1.07349, + "27": 1.05181, + "28": 1.0364, + "29": 1.17111, + "30": 1.02943, + "31": 1.0758, + "32": 1.03304, + "33": 1.04107, + "34": 1.03092, + "35": 1.07869, + "36": 1.02457, + "37": 1.08557, + "38": 1.00729, + "39": 1.07249, + "40": 1.08655, + "41": 1.02362, + "42": 1.02046, + "43": 1.07618, + "44": 1.08709, + "45": 1.00443, + "46": 1.00379, + "47": 1.06019, + "48": 0.98958, + "49": 1.08317, + "50": 0.9932 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json similarity index 100% rename from tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json rename to tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml similarity index 94% rename from tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml rename to tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml index 8832d687004..f4b64722712 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml @@ -42,9 +42,9 @@ MODEL_ARGS: # Data args --seq-length: 4096 --data-cache-path: ${DATA_CACHE_PATH} - --data-path: ${DATA_PATH}/my-gpt3_00_text_document - --vocab-file: ${DATA_PATH}/bpe/vocab.json - --merge-file: ${DATA_PATH}/bpe/merges.txt + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt --split: 949,50,1 # Add network size args --num-layers: 15 diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json deleted file mode 100644 index 30ea509a50b..00000000000 --- a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "lm loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 11.0637, - "5": 9.48263, - "10": 9.04035, - "15": 8.00837, - "20": 7.88364, - "25": 7.67597, - "30": 7.63447, - "35": 7.21393, - "40": 7.55564, - "45": 7.21045, - "50": 7.05439 - } - }, - "num-zeros": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 38802064.0, - "5": 394456256.0, - "10": 571185472.0, - "15": 699100416.0, - "20": 891692160.0, - "25": 748799104.0, - "30": 794511296.0, - "35": 671593792.0, - "40": 421718816.0, - "45": 517934176.0, - "50": 472902496.0 - } - }, - "mem-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 6025468416.0, - "5": 6025470464.0, - "10": 6025470464.0, - "15": 6025470464.0, - "20": 6025470464.0, - "25": 6025470464.0, - "30": 6025470464.0, - "35": 6025470464.0, - "40": 6025470464.0, - "45": 6025470464.0, - "50": 6025470464.0 - } - }, - "mem-max-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 45099868160.0, - "5": 49175810048.0, - "10": 49175810048.0, - "15": 49175810048.0, - "20": 49175810048.0, - "25": 49175810048.0, - "30": 49211260928.0, - "35": 49211260928.0, - "40": 49211260928.0, - "45": 49211260928.0, - "50": 49211260928.0 - } - }, - "mtp_1 loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 11.04508, - "5": 9.76285, - "10": 9.04997, - "15": 7.93865, - "20": 7.79984, - "25": 7.60324, - "30": 7.56633, - "35": 7.13802, - "40": 7.45784, - "45": 7.11892, - "50": 6.9559 - } - }, - "iteration-time": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 52.8667, - "5": 2.06295, - "10": 1.09336, - "15": 1.10509, - "20": 1.08631, - "25": 1.08991, - "30": 1.10548, - "35": 1.10049, - "40": 1.11219, - "45": 1.09542, - "50": 1.09805 - } - } -} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json deleted file mode 100644 index 3687e19e563..00000000000 --- a/tests/functional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "lm loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 11.04266, - "5": 9.38536, - "10": 8.82761, - "15": 7.86966, - "20": 7.72022, - "25": 7.53119, - "30": 7.5026, - "35": 7.10343, - "40": 7.42037, - "45": 7.07056, - "50": 6.90946 - } - }, - "num-zeros": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 844114112.0, - "5": 856834688.0, - "10": 928751040.0, - "15": 952825152.0, - "20": 987111232.0, - "25": 926008384.0, - "30": 864767232.0, - "35": 855095360.0, - "40": 849505920.0, - "45": 847187584.0, - "50": 846195840.0 - } - }, - "mem-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 4419107328.0, - "5": 4419108864.0, - "10": 4419108864.0, - "15": 4419108864.0, - "20": 4419108864.0, - "25": 4419108864.0, - "30": 4419108864.0, - "35": 4419108864.0, - "40": 4419108864.0, - "45": 4419108864.0, - "50": 4419108864.0 - } - }, - "mem-max-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 37959917568.0, - "5": 39583289344.0, - "10": 39583289344.0, - "15": 39583289344.0, - "20": 39583289344.0, - "25": 39583289344.0, - "30": 39583289344.0, - "35": 39583289344.0, - "40": 39583289344.0, - "45": 39583289344.0, - "50": 39583289344.0 - } - }, - "iteration-time": { - "start_step": 1, - "end_step": 50, - "step_interval": 5, - "values": { - "1": 58.78709, - "5": 2.40565, - "10": 1.13046, - "15": 1.39764, - "20": 1.1273, - "25": 1.12154, - "30": 1.03587, - "35": 1.09545, - "40": 1.09901, - "45": 1.00656, - "50": 1.00794 - } - } -} From 0321e20d63bd7c3dbe846b3ad3e5fee03a85115d Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 27 Oct 2025 21:51:03 -0700 Subject: [PATCH 05/47] address comments Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offloading.md | 2 ++ megatron/core/models/gpt/gpt_model.py | 4 +++- .../fine_grained_activation_offload.py | 20 +++++++++---------- megatron/core/tensor_parallel/random.py | 4 ++++ .../core/transformer/transformer_config.py | 7 ++++++- tests/test_utils/recipes/moe.yaml | 4 ++-- ...test_fine_grained_activation_offloading.py | 2 +- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/docs/source/api-guide/fine_grained_activation_offloading.md b/docs/source/api-guide/fine_grained_activation_offloading.md index b4c2ea753fa..969098263fc 100644 --- a/docs/source/api-guide/fine_grained_activation_offloading.md +++ b/docs/source/api-guide/fine_grained_activation_offloading.md @@ -2,6 +2,8 @@ Memory capacity is more and more important with the rising of extreme sparse MoE models like DeepSeek-V3 and Qwen3-235B. Fine-grained recomputing reduces the memory footprint at the cost of extra recomputation, while offloading could utilize the host-device bandwidth to achieve nearly zero-overhead. Fine-grained Activation Offloading targets at offloading the activation at the granularity of specific modules, so that we can calibrate the amount of offloading activation to maximize the training throughput. +Currently, the supported offloading modules are `"attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"`, which could work with fine-grained recomputation to release almost all activations of a transformer layer. + **Features** * Support PP=1/PP/Interleaved PP * Compatible with fine-grained recomputation diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index ae292649561..209fdc9530d 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -417,7 +417,9 @@ def _preprocess( def preprocess_for_fine_grained_offloading(self): """Preprocess for fine-grained activation offloading.""" fine_grained_offloading_init_chunk_handler( - self.vp_stage, self.config.min_offloaded_tensor_size + vp_size=self.config.virtual_pipeline_model_parallel_size, + vp_stage=self.vp_stage, + min_offloaded_tensor_size=self.config.min_offloaded_tensor_size, ) if self.disable_param_offloading: for param in self.decoder.parameters(): diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index be2e5f56fdc..343769fa924 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -73,17 +73,10 @@ def get_instance(cls): def __init__(self): """Initialize the manager with queues and dedicated CUDA streams.""" - from megatron.core import parallel_state - # Queue to store chunk handlers for backward pass self._queue = deque() - if parallel_state.get_virtual_pipeline_model_parallel_world_size() is None: - self._vpp = 1 - else: - self._vpp = parallel_state.get_virtual_pipeline_model_parallel_world_size() - # Cache chunk handlers for each virtual pipeline stage - self._stages = [[] for _ in range(self._vpp)] + self._stages = None # allocate streams and events for synchronization self._d2h_stream = torch.cuda.Stream() self._h2d_stream = torch.cuda.Stream() @@ -150,14 +143,19 @@ def size(self): """Return the number of chunk handlers in the queue.""" return len(self._queue) - def init_model_chunk_offload_handler(self, vp_stage, min_offloaded_tensor_size=1024 * 1024): + def init_model_chunk_offload_handler(self, vp_size, vp_stage, min_offloaded_tensor_size=1024 * 1024): """ Initialize a chunk offload handler for a model chunk (microbatch). Args: + vp_size: Virtual pipeline size vp_stage: Virtual pipeline stage index (None means stage 0) min_offloaded_tensor_size: Minimum tensor size (in elements) to offload """ + if self._stages is None: + self._vpp = vp_size + self._stages = [[] for _ in range(vp_size)] + if vp_stage is None: cur_vpp_rank = 0 else: @@ -596,10 +594,10 @@ def fine_grained_offloading_set_last_layer(is_last_layer): PipelineOffloadManager.get_instance().set_last_layer(is_last_layer) -def fine_grained_offloading_init_chunk_handler(vp_stage, min_offloaded_tensor_size): +def fine_grained_offloading_init_chunk_handler(vp_size, vp_stage, min_offloaded_tensor_size): """Initialize the chunk handler, called at the start of a microbatch forward pass.""" PipelineOffloadManager.get_instance().init_model_chunk_offload_handler( - vp_stage, min_offloaded_tensor_size + vp_size, vp_stage, min_offloaded_tensor_size ) diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 2ae15bef0d9..5a44c38713d 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -510,6 +510,9 @@ def forward(ctx, run_function, checkpoint_without_output_obj, *args): @staticmethod def backward(ctx, *args): """Backward pass.""" + # Get the inputs from the context instead of the saved tensors + # because the saved tensors are already cached by the recomputation. + # This is to avoid double-reloading the inputs in CPU offloading scenario. inputs = ctx.inputs outputs = ctx.outputs torch.autograd.backward(outputs, args) @@ -574,6 +577,7 @@ def _recompute(self, _): recompute_ctx = contextlib.nullcontext() fp8_ctx = contextlib.nullcontext() + # Store the inputs for backward pass inputs = self.ctx.saved_tensors with torch.enable_grad(), fp8_ctx, recompute_ctx: outputs = self.run_function(*inputs) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index c5e984d712c..9af6a3f39d7 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -729,7 +729,10 @@ class TransformerConfig(ModelParallelConfig): # Fine-grained Activation Offloading ##################################### fine_grained_activation_offloading: bool = False - """If True, offload the input of the specified modules to the CPU.""" + """If True, offload the input of the specified modules to the CPU. + Fine-grained activation offloading is a module-level offloading method + instead of a layer-level offloading method like cpu_offloading.""" + offload_modules: Optional[list[str]] = None """The submodules to offload its input. @@ -1035,6 +1038,8 @@ def __post_init__(self): self.recompute_modules.append("moe") if self.fine_grained_activation_offloading: + assert not self.cpu_offloading, \ + "fine_grained_activation_offloading cannot be enabled with cpu_offloading." assert self.offload_modules is not None and len(self.offload_modules) > 0 allowed_modules = { "core_attn", diff --git a/tests/test_utils/recipes/moe.yaml b/tests/test_utils/recipes/moe.yaml index 87585663650..2e3427bd8fc 100644 --- a/tests/test_utils/recipes/moe.yaml +++ b/tests/test_utils/recipes/moe.yaml @@ -180,12 +180,12 @@ products: - environment: [dev] scope: [mr] platforms: [dgx_h100] - - test_case: [gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] + - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] products: - environment: [dev] scope: [mr] platforms: [dgx_h100] - - test_case: [gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] + - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] products: - environment: [dev] scope: [mr] diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index edec95288c2..7c1b7f1fe4b 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -45,7 +45,7 @@ def forward(self, x, use_offload: bool = False): # Initialize a new chunk (microbatch) and enable offload context. with off.get_fine_grained_offloading_context(True): off.fine_grained_offloading_init_chunk_handler( - vp_stage=None, min_offloaded_tensor_size=1 + vp_size=1, vp_stage=None, min_offloaded_tensor_size=1 ) for i, layer in enumerate(self.net): # Group by module; with this linear-only model, each group corresponds to a layer. From 9068805a5352b43e37486f8c2ef141239560f337 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 27 Oct 2025 21:58:19 -0700 Subject: [PATCH 06/47] format Signed-off-by: Hongbin Liu --- .../pipeline_parallel/fine_grained_activation_offload.py | 4 +++- megatron/core/transformer/transformer_config.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 343769fa924..ce9dc4402bf 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -143,7 +143,9 @@ def size(self): """Return the number of chunk handlers in the queue.""" return len(self._queue) - def init_model_chunk_offload_handler(self, vp_size, vp_stage, min_offloaded_tensor_size=1024 * 1024): + def init_model_chunk_offload_handler( + self, vp_size, vp_stage, min_offloaded_tensor_size=1024 * 1024 + ): """ Initialize a chunk offload handler for a model chunk (microbatch). diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 9af6a3f39d7..583f8f859c5 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -732,7 +732,6 @@ class TransformerConfig(ModelParallelConfig): """If True, offload the input of the specified modules to the CPU. Fine-grained activation offloading is a module-level offloading method instead of a layer-level offloading method like cpu_offloading.""" - offload_modules: Optional[list[str]] = None """The submodules to offload its input. @@ -1038,8 +1037,9 @@ def __post_init__(self): self.recompute_modules.append("moe") if self.fine_grained_activation_offloading: - assert not self.cpu_offloading, \ - "fine_grained_activation_offloading cannot be enabled with cpu_offloading." + assert ( + not self.cpu_offloading + ), "fine_grained_activation_offloading cannot be enabled with cpu_offloading." assert self.offload_modules is not None and len(self.offload_modules) > 0 allowed_modules = { "core_attn", From 61af7fde6f42f005ecaa4a76c365b8208779e552 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 28 Oct 2025 00:43:39 -0700 Subject: [PATCH 07/47] update golden values on eos Signed-off-by: Hongbin Liu --- .../golden_values_dev_eos.json | 378 ++++++++++++++---- .../golden_values_dev_eos.json | 289 ++++++++++--- 2 files changed, 548 insertions(+), 119 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json index 30ea509a50b..d7372742ca7 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json @@ -2,109 +2,343 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.0637, - "5": 9.48263, - "10": 9.04035, - "15": 8.00837, - "20": 7.88364, - "25": 7.67597, - "30": 7.63447, - "35": 7.21393, - "40": 7.55564, - "45": 7.21045, - "50": 7.05439 + "1": 11.07546, + "2": 11.03837, + "3": 9.66011, + "4": 9.91381, + "5": 9.32909, + "6": 9.13922, + "7": 9.13574, + "8": 8.65508, + "9": 8.51394, + "10": 8.8409, + "11": 8.29149, + "12": 8.34581, + "13": 8.25518, + "14": 7.73711, + "15": 7.86249, + "16": 7.9371, + "17": 7.89319, + "18": 7.63123, + "19": 7.99731, + "20": 7.74538, + "21": 7.44348, + "22": 7.42249, + "23": 7.29714, + "24": 7.27462, + "25": 7.54574, + "26": 6.96838, + "27": 7.50556, + "28": 7.22743, + "29": 7.36588, + "30": 7.52622, + "31": 7.27026, + "32": 7.45521, + "33": 7.50954, + "34": 7.55686, + "35": 7.10177, + "36": 6.96431, + "37": 7.28463, + "38": 7.0808, + "39": 7.40923, + "40": 7.43338, + "41": 7.38496, + "42": 7.15749, + "43": 7.15858, + "44": 7.28852, + "45": 7.16793, + "46": 6.78468, + "47": 7.4114, + "48": 7.0027, + "49": 7.46249, + "50": 6.92151 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 38802064.0, - "5": 394456256.0, - "10": 571185472.0, - "15": 699100416.0, - "20": 891692160.0, - "25": 748799104.0, - "30": 794511296.0, - "35": 671593792.0, - "40": 421718816.0, - "45": 517934176.0, - "50": 472902496.0 + "1": 911219392.0, + "2": 910960384.0, + "3": 911156352.0, + "4": 912204800.0, + "5": 920796544.0, + "6": 940387968.0, + "7": 990599872.0, + "8": 976457728.0, + "9": 998097664.0, + "10": 995852672.0, + "11": 994583680.0, + "12": 977344896.0, + "13": 1028141824.0, + "14": 1007166208.0, + "15": 987423616.0, + "16": 993054784.0, + "17": 982319168.0, + "18": 998261760.0, + "19": 984696320.0, + "20": 982914752.0, + "21": 979667456.0, + "22": 953988864.0, + "23": 972353984.0, + "24": 964792064.0, + "25": 958512192.0, + "26": 946928512.0, + "27": 948458304.0, + "28": 949643968.0, + "29": 942877440.0, + "30": 935020160.0, + "31": 935327616.0, + "32": 934281088.0, + "33": 921805568.0, + "34": 928189312.0, + "35": 922202496.0, + "36": 924246656.0, + "37": 920661248.0, + "38": 922930752.0, + "39": 922322816.0, + "40": 921856512.0, + "41": 920227968.0, + "42": 918353664.0, + "43": 918607040.0, + "44": 914948032.0, + "45": 914295232.0, + "46": 914344448.0, + "47": 911769536.0, + "48": 912013312.0, + "49": 910349440.0, + "50": 914351552.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 6025468416.0, - "5": 6025470464.0, - "10": 6025470464.0, - "15": 6025470464.0, - "20": 6025470464.0, - "25": 6025470464.0, - "30": 6025470464.0, - "35": 6025470464.0, - "40": 6025470464.0, - "45": 6025470464.0, - "50": 6025470464.0 + "1": 5498353152.0, + "2": 5499147776.0, + "3": 5499940352.0, + "4": 5500732928.0, + "5": 5501525504.0, + "6": 5502318080.0, + "7": 5503110656.0, + "8": 5503903232.0, + "9": 5497958912.0, + "10": 5498751488.0, + "11": 5499544064.0, + "12": 5500336640.0, + "13": 5501129216.0, + "14": 5501921792.0, + "15": 5502714368.0, + "16": 5503506944.0, + "17": 5504299520.0, + "18": 5505092096.0, + "19": 5505884672.0, + "20": 5506677248.0, + "21": 5507469824.0, + "22": 5508262400.0, + "23": 5509054976.0, + "24": 5509847552.0, + "25": 5510640128.0, + "26": 5511432704.0, + "27": 5512225280.0, + "28": 5513017856.0, + "29": 5513810432.0, + "30": 5514603008.0, + "31": 5515395584.0, + "32": 5516188160.0, + "33": 5516980736.0, + "34": 5517773312.0, + "35": 5518565888.0, + "36": 5519358464.0, + "37": 5520151040.0, + "38": 5520943616.0, + "39": 5521736192.0, + "40": 5522528768.0, + "41": 5523321344.0, + "42": 5524113920.0, + "43": 5524906496.0, + "44": 5525699072.0, + "45": 5526491648.0, + "46": 5527284224.0, + "47": 5528076800.0, + "48": 5528869376.0, + "49": 5529661952.0, + "50": 5530454528.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 45099868160.0, - "5": 49175810048.0, - "10": 49175810048.0, - "15": 49175810048.0, - "20": 49175810048.0, - "25": 49175810048.0, - "30": 49211260928.0, - "35": 49211260928.0, - "40": 49211260928.0, - "45": 49211260928.0, - "50": 49211260928.0 + "1": 41739952128.0, + "2": 43687571456.0, + "3": 43687571456.0, + "4": 43983216640.0, + "5": 43983216640.0, + "6": 43983216640.0, + "7": 43983216640.0, + "8": 44024635392.0, + "9": 44041216000.0, + "10": 44041216000.0, + "11": 44041216000.0, + "12": 44041216000.0, + "13": 44041216000.0, + "14": 44041216000.0, + "15": 44041216000.0, + "16": 44041216000.0, + "17": 44041216000.0, + "18": 44041216000.0, + "19": 44041216000.0, + "20": 44041216000.0, + "21": 44041216000.0, + "22": 44041216000.0, + "23": 44041216000.0, + "24": 44041216000.0, + "25": 44041216000.0, + "26": 44041216000.0, + "27": 44041216000.0, + "28": 44041216000.0, + "29": 44041326592.0, + "30": 44162326528.0, + "31": 44220485632.0, + "32": 44270411776.0, + "33": 44293799936.0, + "34": 44293799936.0, + "35": 44293799936.0, + "36": 44293799936.0, + "37": 44293799936.0, + "38": 44293799936.0, + "39": 44293799936.0, + "40": 44293799936.0, + "41": 44293799936.0, + "42": 44293799936.0, + "43": 44293799936.0, + "44": 44293799936.0, + "45": 44293799936.0, + "46": 44293799936.0, + "47": 44293799936.0, + "48": 44293799936.0, + "49": 44293799936.0, + "50": 44293799936.0 } }, "mtp_1 loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.04508, - "5": 9.76285, - "10": 9.04997, - "15": 7.93865, - "20": 7.79984, - "25": 7.60324, - "30": 7.56633, - "35": 7.13802, - "40": 7.45784, - "45": 7.11892, - "50": 6.9559 + "1": 11.08617, + "2": 11.10475, + "3": 10.48001, + "4": 10.13466, + "5": 9.79047, + "6": 9.50601, + "7": 9.5113, + "8": 8.85336, + "9": 8.66683, + "10": 8.95866, + "11": 8.29315, + "12": 8.36982, + "13": 8.25544, + "14": 7.73322, + "15": 7.86639, + "16": 7.92442, + "17": 7.86278, + "18": 7.61012, + "19": 8.00269, + "20": 7.73019, + "21": 7.4165, + "22": 7.41478, + "23": 7.28671, + "24": 7.27903, + "25": 7.54456, + "26": 6.96542, + "27": 7.50538, + "28": 7.20607, + "29": 7.377, + "30": 7.52777, + "31": 7.27094, + "32": 7.4604, + "33": 7.51419, + "34": 7.56867, + "35": 7.09252, + "36": 6.96015, + "37": 7.29846, + "38": 7.0742, + "39": 7.43347, + "40": 7.43116, + "41": 7.40919, + "42": 7.15527, + "43": 7.15652, + "44": 7.30441, + "45": 7.1893, + "46": 6.77296, + "47": 7.45045, + "48": 7.02403, + "49": 7.45719, + "50": 6.92656 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 52.8667, - "5": 2.06295, - "10": 1.09336, - "15": 1.10509, - "20": 1.08631, - "25": 1.08991, - "30": 1.10548, - "35": 1.10049, - "40": 1.11219, - "45": 1.09542, - "50": 1.09805 + "1": 87.63934, + "2": 1.98402, + "3": 3.95877, + "4": 1.64812, + "5": 2.312, + "6": 2.02902, + "7": 1.56333, + "8": 1.66703, + "9": 1.6393, + "10": 1.40472, + "11": 1.086, + "12": 1.34921, + "13": 1.0854, + "14": 1.4242, + "15": 1.09539, + "16": 1.79766, + "17": 1.2562, + "18": 1.08887, + "19": 1.08371, + "20": 1.10071, + "21": 1.25979, + "22": 1.3212, + "23": 1.25044, + "24": 1.05384, + "25": 1.11356, + "26": 1.0605, + "27": 1.03418, + "28": 1.0405, + "29": 1.05174, + "30": 1.04166, + "31": 1.20036, + "32": 1.12936, + "33": 1.02917, + "34": 1.13473, + "35": 1.02829, + "36": 1.04352, + "37": 1.0843, + "38": 1.03714, + "39": 1.04534, + "40": 1.07031, + "41": 1.07618, + "42": 1.03008, + "43": 1.06043, + "44": 1.04049, + "45": 1.02875, + "46": 1.03669, + "47": 1.03128, + "48": 1.02808, + "49": 1.03038, + "50": 1.04621 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json index 3687e19e563..537e20b09d8 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json @@ -2,91 +2,286 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { "1": 11.04266, - "5": 9.38536, - "10": 8.82761, - "15": 7.86966, - "20": 7.72022, - "25": 7.53119, - "30": 7.5026, - "35": 7.10343, - "40": 7.42037, - "45": 7.07056, - "50": 6.90946 + "2": 11.02309, + "3": 9.43552, + "4": 10.04614, + "5": 9.38535, + "6": 9.14543, + "7": 9.21141, + "8": 8.63458, + "9": 8.48937, + "10": 8.82763, + "11": 8.29457, + "12": 8.3282, + "13": 8.23008, + "14": 7.71714, + "15": 7.86981, + "16": 7.92286, + "17": 7.8604, + "18": 7.62039, + "19": 7.98493, + "20": 7.72023, + "21": 7.39758, + "22": 7.39771, + "23": 7.28314, + "24": 7.25048, + "25": 7.53113, + "26": 6.95329, + "27": 7.49432, + "28": 7.20394, + "29": 7.37282, + "30": 7.50232, + "31": 7.25348, + "32": 7.4305, + "33": 7.48364, + "34": 7.53486, + "35": 7.10336, + "36": 6.94516, + "37": 7.26117, + "38": 7.07009, + "39": 7.40543, + "40": 7.42044, + "41": 7.34202, + "42": 7.11816, + "43": 7.11373, + "44": 7.27067, + "45": 7.07036, + "46": 6.77823, + "47": 7.1875, + "48": 6.99998, + "49": 7.45868, + "50": 6.90956 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { "1": 844114112.0, - "5": 856834688.0, - "10": 928751040.0, - "15": 952825152.0, - "20": 987111232.0, - "25": 926008384.0, - "30": 864767232.0, - "35": 855095360.0, - "40": 849505920.0, - "45": 847187584.0, - "50": 846195840.0 + "2": 843855104.0, + "3": 844048640.0, + "4": 842998144.0, + "5": 855786112.0, + "6": 874329728.0, + "7": 925591552.0, + "8": 915644608.0, + "9": 935187584.0, + "10": 927702400.0, + "11": 957888256.0, + "12": 923872512.0, + "13": 969427072.0, + "14": 965228416.0, + "15": 952825344.0, + "16": 943777088.0, + "17": 928845824.0, + "18": 925913856.0, + "19": 955339136.0, + "20": 989208256.0, + "21": 924095424.0, + "22": 908902272.0, + "23": 892664576.0, + "24": 900830400.0, + "25": 928105472.0, + "26": 877724352.0, + "27": 912808320.0, + "28": 904557696.0, + "29": 872625088.0, + "30": 864767104.0, + "31": 868220416.0, + "32": 861931136.0, + "33": 859941312.0, + "34": 855839104.0, + "35": 854046848.0, + "36": 852944896.0, + "37": 851456704.0, + "38": 849532096.0, + "39": 849972608.0, + "40": 849505792.0, + "41": 845780288.0, + "42": 846003328.0, + "43": 846257472.0, + "44": 852034880.0, + "45": 847187456.0, + "46": 855625856.0, + "47": 844661952.0, + "48": 851197248.0, + "49": 851630464.0, + "50": 846195904.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { "1": 4419107328.0, + "2": 4419108864.0, + "3": 4419108864.0, + "4": 4419108864.0, "5": 4419108864.0, + "6": 4419108864.0, + "7": 4419108864.0, + "8": 4419108864.0, + "9": 4419108864.0, "10": 4419108864.0, + "11": 4419108864.0, + "12": 4419108864.0, + "13": 4419108864.0, + "14": 4419108864.0, "15": 4419108864.0, + "16": 4419108864.0, + "17": 4419108864.0, + "18": 4419108864.0, + "19": 4419108864.0, "20": 4419108864.0, + "21": 4419108864.0, + "22": 4419108864.0, + "23": 4419108864.0, + "24": 4419108864.0, "25": 4419108864.0, + "26": 4419108864.0, + "27": 4419108864.0, + "28": 4419108864.0, + "29": 4419108864.0, "30": 4419108864.0, + "31": 4419108864.0, + "32": 4419108864.0, + "33": 4419108864.0, + "34": 4419108864.0, "35": 4419108864.0, + "36": 4419108864.0, + "37": 4419108864.0, + "38": 4419108864.0, + "39": 4419108864.0, "40": 4419108864.0, + "41": 4419108864.0, + "42": 4419108864.0, + "43": 4419108864.0, + "44": 4419108864.0, "45": 4419108864.0, + "46": 4419108864.0, + "47": 4419108864.0, + "48": 4419108864.0, + "49": 4419108864.0, "50": 4419108864.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { "1": 37959917568.0, - "5": 39583289344.0, - "10": 39583289344.0, - "15": 39583289344.0, - "20": 39583289344.0, - "25": 39583289344.0, - "30": 39583289344.0, - "35": 39583289344.0, - "40": 39583289344.0, - "45": 39583289344.0, - "50": 39583289344.0 + "2": 39578677248.0, + "3": 39580196864.0, + "4": 39580196864.0, + "5": 39583309824.0, + "6": 39583309824.0, + "7": 39583309824.0, + "8": 39583309824.0, + "9": 39583309824.0, + "10": 39583309824.0, + "11": 39583309824.0, + "12": 39583309824.0, + "13": 39583309824.0, + "14": 39583309824.0, + "15": 39583309824.0, + "16": 39583309824.0, + "17": 39583309824.0, + "18": 39583309824.0, + "19": 39583309824.0, + "20": 39583309824.0, + "21": 39583309824.0, + "22": 39583309824.0, + "23": 39583309824.0, + "24": 39583309824.0, + "25": 39583309824.0, + "26": 39583309824.0, + "27": 39583309824.0, + "28": 39583309824.0, + "29": 39583309824.0, + "30": 39583309824.0, + "31": 39583309824.0, + "32": 39583309824.0, + "33": 39583309824.0, + "34": 39583309824.0, + "35": 39583309824.0, + "36": 39583309824.0, + "37": 39583309824.0, + "38": 39583309824.0, + "39": 39583309824.0, + "40": 39583309824.0, + "41": 39583309824.0, + "42": 39583309824.0, + "43": 39583309824.0, + "44": 39583309824.0, + "45": 39583309824.0, + "46": 39583309824.0, + "47": 39583309824.0, + "48": 39583309824.0, + "49": 39583309824.0, + "50": 39583309824.0 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 58.78709, - "5": 2.40565, - "10": 1.13046, - "15": 1.39764, - "20": 1.1273, - "25": 1.12154, - "30": 1.03587, - "35": 1.09545, - "40": 1.09901, - "45": 1.00656, - "50": 1.00794 + "1": 67.13422, + "2": 1.95457, + "3": 3.25371, + "4": 2.66673, + "5": 3.05794, + "6": 1.35128, + "7": 1.66174, + "8": 2.19011, + "9": 1.16207, + "10": 1.16456, + "11": 1.26279, + "12": 1.60263, + "13": 1.29219, + "14": 2.93489, + "15": 1.48729, + "16": 1.15146, + "17": 1.27648, + "18": 1.39906, + "19": 1.13846, + "20": 1.14415, + "21": 1.27567, + "22": 1.26287, + "23": 1.11223, + "24": 1.10986, + "25": 1.20096, + "26": 1.13382, + "27": 1.11305, + "28": 1.11424, + "29": 1.22341, + "30": 1.08856, + "31": 1.15539, + "32": 1.10684, + "33": 1.11399, + "34": 1.09048, + "35": 1.1509, + "36": 1.09151, + "37": 1.13904, + "38": 1.06658, + "39": 1.1325, + "40": 1.14715, + "41": 1.07533, + "42": 1.08243, + "43": 1.13881, + "44": 1.14004, + "45": 1.06323, + "46": 1.06103, + "47": 1.11785, + "48": 1.04242, + "49": 1.13933, + "50": 1.0407 } } -} +} \ No newline at end of file From b67a0ed46e01649a4695d44404b5dca12f03b6fd Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 28 Oct 2025 05:57:40 -0700 Subject: [PATCH 08/47] minor fix Signed-off-by: Hongbin Liu --- .../pipeline_parallel/fine_grained_activation_offload.py | 1 + megatron/core/transformer/attention.py | 4 ++-- megatron/core/transformer/transformer_config.py | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index ce9dc4402bf..1e280a09d35 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -155,6 +155,7 @@ def init_model_chunk_offload_handler( min_offloaded_tensor_size: Minimum tensor size (in elements) to offload """ if self._stages is None: + vp_size = 1 if vp_size is None else vp_size self._vpp = vp_size self._stages = [[] for _ in range(vp_size)] diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 97ff68d026c..3707351d7b3 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -752,8 +752,8 @@ def forward( hidden_states, key_value_states, split_qkv=split_qkv ) if self.offload_qkv_linear: - qkv_output, _ = fine_grained_offloading_group_commit( - qkv_output, name="qkv_linear", forced_released_tensors=[hidden_states] + (qkv_output,) = fine_grained_offloading_group_commit( + qkv_output, name="qkv_linear", forced_released_tensors=[] ) attn_mask_type = self.attn_mask_type block_table = None diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 583f8f859c5..c36e5b546be 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -735,11 +735,12 @@ class TransformerConfig(ModelParallelConfig): offload_modules: Optional[list[str]] = None """The submodules to offload its input. - choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". + choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". "attn_norm": offload the input of the normalization in the attention part. + "qkv_linear": offload the input of the qkv linear part. "core_attn": offload the input of the core attention part. - "mlp_norm": offload the input of the normalization in the mlp part. "attn_proj": offload the input of the attn linear projection part. + "mlp_norm": offload the input of the normalization in the mlp part. "expert_fc1": offload the input of the expert fc1 part. "moe_act": offload the input of the moe act part. """ @@ -1048,6 +1049,7 @@ def __post_init__(self): "moe_act", "attn_norm", "mlp_norm", + "qkv_linear", } invalid_modules = set(self.offload_modules) - allowed_modules assert not invalid_modules, ( From a11d35121c1ab2cba4fcee5a4262d8ce8ce26a9c Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 28 Oct 2025 06:14:48 -0700 Subject: [PATCH 09/47] format Signed-off-by: Hongbin Liu --- megatron/core/transformer/transformer_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index c36e5b546be..5c6ac386374 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -735,7 +735,8 @@ class TransformerConfig(ModelParallelConfig): offload_modules: Optional[list[str]] = None """The submodules to offload its input. - choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". + choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", + "mlp_norm", "expert_fc1", "moe_act". "attn_norm": offload the input of the normalization in the attention part. "qkv_linear": offload the input of the qkv linear part. "core_attn": offload the input of the core attention part. From b84b5ff434028e8f035e79ee37e700fdb8c9e12c Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 29 Oct 2025 02:46:14 -0700 Subject: [PATCH 10/47] renaming golden values Signed-off-by: Hongbin Liu --- ...ev_coreweave.json => golden_values_dev_dgxh100_coreweave.json} | 0 ...den_values_dev_eos.json => golden_values_dev_dgxh100_eos.json} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/{golden_values_dev_coreweave.json => golden_values_dev_dgxh100_coreweave.json} (100%) rename tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/{golden_values_dev_eos.json => golden_values_dev_dgxh100_eos.json} (100%) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_coreweave.json similarity index 100% rename from tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_coreweave.json rename to tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_coreweave.json diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_eos.json similarity index 100% rename from tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_eos.json rename to tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_eos.json From c320e5b819f852b8025a5b4ad1847ca44de1b0f8 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 4 Nov 2025 00:30:05 -0800 Subject: [PATCH 11/47] fix bug: accuracy issu because of recomputing and offloading same modul Signed-off-by: Hongbin Liu --- megatron/core/tensor_parallel/random.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 5a44c38713d..69e973142df 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -579,6 +579,14 @@ def _recompute(self, _): # Store the inputs for backward pass inputs = self.ctx.saved_tensors + def detach(t): + if isinstance(t, torch.Tensor): + requires_grad = t.requires_grad + t = t.detach() + t.requires_grad_(requires_grad) + return t + + inputs = tuple(detach(t) for t in inputs) with torch.enable_grad(), fp8_ctx, recompute_ctx: outputs = self.run_function(*inputs) From aa4c86cfda88b3d862960435591c98548424e7c6 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 4 Nov 2025 00:33:06 -0800 Subject: [PATCH 12/47] format Signed-off-by: Hongbin Liu --- megatron/core/tensor_parallel/random.py | 1 + 1 file changed, 1 insertion(+) diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 69e973142df..396e5c54a2d 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -579,6 +579,7 @@ def _recompute(self, _): # Store the inputs for backward pass inputs = self.ctx.saved_tensors + def detach(t): if isinstance(t, torch.Tensor): requires_grad = t.requires_grad From 57878484b79b8fae9ef6d3b7db5084f7ddf1909b Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 4 Nov 2025 19:49:47 -0800 Subject: [PATCH 13/47] update golden values Signed-off-by: Hongbin Liu --- .../golden_values_dev_dgx_h100.json | 344 ++++++++++++++++++ .../golden_values_dev_dgx_h100.json | 287 +++++++++++++++ 2 files changed, 631 insertions(+) create mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json create mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..4b32d4256db --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -0,0 +1,344 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.07559, + "2": 11.03834, + "3": 9.72869, + "4": 9.61678, + "5": 10.63323, + "6": 9.1681, + "7": 9.35196, + "8": 9.05204, + "9": 8.84148, + "10": 9.00321, + "11": 8.49799, + "12": 8.5218, + "13": 8.41649, + "14": 7.9096, + "15": 8.00627, + "16": 8.05394, + "17": 8.0203, + "18": 7.73136, + "19": 8.11676, + "20": 7.83945, + "21": 7.52196, + "22": 7.5295, + "23": 7.38729, + "24": 7.3758, + "25": 7.65255, + "26": 7.04795, + "27": 7.591, + "28": 7.30023, + "29": 7.45656, + "30": 7.60935, + "31": 7.3713, + "32": 7.55298, + "33": 7.59738, + "34": 7.65764, + "35": 7.17916, + "36": 7.04913, + "37": 7.38022, + "38": 7.14883, + "39": 7.50321, + "40": 7.51595, + "41": 7.45139, + "42": 7.21197, + "43": 7.21131, + "44": 7.38058, + "45": 7.16397, + "46": 6.86108, + "47": 7.27247, + "48": 7.10862, + "49": 7.56398, + "50": 7.00523 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 38802120.0, + "2": 38543052.0, + "3": 38738396.0, + "4": 113220144.0, + "5": 344100160.0, + "6": 435062816.0, + "7": 579598912.0, + "8": 819195200.0, + "9": 604910464.0, + "10": 690749824.0, + "11": 744002496.0, + "12": 520212192.0, + "13": 547932992.0, + "14": 585659584.0, + "15": 614149184.0, + "16": 664915328.0, + "17": 592272320.0, + "18": 630225856.0, + "19": 579959808.0, + "20": 800470080.0, + "21": 573941056.0, + "22": 557652032.0, + "23": 797256640.0, + "24": 826380864.0, + "25": 814860160.0, + "26": 617708032.0, + "27": 715680384.0, + "28": 548045824.0, + "29": 736312064.0, + "30": 722163456.0, + "31": 711986176.0, + "32": 674238208.0, + "33": 715239232.0, + "34": 677588288.0, + "35": 473423392.0, + "36": 451352800.0, + "37": 446739392.0, + "38": 567466304.0, + "39": 472519552.0, + "40": 434322048.0, + "41": 554276096.0, + "42": 526187424.0, + "43": 510713152.0, + "44": 522783808.0, + "45": 335511072.0, + "46": 450878784.0, + "47": 450397344.0, + "48": 321720704.0, + "49": 437443680.0, + "50": 419425088.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5498340864.0, + "2": 5499135488.0, + "3": 5499928064.0, + "4": 5500720640.0, + "5": 5501513216.0, + "6": 5502305792.0, + "7": 5497946624.0, + "8": 5498739200.0, + "9": 5499531776.0, + "10": 5500324352.0, + "11": 5501116928.0, + "12": 5498342912.0, + "13": 5499135488.0, + "14": 5499928064.0, + "15": 5500720640.0, + "16": 5501513216.0, + "17": 5502305792.0, + "18": 5503098368.0, + "19": 5503890944.0, + "20": 5504683520.0, + "21": 5505476096.0, + "22": 5506268672.0, + "23": 5507061248.0, + "24": 5507853824.0, + "25": 5508646400.0, + "26": 5509438976.0, + "27": 5510231552.0, + "28": 5511024128.0, + "29": 5511816704.0, + "30": 5512609280.0, + "31": 5513401856.0, + "32": 5514194432.0, + "33": 5514987008.0, + "34": 5515779584.0, + "35": 5516572160.0, + "36": 5517364736.0, + "37": 5518157312.0, + "38": 5518949888.0, + "39": 5519742464.0, + "40": 5520535040.0, + "41": 5521327616.0, + "42": 5522120192.0, + "43": 5522912768.0, + "44": 5523705344.0, + "45": 5524497920.0, + "46": 5525290496.0, + "47": 5526083072.0, + "48": 5526875648.0, + "49": 5527668224.0, + "50": 5528460800.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 41723441152.0, + "2": 43687280640.0, + "3": 43916578816.0, + "4": 43916578816.0, + "5": 43916578816.0, + "6": 43916578816.0, + "7": 43916578816.0, + "8": 43916578816.0, + "9": 43916578816.0, + "10": 43916578816.0, + "11": 43916578816.0, + "12": 44028436480.0, + "13": 44028436480.0, + "14": 44028436480.0, + "15": 44028436480.0, + "16": 44028436480.0, + "17": 44028436480.0, + "18": 44028436480.0, + "19": 44028436480.0, + "20": 44028436480.0, + "21": 44028436480.0, + "22": 44028436480.0, + "23": 44028436480.0, + "24": 44028436480.0, + "25": 44028436480.0, + "26": 44028436480.0, + "27": 44028436480.0, + "28": 44028436480.0, + "29": 44028436480.0, + "30": 44028436480.0, + "31": 44028436480.0, + "32": 44028436480.0, + "33": 44028436480.0, + "34": 44028436480.0, + "35": 44028436480.0, + "36": 44028436480.0, + "37": 44028436480.0, + "38": 44028436480.0, + "39": 44028436480.0, + "40": 44028436480.0, + "41": 44028436480.0, + "42": 44028436480.0, + "43": 44028436480.0, + "44": 44028436480.0, + "45": 44028436480.0, + "46": 44028436480.0, + "47": 44028436480.0, + "48": 44028436480.0, + "49": 44028436480.0, + "50": 44028436480.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.08623, + "2": 11.1047, + "3": 10.54469, + "4": 10.08474, + "5": 9.76549, + "6": 9.56242, + "7": 9.59473, + "8": 8.97686, + "9": 8.83293, + "10": 9.1193, + "11": 8.44318, + "12": 8.49593, + "13": 8.37985, + "14": 7.81516, + "15": 7.95146, + "16": 8.01718, + "17": 7.94503, + "18": 7.68603, + "19": 8.07501, + "20": 7.79558, + "21": 7.46867, + "22": 7.46603, + "23": 7.32734, + "24": 7.32819, + "25": 7.58465, + "26": 6.99257, + "27": 7.53486, + "28": 7.23432, + "29": 7.40501, + "30": 7.55005, + "31": 7.30085, + "32": 7.48028, + "33": 7.53593, + "34": 7.60112, + "35": 7.12344, + "36": 6.99007, + "37": 7.32578, + "38": 7.09623, + "39": 7.45759, + "40": 7.45018, + "41": 7.40101, + "42": 7.14459, + "43": 7.13995, + "44": 7.32066, + "45": 7.0966, + "46": 6.80106, + "47": 7.21219, + "48": 7.05021, + "49": 7.48165, + "50": 6.95118 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 87.66203, + "2": 2.04189, + "3": 3.34278, + "4": 3.72414, + "5": 3.23492, + "6": 1.94546, + "7": 2.14942, + "8": 1.78075, + "9": 1.06029, + "10": 2.13554, + "11": 1.42578, + "12": 1.80986, + "13": 1.06134, + "14": 1.087, + "15": 1.16687, + "16": 1.20412, + "17": 1.06984, + "18": 1.07557, + "19": 1.04081, + "20": 1.21763, + "21": 1.06196, + "22": 1.14038, + "23": 2.25761, + "24": 1.09161, + "25": 1.04319, + "26": 1.40025, + "27": 1.04974, + "28": 1.03984, + "29": 1.05293, + "30": 1.48942, + "31": 1.04785, + "32": 1.0529, + "33": 1.04366, + "34": 1.0633, + "35": 1.0713, + "36": 1.05711, + "37": 1.08085, + "38": 1.07006, + "39": 1.06498, + "40": 1.05913, + "41": 1.0697, + "42": 1.079, + "43": 1.14122, + "44": 1.06478, + "45": 1.04692, + "46": 1.08174, + "47": 1.07595, + "48": 1.10523, + "49": 1.0839, + "50": 1.07754 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..f3ef4646971 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.04276, + "2": 11.02298, + "3": 9.50921, + "4": 10.86244, + "5": 9.36127, + "6": 9.05636, + "7": 9.20064, + "8": 8.98909, + "9": 8.67001, + "10": 9.00892, + "11": 8.50716, + "12": 8.45579, + "13": 8.41197, + "14": 7.92802, + "15": 7.99663, + "16": 8.04156, + "17": 8.06453, + "18": 7.73746, + "19": 8.09946, + "20": 7.85555, + "21": 7.54063, + "22": 7.51142, + "23": 7.39766, + "24": 7.36551, + "25": 7.63399, + "26": 7.04934, + "27": 7.60084, + "28": 7.30223, + "29": 7.47164, + "30": 7.61428, + "31": 7.34981, + "32": 7.53935, + "33": 7.59164, + "34": 7.64951, + "35": 7.18657, + "36": 7.03804, + "37": 7.36778, + "38": 7.14613, + "39": 7.50644, + "40": 7.51103, + "41": 7.44582, + "42": 7.20666, + "43": 7.2123, + "44": 7.37723, + "45": 7.17293, + "46": 6.86188, + "47": 7.2648, + "48": 7.1069, + "49": 7.56115, + "50": 7.00113 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 38808176.0, + "2": 38549232.0, + "3": 38741780.0, + "4": 78604016.0, + "5": 152229680.0, + "6": 299762016.0, + "7": 557587712.0, + "8": 589584384.0, + "9": 482229120.0, + "10": 517739584.0, + "11": 526962624.0, + "12": 476182528.0, + "13": 667453056.0, + "14": 563635200.0, + "15": 592125760.0, + "16": 589362048.0, + "17": 453879424.0, + "18": 444631456.0, + "19": 532791520.0, + "20": 677797248.0, + "21": 545577920.0, + "22": 494731040.0, + "23": 551928576.0, + "24": 489800928.0, + "25": 644993344.0, + "26": 441532864.0, + "27": 467175040.0, + "28": 431687840.0, + "29": 409185824.0, + "30": 583756032.0, + "31": 592451072.0, + "32": 416290048.0, + "33": 391230880.0, + "34": 325273120.0, + "35": 350756576.0, + "36": 331801376.0, + "37": 349196160.0, + "38": 312664800.0, + "39": 419015584.0, + "40": 299035872.0, + "41": 274307296.0, + "42": 296551584.0, + "43": 381740640.0, + "44": 308872480.0, + "45": 263141648.0, + "46": 353360864.0, + "47": 271093472.0, + "48": 346833600.0, + "49": 267589936.0, + "50": 252702768.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4419107328.0, + "2": 4419108864.0, + "3": 4419108864.0, + "4": 4419108864.0, + "5": 4419108864.0, + "6": 4419108864.0, + "7": 4419108864.0, + "8": 4419108864.0, + "9": 4419108864.0, + "10": 4419108864.0, + "11": 4419108864.0, + "12": 4419108864.0, + "13": 4419108864.0, + "14": 4419108864.0, + "15": 4419108864.0, + "16": 4419108864.0, + "17": 4419108864.0, + "18": 4419108864.0, + "19": 4419108864.0, + "20": 4419108864.0, + "21": 4419108864.0, + "22": 4419108864.0, + "23": 4419108864.0, + "24": 4419108864.0, + "25": 4419108864.0, + "26": 4419108864.0, + "27": 4419108864.0, + "28": 4419108864.0, + "29": 4419108864.0, + "30": 4419108864.0, + "31": 4419108864.0, + "32": 4419108864.0, + "33": 4419108864.0, + "34": 4419108864.0, + "35": 4419108864.0, + "36": 4419108864.0, + "37": 4419108864.0, + "38": 4419108864.0, + "39": 4419108864.0, + "40": 4419108864.0, + "41": 4419108864.0, + "42": 4419108864.0, + "43": 4419108864.0, + "44": 4419108864.0, + "45": 4419108864.0, + "46": 4419108864.0, + "47": 4419108864.0, + "48": 4419108864.0, + "49": 4419108864.0, + "50": 4419108864.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 37959917568.0, + "2": 39578673152.0, + "3": 39583825920.0, + "4": 39583825920.0, + "5": 39586181120.0, + "6": 39586181120.0, + "7": 39586181120.0, + "8": 39586181120.0, + "9": 39586181120.0, + "10": 39586181120.0, + "11": 39586181120.0, + "12": 39586181120.0, + "13": 39586181120.0, + "14": 39586181120.0, + "15": 39586181120.0, + "16": 39586181120.0, + "17": 39586181120.0, + "18": 39586181120.0, + "19": 39586181120.0, + "20": 39586181120.0, + "21": 39586181120.0, + "22": 39586181120.0, + "23": 39586181120.0, + "24": 39586181120.0, + "25": 39586181120.0, + "26": 39586181120.0, + "27": 39586181120.0, + "28": 39586181120.0, + "29": 39586181120.0, + "30": 39586181120.0, + "31": 39586181120.0, + "32": 39586181120.0, + "33": 39586181120.0, + "34": 39586181120.0, + "35": 39586181120.0, + "36": 39586181120.0, + "37": 39586181120.0, + "38": 39586181120.0, + "39": 39586181120.0, + "40": 39586181120.0, + "41": 39586181120.0, + "42": 39586181120.0, + "43": 39586181120.0, + "44": 39586181120.0, + "45": 39586181120.0, + "46": 39586181120.0, + "47": 39586181120.0, + "48": 39586181120.0, + "49": 39586181120.0, + "50": 39586181120.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 65.48328, + "2": 1.94615, + "3": 3.94539, + "4": 2.42699, + "5": 1.80319, + "6": 1.79395, + "7": 1.50546, + "8": 2.00251, + "9": 1.2172, + "10": 1.31071, + "11": 1.3171, + "12": 1.10351, + "13": 1.26314, + "14": 1.47608, + "15": 1.19001, + "16": 1.12949, + "17": 1.15105, + "18": 1.06698, + "19": 1.10069, + "20": 1.12463, + "21": 1.35075, + "22": 1.56258, + "23": 1.2368, + "24": 1.13707, + "25": 1.11826, + "26": 1.09445, + "27": 1.08857, + "28": 1.07964, + "29": 1.08505, + "30": 1.24068, + "31": 1.10419, + "32": 1.5164, + "33": 1.10245, + "34": 1.37977, + "35": 1.1165, + "36": 1.1457, + "37": 1.10487, + "38": 1.08491, + "39": 1.08901, + "40": 1.08968, + "41": 1.13702, + "42": 1.09805, + "43": 1.06669, + "44": 1.07791, + "45": 1.08898, + "46": 1.10717, + "47": 1.13008, + "48": 1.05745, + "49": 1.08268, + "50": 1.05678 + } + } +} \ No newline at end of file From 7eaec4d9ffe283a92ec4029799923be628c39b0e Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 5 Nov 2025 01:04:35 -0800 Subject: [PATCH 14/47] update golden values Signed-off-by: Hongbin Liu --- .../golden_values_dev_dgx_h100.json | 392 +++++++++--------- 1 file changed, 196 insertions(+), 196 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json index f3ef4646971..150ba70462f 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -6,54 +6,54 @@ "values": { "1": 11.04276, "2": 11.02298, - "3": 9.50921, - "4": 10.86244, - "5": 9.36127, - "6": 9.05636, - "7": 9.20064, - "8": 8.98909, - "9": 8.67001, - "10": 9.00892, - "11": 8.50716, - "12": 8.45579, - "13": 8.41197, - "14": 7.92802, - "15": 7.99663, - "16": 8.04156, - "17": 8.06453, - "18": 7.73746, - "19": 8.09946, - "20": 7.85555, - "21": 7.54063, - "22": 7.51142, - "23": 7.39766, - "24": 7.36551, - "25": 7.63399, - "26": 7.04934, - "27": 7.60084, - "28": 7.30223, - "29": 7.47164, - "30": 7.61428, - "31": 7.34981, - "32": 7.53935, - "33": 7.59164, - "34": 7.64951, - "35": 7.18657, - "36": 7.03804, - "37": 7.36778, - "38": 7.14613, - "39": 7.50644, - "40": 7.51103, - "41": 7.44582, - "42": 7.20666, - "43": 7.2123, - "44": 7.37723, - "45": 7.17293, - "46": 6.86188, - "47": 7.2648, - "48": 7.1069, - "49": 7.56115, - "50": 7.00113 + "3": 9.50907, + "4": 10.86145, + "5": 9.36104, + "6": 9.05664, + "7": 9.20646, + "8": 9.00188, + "9": 8.69791, + "10": 8.97535, + "11": 8.48206, + "12": 8.44961, + "13": 8.38916, + "14": 7.90422, + "15": 7.98559, + "16": 8.02787, + "17": 8.04894, + "18": 7.72163, + "19": 8.0935, + "20": 7.85609, + "21": 7.53372, + "22": 7.50495, + "23": 7.39733, + "24": 7.36369, + "25": 7.62993, + "26": 7.04703, + "27": 7.59839, + "28": 7.29807, + "29": 7.46826, + "30": 7.60613, + "31": 7.34795, + "32": 7.53766, + "33": 7.58939, + "34": 7.64431, + "35": 7.18358, + "36": 7.036, + "37": 7.36506, + "38": 7.14525, + "39": 7.50347, + "40": 7.50925, + "41": 7.44415, + "42": 7.20526, + "43": 7.21039, + "44": 7.37585, + "45": 7.1698, + "46": 6.8612, + "47": 7.26258, + "48": 7.1033, + "49": 7.55974, + "50": 6.99878 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 38808176.0, - "2": 38549232.0, - "3": 38741780.0, - "4": 78604016.0, - "5": 152229680.0, - "6": 299762016.0, - "7": 557587712.0, - "8": 589584384.0, - "9": 482229120.0, - "10": 517739584.0, - "11": 526962624.0, - "12": 476182528.0, - "13": 667453056.0, - "14": 563635200.0, - "15": 592125760.0, - "16": 589362048.0, - "17": 453879424.0, - "18": 444631456.0, - "19": 532791520.0, - "20": 677797248.0, - "21": 545577920.0, - "22": 494731040.0, - "23": 551928576.0, - "24": 489800928.0, - "25": 644993344.0, - "26": 441532864.0, - "27": 467175040.0, - "28": 431687840.0, - "29": 409185824.0, - "30": 583756032.0, - "31": 592451072.0, - "32": 416290048.0, - "33": 391230880.0, - "34": 325273120.0, - "35": 350756576.0, - "36": 331801376.0, - "37": 349196160.0, - "38": 312664800.0, - "39": 419015584.0, - "40": 299035872.0, - "41": 274307296.0, - "42": 296551584.0, - "43": 381740640.0, - "44": 308872480.0, - "45": 263141648.0, - "46": 353360864.0, - "47": 271093472.0, - "48": 346833600.0, - "49": 267589936.0, - "50": 252702768.0 + "1": 38808152.0, + "2": 38549168.0, + "3": 38741680.0, + "4": 81738424.0, + "5": 161659808.0, + "6": 296608000.0, + "7": 557581568.0, + "8": 592711744.0, + "9": 479088640.0, + "10": 520896096.0, + "11": 555256320.0, + "12": 444724480.0, + "13": 658029440.0, + "14": 585665280.0, + "15": 588986240.0, + "16": 479280192.0, + "17": 494748608.0, + "18": 504398944.0, + "19": 601982144.0, + "20": 787884160.0, + "21": 536156160.0, + "22": 513609344.0, + "23": 577056256.0, + "24": 549563712.0, + "25": 648153280.0, + "26": 498150784.0, + "27": 501770816.0, + "28": 522921920.0, + "29": 462644416.0, + "30": 612066112.0, + "31": 605029312.0, + "32": 454036160.0, + "33": 419547936.0, + "34": 378748896.0, + "35": 385339904.0, + "36": 350676768.0, + "37": 478164480.0, + "38": 337833600.0, + "39": 450472544.0, + "40": 267556496.0, + "41": 280614912.0, + "42": 305998368.0, + "43": 372298848.0, + "44": 261697280.0, + "45": 225394720.0, + "46": 268431392.0, + "47": 217617888.0, + "48": 261904016.0, + "49": 229846288.0, + "50": 214954112.0 } }, "mem-allocated-bytes": { @@ -177,54 +177,54 @@ "values": { "1": 37959917568.0, "2": 39578673152.0, - "3": 39583825920.0, - "4": 39583825920.0, - "5": 39586181120.0, - "6": 39586181120.0, - "7": 39586181120.0, - "8": 39586181120.0, - "9": 39586181120.0, - "10": 39586181120.0, - "11": 39586181120.0, - "12": 39586181120.0, - "13": 39586181120.0, - "14": 39586181120.0, - "15": 39586181120.0, - "16": 39586181120.0, - "17": 39586181120.0, - "18": 39586181120.0, - "19": 39586181120.0, - "20": 39586181120.0, - "21": 39586181120.0, - "22": 39586181120.0, - "23": 39586181120.0, - "24": 39586181120.0, - "25": 39586181120.0, - "26": 39586181120.0, - "27": 39586181120.0, - "28": 39586181120.0, - "29": 39586181120.0, - "30": 39586181120.0, - "31": 39586181120.0, - "32": 39586181120.0, - "33": 39586181120.0, - "34": 39586181120.0, - "35": 39586181120.0, - "36": 39586181120.0, - "37": 39586181120.0, - "38": 39586181120.0, - "39": 39586181120.0, - "40": 39586181120.0, - "41": 39586181120.0, - "42": 39586181120.0, - "43": 39586181120.0, - "44": 39586181120.0, - "45": 39586181120.0, - "46": 39586181120.0, - "47": 39586181120.0, - "48": 39586181120.0, - "49": 39586181120.0, - "50": 39586181120.0 + "3": 39583842304.0, + "4": 39583842304.0, + "5": 39584591872.0, + "6": 39584591872.0, + "7": 39584591872.0, + "8": 39584591872.0, + "9": 39584591872.0, + "10": 39584591872.0, + "11": 39584591872.0, + "12": 39584591872.0, + "13": 39584591872.0, + "14": 39584591872.0, + "15": 39584591872.0, + "16": 39584591872.0, + "17": 39584591872.0, + "18": 39584591872.0, + "19": 39584591872.0, + "20": 39584591872.0, + "21": 39584591872.0, + "22": 39584591872.0, + "23": 39584591872.0, + "24": 39584591872.0, + "25": 39584591872.0, + "26": 39584591872.0, + "27": 39584591872.0, + "28": 39584591872.0, + "29": 39584591872.0, + "30": 39584591872.0, + "31": 39584591872.0, + "32": 39584591872.0, + "33": 39584591872.0, + "34": 39584591872.0, + "35": 39584591872.0, + "36": 39584591872.0, + "37": 39584591872.0, + "38": 39584591872.0, + "39": 39584591872.0, + "40": 39584591872.0, + "41": 39584591872.0, + "42": 39584591872.0, + "43": 39584591872.0, + "44": 39584591872.0, + "45": 39584591872.0, + "46": 39584591872.0, + "47": 39584591872.0, + "48": 39584591872.0, + "49": 39584591872.0, + "50": 39584591872.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 65.48328, - "2": 1.94615, - "3": 3.94539, - "4": 2.42699, - "5": 1.80319, - "6": 1.79395, - "7": 1.50546, - "8": 2.00251, - "9": 1.2172, - "10": 1.31071, - "11": 1.3171, - "12": 1.10351, - "13": 1.26314, - "14": 1.47608, - "15": 1.19001, - "16": 1.12949, - "17": 1.15105, - "18": 1.06698, - "19": 1.10069, - "20": 1.12463, - "21": 1.35075, - "22": 1.56258, - "23": 1.2368, - "24": 1.13707, - "25": 1.11826, - "26": 1.09445, - "27": 1.08857, - "28": 1.07964, - "29": 1.08505, - "30": 1.24068, - "31": 1.10419, - "32": 1.5164, - "33": 1.10245, - "34": 1.37977, - "35": 1.1165, - "36": 1.1457, - "37": 1.10487, - "38": 1.08491, - "39": 1.08901, - "40": 1.08968, - "41": 1.13702, - "42": 1.09805, - "43": 1.06669, - "44": 1.07791, - "45": 1.08898, - "46": 1.10717, - "47": 1.13008, - "48": 1.05745, - "49": 1.08268, - "50": 1.05678 + "1": 65.95827, + "2": 1.9924, + "3": 3.92592, + "4": 2.4652, + "5": 1.84842, + "6": 1.80402, + "7": 1.67822, + "8": 1.88485, + "9": 1.32993, + "10": 1.37648, + "11": 1.18596, + "12": 1.16521, + "13": 1.14524, + "14": 1.34968, + "15": 1.22798, + "16": 1.10709, + "17": 1.2737, + "18": 1.12048, + "19": 1.44431, + "20": 1.22659, + "21": 1.23111, + "22": 1.27597, + "23": 1.25479, + "24": 1.12437, + "25": 1.28457, + "26": 1.26411, + "27": 1.16703, + "28": 1.13595, + "29": 1.24774, + "30": 1.10985, + "31": 1.3919, + "32": 1.10386, + "33": 1.20402, + "34": 1.08667, + "35": 1.10247, + "36": 1.09087, + "37": 1.16339, + "38": 1.12236, + "39": 1.10519, + "40": 1.20224, + "41": 1.11719, + "42": 1.18432, + "43": 1.11065, + "44": 1.14205, + "45": 1.12352, + "46": 1.09449, + "47": 1.10298, + "48": 1.10504, + "49": 1.09853, + "50": 1.0939 } } } \ No newline at end of file From 3c68f794705ab03ed04825881e00c88bbb2727db Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Thu, 6 Nov 2025 02:31:18 -0800 Subject: [PATCH 15/47] update model_config and golden values Signed-off-by: Hongbin Liu --- .../transformer/multi_token_prediction.py | 3 + megatron/training/arguments.py | 2 +- .../golden_values_dev_dgx_h100.json | 600 +++++++++--------- .../model_config.yaml | 6 +- .../golden_values_dev_dgx_h100.json | 500 +++++++-------- .../model_config.yaml | 6 +- 6 files changed, 560 insertions(+), 557 deletions(-) diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py index a619b9ffa55..9db38182d74 100755 --- a/megatron/core/transformer/multi_token_prediction.py +++ b/megatron/core/transformer/multi_token_prediction.py @@ -32,6 +32,9 @@ make_tp_sharded_tensor_for_checkpoint, make_viewless_tensor, ) +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_set_last_layer +) if is_torch_min_version("1.13.0"): dist_all_gather_func = torch.distributed.all_gather_into_tensor diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 24d4f52bac5..3bd4e5b43cc 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -2288,7 +2288,7 @@ def _add_training_args(parser): group.add_argument('--fine-grained-activation-offloading', action='store_true', help='Enable fine-grained activation offloading.') group.add_argument('--offload-modules', nargs='*', type=str, default=[], - help='The submodules to offload its input. Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".') + help='The submodules to offload its input. Choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".') group.add_argument('--min-offloaded-tensor-size', type=int, default=1024*1024, help='The minimum size of the tensor to be offloaded.') return parser diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json index 4b32d4256db..e7f62bbe4af 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.07559, - "2": 11.03834, - "3": 9.72869, - "4": 9.61678, - "5": 10.63323, - "6": 9.1681, - "7": 9.35196, - "8": 9.05204, - "9": 8.84148, - "10": 9.00321, - "11": 8.49799, - "12": 8.5218, - "13": 8.41649, - "14": 7.9096, - "15": 8.00627, - "16": 8.05394, - "17": 8.0203, - "18": 7.73136, - "19": 8.11676, - "20": 7.83945, - "21": 7.52196, - "22": 7.5295, - "23": 7.38729, - "24": 7.3758, - "25": 7.65255, - "26": 7.04795, - "27": 7.591, - "28": 7.30023, - "29": 7.45656, - "30": 7.60935, - "31": 7.3713, - "32": 7.55298, - "33": 7.59738, - "34": 7.65764, - "35": 7.17916, - "36": 7.04913, - "37": 7.38022, - "38": 7.14883, - "39": 7.50321, - "40": 7.51595, - "41": 7.45139, - "42": 7.21197, - "43": 7.21131, - "44": 7.38058, - "45": 7.16397, - "46": 6.86108, - "47": 7.27247, - "48": 7.10862, - "49": 7.56398, - "50": 7.00523 + "1": 11.06715, + "2": 11.06051, + "3": 10.21154, + "4": 9.95175, + "5": 10.12622, + "6": 8.82146, + "7": 9.52879, + "8": 8.442, + "9": 7.84738, + "10": 7.07075, + "11": 9.31042, + "12": 9.16013, + "13": 7.87292, + "14": 8.2102, + "15": 8.22483, + "16": 8.17879, + "17": 8.21121, + "18": 7.50325, + "19": 8.08274, + "20": 7.62562, + "21": 7.95058, + "22": 7.29789, + "23": 7.93775, + "24": 7.44169, + "25": 8.23817, + "26": 7.74959, + "27": 7.69344, + "28": 7.65487, + "29": 7.75173, + "30": 7.56007, + "31": 7.81567, + "32": 6.46589, + "33": 7.20401, + "34": 7.77921, + "35": 7.72944, + "36": 6.71776, + "37": 8.08311, + "38": 7.6137, + "39": 7.96476, + "40": 7.50072, + "41": 7.50304, + "42": 6.11349, + "43": 7.59404, + "44": 7.91361, + "45": 6.83615, + "46": 7.41293, + "47": 7.79226, + "48": 7.87549, + "49": 7.58763, + "50": 6.84525 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 38802120.0, - "2": 38543052.0, - "3": 38738396.0, - "4": 113220144.0, - "5": 344100160.0, - "6": 435062816.0, - "7": 579598912.0, - "8": 819195200.0, - "9": 604910464.0, - "10": 690749824.0, - "11": 744002496.0, - "12": 520212192.0, - "13": 547932992.0, - "14": 585659584.0, - "15": 614149184.0, - "16": 664915328.0, - "17": 592272320.0, - "18": 630225856.0, - "19": 579959808.0, - "20": 800470080.0, - "21": 573941056.0, - "22": 557652032.0, - "23": 797256640.0, - "24": 826380864.0, - "25": 814860160.0, - "26": 617708032.0, - "27": 715680384.0, - "28": 548045824.0, - "29": 736312064.0, - "30": 722163456.0, - "31": 711986176.0, - "32": 674238208.0, - "33": 715239232.0, - "34": 677588288.0, - "35": 473423392.0, - "36": 451352800.0, - "37": 446739392.0, - "38": 567466304.0, - "39": 472519552.0, - "40": 434322048.0, - "41": 554276096.0, - "42": 526187424.0, - "43": 510713152.0, - "44": 522783808.0, - "45": 335511072.0, - "46": 450878784.0, - "47": 450397344.0, - "48": 321720704.0, - "49": 437443680.0, - "50": 419425088.0 + "1": 47165192.0, + "2": 46897912.0, + "3": 52684456.0, + "4": 297127552.0, + "5": 562950784.0, + "6": 668142144.0, + "7": 1027449536.0, + "8": 752259328.0, + "9": 830947776.0, + "10": 718307136.0, + "11": 823731840.0, + "12": 804867840.0, + "13": 639461056.0, + "14": 625408576.0, + "15": 716256960.0, + "16": 870866752.0, + "17": 673817856.0, + "18": 811900096.0, + "19": 892689024.0, + "20": 878114112.0, + "21": 666859968.0, + "22": 792718848.0, + "23": 783683200.0, + "24": 770686976.0, + "25": 651376640.0, + "26": 780070272.0, + "27": 801722496.0, + "28": 670273664.0, + "29": 647960768.0, + "30": 789867776.0, + "31": 801385856.0, + "32": 787688640.0, + "33": 783506816.0, + "34": 792837760.0, + "35": 776103936.0, + "36": 761920512.0, + "37": 775085824.0, + "38": 752868608.0, + "39": 754997184.0, + "40": 745075072.0, + "41": 713941440.0, + "42": 689968512.0, + "43": 663461824.0, + "44": 680285632.0, + "45": 644628992.0, + "46": 641672704.0, + "47": 642439616.0, + "48": 597700608.0, + "49": 603523520.0, + "50": 601014528.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5498340864.0, - "2": 5499135488.0, - "3": 5499928064.0, - "4": 5500720640.0, - "5": 5501513216.0, - "6": 5502305792.0, - "7": 5497946624.0, - "8": 5498739200.0, - "9": 5499531776.0, - "10": 5500324352.0, - "11": 5501116928.0, - "12": 5498342912.0, - "13": 5499135488.0, - "14": 5499928064.0, - "15": 5500720640.0, - "16": 5501513216.0, - "17": 5502305792.0, - "18": 5503098368.0, - "19": 5503890944.0, - "20": 5504683520.0, - "21": 5505476096.0, - "22": 5506268672.0, - "23": 5507061248.0, - "24": 5507853824.0, - "25": 5508646400.0, - "26": 5509438976.0, - "27": 5510231552.0, - "28": 5511024128.0, - "29": 5511816704.0, - "30": 5512609280.0, - "31": 5513401856.0, - "32": 5514194432.0, - "33": 5514987008.0, - "34": 5515779584.0, - "35": 5516572160.0, - "36": 5517364736.0, - "37": 5518157312.0, - "38": 5518949888.0, - "39": 5519742464.0, - "40": 5520535040.0, - "41": 5521327616.0, - "42": 5522120192.0, - "43": 5522912768.0, - "44": 5523705344.0, - "45": 5524497920.0, - "46": 5525290496.0, - "47": 5526083072.0, - "48": 5526875648.0, - "49": 5527668224.0, - "50": 5528460800.0 + "1": 5290944000.0, + "2": 5291148800.0, + "3": 5291351552.0, + "4": 5290946048.0, + "5": 5291148800.0, + "6": 5291351552.0, + "7": 5291554304.0, + "8": 5291757056.0, + "9": 5291959808.0, + "10": 5292162560.0, + "11": 5292365312.0, + "12": 5292568064.0, + "13": 5292770816.0, + "14": 5292973568.0, + "15": 5293176320.0, + "16": 5293379072.0, + "17": 5293581824.0, + "18": 5293784576.0, + "19": 5293987328.0, + "20": 5294190080.0, + "21": 5294392832.0, + "22": 5294595584.0, + "23": 5294798336.0, + "24": 5295001088.0, + "25": 5295203840.0, + "26": 5295406592.0, + "27": 5295609344.0, + "28": 5295812096.0, + "29": 5296014848.0, + "30": 5296217600.0, + "31": 5296420352.0, + "32": 5296623104.0, + "33": 5296825856.0, + "34": 5297028608.0, + "35": 5297231360.0, + "36": 5297434112.0, + "37": 5297636864.0, + "38": 5297839616.0, + "39": 5298042368.0, + "40": 5298245120.0, + "41": 5298447872.0, + "42": 5298650624.0, + "43": 5298853376.0, + "44": 5299056128.0, + "45": 5299258880.0, + "46": 5299461632.0, + "47": 5299664384.0, + "48": 5299867136.0, + "49": 5300069888.0, + "50": 5300272640.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 41723441152.0, - "2": 43687280640.0, - "3": 43916578816.0, - "4": 43916578816.0, - "5": 43916578816.0, - "6": 43916578816.0, - "7": 43916578816.0, - "8": 43916578816.0, - "9": 43916578816.0, - "10": 43916578816.0, - "11": 43916578816.0, - "12": 44028436480.0, - "13": 44028436480.0, - "14": 44028436480.0, - "15": 44028436480.0, - "16": 44028436480.0, - "17": 44028436480.0, - "18": 44028436480.0, - "19": 44028436480.0, - "20": 44028436480.0, - "21": 44028436480.0, - "22": 44028436480.0, - "23": 44028436480.0, - "24": 44028436480.0, - "25": 44028436480.0, - "26": 44028436480.0, - "27": 44028436480.0, - "28": 44028436480.0, - "29": 44028436480.0, - "30": 44028436480.0, - "31": 44028436480.0, - "32": 44028436480.0, - "33": 44028436480.0, - "34": 44028436480.0, - "35": 44028436480.0, - "36": 44028436480.0, - "37": 44028436480.0, - "38": 44028436480.0, - "39": 44028436480.0, - "40": 44028436480.0, - "41": 44028436480.0, - "42": 44028436480.0, - "43": 44028436480.0, - "44": 44028436480.0, - "45": 44028436480.0, - "46": 44028436480.0, - "47": 44028436480.0, - "48": 44028436480.0, - "49": 44028436480.0, - "50": 44028436480.0 + "1": 6180783616.0, + "2": 8225679872.0, + "3": 8225679872.0, + "4": 8225679872.0, + "5": 8225679872.0, + "6": 8225679872.0, + "7": 8225679872.0, + "8": 8225679872.0, + "9": 8225679872.0, + "10": 8225679872.0, + "11": 8239991296.0, + "12": 8239991296.0, + "13": 8239991296.0, + "14": 8239991296.0, + "15": 8239991296.0, + "16": 8239991296.0, + "17": 8244914688.0, + "18": 8244914688.0, + "19": 8244914688.0, + "20": 8265598464.0, + "21": 8265598464.0, + "22": 8265598464.0, + "23": 8265598464.0, + "24": 8265598464.0, + "25": 8265598464.0, + "26": 8265598464.0, + "27": 8265598464.0, + "28": 8265598464.0, + "29": 8271664640.0, + "30": 8316803584.0, + "31": 8316803584.0, + "32": 8316803584.0, + "33": 8316803584.0, + "34": 8316803584.0, + "35": 8316803584.0, + "36": 8316803584.0, + "37": 8316803584.0, + "38": 8316803584.0, + "39": 8318923264.0, + "40": 8318923264.0, + "41": 8318923264.0, + "42": 8318923264.0, + "43": 8318923264.0, + "44": 8318923264.0, + "45": 8318923264.0, + "46": 8318923264.0, + "47": 8318923264.0, + "48": 8318923264.0, + "49": 8318923264.0, + "50": 8318923264.0 } }, "mtp_1 loss": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.08623, - "2": 11.1047, - "3": 10.54469, - "4": 10.08474, - "5": 9.76549, - "6": 9.56242, - "7": 9.59473, - "8": 8.97686, - "9": 8.83293, - "10": 9.1193, - "11": 8.44318, - "12": 8.49593, - "13": 8.37985, - "14": 7.81516, - "15": 7.95146, - "16": 8.01718, - "17": 7.94503, - "18": 7.68603, - "19": 8.07501, - "20": 7.79558, - "21": 7.46867, - "22": 7.46603, - "23": 7.32734, - "24": 7.32819, - "25": 7.58465, - "26": 6.99257, - "27": 7.53486, - "28": 7.23432, - "29": 7.40501, - "30": 7.55005, - "31": 7.30085, - "32": 7.48028, - "33": 7.53593, - "34": 7.60112, - "35": 7.12344, - "36": 6.99007, - "37": 7.32578, - "38": 7.09623, - "39": 7.45759, - "40": 7.45018, - "41": 7.40101, - "42": 7.14459, - "43": 7.13995, - "44": 7.32066, - "45": 7.0966, - "46": 6.80106, - "47": 7.21219, - "48": 7.05021, - "49": 7.48165, - "50": 6.95118 + "1": 11.07395, + "2": 11.0927, + "3": 10.82648, + "4": 10.27524, + "5": 10.45343, + "6": 8.32789, + "7": 9.82687, + "8": 8.01561, + "9": 7.47686, + "10": 6.75778, + "11": 8.92977, + "12": 8.98867, + "13": 7.80263, + "14": 8.02637, + "15": 8.11184, + "16": 8.13967, + "17": 8.13444, + "18": 7.44744, + "19": 8.03657, + "20": 7.53993, + "21": 7.90129, + "22": 7.27518, + "23": 7.88304, + "24": 7.37567, + "25": 8.16836, + "26": 7.69935, + "27": 7.6262, + "28": 7.61271, + "29": 7.69819, + "30": 7.4848, + "31": 7.73967, + "32": 6.36884, + "33": 7.14295, + "34": 7.71844, + "35": 7.63485, + "36": 6.61195, + "37": 8.02821, + "38": 7.57841, + "39": 7.89473, + "40": 7.41461, + "41": 7.42116, + "42": 6.01344, + "43": 7.4906, + "44": 7.86418, + "45": 6.74814, + "46": 7.30484, + "47": 7.72617, + "48": 7.79074, + "49": 7.49049, + "50": 6.75504 } }, "iteration-time": { @@ -289,56 +289,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 87.66203, - "2": 2.04189, - "3": 3.34278, - "4": 3.72414, - "5": 3.23492, - "6": 1.94546, - "7": 2.14942, - "8": 1.78075, - "9": 1.06029, - "10": 2.13554, - "11": 1.42578, - "12": 1.80986, - "13": 1.06134, - "14": 1.087, - "15": 1.16687, - "16": 1.20412, - "17": 1.06984, - "18": 1.07557, - "19": 1.04081, - "20": 1.21763, - "21": 1.06196, - "22": 1.14038, - "23": 2.25761, - "24": 1.09161, - "25": 1.04319, - "26": 1.40025, - "27": 1.04974, - "28": 1.03984, - "29": 1.05293, - "30": 1.48942, - "31": 1.04785, - "32": 1.0529, - "33": 1.04366, - "34": 1.0633, - "35": 1.0713, - "36": 1.05711, - "37": 1.08085, - "38": 1.07006, - "39": 1.06498, - "40": 1.05913, - "41": 1.0697, - "42": 1.079, - "43": 1.14122, - "44": 1.06478, - "45": 1.04692, - "46": 1.08174, - "47": 1.07595, - "48": 1.10523, - "49": 1.0839, - "50": 1.07754 + "1": 90.97535, + "2": 4.15413, + "3": 4.25282, + "4": 5.50314, + "5": 4.36528, + "6": 4.16016, + "7": 4.60989, + "8": 3.68392, + "9": 3.70951, + "10": 3.66417, + "11": 3.64904, + "12": 3.66094, + "13": 3.68824, + "14": 3.64996, + "15": 3.64159, + "16": 3.68269, + "17": 3.66905, + "18": 4.10783, + "19": 3.63362, + "20": 3.65129, + "21": 3.6431, + "22": 3.64946, + "23": 3.6411, + "24": 3.59707, + "25": 3.55364, + "26": 3.61478, + "27": 3.59779, + "28": 3.58741, + "29": 3.62545, + "30": 3.63538, + "31": 3.58264, + "32": 3.65914, + "33": 3.62764, + "34": 3.61962, + "35": 3.57076, + "36": 3.59244, + "37": 3.68499, + "38": 3.6803, + "39": 3.5849, + "40": 3.59019, + "41": 3.62068, + "42": 3.69144, + "43": 3.71863, + "44": 3.67193, + "45": 3.65673, + "46": 3.66919, + "47": 3.58334, + "48": 3.57229, + "49": 3.66195, + "50": 3.64157 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml index d9ec0456190..f508462666a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml @@ -23,8 +23,8 @@ MODEL_ARGS: --use-mcore-models: true --sequence-parallel: true --disable-bias-linear: true - --micro-batch-size: 4 - --global-batch-size: 32 + --micro-batch-size: 1 + --global-batch-size: 8 --train-iters: 50 --exit-duration-in-mins: 230 --no-check-for-nan-in-loss-and-grad: true @@ -36,7 +36,7 @@ MODEL_ARGS: --recompute-granularity: selective --recompute-modules: "[layernorm mla_up_proj mlp moe_act]" --fine-grained-activation-offloading: true - --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm]" + --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm qkv_linear core_attn attn_proj]" # Transformer Engine args --transformer-impl: transformer_engine # Data args diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json index 150ba70462f..1483224813a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.04276, - "2": 11.02298, - "3": 9.50907, - "4": 10.86145, - "5": 9.36104, - "6": 9.05664, - "7": 9.20646, - "8": 9.00188, - "9": 8.69791, - "10": 8.97535, - "11": 8.48206, - "12": 8.44961, - "13": 8.38916, - "14": 7.90422, - "15": 7.98559, - "16": 8.02787, - "17": 8.04894, - "18": 7.72163, - "19": 8.0935, - "20": 7.85609, - "21": 7.53372, - "22": 7.50495, - "23": 7.39733, - "24": 7.36369, - "25": 7.62993, - "26": 7.04703, - "27": 7.59839, - "28": 7.29807, - "29": 7.46826, - "30": 7.60613, - "31": 7.34795, - "32": 7.53766, - "33": 7.58939, - "34": 7.64431, - "35": 7.18358, - "36": 7.036, - "37": 7.36506, - "38": 7.14525, - "39": 7.50347, - "40": 7.50925, - "41": 7.44415, - "42": 7.20526, - "43": 7.21039, - "44": 7.37585, - "45": 7.1698, - "46": 6.8612, - "47": 7.26258, - "48": 7.1033, - "49": 7.55974, - "50": 6.99878 + "1": 11.01686, + "2": 11.06264, + "3": 10.17771, + "4": 10.86294, + "5": 9.81711, + "6": 9.10377, + "7": 9.61048, + "8": 8.39441, + "9": 7.79453, + "10": 7.15206, + "11": 9.06579, + "12": 12.40166, + "13": 8.04847, + "14": 8.24594, + "15": 8.24907, + "16": 8.32751, + "17": 8.35488, + "18": 7.58028, + "19": 8.18771, + "20": 7.71954, + "21": 8.00698, + "22": 7.35089, + "23": 7.95479, + "24": 7.51289, + "25": 8.32529, + "26": 7.78885, + "27": 7.72725, + "28": 7.71319, + "29": 7.77361, + "30": 7.56799, + "31": 7.85271, + "32": 6.52658, + "33": 7.24362, + "34": 7.80331, + "35": 7.74511, + "36": 6.73702, + "37": 8.15605, + "38": 7.62885, + "39": 7.97707, + "40": 7.52037, + "41": 7.52443, + "42": 6.12689, + "43": 7.60467, + "44": 7.96883, + "45": 6.84543, + "46": 7.42548, + "47": 7.82723, + "48": 7.87988, + "49": 7.59963, + "50": 6.85112 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 38808152.0, - "2": 38549168.0, - "3": 38741680.0, - "4": 81738424.0, - "5": 161659808.0, - "6": 296608000.0, - "7": 557581568.0, - "8": 592711744.0, - "9": 479088640.0, - "10": 520896096.0, - "11": 555256320.0, - "12": 444724480.0, - "13": 658029440.0, - "14": 585665280.0, - "15": 588986240.0, - "16": 479280192.0, - "17": 494748608.0, - "18": 504398944.0, - "19": 601982144.0, - "20": 787884160.0, - "21": 536156160.0, - "22": 513609344.0, - "23": 577056256.0, - "24": 549563712.0, - "25": 648153280.0, - "26": 498150784.0, - "27": 501770816.0, - "28": 522921920.0, - "29": 462644416.0, - "30": 612066112.0, - "31": 605029312.0, - "32": 454036160.0, - "33": 419547936.0, - "34": 378748896.0, - "35": 385339904.0, - "36": 350676768.0, - "37": 478164480.0, - "38": 337833600.0, - "39": 450472544.0, - "40": 267556496.0, - "41": 280614912.0, - "42": 305998368.0, - "43": 372298848.0, - "44": 261697280.0, - "45": 225394720.0, - "46": 268431392.0, - "47": 217617888.0, - "48": 261904016.0, - "49": 229846288.0, - "50": 214954112.0 + "1": 47167840.0, + "2": 46900628.0, + "3": 81003512.0, + "4": 243621808.0, + "5": 468555040.0, + "6": 561181184.0, + "7": 958267392.0, + "8": 720794112.0, + "9": 771164224.0, + "10": 718302016.0, + "11": 669618304.0, + "12": 559500096.0, + "13": 642601344.0, + "14": 754397952.0, + "15": 766531584.0, + "16": 697850240.0, + "17": 654906240.0, + "18": 745861824.0, + "19": 738620928.0, + "20": 887555328.0, + "21": 729800064.0, + "22": 666937216.0, + "23": 777389312.0, + "24": 607175552.0, + "25": 855782784.0, + "26": 846129152.0, + "27": 666477056.0, + "28": 830677504.0, + "29": 811523712.0, + "30": 657771072.0, + "31": 609501440.0, + "32": 784538816.0, + "33": 755198720.0, + "34": 729929280.0, + "35": 719482368.0, + "36": 699006208.0, + "37": 727900096.0, + "38": 711973824.0, + "39": 701515264.0, + "40": 682162752.0, + "41": 534678112.0, + "42": 655361792.0, + "43": 663463424.0, + "44": 642541952.0, + "45": 455907168.0, + "46": 613359936.0, + "47": 592108160.0, + "48": 585115008.0, + "49": 559483008.0, + "50": 544390208.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4419107328.0, - "2": 4419108864.0, - "3": 4419108864.0, - "4": 4419108864.0, - "5": 4419108864.0, - "6": 4419108864.0, - "7": 4419108864.0, - "8": 4419108864.0, - "9": 4419108864.0, - "10": 4419108864.0, - "11": 4419108864.0, - "12": 4419108864.0, - "13": 4419108864.0, - "14": 4419108864.0, - "15": 4419108864.0, - "16": 4419108864.0, - "17": 4419108864.0, - "18": 4419108864.0, - "19": 4419108864.0, - "20": 4419108864.0, - "21": 4419108864.0, - "22": 4419108864.0, - "23": 4419108864.0, - "24": 4419108864.0, - "25": 4419108864.0, - "26": 4419108864.0, - "27": 4419108864.0, - "28": 4419108864.0, - "29": 4419108864.0, - "30": 4419108864.0, - "31": 4419108864.0, - "32": 4419108864.0, - "33": 4419108864.0, - "34": 4419108864.0, - "35": 4419108864.0, - "36": 4419108864.0, - "37": 4419108864.0, - "38": 4419108864.0, - "39": 4419108864.0, - "40": 4419108864.0, - "41": 4419108864.0, - "42": 4419108864.0, - "43": 4419108864.0, - "44": 4419108864.0, - "45": 4419108864.0, - "46": 4419108864.0, - "47": 4419108864.0, - "48": 4419108864.0, - "49": 4419108864.0, - "50": 4419108864.0 + "1": 4315544064.0, + "2": 4315545600.0, + "3": 4315545600.0, + "4": 4315545600.0, + "5": 4315545600.0, + "6": 4315545600.0, + "7": 4315545600.0, + "8": 4315545600.0, + "9": 4315545600.0, + "10": 4315545600.0, + "11": 4315545600.0, + "12": 4315545600.0, + "13": 4315545600.0, + "14": 4315545600.0, + "15": 4315545600.0, + "16": 4315545600.0, + "17": 4315545600.0, + "18": 4315545600.0, + "19": 4315545600.0, + "20": 4315545600.0, + "21": 4315545600.0, + "22": 4315545600.0, + "23": 4315545600.0, + "24": 4315545600.0, + "25": 4315545600.0, + "26": 4315545600.0, + "27": 4315545600.0, + "28": 4315545600.0, + "29": 4315545600.0, + "30": 4315545600.0, + "31": 4315545600.0, + "32": 4315545600.0, + "33": 4315545600.0, + "34": 4315545600.0, + "35": 4315545600.0, + "36": 4315545600.0, + "37": 4315545600.0, + "38": 4315545600.0, + "39": 4315545600.0, + "40": 4315545600.0, + "41": 4315545600.0, + "42": 4315545600.0, + "43": 4315545600.0, + "44": 4315545600.0, + "45": 4315545600.0, + "46": 4315545600.0, + "47": 4315545600.0, + "48": 4315545600.0, + "49": 4315545600.0, + "50": 4315545600.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 37959917568.0, - "2": 39578673152.0, - "3": 39583842304.0, - "4": 39583842304.0, - "5": 39584591872.0, - "6": 39584591872.0, - "7": 39584591872.0, - "8": 39584591872.0, - "9": 39584591872.0, - "10": 39584591872.0, - "11": 39584591872.0, - "12": 39584591872.0, - "13": 39584591872.0, - "14": 39584591872.0, - "15": 39584591872.0, - "16": 39584591872.0, - "17": 39584591872.0, - "18": 39584591872.0, - "19": 39584591872.0, - "20": 39584591872.0, - "21": 39584591872.0, - "22": 39584591872.0, - "23": 39584591872.0, - "24": 39584591872.0, - "25": 39584591872.0, - "26": 39584591872.0, - "27": 39584591872.0, - "28": 39584591872.0, - "29": 39584591872.0, - "30": 39584591872.0, - "31": 39584591872.0, - "32": 39584591872.0, - "33": 39584591872.0, - "34": 39584591872.0, - "35": 39584591872.0, - "36": 39584591872.0, - "37": 39584591872.0, - "38": 39584591872.0, - "39": 39584591872.0, - "40": 39584591872.0, - "41": 39584591872.0, - "42": 39584591872.0, - "43": 39584591872.0, - "44": 39584591872.0, - "45": 39584591872.0, - "46": 39584591872.0, - "47": 39584591872.0, - "48": 39584591872.0, - "49": 39584591872.0, - "50": 39584591872.0 + "1": 4919527424.0, + "2": 5861408768.0, + "3": 5861408768.0, + "4": 5863651328.0, + "5": 5863651328.0, + "6": 5863651328.0, + "7": 5863651328.0, + "8": 5863651328.0, + "9": 5863651328.0, + "10": 5863651328.0, + "11": 5863651328.0, + "12": 5863651328.0, + "13": 5863651328.0, + "14": 5863651328.0, + "15": 5863986176.0, + "16": 5865795072.0, + "17": 5865795072.0, + "18": 5865795072.0, + "19": 5865795072.0, + "20": 5865795072.0, + "21": 5866987520.0, + "22": 5866987520.0, + "23": 5866987520.0, + "24": 5866987520.0, + "25": 5866987520.0, + "26": 5866987520.0, + "27": 5866987520.0, + "28": 5866987520.0, + "29": 5866987520.0, + "30": 5866987520.0, + "31": 5866987520.0, + "32": 5866987520.0, + "33": 5866987520.0, + "34": 5866987520.0, + "35": 5866987520.0, + "36": 5866987520.0, + "37": 5866987520.0, + "38": 5866987520.0, + "39": 5866987520.0, + "40": 5866987520.0, + "41": 5866987520.0, + "42": 5866987520.0, + "43": 5866987520.0, + "44": 5866987520.0, + "45": 5866987520.0, + "46": 5866987520.0, + "47": 5866987520.0, + "48": 5866987520.0, + "49": 5866987520.0, + "50": 5866987520.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 65.95827, - "2": 1.9924, - "3": 3.92592, - "4": 2.4652, - "5": 1.84842, - "6": 1.80402, - "7": 1.67822, - "8": 1.88485, - "9": 1.32993, - "10": 1.37648, - "11": 1.18596, - "12": 1.16521, - "13": 1.14524, - "14": 1.34968, - "15": 1.22798, - "16": 1.10709, - "17": 1.2737, - "18": 1.12048, - "19": 1.44431, - "20": 1.22659, - "21": 1.23111, - "22": 1.27597, - "23": 1.25479, - "24": 1.12437, - "25": 1.28457, - "26": 1.26411, - "27": 1.16703, - "28": 1.13595, - "29": 1.24774, - "30": 1.10985, - "31": 1.3919, - "32": 1.10386, - "33": 1.20402, - "34": 1.08667, - "35": 1.10247, - "36": 1.09087, - "37": 1.16339, - "38": 1.12236, - "39": 1.10519, - "40": 1.20224, - "41": 1.11719, - "42": 1.18432, - "43": 1.11065, - "44": 1.14205, - "45": 1.12352, - "46": 1.09449, - "47": 1.10298, - "48": 1.10504, - "49": 1.09853, - "50": 1.0939 + "1": 72.699, + "2": 4.27015, + "3": 3.87365, + "4": 3.67041, + "5": 3.65964, + "6": 3.48532, + "7": 3.47679, + "8": 3.47349, + "9": 3.43879, + "10": 3.47441, + "11": 3.45737, + "12": 3.48691, + "13": 3.54474, + "14": 3.44102, + "15": 3.42127, + "16": 3.45795, + "17": 3.49717, + "18": 3.51293, + "19": 3.5617, + "20": 3.49733, + "21": 3.50336, + "22": 3.62308, + "23": 3.50166, + "24": 3.49075, + "25": 3.50996, + "26": 3.44423, + "27": 3.47323, + "28": 3.53784, + "29": 3.51989, + "30": 3.49211, + "31": 3.49945, + "32": 3.4419, + "33": 3.50458, + "34": 3.47663, + "35": 3.45702, + "36": 3.50281, + "37": 3.44136, + "38": 3.45165, + "39": 3.50095, + "40": 3.50126, + "41": 3.50863, + "42": 3.46684, + "43": 3.55122, + "44": 3.48372, + "45": 3.46903, + "46": 3.47654, + "47": 3.51574, + "48": 3.4895, + "49": 3.49404, + "50": 3.45824 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml index f4b64722712..5c39bcedcc7 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml @@ -23,8 +23,8 @@ MODEL_ARGS: --use-mcore-models: true --sequence-parallel: true --disable-bias-linear: true - --micro-batch-size: 4 - --global-batch-size: 32 + --micro-batch-size: 1 + --global-batch-size: 8 --train-iters: 50 --exit-duration-in-mins: 230 --no-check-for-nan-in-loss-and-grad: true @@ -36,7 +36,7 @@ MODEL_ARGS: --recompute-granularity: selective --recompute-modules: "[layernorm mla_up_proj mlp moe_act]" --fine-grained-activation-offloading: true - --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm]" + --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm qkv_linear core_attn attn_proj]" # Transformer Engine args --transformer-impl: transformer_engine # Data args From ee536776436fe111750713c0f68178043cc02166 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Thu, 6 Nov 2025 02:33:31 -0800 Subject: [PATCH 16/47] format Signed-off-by: Hongbin Liu --- megatron/core/transformer/multi_token_prediction.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py index 9db38182d74..a619b9ffa55 100755 --- a/megatron/core/transformer/multi_token_prediction.py +++ b/megatron/core/transformer/multi_token_prediction.py @@ -32,9 +32,6 @@ make_tp_sharded_tensor_for_checkpoint, make_viewless_tensor, ) -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_set_last_layer -) if is_torch_min_version("1.13.0"): dist_all_gather_func = torch.distributed.all_gather_into_tensor From 1c1d9c8658ca7729121260e3683612355cde8e13 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Thu, 6 Nov 2025 02:43:36 -0800 Subject: [PATCH 17/47] remove old golden values Signed-off-by: Hongbin Liu --- .../golden_values_dev_dgxh100_coreweave.json | 344 ------------------ .../golden_values_dev_dgxh100_eos.json | 344 ------------------ .../golden_values_dev_coreweave.json | 287 --------------- .../golden_values_dev_eos.json | 287 --------------- 4 files changed, 1262 deletions(-) delete mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_coreweave.json delete mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_eos.json delete mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json delete mode 100644 tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_coreweave.json deleted file mode 100644 index b3f192ba287..00000000000 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_coreweave.json +++ /dev/null @@ -1,344 +0,0 @@ -{ - "lm loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 11.07546, - "2": 11.03837, - "3": 9.66011, - "4": 9.91381, - "5": 9.32909, - "6": 9.13922, - "7": 9.13574, - "8": 8.65508, - "9": 8.51394, - "10": 8.8409, - "11": 8.29149, - "12": 8.34581, - "13": 8.25518, - "14": 7.73711, - "15": 7.86249, - "16": 7.9371, - "17": 7.89319, - "18": 7.63123, - "19": 7.99731, - "20": 7.74538, - "21": 7.44348, - "22": 7.42249, - "23": 7.29714, - "24": 7.27462, - "25": 7.54574, - "26": 6.96838, - "27": 7.50556, - "28": 7.22743, - "29": 7.36588, - "30": 7.52622, - "31": 7.27026, - "32": 7.45521, - "33": 7.50954, - "34": 7.55686, - "35": 7.10177, - "36": 6.96431, - "37": 7.28463, - "38": 7.0808, - "39": 7.40923, - "40": 7.43338, - "41": 7.38496, - "42": 7.15749, - "43": 7.15858, - "44": 7.28852, - "45": 7.16793, - "46": 6.78468, - "47": 7.4114, - "48": 7.0027, - "49": 7.46249, - "50": 6.92151 - } - }, - "num-zeros": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 911219392.0, - "2": 910960384.0, - "3": 911156352.0, - "4": 912204800.0, - "5": 920796544.0, - "6": 940387968.0, - "7": 990599872.0, - "8": 976457728.0, - "9": 998097664.0, - "10": 995852672.0, - "11": 994583680.0, - "12": 977344896.0, - "13": 1028141824.0, - "14": 1007166208.0, - "15": 987423616.0, - "16": 993054784.0, - "17": 982319168.0, - "18": 998261760.0, - "19": 984696320.0, - "20": 982914752.0, - "21": 979667456.0, - "22": 953988864.0, - "23": 972353984.0, - "24": 964792064.0, - "25": 958512192.0, - "26": 946928512.0, - "27": 948458304.0, - "28": 949643968.0, - "29": 942877440.0, - "30": 935020160.0, - "31": 935327616.0, - "32": 934281088.0, - "33": 921805568.0, - "34": 928189312.0, - "35": 922202496.0, - "36": 924246656.0, - "37": 920661248.0, - "38": 922930752.0, - "39": 922322816.0, - "40": 921856512.0, - "41": 920227968.0, - "42": 918353664.0, - "43": 918607040.0, - "44": 914948032.0, - "45": 914295232.0, - "46": 914344448.0, - "47": 911769536.0, - "48": 912013312.0, - "49": 910349440.0, - "50": 914351552.0 - } - }, - "mem-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 5498353152.0, - "2": 5499147776.0, - "3": 5499940352.0, - "4": 5500732928.0, - "5": 5501525504.0, - "6": 5502318080.0, - "7": 5503110656.0, - "8": 5503903232.0, - "9": 5497958912.0, - "10": 5498751488.0, - "11": 5499544064.0, - "12": 5500336640.0, - "13": 5501129216.0, - "14": 5501921792.0, - "15": 5502714368.0, - "16": 5503506944.0, - "17": 5504299520.0, - "18": 5505092096.0, - "19": 5505884672.0, - "20": 5506677248.0, - "21": 5507469824.0, - "22": 5508262400.0, - "23": 5509054976.0, - "24": 5509847552.0, - "25": 5510640128.0, - "26": 5511432704.0, - "27": 5512225280.0, - "28": 5513017856.0, - "29": 5513810432.0, - "30": 5514603008.0, - "31": 5515395584.0, - "32": 5516188160.0, - "33": 5516980736.0, - "34": 5517773312.0, - "35": 5518565888.0, - "36": 5519358464.0, - "37": 5520151040.0, - "38": 5520943616.0, - "39": 5521736192.0, - "40": 5522528768.0, - "41": 5523321344.0, - "42": 5524113920.0, - "43": 5524906496.0, - "44": 5525699072.0, - "45": 5526491648.0, - "46": 5527284224.0, - "47": 5528076800.0, - "48": 5528869376.0, - "49": 5529661952.0, - "50": 5530454528.0 - } - }, - "mem-max-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 41739952128.0, - "2": 43687571456.0, - "3": 43687571456.0, - "4": 43983216640.0, - "5": 43983216640.0, - "6": 43983216640.0, - "7": 43983216640.0, - "8": 44024635392.0, - "9": 44041216000.0, - "10": 44041216000.0, - "11": 44041216000.0, - "12": 44041216000.0, - "13": 44041216000.0, - "14": 44041216000.0, - "15": 44041216000.0, - "16": 44041216000.0, - "17": 44041216000.0, - "18": 44041216000.0, - "19": 44041216000.0, - "20": 44041216000.0, - "21": 44041216000.0, - "22": 44041216000.0, - "23": 44041216000.0, - "24": 44041216000.0, - "25": 44041216000.0, - "26": 44041216000.0, - "27": 44041216000.0, - "28": 44041216000.0, - "29": 44041326592.0, - "30": 44162326528.0, - "31": 44220485632.0, - "32": 44270411776.0, - "33": 44293799936.0, - "34": 44293799936.0, - "35": 44293799936.0, - "36": 44293799936.0, - "37": 44293799936.0, - "38": 44293799936.0, - "39": 44293799936.0, - "40": 44293799936.0, - "41": 44293799936.0, - "42": 44293799936.0, - "43": 44293799936.0, - "44": 44293799936.0, - "45": 44293799936.0, - "46": 44293799936.0, - "47": 44293799936.0, - "48": 44293799936.0, - "49": 44293799936.0, - "50": 44293799936.0 - } - }, - "mtp_1 loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 11.08617, - "2": 11.10475, - "3": 10.48001, - "4": 10.13466, - "5": 9.79047, - "6": 9.50601, - "7": 9.5113, - "8": 8.85336, - "9": 8.66683, - "10": 8.95866, - "11": 8.29315, - "12": 8.36982, - "13": 8.25544, - "14": 7.73322, - "15": 7.86639, - "16": 7.92442, - "17": 7.86278, - "18": 7.61012, - "19": 8.00269, - "20": 7.73019, - "21": 7.4165, - "22": 7.41478, - "23": 7.28671, - "24": 7.27903, - "25": 7.54456, - "26": 6.96542, - "27": 7.50538, - "28": 7.20607, - "29": 7.377, - "30": 7.52777, - "31": 7.27094, - "32": 7.4604, - "33": 7.51419, - "34": 7.56867, - "35": 7.09252, - "36": 6.96015, - "37": 7.29846, - "38": 7.0742, - "39": 7.43347, - "40": 7.43116, - "41": 7.40919, - "42": 7.15527, - "43": 7.15652, - "44": 7.30441, - "45": 7.1893, - "46": 6.77296, - "47": 7.45045, - "48": 7.02403, - "49": 7.45719, - "50": 6.92656 - } - }, - "iteration-time": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 64.40054, - "2": 2.16564, - "3": 3.72378, - "4": 1.63174, - "5": 2.30947, - "6": 1.7246, - "7": 1.5089, - "8": 1.60943, - "9": 1.48606, - "10": 1.47162, - "11": 1.05608, - "12": 1.3309, - "13": 1.06824, - "14": 1.41914, - "15": 1.10033, - "16": 1.15759, - "17": 1.23897, - "18": 1.10439, - "19": 1.11869, - "20": 1.09363, - "21": 1.23622, - "22": 1.14797, - "23": 1.23037, - "24": 1.03991, - "25": 1.07795, - "26": 1.04416, - "27": 1.03654, - "28": 1.04098, - "29": 1.03502, - "30": 1.02909, - "31": 1.17935, - "32": 1.14717, - "33": 1.05403, - "34": 1.13894, - "35": 1.04538, - "36": 1.04367, - "37": 1.0843, - "38": 1.04631, - "39": 1.06131, - "40": 1.06988, - "41": 1.09756, - "42": 1.04759, - "43": 1.09649, - "44": 1.05666, - "45": 1.05249, - "46": 1.04539, - "47": 1.04041, - "48": 1.04904, - "49": 1.04777, - "50": 1.06237 - } - } -} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_eos.json deleted file mode 100644 index d7372742ca7..00000000000 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgxh100_eos.json +++ /dev/null @@ -1,344 +0,0 @@ -{ - "lm loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 11.07546, - "2": 11.03837, - "3": 9.66011, - "4": 9.91381, - "5": 9.32909, - "6": 9.13922, - "7": 9.13574, - "8": 8.65508, - "9": 8.51394, - "10": 8.8409, - "11": 8.29149, - "12": 8.34581, - "13": 8.25518, - "14": 7.73711, - "15": 7.86249, - "16": 7.9371, - "17": 7.89319, - "18": 7.63123, - "19": 7.99731, - "20": 7.74538, - "21": 7.44348, - "22": 7.42249, - "23": 7.29714, - "24": 7.27462, - "25": 7.54574, - "26": 6.96838, - "27": 7.50556, - "28": 7.22743, - "29": 7.36588, - "30": 7.52622, - "31": 7.27026, - "32": 7.45521, - "33": 7.50954, - "34": 7.55686, - "35": 7.10177, - "36": 6.96431, - "37": 7.28463, - "38": 7.0808, - "39": 7.40923, - "40": 7.43338, - "41": 7.38496, - "42": 7.15749, - "43": 7.15858, - "44": 7.28852, - "45": 7.16793, - "46": 6.78468, - "47": 7.4114, - "48": 7.0027, - "49": 7.46249, - "50": 6.92151 - } - }, - "num-zeros": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 911219392.0, - "2": 910960384.0, - "3": 911156352.0, - "4": 912204800.0, - "5": 920796544.0, - "6": 940387968.0, - "7": 990599872.0, - "8": 976457728.0, - "9": 998097664.0, - "10": 995852672.0, - "11": 994583680.0, - "12": 977344896.0, - "13": 1028141824.0, - "14": 1007166208.0, - "15": 987423616.0, - "16": 993054784.0, - "17": 982319168.0, - "18": 998261760.0, - "19": 984696320.0, - "20": 982914752.0, - "21": 979667456.0, - "22": 953988864.0, - "23": 972353984.0, - "24": 964792064.0, - "25": 958512192.0, - "26": 946928512.0, - "27": 948458304.0, - "28": 949643968.0, - "29": 942877440.0, - "30": 935020160.0, - "31": 935327616.0, - "32": 934281088.0, - "33": 921805568.0, - "34": 928189312.0, - "35": 922202496.0, - "36": 924246656.0, - "37": 920661248.0, - "38": 922930752.0, - "39": 922322816.0, - "40": 921856512.0, - "41": 920227968.0, - "42": 918353664.0, - "43": 918607040.0, - "44": 914948032.0, - "45": 914295232.0, - "46": 914344448.0, - "47": 911769536.0, - "48": 912013312.0, - "49": 910349440.0, - "50": 914351552.0 - } - }, - "mem-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 5498353152.0, - "2": 5499147776.0, - "3": 5499940352.0, - "4": 5500732928.0, - "5": 5501525504.0, - "6": 5502318080.0, - "7": 5503110656.0, - "8": 5503903232.0, - "9": 5497958912.0, - "10": 5498751488.0, - "11": 5499544064.0, - "12": 5500336640.0, - "13": 5501129216.0, - "14": 5501921792.0, - "15": 5502714368.0, - "16": 5503506944.0, - "17": 5504299520.0, - "18": 5505092096.0, - "19": 5505884672.0, - "20": 5506677248.0, - "21": 5507469824.0, - "22": 5508262400.0, - "23": 5509054976.0, - "24": 5509847552.0, - "25": 5510640128.0, - "26": 5511432704.0, - "27": 5512225280.0, - "28": 5513017856.0, - "29": 5513810432.0, - "30": 5514603008.0, - "31": 5515395584.0, - "32": 5516188160.0, - "33": 5516980736.0, - "34": 5517773312.0, - "35": 5518565888.0, - "36": 5519358464.0, - "37": 5520151040.0, - "38": 5520943616.0, - "39": 5521736192.0, - "40": 5522528768.0, - "41": 5523321344.0, - "42": 5524113920.0, - "43": 5524906496.0, - "44": 5525699072.0, - "45": 5526491648.0, - "46": 5527284224.0, - "47": 5528076800.0, - "48": 5528869376.0, - "49": 5529661952.0, - "50": 5530454528.0 - } - }, - "mem-max-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 41739952128.0, - "2": 43687571456.0, - "3": 43687571456.0, - "4": 43983216640.0, - "5": 43983216640.0, - "6": 43983216640.0, - "7": 43983216640.0, - "8": 44024635392.0, - "9": 44041216000.0, - "10": 44041216000.0, - "11": 44041216000.0, - "12": 44041216000.0, - "13": 44041216000.0, - "14": 44041216000.0, - "15": 44041216000.0, - "16": 44041216000.0, - "17": 44041216000.0, - "18": 44041216000.0, - "19": 44041216000.0, - "20": 44041216000.0, - "21": 44041216000.0, - "22": 44041216000.0, - "23": 44041216000.0, - "24": 44041216000.0, - "25": 44041216000.0, - "26": 44041216000.0, - "27": 44041216000.0, - "28": 44041216000.0, - "29": 44041326592.0, - "30": 44162326528.0, - "31": 44220485632.0, - "32": 44270411776.0, - "33": 44293799936.0, - "34": 44293799936.0, - "35": 44293799936.0, - "36": 44293799936.0, - "37": 44293799936.0, - "38": 44293799936.0, - "39": 44293799936.0, - "40": 44293799936.0, - "41": 44293799936.0, - "42": 44293799936.0, - "43": 44293799936.0, - "44": 44293799936.0, - "45": 44293799936.0, - "46": 44293799936.0, - "47": 44293799936.0, - "48": 44293799936.0, - "49": 44293799936.0, - "50": 44293799936.0 - } - }, - "mtp_1 loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 11.08617, - "2": 11.10475, - "3": 10.48001, - "4": 10.13466, - "5": 9.79047, - "6": 9.50601, - "7": 9.5113, - "8": 8.85336, - "9": 8.66683, - "10": 8.95866, - "11": 8.29315, - "12": 8.36982, - "13": 8.25544, - "14": 7.73322, - "15": 7.86639, - "16": 7.92442, - "17": 7.86278, - "18": 7.61012, - "19": 8.00269, - "20": 7.73019, - "21": 7.4165, - "22": 7.41478, - "23": 7.28671, - "24": 7.27903, - "25": 7.54456, - "26": 6.96542, - "27": 7.50538, - "28": 7.20607, - "29": 7.377, - "30": 7.52777, - "31": 7.27094, - "32": 7.4604, - "33": 7.51419, - "34": 7.56867, - "35": 7.09252, - "36": 6.96015, - "37": 7.29846, - "38": 7.0742, - "39": 7.43347, - "40": 7.43116, - "41": 7.40919, - "42": 7.15527, - "43": 7.15652, - "44": 7.30441, - "45": 7.1893, - "46": 6.77296, - "47": 7.45045, - "48": 7.02403, - "49": 7.45719, - "50": 6.92656 - } - }, - "iteration-time": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 87.63934, - "2": 1.98402, - "3": 3.95877, - "4": 1.64812, - "5": 2.312, - "6": 2.02902, - "7": 1.56333, - "8": 1.66703, - "9": 1.6393, - "10": 1.40472, - "11": 1.086, - "12": 1.34921, - "13": 1.0854, - "14": 1.4242, - "15": 1.09539, - "16": 1.79766, - "17": 1.2562, - "18": 1.08887, - "19": 1.08371, - "20": 1.10071, - "21": 1.25979, - "22": 1.3212, - "23": 1.25044, - "24": 1.05384, - "25": 1.11356, - "26": 1.0605, - "27": 1.03418, - "28": 1.0405, - "29": 1.05174, - "30": 1.04166, - "31": 1.20036, - "32": 1.12936, - "33": 1.02917, - "34": 1.13473, - "35": 1.02829, - "36": 1.04352, - "37": 1.0843, - "38": 1.03714, - "39": 1.04534, - "40": 1.07031, - "41": 1.07618, - "42": 1.03008, - "43": 1.06043, - "44": 1.04049, - "45": 1.02875, - "46": 1.03669, - "47": 1.03128, - "48": 1.02808, - "49": 1.03038, - "50": 1.04621 - } - } -} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json deleted file mode 100644 index 4e979e64295..00000000000 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_coreweave.json +++ /dev/null @@ -1,287 +0,0 @@ -{ - "lm loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 11.04266, - "2": 11.02309, - "3": 9.43552, - "4": 10.04614, - "5": 9.38535, - "6": 9.14543, - "7": 9.21141, - "8": 8.63458, - "9": 8.48937, - "10": 8.82763, - "11": 8.29457, - "12": 8.3282, - "13": 8.23008, - "14": 7.71714, - "15": 7.86981, - "16": 7.92286, - "17": 7.8604, - "18": 7.62039, - "19": 7.98493, - "20": 7.72023, - "21": 7.39758, - "22": 7.39771, - "23": 7.28314, - "24": 7.25048, - "25": 7.53113, - "26": 6.95329, - "27": 7.49432, - "28": 7.20394, - "29": 7.37282, - "30": 7.50232, - "31": 7.25348, - "32": 7.4305, - "33": 7.48364, - "34": 7.53486, - "35": 7.10336, - "36": 6.94516, - "37": 7.26117, - "38": 7.07009, - "39": 7.40543, - "40": 7.42044, - "41": 7.34202, - "42": 7.11816, - "43": 7.11373, - "44": 7.27067, - "45": 7.07036, - "46": 6.77823, - "47": 7.1875, - "48": 6.99998, - "49": 7.45868, - "50": 6.90956 - } - }, - "num-zeros": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 844114112.0, - "2": 843855104.0, - "3": 844048640.0, - "4": 842998144.0, - "5": 855786112.0, - "6": 874329728.0, - "7": 925591552.0, - "8": 915644608.0, - "9": 935187584.0, - "10": 927702400.0, - "11": 957888256.0, - "12": 923872512.0, - "13": 969427072.0, - "14": 965228416.0, - "15": 952825344.0, - "16": 943777088.0, - "17": 928845824.0, - "18": 925913856.0, - "19": 955339136.0, - "20": 989208256.0, - "21": 924095424.0, - "22": 908902272.0, - "23": 892664576.0, - "24": 900830400.0, - "25": 928105472.0, - "26": 877724352.0, - "27": 912808320.0, - "28": 904557696.0, - "29": 872625088.0, - "30": 864767104.0, - "31": 868220416.0, - "32": 861931136.0, - "33": 859941312.0, - "34": 855839104.0, - "35": 854046848.0, - "36": 852944896.0, - "37": 851456704.0, - "38": 849532096.0, - "39": 849972608.0, - "40": 849505792.0, - "41": 845780288.0, - "42": 846003328.0, - "43": 846257472.0, - "44": 852034880.0, - "45": 847187456.0, - "46": 855625856.0, - "47": 844661952.0, - "48": 851197248.0, - "49": 851630464.0, - "50": 846195904.0 - } - }, - "mem-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 4419107328.0, - "2": 4419108864.0, - "3": 4419108864.0, - "4": 4419108864.0, - "5": 4419108864.0, - "6": 4419108864.0, - "7": 4419108864.0, - "8": 4419108864.0, - "9": 4419108864.0, - "10": 4419108864.0, - "11": 4419108864.0, - "12": 4419108864.0, - "13": 4419108864.0, - "14": 4419108864.0, - "15": 4419108864.0, - "16": 4419108864.0, - "17": 4419108864.0, - "18": 4419108864.0, - "19": 4419108864.0, - "20": 4419108864.0, - "21": 4419108864.0, - "22": 4419108864.0, - "23": 4419108864.0, - "24": 4419108864.0, - "25": 4419108864.0, - "26": 4419108864.0, - "27": 4419108864.0, - "28": 4419108864.0, - "29": 4419108864.0, - "30": 4419108864.0, - "31": 4419108864.0, - "32": 4419108864.0, - "33": 4419108864.0, - "34": 4419108864.0, - "35": 4419108864.0, - "36": 4419108864.0, - "37": 4419108864.0, - "38": 4419108864.0, - "39": 4419108864.0, - "40": 4419108864.0, - "41": 4419108864.0, - "42": 4419108864.0, - "43": 4419108864.0, - "44": 4419108864.0, - "45": 4419108864.0, - "46": 4419108864.0, - "47": 4419108864.0, - "48": 4419108864.0, - "49": 4419108864.0, - "50": 4419108864.0 - } - }, - "mem-max-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 37959917568.0, - "2": 39578677248.0, - "3": 39580196864.0, - "4": 39580196864.0, - "5": 39583309824.0, - "6": 39583309824.0, - "7": 39583309824.0, - "8": 39583309824.0, - "9": 39583309824.0, - "10": 39583309824.0, - "11": 39583309824.0, - "12": 39583309824.0, - "13": 39583309824.0, - "14": 39583309824.0, - "15": 39583309824.0, - "16": 39583309824.0, - "17": 39583309824.0, - "18": 39583309824.0, - "19": 39583309824.0, - "20": 39583309824.0, - "21": 39583309824.0, - "22": 39583309824.0, - "23": 39583309824.0, - "24": 39583309824.0, - "25": 39583309824.0, - "26": 39583309824.0, - "27": 39583309824.0, - "28": 39583309824.0, - "29": 39583309824.0, - "30": 39583309824.0, - "31": 39583309824.0, - "32": 39583309824.0, - "33": 39583309824.0, - "34": 39583309824.0, - "35": 39583309824.0, - "36": 39583309824.0, - "37": 39583309824.0, - "38": 39583309824.0, - "39": 39583309824.0, - "40": 39583309824.0, - "41": 39583309824.0, - "42": 39583309824.0, - "43": 39583309824.0, - "44": 39583309824.0, - "45": 39583309824.0, - "46": 39583309824.0, - "47": 39583309824.0, - "48": 39583309824.0, - "49": 39583309824.0, - "50": 39583309824.0 - } - }, - "iteration-time": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 60.48727, - "2": 2.0537, - "3": 3.26481, - "4": 2.56819, - "5": 2.40218, - "6": 1.26492, - "7": 1.5836, - "8": 1.37182, - "9": 1.10133, - "10": 1.10352, - "11": 1.18687, - "12": 1.53724, - "13": 1.25166, - "14": 1.69801, - "15": 1.42166, - "16": 1.104, - "17": 1.22214, - "18": 1.34911, - "19": 1.09323, - "20": 1.08552, - "21": 1.22223, - "22": 1.19712, - "23": 1.05456, - "24": 1.03745, - "25": 1.14154, - "26": 1.07349, - "27": 1.05181, - "28": 1.0364, - "29": 1.17111, - "30": 1.02943, - "31": 1.0758, - "32": 1.03304, - "33": 1.04107, - "34": 1.03092, - "35": 1.07869, - "36": 1.02457, - "37": 1.08557, - "38": 1.00729, - "39": 1.07249, - "40": 1.08655, - "41": 1.02362, - "42": 1.02046, - "43": 1.07618, - "44": 1.08709, - "45": 1.00443, - "46": 1.00379, - "47": 1.06019, - "48": 0.98958, - "49": 1.08317, - "50": 0.9932 - } - } -} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json deleted file mode 100644 index 537e20b09d8..00000000000 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_eos.json +++ /dev/null @@ -1,287 +0,0 @@ -{ - "lm loss": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 11.04266, - "2": 11.02309, - "3": 9.43552, - "4": 10.04614, - "5": 9.38535, - "6": 9.14543, - "7": 9.21141, - "8": 8.63458, - "9": 8.48937, - "10": 8.82763, - "11": 8.29457, - "12": 8.3282, - "13": 8.23008, - "14": 7.71714, - "15": 7.86981, - "16": 7.92286, - "17": 7.8604, - "18": 7.62039, - "19": 7.98493, - "20": 7.72023, - "21": 7.39758, - "22": 7.39771, - "23": 7.28314, - "24": 7.25048, - "25": 7.53113, - "26": 6.95329, - "27": 7.49432, - "28": 7.20394, - "29": 7.37282, - "30": 7.50232, - "31": 7.25348, - "32": 7.4305, - "33": 7.48364, - "34": 7.53486, - "35": 7.10336, - "36": 6.94516, - "37": 7.26117, - "38": 7.07009, - "39": 7.40543, - "40": 7.42044, - "41": 7.34202, - "42": 7.11816, - "43": 7.11373, - "44": 7.27067, - "45": 7.07036, - "46": 6.77823, - "47": 7.1875, - "48": 6.99998, - "49": 7.45868, - "50": 6.90956 - } - }, - "num-zeros": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 844114112.0, - "2": 843855104.0, - "3": 844048640.0, - "4": 842998144.0, - "5": 855786112.0, - "6": 874329728.0, - "7": 925591552.0, - "8": 915644608.0, - "9": 935187584.0, - "10": 927702400.0, - "11": 957888256.0, - "12": 923872512.0, - "13": 969427072.0, - "14": 965228416.0, - "15": 952825344.0, - "16": 943777088.0, - "17": 928845824.0, - "18": 925913856.0, - "19": 955339136.0, - "20": 989208256.0, - "21": 924095424.0, - "22": 908902272.0, - "23": 892664576.0, - "24": 900830400.0, - "25": 928105472.0, - "26": 877724352.0, - "27": 912808320.0, - "28": 904557696.0, - "29": 872625088.0, - "30": 864767104.0, - "31": 868220416.0, - "32": 861931136.0, - "33": 859941312.0, - "34": 855839104.0, - "35": 854046848.0, - "36": 852944896.0, - "37": 851456704.0, - "38": 849532096.0, - "39": 849972608.0, - "40": 849505792.0, - "41": 845780288.0, - "42": 846003328.0, - "43": 846257472.0, - "44": 852034880.0, - "45": 847187456.0, - "46": 855625856.0, - "47": 844661952.0, - "48": 851197248.0, - "49": 851630464.0, - "50": 846195904.0 - } - }, - "mem-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 4419107328.0, - "2": 4419108864.0, - "3": 4419108864.0, - "4": 4419108864.0, - "5": 4419108864.0, - "6": 4419108864.0, - "7": 4419108864.0, - "8": 4419108864.0, - "9": 4419108864.0, - "10": 4419108864.0, - "11": 4419108864.0, - "12": 4419108864.0, - "13": 4419108864.0, - "14": 4419108864.0, - "15": 4419108864.0, - "16": 4419108864.0, - "17": 4419108864.0, - "18": 4419108864.0, - "19": 4419108864.0, - "20": 4419108864.0, - "21": 4419108864.0, - "22": 4419108864.0, - "23": 4419108864.0, - "24": 4419108864.0, - "25": 4419108864.0, - "26": 4419108864.0, - "27": 4419108864.0, - "28": 4419108864.0, - "29": 4419108864.0, - "30": 4419108864.0, - "31": 4419108864.0, - "32": 4419108864.0, - "33": 4419108864.0, - "34": 4419108864.0, - "35": 4419108864.0, - "36": 4419108864.0, - "37": 4419108864.0, - "38": 4419108864.0, - "39": 4419108864.0, - "40": 4419108864.0, - "41": 4419108864.0, - "42": 4419108864.0, - "43": 4419108864.0, - "44": 4419108864.0, - "45": 4419108864.0, - "46": 4419108864.0, - "47": 4419108864.0, - "48": 4419108864.0, - "49": 4419108864.0, - "50": 4419108864.0 - } - }, - "mem-max-allocated-bytes": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 37959917568.0, - "2": 39578677248.0, - "3": 39580196864.0, - "4": 39580196864.0, - "5": 39583309824.0, - "6": 39583309824.0, - "7": 39583309824.0, - "8": 39583309824.0, - "9": 39583309824.0, - "10": 39583309824.0, - "11": 39583309824.0, - "12": 39583309824.0, - "13": 39583309824.0, - "14": 39583309824.0, - "15": 39583309824.0, - "16": 39583309824.0, - "17": 39583309824.0, - "18": 39583309824.0, - "19": 39583309824.0, - "20": 39583309824.0, - "21": 39583309824.0, - "22": 39583309824.0, - "23": 39583309824.0, - "24": 39583309824.0, - "25": 39583309824.0, - "26": 39583309824.0, - "27": 39583309824.0, - "28": 39583309824.0, - "29": 39583309824.0, - "30": 39583309824.0, - "31": 39583309824.0, - "32": 39583309824.0, - "33": 39583309824.0, - "34": 39583309824.0, - "35": 39583309824.0, - "36": 39583309824.0, - "37": 39583309824.0, - "38": 39583309824.0, - "39": 39583309824.0, - "40": 39583309824.0, - "41": 39583309824.0, - "42": 39583309824.0, - "43": 39583309824.0, - "44": 39583309824.0, - "45": 39583309824.0, - "46": 39583309824.0, - "47": 39583309824.0, - "48": 39583309824.0, - "49": 39583309824.0, - "50": 39583309824.0 - } - }, - "iteration-time": { - "start_step": 1, - "end_step": 50, - "step_interval": 1, - "values": { - "1": 67.13422, - "2": 1.95457, - "3": 3.25371, - "4": 2.66673, - "5": 3.05794, - "6": 1.35128, - "7": 1.66174, - "8": 2.19011, - "9": 1.16207, - "10": 1.16456, - "11": 1.26279, - "12": 1.60263, - "13": 1.29219, - "14": 2.93489, - "15": 1.48729, - "16": 1.15146, - "17": 1.27648, - "18": 1.39906, - "19": 1.13846, - "20": 1.14415, - "21": 1.27567, - "22": 1.26287, - "23": 1.11223, - "24": 1.10986, - "25": 1.20096, - "26": 1.13382, - "27": 1.11305, - "28": 1.11424, - "29": 1.22341, - "30": 1.08856, - "31": 1.15539, - "32": 1.10684, - "33": 1.11399, - "34": 1.09048, - "35": 1.1509, - "36": 1.09151, - "37": 1.13904, - "38": 1.06658, - "39": 1.1325, - "40": 1.14715, - "41": 1.07533, - "42": 1.08243, - "43": 1.13881, - "44": 1.14004, - "45": 1.06323, - "46": 1.06103, - "47": 1.11785, - "48": 1.04242, - "49": 1.13933, - "50": 1.0407 - } - } -} \ No newline at end of file From b2a261f4e0e713d4f4ac4b6b256a6f829b2f8349 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Thu, 6 Nov 2025 06:15:51 -0800 Subject: [PATCH 18/47] update golden values Signed-off-by: Hongbin Liu --- .../golden_values_dev_dgx_h100.json | 390 +++++++++--------- 1 file changed, 195 insertions(+), 195 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json index 1483224813a..f31e8584055 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -6,54 +6,54 @@ "values": { "1": 11.01686, "2": 11.06264, - "3": 10.17771, - "4": 10.86294, - "5": 9.81711, - "6": 9.10377, - "7": 9.61048, - "8": 8.39441, - "9": 7.79453, - "10": 7.15206, - "11": 9.06579, - "12": 12.40166, - "13": 8.04847, - "14": 8.24594, - "15": 8.24907, - "16": 8.32751, - "17": 8.35488, - "18": 7.58028, - "19": 8.18771, - "20": 7.71954, - "21": 8.00698, - "22": 7.35089, - "23": 7.95479, - "24": 7.51289, - "25": 8.32529, - "26": 7.78885, - "27": 7.72725, - "28": 7.71319, - "29": 7.77361, - "30": 7.56799, - "31": 7.85271, - "32": 6.52658, - "33": 7.24362, - "34": 7.80331, - "35": 7.74511, - "36": 6.73702, - "37": 8.15605, - "38": 7.62885, - "39": 7.97707, - "40": 7.52037, - "41": 7.52443, - "42": 6.12689, - "43": 7.60467, - "44": 7.96883, - "45": 6.84543, - "46": 7.42548, - "47": 7.82723, - "48": 7.87988, - "49": 7.59963, - "50": 6.85112 + "3": 10.17793, + "4": 10.86283, + "5": 9.81719, + "6": 9.10416, + "7": 9.61067, + "8": 8.39543, + "9": 7.79835, + "10": 7.15295, + "11": 9.06686, + "12": 12.40969, + "13": 8.05055, + "14": 8.2476, + "15": 8.25138, + "16": 8.32761, + "17": 8.33769, + "18": 7.57521, + "19": 8.18843, + "20": 7.70464, + "21": 8.00008, + "22": 7.35567, + "23": 7.9428, + "24": 7.49828, + "25": 8.31989, + "26": 7.79139, + "27": 7.72813, + "28": 7.70354, + "29": 7.77157, + "30": 7.56925, + "31": 7.85097, + "32": 6.53309, + "33": 7.24762, + "34": 7.79993, + "35": 7.74601, + "36": 6.74083, + "37": 8.15463, + "38": 7.62637, + "39": 7.97973, + "40": 7.52426, + "41": 7.52118, + "42": 6.11695, + "43": 7.60509, + "44": 7.96979, + "45": 6.84567, + "46": 7.4309, + "47": 7.82486, + "48": 7.87887, + "49": 7.59924, + "50": 6.85064 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 47167840.0, - "2": 46900628.0, - "3": 81003512.0, - "4": 243621808.0, - "5": 468555040.0, - "6": 561181184.0, - "7": 958267392.0, - "8": 720794112.0, - "9": 771164224.0, - "10": 718302016.0, - "11": 669618304.0, - "12": 559500096.0, - "13": 642601344.0, - "14": 754397952.0, - "15": 766531584.0, - "16": 697850240.0, - "17": 654906240.0, - "18": 745861824.0, - "19": 738620928.0, - "20": 887555328.0, - "21": 729800064.0, - "22": 666937216.0, - "23": 777389312.0, - "24": 607175552.0, - "25": 855782784.0, - "26": 846129152.0, - "27": 666477056.0, - "28": 830677504.0, - "29": 811523712.0, - "30": 657771072.0, - "31": 609501440.0, - "32": 784538816.0, - "33": 755198720.0, - "34": 729929280.0, - "35": 719482368.0, - "36": 699006208.0, - "37": 727900096.0, - "38": 711973824.0, - "39": 701515264.0, - "40": 682162752.0, - "41": 534678112.0, - "42": 655361792.0, - "43": 663463424.0, - "44": 642541952.0, - "45": 455907168.0, - "46": 613359936.0, - "47": 592108160.0, - "48": 585115008.0, - "49": 559483008.0, - "50": 544390208.0 + "1": 47167816.0, + "2": 46900776.0, + "3": 77860808.0, + "4": 237329376.0, + "5": 471709792.0, + "6": 558041536.0, + "7": 948826176.0, + "8": 723939584.0, + "9": 786891776.0, + "10": 734021888.0, + "11": 688478400.0, + "12": 553228736.0, + "13": 608009792.0, + "14": 741806976.0, + "15": 766532736.0, + "16": 685280512.0, + "17": 654899648.0, + "18": 730146112.0, + "19": 751163904.0, + "20": 884406592.0, + "21": 723541120.0, + "22": 805299648.0, + "23": 789975808.0, + "24": 610294016.0, + "25": 830610048.0, + "26": 824111232.0, + "27": 757678144.0, + "28": 774057088.0, + "29": 805232640.0, + "30": 770995712.0, + "31": 801384640.0, + "32": 790830656.0, + "33": 758341184.0, + "34": 726777280.0, + "35": 750934144.0, + "36": 717880064.0, + "37": 740480704.0, + "38": 724556544.0, + "39": 710957376.0, + "40": 716765760.0, + "41": 531516928.0, + "42": 658507328.0, + "43": 676045888.0, + "44": 680286208.0, + "45": 606880576.0, + "46": 641672384.0, + "47": 633002368.0, + "48": 607136576.0, + "49": 430551968.0, + "50": 563263808.0 } }, "mem-allocated-bytes": { @@ -178,53 +178,53 @@ "1": 4919527424.0, "2": 5861408768.0, "3": 5861408768.0, - "4": 5863651328.0, - "5": 5863651328.0, - "6": 5863651328.0, - "7": 5863651328.0, - "8": 5863651328.0, - "9": 5863651328.0, - "10": 5863651328.0, - "11": 5863651328.0, - "12": 5863651328.0, - "13": 5863651328.0, - "14": 5863651328.0, - "15": 5863986176.0, - "16": 5865795072.0, - "17": 5865795072.0, - "18": 5865795072.0, - "19": 5865795072.0, - "20": 5865795072.0, - "21": 5866987520.0, - "22": 5866987520.0, - "23": 5866987520.0, - "24": 5866987520.0, - "25": 5866987520.0, - "26": 5866987520.0, - "27": 5866987520.0, - "28": 5866987520.0, - "29": 5866987520.0, - "30": 5866987520.0, - "31": 5866987520.0, - "32": 5866987520.0, - "33": 5866987520.0, - "34": 5866987520.0, - "35": 5866987520.0, - "36": 5866987520.0, - "37": 5866987520.0, - "38": 5866987520.0, - "39": 5866987520.0, - "40": 5866987520.0, - "41": 5866987520.0, - "42": 5866987520.0, - "43": 5866987520.0, - "44": 5866987520.0, - "45": 5866987520.0, - "46": 5866987520.0, - "47": 5866987520.0, - "48": 5866987520.0, - "49": 5866987520.0, - "50": 5866987520.0 + "4": 5865549824.0, + "5": 5865549824.0, + "6": 5865549824.0, + "7": 5865549824.0, + "8": 5865549824.0, + "9": 5865549824.0, + "10": 5865549824.0, + "11": 5865549824.0, + "12": 5865549824.0, + "13": 5865549824.0, + "14": 5865549824.0, + "15": 5865549824.0, + "16": 5865549824.0, + "17": 5865549824.0, + "18": 5865549824.0, + "19": 5866154496.0, + "20": 5866154496.0, + "21": 5866154496.0, + "22": 5866154496.0, + "23": 5866154496.0, + "24": 5866154496.0, + "25": 5866154496.0, + "26": 5866154496.0, + "27": 5866154496.0, + "28": 5866154496.0, + "29": 5866154496.0, + "30": 5866154496.0, + "31": 5866154496.0, + "32": 5866154496.0, + "33": 5866154496.0, + "34": 5866154496.0, + "35": 5866154496.0, + "36": 5866154496.0, + "37": 5866154496.0, + "38": 5866154496.0, + "39": 5866154496.0, + "40": 5866154496.0, + "41": 5866154496.0, + "42": 5866154496.0, + "43": 5866154496.0, + "44": 5866154496.0, + "45": 5866154496.0, + "46": 5866154496.0, + "47": 5866154496.0, + "48": 5866154496.0, + "49": 5866154496.0, + "50": 5866154496.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 72.699, - "2": 4.27015, - "3": 3.87365, - "4": 3.67041, - "5": 3.65964, - "6": 3.48532, - "7": 3.47679, - "8": 3.47349, - "9": 3.43879, - "10": 3.47441, - "11": 3.45737, - "12": 3.48691, - "13": 3.54474, - "14": 3.44102, - "15": 3.42127, - "16": 3.45795, - "17": 3.49717, - "18": 3.51293, - "19": 3.5617, - "20": 3.49733, - "21": 3.50336, - "22": 3.62308, - "23": 3.50166, - "24": 3.49075, - "25": 3.50996, - "26": 3.44423, - "27": 3.47323, - "28": 3.53784, - "29": 3.51989, - "30": 3.49211, - "31": 3.49945, - "32": 3.4419, - "33": 3.50458, - "34": 3.47663, - "35": 3.45702, - "36": 3.50281, - "37": 3.44136, - "38": 3.45165, - "39": 3.50095, - "40": 3.50126, - "41": 3.50863, - "42": 3.46684, - "43": 3.55122, - "44": 3.48372, - "45": 3.46903, - "46": 3.47654, - "47": 3.51574, - "48": 3.4895, - "49": 3.49404, - "50": 3.45824 + "1": 86.37903, + "2": 4.30499, + "3": 5.51749, + "4": 4.16842, + "5": 5.35652, + "6": 3.7018, + "7": 3.68633, + "8": 3.75304, + "9": 3.67596, + "10": 3.70408, + "11": 3.70621, + "12": 3.71713, + "13": 3.73785, + "14": 3.64923, + "15": 3.63825, + "16": 3.64129, + "17": 3.71791, + "18": 3.69956, + "19": 4.27786, + "20": 4.04035, + "21": 3.67423, + "22": 3.66455, + "23": 3.67758, + "24": 4.16675, + "25": 3.71546, + "26": 3.71205, + "27": 3.71193, + "28": 3.60188, + "29": 3.69233, + "30": 3.68235, + "31": 3.69734, + "32": 3.69173, + "33": 3.64974, + "34": 3.73647, + "35": 3.68627, + "36": 3.70357, + "37": 3.71094, + "38": 3.72508, + "39": 3.70553, + "40": 3.6995, + "41": 3.61312, + "42": 3.63624, + "43": 3.68714, + "44": 3.70371, + "45": 3.67257, + "46": 3.73701, + "47": 3.69639, + "48": 3.65815, + "49": 3.63754, + "50": 3.71569 } } } \ No newline at end of file From 0a9b6940ab11eca577d805ffad28ffd734ecd33d Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 17 Nov 2025 19:53:37 -0800 Subject: [PATCH 19/47] support new TE version Signed-off-by: Hongbin Liu --- megatron/core/extensions/transformer_engine.py | 9 +++++---- .../fine_grained_activation_offload.py | 12 +++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index e807ee54fbf..b87e4194a7c 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -2121,8 +2121,9 @@ def set_save_original_input(module): try: # pylint: disable=unused-import - from transformer_engine.pytorch import cpu_offload - from transformer_engine.pytorch.float8_tensor import Float8Tensor + from transformer_engine.pytorch import cpu_offload_v1 as cpu_offload except ImportError: - Float8Tensor = None - cpu_offload = None + try: + from transformer_engine.pytorch import cpu_offload + except ImportError: + cpu_offload = None diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 1e280a09d35..a818d8486ed 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -200,6 +200,8 @@ def __enter__(self): if cpu_offload is not None: cpu_offload.CPUOffloadEnabled = True + else: + raise RuntimeError("TE CPU offload is not available") self.inside_context = True torch._C._autograd._push_saved_tensors_default_hooks( @@ -213,6 +215,8 @@ def __exit__(self, *args: Any): if cpu_offload is not None: cpu_offload.CPUOffloadEnabled = False + else: + raise RuntimeError("TE CPU offload is not available") self.inside_context = False torch._C._autograd._pop_saved_tensors_default_hooks() @@ -244,24 +248,18 @@ class ChunkOffloadHandler: def offload(src_tensor, pin_memory=True): """Offload.""" debug_rank("--------offload") - from megatron.core.extensions.transformer_engine import Float8Tensor - - fp8_offload = isinstance(src_tensor, Float8Tensor) if Float8Tensor is not None else False if not src_tensor.is_contiguous(): src_tensor = src_tensor.contiguous() cpu_backup = torch.empty( src_tensor.size(), - dtype=torch.uint8 if fp8_offload else src_tensor.dtype, + dtype=src_tensor.dtype, layout=src_tensor.layout, device="cpu", pin_memory=pin_memory, ) - if fp8_offload: - cpu_backup = Float8Tensor.make_like(src_tensor, data=cpu_backup) - cpu_backup.copy_(src_tensor, non_blocking=pin_memory) state = (src_tensor.device, cpu_backup) return state From 950b26f68ed2732b1513d3add4ac9ef492fe6f63 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 17 Nov 2025 20:37:04 -0800 Subject: [PATCH 20/47] move set_ideal_affinity_for_current_gpu to pipeline/util Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offload.py | 35 ++----------------- megatron/core/pipeline_parallel/utils.py | 33 +++++++++++++++++ 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index a818d8486ed..8397c519887 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -7,6 +7,8 @@ import torch +from megatron.core.pipeline_parallel.utils import set_ideal_affinity_for_current_gpu + # CPU offload implementation for pipeline parallelism DEBUG = False DEBUG_RANK = 0 @@ -22,39 +24,6 @@ def debug_rank(message): print(message) -def set_ideal_affinity_for_current_gpu(): - """Set CPU affinity for the current GPU to optimize host-device transfers.""" - import uuid - - try: - import cuda.bindings.driver as cuda_driver - import cuda.bindings.runtime as cuda_runtime - except ImportError: - try: - import cuda.cuda as cuda_driver - import cuda.cudart as cuda_runtime - except ImportError: - # print("cuda-python may not be installed, skipping GPU affinity setting") - warnings.warn("cuda-python may not be installed, skipping GPU affinity setting") - return - try: - import pynvml - except ImportError: - warnings.warn("pynvml is not installed, skipping GPU affinity setting") - return - - # Get current CUDA device ID - err, device_id = cuda_runtime.cudaGetDevice() - assert err == cuda_runtime.cudaError_t.cudaSuccess - # Get device UUID - err, device_uuid = cuda_driver.cuDeviceGetUuid(device_id) - assert err == cuda_driver.CUresult.CUDA_SUCCESS - # Set CPU affinity based on GPU's NUMA node - pynvml.nvmlInit() - handle = pynvml.nvmlDeviceGetHandleByUUID("GPU-" + str(uuid.UUID(bytes=device_uuid.bytes))) - pynvml.nvmlDeviceSetCpuAffinity(handle) - - class PipelineOffloadManager: """ Singleton manager for coordinating activation offloading across pipeline stages. diff --git a/megatron/core/pipeline_parallel/utils.py b/megatron/core/pipeline_parallel/utils.py index fae8e5466da..c50c6ac7964 100644 --- a/megatron/core/pipeline_parallel/utils.py +++ b/megatron/core/pipeline_parallel/utils.py @@ -80,6 +80,39 @@ def make_viewless(e): return e +def set_ideal_affinity_for_current_gpu(): + """Set CPU affinity for the current GPU to optimize host-device transfers.""" + import uuid + + try: + import cuda.bindings.driver as cuda_driver + import cuda.bindings.runtime as cuda_runtime + except ImportError: + try: + import cuda.cuda as cuda_driver + import cuda.cudart as cuda_runtime + except ImportError: + # print("cuda-python may not be installed, skipping GPU affinity setting") + warnings.warn("cuda-python may not be installed, skipping GPU affinity setting") + return + try: + import pynvml + except ImportError: + warnings.warn("pynvml is not installed, skipping GPU affinity setting") + return + + # Get current CUDA device ID + err, device_id = cuda_runtime.cudaGetDevice() + assert err == cuda_runtime.cudaError_t.cudaSuccess + # Get device UUID + err, device_uuid = cuda_driver.cuDeviceGetUuid(device_id) + assert err == cuda_driver.CUresult.CUDA_SUCCESS + # Set CPU affinity based on GPU's NUMA node + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByUUID("GPU-" + str(uuid.UUID(bytes=device_uuid.bytes))) + pynvml.nvmlDeviceSetCpuAffinity(handle) + + @contextmanager def stream_acquire_context(stream, event): """Stream acquire context""" From 495032a8970e91994b7bccadd1d1a966c3689715 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 17 Nov 2025 20:55:18 -0800 Subject: [PATCH 21/47] format Signed-off-by: Hongbin Liu --- .../core/pipeline_parallel/fine_grained_activation_offload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 8397c519887..138dcd8f7b1 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -1,6 +1,5 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -import warnings from collections import deque from contextlib import nullcontext from typing import Any From 2123564aeb3f9ddce7d0e3529ed645f397733467 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Sun, 30 Nov 2025 21:30:56 -0800 Subject: [PATCH 22/47] format Signed-off-by: Hongbin Liu --- megatron/core/extensions/transformer_engine.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index b87e4194a7c..1447840526b 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -2127,3 +2127,8 @@ def set_save_original_input(module): from transformer_engine.pytorch import cpu_offload except ImportError: cpu_offload = None +try: + # pylint: disable=unused-import + from transformer_engine.pytorch.float8_tensor import Float8Tensor +except ImportError: + Float8Tensor = None From 9d23103f3222f13f3808898f69739b0b56523969 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 9 Dec 2025 05:15:06 -0800 Subject: [PATCH 23/47] update golden value Signed-off-by: Hongbin Liu --- .../golden_values_dev_dgx_h100.json | 598 +++++++++--------- .../golden_values_dev_dgx_h100.json | 498 +++++++-------- 2 files changed, 548 insertions(+), 548 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json index e7f62bbe4af..dac60247e7d 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.06715, - "2": 11.06051, - "3": 10.21154, - "4": 9.95175, - "5": 10.12622, - "6": 8.82146, - "7": 9.52879, - "8": 8.442, - "9": 7.84738, - "10": 7.07075, - "11": 9.31042, - "12": 9.16013, - "13": 7.87292, - "14": 8.2102, - "15": 8.22483, - "16": 8.17879, - "17": 8.21121, - "18": 7.50325, - "19": 8.08274, - "20": 7.62562, - "21": 7.95058, - "22": 7.29789, - "23": 7.93775, - "24": 7.44169, - "25": 8.23817, - "26": 7.74959, - "27": 7.69344, - "28": 7.65487, - "29": 7.75173, - "30": 7.56007, - "31": 7.81567, - "32": 6.46589, - "33": 7.20401, - "34": 7.77921, - "35": 7.72944, - "36": 6.71776, - "37": 8.08311, - "38": 7.6137, - "39": 7.96476, - "40": 7.50072, - "41": 7.50304, - "42": 6.11349, - "43": 7.59404, - "44": 7.91361, - "45": 6.83615, - "46": 7.41293, - "47": 7.79226, - "48": 7.87549, - "49": 7.58763, - "50": 6.84525 + "1": 11.06693, + "2": 11.0602, + "3": 10.21194, + "4": 9.95432, + "5": 10.12218, + "6": 8.82302, + "7": 9.52771, + "8": 8.44293, + "9": 7.8503, + "10": 7.06875, + "11": 9.30824, + "12": 9.15133, + "13": 7.87105, + "14": 8.20684, + "15": 8.21969, + "16": 8.17636, + "17": 8.20544, + "18": 7.4911, + "19": 8.08112, + "20": 7.61907, + "21": 7.94753, + "22": 7.29291, + "23": 7.93581, + "24": 7.44247, + "25": 8.2355, + "26": 7.75421, + "27": 7.70115, + "28": 7.654, + "29": 7.75231, + "30": 7.56191, + "31": 7.81571, + "32": 6.47353, + "33": 7.20345, + "34": 7.77999, + "35": 7.72719, + "36": 6.71935, + "37": 8.08764, + "38": 7.6188, + "39": 7.96527, + "40": 7.49853, + "41": 7.50088, + "42": 6.11546, + "43": 7.59151, + "44": 7.91341, + "45": 6.83429, + "46": 7.40916, + "47": 7.78722, + "48": 7.87359, + "49": 7.58606, + "50": 6.84286 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 47165192.0, - "2": 46897912.0, - "3": 52684456.0, - "4": 297127552.0, - "5": 562950784.0, - "6": 668142144.0, - "7": 1027449536.0, - "8": 752259328.0, - "9": 830947776.0, - "10": 718307136.0, - "11": 823731840.0, - "12": 804867840.0, - "13": 639461056.0, - "14": 625408576.0, - "15": 716256960.0, - "16": 870866752.0, - "17": 673817856.0, - "18": 811900096.0, - "19": 892689024.0, - "20": 878114112.0, - "21": 666859968.0, - "22": 792718848.0, - "23": 783683200.0, - "24": 770686976.0, - "25": 651376640.0, - "26": 780070272.0, - "27": 801722496.0, - "28": 670273664.0, - "29": 647960768.0, - "30": 789867776.0, - "31": 801385856.0, - "32": 787688640.0, - "33": 783506816.0, - "34": 792837760.0, - "35": 776103936.0, - "36": 761920512.0, - "37": 775085824.0, - "38": 752868608.0, - "39": 754997184.0, - "40": 745075072.0, - "41": 713941440.0, - "42": 689968512.0, - "43": 663461824.0, - "44": 680285632.0, - "45": 644628992.0, - "46": 641672704.0, - "47": 642439616.0, - "48": 597700608.0, - "49": 603523520.0, - "50": 601014528.0 + "1": 47165232.0, + "2": 46897932.0, + "3": 49540668.0, + "4": 293969504.0, + "5": 559802048.0, + "6": 658711232.0, + "7": 1024298624.0, + "8": 752264576.0, + "9": 846702016.0, + "10": 686872128.0, + "11": 817431744.0, + "12": 804868736.0, + "13": 639449280.0, + "14": 622266240.0, + "15": 694217664.0, + "16": 836259456.0, + "17": 670662656.0, + "18": 692413632.0, + "19": 892693056.0, + "20": 865534016.0, + "21": 685732736.0, + "22": 802160192.0, + "23": 818284096.0, + "24": 792706240.0, + "25": 648186560.0, + "26": 770639552.0, + "27": 776554240.0, + "28": 774063488.0, + "29": 776921920.0, + "30": 786722048.0, + "31": 807679616.0, + "32": 800267392.0, + "33": 802385024.0, + "34": 770817664.0, + "35": 728918528.0, + "36": 724172608.0, + "37": 731043072.0, + "38": 718266048.0, + "39": 723535232.0, + "40": 723057728.0, + "41": 704505472.0, + "42": 674238976.0, + "43": 666610048.0, + "44": 680288064.0, + "45": 638337536.0, + "46": 619652032.0, + "47": 629857024.0, + "48": 603989568.0, + "49": 581503744.0, + "50": 560118272.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5290944000.0, - "2": 5291148800.0, - "3": 5291351552.0, - "4": 5290946048.0, - "5": 5291148800.0, - "6": 5291351552.0, - "7": 5291554304.0, - "8": 5291757056.0, - "9": 5291959808.0, - "10": 5292162560.0, - "11": 5292365312.0, - "12": 5292568064.0, - "13": 5292770816.0, - "14": 5292973568.0, - "15": 5293176320.0, - "16": 5293379072.0, - "17": 5293581824.0, - "18": 5293784576.0, - "19": 5293987328.0, - "20": 5294190080.0, - "21": 5294392832.0, - "22": 5294595584.0, - "23": 5294798336.0, - "24": 5295001088.0, - "25": 5295203840.0, - "26": 5295406592.0, - "27": 5295609344.0, - "28": 5295812096.0, - "29": 5296014848.0, - "30": 5296217600.0, - "31": 5296420352.0, - "32": 5296623104.0, - "33": 5296825856.0, - "34": 5297028608.0, - "35": 5297231360.0, - "36": 5297434112.0, - "37": 5297636864.0, - "38": 5297839616.0, - "39": 5298042368.0, - "40": 5298245120.0, - "41": 5298447872.0, - "42": 5298650624.0, - "43": 5298853376.0, - "44": 5299056128.0, - "45": 5299258880.0, - "46": 5299461632.0, - "47": 5299664384.0, - "48": 5299867136.0, - "49": 5300069888.0, - "50": 5300272640.0 + "1": 5279422976.0, + "2": 5279627776.0, + "3": 5279830528.0, + "4": 5279425024.0, + "5": 5279627776.0, + "6": 5279830528.0, + "7": 5280033280.0, + "8": 5280236032.0, + "9": 5280438784.0, + "10": 5280641536.0, + "11": 5280844288.0, + "12": 5281047040.0, + "13": 5281249792.0, + "14": 5281452544.0, + "15": 5281655296.0, + "16": 5281858048.0, + "17": 5282060800.0, + "18": 5282263552.0, + "19": 5282466304.0, + "20": 5282669056.0, + "21": 5282871808.0, + "22": 5283074560.0, + "23": 5283277312.0, + "24": 5283480064.0, + "25": 5283682816.0, + "26": 5283885568.0, + "27": 5284088320.0, + "28": 5284291072.0, + "29": 5284493824.0, + "30": 5284696576.0, + "31": 5284899328.0, + "32": 5285102080.0, + "33": 5285304832.0, + "34": 5285507584.0, + "35": 5285710336.0, + "36": 5285913088.0, + "37": 5286115840.0, + "38": 5286318592.0, + "39": 5286521344.0, + "40": 5286724096.0, + "41": 5286926848.0, + "42": 5287129600.0, + "43": 5287332352.0, + "44": 5287535104.0, + "45": 5287737856.0, + "46": 5287940608.0, + "47": 5288143360.0, + "48": 5288346112.0, + "49": 5288548864.0, + "50": 5288751616.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6180783616.0, - "2": 8225679872.0, - "3": 8225679872.0, - "4": 8225679872.0, - "5": 8225679872.0, - "6": 8225679872.0, - "7": 8225679872.0, - "8": 8225679872.0, - "9": 8225679872.0, - "10": 8225679872.0, - "11": 8239991296.0, - "12": 8239991296.0, - "13": 8239991296.0, - "14": 8239991296.0, - "15": 8239991296.0, - "16": 8239991296.0, - "17": 8244914688.0, - "18": 8244914688.0, - "19": 8244914688.0, - "20": 8265598464.0, - "21": 8265598464.0, - "22": 8265598464.0, - "23": 8265598464.0, - "24": 8265598464.0, - "25": 8265598464.0, - "26": 8265598464.0, - "27": 8265598464.0, - "28": 8265598464.0, - "29": 8271664640.0, - "30": 8316803584.0, - "31": 8316803584.0, - "32": 8316803584.0, - "33": 8316803584.0, - "34": 8316803584.0, - "35": 8316803584.0, - "36": 8316803584.0, - "37": 8316803584.0, - "38": 8316803584.0, - "39": 8318923264.0, - "40": 8318923264.0, - "41": 8318923264.0, - "42": 8318923264.0, - "43": 8318923264.0, - "44": 8318923264.0, - "45": 8318923264.0, - "46": 8318923264.0, - "47": 8318923264.0, - "48": 8318923264.0, - "49": 8318923264.0, - "50": 8318923264.0 + "1": 6173633024.0, + "2": 8218349056.0, + "3": 8218349056.0, + "4": 8218349056.0, + "5": 8218349056.0, + "6": 8218349056.0, + "7": 8218349056.0, + "8": 8218349056.0, + "9": 8218349056.0, + "10": 8218349056.0, + "11": 8230493696.0, + "12": 8230493696.0, + "13": 8230493696.0, + "14": 8230493696.0, + "15": 8230493696.0, + "16": 8230493696.0, + "17": 8230869504.0, + "18": 8231849472.0, + "19": 8231849472.0, + "20": 8267122176.0, + "21": 8267122176.0, + "22": 8267122176.0, + "23": 8267122176.0, + "24": 8267122176.0, + "25": 8267122176.0, + "26": 8267122176.0, + "27": 8267122176.0, + "28": 8267122176.0, + "29": 8267122176.0, + "30": 8294885888.0, + "31": 8294885888.0, + "32": 8294885888.0, + "33": 8294885888.0, + "34": 8294885888.0, + "35": 8294885888.0, + "36": 8294885888.0, + "37": 8294885888.0, + "38": 8295116288.0, + "39": 8315220480.0, + "40": 8315220480.0, + "41": 8315220480.0, + "42": 8315220480.0, + "43": 8315220480.0, + "44": 8315220480.0, + "45": 8315220480.0, + "46": 8315220480.0, + "47": 8315220480.0, + "48": 8315220480.0, + "49": 8315220480.0, + "50": 8315220480.0 } }, "mtp_1 loss": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.07395, + "1": 11.07401, "2": 11.0927, - "3": 10.82648, - "4": 10.27524, - "5": 10.45343, - "6": 8.32789, - "7": 9.82687, - "8": 8.01561, - "9": 7.47686, - "10": 6.75778, - "11": 8.92977, - "12": 8.98867, - "13": 7.80263, - "14": 8.02637, - "15": 8.11184, - "16": 8.13967, - "17": 8.13444, - "18": 7.44744, - "19": 8.03657, - "20": 7.53993, - "21": 7.90129, - "22": 7.27518, - "23": 7.88304, - "24": 7.37567, - "25": 8.16836, - "26": 7.69935, - "27": 7.6262, - "28": 7.61271, - "29": 7.69819, - "30": 7.4848, - "31": 7.73967, - "32": 6.36884, - "33": 7.14295, - "34": 7.71844, - "35": 7.63485, - "36": 6.61195, - "37": 8.02821, - "38": 7.57841, - "39": 7.89473, - "40": 7.41461, - "41": 7.42116, - "42": 6.01344, - "43": 7.4906, - "44": 7.86418, - "45": 6.74814, - "46": 7.30484, - "47": 7.72617, - "48": 7.79074, - "49": 7.49049, - "50": 6.75504 + "3": 10.82635, + "4": 10.27639, + "5": 10.45337, + "6": 8.32748, + "7": 9.82613, + "8": 8.01543, + "9": 7.47559, + "10": 6.75826, + "11": 8.92939, + "12": 8.98807, + "13": 7.80234, + "14": 8.02541, + "15": 8.1122, + "16": 8.1412, + "17": 8.13081, + "18": 7.44607, + "19": 8.03563, + "20": 7.53922, + "21": 7.9001, + "22": 7.27611, + "23": 7.88426, + "24": 7.37488, + "25": 8.16858, + "26": 7.69965, + "27": 7.62748, + "28": 7.61418, + "29": 7.69669, + "30": 7.48101, + "31": 7.74012, + "32": 6.36863, + "33": 7.13981, + "34": 7.72056, + "35": 7.63371, + "36": 6.61154, + "37": 8.02877, + "38": 7.57825, + "39": 7.89394, + "40": 7.41218, + "41": 7.42138, + "42": 6.01249, + "43": 7.49014, + "44": 7.86865, + "45": 6.7504, + "46": 7.30952, + "47": 7.72961, + "48": 7.7901, + "49": 7.4907, + "50": 6.75604 } }, "iteration-time": { @@ -289,56 +289,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 90.97535, - "2": 4.15413, - "3": 4.25282, - "4": 5.50314, - "5": 4.36528, - "6": 4.16016, - "7": 4.60989, - "8": 3.68392, - "9": 3.70951, - "10": 3.66417, - "11": 3.64904, - "12": 3.66094, - "13": 3.68824, - "14": 3.64996, - "15": 3.64159, - "16": 3.68269, - "17": 3.66905, - "18": 4.10783, - "19": 3.63362, - "20": 3.65129, - "21": 3.6431, - "22": 3.64946, - "23": 3.6411, - "24": 3.59707, - "25": 3.55364, - "26": 3.61478, - "27": 3.59779, - "28": 3.58741, - "29": 3.62545, - "30": 3.63538, - "31": 3.58264, - "32": 3.65914, - "33": 3.62764, - "34": 3.61962, - "35": 3.57076, - "36": 3.59244, - "37": 3.68499, - "38": 3.6803, - "39": 3.5849, - "40": 3.59019, - "41": 3.62068, - "42": 3.69144, - "43": 3.71863, - "44": 3.67193, - "45": 3.65673, - "46": 3.66919, - "47": 3.58334, - "48": 3.57229, - "49": 3.66195, - "50": 3.64157 + "1": 90.26185, + "2": 2.40864, + "3": 2.52164, + "4": 4.93801, + "5": 2.91719, + "6": 2.68307, + "7": 2.58408, + "8": 2.02548, + "9": 2.63344, + "10": 1.98392, + "11": 2.06707, + "12": 1.94587, + "13": 1.957, + "14": 2.00392, + "15": 2.54927, + "16": 2.01775, + "17": 1.96192, + "18": 1.99266, + "19": 1.99493, + "20": 1.95233, + "21": 1.93417, + "22": 1.95913, + "23": 1.94277, + "24": 1.97087, + "25": 1.95352, + "26": 2.00648, + "27": 2.00301, + "28": 1.99205, + "29": 1.99366, + "30": 1.97014, + "31": 1.97902, + "32": 1.97219, + "33": 1.9316, + "34": 1.9719, + "35": 1.97097, + "36": 1.94724, + "37": 1.97172, + "38": 1.99927, + "39": 1.95359, + "40": 1.96791, + "41": 1.9645, + "42": 1.91342, + "43": 1.94576, + "44": 1.96464, + "45": 1.96208, + "46": 1.98727, + "47": 1.97396, + "48": 1.96834, + "49": 1.95939, + "50": 1.96462 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json index f31e8584055..505236b8dfd 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.01686, - "2": 11.06264, - "3": 10.17793, - "4": 10.86283, - "5": 9.81719, - "6": 9.10416, - "7": 9.61067, - "8": 8.39543, - "9": 7.79835, - "10": 7.15295, - "11": 9.06686, - "12": 12.40969, - "13": 8.05055, - "14": 8.2476, - "15": 8.25138, - "16": 8.32761, - "17": 8.33769, - "18": 7.57521, - "19": 8.18843, - "20": 7.70464, - "21": 8.00008, - "22": 7.35567, - "23": 7.9428, - "24": 7.49828, - "25": 8.31989, - "26": 7.79139, - "27": 7.72813, - "28": 7.70354, - "29": 7.77157, - "30": 7.56925, - "31": 7.85097, - "32": 6.53309, - "33": 7.24762, - "34": 7.79993, - "35": 7.74601, - "36": 6.74083, - "37": 8.15463, - "38": 7.62637, - "39": 7.97973, - "40": 7.52426, - "41": 7.52118, - "42": 6.11695, - "43": 7.60509, - "44": 7.96979, - "45": 6.84567, - "46": 7.4309, - "47": 7.82486, - "48": 7.87887, - "49": 7.59924, - "50": 6.85064 + "1": 11.01693, + "2": 11.06263, + "3": 10.17794, + "4": 10.85994, + "5": 9.81692, + "6": 9.10097, + "7": 9.61229, + "8": 8.39605, + "9": 7.79605, + "10": 7.15189, + "11": 9.06657, + "12": 12.42312, + "13": 8.06867, + "14": 8.25318, + "15": 8.25396, + "16": 8.3337, + "17": 8.34707, + "18": 7.58571, + "19": 8.1941, + "20": 7.70065, + "21": 8.00269, + "22": 7.35729, + "23": 7.947, + "24": 7.50771, + "25": 8.3251, + "26": 7.78973, + "27": 7.72902, + "28": 7.7061, + "29": 7.77502, + "30": 7.5685, + "31": 7.85269, + "32": 6.53623, + "33": 7.24568, + "34": 7.80274, + "35": 7.74675, + "36": 6.73775, + "37": 8.15358, + "38": 7.62596, + "39": 7.97835, + "40": 7.52416, + "41": 7.52486, + "42": 6.12066, + "43": 7.60578, + "44": 7.96683, + "45": 6.8492, + "46": 7.42923, + "47": 7.82506, + "48": 7.88008, + "49": 7.59776, + "50": 6.85432 } }, "num-zeros": { @@ -62,55 +62,55 @@ "step_interval": 1, "values": { "1": 47167816.0, - "2": 46900776.0, - "3": 77860808.0, - "4": 237329376.0, - "5": 471709792.0, - "6": 558041536.0, - "7": 948826176.0, - "8": 723939584.0, - "9": 786891776.0, - "10": 734021888.0, - "11": 688478400.0, - "12": 553228736.0, - "13": 608009792.0, - "14": 741806976.0, - "15": 766532736.0, - "16": 685280512.0, - "17": 654899648.0, - "18": 730146112.0, - "19": 751163904.0, - "20": 884406592.0, - "21": 723541120.0, - "22": 805299648.0, - "23": 789975808.0, - "24": 610294016.0, - "25": 830610048.0, - "26": 824111232.0, - "27": 757678144.0, - "28": 774057088.0, - "29": 805232640.0, - "30": 770995712.0, - "31": 801384640.0, - "32": 790830656.0, - "33": 758341184.0, - "34": 726777280.0, - "35": 750934144.0, - "36": 717880064.0, - "37": 740480704.0, - "38": 724556544.0, - "39": 710957376.0, - "40": 716765760.0, - "41": 531516928.0, - "42": 658507328.0, - "43": 676045888.0, - "44": 680286208.0, - "45": 606880576.0, - "46": 641672384.0, - "47": 633002368.0, - "48": 607136576.0, - "49": 430551968.0, - "50": 563263808.0 + "2": 46900688.0, + "3": 71568288.0, + "4": 234203216.0, + "5": 474843232.0, + "6": 558036544.0, + "7": 964556864.0, + "8": 717673920.0, + "9": 815179776.0, + "10": 712034880.0, + "11": 638136896.0, + "12": 559512384.0, + "13": 611146240.0, + "14": 757545728.0, + "15": 769675776.0, + "16": 726164544.0, + "17": 661188736.0, + "18": 730133248.0, + "19": 735484160.0, + "20": 849803264.0, + "21": 814697920.0, + "22": 629166720.0, + "23": 793119360.0, + "24": 597724352.0, + "25": 846342144.0, + "26": 814671808.0, + "27": 628761728.0, + "28": 783493760.0, + "29": 786359488.0, + "30": 764702208.0, + "31": 600060544.0, + "32": 778251840.0, + "33": 764634688.0, + "34": 745651456.0, + "35": 741503424.0, + "36": 705295296.0, + "37": 731045120.0, + "38": 718271552.0, + "39": 736121472.0, + "40": 716764544.0, + "41": 528388096.0, + "42": 680529024.0, + "43": 682338048.0, + "44": 648832640.0, + "45": 619466368.0, + "46": 625945344.0, + "47": 623566592.0, + "48": 613425664.0, + "49": 600379264.0, + "50": 569557440.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4315544064.0, - "2": 4315545600.0, - "3": 4315545600.0, - "4": 4315545600.0, - "5": 4315545600.0, - "6": 4315545600.0, - "7": 4315545600.0, - "8": 4315545600.0, - "9": 4315545600.0, - "10": 4315545600.0, - "11": 4315545600.0, - "12": 4315545600.0, - "13": 4315545600.0, - "14": 4315545600.0, - "15": 4315545600.0, - "16": 4315545600.0, - "17": 4315545600.0, - "18": 4315545600.0, - "19": 4315545600.0, - "20": 4315545600.0, - "21": 4315545600.0, - "22": 4315545600.0, - "23": 4315545600.0, - "24": 4315545600.0, - "25": 4315545600.0, - "26": 4315545600.0, - "27": 4315545600.0, - "28": 4315545600.0, - "29": 4315545600.0, - "30": 4315545600.0, - "31": 4315545600.0, - "32": 4315545600.0, - "33": 4315545600.0, - "34": 4315545600.0, - "35": 4315545600.0, - "36": 4315545600.0, - "37": 4315545600.0, - "38": 4315545600.0, - "39": 4315545600.0, - "40": 4315545600.0, - "41": 4315545600.0, - "42": 4315545600.0, - "43": 4315545600.0, - "44": 4315545600.0, - "45": 4315545600.0, - "46": 4315545600.0, - "47": 4315545600.0, - "48": 4315545600.0, - "49": 4315545600.0, - "50": 4315545600.0 + "1": 4305059328.0, + "2": 4305060864.0, + "3": 4305060864.0, + "4": 4305060864.0, + "5": 4305060864.0, + "6": 4305060864.0, + "7": 4305060864.0, + "8": 4305060864.0, + "9": 4305060864.0, + "10": 4305060864.0, + "11": 4305060864.0, + "12": 4305060864.0, + "13": 4305060864.0, + "14": 4305060864.0, + "15": 4305060864.0, + "16": 4305060864.0, + "17": 4305060864.0, + "18": 4305060864.0, + "19": 4305060864.0, + "20": 4305060864.0, + "21": 4305060864.0, + "22": 4305060864.0, + "23": 4305060864.0, + "24": 4305060864.0, + "25": 4305060864.0, + "26": 4305060864.0, + "27": 4305060864.0, + "28": 4305060864.0, + "29": 4305060864.0, + "30": 4305060864.0, + "31": 4305060864.0, + "32": 4305060864.0, + "33": 4305060864.0, + "34": 4305060864.0, + "35": 4305060864.0, + "36": 4305060864.0, + "37": 4305060864.0, + "38": 4305060864.0, + "39": 4305060864.0, + "40": 4305060864.0, + "41": 4305060864.0, + "42": 4305060864.0, + "43": 4305060864.0, + "44": 4305060864.0, + "45": 4305060864.0, + "46": 4305060864.0, + "47": 4305060864.0, + "48": 4305060864.0, + "49": 4305060864.0, + "50": 4305060864.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4919527424.0, - "2": 5861408768.0, - "3": 5861408768.0, - "4": 5865549824.0, - "5": 5865549824.0, - "6": 5865549824.0, - "7": 5865549824.0, - "8": 5865549824.0, - "9": 5865549824.0, - "10": 5865549824.0, - "11": 5865549824.0, - "12": 5865549824.0, - "13": 5865549824.0, - "14": 5865549824.0, - "15": 5865549824.0, - "16": 5865549824.0, - "17": 5865549824.0, - "18": 5865549824.0, - "19": 5866154496.0, - "20": 5866154496.0, - "21": 5866154496.0, - "22": 5866154496.0, - "23": 5866154496.0, - "24": 5866154496.0, - "25": 5866154496.0, - "26": 5866154496.0, - "27": 5866154496.0, - "28": 5866154496.0, - "29": 5866154496.0, - "30": 5866154496.0, - "31": 5866154496.0, - "32": 5866154496.0, - "33": 5866154496.0, - "34": 5866154496.0, - "35": 5866154496.0, - "36": 5866154496.0, - "37": 5866154496.0, - "38": 5866154496.0, - "39": 5866154496.0, - "40": 5866154496.0, - "41": 5866154496.0, - "42": 5866154496.0, - "43": 5866154496.0, - "44": 5866154496.0, - "45": 5866154496.0, - "46": 5866154496.0, - "47": 5866154496.0, - "48": 5866154496.0, - "49": 5866154496.0, - "50": 5866154496.0 + "1": 4305061888.0, + "2": 5850930176.0, + "3": 5850930176.0, + "4": 5857143296.0, + "5": 5857143296.0, + "6": 5857143296.0, + "7": 5857143296.0, + "8": 5857143296.0, + "9": 5857143296.0, + "10": 5857143296.0, + "11": 5857143296.0, + "12": 5857143296.0, + "13": 5857143296.0, + "14": 5857143296.0, + "15": 5857143296.0, + "16": 5857143296.0, + "17": 5857143296.0, + "18": 5857143296.0, + "19": 5857143296.0, + "20": 5857143296.0, + "21": 5857143296.0, + "22": 5857143296.0, + "23": 5857143296.0, + "24": 5857143296.0, + "25": 5857143296.0, + "26": 5857143296.0, + "27": 5857143296.0, + "28": 5857143296.0, + "29": 5857143296.0, + "30": 5857143296.0, + "31": 5857143296.0, + "32": 5857143296.0, + "33": 5857143296.0, + "34": 5857143296.0, + "35": 5857143296.0, + "36": 5857143296.0, + "37": 5857143296.0, + "38": 5857143296.0, + "39": 5857143296.0, + "40": 5857143296.0, + "41": 5857143296.0, + "42": 5857143296.0, + "43": 5857143296.0, + "44": 5857143296.0, + "45": 5857143296.0, + "46": 5857143296.0, + "47": 5857143296.0, + "48": 5857143296.0, + "49": 5857143296.0, + "50": 5857143296.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 86.37903, - "2": 4.30499, - "3": 5.51749, - "4": 4.16842, - "5": 5.35652, - "6": 3.7018, - "7": 3.68633, - "8": 3.75304, - "9": 3.67596, - "10": 3.70408, - "11": 3.70621, - "12": 3.71713, - "13": 3.73785, - "14": 3.64923, - "15": 3.63825, - "16": 3.64129, - "17": 3.71791, - "18": 3.69956, - "19": 4.27786, - "20": 4.04035, - "21": 3.67423, - "22": 3.66455, - "23": 3.67758, - "24": 4.16675, - "25": 3.71546, - "26": 3.71205, - "27": 3.71193, - "28": 3.60188, - "29": 3.69233, - "30": 3.68235, - "31": 3.69734, - "32": 3.69173, - "33": 3.64974, - "34": 3.73647, - "35": 3.68627, - "36": 3.70357, - "37": 3.71094, - "38": 3.72508, - "39": 3.70553, - "40": 3.6995, - "41": 3.61312, - "42": 3.63624, - "43": 3.68714, - "44": 3.70371, - "45": 3.67257, - "46": 3.73701, - "47": 3.69639, - "48": 3.65815, - "49": 3.63754, - "50": 3.71569 + "1": 72.89456, + "2": 2.94117, + "3": 2.84855, + "4": 2.63467, + "5": 2.44999, + "6": 2.13944, + "7": 2.29092, + "8": 2.26625, + "9": 2.12101, + "10": 2.09852, + "11": 2.08307, + "12": 2.17401, + "13": 2.22063, + "14": 2.09013, + "15": 2.08464, + "16": 2.10017, + "17": 2.16623, + "18": 2.09249, + "19": 2.0884, + "20": 2.221, + "21": 2.09883, + "22": 2.11134, + "23": 2.10284, + "24": 2.09769, + "25": 2.09301, + "26": 2.11065, + "27": 2.10477, + "28": 2.10258, + "29": 2.09663, + "30": 2.13449, + "31": 2.08704, + "32": 2.09973, + "33": 2.22347, + "34": 2.09872, + "35": 2.10521, + "36": 2.13037, + "37": 2.09024, + "38": 2.11762, + "39": 2.09382, + "40": 2.09703, + "41": 2.08655, + "42": 2.1117, + "43": 2.12736, + "44": 2.09903, + "45": 2.09069, + "46": 2.0915, + "47": 2.14487, + "48": 2.09334, + "49": 2.10013, + "50": 2.12076 } } } \ No newline at end of file From 743ac560b12689c09b9165aae83ff84dc148bf18 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 5 Jan 2026 19:03:48 -0800 Subject: [PATCH 24/47] code refactor Signed-off-by: Hongbin Liu --- .../common/model_chunk_schedule_plan.py | 11 +- .../fine_grained_activation_offload.py | 886 +++++++++++++++--- megatron/core/pipeline_parallel/schedules.py | 17 +- megatron/core/pipeline_parallel/utils.py | 14 +- megatron/core/tensor_parallel/__init__.py | 4 + megatron/core/tensor_parallel/random.py | 78 +- megatron/core/transformer/cuda_graphs.py | 7 +- megatron/core/transformer/moe/experts.py | 4 +- megatron/core/transformer/moe/moe_utils.py | 21 +- .../transformer/multi_latent_attention.py | 21 +- .../transformer/multi_token_prediction.py | 5 - .../core/transformer/transformer_block.py | 8 - megatron/training/arguments.py | 7 +- megatron/training/checkpointing.py | 15 +- megatron/training/training.py | 6 + ...test_fine_grained_activation_offloading.py | 66 +- .../unit_tests/tensor_parallel/test_random.py | 145 +++ 17 files changed, 1052 insertions(+), 263 deletions(-) diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py index 74b9a90764d..9faa3540a84 100644 --- a/megatron/core/models/common/model_chunk_schedule_plan.py +++ b/megatron/core/models/common/model_chunk_schedule_plan.py @@ -8,9 +8,6 @@ from megatron.core.enums import Fp8Recipe from megatron.core.fp8_utils import get_fp8_context -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_set_last_layer, -) from megatron.core.pipeline_parallel.utils import ( AbstractSchedulePlan, NoopScheduleNode, @@ -451,10 +448,8 @@ def run( # combined forward and backward pass for overlapped layers for i in range(overlapped_layers): f_layer = f_schedule_plan.get_layer(i) - b_layer = b_schedule_plan.get_layer(b_num_layers - 1 - i) - torch.cuda.nvtx.range_push(f"layer_{i}f-layer_{b_num_layers - 1 - i}b") - if f_layer.layer.config.fine_grained_activation_offloading: - fine_grained_offloading_set_last_layer(i == f_num_layers - 1) + b_layer = b_schedule_plan.pop_layer() + torch.cuda.nvtx.range_push(f"layer_{i}f-layer_{b_schedule_plan.num_layers()}b") f_input, b_grad = TransformerLayerSchedulePlan.run( f_layer, b_layer, @@ -477,8 +472,6 @@ def run( for i in range(overlapped_layers, f_num_layers): f_layer = f_schedule_plan.get_layer(i) torch.cuda.nvtx.range_push(f"layer_{i}f") - if f_layer.layer.config.fine_grained_activation_offloading: - fine_grained_offloading_set_last_layer(i == f_num_layers - 1) f_input, _ = TransformerLayerSchedulePlan.run(f_layer, None, f_input=f_input) torch.cuda.nvtx.range_pop() diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 138dcd8f7b1..164ed3b14a0 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -2,16 +2,16 @@ from collections import deque from contextlib import nullcontext -from typing import Any +from typing import Any, Dict, Tuple import torch -from megatron.core.pipeline_parallel.utils import set_ideal_affinity_for_current_gpu - # CPU offload implementation for pipeline parallelism DEBUG = False DEBUG_RANK = 0 +from megatron.core.transformer.cuda_graphs import is_graph_capturing + def debug_rank(message): """Print debug message for a specific rank when DEBUG is enabled.""" @@ -23,6 +23,359 @@ def debug_rank(message): print(message) +def print_offload_summary_table(total_offload_bytes: Dict[str, int]): + """ + Print an ASCII table summarizing offload bytes across all ranks. + + Gathers offload data from all ranks and prints a formatted table on rank 0, + with rows representing ranks and columns representing groups. + + Args: + total_offload_bytes: Dict mapping group names to offload bytes for this rank. + """ + # pylint: disable=bad-builtin + assert torch.distributed.is_initialized() + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + + # Gather all group names across ranks + local_names = list(total_offload_bytes.keys()) + all_names_list = [None] * world_size + torch.distributed.all_gather_object(all_names_list, local_names) + all_group_names = sorted(set(name for names in all_names_list for name in names)) + + # Gather offload bytes from all ranks: each rank sends a list of bytes per group + local_bytes = [total_offload_bytes.get(name, 0) for name in all_group_names] + all_bytes_list = [None] * world_size + torch.distributed.all_gather_object(all_bytes_list, local_bytes) + + # Print ASCII table on rank 0 + if rank == 0: + # Calculate column widths + col_width = max(12, max((len(name) for name in all_group_names), default=8) + 2) + rank_col_width = max(6, len(f"Rank {world_size - 1}") + 2) + + # Build header + header = "Rank".ljust(rank_col_width) + header += "".join(name.rjust(col_width) for name in all_group_names) + header += "Total".rjust(col_width) + separator = "-" * len(header) + + print("\n" + "=" * len(header)) + print("Activation Offload Summary (MB)".center(len(header))) + print("=" * len(header)) + print(header) + print(separator) + + # Build rows for each rank + grand_total = 0 + col_totals = [0] * len(all_group_names) + for r in range(world_size): + row_bytes = all_bytes_list[r] + row_total = sum(row_bytes) + grand_total += row_total + for i, b in enumerate(row_bytes): + col_totals[i] += b + row_str = f"Rank {r}".ljust(rank_col_width) + for b in row_bytes: + row_str += f"{b / (1024 * 1024):.2f}".rjust(col_width) + row_str += f"{row_total / (1024 * 1024):.2f}".rjust(col_width) + print(row_str) + + # Print totals row + print(separator) + totals_row = "Total".ljust(rank_col_width) + for ct in col_totals: + totals_row += f"{ct / (1024 * 1024):.2f}".rjust(col_width) + totals_row += f"{grand_total / (1024 * 1024):.2f}".rjust(col_width) + print(totals_row) + print("=" * len(header) + "\n") + + torch.distributed.barrier() + + +class GPUTensorPool: + """ + GPU memory pool for efficient allocation and deallocation of tensors. + + Features: + - Supports multiple tensor shapes and dtypes, each with its own pool + - Dynamic allocation: tensors are created on-demand during allocation + - Efficient reuse: freed tensors are returned to the pool for reuse + - Uses queue-based management for O(1) allocation and deallocation + + Example: + pool = GPUTensorPool(device='cuda:0') + tensor = pool.allocate((128, 512), dtype=torch.float32) + # ... use tensor ... + pool.free(tensor, (128, 512), dtype=torch.float32) + """ + + def __init__(self, device: str = 'cuda', pin_memory: bool = False): + """ + Initialize GPU tensor pool. + + Args: + device: GPU device, default 'cuda' + pin_memory: Whether to use pinned memory (mainly for CPU tensors) + """ + self.device = torch.device(device) + self.pin_memory = pin_memory + + # Maintain a separate pool for each (shape, dtype) combination + # Structure: {(shape, dtype): {'free': deque, 'all': list, 'allocated_count': int}} + self._pools: Dict[Tuple, Dict[str, Any]] = {} + + # Statistics + self._stats = { + 'total_allocated': 0, # Total number of tensors ever allocated + 'current_in_use': 0, # Number of tensors currently in use + 'allocation_requests': 0, # Number of allocation requests + 'free_requests': 0, # Number of free requests + 'pool_hits': 0, # Number of times a tensor was reused from pool + 'pool_misses': 0, # Number of times a new tensor was created + } + + debug_rank("GPUTensorPool: Initialized with dynamic allocation") + + def _get_pool_key(self, shape: Tuple, dtype: torch.dtype) -> Tuple: + """Generate a unique key for the pool based on shape and dtype.""" + return (shape, dtype) + + @staticmethod + def _calculate_memory_size(shape: Tuple, dtype: torch.dtype) -> int: + """Calculate memory size in bytes.""" + element_size = torch.tensor([], dtype=dtype).element_size() + numel = 1 + for dim in shape: + numel *= dim + return numel * element_size + + def allocate(self, shape: Tuple, dtype: torch.dtype = torch.float32) -> torch.Tensor: + """ + Allocate a tensor with the specified shape and dtype. + + Args: + shape: Shape of the tensor + dtype: Data type of the tensor, default torch.float32 + + Returns: + Allocated tensor + """ + self._stats['allocation_requests'] += 1 + + pool_key = self._get_pool_key(shape, dtype) + + # Create pool for this (shape, dtype) if it doesn't exist + if pool_key not in self._pools: + self._pools[pool_key] = { + 'free': deque(), # Queue of available tensors + 'all': [], # List of all tensors (for tracking) + 'allocated_count': 0, # Number of allocated tensors + } + + pool = self._pools[pool_key] + + # Try to reuse a tensor from the pool + if len(pool['free']) > 0: + tensor = pool['free'].popleft() + self._stats['pool_hits'] += 1 + debug_rank( + f"GPUTensorPool.allocate: Reused tensor from pool, " + f"shape={shape}, dtype={dtype}, " + f"remaining in pool={len(pool['free'])}" + ) + else: + # Allocate a new tensor + tensor = torch.empty(shape, dtype=dtype, device=self.device, pin_memory=self.pin_memory) + pool['all'].append(tensor) + self._stats['total_allocated'] += 1 + self._stats['pool_misses'] += 1 + + memory_mb = self._calculate_memory_size(shape, dtype) / (1024**2) + debug_rank( + f"GPUTensorPool.allocate: Created new tensor, " + f"shape={shape}, dtype={dtype}, " + f"memory={memory_mb:.2f} MB, " + f"total_created={len(pool['all'])}" + ) + + pool['allocated_count'] += 1 + self._stats['current_in_use'] += 1 + + return tensor + + def free(self, tensor: torch.Tensor): + """ + Return a tensor to the pool for reuse. + + Args: + tensor: Tensor to free + + Raises: + ValueError: If tensor doesn't belong to this pool + """ + self._stats['free_requests'] += 1 + + shape = tensor.shape + dtype = tensor.dtype + + pool_key = self._get_pool_key(shape, dtype) + + if pool_key not in self._pools: + raise ValueError( + f"No pool exists for shape={shape}, dtype={dtype}. " + f"Available pools: {list(self._pools.keys())}" + ) + + pool = self._pools[pool_key] + + # Verify tensor belongs to this pool (use identity check, not value comparison) + tensor_found = any(tensor is t for t in pool['all']) + if not tensor_found: + raise ValueError( + f"Attempting to free a tensor that doesn't belong to this pool " + f"(shape={shape}, dtype={dtype})" + ) + + # Return tensor to the free queue + pool['free'].append(tensor) + pool['allocated_count'] -= 1 + self._stats['current_in_use'] -= 1 + + debug_rank( + f"GPUTensorPool.free: shape={shape}, dtype={dtype}, " + f"available in pool={len(pool['free'])}" + ) + + def get_pool_status(self, shape: Tuple = None, dtype: torch.dtype = None) -> Dict[str, Any]: + """ + Get the status of the memory pool. + + Args: + shape: If specified along with dtype, return status for that specific pool + dtype: Data type (required if shape is specified) + + Returns: + Dictionary containing status information + """ + if shape is not None: + if dtype is None: + raise ValueError("dtype must be specified when shape is provided") + + pool_key = self._get_pool_key(shape, dtype) + + if pool_key not in self._pools: + raise ValueError(f"No pool exists for shape={shape}, dtype={dtype}") + + pool = self._pools[pool_key] + total_count = len(pool['all']) + + return { + 'shape': shape, + 'dtype': dtype, + 'total_count': total_count, + 'allocated_count': pool['allocated_count'], + 'free_count': len(pool['free']), + 'utilization': ( + pool['allocated_count'] / total_count * 100 if total_count > 0 else 0 + ), + } + else: + # Return status for all pools + status = {'global_stats': self._stats.copy(), 'pools': {}} + + for pool_key in self._pools: + shape, dtype = pool_key + status['pools'][pool_key] = self.get_pool_status(shape, dtype) + + return status + + def reset(self): + """Reset the pool, marking all tensors as available.""" + debug_rank("GPUTensorPool: Resetting pool...") + + for pool_key, pool in self._pools.items(): + # Clear and refill the free queue + pool['free'].clear() + for tensor in pool['all']: + pool['free'].append(tensor) + pool['allocated_count'] = 0 + + self._stats['current_in_use'] = 0 + debug_rank("GPUTensorPool: Reset complete") + + def clear(self): + """Clear the pool and release all GPU memory.""" + debug_rank("GPUTensorPool: Clearing pool...") + + for pool_key, pool in self._pools.items(): + # Clear all references, allowing PyTorch GC to reclaim memory + pool['free'].clear() + pool['all'].clear() + + self._pools.clear() + self._stats['current_in_use'] = 0 + + # Trigger GPU cache cleanup + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + debug_rank("GPUTensorPool: Clear complete") + + def __del__(self): + """Destructor to ensure resources are released.""" + self.clear() + + +class OffloadTensorGroup: + """ + A group of tensors to be offloaded together. + """ + + def __init__(self, name): + self._name = name + self._tensors = {} + self._offload_event = torch.cuda.Event() + self._reload_event = torch.cuda.Event() + self.offload = True + self.total_offload_bytes = 0 + self.total_tensor_count = 0 + if name == "expert_fc1" or name == "moe_act": + self.use_cpu_pool = False + else: + self.use_cpu_pool = True + + def push_tensor(self, tag, tensor): + """Push a tensor to the group.""" + self._tensors[tag] = tensor + + def pop_tensor(self, tag): + """Pop a tensor from the group.""" + return self._tensors.pop(tag) + + def record_offload_event(self, stream): + """Record the offload event.""" + self._offload_event.record(stream) + + def wait_offload_event(self, stream): + """Wait for the offload event.""" + stream.wait_event(self._offload_event) + + def record_reload_event(self, stream): + """Record the reload event.""" + self._reload_event.record(stream) + + def wait_reload_event(self, stream): + """Wait for the reload event.""" + stream.wait_event(self._reload_event) + + def update_offload_info(self, tensor): + """Update the offload information.""" + self.total_offload_bytes += tensor.numel() * tensor.element_size() + self.total_tensor_count += 1 + + class PipelineOffloadManager: """ Singleton manager for coordinating activation offloading across pipeline stages. @@ -48,6 +401,20 @@ def __init__(self): # allocate streams and events for synchronization self._d2h_stream = torch.cuda.Stream() self._h2d_stream = torch.cuda.Stream() + # Shared CPU tensor pool for all chunks to improve reuse efficiency + self._cpu_tensor_pool = GPUTensorPool(device="cpu", pin_memory=True) + + self._is_warmup = True + self._cached_chunks_forward = [] + self._cached_chunks_backward = [] + self._cached_chunks_index_backward = 0 + self._cached_chunks_index_forward = 0 + + self.do_offload = True + + # Margin to avoid offloading too many groups so that + self._offload_margin = 0 + self._delayed_offload_groups = [] self.reset() @property @@ -60,14 +427,40 @@ def h2d_stream(self): """Get the host-to-device (CPU to GPU) transfer stream.""" return self._h2d_stream + @property + def cpu_tensor_pool(self): + """Get the shared CPU tensor pool.""" + return self._cpu_tensor_pool + + def push_offload_groups(self, group_hook, forced_released_tensors): + """Push the offload groups to the delayed queue.""" + debug_rank(f"pushing offload groups to the delayed queue") + self._delayed_offload_groups.append((group_hook, forced_released_tensors)) + + def flush_delayed_groups(self): + """Flush the delayed groups.""" + debug_rank("flushing delayed groups") + for group_hook, forced_released_tensors in reversed(self._delayed_offload_groups): + group_hook(forced_released_tensors) + self._delayed_offload_groups = [] + def reset(self): """Reset manager state for a new training iteration.""" - set_ideal_affinity_for_current_gpu() self._inside_context = False self._cur_forward_chunk = None self._cur_backward_chunk = None - # Track the first microbatch of the last virtual pipeline stage - self._is_first_last_vpp_chunk = True + # Reset CPU tensor pool to reuse all CPU tensors for next iteration + if hasattr(self, '_cpu_tensor_pool'): + self._cpu_tensor_pool.reset() + + if self._is_warmup and len(self._cached_chunks_forward) > 0: + self.post_warmup_callback() + self._cached_chunks_index_backward = 0 + self._cached_chunks_index_forward = 0 + + for chunk in self._cached_chunks_forward: + chunk.reset() + self._delayed_offload_groups = [] def flush(self): """Flush all staged chunks to the backward queue in reverse order.""" @@ -84,33 +477,100 @@ def flush(self): for i in range(self._vpp): self._stages[i] = [] + def disable_offload(self): + """Disable the offload.""" + debug_rank("disable_offload") + self.do_offload = False + for chunk in self._cached_chunks_forward: + chunk.do_offload = False + + def enable_offload(self): + """Enable the offload.""" + debug_rank("enable_offload") + self.do_offload = True + for chunk in self._cached_chunks_forward: + chunk.do_offload = True + + def post_warmup_callback(self): + """Callback after warmup.""" + # pylint: disable=bad-builtin + debug_rank("post_warmup_callback") + self._is_warmup = False + assert len(self._cached_chunks_forward) == len( + self._cached_chunks_backward + ), "Cached chunks forward and backward must have the same length" + for chunk in self._cached_chunks_forward: + chunk.is_warmup = False + assert ( + chunk in self._cached_chunks_backward + ), "Chunk not found in cached chunks backward" + # Update the offload margin to the maximum number of deduplicated groups + self._offload_margin = max(self._offload_margin, chunk.get_max_deduplicated_groups()) + debug_rank(f"offload margin {self._offload_margin}") + # Fine the last group with the same name in the cached chunks backward + last_group_with_same_name = {} + for chunk_idx, chunk in enumerate(reversed(self._cached_chunks_backward)): + for group in chunk.offload_groups: + last_group_with_same_name[group._name] = group + # Mark the last group with the same name as not offloadable to make sure + # the reloading won't block the main stream. + for name, group in last_group_with_same_name.items(): + if self._offload_margin > 0: + group.offload = False + self._offload_margin -= 1 + debug_rank(f"setting offload to false for group {name} at chunk index {chunk_idx}") + else: + break + debug_rank(f"offload margin {self._offload_margin}") + assert self._offload_margin == 0, "Offload margin is not 0" + # Dump the offload information + total_tensor_count = {} + total_offload_bytes = {} + for chunk in self._cached_chunks_backward: + for group in chunk.offload_groups: + if group.offload: + if group._name not in total_tensor_count: + total_tensor_count[group._name] = 0 + total_tensor_count[group._name] += group.total_tensor_count + if group._name not in total_offload_bytes: + total_offload_bytes[group._name] = 0 + total_offload_bytes[group._name] += group.total_offload_bytes + print_offload_summary_table(total_offload_bytes) + def push(self, handler): """Add a chunk handler to the backward queue.""" debug_rank(f"pushing handler {handler}") self._queue.append(handler) + if self._is_warmup: + self._cached_chunks_backward.append(handler) - def pop(self): + def pop(self, name=None): """Remove and set the next non-empty chunk as the current backward chunk.""" - assert self.size(), "Cannot pop from empty queue" - while self._queue: - self._cur_backward_chunk = self._queue.popleft() - if not self._cur_backward_chunk.is_empty_chunk(): + self._cur_backward_chunk = None + debug_rank(f"popping backward chunk {self._cached_chunks_index_backward}") + debug_rank(f"cached chunks backward {self._cached_chunks_backward}") + for idx, handler in enumerate( + self._cached_chunks_backward[self._cached_chunks_index_backward :] + ): + self._cached_chunks_index_backward += 1 + if not handler.is_empty_chunk(name): + self._cur_backward_chunk = ( + handler # set the first non-empty chunk as the current backward chunk + ) + debug_rank(f"handler {handler} at index {idx} is not empty") break - debug_rank(f"popping handler {self._cur_backward_chunk}") + assert self._cur_backward_chunk is not None, "No non-empty chunk found" - def front(self): + def front(self, name=None): """Get the first non-empty chunk handler without removing it from the queue.""" - if not self.size(): - return None - for chunk_handler in self._queue: - if not chunk_handler.is_empty_chunk(): - return chunk_handler + for idx, handler in enumerate( + self._cached_chunks_backward[self._cached_chunks_index_backward :] + ): + if not handler.is_empty_chunk(name): + debug_rank(f"front handler {handler} at index {idx}") + return handler return None - def size(self): - """Return the number of chunk handlers in the queue.""" - return len(self._queue) - def init_model_chunk_offload_handler( self, vp_size, vp_stage, min_offloaded_tensor_size=1024 * 1024 ): @@ -122,8 +582,11 @@ def init_model_chunk_offload_handler( vp_stage: Virtual pipeline stage index (None means stage 0) min_offloaded_tensor_size: Minimum tensor size (in elements) to offload """ + if not self._is_warmup: + return + + vp_size = 1 if vp_size is None else vp_size if self._stages is None: - vp_size = 1 if vp_size is None else vp_size self._vpp = vp_size self._stages = [[] for _ in range(vp_size)] @@ -132,26 +595,34 @@ def init_model_chunk_offload_handler( else: cur_vpp_rank = vp_stage - is_first_last_vpp_chunk = self._is_first_last_vpp_chunk # Flush staged chunks when reaching the last virtual pipeline stage if cur_vpp_rank == self._vpp - 1: self.flush() - # Determine if this is the first microbatch of the last virtual pipeline stage - is_first_last_vpp_chunk = is_first_last_vpp_chunk and (cur_vpp_rank == self._vpp - 1) - cur_chunk = ChunkOffloadHandler(is_first_last_vpp_chunk, min_offloaded_tensor_size) + # Use shared CPU tensor pool for better reuse across chunks + cur_chunk = ChunkOffloadHandler(min_offloaded_tensor_size, self._cpu_tensor_pool) + debug_rank(f"init_model_chunk_offload_handler {cur_chunk}") self._stages[cur_vpp_rank].append(cur_chunk) # For the last stage, push immediately and flush if cur_vpp_rank == self._vpp - 1: - self._is_first_last_vpp_chunk = False self.push(cur_chunk) self.flush() self._cur_forward_chunk = cur_chunk cur_chunk.vpp_rank = cur_vpp_rank - - def set_last_layer(self, is_last_layer): - """Mark whether the current forward chunk is processing the last layer.""" - self._cur_forward_chunk.is_last_layer = is_last_layer + self._cached_chunks_forward.append(cur_chunk) + + def pop_forward_chunk(self, name=None): + """Get the next forward pass chunk handler.""" + debug_rank(f"pop_forward_chunk {self._cur_forward_chunk}") + if not self.do_offload: + return self._cur_forward_chunk + while not self._is_warmup and ( + self._cur_forward_chunk is None or self._cur_forward_chunk.finish_all_groups(name) + ): + self._cur_forward_chunk = self._cached_chunks_forward[self._cached_chunks_index_forward] + self._cached_chunks_index_forward += 1 + debug_rank(f"new cur_forward_chunk {self._cur_forward_chunk}") + return self._cur_forward_chunk def cur_forward_chunk(self): """Get the current forward pass chunk handler.""" @@ -164,6 +635,8 @@ def cur_backward_chunk(self): def __enter__(self): """Enter context manager to enable activation offloading hooks.""" debug_rank("----__enter__") + if self._cur_forward_chunk is None or not self.cur_forward_chunk().do_offload: + return from megatron.core.extensions.transformer_engine import cpu_offload if cpu_offload is not None: @@ -179,6 +652,8 @@ def __enter__(self): def __exit__(self, *args: Any): """Exit context manager and restore original tensor saving behavior.""" debug_rank("----__exit__") + if self._cur_forward_chunk is None or not self.cur_forward_chunk().do_offload: + return from megatron.core.extensions.transformer_engine import cpu_offload if cpu_offload is not None: @@ -212,69 +687,101 @@ class ChunkOffloadHandler: Manages tensor groups, coordinates asynchronous GPU-CPU transfers, and handles synchronization. """ - @staticmethod - def offload(src_tensor, pin_memory=True): + def offload(self, src_tensor, pin_memory=True, use_cpu_pool=True): """Offload.""" debug_rank("--------offload") if not src_tensor.is_contiguous(): src_tensor = src_tensor.contiguous() - cpu_backup = torch.empty( - src_tensor.size(), - dtype=src_tensor.dtype, - layout=src_tensor.layout, - device="cpu", - pin_memory=pin_memory, - ) + if use_cpu_pool: + cpu_backup = self.cpu_tensor_pool.allocate(src_tensor.shape, dtype=src_tensor.dtype) + else: + cpu_backup = torch.empty( + src_tensor.shape, dtype=src_tensor.dtype, device="cpu", pin_memory=pin_memory + ) cpu_backup.copy_(src_tensor, non_blocking=pin_memory) - state = (src_tensor.device, cpu_backup) + state = (src_tensor.device, cpu_backup, use_cpu_pool) return state - @staticmethod - def reload(state, non_blocking=None): + def reload(self, state, non_blocking=None): """Reload.""" debug_rank("------reload") - dev, cpu_backup = state + dev, cpu_backup, use_cpu_pool = state if non_blocking is None: non_blocking = cpu_backup.is_pinned() - return cpu_backup.to(dev, non_blocking=non_blocking) + gpu_tensor = torch.empty( + cpu_backup.size(), dtype=cpu_backup.dtype, layout=cpu_backup.layout, device=dev + ) + gpu_tensor.copy_(cpu_backup, non_blocking=non_blocking) + if use_cpu_pool: + self.cpu_tensor_pool.free(cpu_backup) + return gpu_tensor - def __init__(self, is_first_last_vpp_chunk, min_offloaded_tensor_size): + def __init__(self, min_offloaded_tensor_size, cpu_tensor_pool): + self.do_offload = True # Data Structure to maintain reference to activation tensors self._tensor_tag_to_state = {} # Mark the first microbatch of the last virtual pipeline stage - self._is_first_last_vpp_chunk = is_first_last_vpp_chunk + # self._is_first_last_vpp_chunk = is_first_last_vpp_chunk # Group management for batching offload/reload operations self._offloaded_group_index = 0 self._groups_to_offload = [] self._groups_to_reload = [] self._tensor_count_current_group = 0 + self._max_group_size = 0 # Counter for special torch tensor types (FakeTensor, FunctionalTensor) self.torch_tensor_count = 0 self.d2h_stream = PipelineOffloadManager.get_instance().d2h_stream self.h2d_stream = PipelineOffloadManager.get_instance().h2d_stream - self._offload_events = {} - self._reload_events = {} self.min_offloaded_tensor_size = min_offloaded_tensor_size - self.is_last_layer = False + self.cpu_tensor_pool = cpu_tensor_pool + self.offload_groups = [] + self.is_warmup = True - def is_empty_chunk(self): - """Check if this chunk has no tensors to manage.""" - return len(self._tensor_tag_to_state) == 0 + def reset(self): + """Reset the chunk offload handler.""" + self._offloaded_group_index = 0 + self._groups_to_offload = [] + self._groups_to_reload = [] + self._tensor_count_current_group = 0 - def is_first_last_layer(self): - """ - Check if this is the last layer of the first microbatch of the last vp stage. - These tensors should not be offloaded to avoid unnecessary overhead. - """ + def is_empty_chunk(self, name=None): + """Check if this chunk has no tensors to manage.""" + debug_rank(f"------is_empty_chunk {self._max_group_size}") + if name is not None: + for group in self.offload_groups: + debug_rank(f"group name {group._name} need name {name}") + if group._name == name: + return False + return True + return self._max_group_size == 0 + + def finish_all_groups(self, name=None) -> bool: + """Finish all groups.""" debug_rank( - f"------is_first_last_layer {self._is_first_last_vpp_chunk} {self.is_last_layer}" + f"------finish_all_groups {self} {self._max_group_size} {self._offloaded_group_index}" ) - return self._is_first_last_vpp_chunk and self.is_last_layer + # TODO: check if this is correct + # Mark it as finished when all groups are finished and there are no groups to offload or reload + if len(self._groups_to_reload) == 0 and len(self._groups_to_offload) == 0 and self._offloaded_group_index > 0: + return True + assert name is not None, "Name is required" + for group in self.offload_groups[self._offloaded_group_index :]: + if group._name == name: + return False + return True + + def find_next_group(self, name=None): + """Find the next group with the given name.""" + assert name is not None, "Name is required" + for group in self.offload_groups[self._offloaded_group_index :]: + if group._name == name: + return group + return None def tensor_push(self, tensor): """Push tensor to the offload handler.""" @@ -285,27 +792,32 @@ def tensor_push(self, tensor): torch._subclasses.functional_tensor.FunctionalTensor, ), ) + assert not torch_stray_tensor, "Stray tensor should not be offloaded" if not torch_stray_tensor: # Assign unique tag based on group index and position within group tensor_tag = (self._offloaded_group_index, self._tensor_count_current_group) self._tensor_count_current_group += 1 - assert tensor_tag not in self._tensor_tag_to_state, "Duplicate tensor tag" - self._tensor_tag_to_state[tensor_tag] = tensor + # assert tensor_tag not in self._tensor_tag_to_state, "Duplicate tensor tag" + # self._tensor_tag_to_state[tensor_tag] = tensor + self.offload_groups[self._offloaded_group_index - 1].push_tensor(tensor_tag, tensor) else: # Use negative group ID for special tensor types tensor_tag = (-1, self.torch_tensor_count) self.torch_tensor_count += 1 - self._tensor_tag_to_state[tensor_tag] = tensor + # self._tensor_tag_to_state[tensor_tag] = tensor debug_rank(f"--------tensor_push {tensor_tag}") return tensor_tag def tensor_pop(self, tensor_tag): """Pop tensor from the offload handler.""" debug_rank(f"--------tensor_pop {tensor_tag}") - assert tensor_tag in self._tensor_tag_to_state, f"Tag {tensor_tag} not found" - tensor = self._tensor_tag_to_state.pop(tensor_tag) + # assert tensor_tag in self._tensor_tag_to_state, f"Tag {tensor_tag} not found" + # tensor = self._tensor_tag_to_state.pop(tensor_tag) + group_id, idx = tensor_tag + tensor = self.offload_groups[group_id - 1].pop_tensor(tensor_tag) # If tensor is offloaded (stored as tuple), reload it + # assert isinstance(tensor, torch.Tensor), "Tensor is not a tensor" if isinstance(tensor, tuple): tensor = self.reload(tensor) debug_rank(f"--------tensor_pop {tensor.shape}") @@ -313,6 +825,9 @@ def tensor_pop(self, tensor_tag): def tensor_need_offloading_checker(self, tensor): """Check if the tensor needs to be offloaded.""" + debug_rank( + f"tensor_need_offloading_checker {getattr(tensor, 'offloading_activation', None)}" + ) if tensor.numel() < self.min_offloaded_tensor_size: return False # Respect tensor's offload preference if specified @@ -320,83 +835,78 @@ def tensor_need_offloading_checker(self, tensor): return False return True - def bulk_offload_group(self, group_to_offload): + def bulk_offload_group(self): """offload a group of tensors recorded in tensor_push().""" debug_rank("------bulk_offload_group") - assert not self.is_first_last_layer(), "Should not offload first-last layer" - group_id_to_offload, name = group_to_offload - torch.cuda.nvtx.range_push("activation offloading " + name) + group_to_offload = self._groups_to_offload[-1] + torch.cuda.nvtx.range_push("activation offloading " + group_to_offload._name) with torch.cuda.stream(self.d2h_stream): - for tensor_tag, state in self._tensor_tag_to_state.items(): - group_id, _ = tensor_tag - if group_id == group_id_to_offload: - debug_rank(f"------tensor_tag {tensor_tag}") - debug_rank(f"------group_to_offload {group_to_offload}") - assert not isinstance(state, tuple), "Tensor already offloaded" - tensor_on_device = state - if self.tensor_need_offloading_checker(tensor_on_device): - state = self.offload(tensor_on_device) - event = torch.cuda.Event() - event.record(self.d2h_stream) - self._offload_events[name] = event - tensor_on_device.record_stream(self.d2h_stream) - self._tensor_tag_to_state[tensor_tag] = state + # for tensor_tag, state in self._tensor_tag_to_state.items(): + for tensor_tag, tensor_on_device in group_to_offload._tensors.items(): + if self.tensor_need_offloading_checker(tensor_on_device): + state = self.offload( + tensor_on_device, use_cpu_pool=group_to_offload.use_cpu_pool + ) + if self.is_warmup: + group_to_offload.update_offload_info(tensor_on_device) + tensor_on_device.record_stream(self.d2h_stream) + group_to_offload.push_tensor(tensor_tag, state) + group_to_offload.record_offload_event(self.d2h_stream) + self._groups_to_offload.pop() torch.cuda.nvtx.range_pop() - def get_offload_event(self, name): - """Get the CUDA event for a named offload operation.""" - return self._offload_events.get(name, None) - - def get_reload_event(self, name): - """Get the CUDA event for a named reload operation.""" - return self._reload_events.get(name, None) + def get_max_deduplicated_groups(self): + """Get the maximum number of deduplicated groups.""" + count_modules = [] + for group in self.offload_groups: + if group._name not in count_modules: + count_modules.append(group._name) + return len(count_modules) - def bulk_reload_group(self, group_to_reload): + def bulk_reload_group(self): """Bulk reload group.""" debug_rank("----bulk_reload_group") - found_reload_group = False - group_id_to_reload, name = group_to_reload - torch.cuda.nvtx.range_push("activation reloading " + name) + group_to_reload = self._groups_to_reload[-1] + torch.cuda.nvtx.range_push("activation reloading " + group_to_reload._name) with torch.cuda.stream(self.h2d_stream): - for tensor_label, state in self._tensor_tag_to_state.items(): - group_id, _ = tensor_label - if group_id == group_id_to_reload: - debug_rank(f"----tensor_label {tensor_label}") - found_reload_group = True - event = self.get_offload_event(name) - # Only reload if tensor was offloaded (stored as tuple) - if isinstance(state, tuple): - # Wait for offload to complete before reloading - torch.cuda.current_stream().wait_event(event) - recovered_tensor = self.reload(state) - event.record(self.h2d_stream) - self._reload_events[name] = event - debug_rank(f"----recovered_tensor {recovered_tensor.shape}") - self._tensor_tag_to_state[tensor_label] = recovered_tensor + # Wait for offload to complete before reloading + if not is_graph_capturing(): + group_to_reload.wait_offload_event(self.h2d_stream) + for tensor_tag, state in group_to_reload._tensors.items(): + # Only reload if tensor was offloaded (stored as tuple) + if isinstance(state, tuple): + recovered_tensor = self.reload(state) + debug_rank(f"----recovered_tensor {recovered_tensor.shape}") + group_to_reload.push_tensor(tensor_tag, recovered_tensor) + group_to_reload.record_reload_event(self.h2d_stream) + self._groups_to_reload.pop() torch.cuda.nvtx.range_pop() - return found_reload_group def pre_reload_last_layer(self): """Pre-reload the last layer of this chunk to hide reload latency.""" debug_rank("pre_reload_last_layer") - assert not self._is_first_last_vpp_chunk, "Should not pre-reload first chunk" debug_rank(f"len(self._groups_to_reload) {len(self._groups_to_reload)}") if len(self._groups_to_reload) > 0: # Reload the last group (last layer) early - if self.bulk_reload_group(self._groups_to_reload[-1]): - self._groups_to_reload.pop() + self.bulk_reload_group() def should_bulk_offload(self): """Determine if the current group should be offloaded.""" - # Don't offload the first backward chunk's last layer - if self.is_first_last_layer(): + group = self._groups_to_offload[-1] + debug_rank(f"should_bulk_offload {self.is_warmup} {group.offload}") + # Don't offload if the chunk is not in warmup stage + if self.is_warmup: + return True + # Don't offload if the group is marked as not offloadable + if not group.offload: return False # Check if next backward chunk is this chunk (for last pipeline stage) - next_backward_chunk = PipelineOffloadManager.get_instance().front() + next_backward_chunk = PipelineOffloadManager.get_instance().front(name=group._name) if next_backward_chunk is not None and next_backward_chunk is self: - # Don't offload last layer if it's about to be used immediately - if self.is_last_layer: + # Don't offload the last group with the same name if it's about to be used immediately + if self.find_next_group(group._name) is None: + debug_rank(f"next group {group._name} is not found") return False return True @@ -405,9 +915,8 @@ def bulk_offload(self, forced_released_tensors): """Offload a group of tensors and optionally release their GPU memory.""" debug_rank("----bulk_offload") if self.should_bulk_offload(): - group_to_offload = self._groups_to_offload.pop() - self._groups_to_reload.append(group_to_offload) - self.bulk_offload_group(group_to_offload) + self._groups_to_reload.append(self._groups_to_offload[-1]) + self.bulk_offload_group() # Manually release tensors not auto-freed by torch GC if len(forced_released_tensors) > 0: cur_stream = torch.cuda.current_stream() @@ -419,6 +928,8 @@ def bulk_offload(self, forced_released_tensors): def on_group_commit_forward(self, forced_released_tensors): """Called at the end of a layer group's forward pass to trigger offloading.""" + if not self.do_offload: + return debug_rank("--on_group_commit_forward") # Wait for compute to finish before starting offload self.d2h_stream.wait_stream(torch.cuda.current_stream()) @@ -429,9 +940,7 @@ def bulk_reload(self): debug_rank("--bulk_reload") if len(self._groups_to_reload) > 0: # Reload the next layer group - if self.bulk_reload_group(self._groups_to_reload[-1]): - debug_rank(f"--bulk_reload_group {self._groups_to_reload}") - self._groups_to_reload.pop() + self.bulk_reload_group() else: # Pre-load the last layer of the next backward chunk to hide latency next_backward_chunk = PipelineOffloadManager.get_instance().front() @@ -443,40 +952,71 @@ def on_group_commit_backward(self, name): Called at the end of a layer group's backward pass. Ensures correct chunk is active and synchronizes reloads. """ + if not self.do_offload: + return debug_rank("--on_group_commit_backward") cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() # Switch to this chunk if it's not already current if cur_backward_chunk is not self: - PipelineOffloadManager.get_instance().pop() + PipelineOffloadManager.get_instance().pop(name) cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() - assert cur_backward_chunk is self, "Chunk mismatch" + assert cur_backward_chunk is self, f"Chunk mismatch {cur_backward_chunk} {self}" # Wait for reload to complete before using tensors - event = self.get_reload_event(name) - if event is not None: - torch.cuda.current_stream().wait_event(event) - self._offloaded_group_index = self._offloaded_group_index - 1 + if not is_graph_capturing() and len(self._groups_to_reload) > 0: + group_to_reload = self._groups_to_reload[-1] + group_to_reload.wait_reload_event(torch.cuda.current_stream()) def on_group_start_forward(self, name): """ Called at the start of a layer group's forward pass. Increments group index and prepares for offloading. """ - debug_rank(f"--on_group_start_forward") - self._offloaded_group_index = self._offloaded_group_index + 1 + if not self.do_offload: + return + debug_rank(f"--on_group_start_forward {name}") + if self.is_warmup: + self._offloaded_group_index = self._offloaded_group_index + 1 + self.offload_groups.append(OffloadTensorGroup(name)) + self._max_group_size = max(self._max_group_size, self._offloaded_group_index) + debug_rank(f"max group size {self._max_group_size}") + else: + self._offloaded_group_index = self._offloaded_group_index + 1 + for group in self.offload_groups[self._offloaded_group_index - 1 :]: + debug_rank( + f"offloaded group index {self._offloaded_group_index} for group {group._name}" + ) + if group._name == name: + break + self._offloaded_group_index = self._offloaded_group_index + 1 self._tensor_count_current_group = 0 - self._groups_to_offload.append((self._offloaded_group_index, name)) + self._groups_to_offload.append(self.offload_groups[self._offloaded_group_index - 1]) + debug_rank(f"groups to offload {self._groups_to_offload}") def on_group_start_backward(self): """ Called at the start of a layer group's backward pass. Triggers reloading of tensors from CPU. """ + if not self.do_offload: + return debug_rank("--on_group_start_backward") # Wait for compute to finish before starting reload self.h2d_stream.wait_stream(torch.cuda.current_stream()) self.bulk_reload() +def fine_grained_offloading_disable_offload(): + """Disable the offload.""" + debug_rank("fine_grained_offloading_disable_offload") + PipelineOffloadManager.get_instance().disable_offload() + + +def fine_grained_offloading_enable_offload(): + """Enable the offload.""" + debug_rank("fine_grained_offloading_enable_offload") + PipelineOffloadManager.get_instance().enable_offload() + + class FineGrainedOffloadingGroupCommitFunction(torch.autograd.Function): """ Identity operation that marks the end of a layer group for offload synchronization. @@ -488,15 +1028,19 @@ def forward(ctx, *args): # pylint: disable=missing-function-docstring debug_rank("FineGrainedOffloadingGroupCommitFunction forward") - forced_released_tensors = args[-1] - name = args[-2] - cpu_offload_handler = args[-3] - tensor = args[:-3] - cpu_offload_handler.on_group_commit_forward(forced_released_tensors) + delay_offload = args[-1] + forced_released_tensors = args[-2] + name = args[-3] + cpu_offload_handler = args[-4] + tensor = args[:-4] + if delay_offload: + PipelineOffloadManager.get_instance().push_offload_groups( + cpu_offload_handler.on_group_commit_forward, forced_released_tensors + ) + else: + cpu_offload_handler.on_group_commit_forward(forced_released_tensors) ctx.cpu_offload_handler = cpu_offload_handler ctx.name = name - - # return the identical tensor return tensor @staticmethod @@ -506,10 +1050,12 @@ def backward(ctx, *grad_output): cpu_offload_handler = ctx.cpu_offload_handler cpu_offload_handler.on_group_commit_backward(ctx.name) - return grad_output + (None, None, None) + return grad_output + (None, None, None, None) -def fine_grained_offloading_group_commit(*tensor, name, forced_released_tensors=[]): +def fine_grained_offloading_group_commit( + *tensor, name, forced_released_tensors=[], delay_offload=False +): """ Specify the tensors to be released after offloading. forced_released_tensors is a list of tensors to be released after offloading. @@ -517,11 +1063,19 @@ def fine_grained_offloading_group_commit(*tensor, name, forced_released_tensors= Note: specify the tensors only when they are not automatically released by torch gc. """ cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk() + if cur_forward_chunk is None: + return tensor return FineGrainedOffloadingGroupCommitFunction.apply( - *tensor, cur_forward_chunk, name, forced_released_tensors + *tensor, cur_forward_chunk, name, forced_released_tensors, delay_offload ) +def fine_grained_offloading_group_flush_delayed_groups(): + """Flush the delayed groups.""" + debug_rank("fine_grained_offloading_group_flush_delayed_groups") + PipelineOffloadManager.get_instance().flush_delayed_groups() + + class FineGrainedOffloadingGroupStartFunction(torch.autograd.Function): """ Identity operation that marks the start of a layer group for offload/reload. @@ -544,12 +1098,14 @@ def backward(ctx, grad_output): debug_rank("FineGrainedOffloadingGroupStartFunction backward") cpu_offload_handler = ctx.cpu_offload_handler cpu_offload_handler.on_group_start_backward() - return grad_output, None, None + return grad_output, None, None, None def fine_grained_offloading_group_start(tensor, name=None): """Mark the start of a layer group and prepare for offload/reload.""" - cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk() + cur_forward_chunk = PipelineOffloadManager.get_instance().pop_forward_chunk(name=name) + if cur_forward_chunk is None: + return tensor return FineGrainedOffloadingGroupStartFunction.apply(tensor, cur_forward_chunk, name) @@ -558,11 +1114,6 @@ def get_fine_grained_offloading_context(flag): return PipelineOffloadManager.get_instance() if flag else nullcontext() -def fine_grained_offloading_set_last_layer(is_last_layer): - """Set the last layer flag.""" - PipelineOffloadManager.get_instance().set_last_layer(is_last_layer) - - def fine_grained_offloading_init_chunk_handler(vp_size, vp_stage, min_offloaded_tensor_size): """Initialize the chunk handler, called at the start of a microbatch forward pass.""" PipelineOffloadManager.get_instance().init_model_chunk_offload_handler( @@ -573,3 +1124,36 @@ def fine_grained_offloading_init_chunk_handler(vp_size, vp_stage, min_offloaded_ def fine_grained_offloading_reset(): """Reset the chunk handler, called at the start of a training iteration.""" PipelineOffloadManager.get_instance().reset() + + +def fine_grained_offloading_forward_record(event: torch.cuda.Event) -> None: + """Record the forward event for cuda graph capture.""" + d2h_stream = PipelineOffloadManager.get_instance().d2h_stream + torch.cuda.current_stream().record_event(event) + torch.cuda.current_stream().wait_stream(d2h_stream) + + +class FineGrainedOffloadingBackwardRecordFunction(torch.autograd.Function): + """ + Identity operation that marks the end of a layer group for offload synchronization. + Triggers offload during forward and synchronizes reload during backward. + """ + + @staticmethod + def forward(ctx, tensor, event: torch.cuda.Event) -> torch.Tensor: + """Forward pass for cuda graph capture.""" + ctx.event = event + return tensor + + @staticmethod + def backward(ctx, grad_output): + """Record the backward event and wait for the h2d stream on cuda graph stream.""" + h2d_stream = PipelineOffloadManager.get_instance().h2d_stream + torch.cuda.current_stream().record_event(ctx.event) + torch.cuda.current_stream().wait_stream(h2d_stream) + return grad_output, None + + +def fine_grained_offloading_backward_record(tensor, event: torch.cuda.Event) -> torch.Tensor: + """Record the backward event for cuda graph capture.""" + return FineGrainedOffloadingBackwardRecordFunction.apply(tensor, event) diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index 35b2b288d99..98df5158ee7 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -565,9 +565,6 @@ def forward_backward_no_pipelining( if config.timers is not None: config.timers('forward-backward', log_level=1).start(barrier=config.barrier_with_L1_time) - if not forward_only and config.fine_grained_activation_offloading: - fine_grained_offloading_reset() - no_sync_func = config.no_sync_func if no_sync_func is None: no_sync_func = contextlib.nullcontext @@ -648,6 +645,9 @@ def forward_backward_no_pipelining( pg_collection=pg_collection, ) + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + if config.timers is not None: config.timers('forward-backward').stop() @@ -904,9 +904,6 @@ def forward_backward_pipelining_with_interleaving( adjust_tensor_shapes_fn is None ), "adjust_tensor_shapes_fn is not supported for interleaved pipeline parallelism" - if not forward_only and config.fine_grained_activation_offloading: - fine_grained_offloading_reset() - if config.overlap_p2p_comm and config.batch_p2p_comm: raise ValueError("Can not use both overlap_p2p_comm and batch_p2p_comm") @@ -1911,6 +1908,8 @@ def pp_post_backward(input_tensor_grad, vp_stage=None): pg_collection=pg_collection, ) + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() # Restore config.grad_sync_func and config.param_sync_func. if forward_only: config.grad_sync_func, config.param_sync_func = grad_sync_func, param_sync_func @@ -2052,9 +2051,6 @@ def forward_backward_pipelining_without_interleaving( if config.timers is not None: config.timers('forward-backward', log_level=1).start(barrier=config.barrier_with_L1_time) - if not forward_only and config.fine_grained_activation_offloading: - fine_grained_offloading_reset() - # Disable async grad reductions no_sync_func = config.no_sync_func if no_sync_func is None: @@ -2302,6 +2298,9 @@ def enable_grad_sync(): pg_collection=pg_collection, ) + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + if config.timers is not None: config.timers('forward-backward').stop() diff --git a/megatron/core/pipeline_parallel/utils.py b/megatron/core/pipeline_parallel/utils.py index c50c6ac7964..814de85ae70 100644 --- a/megatron/core/pipeline_parallel/utils.py +++ b/megatron/core/pipeline_parallel/utils.py @@ -87,19 +87,13 @@ def set_ideal_affinity_for_current_gpu(): try: import cuda.bindings.driver as cuda_driver import cuda.bindings.runtime as cuda_runtime - except ImportError: + except: try: import cuda.cuda as cuda_driver import cuda.cudart as cuda_runtime - except ImportError: - # print("cuda-python may not be installed, skipping GPU affinity setting") - warnings.warn("cuda-python may not be installed, skipping GPU affinity setting") - return - try: - import pynvml - except ImportError: - warnings.warn("pynvml is not installed, skipping GPU affinity setting") - return + except: + raise RuntimeError("Please install cuda-python to enable GPU affinity setting") + import pynvml # Get current CUDA device ID err, device_id = cuda_runtime.cudaGetDevice() diff --git a/megatron/core/tensor_parallel/__init__.py b/megatron/core/tensor_parallel/__init__.py index 98cc5efec82..2140ee54b37 100644 --- a/megatron/core/tensor_parallel/__init__.py +++ b/megatron/core/tensor_parallel/__init__.py @@ -29,9 +29,11 @@ from .random import ( CheckpointWithoutOutput, checkpoint, + convert_cuda_rng_state, get_cuda_rng_tracker, get_data_parallel_rng_tracker_name, get_expert_parallel_rng_tracker_name, + is_graph_safe_cuda_rng_tracker, model_parallel_cuda_manual_seed, ) from .utils import ( @@ -64,9 +66,11 @@ "scatter_to_sequence_parallel_region", # random.py "checkpoint", + "convert_cuda_rng_state", "get_cuda_rng_tracker", "model_parallel_cuda_manual_seed", "get_expert_parallel_rng_tracker_name", + "is_graph_safe_cuda_rng_tracker", "CheckpointWithoutOutput", # utils.py "split_tensor_along_last_dim", diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 396e5c54a2d..617d2803c12 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -111,6 +111,41 @@ def cb(): _lazy_call(cb) +def convert_cuda_rng_state( + state: Union[torch.Tensor, torch.Generator], to_graphable: bool = False +) -> Union[torch.Tensor, torch.Generator]: + """ + Convert the cuda rng state tensor to the graphable version, + or from the graphable version to the non-graphable tensor version. + """ + if to_graphable: + if isinstance(state, torch.Tensor): + # Convert to the graphable version. + # Store current rng state. + orig_cuda_rng_state = _get_cuda_rng_state(graph_safe=False) + # Set rng state to the desired one + _set_cuda_rng_state(state, graph_safe=False) + # Get the graphable state + graphable_state = _get_cuda_rng_state(clone=True, graph_safe=True) + # And set the state to the original state we started with. + _set_cuda_rng_state(orig_cuda_rng_state, graph_safe=False) + return graphable_state + elif isinstance(state, torch.Generator): + # already graphable, just return it. + return state + else: + raise ValueError(f"Invalid state type: {type(state)}") + else: + if isinstance(state, torch.Tensor): + # already non-graphable, just return it. + return state + elif isinstance(state, torch.Generator): + # Convert to the non-graphable tensor version. + return state.get_state() + else: + raise ValueError(f"Invalid state type: {type(state)}") + + def get_expert_parallel_rng_tracker_name(): """Get the expert parallel rng tracker name""" global _EXPERT_PARALLEL_RNG_TRACKER_NAME @@ -161,6 +196,10 @@ def reset(self): # Seeds are just for book keeping and ensure no seed is set twice. self.seeds_ = set() + # Name of the rng state currently being used in the generator. + # The default one is "default-rng" and won't be pushed to the self.states_ dictionary. + self._current_state_name = "default-rng" + def get_states(self): """Get rng states. Copy the dictionary so we have direct pointers to the states, not just a pointer to the dictionary.""" @@ -207,10 +246,14 @@ def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME): # Check if we have added the state if name not in self.states_: raise Exception('cuda rng state {} is not added'.format(name)) - # Store current rng state. + # Store current rng state and name. Store in self.states_ if it's not the default state. orig_cuda_rng_state = _get_cuda_rng_state(graph_safe=self.use_cudagraphable_rng) - # Set rng state to the desired one + orig_state_name = self._current_state_name + if orig_state_name != "default-rng": + self.states_[orig_state_name] = orig_cuda_rng_state + # Set rng state and name to the desired one. _set_cuda_rng_state(self.states_[name], graph_safe=self.use_cudagraphable_rng) + self._current_state_name = name # Record cpu RNG state cpu_rng_state = torch.get_rng_state() # Do the stuff we wanted to do. @@ -220,10 +263,19 @@ def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME): # Throw a warning if cpu RNG state changed if not torch.all(cpu_rng_state == torch.get_rng_state()).item(): logging.getLogger(__name__).warning('CPU RNG state changed within GPU RNG context') + # Check if the current state name is the same as the desired state name. + if self._current_state_name != name: + raise Exception( + f'current state name {self._current_state_name} is not the same as the desired ' + f'state name {name}.' + ) # Update the current rng state for later use. self.states_[name] = _get_cuda_rng_state(graph_safe=self.use_cudagraphable_rng) - # And set the state to the original state we started with. + # And set the state and name to the original state we started with. + if orig_state_name != "default-rng": + orig_cuda_rng_state = self.states_[orig_state_name] _set_cuda_rng_state(orig_cuda_rng_state, graph_safe=self.use_cudagraphable_rng) + self._current_state_name = orig_state_name # RNG tracker object. @@ -377,10 +429,24 @@ def model_parallel_cuda_manual_seed( _CUDA_RNG_STATE_TRACKER.add(_EXPERT_PARALLEL_RNG_TRACKER_NAME, expert_parallel_seed) +def is_graph_safe_cuda_rng_tracker(cuda_rng_tracker): + """Check if the cuda rng tracker is graph safe version.""" + if HAVE_TE and is_te_min_version("1.5.0"): + from megatron.core.extensions.transformer_engine import TECudaRNGStatesTracker + + if isinstance(cuda_rng_tracker, TECudaRNGStatesTracker): + return True + if getattr(cuda_rng_tracker, "use_cudagraphable_rng", False): + return True + return False + + def _get_all_rng_states(): """Get all the rng states.""" cpu_rng_state = torch.get_rng_state() - cuda_rng_state = _get_cuda_rng_state() + cuda_rng_state = _get_cuda_rng_state( + graph_safe=is_graph_safe_cuda_rng_tracker(get_cuda_rng_tracker()) + ) cuda_rng_state_tracker = get_cuda_rng_tracker().get_states() return cpu_rng_state, cuda_rng_state, cuda_rng_state_tracker @@ -388,7 +454,9 @@ def _get_all_rng_states(): def _set_all_rng_states(cpu_rng_state, cuda_rng_state, cuda_rng_state_tracker): """Set all the rng states.""" torch.set_rng_state(cpu_rng_state) - _set_cuda_rng_state(cuda_rng_state) + _set_cuda_rng_state( + cuda_rng_state, graph_safe=is_graph_safe_cuda_rng_tracker(get_cuda_rng_tracker()) + ) get_cuda_rng_tracker().set_states(cuda_rng_state_tracker) diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index 7b81eb723ed..41570a2ff04 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -1667,7 +1667,12 @@ def create_cudagraphs(self): # Prepare CUDA Graph capturing input data and call `make_graphed_callables`. sample_args, kwargs = self._get_cuda_graph_input_data() - graphs = make_graphed_callables(tuple(self.flattened_callables), sample_args, **kwargs) + if self.config.sequence_parallel: + rng_context = get_cuda_rng_tracker().fork() + else: + rng_context = nullcontext() + with rng_context: + graphs = make_graphed_callables(tuple(self.flattened_callables), sample_args, **kwargs) # Push the captured graphs to the corresponding TransformerBlock. num_layers_accumulated = 0 diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index 1598970a48a..b989300a265 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -1002,7 +1002,9 @@ def glu(x): self.activation_checkpoint.discard_output_and_register_recompute(output) if self.offload_moe_act: (output,) = fine_grained_offloading_group_commit( - output, name="moe_act", forced_released_tensors=[fc1_output] + output, + name="moe_act", + forced_released_tensors=[fc1_output], ) # upad and concat the output diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index f8b7d234fff..8ac7f054349 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -9,8 +9,10 @@ from megatron.core.fp4_utils import get_fp4_align_size from megatron.core.fp8_utils import get_fp8_align_size from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel import get_cuda_rng_tracker, get_expert_parallel_rng_tracker_name from megatron.core.transformer.cuda_graphs import is_graph_capturing from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.utils import internal_api try: import transformer_engine as te # pylint: disable=unused-import @@ -914,6 +916,7 @@ def get_moe_layer_wise_logging_tracker(): return _MOE_LAYER_WISE_LOGGING_TRACKER +@internal_api class RandomSTE(torch.autograd.Function): """ Straight-Through Estimator(STE) function that returns random values @@ -922,26 +925,14 @@ class RandomSTE(torch.autograd.Function): This is used to generate random logits of router for load-balanced benchmark. """ - generator = None - random_logits = None - @staticmethod def forward(ctx, logits): """ Forward pass returns random logits with rank-specific seed. """ - if is_graph_capturing() and RandomSTE.random_logits is not None: - return RandomSTE.random_logits - - if RandomSTE.generator is None: - global_rank = torch.distributed.get_rank() - base_seed = 42 - seed = base_seed + global_rank - RandomSTE.generator = torch.Generator(device=logits.device) - RandomSTE.generator.manual_seed(seed) - - RandomSTE.random_logits = logits.clone().normal_(generator=RandomSTE.generator) - return RandomSTE.random_logits + with get_cuda_rng_tracker().fork(get_expert_parallel_rng_tracker_name()): + random_logits = logits.clone().normal_() + return random_logits @staticmethod def backward(ctx, grad_output): diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index a679fa15bc2..9b51c0407a5 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -243,13 +243,20 @@ def forward( # Get the query, key and value tensors based on the type of attention - # self or cross attn. # query: [96, 1, 16, 128], key:[96, 1, 16, 128], value:[96, 1, 16, 128] - query, key, value = self.get_query_key_value_tensors( - hidden_states, - key_value_states, - position_ids, - packed_seq_params, - inference_context=inference_context, - ) + if self.offload_qkv_linear: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="qkv_linear") + with get_fine_grained_offloading_context(self.offload_qkv_linear): + query, key, value = self.get_query_key_value_tensors( + hidden_states, + key_value_states, + position_ids, + packed_seq_params, + inference_context=inference_context, + ) + if self.offload_qkv_linear: + (query, key, value) = fine_grained_offloading_group_commit( + query, key, value, name="qkv_linear", forced_released_tensors=[hidden_states] + ) # =================================================== # Adjust key, value for inference diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py index d8e29ff8d8d..b2d0165cce8 100755 --- a/megatron/core/transformer/multi_token_prediction.py +++ b/megatron/core/transformer/multi_token_prediction.py @@ -13,9 +13,6 @@ from megatron.core.fp8_utils import get_fp8_context from megatron.core.models.backends import BackendSpecProvider, LocalSpecProvider from megatron.core.packed_seq_params import PackedSeqParams -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_set_last_layer, -) from megatron.core.pipeline_parallel.utils import is_vp_last_stage from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel import ( @@ -1012,8 +1009,6 @@ def forward( hidden_states_list = list(torch.chunk(hidden_states, 1 + offset, dim=0)) hidden_states = hidden_states_list[offset] for layer_number in range(len(self.layers)): - if self.config.fine_grained_activation_offloading: - fine_grained_offloading_set_last_layer(layer_number == len(self.layers) - 1) (hidden_states, input_ids, position_ids) = self.layers[layer_number]( input_ids=input_ids, position_ids=position_ids, diff --git a/megatron/core/transformer/transformer_block.py b/megatron/core/transformer/transformer_block.py index 06e8f1372f4..62b2b3d9b9b 100755 --- a/megatron/core/transformer/transformer_block.py +++ b/megatron/core/transformer/transformer_block.py @@ -16,9 +16,6 @@ from megatron.core.fusions.fused_layer_norm import FusedLayerNorm from megatron.core.inference.contexts import BaseInferenceContext from megatron.core.packed_seq_params import PackedSeqParams -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_set_last_layer, -) from megatron.core.pipeline_parallel.utils import is_vp_first_stage, is_vp_last_stage from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.enums import LayerType @@ -696,11 +693,6 @@ def forward( else: inner_quantization_context = nullcontext() - if self.config.fine_grained_activation_offloading: - fine_grained_offloading_set_last_layer( - l_no == self.num_layers_per_pipeline_rank - 1 - ) - with self.offload_context, inner_quantization_context: hidden_states, context = layer( hidden_states=hidden_states, diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 6062ae7c077..ab0ef8542e3 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1236,7 +1236,10 @@ def validate_args(args, defaults={}): # CUDA Graphs if args.cuda_graph_impl != "none": - if args.transformer_impl == 'transformer_engine' and not args.te_rng_tracker: + if ( + "transformer_engine" in (args.transformer_impl, args.cuda_graph_impl) + and not args.te_rng_tracker + ): args.te_rng_tracker = True warn_rank_0("te_rng_tracker is not enabled, enabling it for CUDA graphs.", args.rank) assert "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", ""), ( @@ -2357,7 +2360,7 @@ def _add_training_args(parser): help='Enable fine-grained activation offloading.') group.add_argument('--offload-modules', nargs='*', type=str, default=[], help='The submodules to offload its input. Choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".') - group.add_argument('--min-offloaded-tensor-size', type=int, default=1024*1024, + group.add_argument('--min-offloaded-tensor-size', type=int, default=10*1024*1024, help='The minimum size of the tensor to be offloaded.') return parser diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py index 853f18f8358..ab85df8d4c8 100644 --- a/megatron/training/checkpointing.py +++ b/megatron/training/checkpointing.py @@ -1712,6 +1712,8 @@ def load_model_state_dict(module, state_dict, strict: bool): # rng states. if not release and not args.finetune and not args.no_load_rng and not ignore_rng_state: try: + cuda_rng_tracker = tensor_parallel.get_cuda_rng_tracker() + graph_safe_rng = tensor_parallel.is_graph_safe_cuda_rng_tracker(cuda_rng_tracker) if 'rng_state' in state_dict: if args.ckpt_format == "fsdp_dtensor": # FSDP DTensor checkpoints store rng_state in a different format. @@ -1737,8 +1739,10 @@ def load_model_state_dict(module, state_dict, strict: bool): # Check for empty states array if not rng_state['rng_tracker_states']: raise KeyError - tensor_parallel.get_cuda_rng_tracker().set_states( - rng_state['rng_tracker_states']) + rng_tracker_states = { + k: tensor_parallel.convert_cuda_rng_state(v, to_graphable=graph_safe_rng) + for k, v in rng_state['rng_tracker_states'].items() + } else: # backward compatability random.setstate(state_dict['random_rng_state']) np.random.set_state(state_dict['np_rng_state']) @@ -1747,8 +1751,11 @@ def load_model_state_dict(module, state_dict, strict: bool): # Check for empty states array if not state_dict['rng_tracker_states']: raise KeyError - tensor_parallel.get_cuda_rng_tracker().set_states( - state_dict['rng_tracker_states']) + rng_tracker_states = { + k: tensor_parallel.convert_cuda_rng_state(v, to_graphable=graph_safe_rng) + for k, v in state_dict['rng_tracker_states'].items() + } + cuda_rng_tracker.set_states(rng_tracker_states) except KeyError: print_rank_0('Unable to load rng state from checkpoint {}. ' 'Specify --no-load-rng or --finetune to prevent ' diff --git a/megatron/training/training.py b/megatron/training/training.py index d4f5d468f5a..210374412d7 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -627,6 +627,12 @@ def pretrain( args = get_args() timers = get_timers() + if args.fine_grained_activation_offloading: + from megatron.core.pipeline_parallel.utils import ( + set_ideal_affinity_for_current_gpu + ) + set_ideal_affinity_for_current_gpu() + if args.log_progress: append_to_progress_log("Starting job") diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 7c1b7f1fe4b..e984b3d38ad 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -1,6 +1,7 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import gc +import os import pytest import torch @@ -19,44 +20,40 @@ def _reset_cuda_memory(): class ToyModel(torch.nn.Module): def __init__(self, hidden_size: int = 2048, num_layers: int = 4, dtype=torch.bfloat16): + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group(backend="nccl") + if torch.cuda.is_available(): + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) super().__init__() layers = [] for _ in range(num_layers): - layers.append( - torch.nn.Linear(hidden_size, hidden_size, bias=True, dtype=dtype, device="cuda") + linear = torch.nn.Linear( + hidden_size, hidden_size, bias=True, dtype=dtype, device="cuda" ) + layers.append(linear) self.net = torch.nn.Sequential(*layers).to(device="cuda", dtype=dtype) self.hidden_size = hidden_size self.num_layers = num_layers self.dtype = dtype - # Prevent weights/bias from being considered activation tensors for offload; - # ensure we only count activation tensors (inputs x) in memory accounting. - for p in self.parameters(): - try: - setattr(p, "offloading_activation", False) - except Exception: - pass - def forward(self, x, use_offload: bool = False): from megatron.core.pipeline_parallel import fine_grained_activation_offload as off if use_offload: # Initialize a new chunk (microbatch) and enable offload context. - with off.get_fine_grained_offloading_context(True): - off.fine_grained_offloading_init_chunk_handler( - vp_size=1, vp_stage=None, min_offloaded_tensor_size=1 - ) - for i, layer in enumerate(self.net): - # Group by module; with this linear-only model, each group corresponds to a layer. - off.fine_grained_offloading_set_last_layer(i == len(self.net) - 1) - x = off.fine_grained_offloading_group_start(x, name=f"layer_{i}") + off.fine_grained_offloading_init_chunk_handler( + vp_size=1, vp_stage=None, min_offloaded_tensor_size=1 + ) + for layer in self.net: + # Group by module; with this linear-only model, each group corresponds to a layer. + x = off.fine_grained_offloading_group_start(x, name=f"linear_layer") + with off.get_fine_grained_offloading_context(True): x = layer(x) - # Commit the group; returns a tuple of tensors - (x,) = off.fine_grained_offloading_group_commit( - x, name=f"layer_{i}", forced_released_tensors=[] - ) - return x + # Commit the group; returns a tuple of tensors + (x,) = off.fine_grained_offloading_group_commit( + x, name=f"linear_layer", forced_released_tensors=[] + ) + return x # Baseline path (no offload hooks) with ( torch.autocast(device_type="cuda", dtype=self.dtype) @@ -68,19 +65,6 @@ def forward(self, x, use_offload: bool = False): return x -@pytest.fixture(autouse=True) -def _monkeypatch_offload_deps(monkeypatch): - # Avoid requiring torch.distributed initialization and NVML in tests - import megatron.core.pipeline_parallel.fine_grained_activation_offload as off - - monkeypatch.setattr(off, "debug_rank", lambda *args, **kwargs: None, raising=False) - monkeypatch.setattr(off, "set_ideal_affinity_for_current_gpu", lambda: None, raising=False) - # Ensure a clean state each test - off.fine_grained_offloading_reset() - yield - off.fine_grained_offloading_reset() - - def test_fine_grained_activation_offload_memory_reduction(): torch.manual_seed(1234) # Use a linear-only stack so theoretical saved memory equals sum of per-layer input x bytes. @@ -112,7 +96,17 @@ def test_fine_grained_activation_offload_memory_reduction(): from megatron.core.pipeline_parallel import fine_grained_activation_offload as off off.fine_grained_offloading_reset() + # warmup + inp_off = inp.detach().clone().requires_grad_(True) + out_off = model(inp_off, use_offload=True) + (out_off.sum()).backward() + torch.cuda.synchronize() + off.fine_grained_offloading_reset() + del inp_off + del out_off _reset_cuda_memory() + torch.cuda.synchronize() + inp_off = inp.detach().clone().requires_grad_(True) offload_mem_before = torch.cuda.memory_allocated() / (1024**2) out_off = model(inp_off, use_offload=True) diff --git a/tests/unit_tests/tensor_parallel/test_random.py b/tests/unit_tests/tensor_parallel/test_random.py index 47b607b8795..a15ad83cb90 100644 --- a/tests/unit_tests/tensor_parallel/test_random.py +++ b/tests/unit_tests/tensor_parallel/test_random.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import pytest import torch @@ -5,6 +7,7 @@ CheckpointWithoutOutput, CudaRNGStatesTracker, checkpoint, + convert_cuda_rng_state, get_cuda_rng_tracker, model_parallel_cuda_manual_seed, ) @@ -33,6 +36,148 @@ def test_cuda_rng_states_tracker(): assert torch.equal(rng_tracker.get_states()['state2'], rng_state) +@pytest.mark.parametrize("use_cudagraphable_rng", [True, False]) +def test_double_fork_cuda_rng_states_tracker(use_cudagraphable_rng): + rng_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=use_cudagraphable_rng) + rng_tracker.add("state1", 1234) + rng_tracker.add("state2", 5678) + randn_double_fork_1 = [] + randn_double_fork_2 = [] + with rng_tracker.fork("state1"): + randn_double_fork_1.append(torch.randn(10, device="cuda")) + with rng_tracker.fork("state2"): + randn_double_fork_2.append(torch.randn(10, device="cuda")) + with rng_tracker.fork("state1"): + randn_double_fork_1.append(torch.randn(10, device="cuda")) + randn_double_fork_2.append(torch.randn(10, device="cuda")) + randn_double_fork_1.append(torch.randn(10, device="cuda")) + if use_cudagraphable_rng: + double_fork_state1 = rng_tracker.get_states()["state1"].get_state() + double_fork_state2 = rng_tracker.get_states()["state2"].get_state() + else: + double_fork_state1 = rng_tracker.get_states()["state1"] + double_fork_state2 = rng_tracker.get_states()["state2"] + + rng_tracker.reset() + rng_tracker.add("state1", 1234) + rng_tracker.add("state2", 5678) + randn_single_fork_1 = [] + randn_single_fork_2 = [] + with rng_tracker.fork("state1"): + randn_single_fork_1.append(torch.randn(10, device="cuda")) + randn_single_fork_1.append(torch.randn(10, device="cuda")) + randn_single_fork_1.append(torch.randn(10, device="cuda")) + with rng_tracker.fork("state2"): + randn_single_fork_2.append(torch.randn(10, device="cuda")) + randn_single_fork_2.append(torch.randn(10, device="cuda")) + if use_cudagraphable_rng: + single_fork_state1 = rng_tracker.get_states()["state1"].get_state() + single_fork_state2 = rng_tracker.get_states()["state2"].get_state() + else: + single_fork_state1 = rng_tracker.get_states()["state1"] + single_fork_state2 = rng_tracker.get_states()["state2"] + + assert torch.equal(randn_double_fork_1[0], randn_single_fork_1[0]) + assert torch.equal(randn_double_fork_1[1], randn_single_fork_1[1]) + assert torch.equal(randn_double_fork_1[2], randn_single_fork_1[2]) + assert torch.equal(randn_double_fork_2[0], randn_single_fork_2[0]) + assert torch.equal(randn_double_fork_2[1], randn_single_fork_2[1]) + assert torch.equal(double_fork_state1, single_fork_state1) + assert torch.equal(double_fork_state2, single_fork_state2) + + +def test_convert_cuda_rng_state(): + ## Get the default rng state + torch.cuda.manual_seed(999) + randn = torch.randn(10, device="cuda") + rng_state = torch.cuda.get_rng_state() + + try: + from megatron.core.extensions.transformer_engine import TECudaRNGStatesTracker + except ImportError: + TECudaRNGStatesTracker = None + + ## from non-graphable RNG to graphable RNG + # get state from non-graphable RNG + tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False) + tracker.add("state1", 123) + for i in range(3): + with tracker.fork("state1"): + randn = torch.randn(10, device="cuda") + state = convert_cuda_rng_state(tracker.states_["state1"], to_graphable=True) + rand_tensors = [] + for i in range(3): + with tracker.fork("state1"): + randn = torch.randn(10, device="cuda") + rand_tensors.append(randn) + + # set state to local graph RNG + cudagraphable_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=True) + cudagraphable_tracker.set_states({"state1": state.clone_state()}) + for i in range(3): + with cudagraphable_tracker.fork("state1"): + randn = torch.randn(10, device="cuda") + assert torch.equal(randn, rand_tensors[i]) + + # set state to TE RNG + if TECudaRNGStatesTracker is not None: + te_tracker = TECudaRNGStatesTracker() + te_tracker.set_states({"state1": state}) + for i in range(3): + with te_tracker.fork("state1"): + randn = torch.randn(10, device="cuda") + assert torch.equal(randn, rand_tensors[i]) + + ## from graphable RNG to non-graphable RNG + # get state from graphable RNG + cudagraphable_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=True) + cudagraphable_tracker.add("state2", 123) + for i in range(3): + with cudagraphable_tracker.fork("state2"): + randn = torch.randn(10, device="cuda") + state = convert_cuda_rng_state(cudagraphable_tracker.states_["state2"], to_graphable=False) + rand_tensors = [] + for i in range(3): + with cudagraphable_tracker.fork("state2"): + randn = torch.randn(10, device="cuda") + rand_tensors.append(randn) + + # set state to non-graphable RNG + tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False) + tracker.set_states({"state2": state}) + for i in range(3): + with tracker.fork("state2"): + randn = torch.randn(10, device="cuda") + assert torch.equal(randn, rand_tensors[i]) + + ## from TE RNG to non-graphable RNG + if TECudaRNGStatesTracker is not None: + # get state from TE RNG + cudagraphable_tracker = TECudaRNGStatesTracker() + cudagraphable_tracker.add("state3", 123) + for i in range(3): + with cudagraphable_tracker.fork("state3"): + randn = torch.randn(10, device="cuda") + state = convert_cuda_rng_state(cudagraphable_tracker.states_["state3"], to_graphable=False) + rand_tensors = [] + for i in range(3): + with cudagraphable_tracker.fork("state3"): + randn = torch.randn(10, device="cuda") + rand_tensors.append(randn) + + # set state to non-graphable RNG + tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False) + tracker.set_states({"state3": state}) + for i in range(3): + with tracker.fork("state3"): + randn = torch.randn(10, device="cuda") + assert torch.equal(randn, rand_tensors[i]) + + ## After all tests, check if the default rng state is still the same. + rng_state_final = torch.cuda.get_rng_state() + assert torch.equal(rng_state, rng_state_final) + + def test_model_parallel_cuda_manual_seed(): Utils.initialize_model_parallel(4, 2) model_parallel_cuda_manual_seed(0, force_reset_rng=True) From 7d8929b506244bd074a23f17bbf91e511c688d92 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 5 Jan 2026 19:20:32 -0800 Subject: [PATCH 25/47] format Signed-off-by: Hongbin Liu --- .../pipeline_parallel/fine_grained_activation_offload.py | 8 ++++++-- megatron/core/transformer/moe/experts.py | 4 +--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 164ed3b14a0..57787e71936 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -766,8 +766,12 @@ def finish_all_groups(self, name=None) -> bool: f"------finish_all_groups {self} {self._max_group_size} {self._offloaded_group_index}" ) # TODO: check if this is correct - # Mark it as finished when all groups are finished and there are no groups to offload or reload - if len(self._groups_to_reload) == 0 and len(self._groups_to_offload) == 0 and self._offloaded_group_index > 0: + # Mark it as finished when there are no groups to offload or reload + if ( + len(self._groups_to_reload) == 0 + and len(self._groups_to_offload) == 0 + and self._offloaded_group_index > 0 + ): return True assert name is not None, "Name is required" for group in self.offload_groups[self._offloaded_group_index :]: diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index fce5ac1963f..d9697e753fb 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -804,9 +804,7 @@ def glu(x): self.activation_checkpoint.discard_output_and_register_recompute(output) if self.offload_moe_act: (output,) = fine_grained_offloading_group_commit( - output, - name="moe_act", - forced_released_tensors=[fc1_output], + output, name="moe_act", forced_released_tensors=[fc1_output] ) # upad and concat the output From 83058e4a0c3cba115fd32571ab9ae86cb51f7f9b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jan 2026 19:12:14 -0800 Subject: [PATCH 26/47] bug fix when cuda graph is disabled and fix for dumping offloading info Signed-off-by: root --- .../fine_grained_activation_offload.py | 19 ++++++++++++++----- megatron/core/transformer/moe/experts.py | 2 +- .../core/transformer/transformer_layer.py | 3 +++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 57787e71936..93e8078f34d 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -526,7 +526,7 @@ def post_warmup_callback(self): # Dump the offload information total_tensor_count = {} total_offload_bytes = {} - for chunk in self._cached_chunks_backward: + for chunk in self._cached_chunks_forward: for group in chunk.offload_groups: if group.offload: if group._name not in total_tensor_count: @@ -535,6 +535,10 @@ def post_warmup_callback(self): if group._name not in total_offload_bytes: total_offload_bytes[group._name] = 0 total_offload_bytes[group._name] += group.total_offload_bytes + # Stop statistics at the first backward chunk after which 1F1B is running, + # where the memory cost will not increase anymore. + if chunk is self._cached_chunks_backward[0]: + break print_offload_summary_table(total_offload_bytes) def push(self, handler): @@ -732,7 +736,7 @@ def __init__(self, min_offloaded_tensor_size, cpu_tensor_pool): self._groups_to_reload = [] self._tensor_count_current_group = 0 self._max_group_size = 0 - + self._reloading_group = [] # Counter for special torch tensor types (FakeTensor, FunctionalTensor) self.torch_tensor_count = 0 self.d2h_stream = PipelineOffloadManager.get_instance().d2h_stream @@ -748,6 +752,7 @@ def reset(self): self._groups_to_offload = [] self._groups_to_reload = [] self._tensor_count_current_group = 0 + self._reloading_group = [] def is_empty_chunk(self, name=None): """Check if this chunk has no tensors to manage.""" @@ -884,6 +889,7 @@ def bulk_reload_group(self): group_to_reload.push_tensor(tensor_tag, recovered_tensor) group_to_reload.record_reload_event(self.h2d_stream) self._groups_to_reload.pop() + self._reloading_group.append(group_to_reload) torch.cuda.nvtx.range_pop() def pre_reload_last_layer(self): @@ -966,9 +972,12 @@ def on_group_commit_backward(self, name): cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() assert cur_backward_chunk is self, f"Chunk mismatch {cur_backward_chunk} {self}" # Wait for reload to complete before using tensors - if not is_graph_capturing() and len(self._groups_to_reload) > 0: - group_to_reload = self._groups_to_reload[-1] - group_to_reload.wait_reload_event(torch.cuda.current_stream()) + if not is_graph_capturing() and len(self._reloading_group) > 0: + for reloading_group in self._reloading_group: + if reloading_group._name == name: + reloading_group.wait_reload_event(torch.cuda.current_stream()) + self._reloading_group.remove(reloading_group) + break def on_group_start_forward(self, name): """ diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index d9697e753fb..f4085d30a08 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -643,7 +643,7 @@ def __init__( set_save_original_input(self.linear_fc2) # This is to avoid the CPU overhead of multiple d2h copies - if self.offload_expert_fc1 and not (self.config.fp8 or self.config.fp4): + if self.offload_expert_fc1: from megatron.core.extensions.transformer_engine import set_save_original_input set_save_original_input(self.linear_fc1) diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index 68897ce8c7f..1fb1a6bbd99 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -757,6 +757,9 @@ def _forward_post_mlp(self, mlp_output_with_bias, residual): Returns: output (Tensor): Transformed hidden states of shape [s, b, h]. """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + ) using_fused_tp_inference_kernel = (not self.training) and ( self.config.inference_fuse_tp_communication ) From 9605a673d29e5b98f6b259110ede7e9763ef566d Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 7 Jan 2026 22:25:10 -0800 Subject: [PATCH 27/47] refactor and update ut Signed-off-by: Hongbin Liu --- .../core/models/gpt/fine_grained_callables.py | 2 +- .../fine_grained_activation_offload.py | 66 ++- megatron/core/transformer/attention.py | 9 +- megatron/core/transformer/moe/experts.py | 9 +- .../transformer/multi_latent_attention.py | 10 +- .../core/transformer/transformer_layer.py | 5 +- ...test_fine_grained_activation_offloading.py | 445 ++++++++++++------ 7 files changed, 359 insertions(+), 187 deletions(-) diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index 230acc36dea..5288e6df8f4 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -471,7 +471,7 @@ def submodule_combine_forward( mlp_output_with_bias, residual, layer.hidden_dropout ) if layer.offload_mlp_norm: - (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states = fine_grained_offloading_group_commit( hidden_states, name="mlp_norm", forced_released_tensors=[residual] ) output = make_viewless_tensor( diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 93e8078f34d..e073c518bd5 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -392,6 +392,12 @@ def get_instance(cls): cls.OFFLOAD_MGR = PipelineOffloadManager() return cls.OFFLOAD_MGR + @classmethod + def reset_instance(cls): + """Reset the singleton instance of PipelineOffloadManager.""" + cls.OFFLOAD_MGR = None + cls.OFFLOAD_MGR = PipelineOffloadManager() + def __init__(self): """Initialize the manager with queues and dedicated CUDA streams.""" # Queue to store chunk handlers for backward pass @@ -462,6 +468,16 @@ def reset(self): chunk.reset() self._delayed_offload_groups = [] + @property + def offload_summary_bytes(self) -> Dict[str, int]: + """Offload summary bytes per group collected after warmup.""" + return self._offload_summary_bytes + + @property + def offload_summary_total_bytes(self) -> int: + """Total offloaded bytes collected after warmup.""" + return self._offload_summary_total_bytes + def flush(self): """Flush all staged chunks to the backward queue in reverse order.""" # Ensure all virtual pipeline stages have the same number of chunks @@ -539,6 +555,9 @@ def post_warmup_callback(self): # where the memory cost will not increase anymore. if chunk is self._cached_chunks_backward[0]: break + # Cache summary for downstream consumers (e.g., unit tests). + self._offload_summary_bytes = dict(total_offload_bytes) + self._offload_summary_total_bytes = int(sum(total_offload_bytes.values())) print_offload_summary_table(total_offload_bytes) def push(self, handler): @@ -1037,22 +1056,17 @@ class FineGrainedOffloadingGroupCommitFunction(torch.autograd.Function): """ @staticmethod - def forward(ctx, *args): + def forward(ctx, tensor, cur_forward_chunk, name, forced_released_tensors, delay_offload): # pylint: disable=missing-function-docstring debug_rank("FineGrainedOffloadingGroupCommitFunction forward") - delay_offload = args[-1] - forced_released_tensors = args[-2] - name = args[-3] - cpu_offload_handler = args[-4] - tensor = args[:-4] if delay_offload: PipelineOffloadManager.get_instance().push_offload_groups( - cpu_offload_handler.on_group_commit_forward, forced_released_tensors + cur_forward_chunk.on_group_commit_forward, forced_released_tensors ) else: - cpu_offload_handler.on_group_commit_forward(forced_released_tensors) - ctx.cpu_offload_handler = cpu_offload_handler + cur_forward_chunk.on_group_commit_forward(forced_released_tensors) + ctx.cpu_offload_handler = cur_forward_chunk ctx.name = name return tensor @@ -1067,7 +1081,7 @@ def backward(ctx, *grad_output): def fine_grained_offloading_group_commit( - *tensor, name, forced_released_tensors=[], delay_offload=False + tensor, name, forced_released_tensors=None, delay_offload=False ): """ Specify the tensors to be released after offloading. @@ -1075,11 +1089,37 @@ def fine_grained_offloading_group_commit( The tensors will be untyped_storage().resize_(0) after offloading. Note: specify the tensors only when they are not automatically released by torch gc. """ + # Be permissive: callers may pass a tuple/list of outputs (e.g., (q, k, v)). + # We only need to insert a single identity op into the autograd graph; applying + # it to the first tensor output is sufficient and keeps callers' code minimal. + if forced_released_tensors is None: + forced_released_tensors = [] + if isinstance(tensor, tuple): + if len(tensor) == 0: + return tensor + committed0 = fine_grained_offloading_group_commit( + tensor[0], + name=name, + forced_released_tensors=forced_released_tensors, + delay_offload=delay_offload, + ) + return (committed0,) + tensor[1:] + if isinstance(tensor, list): + if len(tensor) == 0: + return tensor + committed0 = fine_grained_offloading_group_commit( + tensor[0], + name=name, + forced_released_tensors=forced_released_tensors, + delay_offload=delay_offload, + ) + return [committed0] + tensor[1:] + cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk() if cur_forward_chunk is None: return tensor return FineGrainedOffloadingGroupCommitFunction.apply( - *tensor, cur_forward_chunk, name, forced_released_tensors, delay_offload + tensor, cur_forward_chunk, name, forced_released_tensors, delay_offload ) @@ -1170,3 +1210,7 @@ def backward(ctx, grad_output): def fine_grained_offloading_backward_record(tensor, event: torch.cuda.Event) -> torch.Tensor: """Record the backward event for cuda graph capture.""" return FineGrainedOffloadingBackwardRecordFunction.apply(tensor, event) + +def fine_grained_offloading_reset_instance(): + """Reset the singleton instance of PipelineOffloadManager.""" + PipelineOffloadManager.reset_instance() diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 7d2e48e43c3..596a8203eb1 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -830,7 +830,8 @@ def forward( hidden_states, key_value_states, split_qkv=split_qkv ) if self.offload_qkv_linear: - (qkv_output,) = fine_grained_offloading_group_commit( + # `qkv_output` may be a tuple; commit supports tuple/list and will keep structure. + qkv_output = fine_grained_offloading_group_commit( qkv_output, name="qkv_linear", forced_released_tensors=[] ) attn_mask_type = self.attn_mask_type @@ -1018,7 +1019,7 @@ def forward( core_attn_out[inference_context.padding_slice] = 0.0 if self.offload_core_attention and self.training: - (core_attn_out,) = fine_grained_offloading_group_commit( + core_attn_out = fine_grained_offloading_group_commit( core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] ) if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': @@ -1038,8 +1039,8 @@ def forward( with get_fine_grained_offloading_context(self.offload_attn_proj): output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: - output, bias = fine_grained_offloading_group_commit( - output, bias, name="attn_proj", forced_released_tensors=[core_attn_out] + output = fine_grained_offloading_group_commit( + output, name="attn_proj", forced_released_tensors=[core_attn_out] ) nvtx_range_pop(suffix="linear_proj") diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index f4085d30a08..262ceb55249 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -721,11 +721,8 @@ def forward( permuted_local_hidden_states, tokens_per_expert ) if self.offload_expert_fc1: - fc1_output, bias_parallel = fine_grained_offloading_group_commit( - fc1_output, - bias_parallel, - name="expert_fc1", - forced_released_tensors=[permuted_local_hidden_states], + fc1_output = fine_grained_offloading_group_commit( + fc1_output, name="expert_fc1", forced_released_tensors=[permuted_local_hidden_states] ) def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): @@ -803,7 +800,7 @@ def glu(x): if self.activation_recompute: self.activation_checkpoint.discard_output_and_register_recompute(output) if self.offload_moe_act: - (output,) = fine_grained_offloading_group_commit( + output = fine_grained_offloading_group_commit( output, name="moe_act", forced_released_tensors=[fc1_output] ) diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index 4b4af48c375..384eaf77ef6 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -255,8 +255,8 @@ def forward( inference_context=inference_context, ) if self.offload_qkv_linear: - (query, key, value) = fine_grained_offloading_group_commit( - query, key, value, name="qkv_linear", forced_released_tensors=[hidden_states] + query = fine_grained_offloading_group_commit( + query, name="qkv_linear", forced_released_tensors=[hidden_states] ) # =================================================== @@ -318,7 +318,7 @@ def forward( if not inference_context.is_decode_only(): core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)') if self.offload_core_attention and self.training: - (core_attn_out,) = fine_grained_offloading_group_commit( + core_attn_out = fine_grained_offloading_group_commit( core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] ) @@ -351,8 +351,8 @@ def forward( with get_fine_grained_offloading_context(self.offload_attn_proj): output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: - output, bias = fine_grained_offloading_group_commit( - output, bias, name="attn_proj", forced_released_tensors=[core_attn_out] + output = fine_grained_offloading_group_commit( + output, name="attn_proj", forced_released_tensors=[core_attn_out] ) return output, bias diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index 1fb1a6bbd99..1355dd4a426 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -607,7 +607,7 @@ def _forward_attention( nvtx_range_pop(suffix="self_attn_bda") if self.offload_attn_norm: - (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states = fine_grained_offloading_group_commit( hidden_states, name="attn_norm", forced_released_tensors=[residual] ) @@ -649,7 +649,6 @@ def _forward_mlp(self, hidden_states, inference_context=None): """ from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_commit, fine_grained_offloading_group_start, get_fine_grained_offloading_context, ) @@ -779,7 +778,7 @@ def _forward_post_mlp(self, mlp_output_with_bias, residual): ) nvtx_range_pop(suffix="mlp_bda") if self.offload_mlp_norm: - (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states = fine_grained_offloading_group_commit( hidden_states, name="mlp_norm", forced_released_tensors=[residual] ) diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index e984b3d38ad..7dbb0d77d9e 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -2,180 +2,311 @@ import gc import os +from typing import Dict, List, Optional, Tuple import pytest import torch +from contextlib import nullcontext -EPSILON = 0.1 +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.transformer_config import TransformerConfig, MLATransformerConfig +from megatron.core.utils import is_te_min_version +from tests.unit_tests.test_utilities import Utils +from megatron.core.transformer.enums import AttnBackend -# Skip all tests if CUDA is not available -cuda_available = torch.cuda.is_available() +# Tolerance for memory expectation check (GPU allocator jitter etc). +EPSILON = 0.30 -def _reset_cuda_memory(): + +def _reset_cuda_memory() -> None: gc.collect() - if cuda_available: + if torch.cuda.is_available(): torch.cuda.empty_cache() + torch.cuda.synchronize() -class ToyModel(torch.nn.Module): - def __init__(self, hidden_size: int = 2048, num_layers: int = 4, dtype=torch.bfloat16): - if not torch.distributed.is_initialized(): - torch.distributed.init_process_group(backend="nccl") - if torch.cuda.is_available(): - torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) - super().__init__() - layers = [] - for _ in range(num_layers): - linear = torch.nn.Linear( - hidden_size, hidden_size, bias=True, dtype=dtype, device="cuda" - ) - layers.append(linear) - self.net = torch.nn.Sequential(*layers).to(device="cuda", dtype=dtype) - self.hidden_size = hidden_size - self.num_layers = num_layers - self.dtype = dtype - - def forward(self, x, use_offload: bool = False): - from megatron.core.pipeline_parallel import fine_grained_activation_offload as off - - if use_offload: - # Initialize a new chunk (microbatch) and enable offload context. - off.fine_grained_offloading_init_chunk_handler( - vp_size=1, vp_stage=None, min_offloaded_tensor_size=1 - ) - for layer in self.net: - # Group by module; with this linear-only model, each group corresponds to a layer. - x = off.fine_grained_offloading_group_start(x, name=f"linear_layer") - with off.get_fine_grained_offloading_context(True): - x = layer(x) - # Commit the group; returns a tuple of tensors - (x,) = off.fine_grained_offloading_group_commit( - x, name=f"linear_layer", forced_released_tensors=[] - ) - return x - # Baseline path (no offload hooks) - with ( - torch.autocast(device_type="cuda", dtype=self.dtype) - if self.dtype in (torch.float16, torch.bfloat16) - else torch.cuda.amp.autocast(enabled=False) - ): - for layer in self.net: - x = layer(x) - return x - - -def test_fine_grained_activation_offload_memory_reduction(): - torch.manual_seed(1234) - # Use a linear-only stack so theoretical saved memory equals sum of per-layer input x bytes. - model = ToyModel(hidden_size=2048, num_layers=8, dtype=torch.bfloat16).eval() - - # Create input - inp = torch.randn( - (2048, model.hidden_size), device="cuda", dtype=torch.bfloat16, requires_grad=True +def _build_gpt_model( + *, + seed: int, + num_layers: int, + hidden_size: int, + num_attention_heads: int, + vocab_size: int, + seq_length: int, + num_experts: Optional[int], + fine_grained_activation_offloading: bool, + offload_modules: Optional[List[str]], + min_offloaded_tensor_size: int, + is_mla: bool, +) -> GPTModel: + """Build a GPTModel that uses TE-based transformer layer spec.""" + model_parallel_cuda_manual_seed(seed) + torch.manual_seed(seed) + ConfigClass = MLATransformerConfig if is_mla else TransformerConfig + transformer_config = ConfigClass( + num_layers=num_layers, + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + use_cpu_initialization=True, + attention_backend=AttnBackend.unfused, + # Make sure model weights / activations are BF16 so TE fused attention isn't disabled. + bf16=True, + # params_dtype=torch.bfloat16, + # enable_autocast=True, + # autocast_dtype=torch.bfloat16, + # MoE + num_moe_experts=num_experts, + moe_grouped_gemm=(num_experts is not None), + # Fine-grained activation offloading + fine_grained_activation_offloading=fine_grained_activation_offloading, + offload_modules=offload_modules, + min_offloaded_tensor_size=min_offloaded_tensor_size, ) + gpt_model = GPTModel( + config=transformer_config, + transformer_layer_spec=get_gpt_layer_with_transformer_engine_spec( + num_experts=num_experts, + moe_grouped_gemm=num_experts is not None, + moe_use_legacy_grouped_gemm=False, + multi_latent_attention=is_mla, + ), + vocab_size=vocab_size, + max_sequence_length=seq_length, + ).bfloat16() + return gpt_model + + +def _make_gpt_inputs( + *, + seq_length: int, + micro_batch_size: int, + device: torch.device, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + data = list(range(seq_length)) + input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device) + position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device) + attention_mask = torch.ones((micro_batch_size, 1, seq_length, seq_length), dtype=bool).to( + device + ) + return input_ids, position_ids, attention_mask - # Warmup to stabilize allocator behavior - _reset_cuda_memory() - out = model(inp, use_offload=False) - (out.sum()).backward() - torch.cuda.synchronize() - _reset_cuda_memory() - - # Baseline memory measurement (no offload) - _reset_cuda_memory() - inp_baseline = inp.detach().clone().requires_grad_(True) - baseline_mem_before = torch.cuda.memory_allocated() / (1024**2) - out_base = model(inp_baseline, use_offload=False) - baseline_mem_after = (torch.cuda.memory_allocated() - out_base.nbytes) / (1024**2) - (out_base.sum()).backward() - torch.cuda.synchronize() - baseline_delta = baseline_mem_after - baseline_mem_before - # Offload memory measurement +def _run_one_iter_and_capture( + model: GPTModel, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: torch.Tensor, + enable_offload_reset: bool, +) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], int]: + """ + Run a single forward+backward iteration. + + Returns: + - logits (CPU float32) + - selected grads (CPU float32) + - peak_memory_allocated (bytes) during the iteration + """ from megatron.core.pipeline_parallel import fine_grained_activation_offload as off - off.fine_grained_offloading_reset() - # warmup - inp_off = inp.detach().clone().requires_grad_(True) - out_off = model(inp_off, use_offload=True) - (out_off.sum()).backward() - torch.cuda.synchronize() - off.fine_grained_offloading_reset() - del inp_off - del out_off - _reset_cuda_memory() - torch.cuda.synchronize() + if enable_offload_reset: + off.fine_grained_offloading_reset() + + # for p in model.parameters(): + # if p.grad is not None: + # p.grad = None - inp_off = inp.detach().clone().requires_grad_(True) - offload_mem_before = torch.cuda.memory_allocated() / (1024**2) - out_off = model(inp_off, use_offload=True) - offload_mem_after = (torch.cuda.memory_allocated() - out_off.nbytes) / (1024**2) - (out_off.sum()).backward() + torch.cuda.reset_peak_memory_stats() + logits = model( + input_ids=input_ids, position_ids=position_ids, attention_mask=attention_mask + ) + loss = logits.float().sum() + loss.backward() torch.cuda.synchronize() - offload_delta = offload_mem_after - offload_mem_before - - # Offload should reduce peak cached memory usage after forward - assert ( - offload_delta < baseline_delta - ), f"offload did not reduce memory: off={offload_delta:.2f}MiB base={baseline_delta:.2f}MiB" - - # Theoretical savings: storing per-layer input x (same shape each layer). - bytes_per_elem = inp.element_size() # 2 for bfloat16 - input_bytes = inp.numel() * bytes_per_elem - # -2 because the first and last activations are not offloaded - expected_saved_mib = (model.num_layers - 2) * (input_bytes / (1024**2)) - - # Actual savings ≈ baseline_delta - offload_delta (both exclude output tensor memory). - actual_saved_mib = baseline_delta - offload_delta - - # Allow slack for allocator jitter and extra intermediates; magnitudes should match. - rel_err = abs(actual_saved_mib - expected_saved_mib) / max(expected_saved_mib, 1e-6) - assert ( - rel_err <= EPSILON - ), f"saved mismatch: actual={actual_saved_mib:.2f}MiB expected~={expected_saved_mib:.2f}MiB (rel_err={rel_err:.2f})" - - -def test_fine_grained_activation_offload_output_and_grad_consistency(): - torch.manual_seed(2025) - hidden = 1024 - layers = 3 - - # Create identical models by resetting seed - torch.manual_seed(2025) - model_base = ToyModel(hidden_size=hidden, num_layers=layers, dtype=torch.bfloat16).train() - torch.manual_seed(2025) - model_off = ToyModel(hidden_size=hidden, num_layers=layers, dtype=torch.bfloat16).train() - - # Same input and target - inp = torch.randn((32, hidden), device="cuda", dtype=torch.bfloat16, requires_grad=True) - target = torch.randn_like(inp) - - # Baseline forward/backward - out_base = model_base(inp, use_offload=False) - loss_base = torch.nn.functional.mse_loss(out_base, target) - loss_base.backward() - grads_base = [ - p.grad.detach().clone() if p.grad is not None else None for p in model_base.parameters() - ] - - # Offload forward/backward + peak_bytes = int(torch.cuda.max_memory_allocated()) + + # capture all gradients for correctness + grads: Dict[str, torch.Tensor] = {} + for name, p in model.named_parameters(): + grads[name] = (p.grad.detach().float().cpu() if p.grad is not None else None) + + return logits.detach().float().cpu(), grads, peak_bytes + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for offloading tests.") +@pytest.mark.skipif( + not is_te_min_version("1.13.0"), + reason="Fine-grained activation offloading requires TE-based GPT layer spec (TE 1.13+ in this repo's tests).", +) +@pytest.mark.parametrize( + "is_moe, is_mla, offload_modules", + [ + # Dense GPT modules + (False, True, ["attn_norm"]), + (True, False, ["qkv_linear"]), + (True, False, ["core_attn"]), + # # attn_proj depends on core_attn (validated in TransformerConfig.__post_init__) + (True, True, ["core_attn", "attn_proj"]), + (True, False, ["mlp_norm"]), + (True, False, ["expert_fc1"]), + (True, False, ["moe_act"]), + ], +) +def test_gpt_fine_grained_activation_offloading_correctness_and_memory( + is_moe: bool, is_mla: bool, offload_modules: List[str] +): + """ + Initialize a GPTModel and verify: + - forward output correctness under each offload_modules setting + - backward gradient correctness (subset) + - peak GPU memory is reduced roughly as expected (based on recorded offload bytes) + """ + # setup distributed/model-parallel (same pattern as other UTs) + os.environ.pop("NVTE_FUSED_ATTN", None) + os.environ.pop("NVTE_FLASH_ATTN", None) + os.environ.pop("NVTE_UNFUSED_ATTN", None) + # os.environ["NVTE_FLASH_ATTN"] = "1" + Utils.initialize_model_parallel(1, 1) + torch.cuda.memory._record_memory_history(max_entries=100000) + + seed = 123 + # Choose shapes large enough to make memory deltas stable but still fast. + num_experts = 4 if is_moe else None + num_layers = 8 + hidden_size = 2048 if num_experts is None else 1024 + num_attention_heads = 16 if hidden_size >= 2048 else 8 + vocab_size = 512 + seq_length = 512 + micro_batch_size = 2 + device = torch.device("cuda") + + input_ids, position_ids, attention_mask = _make_gpt_inputs( + seq_length=seq_length, micro_batch_size=micro_batch_size, device=device + ) + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + off.fine_grained_offloading_reset_instance() + + try: + # 1) Baseline run (no offloading) + _reset_cuda_memory() + base_model = _build_gpt_model( + seed=seed, + num_layers=num_layers, + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + vocab_size=vocab_size, + seq_length=seq_length, + num_experts=num_experts, + fine_grained_activation_offloading=False, + offload_modules=None, + min_offloaded_tensor_size=1024 * 1024, + is_mla=is_mla, + ).cuda() + base_model.train() + + # Warmup baseline once for allocator stability + _run_one_iter_and_capture( + base_model, + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + enable_offload_reset=False, + ) + _reset_cuda_memory() + base_logits, base_grads, base_peak = _run_one_iter_and_capture( + base_model, + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + enable_offload_reset=False, + ) + # Free baseline model GPU memory before offload path + del base_model + _reset_cuda_memory() + + # 2) Offload run (warmup to record bytes + steady-state measurement) + off_model = _build_gpt_model( + seed=seed, + num_layers=num_layers, + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + vocab_size=vocab_size, + seq_length=seq_length, + num_experts=num_experts, + fine_grained_activation_offloading=True, + offload_modules=offload_modules, + min_offloaded_tensor_size=1024, # force offloading for UT determinism + is_mla=is_mla, + ).cuda() + off_model.train() - off.fine_grained_offloading_reset() - out_off = model_off(inp.detach().clone().requires_grad_(True), use_offload=True) - loss_off = torch.nn.functional.mse_loss(out_off, target) - loss_off.backward() - grads_off = [ - p.grad.detach().clone() if p.grad is not None else None for p in model_off.parameters() - ] - - # Compare outputs - assert torch.allclose(out_off.float(), out_base.float(), rtol=1e-3, atol=1e-3) - - # Compare gradients parameter-wise - for gb, go in zip(grads_base, grads_off): - if gb is None and go is None: - continue - assert gb is not None and go is not None - assert torch.allclose(go.float(), gb.float(), rtol=1e-3, atol=1e-3) + # Warmup 1 iter to populate cached chunks, then reset to finish warmup bookkeeping. + _run_one_iter_and_capture( + off_model, + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + enable_offload_reset=True, + ) + # Reset once more to trigger post_warmup_callback and apply steady-state offload decisions. + off.fine_grained_offloading_reset() + + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + PipelineOffloadManager, + ) + + mgr = PipelineOffloadManager.get_instance() + expected_offload_bytes = int( + sum(mgr.offload_summary_bytes.get(k, 0) for k in offload_modules) + ) + expected_offload_mib = expected_offload_bytes / (1024**2) + + _reset_cuda_memory() + off_logits, off_grads, off_peak = _run_one_iter_and_capture( + off_model, + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + enable_offload_reset=True, + ) + del off_model + _reset_cuda_memory() + + torch.cuda.memory._dump_snapshot(f"/workspace/pyt_profile/memory_snapshot.pickle") + print(f"Captured memory snapshot at /workspace/pyt_profile/memory_snapshot.pickle") + torch.cuda.memory._record_memory_history(enabled=False) + + # 3) Correctness checks (forward + selected grads) + assert torch.allclose(off_logits, base_logits, rtol=1e-3, atol=1e-3) + assert set(off_grads.keys()) == set(base_grads.keys()) + for name, gb in base_grads.items(): + go = off_grads[name] + if gb is None or go is None: + assert gb is None and go is None, f"Grad None mismatch for {name}" + continue + assert torch.allclose(go, gb, rtol=1e-3, atol=1e-3), f"Grad mismatch for {name}" + + # 4) Memory checks (peak allocated over forward+backward) + saved_mib = (base_peak - off_peak) / (1024**2) + assert saved_mib > 0.0, ( + f"Expected GPU peak memory reduction for offload_modules={offload_modules}, " + f"but got saved={saved_mib:.2f}MiB (base={base_peak/(1024**2):.2f}MiB, " + f"off={off_peak/(1024**2):.2f}MiB)" + ) + + # If expectation is large enough, enforce approximate match. + # For tiny expectations, allocator noise may dominate; we only require a positive reduction. + if expected_offload_mib >= 2.0: + rel_err = abs(saved_mib - expected_offload_mib) / max(expected_offload_mib, 1e-6) + assert rel_err <= EPSILON, ( + f"Memory saving mismatch for offload_modules={offload_modules}: " + f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB " + f"(rel_err={rel_err:.2f})" + ) + print(f"Rank {torch.distributed.get_rank()}: Saved {saved_mib:.2f}MiB, expected {expected_offload_mib:.2f}MiB") + finally: + Utils.destroy_model_parallel() From cb612c7f528b2f1319b866c07cb82b7ed47e23c5 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 7 Jan 2026 22:37:30 -0800 Subject: [PATCH 28/47] format Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offload.py | 1 + megatron/core/transformer/moe/experts.py | 4 ++- .../core/transformer/transformer_layer.py | 1 + ...test_fine_grained_activation_offloading.py | 25 ++++++++----------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index e073c518bd5..baabe29838a 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -1211,6 +1211,7 @@ def fine_grained_offloading_backward_record(tensor, event: torch.cuda.Event) -> """Record the backward event for cuda graph capture.""" return FineGrainedOffloadingBackwardRecordFunction.apply(tensor, event) + def fine_grained_offloading_reset_instance(): """Reset the singleton instance of PipelineOffloadManager.""" PipelineOffloadManager.reset_instance() diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index 262ceb55249..59e29a2ee68 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -722,7 +722,9 @@ def forward( ) if self.offload_expert_fc1: fc1_output = fine_grained_offloading_group_commit( - fc1_output, name="expert_fc1", forced_released_tensors=[permuted_local_hidden_states] + fc1_output, + name="expert_fc1", + forced_released_tensors=[permuted_local_hidden_states], ) def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index 1355dd4a426..cb263aba17c 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -759,6 +759,7 @@ def _forward_post_mlp(self, mlp_output_with_bias, residual): from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, ) + using_fused_tp_inference_kernel = (not self.training) and ( self.config.inference_fuse_tp_communication ) diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 7dbb0d77d9e..302af7a2c27 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -2,20 +2,19 @@ import gc import os +from contextlib import nullcontext from typing import Dict, List, Optional, Tuple import pytest import torch -from contextlib import nullcontext from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec from megatron.core.models.gpt.gpt_model import GPTModel from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed -from megatron.core.transformer.transformer_config import TransformerConfig, MLATransformerConfig +from megatron.core.transformer.enums import AttnBackend +from megatron.core.transformer.transformer_config import MLATransformerConfig, TransformerConfig from megatron.core.utils import is_te_min_version from tests.unit_tests.test_utilities import Utils -from megatron.core.transformer.enums import AttnBackend - # Tolerance for memory expectation check (GPU allocator jitter etc). EPSILON = 0.30 @@ -72,7 +71,7 @@ def _build_gpt_model( moe_grouped_gemm=num_experts is not None, moe_use_legacy_grouped_gemm=False, multi_latent_attention=is_mla, - ), + ), vocab_size=vocab_size, max_sequence_length=seq_length, ).bfloat16() @@ -80,10 +79,7 @@ def _build_gpt_model( def _make_gpt_inputs( - *, - seq_length: int, - micro_batch_size: int, - device: torch.device, + *, seq_length: int, micro_batch_size: int, device: torch.device ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: data = list(range(seq_length)) input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device) @@ -120,9 +116,7 @@ def _run_one_iter_and_capture( # p.grad = None torch.cuda.reset_peak_memory_stats() - logits = model( - input_ids=input_ids, position_ids=position_ids, attention_mask=attention_mask - ) + logits = model(input_ids=input_ids, position_ids=position_ids, attention_mask=attention_mask) loss = logits.float().sum() loss.backward() torch.cuda.synchronize() @@ -131,7 +125,7 @@ def _run_one_iter_and_capture( # capture all gradients for correctness grads: Dict[str, torch.Tensor] = {} for name, p in model.named_parameters(): - grads[name] = (p.grad.detach().float().cpu() if p.grad is not None else None) + grads[name] = p.grad.detach().float().cpu() if p.grad is not None else None return logits.detach().float().cpu(), grads, peak_bytes @@ -188,6 +182,7 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( ) from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + off.fine_grained_offloading_reset_instance() try: @@ -307,6 +302,8 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB " f"(rel_err={rel_err:.2f})" ) - print(f"Rank {torch.distributed.get_rank()}: Saved {saved_mib:.2f}MiB, expected {expected_offload_mib:.2f}MiB") + print( + f"Rank {torch.distributed.get_rank()}: Saved {saved_mib:.2f}MiB, expected {expected_offload_mib:.2f}MiB" + ) finally: Utils.destroy_model_parallel() From 64ae36142b079e62536386588e2ac0460f99a4f7 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 7 Jan 2026 22:51:10 -0800 Subject: [PATCH 29/47] remove unrelated changes Signed-off-by: Hongbin Liu --- megatron/core/tensor_parallel/__init__.py | 4 - megatron/core/tensor_parallel/random.py | 80 +--------- megatron/core/transformer/cuda_graphs.py | 7 +- megatron/core/transformer/moe/moe_utils.py | 20 ++- .../transformer/multi_token_prediction.py | 2 +- megatron/training/arguments.py | 5 +- megatron/training/checkpointing.py | 15 +- megatron/training/training.py | 3 - .../unit_tests/tensor_parallel/test_random.py | 145 ------------------ 9 files changed, 28 insertions(+), 253 deletions(-) diff --git a/megatron/core/tensor_parallel/__init__.py b/megatron/core/tensor_parallel/__init__.py index 2140ee54b37..98cc5efec82 100644 --- a/megatron/core/tensor_parallel/__init__.py +++ b/megatron/core/tensor_parallel/__init__.py @@ -29,11 +29,9 @@ from .random import ( CheckpointWithoutOutput, checkpoint, - convert_cuda_rng_state, get_cuda_rng_tracker, get_data_parallel_rng_tracker_name, get_expert_parallel_rng_tracker_name, - is_graph_safe_cuda_rng_tracker, model_parallel_cuda_manual_seed, ) from .utils import ( @@ -66,11 +64,9 @@ "scatter_to_sequence_parallel_region", # random.py "checkpoint", - "convert_cuda_rng_state", "get_cuda_rng_tracker", "model_parallel_cuda_manual_seed", "get_expert_parallel_rng_tracker_name", - "is_graph_safe_cuda_rng_tracker", "CheckpointWithoutOutput", # utils.py "split_tensor_along_last_dim", diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 5d5389a52d2..1ea726cb8fa 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. # Parts of the code here are adapted from PyTorch # repo: https://github.com/pytorch/pytorch @@ -111,41 +111,6 @@ def cb(): _lazy_call(cb) -def convert_cuda_rng_state( - state: Union[torch.Tensor, torch.Generator], to_graphable: bool = False -) -> Union[torch.Tensor, torch.Generator]: - """ - Convert the cuda rng state tensor to the graphable version, - or from the graphable version to the non-graphable tensor version. - """ - if to_graphable: - if isinstance(state, torch.Tensor): - # Convert to the graphable version. - # Store current rng state. - orig_cuda_rng_state = _get_cuda_rng_state(graph_safe=False) - # Set rng state to the desired one - _set_cuda_rng_state(state, graph_safe=False) - # Get the graphable state - graphable_state = _get_cuda_rng_state(clone=True, graph_safe=True) - # And set the state to the original state we started with. - _set_cuda_rng_state(orig_cuda_rng_state, graph_safe=False) - return graphable_state - elif isinstance(state, torch.Generator): - # already graphable, just return it. - return state - else: - raise ValueError(f"Invalid state type: {type(state)}") - else: - if isinstance(state, torch.Tensor): - # already non-graphable, just return it. - return state - elif isinstance(state, torch.Generator): - # Convert to the non-graphable tensor version. - return state.get_state() - else: - raise ValueError(f"Invalid state type: {type(state)}") - - def get_expert_parallel_rng_tracker_name(): """Get the expert parallel rng tracker name""" global _EXPERT_PARALLEL_RNG_TRACKER_NAME @@ -196,10 +161,6 @@ def reset(self): # Seeds are just for book keeping and ensure no seed is set twice. self.seeds_ = set() - # Name of the rng state currently being used in the generator. - # The default one is "default-rng" and won't be pushed to the self.states_ dictionary. - self._current_state_name = "default-rng" - def get_states(self): """Get rng states. Copy the dictionary so we have direct pointers to the states, not just a pointer to the dictionary.""" @@ -246,14 +207,10 @@ def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME): # Check if we have added the state if name not in self.states_: raise Exception('cuda rng state {} is not added'.format(name)) - # Store current rng state and name. Store in self.states_ if it's not the default state. + # Store current rng state. orig_cuda_rng_state = _get_cuda_rng_state(graph_safe=self.use_cudagraphable_rng) - orig_state_name = self._current_state_name - if orig_state_name != "default-rng": - self.states_[orig_state_name] = orig_cuda_rng_state - # Set rng state and name to the desired one. + # Set rng state to the desired one _set_cuda_rng_state(self.states_[name], graph_safe=self.use_cudagraphable_rng) - self._current_state_name = name # Record cpu RNG state cpu_rng_state = torch.get_rng_state() # Do the stuff we wanted to do. @@ -263,19 +220,10 @@ def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME): # Throw a warning if cpu RNG state changed if not torch.all(cpu_rng_state == torch.get_rng_state()).item(): logging.getLogger(__name__).warning('CPU RNG state changed within GPU RNG context') - # Check if the current state name is the same as the desired state name. - if self._current_state_name != name: - raise Exception( - f'current state name {self._current_state_name} is not the same as the desired ' - f'state name {name}.' - ) # Update the current rng state for later use. self.states_[name] = _get_cuda_rng_state(graph_safe=self.use_cudagraphable_rng) - # And set the state and name to the original state we started with. - if orig_state_name != "default-rng": - orig_cuda_rng_state = self.states_[orig_state_name] + # And set the state to the original state we started with. _set_cuda_rng_state(orig_cuda_rng_state, graph_safe=self.use_cudagraphable_rng) - self._current_state_name = orig_state_name # RNG tracker object. @@ -429,24 +377,10 @@ def model_parallel_cuda_manual_seed( _CUDA_RNG_STATE_TRACKER.add(_EXPERT_PARALLEL_RNG_TRACKER_NAME, expert_parallel_seed) -def is_graph_safe_cuda_rng_tracker(cuda_rng_tracker): - """Check if the cuda rng tracker is graph safe version.""" - if HAVE_TE and is_te_min_version("1.5.0"): - from megatron.core.extensions.transformer_engine import TECudaRNGStatesTracker - - if isinstance(cuda_rng_tracker, TECudaRNGStatesTracker): - return True - if getattr(cuda_rng_tracker, "use_cudagraphable_rng", False): - return True - return False - - def _get_all_rng_states(): """Get all the rng states.""" cpu_rng_state = torch.get_rng_state() - cuda_rng_state = _get_cuda_rng_state( - graph_safe=is_graph_safe_cuda_rng_tracker(get_cuda_rng_tracker()) - ) + cuda_rng_state = _get_cuda_rng_state() cuda_rng_state_tracker = get_cuda_rng_tracker().get_states() return cpu_rng_state, cuda_rng_state, cuda_rng_state_tracker @@ -454,9 +388,7 @@ def _get_all_rng_states(): def _set_all_rng_states(cpu_rng_state, cuda_rng_state, cuda_rng_state_tracker): """Set all the rng states.""" torch.set_rng_state(cpu_rng_state) - _set_cuda_rng_state( - cuda_rng_state, graph_safe=is_graph_safe_cuda_rng_tracker(get_cuda_rng_tracker()) - ) + _set_cuda_rng_state(cuda_rng_state) get_cuda_rng_tracker().set_states(cuda_rng_state_tracker) diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index a1cb764b01d..5fb9dfe15db 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -1911,12 +1911,7 @@ def create_cudagraphs(self): # Prepare CUDA Graph capturing input data and call `make_graphed_callables`. sample_args, kwargs = self._get_cuda_graph_input_data() - if self.config.sequence_parallel: - rng_context = get_cuda_rng_tracker().fork() - else: - rng_context = nullcontext() - with rng_context: - graphs = make_graphed_callables(tuple(self.flattened_callables), sample_args, **kwargs) + graphs = make_graphed_callables(tuple(self.flattened_callables), sample_args, **kwargs) # Push the captured graphs to the corresponding TransformerBlock. num_layers_accumulated = 0 diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 81fcc94dee3..6f9fa3099f4 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -11,7 +11,6 @@ from megatron.core.fp4_utils import get_fp4_align_size from megatron.core.fp8_utils import get_fp8_align_size from megatron.core.process_groups_config import ProcessGroupCollection -from megatron.core.tensor_parallel import get_cuda_rng_tracker, get_expert_parallel_rng_tracker_name from megatron.core.transformer.cuda_graphs import is_graph_capturing from megatron.core.transformer.enums import CudaGraphScope from megatron.core.transformer.transformer_config import TransformerConfig @@ -919,7 +918,6 @@ def get_moe_layer_wise_logging_tracker(): return _MOE_LAYER_WISE_LOGGING_TRACKER -@internal_api class RandomSTE(torch.autograd.Function): """ Straight-Through Estimator(STE) function that returns random values @@ -928,14 +926,26 @@ class RandomSTE(torch.autograd.Function): This is used to generate random logits of router for load-balanced benchmark. """ + generator = None + random_logits = None + @staticmethod def forward(ctx, logits): """ Forward pass returns random logits with rank-specific seed. """ - with get_cuda_rng_tracker().fork(get_expert_parallel_rng_tracker_name()): - random_logits = logits.clone().normal_() - return random_logits + if is_graph_capturing() and RandomSTE.random_logits is not None: + return RandomSTE.random_logits + + if RandomSTE.generator is None: + global_rank = torch.distributed.get_rank() + base_seed = 42 + seed = base_seed + global_rank + RandomSTE.generator = torch.Generator(device=logits.device) + RandomSTE.generator.manual_seed(seed) + + RandomSTE.random_logits = logits.clone().normal_(generator=RandomSTE.generator) + return RandomSTE.random_logits @staticmethod def backward(ctx, grad_output): diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py index 7c0438c6227..124430e107a 100755 --- a/megatron/core/transformer/multi_token_prediction.py +++ b/megatron/core/transformer/multi_token_prediction.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. import warnings from contextlib import nullcontext diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 12ffca619ab..c4ae8d53542 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1255,10 +1255,7 @@ def validate_args(args, defaults={}): # CUDA Graphs if args.cuda_graph_impl != "none": - if ( - "transformer_engine" in (args.transformer_impl, args.cuda_graph_impl) - and not args.te_rng_tracker - ): + if args.transformer_impl == 'transformer_engine' and not args.te_rng_tracker: args.te_rng_tracker = True warn_rank_0("te_rng_tracker is not enabled, enabling it for CUDA graphs.", args.rank) assert ( diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py index 0a96d088728..e49e9648d62 100644 --- a/megatron/training/checkpointing.py +++ b/megatron/training/checkpointing.py @@ -1766,8 +1766,6 @@ def load_model_state_dict(module, state_dict, strict: bool): # rng states. if not release and not args.finetune and not args.no_load_rng and not ignore_rng_state: try: - cuda_rng_tracker = tensor_parallel.get_cuda_rng_tracker() - graph_safe_rng = tensor_parallel.is_graph_safe_cuda_rng_tracker(cuda_rng_tracker) if 'rng_state' in state_dict: if args.ckpt_format == "fsdp_dtensor": # FSDP DTensor checkpoints store rng_state in a different format. @@ -1793,10 +1791,8 @@ def load_model_state_dict(module, state_dict, strict: bool): # Check for empty states array if not rng_state['rng_tracker_states']: raise KeyError - rng_tracker_states = { - k: tensor_parallel.convert_cuda_rng_state(v, to_graphable=graph_safe_rng) - for k, v in rng_state['rng_tracker_states'].items() - } + tensor_parallel.get_cuda_rng_tracker().set_states( + rng_state['rng_tracker_states']) else: # backward compatability random.setstate(state_dict['random_rng_state']) np.random.set_state(state_dict['np_rng_state']) @@ -1805,11 +1801,8 @@ def load_model_state_dict(module, state_dict, strict: bool): # Check for empty states array if not state_dict['rng_tracker_states']: raise KeyError - rng_tracker_states = { - k: tensor_parallel.convert_cuda_rng_state(v, to_graphable=graph_safe_rng) - for k, v in state_dict['rng_tracker_states'].items() - } - cuda_rng_tracker.set_states(rng_tracker_states) + tensor_parallel.get_cuda_rng_tracker().set_states( + state_dict['rng_tracker_states']) except KeyError: print_rank_0('Unable to load rng state from checkpoint {}. ' 'Specify --no-load-rng or --finetune to prevent ' diff --git a/megatron/training/training.py b/megatron/training/training.py index b2de8dd16dc..aaee7f281e9 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -646,9 +646,6 @@ def pretrain( ) set_ideal_affinity_for_current_gpu() - if args.batch_invariant_mode: - print_rank_0("Enabling batch invariant mode globally",flush=True) - enable_batch_invariant_mode() if args.log_progress: append_to_progress_log("Starting job") diff --git a/tests/unit_tests/tensor_parallel/test_random.py b/tests/unit_tests/tensor_parallel/test_random.py index a15ad83cb90..47b607b8795 100644 --- a/tests/unit_tests/tensor_parallel/test_random.py +++ b/tests/unit_tests/tensor_parallel/test_random.py @@ -1,5 +1,3 @@ -# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - import pytest import torch @@ -7,7 +5,6 @@ CheckpointWithoutOutput, CudaRNGStatesTracker, checkpoint, - convert_cuda_rng_state, get_cuda_rng_tracker, model_parallel_cuda_manual_seed, ) @@ -36,148 +33,6 @@ def test_cuda_rng_states_tracker(): assert torch.equal(rng_tracker.get_states()['state2'], rng_state) -@pytest.mark.parametrize("use_cudagraphable_rng", [True, False]) -def test_double_fork_cuda_rng_states_tracker(use_cudagraphable_rng): - rng_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=use_cudagraphable_rng) - rng_tracker.add("state1", 1234) - rng_tracker.add("state2", 5678) - randn_double_fork_1 = [] - randn_double_fork_2 = [] - with rng_tracker.fork("state1"): - randn_double_fork_1.append(torch.randn(10, device="cuda")) - with rng_tracker.fork("state2"): - randn_double_fork_2.append(torch.randn(10, device="cuda")) - with rng_tracker.fork("state1"): - randn_double_fork_1.append(torch.randn(10, device="cuda")) - randn_double_fork_2.append(torch.randn(10, device="cuda")) - randn_double_fork_1.append(torch.randn(10, device="cuda")) - if use_cudagraphable_rng: - double_fork_state1 = rng_tracker.get_states()["state1"].get_state() - double_fork_state2 = rng_tracker.get_states()["state2"].get_state() - else: - double_fork_state1 = rng_tracker.get_states()["state1"] - double_fork_state2 = rng_tracker.get_states()["state2"] - - rng_tracker.reset() - rng_tracker.add("state1", 1234) - rng_tracker.add("state2", 5678) - randn_single_fork_1 = [] - randn_single_fork_2 = [] - with rng_tracker.fork("state1"): - randn_single_fork_1.append(torch.randn(10, device="cuda")) - randn_single_fork_1.append(torch.randn(10, device="cuda")) - randn_single_fork_1.append(torch.randn(10, device="cuda")) - with rng_tracker.fork("state2"): - randn_single_fork_2.append(torch.randn(10, device="cuda")) - randn_single_fork_2.append(torch.randn(10, device="cuda")) - if use_cudagraphable_rng: - single_fork_state1 = rng_tracker.get_states()["state1"].get_state() - single_fork_state2 = rng_tracker.get_states()["state2"].get_state() - else: - single_fork_state1 = rng_tracker.get_states()["state1"] - single_fork_state2 = rng_tracker.get_states()["state2"] - - assert torch.equal(randn_double_fork_1[0], randn_single_fork_1[0]) - assert torch.equal(randn_double_fork_1[1], randn_single_fork_1[1]) - assert torch.equal(randn_double_fork_1[2], randn_single_fork_1[2]) - assert torch.equal(randn_double_fork_2[0], randn_single_fork_2[0]) - assert torch.equal(randn_double_fork_2[1], randn_single_fork_2[1]) - assert torch.equal(double_fork_state1, single_fork_state1) - assert torch.equal(double_fork_state2, single_fork_state2) - - -def test_convert_cuda_rng_state(): - ## Get the default rng state - torch.cuda.manual_seed(999) - randn = torch.randn(10, device="cuda") - rng_state = torch.cuda.get_rng_state() - - try: - from megatron.core.extensions.transformer_engine import TECudaRNGStatesTracker - except ImportError: - TECudaRNGStatesTracker = None - - ## from non-graphable RNG to graphable RNG - # get state from non-graphable RNG - tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False) - tracker.add("state1", 123) - for i in range(3): - with tracker.fork("state1"): - randn = torch.randn(10, device="cuda") - state = convert_cuda_rng_state(tracker.states_["state1"], to_graphable=True) - rand_tensors = [] - for i in range(3): - with tracker.fork("state1"): - randn = torch.randn(10, device="cuda") - rand_tensors.append(randn) - - # set state to local graph RNG - cudagraphable_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=True) - cudagraphable_tracker.set_states({"state1": state.clone_state()}) - for i in range(3): - with cudagraphable_tracker.fork("state1"): - randn = torch.randn(10, device="cuda") - assert torch.equal(randn, rand_tensors[i]) - - # set state to TE RNG - if TECudaRNGStatesTracker is not None: - te_tracker = TECudaRNGStatesTracker() - te_tracker.set_states({"state1": state}) - for i in range(3): - with te_tracker.fork("state1"): - randn = torch.randn(10, device="cuda") - assert torch.equal(randn, rand_tensors[i]) - - ## from graphable RNG to non-graphable RNG - # get state from graphable RNG - cudagraphable_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=True) - cudagraphable_tracker.add("state2", 123) - for i in range(3): - with cudagraphable_tracker.fork("state2"): - randn = torch.randn(10, device="cuda") - state = convert_cuda_rng_state(cudagraphable_tracker.states_["state2"], to_graphable=False) - rand_tensors = [] - for i in range(3): - with cudagraphable_tracker.fork("state2"): - randn = torch.randn(10, device="cuda") - rand_tensors.append(randn) - - # set state to non-graphable RNG - tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False) - tracker.set_states({"state2": state}) - for i in range(3): - with tracker.fork("state2"): - randn = torch.randn(10, device="cuda") - assert torch.equal(randn, rand_tensors[i]) - - ## from TE RNG to non-graphable RNG - if TECudaRNGStatesTracker is not None: - # get state from TE RNG - cudagraphable_tracker = TECudaRNGStatesTracker() - cudagraphable_tracker.add("state3", 123) - for i in range(3): - with cudagraphable_tracker.fork("state3"): - randn = torch.randn(10, device="cuda") - state = convert_cuda_rng_state(cudagraphable_tracker.states_["state3"], to_graphable=False) - rand_tensors = [] - for i in range(3): - with cudagraphable_tracker.fork("state3"): - randn = torch.randn(10, device="cuda") - rand_tensors.append(randn) - - # set state to non-graphable RNG - tracker = CudaRNGStatesTracker(use_cudagraphable_rng=False) - tracker.set_states({"state3": state}) - for i in range(3): - with tracker.fork("state3"): - randn = torch.randn(10, device="cuda") - assert torch.equal(randn, rand_tensors[i]) - - ## After all tests, check if the default rng state is still the same. - rng_state_final = torch.cuda.get_rng_state() - assert torch.equal(rng_state, rng_state_final) - - def test_model_parallel_cuda_manual_seed(): Utils.initialize_model_parallel(4, 2) model_parallel_cuda_manual_seed(0, force_reset_rng=True) From a8dfa651bc06c7b1bf62c3f897527f44070a2e94 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 7 Jan 2026 23:28:35 -0800 Subject: [PATCH 30/47] fix ut Signed-off-by: Hongbin Liu --- tests/unit_tests/models/test_mamba_moe_model.py | 3 +++ .../test_fine_grained_activation_offloading.py | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/unit_tests/models/test_mamba_moe_model.py b/tests/unit_tests/models/test_mamba_moe_model.py index 94120b7c1bf..dbb28a2a7c4 100644 --- a/tests/unit_tests/models/test_mamba_moe_model.py +++ b/tests/unit_tests/models/test_mamba_moe_model.py @@ -414,6 +414,9 @@ def create_test_args(self): args.te_rng_tracker = True args.tensor_model_parallel_size = 2 args.vocab_size = 131072 + args.fine_grained_activation_offloading = False + args.min_offloaded_tensor_size = 1024 * 1024 + args.offload_modules = [] validate_args(args) set_global_variables(args, False) diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 302af7a2c27..34c1e523dcb 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -164,7 +164,6 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( os.environ.pop("NVTE_UNFUSED_ATTN", None) # os.environ["NVTE_FLASH_ATTN"] = "1" Utils.initialize_model_parallel(1, 1) - torch.cuda.memory._record_memory_history(max_entries=100000) seed = 123 # Choose shapes large enough to make memory deltas stable but still fast. @@ -271,10 +270,6 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( del off_model _reset_cuda_memory() - torch.cuda.memory._dump_snapshot(f"/workspace/pyt_profile/memory_snapshot.pickle") - print(f"Captured memory snapshot at /workspace/pyt_profile/memory_snapshot.pickle") - torch.cuda.memory._record_memory_history(enabled=False) - # 3) Correctness checks (forward + selected grads) assert torch.allclose(off_logits, base_logits, rtol=1e-3, atol=1e-3) assert set(off_grads.keys()) == set(base_grads.keys()) From 2c63e1d6eac9f1d5a530cd8d5e5ddc5c9a2f9304 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 7 Jan 2026 23:51:30 -0800 Subject: [PATCH 31/47] fix ut Signed-off-by: Hongbin Liu --- tests/unit_tests/models/test_mamba_moe_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/models/test_mamba_moe_model.py b/tests/unit_tests/models/test_mamba_moe_model.py index dbb28a2a7c4..6ed97302d86 100644 --- a/tests/unit_tests/models/test_mamba_moe_model.py +++ b/tests/unit_tests/models/test_mamba_moe_model.py @@ -251,6 +251,9 @@ "wgrad_deferral_limit": 0, "window_attn_skip_freq": None, "window_size": None, + "fine_grained_activation_offloading": False, + "min_offloaded_tensor_size": 1024 * 1024, + "offload_modules": [], } # Fields to ignore entirely (ephemeral, environment-specific, very large). SKIP_FIELDS = set() @@ -414,9 +417,6 @@ def create_test_args(self): args.te_rng_tracker = True args.tensor_model_parallel_size = 2 args.vocab_size = 131072 - args.fine_grained_activation_offloading = False - args.min_offloaded_tensor_size = 1024 * 1024 - args.offload_modules = [] validate_args(args) set_global_variables(args, False) From f52069e765d269887015523892b93d25f1d6215e Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Thu, 8 Jan 2026 00:20:27 -0800 Subject: [PATCH 32/47] fix ut Signed-off-by: Hongbin Liu --- tests/unit_tests/models/test_mamba_moe_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/models/test_mamba_moe_model.py b/tests/unit_tests/models/test_mamba_moe_model.py index 6ed97302d86..e162506047c 100644 --- a/tests/unit_tests/models/test_mamba_moe_model.py +++ b/tests/unit_tests/models/test_mamba_moe_model.py @@ -252,7 +252,7 @@ "window_attn_skip_freq": None, "window_size": None, "fine_grained_activation_offloading": False, - "min_offloaded_tensor_size": 1024 * 1024, + "min_offloaded_tensor_size": 10 * 1024 * 1024, "offload_modules": [], } # Fields to ignore entirely (ephemeral, environment-specific, very large). From 0643e558d0bf986d59150a77e4e5c01187cf3ac4 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Fri, 9 Jan 2026 02:32:11 -0800 Subject: [PATCH 33/47] update ut Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offload.py | 5 +- ...test_fine_grained_activation_offloading.py | 271 +++++++++++++++++- 2 files changed, 265 insertions(+), 11 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index baabe29838a..b0f8c0e4cf0 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -973,7 +973,8 @@ def bulk_reload(self): else: # Pre-load the last layer of the next backward chunk to hide latency next_backward_chunk = PipelineOffloadManager.get_instance().front() - if next_backward_chunk is not None: + if next_backward_chunk is not None \ + and next_backward_chunk._offloaded_group_index == next_backward_chunk._max_group_size: next_backward_chunk.pre_reload_last_layer() def on_group_commit_backward(self, name): @@ -1031,7 +1032,7 @@ def on_group_start_backward(self): """ if not self.do_offload: return - debug_rank("--on_group_start_backward") + debug_rank(f"--on_group_start_backward {self}") # Wait for compute to finish before starting reload self.h2d_stream.wait_stream(torch.cuda.current_stream()) self.bulk_reload() diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 34c1e523dcb..c14299a4c29 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -18,6 +18,8 @@ # Tolerance for memory expectation check (GPU allocator jitter etc). EPSILON = 0.30 +EPSILON_A2A = 0.30 +DELTA = 20 # MiB def _reset_cuda_memory() -> None: @@ -51,11 +53,10 @@ def _build_gpt_model( num_attention_heads=num_attention_heads, use_cpu_initialization=True, attention_backend=AttnBackend.unfused, - # Make sure model weights / activations are BF16 so TE fused attention isn't disabled. bf16=True, - # params_dtype=torch.bfloat16, - # enable_autocast=True, - # autocast_dtype=torch.bfloat16, + # Recompute + recompute_modules=["layernorm", "moe_act"] if num_experts is not None else ["layernorm"], + recompute_granularity="selective", # MoE num_moe_experts=num_experts, moe_grouped_gemm=(num_experts is not None), @@ -147,6 +148,8 @@ def _run_one_iter_and_capture( (True, False, ["mlp_norm"]), (True, False, ["expert_fc1"]), (True, False, ["moe_act"]), + (True, True, ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), + (True, False, ["core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), ], ) def test_gpt_fine_grained_activation_offloading_correctness_and_memory( @@ -163,7 +166,7 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( os.environ.pop("NVTE_FLASH_ATTN", None) os.environ.pop("NVTE_UNFUSED_ATTN", None) # os.environ["NVTE_FLASH_ATTN"] = "1" - Utils.initialize_model_parallel(1, 1) + Utils.initialize_model_parallel(tensor_model_parallel_size=1, pipeline_model_parallel_size=1) seed = 123 # Choose shapes large enough to make memory deltas stable but still fast. @@ -171,8 +174,8 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( num_layers = 8 hidden_size = 2048 if num_experts is None else 1024 num_attention_heads = 16 if hidden_size >= 2048 else 8 - vocab_size = 512 - seq_length = 512 + vocab_size = 1024 + seq_length = 1024 micro_batch_size = 2 device = torch.device("cuda") @@ -292,13 +295,263 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( # For tiny expectations, allocator noise may dominate; we only require a positive reduction. if expected_offload_mib >= 2.0: rel_err = abs(saved_mib - expected_offload_mib) / max(expected_offload_mib, 1e-6) - assert rel_err <= EPSILON, ( + abs_err = abs(saved_mib - expected_offload_mib) + assert rel_err <= EPSILON and abs_err <= DELTA, ( f"Memory saving mismatch for offload_modules={offload_modules}: " f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB " - f"(rel_err={rel_err:.2f})" + f"(rel_err={rel_err:.2f}, abs_err={abs_err:.2f})" ) print( f"Rank {torch.distributed.get_rank()}: Saved {saved_mib:.2f}MiB, expected {expected_offload_mib:.2f}MiB" ) finally: Utils.destroy_model_parallel() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for offloading tests.") +@pytest.mark.skipif( + not is_te_min_version("1.13.0"), + reason="Requires TE-based GPT layer spec (TE 1.13+ in this repo's tests).", +) +@pytest.mark.parametrize( + "dispatcher_backend, is_mla, offload_modules", + [ + ("alltoall", True, ["attn_norm"]), + ("alltoall", True, ["core_attn"]), + ("alltoall", True, ["attn_norm", "core_attn", "attn_proj"]), + ("alltoall", True, ["mlp_norm"]), + ("alltoall", False, ["expert_fc1"]), + ("alltoall", False, ["moe_act"]), + ("alltoall", False, ["mlp_norm", "expert_fc1", "moe_act"]), + ("alltoall", True, ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), + ("alltoall", False, ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), + ], +) +def test_fine_grained_activation_offload_with_ep_a2a_overlap_compatibility( + dispatcher_backend: str, is_mla: bool, offload_modules: List[str], +): + """ + Compatibility test for: + - fine-grained activation offloading + - EP all-to-all overlap (overlap_moe_expert_parallel_comm) + - memory saving roughly matches expected offload bytes (when expectation is large enough) + + The EP A2A overlap initialization pattern is aligned with + `tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py`. + """ + from megatron.core.pipeline_parallel.utils import set_streams + from megatron.core.models.common.model_chunk_schedule_plan import TransformerModelChunkSchedulePlan + from tests.unit_tests.a2a_overlap.utils import deterministic_mode + + # EP overlap requires distributed initialization with EP groups. + ep_size = 4 + if Utils.world_size % ep_size != 0: + pytest.skip( + f"Skipping: WORLD_SIZE={Utils.world_size} must be divisible by ep_size={ep_size}." + ) + + seed = 123 + num_experts = 8 # must be divisible by ep_size + if num_experts % ep_size != 0: + pytest.skip(f"Skipping: num_moe_experts={num_experts} must be divisible by ep_size={ep_size}.") + + # Small shapes to keep this compatibility test fast. + num_layers = 8 + hidden_size = 1024 + num_attention_heads = 16 + vocab_size = 1024 + seq_length = 1024 + micro_batch_size = 2 + device = torch.device("cuda") + + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + + def _make_schedule_inputs() -> Dict[str, torch.Tensor]: + data = list(range(seq_length)) + input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device) + position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to( + device + ) + attention_mask = torch.ones((micro_batch_size, 1, seq_length, seq_length), dtype=bool).to( + device + ) + labels = input_ids.clone() + return { + "input_ids": input_ids, + "labels": labels, + "position_ids": position_ids, + "attention_mask": attention_mask, + } + + def _capture_params(model: torch.nn.Module) -> Dict[str, torch.Tensor]: + params: Dict[str, torch.Tensor] = {} + for name, p in model.named_parameters(): + params[name] = p.detach().clone() + return params + + def _restore_params(model: torch.nn.Module, params: Dict[str, torch.Tensor]) -> None: + for name, p in model.named_parameters(): + p.data.copy_(params[name]) + + def _build_overlap_moe_gpt(*, enable_offload: bool, is_mla: bool, dispatcher_backend: str) -> GPTModel: + model_parallel_cuda_manual_seed(seed) + torch.manual_seed(seed) + ConfigClass = MLATransformerConfig if is_mla else TransformerConfig + transformer_config = ConfigClass( + num_layers=num_layers, + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + use_cpu_initialization=True, + attention_backend=AttnBackend.unfused, + # Recompute + recompute_modules=["layernorm", "moe_act"], + recompute_granularity="selective", + bf16=True, + # MoE + EP overlap + num_moe_experts=num_experts, + moe_grouped_gemm=True, + expert_model_parallel_size=ep_size, + moe_token_dispatcher_type="alltoall" if dispatcher_backend == "alltoall" else "flex", + moe_flex_dispatcher_backend=dispatcher_backend, + moe_router_dtype="fp32" if dispatcher_backend == "hybridep" else "fp64", + overlap_moe_expert_parallel_comm=True, + delay_wgrad_compute=True, + # Fine-grained activation offloading + fine_grained_activation_offloading=enable_offload, + offload_modules=offload_modules if enable_offload else None, + min_offloaded_tensor_size=1024, # force offloading to exercise the code path + ) + return ( + GPTModel( + config=transformer_config, + transformer_layer_spec=get_gpt_layer_with_transformer_engine_spec( + num_experts=num_experts, + moe_grouped_gemm=True, + moe_use_legacy_grouped_gemm=False, + multi_latent_attention=is_mla, + ), + vocab_size=vocab_size, + max_sequence_length=seq_length, + ) + .bfloat16() + .cuda() + ) + + def _run_schedule_1f1b_two_microbatches( + model: GPTModel, *, enable_offload_reset: bool + ) -> Tuple[List[torch.Tensor], Dict[str, torch.Tensor], int]: + """ + Run a minimal 1F1B schedule (2 microbatches) using ModelChunkSchedulePlan.run(). + This is the execution path that exercises EP A2A overlap scheduling. + """ + if enable_offload_reset: + off.fine_grained_offloading_reset() + + data0 = _make_schedule_inputs() + data1 = _make_schedule_inputs() + plan0 = model.build_schedule_plan(**data0) + + torch.cuda.reset_peak_memory_stats() + out0 = TransformerModelChunkSchedulePlan.run(plan0, None) + plan1 = model.build_schedule_plan(**data1) + out1 = TransformerModelChunkSchedulePlan.run(plan1, plan0, b_grad=torch.ones_like(out0)) + TransformerModelChunkSchedulePlan.run(None, plan1, b_grad=torch.ones_like(out1)) + torch.cuda.synchronize() + peak_bytes = int(torch.cuda.max_memory_allocated()) + + # capture outputs and grads + outputs = [out0.detach().float().cpu(), out1.detach().float().cpu()] + grads: Dict[str, torch.Tensor] = {} + for name, p in model.named_parameters(): + grads[name] = p.grad.detach().float().cpu() if p.grad is not None else None + return outputs, grads, peak_bytes + + # setup distributed/model-parallel + os.environ.pop("NVTE_FUSED_ATTN", None) + os.environ.pop("NVTE_FLASH_ATTN", None) + os.environ.pop("NVTE_UNFUSED_ATTN", None) + + Utils.initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + expert_model_parallel_size=ep_size, + ) + set_streams() + + off.fine_grained_offloading_reset_instance() + + try: + with deterministic_mode(): + # Baseline: EP overlap on, offload off. + _reset_cuda_memory() + base_model = _build_overlap_moe_gpt(enable_offload=False, is_mla=is_mla, dispatcher_backend=dispatcher_backend) + base_model.train() + base_params = _capture_params(base_model) + # Warmup once for allocator stability / graph caching + _run_schedule_1f1b_two_microbatches(base_model, enable_offload_reset=False) + _reset_cuda_memory() + base_outs, base_grads, base_peak = _run_schedule_1f1b_two_microbatches( + base_model, enable_offload_reset=False + ) + del base_model + _reset_cuda_memory() + + # Offload: EP overlap on, fine-grained offload on. + off_model = _build_overlap_moe_gpt(enable_offload=True, is_mla=is_mla, dispatcher_backend=dispatcher_backend) + _restore_params(off_model, base_params) + off_model.train() + # Warmup once to populate cached chunks, then reset to apply steady-state offload decisions. + off.fine_grained_offloading_reset() + _run_schedule_1f1b_two_microbatches(off_model, enable_offload_reset=False) + off.fine_grained_offloading_reset() + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + PipelineOffloadManager, + ) + + mgr = PipelineOffloadManager.get_instance() + expected_offload_bytes = int( + sum(mgr.offload_summary_bytes.get(k, 0) for k in offload_modules) + ) + expected_offload_mib = expected_offload_bytes / (1024**2) + + _reset_cuda_memory() + off_outs, off_grads, off_peak = _run_schedule_1f1b_two_microbatches( + off_model, enable_offload_reset=True + ) + del off_model + _reset_cuda_memory() + + # Correctness (forward outputs + all grads) + assert len(off_outs) == len(base_outs) == 2 + for i in range(2): + assert torch.allclose(off_outs[i], base_outs[i], rtol=1e-3, atol=1e-3) + assert set(off_grads.keys()) == set(base_grads.keys()) + for name, gb in base_grads.items(): + go = off_grads[name] + if gb is None or go is None: + assert gb is None and go is None, f"Grad None mismatch for {name}" + continue + assert torch.allclose(go, gb, rtol=1e-3, atol=1e-3), f"Rank {torch.distributed.get_rank()}: Grad mismatch for {name}" + + # Memory checks (peak allocated during the scheduled 1F1B run) + saved_mib = (base_peak - off_peak) / (1024**2) + assert saved_mib > 0.0, ( + f"Expected GPU peak memory reduction for offload_modules={offload_modules}, " + f"but got saved={saved_mib:.2f}MiB (base={base_peak/(1024**2):.2f}MiB, " + f"off={off_peak/(1024**2):.2f}MiB)" + ) + # If expectation is large enough, enforce approximate match. + if expected_offload_mib >= 2.0: + rel_err = abs(saved_mib - expected_offload_mib) / max(expected_offload_mib, 1e-6) + abs_err = abs(saved_mib - expected_offload_mib) + print( + f"Rank {torch.distributed.get_rank()}: Saved {saved_mib:.2f}MiB, expected {expected_offload_mib:.2f}MiB" + ) + if abs_err > DELTA: + assert rel_err <= EPSILON_A2A, ( + f"Memory saving mismatch for offload_modules={offload_modules}: " + f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB " + f"(rel_err={rel_err:.2f}, abs_err={abs_err:.2f})" + ) + finally: + Utils.destroy_model_parallel() From d57ccb0020258cc50af033061fac83d62d0ce01c Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Fri, 9 Jan 2026 02:33:40 -0800 Subject: [PATCH 34/47] format Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offload.py | 7 ++- ...test_fine_grained_activation_offloading.py | 52 +++++++++++++------ 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index b0f8c0e4cf0..bbb1f7e32e4 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -973,8 +973,11 @@ def bulk_reload(self): else: # Pre-load the last layer of the next backward chunk to hide latency next_backward_chunk = PipelineOffloadManager.get_instance().front() - if next_backward_chunk is not None \ - and next_backward_chunk._offloaded_group_index == next_backward_chunk._max_group_size: + if ( + next_backward_chunk is not None + and next_backward_chunk._offloaded_group_index + == next_backward_chunk._max_group_size + ): next_backward_chunk.pre_reload_last_layer() def on_group_commit_backward(self, name): diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index c14299a4c29..3a9bc4ddf35 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -19,7 +19,7 @@ # Tolerance for memory expectation check (GPU allocator jitter etc). EPSILON = 0.30 EPSILON_A2A = 0.30 -DELTA = 20 # MiB +DELTA = 20 # MiB def _reset_cuda_memory() -> None: @@ -323,12 +323,20 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( ("alltoall", False, ["expert_fc1"]), ("alltoall", False, ["moe_act"]), ("alltoall", False, ["mlp_norm", "expert_fc1", "moe_act"]), - ("alltoall", True, ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), - ("alltoall", False, ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), + ( + "alltoall", + True, + ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"], + ), + ( + "alltoall", + False, + ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"], + ), ], ) def test_fine_grained_activation_offload_with_ep_a2a_overlap_compatibility( - dispatcher_backend: str, is_mla: bool, offload_modules: List[str], + dispatcher_backend: str, is_mla: bool, offload_modules: List[str] ): """ Compatibility test for: @@ -339,8 +347,10 @@ def test_fine_grained_activation_offload_with_ep_a2a_overlap_compatibility( The EP A2A overlap initialization pattern is aligned with `tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py`. """ + from megatron.core.models.common.model_chunk_schedule_plan import ( + TransformerModelChunkSchedulePlan, + ) from megatron.core.pipeline_parallel.utils import set_streams - from megatron.core.models.common.model_chunk_schedule_plan import TransformerModelChunkSchedulePlan from tests.unit_tests.a2a_overlap.utils import deterministic_mode # EP overlap requires distributed initialization with EP groups. @@ -353,7 +363,9 @@ def test_fine_grained_activation_offload_with_ep_a2a_overlap_compatibility( seed = 123 num_experts = 8 # must be divisible by ep_size if num_experts % ep_size != 0: - pytest.skip(f"Skipping: num_moe_experts={num_experts} must be divisible by ep_size={ep_size}.") + pytest.skip( + f"Skipping: num_moe_experts={num_experts} must be divisible by ep_size={ep_size}." + ) # Small shapes to keep this compatibility test fast. num_layers = 8 @@ -369,8 +381,8 @@ def test_fine_grained_activation_offload_with_ep_a2a_overlap_compatibility( def _make_schedule_inputs() -> Dict[str, torch.Tensor]: data = list(range(seq_length)) input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device) - position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to( - device + position_ids = ( + torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).to(device) ) attention_mask = torch.ones((micro_batch_size, 1, seq_length, seq_length), dtype=bool).to( device @@ -393,7 +405,9 @@ def _restore_params(model: torch.nn.Module, params: Dict[str, torch.Tensor]) -> for name, p in model.named_parameters(): p.data.copy_(params[name]) - def _build_overlap_moe_gpt(*, enable_offload: bool, is_mla: bool, dispatcher_backend: str) -> GPTModel: + def _build_overlap_moe_gpt( + *, enable_offload: bool, is_mla: bool, dispatcher_backend: str + ) -> GPTModel: model_parallel_cuda_manual_seed(seed) torch.manual_seed(seed) ConfigClass = MLATransformerConfig if is_mla else TransformerConfig @@ -484,7 +498,9 @@ def _run_schedule_1f1b_two_microbatches( with deterministic_mode(): # Baseline: EP overlap on, offload off. _reset_cuda_memory() - base_model = _build_overlap_moe_gpt(enable_offload=False, is_mla=is_mla, dispatcher_backend=dispatcher_backend) + base_model = _build_overlap_moe_gpt( + enable_offload=False, is_mla=is_mla, dispatcher_backend=dispatcher_backend + ) base_model.train() base_params = _capture_params(base_model) # Warmup once for allocator stability / graph caching @@ -497,7 +513,9 @@ def _run_schedule_1f1b_two_microbatches( _reset_cuda_memory() # Offload: EP overlap on, fine-grained offload on. - off_model = _build_overlap_moe_gpt(enable_offload=True, is_mla=is_mla, dispatcher_backend=dispatcher_backend) + off_model = _build_overlap_moe_gpt( + enable_offload=True, is_mla=is_mla, dispatcher_backend=dispatcher_backend + ) _restore_params(off_model, base_params) off_model.train() # Warmup once to populate cached chunks, then reset to apply steady-state offload decisions. @@ -531,7 +549,9 @@ def _run_schedule_1f1b_two_microbatches( if gb is None or go is None: assert gb is None and go is None, f"Grad None mismatch for {name}" continue - assert torch.allclose(go, gb, rtol=1e-3, atol=1e-3), f"Rank {torch.distributed.get_rank()}: Grad mismatch for {name}" + assert torch.allclose( + go, gb, rtol=1e-3, atol=1e-3 + ), f"Rank {torch.distributed.get_rank()}: Grad mismatch for {name}" # Memory checks (peak allocated during the scheduled 1F1B run) saved_mib = (base_peak - off_peak) / (1024**2) @@ -549,9 +569,9 @@ def _run_schedule_1f1b_two_microbatches( ) if abs_err > DELTA: assert rel_err <= EPSILON_A2A, ( - f"Memory saving mismatch for offload_modules={offload_modules}: " - f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB " - f"(rel_err={rel_err:.2f}, abs_err={abs_err:.2f})" - ) + f"Memory saving mismatch for offload_modules={offload_modules}: " + f"saved={saved_mib:.2f}MiB expected~={expected_offload_mib:.2f}MiB " + f"(rel_err={rel_err:.2f}, abs_err={abs_err:.2f})" + ) finally: Utils.destroy_model_parallel() From ff4a2cd2e576424516890ad67163b2e9c34a50de Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Fri, 9 Jan 2026 03:21:33 -0800 Subject: [PATCH 35/47] update golden values Signed-off-by: Hongbin Liu --- .../golden_values_dev_dgx_h100.json | 482 +++++++++--------- ...test_fine_grained_activation_offloading.py | 8 +- 2 files changed, 243 insertions(+), 247 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json index 505236b8dfd..9750a5861c2 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -8,52 +8,52 @@ "2": 11.06263, "3": 10.17794, "4": 10.85994, - "5": 9.81692, - "6": 9.10097, - "7": 9.61229, - "8": 8.39605, - "9": 7.79605, - "10": 7.15189, - "11": 9.06657, - "12": 12.42312, - "13": 8.06867, - "14": 8.25318, - "15": 8.25396, - "16": 8.3337, - "17": 8.34707, - "18": 7.58571, - "19": 8.1941, - "20": 7.70065, - "21": 8.00269, - "22": 7.35729, - "23": 7.947, - "24": 7.50771, - "25": 8.3251, - "26": 7.78973, - "27": 7.72902, - "28": 7.7061, - "29": 7.77502, - "30": 7.5685, - "31": 7.85269, - "32": 6.53623, - "33": 7.24568, - "34": 7.80274, - "35": 7.74675, - "36": 6.73775, - "37": 8.15358, - "38": 7.62596, - "39": 7.97835, - "40": 7.52416, - "41": 7.52486, - "42": 6.12066, - "43": 7.60578, - "44": 7.96683, - "45": 6.8492, - "46": 7.42923, - "47": 7.82506, - "48": 7.88008, - "49": 7.59776, - "50": 6.85432 + "5": 9.81693, + "6": 9.10087, + "7": 9.61225, + "8": 8.39617, + "9": 7.79702, + "10": 7.15217, + "11": 9.06659, + "12": 12.29047, + "13": 8.0613, + "14": 8.25238, + "15": 8.2457, + "16": 8.31701, + "17": 8.33964, + "18": 7.58044, + "19": 8.18777, + "20": 7.71116, + "21": 7.99992, + "22": 7.3454, + "23": 7.94728, + "24": 7.50571, + "25": 8.32022, + "26": 7.78539, + "27": 7.72311, + "28": 7.70481, + "29": 7.7694, + "30": 7.56679, + "31": 7.84739, + "32": 6.52324, + "33": 7.24018, + "34": 7.796, + "35": 7.73916, + "36": 6.73059, + "37": 8.15243, + "38": 7.62149, + "39": 7.97743, + "40": 7.51901, + "41": 7.51939, + "42": 6.1129, + "43": 7.60283, + "44": 7.96375, + "45": 6.84546, + "46": 7.42487, + "47": 7.82289, + "48": 7.87871, + "49": 7.59657, + "50": 6.84804 } }, "num-zeros": { @@ -65,52 +65,52 @@ "2": 46900688.0, "3": 71568288.0, "4": 234203216.0, - "5": 474843232.0, - "6": 558036544.0, + "5": 474837472.0, + "6": 561190528.0, "7": 964556864.0, - "8": 717673920.0, - "9": 815179776.0, - "10": 712034880.0, - "11": 638136896.0, - "12": 559512384.0, - "13": 611146240.0, - "14": 757545728.0, - "15": 769675776.0, - "16": 726164544.0, - "17": 661188736.0, - "18": 730133248.0, - "19": 735484160.0, - "20": 849803264.0, - "21": 814697920.0, - "22": 629166720.0, - "23": 793119360.0, - "24": 597724352.0, - "25": 846342144.0, - "26": 814671808.0, - "27": 628761728.0, - "28": 783493760.0, - "29": 786359488.0, - "30": 764702208.0, - "31": 600060544.0, - "32": 778251840.0, - "33": 764634688.0, - "34": 745651456.0, - "35": 741503424.0, - "36": 705295296.0, - "37": 731045120.0, - "38": 718271552.0, - "39": 736121472.0, - "40": 716764544.0, - "41": 528388096.0, - "42": 680529024.0, - "43": 682338048.0, - "44": 648832640.0, - "45": 619466368.0, - "46": 625945344.0, - "47": 623566592.0, - "48": 613425664.0, - "49": 600379264.0, - "50": 569557440.0 + "8": 720809472.0, + "9": 815181312.0, + "10": 708885056.0, + "11": 635007360.0, + "12": 553207296.0, + "13": 683492160.0, + "14": 741814912.0, + "15": 782267840.0, + "16": 723021312.0, + "17": 651756800.0, + "18": 742705920.0, + "19": 741724544.0, + "20": 868679872.0, + "21": 861892736.0, + "22": 651191232.0, + "23": 767955264.0, + "24": 594551424.0, + "25": 824319360.0, + "26": 783214656.0, + "27": 625579968.0, + "28": 811807360.0, + "29": 795796608.0, + "30": 777285056.0, + "31": 785653312.0, + "32": 778247296.0, + "33": 745758336.0, + "34": 742508544.0, + "35": 725768768.0, + "36": 692716544.0, + "37": 696440960.0, + "38": 699391552.0, + "39": 710961152.0, + "40": 704181952.0, + "41": 534684160.0, + "42": 655362624.0, + "43": 666612480.0, + "44": 639391360.0, + "45": 610026240.0, + "46": 613361920.0, + "47": 604690432.0, + "48": 591405696.0, + "49": 562632448.0, + "50": 544389888.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4305059328.0, - "2": 4305060864.0, - "3": 4305060864.0, - "4": 4305060864.0, - "5": 4305060864.0, - "6": 4305060864.0, - "7": 4305060864.0, - "8": 4305060864.0, - "9": 4305060864.0, - "10": 4305060864.0, - "11": 4305060864.0, - "12": 4305060864.0, - "13": 4305060864.0, - "14": 4305060864.0, - "15": 4305060864.0, - "16": 4305060864.0, - "17": 4305060864.0, - "18": 4305060864.0, - "19": 4305060864.0, - "20": 4305060864.0, - "21": 4305060864.0, - "22": 4305060864.0, - "23": 4305060864.0, - "24": 4305060864.0, - "25": 4305060864.0, - "26": 4305060864.0, - "27": 4305060864.0, - "28": 4305060864.0, - "29": 4305060864.0, - "30": 4305060864.0, - "31": 4305060864.0, - "32": 4305060864.0, - "33": 4305060864.0, - "34": 4305060864.0, - "35": 4305060864.0, - "36": 4305060864.0, - "37": 4305060864.0, - "38": 4305060864.0, - "39": 4305060864.0, - "40": 4305060864.0, - "41": 4305060864.0, - "42": 4305060864.0, - "43": 4305060864.0, - "44": 4305060864.0, - "45": 4305060864.0, - "46": 4305060864.0, - "47": 4305060864.0, - "48": 4305060864.0, - "49": 4305060864.0, - "50": 4305060864.0 + "1": 4313315840.0, + "2": 4313317376.0, + "3": 4313317376.0, + "4": 4313317376.0, + "5": 4313317376.0, + "6": 4313317376.0, + "7": 4313317376.0, + "8": 4313317376.0, + "9": 4313317376.0, + "10": 4313317376.0, + "11": 4313317376.0, + "12": 4313317376.0, + "13": 4313317376.0, + "14": 4313317376.0, + "15": 4313317376.0, + "16": 4313317376.0, + "17": 4313317376.0, + "18": 4313317376.0, + "19": 4313317376.0, + "20": 4313317376.0, + "21": 4313317376.0, + "22": 4313317376.0, + "23": 4313317376.0, + "24": 4313317376.0, + "25": 4313317376.0, + "26": 4313317376.0, + "27": 4313317376.0, + "28": 4313317376.0, + "29": 4313317376.0, + "30": 4313317376.0, + "31": 4313317376.0, + "32": 4313317376.0, + "33": 4313317376.0, + "34": 4313317376.0, + "35": 4313317376.0, + "36": 4313317376.0, + "37": 4313317376.0, + "38": 4313317376.0, + "39": 4313317376.0, + "40": 4313317376.0, + "41": 4313317376.0, + "42": 4313317376.0, + "43": 4313317376.0, + "44": 4313317376.0, + "45": 4313317376.0, + "46": 4313317376.0, + "47": 4313317376.0, + "48": 4313317376.0, + "49": 4313317376.0, + "50": 4313317376.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4305061888.0, - "2": 5850930176.0, - "3": 5850930176.0, - "4": 5857143296.0, - "5": 5857143296.0, - "6": 5857143296.0, - "7": 5857143296.0, - "8": 5857143296.0, - "9": 5857143296.0, - "10": 5857143296.0, - "11": 5857143296.0, - "12": 5857143296.0, - "13": 5857143296.0, - "14": 5857143296.0, - "15": 5857143296.0, - "16": 5857143296.0, - "17": 5857143296.0, - "18": 5857143296.0, - "19": 5857143296.0, - "20": 5857143296.0, - "21": 5857143296.0, - "22": 5857143296.0, - "23": 5857143296.0, - "24": 5857143296.0, - "25": 5857143296.0, - "26": 5857143296.0, - "27": 5857143296.0, - "28": 5857143296.0, - "29": 5857143296.0, - "30": 5857143296.0, - "31": 5857143296.0, - "32": 5857143296.0, - "33": 5857143296.0, - "34": 5857143296.0, - "35": 5857143296.0, - "36": 5857143296.0, - "37": 5857143296.0, - "38": 5857143296.0, - "39": 5857143296.0, - "40": 5857143296.0, - "41": 5857143296.0, - "42": 5857143296.0, - "43": 5857143296.0, - "44": 5857143296.0, - "45": 5857143296.0, - "46": 5857143296.0, - "47": 5857143296.0, - "48": 5857143296.0, - "49": 5857143296.0, - "50": 5857143296.0 + "1": 4763568128.0, + "2": 7505469952.0, + "3": 7505469952.0, + "4": 7505469952.0, + "5": 7505469952.0, + "6": 7505469952.0, + "7": 7505469952.0, + "8": 7505469952.0, + "9": 7505469952.0, + "10": 7505469952.0, + "11": 7505469952.0, + "12": 7505469952.0, + "13": 7505469952.0, + "14": 7505469952.0, + "15": 7505469952.0, + "16": 7505469952.0, + "17": 7505469952.0, + "18": 7505469952.0, + "19": 7505469952.0, + "20": 7505469952.0, + "21": 7505469952.0, + "22": 7505469952.0, + "23": 7505469952.0, + "24": 7505469952.0, + "25": 7505469952.0, + "26": 7505469952.0, + "27": 7505469952.0, + "28": 7505469952.0, + "29": 7505469952.0, + "30": 7505469952.0, + "31": 7505469952.0, + "32": 7505469952.0, + "33": 7505469952.0, + "34": 7505469952.0, + "35": 7505469952.0, + "36": 7505469952.0, + "37": 7505469952.0, + "38": 7505469952.0, + "39": 7505469952.0, + "40": 7533150720.0, + "41": 7533150720.0, + "42": 7540405760.0, + "43": 7540405760.0, + "44": 7540405760.0, + "45": 7540405760.0, + "46": 7540405760.0, + "47": 7540405760.0, + "48": 7540405760.0, + "49": 7543386624.0, + "50": 7557736448.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 72.89456, - "2": 2.94117, - "3": 2.84855, - "4": 2.63467, - "5": 2.44999, - "6": 2.13944, - "7": 2.29092, - "8": 2.26625, - "9": 2.12101, - "10": 2.09852, - "11": 2.08307, - "12": 2.17401, - "13": 2.22063, - "14": 2.09013, - "15": 2.08464, - "16": 2.10017, - "17": 2.16623, - "18": 2.09249, - "19": 2.0884, - "20": 2.221, - "21": 2.09883, - "22": 2.11134, - "23": 2.10284, - "24": 2.09769, - "25": 2.09301, - "26": 2.11065, - "27": 2.10477, - "28": 2.10258, - "29": 2.09663, - "30": 2.13449, - "31": 2.08704, - "32": 2.09973, - "33": 2.22347, - "34": 2.09872, - "35": 2.10521, - "36": 2.13037, - "37": 2.09024, - "38": 2.11762, - "39": 2.09382, - "40": 2.09703, - "41": 2.08655, - "42": 2.1117, - "43": 2.12736, - "44": 2.09903, - "45": 2.09069, - "46": 2.0915, - "47": 2.14487, - "48": 2.09334, - "49": 2.10013, - "50": 2.12076 + "1": 73.13358, + "2": 2.30272, + "3": 2.54126, + "4": 2.42795, + "5": 2.21903, + "6": 1.98058, + "7": 2.03904, + "8": 1.9768, + "9": 2.00213, + "10": 1.97759, + "11": 1.9686, + "12": 2.04098, + "13": 2.0766, + "14": 1.97492, + "15": 2.00924, + "16": 1.97589, + "17": 2.02471, + "18": 1.9905, + "19": 2.03777, + "20": 1.97139, + "21": 2.09691, + "22": 2.00009, + "23": 1.98721, + "24": 1.98177, + "25": 1.97569, + "26": 2.02083, + "27": 1.99751, + "28": 2.05914, + "29": 1.97108, + "30": 1.98538, + "31": 1.97162, + "32": 1.98459, + "33": 1.99037, + "34": 2.01573, + "35": 2.05758, + "36": 1.98037, + "37": 2.00516, + "38": 1.9873, + "39": 1.97752, + "40": 2.0009, + "41": 1.98042, + "42": 1.9972, + "43": 1.98166, + "44": 1.9771, + "45": 1.99252, + "46": 1.97768, + "47": 1.98175, + "48": 1.99748, + "49": 1.98307, + "50": 1.99372 } } } \ No newline at end of file diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 3a9bc4ddf35..ee5f46bf706 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -132,10 +132,6 @@ def _run_one_iter_and_capture( @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for offloading tests.") -@pytest.mark.skipif( - not is_te_min_version("1.13.0"), - reason="Fine-grained activation offloading requires TE-based GPT layer spec (TE 1.13+ in this repo's tests).", -) @pytest.mark.parametrize( "is_moe, is_mla, offload_modules", [ @@ -310,8 +306,8 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for offloading tests.") @pytest.mark.skipif( - not is_te_min_version("1.13.0"), - reason="Requires TE-based GPT layer spec (TE 1.13+ in this repo's tests).", + not is_te_min_version("1.9.0.dev0"), + reason="EP A2A overlap requires TE 1.9.0.dev0+ in this repo's tests.", ) @pytest.mark.parametrize( "dispatcher_backend, is_mla, offload_modules", From 9555702a67c7fe21c6af5d399d75467239bb74d2 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Fri, 9 Jan 2026 04:07:04 -0800 Subject: [PATCH 36/47] fix ut Signed-off-by: Hongbin Liu --- .../test_fine_grained_activation_offloading.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index ee5f46bf706..88d76bf5d80 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -144,8 +144,6 @@ def _run_one_iter_and_capture( (True, False, ["mlp_norm"]), (True, False, ["expert_fc1"]), (True, False, ["moe_act"]), - (True, True, ["attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), - (True, False, ["core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"]), ], ) def test_gpt_fine_grained_activation_offloading_correctness_and_memory( From fd2d3fd7b12fe569553c7a27d1885abeb1ad90b8 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Sun, 11 Jan 2026 19:18:25 -0800 Subject: [PATCH 37/47] add version check Signed-off-by: Hongbin Liu --- megatron/training/arguments.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index f1762690bdb..11f5bd4f77f 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1253,6 +1253,9 @@ def validate_args(args, defaults={}): if args.fine_grained_activation_offloading: assert args.transformer_impl == 'transformer_engine', \ "Fine-grained activation offloading is only supported with transformer_engine implementation" + if is_te_min_version("2.10.0"): + assert os.getenv("NVTE_CPU_OFFLOAD_V1", "0") == "1", \ + "For fine-grained activation offloading with TE >= 2.10.0, NVTE_CPU_OFFLOAD_V1 should be set to 1 to avoid offloading weights." if args.mtp_num_layers: assert not args.use_legacy_models, "The legacy Megatron models does not support Multi-Token Prediction (MTP)." From 84d13e9136be6c32d7b4145ee32b257775f180d8 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 12 Jan 2026 01:15:16 -0800 Subject: [PATCH 38/47] minor refactor for fine_grained_activation_offload.py Signed-off-by: Hongbin Liu --- .../common/model_chunk_schedule_plan.py | 2 +- .../fine_grained_activation_offload.py | 94 +++++++++---------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py index 1ac76fc4cd9..9b2eb9cdde2 100644 --- a/megatron/core/models/common/model_chunk_schedule_plan.py +++ b/megatron/core/models/common/model_chunk_schedule_plan.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. from contextlib import nullcontext from typing import Optional diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index bbb1f7e32e4..a8cf2207e89 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -341,6 +341,9 @@ def __init__(self, name): self.offload = True self.total_offload_bytes = 0 self.total_tensor_count = 0 + # Using memory pool is for the compatibility with cuda graph. + # Shapes of tensors for expert_fc1 and moe_act are not known in advance, + # so we do not use CPU pool for them. if name == "expert_fc1" or name == "moe_act": self.use_cpu_pool = False else: @@ -410,16 +413,23 @@ def __init__(self): # Shared CPU tensor pool for all chunks to improve reuse efficiency self._cpu_tensor_pool = GPUTensorPool(device="cpu", pin_memory=True) + # Whether the manager is in warmup phase. self._is_warmup = True + # Cache OffloadChunkHandler objects for each virtual pipeline stage and each forward pass. self._cached_chunks_forward = [] + # Cache OffloadChunkHandler objects for each virtual pipeline stage and each backward pass. self._cached_chunks_backward = [] + # Index of the current backward chunk in the cached chunks backward. self._cached_chunks_index_backward = 0 + # Index of the current forward chunk in the cached chunks forward. self._cached_chunks_index_forward = 0 self.do_offload = True - # Margin to avoid offloading too many groups so that + # Do not offload the last X groups so that the reloading won't block the computing stream. self._offload_margin = 0 + # Sometimes we need to delay the offloading and launch it later. + # The delayed offload groups are stored in a queue. self._delayed_offload_groups = [] self.reset() @@ -446,6 +456,7 @@ def push_offload_groups(self, group_hook, forced_released_tensors): def flush_delayed_groups(self): """Flush the delayed groups.""" debug_rank("flushing delayed groups") + # Flush the delayed groups in reverse order to maintain the order of the groups. for group_hook, forced_released_tensors in reversed(self._delayed_offload_groups): group_hook(forced_released_tensors) self._delayed_offload_groups = [] @@ -459,6 +470,7 @@ def reset(self): if hasattr(self, '_cpu_tensor_pool'): self._cpu_tensor_pool.reset() + # Call post_warmup_callback after warmup to collect the offload information. if self._is_warmup and len(self._cached_chunks_forward) > 0: self.post_warmup_callback() self._cached_chunks_index_backward = 0 @@ -523,7 +535,7 @@ def post_warmup_callback(self): # Update the offload margin to the maximum number of deduplicated groups self._offload_margin = max(self._offload_margin, chunk.get_max_deduplicated_groups()) debug_rank(f"offload margin {self._offload_margin}") - # Fine the last group with the same name in the cached chunks backward + # Find the last group with the same name in the cached chunks backward last_group_with_same_name = {} for chunk_idx, chunk in enumerate(reversed(self._cached_chunks_backward)): for group in chunk.offload_groups: @@ -567,8 +579,8 @@ def push(self, handler): if self._is_warmup: self._cached_chunks_backward.append(handler) - def pop(self, name=None): - """Remove and set the next non-empty chunk as the current backward chunk.""" + def pop_backward_chunk(self, name=None): + """Get the next non-empty backward chunk containing the group with the given name.""" self._cur_backward_chunk = None debug_rank(f"popping backward chunk {self._cached_chunks_index_backward}") debug_rank(f"cached chunks backward {self._cached_chunks_backward}") @@ -584,8 +596,8 @@ def pop(self, name=None): break assert self._cur_backward_chunk is not None, "No non-empty chunk found" - def front(self, name=None): - """Get the first non-empty chunk handler without removing it from the queue.""" + def front_backward_chunk(self, name=None): + """Get the first non-empty backward chunk containing the group with the given name.""" for idx, handler in enumerate( self._cached_chunks_backward[self._cached_chunks_index_backward :] ): @@ -744,17 +756,19 @@ def reload(self, state, non_blocking=None): def __init__(self, min_offloaded_tensor_size, cpu_tensor_pool): self.do_offload = True - # Data Structure to maintain reference to activation tensors - self._tensor_tag_to_state = {} - # Mark the first microbatch of the last virtual pipeline stage - # self._is_first_last_vpp_chunk = is_first_last_vpp_chunk # Group management for batching offload/reload operations + self.offload_groups = [] self._offloaded_group_index = 0 + # Groups to be offloaded. self._groups_to_offload = [] + # Groups to be reloaded. self._groups_to_reload = [] + # Tensor count for the current group. self._tensor_count_current_group = 0 + # Maximum number of groups to offload or reload. self._max_group_size = 0 + # Groups being reloaded. self._reloading_group = [] # Counter for special torch tensor types (FakeTensor, FunctionalTensor) self.torch_tensor_count = 0 @@ -762,7 +776,6 @@ def __init__(self, min_offloaded_tensor_size, cpu_tensor_pool): self.h2d_stream = PipelineOffloadManager.get_instance().h2d_stream self.min_offloaded_tensor_size = min_offloaded_tensor_size self.cpu_tensor_pool = cpu_tensor_pool - self.offload_groups = [] self.is_warmup = True def reset(self): @@ -773,15 +786,18 @@ def reset(self): self._tensor_count_current_group = 0 self._reloading_group = [] + def find_group_with_name(self, name: str, start_index: int = 0): + """Find the group with the given name starting from the given index.""" + return next( + (group for group in self.offload_groups[start_index:] if group._name == name), + None, + ) + def is_empty_chunk(self, name=None): """Check if this chunk has no tensors to manage.""" debug_rank(f"------is_empty_chunk {self._max_group_size}") if name is not None: - for group in self.offload_groups: - debug_rank(f"group name {group._name} need name {name}") - if group._name == name: - return False - return True + return self.find_group_with_name(name) is None return self._max_group_size == 0 def finish_all_groups(self, name=None) -> bool: @@ -798,18 +814,12 @@ def finish_all_groups(self, name=None) -> bool: ): return True assert name is not None, "Name is required" - for group in self.offload_groups[self._offloaded_group_index :]: - if group._name == name: - return False - return True + return self.find_group_with_name(name, self._offloaded_group_index) is None def find_next_group(self, name=None): """Find the next group with the given name.""" assert name is not None, "Name is required" - for group in self.offload_groups[self._offloaded_group_index :]: - if group._name == name: - return group - return None + return self.find_group_with_name(name, self._offloaded_group_index) def tensor_push(self, tensor): """Push tensor to the offload handler.""" @@ -822,30 +832,19 @@ def tensor_push(self, tensor): ) assert not torch_stray_tensor, "Stray tensor should not be offloaded" - if not torch_stray_tensor: - # Assign unique tag based on group index and position within group - tensor_tag = (self._offloaded_group_index, self._tensor_count_current_group) - self._tensor_count_current_group += 1 - # assert tensor_tag not in self._tensor_tag_to_state, "Duplicate tensor tag" - # self._tensor_tag_to_state[tensor_tag] = tensor - self.offload_groups[self._offloaded_group_index - 1].push_tensor(tensor_tag, tensor) - else: - # Use negative group ID for special tensor types - tensor_tag = (-1, self.torch_tensor_count) - self.torch_tensor_count += 1 - # self._tensor_tag_to_state[tensor_tag] = tensor + # Assign unique tag based on group index and position within group + tensor_tag = (self._offloaded_group_index, self._tensor_count_current_group) + self._tensor_count_current_group += 1 + self.offload_groups[self._offloaded_group_index - 1].push_tensor(tensor_tag, tensor) debug_rank(f"--------tensor_push {tensor_tag}") return tensor_tag def tensor_pop(self, tensor_tag): """Pop tensor from the offload handler.""" debug_rank(f"--------tensor_pop {tensor_tag}") - # assert tensor_tag in self._tensor_tag_to_state, f"Tag {tensor_tag} not found" - # tensor = self._tensor_tag_to_state.pop(tensor_tag) group_id, idx = tensor_tag tensor = self.offload_groups[group_id - 1].pop_tensor(tensor_tag) # If tensor is offloaded (stored as tuple), reload it - # assert isinstance(tensor, torch.Tensor), "Tensor is not a tensor" if isinstance(tensor, tuple): tensor = self.reload(tensor) debug_rank(f"--------tensor_pop {tensor.shape}") @@ -869,7 +868,6 @@ def bulk_offload_group(self): group_to_offload = self._groups_to_offload[-1] torch.cuda.nvtx.range_push("activation offloading " + group_to_offload._name) with torch.cuda.stream(self.d2h_stream): - # for tensor_tag, state in self._tensor_tag_to_state.items(): for tensor_tag, tensor_on_device in group_to_offload._tensors.items(): if self.tensor_need_offloading_checker(tensor_on_device): state = self.offload( @@ -908,6 +906,7 @@ def bulk_reload_group(self): group_to_reload.push_tensor(tensor_tag, recovered_tensor) group_to_reload.record_reload_event(self.h2d_stream) self._groups_to_reload.pop() + # Add the group to the reloading group to wait for the reload event. self._reloading_group.append(group_to_reload) torch.cuda.nvtx.range_pop() @@ -921,6 +920,7 @@ def pre_reload_last_layer(self): def should_bulk_offload(self): """Determine if the current group should be offloaded.""" + assert len(self._groups_to_offload) > 0, "No groups to offload" group = self._groups_to_offload[-1] debug_rank(f"should_bulk_offload {self.is_warmup} {group.offload}") # Don't offload if the chunk is not in warmup stage @@ -931,7 +931,8 @@ def should_bulk_offload(self): return False # Check if next backward chunk is this chunk (for last pipeline stage) - next_backward_chunk = PipelineOffloadManager.get_instance().front(name=group._name) + next_backward_chunk = \ + PipelineOffloadManager.get_instance().front_backward_chunk(group._name) if next_backward_chunk is not None and next_backward_chunk is self: # Don't offload the last group with the same name if it's about to be used immediately if self.find_next_group(group._name) is None: @@ -972,7 +973,8 @@ def bulk_reload(self): self.bulk_reload_group() else: # Pre-load the last layer of the next backward chunk to hide latency - next_backward_chunk = PipelineOffloadManager.get_instance().front() + next_backward_chunk = PipelineOffloadManager.get_instance().front_backward_chunk() + # Don't pre-reload the last layer if the next backward chunk hasn't finished fprop yet. if ( next_backward_chunk is not None and next_backward_chunk._offloaded_group_index @@ -991,7 +993,7 @@ def on_group_commit_backward(self, name): cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() # Switch to this chunk if it's not already current if cur_backward_chunk is not self: - PipelineOffloadManager.get_instance().pop(name) + PipelineOffloadManager.get_instance().pop_backward_chunk(name) cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() assert cur_backward_chunk is self, f"Chunk mismatch {cur_backward_chunk} {self}" # Wait for reload to complete before using tensors @@ -1010,17 +1012,13 @@ def on_group_start_forward(self, name): if not self.do_offload: return debug_rank(f"--on_group_start_forward {name}") + self._offloaded_group_index = self._offloaded_group_index + 1 if self.is_warmup: - self._offloaded_group_index = self._offloaded_group_index + 1 self.offload_groups.append(OffloadTensorGroup(name)) self._max_group_size = max(self._max_group_size, self._offloaded_group_index) debug_rank(f"max group size {self._max_group_size}") else: - self._offloaded_group_index = self._offloaded_group_index + 1 for group in self.offload_groups[self._offloaded_group_index - 1 :]: - debug_rank( - f"offloaded group index {self._offloaded_group_index} for group {group._name}" - ) if group._name == name: break self._offloaded_group_index = self._offloaded_group_index + 1 From 483d87acf5f88a0ff0edafdcca08f10afe52daf3 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Mon, 12 Jan 2026 01:20:25 -0800 Subject: [PATCH 39/47] format Signed-off-by: Hongbin Liu --- .../pipeline_parallel/fine_grained_activation_offload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index a8cf2207e89..fa23dd2417f 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -789,8 +789,7 @@ def reset(self): def find_group_with_name(self, name: str, start_index: int = 0): """Find the group with the given name starting from the given index.""" return next( - (group for group in self.offload_groups[start_index:] if group._name == name), - None, + (group for group in self.offload_groups[start_index:] if group._name == name), None ) def is_empty_chunk(self, name=None): @@ -931,8 +930,9 @@ def should_bulk_offload(self): return False # Check if next backward chunk is this chunk (for last pipeline stage) - next_backward_chunk = \ - PipelineOffloadManager.get_instance().front_backward_chunk(group._name) + next_backward_chunk = PipelineOffloadManager.get_instance().front_backward_chunk( + group._name + ) if next_backward_chunk is not None and next_backward_chunk is self: # Don't offload the last group with the same name if it's about to be used immediately if self.find_next_group(group._name) is None: From 2f42e9168f22de1d9eeee0b81ddbe3a72d7f7c89 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 13 Jan 2026 17:58:39 -0800 Subject: [PATCH 40/47] fix doc Signed-off-by: Hongbin Liu --- .../offloading_and_recomputing.png | Bin .../features/fine_grained_activation_offloading.md | 2 +- megatron/core/transformer/moe/README.md | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename docs/{source => }/images/fine_grained_activation_offloading/offloading_and_recomputing.png (100%) diff --git a/docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png b/docs/images/fine_grained_activation_offloading/offloading_and_recomputing.png similarity index 100% rename from docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png rename to docs/images/fine_grained_activation_offloading/offloading_and_recomputing.png diff --git a/docs/user-guide/features/fine_grained_activation_offloading.md b/docs/user-guide/features/fine_grained_activation_offloading.md index 969098263fc..53211d1d06c 100644 --- a/docs/user-guide/features/fine_grained_activation_offloading.md +++ b/docs/user-guide/features/fine_grained_activation_offloading.md @@ -28,4 +28,4 @@ Currently, the supported offloading modules are `"attn_norm", "core_attn", "attn - For other modules, use offloading to reduce memory footprint; - Make sure the offloading/reloading could be overlapped with computing; -![Fine-grained Activation Offloading and Fine-grained Recomputation](../images/fine_grained_activation_offloading/offloading_and_recomputing.png) +![Fine-grained Activation Offloading and Fine-grained Recomputation](../../images/fine_grained_activation_offloading/offloading_and_recomputing.png) diff --git a/megatron/core/transformer/moe/README.md b/megatron/core/transformer/moe/README.md index f46cc5ef838..5dd5da649d0 100644 --- a/megatron/core/transformer/moe/README.md +++ b/megatron/core/transformer/moe/README.md @@ -222,7 +222,7 @@ Offload the input activation at the granularity of modules # Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". --offload-modules expert_fc1 ``` -For more details, please refer to the ```docs/source/api-guide/fine_grained_activation_offloading.md``` +For more details, please refer to the ```docs/user-guide/features/fine_grained_activation_offloading.md``` ### MoE Related Arguments | Item | Description | From 469cef09ee1ee92690d8c614cdec439a475ac28d Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Tue, 13 Jan 2026 18:47:12 -0800 Subject: [PATCH 41/47] code refactor Signed-off-by: Hongbin Liu --- .../core/models/gpt/fine_grained_callables.py | 8 ++- megatron/core/models/gpt/gpt_model.py | 10 +-- .../fine_grained_activation_offload.py | 64 +++++++++++++------ megatron/core/pipeline_parallel/schedules.py | 8 +-- megatron/core/transformer/attention.py | 10 +-- megatron/core/transformer/moe/experts.py | 10 +-- .../transformer/multi_latent_attention.py | 10 +-- .../core/transformer/transformer_layer.py | 16 +++-- ...test_fine_grained_activation_offloading.py | 18 +++--- 9 files changed, 96 insertions(+), 58 deletions(-) diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index 5288e6df8f4..6d32ff0027c 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -8,10 +8,12 @@ import torch from megatron.core import tensor_parallel +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + FineGrainedActivationOffloadingInterface as off_interface, +) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, fine_grained_offloading_group_start, - get_fine_grained_offloading_context, ) from megatron.core.pipeline_parallel.utils import ScheduleNode, make_viewless from megatron.core.transformer.module import float16_to_fp32 @@ -382,12 +384,12 @@ def submodule_post_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor) hidden_states = fine_grained_offloading_group_start(hidden_states, name="mlp_norm") if layer.recompute_pre_mlp_layernorm: layer.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with get_fine_grained_offloading_context(layer.offload_mlp_norm): + with off_interface.get_context(layer.offload_mlp_norm): pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint( layer.pre_mlp_layernorm, hidden_states ) else: - with get_fine_grained_offloading_context(layer.offload_mlp_norm): + with off_interface.get_context(layer.offload_mlp_norm): pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states) probs, routing_map = layer.mlp.route(pre_mlp_layernorm_output) diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index 58a6b48acc1..be0816d2e1d 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -19,7 +19,7 @@ from megatron.core.models.common.language_module.language_module import LanguageModule from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_init_chunk_handler, + FineGrainedActivationOffloadingInterface as off_interface, ) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.quantization.utils import get_quant_config_or_none @@ -427,20 +427,20 @@ def _preprocess( def preprocess_for_fine_grained_offloading(self): """Preprocess for fine-grained activation offloading.""" - fine_grained_offloading_init_chunk_handler( + off_interface.init_chunk_handler( vp_size=self.config.virtual_pipeline_model_parallel_size, vp_stage=self.vp_stage, min_offloaded_tensor_size=self.config.min_offloaded_tensor_size, ) if self.disable_param_offloading: for param in self.decoder.parameters(): - param.offloading_activation = False + off_interface.mark_not_offloadable(param) if self.mtp_process: for param in self.mtp.parameters(): - param.offloading_activation = False + off_interface.mark_not_offloadable(param) if self.post_process: for param in self.output_layer.parameters(): - param.offloading_activation = False + off_interface.mark_not_offloadable(param) self.disable_param_offloading = False def forward( diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index fa23dd2417f..4e6c8219ee2 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -667,6 +667,11 @@ def cur_backward_chunk(self): """Get the current backward pass chunk handler.""" return self._cur_backward_chunk + def mark_not_offloadable(self, tensor: torch.Tensor): + """Mark the current forward chunk as not offloadable.""" + if tensor is not None: + tensor.offloading_activation = False + def __enter__(self): """Enter context manager to enable activation offloading hooks.""" debug_rank("----__enter__") @@ -1164,23 +1169,6 @@ def fine_grained_offloading_group_start(tensor, name=None): return FineGrainedOffloadingGroupStartFunction.apply(tensor, cur_forward_chunk, name) -def get_fine_grained_offloading_context(flag): - """Get the fine-grained offload context""" - return PipelineOffloadManager.get_instance() if flag else nullcontext() - - -def fine_grained_offloading_init_chunk_handler(vp_size, vp_stage, min_offloaded_tensor_size): - """Initialize the chunk handler, called at the start of a microbatch forward pass.""" - PipelineOffloadManager.get_instance().init_model_chunk_offload_handler( - vp_size, vp_stage, min_offloaded_tensor_size - ) - - -def fine_grained_offloading_reset(): - """Reset the chunk handler, called at the start of a training iteration.""" - PipelineOffloadManager.get_instance().reset() - - def fine_grained_offloading_forward_record(event: torch.cuda.Event) -> None: """Record the forward event for cuda graph capture.""" d2h_stream = PipelineOffloadManager.get_instance().d2h_stream @@ -1214,6 +1202,42 @@ def fine_grained_offloading_backward_record(tensor, event: torch.cuda.Event) -> return FineGrainedOffloadingBackwardRecordFunction.apply(tensor, event) -def fine_grained_offloading_reset_instance(): - """Reset the singleton instance of PipelineOffloadManager.""" - PipelineOffloadManager.reset_instance() +class FineGrainedActivationOffloadingInterface: + """Interface for fine-grained activation offloading.""" + + def __init__(self): + pass + + @staticmethod + def init_chunk_handler(vp_size, vp_stage, min_offloaded_tensor_size): + """Initialize the chunk handler, called at the start of a microbatch forward pass.""" + PipelineOffloadManager.get_instance().init_model_chunk_offload_handler( + vp_size, vp_stage, min_offloaded_tensor_size + ) + + @staticmethod + def get_context(flag): + """Get the fine-grained offload context""" + return PipelineOffloadManager.get_instance() if flag else nullcontext() + + @staticmethod + def mark_not_offloadable(tensor: torch.Tensor): + """Mark the tensor as not offloadable.""" + PipelineOffloadManager.get_instance().mark_not_offloadable(tensor) + + @staticmethod + def forward_record(event: torch.cuda.Event) -> None: + """Record the forward event for cuda graph capture.""" + d2h_stream = PipelineOffloadManager.get_instance().d2h_stream + torch.cuda.current_stream().record_event(event) + torch.cuda.current_stream().wait_stream(d2h_stream) + + @staticmethod + def reset(): + """Reset the chunk handler.""" + PipelineOffloadManager.get_instance().reset() + + @staticmethod + def reset_instance(): + """Reset the singleton instance.""" + PipelineOffloadManager.reset_instance() diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index e4dbc2b3401..c447d2a596b 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -10,7 +10,7 @@ from megatron.core import parallel_state from megatron.core.enums import ModelType from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_reset, + FineGrainedActivationOffloadingInterface as off_interface, ) from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator from megatron.core.pipeline_parallel.utils import ( @@ -649,7 +649,7 @@ def forward_backward_no_pipelining( ) if not forward_only and config.fine_grained_activation_offloading: - fine_grained_offloading_reset() + off_interface.reset() if config.timers is not None: config.timers('forward-backward').stop() @@ -1912,7 +1912,7 @@ def pp_post_backward(input_tensor_grad, vp_stage=None): ) if not forward_only and config.fine_grained_activation_offloading: - fine_grained_offloading_reset() + off_interface.reset() # Restore config.grad_sync_func and config.param_sync_func. if forward_only: config.grad_sync_func, config.param_sync_func = grad_sync_func, param_sync_func @@ -2302,7 +2302,7 @@ def enable_grad_sync(): ) if not forward_only and config.fine_grained_activation_offloading: - fine_grained_offloading_reset() + off_interface.reset() if config.timers is not None: config.timers('forward-backward').stop() diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 31c0afb91a4..ec55cad093e 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -23,10 +23,12 @@ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + FineGrainedActivationOffloadingInterface as off_interface, +) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, fine_grained_offloading_group_start, - get_fine_grained_offloading_context, ) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.mappings import all_gather_last_dim_from_tensor_parallel_region @@ -828,7 +830,7 @@ def forward( if self.offload_qkv_linear: hidden_states = fine_grained_offloading_group_start(hidden_states, name="qkv_linear") - with get_fine_grained_offloading_context(self.offload_qkv_linear): + with off_interface.get_context(self.offload_qkv_linear): qkv_output = self.get_query_key_value_tensors( hidden_states, key_value_states, @@ -997,7 +999,7 @@ def forward( query = fine_grained_offloading_group_start(query, name="core_attn") if inference_context is None or inference_context.is_static_batching(): # Static batching attention kernel. - with get_fine_grained_offloading_context(self.offload_core_attention): + with off_interface.get_context(self.offload_core_attention): core_attn_out = self.core_attention( query, key, @@ -1056,7 +1058,7 @@ def forward( nvtx_range_push(suffix="linear_proj") if self.offload_attn_proj: core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") - with get_fine_grained_offloading_context(self.offload_attn_proj): + with off_interface.get_context(self.offload_attn_proj): output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: output = fine_grained_offloading_group_commit( diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index c9f8842dda0..5f7de2e7141 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -25,10 +25,12 @@ from megatron.core.fusions.fused_bias_swiglu import weighted_bias_swiglu_impl from megatron.core.fusions.fused_weighted_squared_relu import weighted_squared_relu_impl from megatron.core.jit import jit_fuser +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + FineGrainedActivationOffloadingInterface as off_interface, +) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, fine_grained_offloading_group_start, - get_fine_grained_offloading_context, ) from megatron.core.tensor_parallel.layers import ( _initialize_affine_weight_cpu, @@ -716,7 +718,7 @@ def forward( permuted_local_hidden_states = fine_grained_offloading_group_start( permuted_local_hidden_states, name="expert_fc1" ) - with get_fine_grained_offloading_context(self.offload_expert_fc1): + with off_interface.get_context(self.offload_expert_fc1): fc1_output, bias_parallel = self.linear_fc1( permuted_local_hidden_states, tokens_per_expert ) @@ -790,12 +792,12 @@ def glu(x): if self.activation_recompute: self.activation_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with get_fine_grained_offloading_context(self.offload_moe_act): + with off_interface.get_context(self.offload_moe_act): bias_act_output = self.activation_checkpoint.checkpoint( bias_act_func, fc1_output, bias_parallel, permuted_probs ) else: - with get_fine_grained_offloading_context(self.offload_moe_act): + with off_interface.get_context(self.offload_moe_act): bias_act_output = bias_act_func(fc1_output, bias_parallel, permuted_probs) output, output_bias = self.linear_fc2(bias_act_output, tokens_per_expert) diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index 384eaf77ef6..2a5c75b95bb 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -22,10 +22,12 @@ _yarn_get_mscale, apply_rotary_pos_emb, ) +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + FineGrainedActivationOffloadingInterface as off_interface, +) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, fine_grained_offloading_group_start, - get_fine_grained_offloading_context, ) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.layers import ColumnParallelLinear @@ -246,7 +248,7 @@ def forward( # query: [96, 1, 16, 128], key:[96, 1, 16, 128], value:[96, 1, 16, 128] if self.offload_qkv_linear: hidden_states = fine_grained_offloading_group_start(hidden_states, name="qkv_linear") - with get_fine_grained_offloading_context(self.offload_qkv_linear): + with off_interface.get_context(self.offload_qkv_linear): query, key, value = self.get_query_key_value_tensors( hidden_states, key_value_states, @@ -288,7 +290,7 @@ def forward( query = fine_grained_offloading_group_start(query, name="core_attn") if inference_context is None or inference_context.is_static_batching(): - with get_fine_grained_offloading_context(self.offload_core_attention): + with off_interface.get_context(self.offload_core_attention): core_attn_out = self.core_attention( query, key, @@ -348,7 +350,7 @@ def forward( # ================= if self.offload_attn_proj: core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") - with get_fine_grained_offloading_context(self.offload_attn_proj): + with off_interface.get_context(self.offload_attn_proj): output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: output = fine_grained_offloading_group_commit( diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index cb263aba17c..86cefcae298 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -535,10 +535,12 @@ def _forward_attention( context (Tensor): Updated context tensor if cross-attention is used, otherwise None. """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + FineGrainedActivationOffloadingInterface as off_interface, + ) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, fine_grained_offloading_group_start, - get_fine_grained_offloading_context, ) inference_context = deprecate_inference_params(inference_context, inference_params) @@ -551,12 +553,12 @@ def _forward_attention( # Optional Input Layer norm if self.recompute_input_layernorm: self.input_layernorm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with get_fine_grained_offloading_context(self.offload_attn_norm): + with off_interface.get_context(self.offload_attn_norm): input_layernorm_output = self.input_layernorm_checkpoint.checkpoint( self.input_layernorm, hidden_states ) else: - with get_fine_grained_offloading_context(self.offload_attn_norm): + with off_interface.get_context(self.offload_attn_norm): input_layernorm_output = self.input_layernorm(hidden_states) using_fused_tp_inference_kernel = (not self.training) and ( @@ -648,9 +650,11 @@ def _forward_mlp(self, hidden_states, inference_context=None): output (Tensor): Transformed hidden states of shape [s, b, h]. """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + FineGrainedActivationOffloadingInterface as off_interface, + ) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_start, - get_fine_grained_offloading_context, ) # Residual connection. @@ -661,12 +665,12 @@ def _forward_mlp(self, hidden_states, inference_context=None): # Optional Layer norm post the cross-attention. if self.recompute_pre_mlp_layernorm: self.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with get_fine_grained_offloading_context(self.offload_mlp_norm): + with off_interface.get_context(self.offload_mlp_norm): pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint( self.pre_mlp_layernorm, hidden_states ) else: - with get_fine_grained_offloading_context(self.offload_mlp_norm): + with off_interface.get_context(self.offload_mlp_norm): pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) nvtx_range_push(suffix="mlp") diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 88d76bf5d80..558c6934a0c 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -10,6 +10,9 @@ from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + FineGrainedActivationOffloadingInterface as off_interface, +) from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed from megatron.core.transformer.enums import AttnBackend from megatron.core.transformer.transformer_config import MLATransformerConfig, TransformerConfig @@ -107,10 +110,9 @@ def _run_one_iter_and_capture( - selected grads (CPU float32) - peak_memory_allocated (bytes) during the iteration """ - from megatron.core.pipeline_parallel import fine_grained_activation_offload as off if enable_offload_reset: - off.fine_grained_offloading_reset() + off_interface.reset() # for p in model.parameters(): # if p.grad is not None: @@ -179,7 +181,7 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( from megatron.core.pipeline_parallel import fine_grained_activation_offload as off - off.fine_grained_offloading_reset_instance() + off_interface.reset_instance() try: # 1) Baseline run (no offloading) @@ -244,7 +246,7 @@ def test_gpt_fine_grained_activation_offloading_correctness_and_memory( enable_offload_reset=True, ) # Reset once more to trigger post_warmup_callback and apply steady-state offload decisions. - off.fine_grained_offloading_reset() + off_interface.reset() from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( PipelineOffloadManager, @@ -453,7 +455,7 @@ def _run_schedule_1f1b_two_microbatches( This is the execution path that exercises EP A2A overlap scheduling. """ if enable_offload_reset: - off.fine_grained_offloading_reset() + off_interface.reset() data0 = _make_schedule_inputs() data1 = _make_schedule_inputs() @@ -486,7 +488,7 @@ def _run_schedule_1f1b_two_microbatches( ) set_streams() - off.fine_grained_offloading_reset_instance() + off_interface.reset_instance() try: with deterministic_mode(): @@ -513,9 +515,9 @@ def _run_schedule_1f1b_two_microbatches( _restore_params(off_model, base_params) off_model.train() # Warmup once to populate cached chunks, then reset to apply steady-state offload decisions. - off.fine_grained_offloading_reset() + off_interface.reset() _run_schedule_1f1b_two_microbatches(off_model, enable_offload_reset=False) - off.fine_grained_offloading_reset() + off_interface.reset() from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( PipelineOffloadManager, ) From b93d212472683cb082edc7244129482178ea14c0 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 14 Jan 2026 01:48:42 -0800 Subject: [PATCH 42/47] remove group_start() calls Signed-off-by: Hongbin Liu --- .../fine_grained_activation_offload.py | 18 ++++++++++++++++-- megatron/core/transformer/attention.py | 14 +++++--------- megatron/core/transformer/moe/experts.py | 14 ++++---------- .../transformer/multi_latent_attention.py | 15 +++++---------- .../core/transformer/transformer_layer.py | 19 ++++++++----------- 5 files changed, 38 insertions(+), 42 deletions(-) diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 4e6c8219ee2..47bfc0518c4 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -1205,8 +1205,22 @@ def fine_grained_offloading_backward_record(tensor, event: torch.cuda.Event) -> class FineGrainedActivationOffloadingInterface: """Interface for fine-grained activation offloading.""" - def __init__(self): - pass + def __init__(self, offload: bool, tensor: torch.Tensor, name: str): + self.offload = offload + self.tensor = tensor + self.name = name + + def __enter__(self): + """Enter context manager to enable activation offloading hooks.""" + if self.offload: + self.tensor = fine_grained_offloading_group_start(self.tensor, self.name) + PipelineOffloadManager.get_instance().__enter__() + return self.tensor + + def __exit__(self, *args: Any): + """Exit context manager to disable activation offloading hooks.""" + if self.offload: + PipelineOffloadManager.get_instance().__exit__() @staticmethod def init_chunk_handler(vp_size, vp_stage, min_offloaded_tensor_size): diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 854e87a7f70..75c2618a392 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -806,9 +806,8 @@ def forward( self.config.fused_single_qkv_rope and split_qkv ), "fused_single_qkv_rope requested but not available/supported for the config." - if self.offload_qkv_linear: - hidden_states = fine_grained_offloading_group_start(hidden_states, name="qkv_linear") - with off_interface.get_context(self.offload_qkv_linear): + with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") \ + as hidden_states: qkv_output = self.get_query_key_value_tensors( hidden_states, key_value_states, @@ -973,11 +972,10 @@ def forward( packed_seq_params=packed_seq_params, ) else: - if self.offload_core_attention and self.training: - query = fine_grained_offloading_group_start(query, name="core_attn") if inference_context is None or inference_context.is_static_batching(): # Static batching attention kernel. - with off_interface.get_context(self.offload_core_attention): + with off_interface(self.offload_core_attention and self.training, query, \ + "core_attn") as query: core_attn_out = self.core_attention( query, key, @@ -1034,9 +1032,7 @@ def forward( # Output. [sq, b, h] # ================= nvtx_range_push(suffix="linear_proj") - if self.offload_attn_proj: - core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") - with off_interface.get_context(self.offload_attn_proj): + with off_interface(self.offload_attn_proj, core_attn_out, "attn_proj") as core_attn_out: output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: output = fine_grained_offloading_group_commit( diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index 1fe4703fef2..df561c4e048 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -668,11 +668,8 @@ def forward( # Probs already applied, so reset to 1. permuted_probs = torch.ones_like(permuted_probs) - if self.offload_expert_fc1: - permuted_local_hidden_states = fine_grained_offloading_group_start( - permuted_local_hidden_states, name="expert_fc1" - ) - with off_interface.get_context(self.offload_expert_fc1): + with off_interface(self.offload_expert_fc1, permuted_local_hidden_states, "expert_fc1") \ + as permuted_local_hidden_states: fc1_output, bias_parallel = self.linear_fc1( permuted_local_hidden_states, tokens_per_expert ) @@ -741,17 +738,14 @@ def glu(x): intermediate_parallel = intermediate_parallel.to(original_dtype) return intermediate_parallel - if self.offload_moe_act: - fc1_output = fine_grained_offloading_group_start(fc1_output, name="moe_act") - if self.activation_recompute: self.activation_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with off_interface.get_context(self.offload_moe_act): + with off_interface(self.offload_moe_act, fc1_output, "moe_act") as fc1_output: bias_act_output = self.activation_checkpoint.checkpoint( bias_act_func, fc1_output, bias_parallel, permuted_probs ) else: - with off_interface.get_context(self.offload_moe_act): + with off_interface(self.offload_moe_act, fc1_output, "moe_act") as fc1_output: bias_act_output = bias_act_func(fc1_output, bias_parallel, permuted_probs) output, output_bias = self.linear_fc2(bias_act_output, tokens_per_expert) diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index e7550789ab4..563c057d5a6 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -247,9 +247,8 @@ def forward( # Get the query, key and value tensors based on the type of attention - # self or cross attn. # query: [96, 1, 16, 128], key:[96, 1, 16, 128], value:[96, 1, 16, 128] - if self.offload_qkv_linear: - hidden_states = fine_grained_offloading_group_start(hidden_states, name="qkv_linear") - with off_interface.get_context(self.offload_qkv_linear): + with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") \ + as hidden_states: query, key, value = self.get_query_key_value_tensors( hidden_states, key_value_states, @@ -287,11 +286,9 @@ def forward( query, key, value, attention_mask, packed_seq_params=packed_seq_params ) else: - if self.offload_core_attention and self.training: - query = fine_grained_offloading_group_start(query, name="core_attn") - if inference_context is None or inference_context.is_static_batching(): - with off_interface.get_context(self.offload_core_attention): + with off_interface(self.offload_core_attention and self.training, query, \ + "core_attn") as query: core_attn_out = self.core_attention( query, key, @@ -349,9 +346,7 @@ def forward( # ================= # Output. [sq, b, h] # ================= - if self.offload_attn_proj: - core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") - with off_interface.get_context(self.offload_attn_proj): + with off_interface(self.offload_attn_proj, core_attn_out, "attn_proj") as core_attn_out: output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: output = fine_grained_offloading_group_commit( diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index 86cefcae298..ddef3831df1 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -548,17 +548,17 @@ def _forward_attention( # Residual connection. residual = hidden_states - if self.offload_attn_norm: - hidden_states = fine_grained_offloading_group_start(hidden_states, name="attn_norm") # Optional Input Layer norm if self.recompute_input_layernorm: self.input_layernorm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with off_interface.get_context(self.offload_attn_norm): + with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") \ + as hidden_states: input_layernorm_output = self.input_layernorm_checkpoint.checkpoint( self.input_layernorm, hidden_states ) else: - with off_interface.get_context(self.offload_attn_norm): + with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") \ + as hidden_states: input_layernorm_output = self.input_layernorm(hidden_states) using_fused_tp_inference_kernel = (not self.training) and ( @@ -653,24 +653,21 @@ def _forward_mlp(self, hidden_states, inference_context=None): from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( FineGrainedActivationOffloadingInterface as off_interface, ) - from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_start, - ) # Residual connection. residual = hidden_states - if self.offload_mlp_norm: - hidden_states = fine_grained_offloading_group_start(hidden_states, name="mlp_norm") # Optional Layer norm post the cross-attention. if self.recompute_pre_mlp_layernorm: self.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with off_interface.get_context(self.offload_mlp_norm): + with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") \ + as hidden_states: pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint( self.pre_mlp_layernorm, hidden_states ) else: - with off_interface.get_context(self.offload_mlp_norm): + with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") \ + as hidden_states: pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) nvtx_range_push(suffix="mlp") From 16d4114ad65c67dfedb344cbbe80894456e54d38 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 14 Jan 2026 01:51:19 -0800 Subject: [PATCH 43/47] format Signed-off-by: Hongbin Liu --- megatron/core/transformer/attention.py | 9 ++++----- megatron/core/transformer/moe/experts.py | 6 +++--- megatron/core/transformer/multi_latent_attention.py | 9 ++++----- megatron/core/transformer/transformer_layer.py | 12 ++++-------- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 75c2618a392..14dce873b8e 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -28,7 +28,6 @@ ) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, - fine_grained_offloading_group_start, ) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.mappings import all_gather_last_dim_from_tensor_parallel_region @@ -806,8 +805,7 @@ def forward( self.config.fused_single_qkv_rope and split_qkv ), "fused_single_qkv_rope requested but not available/supported for the config." - with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") \ - as hidden_states: + with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") as hidden_states: qkv_output = self.get_query_key_value_tensors( hidden_states, key_value_states, @@ -974,8 +972,9 @@ def forward( else: if inference_context is None or inference_context.is_static_batching(): # Static batching attention kernel. - with off_interface(self.offload_core_attention and self.training, query, \ - "core_attn") as query: + with off_interface( + self.offload_core_attention and self.training, query, "core_attn" + ) as query: core_attn_out = self.core_attention( query, key, diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index df561c4e048..a6d5098aadd 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -30,7 +30,6 @@ ) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, - fine_grained_offloading_group_start, ) from megatron.core.tensor_parallel.layers import ( _initialize_affine_weight_cpu, @@ -668,8 +667,9 @@ def forward( # Probs already applied, so reset to 1. permuted_probs = torch.ones_like(permuted_probs) - with off_interface(self.offload_expert_fc1, permuted_local_hidden_states, "expert_fc1") \ - as permuted_local_hidden_states: + with off_interface( + self.offload_expert_fc1, permuted_local_hidden_states, "expert_fc1" + ) as permuted_local_hidden_states: fc1_output, bias_parallel = self.linear_fc1( permuted_local_hidden_states, tokens_per_expert ) diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index 563c057d5a6..2cc083a300b 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -27,7 +27,6 @@ ) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, - fine_grained_offloading_group_start, ) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.layers import ColumnParallelLinear @@ -247,8 +246,7 @@ def forward( # Get the query, key and value tensors based on the type of attention - # self or cross attn. # query: [96, 1, 16, 128], key:[96, 1, 16, 128], value:[96, 1, 16, 128] - with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") \ - as hidden_states: + with off_interface(self.offload_qkv_linear, hidden_states, "qkv_linear") as hidden_states: query, key, value = self.get_query_key_value_tensors( hidden_states, key_value_states, @@ -287,8 +285,9 @@ def forward( ) else: if inference_context is None or inference_context.is_static_batching(): - with off_interface(self.offload_core_attention and self.training, query, \ - "core_attn") as query: + with off_interface( + self.offload_core_attention and self.training, query, "core_attn" + ) as query: core_attn_out = self.core_attention( query, key, diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index ddef3831df1..d4e4ee606c7 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -551,14 +551,12 @@ def _forward_attention( # Optional Input Layer norm if self.recompute_input_layernorm: self.input_layernorm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") \ - as hidden_states: + with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") as hidden_states: input_layernorm_output = self.input_layernorm_checkpoint.checkpoint( self.input_layernorm, hidden_states ) else: - with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") \ - as hidden_states: + with off_interface(self.offload_attn_norm, hidden_states, "attn_norm") as hidden_states: input_layernorm_output = self.input_layernorm(hidden_states) using_fused_tp_inference_kernel = (not self.training) and ( @@ -660,14 +658,12 @@ def _forward_mlp(self, hidden_states, inference_context=None): # Optional Layer norm post the cross-attention. if self.recompute_pre_mlp_layernorm: self.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") \ - as hidden_states: + with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") as hidden_states: pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint( self.pre_mlp_layernorm, hidden_states ) else: - with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") \ - as hidden_states: + with off_interface(self.offload_mlp_norm, hidden_states, "mlp_norm") as hidden_states: pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) nvtx_range_push(suffix="mlp") From 6263630406e02c398e1cc3d8c1cf6faa44324161 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 14 Jan 2026 02:05:47 -0800 Subject: [PATCH 44/47] add comments Signed-off-by: Hongbin Liu --- megatron/core/models/gpt/fine_grained_callables.py | 9 ++++----- megatron/core/pipeline_parallel/utils.py | 11 ++++++++++- megatron/core/transformer/moe/experts.py | 3 +++ megatron/core/transformer/transformer_layer.py | 5 ++++- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index 6d32ff0027c..2ce2902926a 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -13,7 +13,6 @@ ) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, - fine_grained_offloading_group_start, ) from megatron.core.pipeline_parallel.utils import ScheduleNode, make_viewless from megatron.core.transformer.module import float16_to_fp32 @@ -380,16 +379,14 @@ def submodule_post_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor) Run forward pass for computations between attention and dispatch: pre mlp layernorm->router->dispatch preprocess """ - if layer.offload_mlp_norm: - hidden_states = fine_grained_offloading_group_start(hidden_states, name="mlp_norm") if layer.recompute_pre_mlp_layernorm: layer.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - with off_interface.get_context(layer.offload_mlp_norm): + with off_interface(layer.offload_mlp_norm, hidden_states, "mlp_norm") as hidden_states: pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint( layer.pre_mlp_layernorm, hidden_states ) else: - with off_interface.get_context(layer.offload_mlp_norm): + with off_interface(layer.offload_mlp_norm, hidden_states, "mlp_norm") as hidden_states: pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states) probs, routing_map = layer.mlp.route(pre_mlp_layernorm_output) @@ -472,6 +469,8 @@ def submodule_combine_forward( hidden_states = layer.mlp_bda(layer.training, layer.config.bias_dropout_fusion)( mlp_output_with_bias, residual, layer.hidden_dropout ) + # Delay the offload of the mlp norm until after the mlp_bda has been computed + # because the residual is needed in the mlp_bda. if layer.offload_mlp_norm: hidden_states = fine_grained_offloading_group_commit( hidden_states, name="mlp_norm", forced_released_tensors=[residual] diff --git a/megatron/core/pipeline_parallel/utils.py b/megatron/core/pipeline_parallel/utils.py index c6b7f4c6419..03c5f01f443 100644 --- a/megatron/core/pipeline_parallel/utils.py +++ b/megatron/core/pipeline_parallel/utils.py @@ -1,5 +1,6 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +import logging from abc import ABC, abstractmethod from contextlib import contextmanager from typing import Callable, Optional @@ -7,7 +8,9 @@ import torch from torch.autograd import Variable -from megatron.core.utils import get_pg_rank, get_pg_size, make_viewless_tensor +from megatron.core.utils import get_pg_rank, get_pg_size, log_single_rank, make_viewless_tensor + +logger = logging.getLogger(__name__) def is_pp_first_stage(pp_group: torch.distributed.ProcessGroup): @@ -106,6 +109,12 @@ def set_ideal_affinity_for_current_gpu(): handle = pynvml.nvmlDeviceGetHandleByUUID("GPU-" + str(uuid.UUID(bytes=device_uuid.bytes))) pynvml.nvmlDeviceSetCpuAffinity(handle) + log_single_rank( + logger, + logging.WARNING, + f"Set CPU affinity for all GPUs for optimal host-device transfer performance", + ) + @contextmanager def stream_acquire_context(stream, event): diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index a6d5098aadd..24c87c52ec9 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -751,6 +751,9 @@ def glu(x): output, output_bias = self.linear_fc2(bias_act_output, tokens_per_expert) if self.activation_recompute: self.activation_checkpoint.discard_output_and_register_recompute(output) + + # Delay the offload of the moe act until after the linear_fc2 has been computed + # to make sure the fc1_output is reloaded to GPU before recomputing moe_act. if self.offload_moe_act: output = fine_grained_offloading_group_commit( output, name="moe_act", forced_released_tensors=[fc1_output] diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index d4e4ee606c7..f9797fee402 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -540,7 +540,6 @@ def _forward_attention( ) from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( fine_grained_offloading_group_commit, - fine_grained_offloading_group_start, ) inference_context = deprecate_inference_params(inference_context, inference_params) @@ -606,6 +605,8 @@ def _forward_attention( ) nvtx_range_pop(suffix="self_attn_bda") + # Delay the offload of the attention norm until after the self_attn_bda has been computed + # because the residual is needed in the self_attn_bda. if self.offload_attn_norm: hidden_states = fine_grained_offloading_group_commit( hidden_states, name="attn_norm", forced_released_tensors=[residual] @@ -775,6 +776,8 @@ def _forward_post_mlp(self, mlp_output_with_bias, residual): mlp_output_with_bias, residual, self.hidden_dropout ) nvtx_range_pop(suffix="mlp_bda") + # Delay the offload of the mlp norm until after the mlp_bda has been computed + # because the residual is needed in the mlp_bda. if self.offload_mlp_norm: hidden_states = fine_grained_offloading_group_commit( hidden_states, name="mlp_norm", forced_released_tensors=[residual] From 9423c6b3b5eefad443e90bbdfbb62a83fe30bf75 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Wed, 14 Jan 2026 02:51:24 -0800 Subject: [PATCH 45/47] fix min_offload_size and update golden values Signed-off-by: Hongbin Liu --- megatron/training/arguments.py | 2 +- .../golden_values_dev_dgx_h100.json | 592 +++++++++--------- .../model_config.yaml | 7 +- .../golden_values_dev_dgx_h100.json | 496 +++++++-------- .../model_config.yaml | 7 +- .../unit_tests/models/test_mamba_moe_model.py | 2 +- 6 files changed, 554 insertions(+), 552 deletions(-) diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 340238e4094..57967e9c1cc 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -2372,7 +2372,7 @@ def _add_training_args(parser): help='Enable fine-grained activation offloading.') group.add_argument('--offload-modules', nargs='*', type=str, default=[], help='The submodules to offload its input. Choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".') - group.add_argument('--min-offloaded-tensor-size', type=int, default=10*1024*1024, + group.add_argument('--min-offloaded-tensor-size', type=int, default=1024*1024, help='The minimum size of the tensor to be offloaded.') group.add_argument('--batch-invariant-mode', action='store_true', help='Use batch-invariant kernels for deterministic forward execution regardless ' diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json index dac60247e7d..d5ced620365 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -6,54 +6,54 @@ "values": { "1": 11.06693, "2": 11.0602, - "3": 10.21194, - "4": 9.95432, - "5": 10.12218, - "6": 8.82302, - "7": 9.52771, - "8": 8.44293, - "9": 7.8503, - "10": 7.06875, - "11": 9.30824, - "12": 9.15133, - "13": 7.87105, - "14": 8.20684, - "15": 8.21969, - "16": 8.17636, - "17": 8.20544, - "18": 7.4911, - "19": 8.08112, - "20": 7.61907, - "21": 7.94753, - "22": 7.29291, - "23": 7.93581, - "24": 7.44247, - "25": 8.2355, - "26": 7.75421, - "27": 7.70115, - "28": 7.654, - "29": 7.75231, - "30": 7.56191, - "31": 7.81571, - "32": 6.47353, - "33": 7.20345, - "34": 7.77999, - "35": 7.72719, - "36": 6.71935, - "37": 8.08764, - "38": 7.6188, - "39": 7.96527, - "40": 7.49853, - "41": 7.50088, - "42": 6.11546, - "43": 7.59151, - "44": 7.91341, - "45": 6.83429, - "46": 7.40916, - "47": 7.78722, - "48": 7.87359, - "49": 7.58606, - "50": 6.84286 + "3": 10.16141, + "4": 10.11145, + "5": 10.47957, + "6": 10.21751, + "7": 10.56153, + "8": 12.79501, + "9": 12.96949, + "10": 13.32223, + "11": 11.63359, + "12": 11.4938, + "13": 12.46292, + "14": 12.13415, + "15": 11.90295, + "16": 12.01307, + "17": 12.17443, + "18": 12.64978, + "19": 11.81295, + "20": 12.18673, + "21": 11.24306, + "22": 11.54156, + "23": 10.98412, + "24": 11.01925, + "25": 10.73001, + "26": 10.72806, + "27": 10.79039, + "28": 10.714, + "29": 10.73974, + "30": 10.75246, + "31": 10.68874, + "32": 10.65791, + "33": 10.81137, + "34": 10.79058, + "35": 10.75368, + "36": 10.64393, + "37": 10.87492, + "38": 10.90591, + "39": 10.78825, + "40": 10.75548, + "41": 10.8955, + "42": 10.70411, + "43": 10.66907, + "44": 10.72512, + "45": 10.54927, + "46": 10.46973, + "47": 10.66311, + "48": 10.62453, + "49": 10.61656, + "50": 10.21176 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 47165232.0, - "2": 46897932.0, - "3": 49540668.0, - "4": 293969504.0, - "5": 559802048.0, - "6": 658711232.0, - "7": 1024298624.0, - "8": 752264576.0, - "9": 846702016.0, - "10": 686872128.0, - "11": 817431744.0, - "12": 804868736.0, - "13": 639449280.0, - "14": 622266240.0, - "15": 694217664.0, - "16": 836259456.0, - "17": 670662656.0, - "18": 692413632.0, - "19": 892693056.0, - "20": 865534016.0, - "21": 685732736.0, - "22": 802160192.0, - "23": 818284096.0, - "24": 792706240.0, - "25": 648186560.0, - "26": 770639552.0, - "27": 776554240.0, - "28": 774063488.0, - "29": 776921920.0, - "30": 786722048.0, - "31": 807679616.0, - "32": 800267392.0, - "33": 802385024.0, - "34": 770817664.0, - "35": 728918528.0, - "36": 724172608.0, - "37": 731043072.0, - "38": 718266048.0, - "39": 723535232.0, - "40": 723057728.0, - "41": 704505472.0, - "42": 674238976.0, - "43": 666610048.0, - "44": 680288064.0, - "45": 638337536.0, - "46": 619652032.0, - "47": 629857024.0, - "48": 603989568.0, - "49": 581503744.0, - "50": 560118272.0 + "1": 47165216.0, + "2": 46897552.0, + "3": 52682736.0, + "4": 70585808.0, + "5": 1850183680.0, + "6": 171098656.0, + "7": 436105120.0, + "8": 1850183680.0, + "9": 1850183680.0, + "10": 1850183680.0, + "11": 1850183680.0, + "12": 1850183680.0, + "13": 1850183680.0, + "14": 1850183680.0, + "15": 555857088.0, + "16": 1850183680.0, + "17": 1850183680.0, + "18": 1850183680.0, + "19": 886404992.0, + "20": 654826944.0, + "21": 603993664.0, + "22": 726709632.0, + "23": 566656896.0, + "24": 1850183680.0, + "25": 799245696.0, + "26": 978252032.0, + "27": 1850183680.0, + "28": 906183104.0, + "29": 1850183680.0, + "30": 1850183680.0, + "31": 810874112.0, + "32": 1850183680.0, + "33": 1850183680.0, + "34": 553779584.0, + "35": 565382400.0, + "36": 585787712.0, + "37": 627284160.0, + "38": 331368192.0, + "39": 638619264.0, + "40": 1850183680.0, + "41": 1850183680.0, + "42": 1850183680.0, + "43": 1850183680.0, + "44": 1850183680.0, + "45": 1850183680.0, + "46": 1850183680.0, + "47": 434842944.0, + "48": 1850183680.0, + "49": 575219328.0, + "50": 1850183680.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5279422976.0, - "2": 5279627776.0, - "3": 5279830528.0, - "4": 5279425024.0, - "5": 5279627776.0, - "6": 5279830528.0, - "7": 5280033280.0, - "8": 5280236032.0, - "9": 5280438784.0, - "10": 5280641536.0, - "11": 5280844288.0, - "12": 5281047040.0, - "13": 5281249792.0, - "14": 5281452544.0, - "15": 5281655296.0, - "16": 5281858048.0, - "17": 5282060800.0, - "18": 5282263552.0, - "19": 5282466304.0, - "20": 5282669056.0, - "21": 5282871808.0, - "22": 5283074560.0, - "23": 5283277312.0, - "24": 5283480064.0, - "25": 5283682816.0, - "26": 5283885568.0, - "27": 5284088320.0, - "28": 5284291072.0, - "29": 5284493824.0, - "30": 5284696576.0, - "31": 5284899328.0, - "32": 5285102080.0, - "33": 5285304832.0, - "34": 5285507584.0, - "35": 5285710336.0, - "36": 5285913088.0, - "37": 5286115840.0, - "38": 5286318592.0, - "39": 5286521344.0, - "40": 5286724096.0, - "41": 5286926848.0, - "42": 5287129600.0, - "43": 5287332352.0, - "44": 5287535104.0, - "45": 5287737856.0, - "46": 5287940608.0, - "47": 5288143360.0, - "48": 5288346112.0, - "49": 5288548864.0, - "50": 5288751616.0 + "1": 5283616256.0, + "2": 5288015360.0, + "3": 5288218112.0, + "4": 5288420864.0, + "5": 5288623616.0, + "6": 5287812608.0, + "7": 5288015360.0, + "8": 5288218112.0, + "9": 5287711232.0, + "10": 5287913984.0, + "11": 5288116736.0, + "12": 5288319488.0, + "13": 5288522240.0, + "14": 5288724992.0, + "15": 5288927744.0, + "16": 5289130496.0, + "17": 5289333248.0, + "18": 5289536000.0, + "19": 5289738752.0, + "20": 5289941504.0, + "21": 5290144256.0, + "22": 5290347008.0, + "23": 5290549760.0, + "24": 5290752512.0, + "25": 5290955264.0, + "26": 5291158016.0, + "27": 5291360768.0, + "28": 5291563520.0, + "29": 5291766272.0, + "30": 5291969024.0, + "31": 5292171776.0, + "32": 5292374528.0, + "33": 5292577280.0, + "34": 5292780032.0, + "35": 5292982784.0, + "36": 5293185536.0, + "37": 5293388288.0, + "38": 5293591040.0, + "39": 5293793792.0, + "40": 5293996544.0, + "41": 5294199296.0, + "42": 5294402048.0, + "43": 5294604800.0, + "44": 5294807552.0, + "45": 5295010304.0, + "46": 5295213056.0, + "47": 5295415808.0, + "48": 5295618560.0, + "49": 5295821312.0, + "50": 5296024064.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6173633024.0, - "2": 8218349056.0, - "3": 8218349056.0, - "4": 8218349056.0, - "5": 8218349056.0, - "6": 8218349056.0, - "7": 8218349056.0, - "8": 8218349056.0, - "9": 8218349056.0, - "10": 8218349056.0, - "11": 8230493696.0, - "12": 8230493696.0, - "13": 8230493696.0, - "14": 8230493696.0, - "15": 8230493696.0, - "16": 8230493696.0, - "17": 8230869504.0, - "18": 8231849472.0, - "19": 8231849472.0, - "20": 8267122176.0, - "21": 8267122176.0, - "22": 8267122176.0, - "23": 8267122176.0, - "24": 8267122176.0, - "25": 8267122176.0, - "26": 8267122176.0, - "27": 8267122176.0, - "28": 8267122176.0, - "29": 8267122176.0, - "30": 8294885888.0, - "31": 8294885888.0, - "32": 8294885888.0, - "33": 8294885888.0, - "34": 8294885888.0, - "35": 8294885888.0, - "36": 8294885888.0, - "37": 8294885888.0, - "38": 8295116288.0, - "39": 8315220480.0, - "40": 8315220480.0, - "41": 8315220480.0, - "42": 8315220480.0, - "43": 8315220480.0, - "44": 8315220480.0, - "45": 8315220480.0, - "46": 8315220480.0, - "47": 8315220480.0, - "48": 8315220480.0, - "49": 8315220480.0, - "50": 8315220480.0 + "1": 5283618816.0, + "2": 8185453056.0, + "3": 8185453056.0, + "4": 8185453056.0, + "5": 8195318272.0, + "6": 8195318272.0, + "7": 8195318272.0, + "8": 8195318272.0, + "9": 8195318272.0, + "10": 8195318272.0, + "11": 8195318272.0, + "12": 8195318272.0, + "13": 8195318272.0, + "14": 8195318272.0, + "15": 8195318272.0, + "16": 8199233024.0, + "17": 8199233024.0, + "18": 8199233024.0, + "19": 8199233024.0, + "20": 8199233024.0, + "21": 8238446080.0, + "22": 8238446080.0, + "23": 8238446080.0, + "24": 8238446080.0, + "25": 8247293440.0, + "26": 8247293440.0, + "27": 8247293440.0, + "28": 8250185216.0, + "29": 8255527424.0, + "30": 8255527424.0, + "31": 8255527424.0, + "32": 8255527424.0, + "33": 8255527424.0, + "34": 8255527424.0, + "35": 8255527424.0, + "36": 8255527424.0, + "37": 8255527424.0, + "38": 8255527424.0, + "39": 8255527424.0, + "40": 8255527424.0, + "41": 8255527424.0, + "42": 8255527424.0, + "43": 8255527424.0, + "44": 8255527424.0, + "45": 8255527424.0, + "46": 8255527424.0, + "47": 8255527424.0, + "48": 8255527424.0, + "49": 8255527424.0, + "50": 8255527424.0 } }, "mtp_1 loss": { @@ -234,54 +234,54 @@ "values": { "1": 11.07401, "2": 11.0927, - "3": 10.82635, - "4": 10.27639, - "5": 10.45337, - "6": 8.32748, - "7": 9.82613, - "8": 8.01543, - "9": 7.47559, - "10": 6.75826, - "11": 8.92939, - "12": 8.98807, - "13": 7.80234, - "14": 8.02541, - "15": 8.1122, - "16": 8.1412, - "17": 8.13081, - "18": 7.44607, - "19": 8.03563, - "20": 7.53922, - "21": 7.9001, - "22": 7.27611, - "23": 7.88426, - "24": 7.37488, - "25": 8.16858, - "26": 7.69965, - "27": 7.62748, - "28": 7.61418, - "29": 7.69669, - "30": 7.48101, - "31": 7.74012, - "32": 6.36863, - "33": 7.13981, - "34": 7.72056, - "35": 7.63371, - "36": 6.61154, - "37": 8.02877, - "38": 7.57825, - "39": 7.89394, - "40": 7.41218, - "41": 7.42138, - "42": 6.01249, - "43": 7.49014, - "44": 7.86865, - "45": 6.7504, - "46": 7.30952, - "47": 7.72961, - "48": 7.7901, - "49": 7.4907, - "50": 6.75604 + "3": 10.83159, + "4": 10.61397, + "5": 10.85768, + "6": 9.79263, + "7": 10.90607, + "8": 10.19798, + "9": 9.82717, + "10": 9.23805, + "11": 11.0712, + "12": 11.11709, + "13": 10.03407, + "14": 10.27606, + "15": 10.73067, + "16": 10.91485, + "17": 10.76886, + "18": 10.49659, + "19": 10.96955, + "20": 10.45905, + "21": 10.91629, + "22": 10.05081, + "23": 10.44411, + "24": 9.74826, + "25": 10.81497, + "26": 10.38519, + "27": 10.31999, + "28": 10.27887, + "29": 10.40945, + "30": 10.20684, + "31": 10.54594, + "32": 8.85942, + "33": 9.75619, + "34": 10.56214, + "35": 10.59167, + "36": 9.30537, + "37": 10.59407, + "38": 10.2994, + "39": 10.69954, + "40": 10.37003, + "41": 10.248, + "42": 8.56376, + "43": 10.49224, + "44": 10.57211, + "45": 9.36238, + "46": 10.2179, + "47": 10.63449, + "48": 10.56697, + "49": 10.44093, + "50": 9.49252 } }, "iteration-time": { @@ -289,56 +289,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 90.26185, - "2": 2.40864, - "3": 2.52164, - "4": 4.93801, - "5": 2.91719, - "6": 2.68307, - "7": 2.58408, - "8": 2.02548, - "9": 2.63344, - "10": 1.98392, - "11": 2.06707, - "12": 1.94587, - "13": 1.957, - "14": 2.00392, - "15": 2.54927, - "16": 2.01775, - "17": 1.96192, - "18": 1.99266, - "19": 1.99493, - "20": 1.95233, - "21": 1.93417, - "22": 1.95913, - "23": 1.94277, - "24": 1.97087, - "25": 1.95352, - "26": 2.00648, - "27": 2.00301, - "28": 1.99205, - "29": 1.99366, - "30": 1.97014, - "31": 1.97902, - "32": 1.97219, - "33": 1.9316, - "34": 1.9719, - "35": 1.97097, - "36": 1.94724, - "37": 1.97172, - "38": 1.99927, - "39": 1.95359, - "40": 1.96791, - "41": 1.9645, - "42": 1.91342, - "43": 1.94576, - "44": 1.96464, - "45": 1.96208, - "46": 1.98727, - "47": 1.97396, - "48": 1.96834, - "49": 1.95939, - "50": 1.96462 + "1": 71.30157, + "2": 2.34464, + "3": 2.38747, + "4": 2.10322, + "5": 2.12945, + "6": 2.0424, + "7": 2.12036, + "8": 2.0147, + "9": 2.04925, + "10": 2.02797, + "11": 1.95087, + "12": 2.04985, + "13": 1.94106, + "14": 1.90425, + "15": 1.89051, + "16": 1.89398, + "17": 1.94082, + "18": 1.93176, + "19": 1.94027, + "20": 1.90271, + "21": 1.91097, + "22": 1.90382, + "23": 1.93889, + "24": 1.90551, + "25": 1.90947, + "26": 1.92126, + "27": 1.89917, + "28": 1.89866, + "29": 1.93981, + "30": 1.90782, + "31": 1.91244, + "32": 1.93864, + "33": 1.93947, + "34": 1.96882, + "35": 1.89751, + "36": 1.94038, + "37": 1.90603, + "38": 1.94988, + "39": 1.89874, + "40": 1.90233, + "41": 1.92861, + "42": 1.93931, + "43": 1.91212, + "44": 1.92615, + "45": 1.89555, + "46": 1.94522, + "47": 1.9103, + "48": 1.94689, + "49": 1.9355, + "50": 1.89832 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml index f508462666a..a37dd0dc658 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml @@ -5,6 +5,10 @@ ENV_VARS: NCCL_NVLS_ENABLE: 0 PYTHONWARNINGS: ignore NCCL_DEBUG: VERSION + NVTE_CPU_OFFLOAD_V1: 1 + NVTE_FUSED_ATTN: 0 + NCCL_ALGO: ^NVLS + CUBLAS_WORKSPACE_CONFIG: ':4096:8' MODEL_ARGS: # Distributed args --distributed-timeout-minutes: 60 @@ -29,8 +33,6 @@ MODEL_ARGS: --exit-duration-in-mins: 230 --no-check-for-nan-in-loss-and-grad: true --no-rope-fusion: true - --cross-entropy-loss-fusion: true - --cross-entropy-fusion-impl: native --manual-gc: true --manual-gc-interval: 100 --recompute-granularity: selective @@ -133,7 +135,6 @@ TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular METRICS: - "iteration-time" - "lm loss" - - "num-zeros" - "mem-allocated-bytes" - "mem-max-allocated-bytes" - "mtp_1 loss" diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json index 9750a5861c2..c133e93e5f7 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -6,54 +6,54 @@ "values": { "1": 11.01693, "2": 11.06263, - "3": 10.17794, - "4": 10.85994, - "5": 9.81693, - "6": 9.10087, - "7": 9.61225, - "8": 8.39617, - "9": 7.79702, - "10": 7.15217, - "11": 9.06659, - "12": 12.29047, - "13": 8.0613, - "14": 8.25238, - "15": 8.2457, - "16": 8.31701, - "17": 8.33964, - "18": 7.58044, - "19": 8.18777, - "20": 7.71116, - "21": 7.99992, - "22": 7.3454, - "23": 7.94728, - "24": 7.50571, - "25": 8.32022, - "26": 7.78539, - "27": 7.72311, - "28": 7.70481, - "29": 7.7694, - "30": 7.56679, - "31": 7.84739, - "32": 6.52324, - "33": 7.24018, - "34": 7.796, - "35": 7.73916, - "36": 6.73059, - "37": 8.15243, - "38": 7.62149, - "39": 7.97743, - "40": 7.51901, - "41": 7.51939, - "42": 6.1129, - "43": 7.60283, - "44": 7.96375, - "45": 6.84546, - "46": 7.42487, - "47": 7.82289, - "48": 7.87871, - "49": 7.59657, - "50": 6.84804 + "3": 10.0893, + "4": 9.64622, + "5": 10.351, + "6": 8.80033, + "7": 10.38861, + "8": 9.08827, + "9": 9.2025, + "10": 8.69816, + "11": 10.71757, + "12": 10.72938, + "13": 9.8103, + "14": 10.16776, + "15": 10.34088, + "16": 10.39001, + "17": 10.33746, + "18": 10.03205, + "19": 10.40886, + "20": 10.17433, + "21": 10.41968, + "22": 9.93423, + "23": 10.27377, + "24": 10.01972, + "25": 10.6325, + "26": 10.27763, + "27": 10.2853, + "28": 10.30143, + "29": 10.34056, + "30": 10.23697, + "31": 10.43065, + "32": 9.49143, + "33": 9.97924, + "34": 10.48027, + "35": 10.40919, + "36": 9.84774, + "37": 10.5738, + "38": 10.35817, + "39": 10.53096, + "40": 10.45813, + "41": 11.01492, + "42": 11.30727, + "43": 10.54763, + "44": 10.72116, + "45": 11.32983, + "46": 10.88386, + "47": 10.6974, + "48": 10.6521, + "49": 10.74413, + "50": 11.16561 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 47167816.0, - "2": 46900688.0, - "3": 71568288.0, - "4": 234203216.0, - "5": 474837472.0, - "6": 561190528.0, - "7": 964556864.0, - "8": 720809472.0, - "9": 815181312.0, - "10": 708885056.0, - "11": 635007360.0, - "12": 553207296.0, - "13": 683492160.0, - "14": 741814912.0, - "15": 782267840.0, - "16": 723021312.0, - "17": 651756800.0, - "18": 742705920.0, - "19": 741724544.0, - "20": 868679872.0, - "21": 861892736.0, - "22": 651191232.0, - "23": 767955264.0, - "24": 594551424.0, - "25": 824319360.0, - "26": 783214656.0, - "27": 625579968.0, - "28": 811807360.0, - "29": 795796608.0, - "30": 777285056.0, - "31": 785653312.0, - "32": 778247296.0, - "33": 745758336.0, - "34": 742508544.0, - "35": 725768768.0, - "36": 692716544.0, - "37": 696440960.0, - "38": 699391552.0, - "39": 710961152.0, - "40": 704181952.0, - "41": 534684160.0, - "42": 655362624.0, - "43": 666612480.0, - "44": 639391360.0, - "45": 610026240.0, - "46": 613361920.0, - "47": 604690432.0, - "48": 591405696.0, - "49": 562632448.0, - "50": 544389888.0 + "1": 47167796.0, + "2": 46899648.0, + "3": 58981488.0, + "4": 1722086400.0, + "5": 172862128.0, + "6": 167948304.0, + "7": 461255808.0, + "8": 1722086400.0, + "9": 1722086400.0, + "10": 155206176.0, + "11": 276392000.0, + "12": 449419776.0, + "13": 1722086400.0, + "14": 704085952.0, + "15": 684765632.0, + "16": 568911360.0, + "17": 1722086400.0, + "18": 944021952.0, + "19": 663072256.0, + "20": 1722086400.0, + "21": 802140416.0, + "22": 1722086400.0, + "23": 1722086400.0, + "24": 1722086400.0, + "25": 802352640.0, + "26": 1006560192.0, + "27": 710512832.0, + "28": 959659776.0, + "29": 1722086400.0, + "30": 962886400.0, + "31": 1722086400.0, + "32": 630419136.0, + "33": 1722086400.0, + "34": 585257664.0, + "35": 857957504.0, + "36": 1722086400.0, + "37": 674456896.0, + "38": 1722086400.0, + "39": 635463424.0, + "40": 559496704.0, + "41": 1722086400.0, + "42": 1722086400.0, + "43": 581694144.0, + "44": 592225088.0, + "45": 484204448.0, + "46": 493839392.0, + "47": 1722086400.0, + "48": 500215200.0, + "49": 499718016.0, + "50": 1722086400.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4313315840.0, - "2": 4313317376.0, - "3": 4313317376.0, - "4": 4313317376.0, - "5": 4313317376.0, - "6": 4313317376.0, - "7": 4313317376.0, - "8": 4313317376.0, - "9": 4313317376.0, - "10": 4313317376.0, - "11": 4313317376.0, - "12": 4313317376.0, - "13": 4313317376.0, - "14": 4313317376.0, - "15": 4313317376.0, - "16": 4313317376.0, - "17": 4313317376.0, - "18": 4313317376.0, - "19": 4313317376.0, - "20": 4313317376.0, - "21": 4313317376.0, - "22": 4313317376.0, - "23": 4313317376.0, - "24": 4313317376.0, - "25": 4313317376.0, - "26": 4313317376.0, - "27": 4313317376.0, - "28": 4313317376.0, - "29": 4313317376.0, - "30": 4313317376.0, - "31": 4313317376.0, - "32": 4313317376.0, - "33": 4313317376.0, - "34": 4313317376.0, - "35": 4313317376.0, - "36": 4313317376.0, - "37": 4313317376.0, - "38": 4313317376.0, - "39": 4313317376.0, - "40": 4313317376.0, - "41": 4313317376.0, - "42": 4313317376.0, - "43": 4313317376.0, - "44": 4313317376.0, - "45": 4313317376.0, - "46": 4313317376.0, - "47": 4313317376.0, - "48": 4313317376.0, - "49": 4313317376.0, - "50": 4313317376.0 + "1": 4313446912.0, + "2": 4313448448.0, + "3": 4313448448.0, + "4": 4313448448.0, + "5": 4313448448.0, + "6": 4313448448.0, + "7": 4313448448.0, + "8": 4313448448.0, + "9": 4313448448.0, + "10": 4313448448.0, + "11": 4313448448.0, + "12": 4313448448.0, + "13": 4313448448.0, + "14": 4313448448.0, + "15": 4313448448.0, + "16": 4313448448.0, + "17": 4313448448.0, + "18": 4313448448.0, + "19": 4313448448.0, + "20": 4313448448.0, + "21": 4313448448.0, + "22": 4313448448.0, + "23": 4313448448.0, + "24": 4313448448.0, + "25": 4313448448.0, + "26": 4313448448.0, + "27": 4313448448.0, + "28": 4313448448.0, + "29": 4313448448.0, + "30": 4313448448.0, + "31": 4313448448.0, + "32": 4313448448.0, + "33": 4313448448.0, + "34": 4313448448.0, + "35": 4313448448.0, + "36": 4313448448.0, + "37": 4313448448.0, + "38": 4313448448.0, + "39": 4313448448.0, + "40": 4313448448.0, + "41": 4313448448.0, + "42": 4313448448.0, + "43": 4313448448.0, + "44": 4313448448.0, + "45": 4313448448.0, + "46": 4313448448.0, + "47": 4313448448.0, + "48": 4313448448.0, + "49": 4313448448.0, + "50": 4313448448.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4763568128.0, - "2": 7505469952.0, - "3": 7505469952.0, - "4": 7505469952.0, - "5": 7505469952.0, - "6": 7505469952.0, - "7": 7505469952.0, - "8": 7505469952.0, - "9": 7505469952.0, - "10": 7505469952.0, - "11": 7505469952.0, - "12": 7505469952.0, - "13": 7505469952.0, - "14": 7505469952.0, - "15": 7505469952.0, - "16": 7505469952.0, - "17": 7505469952.0, - "18": 7505469952.0, - "19": 7505469952.0, - "20": 7505469952.0, - "21": 7505469952.0, - "22": 7505469952.0, - "23": 7505469952.0, - "24": 7505469952.0, - "25": 7505469952.0, - "26": 7505469952.0, - "27": 7505469952.0, - "28": 7505469952.0, - "29": 7505469952.0, - "30": 7505469952.0, - "31": 7505469952.0, - "32": 7505469952.0, - "33": 7505469952.0, - "34": 7505469952.0, - "35": 7505469952.0, - "36": 7505469952.0, - "37": 7505469952.0, - "38": 7505469952.0, - "39": 7505469952.0, - "40": 7533150720.0, - "41": 7533150720.0, - "42": 7540405760.0, - "43": 7540405760.0, - "44": 7540405760.0, - "45": 7540405760.0, - "46": 7540405760.0, - "47": 7540405760.0, - "48": 7540405760.0, - "49": 7543386624.0, - "50": 7557736448.0 + "1": 4313449472.0, + "2": 7108272640.0, + "3": 7108272640.0, + "4": 7108272640.0, + "5": 7128937984.0, + "6": 7128937984.0, + "7": 7128937984.0, + "8": 7128937984.0, + "9": 7128937984.0, + "10": 7128937984.0, + "11": 7128937984.0, + "12": 7128937984.0, + "13": 7128937984.0, + "14": 7128937984.0, + "15": 7128937984.0, + "16": 7128937984.0, + "17": 7128937984.0, + "18": 7128937984.0, + "19": 7128937984.0, + "20": 7128937984.0, + "21": 7128937984.0, + "22": 7128937984.0, + "23": 7128937984.0, + "24": 7128937984.0, + "25": 7128937984.0, + "26": 7128937984.0, + "27": 7128937984.0, + "28": 7128937984.0, + "29": 7128937984.0, + "30": 7128937984.0, + "31": 7128937984.0, + "32": 7128937984.0, + "33": 7128937984.0, + "34": 7128937984.0, + "35": 7128937984.0, + "36": 7128937984.0, + "37": 7128937984.0, + "38": 7128937984.0, + "39": 7128937984.0, + "40": 7128937984.0, + "41": 7128937984.0, + "42": 7128937984.0, + "43": 7129990656.0, + "44": 7129990656.0, + "45": 7129990656.0, + "46": 7129990656.0, + "47": 7129990656.0, + "48": 7129990656.0, + "49": 7129990656.0, + "50": 7129990656.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 73.13358, - "2": 2.30272, - "3": 2.54126, - "4": 2.42795, - "5": 2.21903, - "6": 1.98058, - "7": 2.03904, - "8": 1.9768, - "9": 2.00213, - "10": 1.97759, - "11": 1.9686, - "12": 2.04098, - "13": 2.0766, - "14": 1.97492, - "15": 2.00924, - "16": 1.97589, - "17": 2.02471, - "18": 1.9905, - "19": 2.03777, - "20": 1.97139, - "21": 2.09691, - "22": 2.00009, - "23": 1.98721, - "24": 1.98177, - "25": 1.97569, - "26": 2.02083, - "27": 1.99751, - "28": 2.05914, - "29": 1.97108, - "30": 1.98538, - "31": 1.97162, - "32": 1.98459, - "33": 1.99037, - "34": 2.01573, - "35": 2.05758, - "36": 1.98037, - "37": 2.00516, - "38": 1.9873, - "39": 1.97752, - "40": 2.0009, - "41": 1.98042, - "42": 1.9972, - "43": 1.98166, - "44": 1.9771, - "45": 1.99252, - "46": 1.97768, - "47": 1.98175, - "48": 1.99748, - "49": 1.98307, - "50": 1.99372 + "1": 71.71892, + "2": 2.43398, + "3": 2.53134, + "4": 2.58837, + "5": 2.45297, + "6": 2.23546, + "7": 2.09269, + "8": 2.17361, + "9": 2.21504, + "10": 2.09975, + "11": 2.0732, + "12": 2.12061, + "13": 2.19105, + "14": 2.07416, + "15": 2.05962, + "16": 2.06449, + "17": 2.06534, + "18": 2.06832, + "19": 2.09788, + "20": 2.06188, + "21": 2.06072, + "22": 2.0677, + "23": 2.05993, + "24": 2.06692, + "25": 2.10922, + "26": 2.06561, + "27": 2.06369, + "28": 2.08584, + "29": 2.0623, + "30": 2.06367, + "31": 2.06523, + "32": 2.06471, + "33": 2.06243, + "34": 2.05839, + "35": 2.0663, + "36": 2.07558, + "37": 2.08622, + "38": 2.07519, + "39": 2.07009, + "40": 2.07146, + "41": 2.09338, + "42": 2.08324, + "43": 2.08632, + "44": 2.07644, + "45": 2.0922, + "46": 2.07436, + "47": 2.07246, + "48": 2.07957, + "49": 2.08348, + "50": 2.09287 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml index 5c39bcedcc7..bc2b4a5ab7f 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml @@ -5,6 +5,10 @@ ENV_VARS: NCCL_NVLS_ENABLE: 0 PYTHONWARNINGS: ignore NCCL_DEBUG: VERSION + NVTE_CPU_OFFLOAD_V1: 1 + NVTE_FUSED_ATTN: 0 + NCCL_ALGO: ^NVLS + CUBLAS_WORKSPACE_CONFIG: ':4096:8' MODEL_ARGS: # Distributed args --distributed-timeout-minutes: 60 @@ -29,8 +33,6 @@ MODEL_ARGS: --exit-duration-in-mins: 230 --no-check-for-nan-in-loss-and-grad: true --no-rope-fusion: true - --cross-entropy-loss-fusion: true - --cross-entropy-fusion-impl: native --manual-gc: true --manual-gc-interval: 100 --recompute-granularity: selective @@ -129,6 +131,5 @@ TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular METRICS: - "iteration-time" - "lm loss" - - "num-zeros" - "mem-allocated-bytes" - "mem-max-allocated-bytes" diff --git a/tests/unit_tests/models/test_mamba_moe_model.py b/tests/unit_tests/models/test_mamba_moe_model.py index 5e116843225..770bc312aeb 100644 --- a/tests/unit_tests/models/test_mamba_moe_model.py +++ b/tests/unit_tests/models/test_mamba_moe_model.py @@ -254,7 +254,7 @@ "window_attn_skip_freq": None, "window_size": None, "fine_grained_activation_offloading": False, - "min_offloaded_tensor_size": 10 * 1024 * 1024, + "min_offloaded_tensor_size": 1024 * 1024, "offload_modules": [], } # Fields to ignore entirely (ephemeral, environment-specific, very large). From cc28dd74ae830803aceafe1bf8f1e0924b66e417 Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Thu, 15 Jan 2026 01:48:58 -0800 Subject: [PATCH 46/47] rename group_commit Signed-off-by: Hongbin Liu --- megatron/core/models/gpt/fine_grained_callables.py | 5 +---- .../pipeline_parallel/fine_grained_activation_offload.py | 7 +++++++ megatron/core/transformer/attention.py | 9 +++------ megatron/core/transformer/moe/experts.py | 7 ++----- megatron/core/transformer/multi_latent_attention.py | 9 +++------ megatron/core/transformer/transformer_layer.py | 9 +++------ 6 files changed, 19 insertions(+), 27 deletions(-) diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index 2ce2902926a..be6917bbd5a 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -11,9 +11,6 @@ from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( FineGrainedActivationOffloadingInterface as off_interface, ) -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_commit, -) from megatron.core.pipeline_parallel.utils import ScheduleNode, make_viewless from megatron.core.transformer.module import float16_to_fp32 from megatron.core.transformer.moe.moe_layer import MoELayer @@ -472,7 +469,7 @@ def submodule_combine_forward( # Delay the offload of the mlp norm until after the mlp_bda has been computed # because the residual is needed in the mlp_bda. if layer.offload_mlp_norm: - hidden_states = fine_grained_offloading_group_commit( + hidden_states = off_interface.group_commit( hidden_states, name="mlp_norm", forced_released_tensors=[residual] ) output = make_viewless_tensor( diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py index 47bfc0518c4..9fbc657d574 100644 --- a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -1234,6 +1234,13 @@ def get_context(flag): """Get the fine-grained offload context""" return PipelineOffloadManager.get_instance() if flag else nullcontext() + @staticmethod + def group_commit(tensor, name, forced_released_tensors=None, delay_offload=False): + """Group commit the tensors.""" + return fine_grained_offloading_group_commit( + tensor, name, forced_released_tensors, delay_offload + ) + @staticmethod def mark_not_offloadable(tensor: torch.Tensor): """Mark the tensor as not offloadable.""" diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 14dce873b8e..217a6937b13 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -26,9 +26,6 @@ from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( FineGrainedActivationOffloadingInterface as off_interface, ) -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_commit, -) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.mappings import all_gather_last_dim_from_tensor_parallel_region from megatron.core.transformer.identity_op import IdentityOp @@ -814,7 +811,7 @@ def forward( ) if self.offload_qkv_linear: # `qkv_output` may be a tuple; commit supports tuple/list and will keep structure. - qkv_output = fine_grained_offloading_group_commit( + qkv_output = off_interface.group_commit( qkv_output, name="qkv_linear", forced_released_tensors=[] ) attn_mask_type = self.attn_mask_type @@ -1010,7 +1007,7 @@ def forward( core_attn_out[inference_context.padding_slice] = 0.0 if self.offload_core_attention and self.training: - core_attn_out = fine_grained_offloading_group_commit( + core_attn_out = off_interface.group_commit( core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] ) if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': @@ -1034,7 +1031,7 @@ def forward( with off_interface(self.offload_attn_proj, core_attn_out, "attn_proj") as core_attn_out: output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: - output = fine_grained_offloading_group_commit( + output = off_interface.group_commit( output, name="attn_proj", forced_released_tensors=[core_attn_out] ) nvtx_range_pop(suffix="linear_proj") diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index 24c87c52ec9..62fb7a148c8 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -28,9 +28,6 @@ from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( FineGrainedActivationOffloadingInterface as off_interface, ) -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_commit, -) from megatron.core.tensor_parallel.layers import ( _initialize_affine_weight_cpu, _initialize_affine_weight_gpu, @@ -674,7 +671,7 @@ def forward( permuted_local_hidden_states, tokens_per_expert ) if self.offload_expert_fc1: - fc1_output = fine_grained_offloading_group_commit( + fc1_output = off_interface.group_commit( fc1_output, name="expert_fc1", forced_released_tensors=[permuted_local_hidden_states], @@ -755,7 +752,7 @@ def glu(x): # Delay the offload of the moe act until after the linear_fc2 has been computed # to make sure the fc1_output is reloaded to GPU before recomputing moe_act. if self.offload_moe_act: - output = fine_grained_offloading_group_commit( + output = off_interface.group_commit( output, name="moe_act", forced_released_tensors=[fc1_output] ) output = self._apply_bias(output, output_bias, tokens_per_expert, permuted_probs) diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index 2cc083a300b..d25f417fc50 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -25,9 +25,6 @@ from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( FineGrainedActivationOffloadingInterface as off_interface, ) -from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_commit, -) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.layers import ColumnParallelLinear from megatron.core.tensor_parallel.mappings import ( @@ -255,7 +252,7 @@ def forward( inference_context=inference_context, ) if self.offload_qkv_linear: - query = fine_grained_offloading_group_commit( + query = off_interface.group_commit( query, name="qkv_linear", forced_released_tensors=[hidden_states] ) @@ -317,7 +314,7 @@ def forward( if not inference_context.is_decode_only(): core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)') if self.offload_core_attention and self.training: - core_attn_out = fine_grained_offloading_group_commit( + core_attn_out = off_interface.group_commit( core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] ) @@ -348,7 +345,7 @@ def forward( with off_interface(self.offload_attn_proj, core_attn_out, "attn_proj") as core_attn_out: output, bias = self.linear_proj(core_attn_out) if self.offload_attn_proj: - output = fine_grained_offloading_group_commit( + output = off_interface.group_commit( output, name="attn_proj", forced_released_tensors=[core_attn_out] ) diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index f9797fee402..fac023032ef 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -538,9 +538,6 @@ def _forward_attention( from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( FineGrainedActivationOffloadingInterface as off_interface, ) - from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_commit, - ) inference_context = deprecate_inference_params(inference_context, inference_params) @@ -608,7 +605,7 @@ def _forward_attention( # Delay the offload of the attention norm until after the self_attn_bda has been computed # because the residual is needed in the self_attn_bda. if self.offload_attn_norm: - hidden_states = fine_grained_offloading_group_commit( + hidden_states = off_interface.group_commit( hidden_states, name="attn_norm", forced_released_tensors=[residual] ) @@ -755,7 +752,7 @@ def _forward_post_mlp(self, mlp_output_with_bias, residual): output (Tensor): Transformed hidden states of shape [s, b, h]. """ from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( - fine_grained_offloading_group_commit, + FineGrainedActivationOffloadingInterface as off_interface, ) using_fused_tp_inference_kernel = (not self.training) and ( @@ -779,7 +776,7 @@ def _forward_post_mlp(self, mlp_output_with_bias, residual): # Delay the offload of the mlp norm until after the mlp_bda has been computed # because the residual is needed in the mlp_bda. if self.offload_mlp_norm: - hidden_states = fine_grained_offloading_group_commit( + hidden_states = off_interface.group_commit( hidden_states, name="mlp_norm", forced_released_tensors=[residual] ) From 871bdafb6041212cd14c5455693001401d8d14cc Mon Sep 17 00:00:00 2001 From: Hongbin Liu Date: Thu, 15 Jan 2026 05:56:01 -0800 Subject: [PATCH 47/47] fix test_mamba_moe_model.py Signed-off-by: Hongbin Liu --- tests/unit_tests/models/test_mamba_moe_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit_tests/models/test_mamba_moe_model.py b/tests/unit_tests/models/test_mamba_moe_model.py index 770bc312aeb..c4cfbc2e56c 100644 --- a/tests/unit_tests/models/test_mamba_moe_model.py +++ b/tests/unit_tests/models/test_mamba_moe_model.py @@ -182,6 +182,7 @@ "mrope_section": None, "mtp_loss_scaling_factor": 0.1, "mtp_num_layers": None, + "mtp_standalone": False, "multi_latent_attention": False, "no_rope_freq": None, "no_sync_func": None,