From 154c2d8fa6cd2008eafe3a5906c921c62c97778a Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Wed, 4 Sep 2019 12:23:05 +0800 Subject: [PATCH 1/5] feat: Add suggestion redesign doc Signed-off-by: Ce Gao --- docs/images/suggestion-workflow.png | Bin 0 -> 39072 bytes docs/proposals/suggestion.md | 418 ++++++++++++++++++++++++++++ 2 files changed, 418 insertions(+) create mode 100644 docs/images/suggestion-workflow.png create mode 100644 docs/proposals/suggestion.md diff --git a/docs/images/suggestion-workflow.png b/docs/images/suggestion-workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..b695f78c6963ae067666830c3a865999215e96fe GIT binary patch literal 39072 zcmeFZcRZH;|2}>pWrVDV5JL92tn9sIOB8Ws?_|%ivMciwa|=-{E#$DMua9E&U-p64jg9O zkc@{Wuk@`l?zy*6%l7uENIh5IqL|7K$I`AjA+g6aNGQ?V3 z>}giNHn?>3OcW~YXJtQmRu*&T1$+wnS!LteBg8(gzuDK};e%<;x_zmfmzt$! z?TMZ|aJKFkbXL;bsgk#8a!EgL0mIkrs%ubFf&21kd6(&Q);O;`zAxnLw`kRt(LHjV1Gu-r-@Yfo|FQFRdI| zaw65JqNH@t2E58IjvB2*R_!cKExopF6bkO7k*fFeu~_UPImPh}6Ie%WN4X`!NAM+H zy`<0jwF!&^iQ9%gGp3UwbIdC2YkZ`L1ZE^TtfQkNBr-BGetBiZUPwsD@YR<`VXDko zngOWUDhrI<+8=zpwTj`E@8hj80`IA3vqIE;kBg{@uJ1ea@jkpXl{sVmpuqgLS7Tmm^LD{vNoAkAIw^a*EyCky_v$GD<+Y&Vm&QWz3!yQ#%%5)`_$wZ{ zuk5J%n=L+BVBX$d4zZ8&V$I4n_Xz>>%IH;2cBtBp8S|pVT|4zId3bP4VL9;<#Isee z%Fgb_rvh0ToYcy-b_v{^t5>;i_A^6cnb-IT8Izdt^1YtM;Gi~Juc}+Fp+2svt?z)9 z;N?M+V;D{J2!38do;ihRSW*3vxW-5|4O`j%MqvcZeziCTuGD#~xnG|F@42r#%4LQy zW9kUvVNqeVZ-NCi)#ZsY-buY>m-Y&tkvq9*#sS??$uuuMs8-D=`$D7gmJ~y?Klcld zR{ssC4kqeE6+g+0c;?H_nwa;%Q;qhp{Y&HA`b*fxk7%`wyQx06YL1zz>W*=QYPv9& z>Q2#m&Do!o7T^WRh0B3}S=SUG*rpLAKbR}R9v^o~yHe*PNzTJYwHK zis)=EjYw@;%0wEcncPY#wSQ*leQ4a!bF<8Lj^&4Bt)YxI+cA3Z$81j$ld50}&5a{<%4&c-Xpz(mv~P_fQ?T;mQ`vEJtz} z|H{b2zNAcKjU%aVnr(c%f=i1tJrB*X4Xi5sFDsxWr^p}6Dl!BW>k_MQ2KVdAg3Tu)X< zu05!BZ3e$p(XU(jcxQj#o!%`0-SDJly_|8os-kUas0<`i1XTaUk6WJpDxO2H$wz*BP$ZFa5>f*BS=iB(*X)M_C)aGoR)vD zIfA6PO<{xOZ))nLeDK}#IYqH-zC1af3sH^NvBT+C=?nPyO)sCZHoh}1l5c8W!F*l! zgc5ay$P>Xvx)#(9VV>1S4cI;?7GlSc&3~LZkYx#dUG)SPb$a<4(F(x zviY@F0?Su->0?v!`cyr;=7*8FXWPB&G$f(@me&OLzKwhMnzwX3z=ZC=HmC%7oj&Cl zc}39m)7yM=x}YYqabY0KJhANii;GFFG!m{+V~swU*SZ~C`ncAsX&w$*rL*@M4JJ(!>ikq!NkLYE*rOk@O@Pk25s(Jj(st#aDf-6Fq`Eo z_A+M3r6Axe+N3wQZeHQ6a=l9NDA8=O&sR-qKsShwQcOBMAL6Mtm(g7s@xpmom?S^f zJT${ji=ANKi5CG%>LQ{Ybmiw#n~hrkB|1@lt;1(^CuB0Kmh`?DQ!uP ztt|`8^TDR(aEwZ^-ZKCPYduv>Q?@mzmO4UUYK&COx6X%bImI#ntglxI11=r~?g(b_&DTGu_6)X2KOCQ;M zpv80f@!BtW6slLui_OG&O8d2l8@kgR$2KI|t57zIg0szVcScKtUVZ(qDtNm11Ms(& z2;J;sFNFIDB6`Coup{Cm^1G4DOyb|gzAWACoxZY4nG49gBnvNnpv&HJsJ{) z1wUgI$F?lmbtjEBQ@IJ1)tK2d>li*hU27z)sbBYgvNeAZN$z!A6S&p?0$j@1=6B_->&_kg;ELxTl;@^~nlm$wDWF=&ulGTi zVPO*x#vL3UO`Qa;CAF)-aqzY>LtFA2DJX17ysYnatB@W)Cb;Cw)~|2B@s(FZr(W!I ztxZScuy`<&+jJIva%;ez;3;>IR=nM$c&k=daXJu-^ zw~_gMwvFaU@<@_=mfHG{4aRQvt3;EfF>NtsS-vM2i+y?icyLeW(>toro$k$Y2djSl z?UT2h^>nD|L|CpbkQrqfL>XEPb>TP3*M`u9t<8(>CVHHJ)G*+K;U7G?qcN?5IzeSI z@E%^(svg(b6~9raOXCG1N|&PGg2J|&&>etD8z7P&wwzLVv}=RNelzGDdZ;$o5_uvl zV`kN`iD@yMpFLlV)_{gZll^nqb+xoIb9|CRg`5|$I&V_&XyL|n4hmHoc^oh_uw65% zgWyr{fBe{hCRHi}wQm2ys_6IbG-C{zaGaKnDX&^nxzZX+RwvS>a1?%u z3)JbiM6_v>qIhZJ2w3Z5%RFoJP!QV7W3Kw!w;deF4+*Y^(;CyuERcF4UJa~}cp!Qz ze4^%-lb#QpmF(l=_=f$}ZySQM7&7N(K~)gEBtH`Fx0k!|fWV=sN#0jaiBwm|rtsS#9#ORjLra7D!4Z2-P9u#@5;*h9b|5QJ zSDmz{@o@aNUKM_=zKnIa55MYUNAb(VA+iz`o`K5jDBc$x-IM2FBd|KFWb1EV%Y==~y<8-$YiE_H_1L)p z-2to2>uM9C!)&0D#p27DKc=OIZT4m8u{4F(<^kg;-+dZdBQOlF;)n1tSW#ZQM|En^ zcp%vOAIXPB15m9;h_c?nHT~d=c$5NL&1HkC5PF|re8FYsu|`<{vm zetF&X!P=a5rEg|)PC(3{o&s#vkCd;3qV}eT-IOpR~8XVS~8;-4K<8 z=2yEZ+Y>7#N@v*vIstZGJARwDW>z||WNe$57gwksYDcm>6d!`zjd*#tm?kiP(>zuV z87Y&VUlUwuS6lTAE~N4Jdi(VxWB0E-VBb(nIX?8CqrGw4}e$t-EqUA z5>8?o-#QX4@z-=eSa0!St+6MhSGGM*f%9n2epb1MJ)paxZG+{LbQjWD_xL)qHhkM# zYs^;Dkvy6Yl@qcU#`S&5VKjy-EB;KWUPPzqYqt1tbw*!_KMmo?!C zzyeTA(d+#>@nk~ebtoEI=wMfKEz{wp3t?tkN2?Rp_75?Q)fEI$r%IN!P6BLL9D{eD z%Zl)XJDGq+=YV|cAkG2}yAi9tz3ir$3j70PS2ZyMK+aI@8gEcQ1O#^Zl@rcT!Qq75}znFVu_xPwqa*j^7B)5kS`Q;c+@|N%T}vHSR^Q z59Txo9Qnx5jH8j05Jf%>O(!SfmzvMZcUxvnxg?vvc@O;vk32w>BxoT-N#lAP?*7%OSf|Bi4M4 znibg-vU_B*meu&n0i6(<3qcn$D(&rW+Nu!66%Mao6RI21;GHIiGjBu`05w4<_PC(K zv?_xK1RU}_;PyXk%+Ge?iD7t6KtEm*74@VuU8?`qF0@N!_{5iR^XX%i*J6ZJLaq{f zcdKCrySgO#I4CNLBCV)VBvhJoWgC!rRoTU^d&62?X})99h}(lvNQtRO0wlYjHTbPk z09_h=vS__!qWEEYPQJ-6#@a44J^dq!+jZf7nS<3+{AA&WAocQSm4}q;h$P~zi(bNM zr5vY#MqIW)98vsVGs%wngy9>j)H-tXBMhYQ;vt;y*=8mst*8q^uvc*nFIK z0iMV5(3Ui+3N6KXo=xQFtK<6#fkNj-`N%*zUi6~CsAqJ%QhF_Yu1OYS?t?qjRMtgq z3d%k27@H_KKgZ!4mTSw8`ZiQ?*MRJ@QRAe_TQ7EwER*T zx9GUHGAd;3-Xj-ABW;aMUEUY&7;QE|s%Af#UA-8;kyv$K2{;*@FW&tcjr5=j5Tjuw z(PMtW=4yXk^p7FDxl6D*Y9vdxmm)=f4+_Y2V~+2QAE!9m%l%|j!nHTqpyg<9*F=Tl z@W%jpWb>An?cDffw-|6~A!;%^yZEBpNCIy^1F5&TB2-?KcUhrRCo<3b) z-$ov7iC2Webh3I3PyxKyR|A%q?F06;iWVnD>G^8tW9b z!mmRm_3A8|$sAU??(NqYd)mcPOdO77!WN1vz@9PVu@9R6NCQc6xxM~omhH*zl@z-! z2v9D$ySwc>VmYA^Xx5h3U&K}9m^!y<>m$itl*iHvpLx>SPFEUDE?iWDHLdRS+bt(7 zdEX(;(GcJ-tYY4n+2!RzuKSWjVp*cVpKr&*)W0oe!x~W@Q}Q)Vz?x4hM_DaB2W&Xz zFqqaLR$t*=vSh=>fZpsJnI-GGiK>dP4ywP-hg$POS^Y=}!A`0V1$2&G_GSKYq)k(h zOXKzkaH?+QtAy8nlX)=t{(WuQ7Agp=t;$};^>p_DH=ymocAhSZwC17R(SH>ulp6`R z8c!EBV?R`9Jt-fQX0S)NEVs&xRymhbWV0KTe*&UKXN&{RJUysl%?yiWw;!8~gq zi@o1&S*97vgTn2K!t47voA8)fMsy;<7J>N^vu-8m=Ln@xEi!+S|*n4vOclK{F9Fv|} z*o>0{)zUxVgQJ6N?|X|OJUKnbF5++oFk8mGue{z{d{q|QTkca0yaGTE-+aXrVNS?n zhLTAPAgCVMj!Y&mI0Zh@8t=ZF<&m|Qm{&(#F#yUMiivXbUaXy7nz9{o?z>>NmYS>cG?jPFcEQ zuUVs$)S+1u-S36nS*X%^7=G0W+ht8eVvAojyZDClYjdw?^pPSu)K zOARRo6Py^Bp%E~$D3kg1!SL0V7jR8qS0L0u{wwyjznI8tODD3GZS5PF|F~&>YBpmh z2{slgIrN#!zokQ6p+RNX1YkYts_zo&!6T?VaD|-cIoYT%u7D74drZkYCg5o>k;BeJ zPR~+XQf-Oy-3nL>vb*JIso4AR)>D;M7az=pvEM89L=biux!=m#S?ncn-^%^6z#S? zF{Ivj%h{gy@i61L6QvJyS5Noj&Zknf(oKCEsWZ$d*%@<{T;y_56a`RMyKbj{DOKj^ zz=U@+(s=799NzGcll;kgHMw^f)Dawzr^G%_8Wqb z$;_B%8rODaZy65*rT9FRlGAIz9Kl-;`-++(W-D2mWYa}*K|u+yK}Ir-OxPZKIrtAc zBUKW)3F}Vl{+Q#b2L;Mka#cL{Q|>1=0))bG{U=USCaR@L`mBEt|NznNqfY+ zUfVgLN)P9;Z)rJP;&ivophln7yKhKwsEm~WD8FFfW=t7g1me#==r+X!au=41mQu0f zDPle@Q&tO9cG^U|=U>wkB36Gg|6Ou!OOn1N@*{8wU%Yc>rx@WQ87=o|ULw`#jJe$w zQ*uJiqs446m{H;l>)u5PydRuQsBHrq(x*fTHNG4qhhMnK%>arf>sAP$Zrt8}aT?lR zxtmcIYNqtS&JajzH;=P#^wtH-8+HhXJ*YLf4?n;7E1aLr4`kGkSwM1wEKkI%WQGORjdAvEbs#`@A^m;)ztD-}L*;J))b9Cp2KCspe0re8Q+ZJ|cmcvt)8d|EckeIv({OyS) zvU5ox?!a}ma9@fwiu7lYcfYIRTsZ+NG|#$*YvgO>wzr1ctd6;x(m@)xCR9uem^=z; z#_DQcmX&xRiWn3_-0DuYES7q%(lGgk0|zQ2Pzx7?5t?=7%}gK`QC4LOnJ^ACz|w~a z2$L8B@1u}ZN&<5@^uBP_6yWIPVr2PW&FQx`w&Wfv+yIAO?>(I)YCx5iopNP1{hEEk zJH7VejrV55Y3@9VhQBMB7*?sRe0Wi=hLG9o)Juy9ENg~$@NPyDB{(rzHp&fQtDK42 zyH8?`mCpvgCNQ_pWo;Wfb*gEgPKzxFu%1_-9;w+E2hZ3SxXNafW?IEUiK^PAwD*O) zns!*5v)aahztkl$u|D~XcfakZhdC%UD68^s~M3;Vw6yS25qm1hezSh{~iFjg69IY~h zjxfA|y%by)gAWSCH`ot8X%kE7k%1C6Y^lae5Lel8{G)nt4W4=u_Y5c zyMguZuNNGB5iEqc%J;#?_B=%mf+4kwOI$2{6G>Ac(iTO%9*wnKi~s`zpUX_8pe&ZY zagp%t?_L1xTl=Vo3~gQ@dGb!_LcUwp&#S;#H?cu2_xxjQ5V@2(bn5Z%|NInVA1|qy z2yq+t1RBXJ7ykX#hyVY2SpWY}{@Z5#e`;0qq?E7d%Gxfekq0m8iKYJ$Vh_p~&_xTF zso2u9v)eIf5fwX3;nL1*sa+RHeuW`Aj?g^@bRChNr)?w}-wuh9=|XX8pk$97D{xwD z3Y8trC3;%*Dchf+>^ju!GC7P~*rlK=jQ|(7>G)s^4MIR8CZnfE?y9g|T0ucu)=K|PJ})1_pXV{)k!1t9Q&Ge7hvCUx-)SK-^71fb z_%(>1bTGd0##D2zQN5VmSPeGhd01G}xbIoS#Kc1tI1i?yqhkWpJn|vr)vH$%6BCeZ z&5-x+1x4J}A>S(P<=xx_$GrC*>>nM`3ky?9e1IqN+fqnLNlnbo2DP=R)FSEl_(!-&703Z7QUX!|;6J`#MATWg0v07Zn=Z+5KRC5?wOj6#^kRC+B&W=MwL}Z&Htglbo z>u9AC48LK%I};5eA9=muooQ>a+lEPl3RwP+?d^CVh%Ts)MPH?$025mjk&%@}gFJrx z*kQKqnno17?%4iY1s)+GVWZ1cHob?T;`tzc*xsfx(6tgW%AZ0Yg1s`p7zkM^vxu?z&t5<191xYI_BjJIo6qTqa<%N-PPA%l5BD0m_xjeQl zzSsL2Jwj+|Y0*Z1xk5=92Bs||C)YSWZe%{jGAjvHwMA^toW&dYG;DQ>M;SK-8E?(D zyRA<=9jmz;OEHj|i>}?WiLENp@N)u8X=%|QSv1pAv6&(uX=Xc!r&8f$;M3dXk8#+WUCsE2*Hc@Dkru zD=20R!OGHTwLl!(J#+{iH1^dibWTprZr8Q%A2Ol#q+4mM?dqpzr$_dW%*^O++<=IB z?)D|^l=h)gwoHLa!EE2`z$5G5)f?#7~yRCkhdy$lc3)!Db_3bLS z2fa+r1(Enr0=&bGogGp*GXVj?kGVN|PELZ`w{K?{KfZVGo?eLwreA(i9xfTj$KGDz zz12|+NTIRM?(Xh;mlX=g#PV<{Ik!nbM@_Q0?^rIJg_*e=gRI5>^P#!XzN zUrF9qS2uNb63EKRLi|2`lDS2K1HQh$cf`1M?HU$9wjW=jxlub@=S>NIsjj;4~MoqSJOZ*%VAjLK48Ng+T2k>)p2`yWEdH zUnxxY*J+TN!~D$-PBn>N&}*Ly za^v>xYoJfAe|VphnOXAQJ-DKrBuD&RtH(hAegNWz^~`o8hXE8$s)<@+zrfRJ63T7T@1%7rT;4(&^nwsjW zv^VM&PUC?>35kjQ_xIgoWMlxq^!*smF=P@GqXrM0A1a}LS6a$#+<0Moz8j;vyPI*% ze5A}03mf}sQW7P&6%C@OsOaF}0J59R{rlHJr(pm3Z3%56)IJ3r{7l<3#y(FjX`iEE zLl3dKHm{Zyl$ewh6Y}}Xmj;k#)3jM+&8(IN3L6ifK79%rLW|)tUmRNXrdPc=! z+zM-nU2Iz)$U|j4yzR)Zv8;qtR1u2NjFW+6h95yN$uQ;!iE3@lb&|3Bu1}yuC8*hEq$xx#HSX~ z%R7mQi<4rE#%=}0N&yrU|0iwF&5CQf2CL^ra%fKp>wsnlWQNH|B$e>Q>S`niH#A5_ zMh2ic96Y!CL2;Uno15F=@#AMo*+eVyy()MB5mS8*nKd;vCuU|C4o?A;shoO~iqiMySmM0ZzbEfT_mwUFq-L zHs~NX=;<-uzI}^@g(dM1NOM>l6TN_mX*^VHoWNs>^L=!5a&9g-BZJP^vh_Bi=z~78WT$M6Xj(y^Mh`^LxkXhvw$N0PrCaCMI+wIGEsz9JXsFWOObMb0vSuHNna>B1b=N z7v;`(_2e)n)o>}~(A^A`K04pi116y-^L}S1YrbaVE~cIz3t|(6N1x|Qp&0ca;*n#7 z=qdHUkE>dggAc=Z9?h7|^Y`*=xst~AuHX22=3S6Cc6`FpP$D`C#Q)aDKiORe z>JH}zdVPq+j_Df(m!0BTDRvp|o4kctEIh z`TE5RzjWzflePqIkQSvMJ-P{kKWKaV$@};3E!vgzbo!bYRjFPLzq!l#`nHqY{_E&y ze@Xv~##5g{+_u^U+F#{_>gf~zl-`Z+5=Jbtf-)m8Ak)Z*rcWWnvUA8bIFyivhCpJE zWUW7)hu9_-EXpS@A=e|crKQI@{l>OvJrg_Kdyi&O--5f z6&jEU*nPi}OgUI&)N*=qgkqs6>U;C%1v@*t&mbWVw6?X$sHj|?nw|zkl2%2F;JrmU#-n636xm{i}GT(n;R;U6Y@vpt6u=u^#5m}tU38nXe zGPuTWP{He#aw}Hp6B3CIdFjIWK1C_J|4_B>oBQPG7g}j&+2WJlNA{8QMy)Al-en|{NF z96~PcBQoYbNA)uOd1$EAP%9e8LK?uaWGo|@CA$@>pV$<=vckhZ# znsFd!_*>PkYt*QN`JmcmfsbEPKnN~#PIyi2ZAdenB z!ip9H8NdY4zT(nSX%mx-Vi7?>@@OVCBuXaiEb27GfU9n>v8m{}9PQd><>X|7#6r&V z2rD!!Y;tJ{2rzvtIc4H@Y^IK6VWz_x$7wkp)RO}qrQ2zTNIaN2_m#GL#uA(Fw+xWv z-v7suo9{@GNekvXcbD4ova=(?_hoP@U5A!yc>r z->5k-!P1O<_`*I!7BZ`U`0=MSvNT=@qc^5j(0W$Pk!!~<-Y<$kt&p(`o44c5HVjLp zD+8Eo&Q61GA2@FQP=rW$cnICUe}DUq_t?*qjX-W`kHVBtC$L}k4P`m9$VaA7B9SP- zS!49;NY4Q%R`^ZsDf|@E1?Ly*U$Yn@{1X(_mP{_h)VJQGr9}hi-wZU_m8(~SK7UrY zF6`2_Ox6@iW}$W4VFCh#%#CwF#VV6?C)r_ZR<(3ai>BQ9(f|o>6oaL{bAgUlPbY7I zPQBIL0m#3(6bX=$pVB1#yWW{9PbS}?myZm{$zcN2-Pz~Tzqp+nc%x!t@l8>NY}nOC z7myXv)YQ~a{0(pzz*kY)-v1H-pi11fBfJ3t0VtsgbU5VHdoE_gB5yC#`T<0*)C{kt zrUpd^E605(!P^UZCYc5b@f-0(k*aU00VnN~>`Nc@YQl0su3tB@(1fG?rK#2$$!sO+&)sB0MxA z0u%CMb{6DW=`xklL8b93Ac`z{vasz&t1uyY#ShU8-DgOnqwJldEKSppRH@)in4h6( zrUU}v{d;nd!01!Twymj;VqXQwCnu24z=yKo@D4A(|G-@W6xaOo24YYa1LK|z)X zuw`UmupC|KtE76MZS$TvR%!BVZ`_y0d9kknY=r~R=VkWe)ByeDH8qL!%dG;ji5YKj zaNv6%?KT5K7`O)aAgcrm2_h(H9-j?j1F#anYl4t7i=pp{^YP(QQN~(JOA9?fe=1=Y z85b9RMwJ9VYEh3HLP8@yMCPlS3^w$5kSTPk+1YtjPn8ch@>0#<1vq(mc^QWFAR@a< zsJTrph#mYOhDgZB1f&J0uK_48FsQtfWeg}cuw;UR325kecr+CTO%Z+L?B`HR%>Kp# z@uV>6_3O*__V&iLz=F{)f@c`Z-C_~_lamYgR?UBzD_#Oz zTvrYb4um+nkgFNGH-R6-u3rX-egNS5fUV26_T!~~|9+p%ph86*?t&RKGXxcumMhfkLSH`R4!^(IEh-gVWM(bnmr9(GWm}-aFD?zkYrFXssS) z?eM@a13>3xCOBNZxKxdMCs9dSOZtV+K#`I2(rio&iJQB7NaP$~t^sF2uj9J9x-O4a zlL5|k5kh_x`F1V^2NxIkAH_Z=9yZ@9ROjsT_D6w?{Wv^)Z_Q=rd@*QfXu9o-b)m8- z_+Tut1a0L&t({Nba(Q*`KRNyr!?Xz796HjqXPv3yMrFqh{+H%^vT=`3PBuEl&#)GN z==R)QCKNk5I)e4*>*!R4y%|3s%p+`-kAYyiAM_mlFA zXxC5{6iv$gn&lMCIO0&J(vE|f`RTtHGVS3CxJy1&%zM3s#usu+ped*>)1C+$Oy<$i(F2)8k_=KvpM#p=rjgMaFUe3f>mTinMTM6y;$ptJ2^!tjTp!7%DWVbeL?! zOeRy3b`^JB9zp}_n7%;-3_8H2j78g#K_%6>q}@ASa3F*tqjb^y4*>G;oeXQEJMz}vQhx3YL?jGP&{&PJb@l&wf_FA@@NsYnoBx#9=xw2)mb#|Hpm@_|>C zz-vi_xRV?z=6(24JKs}<6$4Lr>dzFk@Y>G=Xm+=`WLE$cxw*G)GBH{6QA3(s#kI7w zWL<;2y}f1e1lp^UfnOxIR%4a%W-{)@iyORL^~amas51^+C-qw-jY44iLAax*iDu=# zb9LnFRZ>z4$7kjlaCoCa&BFg3Bnens1aE8{B2zAcXeN4lT3NiE=U(>Ls;lSXWn!gr ze85Al2r*RaS-I}}_%RPahliW4Z=6I}aYe6mGV%VcOr_TP<^f%30n0@QAbx$_fz7Dy zy5DIv{BYH0tlE`8Qc{wf--aBa0!Ig1x_@wxo|}v1w{JHG5HWnrUB_v@ix!BNGP_X< zK!T)Dk{(52z&1_-&$^~Y#Ac)n8z87GFo##i>j^&GkId7`G3m)t)F-IEJIi6$EL^;} z)7aR!-XXkpgN{zhZRo+)5rAkw$Nhoz+SAj6I(pzhq1Yr~zXHqzRjcb()XK;NQA-wf zRS=X4hmHj(eA6_2fQzC);8U#K%v6b}sX0`_#Ju($8p;x-oFmOm)BgeQUm1-?SmRq? zCM8)f*!11XVs0R=NHcO~wN{=Rr^s97F1*aU{9MfbYhUbtGr7Ec z`KPWi5EACQXsNXRtZ-i5x%2?s#(!;!mwpQn2KG47h$Y?eB2&6 zY!4_9a@Y=AJYU8uz5B|_lw{Ufh^X5rFU;bZ(kC^Nt6@&zCE`2tkCV9L^c2~d*(iM% z6ze<5mES!(*3gq{RlB+Rne;k}uU1x&N0}5^LaNpXG)3V zVq@WDG}f6cz%s(85woE(3#EQON0jLI?3_6lsz@031cqs3w)be;l;x>vvqm_B5@J`M zX6a-hw&j;gR|hpkwbDH(c0Ko2X+eetdRO|%6JE29*Lkj~|91OvAa4IR_EMRdUSFjs zWT?(t807Pf)#^21M$-%r4=X4sJ@fa+h*4qyCe->w1BU;%U&MvWWmykcUCE&IWBRM+ za|3D`gez*yKT`Y{BfC{5kCy#FqA>eyP$fmrD+0E+3^#mo`YUX|T>&b}#y&5-eh+}6=x-AJQ zX*#fY-a9YFtydQ`s&p6AbNsGhStO+HPsv2^w`xXl^CvCvYgj2L`;b3;bldU^!PS1E zD@l34^1uXXK?ImXL7-kS4YGyMUVLIA4)C9V^8^GKDp7Bn%v>U%PE(HOSl`=g5kEVo z&jKZ=KOLiM-wl8z(Rgr>A7dZvC!U=W4K{ zmB7*c>d#Io!G@vCkj@lQ=T%N1tqctfGs=WrSD%ZueERee_z*0yN(z7-T)a%~rzU>V zH?fN{g{G%7p)`qRU%U8UMMukiyK$Z&^>Bqa<%BryJbC)g2fTm`O5ed0=ko+Pcxs}- zAL0iX74V=Zi|S8%MvN2{vBB%Ijw>U4kdMQq=G;Ix8y|q0LkP%T^nhyP1WKWp0;;A1 zSuyrGtN@!uM=={88yiTQD?MMJ{`sjcr+glqCzR%GDNGrEx&&@*YTidp3wQRqo z*B^F`-S_v#4wPTMe3|dfAwWg*?l=3U?ry?2x`pTv6ubhzei3%h`cLkihJ7oy8%g=I zNkBTn%CZO9{`@|NJg^3A*D5pQL%kQd`rx*h8SbM70pd6@jwo)vqJdXNQsP+*C`eZz z=iuD-tAK+FEg9;SQB=(Z@&XPINk}071V}s`9Xg;N5WlRfn}VQ+E>D{dNC8&e0zUv~ zs9H~wfW67*7ql;0XCQtI%5fh-EcE86$tx&eQ2-GF06WRk z;d>EeUyZ)94>s6slp>}zeqYmtF=M7A+SUn5RH})*l2TIWkhH8Ul#wLGsiUt?AqDVl zT&u)fY0JsZ?(IVa1U0DNA`8?tP(SkX>*|dIP38TX&0Ft%_p2Z)icfz%h$twCYU8*2 zCfRgcurEdM-1EF*2}%{x*_x8IDx(Tf5fQS$zRo3k^Ull)v@T~%r^9vi^#QCQ#O}n2 zv3<9=ry=bZWM%cx%CL~3KEQ`1({zs_5?Q{JX^ZJ`BSp;u9p}6ag%>T5RA273QG@_!l~9d$h3cZ2{lH!co7^3F8O!XVKS6i%Ol+RUNgy~ARM%xn%k z%UlHHu0GMlyzsaKK5}txB!B-vR0V0SV+#{#oW!ehCY;5u7;<*vkv_8N$ z;kE3!5YO>I>c1h?gTJeUbDgR3pvVSl^FVf%SrEeW^d$;u@%sC)QO_*7%3fXiSW7m6 zkizBD@5X>jcw5jDsWUOyvORK03aM8+@_8G$l{eyByQ`e7QAGChT$W<=1Sn*OhhIi{ zIsf!`jBEc<`vZl1Q4vv5ZuiYcK%8GH)Gr4X6c6|}vABrBw@0R?XnwZ}K|vV!m;TAg zR7AwY?4!lyadJtD5MJ$G$bCyi@=y9@;LSqRapsfI;oNp6hdEK_bi&FXWw69r;;JXL zq~!+(-c!W5o|c|?aY9MC@y0*~!%w}Ci^k%a?wiP6;m5uEoO?H2$9x1`Z_!PAwwS9a zqT(>Tq`N%!9z}j0nUjU0-X+v+c@%#CjHEKo?UL}4OW`H-;U$g~-{wD&8#Iy|w2_DP zD6D-XPip<@S!_6{VBble)Ic6)9v*0sB@Vv-OrG?S9Jfayo*N1NatXcB+n1}yZqMo|kqOVZ2ejycGi(EawMB6c=+KX*aL;}rU)mAuw4#6_4b8%xJzop(xq$is zGJzFVkS*tdfz(@SIk3%tX$WrkvEj4K+q1-lq!2qzx$Uu=-)r%v^J}o>X)`%4RN63H zN}3PPgdNWz<4`Xw?h(QY@~`;a4fMGITPA`nbHQw4ko+S&NJc0U!GkpHMNQ_&FLv*t ztVLEeHoCG zj<}IN#^ERN+;RCYS6j%dJIJdW$iL;jr2a@=4Soclk9bM_iTo&-TX%QgHTT-S+o)%F z2|L9C2IyFaBClM#c5hfN2@#UmVMQ2yM+6lUA0dd@e*I15`4&|4=wSRK8)ctKiNE63 zZTyNCS<1j;iy_iL?z!oh>;5Ws&xG&M45NaosP5QO5WvC2{Oo3x7z)&fJ`91SV`yzz zu^%vO!`Ss-!Sm7nl}7dh#S*!7zmgw?a}U($I;qY#=0*bjaM$|4kGtY|EP<#ttdrcy z{d$bu@DpoGKjQxLvHgr}zuO$BIC~Rb5?fF+7*Kp!&iNSpC5b-$I3A>r*-U3{9(-Rg zqK7@z?(aQCaj1-o5)6&xj^QO|eJcFe#JS;dp4VU>3XTsr8xOh6Kip))g)?JI|BIYg zrCm&v;o2-l5YTHi<84o{SH6J1;?n-ZC1oX6pGVaYADT2zH+|N=C;EhN@+yxOQP5Y0 zKkTZ|Dh&oHhU!n=dLQyK(i3&D@a!jkU-f70rr6f--II%0Qc}QpJ1+}?gq@52=qCw8 zrv2sY{)kKD(K;cA^|F-sl()wyiU$LGxw5<@;L+m0<6IU`ffAfu&M!bq;NQJiWvJp&6>?iADc4|m+%4s}j!w zHV@FwOD*TlMKOZtA%c#Jn~2E!iAn)S9{p zTW()Geev7x9EUp1s(^H!RJ&oM1J%*h4FZ2Y2Om)9>KFXh18k@svDd>;TvA^cNE4u= z>-UdQBdmbO!~m=E5~^y`+&l?%gh!>; zOO$E=UMQ%Mll1_F9tHFo8br6q5D#4WTEv}5z<%u03tDNAbON0R3+`cn5(w&UU^1Xu zqjU@PO`m`UrcvM(XlQ68fQ~P~vbe!wB-eAZv*Up(DqU}-_mN9La4-zKChvB#Gq|x- zRFCfGP>`dV6a+fE-hcQ&3x)PhoLX(&rv-I7=f8VOKgi}N@?auFw~I3J^1&-Vpk)FS za(W)7#40g>$}|K5tQ63f6$BawsKvanAPUOL(iRrC{2oKgotJbct3mCb4kK&WT1!(i z!&SHnrN}7wZJq^$R0x!WLk1!uB1nDwgM)(`PWop*n5_I6MEOsCHYtKA z?zWm=QtHL*H6jZ>uX!gIl^X$gEwdzk4SdzvdwJR>fD1Lw0lWP3BOa8v1BI=ZFJA&x zM;2Ff{{s=|m_T_^hHjIXHbcdddI(G2a9epY5Ng6=XU9&!>Ppkdlmn|W@WECU8vBP? zixc#vi%0t4XeQoh9W)fcKoxta#vMv6;+6)bS)9({*J5U7&Nv|@Bb(04NofRrT3Mts zh@?k6@ZWPnHC3J;{lD92dR`fnQ5;nDJZT6YZ00C`;rkRcMJ!&QbzK@@zyIDrvh{K< z-qZN_c#9LD7a)W6Wgix%yx_WwlV;bUD zP{8&QzPYhMH~_TeHaj$~{$FDBR8HcXiskCsFWd05O5R}Y&uSgsY7u`YsdX^u>^X;? zVHJ&EO+*s2a0Kq>l2cY8EMLu<>mE3BVWrCBq#@jMFO=O(rxTh z)fQ8Ca_{iCgc0RF{Y1|5W@=;VGKxdgQWtco#Ky)NA8gK`unfq=E7a7HGp+F$kblrQ z?0WkD=p{hEm3oWz)-5ccU7A3n59sv37Z(=?76lpv@V#{PTWRXI9HxXKxyoo15G_-?H4R8;N`G|t(L_9A(j2SHJ-PKVRkLj z5(*moUFTM>hxtxNR6Dtb*NU4DCNP&ec`q&=QiDq6f7Yx)tNa&s9e%Ak!ckhg?^=EL zYo2}(9^Y4m1`Y!y~Te8rHj_dl}V{gGw~-;2D4C-K&f6DM+oTmT=2hYV~wP zp7mzS04yu%(VwFQ3liWCUV%U}f$kKA=+ioYAdK>Pz5awj4BHnQ_GOIDK1pP&1L_QkKa0zL$KLQOTzz{}70n59IcSSRNl z)&F_;pYT_>TgBY{y3)rbavjHvp^ESj)mHh0>c^k$bf^s}Eos-vtkJZ~37VW#=VI-S z=psOMXq;zy&;fl~UwcFly4bN5c-GFBv_D@JiLJK7sDzlh+ILf}KXx>kRzN}SG5m7O zg)F#*@&#-<-XtH!J2wcOoa`;9k%h8DN%vLU87>6Lx7|z>xxj6G@%@Vj8Hx;oA{}l1 z8HB>Fo&)m~D%v$0@^jv2H@p#)6!Sqrt7~&LMmi5j4|zw8I_A7O54?_j4g;Rw5JAu52QCL`b3$*db%3`8gCS0n~AkNOtpmpH?wD;!W zP`BaV@Yst&5kf?piHgcnSt3-jQz?ZgWM4|Q?6Nm0Wug$7XhTAl3~6W~ON#7FC9?0k z=e&j@-S_<*?|VGY`_KDx_@g5;-}x@rbuORf9Pcty-=fz2*4EpA9;v9Q2>={V07TTl z*kQc(_gZJwpGnh8h4iQQ!*fZ|BUk}ZTMcoW}rAp`8J2T@^Yb8fz%?kz8iku zynlb-$KB#TrlJu4HL+16A zy8$Z4%D?hgR8{#v1Vr+PmoFQ+438XP0%e9Klz89=xC2Vw+GD3rhk?KyN-$${^JVMS ztphw-0VvNq>RX^#^O=}%rFFG!Zf~%WL=sz2P3j&$elX_{W!=cP?tI94YIUScR_*@iX({F8Z!ETEc+nLbEc0uL6#r%*0%PGO`#oGoto@(h2aL7W$VlxNek~y^sD)$)K1qoLGwBrElz5zlEjFw9-D>E}52*NdM z)_@SqzrLFuQ~l|a0Q4mi(JPq)XrTY*XNxT^R_ zy1IU_t1$(@iH8Equoi08-NDJ4awUrb>Rh-?!0!b`Cq($C1}_;gKf1bJLtWR8SWHRD z8t2$Z1J}21yp&ZRBzER{Z=#;6+82G>m*NC`UAh?yBO@aq$rkJfmITt1Z%EakUlzU8 zb7}=Mb3DkAp^_XFCMZnvMwear{@s*VKCq3z&CLzUuwXIrIMng>K%JQkcYcE2keC+5 z(V@q>s&d)F-lEevl7^3>w~cNF>D}~5N3X}|zJ|-Ajmlcr@7(FinueN;%&{lIB?N19 zKf@L(%B19EDh$zhYjt`5wmZ}^H5C{hYIZmYsGRVq zs9=cdymB55w)*htw{{p_1xfqv-MjsU_CS}c1WK8zj~^?WnwT-QS+=qL4R3F32^-Z< zHx29eh}>aVe31XLAKHWLf`u+O2SUWtF#_S2^j8}efL6X zr1h8ylVERW#L=*p&Qw;)g-FJ=6mEr?k?7UNf?K%S&%GYc9CryxYS2n)=65AOr_J&MSQr{>f$xZ|r}Hy89371{YzzXqCqNLNXIu-Rp0`+B&la033# zDkTp1ciG;Bl`R&|7CNZSwD4Z(d&>%USiXZ4782C3n=XR}yD|J~hX|8XEZ&-E9#A7c zjUT6Hm)7%d2|9@GhU!@VA8o$Sw0PvmQkz&eiytofxZ&NQmp*F;i&*4BXJ%%wVd7O{ch42}u%&l%LKcco zDp9nF!o)$*1#X1mnF#()I8<`s7G17JX{KSIIsHPeM>mSfBqSLWi>;aUHbu}wQepmIRjE-? z+q;*VZI5;B z#De&xF$K@)RC@I2QIq1~S4=w)WW~VDE85iCc&V8L)LB0Sx($FQ-MxF4i;quPUY-ic z%-~kHkT?y8_-svI?Gd8=YvCHh!)X#E)qjtSi15qp$zj^()P)g!rohI}Z~ItVd_MMP zB^-Bg_XN+GL@*YU`o&5}NC<wiotRR#vb7BT1A6#kw8nYcc!H%(gi7Gz490a$(5M&MuzJ8>%|eL@rY;4vXAD+QT*cSBXY~XPu`M zeCZsd`+Nl;x4o%0=j3lAs<1}q0HZEX)i>s41*hs2o} zuM%?Md~U%%G3Syr$Y&41Qg$NSRmGkYiPbeK$)1`jWp{6C9SjGQ4B1*o8!z#LFU6)S z9UNN%SnP6;Oq!oRFAw$+Kzl*D7zh9lPs*dGPnAtgH-o5G4ARKetIWth1@N`{1GXc# zMC31DmN|LyB&v3xJ%RRz#;e)UCW{C7VZ5BD?88ToN?up&`|!_6iK};ADIO$@`%F(k zEww9VA9$4eJuE*z7HjH-CouLS0f)qL0|O51?Cc1iLhJCA7(-B?!mEYWwPT}0zEX8; zS!YthArRt!eb1LHFK9hwM(A zQgwpG{Ub(?w6~VEPxG}6Ru1Ns=YJ?N4pZBGmK7+WUh|U5TEaeQQaqYlOtj0a7ny{yrQxiKunNa z&gftXd6ZmvK5*gSAK^dULmuukJ6Fxl{^5vBTA7B%9a~CsLu}ddtB|*2*D*M!l3cfz zZ->7Zv=29}H-$!Dk!l_?`M#*`xJ-LgW|f-!eVjE`Pqiax>7F`xpUvjFa8*er`yU;Y zn%4aPOQ1Af8A^@8mH0kI6vvWUw{$zU54M{^$ zMTODW#Nh+WD!>ye&$nVo8G3=559C{$A&eeNA!?EreJV82Xs#tof~SN z?ki$RAQC-K_O9(Pi5|mk3SZsvg3W*7uJdDzvg%vyr1}8v=YCrN!2MMa^U>XuQa$Fi zTS7u`+qSZ??95CKpkH*#bWC-nLa=b!y_b94z&qj7-vH8F`<4Rl)I3kAu$kP`+c~BA z%AYswLRbIs{9tsCy}V8Wdy4PgozO*U)Xvu%6A;8+qxNU_?yr$65F!uqD(`+rOG{xu z={2>qyg?%TqR|Gk|Mcl~fR`Ahe`&FkvV;657DW1C4BLbWdJF#&GG+I@e-G#1a$#>+ zO?7p3fpMA^-2!+i6;YO0mRv#*{7+M6q%ER?|N3nh4%-*tlRUdcGFU2D%D6C(YLyISf_ z!^rnoq3`tuXgF~&Hh>)C)^l4_%r5&>AP!dopGAd%j{tY1SY|-Z&SA)=D91+@fnfWY|C|FpzM@M2L_0^)uESHevbhGkT#%V2}vBY>mw*#7nv6r7A1bTvh zMG(WG)4H?K*kW_NGnHf3v#STO1`HR^eqR(N_-$Vff5sbIfKUyxxajIh=rzgvH0EL~g z0J7gFo_mIt!g^4g7FeddL@qHAy;KZTBvf+cS@?%3DJ8*cghOjCLo2LYxVthlsCa6; zJ(dg`umr9r&2!d+c=UBpaPpNa^u+Rg&^-3MATz`Awk-9q%WVtDM~?~*sH z^Gt2et;1Quh^;CPqwHMl&)pT0!fK zQ9j20y*w6Yn?G(^+AJd~+A^Azed_7q`YAnCr3YWk$>NT}Kb>PamjuItH$?4T7*7BN zE_k+tz;XHT=%Xn{|68{>Ks$r-xWB)@l;=!d+LN3dZqQ7EF^_j}M+=gQ0g40Ig)I2| z$E|swf>l+742Xc`)_ z7@ei-3_r+MNKlLJ10bXo{w?NV8*fbx8&AN=D!>`Nd;cDVCUBY2P38XV> zx2Gr4>C0s&MbLKj%ouHC`J*Vgt$F3`g}N!zgfLD~TXf$Z{2kKqq6gvwU3*`ShR!~P zjZ*G8GZ>et6mp2aUR$-Do;|&?*z)_a&;7C*b6a|GIt`5$(0cvkbnfS|{5R0nOs#GU z`t{CH8;@O$f4*~3FwkeaU+j5*45znhIIDNfW#Y3ZV;1N1m(Pp|<)3s({Tek`Q9OV! zONI#Ro&bl7 z_Nr&Tb*kRUj?5>n-A6yU{+>@%#j5*ma1&tNYd?IJVV<7;urtr-G zhfb3xI0bdn@B@VlL;>E*vccdP4>wCmnmTW|6N z*!Wra!ZF$nQ_LC!$L#n-W`s1w;Ew}p@R?aqR?97-1-4RcaSR+PJhWjB>fT^^A=>bq z{#!|O;dTA9PhlfRrsAO^PK_R4argPE4&U%V^M~%i$(2eGoTx%)%)MP+ahNBcp*V9i zn7rYl=IT}coy6zXrwqRwYv70r955F9=DSezu^H&fPW2stktpkdV}~5UBVh$p#Lki4 zW`13Ve5)@zhlYpghU+@GWIliq242^aMV#&q^&HXlb0g-#ToGW|m4P4!GZ79FXd7_W z0yrgXVrG`Lsy&~`_=9(uZO;zz-!Uaoa5AnFUOCp+;SvT4kI@P^f22krm743B^Hx@h zCr?K855kmiANWSBR_K`pHg)c@9+I-Ov<$y?O;k8`-9x(`&VMSct`$3sG^@XY^&ET0 zdeiBf0TQySJ7_+ZZmm~wmi@$<0$m*`g5`H@@Z|6gk%W+l|8Ls;xRNixKfx%FSLA;1 z-!Ogq_RUU;*KY6KVoU1FJt>~kKMxxkYH1;GjvvY|grP!UaGdH3kb4bSZfFgCt|7Fj zvOShGVJQg-MZLZG9@Bemh|^9(BO~195T;fZ6&1;cojWnWqNzosnjMdHvU1rlE2li+ zwUI+7r|qI>CtC6g?G0Q#=roikPl5BUC+FvOG;9Xb;*ia{7!Vk1tP~c_mjk~G4~ZQE z%osFG7%r^e#N_0e^e<2lht+J^yjfjJ9q>v>@Fg0a6FSb)e0kF&^~HnW4I2`cR`%@a z^_-nNYzW`CqCtacavxNC{rYtaNCAKXfJ{#Uj8~y<$BmZ~6&`D?f!)GO+VTN)pXxSh z108K;twhsg>~z+Rv2Wpb$B2d}LO$S{X$p8W6`3-}p`|EHDnXppE1N2&5ZmwpfRF?L z8!>e6_fIR+H~-S@1g5R=HM{4Lg4piKIbN>Ik32N}A)=^N^kg&WQ2hZuF#=zPbBi}a zUonrZ>R5JR{tSGtN5AE!seYViC^DBmH{47~G7ezx${VS(JOnA$hmNV@%a=VplaW># ztv5e#YEH@-D0V(PYa#+2QHoT3@FawY%lS_{++)fIU1U=x2Y(75_vmu}dA@7>fr|Ug zw`BYBL=i$@pb-0w%^eBH-H`>;B=P>edsLvm@JJy#d<*VxP6$bcDGy+tU4f}+YLX@3 zxu<3Ln~{1V0FZ138TAx!uL=TLm}S#QjX_NRD{^JnP*^%j=tj z)oVv^KW9qauJyUdmbZ`R^^`Y{(!3YepPi9wZK#=LYi`Js=ozmTS&ZUtR;}_*Q+fw5 zntjdbOK0rtSbo46^jw$dnS)1cZwP}Zp_uW~&t2DC- z%T=&V%rcCr(8K02E6Ze#j~f|g{bGkV=OyvNz$GA-U-afrrh+nyJpO+bve?24?$6uJ znE<;$HE#q&h0u;22LU(YbL$ej1D@8FcKXQ)7>IMGEJ_^uBy$UkS1_5t4r&F#En67C z{c`TmXmo=l- z&2a(=d?8r=1<3b?f!-VGO#TLEA~2k@WGmr4I^za!4}{A~m8oPIZ>=hMl5(hr!GMAC zwuRgjHQsgQd8>>N<(=te23&6;#9pu*3>mWjDSUm>`}dCAUQ0_?pQB@wBCR{XWTyYN z=7TS3wr}(a4~L>_-`6ca^psuWHXhMrXc7q;%8yGbrfg&MCiJ>SAWkFFFPdbGMrLK* zw0dGCED!aH`>gog^75HM-1nJ_6=}S)cIz2W!vTa6dWY6-4{zdR|D>ln-trr>JC?ef zG4tYsh8}6X0V~L|?;(AP*XP8}d_2SwGqoOJE4!YL9~L1vuM2gp*X?V6W4&r^t4N#w z{$(B~bPz0FxLL@f|D0B0z!!Yvner=i_#W9$-Sq4Lm%`&gQ`8pfumeG*Q~(+`0babW6wT8 z@vyCm97wy6wTx~QdBG{IX@vT`_`+zi4WQlwvj+xe%JXt_av&H5!C?RqXTwxfSNq+$ z!)wAHW^j98Iy*Plt#sD+!{|%!(k=?#zdlaPQH->Z9pF=z-=O`)Gw-%xd}USoKAD}T zO6^i?XYF^h^e+{lf9UdsrV@mCuWYi4Y_ebgSb4AK!*v#7-zJ+9VjU@s+{*X)vCM#A z>A0jilc4y+hVuad-XUE3RWxbixygHNv88>^*mo495#ZB4^sPRixksKgz%OAt zLu>Z1Ipgr5LtdZ_DKjmCZ^-Ps5j#Z-J-X+3q-PL>H80}eJb+U*3iH1tId&L?_Vn0b zzUMBd7S(Rxm9i_S-Gb0ob@h>Y4e&dxmAnc_jg(Ut7bfoI%gd^B=L{StFTB<2-DM3x z$il_X$B&)%d9~4>n%WB*HtE6AT^=Q+Av^Lx!4({LHAPc31MdngW!UV@%qS6ulbrw+ zg`pu%Lw0;)OYe(Q+nuFj;+o{d#cz@#Gz!W80Mpj9v9VRw*N4Q!@U2*}Vm-8#0nF>D zF}g>Orr`yn&Ma&@#;sbqB82C=c?JhSi1z{#EWoB;FO|BFe~TPz_nc|>l4QmI_)g1* zV}S8%5LW_~3-#3S**F4wI^{v+&f^&0kw!zS90vr|yKOYV6*=K93Y(W9HPQeSO$v@S6I7;t*OiGzJR%8j_huk00N#YZn`|a$MZp3I`7|z~Dnre$YSo z7zAFNxcq!R@22|EJx~(Bohpr$Hq5-5SDgEia?;_rRi|X6VKwf)Vm5w`MbTdOfsalcDXeO3OUMUo> zv~iRMu^fXdg`86v3)ygpIrLG}HKApMSpeKAgWCw3bKe#^L+8GE!r;SMGJ^HOS{BGM{O!2ICJ~kK;R)9*Hrv`+xTgRH?-l@)XMo9L+^xB(BL^JFZFVrwxN0gv&@->s&~-a zMqN}|Iuv+VB>9C3^bokiO>i<-AmdPRxsiQ>Y#dnMFzv|mqGRG{3EC1<0Hs;+Ix?i7 z(01@&?P-0{+1Z(T_U&qb_KLcl9G|{1Ij*wp>d^YYS3wKCgb0C_Jo{X7Cm~nk%XiMB zJ-RB5+A61knAKL()cqw1iQoPUNjAE1p!oQKbg>;~FSb`!p@1eC$NE8ERUUg$?bjO? zORpeuu&N~Y4&JLVnf9DnLngG6bB(vQky{R2HpH%ZFr%mH!cP^lbq+MQu)NQ(zx9{2 zMq4#|tm3DA$!!0FNI@BTSp`q&W7poiP^u|Pm;z9RpFMG2{<2C1bZloOEW){%_R zt_77s`CYixn^CQ4?hfm4Wn%R;?T32(4Tqyl_Jy-rt-JwZ6Am`&LZsWt9ap-!FGAb! zksODXf%GLtBamkC;eS!c#tQ4X+7n}(C5cHlSX=hQ?4uJ!YuJdfkm)5_!+6$Z#V*NP zICM_7Wb0s%T4#pCXPXd(Po8@#G=+5NZF71f1~zKJ_zE}^I6>8Afq?|CcJ(UmNTRMHmDCM2mC*I9zv;c>daAr)je9$m$W0= z2C`VKy*Rug>!yjo=ru%5FBaFmfV=& z+!Ja+k@lzxINw7WvEK(GRR{2U1OnO*nWzmE-7!Pb{d38nd2F5@;JGyE?9|(YMlL1f z=I;C2ysmsjSB#OIJOUtlw?7r-vn)1@#OdRb~3Qi z2SB!mK};dc1xQLtLZeqfOGn}4qQv0o3o;2vn_e>F%H4U6(w|Xb!0`w2;i3BP+nl{% zIp^2Q%mLjd^&oc%^H~Nj{Wh5kxEzT%^`3}91FWn-B>iCP~)1bA$w^|3dw5U^-p&-wyyVPiv&0X_nr#Dq0dCY~ArXn#fC%pj1a8O{tu zjy)ny2%|YRO=&O+a-oXgnq1KxlN8EBR}P4;w5WeeWCBu|k^J_HemJsCv72{@Lnt8x zYpdlr2eeAHNEo#lcY92bmoD3#Wa{263xtOS^ku(zVeEoVB zI%k-k+B*Z6#CFp@p#?=#^hy2(3;RMrs=dX!m9O?fa5A_Dq0x?i@PGkk;(FBc@bIL; zJSuVqqVaDUmAW^K0ZGWqV*w~FVE5s4MP-JV04vhN z=-l?5HNXzzEUdKFe3p=lVx=4vZ(Q&5M=;6|y}L&$C&^e$+|9X%y%urbcKq2Gl{dnd@owHIYtre#zT0cLv$BEk3gl^kSsH786(>Ec!uSM$G2Ec;>cc zg&gdQZonO1SK|g+lqgtK1;)Oa|NWJd=cYWnos^aZ>`ACjty1Sq;$fOfky4{U5 zY1lS5pohKo(=XL1+?Z$aPR#V;qGw*cbYrLF%y@KykKvFQV9Y>b>>oSbWrmdBFcknC z6aL^s00k$^AHA27vZEyvIw?RGjlloH1wZLBASi%oDu^+BFuccrMQ%zIu%|8VYJs_C z+QIGW!P-{(tbb%j4mbg7FHqbRwHCmfF3Zqwz_*nk5fJz2|4!{`_NvD3h1=#tO0KaP zBzt3NiM-nuo)APinDxCLnLVK0E?p726NBNBJgg|MOHBKJM9z91hw2U`BtQh)(#vIc z#dvlmKR^%!s10k&lOI0x0l^TAjxx$PnP%6jkk|K$3C+~%&M4cRwH5pO{) z^Vhy&&2a2{lGVhmquTu?odH7fAvHBRQSB9^MZum0Nx6RdZX>`W_l4VG3_uZZM=&Nj z%1~@gXFDXw-I9`%mv7mMeS**plPqZK$_s<*UTF0#S$3}VQ;2YtywyD2bLquoPcBs` z`zC~<)l>BJmr|iquByv&A9n!y8{-s(zC;i*JLf#>{HuecHA4G7k}ee8U_1C#yNDLH&6y#6|v$*s?ml* zvAa%)eRIg3s+)rLN=sk=y>YOr@6xF8O7%nG?EQ!+$-Ca*dqV;(FW|XB9t09iG(4ZE z0LleA)ZGaS9{^?3hgdmVs|6WNNyD2qn1ThjSP67BSNY|7vYxzrxejI;p|RMYlU10w z1HB2FsQ@w~P$@BBy6UsaoqG_kHTmR3da*MRUYw5Kxpo(bwjq`pUcL3Z&dLyFbF_P) z-Kn2gEuL6S-nIAF7S}2(1<51qlpRA0B`ODm$CpagPkIezGeEI_{XasXv|N_z_(A1nycgN2?E_Axj{Xdl0+Eg9h{K$lp(Cb62hHBpwl zp^IgTWH{yD?Axvm@p%y31nehT@xDPS$=egFUA`IWsy!;hZGQbvp&I4*AMwGY>e5zJ zZU6IySLo|bM{9Rhu3anM-+=W(P6DW*4Sp;tN=m=-P7zN8(*!aV`Dqt(mW}>d$0q z(;S$U#h_dI>jR(TA)7heQgo0TTjun*a;E{~;yijJWDi&Mdb7hDM}2WdCEVM{n}AhY zTYxgz+#4h2Qj_gg;@*3Jhw>L*6+c;gjz2PDo`66KH{i z2eY%ewe>kMr8DP{0P9yF1o<9*x0#1W&;WLJ0JNf*IM4)vP-CeUrdZnb+`G|sz|0`- z4>e6sPlNmuOijf<0iimfs-*{A((m{VMik^lfh*3=YB_IXBT-xpVpbTy0>&x$1iKfq9(0tOjQVNO7Y_UjD%aJ> zS-C&t1rmj=I!sW~=1Xd^&pw3)h(Oq1hLN&wqa7F=4D0?^QbA93l@jakvK#tL_~m$h z=$-yzp&<|1;xDmcJqW2_jB^mk4OgsOdG8+-1-SjabJP-Wph!*uK#4h4{dwTOt^~e6 z7F{BSB3qg-2*E0Y+%k0At`Bsu%e8X6oo?Aso2LBU8H?b6-wPbVE&QG|(tf&SZXZ!@ zOu_wAbA{n5Ek7}A4MdoeCco<4`R7J`!vV_0KkMuHyMd4RNhvfJh4dvK^FJr>KX>5& z-*>>TFHYzjNbP=S0O(`kmq~=+*W&&W_bbR5AulZa?4|>L*ZlKuqXC#QkT1gZvX-MDD(<@wqTC@8mRRgs4~f znEZV{5PF3A5)e|6q<>NA0gxr=g(#4(J$}r{8jm=^_@^H zyPn*$<6lmxaP!zIETPGB_A-FdP^GiOWa1J)w$PL-lT&j}!sb$4@l`uNO_P?m9~aId zc382I+~4BXJju8H)C;3f6PUm&W!K`>zbhnJu5Hmng)7*>9}}ucnIi(7IX>0EmJA1(C@hcZZRu@y##Xj}tz` z$S^^b0b}f_F;UUc00W)L`@G1t_qPpNHDxuU?^>Lc7W;4Zbbg^Iq%NPa+Blw)tOWuP z7-B({@^XY5yk2QA803PJ6F2#{2anOD9{e8mCF?G=fYuqK9GncQRTDR8bmt`5NR1C`Cnh)bTlRt3M-YsWFQ}9NWCK4d(mjlfIFU!8 z4!H)y8e!-oI|oM__%2!>hY-%rU(@`~T3<^mXWV>yXY`5Ma!Pn%r-v;dLNI+R(q$GO zxOAVxMd#Vyr~?G&%^BKz9H_L$qGgOp9WNL4r=72xyJ4OA~)xPZQC z0H*5{78DfV#S#ew)M}>WPgqp=zvDD?N7|+UK3DUq8u*489zaEvK~_}Iz2F!N0hg&u z)hLK9Uy*TgSIuHv=sgD|J$Lo|B;?F}Hflx$1!2qub^ejQ4bOIdf~xbE-;2D*;p;MF z8Y_nXlDtn@YzbM zOB<5B*pSsPJzT8=6IMY3Jp9XS3txrR2d*>jSsWIFXdzgBTH4ykqdT@nXQ^2%%oKNxtlC#+llalaudFUhrul^>`=I^SO;eQ}Ai~Y9dHa18?i)r~w zU!H)L@{&UTwze1@hGOMfhpA|63 znZ#fNf-U$|u2WeG-oo6x07-QELE_JGsllEBrYHG>$De48YzkVc;^2#z(-Z!Zomc8o z!UD#NG9AIHz}ODPTEE|6^ zF5=-)l}25aJqlwlL23=$!3OX!fw&Gc6#}kZahv)Ea8FIwO@WuTwdQ-)66005vzU0L z^MP7JkNK2FS+TE9sDcCZCpdP78+)g$mznen6Rz^f1cUJrFngqy^dO(NwJia667mOv zpqzrhe|~(z7kuFyV3_lej7tKw@CKYT^@5S2c-yDHVg|-5Cugt7iD$Ghch^HQv2Wli zh?{y2fT;`oRH4zTz|J7=@66PA($l9Sxm2WR+x$JP=#1yWiF~UNVMdpN#bY^j00)k{ zw&Boyf!{9B`Gila^8J8|ka*Px$(+Ce*_or*Pn=)Lv&U7Y&o+49eyvOM_TPfL&}|lp zj?v=LJ>uG`r;mf*a&Mxp!ZjR&S$7IY;J*ZKDd3CB5Qwi5&96p-N~6D@pTHebL}XS{ z4!Fz9xGAB!3=kk4%)LSCkDV$a=iId8Pr1IV*oA zvIL`>oR~Iz7a65kXv$F7P0V4YuE+p(9#=*92<&?tC3h4{X|i7 z=qT>~`P=@y^fJnDTUh3`B1>+|-#PvF75<%){;L%K{S37HQsCi}g&Ln-z=~4y&b_vEr4Ty)5UAON6z1O05k@)S l{72!x`vxAug2$X*_oCWYyTJ1>2n>Tctfa2^RKdvSe*w7Ot)u_| literal 0 HcmV?d00001 diff --git a/docs/proposals/suggestion.md b/docs/proposals/suggestion.md new file mode 100644 index 00000000000..be6de9d4913 --- /dev/null +++ b/docs/proposals/suggestion.md @@ -0,0 +1,418 @@ +# Suggestion CRD Design Document + +Table of Contents +================= + + * [Suggestion CRD Design Document](#suggestion-crd-design-document) + * [Table of Contents](#table-of-contents) + * [Background](#background) + * [Goals](#goals) + * [Non-Goals](#non-goals) + * [Design](#design) + * [Kubernetes API](#kubernetes-api) + * [GRPC API](#grpc-api) + * [Workflow](#workflow) + * [Example](#example) + * [Algorithm Supports](#algorithm-supports) + * [Random](#random) + * [Grid](#grid) + * [Bayes Optimization](#bayes-optimization) + * [HyperBand](#hyperband) + * [BOHB](#bohb) + * [TPE](#tpe) + * [Anneal](#anneal) + * [SMAC](#smac) + +Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc) + +## Background + +Katib makes suggestions long-running in v1alpha2 and v1alpha3. And the suggestions need to communicate with katib manager to get experiments and trials from katib-db. This design hurts high availability. + +Thus we proposed a new design to implement a CRD for suggestion and remove katib-db from main workflow. The new design simplifies the implmentation of experiment and trial controller, and makes katib Kubernetes native. + +This document is to illustrate the details of the new design. + +## Goals + +- Propose the Suggestion CRD. +- Propose new GRPC API for Suggestion service. +- Suggest the approaches to implement suggestion algorithms. + +## Non-Goals + +- Metrics collection (See [Metrics Collector Design Document](./metrics-collector.md)) +- Database-related refactor + +## Design + +### Kubernetes API + +```go +type SuggestionSpec struct { + //Name of the algorithm + AlgorithmName string `json:"algorithm_name"` + + // Number of suggestions requested + Suggestions int `json:"suggestions"` + + //Algorithm settings set by the user in the experiment config + AlgorithmSettings []AlgorithmSetting `json:"algorithm_settings,omitempty"` +} + +type AlgorithmSetting struct { + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` +} + +type SuggestionStatus struct { + // Suggestion results + Assignments []TrialAssignment `json:"assignments,omitempty"` + + // Algorithm settings set by the algorithm. + AlgorithmSettings []AlgorithmSetting `json:"algorithm_settings,omitempty"` + + Conditions []SuggestionCondition `json:"conditions,omitempty"` + // include all common fields + +} + +type TrialAssignment struct { + // Suggestion results + Assignments []ParameterAssignment `json:"assignments,omitempty"` +} + +type ParameterAssignment struct { + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` +} +``` + +### GRPC API + +```protobuf +syntax = "proto3"; + +package api.v1.alpha3; + +import "google/api/annotations.proto"; + +service Suggestion { + rpc GetSuggestions(GetSuggestionsRequest) returns (GetSuggestionsReply); +} + +message GetSuggestionsRequest { + Experiment experiment = 1; + repeated Trial trials = 2; // all completed trials owned by the experiment. + int32 request_number = 3; +} + +message GetSuggestionsReply { + repeated Trial trials = 1; // trials should be created in the next run. + AlgorithmSpec algorithm = 2; +} + +message Experiment { + string name = 1; + ExperimentSpec experiment_spec = 2; +} + +message ExperimentSpec { + AlgorithmSpec algorithm = 3; + ParameterSpecs parameter_specs = 1; + ObjectiveSpec objective = 2; +} + +message ParameterSpecs { + repeated ParameterSpec parameters = 1; +} + +message AlgorithmSpec { + string algorithm_name = 1; + repeated AlgorithmSetting algorithm_setting = 2; +} + +message AlgorithmSetting { + string name = 1; + string value = 2; +} + +message ParameterSpec { + string name = 1; /// Name of the parameter. + ParameterType parameter_type = 2; /// Type of the parameter. + FeasibleSpace feasible_space = 3; /// FeasibleSpace for the parameter. +} + +message FeasibleSpace { + string max = 1; /// Max Value + string min = 2; /// Minimum Value + repeated string list = 3; /// List of Values. + string step = 4; /// Step for double or int parameter +} + +enum ParameterType { + UNKNOWN_TYPE = 0; /// Undefined type and not used. + DOUBLE = 1; /// Double float type. Use "Max/Min". + INT = 2; /// Int type. Use "Max/Min". + DISCRETE = 3; /// Discrete number type. Use "List" as float. + CATEGORICAL = 4; /// Categorical type. Use "List" as string. +} + +enum ObjectiveType { + UNKNOWN = 0; /// Undefined type and not used. + MINIMIZE = 1; /// Minimize + MAXIMIZE = 2; /// Maximize +} + +message ObjectiveSpec { + ObjectiveType type = 1; + double goal = 2; + string objective_metric_name = 3; +} + +message Trial { + string name = 1; + TrialSpec spec = 2; + TrialStatus status = 3; +} + +message TrialSpec { + ParameterAssignments parameter_assignments = 2; + string run_spec = 3; +} + +message ParameterAssignments { + repeated ParameterAssignment assignments = 1; +} + +message ParameterAssignment { + string name = 1; + string value = 2; +} + +message TrialStatus { + Observation observation = 4; // The best observation in logs. +} + +message Observation { + repeated Metric metrics = 1; +} + +message Metric { + string name = 1; + string value = 2; +} +``` + +### Workflow + +![](../images/suggestion-workflow.png) + +When the user creates a Experiment, we will create a Suggestion for the Experiment. When the Experiment needs some suggestions, Experiment controller updates the `Suggestions`, then Suggestion controller communicates with the Suggestion to get parameter assignments and set them in Suggestion status. + +#### Example + +Now the workflow will be illustrated with an example. + +```yaml +apiVersion: "kubeflow.org/v1alpha2" +kind: Experiment +metadata: + namespace: kubeflow + name: random-experiment +spec: + parallelTrialCount: 3 + maxTrialCount: 12 + maxFailedTrialCount: 3 + objective: + type: maximize + goal: 0.99 + objectiveMetricName: Validation-accuracy + additionalMetricNames: + - accuracy + algorithm: + algorithmName: random + trialTemplate: + goTemplate: + rawTemplate: |- + apiVersion: batch/v1 + kind: Job + metadata: + name: {{.Trial}} + namespace: {{.NameSpace}} + spec: + template: + spec: + containers: + - name: {{.Trial}} + image: katib/mxnet-mnist-example + command: + - "python" + - "/mxnet/example/image-classification/train_mnist.py" + - "--batch-size=64" + {{- with .HyperParameters}} + {{- range .}} + - "{{.Name}}={{.Value}}" + {{- end}} + {{- end}} + restartPolicy: Never + parameters: + - name: --lr + parameterType: double + feasibleSpace: + min: "0.01" + max: "0.03" + - name: --num-layers + parameterType: int + feasibleSpace: + min: "2" + max: "5" + - name: --optimizer + parameterType: categorical + feasibleSpace: + list: + - sgd + - adam + - ftrl +``` + +Now, we will create a Suggestion for the Experiment: + +```yaml +apiVersion: "kubeflow.org/v1alpha2" +kind: Suggestion +metadata: + namespace: kubeflow + name: random-experiment +spec: + algorithmName: random + suggestions: 0 +``` + +Then, Experiment controller needs 3 parallel trials to run. It updates the Suggestions: + +```yaml +apiVersion: "kubeflow.org/v1alpha2" +kind: Suggestion +metadata: + namespace: kubeflow + name: random-experiment +spec: + algorithmName: random + suggestions: 3 +``` + +After that, Suggestion controller communicates with the Suggestion via GRPC and updates the status: + +```yaml +apiVersion: "kubeflow.org/v1alpha2" +kind: Suggestion +metadata: + namespace: kubeflow + name: random-experiment +spec: + algorithmName: random + suggestions: 3 +status: + assignments: + - assignments: + - name: --lr + value: 0.02 + - name: --num-layers + value: 4 + - name: --optimizer + value: sgd + - assignments: + - name: --lr + value: 0.021 + - name: --num-layers + value: 3 + - name: --optimizer + value: adam + - assignments: + - name: --lr + value: 0.03 + - name: --num-layers + value: 5 + - name: --optimizer + value: adam +``` + +Then Experiment controller creates the trial. When there is one trial finished, Experiment controller will ask Suggestion controller for a new suggestion: + +```yaml +apiVersion: "kubeflow.org/v1alpha2" +kind: Suggestion +metadata: + namespace: kubeflow + name: random-experiment +spec: + algorithmName: random + suggestions: 4 +status: + assignments: + - assignments: + - name: --lr + value: 0.02 + - name: --num-layers + value: 4 + - name: --optimizer + value: sgd + - assignments: + - name: --lr + value: 0.021 + - name: --num-layers + value: 3 + - name: --optimizer + value: adam + - assignments: + - name: --lr + value: 0.03 + - name: --num-layers + value: 5 + - name: --optimizer + value: adam + - assignments: + - name: --lr + value: 0.012 + - name: --num-layers + value: 4 + - name: --optimizer + value: adam +``` + +## Algorithm Supports + +### Random + +We can use the implementation in katib or [hyperopt](https://github.com/hyperopt/hyperopt). + +### Grid + +We can use the length of the trials to know which grid we are in. Please refer to the [implementation in advisor](https://github.com/tobegit3hub/advisor/blob/master/advisor_server/suggestion/algorithm/grid_search.py). + +Or we can use [chocolate](https://github.com/AIworx-Labs/chocolate). + +### Bayes Optimization + +We can use [skopt](https://github.com/scikit-optimize/scikit-optimize) to run bayes optimization. + +### HyperBand + +We can use [HpBandSter](https://github.com/automl/HpBandSter) to run HyperBand. + +### BOHB + +We can use [HpBandSter](https://github.com/automl/HpBandSter) to run BOHB. + +### TPE + +We can use [hyperopt](https://github.com/hyperopt/hyperopt) to run TPE. + +### Anneal + +We can use [hyperopt](https://github.com/hyperopt/hyperopt) to run Anneal. + +### SMAC + +We can use [SMAC3](https://github.com/automl/SMAC3) to run SMAC. From 1ede573521799b47af01a0a1faf4a7ab2d00be75 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Tue, 17 Sep 2019 16:04:44 +0800 Subject: [PATCH 2/5] fix: Update K8s API Signed-off-by: Ce Gao --- docs/proposals/suggestion.md | 39 +++++++++++------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/docs/proposals/suggestion.md b/docs/proposals/suggestion.md index be6de9d4913..32abd336fa6 100644 --- a/docs/proposals/suggestion.md +++ b/docs/proposals/suggestion.md @@ -49,42 +49,27 @@ This document is to illustrate the details of the new design. ### Kubernetes API ```go +// SuggestionSpec defines the desired state of Suggestion type SuggestionSpec struct { - //Name of the algorithm - AlgorithmName string `json:"algorithm_name"` - // Number of suggestions requested - Suggestions int `json:"suggestions"` + Requests int32 `json:"requests,omitempty"` //Algorithm settings set by the user in the experiment config - AlgorithmSettings []AlgorithmSetting `json:"algorithm_settings,omitempty"` -} - -type AlgorithmSetting struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` + AlgorithmSpec *common.AlgorithmSpec `json:"algorithmSpec,omitempty"` } +// SuggestionStatus defines the observed state of Suggestion type SuggestionStatus struct { // Suggestion results - Assignments []TrialAssignment `json:"assignments,omitempty"` - - // Algorithm settings set by the algorithm. - AlgorithmSettings []AlgorithmSetting `json:"algorithm_settings,omitempty"` - - Conditions []SuggestionCondition `json:"conditions,omitempty"` - // include all common fields - + Suggestions []TrialAssignment `json:"suggestions,omitempty"` } type TrialAssignment struct { // Suggestion results - Assignments []ParameterAssignment `json:"assignments,omitempty"` -} + ParameterAssignments []common.ParameterAssignment `json:"parameterAssignments,omitempty"` -type ParameterAssignment struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` + //Name of the suggestion + Name string `json:"name,omitempty"` } ``` @@ -286,7 +271,7 @@ metadata: name: random-experiment spec: algorithmName: random - suggestions: 0 + requests: 0 ``` Then, Experiment controller needs 3 parallel trials to run. It updates the Suggestions: @@ -299,7 +284,7 @@ metadata: name: random-experiment spec: algorithmName: random - suggestions: 3 + requests: 3 ``` After that, Suggestion controller communicates with the Suggestion via GRPC and updates the status: @@ -312,7 +297,7 @@ metadata: name: random-experiment spec: algorithmName: random - suggestions: 3 + requests: 3 status: assignments: - assignments: @@ -348,7 +333,7 @@ metadata: name: random-experiment spec: algorithmName: random - suggestions: 4 + requests: 4 status: assignments: - assignments: From 7956726d893baa156681531a5f27fce130f76849 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Mon, 30 Sep 2019 16:49:43 +0800 Subject: [PATCH 3/5] chore: Update TOC Signed-off-by: Ce Gao --- docs/developer-guide.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 0a12b545d83..5ef97186652 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -1,13 +1,18 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Developer Guide](#developer-guide) - - [Requirements](#requirements) - - [Build from source code](#build-from-source-code) - - [Implement new suggestion algorithm](#implement-new-suggestion-algorithm) - - +Table of Contents +================= + + * [Developer Guide](#developer-guide) + * [Requirements](#requirements) + * [Build from source code](#build-from-source-code) + * [Implement a new algorithm and use it in katib](#implement-a-new-algorithm-and-use-it-in-katib) + * [Implement the algorithm](#implement-the-algorithm) + * [Make the algorithm a GRPC server](#make-the-algorithm-a-grpc-server) + * [Use the algorithm in katib.](#use-the-algorithm-in-katib) + * [Contribute the algorithm to katib](#contribute-the-algorithm-to-katib) + * [Unit Test](#unit-test) + * [E2E Test (Optional)](#e2e-test-optional) + +Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc) # Developer Guide From e8758c9b724f26c38466c48a7326f90b04927fbe Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Mon, 30 Sep 2019 16:59:50 +0800 Subject: [PATCH 4/5] fix: Address comments Signed-off-by: Ce Gao --- docs/proposals/suggestion.md | 43 ++++++++----------- .../suggestions/v1alpha3/suggestion_types.go | 1 + 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/docs/proposals/suggestion.md b/docs/proposals/suggestion.md index 32abd336fa6..2fd06252733 100644 --- a/docs/proposals/suggestion.md +++ b/docs/proposals/suggestion.md @@ -27,7 +27,7 @@ Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc) ## Background -Katib makes suggestions long-running in v1alpha2 and v1alpha3. And the suggestions need to communicate with katib manager to get experiments and trials from katib-db. This design hurts high availability. +Katib makes suggestions long-running in v1alpha3 and v1alpha3. And the suggestions need to communicate with katib manager to get experiments and trials from katib-db. This design hurts high availability. Thus we proposed a new design to implement a CRD for suggestion and remove katib-db from main workflow. The new design simplifies the implmentation of experiment and trial controller, and makes katib Kubernetes native. @@ -51,19 +51,24 @@ This document is to illustrate the details of the new design. ```go // SuggestionSpec defines the desired state of Suggestion type SuggestionSpec struct { + AlgorithmName string `json:"algorithmName"` // Number of suggestions requested Requests int32 `json:"requests,omitempty"` - - //Algorithm settings set by the user in the experiment config - AlgorithmSpec *common.AlgorithmSpec `json:"algorithmSpec,omitempty"` } // SuggestionStatus defines the observed state of Suggestion type SuggestionStatus struct { + // Algorithmsettings set by the algorithm services. + AlgorithmSettings []common.AlgorithmSetting `json:"algorithmSettings,omitempty"` + + // Number of suggestion results + SuggestionCount int32 `json:"suggestionCount,omitempty"` + // Suggestion results Suggestions []TrialAssignment `json:"suggestions,omitempty"` } +// TrialAssignment is the assignment for one trial. type TrialAssignment struct { // Suggestion results ParameterAssignments []common.ParameterAssignment `json:"parameterAssignments,omitempty"` @@ -89,11 +94,14 @@ service Suggestion { message GetSuggestionsRequest { Experiment experiment = 1; repeated Trial trials = 2; // all completed trials owned by the experiment. - int32 request_number = 3; + int32 request_number = 3; ///The number of Suggestion you request at one time. When you set 3 to request_number, you can get three Suggestions at one time. } message GetSuggestionsReply { - repeated Trial trials = 1; // trials should be created in the next run. + message ParameterAssignments{ + repeated ParameterAssignment assignments = 1; + } + repeated ParameterAssignments parameter_assignments = 1; AlgorithmSpec algorithm = 2; } @@ -200,7 +208,7 @@ When the user creates a Experiment, we will create a Suggestion for the Experime Now the workflow will be illustrated with an example. ```yaml -apiVersion: "kubeflow.org/v1alpha2" +apiVersion: "kubeflow.org/v1alpha3" kind: Experiment metadata: namespace: kubeflow @@ -261,23 +269,10 @@ spec: - ftrl ``` -Now, we will create a Suggestion for the Experiment: - -```yaml -apiVersion: "kubeflow.org/v1alpha2" -kind: Suggestion -metadata: - namespace: kubeflow - name: random-experiment -spec: - algorithmName: random - requests: 0 -``` - -Then, Experiment controller needs 3 parallel trials to run. It updates the Suggestions: +Then, Experiment controller needs 3 parallel trials to run. It creates the Suggestions: ```yaml -apiVersion: "kubeflow.org/v1alpha2" +apiVersion: "kubeflow.org/v1alpha3" kind: Suggestion metadata: namespace: kubeflow @@ -290,7 +285,7 @@ spec: After that, Suggestion controller communicates with the Suggestion via GRPC and updates the status: ```yaml -apiVersion: "kubeflow.org/v1alpha2" +apiVersion: "kubeflow.org/v1alpha3" kind: Suggestion metadata: namespace: kubeflow @@ -326,7 +321,7 @@ status: Then Experiment controller creates the trial. When there is one trial finished, Experiment controller will ask Suggestion controller for a new suggestion: ```yaml -apiVersion: "kubeflow.org/v1alpha2" +apiVersion: "kubeflow.org/v1alpha3" kind: Suggestion metadata: namespace: kubeflow diff --git a/pkg/apis/controller/suggestions/v1alpha3/suggestion_types.go b/pkg/apis/controller/suggestions/v1alpha3/suggestion_types.go index d6e1885cd77..271b4cc247c 100644 --- a/pkg/apis/controller/suggestions/v1alpha3/suggestion_types.go +++ b/pkg/apis/controller/suggestions/v1alpha3/suggestion_types.go @@ -60,6 +60,7 @@ type SuggestionStatus struct { Conditions []SuggestionCondition `json:"conditions,omitempty"` } +// TrialAssignment is the assignment for one trial. type TrialAssignment struct { // Suggestion results ParameterAssignments []common.ParameterAssignment `json:"parameterAssignments,omitempty"` From eb8f2a99daa9f78f345c7cb30f262a82aef1ffe8 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Mon, 30 Sep 2019 17:01:21 +0800 Subject: [PATCH 5/5] fix: Fix field Signed-off-by: Ce Gao --- docs/proposals/suggestion.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposals/suggestion.md b/docs/proposals/suggestion.md index 2fd06252733..12ee15b5751 100644 --- a/docs/proposals/suggestion.md +++ b/docs/proposals/suggestion.md @@ -294,7 +294,7 @@ spec: algorithmName: random requests: 3 status: - assignments: + suggestions: - assignments: - name: --lr value: 0.02 @@ -330,7 +330,7 @@ spec: algorithmName: random requests: 4 status: - assignments: + suggestions: - assignments: - name: --lr value: 0.02