From 605562467d41836fc1bbcdc683d4277cd34c6a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 20 Dec 2019 10:51:13 +0100 Subject: [PATCH 01/61] Flatten fork code for testnet2 --- build/bootstrap/bootstrappers.pi | 6 - build/forks.go | 3 +- build/genesis/devnet.car | Bin 36190 -> 0 bytes build/version.go | 2 +- chain/actors/actor_miner.go | 220 +------------------------------ chain/stmgr/fork_no_p_eps.go | 127 ------------------ chain/stmgr/forks.go | 11 +- chain/sync.go | 17 +-- 8 files changed, 13 insertions(+), 373 deletions(-) delete mode 100644 build/bootstrap/bootstrappers.pi delete mode 100644 build/genesis/devnet.car delete mode 100644 chain/stmgr/fork_no_p_eps.go diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi deleted file mode 100644 index 9c619b1bb..000000000 --- a/build/bootstrap/bootstrappers.pi +++ /dev/null @@ -1,6 +0,0 @@ -/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWHwGBSiLR5ts7KW9MgH4BMzC2iXe18kwAQ8Ee3LUd1jeR -/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWCLFaawdhLGcSpiqg43DtZ9QzPQ6HcB8Vvyu2Cnta8UWc -/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWMmaL7eaUCF6tVAghVmgozxz4uztbuFUQv6dyFpHRarHR -/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5 -/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8 -/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm diff --git a/build/forks.go b/build/forks.go index 8e8f53abe..9cc7b0c9d 100644 --- a/build/forks.go +++ b/build/forks.go @@ -1,4 +1,3 @@ package build -const ForkCCM = 1750 -const ForkNoPowerEPSUpdates = 16450 +// No forks yet \o/ diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car deleted file mode 100644 index d0aef24863cdef23849a46eb87279c1d1f1472ab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36190 zcmcKj1zc6x_XZAMy1V;IBOxUsl2QT^g21Je?h-`0I|QUfQji7_5Rj0Pln{^(K@dT@ z8~@iCum3yu{eIr_&Tn}6_?fxRES_hd{j9yuzH6CkKbe&i><9A#5O|M_G%= z;A7fBzu||zS`?n#4)(ULlV&?1&er##w42J%(v9dRX zS{T`zI@tO7NrK>pt#6X7n9{iUeb(lBvzBu3M zK6l}hI(~8f@3=DeX(9smrxWO6L(I%Jb+;AaOrJX7W6jA51=sppCiZFNZ(Qp)1hH-~ zmB<*lZGl^QB$bh!tt{+~9=kZ1L4T+Xw%JlNkd18}Ol-jL|04?f@2LVBWB`PLuo;8| z`u&4M>PIN~_h(e-e?;;Fp~hqz@%aZBgHVIUud0HOG(p(rR<_spu9-MF*u#xmwtS!! z+`p#hMQi-wGnJU|;%aVm7@1Sr_lIfN<%=!NAj!XQOh64NP$0EmKsbK`*;(0}Il&>l z?{%MFOQN0^<#rq_Cd;(`MwRF7}V%P=~Eqsq&l@k)XKw&?WtB^!e* z0It}Pp1R5GYJHOC@w^qg{-a^BP^4$c-&JTJgipsV;&F;W|Ko~H{__?886MJqhf)Ti zKcU^Zp`f3#=J$L0OhB{0fd9WKO`$X*F^X(?FeKu|9}xRpAzT--L4re+p0I?u^i~k* z_qqZ4#;IL5pC0b~uemh)&lB*!yyusn z-}~j0xWE0}$jQbG-hcEyu3)%h>{i4z9KKV=3@BbGvP)zuAm`gq3zLpr{lzf63T_j8uy4o~0~K8GgZ6{>IasrYr-0@c+%US}2QM-seZ%mxA_)0vIk0G1j3EUReN(^ zIS3R@=O64~sIip^vNQUQOb!xGXx1T03qh=J$> zr8J@5^XGgqaD`B7%Cv4$%Yo5gvR|ovt1>)XQ5@wV4aHJ+Ws|!LoGHFGZ>m6{kiIa1 z)#JmF>t1v_or2m%PX&CRU~&MYm)A??;Lx-|9pB?D;2%lV=w!SpVr?l1T@eJvVFm&!@D6dr?XP2#r=gusFM4$BP2c7{`1x6Qp9KaBWQPFjFm*@prsXgxV`bJz z&eq^$hN;a<-_92yZjFx1qE#b&eoeoRVL*m~QuKa70T8f>I>)r0f1l(o-z&rSkr#sc zstGMO@9TEFE5GAF9C?6bd0tP9y({UM1}~njM!W9&UF%_UV`b1e`hjA>QAYn^*ZZ&zjvztL>|X z_+|dA;Bnhz$B9qxS{R@pLRVEct~66pJldC?#?1?Ey52J#78AJ_aOOEFkQ^Gt4gohQ zNW0R~uAct%`gZ=9&d)0?tHivi<91z%C0@RH?IZ{a^0r2(hF(Z9j6}~iH@V3gyelDm z)^(P|^2B>kxs=KGE(AP*4*3yr-RMg-`^X6P=^XK=TdwOM^p!DU@Ui@Pd;s@vxq+Mw zK^MKrCny3r=^L7*VP~Oss%3ggkdadQ%5fb_k+;h>X{#`~f$R`q?|$UGCh<{MBi&rb zlV5$YN5YE!_Hv2d_p-232OV!Ju0L`E8N%eY^!#+Ej^?%XBev`W2Cwj%lq7Pa+p~SR zM(Jgb2JK?eA>dcO8{1yfSB^b5=9ItmjOKW=RWrA9zp~G@v3|er<$eV^$@_aYH6CI- z`Mp$1efI`eG5eT4`&L$pPIqkGS=Tv~REB`>q7F9`ZzY8jLNfSeh_up*aw}uoo-MF8 zTU;%x0B`X_K>{OcXx-AxO!4u9#g-GOLo{A8_eLr;zxi|`)gD*kDhvS|o3|LT1{f#J z-D0@DZtpiuY`dGOk@0dV$p$>sp>-V}3X(v8C-h3NZTmZ_@W?h7^B~Ww1y-(8qSO-=`W&xm*Ui$*XhElFu8&35MbY>pk{cW^erAMXr&JauMTGT z+ecOr;ma82a7V5eUQ_=gH;^GrZj#~3fe~R4HtS#~I~B^EXpTJ{zKG%(8c!7ki@t&z z`VerHT$oit^U240+D?5g>`*6mL(~v#uW3&wa|{LnPZAdBB$KLjtLzjxEnBMDd=Z-^ zVEN(WDeDj@Suo$7S^mt#4Z1}<(1;!*u=!Lx+eXbdxI;Eu*&%npUh`u>tk7d%L)|E@YI%ceTSTH#=XFJQ4?ujN)G`?o<(ip494_WgX|*|Rm%3c z)^DCUl~1!CQZXIw&IW6K!z~6Y2sQ8RqbuHuIF9vz5g?%r|p#UZD`SLFVoa_-WGbh>JV^HJXS^H zRIVpIr&r)BpCkcoAIgfpuO_kQXD_dxV5yNoCwayq!LKE-jBcPja3>#STVn1Ra~p0@ zb!)wVxkRq;yfg$Hx=0ODdUUFk_{h3k=h~tr{VnIN*chDjZ=}W}LqU9;tcj-S zP%4zl3FckyhRWM#eA=-c&&4azXuE-7$lwS8-=bH>-gC;hIrgosq-yNBwS)g5SM^D@ ziWrAABZN(m4+4l-+K`}Pr_|{8ilkVTESF=jtx7M03CD?4kdb)*{1o5_8&d_Is5|#D#{+0 z1`joPWn35W@3FX_n7asfd~j(Ee}4bnLdnJ3^J?Qn7W8KZ@3V(L%Qd&$ssx7%D6zwt zHqtA~2pQH97`{~C#KY1&CSsWL^tO%NSucagO+ zpnmQRg9WlffL(bnK0Hk(86snEkfgZBs@OTY8*++$V&Or`h@s6Zit)#ffec}OtXi`2 zcqYA3IcqLDdyjpwQ~0SQa!VG4ymU93yDR1d2zm=D^yf@^P#u<;?-nP_U!{GDp(Rj4 zL)JL}VSi$NYaA2mJujz3K7#IViCO64pKi;62)d{{<<#ug14g>(OI5C46M}&02cQ1ikORKAto&GjZ8b^^3gX~Kul~d}XkdpF+c;u9lT^K1m+_wc znR^MH)ZW=ir62@+k{0u9nhL}BS#svZt%cL6)sOLwp9$h zzLnA?T#YxoWux`@0L4mvY1F%|3?vHyU%b2E>cQ zdK|yy2HG$XbP16DzPUg8(CvtEHbd4rD~gI?ULM&&+gSQKZ4|z|nHfxOAUgzD8*rlF z5A>Pic>kE~-&m%Kt zl0%a%$C5dLD)Thv^^6V#O#c&GFmt>>yso44fR6dB$j5rPkS7=ycO#y8P;4lH4m!zb z*H_=x)qX|hUr*{51mBO%xFlN2X?O)$f7*cjDRklZ7aM%MG;Y zBk1DB4dD+^nu{Vc!aUflVqMigQhabD%!IuS?}a-#nQkadZXi1Z*oNWzFQpkC7_hy< z3B1nL{9f~_=)}DqSCg#3?X|7R@zpyjn?_G zOq(WzUPbRrH|6}sHC{$Mxzx~JC)Gh$VJJxaLvkd|0WUD$_jUv88z!o$(#B7_CrXVq z)#EHkP&Or@-b+ug`-qHHyOephcF9H}tR&V+&)ngSN*Ae}wxc5+zyI$@2~bESPzo&o z?o1%i9iVo1Wre>E*Xq!DUSQ~YuoS;q;#QLnF?#E$Lf z!YbO+?dFEnKc(LE>n_l&u)$md2ok|HP>e*g-&BQBG4?mPa`BjI7s}cb+H8L1EOInd zTA6j6g24jWA;9jxnw}If>rtM`57ygmyxdd9^u3XQIh(PU++(3pfxntY&X=a2S zxpJ$=s8^N8ZW0(RAoFkNG4ERVqTaI?C4+_|*z`iz24&I*8U(I*m*BSsqNG2s4&OXi zjpI(w&XE0eD==JKCQ!-Pae;{2$z~BLmN<$Y6z=hCE8NHB4U zwz~LORzmq;wXZ=j`>{PfoqO5sw;}<*K+taGi!~XwY$kLn$v^luIF$q$^wpR}kt%fy zfAZoU1}6k8uvMh;)!n=~`1*2<5D}?j&PPp@fShX+4l0F$vjGmjQ`wJMV0^|rX z-ize44phX-(wXr7AAZHk(ag;0F|VMYAPqEoiTNvoN&o+m4g5!H@OR`NC@q);mpb^B z{`(u_*C)-tW41s^K=|#o>-?wnE=5R#tdvK0bolxHyXwQNHOKPDoEf1w;#5tqs8E)p zf5%pVOamZPexSc1`M<99uOC9x-yeehN{YJiq2#R}_B7I=A2y9&1zpJ;GoI@AoQ(?) zo+`=lygT+ry>ve%yWUuv7=H#kSf%-9w+*E!>^J|dWR&KA{iX}@RfQihdrg()PWqridU_9T9v_?QhOmH#2Z)?4eft+L4m zokCg;vN&jVMG~4;;r`n%fPe!*pg>7KKiE0~0{^v+{;izwTY z%YtZM<|26yW$5P?F@4)z$!qSz59<6rIG}zvlhYgAorD|y3yZ@zQ}6~yW_pVeDoHLW z!)@ijZ3a{n`_bl3?86yyI8!Qt7i#@V1U+*{r*~=c+r}cNKN_hG%f5O~Dg4%oDv+4GU$lNE6V{T0kUsh;~v+N`JH~ zl`nzdYd;FH_f&JrBvb;`Qv&QfHF%!jtN5OXKHQE^HQpRj^-6->#eoKja2KBsF?9UI zy7+UrlJ#D{&(Fct+|Vv+p9jNBLyWc8k4fZUut0VQuzF1e**#zU@v<}v`Q zo>yeB&?{D+Q=1RDZv8nN02#s#2kT`7vy9Q*pLco6Pq6IUqbR-DtyOzxQm4W)~J8v1~P=nO;Nb7_7FXf()7NFY5`XDg-hLbd|>5$TdtE?!k4>zMiB7y zT(dVW2Fn~<_I5OFe2C|HncTz9TzaRkM{JZI(6@dm^|0*LaLH3y^z!#HU)-h3c-_Qz z3pPK_e__S))o^^9rvU-09S-(=(OW;dedHXWb0n38dq`X;u-r>2OwLhPnpngE1^LFX zvGM+pAxsEqE>qHc@rQDnkl)iIBU8uMEcC`RBG81#45^+{-xV=Tjy*vtca%AX@9Wk< z{Lf)tc+J1M#@hLX@V6mwCNM@ z%?QCiaswH{FGSZ)k-p{HnQ}w2~7hici#zKAKQt=Ge5R?Vq6IA6Z6Ux_H>glX0F$(&(+eCZaFvFxI6`z&f6* zgcQvR+a@O9tHb05vO|Ey5#e~s#_@D~1g+X)JG&H}W=+48hj!F>MoS9+!rsL5kK90p zFu65NUl{49YG7_}2wk05$e%g#o%Xi!muh^5r)apD?F@|^3TZ=>sPtbikC(lL$cNs$ z+rOiwQ&|x6Ksq|kL?M*wS9rEk@^O6Nt{CaHSLcG~^k~)RC?wbtFA9}F`5o<5oC$^y zuucI|bxL^V8h_>ljrUwY=WUK7jV9h{ohrS11sn0%Tu>0fsPF*Bt)cC2@uV>mRRcG< zx@M}@zZUAeqR;vEDNG!iSY0QR^%1n(bfsw}7)_$pRlP9mQ1Y|CZ5o3+W&yU>C5D2G zEg;LvpDEsLIxgIqJ1G-1Uh;HG-WI<0K;&h9${Pn%2>38gzzicf3s=Tw)j5hk1pWG^ zhwkcJ1vQ>#TMdgdYobmICkwa&PYgyBpvT+~QK#95l%;~B6u^+ask?78n&rhmhpVO5ZD z{(7(pb2y-ZA~@WSpF`s^M#V)W)|;Zf(=W+3TAQ33(A1gPANVdG+)asu!2;PKz@{kN zDGkFoDtPdXizohMflsm0L*F7aa5BhZat-smLGF(q0~x~n*xEL7-e?-hAq&Yph&e`U z1@BR>Y2aN>+D4FIM;+}$XhwZSx`=qc^GW=OB~kSQ*Zz58WSe|9pCXQgzs)kFiq05sbG&Fo1(&9xMX^w^;1nbir3$>6W@BS|7is zFWXP^5F_nYPX^Co@Pn+IBv6o;oi*_N$Xkm7qv?$#1g_jtcp7v=-^O@2W9}DymyK0` z#yd-dibrl^zewK!?G%;d%vK+&9`F|2#7&C~`KgiU`zvD{gsw*MmUjgEC}VyMWGTxd z&Hg0uON`w`Kv22z{$4xurkgI$=Sg(B(3szTyFz$(>Hgy|MVcVm4$@KEVXfnj_FTWW zEg)w@&?N>#cs&G{P{EB=TybrRC_Bk+dO&ACBCsGm_MlmmQvoJ7kR1Xnni&sg5z-Z9 zUbM*%woy!d8BIRTo=7CN!(W=5{ zzmg8@PRZ;FUiZV}Y;|g5NCzry4ccvnnx}nGxnIT-4_*+?Ash95Y{z{Ws&veQpMB`c znZ#L(0m-iD{FNjt971o7k={&=uk?G4kxn!qIQ%O386(}ki>&^~pp!!jXj+~HFRPfG zMc(Fxuv=XxXG&;O9nQ1zD`t0M$YwLs^M6G~m9E}Q7}{KcphX(&?R-7uR`1B?_`J5@ zx+l2ivd6q`4FPZV^gUZ8t_yH{L*ZpQ;Hu(>`GoEIPAl$2uWphh?qlNLaszD`2)d9e zZFav)RX=PLpD7gkqRS{Kd;_|6M zn#ZqtnS|_X8EDetqgCT_&w}(-)~p07lzg=$Dy^lu<%z3QSRvrjSJ5SMc_7ZLi{X~2 z;yGv}```{}n%7Q`nM8C(nhy;Mf|q`9sws%7_U2gboBx3A^DO!d2^ulCtA^{**w{SH z(61b3!wnKtSF@3Mo9pM8)V&(jbOf*sg3fZw<2`nJ=+o(b%MG;YBj^%HERZDoBrMdQ zDOd1&iH!*V#o_vv&~ms(3D+8hcjYIT+(32+u({mHWt`edJ6Xn#nLYy#U1_AdSn1P0 zsFLt%GSzPdKlvj!kReQN5jhvN=y-RlV8+g8pEa)pZU`ERyvMTkOPzJ)4vrImrhBdG zy_QX$HOh*Ft@_KJirl4q+wd{Y%X+zvCR(#oTK3nPQr5AwAxN`&eITvqO z`CS3|l^gfU)w0W-jP{lds%0X>rTJVE2=KOP5U2=h2b(Y-;^k4H~IMp5^2K9HR8 zKT3)3vByeO4aF7N?~8>VgY7sM1lGoPy)y`zCWtN4tYvp?or#7Xk*cg8k_^tC|610b zsxEs=@9jAG4i|@Iv>PUqFwBX@V;38ui*;y8%l&zvrISg=nZvpQEVx4#BP;2nqZVhU zIn26L;qrQWPb54dz)%n^AHAB}dQm&=dOuy$Kj}m}$(#{q2}@FpWgaHKoEV1|>QrlH zA6k79EqK{&?xQEL*MTg6ayv+K=|idvks2fJA|VvSS>c^=#7MTsmllT{F_f7vH%0f! z@lWEyr3;;ZUg=zc?w5qMrB?%;xsYGHRuGVqI^0qIvYj#Tj5PAyA)|nLvx@ld4FO43~S5Hb8jBO-u@Z;SVvR@NKLtVYD*5X%wrSzF&}dh5 zy6PG0v>!G@Cz;S{{_W=+!3j`vuEpHejVdZxwuN3FVsdhc$}x%1pkJw}1@v!1C{lHM zkA|ykt$*NaeA@jY3+j9|wQXmWSi|)zok`oVXd3k*={8+_u^4iJx`(4Tg@EbR5?#=< z<}8tD&tEmWb~6ts%RE0;>0-rD87|2_^alT+C-e8yR?Lybj}@SX#xy|wnIq0GU68*z zyu95P;E&Huf*S`)0pYc!+TpYIDg4U9>og-J^?&19_SH1_U`q9|=CyRZx8PH(leF_$ zM|q|N=zd8>)u7>L&F%8&&by0f6~^fsKT}qjcoM5vUYy9!6>xCE@(w_c6Ggy3)d3Xq zhudEMg(*;q?RVaR7V$}=-_NN0_0*B?-$@CeCywBs@u0l0?lsEEvBA9|swfhjr{vGM z)AK^zPc1gh?K7rk1TNgg_|CnN8GgDYjp5}zS^Yd4g5e<)>8HsXi|2KYBO2qtO6kV& zH$M<&fTUmW@0}e8GZ2~5El>&tKE0MlxH;bjv0lLNJ5mv;f*Q>khCS~`5eqVRQ}wY~Mv{&`!w#4F*F0l<(Vu>SD1VO_2s2RXC;THU zzGv16WQ(hoHSOZ0tzVH^1Phir>R){x6?ntxW{XMMAn}ZY$)`k9%oe6 zFF&v`zp!xNF#E5Z7RDCzAODBUm^AcpMk*$D=pRW* zBq$e9H2yNVfP(gy$psYKzf3NmI5|95d`n9zhjPesyxB$1t=@+3S5&KRk3B?^8K4&J zS+2ogjs5)4>%pF@`k>!#_D1c17NTB2;q=So0*bm{CKpg}{?fQ$ixFtwf-Od%s5tz< z>X1<4?U8e?dyi_opNol+_2y9KjcIbBj0|FZkGtKGk7ifHqB5#QbA?gJb{k}{tpdk5WaskECFOBQZVgxuDz!oD=tQUS@MFZ|p z?mv9WL+huOQ5uO4mo7HCaU!ZOu5jm7ZRiy>k#~NbT0+m`374;S<;uR&GdyUri>6w+^ZvNxg1-qjGD9G zaXZ*%-szhP#Me1eA?gJb)4fbCpwRDSasfq%FO3Vf7=g~-u*C=znuQ-&&P&|49aw7J z(Ng)K&)Fed!KDs$PHQ<`UFLGA)i z!%3Q^J1^+v=*}*V9|Z;#Rb)wj&cm~7ew%-D%x8x!?r~zJE~2aD5o|HO&{0vSh}lkHG`7UklzLiQjDos} z+B`T}*3=k8;#qwIeqjH$mRuUthYTOU8FBC+^%%p1*m}*=J^J21(X2=vo7VcZ@#BCd z>zGR}=VfwXqhUBtQ*4)=5*0ICZ(bTDT87AMcurkN7o5E1B2gBtyI#EKUoQ7?r>;mWv8YC~VVuM(;({IlH~fzRq>Ug^8Jv=cj*fB1)9|CQ@yc=byzv0hh* z->x0G38_ubVE3C@zVTkhyC$236v6-S!~e=v9y(9)*1Za+tM+c2d4{@ySwW`U?9G&n zj*&B8hNuw2OPfR3Vg!oN!WQFt5Po8HT(}JjL3Y5PnL)P_(*~<$<7=&ALlSoqNT4wM zz)C#wmnMF2*5CA*Q+r|MJ&DVRwoeWjt*L#3{2+_s?XP=hG$ z7gC*?0#d#Mp)ZVV4~X1_2Zbl^GSNBLuVpS1v6~)XT_%@7Yi>1#_BRUh@e{6F8r1n7 zG2z$lx_;(Ul{>$C6q3$-nOt9{C<~GL&ldW{RR^{z^Y4-7bFb#3ofJBmDP0YFJV<|; zToi%S0pOwe=ewiAi+xMMf+)9t&e#`JM_%dY%EY@o6;Gug}D`Xm2y8K$pfgO7WyRI# zM)k&%bn=^qQ=6NdvpcRn@H&ovA5#7gr;UkHD!@!1>SJy^X351Hq6oG;NS44p&9^oD zq=eQ@)1hiPbZ<(BWLgrv_O-Fkg(IT4A8x2DMuoB2+>l;drNBwhkt6MjinvKzZ1j!Q z?*kGw4g4hhSFYtag;K+_;SEW7OVmatHq_?_J1uMdCc0{&2I4f?#sZhg#c6JYgwxMw z43=vd;JJg|g)!M2+3w1b$w?h{%}9HQ`!c!K(JB}0Tjru4E4Qmx2u?iPh%JuvJsc*m03eAi3%reK#414o%(o$^NiCzGNV_%Qz;n8`2 ztB+ZzeiKozRXcLOp%1&BG*v8uArfM`w~|1VU-JicJ`5$v_$s=Pz+cyY%TL znenRW+55r4Ho@{V0@hQ)A4B9OyF^$l0YsO{1+j~jMq9c@4L*p>GnYwRUB^YszD0H6 z=fe=0A=1H&b7@@TzaP2;$_fWc`G&&fFHcJ?#!a5l*4TIl+N?hHs7@W$Gh%t7yI=ER z8d<62`~jRPhQjOVyfZo9Q%xam!6J5FK7~(MOl%xdvk}2^$8pVhaHckR4#nJ)K0~!J z5F_Nb18bHL@||q-JEZ90qWp!oec(IgCi-t~$onbW+@M4y8Zq)Qz0u+&d4xqPXtha3 ze419M1c%zA>|R^46qGDG*Ao9~-qT&Gl_F?I*XjHewNqv)K)_K>-&4hPOA$24E+QFvtEj$ zVpVtkWN?9|3bxHy_VjPw1 z5vlLL_B%0I+L<}H^OWit6~)fmRy@9SC_~66{~=jGUMwU)cRzVb`+f`JvdOWem5V=I zMYW`mK@FYbasSfl?at_y{!H*moMVv(7y4y#CDT67^x?fCO@*w}9zIqvf8X~7@)A>9 zx2s z=^IkTRCnClr8!Lcw75;Tmx?(cEW5)u!mU2LT~X%O{VzO@|4YU8LZd4QS{M`nTNy3t=eh7?=b zs=Mj=ME>(XZcG1_%O;V@R|&-iw0xH0!1l65fzJ11@s5&1(5$FK)`=L$3uH29{LhQ0}U@!lxCYNBcJ@4&2ee zQoU76U(E9|uy%f!dMj@P@_z_81usqesR+53m36x~3LcLN^C{ zSDx>!um~A0U47(Qr~6ZW?CMEvcqzJ-?LVCMzs}*)W`-szrpKpD*J|=^zmP*&^Vi*5 zyhu%L#kCNL?DrCbpToD3_v&sNmfO7@#gY@`li{-yl#Xx^s1^EYFlhhT|4LCS&eO zO6gMYi#7QcfeRjbtbcs-|4-KiTZ}-rd)Q(GYLN>+3IDcxgEX`$KYVBmsFUtxasf5m zy-Y5kzPy*o1=Qmfo~tyC9PKN$IapogegI+6I-k?rx$kS!zSo47;aul5$F@nSJ?bMW zkzoM}V}p;tq~nNs0ky)tOfH~~xtGZW)L8e@xL}JB=s*oyj6fZ2;Rp7dDRSq^w{4dD zkm=eH&2JJ-xF=e5ZS2<3osP^ah26ouM3Nj${>-DjNmNQMl*wL*;sQ0ky-Y5k{;1Zt}aKd{;vm%y-5W0QL0Y5iP> zI_4Vu;l^kAHajL{p<&;#r;XKfpV#yA(v4TPl23%kff^CT1?s+fnOs0kS}&6esF&+y zasl;4h3BfE`!4TJbgFiBI(JVqP!nsPS1m%|=4)w>If;TEs+%cpw+j{mUvEvP+ep@u zbyl|_>IKw3^)k7Dx~yI%7f>_SOXGqqMxd}OY%v10AcY@T)NG2LznHmZvSOQ6S$Tt< z7DH<$Uj`G!-0nNHjAp0Er31UI_zsUXH|1@{w++m$GhtiOk`#HRWY&(2`z{9Q zBkBdzYVfI5<1CKpiS(o5rldnMRkZ%+IBam&9xhW>tr5$FwS!EjlZ#@1e~3(Og* z_333C7K!ke%YT4~?vlN?zmhBFl$d38&Xp2+c5l>_rvr;~RwDpN8A%6Q*$ zo%p2*Y{c78pl|s?X@)-L1~UKq`8go$00_O`FLT(BAMC?l|Azi8@UI4*CeSbu((g~R zNU%?R$VfVW(*TVt5QGGMj@b|X6;?(^v~x2fXxzllZiH?{wKdEKRrg2NPbFJDu#`I; ze(w1D0)Wi^#RVMvxd7OwKI8*fk4vBz)&1jXUkK$_EQM_7aT~CT5qd{5-0<5kq+R@rXK~n@k60u7 zK1%!~ST-Oapz^)PxUMq$!w$Y}0!EDX580WwxyF-0c5KU|ME)y{MqUmkWT_GELWT0F zo_~5=0?{M1llp%0h)=5_Zn|IR#ZK%vr6{J!-0d4RHlmD!!W@ ze!0FwjqCv5c`Eh#$<4r{hVFSfeSI|0XGSLb=meu<<9DiBH-aV#o}I#;HV4LZt*C$36AY;-gZE=SptumRE~7*_l=}QiuH|Put0VQuw%uXd)L2>)b$OLewO{P zd``_>@gbF8DkCebzLMmp<^S7X2*?oDARwjdLQ4^6;pR#R3G-aHPC#xbdVblrnVTXE zogHA^6eI{3sCEa1?0kks}p5t#`;eJ*CZ<-R4(i~bQFcCrDmH<$c z2?dGfl|N8bh-p1cIvf{Bo$rqd$39r7Ajn0-#^z|{P8EQFfvPQ`AUgrwWn=dU%xTBM zhW08e2A_g=@4hCyaI@K2ySW}VMh$Jh2vA1(yB`FQvmxlBlZX$lH@R`Qd%yjFevJ;y z_ej>e!GW;*PGnl)uoO`iOl}}M1XwLO_n{&x?`_7E&Gfp%KJdyXbB+77w{$cbm>y@n z8h85B4+6*#Cbw4zK9}>=#lTk=cb|W}H;_R%M&5ajL-s}_^x(?+!wYCPQlQdL=p@e# zf7CMCH+TOMqiUOKvS28z5pIDt$IkOgWKfU0u3r!OP8xukPAEu8@4yez2^rcF1%KKS zx)!&f-f7KKQ8wBe7Rcb-{=3G|hBE-gnoy7$rVWDH2Mp@jG^Vi?mhYM}* zHild?xIQ2SMf*roe5GI zU~&W5A;5O5x(-gzswdagigc2+Q$3S>zkADj??p+f041imC(`^Mxq%E}a$oCu86d@i_W=@_H9?%;x&cr;+rr#2pFiW6V~Gr=%9meC^0AvGkiywE`m3YIMbu(1ujV}Rmx(iJ<66D;fB04 z_2tiQT#Zlt{OOX!ru1MA2Q*Lwhs)g<%|@pE9@37iTXvg`u+H@p=#!4Y9X{6tMdjc` zbYvJTkR1Z-j8;!fcYd|;!q6cxiHFT}oe;)8v+r2#{_a$r0vFloA3p{%g!yqjlRw2x z%W3zViwJ=rX+22i`W(+pG4sUJo(EMn?H?XNUp)&@HVgG0&Q#GxH;Fg1A?Sw9AD455 z`g^9InU=@wzQ%7GqZm?vE(V}_SSZM(X=Wzw-b>{akA^~IIppOzdK#lNh2T#$(QJx| z3SXcj0V;Hbf=E}|7c#SKhbsTL)$-6J|H%AX(fE*x?_7OLp%wo^J3e%*02Qpiw=E!N zL(oNm*N4MHti^S9c+^c6WyUr9Vj=P<5l<;6(x#|7gxMG-H;^3yYTR^Uq+PU=G9`)hsaz!zt;7onzJKHfGK9%JdR(!ptdU#wfm^he4Tx&>ojS1+c&62HG$XbTN0Gm?`s3df5ju>#PiyDJPez zJQU4t`?{XBA+o*slolp8kR1Xn6}#L4xdb_R>3I`+z|7-hBNo(%f`j$>KGgwwN#ZZp z|Hutw2$LI06iffi-q=opG-0aq=BV6H3I!r!YL-|7zPN`hUfy@0-zNa7MWK^yW!A|u zrW&=!bYM#&fB5n{R?G?-dE)jl+Kg1}StXuH38=8IAEQka~!RktiM8_Hz53{Dp59Ol}}M z1lU9ueR`sSD@AXJDXM(5U+FK0hy*kpbW`w3tW`Nxe=YwbH;^GrZq=BkY9oVX+qQ|J zo#2b9QX8!!LjS6{R37S^Bah!Znn1un#ir0n#;a(G3$g9;lgCSzZDe_17^plH z3euMLKns=N-KRE!#_9L7G!`3+hSl_fLYoc6H^d72@}PGFpt?{0-%&)MkP1SN%UZ&q zBtxd_xH~?ahBl#R(e`ZMYa3BX6J1PV&v6{l7vPQ$x0r|&$!c1>E8p-j5k?#B4Uml+enGiBKPHi#zhe>* z*1zI1xsx$y4Q^h$BdY>^GZ;W=D-@(Q@$`0Gr_^|?mbNR+?33p>h6`yjE7$>&!aJm+ zT(!_U8&J(E6r^s=ei>D(Rm#d1SsaV1z~Ldm2lB2&qX#@+HlN}j|N80#RE+w2g8_02 z1YIUQWpAOf-WYI@rD|X_;2}!KW*mL!*WsLMg#KZ3zG4z4H;^3yY&K!ZCRzc0#e;7= zBIc`o8om{jAvP}1hXP1MIC>42D*wn0WC)Wxw42j_|4mL7YL4l9{_*~ZPYEr93mZ3E zrMq{FW`jhbFS-CKpoLDdRoAG%nyxM5`TUl5AhBiHWKA;OT?YBhSn|z|MxyU15HL_> zEEL3?qDUiC=;S2*%9wF;=epW-#f*lDQ?HTPQD&38PAd_V2B72>3IYa+1o1mA?Gv4a zW_63vCzaim?lqD&-HaC#dlTRP4*JyzsFoG>?kS*<^RIVL{oALl{|~?N``uFl0)JK4 z|MduVRzFNiRV&nBysqI@)XH9k!lP#$_yat8&_C8iVoBQF0hT5J9bQ=m!u z-G$Tp?^MWB2YjqKS)t%sf6K%^t^AE^{e~dc4W<$q1Gg=3OOGUc+sc122q4q{2XQ=# Ax&QzG diff --git a/build/version.go b/build/version.go index c93598684..174910e7b 100644 --- a/build/version.go +++ b/build/version.go @@ -5,7 +5,7 @@ import "fmt" var CurrentCommit string // BuildVersion is the local build version, set by build system -const BuildVersion = "0.1.5" +const BuildVersion = "0.2.0" var UserVersion = BuildVersion + CurrentCommit diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 53a8d303d..d88646f20 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -140,10 +140,7 @@ func (sma StorageMinerActor) Exports() []interface{} { return []interface{}{ 1: sma.StorageMinerConstructor, 2: sma.PreCommitSector, - 3: withUpdates( - update{start: 0, method: sma.ProveCommitSectorV0}, - update{start: build.ForkNoPowerEPSUpdates, method: sma.ProveCommitSectorV1}, - ), + 3: sma.ProveCommitSector, 4: sma.SubmitFallbackPoSt, //5: sma.SlashStorageFault, //6: sma.GetCurrentProvingSet, @@ -158,10 +155,7 @@ func (sma StorageMinerActor) Exports() []interface{} { //15: sma.ChangeWorker, 16: sma.IsSlashed, 17: sma.CheckMiner, - 18: withUpdates( - update{start: 0, method: sma.DeclareFaultsV0}, - update{start: build.ForkNoPowerEPSUpdates, method: sma.DeclareFaultsV1}, - ), + 18: sma.DeclareFaults, 19: sma.SlashConsensusFault, 20: sma.SubmitElectionPoSt, } @@ -297,110 +291,7 @@ type SectorProveCommitInfo struct { DealIDs []uint64 } -func (sma StorageMinerActor) ProveCommitSectorV0(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { - ctx := vmctx.Context() - oldstate, self, err := loadState(vmctx) - if err != nil { - return nil, err - } - - mi, err := loadMinerInfo(vmctx, self) - if err != nil { - return nil, err - } - - us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)] - if !ok { - return nil, aerrors.New(1, "no pre-commitment found for sector") - } - - if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() { - return nil, aerrors.New(2, "too early for proof submission") - } - - delete(self.PreCommittedSectors, uintToStringKey(params.SectorID)) - - // TODO: ensure normalization to ID address - maddr := vmctx.Message().To - - ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback) - if err != nil { - return nil, aerrors.Wrap(err, "failed to get ticket randomness") - } - - seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay) - if err != nil { - return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment") - } - - enc, err := SerializeParams(&ComputeDataCommitmentParams{ - DealIDs: params.DealIDs, - SectorSize: mi.SectorSize, - }) - if err != nil { - return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams") - } - - commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc) - if err != nil { - return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs) - } - - if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { - return nil, err - } else if !ok { - return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof)) - } - - // Note: There must exist a unique index in the miner's sector set for each - // sector ID. The `faults`, `recovered`, and `done` parameters of the - // SubmitPoSt method express indices into this sector set. - nssroot, err := AddToSectorSet(ctx, types.WrapStorage(vmctx.Storage()), self.Sectors, params.SectorID, us.Info.CommR, commD) - if err != nil { - return nil, err - } - self.Sectors = nssroot - - // if miner is not mining, start their proving period now - // Note: As written here, every miners first PoSt will only be over one sector. - // We could set up a 'grace period' for starting mining that would allow miners - // to submit several sectors for their first proving period. Alternatively, we - // could simply make the 'PreCommitSector' call take multiple sectors at a time. - // - // Note: Proving period is a function of sector size; small sectors take less - // time to prove than large sectors do. Sector size is selected when pledging. - pss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet) - if lerr != nil { - return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") - } - - if pss.Count == 0 { - self.ProvingSet = self.Sectors - // TODO: probably want to wait until the miner is above a certain - // threshold before starting this - self.ElectionPeriodStart = vmctx.BlockHeight() - } - - nstate, err := vmctx.Storage().Put(self) - if err != nil { - return nil, err - } - if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { - return nil, err - } - - activateParams, err := SerializeParams(&ActivateStorageDealsParams{ - Deals: params.DealIDs, - }) - if err != nil { - return nil, err - } - - _, err = vmctx.Send(StorageMarketAddress, SMAMethods.ActivateStorageDeals, types.NewInt(0), activateParams) - return nil, aerrors.Wrapf(err, "calling ActivateStorageDeals failed") -} - -func (sma StorageMinerActor) ProveCommitSectorV1(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { +func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { ctx := vmctx.Context() oldstate, self, err := loadState(vmctx) if err != nil { @@ -621,7 +512,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM } if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize, - sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, 0); !ok || lerr != nil { // TODO: FORK - set faults to len(faults) + sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, len(faults)); !ok || lerr != nil { if lerr != nil { // TODO: study PoST errors return nil, aerrors.Absorb(lerr, 4, "PoST error") @@ -923,33 +814,7 @@ type DeclareFaultsParams struct { Faults types.BitField } -func (sma StorageMinerActor) DeclareFaultsV0(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { - oldstate, self, aerr := loadState(vmctx) - if aerr != nil { - return nil, aerr - } - - nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet) - if err != nil { - return nil, aerrors.Absorb(err, 1, "failed to merge bitfields") - } - - self.FaultSet = nfaults - - self.LastFaultSubmission = vmctx.BlockHeight() - - nstate, aerr := vmctx.Storage().Put(self) - if err != nil { // TODO: FORK: should be aerr - return nil, aerr - } - if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { - return nil, err - } - - return nil, nil -} - -func (sma StorageMinerActor) DeclareFaultsV1(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { +func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { oldstate, self, aerr := loadState(vmctx) if aerr != nil { return nil, aerr @@ -1060,81 +925,6 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM } func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError { - if vmctx.BlockHeight() >= build.ForkNoPowerEPSUpdates { - return onSuccessfulPoStV1(self, vmctx) - } - - return onSuccessfulPoStV0(self, vmctx) -} - -func onSuccessfulPoStV0(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError { - var mi MinerInfo - if err := vmctx.Storage().Get(self.Info, &mi); err != nil { - return err - } - - pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet) - if nerr != nil { - return aerrors.HandleExternalError(nerr, "failed to load proving set") - } - - faults, nerr := self.FaultSet.All() - if nerr != nil { - return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") - } - - self.FaultSet = types.NewBitField() - - oldPower := self.Power - newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize)) - - // If below the minimum size requirement, miners have zero power - if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) { - newPower = types.NewInt(0) - } - - self.Power = newPower - - delta := types.BigSub(self.Power, oldPower) - if self.SlashedAt != 0 { - self.SlashedAt = 0 - delta = self.Power - } - - prevSlashingDeadline := self.ElectionPeriodStart + build.SlashablePowerDelay - if !self.Active && newPower.GreaterThan(types.NewInt(0)) { - self.Active = true - prevSlashingDeadline = 0 - } - - if !(oldPower.IsZero() && newPower.IsZero()) { - enc, err := SerializeParams(&UpdateStorageParams{ - Delta: delta, - NextSlashDeadline: vmctx.BlockHeight() + build.SlashablePowerDelay, - PreviousSlashDeadline: prevSlashingDeadline, - }) - if err != nil { - return err - } - - _, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc) - if err != nil { - return aerrors.Wrap(err, "updating storage failed") - } - } - - ncid, err := RemoveFromSectorSet(vmctx.Context(), vmctx.Storage(), self.Sectors, faults) - if err != nil { - return err - } - - self.Sectors = ncid - self.ProvingSet = ncid - self.ElectionPeriodStart = vmctx.BlockHeight() - return nil -} - -func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError { // TODO: some sector upkeep stuff that is very haphazard and unclear in the spec var mi MinerInfo diff --git a/chain/stmgr/fork_no_p_eps.go b/chain/stmgr/fork_no_p_eps.go deleted file mode 100644 index f74f20cdd..000000000 --- a/chain/stmgr/fork_no_p_eps.go +++ /dev/null @@ -1,127 +0,0 @@ -package stmgr - -import ( - "context" - - "github.com/filecoin-project/go-amt-ipld" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-hamt-ipld" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/address" - "github.com/filecoin-project/lotus/chain/state" - "github.com/filecoin-project/lotus/chain/types" -) - -func (sm *StateManager) forkNoPowerEPS(ctx context.Context, pstate cid.Cid) (cid.Cid, error) { - cst := hamt.CSTFromBstore(sm.cs.Blockstore()) - st, err := state.LoadStateTree(cst, pstate) - if err != nil { - return cid.Undef, xerrors.Errorf("loading parent state tree: %w", err) - } - - if err := st.MutateActor(actors.StoragePowerAddress, func(spa *types.Actor) error { - var head actors.StoragePowerState - if err := cst.Get(ctx, spa.Head, &head); err != nil { - return xerrors.Errorf("reading StoragePower state: %w", err) - } - - buckets, err := amt.LoadAMT(amt.WrapBlockstore(sm.cs.Blockstore()), head.ProvingBuckets) - if err != nil { - return xerrors.Errorf("opening proving buckets AMT: %w", err) - } - - fixedBuckets := map[uint64]map[address.Address]struct{}{} - - if err := buckets.ForEach(func(bucketId uint64, ent *typegen.Deferred) error { - var bcid cid.Cid - if err := cbor.DecodeInto(ent.Raw, &bcid); err != nil { - return xerrors.Errorf("decoding bucket cid: %w", err) - } - - bucket, err := hamt.LoadNode(ctx, cst, bcid) - if err != nil { - return xerrors.Errorf("loading bucket hamt: %w", err) - } - - return bucket.ForEach(ctx, func(abytes string, _ interface{}) error { - addr, err := address.NewFromBytes([]byte(abytes)) - if err != nil { - return xerrors.Errorf("parsing address in proving bucket: %w", err) - } - - // now find the correct bucket - miner, err := st.GetActor(addr) - if err != nil { - return xerrors.Errorf("getting miner %s: %w", addr, err) - } - - var minerHead actors.StorageMinerActorState - if err := cst.Get(ctx, miner.Head, &minerHead); err != nil { - return xerrors.Errorf("reading miner %s state: %w", addr, err) - } - - correctBucket := minerHead.ElectionPeriodStart % build.SlashablePowerDelay - if correctBucket != bucketId { - log.Warnf("miner %s was in wrong proving bucket %d, putting in %d (eps: %d)", addr, bucketId, correctBucket, minerHead.ElectionPeriodStart) - } - - if _, ok := fixedBuckets[correctBucket]; !ok { - fixedBuckets[correctBucket] = map[address.Address]struct{}{} - } - fixedBuckets[correctBucket][addr] = struct{}{} - - return nil - }) - }); err != nil { - return err - } - - // ///// - // Write fixed buckets - - fixed := amt.NewAMT(amt.WrapBlockstore(sm.cs.Blockstore())) - - for bucketId, addrss := range fixedBuckets { - bucket := hamt.NewNode(cst) - for addr := range addrss { - if err := bucket.Set(ctx, string(addr.Bytes()), actors.CborNull); err != nil { - return xerrors.Errorf("setting address in bucket: %w", err) - } - } - - if err := bucket.Flush(ctx); err != nil { - return xerrors.Errorf("flushing bucket amt: %w", err) - } - - bcid, err := cst.Put(context.TODO(), bucket) - if err != nil { - return xerrors.Errorf("put bucket: %w", err) - } - - if err := fixed.Set(bucketId, bcid); err != nil { - return xerrors.Errorf("set bucket: %w", err) - } - } - - head.ProvingBuckets, err = fixed.Flush() - if err != nil { - return xerrors.Errorf("flushing bucket amt: %w", err) - } - - spa.Head, err = cst.Put(ctx, &head) - if err != nil { - return xerrors.Errorf("putting actor head: %w", err) - } - - return nil - }); err != nil { - return cid.Undef, err - } - - return st.Flush() -} diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 0f92f3438..c1f48b8f6 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -2,22 +2,13 @@ package stmgr import ( "context" - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/build" + "github.com/ipfs/go-cid" ) func (sm *StateManager) handleStateForks(ctx context.Context, pstate cid.Cid, height, parentH uint64) (_ cid.Cid, err error) { for i := parentH; i < height; i++ { switch i { - case build.ForkNoPowerEPSUpdates: - pstate, err = sm.forkNoPowerEPS(ctx, pstate) - if err != nil { - return cid.Undef, xerrors.Errorf("executing state fork in epoch %d: %w", i, err) - } - - log.Infof("forkNoPowerEPS state: %s", pstate) } } diff --git a/chain/sync.go b/chain/sync.go index 3c80b3275..2f48962f6 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -532,13 +532,9 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err snum := types.BigDiv(mpow, types.NewInt(ssize)) - // FORK START - if h.Height > build.ForkCCM { - if len(h.EPostProof.Candidates) == 0 { - return xerrors.Errorf("no candidates") - } + if len(h.EPostProof.Candidates) == 0 { + return xerrors.Errorf("no candidates") } - // FORK END for _, t := range h.EPostProof.Candidates { if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) { @@ -661,13 +657,10 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc SectorChallengeIndex: t.ChallengeIndex, }) } - // FORK START - if h.Height > build.ForkCCM { - if len(winners) == 0 { - return xerrors.Errorf("no candidates") - } + + if len(winners) == 0 { + return xerrors.Errorf("no candidates") } - // FORK END sectorInfo, err := stmgr.GetSectorsForElectionPost(ctx, syncer.sm, baseTs, h.Miner) if err != nil { From a109ae1490a254831bf8cb95549077f68dcfbf8c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 7 Jan 2020 12:41:26 -0800 Subject: [PATCH 02/61] introduce a limit on the number of messages a block can have --- build/params_shared.go | 5 +++++ chain/sub/incoming.go | 6 ++++++ chain/sync.go | 8 ++++++++ go.sum | 5 +++++ miner/miner.go | 12 ++++++++---- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/build/params_shared.go b/build/params_shared.go index ec4982cc1..0357bf03a 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -97,3 +97,8 @@ const BadBlockCacheSize = 1 << 15 // assuming 4000 messages per round, this lets us not lose any messages across a // 10 block reorg. const BlsSignatureCacheSize = 40000 + +// /////// +// Limits + +const BlockMessageLimit = 512 diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 79dc29b80..7af740c33 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -8,6 +8,7 @@ import ( connmgr "github.com/libp2p/go-libp2p-core/connmgr" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" @@ -33,6 +34,11 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha continue } + if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit { + log.Warnf("received block with too many messages over pubsub") + continue + } + go func() { log.Infof("New block over pubsub: %s", blk.Cid()) diff --git a/chain/sync.go b/chain/sync.go index 19d83df27..fa1a0ec7f 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -189,6 +189,10 @@ func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHe } func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { + if msgc := len(fblk.BlsMessages) + len(fblk.SecpkMessages); msgc > build.BlockMessageLimit { + return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc) + } + var bcids, scids []cbg.CBORMarshaler for _, m := range fblk.BlsMessages { c := cbg.CborCid(m.Cid()) @@ -295,6 +299,10 @@ func zipTipSetAndMessages(bs amt.Blocks, ts *types.TipSet, allbmsgs []*types.Mes bmsgCids = append(bmsgCids, &c) } + if msgc := len(bmsgCids) + len(smsgCids); msgc > build.BlockMessageLimit { + return nil, fmt.Errorf("block %q has too many messages (%d)", b.Cid(), msgc) + } + mrcid, err := computeMsgMeta(bs, bmsgCids, smsgCids) if err != nil { return nil, err diff --git a/go.sum b/go.sum index 34d5667dc..9fe0eb6e1 100644 --- a/go.sum +++ b/go.sum @@ -23,7 +23,9 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -584,6 +586,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -592,6 +595,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -797,6 +801,7 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/miner/miner.go b/miner/miner.go index 4c6c3d2a0..639ab6c06 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -17,8 +17,6 @@ import ( "golang.org/x/xerrors" ) -const MaxMessagesPerBlock = 4000 - var log = logging.Logger("miner") type waitFunc func(ctx context.Context, baseTime uint64) error @@ -363,6 +361,11 @@ func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *type return nil, xerrors.Errorf("message filtering failed: %w", err) } + if len(msgs) > build.BlockMessageLimit { + log.Error("selectMessages returned too many messages: ", len(msgs)) + msgs = msgs[:build.BlockMessageLimit] + } + uts := base.ts.MinTimestamp() + uint64(build.BlockDelay*(base.nullRounds+1)) nheight := base.ts.Height() + base.nullRounds + 1 @@ -383,12 +386,13 @@ func countFrom(msgs []*types.SignedMessage, from address.Address) (out int) { } func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs []*types.SignedMessage) ([]*types.SignedMessage, error) { - out := make([]*types.SignedMessage, 0, len(msgs)) + out := make([]*types.SignedMessage, 0, build.BlockMessageLimit) inclNonces := make(map[address.Address]uint64) inclBalances := make(map[address.Address]types.BigInt) inclCount := make(map[address.Address]int) for _, msg := range msgs { + if msg.Message.To == address.Undef { log.Warnf("message in mempool had bad 'To' address") continue @@ -426,7 +430,7 @@ func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs inclCount[from]++ out = append(out, msg) - if len(out) >= MaxMessagesPerBlock { + if len(out) >= build.BlockMessageLimit { break } } From dbc729d049f1d2bc2a7971694cea349f6c9eafb4 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 6 Jan 2020 20:53:03 +0100 Subject: [PATCH 03/61] Validate length of RLE Signed-off-by: Jakub Sztandera --- chain/actors/actor_miner.go | 14 ++++++++++++-- chain/types/bitfield.go | 21 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index a55010669..cd282c949 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -467,7 +467,12 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") } - faults, nerr := self.FaultSet.AllMap() + ss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if lerr != nil { + return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") + } + + faults, nerr := self.FaultSet.AllMap(ss.Count) if nerr != nil { return nil, aerrors.Absorb(err, 5, "RLE+ invalid") } @@ -937,7 +942,12 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro return aerrors.HandleExternalError(nerr, "failed to load proving set") } - faults, nerr := self.FaultSet.All() + ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if nerr != nil { + return aerrors.HandleExternalError(nerr, "failed to load proving set") + } + + faults, nerr := self.FaultSet.All(ss.Count) if nerr != nil { return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") } diff --git a/chain/types/bitfield.go b/chain/types/bitfield.go index d959e4f84..a9498d3b9 100644 --- a/chain/types/bitfield.go +++ b/chain/types/bitfield.go @@ -1,6 +1,7 @@ package types import ( + "errors" "fmt" "io" @@ -9,6 +10,8 @@ import ( "golang.org/x/xerrors" ) +var ErrBitFieldTooMany = errors.New("to many items in RLE") + type BitField struct { rle rlepluslazy.RLE @@ -106,7 +109,14 @@ func (bf BitField) Count() (uint64, error) { } // All returns all set bits -func (bf BitField) All() ([]uint64, error) { +func (bf BitField) All(max uint64) ([]uint64, error) { + c, err := bf.Count() + if err != nil { + return nil, xerrors.Errorf("count errror: %w", err) + } + if c > max { + return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany) + } runs, err := bf.sum() if err != nil { @@ -121,7 +131,14 @@ func (bf BitField) All() ([]uint64, error) { return res, nil } -func (bf BitField) AllMap() (map[uint64]bool, error) { +func (bf BitField) AllMap(max uint64) (map[uint64]bool, error) { + c, err := bf.Count() + if err != nil { + return nil, xerrors.Errorf("count errror: %w", err) + } + if c > max { + return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany) + } runs, err := bf.sum() if err != nil { From a63c34a9023c91156b8b37653ddb977d91cd232f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 6 Jan 2020 21:12:16 +0100 Subject: [PATCH 04/61] Handle faults in ss and pss correctly Signed-off-by: Jakub Sztandera --- chain/actors/actor_miner.go | 53 ++++++++++++++++++++++++++++++++----- chain/types/blockheader.go | 2 +- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index cd282c949..2c67ba74a 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -477,9 +477,11 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.Absorb(err, 5, "RLE+ invalid") } + activeFaults := uint64(0) var sectorInfos []ffi.PublicSectorInfo if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error { if faults[id] { + activeFaults++ return nil } @@ -517,7 +519,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM } if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize, - sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, len(faults)); !ok || lerr != nil { + sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil { if lerr != nil { // TODO: study PoST errors return nil, aerrors.Absorb(lerr, 4, "PoST error") @@ -528,7 +530,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM } // Post submission is successful! - if err := onSuccessfulPoSt(self, vmctx); err != nil { + if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { return nil, err } @@ -839,6 +841,20 @@ func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMConte return nil, aerrors.Absorb(err, 1, "failed to merge bitfields") } + ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if nerr != nil { + return nil, aerrors.HandleExternalError(nerr, "failed to load sector set") + } + + cf, nerr := nfaults.Count() + if nerr != nil { + return nil, aerrors.Absorb(nerr, 2, "could not decode RLE+") + } + + if cf > ss.Count { + return nil, aerrors.New(3, "too many declared faults") + } + self.FaultSet = nfaults self.LastFaultSubmission = vmctx.BlockHeight() @@ -914,7 +930,32 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.New(1, "slashed miners can't perform election PoSt") } - if err := onSuccessfulPoSt(self, vmctx); err != nil { + pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet) + if nerr != nil { + return nil, aerrors.HandleExternalError(nerr, "failed to load proving set") + } + + ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if nerr != nil { + return nil, aerrors.HandleExternalError(nerr, "failed to load proving set") + } + + faults, nerr := self.FaultSet.AllMap(ss.Count) + if nerr != nil { + return nil, aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") + } + + activeFaults := uint64(0) + if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error { + if faults[id] { + activeFaults++ + } + return nil + }); err != nil { + return nil, aerrors.Absorb(err, 2, "could not decode sectorset") + } + + if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { // TODO return nil, err } @@ -929,7 +970,7 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM return nil, nil } -func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError { +func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext, activeFaults uint64) aerrors.ActorError { // TODO: some sector upkeep stuff that is very haphazard and unclear in the spec var mi MinerInfo @@ -944,7 +985,7 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) if nerr != nil { - return aerrors.HandleExternalError(nerr, "failed to load proving set") + return aerrors.HandleExternalError(nerr, "failed to load sector set") } faults, nerr := self.FaultSet.All(ss.Count) @@ -955,7 +996,7 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro self.FaultSet = types.NewBitField() oldPower := self.Power - newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize)) + newPower := types.BigMul(types.NewInt(pss.Count-activeFaults), types.NewInt(mi.SectorSize)) // If below the minimum size requirement, miners have zero power if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) { diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 23fdab0f7..6b3f5c679 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -216,7 +216,7 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big return lhs.Cmp(rhs) < 0 } -func ElectionPostChallengeCount(sectors uint64, faults int) uint64 { +func ElectionPostChallengeCount(sectors uint64, faults uint64) uint64 { return sectorbuilder.ElectionPostChallengeCount(sectors, faults) } diff --git a/go.mod b/go.mod index 3283f456b..42b472943 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 - github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 + github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 9fe0eb6e1..91d98a05e 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,8 @@ github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1: github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab h1:bsrBNO1LwnhOLxPEXlSPal/WuY61mLJUCHYyD0NayHg= +github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= From 65724133ea898b5b55fe0c23001c1d64177360ff Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 7 Jan 2020 19:23:49 +0100 Subject: [PATCH 05/61] Add tests , Signed-off-by: Jakub Sztandera --- chain/actors/actor_miner.go | 10 ++++---- chain/actors/actor_miner_test.go | 40 ++++++++++++++++++++++++++++++++ chain/types/bitfield.go | 19 +++++++++++---- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 2c67ba74a..a4a11b178 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -472,7 +472,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") } - faults, nerr := self.FaultSet.AllMap(ss.Count) + faults, nerr := self.FaultSet.AllMap(2 * ss.Count) if nerr != nil { return nil, aerrors.Absorb(err, 5, "RLE+ invalid") } @@ -851,8 +851,8 @@ func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMConte return nil, aerrors.Absorb(nerr, 2, "could not decode RLE+") } - if cf > ss.Count { - return nil, aerrors.New(3, "too many declared faults") + if cf > 2*ss.Count { + return nil, aerrors.Newf(3, "too many declared faults: %d > %d", cf, 2*ss.Count) } self.FaultSet = nfaults @@ -940,7 +940,7 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.HandleExternalError(nerr, "failed to load proving set") } - faults, nerr := self.FaultSet.AllMap(ss.Count) + faults, nerr := self.FaultSet.AllMap(2 * ss.Count) if nerr != nil { return nil, aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") } @@ -988,7 +988,7 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext, activ return aerrors.HandleExternalError(nerr, "failed to load sector set") } - faults, nerr := self.FaultSet.All(ss.Count) + faults, nerr := self.FaultSet.All(2 * ss.Count) if nerr != nil { return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") } diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go index a522fe349..7aa4d73e8 100644 --- a/chain/actors/actor_miner_test.go +++ b/chain/actors/actor_miner_test.go @@ -3,6 +3,7 @@ package actors_test import ( "bytes" "context" + "math" "math/rand" "testing" @@ -13,8 +14,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/rlepluslazy" hamt "github.com/ipfs/go-hamt-ipld" blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/stretchr/testify/assert" cbg "github.com/whyrusleeping/cbor-gen" ) @@ -50,8 +53,24 @@ func TestMinerCommitSectors(t *testing.T) { addSectorToMiner(h, t, minerAddr, worker, client, 1) assertSectorIDs(h, t, minerAddr, []uint64{1}) + } +type badRuns struct { + done bool +} + +func (br *badRuns) HasNext() bool { + return !br.done +} + +func (br *badRuns) NextRun() (rlepluslazy.Run, error) { + br.done = true + return rlepluslazy.Run{true, math.MaxInt64}, nil +} + +var _ rlepluslazy.RunIterator = (*badRuns)(nil) + func TestMinerSubmitBadFault(t *testing.T) { var worker, client address.Address var minerAddr address.Address @@ -99,11 +118,32 @@ func TestMinerSubmitBadFault(t *testing.T) { badnum-- bf = types.NewBitField() bf.Set(badnum) + bf.Set(badnum - 1) ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) ApplyOK(t, ret) ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil) + ApplyOK(t, ret) + assertSectorIDs(h, t, minerAddr, []uint64{1}) + + bf.Set(badnum - 2) + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) + if ret.ExitCode != 3 { + t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr) + } + assertSectorIDs(h, t, minerAddr, []uint64{1}) + + rle, err := rlepluslazy.EncodeRuns(&badRuns{}, []byte{}) + assert.NoError(t, err) + + bf, err = types.NewBitFieldFromBytes(rle) + assert.NoError(t, err) + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) + if ret.ExitCode != 3 { + t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr) + } + assertSectorIDs(h, t, minerAddr, []uint64{1}) bf = types.NewBitField() bf.Set(1) diff --git a/chain/types/bitfield.go b/chain/types/bitfield.go index a9498d3b9..81031085b 100644 --- a/chain/types/bitfield.go +++ b/chain/types/bitfield.go @@ -19,14 +19,23 @@ type BitField struct { } func NewBitField() BitField { - rle, err := rlepluslazy.FromBuf([]byte{}) + bf, err := NewBitFieldFromBytes([]byte{}) if err != nil { - panic(err) + panic(fmt.Sprintf("creating empty rle: %+v", err)) } - return BitField{ - rle: rle, - bits: make(map[uint64]struct{}), + return bf +} + +func NewBitFieldFromBytes(rle []byte) (BitField, error) { + bf := BitField{} + rlep, err := rlepluslazy.FromBuf(rle) + if err != nil { + return BitField{}, xerrors.Errorf("could not decode rle+: %w", err) } + bf.rle = rlep + bf.bits = make(map[uint64]struct{}) + return bf, nil + } func BitFieldFromSet(setBits []uint64) BitField { From 5e775929be5ee2044e20f03c6c376d994be2eb64 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 7 Jan 2020 19:59:31 +0100 Subject: [PATCH 06/61] Cleanup handing and add tests Signed-off-by: Jakub Sztandera --- chain/actors/actor_miner.go | 17 +++++++++++++++++ chain/actors/actor_miner_test.go | 21 ++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index a4a11b178..6db0c3e4d 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -946,6 +946,23 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM } activeFaults := uint64(0) + for f := range faults { + if f > amt.MaxIndex { + continue + } + + var comms [][]byte + err := pss.Get(f, &comms) + if err != nil { + var notfound *amt.ErrNotFound + if xerrors.As(err, ¬found) { + activeFaults++ + } else { + return nil, aerrors.HandleExternalError(err, "failed to find sector in sector set") + } + } + + } if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error { if faults[id] { activeFaults++ diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go index 7aa4d73e8..a840ccf60 100644 --- a/chain/actors/actor_miner_test.go +++ b/chain/actors/actor_miner_test.go @@ -104,6 +104,10 @@ func TestMinerSubmitBadFault(t *testing.T) { assertSectorIDs(h, t, minerAddr, []uint64{1}) + st, err := getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr) + assert.NoError(t, err) + expectedPower := st.Power + bf := types.NewBitField() bf.Set(6) ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) @@ -127,6 +131,13 @@ func TestMinerSubmitBadFault(t *testing.T) { ApplyOK(t, ret) assertSectorIDs(h, t, minerAddr, []uint64{1}) + st, err = getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr) + assert.NoError(t, err) + currentPower := st.Power + if types.BigCmp(expectedPower, currentPower) != 0 { + t.Errorf("power changed and shouldn't have: %s != %s", expectedPower, currentPower) + } + bf.Set(badnum - 2) ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) if ret.ExitCode != 3 { @@ -215,7 +226,7 @@ func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint } } -func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) { +func getMinerState(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) (*actors.StorageMinerActorState, error) { mact, err := st.GetActor(maddr) if err != nil { return nil, err @@ -227,6 +238,14 @@ func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Bl if err := cst.Get(ctx, mact.Head, &mstate); err != nil { return nil, err } + return &mstate, nil +} + +func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) { + mstate, err := getMinerState(ctx, st, bs, maddr) + if err != nil { + return nil, err + } return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors) } From f572cdea47b8711b8b0c36a8282ad5991c12cd0e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 7 Jan 2020 22:51:33 +0100 Subject: [PATCH 07/61] Fix logic, improve tests Signed-off-by: Jakub Sztandera --- chain/actors/actor_miner.go | 14 +++----------- chain/actors/actor_miner_test.go | 29 +++++++++++++++-------------- chain/actors/actor_storagepower.go | 2 +- chain/actors/harness2_test.go | 28 +++++++++++++++++++++++++++- 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 6db0c3e4d..e5e8b3b1e 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -955,21 +955,13 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM err := pss.Get(f, &comms) if err != nil { var notfound *amt.ErrNotFound - if xerrors.As(err, ¬found) { - activeFaults++ - } else { + if !xerrors.As(err, ¬found) { return nil, aerrors.HandleExternalError(err, "failed to find sector in sector set") } + continue } - } - if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error { - if faults[id] { - activeFaults++ - } - return nil - }); err != nil { - return nil, aerrors.Absorb(err, 2, "could not decode sectorset") + activeFaults++ } if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { // TODO diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go index a840ccf60..f76509373 100644 --- a/chain/actors/actor_miner_test.go +++ b/chain/actors/actor_miner_test.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/stmgr" @@ -72,20 +73,18 @@ func (br *badRuns) NextRun() (rlepluslazy.Run, error) { var _ rlepluslazy.RunIterator = (*badRuns)(nil) func TestMinerSubmitBadFault(t *testing.T) { + oldSS, oldMin := build.SectorSizes, build.MinimumMinerPower + build.SectorSizes, build.MinimumMinerPower = []uint64{1024}, 1024 + defer func() { + build.SectorSizes, build.MinimumMinerPower = oldSS, oldMin + }() + var worker, client address.Address var minerAddr address.Address opts := []HarnessOpt{ HarnessAddr(&worker, 1000000), HarnessAddr(&client, 1000000), - HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid, - func() cbg.CBORMarshaler { - return &actors.StorageMinerConstructorParams{ - Owner: worker, - Worker: worker, - SectorSize: 1024, - PeerID: "fakepeerid", - } - }), + HarnessAddMiner(&minerAddr, &worker), } h := NewHarness(t, opts...) @@ -104,10 +103,6 @@ func TestMinerSubmitBadFault(t *testing.T) { assertSectorIDs(h, t, minerAddr, []uint64{1}) - st, err := getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr) - assert.NoError(t, err) - expectedPower := st.Power - bf := types.NewBitField() bf.Set(6) ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) @@ -115,9 +110,15 @@ func TestMinerSubmitBadFault(t *testing.T) { ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil) ApplyOK(t, ret) - assertSectorIDs(h, t, minerAddr, []uint64{1}) + st, err := getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr) + assert.NoError(t, err) + expectedPower := st.Power + if types.BigCmp(expectedPower, types.NewInt(1024)) != 0 { + t.Errorf("Expected power of 1024, got %s", expectedPower) + } + badnum := uint64(0) badnum-- bf = types.NewBitField() diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index 09a3eb30a..d3c2ad674 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -67,7 +67,7 @@ type CreateStorageMinerParams struct { func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VMContext, params *CreateStorageMinerParams) ([]byte, ActorError) { if !build.SupportedSectorSize(params.SectorSize) { - return nil, aerrors.New(1, "Unsupported sector size") + return nil, aerrors.Newf(1, "Unsupported sector size: %d", params.SectorSize) } var self StoragePowerState diff --git a/chain/actors/harness2_test.go b/chain/actors/harness2_test.go index d5d34795b..2739d0cc8 100644 --- a/chain/actors/harness2_test.go +++ b/chain/actors/harness2_test.go @@ -115,6 +115,32 @@ func HarnessActor(actor *address.Address, creator *address.Address, code cid.Cid } +func HarnessAddMiner(addr *address.Address, creator *address.Address) HarnessOpt { + return func(t testing.TB, h *Harness) error { + if h.Stage != HarnessPostInit { + return nil + } + if !addr.Empty() { + return xerrors.New("actor address should be empty") + } + ret, _ := h.InvokeWithValue(t, *creator, actors.StoragePowerAddress, + actors.SPAMethods.CreateStorageMiner, types.NewInt(3000), &actors.StorageMinerConstructorParams{ + Owner: *creator, + Worker: *creator, + SectorSize: 1024, + PeerID: "fakepeerid", + }) + + if ret.ExitCode != 0 { + return xerrors.Errorf("creating actor: %w", ret.ActorErr) + } + var err error + *addr, err = address.NewFromBytes(ret.Return) + return err + + } +} + func HarnessCtx(ctx context.Context) HarnessOpt { return func(t testing.TB, h *Harness) error { h.ctx = ctx @@ -177,7 +203,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { for _, opt := range options { err := opt(t, h) if err != nil { - t.Fatalf("Applying options: %v", err) + t.Fatalf("Applying options: %+v", err) } } From 64496690bba8efcd2cd7b40ca6f07ff187696017 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 7 Jan 2020 23:09:46 +0100 Subject: [PATCH 08/61] go mod tidy Signed-off-by: Jakub Sztandera --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 91d98a05e..3c8f189b2 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,6 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab h1:bsrBNO1LwnhOLxPEXlSPal/WuY61mLJUCHYyD0NayHg= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= From e41dbdad902190f33775474a2c432e929c71e17b Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 7 Jan 2020 19:46:55 -0800 Subject: [PATCH 09/61] reduce most vm errors to non-fatal, explicitly mark disk issues as fatal --- chain/actors/actor_paych.go | 2 +- chain/actors/aerrors/wrap.go | 3 ++- chain/vm/vm.go | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/chain/actors/actor_paych.go b/chain/actors/actor_paych.go index 32181e592..f9e55b7cf 100644 --- a/chain/actors/actor_paych.go +++ b/chain/actors/actor_paych.go @@ -116,7 +116,7 @@ func (pca PaymentChannelActor) UpdateChannelState(act *types.Actor, vmctx types. vb, nerr := sv.SigningBytes() if nerr != nil { - return nil, aerrors.Escalate(nerr, "failed to serialize signedvoucher") + return nil, aerrors.Newf(1, nerr, "failed to serialize signedvoucher") } if err := vmctx.VerifySignature(sv.Signature, self.From, vb); err != nil { diff --git a/chain/actors/aerrors/wrap.go b/chain/actors/aerrors/wrap.go index da12cb7c6..b3dea8f09 100644 --- a/chain/actors/aerrors/wrap.go +++ b/chain/actors/aerrors/wrap.go @@ -171,7 +171,8 @@ func HandleExternalError(err error, msg string) ActorError { } return &actorError{ - fatal: true, + fatal: false, + retCode: 219, msg: msg, frame: xerrors.Caller(1), diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 6fa6a432c..0912d9a67 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -259,7 +259,7 @@ func (bs *gasChargingBlocks) GetBlock(ctx context.Context, c cid.Cid) (block.Blo } blk, err := bs.under.GetBlock(ctx, c) if err != nil { - return nil, err + return nil, aerrors.Escalate(err, "failed to get block from blockstore") } if err := bs.chargeGas(uint64(len(blk.RawData())) * gasGetPerByte); err != nil { return nil, err @@ -272,7 +272,10 @@ func (bs *gasChargingBlocks) AddBlock(blk block.Block) error { if err := bs.chargeGas(gasPutObj + uint64(len(blk.RawData()))*gasPutPerByte); err != nil { return err } - return bs.under.AddBlock(blk) + if err := bs.under.AddBlock(blk); err != nil { + return aerrors.Escalate(err, "failed to write data to disk") + } + return nil } func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Message, origin address.Address, usedGas types.BigInt) *VMContext { From 3965e1053e8159d4c88edf52bd54738def73bf17 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 7 Jan 2020 21:05:45 -0800 Subject: [PATCH 10/61] fix build --- chain/actors/actor_paych.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/actors/actor_paych.go b/chain/actors/actor_paych.go index f9e55b7cf..05d9a9913 100644 --- a/chain/actors/actor_paych.go +++ b/chain/actors/actor_paych.go @@ -116,7 +116,7 @@ func (pca PaymentChannelActor) UpdateChannelState(act *types.Actor, vmctx types. vb, nerr := sv.SigningBytes() if nerr != nil { - return nil, aerrors.Newf(1, nerr, "failed to serialize signedvoucher") + return nil, aerrors.Absorb(nerr, 1, "failed to serialize signedvoucher") } if err := vmctx.VerifySignature(sv.Signature, self.From, vb); err != nil { From 61e2568b8d932fa8b243b8a551d59fee325ad69a Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 29 Oct 2019 19:42:16 -0700 Subject: [PATCH 11/61] feat(datatransfer): implement and extract feat(datatransfer): setup implementation path Sets up a path to implementation, offering both the dagservice implementation and a future graphsync implement, establishes message interfaces and network layer, and isolates the datatransfer module from the app WIP using CBOR encoding for dataxfermsg * Bring cbor-gen stuff into datatransfer package * make transferRequest private struct * add transferResponse + funcs * Rename VoucherID to VoucherType * more tests passing WIP trying out some stuff * Embed request/response in message so all the interfaces work AND the CBOR unmarshaling works: this is more like the spec anyway * get rid of pb stuff all message tests passing, some others in datatransfer Some cleanup for PR Cleanup for PR, clarifying and additional comments mod tidy Respond to PR comments: * Make DataTransferRequest/Response be returned in from Net * Regenerate cbor_gen and fix the generator caller so it works better * Please the linters Fix tests Initiate push and pull requests (#536) * add issue link for data TransferID generation * comment out failing but not relevant tests * finish voucher rename from Identifier --> Type tests passing cleanup for PR remove unused fmt import in graphsync_test a better reflection send data transfer response other tests passing feat(datatransfer): milestone 2 infrastructure Setup test path for all tickets for milestone 2 responses alert subscribers when request is not accepted (#607) Graphsync response is scheduled when a valid push request is received (#625) fix(datatransfer): fix tests fix an error with read buffers in tests fix(deps): fix go.sum Feat/dt graphsync pullreqs (#627) * graphsync responses to pull requests Feat/dt initiator cleanup (#645) * ChannelID.To --> ChannelID.Initiator * We now store our peer ID (from host.ID()) so it can be used when creating ChannelIDs. * InProgressChannels returns all of impl.channels, currently just for testing * Implements go-data-transfer issue * Some assertions were changed based on the above. * Renamed some variables and added some assertions based on the new understanding * Updated SHA for graphsync module * Updated fakeGraphSync test structs to use new interfaces from new SHA above Techdebt/dt split graphsync impl receiver (#651) * Split up graphsyncImpl and graphsyncReceiver * rename graphsync to utils DTM sends data over graphsync for validated push requests (#665) * create channels when a request is received. register push request hook with graphsync. fix tests. * better NewReaders * use mutex lock around impl.channels access * fix(datatransfer): fix test uncertainty * fix a data race and also don't use random bytes in basic block which can fail * privatize 3 funcs with @hannahhoward Feat/dt gs pullrequests (#693) * Implements DTM Sends Data Over Graphsync For Validated Pull Requests * rename a field in a test struct * refactor a couple of private functions (one was refactored out of existence) Feat/dt subscribe, file Xfer round trip (#720) Implements the rest of Subscriber Is Notified When Request Completed #24: * send a graphsync message within a go func and consume responses until error or transfer is complete. * notify subscribers of results. * Rename datatransfer.Event to EventCode. * datatransfer.Event is now a struct that includes a message and a timestamp as well as the Event.Code int, formerly Event, update all uses * Add extension data to graphsync request hook, gsReq * rename sendRequest to sendDtRequest, to distinguish it from sendGsRequest, where Dt = datatransfer, Gs = graphsync * use a mutex lock for last transfer ID * obey the linter Don't respond with error in gsReqRcdHook when we can't find the datatransfer extension. (#754) * update to correct graphsync version, update tests & code to call the new graphsync hooks * getExtensionData returns an empty struct + nil if we can't find our extension * Don't respond with error when we can't find the extension. * Test for same * mod tidy minor fix to go.sum feat(datatransfer): switch to graphsync implementation Move over to real graphsync implementation of data transfer, add constructors for graphsync instances on client and miner side fix(datatransfer): Fix validators Validators were checking payload cid against commP -- which are not the same any more. Added a payloadCid to client deal to maintain the record, fixed validator logic Feat/dt extraction use go-fil-components/datatransfer (#770) * Initial commit after changing to go-fil-components/datatransfer * blow away the datatransfer dir * use go-fil-components master after its PR #1 was merged * go mod tidy use a package updates after rebase with master --- chain/deals/cbor_gen.go | 22 +++- chain/deals/client.go | 6 +- chain/deals/client_utils.go | 7 +- chain/deals/provider.go | 4 +- chain/deals/provider_utils.go | 7 +- chain/deals/request_validation_test.go | 50 +++---- chain/deals/types.go | 4 +- datatransfer/dagservice_impl.go | 78 ----------- datatransfer/types.go | 173 ------------------------- go.mod | 7 +- go.sum | 44 ++++++- node/builder.go | 4 +- node/modules/client.go | 25 +++- node/modules/dtypes/storage.go | 9 +- node/modules/storageminer.go | 36 ++++- 15 files changed, 153 insertions(+), 323 deletions(-) delete mode 100644 datatransfer/dagservice_impl.go delete mode 100644 datatransfer/types.go diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 2a5f1f8a9..bd56e0551 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -526,7 +526,7 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{135}); err != nil { + if _, err := w.Write([]byte{136}); err != nil { return err } @@ -568,6 +568,12 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } + // t.PayloadCid (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.PayloadCid); err != nil { + return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err) + } + // t.PublishMessage (types.SignedMessage) (struct) if err := t.PublishMessage.MarshalCBOR(w); err != nil { return err @@ -586,7 +592,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 7 { + if extra != 8 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -650,6 +656,18 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.DealID = uint64(extra) + // t.PayloadCid (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err) + } + + t.PayloadCid = c + + } // t.PublishMessage (types.SignedMessage) (struct) { diff --git a/chain/deals/client.go b/chain/deals/client.go index 709432f63..798296135 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -23,6 +23,7 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/retrieval/discovery" ) @@ -35,6 +36,7 @@ type ClientDeal struct { Miner peer.ID MinerWorker address.Address DealID uint64 + PayloadCid cid.Cid PublishMessage *types.SignedMessage @@ -244,8 +246,8 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro State: api.DealUnknown, Miner: p.MinerID, MinerWorker: p.MinerWorker, - - s: s, + PayloadCid: p.Data, + s: s, } c.incoming <- deal diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 944846ed0..5c875749a 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -1,7 +1,6 @@ package deals import ( - "bytes" "context" "runtime" @@ -13,10 +12,10 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-components/datatransfer" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -148,7 +147,7 @@ func (c *ClientRequestValidator) ValidatePull( Selector ipld.Node) error { dealVoucher, ok := voucher.(*StorageDataTransferVoucher) if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType) + return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) } var deal ClientDeal @@ -159,7 +158,7 @@ func (c *ClientRequestValidator) ValidatePull( if deal.Miner != receiver { return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) } - if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) { + if !deal.PayloadCid.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 9e45ef8b2..7c0bde0bf 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -14,11 +14,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-components/datatransfer" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -225,7 +225,7 @@ func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState da var next api.DealState var err error var mut func(*MinerDeal) - switch event { + switch event.Code { case datatransfer.Complete: next = api.DealStaged mut = func(deal *MinerDeal) { diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index e1fcfcf32..35614f32d 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -1,12 +1,11 @@ package deals import ( - "bytes" "context" "runtime" + "github.com/filecoin-project/go-fil-components/datatransfer" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipld/go-ipld-prime" @@ -163,7 +162,7 @@ func (m *ProviderRequestValidator) ValidatePush( Selector ipld.Node) error { dealVoucher, ok := voucher.(*StorageDataTransferVoucher) if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType) + return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) } var deal MinerDeal @@ -175,7 +174,7 @@ func (m *ProviderRequestValidator) ValidatePush( return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer) } - if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) { + if !deal.Ref.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { diff --git a/chain/deals/request_validation_test.go b/chain/deals/request_validation_test.go index 6da68d219..f6e6a0e33 100644 --- a/chain/deals/request_validation_test.go +++ b/chain/deals/request_validation_test.go @@ -35,7 +35,7 @@ func (wrongDTType) FromBytes([]byte) error { return fmt.Errorf("not implemented") } -func (wrongDTType) Identifier() string { +func (wrongDTType) Type() string { return "WrongDTTYPE" } @@ -76,6 +76,7 @@ func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, erro return deals.ClientDeal{ Proposal: newProposal, ProposalCid: proposalNd.Cid(), + PayloadCid: blockGenerator.Next().Cid(), Miner: minerID, MinerWorker: minerAddr, State: state, @@ -91,10 +92,7 @@ func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error if err != nil { return deals.MinerDeal{}, err } - ref, err := cid.Cast(newProposal.PieceRef) - if err != nil { - return deals.MinerDeal{}, err - } + ref := blockGenerator.Next().Cid() return deals.MinerDeal{ Proposal: newProposal, @@ -143,11 +141,8 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) { + payloadCid := clientDeal.PayloadCid + if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrWrongPeer) { t.Fatal("Pull should fail if miner address is incorrect") } }) @@ -171,11 +166,8 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) { + payloadCid := clientDeal.PayloadCid + if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrInacceptableDealState) { t.Fatal("Pull should fail if deal is in a state that cannot be data transferred") } }) @@ -187,11 +179,8 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil) != nil { + payloadCid := clientDeal.PayloadCid + if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil) != nil { t.Fatal("Pull should should succeed when all parameters are correct") } }) @@ -236,11 +225,8 @@ func TestProviderRequestValidation(t *testing.T) { if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) { + ref := minerDeal.Ref + if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrWrongPeer) { t.Fatal("Push should fail if miner address is incorrect") } }) @@ -264,11 +250,8 @@ func TestProviderRequestValidation(t *testing.T) { if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) { + ref := minerDeal.Ref + if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrInacceptableDealState) { t.Fatal("Push should fail if deal is in a state that cannot be data transferred") } }) @@ -280,11 +263,8 @@ func TestProviderRequestValidation(t *testing.T) { if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil) != nil { + ref := minerDeal.Ref + if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil) != nil { t.Fatal("Push should should succeed when all parameters are correct") } }) diff --git a/chain/deals/types.go b/chain/deals/types.go index 8a368e80d..4ad60a4bc 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -108,7 +108,7 @@ func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error { return dv.UnmarshalCBOR(r) } -// Identifier is the unique string identifier for a StorageDataTransferVoucher -func (dv *StorageDataTransferVoucher) Identifier() string { +// Type is the unique string identifier for a StorageDataTransferVoucher +func (dv *StorageDataTransferVoucher) Type() string { return "StorageDataTransferVoucher" } diff --git a/datatransfer/dagservice_impl.go b/datatransfer/dagservice_impl.go deleted file mode 100644 index 98c15d062..000000000 --- a/datatransfer/dagservice_impl.go +++ /dev/null @@ -1,78 +0,0 @@ -package datatransfer - -import ( - "context" - "reflect" - - "github.com/ipfs/go-cid" - ipldformat "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - ipld "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" -) - -// This file implements a VERY simple, incomplete version of the data transfer -// module that allows us to make the necessary insertions of data transfer -// functionality into the storage market -// It does not: -// -- actually validate requests -// -- support Push requests -// -- support multiple subscribers -// -- do any actual network coordination or use Graphsync - -type dagserviceImpl struct { - dag ipldformat.DAGService - subscriber Subscriber -} - -// NewDAGServiceDataTransfer returns a data transfer manager based on -// an IPLD DAGService -func NewDAGServiceDataTransfer(dag ipldformat.DAGService) Manager { - return &dagserviceImpl{dag, nil} -} - -// RegisterVoucherType registers a validator for the given voucher type -// will error if voucher type does not implement voucher -// or if there is a voucher type registered with an identical identifier -func (impl *dagserviceImpl) RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error { - return nil -} - -// open a data transfer that will send data to the recipient peer and -// transfer parts of the piece that match the selector -func (impl *dagserviceImpl) OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) { - return ChannelID{}, xerrors.Errorf("not implemented") -} - -// open a data transfer that will request data from the sending peer and -// transfer parts of the piece that match the selector -func (impl *dagserviceImpl) OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) { - ctx, cancel := context.WithCancel(ctx) - go func() { - defer cancel() - err := merkledag.FetchGraph(ctx, baseCid, impl.dag) - var event Event - if err != nil { - event = Error - } else { - event = Complete - } - impl.subscriber(event, ChannelState{Channel: Channel{voucher: voucher}}) - }() - return ChannelID{}, nil -} - -// close an open channel (effectively a cancel) -func (impl *dagserviceImpl) CloseDataTransferChannel(x ChannelID) {} - -// get status of a transfer -func (impl *dagserviceImpl) TransferChannelStatus(x ChannelID) Status { return ChannelNotFoundError } - -// get notified when certain types of events happen -func (impl *dagserviceImpl) SubscribeToEvents(subscriber Subscriber) { - impl.subscriber = subscriber -} - -// get all in progress transfers -func (impl *dagserviceImpl) InProgressChannels() map[ChannelID]ChannelState { return nil } diff --git a/datatransfer/types.go b/datatransfer/types.go deleted file mode 100644 index 684b7f7a8..000000000 --- a/datatransfer/types.go +++ /dev/null @@ -1,173 +0,0 @@ -package datatransfer - -import ( - "context" - "reflect" - - "github.com/ipfs/go-cid" - ipld "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" -) - -// Voucher is used to validate -// a data transfer request against the underlying storage or retrieval deal -// that precipitated it. The only requirement is a voucher can read and write -// from bytes, and has a string identifier type -type Voucher interface { - // ToBytes converts the Voucher to raw bytes - ToBytes() ([]byte, error) - // FromBytes reads a Voucher from raw bytes - FromBytes([]byte) error - // Identifier is a unique string identifier for this voucher type - Identifier() string -} - -// Status is the status of transfer for a given channel -type Status int - -const ( - // Ongoing means the data transfer is in progress - Ongoing Status = iota - - // Completed means the data transfer is completed successfully - Completed - - // Failed means the data transfer failed - Failed - - // ChannelNotFoundError means the searched for data transfer does not exist - ChannelNotFoundError -) - -// TransferID is an identifier for a data transfer, shared between -// request/responder and unique to the requester -type TransferID uint64 - -// ChannelID is a unique identifier for a channel, distinct by both the other -// party's peer ID + the transfer ID -type ChannelID struct { - to peer.ID - id TransferID -} - -// Channel represents all the parameters for a single data transfer -type Channel struct { - // an identifier for this channel shared by request and responder, set by requester through protocol - transferID TransferID - // base CID for the piece being transferred - baseCid cid.Cid - // portion of Piece to return, specified by an IPLD selector - selector ipld.Node - // used to verify this channel - voucher Voucher - // the party that is sending the data (not who initiated the request) - sender peer.ID - // the party that is receiving the data (not who initiated the request) - recipient peer.ID - // expected amount of data to be transferred - totalSize uint64 -} - -// TransferID returns the transfer id for this channel -func (c Channel) TransferID() TransferID { return c.transferID } - -// BaseCID returns the CID that is at the root of this data transfer -func (c Channel) BaseCID() cid.Cid { return c.baseCid } - -// Selector returns the IPLD selector for this data transfer (represented as -// an IPLD node) -func (c Channel) Selector() ipld.Node { return c.selector } - -// Voucher returns the voucher for this data transfer -func (c Channel) Voucher() Voucher { return c.voucher } - -// Sender returns the peer id for the node that is sending data -func (c Channel) Sender() peer.ID { return c.sender } - -// Recipient returns the peer id for the node that is receiving data -func (c Channel) Recipient() peer.ID { return c.recipient } - -// TotalSize returns the total size for the data being transferred -func (c Channel) TotalSize() uint64 { return c.totalSize } - -// ChannelState is immutable channel data plus mutable state -type ChannelState struct { - Channel - // total bytes sent from this node (0 if receiver) - sent uint64 - // total bytes received by this node (0 if sender) - received uint64 -} - -// Sent returns the number of bytes sent -func (c ChannelState) Sent() uint64 { return c.sent } - -// Received returns the number of bytes received -func (c ChannelState) Received() uint64 { return c.received } - -// Event is a name for an event that occurs on a data transfer channel -type Event int - -const ( - // Open is an event occurs when a channel is first opened - Open Event = iota - - // Progress is an event that gets emitted every time more data is transferred - Progress - - // Error is an event that emits when an error occurs in a data transfer - Error - - // Complete is emitted when a data transfer is complete - Complete -) - -// Subscriber is a callback that is called when events are emitted -type Subscriber func(event Event, channelState ChannelState) - -// RequestValidator is an interface implemented by the client of the -// data transfer module to validate requests -type RequestValidator interface { - // ValidatePush validates a push request received from the peer that will send data - ValidatePush( - sender peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error - // ValidatePull validates a pull request received from the peer that will receive data - ValidatePull( - receiver peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error -} - -// Manager is the core interface presented by all implementations of -// of the data transfer sub system -type Manager interface { - // RegisterVoucherType registers a validator for the given voucher type - // will error if voucher type does not implement voucher - // or if there is a voucher type registered with an identical identifier - RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error - - // open a data transfer that will send data to the recipient peer and - // open a data transfer that will send data to the recipient peer and - // transfer parts of the piece that match the selector - OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // open a data transfer that will request data from the sending peer and - // transfer parts of the piece that match the selector - OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // close an open channel (effectively a cancel) - CloseDataTransferChannel(x ChannelID) - - // get status of a transfer - TransferChannelStatus(x ChannelID) Status - - // get notified when certain types of events happen - SubscribeToEvents(subscriber Subscriber) - - // get all in progress transfers - InProgressChannels() map[ChannelID]ChannelState -} diff --git a/go.mod b/go.mod index 541d7f8d1..1dad59831 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,12 @@ require ( github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 + github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect - github.com/google/go-cmp v0.3.1 // indirect github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.1 github.com/hashicorp/go-multierror v1.0.0 @@ -35,6 +35,7 @@ require ( github.com/ipfs/go-ds-badger v0.0.7 github.com/ipfs/go-filestore v0.0.2 github.com/ipfs/go-fs-lock v0.0.1 + github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 github.com/ipfs/go-ipfs-blockstore v0.1.1 github.com/ipfs/go-ipfs-blocksutil v0.0.1 @@ -50,7 +51,7 @@ require ( github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb - github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b + github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 github.com/libp2p/go-libp2p v0.4.2 github.com/libp2p/go-libp2p-circuit v0.1.4 github.com/libp2p/go-libp2p-connmgr v0.1.0 @@ -79,8 +80,6 @@ require ( github.com/multiformats/go-multiaddr-net v0.1.1 github.com/multiformats/go-multihash v0.0.10 github.com/multiformats/go-varint v0.0.2 - github.com/onsi/ginkgo v1.9.0 // indirect - github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/prometheus/common v0.2.0 diff --git a/go.sum b/go.sum index 32e0a5717..81f67fff2 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -93,6 +94,9 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 h1:aDoskukzly9o/fFbHGOZVZDOJc3eeo6IQwSp5bvkfb8= +github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701/go.mod h1:VEQZ6UfSBYB3wCl19LZUScGfNaIakrBlMSxiVATWPMg= +github.com/filecoin-project/go-fil-filestore v0.0.0-20191202230242-40c6a5a2306c/go.mod h1:pPW5DjeA3Fk89gLGxMYH7FFd8TzZ4H3NAryc7UWQVCg= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -126,8 +130,9 @@ github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -209,6 +214,8 @@ github.com/ipfs/go-filestore v0.0.2 h1:pcYwpjtXXwirtbjBXKVJM9CTa9F7/8v1EkfnDaHTO github.com/ipfs/go-filestore v0.0.2/go.mod h1:KnZ41qJsCt2OX2mxZS0xsK3Psr0/oB93HMMssLujjVc= github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0= github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y= +github.com/ipfs/go-graphsync v0.0.4 h1:iF98+J8pcqvEb48IM0TemqeGARsCDtwQ73P9ejMZIuU= +github.com/ipfs/go-graphsync v0.0.4/go.mod h1:6UACBjfOXEa8rQL3Q/JpZpWS0nZDCLx134WUkjrmFpQ= github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 h1:0xpCaaXvN8bPyws3ObiCn7G0KIfgbS9E132eL57dHx8= github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1/go.mod h1:8yRx0xLUps1Xq8ZDnIwIVdQRp7JjA55gGvCiRHT91Vk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= @@ -269,8 +276,10 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb h1:tmWYgjltxwM7PD github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b h1:ACSEK4f1SDQC+FJ4B4pqHFW14d7kEW2ufwXA/c7eLP0= -github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -281,6 +290,8 @@ github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+ github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs= github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2 h1:vhC1OXXiT9R2pczegwz6moDvuRpggaroAXhPIseh57A= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= @@ -319,6 +330,8 @@ github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40J github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= +github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= +github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= @@ -326,6 +339,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= +github.com/libp2p/go-libp2p v0.2.1/go.mod h1:HZbtEOrgZN4F1fGZVvkV+930Wx3DkqlpBlO8dIoZWds= +github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0= github.com/libp2p/go-libp2p v0.4.2 h1:p0cthB0jDNHO4gH2HzS8/nAMMXbfUlFHs0jwZ4U+F2g= github.com/libp2p/go-libp2p v0.4.2/go.mod h1:MNmgUxUw5pMsdOzMlT0EE7oKjRasl+WyVwM0IBlpKgQ= github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= @@ -334,10 +349,12 @@ github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Z github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= +github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8= github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-connmgr v0.1.0 h1:vp0t0F0EuT3rrlTtnMnIyyzCnly7nIlRoEbhJpgp0qU= @@ -346,6 +363,8 @@ github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGc github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco= github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE= github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= +github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= +github.com/libp2p/go-libp2p-core v0.0.9/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8= @@ -391,6 +410,8 @@ github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMg github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= +github.com/libp2p/go-libp2p-peerstore v0.1.2/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= +github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k= github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= @@ -413,11 +434,15 @@ github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= +github.com/libp2p/go-libp2p-secio v0.1.1/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= +github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= +github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwLx73ln1FTefR91Q= +github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -461,6 +486,7 @@ github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FW github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= +github.com/libp2p/go-stream-muxer v0.0.1 h1:Ce6e2Pyu+b5MC1k3eeFtAax0pW4gc6MosYSLV05UeLw= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ= github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw= @@ -528,6 +554,7 @@ github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oP github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= @@ -541,6 +568,7 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= @@ -607,6 +635,7 @@ github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHei github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= @@ -672,6 +701,8 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -707,6 +738,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -730,10 +763,13 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -760,6 +796,7 @@ golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -800,6 +837,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/node/builder.go b/node/builder.go index ee34b3600..ee15f8e70 100644 --- a/node/builder.go +++ b/node/builder.go @@ -243,8 +243,9 @@ func Online() Option { Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), + Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), Override(new(dtypes.StagingDAG), modules.StagingDAG), - + Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), Override(new(*retrieval.Miner), retrieval.NewMiner), Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), @@ -372,6 +373,7 @@ func Repo(r repo.Repo) Option { Override(new(dtypes.ClientFilestore), modules.ClientFstore), Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), Override(new(dtypes.ClientDAG), modules.ClientDAG), + Override(new(dtypes.ClientGraphsync), modules.ClientGraphsync), Override(new(ci.PrivKey), lp2p.PrivKey), Override(new(ci.PubKey), ci.PrivKey.GetPublic), diff --git a/node/modules/client.go b/node/modules/client.go index 205d39a6d..dc17996c9 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -9,6 +9,10 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" + graphsync "github.com/ipfs/go-graphsync/impl" + "github.com/ipfs/go-graphsync/ipldbridge" + gsnet "github.com/ipfs/go-graphsync/network" + "github.com/ipfs/go-graphsync/storeutil" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" @@ -20,8 +24,8 @@ import ( "github.com/ipfs/go-merkledag" "go.uber.org/fx" + "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" "github.com/filecoin-project/lotus/chain/deals" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" ) @@ -49,15 +53,15 @@ func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore { // request validator with the data transfer module as the validator for // StorageDataTransferVoucher types func RegisterClientValidator(crv *deals.ClientRequestValidator, dtm dtypes.ClientDataTransfer) { - if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), crv); err != nil { + if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), crv); err != nil { panic(err) } } // NewClientDAGServiceDataTransfer returns a data transfer manager that just // uses the clients's Client DAG service for transfers -func NewClientDAGServiceDataTransfer(dag dtypes.ClientDAG) dtypes.ClientDataTransfer { - return datatransfer.NewDAGServiceDataTransfer(dag) +func NewClientDAGServiceDataTransfer(h host.Host, gs dtypes.ClientGraphsync) dtypes.ClientDataTransfer { + return graphsyncimpl.NewGraphSyncDataTransfer(h, gs) } // NewClientDealStore creates a statestore for the client to store its deals @@ -65,6 +69,7 @@ func NewClientDealStore(ds dtypes.MetadataDS) dtypes.ClientDealStore { return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) } +// ClientDAG is a DAGService for the ClientBlockstore func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG { bitswapNetwork := network.NewFromIpfsHost(h, rt) exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) @@ -80,3 +85,15 @@ func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlocks return dag } + +// ClientGraphsync creates a graphsync instance which reads and writes blocks +// to the ClientBlockstore +func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, h host.Host) dtypes.ClientGraphsync { + graphsyncNetwork := gsnet.NewFromLibp2pHost(h) + ipldBridge := ipldbridge.NewIPLDBridge() + loader := storeutil.LoaderForBlockstore(ibs) + storer := storeutil.StorerForBlockstore(ibs) + gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer) + + return gs +} diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index 32b97767d..4c09a61e5 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -1,15 +1,15 @@ package dtypes import ( + "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-statestore" bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-filestore" + "github.com/ipfs/go-graphsync" blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" ipld "github.com/ipfs/go-ipld-format" - - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/datatransfer" ) // MetadataDS stores metadata @@ -26,6 +26,7 @@ type ChainBlockService bserv.BlockService type ClientFilestore *filestore.Filestore type ClientBlockstore blockstore.Blockstore type ClientDAG ipld.DAGService +type ClientGraphsync graphsync.GraphExchange type ClientDealStore *statestore.StateStore // ClientDataTransfer is a data transfer manager for the client @@ -37,3 +38,5 @@ type ProviderDealStore *statestore.StateStore type ProviderDataTransfer datatransfer.Manager type StagingDAG ipld.DAGService +type StagingBlockstore blockstore.Blockstore +type StagingGraphsync graphsync.GraphExchange diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 23fb35455..13c8a749e 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -11,6 +11,10 @@ import ( "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + graphsync "github.com/ipfs/go-graphsync/impl" + "github.com/ipfs/go-graphsync/ipldbridge" + gsnet "github.com/ipfs/go-graphsync/network" + "github.com/ipfs/go-graphsync/storeutil" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/host" @@ -20,13 +24,13 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + dtgraphsync "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -142,15 +146,15 @@ func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *de // request validator with the data transfer module as the validator for // StorageDataTransferVoucher types func RegisterProviderValidator(mrv *deals.ProviderRequestValidator, dtm dtypes.ProviderDataTransfer) { - if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), mrv); err != nil { + if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), mrv); err != nil { panic(err) } } // NewProviderDAGServiceDataTransfer returns a data transfer manager that just // uses the provider's Staging DAG service for transfers -func NewProviderDAGServiceDataTransfer(dag dtypes.StagingDAG) dtypes.ProviderDataTransfer { - return datatransfer.NewDAGServiceDataTransfer(dag) +func NewProviderDAGServiceDataTransfer(h host.Host, gs dtypes.StagingGraphsync) dtypes.ProviderDataTransfer { + return dtgraphsync.NewGraphSyncDataTransfer(h, gs) } // NewProviderDealStore creates a statestore for the client to store its deals @@ -158,7 +162,9 @@ func NewProviderDealStore(ds dtypes.MetadataDS) dtypes.ProviderDealStore { return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) } -func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt routing.Routing, h host.Host) (dtypes.StagingDAG, error) { +// StagingBlockstore creates a blockstore for staging blocks for a miner +// in a storage deal, prior to sealing +func StagingBlockstore(r repo.LockedRepo) (dtypes.StagingBlockstore, error) { stagingds, err := r.Datastore("/staging") if err != nil { return nil, err @@ -167,8 +173,14 @@ func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt bs := blockstore.NewBlockstore(stagingds) ibs := blockstore.NewIdStore(bs) + return ibs, nil +} + +// StagingDAG is a DAGService for the StagingBlockstore +func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, rt routing.Routing, h host.Host) (dtypes.StagingDAG, error) { + bitswapNetwork := network.NewFromIpfsHost(h, rt) - exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs) + exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) bsvc := blockservice.New(ibs, exch) dag := merkledag.NewDAGService(bsvc) @@ -182,6 +194,18 @@ func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt return dag, nil } +// StagingGraphsync creates a graphsync instance which reads and writes blocks +// to the StagingBlockstore +func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync { + graphsyncNetwork := gsnet.NewFromLibp2pHost(h) + ipldBridge := ipldbridge.NewIPLDBridge() + loader := storeutil.LoaderForBlockstore(ibs) + storer := storeutil.StorerForBlockstore(ibs) + gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer) + + return gs +} + func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, epp gen.ElectionPoStProver) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { From 5312340bf9b176403560098fcb17484d6a8b61cb Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 18 Dec 2019 17:05:07 -0800 Subject: [PATCH 12/61] refactor(datatransfer): use independent repo Use independent repo for go-data-transfer --- chain/deals/client_utils.go | 2 +- chain/deals/provider.go | 2 +- chain/deals/provider_utils.go | 2 +- go.mod | 2 +- go.sum | 20 +++++++++++++++++++- node/modules/client.go | 2 +- node/modules/dtypes/storage.go | 5 +++-- node/modules/storageminer.go | 2 +- 8 files changed, 28 insertions(+), 9 deletions(-) diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 5c875749a..f2d150a23 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -12,8 +12,8 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-components/datatransfer" cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/lib/padreader" diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 7c0bde0bf..0b1fa3900 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 35614f32d..a02c6daf9 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -4,7 +4,7 @@ import ( "context" "runtime" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipld/go-ipld-prime" diff --git a/go.mod b/go.mod index 1dad59831..068fb7b29 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 + github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 diff --git a/go.sum b/go.sum index 81f67fff2..f87498af4 100644 --- a/go.sum +++ b/go.sum @@ -54,7 +54,10 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= @@ -98,6 +101,8 @@ github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701/go.mod h1:VEQZ6UfSBYB3wCl19LZUScGfNaIakrBlMSxiVATWPMg= github.com/filecoin-project/go-fil-filestore v0.0.0-20191202230242-40c6a5a2306c/go.mod h1:pPW5DjeA3Fk89gLGxMYH7FFd8TzZ4H3NAryc7UWQVCg= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -155,6 +160,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c h1:+MSf4NEnLCYZoAgK6fqwc7NH88nM8haFSxKGUGIG3vA= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -624,10 +631,15 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -660,7 +672,10 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= +github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= @@ -675,6 +690,7 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CH github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 h1:efb/4CnrubzNGqQOeHErxyQ6rIsJb7GcgeSDF7fqWeI= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= @@ -751,6 +767,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -819,8 +836,9 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/node/modules/client.go b/node/modules/client.go index dc17996c9..f01bbf8c3 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -24,7 +24,7 @@ import ( "github.com/ipfs/go-merkledag" "go.uber.org/fx" - "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" + "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index 4c09a61e5..d80f86d24 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -1,8 +1,6 @@ package dtypes import ( - "github.com/filecoin-project/go-fil-components/datatransfer" - "github.com/filecoin-project/go-statestore" bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-filestore" @@ -10,6 +8,9 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" ipld "github.com/ipfs/go-ipld-format" + + "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-statestore" ) // MetadataDS stores metadata diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 13c8a749e..9725d3440 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -24,7 +24,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - dtgraphsync "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" + dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" From bf9aa4465bbafa8305a062583bc5ac6ad41413a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jan 2020 14:20:43 +0100 Subject: [PATCH 13/61] mod tidy --- go.sum | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/go.sum b/go.sum index f87498af4..3cc9acf70 100644 --- a/go.sum +++ b/go.sum @@ -97,13 +97,10 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 h1:aDoskukzly9o/fFbHGOZVZDOJc3eeo6IQwSp5bvkfb8= -github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701/go.mod h1:VEQZ6UfSBYB3wCl19LZUScGfNaIakrBlMSxiVATWPMg= -github.com/filecoin-project/go-fil-filestore v0.0.0-20191202230242-40c6a5a2306c/go.mod h1:pPW5DjeA3Fk89gLGxMYH7FFd8TzZ4H3NAryc7UWQVCg= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab h1:bsrBNO1LwnhOLxPEXlSPal/WuY61mLJUCHYyD0NayHg= @@ -637,9 +634,9 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= From 886898b137bf098dfc34422479650508b8fbebac Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 8 Jan 2020 19:56:45 +0100 Subject: [PATCH 14/61] Update to badger v2 Signed-off-by: Jakub Sztandera --- cmd/lotus-seed/main.go | 2 +- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 2 +- go.mod | 4 ++-- go.sum | 16 ++++++++++++++++ node/node_test.go | 2 +- node/repo/fsrepo.go | 2 +- 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index ee1481d3b..da715af3b 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -8,7 +8,7 @@ import ( "encoding/json" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - badger "github.com/ipfs/go-ds-badger" + badger "github.com/ipfs/go-ds-badger2" logging "github.com/ipfs/go-log" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index e98abdb0b..c747b7eef 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -11,7 +11,7 @@ import ( "path/filepath" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - badger "github.com/ipfs/go-ds-badger" + badger "github.com/ipfs/go-ds-badger2" logging "github.com/ipfs/go-log" "golang.org/x/xerrors" diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9bf52fea8..771e21702 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -13,7 +13,7 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger" + badger "github.com/ipfs/go-ds-badger2" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/mitchellh/go-homedir" diff --git a/go.mod b/go.mod index 068fb7b29..346fd3368 100644 --- a/go.mod +++ b/go.mod @@ -31,8 +31,8 @@ require ( github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c github.com/ipfs/go-car v0.0.2 github.com/ipfs/go-cid v0.0.4 - github.com/ipfs/go-datastore v0.1.1 - github.com/ipfs/go-ds-badger v0.0.7 + github.com/ipfs/go-datastore v0.3.1 + github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 github.com/ipfs/go-filestore v0.0.2 github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-graphsync v0.0.4 diff --git a/go.sum b/go.sum index 3cc9acf70..f96fb52cb 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= @@ -14,6 +16,8 @@ github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtix github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K172oDhSKU0dJ/miJramo9NITOMyZQ= github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= @@ -45,6 +49,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -72,6 +78,10 @@ github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzA github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger/v2 v2.0.1 h1:+D6dhIqC6jIeCclnxMHqk4HPuXgrRN5UfBsLR4dNQ3A= +github.com/dgraph-io/badger/v2 v2.0.1/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04= +github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e h1:aeUNgwup7PnDOBAD1BOKAqzb/W/NksOj6r3dwKKuqfg= +github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -136,6 +146,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -206,12 +218,16 @@ github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-badger v0.0.7 h1:NMyh88Q50HG6/S2YD58DLkq0c0/ZQPMbSojONH+PRf4= github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= +github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 h1:GSygNXXIRXFVntMR3Yr30imf40H9cYDAw72Qb0za2nE= +github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521/go.mod h1:oDKWqmQbLyyml3mgMtHgB7qAf6cQOEsJPRLFzfwKQ5Q= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-filestore v0.0.2 h1:pcYwpjtXXwirtbjBXKVJM9CTa9F7/8v1EkfnDaHTO3s= diff --git a/node/node_test.go b/node/node_test.go index f8592b889..4f9386af9 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -15,7 +15,7 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger" + badger "github.com/ipfs/go-ds-badger2" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index b41024e5d..120a970cf 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -12,7 +12,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - badger "github.com/ipfs/go-ds-badger" + badger "github.com/ipfs/go-ds-badger2" fslock "github.com/ipfs/go-fs-lock" logging "github.com/ipfs/go-log" "github.com/mitchellh/go-homedir" From b5328d01cfbc52ef3d122a32806e234b474da6ae Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 8 Jan 2020 20:06:36 +0100 Subject: [PATCH 15/61] Update to go-log v1.0.1 Signed-off-by: Jakub Sztandera --- build/genesis.go | 4 ++-- go.mod | 2 +- go.sum | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build/genesis.go b/build/genesis.go index cdcf0402e..08734644a 100644 --- a/build/genesis.go +++ b/build/genesis.go @@ -11,12 +11,12 @@ var log = logging.Logger("build") func MaybeGenesis() []byte { builtinGen, err := rice.FindBox("genesis") if err != nil { - log.Warn("loading built-in genesis: %s", err) + log.Warnf("loading built-in genesis: %s", err) return nil } genBytes, err := builtinGen.Bytes("devnet.car") if err != nil { - log.Warn("loading built-in genesis: %s", err) + log.Warnf("loading built-in genesis: %s", err) } return genBytes diff --git a/go.mod b/go.mod index 346fd3368..4337ffd48 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipld-cbor v0.0.3 github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log v1.0.0 + github.com/ipfs/go-log v1.0.1 github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb diff --git a/go.sum b/go.sum index f96fb52cb..1dbf549c1 100644 --- a/go.sum +++ b/go.sum @@ -277,6 +277,12 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= +github.com/ipfs/go-log v1.0.1-alpha-2 h1:NZuCXEh6R3i1NQE3yDug93bfRSPzsoX7MxVylNLnQQA= +github.com/ipfs/go-log v1.0.1-alpha-2/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= From 1ed62628a7dcef15b28a736a31e8fd1b7f3db430 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 8 Jan 2020 20:10:57 +0100 Subject: [PATCH 16/61] Update go-log to v2 Signed-off-by: Jakub Sztandera --- api/test/deals.go | 2 +- build/genesis.go | 2 +- chain/actors/actor_init.go | 2 +- chain/blocksync/blocksync.go | 2 +- chain/deals/client.go | 2 +- chain/events/events.go | 2 +- chain/gen/gen.go | 2 +- chain/messagepool/messagepool.go | 2 +- chain/metrics/consensus.go | 2 +- chain/state/statetree.go | 2 +- chain/stmgr/stmgr.go | 2 +- chain/store/store.go | 2 +- chain/sub/incoming.go | 2 +- chain/sync.go | 2 +- chain/sync_test.go | 2 +- chain/types/tipset.go | 2 +- chain/vm/vm.go | 2 +- chain/wallet/wallet.go | 2 +- cli/cmd.go | 2 +- cmd/lotus-bench/main.go | 2 +- cmd/lotus-chainwatch/main.go | 2 +- cmd/lotus-fountain/main.go | 2 +- cmd/lotus-seal-worker/main.go | 2 +- cmd/lotus-seed/main.go | 2 +- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-shed/main.go | 2 +- cmd/lotus-storage-miner/main.go | 2 +- cmd/lotus/main.go | 2 +- cmd/lotus/rpc.go | 2 +- go.mod | 3 ++- go.sum | 3 +-- lib/auth/handler.go | 2 +- lib/jsonrpc/client.go | 2 +- lib/tarutil/systar.go | 2 +- miner/miner.go | 2 +- node/fxlog.go | 2 +- node/hello/hello.go | 2 +- node/impl/full.go | 2 +- node/modules/core.go | 2 +- node/modules/lp2p/libp2p.go | 2 +- node/modules/testing/genesis.go | 2 +- node/node_test.go | 2 +- node/repo/fsrepo.go | 2 +- paych/paych.go | 2 +- peermgr/peermgr.go | 2 +- retrieval/client.go | 2 +- retrieval/discovery/local.go | 2 +- storage/miner.go | 2 +- storage/sectorblocks/blockstore.go | 2 +- tools/stats/main.go | 2 +- tracing/setup.go | 2 +- 51 files changed, 52 insertions(+), 52 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index b9ff2611f..76a739605 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -11,7 +11,7 @@ import ( "testing" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" diff --git a/build/genesis.go b/build/genesis.go index 08734644a..dc4ded273 100644 --- a/build/genesis.go +++ b/build/genesis.go @@ -2,7 +2,7 @@ package build import ( rice "github.com/GeertJohan/go.rice" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) // moved from now-defunct build/paramfetch.go diff --git a/chain/actors/actor_init.go b/chain/actors/actor_init.go index 45da1401e..2bb31ce7f 100644 --- a/chain/actors/actor_init.go +++ b/chain/actors/actor_init.go @@ -15,7 +15,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" ) diff --git a/chain/blocksync/blocksync.go b/chain/blocksync/blocksync.go index 429ad2ce1..b810c7783 100644 --- a/chain/blocksync/blocksync.go +++ b/chain/blocksync/blocksync.go @@ -14,7 +14,7 @@ import ( "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" ) diff --git a/chain/deals/client.go b/chain/deals/client.go index 798296135..3af97315f 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" diff --git a/chain/events/events.go b/chain/events/events.go index 683dab5db..8a860c5ad 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -6,7 +6,7 @@ import ( "time" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d476a0937..0b396ad06 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -34,7 +34,7 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("gen") diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 8bbf3823a..1ffd46907 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -13,7 +13,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-datastore/query" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" lps "github.com/whyrusleeping/pubsub" "go.uber.org/multierr" diff --git a/chain/metrics/consensus.go b/chain/metrics/consensus.go index b0352860e..0635b08fd 100644 --- a/chain/metrics/consensus.go +++ b/chain/metrics/consensus.go @@ -6,7 +6,7 @@ import ( "time" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" diff --git a/chain/state/statetree.go b/chain/state/statetree.go index fc3da47e4..384af58bb 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/go-cid" hamt "github.com/ipfs/go-hamt-ipld" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b880492f2..34204fb3e 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -18,7 +18,7 @@ import ( bls "github.com/filecoin-project/filecoin-ffi" "github.com/ipfs/go-cid" hamt "github.com/ipfs/go-hamt-ipld" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" ) diff --git a/chain/store/store.go b/chain/store/store.go index d59126518..07d1c692c 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -23,7 +23,7 @@ import ( dstore "github.com/ipfs/go-datastore" hamt "github.com/ipfs/go-hamt-ipld" bstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" pubsub "github.com/whyrusleeping/pubsub" "golang.org/x/xerrors" diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 7af740c33..fc554f4b3 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -4,7 +4,7 @@ import ( "context" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" connmgr "github.com/libp2p/go-libp2p-core/connmgr" pubsub "github.com/libp2p/go-libp2p-pubsub" diff --git a/chain/sync.go b/chain/sync.go index 8efd19855..16fb6278b 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -18,7 +18,7 @@ import ( dstore "github.com/ipfs/go-datastore" hamt "github.com/ipfs/go-hamt-ipld" bstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/chain/sync_test.go b/chain/sync_test.go index c9b8e7bbb..9240dea21 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" diff --git a/chain/types/tipset.go b/chain/types/tipset.go index a4fb209cb..b320f3e30 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -8,7 +8,7 @@ import ( "sort" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" ) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 0912d9a67..a038211b1 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -10,7 +10,7 @@ import ( cid "github.com/ipfs/go-cid" hamt "github.com/ipfs/go-hamt-ipld" blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "go.opencensus.io/trace" "golang.org/x/xerrors" diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 9472d36e0..3a70b506a 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -9,7 +9,7 @@ import ( bls "github.com/filecoin-project/filecoin-ffi" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/minio/blake2b-simd" "golang.org/x/xerrors" diff --git a/cli/cmd.go b/cli/cmd.go index 9eff3e541..0c6c51e82 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -9,7 +9,7 @@ import ( "strings" "syscall" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 57b963041..6f7a3fd89 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -17,7 +17,7 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go index f808b1e81..0b7365d8b 100644 --- a/cmd/lotus-chainwatch/main.go +++ b/cmd/lotus-chainwatch/main.go @@ -5,7 +5,7 @@ import ( "net/http" "os" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 33abeb76b..18e61c97c 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -11,7 +11,7 @@ import ( rice "github.com/GeertJohan/go.rice" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" peer "github.com/libp2p/go-libp2p-peer" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 57d2bf655..29f88c258 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -5,7 +5,7 @@ import ( "github.com/mitchellh/go-homedir" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index da715af3b..3918e004f 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -9,7 +9,7 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" badger "github.com/ipfs/go-ds-badger2" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index c747b7eef..0b62c38bb 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -12,7 +12,7 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" badger "github.com/ipfs/go-ds-badger2" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 79be74d31..fe8abb233 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -3,7 +3,7 @@ package main import ( "os" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/build" diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index 00c170703..8e4b2da34 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -3,7 +3,7 @@ package main import ( "os" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 2892d9358..79f7f7958 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -4,7 +4,7 @@ import ( "context" "os" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index 5178e1b6c..0ec44b148 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lotus/node/impl" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" "golang.org/x/xerrors" diff --git a/go.mod b/go.mod index 4337ffd48..943b1b5a2 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,8 @@ require ( github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipld-cbor v0.0.3 github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log v1.0.1 + github.com/ipfs/go-log v1.0.1 // indirect + github.com/ipfs/go-log/v2 v2.0.1 github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb diff --git a/go.sum b/go.sum index 1dbf549c1..b27f2ee30 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,6 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= -github.com/ipfs/go-log v1.0.1-alpha-2 h1:NZuCXEh6R3i1NQE3yDug93bfRSPzsoX7MxVylNLnQQA= -github.com/ipfs/go-log v1.0.1-alpha-2/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= @@ -807,6 +805,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/lib/auth/handler.go b/lib/auth/handler.go index b6112ce8c..61eb0ff03 100644 --- a/lib/auth/handler.go +++ b/lib/auth/handler.go @@ -5,7 +5,7 @@ import ( "net/http" "strings" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index 45b52c0e0..018a9f681 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -12,7 +12,7 @@ import ( "sync/atomic" "github.com/gorilla/websocket" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "go.opencensus.io/trace/propagation" "golang.org/x/xerrors" diff --git a/lib/tarutil/systar.go b/lib/tarutil/systar.go index 67a87d321..a94354731 100644 --- a/lib/tarutil/systar.go +++ b/lib/tarutil/systar.go @@ -8,7 +8,7 @@ import ( "os" "path/filepath" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("tarutil") diff --git a/miner/miner.go b/miner/miner.go index 639ab6c06..d9d4880f3 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "golang.org/x/xerrors" ) diff --git a/node/fxlog.go b/node/fxlog.go index 3f0e18d64..d5d1a3e1f 100644 --- a/node/fxlog.go +++ b/node/fxlog.go @@ -1,7 +1,7 @@ package node import ( - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.uber.org/fx" ) diff --git a/node/hello/hello.go b/node/hello/hello.go index d05e0a0c8..8dabd3de8 100644 --- a/node/hello/hello.go +++ b/node/hello/hello.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" diff --git a/node/impl/full.go b/node/impl/full.go index 2c01674fb..8e7456604 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -1,7 +1,7 @@ package impl import ( - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/node/impl/client" "github.com/filecoin-project/lotus/node/impl/market" diff --git a/node/modules/core.go b/node/modules/core.go index 186ad9936..23adc23bc 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" "github.com/gbrlsnchs/jwt/v3" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" "golang.org/x/xerrors" diff --git a/node/modules/lp2p/libp2p.go b/node/modules/lp2p/libp2p.go index f14368aa4..c74d05e60 100644 --- a/node/modules/lp2p/libp2p.go +++ b/node/modules/lp2p/libp2p.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "golang.org/x/xerrors" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p" connmgr "github.com/libp2p/go-libp2p-connmgr" "github.com/libp2p/go-libp2p-core/crypto" diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index 6d13612ab..bfb6e26c5 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -13,7 +13,7 @@ import ( "github.com/ipfs/go-car" "github.com/ipfs/go-cid" offline "github.com/ipfs/go-ipfs-exchange-offline" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/peer" "github.com/mitchellh/go-homedir" diff --git a/node/node_test.go b/node/node_test.go index 4f9386af9..3903e2dae 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -16,7 +16,7 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 120a970cf..b0d9de1ec 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -14,7 +14,7 @@ import ( "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger2" fslock "github.com/ipfs/go-fs-lock" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-base32" "github.com/multiformats/go-multiaddr" diff --git a/paych/paych.go b/paych/paych.go index 255d6c786..28f4c0fca 100644 --- a/paych/paych.go +++ b/paych/paych.go @@ -9,7 +9,7 @@ import ( "golang.org/x/xerrors" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.uber.org/fx" "github.com/filecoin-project/go-address" diff --git a/peermgr/peermgr.go b/peermgr/peermgr.go index fe87e723f..3d600fcbc 100644 --- a/peermgr/peermgr.go +++ b/peermgr/peermgr.go @@ -13,7 +13,7 @@ import ( peer "github.com/libp2p/go-libp2p-core/peer" dht "github.com/libp2p/go-libp2p-kad-dht" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("peermgr") diff --git a/retrieval/client.go b/retrieval/client.go index bd06c4d3b..496781222 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -6,7 +6,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" diff --git a/retrieval/discovery/local.go b/retrieval/discovery/local.go index 1ca99db9c..984e47da5 100644 --- a/retrieval/discovery/local.go +++ b/retrieval/discovery/local.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/go-datastore/namespace" dshelp "github.com/ipfs/go-ipfs-ds-help" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/node/modules/dtypes" ) diff --git a/storage/miner.go b/storage/miner.go index 3265aa2ae..53303a504 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/host" "golang.org/x/xerrors" diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index b0807ef3a..3cf497d59 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -9,7 +9,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" ) diff --git a/tools/stats/main.go b/tools/stats/main.go index 7c0f12e97..fb383e8e1 100644 --- a/tools/stats/main.go +++ b/tools/stats/main.go @@ -6,7 +6,7 @@ import ( "os" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("stats") diff --git a/tracing/setup.go b/tracing/setup.go index ee8e4c8dd..141683b39 100644 --- a/tracing/setup.go +++ b/tracing/setup.go @@ -4,7 +4,7 @@ import ( "os" "contrib.go.opencensus.io/exporter/jaeger" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" ) From 6ce9a7f22535009328dbec792e04a1159fa99f46 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 8 Jan 2020 11:16:09 -0800 Subject: [PATCH 17/61] switch default key type to secp256k1 --- cli/wallet.go | 2 +- documentation/en/join-testnet.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/wallet.go b/cli/wallet.go index 1f950adb8..ad600eadf 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -41,7 +41,7 @@ var walletNew = &cli.Command{ t := cctx.Args().First() if t == "" { - t = "bls" + t = "secp256k1" } nk, err := api.WalletNew(ctx, t) diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md index ae97db49f..4d48af20c 100644 --- a/documentation/en/join-testnet.md +++ b/documentation/en/join-testnet.md @@ -47,16 +47,16 @@ lotus sync wait ## Create your first address -Initialize a wallet using BLS signature formats: +Initialize a new wallet: ```sh -lotus wallet new bls +lotus wallet new ``` Here is an example of the response: ```sh -t3vhfme4qfvegqaz7m7q6o6afjcs67n6kpzv7t2eozio4chwpafwa2y4l7zhwd5eom7jmihzdg4s52dpvnclza +t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi ``` - Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html) to add funds. From 8cfb4aafcd03d4c2d3b4772ca72df2f2833131cb Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 8 Jan 2020 20:24:44 +0100 Subject: [PATCH 18/61] Add fork signaling to blockheader Signed-off-by: Jakub Sztandera --- api/cbor_gen.go | 36 ++++++++++++ chain/blocksync/cbor_gen.go | 2 +- chain/types/blockheader.go | 2 + chain/types/cbor_gen.go | 21 ++++++- go.mod | 2 +- go.sum | 2 + storage/cbor_gen.go | 112 ++++++++++++++++++++++++++++++++++++ 7 files changed, 172 insertions(+), 5 deletions(-) diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 8d309a6cc..a65f97aaf 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -23,6 +23,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } // t.Channel (address.Address) (struct) + if len("Channel") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Channel\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Channel")))); err != nil { return err } @@ -35,6 +39,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } // t.ChannelMessage (cid.Cid) (struct) + if len("ChannelMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"ChannelMessage\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("ChannelMessage")))); err != nil { return err } @@ -53,6 +61,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } // t.Vouchers ([]*types.SignedVoucher) (slice) + if len("Vouchers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Vouchers\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Vouchers")))); err != nil { return err } @@ -60,6 +72,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.Vouchers) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Vouchers was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { return err } @@ -200,6 +216,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { } // t.SectorID (uint64) (uint64) + if len("SectorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SectorID\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorID")))); err != nil { return err } @@ -212,6 +232,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { } // t.Offset (uint64) (uint64) + if len("Offset") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Offset\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Offset")))); err != nil { return err } @@ -224,6 +248,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { } // t.Size (uint64) (uint64) + if len("Size") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Size\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Size")))); err != nil { return err } @@ -336,6 +364,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error { } // t.Refs ([]api.SealedRef) (slice) + if len("Refs") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Refs\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Refs")))); err != nil { return err } @@ -343,6 +375,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error { return err } + if len(t.Refs) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Refs was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Refs)))); err != nil { return err } diff --git a/chain/blocksync/cbor_gen.go b/chain/blocksync/cbor_gen.go index 896d8f961..01db7d83a 100644 --- a/chain/blocksync/cbor_gen.go +++ b/chain/blocksync/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "github.com/filecoin-project/lotus/chain/types" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 6b3f5c679..c91d029fc 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -59,6 +59,8 @@ type BlockHeader struct { Timestamp uint64 BlockSig *Signature + + ForkSignaling uint64 } func (b *BlockHeader) ToStorageBlock() (block.Block, error) { diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 407f89809..45c186435 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -19,7 +19,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{140}); err != nil { + if _, err := w.Write([]byte{141}); err != nil { return err } @@ -94,6 +94,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { if err := t.BlockSig.MarshalCBOR(w); err != nil { return err } + + // t.ForkSignaling (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil { + return err + } return nil } @@ -108,7 +113,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 12 { + if extra != 13 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -272,6 +277,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } + // t.ForkSignaling (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ForkSignaling = uint64(extra) return nil } diff --git a/go.mod b/go.mod index 346fd3368..efd798b03 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/prometheus/common v0.2.0 github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 + github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.2 diff --git a/go.sum b/go.sum index f96fb52cb..df781631e 100644 --- a/go.sum +++ b/go.sum @@ -706,6 +706,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:x github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 h1:efb/4CnrubzNGqQOeHErxyQ6rIsJb7GcgeSDF7fqWeI= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe h1:n7En1uyDtknjLRDXebWlPGJoHvwL8AkNcSQzuOoYYYQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index 43262ad2b..d2a6b9ee3 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -22,6 +22,10 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { } // t.BlockHeight (uint64) (uint64) + if len("BlockHeight") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"BlockHeight\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("BlockHeight")))); err != nil { return err } @@ -34,6 +38,10 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { } // t.TicketBytes ([]uint8) (slice) + if len("TicketBytes") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TicketBytes\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("TicketBytes")))); err != nil { return err } @@ -41,6 +49,10 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { return err } + if len(t.TicketBytes) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.TicketBytes was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { return err } @@ -133,6 +145,10 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { } // t.BlockHeight (uint64) (uint64) + if len("BlockHeight") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"BlockHeight\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("BlockHeight")))); err != nil { return err } @@ -145,6 +161,10 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { } // t.TicketBytes ([]uint8) (slice) + if len("TicketBytes") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TicketBytes\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("TicketBytes")))); err != nil { return err } @@ -152,6 +172,10 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { return err } + if len(t.TicketBytes) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.TicketBytes was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { return err } @@ -244,6 +268,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { } // t.DealID (uint64) (uint64) + if len("DealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealID\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("DealID")))); err != nil { return err } @@ -256,6 +284,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { } // t.Size (uint64) (uint64) + if len("Size") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Size\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Size")))); err != nil { return err } @@ -268,6 +300,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { } // t.CommP ([]uint8) (slice) + if len("CommP") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommP\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommP")))); err != nil { return err } @@ -275,6 +311,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { return err } + if len(t.CommP) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.CommP was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommP)))); err != nil { return err } @@ -390,6 +430,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.State (uint64) (uint64) + if len("State") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"State\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("State")))); err != nil { return err } @@ -402,6 +446,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.SectorID (uint64) (uint64) + if len("SectorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SectorID\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorID")))); err != nil { return err } @@ -414,6 +462,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Nonce (uint64) (uint64) + if len("Nonce") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Nonce\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Nonce")))); err != nil { return err } @@ -426,6 +478,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Pieces ([]storage.Piece) (slice) + if len("Pieces") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Pieces\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Pieces")))); err != nil { return err } @@ -433,6 +489,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.Pieces) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Pieces was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil { return err } @@ -443,6 +503,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.CommD ([]uint8) (slice) + if len("CommD") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommD\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommD")))); err != nil { return err } @@ -450,6 +514,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.CommD) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.CommD was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { return err } @@ -458,6 +526,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.CommR ([]uint8) (slice) + if len("CommR") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommR\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommR")))); err != nil { return err } @@ -465,6 +537,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.CommR) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.CommR was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { return err } @@ -473,6 +549,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Proof ([]uint8) (slice) + if len("Proof") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Proof\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Proof")))); err != nil { return err } @@ -480,6 +560,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.Proof) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Proof was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -488,6 +572,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Ticket (storage.SealTicket) (struct) + if len("Ticket") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Ticket\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Ticket")))); err != nil { return err } @@ -500,6 +588,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.PreCommitMessage (cid.Cid) (struct) + if len("PreCommitMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommitMessage\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitMessage")))); err != nil { return err } @@ -518,6 +610,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Seed (storage.SealSeed) (struct) + if len("Seed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Seed\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Seed")))); err != nil { return err } @@ -530,6 +626,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.CommitMessage (cid.Cid) (struct) + if len("CommitMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommitMessage\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommitMessage")))); err != nil { return err } @@ -548,6 +648,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.FaultReportMsg (cid.Cid) (struct) + if len("FaultReportMsg") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"FaultReportMsg\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("FaultReportMsg")))); err != nil { return err } @@ -566,6 +670,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.LastErr (string) (string) + if len("LastErr") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"LastErr\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("LastErr")))); err != nil { return err } @@ -573,6 +681,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.LastErr) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.LastErr was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.LastErr)))); err != nil { return err } From f36d81070b1a9a4c11f02beeac650c2c052d1447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jan 2020 21:15:55 +0100 Subject: [PATCH 19/61] mod tidy --- go.sum | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index 9ba55670b..78503bdee 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -20,6 +21,7 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -88,12 +90,12 @@ github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc/go.mo github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= -github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= @@ -285,6 +287,7 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -550,6 +553,7 @@ github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -628,7 +632,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= From da38822dfdb8e609360e19eec9e5f71b3a8fbfc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jan 2020 21:31:35 +0100 Subject: [PATCH 20/61] paramfetch: bring parameters.json back to this repo --- build/parameters.go | 5 ++ build/proof-params/parameters.json | 103 +++++++++++++++++++++++++++++ cli/params.go | 4 +- cmd/lotus-bench/main.go | 2 +- cmd/lotus-seal-worker/sub.go | 5 +- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus/daemon.go | 2 +- go.mod | 11 ++- go.sum | 25 +++++++ node/modules/storageminer.go | 2 +- 10 files changed, 151 insertions(+), 10 deletions(-) create mode 100644 build/parameters.go create mode 100644 build/proof-params/parameters.json diff --git a/build/parameters.go b/build/parameters.go new file mode 100644 index 000000000..433fa497b --- /dev/null +++ b/build/parameters.go @@ -0,0 +1,5 @@ +package build + +import rice "github.com/GeertJohan/go.rice" + +var ParametersJson = rice.MustFindBox("proof-params").MustBytes("parameters.json") diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json new file mode 100644 index 000000000..7e1988a41 --- /dev/null +++ b/build/proof-params/parameters.json @@ -0,0 +1,103 @@ +{ + "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.params": { + "cid": "QmX7tYeNPWae2fjZ3Am6GB9dmHvLqvoz8dKo3PR98VYxH9", + "digest": "39a9edec3355516674f0d12b926be493", + "sector_size": 34359738368 + }, + "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.vk": { + "cid": "QmbNGx7pNbGiEr8ykoHxVXHW2LNSmGdsxKtj1onZCyguCX", + "digest": "0227ae7df4f2affe529ebafbbc7540ee", + "sector_size": 34359738368 + }, + "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.params": { + "cid": "QmRGZsNp4mp1cZshcXqt3VMuWscAEsiMa2iepF4CsWWoiv", + "digest": "991041a354b12c280542741f58c7f2ca", + "sector_size": 1024 + }, + "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.vk": { + "cid": "QmWpmrhCGVcfqLyqp5oGAnhPmCE5hGTPaauHi25mpQwRSU", + "digest": "91fac550e1f9bccab213830bb0c85bd6", + "sector_size": 1024 + }, + "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.params": { + "cid": "QmenSZXh1EsSyHiSRvA6wb8yaPhYBTjrKehJw96Px5HnN4", + "digest": "6322eacd2773163ddd51f9ca7d645fc4", + "sector_size": 1073741824 + }, + "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.vk": { + "cid": "QmPvZoMKofw6eDhDg5ESJA2QAZP8HvM6qMQk7fw4pq9bQf", + "digest": "0df62745fceac922e3e70847cfc70b52", + "sector_size": 1073741824 + }, + "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.params": { + "cid": "QmVibFqzkZoL8cwQmzj8njPokCQGCCx4pBcUH77bzgJgV9", + "digest": "de9d71e672f286706a1673bd57abdaac", + "sector_size": 16777216 + }, + "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.vk": { + "cid": "QmZa5FX27XyiEXQQLQpHqtMJKLzrcY8wMuj3pxzmSimSyu", + "digest": "7f796d3a0f13499181e44b5eee0cc744", + "sector_size": 16777216 + }, + "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.params": { + "cid": "Qmbt2SWWAmMcYoY3DAiRDXA8fAuqdqRLWucJMSxYmzBCmN", + "digest": "151ae0ae183fc141e8c2bebc28e5cc10", + "sector_size": 268435456 + }, + "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.vk": { + "cid": "QmUxvPu4xdVmjMFihUKoYyEdXBqxsXkvmxRweU7KouWHji", + "digest": "95eb89588e9d1832aca044c3a13178af", + "sector_size": 268435456 + }, + "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": { + "cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX", + "digest": "fcd50e2e08a8560a6bb3418e883567ed", + "sector_size": 268435456 + }, + "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": { + "cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y", + "digest": "872e244d86499fd659082e3bcf3f13e7", + "sector_size": 268435456 + }, + "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": { + "cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN", + "digest": "0a338b941c5f17946340de5fc95cab30", + "sector_size": 34359738368 + }, + "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": { + "cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF", + "digest": "c752e070a6b7aa8b79aa661a6b600b55", + "sector_size": 34359738368 + }, + "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": { + "cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186", + "digest": "efa407fd09202dffd15799a8518e73d3", + "sector_size": 1024 + }, + "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": { + "cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG", + "digest": "b2f09a0ccb62da28c890d5b881c8dcd2", + "sector_size": 1024 + }, + "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": { + "cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q", + "digest": "b53c1916a63839ec345aa2224e9198b7", + "sector_size": 1073741824 + }, + "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": { + "cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA", + "digest": "20d58a3fae7343481f8298a2dd493dd7", + "sector_size": 1073741824 + }, + "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": { + "cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU", + "digest": "7698426202c7e07b26ef056d31485b3a", + "sector_size": 16777216 + }, + "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": { + "cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax", + "digest": "834408e5c3fce6ec5d1bf64e64cee94e", + "sector_size": 16777216 + } +} + diff --git a/cli/params.go b/cli/params.go index f83b95bef..440c2bbdf 100644 --- a/cli/params.go +++ b/cli/params.go @@ -5,6 +5,8 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + + "github.com/filecoin-project/lotus/build" ) var fetchParamCmd = &cli.Command{ @@ -22,7 +24,7 @@ var fetchParamCmd = &cli.Command{ return err } sectorSize := uint64(sectorSizeInt) - err = paramfetch.GetParams(sectorSize) + err = paramfetch.GetParams(build.ParametersJson, sectorSize) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 57b963041..b0c90b58c 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -155,7 +155,7 @@ func main() { } } - if err := paramfetch.GetParams(sectorSize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, sectorSize); err != nil { return xerrors.Errorf("getting params: %w", err) } sb, err := sectorbuilder.New(cfg, mds) diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index fa1b00a8a..d79a8ed83 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -5,10 +5,11 @@ import ( "net/http" paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" - "github.com/filecoin-project/go-sectorbuilder" lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" ) type worker struct { @@ -40,7 +41,7 @@ func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, aut return err } - if err := paramfetch.GetParams(ssize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil { return xerrors.Errorf("get params: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9bf52fea8..e4bd00738 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -93,7 +93,7 @@ var initCmd = &cli.Command{ } log.Info("Checking proof parameters") - if err := paramfetch.GetParams(ssize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 4675453f7..59b17ceab 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -68,7 +68,7 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := paramfetch.GetParams(0); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, 0); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/go.mod b/go.mod index e9e85962d..fa68fc620 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 + github.com/filecoin-project/go-paramfetch v0.0.1 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 @@ -46,7 +46,8 @@ require ( github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipld-cbor v0.0.3 github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log v1.0.0 + github.com/ipfs/go-log v1.0.1 + github.com/ipfs/go-log/v2 v2.0.2 // indirect github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb @@ -90,13 +91,17 @@ require ( github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.2 + go.uber.org/atomic v1.5.1 // indirect go.uber.org/dig v1.7.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/goleak v0.10.0 // indirect go.uber.org/multierr v1.4.0 - go.uber.org/zap v1.10.0 + go.uber.org/zap v1.13.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect + golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect + golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + golang.org/x/tools v0.0.0-20200108195415-316d2f248479 // indirect golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 diff --git a/go.sum b/go.sum index 78503bdee..40dc30699 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuYCW0wUb26AvnD4WK59M= github.com/filecoin-project/chain-validation v0.0.3/go.mod h1:NCEGFjcWRjb8akWFSOXvU6n2efkWIqAeOKU6o5WBGQw= @@ -97,6 +98,8 @@ github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1: github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= +github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE= +github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= @@ -250,6 +253,12 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= +github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw= +github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= @@ -490,6 +499,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -678,6 +689,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/dig v1.7.0 h1:E5/L92iQTNJTjfgJF2KgU+/JpMaiuvK2DHLBj0+kSZk= go.uber.org/dig v1.7.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg= go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY= @@ -685,12 +698,15 @@ go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWAah4XYJSfw= go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -717,7 +733,10 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -742,6 +761,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -765,6 +785,8 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -784,6 +806,9 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200108195415-316d2f248479 h1:csuS+MHeEA2eWhyjQCMaPMq4z1+/PohkBSjJZHSIbOE= +golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 23fb35455..f93d8c945 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -45,7 +45,7 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { } func GetParams(sbc *sectorbuilder.Config) error { - if err := paramfetch.GetParams(sbc.SectorSize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, sbc.SectorSize); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } From 13c8fe3fb330ad06e5d5c77e3f3271dc29ed6840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jan 2020 21:31:44 +0100 Subject: [PATCH 21/61] mod tidy --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 40dc30699..2caf516d4 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuYCW0wUb26AvnD4WK59M= @@ -499,6 +500,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= From 5b74a71dd3ad794dcd7544d2ac89056f295ce0d6 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 9 Dec 2019 20:19:59 -0800 Subject: [PATCH 22/61] feat(retrievalmarket): extract skeleton interfaces Define all types to spec, modify interfaces, wrap old code fix(builder): use client blockstore for retrieval feat(retrieval): add node implementations add node adapters for client & provider so that retrieval can be extracted --- chain/deals/client.go | 4 +- gen/main.go | 16 - node/builder.go | 8 +- node/impl/client/client.go | 65 +++- node/modules/client.go | 11 + node/modules/services.go | 3 +- node/modules/storageminer.go | 16 +- retrieval/cbor-gen/main.go | 20 ++ retrieval/client.go | 272 ---------------- retrieval/discovery/discovery.go | 18 +- retrieval/discovery/local.go | 13 +- retrieval/{ => impl}/cbor_gen.go | 26 +- retrieval/impl/client.go | 381 +++++++++++++++++++++++ retrieval/{miner.go => impl/provider.go} | 123 ++++++-- retrieval/impl/run_cbor_gen.go | 33 ++ retrieval/impl/types.go | 67 ++++ retrieval/{ => impl}/verify.go | 24 +- retrieval/types.go | 368 +++++++++++++++++++--- retrievaladapter/client.go | 41 +++ retrievaladapter/provider.go | 23 ++ 20 files changed, 1111 insertions(+), 421 deletions(-) create mode 100644 retrieval/cbor-gen/main.go delete mode 100644 retrieval/client.go rename retrieval/{ => impl}/cbor_gen.go (92%) create mode 100644 retrieval/impl/client.go rename retrieval/{miner.go => impl/provider.go} (54%) create mode 100644 retrieval/impl/run_cbor_gen.go create mode 100644 retrieval/impl/types.go rename retrieval/{ => impl}/verify.go (83%) create mode 100644 retrievaladapter/client.go create mode 100644 retrievaladapter/provider.go diff --git a/chain/deals/client.go b/chain/deals/client.go index 3af97315f..4e1d3df4a 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -23,7 +23,7 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules/dtypes" - + retrievalmarket "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" ) @@ -252,7 +252,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro c.incoming <- deal - return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{ + return deal.ProposalCid, c.discovery.AddPeer(p.Data, retrievalmarket.RetrievalPeer{ Address: dealProposal.Provider, ID: deal.Miner, }) diff --git a/gen/main.go b/gen/main.go index 52fcf4665..4f90dc236 100644 --- a/gen/main.go +++ b/gen/main.go @@ -12,7 +12,6 @@ import ( "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/paych" - "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage" ) @@ -59,21 +58,6 @@ func main() { os.Exit(1) } - err = gen.WriteTupleEncodersToFile("./retrieval/cbor_gen.go", "retrieval", - retrieval.RetParams{}, - - retrieval.Query{}, - retrieval.QueryResponse{}, - retrieval.Unixfs0Offer{}, - retrieval.DealProposal{}, - retrieval.DealResponse{}, - retrieval.Block{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - err = gen.WriteTupleEncodersToFile("./chain/blocksync/cbor_gen.go", "blocksync", blocksync.BlockSyncRequest{}, blocksync.BlockSyncResponse{}, diff --git a/node/builder.go b/node/builder.go index ee15f8e70..76e9289fa 100644 --- a/node/builder.go +++ b/node/builder.go @@ -43,7 +43,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/peermgr" - "github.com/filecoin-project/lotus/retrieval" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -221,9 +221,9 @@ func Online() Option { Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), Override(new(*discovery.Local), discovery.NewLocal), - Override(new(discovery.PeerResolver), modules.RetrievalResolver), + Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver), - Override(new(*retrieval.Client), retrieval.NewClient), + Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), Override(new(dtypes.ClientDealStore), modules.NewClientDealStore), Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator), @@ -246,7 +246,7 @@ func Online() Option { Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), Override(new(dtypes.StagingDAG), modules.StagingDAG), Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), - Override(new(*retrieval.Miner), retrieval.NewMiner), + Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a0c420e7e..57df600bd 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1,6 +1,7 @@ package client import ( + "bytes" "context" "errors" "io" @@ -17,6 +18,7 @@ import ( files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + unixfile "github.com/ipfs/go-unixfs/file" "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" "github.com/libp2p/go-libp2p-core/peer" @@ -31,8 +33,7 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" ) type API struct { @@ -44,8 +45,8 @@ type API struct { paych.PaychAPI DealClient *deals.Client - RetDiscovery discovery.PeerResolver - Retrieval *retrieval.Client + RetDiscovery retrievalmarket.PeerResolver + Retrieval retrievalmarket.RetrievalClient Chain *store.ChainStore LocalDAG dtypes.ClientDAG @@ -153,7 +154,18 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe out := make([]api.QueryOffer, len(peers)) for k, p := range peers { - out[k] = a.Retrieval.Query(ctx, p, root) + queryResponse, err := a.Retrieval.Query(ctx, p, root.Bytes(), retrievalmarket.QueryParams{}) + if err != nil { + out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } else { + out[k] = api.QueryOffer{ + Root: root, + Size: queryResponse.Size, + MinPrice: queryResponse.PieceRetrievalPrice(), + Miner: p.Address, // TODO: check + MinerPeerID: p.ID, + } + } } return out, nil @@ -263,18 +275,43 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path order.MinerPeerID = pid } - outFile, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) - if err != nil { - return err + retrievalResult := make(chan error, 1) + + unsubscribe := a.Retrieval.SubscribeToEvents(func(event retrievalmarket.ClientEvent, state retrievalmarket.ClientDealState) { + if bytes.Equal(state.PieceCID, order.Root.Bytes()) { + switch event { + case retrievalmarket.ClientEventError: + retrievalResult <- xerrors.New("Retrieval Error") + case retrievalmarket.ClientEventComplete: + retrievalResult <- nil + } + } + }) + + a.Retrieval.Retrieve( + ctx, order.Root.Bytes(), retrievalmarket.Params{ + PricePerByte: types.BigDiv(order.Total, types.NewInt(order.Size)), + }, order.Total, order.MinerPeerID, order.Client, order.Miner) + select { + case <-ctx.Done(): + return xerrors.New("Retrieval Timed Out") + case err := <-retrievalResult: + if err != nil { + return xerrors.Errorf("RetrieveUnixfs: %w", err) + } } - err = a.Retrieval.RetrieveUnixfs(ctx, order.Root, order.Size, order.Total, order.MinerPeerID, order.Client, order.Miner, outFile) - if err != nil { - _ = outFile.Close() - return xerrors.Errorf("RetrieveUnixfs: %w", err) - } + unsubscribe() - return outFile.Close() + nd, err := a.LocalDAG.Get(ctx, order.Root) + if err != nil { + return xerrors.Errorf("ClientRetrieve: %w", err) + } + file, err := unixfile.NewUnixfsFile(ctx, a.LocalDAG, nd) + if err != nil { + return xerrors.Errorf("ClientRetrieve: %w", err) + } + return files.WriteTo(file, path) } func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { diff --git a/node/modules/client.go b/node/modules/client.go index f01bbf8c3..6047df7af 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -2,11 +2,15 @@ package modules import ( "context" + "github.com/filecoin-project/lotus/retrievaladapter" "path/filepath" "reflect" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/node/modules/helpers" + "github.com/filecoin-project/lotus/paych" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" + retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" graphsync "github.com/ipfs/go-graphsync/impl" @@ -26,6 +30,7 @@ import ( "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/lotus/chain/deals" + payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" ) @@ -97,3 +102,9 @@ func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Client return gs } + +// RetrievalClient creates a new retrieval client attached to the client blockstore +func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClient { + adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi) + return retrievalimpl.NewClient(h, bs, adapter) +} diff --git a/node/modules/services.go b/node/modules/services.go index 92b6264e8..9949d0d69 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" ) @@ -80,6 +81,6 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c *deals.Client) { }) } -func RetrievalResolver(l *discovery.Local) discovery.PeerResolver { +func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { return discovery.Multi(l) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 9725d3440..37902dd08 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -35,8 +35,11 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/retrieval" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" + retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" + "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sectorblocks" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -115,11 +118,10 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h return sm, nil } -func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) { +func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider) { lc.Append(fx.Hook{ OnStart: func(context.Context) error { - host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleQueryStream) - host.SetStreamHandler(retrieval.ProtocolID, m.HandleDealStream) + m.Start(host) return nil }, }) @@ -261,3 +263,9 @@ func SealTicketGen(api api.FullNode) storage.TicketFn { }, nil } } + +// RetrievalProvider creates a new retrieval provider attached to the provider blockstore +func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider { + adapter := retrievaladapter.NewRetrievalProviderNode(full) + return retrievalimpl.NewProvider(sblks, adapter) +} diff --git a/retrieval/cbor-gen/main.go b/retrieval/cbor-gen/main.go new file mode 100644 index 000000000..cc3fe9887 --- /dev/null +++ b/retrieval/cbor-gen/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "fmt" + "os" + + retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" +) + +// main func has ONE JOB +func main() { + fmt.Print("Generating Cbor Marshal/Unmarshal...") + + if err := retrievalimpl.RunCborGen(); err != nil { + fmt.Println("Failed: ") + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Done.") +} diff --git a/retrieval/client.go b/retrieval/client.go deleted file mode 100644 index 496781222..000000000 --- a/retrieval/client.go +++ /dev/null @@ -1,272 +0,0 @@ -package retrieval - -import ( - "context" - "io" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - payapi "github.com/filecoin-project/lotus/node/impl/paych" - "github.com/filecoin-project/lotus/paych" - "github.com/filecoin-project/lotus/retrieval/discovery" -) - -var log = logging.Logger("retrieval") - -type Client struct { - h host.Host - - pmgr *paych.Manager - payapi payapi.PaychAPI -} - -func NewClient(h host.Host, pmgr *paych.Manager, payapi payapi.PaychAPI) *Client { - return &Client{h: h, pmgr: pmgr, payapi: payapi} -} - -func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid.Cid) api.QueryOffer { - s, err := c.h.NewStream(ctx, p.ID, QueryProtocolID) - if err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - defer s.Close() - - err = cborutil.WriteCborRPC(s, &Query{ - Piece: data, - }) - if err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - - var resp QueryResponse - if err := resp.UnmarshalCBOR(s); err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - - return api.QueryOffer{ - Root: data, - Size: resp.Size, - MinPrice: resp.MinPrice, - Miner: p.Address, // TODO: check - MinerPeerID: p.ID, - } -} - -type clientStream struct { - payapi payapi.PaychAPI - stream network.Stream - peeker cbg.BytePeeker - - root cid.Cid - size types.BigInt - offset uint64 - - paych address.Address - lane uint64 - total types.BigInt - transferred types.BigInt - - windowSize uint64 // how much we "trust" the peer - verifier BlockVerifier -} - -// C > S -// -// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size -// -// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} -// < Resp{Accept} -// < ..(Intermediate Block) -// < ..Blocks -// < ..(Intermediate Block) -// < ..Blocks -// > DealProposal(...) -// < ... -func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr address.Address, out io.Writer) error { - s, err := c.h.NewStream(ctx, miner, ProtocolID) - if err != nil { - return xerrors.Errorf("failed to open stream to miner for retrieval query: %w", err) - } - defer s.Close() - - initialOffset := uint64(0) // TODO: Check how much data we have locally - // TODO: Support in handler - // TODO: Allow client to specify this - - paych, _, err := c.pmgr.GetPaych(ctx, client, minerAddr, total) - if err != nil { - return xerrors.Errorf("getting payment channel: %w", err) - } - lane, err := c.pmgr.AllocateLane(paych) - if err != nil { - return xerrors.Errorf("allocating payment lane: %w", err) - } - - cst := clientStream{ - payapi: c.payapi, - stream: s, - peeker: cbg.GetPeeker(s), - - root: root, - size: types.NewInt(size), - offset: initialOffset, - - paych: paych, - lane: lane, - total: total, - transferred: types.NewInt(0), - - windowSize: build.UnixfsChunkSize, - verifier: &UnixFs0Verifier{Root: root}, - } - - for cst.offset != size+initialOffset { - toFetch := cst.windowSize - if toFetch+cst.offset > size { - toFetch = size - cst.offset - } - log.Infof("Retrieve %dB @%d", toFetch, cst.offset) - - err := cst.doOneExchange(ctx, toFetch, out) - if err != nil { - return xerrors.Errorf("retrieval exchange: %w", err) - } - - cst.offset += toFetch - } - return nil -} - -func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out io.Writer) error { - payAmount := types.BigDiv(types.BigMul(cst.total, types.NewInt(toFetch)), cst.size) - - payment, err := cst.setupPayment(ctx, payAmount) - if err != nil { - return xerrors.Errorf("setting up retrieval payment: %w", err) - } - - deal := &DealProposal{ - Payment: payment, - Ref: cst.root, - Params: RetParams{ - Unixfs0: &Unixfs0Offer{ - Offset: cst.offset, - Size: toFetch, - }, - }, - } - - if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { - return xerrors.Errorf("sending incremental retrieval request: %w", err) - } - - var resp DealResponse - if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { - return xerrors.Errorf("reading retrieval response: %w", err) - } - - if resp.Status != Accepted { - cst.windowSize = build.UnixfsChunkSize - // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) - - if resp.Status == Error { - return xerrors.Errorf("storage deal error: %s", resp.Message) - } - if resp.Status == Rejected { - return xerrors.Errorf("storage deal rejected: %s", resp.Message) - } - return xerrors.New("storage deal response had no Accepted section") - } - - log.Info("Retrieval accepted, fetching blocks") - - return cst.fetchBlocks(toFetch, out) - - // TODO: maybe increase miner window size after success -} - -func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { - blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - - for i := uint64(0); i < blocksToFetch; { - log.Infof("block %d of %d", i+1, blocksToFetch) - - var block Block - if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil { - return xerrors.Errorf("reading fetchBlock response: %w", err) - } - - dataBlocks, err := cst.consumeBlockMessage(block, out) - if err != nil { - return xerrors.Errorf("consuming retrieved blocks: %w", err) - } - - i += dataBlocks - } - - return nil -} - -func (cst *clientStream) consumeBlockMessage(block Block, out io.Writer) (uint64, error) { - prefix, err := cid.PrefixFromBytes(block.Prefix) - if err != nil { - return 0, err - } - - cid, err := prefix.Sum(block.Data) - if err != nil { - return 0, err - } - - blk, err := blocks.NewBlockWithCid(block.Data, cid) - if err != nil { - return 0, err - } - - internal, err := cst.verifier.Verify(context.TODO(), blk, out) - if err != nil { - log.Warnf("block verify failed: %s", err) - return 0, err - } - - // TODO: Smarter out, maybe add to filestore automagically - // (Also, persist intermediate nodes) - - if internal { - return 0, nil - } - - return 1, nil -} - -func (cst *clientStream) setupPayment(ctx context.Context, toSend types.BigInt) (api.PaymentInfo, error) { - amount := types.BigAdd(cst.transferred, toSend) - - sv, err := cst.payapi.PaychVoucherCreate(ctx, cst.paych, amount, cst.lane) - if err != nil { - return api.PaymentInfo{}, err - } - - cst.transferred = amount - - return api.PaymentInfo{ - Channel: cst.paych, - ChannelMessage: nil, - Vouchers: []*types.SignedVoucher{sv}, - }, nil -} diff --git a/retrieval/discovery/discovery.go b/retrieval/discovery/discovery.go index 9933ad793..93da56b60 100644 --- a/retrieval/discovery/discovery.go +++ b/retrieval/discovery/discovery.go @@ -1,25 +1,15 @@ package discovery import ( - "github.com/filecoin-project/go-address" - "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - "github.com/libp2p/go-libp2p-core/peer" + + retrievalmarket "github.com/filecoin-project/lotus/retrieval" ) func init() { - cbor.RegisterCborType(RetrievalPeer{}) + cbor.RegisterCborType(retrievalmarket.RetrievalPeer{}) } -type RetrievalPeer struct { - Address address.Address - ID peer.ID // optional -} - -type PeerResolver interface { - GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel -} - -func Multi(r PeerResolver) PeerResolver { // TODO: actually support multiple mechanisms +func Multi(r retrievalmarket.PeerResolver) retrievalmarket.PeerResolver { // TODO: actually support multiple mechanisms return r } diff --git a/retrieval/discovery/local.go b/retrieval/discovery/local.go index 984e47da5..c777a0828 100644 --- a/retrieval/discovery/local.go +++ b/retrieval/discovery/local.go @@ -9,6 +9,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/node/modules/dtypes" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" ) var log = logging.Logger("ret-discovery") @@ -21,7 +22,7 @@ func NewLocal(ds dtypes.MetadataDS) *Local { return &Local{ds: namespace.Wrap(ds, datastore.NewKey("/deals/local"))} } -func (l *Local) AddPeer(cid cid.Cid, peer RetrievalPeer) error { +func (l *Local) AddPeer(cid cid.Cid, peer retrievalmarket.RetrievalPeer) error { // TODO: allow multiple peers here // (implement an util for tracking map[thing][]otherThing, use in sectorBlockstore too) @@ -35,19 +36,19 @@ func (l *Local) AddPeer(cid cid.Cid, peer RetrievalPeer) error { return l.ds.Put(dshelp.CidToDsKey(cid), entry) } -func (l *Local) GetPeers(data cid.Cid) ([]RetrievalPeer, error) { +func (l *Local) GetPeers(data cid.Cid) ([]retrievalmarket.RetrievalPeer, error) { entry, err := l.ds.Get(dshelp.CidToDsKey(data)) if err == datastore.ErrNotFound { - return []RetrievalPeer{}, nil + return []retrievalmarket.RetrievalPeer{}, nil } if err != nil { return nil, err } - var peer RetrievalPeer + var peer retrievalmarket.RetrievalPeer if err := cbor.DecodeInto(entry, &peer); err != nil { return nil, err } - return []RetrievalPeer{peer}, nil + return []retrievalmarket.RetrievalPeer{peer}, nil } -var _ PeerResolver = &Local{} +var _ retrievalmarket.PeerResolver = &Local{} diff --git a/retrieval/cbor_gen.go b/retrieval/impl/cbor_gen.go similarity index 92% rename from retrieval/cbor_gen.go rename to retrieval/impl/cbor_gen.go index c80ab66d6..f2778544c 100644 --- a/retrieval/cbor_gen.go +++ b/retrieval/impl/cbor_gen.go @@ -1,4 +1,4 @@ -package retrieval +package retrievalimpl import ( "fmt" @@ -67,7 +67,7 @@ func (t *RetParams) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *Query) MarshalCBOR(w io.Writer) error { +func (t *OldQuery) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -85,7 +85,7 @@ func (t *Query) MarshalCBOR(w io.Writer) error { return nil } -func (t *Query) UnmarshalCBOR(r io.Reader) error { +func (t *OldQuery) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -115,7 +115,7 @@ func (t *Query) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *QueryResponse) MarshalCBOR(w io.Writer) error { +func (t *OldQueryResponse) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -124,7 +124,7 @@ func (t *QueryResponse) MarshalCBOR(w io.Writer) error { return err } - // t.Status (retrieval.QueryResponseStatus) (uint64) + // t.t.Status (retrieval.OldQueryResponseStatus) (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { return err } @@ -141,7 +141,7 @@ func (t *QueryResponse) MarshalCBOR(w io.Writer) error { return nil } -func (t *QueryResponse) UnmarshalCBOR(r io.Reader) error { +func (t *OldQueryResponse) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -156,7 +156,7 @@ func (t *QueryResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Status (retrieval.QueryResponseStatus) (uint64) + // t.t.Status (retrieval.OldQueryResponseStatus) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -165,8 +165,8 @@ func (t *QueryResponse) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Status = QueryResponseStatus(extra) - // t.Size (uint64) (uint64) + t.Status = OldQueryResponseStatus(extra) + // t.t.Size (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -247,7 +247,7 @@ func (t *Unixfs0Offer) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *DealProposal) MarshalCBOR(w io.Writer) error { +func (t *OldDealProposal) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -274,7 +274,7 @@ func (t *DealProposal) MarshalCBOR(w io.Writer) error { return nil } -func (t *DealProposal) UnmarshalCBOR(r io.Reader) error { +func (t *OldDealProposal) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -322,7 +322,7 @@ func (t *DealProposal) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *DealResponse) MarshalCBOR(w io.Writer) error { +func (t *OldDealResponse) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -350,7 +350,7 @@ func (t *DealResponse) MarshalCBOR(w io.Writer) error { return nil } -func (t *DealResponse) UnmarshalCBOR(r io.Reader) error { +func (t *OldDealResponse) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) diff --git a/retrieval/impl/client.go b/retrieval/impl/client.go new file mode 100644 index 000000000..96bad3503 --- /dev/null +++ b/retrieval/impl/client.go @@ -0,0 +1,381 @@ +package retrievalimpl + +import ( + "context" + "reflect" + "sync" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" +) + +var log = logging.Logger("retrieval") + +type client struct { + h host.Host + bs blockstore.Blockstore + node retrievalmarket.RetrievalClientNode + // The parameters should be replaced by RetrievalClientNode + + nextDealLk sync.Mutex + nextDealID retrievalmarket.DealID + subscribers []retrievalmarket.ClientSubscriber +} + +// NewClient creates a new retrieval client +func NewClient(h host.Host, bs blockstore.Blockstore, node retrievalmarket.RetrievalClientNode) retrievalmarket.RetrievalClient { + return &client{h: h, bs: bs, node: node} +} + +// V0 + +// TODO: Implement for retrieval provider V0 epic +// https://github.com/filecoin-project/go-retrieval-market-project/issues/12 +func (c *client) FindProviders(pieceCID []byte) []retrievalmarket.RetrievalPeer { + panic("not implemented") +} + +// TODO: Update to match spec for V0 epic +// https://github.com/filecoin-project/go-retrieval-market-project/issues/8 +func (c *client) Query(ctx context.Context, p retrievalmarket.RetrievalPeer, pieceCID []byte, params retrievalmarket.QueryParams) (retrievalmarket.QueryResponse, error) { + cid, err := cid.Cast(pieceCID) + if err != nil { + log.Warn(err) + return retrievalmarket.QueryResponseUndefined, err + } + + s, err := c.h.NewStream(ctx, p.ID, retrievalmarket.QueryProtocolID) + if err != nil { + log.Warn(err) + return retrievalmarket.QueryResponseUndefined, err + } + defer s.Close() + + err = cborutil.WriteCborRPC(s, &OldQuery{ + Piece: cid, + }) + if err != nil { + log.Warn(err) + return retrievalmarket.QueryResponseUndefined, err + } + + var oldResp OldQueryResponse + if err := oldResp.UnmarshalCBOR(s); err != nil { + log.Warn(err) + return retrievalmarket.QueryResponseUndefined, err + } + + resp := retrievalmarket.QueryResponse{ + Status: retrievalmarket.QueryResponseStatus(oldResp.Status), + Size: oldResp.Size, + MinPricePerByte: types.BigDiv(oldResp.MinPrice, types.NewInt(oldResp.Size)), + } + return resp, nil +} + +// TODO: Update to match spec for V0 Epic: +// https://github.com/filecoin-project/go-retrieval-market-project/issues/9 +func (c *client) Retrieve(ctx context.Context, pieceCID []byte, params retrievalmarket.Params, totalFunds types.BigInt, miner peer.ID, clientWallet retrievalmarket.Address, minerWallet retrievalmarket.Address) retrievalmarket.DealID { + /* The implementation of this function is just wrapper for the old code which retrieves UnixFS pieces + -- it will be replaced when we do the V0 implementation of the module */ + c.nextDealLk.Lock() + c.nextDealID++ + dealID := c.nextDealID + c.nextDealLk.Unlock() + + dealState := retrievalmarket.ClientDealState{ + DealProposal: retrievalmarket.DealProposal{ + PieceCID: pieceCID, + ID: dealID, + Params: params, + }, + Status: retrievalmarket.DealStatusFailed, + Sender: miner, + } + + go func() { + evt := retrievalmarket.ClientEventError + converted, err := cid.Cast(pieceCID) + + if err == nil { + err = c.retrieveUnixfs(ctx, converted, types.BigDiv(totalFunds, params.PricePerByte).Uint64(), totalFunds, miner, clientWallet, minerWallet) + if err == nil { + evt = retrievalmarket.ClientEventComplete + dealState.Status = retrievalmarket.DealStatusCompleted + } + } + + c.notifySubscribers(evt, dealState) + }() + + return dealID +} + +// unsubscribeAt returns a function that removes an item from the subscribers list by comparing +// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order. +// Subsequent, repeated calls to the func with the same Subscriber are a no-op. +func (c *client) unsubscribeAt(sub retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { + return func() { + curLen := len(c.subscribers) + for i, el := range c.subscribers { + if reflect.ValueOf(sub) == reflect.ValueOf(el) { + c.subscribers[i] = c.subscribers[curLen-1] + c.subscribers = c.subscribers[:curLen-1] + return + } + } + } +} + +func (c *client) notifySubscribers(evt retrievalmarket.ClientEvent, ds retrievalmarket.ClientDealState) { + for _, cb := range c.subscribers { + cb(evt, ds) + } +} + +func (c *client) SubscribeToEvents(subscriber retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { + c.subscribers = append(c.subscribers, subscriber) + return c.unsubscribeAt(subscriber) +} + +// V1 +func (c *client) AddMoreFunds(id retrievalmarket.DealID, amount types.BigInt) error { + panic("not implemented") +} + +func (c *client) CancelDeal(id retrievalmarket.DealID) error { + panic("not implemented") +} + +func (c *client) RetrievalStatus(id retrievalmarket.DealID) { + panic("not implemented") +} + +func (c *client) ListDeals() map[retrievalmarket.DealID]retrievalmarket.ClientDealState { + panic("not implemented") +} + +type clientStream struct { + node retrievalmarket.RetrievalClientNode + stream network.Stream + peeker cbg.BytePeeker + + root cid.Cid + size types.BigInt + offset uint64 + + paych retrievalmarket.Address + lane uint64 + total types.BigInt + transferred types.BigInt + + windowSize uint64 // how much we "trust" the peer + verifier BlockVerifier + bs blockstore.Blockstore +} + +/* This is the old retrieval code that is NOT spec compliant */ + +// C > S +// +// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size +// +// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} +// < Resp{Accept} +// < ..(Intermediate Block) +// < ..Blocks +// < ..(Intermediate Block) +// < ..Blocks +// > DealProposal(...) +// < ... +func (c *client) retrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr retrievalmarket.Address) error { + s, err := c.h.NewStream(ctx, miner, retrievalmarket.ProtocolID) + if err != nil { + return err + } + defer s.Close() + + initialOffset := uint64(0) // TODO: Check how much data we have locally + // TODO: Support in handler + // TODO: Allow client to specify this + + paych, err := c.node.GetOrCreatePaymentChannel(ctx, client, minerAddr, total) + if err != nil { + return xerrors.Errorf("getting payment channel: %w", err) + } + lane, err := c.node.AllocateLane(paych) + if err != nil { + return xerrors.Errorf("allocating payment lane: %w", err) + } + + cst := clientStream{ + node: c.node, + stream: s, + peeker: cbg.GetPeeker(s), + + root: root, + size: types.NewInt(size), + offset: initialOffset, + + paych: paych, + lane: lane, + total: total, + transferred: types.NewInt(0), + + windowSize: build.UnixfsChunkSize, + verifier: &UnixFs0Verifier{Root: root}, + bs: c.bs, + } + + for cst.offset != size+initialOffset { + toFetch := cst.windowSize + if toFetch+cst.offset > size { + toFetch = size - cst.offset + } + log.Infof("Retrieve %dB @%d", toFetch, cst.offset) + + err := cst.doOneExchange(ctx, toFetch) + if err != nil { + return xerrors.Errorf("retrieval exchange: %w", err) + } + + cst.offset += toFetch + } + log.Info("RETRIEVE SUCCESSFUL") + return nil +} + +func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64) error { + payAmount := types.BigDiv(types.BigMul(cst.total, types.NewInt(toFetch)), cst.size) + + payment, err := cst.setupPayment(ctx, payAmount) + if err != nil { + return xerrors.Errorf("setting up retrieval payment: %w", err) + } + + deal := &OldDealProposal{ + Payment: payment, + Ref: cst.root, + Params: RetParams{ + Unixfs0: &Unixfs0Offer{ + Offset: cst.offset, + Size: toFetch, + }, + }, + } + + if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { + return err + } + + var resp OldDealResponse + if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { + log.Error(err) + return err + } + + if resp.Status != Accepted { + cst.windowSize = build.UnixfsChunkSize + // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) + + if resp.Status == Error { + return xerrors.Errorf("storage deal error: %s", resp.Message) + } + if resp.Status == Rejected { + return xerrors.Errorf("storage deal rejected: %s", resp.Message) + } + return xerrors.New("storage deal response had no Accepted section") + } + + log.Info("Retrieval accepted, fetching blocks") + + return cst.fetchBlocks(toFetch) + + // TODO: maybe increase miner window size after success +} + +func (cst *clientStream) fetchBlocks(toFetch uint64) error { + blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize + + for i := uint64(0); i < blocksToFetch; { + log.Infof("block %d of %d", i+1, blocksToFetch) + + var block Block + if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil { + return xerrors.Errorf("reading fetchBlock response: %w", err) + } + + dataBlocks, err := cst.consumeBlockMessage(block) + if err != nil { + return xerrors.Errorf("consuming retrieved blocks: %w", err) + } + + i += dataBlocks + } + + return nil +} + +func (cst *clientStream) consumeBlockMessage(block Block) (uint64, error) { + prefix, err := cid.PrefixFromBytes(block.Prefix) + if err != nil { + return 0, err + } + + cid, err := prefix.Sum(block.Data) + + blk, err := blocks.NewBlockWithCid(block.Data, cid) + if err != nil { + return 0, err + } + + internal, err := cst.verifier.Verify(context.TODO(), blk) + if err != nil { + log.Warnf("block verify failed: %s", err) + return 0, err + } + + // TODO: Smarter out, maybe add to filestore automagically + // (Also, persist intermediate nodes) + err = cst.bs.Put(blk) + if err != nil { + log.Warnf("block write failed: %s", err) + return 0, err + } + + if internal { + return 0, nil + } + + return 1, nil +} + +func (cst *clientStream) setupPayment(ctx context.Context, toSend types.BigInt) (api.PaymentInfo, error) { + amount := types.BigAdd(cst.transferred, toSend) + + sv, err := cst.node.CreatePaymentVoucher(ctx, cst.paych, amount, cst.lane) + if err != nil { + return api.PaymentInfo{}, err + } + + cst.transferred = amount + + return api.PaymentInfo{ + Channel: cst.paych, + ChannelMessage: nil, + Vouchers: []*types.SignedVoucher{sv}, + }, nil +} diff --git a/retrieval/miner.go b/retrieval/impl/provider.go similarity index 54% rename from retrieval/miner.go rename to retrieval/impl/provider.go index c9b54c5ec..ef1de4dde 100644 --- a/retrieval/miner.go +++ b/retrieval/impl/provider.go @@ -1,76 +1,144 @@ -package retrieval +package retrievalimpl import ( "context" "io" + "reflect" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" + "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage/sectorblocks" ) -type RetrMinerApi interface { +// RetrMinerAPI are the node functions needed by a retrieval provider +type RetrMinerAPI interface { PaychVoucherAdd(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) } -type Miner struct { - sectorBlocks *sectorblocks.SectorBlocks - full RetrMinerApi +type provider struct { - pricePerByte types.BigInt - // TODO: Unseal price + // TODO: Replace with RetrievalProviderNode & FileStore for https://github.com/filecoin-project/go-retrieval-market-project/issues/9 + sectorBlocks *sectorblocks.SectorBlocks + + // TODO: Replace with RetrievalProviderNode for + // https://github.com/filecoin-project/go-retrieval-market-project/issues/4 + node retrievalmarket.RetrievalProviderNode + + pricePerByte retrievalmarket.BigInt + + subscribers []retrievalmarket.ProviderSubscriber } -func NewMiner(sblks *sectorblocks.SectorBlocks, full api.FullNode) *Miner { - return &Miner{ +// NewProvider returns a new retrieval provider +func NewProvider(sblks *sectorblocks.SectorBlocks, node retrievalmarket.RetrievalProviderNode) retrievalmarket.RetrievalProvider { + return &provider{ sectorBlocks: sblks, - full: full, + node: node, pricePerByte: types.NewInt(2), // TODO: allow setting } } +// Start begins listening for deals on the given host +func (p *provider) Start(host host.Host) { + host.SetStreamHandler(retrievalmarket.QueryProtocolID, p.handleQueryStream) + host.SetStreamHandler(retrievalmarket.ProtocolID, p.handleDealStream) +} + +// V0 +// SetPricePerByte sets the price per byte a miner charges for retrievals +func (p *provider) SetPricePerByte(price retrievalmarket.BigInt) { + p.pricePerByte = price +} + +// SetPaymentInterval sets the maximum number of bytes a a provider will send before +// requesting further payment, and the rate at which that value increases +// TODO: Implement for https://github.com/filecoin-project/go-retrieval-market-project/issues/7 +func (p *provider) SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64) { + panic("not implemented") +} + +// unsubscribeAt returns a function that removes an item from the subscribers list by comparing +// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order. +// Subsequent, repeated calls to the func with the same Subscriber are a no-op. +func (p *provider) unsubscribeAt(sub retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { + return func() { + curLen := len(p.subscribers) + for i, el := range p.subscribers { + if reflect.ValueOf(sub) == reflect.ValueOf(el) { + p.subscribers[i] = p.subscribers[curLen-1] + p.subscribers = p.subscribers[:curLen-1] + return + } + } + } +} + +func (p *provider) notifySubscribers(evt retrievalmarket.ProviderEvent, ds retrievalmarket.ProviderDealState) { + for _, cb := range p.subscribers { + cb(evt, ds) + } +} + +// SubscribeToEvents listens for events that happen related to client retrievals +// TODO: Implement updates as part of https://github.com/filecoin-project/go-retrieval-market-project/issues/7 +func (p *provider) SubscribeToEvents(subscriber retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { + p.subscribers = append(p.subscribers, subscriber) + return p.unsubscribeAt(subscriber) +} + +// V1 +func (p *provider) SetPricePerUnseal(price retrievalmarket.BigInt) { + panic("not implemented") +} + +func (p *provider) ListDeals() map[retrievalmarket.ProviderDealID]retrievalmarket.ProviderDealState { + panic("not implemented") +} + func writeErr(stream network.Stream, err error) { log.Errorf("Retrieval deal error: %+v", err) - _ = cborutil.WriteCborRPC(stream, &DealResponse{ + _ = cborutil.WriteCborRPC(stream, &OldDealResponse{ Status: Error, Message: err.Error(), }) } -func (m *Miner) HandleQueryStream(stream network.Stream) { +// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/8 +func (p *provider) handleQueryStream(stream network.Stream) { defer stream.Close() - var query Query + var query OldQuery if err := cborutil.ReadCborRPC(stream, &query); err != nil { writeErr(stream, err) return } - size, err := m.sectorBlocks.GetSize(query.Piece) + size, err := p.sectorBlocks.GetSize(query.Piece) if err != nil && err != sectorblocks.ErrNotFound { log.Errorf("Retrieval query: GetRefs: %s", err) return } - answer := &QueryResponse{ + answer := &OldQueryResponse{ Status: Unavailable, } if err == nil { answer.Status = Available // TODO: get price, look for already unsealed ref to reduce work - answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), m.pricePerByte) + answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), p.pricePerByte) answer.Size = uint64(size) // TODO: verify on intermediate } @@ -81,7 +149,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { } type handlerDeal struct { - m *Miner + p *provider stream network.Stream ufsr sectorblocks.UnixfsReader @@ -90,11 +158,12 @@ type handlerDeal struct { size uint64 } -func (m *Miner) HandleDealStream(stream network.Stream) { +// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/7 +func (p *provider) handleDealStream(stream network.Stream) { defer stream.Close() hnd := &handlerDeal{ - m: m, + p: p, stream: stream, } @@ -113,7 +182,7 @@ func (m *Miner) HandleDealStream(stream network.Stream) { } func (hnd *handlerDeal) handleNext() (bool, error) { - var deal DealProposal + var deal OldDealProposal if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil { if err == io.EOF { // client sent all deals err = nil @@ -131,8 +200,8 @@ func (hnd *handlerDeal) handleNext() (bool, error) { return false, xerrors.Errorf("expected one signed voucher, got %d", len(deal.Payment.Vouchers)) } - expPayment := types.BigMul(hnd.m.pricePerByte, types.NewInt(deal.Params.Unixfs0.Size)) - if _, err := hnd.m.full.PaychVoucherAdd(context.TODO(), deal.Payment.Channel, deal.Payment.Vouchers[0], nil, expPayment); err != nil { + expPayment := types.BigMul(hnd.p.pricePerByte, types.NewInt(deal.Params.Unixfs0.Size)) + if _, err := hnd.p.node.SavePaymentVoucher(context.TODO(), deal.Payment.Channel, deal.Payment.Vouchers[0], nil, expPayment); err != nil { return false, xerrors.Errorf("processing retrieval payment: %w", err) } @@ -156,7 +225,7 @@ func (hnd *handlerDeal) handleNext() (bool, error) { return true, nil } -func (hnd *handlerDeal) openFile(deal DealProposal) error { +func (hnd *handlerDeal) openFile(deal OldDealProposal) error { unixfs0 := deal.Params.Unixfs0 if unixfs0.Offset != 0 { @@ -165,7 +234,7 @@ func (hnd *handlerDeal) openFile(deal DealProposal) error { } hnd.at = unixfs0.Offset - bstore := hnd.m.sectorBlocks.SealedBlockstore(func() error { + bstore := hnd.p.sectorBlocks.SealedBlockstore(func() error { return nil // TODO: approve unsealing based on amount paid }) @@ -197,10 +266,10 @@ func (hnd *handlerDeal) openFile(deal DealProposal) error { return nil } -func (hnd *handlerDeal) accept(deal DealProposal) error { +func (hnd *handlerDeal) accept(deal OldDealProposal) error { unixfs0 := deal.Params.Unixfs0 - resp := &DealResponse{ + resp := &OldDealResponse{ Status: Accepted, } if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil { diff --git a/retrieval/impl/run_cbor_gen.go b/retrieval/impl/run_cbor_gen.go new file mode 100644 index 000000000..8dbd6f19c --- /dev/null +++ b/retrieval/impl/run_cbor_gen.go @@ -0,0 +1,33 @@ +package retrievalimpl + +import ( + "fmt" + "os" + + cborgen "github.com/whyrusleeping/cbor-gen" +) + +func RunCborGen() error { + genName := "./impl/cbor_gen.go" + reName := "./impl/cbor_gen_old.go" + if err := os.Rename(genName, reName); err != nil { + return fmt.Errorf("could not rename %s to %s", genName, reName) + } + if err := cborgen.WriteTupleEncodersToFile( + genName, + "retrievalimpl", + RetParams{}, + OldQuery{}, + OldQueryResponse{}, + Unixfs0Offer{}, + OldDealProposal{}, + OldDealResponse{}, + Block{}, + ); err != nil { + return err + } + if err := os.Remove(reName); err != nil { + return err + } + return nil +} diff --git a/retrieval/impl/types.go b/retrieval/impl/types.go new file mode 100644 index 000000000..3af74f993 --- /dev/null +++ b/retrieval/impl/types.go @@ -0,0 +1,67 @@ +package retrievalimpl + +import ( + "github.com/filecoin-project/lotus/api" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/types" +) + +/* These types are all the types provided by Lotus, which diverge even from +spec V0 -- prior to the "update to spec epic", we are using these types internally +and switching to spec at the boundaries of the module */ + +type OldQueryResponseStatus uint64 + +const ( + Available OldQueryResponseStatus = iota + Unavailable +) + +const ( + Accepted = iota + Error + Rejected + Unsealing +) + +type OldQuery struct { + Piece cid.Cid + // TODO: payment +} + +type OldQueryResponse struct { + Status OldQueryResponseStatus + + Size uint64 // TODO: spec + // TODO: unseal price (+spec) + // TODO: sectors to unseal + // TODO: address to send money for the deal? + MinPrice types.BigInt +} + +type Unixfs0Offer struct { + Offset uint64 + Size uint64 +} + +type RetParams struct { + Unixfs0 *Unixfs0Offer +} + +type OldDealProposal struct { + Payment api.PaymentInfo + + Ref cid.Cid + Params RetParams +} + +type OldDealResponse struct { + Status uint64 + Message string +} + +type Block struct { // TODO: put in spec + Prefix []byte // TODO: fix cid.Prefix marshaling somehow + Data []byte +} diff --git a/retrieval/verify.go b/retrieval/impl/verify.go similarity index 83% rename from retrieval/verify.go rename to retrieval/impl/verify.go index 88fa0dfed..892876ef5 100644 --- a/retrieval/verify.go +++ b/retrieval/impl/verify.go @@ -1,8 +1,7 @@ -package retrieval +package retrievalimpl import ( "context" - "io" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -16,13 +15,13 @@ import ( ) type BlockVerifier interface { - Verify(context.Context, blocks.Block, io.Writer) (internal bool, err error) + Verify(context.Context, blocks.Block) (internal bool, err error) } type OptimisticVerifier struct { } -func (o *OptimisticVerifier) Verify(context.Context, blocks.Block, io.Writer) (bool, error) { +func (o *OptimisticVerifier) Verify(context.Context, blocks.Block) (bool, error) { // It's probably fine return false, nil } @@ -37,11 +36,11 @@ type UnixFs0Verifier struct { sub *UnixFs0Verifier } -func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.Writer) (last bool, internal bool, err error) { +func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block) (last bool, internal bool, err error) { if b.sub != nil { // TODO: check links here (iff b.sub.sub == nil) - subLast, internal, err := b.sub.verify(ctx, blk, out) + subLast, internal, err := b.sub.verify(ctx, blk) if err != nil { return false, false, err } @@ -57,7 +56,7 @@ func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.W return false, false, xerrors.New("unixfs verifier: too many nodes in level") } - links, err := b.checkInternal(blk, out) + links, err := b.checkInternal(blk) if err != nil { return false, false, err } @@ -85,7 +84,7 @@ func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.W return b.seen == b.expect, false, nil } -func (b *UnixFs0Verifier) checkInternal(blk blocks.Block, out io.Writer) (int, error) { +func (b *UnixFs0Verifier) checkInternal(blk blocks.Block) (int, error) { nd, err := ipld.Decode(blk) if err != nil { log.Warnf("IPLD Decode failed: %s", err) @@ -112,21 +111,20 @@ func (b *UnixFs0Verifier) checkInternal(blk blocks.Block, out io.Writer) (int, e return len(nd.Links()), nil case *merkledag.RawNode: - _, err := out.Write(nd.RawData()) - return 0, err + return 0, nil default: return 0, xerrors.New("verifier: unknown node type") } } -func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block, w io.Writer) (bool, error) { +func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block) (bool, error) { // root is special if b.rootBlk == nil { if !b.Root.Equals(blk.Cid()) { return false, xerrors.Errorf("unixfs verifier: root block CID didn't match: valid %s, got %s", b.Root, blk.Cid()) } b.rootBlk = blk - links, err := b.checkInternal(blk, w) + links, err := b.checkInternal(blk) if err != nil { return false, err } @@ -135,7 +133,7 @@ func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block, w io.Wri return links != 0, nil } - _, internal, err := b.verify(ctx, blk, w) + _, internal, err := b.verify(ctx, blk) return internal, err } diff --git a/retrieval/types.go b/retrieval/types.go index 28343e6b2..f82279a9d 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -1,66 +1,364 @@ -package retrieval +package retrievalmarket import ( - "github.com/filecoin-project/lotus/api" - "github.com/ipfs/go-cid" + "context" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/peer" ) -const ProtocolID = "/fil/retrieval/-1.0.0" // TODO: spec -const QueryProtocolID = "/fil/retrieval/qry/-1.0.0" // TODO: spec +// type aliases +// TODO: Remove and use native types or extract for +// https://github.com/filecoin-project/go-retrieval-market-project/issues/5 +// BigInt is used for token amounts in retrieval deals +type BigInt = types.BigInt + +// Address is an address in the filecoin network +type Address = address.Address + +// SignedVoucher is a signed payment voucher +type SignedVoucher = types.SignedVoucher + +// ProtocolID is the protocol for proposing / responding to retrieval deals +const ProtocolID = "/fil/retrieval/0.0.1" + +// QueryProtocolID is the protocol for querying infromation about retrieval +// deal parameters +const QueryProtocolID = "/fil/retrieval/qry/0.0.1" // TODO: spec + +// Unsubscribe is a function that unsubscribes a subscriber for either the +// client or the provider +type Unsubscribe func() + +// ClientDealState is the current state of a deal from the point of view +// of a retrieval client +type ClientDealState struct { + DealProposal + Status DealStatus + Sender peer.ID + TotalReceived uint64 + FundsSpent BigInt +} + +// ClientEvent is an event that occurs in a deal lifecycle on the client +type ClientEvent uint64 + +const ( + // ClientEventOpen indicates a deal was initiated + ClientEventOpen ClientEvent = iota + + // ClientEventFundsExpended indicates a deal has run out of funds in the payment channel + // forcing the client to add more funds to continue the deal + ClientEventFundsExpended // when totalFunds is expended + + // ClientEventProgress indicates more data was received for a retrieval + ClientEventProgress + + // ClientEventError indicates an error occurred during a deal + ClientEventError + + // ClientEventComplete indicates a deal has completed + ClientEventComplete +) + +// ClientSubscriber is a callback that is registered to listen for retrieval events +type ClientSubscriber func(event ClientEvent, state ClientDealState) + +// RetrievalClient is a client interface for making retrieval deals +type RetrievalClient interface { + // V0 + + // Find Providers finds retrieval providers who may be storing a given piece + FindProviders(pieceCID []byte) []RetrievalPeer + + // Query asks a provider for information about a piece it is storing + Query( + ctx context.Context, + p RetrievalPeer, + pieceCID []byte, + params QueryParams, + ) (QueryResponse, error) + + // Retrieve retrieves all or part of a piece with the given retrieval parameters + Retrieve( + ctx context.Context, + pieceCID []byte, + params Params, + totalFunds BigInt, + miner peer.ID, + clientWallet Address, + minerWallet Address, + ) DealID + + // SubscribeToEvents listens for events that happen related to client retrievals + SubscribeToEvents(subscriber ClientSubscriber) Unsubscribe + + // V1 + AddMoreFunds(id DealID, amount BigInt) error + CancelDeal(id DealID) error + RetrievalStatus(id DealID) + ListDeals() map[DealID]ClientDealState +} + +// RetrievalClientNode are the node depedencies for a RetrevalClient +type RetrievalClientNode interface { + + // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist + // between a client and a miner and insures the client has the given amount of funds available in the channel + GetOrCreatePaymentChannel(ctx context.Context, clientAddress Address, minerAddress Address, clientFundsAvailable BigInt) (Address, error) + + // Allocate late creates a lane within a payment channel so that calls to + // CreatePaymentVoucher will automatically make vouchers only for the difference + // in total + AllocateLane(paymentChannel Address) (uint64, error) + + // CreatePaymentVoucher creates a new payment voucher in the given lane for a + // given payment channel so that all the payment vouchers in the lane add up + // to the given amount (so the payment voucher will be for the difference) + CreatePaymentVoucher(ctx context.Context, paymentChannel Address, amount BigInt, lane uint64) (*SignedVoucher, error) +} + +// ProviderDealState is the current state of a deal from the point of view +// of a retrieval provider +type ProviderDealState struct { + DealProposal + Status DealStatus + Receiver peer.ID + TotalSent uint64 + FundsReceived BigInt +} + +// ProviderEvent is an event that occurs in a deal lifecycle on the provider +type ProviderEvent uint64 + +const ( + + // ProviderEventOpen indicates a new deal was received from a client + ProviderEventOpen ProviderEvent = iota + + // ProviderEventProgress indicates more data was sent to a client + ProviderEventProgress + + // ProviderEventError indicates an error occurred in processing a deal for a client + ProviderEventError + + // ProviderEventComplete indicates a retrieval deal was completed for a client + ProviderEventComplete +) + +// ProviderDealID is a unique identifier for a deal on a provider -- it is +// a combination of DealID set by the client and the peer ID of the client +type ProviderDealID struct { + From peer.ID + ID DealID +} + +// ProviderSubscriber is a callback that is registered to listen for retrieval events on a provider +type ProviderSubscriber func(event ProviderEvent, state ProviderDealState) + +// RetrievalProvider is an interface by which a provider configures their +// retrieval operations and monitors deals received and process +type RetrievalProvider interface { + // Start begins listening for deals on the given host + Start(host.Host) + + // V0 + + // SetPricePerByte sets the price per byte a miner charges for retrievals + SetPricePerByte(price BigInt) + + // SetPaymentInterval sets the maximum number of bytes a a provider will send before + // requesting further payment, and the rate at which that value increases + SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64) + + // SubscribeToEvents listens for events that happen related to client retrievals + SubscribeToEvents(subscriber ProviderSubscriber) Unsubscribe + + // V1 + SetPricePerUnseal(price BigInt) + ListDeals() map[ProviderDealID]ProviderDealState +} + +// RetrievalProviderNode are the node depedencies for a RetrevalProvider +type RetrievalProviderNode interface { + SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *SignedVoucher, proof []byte, expectedAmount BigInt) (BigInt, error) +} + +// PeerResolver is an interface for looking up providers that may have a piece +type PeerResolver interface { + GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel +} + +// RetrievalPeer is a provider address/peer.ID pair (everything needed to make +// deals for with a miner) +type RetrievalPeer struct { + Address Address + ID peer.ID // optional +} + +// QueryResponseStatus indicates whether a queried piece is available type QueryResponseStatus uint64 const ( - Available QueryResponseStatus = iota - Unavailable + // QueryResponseAvailable indicates a provider has a piece and is prepared to + // return it + QueryResponseAvailable QueryResponseStatus = iota + + // QueryResponseUnavailable indicates a provider either does not have or cannot + // serve the queried piece to the client + QueryResponseUnavailable ) +// QueryItemStatus (V1) indicates whether the requested part of a piece (payload or selector) +// is available for retrieval +type QueryItemStatus uint64 + const ( - Accepted = iota - Error - Rejected - Unsealing + // QueryItemAvailable indicates requested part of the piece is available to be + // served + QueryItemAvailable QueryItemStatus = iota + + // QueryItemUnavailable indicates the piece either does not contain the requested + // item or it cannot be served + QueryItemUnavailable + + // QueryItemUnknown indicates the provider cannot determine if the given item + // is part of the requested piece (for example, if the piece is sealed and the + // miner does not maintain a payload CID index) + QueryItemUnknown ) -type Query struct { - Piece cid.Cid - // TODO: payment +// QueryParams indicate what specific information about a piece that a retrieval +// client is interested in, as well as specific parameters the client is seeking +// for the retrieval deal +type QueryParams struct { + PayloadCID cid.Cid // optional, query if miner has this cid in this piece. some miners may not be able to respond. + Selector ipld.Node // optional, query if miner has this cid in this piece. some miners may not be able to respond. + MaxPricePerByte BigInt // optional, tell miner uninterested if more expensive than this + MinPaymentInterval uint64 // optional, tell miner uninterested unless payment interval is greater than this + MinPaymentIntervalIncrease uint64 // optional, tell miner uninterested unless payment interval increase is greater than this } +// Query is a query to a given provider to determine information about a piece +// they may have available for retrieval +type Query struct { + PieceCID []byte // V0 + // QueryParams // V1 +} + +// QueryResponse is a miners response to a given retrieval query type QueryResponse struct { Status QueryResponseStatus + //PayloadCIDFound QueryItemStatus // V1 - if a PayloadCid was requested, the result + //SelectorFound QueryItemStatus // V1 - if a Selector was requested, the result - Size uint64 // TODO: spec - // TODO: unseal price (+spec) - // TODO: sectors to unseal - // TODO: address to send money for the deal? - MinPrice types.BigInt + Size uint64 // Total size of piece in bytes + //ExpectedPayloadSize uint64 // V1 - optional, if PayloadCID + selector are specified and miner knows, can offer an expected size + + PaymentAddress Address // address to send funds to -- may be different than miner addr + MinPricePerByte BigInt + MaxPaymentInterval uint64 + MaxPaymentIntervalIncrease uint64 } -type Unixfs0Offer struct { - Offset uint64 - Size uint64 +// QueryResponseUndefined is an empty QueryResponse +var QueryResponseUndefined = QueryResponse{} + +// PieceRetrievalPrice is the total price to retrieve the piece (size * MinPricePerByte) +func (qr QueryResponse) PieceRetrievalPrice() BigInt { + return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.Size)) } -type RetParams struct { - Unixfs0 *Unixfs0Offer +// PayloadRetrievalPrice is the expected price to retrieve just the given payload +// & selector (V1) +//func (qr QueryResponse) PayloadRetrievalPrice() BigInt { +// return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.ExpectedPayloadSize)) +//} + +// DealStatus is the status of a retrieval deal returned by a provider +// in a DealResponse +type DealStatus uint64 + +const ( + // DealStatusAccepted means a deal has been accepted by a provider + // and its is ready to proceed with retrieval + DealStatusAccepted DealStatus = iota + + // DealStatusFailed indicates something went wrong during a retrieval + DealStatusFailed + + // DealStatusRejected indicates the provider rejected a client's deal proposal + // for some reason + DealStatusRejected + + // DealStatusUnsealing indicates the provider is currently unsealing the sector + // needed to serve the retrieval deal + DealStatusUnsealing + + // DealStatusFundsNeeded indicates the provider is awaiting a payment voucher to + // continue processing the deal + DealStatusFundsNeeded + + // DealStatusOngoing indicates the provider is continuing to process a deal + DealStatusOngoing + + // DealStatusFundsNeededLastPayment indicates the provider is awaiting funds for + // a final payment in order to complete a deal + DealStatusFundsNeededLastPayment + + // DealStatusCompleted indicates a deal is complete + DealStatusCompleted + + // DealStatusDealNotFound indicates an update was received for a deal that could + // not be identified + DealStatusDealNotFound +) + +// Params are the parameters requested for a retrieval deal proposal +type Params struct { + //PayloadCID cid.Cid // V1 + //Selector ipld.Node // V1 + PricePerByte BigInt + PaymentInterval uint64 + PaymentIntervalIncrease uint64 } +// DealID is an identifier for a retrieval deal (unique to a client) +type DealID uint64 + +// DealProposal is a proposal for a new retrieval deal type DealProposal struct { - Payment api.PaymentInfo - - Ref cid.Cid - Params RetParams + PieceCID []byte + ID DealID + Params } -type DealResponse struct { - Status uint64 - Message string -} - -type Block struct { // TODO: put in spec - Prefix []byte // TODO: fix cid.Prefix marshaling somehow +// Block is an IPLD block in bitswap format +type Block struct { + Prefix []byte Data []byte } + +// DealResponse is a response to a retrieval deal proposal +type DealResponse struct { + Status DealStatus + ID DealID + + // payment required to proceed + PaymentOwed BigInt + + Message string + Blocks []Block // V0 only +} + +// DealPayment is a payment for an in progress retrieval deal +type DealPayment struct { + ID DealID + PaymentChannel address.Address + PaymentVoucher *types.SignedVoucher +} diff --git a/retrievaladapter/client.go b/retrievaladapter/client.go new file mode 100644 index 000000000..ff7d2f0fb --- /dev/null +++ b/retrievaladapter/client.go @@ -0,0 +1,41 @@ +package retrievaladapter + +import ( + "context" + + payapi "github.com/filecoin-project/lotus/node/impl/paych" + "github.com/filecoin-project/lotus/paych" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" +) + +type retrievalClientNode struct { + pmgr *paych.Manager + payapi payapi.PaychAPI +} + +// NewRetrievalClientNode returns a new node adapter for a retrieval client that talks to the +// Lotus Node +func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClientNode { + return &retrievalClientNode{pmgr: pmgr, payapi: payapi} +} + +// GetOrCreatePaymentChannel sets up a new payment channel if one does not exist +// between a client and a miner and insures the client has the given amount of funds available in the channel +func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress retrievalmarket.Address, minerAddress retrievalmarket.Address, clientFundsAvailable retrievalmarket.BigInt) (retrievalmarket.Address, error) { + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, clientFundsAvailable) + return paych, err +} + +// Allocate late creates a lane within a payment channel so that calls to +// CreatePaymentVoucher will automatically make vouchers only for the difference +// in total +func (rcn *retrievalClientNode) AllocateLane(paymentChannel retrievalmarket.Address) (uint64, error) { + return rcn.pmgr.AllocateLane(paymentChannel) +} + +// CreatePaymentVoucher creates a new payment voucher in the given lane for a +// given payment channel so that all the payment vouchers in the lane add up +// to the given amount (so the payment voucher will be for the difference) +func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel retrievalmarket.Address, amount retrievalmarket.BigInt, lane uint64) (*retrievalmarket.SignedVoucher, error) { + return rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, amount, lane) +} diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go new file mode 100644 index 000000000..f8d188a95 --- /dev/null +++ b/retrievaladapter/provider.go @@ -0,0 +1,23 @@ +package retrievaladapter + +import ( + "context" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/go-address" + retrievalmarket "github.com/filecoin-project/lotus/retrieval" +) + +type retrievalProviderNode struct { + full api.FullNode +} + +// NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the +// Lotus Node +func NewRetrievalProviderNode(full api.FullNode) retrievalmarket.RetrievalProviderNode { + return &retrievalProviderNode{full} +} + +func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievalmarket.SignedVoucher, proof []byte, expectedAmount retrievalmarket.BigInt) (retrievalmarket.BigInt, error) { + return rpn.full.PaychVoucherAdd(ctx, paymentChannel, voucher, proof, expectedAmount) +} From 7ba34535b7b6ee2adfe67082e885ecd4c4b5d559 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 8 Jan 2020 13:40:51 -0800 Subject: [PATCH 23/61] Set hard limit on maximum randomness lookback for seal proof submissions --- build/params_shared.go | 3 +++ chain/actors/actor_miner.go | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/build/params_shared.go b/build/params_shared.go index 0357bf03a..fd9dbf65d 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -56,6 +56,9 @@ const SealRandomnessLookback = Finality // Epochs const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 +// Maximum lookback that randomness can be sourced from for a seal proof submission +const MaxSealLookback = SealRandomnessLookbackLimit + 2000 + // ///// // Mining diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index e5e8b3b1e..c6bb7ae6d 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -321,6 +321,14 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC // TODO: ensure normalization to ID address maddr := vmctx.Message().To + if vmctx.BlockHeight()-us.Info.SealEpoch > build.MaxSealLookback { + return nil, aerrors.Newf(5, "source randomness for sector SealEpoch too far in past (epoch %d)", us.Info.SealEpoch) + } + + if vmctx.BlockHeight()-us.ReceivedEpoch > build.MaxSealLookback { + return nil, aerrors.Newf(6, "source randomness for sector ReceivedEpoch too far in past (epoch %d)", us.ReceivedEpoch) + } + ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback) if err != nil { return nil, aerrors.Wrap(err, "failed to get ticket randomness") From d08898ef37dedbb23d69379656d1898c4d1a3096 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 8 Jan 2020 17:10:17 -0800 Subject: [PATCH 24/61] make storage miner acccept an interface to the sectorbuilder --- node/builder.go | 2 +- storage/fpost_sched.go | 3 +-- storage/miner.go | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/node/builder.go b/node/builder.go index ee15f8e70..0e28d7b0c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -238,7 +238,7 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, - Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder), + Override(new(storage.SectorBuilder), modules.SectorBuilder), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), diff --git a/storage/fpost_sched.go b/storage/fpost_sched.go index fc192ebee..30dadbd56 100644 --- a/storage/fpost_sched.go +++ b/storage/fpost_sched.go @@ -4,7 +4,6 @@ import ( "context" "sync" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -20,7 +19,7 @@ const StartConfidence = 4 // TODO: config type fpostScheduler struct { api storageMinerApi - sb *sectorbuilder.SectorBuilder + sb SectorBuilder actor address.Address worker address.Address diff --git a/storage/miner.go b/storage/miner.go index 53303a504..c7d44d885 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,6 +3,7 @@ package storage import ( "context" "errors" + "io" "time" "github.com/ipfs/go-cid" @@ -36,7 +37,7 @@ type Miner struct { worker address.Address // Sealing - sb *sectorbuilder.SectorBuilder + sb SectorBuilder sectors *statestore.StateStore tktFn TicketFn @@ -71,7 +72,18 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb *sectorbuilder.SectorBuilder, tktFn TicketFn) (*Miner, error) { +type SectorBuilder interface { + RateLimit() func() + AddPiece(uint64, uint64, io.Reader, []uint64) (sectorbuilder.PublicPieceInfo, error) + SectorSize() uint64 + AcquireSectorId() (uint64, error) + Scrub(sectorbuilder.SortedPublicSectorInfo) []*sectorbuilder.Fault + GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error) + SealPreCommit(uint64, sectorbuilder.SealTicket, []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) + SealCommit(uint64, sectorbuilder.SealTicket, sectorbuilder.SealSeed, []sectorbuilder.PublicPieceInfo, sectorbuilder.RawSealPreCommitOutput) ([]byte, error) +} + +func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb SectorBuilder, tktFn TicketFn) (*Miner, error) { return &Miner{ api: api, From 99309ec0dcd6cc4308c29299337cb819545647bc Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 8 Jan 2020 19:46:11 -0800 Subject: [PATCH 25/61] fix(retrievalmarket): add mutex for subscribers Add a mutex to protect access to the subscriber list for retrieval market --- retrieval/impl/client.go | 14 +++++++++++--- retrieval/impl/provider.go | 13 +++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/retrieval/impl/client.go b/retrieval/impl/client.go index 96bad3503..5087a893d 100644 --- a/retrieval/impl/client.go +++ b/retrieval/impl/client.go @@ -30,9 +30,10 @@ type client struct { node retrievalmarket.RetrievalClientNode // The parameters should be replaced by RetrievalClientNode - nextDealLk sync.Mutex - nextDealID retrievalmarket.DealID - subscribers []retrievalmarket.ClientSubscriber + nextDealLk sync.Mutex + nextDealID retrievalmarket.DealID + subscribersLk sync.RWMutex + subscribers []retrievalmarket.ClientSubscriber } // NewClient creates a new retrieval client @@ -129,6 +130,8 @@ func (c *client) Retrieve(ctx context.Context, pieceCID []byte, params retrieval // Subsequent, repeated calls to the func with the same Subscriber are a no-op. func (c *client) unsubscribeAt(sub retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { return func() { + c.subscribersLk.Lock() + defer c.subscribersLk.Unlock() curLen := len(c.subscribers) for i, el := range c.subscribers { if reflect.ValueOf(sub) == reflect.ValueOf(el) { @@ -141,13 +144,18 @@ func (c *client) unsubscribeAt(sub retrievalmarket.ClientSubscriber) retrievalma } func (c *client) notifySubscribers(evt retrievalmarket.ClientEvent, ds retrievalmarket.ClientDealState) { + c.subscribersLk.RLock() + defer c.subscribersLk.RUnlock() for _, cb := range c.subscribers { cb(evt, ds) } } func (c *client) SubscribeToEvents(subscriber retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { + c.subscribersLk.Lock() c.subscribers = append(c.subscribers, subscriber) + c.subscribersLk.Unlock() + return c.unsubscribeAt(subscriber) } diff --git a/retrieval/impl/provider.go b/retrieval/impl/provider.go index ef1de4dde..9e7254b60 100644 --- a/retrieval/impl/provider.go +++ b/retrieval/impl/provider.go @@ -4,6 +4,7 @@ import ( "context" "io" "reflect" + "sync" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" @@ -14,7 +15,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" retrievalmarket "github.com/filecoin-project/lotus/retrieval" @@ -37,7 +38,8 @@ type provider struct { pricePerByte retrievalmarket.BigInt - subscribers []retrievalmarket.ProviderSubscriber + subscribersLk sync.RWMutex + subscribers []retrievalmarket.ProviderSubscriber } // NewProvider returns a new retrieval provider @@ -74,6 +76,8 @@ func (p *provider) SetPaymentInterval(paymentInterval uint64, paymentIntervalInc // Subsequent, repeated calls to the func with the same Subscriber are a no-op. func (p *provider) unsubscribeAt(sub retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { return func() { + p.subscribersLk.Lock() + defer p.subscribersLk.Unlock() curLen := len(p.subscribers) for i, el := range p.subscribers { if reflect.ValueOf(sub) == reflect.ValueOf(el) { @@ -86,6 +90,8 @@ func (p *provider) unsubscribeAt(sub retrievalmarket.ProviderSubscriber) retriev } func (p *provider) notifySubscribers(evt retrievalmarket.ProviderEvent, ds retrievalmarket.ProviderDealState) { + p.subscribersLk.RLock() + defer p.subscribersLk.RUnlock() for _, cb := range p.subscribers { cb(evt, ds) } @@ -94,7 +100,10 @@ func (p *provider) notifySubscribers(evt retrievalmarket.ProviderEvent, ds retri // SubscribeToEvents listens for events that happen related to client retrievals // TODO: Implement updates as part of https://github.com/filecoin-project/go-retrieval-market-project/issues/7 func (p *provider) SubscribeToEvents(subscriber retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { + p.subscribersLk.Lock() p.subscribers = append(p.subscribers, subscriber) + p.subscribersLk.Unlock() + return p.unsubscribeAt(subscriber) } From 60969c69a7f132e9c1eb87201bc2fdee6bd64af1 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 8 Jan 2020 20:27:14 -0800 Subject: [PATCH 26/61] implement a mocked out sectorbuilder, probably not useful yet --- storage/sbmock/sbmock.go | 121 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 storage/sbmock/sbmock.go diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go new file mode 100644 index 000000000..a46ec5bfa --- /dev/null +++ b/storage/sbmock/sbmock.go @@ -0,0 +1,121 @@ +package sbmock + +import ( + "io" + "io/ioutil" + "math/rand" + "sync" + + "github.com/filecoin-project/go-sectorbuilder" + "golang.org/x/xerrors" +) + +func randComm() [sectorbuilder.CommLen]byte { + var out [sectorbuilder.CommLen]byte + rand.Read(out[:]) + return out +} + +type SBMock struct { + sectors map[uint64]*sectorState + sectorSize uint64 + nextSectorID uint64 + rateLimit chan struct{} + + lk sync.Mutex +} + +func NewMockSectorBuilder(threads int, ssize uint64) *SBMock { + return &SBMock{ + sectors: make(map[uint64]*sectorState), + sectorSize: ssize, + nextSectorID: 0, + rateLimit: make(chan struct{}, threads), + } +} + +type sectorState struct { + pieces [][]byte +} + +func (sb *SBMock) RateLimit() func() { + sb.rateLimit <- struct{}{} + + // TODO: probably want to copy over rate limit code + return func() { + <-sb.rateLimit + } +} + +func (sb *SBMock) AddPiece(size uint64, sectorId uint64, r io.Reader, existingPieces []uint64) (sectorbuilder.PublicPieceInfo, error) { + sb.lk.Lock() + defer sb.lk.Unlock() + + ss, ok := sb.sectors[sectorId] + if !ok { + ss = §orState{} + sb.sectors[sectorId] = ss + } + + b, err := ioutil.ReadAll(r) + if err != nil { + return sectorbuilder.PublicPieceInfo{}, err + } + + ss.pieces = append(ss.pieces, b) + return sectorbuilder.PublicPieceInfo{ + Size: size, + // TODO: should we compute a commP? maybe do it when we need it + }, nil +} + +func (sb *SBMock) SectorSize() uint64 { + return sb.sectorSize +} + +func (sb *SBMock) AcquireSectorId() (uint64, error) { + sb.lk.Lock() + sb.lk.Unlock() + id := sb.nextSectorID + sb.nextSectorID++ + return id, nil +} + +func (sb *SBMock) Scrub(sectorbuilder.SortedPublicSectorInfo) []*sectorbuilder.Fault { + return nil +} + +func (sb *SBMock) GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error) { + panic("NYI") +} + +func (sb *SBMock) SealPreCommit(sid uint64, ticket sectorbuilder.SealTicket, pieces []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) { + _, ok := sb.sectors[sid] + if !ok { + return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("no sector with id %d in sectorbuilder", sid) + } + + ussize := sectorbuilder.UserBytesForSectorSize(sb.sectorSize) + + // TODO: verify pieces in sinfo.pieces match passed in pieces + + var sum uint64 + for _, p := range pieces { + sum += p.Size + } + + if sum != ussize { + return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match up: %d != %d", sum, ussize) + } + + return sectorbuilder.RawSealPreCommitOutput{ + CommD: randComm(), + CommR: randComm(), + }, nil +} + +func (sb *SBMock) SealCommit(sid uint64, ticket sectorbuilder.SealTicket, seed sectorbuilder.SealSeed, pieces []sectorbuilder.PublicPieceInfo, precommit sectorbuilder.RawSealPreCommitOutput) ([]byte, error) { + buf := make([]byte, 32) + rand.Read(buf) + return buf, nil +} From cc56389dbb5060f46b6af3b5d1bd5c81105d5415 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 8 Jan 2020 20:51:18 -0800 Subject: [PATCH 27/61] attempt to fix the DI loading, still not working --- node/modules/storageminer.go | 2 +- storage/miner.go | 4 ++-- storage/sectorblocks/blocks.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index b8e3f8a69..ccdff10cc 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -92,7 +92,7 @@ func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit } } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb storage.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err diff --git a/storage/miner.go b/storage/miner.go index c7d44d885..4a35e1dd4 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -159,8 +159,8 @@ type SectorBuilderEpp struct { sb *sectorbuilder.SectorBuilder } -func NewElectionPoStProver(sb *sectorbuilder.SectorBuilder) *SectorBuilderEpp { - return &SectorBuilderEpp{sb} +func NewElectionPoStProver(sb SectorBuilder) *SectorBuilderEpp { + return &SectorBuilderEpp{sb.(*sectorbuilder.SectorBuilder)} } var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil) diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 9d12e0537..4b80d0fb6 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -47,10 +47,10 @@ type SectorBlocks struct { keyLk sync.Mutex } -func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { +func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb storage.SectorBuilder) *SectorBlocks { sbc := &SectorBlocks{ Miner: miner, - sb: sb, + sb: sb.(*sectorbuilder.SectorBuilder), intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)), From fac47bfc54c34581567b6c946fd32989816d37a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 9 Jan 2020 14:23:44 +0100 Subject: [PATCH 28/61] Pass correct datastore into sectorbuilder --- cmd/lotus-seed/main.go | 6 ++++-- cmd/lotus-seed/seed/seed.go | 4 +++- cmd/lotus-storage-miner/init.go | 5 +++-- node/modules/storageminer.go | 2 +- node/node_test.go | 12 +++++------- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index ee1481d3b..82abbe047 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -8,6 +8,8 @@ import ( "encoding/json" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger" logging "github.com/ipfs/go-log" "github.com/mitchellh/go-homedir" @@ -196,7 +198,7 @@ var aggregateSectorDirsCmd = &cli.Command{ SectorSize: ssize, Dir: destdir, WorkerThreads: 2, - }, agmds) + }, namespace.Wrap(agmds, datastore.NewKey("/sectorbuilder"))) if err != nil { return err } @@ -257,7 +259,7 @@ var aggregateSectorDirsCmd = &cli.Command{ SectorSize: genm.SectorSize, Dir: dir, WorkerThreads: 2, - }, mds) + }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { return err } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index e98abdb0b..e42593bf4 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -11,6 +11,8 @@ import ( "path/filepath" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger" logging "github.com/ipfs/go-log" "golang.org/x/xerrors" @@ -43,7 +45,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb return nil, err } - sb, err := sectorbuilder.New(cfg, mds) + sb, err := sectorbuilder.New(cfg, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { return nil, err } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index e4bd00738..3e16e18e9 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -13,6 +13,7 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" @@ -174,7 +175,7 @@ var initCmd = &cli.Command{ SectorSize: ssize, WorkerThreads: 2, Dir: pssb, - }, oldmds) + }, namespace.Wrap(oldmds, datastore.NewKey("/sectorbuilder"))) if err != nil { return xerrors.Errorf("failed to open up preseal sectorbuilder: %w", err) } @@ -183,7 +184,7 @@ var initCmd = &cli.Command{ SectorSize: ssize, WorkerThreads: 2, Dir: lr.Path(), - }, mds) + }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { return xerrors.Errorf("failed to open up sectorbuilder: %w", err) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index f93d8c945..436686995 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -206,7 +206,7 @@ func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, } func SectorBuilder(cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) { - sb, err := sectorbuilder.New(cfg, ds) + sb, err := sectorbuilder.New(cfg, namespace.Wrap(ds, datastore.NewKey("/sectorbuilder"))) if err != nil { return nil, err } diff --git a/node/node_test.go b/node/node_test.go index f8592b889..4c372aa1a 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -9,20 +9,18 @@ import ( "path/filepath" "testing" - "github.com/filecoin-project/lotus/build" - - "github.com/libp2p/go-libp2p-core/crypto" - + "github.com/filecoin-project/go-address" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/build" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger" logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/api/test" @@ -231,7 +229,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te WorkerThreads: 2, Miner: genMiner, Dir: psd, - }, mds) + }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { t.Fatal(err) } From 5e16aab51b4c6a51a3ca735b7e7b580c24a14775 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 9 Jan 2020 10:28:14 -0800 Subject: [PATCH 29/61] fix tests, expand interfaces a little --- node/impl/storminer.go | 5 +++-- node/node_test.go | 2 +- storage/miner.go | 6 ++++++ storage/sbmock/sbmock.go | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 5568ed529..ac149b64b 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,12 +3,13 @@ package impl import ( "context" "encoding/json" - "github.com/filecoin-project/lotus/api/apistruct" "io" "mime" "net/http" "os" + "github.com/filecoin-project/lotus/api/apistruct" + "github.com/gorilla/mux" files "github.com/ipfs/go-ipfs-files" @@ -25,7 +26,7 @@ type StorageMinerAPI struct { CommonAPI SectorBuilderConfig *sectorbuilder.Config - SectorBuilder *sectorbuilder.SectorBuilder + SectorBuilder storage.SectorBuilder SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner diff --git a/node/node_test.go b/node/node_test.go index 3903e2dae..b229a2fef 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -236,7 +236,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te t.Fatal(err) } - if err := sma.SectorBuilder.ImportFrom(osb, false); err != nil { + if err := sma.SectorBuilder.(*sectorbuilder.SectorBuilder).ImportFrom(osb, false); err != nil { t.Fatal(err) } diff --git a/storage/miner.go b/storage/miner.go index 4a35e1dd4..ed2b8876f 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -81,6 +81,12 @@ type SectorBuilder interface { GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error) SealPreCommit(uint64, sectorbuilder.SealTicket, []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) SealCommit(uint64, sectorbuilder.SealTicket, sectorbuilder.SealSeed, []sectorbuilder.PublicPieceInfo, sectorbuilder.RawSealPreCommitOutput) ([]byte, error) + + // Not so sure about these being on the interface + GetPath(string, string) (string, error) + WorkerStats() sectorbuilder.WorkerStats + AddWorker(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) + TaskDone(context.Context, uint64, sectorbuilder.SealRes) error } func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb SectorBuilder, tktFn TicketFn) (*Miner, error) { diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index a46ec5bfa..dd886fe30 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -1,6 +1,7 @@ package sbmock import ( + "context" "io" "io/ioutil" "math/rand" @@ -119,3 +120,19 @@ func (sb *SBMock) SealCommit(sid uint64, ticket sectorbuilder.SealTicket, seed s rand.Read(buf) return buf, nil } + +func (sb *SBMock) GetPath(string, string) (string, error) { + panic("nyi") +} + +func (sb *SBMock) WorkerStats() sectorbuilder.WorkerStats { + panic("nyi") +} + +func (sb *SBMock) AddWorker(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { + panic("nyi") +} + +func (sb *SBMock) TaskDone(context.Context, uint64, sectorbuilder.SealRes) error { + panic("nyi") +} From bda85e40974066ccbd967c4478fd56e69af51a02 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 9 Jan 2020 11:23:42 -0800 Subject: [PATCH 30/61] more features, better locking --- storage/sbmock/sbmock.go | 87 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index dd886fe30..45f83d76d 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -2,10 +2,12 @@ package sbmock import ( "context" + "fmt" "io" "io/ioutil" "math/rand" "sync" + "time" "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" @@ -23,6 +25,9 @@ type SBMock struct { nextSectorID uint64 rateLimit chan struct{} + preCommitDelay time.Duration + commitDelay time.Duration + lk sync.Mutex } @@ -35,8 +40,19 @@ func NewMockSectorBuilder(threads int, ssize uint64) *SBMock { } } +const ( + statePacking = iota + statePreCommit + stateCommit +) + type sectorState struct { pieces [][]byte + failed bool + + state int + + lk sync.Mutex } func (sb *SBMock) RateLimit() func() { @@ -50,13 +66,16 @@ func (sb *SBMock) RateLimit() func() { func (sb *SBMock) AddPiece(size uint64, sectorId uint64, r io.Reader, existingPieces []uint64) (sectorbuilder.PublicPieceInfo, error) { sb.lk.Lock() - defer sb.lk.Unlock() - ss, ok := sb.sectors[sectorId] if !ok { - ss = §orState{} + ss = §orState{ + state: statePacking, + } sb.sectors[sectorId] = ss } + sb.lk.Unlock() + ss.lk.Lock() + defer ss.lk.Unlock() b, err := ioutil.ReadAll(r) if err != nil { @@ -76,7 +95,7 @@ func (sb *SBMock) SectorSize() uint64 { func (sb *SBMock) AcquireSectorId() (uint64, error) { sb.lk.Lock() - sb.lk.Unlock() + defer sb.lk.Unlock() id := sb.nextSectorID sb.nextSectorID++ return id, nil @@ -91,11 +110,16 @@ func (sb *SBMock) GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [se } func (sb *SBMock) SealPreCommit(sid uint64, ticket sectorbuilder.SealTicket, pieces []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) { - _, ok := sb.sectors[sid] + sb.lk.Lock() + ss, ok := sb.sectors[sid] + sb.lk.Unlock() if !ok { return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("no sector with id %d in sectorbuilder", sid) } + ss.lk.Lock() + defer ss.lk.Unlock() + ussize := sectorbuilder.UserBytesForSectorSize(sb.sectorSize) // TODO: verify pieces in sinfo.pieces match passed in pieces @@ -109,6 +133,14 @@ func (sb *SBMock) SealPreCommit(sid uint64, ticket sectorbuilder.SealTicket, pie return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match up: %d != %d", sum, ussize) } + if ss.state != statePacking { + return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("cannot call pre-seal on sector not in 'packing' state") + } + + time.Sleep(sb.preCommitDelay) + + ss.state = statePreCommit + return sectorbuilder.RawSealPreCommitOutput{ CommD: randComm(), CommR: randComm(), @@ -116,6 +148,25 @@ func (sb *SBMock) SealPreCommit(sid uint64, ticket sectorbuilder.SealTicket, pie } func (sb *SBMock) SealCommit(sid uint64, ticket sectorbuilder.SealTicket, seed sectorbuilder.SealSeed, pieces []sectorbuilder.PublicPieceInfo, precommit sectorbuilder.RawSealPreCommitOutput) ([]byte, error) { + sb.lk.Lock() + ss, ok := sb.sectors[sid] + sb.lk.Unlock() + if !ok { + return nil, xerrors.Errorf("no such sector %d", sid) + } + ss.lk.Lock() + defer ss.lk.Unlock() + + if ss.failed { + return nil, xerrors.Errorf("[mock] cannot commit failed sector %d", sid) + } + + if ss.state != statePreCommit { + return nil, xerrors.Errorf("cannot commit sector that has not been precommitted") + } + + time.Sleep(sb.commitDelay) + buf := make([]byte, 32) rand.Read(buf) return buf, nil @@ -136,3 +187,29 @@ func (sb *SBMock) AddWorker(context.Context, sectorbuilder.WorkerCfg) (<-chan se func (sb *SBMock) TaskDone(context.Context, uint64, sectorbuilder.SealRes) error { panic("nyi") } + +// Test Instrumentation Methods + +func (sb *SBMock) FailSector(sid uint64) error { + sb.lk.Lock() + defer sb.lk.Unlock() + ss, ok := sb.sectors[sid] + if !ok { + return fmt.Errorf("no such sector in sectorbuilder") + } + + ss.failed = true + return nil +} + +func (sb *SBMock) SetPreCommitDelay(d time.Duration) { + sb.lk.Lock() + defer sb.lk.Unlock() + sb.preCommitDelay = d +} + +func (sb *SBMock) SetCommitDelay(d time.Duration) { + sb.lk.Lock() + defer sb.lk.Unlock() + sb.commitDelay = d +} From 7534ac1e1af87804ec9b046734d2665c6fdc0ac5 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 9 Jan 2020 12:50:45 -0800 Subject: [PATCH 31/61] add ability to control seal operations completion time --- cmd/lotus-bench/main.go | 4 +- cmd/lotus-seal-worker/sub.go | 4 +- cmd/lotus-seed/seed/seed.go | 2 +- go.mod | 4 +- go.sum | 10 +++-- storage/miner.go | 4 +- storage/sbmock/sbmock.go | 73 +++++++++++++++++++++++++++-------- storage/sbmock/sbmock_test.go | 45 +++++++++++++++++++++ storage/sector_states.go | 4 +- 9 files changed, 118 insertions(+), 32 deletions(-) create mode 100644 storage/sbmock/sbmock_test.go diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 558906963..6bd212c78 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -188,7 +188,7 @@ func main() { log.Info("Running replication...") pieces := []sectorbuilder.PublicPieceInfo{pi} - pco, err := sb.SealPreCommit(i, ticket, pieces) + pco, err := sb.SealPreCommit(context.TODO(), i, ticket, pieces) if err != nil { return xerrors.Errorf("commit: %w", err) } @@ -206,7 +206,7 @@ func main() { } log.Info("Generating PoRep for sector") - proof, err := sb.SealCommit(i, ticket, seed, pieces, pco) + proof, err := sb.SealCommit(context.TODO(), i, ticket, seed, pieces, pco) if err != nil { return err } diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index d79a8ed83..1043aa5ea 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -103,7 +103,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) switch task.Type { case sectorbuilder.WorkerPreCommit: - rspco, err := w.sb.SealPreCommit(task.SectorID, task.SealTicket, task.Pieces) + rspco, err := w.sb.SealPreCommit(ctx, task.SectorID, task.SealTicket, task.Pieces) if err != nil { return errRes(xerrors.Errorf("precomitting: %w", err)) } @@ -117,7 +117,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) return errRes(xerrors.Errorf("pushing precommited data: %w", err)) } case sectorbuilder.WorkerCommit: - proof, err := w.sb.SealCommit(task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco) + proof, err := w.sb.SealCommit(ctx, task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco) if err != nil { return errRes(xerrors.Errorf("comitting: %w", err)) } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 0b62c38bb..42f864785 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -69,7 +69,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb fmt.Printf("sector-id: %d, piece info: %v", sid, pi) - pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{pi}) + pco, err := sb.SealPreCommit(context.TODO(), sid, ticket, []sectorbuilder.PublicPieceInfo{pi}) if err != nil { return nil, xerrors.Errorf("commit: %w", err) } diff --git a/go.mod b/go.mod index 0c6325b02..ee7c4d175 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-paramfetch v0.0.1 - github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab + github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect @@ -47,7 +47,7 @@ require ( github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipld-cbor v0.0.3 github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log v1.0.1 // indirect + github.com/ipfs/go-log v1.0.1 github.com/ipfs/go-log/v2 v2.0.2 github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 diff --git a/go.sum b/go.sum index 5fd3476af..b31495db1 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyC github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab h1:bsrBNO1LwnhOLxPEXlSPal/WuY61mLJUCHYyD0NayHg= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4= +github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -283,12 +285,12 @@ github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9 github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= -github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= -github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= -github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= +github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw= github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= @@ -879,9 +881,9 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200108195415-316d2f248479 h1:csuS+MHeEA2eWhyjQCMaPMq4z1+/PohkBSjJZHSIbOE= golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/storage/miner.go b/storage/miner.go index ed2b8876f..90c4d9fac 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -79,8 +79,8 @@ type SectorBuilder interface { AcquireSectorId() (uint64, error) Scrub(sectorbuilder.SortedPublicSectorInfo) []*sectorbuilder.Fault GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error) - SealPreCommit(uint64, sectorbuilder.SealTicket, []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) - SealCommit(uint64, sectorbuilder.SealTicket, sectorbuilder.SealSeed, []sectorbuilder.PublicPieceInfo, sectorbuilder.RawSealPreCommitOutput) ([]byte, error) + SealPreCommit(context.Context, uint64, sectorbuilder.SealTicket, []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) + SealCommit(context.Context, uint64, sectorbuilder.SealTicket, sectorbuilder.SealSeed, []sectorbuilder.PublicPieceInfo, sectorbuilder.RawSealPreCommitOutput) ([]byte, error) // Not so sure about these being on the interface GetPath(string, string) (string, error) diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index 45f83d76d..73905db28 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -1,13 +1,13 @@ package sbmock import ( + "bytes" "context" "fmt" "io" "io/ioutil" "math/rand" "sync" - "time" "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" @@ -25,9 +25,6 @@ type SBMock struct { nextSectorID uint64 rateLimit chan struct{} - preCommitDelay time.Duration - commitDelay time.Duration - lk sync.Mutex } @@ -102,14 +99,34 @@ func (sb *SBMock) AcquireSectorId() (uint64, error) { } func (sb *SBMock) Scrub(sectorbuilder.SortedPublicSectorInfo) []*sectorbuilder.Fault { - return nil + sb.lk.Lock() + mcopy := make(map[uint64]*sectorState) + for k, v := range sb.sectors { + mcopy[k] = v + } + sb.lk.Unlock() + + var out []*sectorbuilder.Fault + for sid, ss := range mcopy { + ss.lk.Lock() + if ss.failed { + out = append(out, §orbuilder.Fault{ + SectorID: sid, + Err: fmt.Errorf("mock sector failed"), + }) + + } + ss.lk.Unlock() + } + + return out } func (sb *SBMock) GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error) { panic("NYI") } -func (sb *SBMock) SealPreCommit(sid uint64, ticket sectorbuilder.SealTicket, pieces []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) { +func (sb *SBMock) SealPreCommit(ctx context.Context, sid uint64, ticket sectorbuilder.SealTicket, pieces []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() @@ -137,7 +154,7 @@ func (sb *SBMock) SealPreCommit(sid uint64, ticket sectorbuilder.SealTicket, pie return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("cannot call pre-seal on sector not in 'packing' state") } - time.Sleep(sb.preCommitDelay) + opFinishWait(ctx) ss.state = statePreCommit @@ -147,7 +164,7 @@ func (sb *SBMock) SealPreCommit(sid uint64, ticket sectorbuilder.SealTicket, pie }, nil } -func (sb *SBMock) SealCommit(sid uint64, ticket sectorbuilder.SealTicket, seed sectorbuilder.SealSeed, pieces []sectorbuilder.PublicPieceInfo, precommit sectorbuilder.RawSealPreCommitOutput) ([]byte, error) { +func (sb *SBMock) SealCommit(ctx context.Context, sid uint64, ticket sectorbuilder.SealTicket, seed sectorbuilder.SealSeed, pieces []sectorbuilder.PublicPieceInfo, precommit sectorbuilder.RawSealPreCommitOutput) ([]byte, error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() @@ -165,7 +182,7 @@ func (sb *SBMock) SealCommit(sid uint64, ticket sectorbuilder.SealTicket, seed s return nil, xerrors.Errorf("cannot commit sector that has not been precommitted") } - time.Sleep(sb.commitDelay) + opFinishWait(ctx) buf := make([]byte, 32) rand.Read(buf) @@ -202,14 +219,36 @@ func (sb *SBMock) FailSector(sid uint64) error { return nil } -func (sb *SBMock) SetPreCommitDelay(d time.Duration) { - sb.lk.Lock() - defer sb.lk.Unlock() - sb.preCommitDelay = d +func opFinishWait(ctx context.Context) { + val, ok := ctx.Value("opfinish").(chan struct{}) + if !ok { + return + } + <-val } -func (sb *SBMock) SetCommitDelay(d time.Duration) { - sb.lk.Lock() - defer sb.lk.Unlock() - sb.commitDelay = d +func AddOpFinish(ctx context.Context) (context.Context, func()) { + done := make(chan struct{}) + + return context.WithValue(ctx, "opfinish", done), func() { + close(done) + } +} + +func (sb *SBMock) StageFakeData() (uint64, []sectorbuilder.PublicPieceInfo, error) { + usize := sectorbuilder.UserBytesForSectorSize(sb.sectorSize) + sid, err := sb.AcquireSectorId() + if err != nil { + return 0, nil, err + } + + buf := make([]byte, usize) + rand.Read(buf) + + pi, err := sb.AddPiece(usize, sid, bytes.NewReader(buf), nil) + if err != nil { + return 0, nil, err + } + + return sid, []sectorbuilder.PublicPieceInfo{pi}, nil } diff --git a/storage/sbmock/sbmock_test.go b/storage/sbmock/sbmock_test.go new file mode 100644 index 000000000..d07a4eee4 --- /dev/null +++ b/storage/sbmock/sbmock_test.go @@ -0,0 +1,45 @@ +package sbmock + +import ( + "context" + "testing" + "time" + + "github.com/filecoin-project/go-sectorbuilder" +) + +func TestOpFinish(t *testing.T) { + sb := NewMockSectorBuilder(1, 1024) + + sid, pieces, err := sb.StageFakeData() + if err != nil { + t.Fatal(err) + } + + ctx, done := AddOpFinish(context.TODO()) + + finished := make(chan struct{}) + go func() { + _, err := sb.SealPreCommit(ctx, sid, sectorbuilder.SealTicket{}, pieces) + if err != nil { + t.Error(err) + return + } + + close(finished) + }() + + select { + case <-finished: + t.Fatal("should not finish until we tell it to") + case <-time.After(time.Second / 2): + } + + done() + + select { + case <-finished: + case <-time.After(time.Second / 2): + t.Fatal("should finish after we tell it to") + } +} diff --git a/storage/sector_states.go b/storage/sector_states.go index 0b86a3059..f3c2a4b44 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -69,7 +69,7 @@ func (m *Miner) handleUnsealed(ctx context.Context, sector SectorInfo) *sectorUp return sector.upd().fatal(err) } - rspco, err := m.sb.SealPreCommit(sector.SectorID, *ticket, sector.pieceInfos()) + rspco, err := m.sb.SealPreCommit(ctx, sector.SectorID, *ticket, sector.pieceInfos()) if err != nil { return sector.upd().to(api.SealFailed).error(xerrors.Errorf("seal pre commit failed: %w", err)) } @@ -173,7 +173,7 @@ func (m *Miner) handlePreCommitted(ctx context.Context, sector SectorInfo) *sect func (m *Miner) handleCommitting(ctx context.Context, sector SectorInfo) *sectorUpdate { log.Info("scheduling seal proof computation...") - proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.rspco()) + proof, err := m.sb.SealCommit(ctx, sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.rspco()) if err != nil { return sector.upd().to(api.SealCommitFailed).error(xerrors.Errorf("computing seal proof failed: %w", err)) } From 45e576addff425488d949dacc1df2daf4da645e0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 9 Jan 2020 12:59:37 -0800 Subject: [PATCH 32/61] mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index b31495db1..589a5d544 100644 --- a/go.sum +++ b/go.sum @@ -118,8 +118,6 @@ github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go. github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE= github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab h1:bsrBNO1LwnhOLxPEXlSPal/WuY61mLJUCHYyD0NayHg= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= From 960b6012c2904a773326d5e80b17d62c73cdf83d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 9 Jan 2020 22:53:41 +0100 Subject: [PATCH 33/61] Count coverage of integration tests Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9aa747df..b2434cfbe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,7 +100,7 @@ jobs: description: gotestsum format. https://github.com/gotestyourself/gotestsum#format coverage: type: string - default: -coverprofile=coverage.txt + default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/... description: Coverage flag. Set to the empty string to disable. codecov-upload: type: boolean From 61849ef7eaeb196728ae4fe33f2186d4c6305dbc Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 9 Jan 2020 23:12:33 +0100 Subject: [PATCH 34/61] Enable codecov Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2434cfbe..ea3ff8f42 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -253,7 +253,8 @@ workflows: jobs: - lint-changes: args: "--new-from-rev origin/master" - - test + - test: + codecov-upload: true - test-short: go-test-flags: "--timeout 10m --short" - mod-tidy-check From a82a0769faf2c1fa2d2c3b58800e2793a47192ab Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 9 Jan 2020 23:13:07 +0100 Subject: [PATCH 35/61] Disable buildall on macos (it takes forever) Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea3ff8f42..9bcd6e8d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -190,7 +190,7 @@ jobs: - install-deps - go/mod-download - run: - command: make buildall + command: make build no_output_timeout: 30m - store_artifacts: path: lotus From 6c9289df0a746afaa13e6e181c31b8cb4d70c966 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 9 Jan 2020 14:18:06 -0800 Subject: [PATCH 36/61] don't require signatures for self deals --- chain/actors/actor_storagemarket.go | 22 ++++++++++++++-------- chain/deals/provider_utils.go | 8 ++++++-- storage/garbage.go | 7 +------ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index e4ad58cb5..a3fb269bb 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -122,15 +122,21 @@ func (sdp *StorageDealProposal) Cid() (cid.Cid, error) { return nd.Cid(), nil } -func (sdp *StorageDealProposal) Verify() error { - unsigned := *sdp - unsigned.ProposerSignature = nil - var buf bytes.Buffer - if err := unsigned.MarshalCBOR(&buf); err != nil { - return err +func (sdp *StorageDealProposal) Verify(worker address.Address) error { + if sdp.Client != worker || worker == address.Undef { + unsigned := *sdp + unsigned.ProposerSignature = nil + var buf bytes.Buffer + if err := unsigned.MarshalCBOR(&buf); err != nil { + return err + } + + if err := sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()); err != nil { + return err + } } - return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()) + return nil } type OnChainDeal struct { @@ -396,7 +402,7 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe return aerrors.New(2, "Deals must be submitted by the miner worker") } - if err := deal.Verify(); err != nil { + if err := deal.Verify(providerWorker); err != nil { return aerrors.Absorb(err, 3, "verifying proposer signature") } diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index a02c6daf9..e52268dde 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -4,7 +4,7 @@ import ( "context" "runtime" - "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipld/go-ipld-prime" @@ -56,7 +56,11 @@ func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { return proposal, err } - if err := proposal.DealProposal.Verify(); err != nil { + if proposal.DealProposal.ProposerSignature == nil { + return proposal, xerrors.Errorf("incoming deal proposal has no signature") + } + + if err := proposal.DealProposal.Verify(address.Undef); err != nil { return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) } diff --git a/storage/garbage.go b/storage/garbage.go index d128413bd..a5470cdaa 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -10,7 +10,6 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) @@ -39,11 +38,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece Duration: math.MaxUint64 / 2, // /2 because overflows StoragePricePerEpoch: types.NewInt(0), StorageCollateral: types.NewInt(0), - ProposerSignature: nil, - } - - if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &sdp); err != nil { - return nil, xerrors.Errorf("signing storage deal failed: ", err) + ProposerSignature: nil, // nil because self dealing } deals[i] = sdp From d309b56ad6c86ae40af56db9f89641258d4abbf1 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 9 Jan 2020 23:27:47 +0100 Subject: [PATCH 37/61] Codecov ignore cbor gen Signed-off-by: Jakub Sztandera --- .codecov.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index db2472009..cf409a6b6 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1 +1,3 @@ comment: off +ignore: + - "cbor_gen.go" From da4528932ad31d8afe6947ef0cda3c6d30065e23 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 4 Nov 2019 11:57:54 -0800 Subject: [PATCH 38/61] feat(storagemarket): initial extraction Types for storage market Modify deals.Provider to implement storagemarket.StorageProvider Inject storagemarket.StorageProvider Storage Provider interfaces Storage Client interfaces Add ValidatePublishedDeal to ClientNodeAdapter Remove FundManager from client Remove Wallet from client Remove StateManager, Events, Wallet from client Rebasing - Copy types.BigInt, use TokenAmount/BigInt for token amounts - Remove auto-imported log package - Move `checkAskSignature` to a client file. - Plumb contexts through fix(storagemarket): use publish cids Switch back to publish message cids to reduce the dependency surface area --- chain/deals/cbor_gen.go | 165 ++--------- chain/deals/client.go | 66 ++--- chain/deals/client_states.go | 158 +--------- chain/deals/client_storagemarket.go | 118 ++++++++ chain/deals/provider.go | 29 +- chain/deals/provider_asks.go | 35 +-- chain/deals/provider_states.go | 105 ++----- chain/deals/provider_storagemarket.go | 65 +++++ chain/deals/provider_utils.go | 41 +-- chain/deals/request_validation_test.go | 15 +- chain/deals/types.go | 8 +- chain/types/cbor_gen.go | 2 +- gen/main.go | 10 + node/builder.go | 8 +- node/impl/client/client.go | 38 +-- node/modules/services.go | 4 +- node/modules/storageminer.go | 7 +- retrieval/impl/cbor_gen.go | 14 +- retrievaladapter/provider.go | 2 +- storagemarket/bigint.go | 241 ++++++++++++++++ storagemarket/cbor_gen.go | 352 +++++++++++++++++++++++ storagemarket/types.go | 199 +++++++++++++ storagemarketadapter/client_adapter.go | 328 +++++++++++++++++++++ storagemarketadapter/provider_adapter.go | 196 +++++++++++++ 24 files changed, 1684 insertions(+), 522 deletions(-) create mode 100644 chain/deals/client_storagemarket.go create mode 100644 chain/deals/provider_storagemarket.go create mode 100644 storagemarket/bigint.go create mode 100644 storagemarket/cbor_gen.go create mode 100644 storagemarket/types.go create mode 100644 storagemarketadapter/client_adapter.go create mode 100644 storagemarketadapter/provider_adapter.go diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index bd56e0551..5abb92825 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -219,10 +219,18 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) } - // t.StorageDealSubmission (types.SignedMessage) (struct) - if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil { - return err + // t.PublishMessage (cid.Cid) (struct) + + if t.PublishMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) + } } + return nil } @@ -273,7 +281,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.Proposal = c } - // t.StorageDealSubmission (types.SignedMessage) (struct) + // t.PublishMessage (cid.Cid) (struct) { @@ -287,10 +295,13 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return err } } else { - t.StorageDealSubmission = new(types.SignedMessage) - if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil { - return err + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) } + + t.PublishMessage = &c } } @@ -526,56 +537,12 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{136}); err != nil { + if _, err := w.Write([]byte{129}); err != nil { return err } - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Miner (peer.ID) (string) - if len(t.Miner) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Miner was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Miner)); err != nil { - return err - } - - // t.MinerWorker (address.Address) (struct) - if err := t.MinerWorker.MarshalCBOR(w); err != nil { - return err - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.PayloadCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.PayloadCid); err != nil { - return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err) - } - - // t.PublishMessage (types.SignedMessage) (struct) - if err := t.PublishMessage.MarshalCBOR(w); err != nil { + // t.ClientDeal (storagemarket.ClientDeal) (struct) + if err := t.ClientDeal.MarshalCBOR(w); err != nil { return err } return nil @@ -592,102 +559,18 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 8 { + if extra != 1 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.ProposalCid (cid.Cid) (struct) + // t.ClientDeal (storagemarket.ClientDeal) (struct) { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { + if err := t.ClientDeal.UnmarshalCBOR(br); err != nil { return err } - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Miner (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Miner = peer.ID(sval) - } - // t.MinerWorker (address.Address) (struct) - - { - - if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.PayloadCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err) - } - - t.PayloadCid = c - - } - // t.PublishMessage (types.SignedMessage) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.PublishMessage = new(types.SignedMessage) - if err := t.PublishMessage.UnmarshalCBOR(br); err != nil { - return err - } - } - } return nil } diff --git a/chain/deals/client.go b/chain/deals/client.go index 4e1d3df4a..879e408e2 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -15,39 +15,24 @@ import ( "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/events" - "github.com/filecoin-project/lotus/chain/market" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules/dtypes" retrievalmarket "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" + "github.com/filecoin-project/lotus/storagemarket" ) var log = logging.Logger("deals") type ClientDeal struct { - ProposalCid cid.Cid - Proposal actors.StorageDealProposal - State api.DealState - Miner peer.ID - MinerWorker address.Address - DealID uint64 - PayloadCid cid.Cid - - PublishMessage *types.SignedMessage + storagemarket.ClientDeal s inet.Stream } type Client struct { - sm *stmgr.StateManager - chain *store.ChainStore - h host.Host - w *wallet.Wallet + h host.Host + // dataTransfer // TODO: once the data transfer module is complete, the // client will listen to events on the data transfer module @@ -56,8 +41,8 @@ type Client struct { dataTransfer dtypes.ClientDataTransfer dag dtypes.ClientDAG discovery *discovery.Local - events *events.Events - fm *market.FundMgr + + node storagemarket.StorageClientNode deals *statestore.StateStore conns map[cid.Cid]inet.Stream @@ -76,22 +61,13 @@ type clientDealUpdate struct { mut func(*ClientDeal) } -type clientApi struct { - full.ChainAPI - full.StateAPI -} - -func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, fm *market.FundMgr, deals dtypes.ClientDealStore, chainapi full.ChainAPI, stateapi full.StateAPI) *Client { +func NewClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) *Client { c := &Client{ - sm: sm, - chain: chain, h: h, - w: w, dataTransfer: dataTransfer, dag: dag, discovery: discovery, - fm: fm, - events: events.NewEvents(context.TODO(), &clientApi{chainapi, stateapi}), + node: scn, deals: deals, conns: map[cid.Cid]inet.Stream{}, @@ -196,7 +172,8 @@ type ClientDealProposal struct { } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil { + amount := types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration)) + if err := c.node.EnsureFunds(ctx, p.Client, storagemarket.TokenAmount(amount)); err != nil { return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) } @@ -216,7 +193,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc } - if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil { + if err := c.node.SignProposal(ctx, p.Client, dealProposal); err != nil { return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) } @@ -225,7 +202,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) } - s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID) + s, err := c.h.NewStream(ctx, p.MinerID, storagemarket.DealProtocolID) if err != nil { return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) } @@ -241,13 +218,16 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro } deal := &ClientDeal{ - ProposalCid: proposalNd.Cid(), - Proposal: *dealProposal, - State: api.DealUnknown, - Miner: p.MinerID, - MinerWorker: p.MinerWorker, - PayloadCid: p.Data, - s: s, + ClientDeal: storagemarket.ClientDeal{ + ProposalCid: proposalNd.Cid(), + Proposal: *dealProposal, + State: api.DealUnknown, + Miner: p.MinerID, + MinerWorker: p.MinerWorker, + PayloadCid: p.Data, + }, + + s: s, } c.incoming <- deal @@ -259,7 +239,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro } func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) { - s, err := c.h.NewStream(ctx, p, AskProtocolID) + s, err := c.h.NewStream(ctx, p, storagemarket.AskProtocolID) if err != nil { return nil, xerrors.Errorf("failed to open stream to miner: %w", err) } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index d19765f60..e82beb126 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -1,16 +1,11 @@ package deals import ( - "bytes" "context" "golang.org/x/xerrors" - "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" ) @@ -57,70 +52,20 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), e } return func(info *ClientDeal) { - info.PublishMessage = resp.StorageDealSubmission + info.PublishMessage = resp.PublishMessage }, nil } func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { log.Infow("DEAL ACCEPTED!") - pubmsg := deal.PublishMessage.Message - pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) + dealId, err := c.node.ValidatePublishedDeal(ctx, deal.ClientDeal) if err != nil { - return nil, xerrors.Errorf("getting miner worker failed: %w", err) - } - - if pubmsg.From != pw { - return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) - } - - if pubmsg.To != actors.StorageMarketAddress { - return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) - } - - if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { - return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) - } - - var params actors.PublishStorageDealsParams - if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { - return nil, err - } - - dealIdx := -1 - for i, storageDeal := range params.Deals { - // TODO: make it less hacky - sd := storageDeal - eq, err := cborutil.Equals(&deal.Proposal, &sd) - if err != nil { - return nil, err - } - if eq { - dealIdx = i - break - } - } - - if dealIdx == -1 { - return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid()) - } - - // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid()) - if err != nil { - return nil, xerrors.Errorf("waiting for deal publish message: %w", err) - } - if ret.ExitCode != 0 { - return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) - } - - var res actors.PublishStorageDealResponse - if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { return nil, err } return func(info *ClientDeal) { - info.DealID = res.DealIDs[dealIdx] + info.DealID = dealId }, nil } @@ -131,103 +76,22 @@ func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal) } func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { - sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) - if err != nil { - // TODO: This may be fine for some errors - return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) - } - - if sd.ActivationEpoch > 0 { - select { - case c.updated <- clientDealUpdate{ - newState: api.DealComplete, - id: deal.ProposalCid, - }: - case <-c.stop: - } - - return true, false, nil - } - - return false, true, nil - } - - called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) { - defer func() { - if err != nil { - select { - case c.updated <- clientDealUpdate{ - newState: api.DealComplete, - id: deal.ProposalCid, - err: xerrors.Errorf("handling applied event: %w", err), - }: - case <-c.stop: - } - } - }() - - if msg == nil { - log.Error("timed out waiting for deal activation... what now?") - return false, nil - } - - sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) - if err != nil { - return false, xerrors.Errorf("failed to look up deal on chain: %w", err) - } - - if sd.ActivationEpoch == 0 { - return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height()) - } - - log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch) - + cb := func(err error) { select { case c.updated <- clientDealUpdate{ newState: api.DealComplete, id: deal.ProposalCid, + err: err, }: case <-c.stop: } - - return false, nil } - revert := func(ctx context.Context, ts *types.TipSet) error { - log.Warn("deal activation reverted; TODO: actually handle this!") - // TODO: Just go back to DealSealing? - return nil - } + err := c.node.OnDealSectorCommitted(ctx, deal.Proposal.Provider, deal.DealID, cb) - matchEvent := func(msg *types.Message) (bool, error) { - if msg.To != deal.Proposal.Provider { - return false, nil - } - - if msg.Method != actors.MAMethods.ProveCommitSector { - return false, nil - } - - var params actors.SectorProveCommitInfo - if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { - return false, err - } - - var found bool - for _, dealID := range params.DealIDs { - if dealID == deal.DealID { - found = true - break - } - } - - return found, nil - } - - if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil { - return nil, xerrors.Errorf("failed to set up called handler") - } - - return nil, nil + return nil, err +} + +func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { + return c.node.ValidateAskSignature(ask) } diff --git a/chain/deals/client_storagemarket.go b/chain/deals/client_storagemarket.go new file mode 100644 index 000000000..08dacc7a3 --- /dev/null +++ b/chain/deals/client_storagemarket.go @@ -0,0 +1,118 @@ +package deals + +// this file implements storagemarket.StorageClient + +import ( + "context" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/storagemarket" +) + +func (c *Client) ListProviders(ctx context.Context) (<-chan storagemarket.StorageProviderInfo, error) { + providers, err := c.node.ListStorageProviders(ctx) + if err != nil { + return nil, err + } + + out := make(chan storagemarket.StorageProviderInfo) + + go func() { + for _, p := range providers { + select { + case out <- *p: + case <-ctx.Done(): + return + } + + } + }() + + return out, nil +} + +func (c *Client) ListDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) { + return c.node.ListClientDeals(ctx, addr) +} + +func (c *Client) ListInProgressDeals(ctx context.Context) ([]storagemarket.ClientDeal, error) { + deals, err := c.List() + if err != nil { + return nil, err + } + + out := make([]storagemarket.ClientDeal, len(deals)) + for k, v := range deals { + out[k] = storagemarket.ClientDeal{ + ProposalCid: v.ProposalCid, + Proposal: v.Proposal, + State: v.State, + Miner: v.Miner, + MinerWorker: v.MinerWorker, + DealID: v.DealID, + PublishMessage: v.PublishMessage, + } + } + + return out, nil +} + +func (c *Client) GetInProgressDeal(ctx context.Context, cid cid.Cid) (storagemarket.ClientDeal, error) { + deals, err := c.ListInProgressDeals(ctx) + if err != nil { + return storagemarket.ClientDeal{}, err + } + + for _, deal := range deals { + if deal.ProposalCid == cid { + return deal, nil + } + } + + return storagemarket.ClientDeal{}, xerrors.Errorf("couldn't find client deal") +} + +func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderInfo) (*storagemarket.StorageAsk, error) { + return c.QueryAsk(ctx, info.PeerID, info.Address) +} + +func (c *Client) ProposeStorageDeal(ctx context.Context, addr address.Address, info *storagemarket.StorageProviderInfo, payloadCid cid.Cid, proposalExpiration storagemarket.Epoch, duration storagemarket.Epoch, price storagemarket.TokenAmount, collateral storagemarket.TokenAmount) (*storagemarket.ProposeStorageDealResult, error) { + + proposal := ClientDealProposal{ + Data: payloadCid, + PricePerEpoch: types.BigInt(price), + ProposalExpiration: uint64(proposalExpiration), + Duration: uint64(duration), + Client: addr, + ProviderAddress: info.Address, + MinerWorker: info.Worker, + MinerID: info.PeerID, + } + + proposalCid, err := c.Start(ctx, proposal) + + result := &storagemarket.ProposeStorageDealResult{ + ProposalCid: proposalCid, + } + + return result, err +} + +func (c *Client) GetPaymentEscrow(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { + + balance, err := c.node.GetBalance(ctx, addr) + + return balance, err +} + +func (c *Client) AddPaymentEscrow(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { + + return c.node.AddFunds(ctx, addr, amount) +} + +var _ storagemarket.StorageClient = &Client{} diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 0b1fa3900..1a020de71 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -5,9 +5,10 @@ import ( "errors" "sync" - cid "github.com/ipfs/go-cid" - datastore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" @@ -20,8 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storagemarket" ) var ProviderDsPrefix = "/deals/provider" @@ -47,9 +47,7 @@ type Provider struct { ask *types.SignedStorageAsk askLk sync.Mutex - secb *sectorblocks.SectorBlocks - sminer *storage.Miner - full api.FullNode + spn storagemarket.StorageProviderNode // TODO: This will go away once storage market module + CAR // is implemented @@ -83,7 +81,7 @@ var ( ErrDataTransferFailed = errors.New("deal data transfer failed") ) -func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, fullNode api.FullNode) (*Provider, error) { +func NewProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err @@ -94,11 +92,9 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks } h := &Provider{ - sminer: sminer, dag: dag, dataTransfer: dataTransfer, - full: fullNode, - secb: secb, + spn: spn, pricePerByteBlock: types.NewInt(3), // TODO: allow setting minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) @@ -135,9 +131,12 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks return h, nil } -func (p *Provider) Run(ctx context.Context) { +func (p *Provider) Run(ctx context.Context, host host.Host) { // TODO: restore state + host.SetStreamHandler(storagemarket.DealProtocolID, p.HandleStream) + host.SetStreamHandler(storagemarket.AskProtocolID, p.HandleAskStream) + go func() { defer log.Warn("quitting deal provider loop") defer close(p.stopped) @@ -162,7 +161,7 @@ func (p *Provider) onIncoming(deal MinerDeal) { if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil { // This can happen when client re-sends proposal - p.failDeal(deal.ProposalCid, err) + p.failDeal(context.TODO(), deal.ProposalCid, err) log.Errorf("deal tracking failed: %s", err) return } @@ -180,7 +179,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState]) if update.err != nil { log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err) - p.failDeal(update.id, update.err) + p.failDeal(ctx, update.id, update.err) return } var deal MinerDeal @@ -193,7 +192,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { return nil }) if err != nil { - p.failDeal(update.id, err) + p.failDeal(ctx, update.id, err) return } diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go index 1170eb28b..a0cdd9c71 100644 --- a/chain/deals/provider_asks.go +++ b/chain/deals/provider_asks.go @@ -5,13 +5,13 @@ import ( "context" "time" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/types" - datastore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore" inet "github.com/libp2p/go-libp2p-core/network" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/lotus/chain/types" ) func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error { @@ -41,7 +41,7 @@ func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error { return p.saveAsk(ssa) } -func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk { +func (p *Provider) GetAsk(m address.Address) *types.SignedStorageAsk { p.askLk.Lock() defer p.askLk.Unlock() if m != p.actor { @@ -69,7 +69,7 @@ func (p *Provider) HandleAskStream(s inet.Stream) { func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse { return &AskResponse{ - Ask: p.getAsk(ar.Miner), + Ask: p.GetAsk(ar.Miner), } } @@ -112,12 +112,12 @@ func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) return nil, err } - worker, err := p.getWorker(p.actor) + worker, err := p.spn.GetMinerWorker(context.TODO(), p.actor) if err != nil { return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err) } - sig, err := p.full.WalletSign(context.TODO(), worker, b) + sig, err := p.spn.SignBytes(context.TODO(), worker, b) if err != nil { return nil, err } @@ -141,20 +141,3 @@ func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { p.ask = a return nil } - -func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { - tss := c.sm.ChainStore().GetHeaviestTipSet().ParentState() - - w, err := stmgr.GetMinerWorkerRaw(context.TODO(), c.sm, tss, ask.Ask.Miner) - if err != nil { - return xerrors.Errorf("failed to get worker for miner in ask", err) - } - - sigb, err := cborutil.Dump(ask.Ask) - if err != nil { - return xerrors.Errorf("failed to re-serialize ask") - } - - return ask.Signature.Verify(w, sigb) - -} diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 4faa44973..76fa9a99b 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -1,21 +1,17 @@ package deals import ( - "bytes" "context" ipldfree "github.com/ipld/go-ipld-prime/impl/free" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" - unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storagemarket" ) type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) @@ -43,7 +39,7 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle // ACCEPTED func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - head, err := p.full.ChainHead(ctx) + head, err := p.spn.MostRecentStateId(ctx) if err != nil { return nil, err } @@ -63,7 +59,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) } // check market funds - clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil) + clientMarketBalance, err := p.spn.GetBalance(ctx, deal.Proposal.Client) if err != nil { return nil, xerrors.Errorf("getting client market balance failed: %w", err) } @@ -74,59 +70,36 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.New("clientMarketBalance.Available too small") } - waddr, err := p.full.StateMinerWorker(ctx, deal.Proposal.Provider, nil) + waddr, err := p.spn.GetMinerWorker(ctx, deal.Proposal.Provider) if err != nil { return nil, err } // TODO: check StorageCollateral (may be too large (or too small)) - if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil { + if err := p.spn.EnsureFunds(ctx, waddr, storagemarket.TokenAmount(deal.Proposal.StorageCollateral)); err != nil { return nil, err } - log.Info("publishing deal") - - params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ - Deals: []actors.StorageDealProposal{deal.Proposal}, - }) - if err != nil { - return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err) + smDeal := storagemarket.MinerDeal{ + Client: deal.Client, + Proposal: deal.Proposal, + ProposalCid: deal.ProposalCid, + State: deal.State, + Ref: deal.Ref, + SectorID: deal.SectorID, } - // TODO: We may want this to happen after fetching data - smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: waddr, - Value: types.NewInt(0), - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.PublishStorageDeals, - Params: params, - }) + dealId, mcid, err := p.spn.PublishDeals(ctx, smDeal) if err != nil { return nil, err } - r, err := p.full.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - return nil, err - } - if r.Receipt.ExitCode != 0 { - return nil, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode) - } - var resp actors.PublishStorageDealResponse - if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { - return nil, err - } - if len(resp.DealIDs) != 1 { - return nil, xerrors.Errorf("got unexpected number of DealIDs from SMA") - } - log.Infof("fetching data for a deal %d", resp.DealIDs[0]) - err = p.sendSignedResponse(&Response{ + log.Infof("fetching data for a deal %d", dealId) + err = p.sendSignedResponse(ctx, &Response{ State: api.DealAccepted, - Proposal: deal.ProposalCid, - StorageDealSubmission: smsg, + Proposal: deal.ProposalCid, + PublishMessage: &mcid, }) if err != nil { return nil, err @@ -148,7 +121,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // (see onDataTransferEvent) _, err = p.dataTransfer.OpenPullDataChannel(ctx, deal.Client, - &StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: resp.DealIDs[0]}, + &StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: uint64(dealId)}, deal.Ref, allSelector, ) @@ -162,39 +135,23 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // STAGED func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - root, err := p.dag.Get(ctx, deal.Ref) - if err != nil { - return nil, xerrors.Errorf("failed to get file root for deal: %s", err) - } + sectorID, err := p.spn.OnDealComplete( + ctx, + storagemarket.MinerDeal{ + Client: deal.Client, + Proposal: deal.Proposal, + ProposalCid: deal.ProposalCid, + State: deal.State, + Ref: deal.Ref, + DealID: deal.DealID, + }, + "", + ) - // TODO: abstract this away into ReadSizeCloser + implement different modes - n, err := unixfile.NewUnixfsFile(ctx, p.dag, root) if err != nil { - return nil, xerrors.Errorf("cannot open unixfs file: %s", err) + return nil, err } - uf, ok := n.(sectorblocks.UnixfsReader) - if !ok { - // we probably got directory, unsupported for now - return nil, xerrors.Errorf("unsupported unixfs file type") - } - - // TODO: uf.Size() is user input, not trusted - // This won't be useful / here after we migrate to putting CARs into sectors - size, err := uf.Size() - if err != nil { - return nil, xerrors.Errorf("getting unixfs file size: %w", err) - } - if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize { - return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size") - } - - sectorID, err := p.secb.AddUnixfsPiece(ctx, uf, deal.DealID) - if err != nil { - return nil, xerrors.Errorf("AddPiece failed: %s", err) - } - log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID) - return func(deal *MinerDeal) { deal.SectorID = sectorID }, nil diff --git a/chain/deals/provider_storagemarket.go b/chain/deals/provider_storagemarket.go new file mode 100644 index 000000000..4d9863bd9 --- /dev/null +++ b/chain/deals/provider_storagemarket.go @@ -0,0 +1,65 @@ +package deals + +// this file implements storagemarket.StorageClient + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/storagemarket" +) + +func (p *Provider) AddAsk(price storagemarket.TokenAmount, ttlsecs int64) error { + return p.SetPrice(types.BigInt(price), ttlsecs) +} + +func (p *Provider) ListAsks(addr address.Address) []*types.SignedStorageAsk { + ask := p.GetAsk(addr) + + if ask != nil { + return []*types.SignedStorageAsk{ask} + } + + return nil +} + +func (p *Provider) ListDeals(ctx context.Context) ([]actors.OnChainDeal, error) { + return p.spn.ListProviderDeals(ctx, p.actor) +} + +func (p *Provider) AddStorageCollateral(ctx context.Context, amount storagemarket.TokenAmount) error { + return p.spn.AddFunds(ctx, p.actor, amount) +} + +func (p *Provider) GetStorageCollateral(ctx context.Context) (storagemarket.Balance, error) { + balance, err := p.spn.GetBalance(ctx, p.actor) + + return balance, err +} + +func (p *Provider) ListIncompleteDeals() ([]storagemarket.MinerDeal, error) { + var out []storagemarket.MinerDeal + + var deals []MinerDeal + if err := p.deals.List(&deals); err != nil { + return nil, err + } + + for _, deal := range deals { + out = append(out, storagemarket.MinerDeal{ + Client: deal.Client, + Proposal: deal.Proposal, + ProposalCid: deal.ProposalCid, + State: deal.State, + Ref: deal.Ref, + DealID: deal.DealID, + SectorID: deal.SectorID, + }) + } + + return out, nil +} + +var _ storagemarket.StorageProvider = &Provider{} diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index e52268dde..54b838a43 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -4,24 +4,21 @@ import ( "context" "runtime" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/ipld/go-ipld-prime" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func (p *Provider) failDeal(id cid.Cid, cerr error) { +func (p *Provider) failDeal(ctx context.Context, id cid.Cid, cerr error) { if err := p.deals.End(id); err != nil { log.Warnf("deals.End: %s", err) } @@ -33,7 +30,7 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { log.Warnf("deal %s failed: %s", id, cerr) - err := p.sendSignedResponse(&Response{ + err := p.sendSignedResponse(ctx, &Response{ State: api.DealFailed, Message: cerr.Error(), Proposal: id, @@ -72,7 +69,7 @@ func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { return } -func (p *Provider) sendSignedResponse(resp *Response) error { +func (p *Provider) sendSignedResponse(ctx context.Context, resp *Response) error { s, ok := p.conns[resp.Proposal] if !ok { return xerrors.New("couldn't send response: not connected") @@ -83,12 +80,12 @@ func (p *Provider) sendSignedResponse(resp *Response) error { return xerrors.Errorf("serializing response: %w", err) } - worker, err := p.getWorker(p.actor) + worker, err := p.spn.GetMinerWorker(ctx, p.actor) if err != nil { return err } - sig, err := p.full.WalletSign(context.TODO(), worker, msg) + sig, err := p.spn.SignBytes(ctx, worker, msg) if err != nil { return xerrors.Errorf("failed to sign response message: %w", err) } @@ -118,24 +115,6 @@ func (p *Provider) disconnect(deal MinerDeal) error { return err } -func (p *Provider) getWorker(miner address.Address) (address.Address, error) { - getworker := &types.Message{ - To: miner, - From: miner, - Method: actors.MAMethods.GetWorkerAddr, - } - r, err := p.full.StateCall(context.TODO(), getworker, nil) - if err != nil { - return address.Undef, xerrors.Errorf("getting worker address: %w", err) - } - - if r.ExitCode != 0 { - return address.Undef, xerrors.Errorf("getWorker call failed: %d", r.ExitCode) - } - - return address.NewFromBytes(r.Return) -} - var _ datatransfer.RequestValidator = &ProviderRequestValidator{} // ProviderRequestValidator validates data transfer requests for the provider diff --git a/chain/deals/request_validation_test.go b/chain/deals/request_validation_test.go index f6e6a0e33..7607eacb5 100644 --- a/chain/deals/request_validation_test.go +++ b/chain/deals/request_validation_test.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/storagemarket" ) var blockGenerator = blocksutil.NewBlockGenerator() @@ -74,12 +75,14 @@ func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, erro } return deals.ClientDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - PayloadCid: blockGenerator.Next().Cid(), - Miner: minerID, - MinerWorker: minerAddr, - State: state, + ClientDeal: storagemarket.ClientDeal{ + Proposal: newProposal, + ProposalCid: proposalNd.Cid(), + PayloadCid: blockGenerator.Next().Cid(), + Miner: minerID, + MinerWorker: minerAddr, + State: state, + }, }, nil } diff --git a/chain/deals/types.go b/chain/deals/types.go index 4ad60a4bc..a35f537ab 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -4,12 +4,13 @@ import ( "bytes" "errors" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" ) var ( @@ -41,9 +42,6 @@ var ( DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown} ) -const DealProtocolID = "/fil/storage/mk/1.0.1" -const AskProtocolID = "/fil/storage/ask/1.0.1" - type Proposal struct { DealProposal *actors.StorageDealProposal @@ -58,7 +56,7 @@ type Response struct { Proposal cid.Cid // DealAccepted - StorageDealSubmission *types.SignedMessage + PublishMessage *cid.Cid } // TODO: Do we actually need this to be signed? diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 407f89809..73352df61 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/gen/main.go b/gen/main.go index 4f90dc236..0cb569148 100644 --- a/gen/main.go +++ b/gen/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/filecoin-project/lotus/storagemarket" "os" gen "github.com/whyrusleeping/cbor-gen" @@ -122,6 +123,15 @@ func main() { os.Exit(1) } + err = gen.WriteTupleEncodersToFile("./storagemarket/cbor_gen.go", "storagemarket", + storagemarket.ClientDeal{}, + storagemarket.MinerDeal{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals", deals.AskRequest{}, deals.AskResponse{}, diff --git a/node/builder.go b/node/builder.go index 8169d3829..dcce895b7 100644 --- a/node/builder.go +++ b/node/builder.go @@ -47,6 +47,8 @@ import ( "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storagemarket" + "github.com/filecoin-project/lotus/storagemarketadapter" ) // special is a type used to give keys to modules which @@ -227,7 +229,8 @@ func Online() Option { Override(new(dtypes.ClientDealStore), modules.NewClientDealStore), Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator), - Override(new(*deals.Client), deals.NewClient), + Override(new(storagemarket.StorageClient), deals.NewClient), + Override(new(storagemarket.StorageClientNode), storagemarketadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), Override(RunDealClientKey, modules.RunDealClient), @@ -250,7 +253,8 @@ func Online() Option { Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator), - Override(new(*deals.Provider), deals.NewProvider), + Override(new(storagemarket.StorageProvider), deals.NewProvider), + Override(new(storagemarket.StorageProviderNode), storagemarketadapter.NewProviderNodeAdapter), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(GetParamsKey, modules.GetParams), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 57df600bd..58f250f67 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -27,13 +27,13 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" retrievalmarket "github.com/filecoin-project/lotus/retrieval" + "github.com/filecoin-project/lotus/storagemarket" ) type API struct { @@ -44,7 +44,7 @@ type API struct { full.WalletAPI paych.PaychAPI - DealClient *deals.Client + SMDealClient storagemarket.StorageClient RetDiscovery retrievalmarket.PeerResolver Retrieval retrievalmarket.RetrievalClient Chain *store.ChainStore @@ -72,28 +72,30 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad if err != nil { return nil, xerrors.Errorf("failed getting miner worker: %w", err) } - - proposal := deals.ClientDealProposal{ - Data: data, - PricePerEpoch: epochPrice, - ProposalExpiration: math.MaxUint64, // TODO: set something reasonable - Duration: blocksDuration, - Client: addr, - ProviderAddress: miner, - MinerWorker: mw, - MinerID: pid, + providerInfo := storagemarket.StorageProviderInfo{ + Address: miner, + Worker: mw, + PeerID: pid, } + result, err := a.SMDealClient.ProposeStorageDeal( + ctx, + addr, + &providerInfo, + data, + storagemarket.Epoch(math.MaxUint64), + storagemarket.Epoch(blocksDuration), + storagemarket.TokenAmount(epochPrice), + storagemarket.TokenAmount(storagemarket.EmptyInt)) - c, err := a.DealClient.Start(ctx, proposal) if err != nil { return nil, xerrors.Errorf("failed to start deal: %w", err) } - return &c, nil + return &result.ProposalCid, nil } func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { - deals, err := a.DealClient.List() + deals, err := a.SMDealClient.ListInProgressDeals(ctx) if err != nil { return nil, err } @@ -117,10 +119,11 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { } func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) { - v, err := a.DealClient.GetDeal(d) + v, err := a.SMDealClient.GetInProgressDeal(ctx, d) if err != nil { return nil, err } + return &api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, @@ -315,5 +318,6 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path } func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { - return a.DealClient.QueryAsk(ctx, p, miner) + info := storagemarket.StorageProviderInfo{Address: miner, PeerID: p} + return a.SMDealClient.GetAsk(ctx, info) } diff --git a/node/modules/services.go b/node/modules/services.go index 9949d0d69..bd0978085 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/sub" "github.com/filecoin-project/lotus/node/hello" @@ -18,6 +17,7 @@ import ( "github.com/filecoin-project/lotus/peermgr" retrievalmarket "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" + "github.com/filecoin-project/lotus/storagemarket" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { @@ -66,7 +66,7 @@ func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, pubsub *pu go sub.HandleIncomingMessages(ctx, mpool, msgsub) } -func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c *deals.Client) { +func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c storagemarket.StorageClient) { ctx := helpers.LifecycleCtx(mctx, lc) lc.Append(fx.Hook{ diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 2a897a9a3..d868414d9 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -40,6 +40,7 @@ import ( "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storagemarket" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -127,14 +128,12 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.Retrieva }) } -func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Provider) { +func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider) { ctx := helpers.LifecycleCtx(mctx, lc) lc.Append(fx.Hook{ OnStart: func(context.Context) error { - h.Run(ctx) - host.SetStreamHandler(deals.DealProtocolID, h.HandleStream) - host.SetStreamHandler(deals.AskProtocolID, h.HandleAskStream) + h.Run(ctx, host) return nil }, OnStop: func(context.Context) error { diff --git a/retrieval/impl/cbor_gen.go b/retrieval/impl/cbor_gen.go index f2778544c..220577089 100644 --- a/retrieval/impl/cbor_gen.go +++ b/retrieval/impl/cbor_gen.go @@ -21,7 +21,7 @@ func (t *RetParams) MarshalCBOR(w io.Writer) error { return err } - // t.Unixfs0 (retrieval.Unixfs0Offer) (struct) + // t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct) if err := t.Unixfs0.MarshalCBOR(w); err != nil { return err } @@ -43,7 +43,7 @@ func (t *RetParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Unixfs0 (retrieval.Unixfs0Offer) (struct) + // t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct) { @@ -124,7 +124,7 @@ func (t *OldQueryResponse) MarshalCBOR(w io.Writer) error { return err } - // t.t.Status (retrieval.OldQueryResponseStatus) (uint64) + // t.Status (retrievalimpl.OldQueryResponseStatus) (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { return err } @@ -156,7 +156,7 @@ func (t *OldQueryResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Status (retrieval.OldQueryResponseStatus) (uint64) + // t.Status (retrievalimpl.OldQueryResponseStatus) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -166,7 +166,7 @@ func (t *OldQueryResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.Status = OldQueryResponseStatus(extra) - // t.t.Size (uint64) (uint64) + // t.Size (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -267,7 +267,7 @@ func (t *OldDealProposal) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Ref: %w", err) } - // t.Params (retrieval.RetParams) (struct) + // t.Params (retrievalimpl.RetParams) (struct) if err := t.Params.MarshalCBOR(w); err != nil { return err } @@ -310,7 +310,7 @@ func (t *OldDealProposal) UnmarshalCBOR(r io.Reader) error { t.Ref = c } - // t.Params (retrieval.RetParams) (struct) + // t.Params (retrievalimpl.RetParams) (struct) { diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go index f8d188a95..06757593b 100644 --- a/retrievaladapter/provider.go +++ b/retrievaladapter/provider.go @@ -3,8 +3,8 @@ package retrievaladapter import ( "context" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" retrievalmarket "github.com/filecoin-project/lotus/retrieval" ) diff --git a/storagemarket/bigint.go b/storagemarket/bigint.go new file mode 100644 index 000000000..f36e242b0 --- /dev/null +++ b/storagemarket/bigint.go @@ -0,0 +1,241 @@ +// Copied from lotus until this can be extracted into shared types + +package storagemarket + +import ( + "encoding/json" + "fmt" + "io" + "math/big" + + "github.com/filecoin-project/lotus/build" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/polydawn/refmt/obj/atlas" + + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" +) + +const BigIntMaxSerializedLen = 128 // is this big enough? or too big? + +var TotalFilecoinInt = FromFil(build.TotalFilecoin) + +func init() { + cbor.RegisterCborType(atlas.BuildEntry(BigInt{}).Transform(). + TransformMarshal(atlas.MakeMarshalTransformFunc( + func(i BigInt) ([]byte, error) { + return i.cborBytes(), nil + })). + TransformUnmarshal(atlas.MakeUnmarshalTransformFunc( + func(x []byte) (BigInt, error) { + return fromCborBytes(x) + })). + Complete()) +} + +var EmptyInt = BigInt{} + +type BigInt struct { + *big.Int +} + +func NewInt(i uint64) BigInt { + return BigInt{big.NewInt(0).SetUint64(i)} +} + +func FromFil(i uint64) BigInt { + return BigMul(NewInt(i), NewInt(build.FilecoinPrecision)) +} + +func BigFromBytes(b []byte) BigInt { + i := big.NewInt(0).SetBytes(b) + return BigInt{i} +} + +func BigFromString(s string) (BigInt, error) { + v, ok := big.NewInt(0).SetString(s, 10) + if !ok { + return BigInt{}, fmt.Errorf("failed to parse string as a big int") + } + + return BigInt{v}, nil +} + +func BigMul(a, b BigInt) BigInt { + return BigInt{big.NewInt(0).Mul(a.Int, b.Int)} +} + +func BigDiv(a, b BigInt) BigInt { + return BigInt{big.NewInt(0).Div(a.Int, b.Int)} +} + +func BigMod(a, b BigInt) BigInt { + return BigInt{big.NewInt(0).Mod(a.Int, b.Int)} +} + +func BigAdd(a, b BigInt) BigInt { + return BigInt{big.NewInt(0).Add(a.Int, b.Int)} +} + +func BigSub(a, b BigInt) BigInt { + return BigInt{big.NewInt(0).Sub(a.Int, b.Int)} +} + +func BigCmp(a, b BigInt) int { + return a.Int.Cmp(b.Int) +} + +func (bi BigInt) Nil() bool { + return bi.Int == nil +} + +// LessThan returns true if bi < o +func (bi BigInt) LessThan(o BigInt) bool { + return BigCmp(bi, o) < 0 +} + +// GreaterThan returns true if bi > o +func (bi BigInt) GreaterThan(o BigInt) bool { + return BigCmp(bi, o) > 0 +} + +// Equals returns true if bi == o +func (bi BigInt) Equals(o BigInt) bool { + return BigCmp(bi, o) == 0 +} + +func (bi *BigInt) MarshalJSON() ([]byte, error) { + return json.Marshal(bi.String()) +} + +func (bi *BigInt) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + i, ok := big.NewInt(0).SetString(s, 10) + if !ok { + if string(s) == "" { + return nil + } + return xerrors.Errorf("failed to parse bigint string: '%s'", string(b)) + } + + bi.Int = i + return nil +} + +func (bi *BigInt) Scan(value interface{}) error { + switch value := value.(type) { + case string: + i, ok := big.NewInt(0).SetString(value, 10) + if !ok { + if value == "" { + return nil + } + return xerrors.Errorf("failed to parse bigint string: '%s'", value) + } + + bi.Int = i + + return nil + case int64: + bi.Int = big.NewInt(value) + return nil + default: + return xerrors.Errorf("non-string types unsupported: %T", value) + } +} + +func (bi *BigInt) cborBytes() []byte { + if bi.Int == nil { + return []byte{} + } + + switch { + case bi.Sign() > 0: + return append([]byte{0}, bi.Bytes()...) + case bi.Sign() < 0: + return append([]byte{1}, bi.Bytes()...) + default: // bi.Sign() == 0: + return []byte{} + } +} + +func fromCborBytes(buf []byte) (BigInt, error) { + if len(buf) == 0 { + return NewInt(0), nil + } + + var negative bool + switch buf[0] { + case 0: + negative = false + case 1: + negative = true + default: + return EmptyInt, fmt.Errorf("big int prefix should be either 0 or 1, got %d", buf[0]) + } + + i := big.NewInt(0).SetBytes(buf[1:]) + if negative { + i.Neg(i) + } + + return BigInt{i}, nil +} + +func (bi *BigInt) MarshalCBOR(w io.Writer) error { + if bi.Int == nil { + zero := NewInt(0) + return zero.MarshalCBOR(w) + } + + enc := bi.cborBytes() + + header := cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(enc))) + if _, err := w.Write(header); err != nil { + return err + } + + if _, err := w.Write(enc); err != nil { + return err + } + + return nil +} + +func (bi *BigInt) UnmarshalCBOR(br io.Reader) error { + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + + if maj != cbg.MajByteString { + return fmt.Errorf("cbor input for fil big int was not a byte string (%x)", maj) + } + + if extra == 0 { + bi.Int = big.NewInt(0) + return nil + } + + if extra > BigIntMaxSerializedLen { + return fmt.Errorf("big integer byte array too long") + } + + buf := make([]byte, extra) + if _, err := io.ReadFull(br, buf); err != nil { + return err + } + + i, err := fromCborBytes(buf) + if err != nil { + return err + } + + *bi = i + + return nil +} diff --git a/storagemarket/cbor_gen.go b/storagemarket/cbor_gen.go new file mode 100644 index 000000000..e4a9efcfa --- /dev/null +++ b/storagemarket/cbor_gen.go @@ -0,0 +1,352 @@ +package storagemarket + +import ( + "fmt" + "io" + + "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +var _ = xerrors.Errorf + +func (t *ClientDeal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{136}); err != nil { + return err + } + + // t.ProposalCid (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.ProposalCid); err != nil { + return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) + } + + // t.Proposal (actors.StorageDealProposal) (struct) + if err := t.Proposal.MarshalCBOR(w); err != nil { + return err + } + + // t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { + return err + } + + // t.Miner (peer.ID) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Miner)); err != nil { + return err + } + + // t.MinerWorker (address.Address) (struct) + if err := t.MinerWorker.MarshalCBOR(w); err != nil { + return err + } + + // t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { + return err + } + + // t.PayloadCid (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.PayloadCid); err != nil { + return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err) + } + + // t.PublishMessage (cid.Cid) (struct) + + if t.PublishMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) + } + } + + return nil +} + +func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 8 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.ProposalCid (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) + } + + t.ProposalCid = c + + } + // t.Proposal (actors.StorageDealProposal) (struct) + + { + + if err := t.Proposal.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.State (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = uint64(extra) + // t.Miner (peer.ID) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Miner = peer.ID(sval) + } + // t.MinerWorker (address.Address) (struct) + + { + + if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.DealID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + // t.PayloadCid (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err) + } + + t.PayloadCid = c + + } + // t.PublishMessage (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + } + + t.PublishMessage = &c + } + + } + return nil +} + +func (t *MinerDeal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{136}); err != nil { + return err + } + + // t.ProposalCid (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.ProposalCid); err != nil { + return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) + } + + // t.Proposal (actors.StorageDealProposal) (struct) + if err := t.Proposal.MarshalCBOR(w); err != nil { + return err + } + + // t.Miner (peer.ID) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Miner)); err != nil { + return err + } + + // t.Client (peer.ID) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Client)); err != nil { + return err + } + + // t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { + return err + } + + // t.Ref (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Ref); err != nil { + return xerrors.Errorf("failed to write cid field t.Ref: %w", err) + } + + // t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { + return err + } + + // t.SectorID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { + return err + } + return nil +} + +func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 8 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.ProposalCid (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) + } + + t.ProposalCid = c + + } + // t.Proposal (actors.StorageDealProposal) (struct) + + { + + if err := t.Proposal.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.Miner (peer.ID) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Miner = peer.ID(sval) + } + // t.Client (peer.ID) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Client = peer.ID(sval) + } + // t.State (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = uint64(extra) + // t.Ref (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Ref: %w", err) + } + + t.Ref = c + + } + // t.DealID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + // t.SectorID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = uint64(extra) + return nil +} diff --git a/storagemarket/types.go b/storagemarket/types.go new file mode 100644 index 000000000..1497063c4 --- /dev/null +++ b/storagemarket/types.go @@ -0,0 +1,199 @@ +package storagemarket + +import ( + "context" + + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/peer" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +const DealProtocolID = "/fil/storage/mk/1.0.1" +const AskProtocolID = "/fil/storage/ask/1.0.1" + +// type shims - used during migration into separate module +type Balance = actors.StorageParticipantBalance +type DealID uint64 +type Signature = types.Signature +type StorageDeal = actors.OnChainDeal +type StorageAsk = types.SignedStorageAsk +type StateKey = *types.TipSet +type Epoch uint64 +type TokenAmount BigInt + +// Duplicated from deals package for now +type MinerDeal struct { + ProposalCid cid.Cid + Proposal actors.StorageDealProposal + Miner peer.ID + Client peer.ID + State api.DealState + + Ref cid.Cid + + DealID uint64 + SectorID uint64 // Set when sm >= DealStaged +} + +type ClientDeal struct { + ProposalCid cid.Cid + Proposal actors.StorageDealProposal + State api.DealState + Miner peer.ID + MinerWorker address.Address + DealID uint64 + PayloadCid cid.Cid + PublishMessage *cid.Cid +} + +// The interface provided for storage providers +type StorageProvider interface { + Run(ctx context.Context, host host.Host) + + Stop() + + AddAsk(price TokenAmount, ttlsecs int64) error + + // ListAsks lists current asks + ListAsks(addr address.Address) []*StorageAsk + + // ListDeals lists on-chain deals associated with this provider + ListDeals(ctx context.Context) ([]StorageDeal, error) + + // ListIncompleteDeals lists deals that are in progress or rejected + ListIncompleteDeals() ([]MinerDeal, error) + + // AddStorageCollateral adds storage collateral + AddStorageCollateral(ctx context.Context, amount TokenAmount) error + + // GetStorageCollateral returns the current collateral balance + GetStorageCollateral(ctx context.Context) (Balance, error) +} + +// Node dependencies for a StorageProvider +type StorageProviderNode interface { + MostRecentStateId(ctx context.Context) (StateKey, error) + + // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. + AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error + + // Ensures that a storage market participant has a certain amount of available funds + EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error + + // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. + GetBalance(ctx context.Context, addr address.Address) (Balance, error) + + // Publishes deal on chain + PublishDeals(ctx context.Context, deal MinerDeal) (DealID, cid.Cid, error) + + // ListProviderDeals lists all deals associated with a storage provider + ListProviderDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) + + // Called when a deal is complete and on chain, and data has been transferred and is ready to be added to a sector + // returns sector id + OnDealComplete(ctx context.Context, deal MinerDeal, piecePath string) (uint64, error) + + // returns the worker address associated with a miner + GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) + + // Signs bytes + SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) +} + +type DealSectorCommittedCallback func(error) + +// Node dependencies for a StorageClient +type StorageClientNode interface { + MostRecentStateId(ctx context.Context) (StateKey, error) + + // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. + AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error + + EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error + + // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. + GetBalance(ctx context.Context, addr address.Address) (Balance, error) + + //// ListClientDeals lists all on-chain deals associated with a storage client + ListClientDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) + + // GetProviderInfo returns information about a single storage provider + //GetProviderInfo(stateId StateID, addr Address) *StorageProviderInfo + + // GetStorageProviders returns information about known miners + ListStorageProviders(ctx context.Context) ([]*StorageProviderInfo, error) + + // Subscribes to storage market actor state changes for a given address. + // TODO: Should there be a timeout option for this? In the case that we are waiting for funds to be deposited and it never happens? + //SubscribeStorageMarketEvents(addr Address, handler StorageMarketEventHandler) (SubID, error) + + // Cancels a subscription + //UnsubscribeStorageMarketEvents(subId SubID) + ValidatePublishedDeal(ctx context.Context, deal ClientDeal) (uint64, error) + + // SignProposal signs a proposal + SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error + + GetDefaultWalletAddress(ctx context.Context) (address.Address, error) + + OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb DealSectorCommittedCallback) error + + ValidateAskSignature(ask *StorageAsk) error +} + +type StorageClientProofs interface { + //GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (CommP, error) +} + +// Closely follows the MinerInfo struct in the spec +type StorageProviderInfo struct { + Address address.Address // actor address + Owner address.Address + Worker address.Address // signs messages + SectorSize uint64 + PeerID peer.ID + // probably more like how much storage power, available collateral etc +} + +type ProposeStorageDealResult struct { + ProposalCid cid.Cid +} + +// The interface provided by the module to the outside world for storage clients. +type StorageClient interface { + Run(ctx context.Context) + + Stop() + + // ListProviders queries chain state and returns active storage providers + ListProviders(ctx context.Context) (<-chan StorageProviderInfo, error) + + // ListDeals lists on-chain deals associated with this provider + ListDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) + + // ListInProgressDeals lists deals that are in progress or rejected + ListInProgressDeals(ctx context.Context) ([]ClientDeal, error) + + // ListInProgressDeals lists deals that are in progress or rejected + GetInProgressDeal(ctx context.Context, cid cid.Cid) (ClientDeal, error) + + // GetAsk returns the current ask for a storage provider + GetAsk(ctx context.Context, info StorageProviderInfo) (*StorageAsk, error) + + //// FindStorageOffers lists providers and queries them to find offers that satisfy some criteria based on price, duration, etc. + //FindStorageOffers(criteria AskCriteria, limit uint) []*StorageOffer + + // ProposeStorageDeal initiates deal negotiation with a Storage Provider + ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, payloadCid cid.Cid, proposalExpiration Epoch, duration Epoch, price TokenAmount, collateral TokenAmount) (*ProposeStorageDealResult, error) + + // GetPaymentEscrow returns the current funds available for deal payment + GetPaymentEscrow(ctx context.Context, addr address.Address) (Balance, error) + + // AddStorageCollateral adds storage collateral + AddPaymentEscrow(ctx context.Context, addr address.Address, amount TokenAmount) error +} diff --git a/storagemarketadapter/client_adapter.go b/storagemarketadapter/client_adapter.go new file mode 100644 index 000000000..9e23771f5 --- /dev/null +++ b/storagemarketadapter/client_adapter.go @@ -0,0 +1,328 @@ +package storagemarketadapter + +// this file implements storagemarket.StorageClientNode + +import ( + "bytes" + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/events" + "github.com/filecoin-project/lotus/chain/market" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/storagemarket" +) + +type ClientNodeAdapter struct { + full.StateAPI + full.ChainAPI + full.MpoolAPI + + sm *stmgr.StateManager + cs *store.ChainStore + fm *market.FundMgr + ev *events.Events +} + +type clientApi struct { + full.ChainAPI + full.StateAPI +} + +func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr) storagemarket.StorageClientNode { + return &ClientNodeAdapter{ + StateAPI: state, + ChainAPI: chain, + MpoolAPI: mpool, + + sm: sm, + cs: cs, + fm: fm, + ev: events.NewEvents(context.TODO(), &clientApi{chain, state}), + } +} + +func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storagemarket.StorageProviderInfo, error) { + ts, err := n.ChainHead(ctx) + if err != nil { + return nil, err + } + + addresses, err := n.StateListMiners(ctx, ts) + if err != nil { + return nil, err + } + + var out []*storagemarket.StorageProviderInfo + + for _, addr := range addresses { + workerAddr, err := n.StateMinerWorker(ctx, addr, ts) + if err != nil { + return nil, err + } + + sectorSize, err := n.StateMinerSectorSize(ctx, addr, ts) + if err != nil { + return nil, err + } + + peerId, err := n.StateMinerPeerID(ctx, addr, ts) + if err != nil { + return nil, err + } + + out = append(out, &storagemarket.StorageProviderInfo{ + Address: addr, + Worker: workerAddr, + SectorSize: sectorSize, + PeerID: peerId, + }) + } + + return out, nil +} + +func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) { + allDeals, err := n.StateMarketDeals(ctx, nil) + if err != nil { + return nil, err + } + + var out []actors.OnChainDeal + + for _, deal := range allDeals { + if deal.Client == addr { + out = append(out, deal) + } + } + + return out, nil +} + +func (n *ClientNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { + return n.ChainHead(ctx) +} + +// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. +func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { + // (Provider Node API) + smsg, err := n.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: addr, + Value: types.BigInt(amount), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return err + } + + r, err := n.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return err + } + + if r.Receipt.ExitCode != 0 { + return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode) + } + + return nil +} + +func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { + return n.fm.EnsureAvailable(ctx, addr, types.BigInt(amount)) +} + +func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { + bal, err := n.StateMarketBalance(ctx, addr, nil) + if err != nil { + return storagemarket.Balance{}, err + } + + return bal, nil +} + +// ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal +// returns the Deal id if there is no error +func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal storagemarket.ClientDeal) (uint64, error) { + log.Infow("DEAL ACCEPTED!") + + pubmsg, err := c.cs.GetMessage(*deal.PublishMessage) + if err != nil { + return 0, xerrors.Errorf("getting deal pubsish message: %w", err) + } + + pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) + if err != nil { + return 0, xerrors.Errorf("getting miner worker failed: %w", err) + } + + if pubmsg.From != pw { + return 0, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) + } + + if pubmsg.To != actors.StorageMarketAddress { + return 0, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) + } + + if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { + return 0, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) + } + + var params actors.PublishStorageDealsParams + if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { + return 0, err + } + + dealIdx := -1 + for i, storageDeal := range params.Deals { + // TODO: make it less hacky + sd := storageDeal + eq, err := cborutil.Equals(&deal.Proposal, &sd) + if err != nil { + return 0, err + } + if eq { + dealIdx = i + break + } + } + + if dealIdx == -1 { + return 0, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage) + } + + // TODO: timeout + _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage) + if err != nil { + return 0, xerrors.Errorf("waiting for deal publish message: %w", err) + } + if ret.ExitCode != 0 { + return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) + } + + var res actors.PublishStorageDealResponse + if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { + return 0, err + } + + return res.DealIDs[dealIdx], nil +} + +func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb storagemarket.DealSectorCommittedCallback) error { + checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts) + if err != nil { + // TODO: This may be fine for some errors + return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } + + if sd.ActivationEpoch > 0 { + cb(nil) + return true, false, nil + } + + return false, true, nil + } + + called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) { + defer func() { + if err != nil { + cb(xerrors.Errorf("handling applied event: %w", err)) + } + }() + + if msg == nil { + log.Error("timed out waiting for deal activation... what now?") + return false, nil + } + + sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts) + if err != nil { + return false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } + + if sd.ActivationEpoch == 0 { + return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", dealId, ts.ParentState(), ts.Height()) + } + + log.Infof("Storage deal %d activated at epoch %d", dealId, sd.ActivationEpoch) + + cb(nil) + + return false, nil + } + + revert := func(ctx context.Context, ts *types.TipSet) error { + log.Warn("deal activation reverted; TODO: actually handle this!") + // TODO: Just go back to DealSealing? + return nil + } + + matchEvent := func(msg *types.Message) (bool, error) { + if msg.To != provider { + return false, nil + } + + if msg.Method != actors.MAMethods.ProveCommitSector { + return false, nil + } + + var params actors.SectorProveCommitInfo + if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { + return false, err + } + + var found bool + for _, dealID := range params.DealIDs { + if dealID == dealId { + found = true + break + } + } + + return found, nil + } + + if err := c.ev.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil { + return xerrors.Errorf("failed to set up called handler") + } + + return nil +} + +func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error { + return api.SignWith(ctx, n.Wallet.Sign, signer, proposal) +} + +func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { + return n.Wallet.GetDefault() +} + +func (n *ClientNodeAdapter) ValidateAskSignature(ask *types.SignedStorageAsk) error { + tss := n.cs.GetHeaviestTipSet().ParentState() + + w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner) + if err != nil { + return xerrors.Errorf("failed to get worker for miner in ask", err) + } + + sigb, err := cborutil.Dump(ask.Ask) + if err != nil { + return xerrors.Errorf("failed to re-serialize ask") + } + + return ask.Signature.Verify(w, sigb) +} + +var _ storagemarket.StorageClientNode = &ClientNodeAdapter{} diff --git a/storagemarketadapter/provider_adapter.go b/storagemarketadapter/provider_adapter.go new file mode 100644 index 000000000..0691bee36 --- /dev/null +++ b/storagemarketadapter/provider_adapter.go @@ -0,0 +1,196 @@ +package storagemarketadapter + +// this file implements storagemarket.StorageProviderNode + +import ( + "bytes" + "context" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" + unixfile "github.com/ipfs/go-unixfs/file" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/padreader" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storagemarket" +) + +var log = logging.Logger("provideradapter") + +type ProviderNodeAdapter struct { + api.FullNode + + // this goes away with the data transfer module + dag dtypes.StagingDAG + + secb *sectorblocks.SectorBlocks +} + +func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { + return &ProviderNodeAdapter{ + FullNode: full, + dag: dag, + secb: secb, + } +} + +func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { + log.Info("publishing deal") + + worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil) + if err != nil { + return 0, cid.Undef, err + } + + params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ + Deals: []actors.StorageDealProposal{deal.Proposal}, + }) + + if err != nil { + return 0, cid.Undef, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err) + } + + // TODO: We may want this to happen after fetching data + smsg, err := n.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: worker, + Value: types.NewInt(0), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.PublishStorageDeals, + Params: params, + }) + if err != nil { + return 0, cid.Undef, err + } + r, err := n.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return 0, cid.Undef, err + } + if r.Receipt.ExitCode != 0 { + return 0, cid.Undef, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode) + } + var resp actors.PublishStorageDealResponse + if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { + return 0, cid.Undef, err + } + if len(resp.DealIDs) != 1 { + return 0, cid.Undef, xerrors.Errorf("got unexpected number of DealIDs from") + } + + return storagemarket.DealID(resp.DealIDs[0]), smsg.Cid(), nil +} + +func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, piecePath string) (uint64, error) { + root, err := n.dag.Get(ctx, deal.Ref) + if err != nil { + return 0, xerrors.Errorf("failed to get file root for deal: %s", err) + } + + // TODO: abstract this away into ReadSizeCloser + implement different modes + node, err := unixfile.NewUnixfsFile(ctx, n.dag, root) + if err != nil { + return 0, xerrors.Errorf("cannot open unixfs file: %s", err) + } + + uf, ok := node.(sectorblocks.UnixfsReader) + if !ok { + // we probably got directory, unsupported for now + return 0, xerrors.Errorf("unsupported unixfs file type") + } + + // TODO: uf.Size() is user input, not trusted + // This won't be useful / here after we migrate to putting CARs into sectors + size, err := uf.Size() + if err != nil { + return 0, xerrors.Errorf("getting unixfs file size: %w", err) + } + if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize { + return 0, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size") + } + + sectorID, err := n.secb.AddUnixfsPiece(ctx, uf, deal.DealID) + if err != nil { + return 0, xerrors.Errorf("AddPiece failed: %s", err) + } + log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID) + + return sectorID, nil +} + +func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) { + allDeals, err := n.StateMarketDeals(ctx, nil) + if err != nil { + return nil, err + } + + var out []actors.OnChainDeal + + for _, deal := range allDeals { + if deal.Provider == addr { + out = append(out, deal) + } + } + + return out, nil +} + +func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { + return n.StateMinerWorker(ctx, miner, nil) +} + +func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) { + return n.WalletSign(ctx, signer, b) +} + +func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt storagemarket.TokenAmount) error { + return n.MarketEnsureAvailable(ctx, addr, types.BigInt(amt)) +} + +func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { + return n.ChainHead(ctx) +} + +// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. +func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { + // (Provider Node API) + smsg, err := n.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: addr, + Value: types.BigInt(amount), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return err + } + + r, err := n.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return err + } + + if r.Receipt.ExitCode != 0 { + return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode) + } + + return nil +} + +func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { + bal, err := n.StateMarketBalance(ctx, addr, nil) + if err != nil { + return storagemarket.Balance{}, err + } + + return bal, nil +} + +var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{} From ccf359d057ddbf75b552b53c6f2b45edfe965bb1 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 16 Dec 2019 19:17:46 -0800 Subject: [PATCH 39/61] feat(retrieval): extract retrievalmarket Extract retrieval market and modify shared types --- chain/deals/client.go | 8 +- go.mod | 3 +- go.sum | 7 +- node/builder.go | 6 +- node/impl/client/client.go | 15 +- node/modules/client.go | 5 +- node/modules/services.go | 9 +- node/modules/storageminer.go | 9 +- retrieval/cbor-gen/main.go | 20 -- retrieval/discovery/discovery.go | 15 - retrieval/discovery/local.go | 54 ---- retrieval/impl/cbor_gen.go | 476 ------------------------------- retrieval/impl/client.go | 389 ------------------------- retrieval/impl/provider.go | 323 --------------------- retrieval/impl/run_cbor_gen.go | 33 --- retrieval/impl/types.go | 67 ----- retrieval/impl/verify.go | 141 --------- retrieval/types.go | 364 ----------------------- retrievaladapter/client.go | 20 +- retrievaladapter/converters.go | 45 +++ retrievaladapter/provider.go | 35 ++- 21 files changed, 127 insertions(+), 1917 deletions(-) delete mode 100644 retrieval/cbor-gen/main.go delete mode 100644 retrieval/discovery/discovery.go delete mode 100644 retrieval/discovery/local.go delete mode 100644 retrieval/impl/cbor_gen.go delete mode 100644 retrieval/impl/client.go delete mode 100644 retrieval/impl/provider.go delete mode 100644 retrieval/impl/run_cbor_gen.go delete mode 100644 retrieval/impl/types.go delete mode 100644 retrieval/impl/verify.go delete mode 100644 retrieval/types.go create mode 100644 retrievaladapter/converters.go diff --git a/chain/deals/client.go b/chain/deals/client.go index 879e408e2..02c9bcce8 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -11,15 +11,15 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" - "github.com/filecoin-project/lotus/storagemarket" + "github.com/filecoin-project/lotus/storagemarket" ) var log = logging.Logger("deals") diff --git a/go.mod b/go.mod index ee7c4d175..665c367c4 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce + github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 github.com/filecoin-project/go-paramfetch v0.0.1 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 @@ -29,7 +30,7 @@ require ( github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c - github.com/ipfs/go-car v0.0.2 + github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 github.com/ipfs/go-cid v0.0.4 github.com/ipfs/go-datastore v0.3.1 github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 diff --git a/go.sum b/go.sum index 589a5d544..dc1b26e42 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= +github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 h1:/L916kY3hyq8w18rLO9VMSHqw25/9pwRB3nVW6b+Sm4= +github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133/go.mod h1:7M0YUI2CSVmqEmXNeXNq5L/pjk7C1Q5ifhirfMADD/k= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -120,6 +122,7 @@ github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyC github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -210,8 +213,8 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= -github.com/ipfs/go-car v0.0.2 h1:j02lzgeijorstzoMl3nQmvvb8wjJUVCiOAl8XEwYMCQ= -github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5JwQ= +github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 h1:Nq8xEW+2KZq7IkRlkOh0rTEUI8FgunhMoLj5EMkJzbQ= +github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= diff --git a/node/builder.go b/node/builder.go index dcce895b7..d263e04fa 100644 --- a/node/builder.go +++ b/node/builder.go @@ -19,6 +19,8 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" @@ -43,8 +45,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/peermgr" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/filecoin-project/lotus/storagemarket" @@ -222,7 +222,7 @@ func Online() Option { Override(RunPeerMgrKey, modules.RunPeerMgr), Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - Override(new(*discovery.Local), discovery.NewLocal), + Override(new(*discovery.Local), modules.NewLocalDiscovery), Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver), Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 58f250f67..a8b7ff17f 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -25,6 +25,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-components/retrievalmarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" @@ -32,7 +33,7 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" + "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storagemarket" ) @@ -164,7 +165,7 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, - MinPrice: queryResponse.PieceRetrievalPrice(), + MinPrice: retrievaladapter.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), Miner: p.Address, // TODO: check MinerPeerID: p.ID, } @@ -292,9 +293,13 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path }) a.Retrieval.Retrieve( - ctx, order.Root.Bytes(), retrievalmarket.Params{ - PricePerByte: types.BigDiv(order.Total, types.NewInt(order.Size)), - }, order.Total, order.MinerPeerID, order.Client, order.Miner) + ctx, + order.Root.Bytes(), + retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), + retrievaladapter.ToSharedTokenAmount(order.Total), + order.MinerPeerID, + order.Client, + order.Miner) select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") diff --git a/node/modules/client.go b/node/modules/client.go index 6047df7af..67c6322ab 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -6,11 +6,12 @@ import ( "path/filepath" "reflect" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/paych" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" + "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" graphsync "github.com/ipfs/go-graphsync/impl" diff --git a/node/modules/services.go b/node/modules/services.go index bd0978085..bc99d49f3 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -8,15 +8,16 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/sub" "github.com/filecoin-project/lotus/node/hello" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storagemarket" ) @@ -81,6 +82,10 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c storagemarket.Sto }) } +func NewLocalDiscovery(ds dtypes.MetadataDS) *discovery.Local { + return discovery.NewLocal(ds) +} + func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { return discovery.Multi(l) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index d868414d9..876c2f368 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -25,6 +25,8 @@ import ( "github.com/filecoin-project/go-address" dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" @@ -35,8 +37,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" + "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -265,6 +266,6 @@ func SealTicketGen(api api.FullNode) storage.TicketFn { // RetrievalProvider creates a new retrieval provider attached to the provider blockstore func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider { - adapter := retrievaladapter.NewRetrievalProviderNode(full) - return retrievalimpl.NewProvider(sblks, adapter) + adapter := retrievaladapter.NewRetrievalProviderNode(sblks, full) + return retrievalimpl.NewProvider(adapter) } diff --git a/retrieval/cbor-gen/main.go b/retrieval/cbor-gen/main.go deleted file mode 100644 index cc3fe9887..000000000 --- a/retrieval/cbor-gen/main.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "fmt" - "os" - - retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" -) - -// main func has ONE JOB -func main() { - fmt.Print("Generating Cbor Marshal/Unmarshal...") - - if err := retrievalimpl.RunCborGen(); err != nil { - fmt.Println("Failed: ") - fmt.Println(err) - os.Exit(1) - } - fmt.Println("Done.") -} diff --git a/retrieval/discovery/discovery.go b/retrieval/discovery/discovery.go deleted file mode 100644 index 93da56b60..000000000 --- a/retrieval/discovery/discovery.go +++ /dev/null @@ -1,15 +0,0 @@ -package discovery - -import ( - cbor "github.com/ipfs/go-ipld-cbor" - - retrievalmarket "github.com/filecoin-project/lotus/retrieval" -) - -func init() { - cbor.RegisterCborType(retrievalmarket.RetrievalPeer{}) -} - -func Multi(r retrievalmarket.PeerResolver) retrievalmarket.PeerResolver { // TODO: actually support multiple mechanisms - return r -} diff --git a/retrieval/discovery/local.go b/retrieval/discovery/local.go deleted file mode 100644 index c777a0828..000000000 --- a/retrieval/discovery/local.go +++ /dev/null @@ -1,54 +0,0 @@ -package discovery - -import ( - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - dshelp "github.com/ipfs/go-ipfs-ds-help" - cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log/v2" - - "github.com/filecoin-project/lotus/node/modules/dtypes" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" -) - -var log = logging.Logger("ret-discovery") - -type Local struct { - ds datastore.Datastore -} - -func NewLocal(ds dtypes.MetadataDS) *Local { - return &Local{ds: namespace.Wrap(ds, datastore.NewKey("/deals/local"))} -} - -func (l *Local) AddPeer(cid cid.Cid, peer retrievalmarket.RetrievalPeer) error { - // TODO: allow multiple peers here - // (implement an util for tracking map[thing][]otherThing, use in sectorBlockstore too) - - log.Warn("Tracking multiple retrieval peers not implemented") - - entry, err := cbor.DumpObject(peer) - if err != nil { - return err - } - - return l.ds.Put(dshelp.CidToDsKey(cid), entry) -} - -func (l *Local) GetPeers(data cid.Cid) ([]retrievalmarket.RetrievalPeer, error) { - entry, err := l.ds.Get(dshelp.CidToDsKey(data)) - if err == datastore.ErrNotFound { - return []retrievalmarket.RetrievalPeer{}, nil - } - if err != nil { - return nil, err - } - var peer retrievalmarket.RetrievalPeer - if err := cbor.DecodeInto(entry, &peer); err != nil { - return nil, err - } - return []retrievalmarket.RetrievalPeer{peer}, nil -} - -var _ retrievalmarket.PeerResolver = &Local{} diff --git a/retrieval/impl/cbor_gen.go b/retrieval/impl/cbor_gen.go deleted file mode 100644 index 220577089..000000000 --- a/retrieval/impl/cbor_gen.go +++ /dev/null @@ -1,476 +0,0 @@ -package retrievalimpl - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *RetParams) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct) - if err := t.Unixfs0.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *RetParams) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Unixfs0 = new(Unixfs0Offer) - if err := t.Unixfs0.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *OldQuery) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Piece (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Piece); err != nil { - return xerrors.Errorf("failed to write cid field t.Piece: %w", err) - } - - return nil -} - -func (t *OldQuery) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Piece (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Piece: %w", err) - } - - t.Piece = c - - } - return nil -} - -func (t *OldQueryResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.Status (retrievalimpl.OldQueryResponseStatus) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { - return err - } - - // t.Size (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { - return err - } - - // t.MinPrice (types.BigInt) (struct) - if err := t.MinPrice.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *OldQueryResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Status (retrievalimpl.OldQueryResponseStatus) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Status = OldQueryResponseStatus(extra) - // t.Size (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) - // t.MinPrice (types.BigInt) (struct) - - { - - if err := t.MinPrice.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *Unixfs0Offer) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Offset (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil { - return err - } - - // t.Size (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { - return err - } - return nil -} - -func (t *Unixfs0Offer) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Offset (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Offset = uint64(extra) - // t.Size (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) - return nil -} - -func (t *OldDealProposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.Payment (api.PaymentInfo) (struct) - if err := t.Payment.MarshalCBOR(w); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.Params (retrievalimpl.RetParams) (struct) - if err := t.Params.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *OldDealProposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Payment (api.PaymentInfo) (struct) - - { - - if err := t.Payment.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.Params (retrievalimpl.RetParams) (struct) - - { - - if err := t.Params.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *OldDealResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Status (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { - return err - } - - // t.Message (string) (string) - if len(t.Message) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Message was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Message)); err != nil { - return err - } - return nil -} - -func (t *OldDealResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Status (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Status = uint64(extra) - // t.Message (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Message = string(sval) - } - return nil -} - -func (t *Block) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Prefix ([]uint8) (slice) - if len(t.Prefix) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Prefix was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Prefix)))); err != nil { - return err - } - if _, err := w.Write(t.Prefix); err != nil { - return err - } - - // t.Data ([]uint8) (slice) - if len(t.Data) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Data was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { - return err - } - if _, err := w.Write(t.Data); err != nil { - return err - } - return nil -} - -func (t *Block) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Prefix ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Prefix: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Prefix = make([]byte, extra) - if _, err := io.ReadFull(br, t.Prefix); err != nil { - return err - } - // t.Data ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Data: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Data = make([]byte, extra) - if _, err := io.ReadFull(br, t.Data); err != nil { - return err - } - return nil -} diff --git a/retrieval/impl/client.go b/retrieval/impl/client.go deleted file mode 100644 index 5087a893d..000000000 --- a/retrieval/impl/client.go +++ /dev/null @@ -1,389 +0,0 @@ -package retrievalimpl - -import ( - "context" - "reflect" - "sync" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" - - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" -) - -var log = logging.Logger("retrieval") - -type client struct { - h host.Host - bs blockstore.Blockstore - node retrievalmarket.RetrievalClientNode - // The parameters should be replaced by RetrievalClientNode - - nextDealLk sync.Mutex - nextDealID retrievalmarket.DealID - subscribersLk sync.RWMutex - subscribers []retrievalmarket.ClientSubscriber -} - -// NewClient creates a new retrieval client -func NewClient(h host.Host, bs blockstore.Blockstore, node retrievalmarket.RetrievalClientNode) retrievalmarket.RetrievalClient { - return &client{h: h, bs: bs, node: node} -} - -// V0 - -// TODO: Implement for retrieval provider V0 epic -// https://github.com/filecoin-project/go-retrieval-market-project/issues/12 -func (c *client) FindProviders(pieceCID []byte) []retrievalmarket.RetrievalPeer { - panic("not implemented") -} - -// TODO: Update to match spec for V0 epic -// https://github.com/filecoin-project/go-retrieval-market-project/issues/8 -func (c *client) Query(ctx context.Context, p retrievalmarket.RetrievalPeer, pieceCID []byte, params retrievalmarket.QueryParams) (retrievalmarket.QueryResponse, error) { - cid, err := cid.Cast(pieceCID) - if err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - - s, err := c.h.NewStream(ctx, p.ID, retrievalmarket.QueryProtocolID) - if err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - defer s.Close() - - err = cborutil.WriteCborRPC(s, &OldQuery{ - Piece: cid, - }) - if err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - - var oldResp OldQueryResponse - if err := oldResp.UnmarshalCBOR(s); err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - - resp := retrievalmarket.QueryResponse{ - Status: retrievalmarket.QueryResponseStatus(oldResp.Status), - Size: oldResp.Size, - MinPricePerByte: types.BigDiv(oldResp.MinPrice, types.NewInt(oldResp.Size)), - } - return resp, nil -} - -// TODO: Update to match spec for V0 Epic: -// https://github.com/filecoin-project/go-retrieval-market-project/issues/9 -func (c *client) Retrieve(ctx context.Context, pieceCID []byte, params retrievalmarket.Params, totalFunds types.BigInt, miner peer.ID, clientWallet retrievalmarket.Address, minerWallet retrievalmarket.Address) retrievalmarket.DealID { - /* The implementation of this function is just wrapper for the old code which retrieves UnixFS pieces - -- it will be replaced when we do the V0 implementation of the module */ - c.nextDealLk.Lock() - c.nextDealID++ - dealID := c.nextDealID - c.nextDealLk.Unlock() - - dealState := retrievalmarket.ClientDealState{ - DealProposal: retrievalmarket.DealProposal{ - PieceCID: pieceCID, - ID: dealID, - Params: params, - }, - Status: retrievalmarket.DealStatusFailed, - Sender: miner, - } - - go func() { - evt := retrievalmarket.ClientEventError - converted, err := cid.Cast(pieceCID) - - if err == nil { - err = c.retrieveUnixfs(ctx, converted, types.BigDiv(totalFunds, params.PricePerByte).Uint64(), totalFunds, miner, clientWallet, minerWallet) - if err == nil { - evt = retrievalmarket.ClientEventComplete - dealState.Status = retrievalmarket.DealStatusCompleted - } - } - - c.notifySubscribers(evt, dealState) - }() - - return dealID -} - -// unsubscribeAt returns a function that removes an item from the subscribers list by comparing -// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order. -// Subsequent, repeated calls to the func with the same Subscriber are a no-op. -func (c *client) unsubscribeAt(sub retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { - return func() { - c.subscribersLk.Lock() - defer c.subscribersLk.Unlock() - curLen := len(c.subscribers) - for i, el := range c.subscribers { - if reflect.ValueOf(sub) == reflect.ValueOf(el) { - c.subscribers[i] = c.subscribers[curLen-1] - c.subscribers = c.subscribers[:curLen-1] - return - } - } - } -} - -func (c *client) notifySubscribers(evt retrievalmarket.ClientEvent, ds retrievalmarket.ClientDealState) { - c.subscribersLk.RLock() - defer c.subscribersLk.RUnlock() - for _, cb := range c.subscribers { - cb(evt, ds) - } -} - -func (c *client) SubscribeToEvents(subscriber retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { - c.subscribersLk.Lock() - c.subscribers = append(c.subscribers, subscriber) - c.subscribersLk.Unlock() - - return c.unsubscribeAt(subscriber) -} - -// V1 -func (c *client) AddMoreFunds(id retrievalmarket.DealID, amount types.BigInt) error { - panic("not implemented") -} - -func (c *client) CancelDeal(id retrievalmarket.DealID) error { - panic("not implemented") -} - -func (c *client) RetrievalStatus(id retrievalmarket.DealID) { - panic("not implemented") -} - -func (c *client) ListDeals() map[retrievalmarket.DealID]retrievalmarket.ClientDealState { - panic("not implemented") -} - -type clientStream struct { - node retrievalmarket.RetrievalClientNode - stream network.Stream - peeker cbg.BytePeeker - - root cid.Cid - size types.BigInt - offset uint64 - - paych retrievalmarket.Address - lane uint64 - total types.BigInt - transferred types.BigInt - - windowSize uint64 // how much we "trust" the peer - verifier BlockVerifier - bs blockstore.Blockstore -} - -/* This is the old retrieval code that is NOT spec compliant */ - -// C > S -// -// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size -// -// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} -// < Resp{Accept} -// < ..(Intermediate Block) -// < ..Blocks -// < ..(Intermediate Block) -// < ..Blocks -// > DealProposal(...) -// < ... -func (c *client) retrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr retrievalmarket.Address) error { - s, err := c.h.NewStream(ctx, miner, retrievalmarket.ProtocolID) - if err != nil { - return err - } - defer s.Close() - - initialOffset := uint64(0) // TODO: Check how much data we have locally - // TODO: Support in handler - // TODO: Allow client to specify this - - paych, err := c.node.GetOrCreatePaymentChannel(ctx, client, minerAddr, total) - if err != nil { - return xerrors.Errorf("getting payment channel: %w", err) - } - lane, err := c.node.AllocateLane(paych) - if err != nil { - return xerrors.Errorf("allocating payment lane: %w", err) - } - - cst := clientStream{ - node: c.node, - stream: s, - peeker: cbg.GetPeeker(s), - - root: root, - size: types.NewInt(size), - offset: initialOffset, - - paych: paych, - lane: lane, - total: total, - transferred: types.NewInt(0), - - windowSize: build.UnixfsChunkSize, - verifier: &UnixFs0Verifier{Root: root}, - bs: c.bs, - } - - for cst.offset != size+initialOffset { - toFetch := cst.windowSize - if toFetch+cst.offset > size { - toFetch = size - cst.offset - } - log.Infof("Retrieve %dB @%d", toFetch, cst.offset) - - err := cst.doOneExchange(ctx, toFetch) - if err != nil { - return xerrors.Errorf("retrieval exchange: %w", err) - } - - cst.offset += toFetch - } - log.Info("RETRIEVE SUCCESSFUL") - return nil -} - -func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64) error { - payAmount := types.BigDiv(types.BigMul(cst.total, types.NewInt(toFetch)), cst.size) - - payment, err := cst.setupPayment(ctx, payAmount) - if err != nil { - return xerrors.Errorf("setting up retrieval payment: %w", err) - } - - deal := &OldDealProposal{ - Payment: payment, - Ref: cst.root, - Params: RetParams{ - Unixfs0: &Unixfs0Offer{ - Offset: cst.offset, - Size: toFetch, - }, - }, - } - - if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { - return err - } - - var resp OldDealResponse - if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { - log.Error(err) - return err - } - - if resp.Status != Accepted { - cst.windowSize = build.UnixfsChunkSize - // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) - - if resp.Status == Error { - return xerrors.Errorf("storage deal error: %s", resp.Message) - } - if resp.Status == Rejected { - return xerrors.Errorf("storage deal rejected: %s", resp.Message) - } - return xerrors.New("storage deal response had no Accepted section") - } - - log.Info("Retrieval accepted, fetching blocks") - - return cst.fetchBlocks(toFetch) - - // TODO: maybe increase miner window size after success -} - -func (cst *clientStream) fetchBlocks(toFetch uint64) error { - blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - - for i := uint64(0); i < blocksToFetch; { - log.Infof("block %d of %d", i+1, blocksToFetch) - - var block Block - if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil { - return xerrors.Errorf("reading fetchBlock response: %w", err) - } - - dataBlocks, err := cst.consumeBlockMessage(block) - if err != nil { - return xerrors.Errorf("consuming retrieved blocks: %w", err) - } - - i += dataBlocks - } - - return nil -} - -func (cst *clientStream) consumeBlockMessage(block Block) (uint64, error) { - prefix, err := cid.PrefixFromBytes(block.Prefix) - if err != nil { - return 0, err - } - - cid, err := prefix.Sum(block.Data) - - blk, err := blocks.NewBlockWithCid(block.Data, cid) - if err != nil { - return 0, err - } - - internal, err := cst.verifier.Verify(context.TODO(), blk) - if err != nil { - log.Warnf("block verify failed: %s", err) - return 0, err - } - - // TODO: Smarter out, maybe add to filestore automagically - // (Also, persist intermediate nodes) - err = cst.bs.Put(blk) - if err != nil { - log.Warnf("block write failed: %s", err) - return 0, err - } - - if internal { - return 0, nil - } - - return 1, nil -} - -func (cst *clientStream) setupPayment(ctx context.Context, toSend types.BigInt) (api.PaymentInfo, error) { - amount := types.BigAdd(cst.transferred, toSend) - - sv, err := cst.node.CreatePaymentVoucher(ctx, cst.paych, amount, cst.lane) - if err != nil { - return api.PaymentInfo{}, err - } - - cst.transferred = amount - - return api.PaymentInfo{ - Channel: cst.paych, - ChannelMessage: nil, - Vouchers: []*types.SignedVoucher{sv}, - }, nil -} diff --git a/retrieval/impl/provider.go b/retrieval/impl/provider.go deleted file mode 100644 index 9e7254b60..000000000 --- a/retrieval/impl/provider.go +++ /dev/null @@ -1,323 +0,0 @@ -package retrievalimpl - -import ( - "context" - "io" - "reflect" - "sync" - - "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-merkledag" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/storage/sectorblocks" -) - -// RetrMinerAPI are the node functions needed by a retrieval provider -type RetrMinerAPI interface { - PaychVoucherAdd(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) -} - -type provider struct { - - // TODO: Replace with RetrievalProviderNode & FileStore for https://github.com/filecoin-project/go-retrieval-market-project/issues/9 - sectorBlocks *sectorblocks.SectorBlocks - - // TODO: Replace with RetrievalProviderNode for - // https://github.com/filecoin-project/go-retrieval-market-project/issues/4 - node retrievalmarket.RetrievalProviderNode - - pricePerByte retrievalmarket.BigInt - - subscribersLk sync.RWMutex - subscribers []retrievalmarket.ProviderSubscriber -} - -// NewProvider returns a new retrieval provider -func NewProvider(sblks *sectorblocks.SectorBlocks, node retrievalmarket.RetrievalProviderNode) retrievalmarket.RetrievalProvider { - return &provider{ - sectorBlocks: sblks, - node: node, - - pricePerByte: types.NewInt(2), // TODO: allow setting - } -} - -// Start begins listening for deals on the given host -func (p *provider) Start(host host.Host) { - host.SetStreamHandler(retrievalmarket.QueryProtocolID, p.handleQueryStream) - host.SetStreamHandler(retrievalmarket.ProtocolID, p.handleDealStream) -} - -// V0 -// SetPricePerByte sets the price per byte a miner charges for retrievals -func (p *provider) SetPricePerByte(price retrievalmarket.BigInt) { - p.pricePerByte = price -} - -// SetPaymentInterval sets the maximum number of bytes a a provider will send before -// requesting further payment, and the rate at which that value increases -// TODO: Implement for https://github.com/filecoin-project/go-retrieval-market-project/issues/7 -func (p *provider) SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64) { - panic("not implemented") -} - -// unsubscribeAt returns a function that removes an item from the subscribers list by comparing -// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order. -// Subsequent, repeated calls to the func with the same Subscriber are a no-op. -func (p *provider) unsubscribeAt(sub retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { - return func() { - p.subscribersLk.Lock() - defer p.subscribersLk.Unlock() - curLen := len(p.subscribers) - for i, el := range p.subscribers { - if reflect.ValueOf(sub) == reflect.ValueOf(el) { - p.subscribers[i] = p.subscribers[curLen-1] - p.subscribers = p.subscribers[:curLen-1] - return - } - } - } -} - -func (p *provider) notifySubscribers(evt retrievalmarket.ProviderEvent, ds retrievalmarket.ProviderDealState) { - p.subscribersLk.RLock() - defer p.subscribersLk.RUnlock() - for _, cb := range p.subscribers { - cb(evt, ds) - } -} - -// SubscribeToEvents listens for events that happen related to client retrievals -// TODO: Implement updates as part of https://github.com/filecoin-project/go-retrieval-market-project/issues/7 -func (p *provider) SubscribeToEvents(subscriber retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { - p.subscribersLk.Lock() - p.subscribers = append(p.subscribers, subscriber) - p.subscribersLk.Unlock() - - return p.unsubscribeAt(subscriber) -} - -// V1 -func (p *provider) SetPricePerUnseal(price retrievalmarket.BigInt) { - panic("not implemented") -} - -func (p *provider) ListDeals() map[retrievalmarket.ProviderDealID]retrievalmarket.ProviderDealState { - panic("not implemented") -} - -func writeErr(stream network.Stream, err error) { - log.Errorf("Retrieval deal error: %+v", err) - _ = cborutil.WriteCborRPC(stream, &OldDealResponse{ - Status: Error, - Message: err.Error(), - }) -} - -// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/8 -func (p *provider) handleQueryStream(stream network.Stream) { - defer stream.Close() - - var query OldQuery - if err := cborutil.ReadCborRPC(stream, &query); err != nil { - writeErr(stream, err) - return - } - - size, err := p.sectorBlocks.GetSize(query.Piece) - if err != nil && err != sectorblocks.ErrNotFound { - log.Errorf("Retrieval query: GetRefs: %s", err) - return - } - - answer := &OldQueryResponse{ - Status: Unavailable, - } - if err == nil { - answer.Status = Available - - // TODO: get price, look for already unsealed ref to reduce work - answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), p.pricePerByte) - answer.Size = uint64(size) // TODO: verify on intermediate - } - - if err := cborutil.WriteCborRPC(stream, answer); err != nil { - log.Errorf("Retrieval query: WriteCborRPC: %s", err) - return - } -} - -type handlerDeal struct { - p *provider - stream network.Stream - - ufsr sectorblocks.UnixfsReader - open cid.Cid - at uint64 - size uint64 -} - -// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/7 -func (p *provider) handleDealStream(stream network.Stream) { - defer stream.Close() - - hnd := &handlerDeal{ - p: p, - - stream: stream, - } - - var err error - more := true - - for more { - more, err = hnd.handleNext() // TODO: 'more' bool - if err != nil { - writeErr(stream, err) - return - } - } - -} - -func (hnd *handlerDeal) handleNext() (bool, error) { - var deal OldDealProposal - if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil { - if err == io.EOF { // client sent all deals - err = nil - } - return false, err - } - - if deal.Params.Unixfs0 == nil { - return false, xerrors.New("unknown deal type") - } - - unixfs0 := deal.Params.Unixfs0 - - if len(deal.Payment.Vouchers) != 1 { - return false, xerrors.Errorf("expected one signed voucher, got %d", len(deal.Payment.Vouchers)) - } - - expPayment := types.BigMul(hnd.p.pricePerByte, types.NewInt(deal.Params.Unixfs0.Size)) - if _, err := hnd.p.node.SavePaymentVoucher(context.TODO(), deal.Payment.Channel, deal.Payment.Vouchers[0], nil, expPayment); err != nil { - return false, xerrors.Errorf("processing retrieval payment: %w", err) - } - - // If the file isn't open (new deal stream), isn't the right file, or isn't - // at the right offset, (re)open it - if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { - log.Infof("opening file for sending (open '%s') (@%d, want %d)", deal.Ref, hnd.at, unixfs0.Offset) - if err := hnd.openFile(deal); err != nil { - return false, err - } - } - - if unixfs0.Offset+unixfs0.Size > hnd.size { - return false, xerrors.Errorf("tried to read too much %d+%d > %d", unixfs0.Offset, unixfs0.Size, hnd.size) - } - - err := hnd.accept(deal) - if err != nil { - return false, err - } - return true, nil -} - -func (hnd *handlerDeal) openFile(deal OldDealProposal) error { - unixfs0 := deal.Params.Unixfs0 - - if unixfs0.Offset != 0 { - // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs - return xerrors.New("sending merkle proofs for nonzero offset not supported yet") - } - hnd.at = unixfs0.Offset - - bstore := hnd.p.sectorBlocks.SealedBlockstore(func() error { - return nil // TODO: approve unsealing based on amount paid - }) - - ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) - rootNd, err := ds.Get(context.TODO(), deal.Ref) - if err != nil { - return err - } - - fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd) - if err != nil { - return err - } - - var ok bool - hnd.ufsr, ok = fsr.(sectorblocks.UnixfsReader) - if !ok { - return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Ref) - } - - isize, err := hnd.ufsr.Size() - if err != nil { - return err - } - hnd.size = uint64(isize) - - hnd.open = deal.Ref - - return nil -} - -func (hnd *handlerDeal) accept(deal OldDealProposal) error { - unixfs0 := deal.Params.Unixfs0 - - resp := &OldDealResponse{ - Status: Accepted, - } - if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil { - log.Errorf("Retrieval query: Write Accepted resp: %s", err) - return err - } - - blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - for i := uint64(0); i < blocksToSend; { - data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) - if err != nil { - return err - } - - log.Infof("sending block for a deal: %s", nd.Cid()) - - if offset != unixfs0.Offset { - return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", unixfs0.Offset, offset) - } - - /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) - writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data))) - return - }*/ - - block := &Block{ - Prefix: nd.Cid().Prefix().Bytes(), - Data: nd.RawData(), - } - - if err := cborutil.WriteCborRPC(hnd.stream, block); err != nil { - return err - } - - if len(data) > 0 { // don't count internal nodes - hnd.at += uint64(len(data)) - i++ - } - } - - return nil -} diff --git a/retrieval/impl/run_cbor_gen.go b/retrieval/impl/run_cbor_gen.go deleted file mode 100644 index 8dbd6f19c..000000000 --- a/retrieval/impl/run_cbor_gen.go +++ /dev/null @@ -1,33 +0,0 @@ -package retrievalimpl - -import ( - "fmt" - "os" - - cborgen "github.com/whyrusleeping/cbor-gen" -) - -func RunCborGen() error { - genName := "./impl/cbor_gen.go" - reName := "./impl/cbor_gen_old.go" - if err := os.Rename(genName, reName); err != nil { - return fmt.Errorf("could not rename %s to %s", genName, reName) - } - if err := cborgen.WriteTupleEncodersToFile( - genName, - "retrievalimpl", - RetParams{}, - OldQuery{}, - OldQueryResponse{}, - Unixfs0Offer{}, - OldDealProposal{}, - OldDealResponse{}, - Block{}, - ); err != nil { - return err - } - if err := os.Remove(reName); err != nil { - return err - } - return nil -} diff --git a/retrieval/impl/types.go b/retrieval/impl/types.go deleted file mode 100644 index 3af74f993..000000000 --- a/retrieval/impl/types.go +++ /dev/null @@ -1,67 +0,0 @@ -package retrievalimpl - -import ( - "github.com/filecoin-project/lotus/api" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/chain/types" -) - -/* These types are all the types provided by Lotus, which diverge even from -spec V0 -- prior to the "update to spec epic", we are using these types internally -and switching to spec at the boundaries of the module */ - -type OldQueryResponseStatus uint64 - -const ( - Available OldQueryResponseStatus = iota - Unavailable -) - -const ( - Accepted = iota - Error - Rejected - Unsealing -) - -type OldQuery struct { - Piece cid.Cid - // TODO: payment -} - -type OldQueryResponse struct { - Status OldQueryResponseStatus - - Size uint64 // TODO: spec - // TODO: unseal price (+spec) - // TODO: sectors to unseal - // TODO: address to send money for the deal? - MinPrice types.BigInt -} - -type Unixfs0Offer struct { - Offset uint64 - Size uint64 -} - -type RetParams struct { - Unixfs0 *Unixfs0Offer -} - -type OldDealProposal struct { - Payment api.PaymentInfo - - Ref cid.Cid - Params RetParams -} - -type OldDealResponse struct { - Status uint64 - Message string -} - -type Block struct { // TODO: put in spec - Prefix []byte // TODO: fix cid.Prefix marshaling somehow - Data []byte -} diff --git a/retrieval/impl/verify.go b/retrieval/impl/verify.go deleted file mode 100644 index 892876ef5..000000000 --- a/retrieval/impl/verify.go +++ /dev/null @@ -1,141 +0,0 @@ -package retrievalimpl - -import ( - "context" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-unixfs" - pb "github.com/ipfs/go-unixfs/pb" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" -) - -type BlockVerifier interface { - Verify(context.Context, blocks.Block) (internal bool, err error) -} - -type OptimisticVerifier struct { -} - -func (o *OptimisticVerifier) Verify(context.Context, blocks.Block) (bool, error) { - // It's probably fine - return false, nil -} - -type UnixFs0Verifier struct { - Root cid.Cid - rootBlk blocks.Block - - expect int - seen int - - sub *UnixFs0Verifier -} - -func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block) (last bool, internal bool, err error) { - if b.sub != nil { - // TODO: check links here (iff b.sub.sub == nil) - - subLast, internal, err := b.sub.verify(ctx, blk) - if err != nil { - return false, false, err - } - if subLast { - b.sub = nil - b.seen++ - } - - return b.seen == b.expect, internal, nil - } - - if b.seen >= b.expect { // this is probably impossible - return false, false, xerrors.New("unixfs verifier: too many nodes in level") - } - - links, err := b.checkInternal(blk) - if err != nil { - return false, false, err - } - - if links > 0 { // TODO: check if all links are intermediate (or all aren't) - if links > build.UnixfsLinksPerLevel { - return false, false, xerrors.New("unixfs verifier: too many links in intermediate node") - } - - if b.seen+1 == b.expect && links != build.UnixfsLinksPerLevel { - return false, false, xerrors.New("unixfs verifier: too few nodes in level") - } - - b.sub = &UnixFs0Verifier{ - Root: blk.Cid(), - rootBlk: blk, - expect: links, - } - - // don't mark as seen yet - return false, true, nil - } - - b.seen++ - return b.seen == b.expect, false, nil -} - -func (b *UnixFs0Verifier) checkInternal(blk blocks.Block) (int, error) { - nd, err := ipld.Decode(blk) - if err != nil { - log.Warnf("IPLD Decode failed: %s", err) - return 0, err - } - - // TODO: check size - switch nd := nd.(type) { - case *merkledag.ProtoNode: - fsn, err := unixfs.FSNodeFromBytes(nd.Data()) - if err != nil { - log.Warnf("unixfs.FSNodeFromBytes failed: %s", err) - return 0, err - } - if fsn.Type() != pb.Data_File { - return 0, xerrors.New("internal nodes must be a file") - } - if len(fsn.Data()) > 0 { - return 0, xerrors.New("internal node with data") - } - if len(nd.Links()) == 0 { - return 0, xerrors.New("internal node with no links") - } - return len(nd.Links()), nil - - case *merkledag.RawNode: - return 0, nil - default: - return 0, xerrors.New("verifier: unknown node type") - } -} - -func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block) (bool, error) { - // root is special - if b.rootBlk == nil { - if !b.Root.Equals(blk.Cid()) { - return false, xerrors.Errorf("unixfs verifier: root block CID didn't match: valid %s, got %s", b.Root, blk.Cid()) - } - b.rootBlk = blk - links, err := b.checkInternal(blk) - if err != nil { - return false, err - } - - b.expect = links - return links != 0, nil - } - - _, internal, err := b.verify(ctx, blk) - return internal, err -} - -var _ BlockVerifier = &OptimisticVerifier{} -var _ BlockVerifier = &UnixFs0Verifier{} diff --git a/retrieval/types.go b/retrieval/types.go deleted file mode 100644 index f82279a9d..000000000 --- a/retrieval/types.go +++ /dev/null @@ -1,364 +0,0 @@ -package retrievalmarket - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/peer" -) - -// type aliases -// TODO: Remove and use native types or extract for -// https://github.com/filecoin-project/go-retrieval-market-project/issues/5 - -// BigInt is used for token amounts in retrieval deals -type BigInt = types.BigInt - -// Address is an address in the filecoin network -type Address = address.Address - -// SignedVoucher is a signed payment voucher -type SignedVoucher = types.SignedVoucher - -// ProtocolID is the protocol for proposing / responding to retrieval deals -const ProtocolID = "/fil/retrieval/0.0.1" - -// QueryProtocolID is the protocol for querying infromation about retrieval -// deal parameters -const QueryProtocolID = "/fil/retrieval/qry/0.0.1" // TODO: spec - -// Unsubscribe is a function that unsubscribes a subscriber for either the -// client or the provider -type Unsubscribe func() - -// ClientDealState is the current state of a deal from the point of view -// of a retrieval client -type ClientDealState struct { - DealProposal - Status DealStatus - Sender peer.ID - TotalReceived uint64 - FundsSpent BigInt -} - -// ClientEvent is an event that occurs in a deal lifecycle on the client -type ClientEvent uint64 - -const ( - // ClientEventOpen indicates a deal was initiated - ClientEventOpen ClientEvent = iota - - // ClientEventFundsExpended indicates a deal has run out of funds in the payment channel - // forcing the client to add more funds to continue the deal - ClientEventFundsExpended // when totalFunds is expended - - // ClientEventProgress indicates more data was received for a retrieval - ClientEventProgress - - // ClientEventError indicates an error occurred during a deal - ClientEventError - - // ClientEventComplete indicates a deal has completed - ClientEventComplete -) - -// ClientSubscriber is a callback that is registered to listen for retrieval events -type ClientSubscriber func(event ClientEvent, state ClientDealState) - -// RetrievalClient is a client interface for making retrieval deals -type RetrievalClient interface { - // V0 - - // Find Providers finds retrieval providers who may be storing a given piece - FindProviders(pieceCID []byte) []RetrievalPeer - - // Query asks a provider for information about a piece it is storing - Query( - ctx context.Context, - p RetrievalPeer, - pieceCID []byte, - params QueryParams, - ) (QueryResponse, error) - - // Retrieve retrieves all or part of a piece with the given retrieval parameters - Retrieve( - ctx context.Context, - pieceCID []byte, - params Params, - totalFunds BigInt, - miner peer.ID, - clientWallet Address, - minerWallet Address, - ) DealID - - // SubscribeToEvents listens for events that happen related to client retrievals - SubscribeToEvents(subscriber ClientSubscriber) Unsubscribe - - // V1 - AddMoreFunds(id DealID, amount BigInt) error - CancelDeal(id DealID) error - RetrievalStatus(id DealID) - ListDeals() map[DealID]ClientDealState -} - -// RetrievalClientNode are the node depedencies for a RetrevalClient -type RetrievalClientNode interface { - - // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist - // between a client and a miner and insures the client has the given amount of funds available in the channel - GetOrCreatePaymentChannel(ctx context.Context, clientAddress Address, minerAddress Address, clientFundsAvailable BigInt) (Address, error) - - // Allocate late creates a lane within a payment channel so that calls to - // CreatePaymentVoucher will automatically make vouchers only for the difference - // in total - AllocateLane(paymentChannel Address) (uint64, error) - - // CreatePaymentVoucher creates a new payment voucher in the given lane for a - // given payment channel so that all the payment vouchers in the lane add up - // to the given amount (so the payment voucher will be for the difference) - CreatePaymentVoucher(ctx context.Context, paymentChannel Address, amount BigInt, lane uint64) (*SignedVoucher, error) -} - -// ProviderDealState is the current state of a deal from the point of view -// of a retrieval provider -type ProviderDealState struct { - DealProposal - Status DealStatus - Receiver peer.ID - TotalSent uint64 - FundsReceived BigInt -} - -// ProviderEvent is an event that occurs in a deal lifecycle on the provider -type ProviderEvent uint64 - -const ( - - // ProviderEventOpen indicates a new deal was received from a client - ProviderEventOpen ProviderEvent = iota - - // ProviderEventProgress indicates more data was sent to a client - ProviderEventProgress - - // ProviderEventError indicates an error occurred in processing a deal for a client - ProviderEventError - - // ProviderEventComplete indicates a retrieval deal was completed for a client - ProviderEventComplete -) - -// ProviderDealID is a unique identifier for a deal on a provider -- it is -// a combination of DealID set by the client and the peer ID of the client -type ProviderDealID struct { - From peer.ID - ID DealID -} - -// ProviderSubscriber is a callback that is registered to listen for retrieval events on a provider -type ProviderSubscriber func(event ProviderEvent, state ProviderDealState) - -// RetrievalProvider is an interface by which a provider configures their -// retrieval operations and monitors deals received and process -type RetrievalProvider interface { - // Start begins listening for deals on the given host - Start(host.Host) - - // V0 - - // SetPricePerByte sets the price per byte a miner charges for retrievals - SetPricePerByte(price BigInt) - - // SetPaymentInterval sets the maximum number of bytes a a provider will send before - // requesting further payment, and the rate at which that value increases - SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64) - - // SubscribeToEvents listens for events that happen related to client retrievals - SubscribeToEvents(subscriber ProviderSubscriber) Unsubscribe - - // V1 - SetPricePerUnseal(price BigInt) - ListDeals() map[ProviderDealID]ProviderDealState -} - -// RetrievalProviderNode are the node depedencies for a RetrevalProvider -type RetrievalProviderNode interface { - SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *SignedVoucher, proof []byte, expectedAmount BigInt) (BigInt, error) -} - -// PeerResolver is an interface for looking up providers that may have a piece -type PeerResolver interface { - GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel -} - -// RetrievalPeer is a provider address/peer.ID pair (everything needed to make -// deals for with a miner) -type RetrievalPeer struct { - Address Address - ID peer.ID // optional -} - -// QueryResponseStatus indicates whether a queried piece is available -type QueryResponseStatus uint64 - -const ( - // QueryResponseAvailable indicates a provider has a piece and is prepared to - // return it - QueryResponseAvailable QueryResponseStatus = iota - - // QueryResponseUnavailable indicates a provider either does not have or cannot - // serve the queried piece to the client - QueryResponseUnavailable -) - -// QueryItemStatus (V1) indicates whether the requested part of a piece (payload or selector) -// is available for retrieval -type QueryItemStatus uint64 - -const ( - // QueryItemAvailable indicates requested part of the piece is available to be - // served - QueryItemAvailable QueryItemStatus = iota - - // QueryItemUnavailable indicates the piece either does not contain the requested - // item or it cannot be served - QueryItemUnavailable - - // QueryItemUnknown indicates the provider cannot determine if the given item - // is part of the requested piece (for example, if the piece is sealed and the - // miner does not maintain a payload CID index) - QueryItemUnknown -) - -// QueryParams indicate what specific information about a piece that a retrieval -// client is interested in, as well as specific parameters the client is seeking -// for the retrieval deal -type QueryParams struct { - PayloadCID cid.Cid // optional, query if miner has this cid in this piece. some miners may not be able to respond. - Selector ipld.Node // optional, query if miner has this cid in this piece. some miners may not be able to respond. - MaxPricePerByte BigInt // optional, tell miner uninterested if more expensive than this - MinPaymentInterval uint64 // optional, tell miner uninterested unless payment interval is greater than this - MinPaymentIntervalIncrease uint64 // optional, tell miner uninterested unless payment interval increase is greater than this -} - -// Query is a query to a given provider to determine information about a piece -// they may have available for retrieval -type Query struct { - PieceCID []byte // V0 - // QueryParams // V1 -} - -// QueryResponse is a miners response to a given retrieval query -type QueryResponse struct { - Status QueryResponseStatus - //PayloadCIDFound QueryItemStatus // V1 - if a PayloadCid was requested, the result - //SelectorFound QueryItemStatus // V1 - if a Selector was requested, the result - - Size uint64 // Total size of piece in bytes - //ExpectedPayloadSize uint64 // V1 - optional, if PayloadCID + selector are specified and miner knows, can offer an expected size - - PaymentAddress Address // address to send funds to -- may be different than miner addr - MinPricePerByte BigInt - MaxPaymentInterval uint64 - MaxPaymentIntervalIncrease uint64 -} - -// QueryResponseUndefined is an empty QueryResponse -var QueryResponseUndefined = QueryResponse{} - -// PieceRetrievalPrice is the total price to retrieve the piece (size * MinPricePerByte) -func (qr QueryResponse) PieceRetrievalPrice() BigInt { - return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.Size)) -} - -// PayloadRetrievalPrice is the expected price to retrieve just the given payload -// & selector (V1) -//func (qr QueryResponse) PayloadRetrievalPrice() BigInt { -// return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.ExpectedPayloadSize)) -//} - -// DealStatus is the status of a retrieval deal returned by a provider -// in a DealResponse -type DealStatus uint64 - -const ( - // DealStatusAccepted means a deal has been accepted by a provider - // and its is ready to proceed with retrieval - DealStatusAccepted DealStatus = iota - - // DealStatusFailed indicates something went wrong during a retrieval - DealStatusFailed - - // DealStatusRejected indicates the provider rejected a client's deal proposal - // for some reason - DealStatusRejected - - // DealStatusUnsealing indicates the provider is currently unsealing the sector - // needed to serve the retrieval deal - DealStatusUnsealing - - // DealStatusFundsNeeded indicates the provider is awaiting a payment voucher to - // continue processing the deal - DealStatusFundsNeeded - - // DealStatusOngoing indicates the provider is continuing to process a deal - DealStatusOngoing - - // DealStatusFundsNeededLastPayment indicates the provider is awaiting funds for - // a final payment in order to complete a deal - DealStatusFundsNeededLastPayment - - // DealStatusCompleted indicates a deal is complete - DealStatusCompleted - - // DealStatusDealNotFound indicates an update was received for a deal that could - // not be identified - DealStatusDealNotFound -) - -// Params are the parameters requested for a retrieval deal proposal -type Params struct { - //PayloadCID cid.Cid // V1 - //Selector ipld.Node // V1 - PricePerByte BigInt - PaymentInterval uint64 - PaymentIntervalIncrease uint64 -} - -// DealID is an identifier for a retrieval deal (unique to a client) -type DealID uint64 - -// DealProposal is a proposal for a new retrieval deal -type DealProposal struct { - PieceCID []byte - ID DealID - Params -} - -// Block is an IPLD block in bitswap format -type Block struct { - Prefix []byte - Data []byte -} - -// DealResponse is a response to a retrieval deal proposal -type DealResponse struct { - Status DealStatus - ID DealID - - // payment required to proceed - PaymentOwed BigInt - - Message string - Blocks []Block // V0 only -} - -// DealPayment is a payment for an in progress retrieval deal -type DealPayment struct { - ID DealID - PaymentChannel address.Address - PaymentVoucher *types.SignedVoucher -} diff --git a/retrievaladapter/client.go b/retrievaladapter/client.go index ff7d2f0fb..ae62632bd 100644 --- a/retrievaladapter/client.go +++ b/retrievaladapter/client.go @@ -3,9 +3,13 @@ package retrievaladapter import ( "context" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" + payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" ) type retrievalClientNode struct { @@ -21,21 +25,25 @@ func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retriev // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // between a client and a miner and insures the client has the given amount of funds available in the channel -func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress retrievalmarket.Address, minerAddress retrievalmarket.Address, clientFundsAvailable retrievalmarket.BigInt) (retrievalmarket.Address, error) { - paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, clientFundsAvailable) +func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) { + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, FromSharedTokenAmount(clientFundsAvailable)) return paych, err } // Allocate late creates a lane within a payment channel so that calls to // CreatePaymentVoucher will automatically make vouchers only for the difference // in total -func (rcn *retrievalClientNode) AllocateLane(paymentChannel retrievalmarket.Address) (uint64, error) { +func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) { return rcn.pmgr.AllocateLane(paymentChannel) } // CreatePaymentVoucher creates a new payment voucher in the given lane for a // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) -func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel retrievalmarket.Address, amount retrievalmarket.BigInt, lane uint64) (*retrievalmarket.SignedVoucher, error) { - return rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, amount, lane) +func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) { + voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, FromSharedTokenAmount(amount), lane) + if err != nil { + return nil, err + } + return ToSharedSignedVoucher(voucher) } diff --git a/retrievaladapter/converters.go b/retrievaladapter/converters.go new file mode 100644 index 000000000..069c3ea34 --- /dev/null +++ b/retrievaladapter/converters.go @@ -0,0 +1,45 @@ +package retrievaladapter + +import ( + "bytes" + sharedamount "github.com/filecoin-project/go-fil-components/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-components/shared/types" + "github.com/filecoin-project/lotus/chain/types" +) + + +func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { + return types.BigInt{Int: in.Int} +} + +func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { + return sharedamount.TokenAmount{Int: in.Int} +} + +func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go index 06757593b..fffd2c7b5 100644 --- a/retrievaladapter/provider.go +++ b/retrievaladapter/provider.go @@ -4,20 +4,43 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" "github.com/filecoin-project/lotus/api" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" + "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" ) type retrievalProviderNode struct { - full api.FullNode + sectorBlocks *sectorblocks.SectorBlocks + full api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(full api.FullNode) retrievalmarket.RetrievalProviderNode { - return &retrievalProviderNode{full} +func NewRetrievalProviderNode(sectorBlocks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProviderNode { + return &retrievalProviderNode{sectorBlocks, full} } -func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievalmarket.SignedVoucher, proof []byte, expectedAmount retrievalmarket.BigInt) (retrievalmarket.BigInt, error) { - return rpn.full.PaychVoucherAdd(ctx, paymentChannel, voucher, proof, expectedAmount) +func (rpn *retrievalProviderNode) GetPieceSize(pieceCid []byte) (uint64, error) { + asCid, err := cid.Cast(pieceCid) + if err != nil { + return 0, err + } + return rpn.sectorBlocks.GetSize(asCid) +} + +func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) blockstore.Blockstore { + return rpn.sectorBlocks.SealedBlockstore(approveUnseal) +} + +func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) { + localVoucher, err := FromSharedSignedVoucher(voucher) + if err != nil { + return retrievaltoken.FromInt(0), err + } + added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, FromSharedTokenAmount(expectedAmount)) + return ToSharedTokenAmount(added), err } From 0a4eeb7cb1e4ae1873e6cc8f2c77cfbb798ff5b6 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 10 Jan 2020 09:13:12 -0800 Subject: [PATCH 40/61] refactor(retrievalmarket): switch to go-fil-markets repo Switch go-fil-components to go-fil-markets --- chain/deals/client.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- node/builder.go | 4 ++-- node/impl/client/client.go | 2 +- node/modules/client.go | 4 ++-- node/modules/services.go | 4 ++-- node/modules/storageminer.go | 4 ++-- retrievaladapter/client.go | 6 +++--- retrievaladapter/converters.go | 4 ++-- retrievaladapter/provider.go | 6 +++--- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/chain/deals/client.go b/chain/deals/client.go index 02c9bcce8..e1067961c 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" diff --git a/go.mod b/go.mod index 665c367c4..1bcb04ec4 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce - github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 + github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 github.com/filecoin-project/go-paramfetch v0.0.1 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 diff --git a/go.sum b/go.sum index dc1b26e42..0e14ed7b1 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= -github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 h1:/L916kY3hyq8w18rLO9VMSHqw25/9pwRB3nVW6b+Sm4= -github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133/go.mod h1:7M0YUI2CSVmqEmXNeXNq5L/pjk7C1Q5ifhirfMADD/k= +github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= +github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= diff --git a/node/builder.go b/node/builder.go index d263e04fa..ee7286908 100644 --- a/node/builder.go +++ b/node/builder.go @@ -19,8 +19,8 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a8b7ff17f..09bd5d9ee 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -25,7 +25,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" diff --git a/node/modules/client.go b/node/modules/client.go index 67c6322ab..a03ea3691 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -6,8 +6,8 @@ import ( "path/filepath" "reflect" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/paych" diff --git a/node/modules/services.go b/node/modules/services.go index bc99d49f3..bf1985041 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -8,8 +8,8 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/messagepool" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 876c2f368..1197870d5 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -25,8 +25,8 @@ import ( "github.com/filecoin-project/go-address" dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" diff --git a/retrievaladapter/client.go b/retrievaladapter/client.go index ae62632bd..8f0289921 100644 --- a/retrievaladapter/client.go +++ b/retrievaladapter/client.go @@ -4,9 +4,9 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" - retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" diff --git a/retrievaladapter/converters.go b/retrievaladapter/converters.go index 069c3ea34..7ea3ffec0 100644 --- a/retrievaladapter/converters.go +++ b/retrievaladapter/converters.go @@ -2,8 +2,8 @@ package retrievaladapter import ( "bytes" - sharedamount "github.com/filecoin-project/go-fil-components/shared/tokenamount" - sharedtypes "github.com/filecoin-project/go-fil-components/shared/types" + sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/lotus/chain/types" ) diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go index fffd2c7b5..c5b238eda 100644 --- a/retrievaladapter/provider.go +++ b/retrievaladapter/provider.go @@ -4,9 +4,9 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" - retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/ipfs/go-cid" From 65ecb33630a7ff9077d8e26d059500a47fb72d22 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 17 Dec 2019 02:46:39 -0800 Subject: [PATCH 41/61] feat(storagemarket): extract storage market remove all shared components, point at go-fil-components repo --- chain/deals/cbor_gen.go | 784 ----------------------- chain/deals/client.go | 293 --------- chain/deals/client_states.go | 97 --- chain/deals/client_storagemarket.go | 118 ---- chain/deals/client_utils.go | 170 ----- chain/deals/provider.go | 293 --------- chain/deals/provider_asks.go | 143 ----- chain/deals/provider_states.go | 172 ----- chain/deals/provider_storagemarket.go | 65 -- chain/deals/provider_utils.go | 180 ------ chain/deals/request_validation_test.go | 274 -------- chain/deals/types.go | 112 ---- cmd/lotus-storage-miner/init.go | 26 +- gen/main.go | 27 - go.mod | 1 - lib/sharedutils/converters.go | 101 +++ node/builder.go | 12 +- node/impl/client/client.go | 43 +- node/modules/client.go | 18 +- node/modules/services.go | 2 +- node/modules/storageminer.go | 15 +- retrievaladapter/client.go | 7 +- retrievaladapter/converters.go | 45 -- retrievaladapter/provider.go | 7 +- storagemarket/bigint.go | 241 ------- storagemarket/cbor_gen.go | 352 ---------- storagemarket/types.go | 199 ------ storagemarketadapter/client_adapter.go | 55 +- storagemarketadapter/provider_adapter.go | 48 +- storagemarketadapter/utilities.go | 79 +++ 30 files changed, 325 insertions(+), 3654 deletions(-) delete mode 100644 chain/deals/cbor_gen.go delete mode 100644 chain/deals/client.go delete mode 100644 chain/deals/client_states.go delete mode 100644 chain/deals/client_storagemarket.go delete mode 100644 chain/deals/client_utils.go delete mode 100644 chain/deals/provider.go delete mode 100644 chain/deals/provider_asks.go delete mode 100644 chain/deals/provider_states.go delete mode 100644 chain/deals/provider_storagemarket.go delete mode 100644 chain/deals/provider_utils.go delete mode 100644 chain/deals/request_validation_test.go delete mode 100644 chain/deals/types.go create mode 100644 lib/sharedutils/converters.go delete mode 100644 retrievaladapter/converters.go delete mode 100644 storagemarket/bigint.go delete mode 100644 storagemarket/cbor_gen.go delete mode 100644 storagemarket/types.go create mode 100644 storagemarketadapter/utilities.go diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go deleted file mode 100644 index 5abb92825..000000000 --- a/chain/deals/cbor_gen.go +++ /dev/null @@ -1,784 +0,0 @@ -package deals - -import ( - "fmt" - "io" - - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *AskRequest) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Miner (address.Address) (struct) - if err := t.Miner.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *AskRequest) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Miner (address.Address) (struct) - - { - - if err := t.Miner.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *AskResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Ask (types.SignedStorageAsk) (struct) - if err := t.Ask.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Ask (types.SignedStorageAsk) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Ask = new(types.SignedStorageAsk) - if err := t.Ask.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *Proposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.DealProposal (actors.StorageDealProposal) (struct) - if err := t.DealProposal.MarshalCBOR(w); err != nil { - return err - } - - // t.Piece (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Piece); err != nil { - return xerrors.Errorf("failed to write cid field t.Piece: %w", err) - } - - return nil -} - -func (t *Proposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.DealProposal (actors.StorageDealProposal) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.DealProposal = new(actors.StorageDealProposal) - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - // t.Piece (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Piece: %w", err) - } - - t.Piece = c - - } - return nil -} - -func (t *Response) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{132}); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Message (string) (string) - if len(t.Message) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Message was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Message)); err != nil { - return err - } - - // t.Proposal (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Proposal); err != nil { - return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) - } - - // t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } - } - - return nil -} - -func (t *Response) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 4 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Message (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Message = string(sval) - } - // t.Proposal (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Proposal: %w", err) - } - - t.Proposal = c - - } - // t.PublishMessage (cid.Cid) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) - } - - t.PublishMessage = &c - } - - } - return nil -} - -func (t *SignedResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Response (deals.Response) (struct) - if err := t.Response.MarshalCBOR(w); err != nil { - return err - } - - // t.Signature (types.Signature) (struct) - if err := t.Signature.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Response (deals.Response) (struct) - - { - - if err := t.Response.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Signature (types.Signature) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Signature = new(types.Signature) - if err := t.Signature.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{136}); err != nil { - return err - } - - // t.Data (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Data); err != nil { - return xerrors.Errorf("failed to write cid field t.Data: %w", err) - } - - // t.PricePerEpoch (types.BigInt) (struct) - if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { - return err - } - - // t.ProposalExpiration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { - return err - } - - // t.Duration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { - return err - } - - // t.ProviderAddress (address.Address) (struct) - if err := t.ProviderAddress.MarshalCBOR(w); err != nil { - return err - } - - // t.Client (address.Address) (struct) - if err := t.Client.MarshalCBOR(w); err != nil { - return err - } - - // t.MinerWorker (address.Address) (struct) - if err := t.MinerWorker.MarshalCBOR(w); err != nil { - return err - } - - // t.MinerID (peer.ID) (string) - if len(t.MinerID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.MinerID was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.MinerID)); err != nil { - return err - } - return nil -} - -func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 8 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Data (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Data: %w", err) - } - - t.Data = c - - } - // t.PricePerEpoch (types.BigInt) (struct) - - { - - if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ProposalExpiration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ProposalExpiration = uint64(extra) - // t.Duration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Duration = uint64(extra) - // t.ProviderAddress (address.Address) (struct) - - { - - if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Client (address.Address) (struct) - - { - - if err := t.Client.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinerWorker (address.Address) (struct) - - { - - if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinerID (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.MinerID = peer.ID(sval) - } - return nil -} - -func (t *ClientDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.ClientDeal (storagemarket.ClientDeal) (struct) - if err := t.ClientDeal.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.ClientDeal (storagemarket.ClientDeal) (struct) - - { - - if err := t.ClientDeal.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *MinerDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{135}); err != nil { - return err - } - - // t.Client (peer.ID) (string) - if len(t.Client) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Client was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Client)); err != nil { - return err - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.SectorID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { - return err - } - return nil -} - -func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 7 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Client (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Client = peer.ID(sval) - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.SectorID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = uint64(extra) - return nil -} - -func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Proposal (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Proposal); err != nil { - return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - return nil -} - -func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Proposal (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Proposal: %w", err) - } - - t.Proposal = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - return nil -} diff --git a/chain/deals/client.go b/chain/deals/client.go deleted file mode 100644 index e1067961c..000000000 --- a/chain/deals/client.go +++ /dev/null @@ -1,293 +0,0 @@ -package deals - -import ( - "context" - - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p-core/host" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storagemarket" -) - -var log = logging.Logger("deals") - -type ClientDeal struct { - storagemarket.ClientDeal - - s inet.Stream -} - -type Client struct { - h host.Host - - // dataTransfer - // TODO: once the data transfer module is complete, the - // client will listen to events on the data transfer module - // Because we are using only a fake DAGService - // implementation, there's no validation or events on the client side - dataTransfer dtypes.ClientDataTransfer - dag dtypes.ClientDAG - discovery *discovery.Local - - node storagemarket.StorageClientNode - - deals *statestore.StateStore - conns map[cid.Cid]inet.Stream - - incoming chan *ClientDeal - updated chan clientDealUpdate - - stop chan struct{} - stopped chan struct{} -} - -type clientDealUpdate struct { - newState api.DealState - id cid.Cid - err error - mut func(*ClientDeal) -} - -func NewClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) *Client { - c := &Client{ - h: h, - dataTransfer: dataTransfer, - dag: dag, - discovery: discovery, - node: scn, - - deals: deals, - conns: map[cid.Cid]inet.Stream{}, - - incoming: make(chan *ClientDeal, 16), - updated: make(chan clientDealUpdate, 16), - - stop: make(chan struct{}), - stopped: make(chan struct{}), - } - - return c -} - -func (c *Client) Run(ctx context.Context) { - go func() { - defer close(c.stopped) - - for { - select { - case deal := <-c.incoming: - c.onIncoming(deal) - case update := <-c.updated: - c.onUpdated(ctx, update) - case <-c.stop: - return - } - } - }() -} - -func (c *Client) onIncoming(deal *ClientDeal) { - log.Info("incoming deal") - - if _, ok := c.conns[deal.ProposalCid]; ok { - log.Errorf("tracking deal connection: already tracking connection for deal %s", deal.ProposalCid) - return - } - c.conns[deal.ProposalCid] = deal.s - - if err := c.deals.Begin(deal.ProposalCid, deal); err != nil { - // We may have re-sent the proposal - log.Errorf("deal tracking failed: %s", err) - c.failDeal(deal.ProposalCid, err) - return - } - - go func() { - c.updated <- clientDealUpdate{ - newState: api.DealUnknown, - id: deal.ProposalCid, - err: nil, - } - }() -} - -func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { - log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState]) - var deal ClientDeal - err := c.deals.Mutate(update.id, func(d *ClientDeal) error { - d.State = update.newState - if update.mut != nil { - update.mut(d) - } - deal = *d - return nil - }) - if update.err != nil { - log.Errorf("deal %s failed: %s", update.id, update.err) - c.failDeal(update.id, update.err) - return - } - if err != nil { - c.failDeal(update.id, err) - return - } - - switch update.newState { - case api.DealUnknown: // new - c.handle(ctx, deal, c.new, api.DealAccepted) - case api.DealAccepted: - c.handle(ctx, deal, c.accepted, api.DealStaged) - case api.DealStaged: - c.handle(ctx, deal, c.staged, api.DealSealing) - case api.DealSealing: - c.handle(ctx, deal, c.sealing, api.DealNoUpdate) - // TODO: DealComplete -> watch for faults, expiration, etc. - } -} - -type ClientDealProposal struct { - Data cid.Cid - - PricePerEpoch types.BigInt - ProposalExpiration uint64 - Duration uint64 - - ProviderAddress address.Address - Client address.Address - MinerWorker address.Address - MinerID peer.ID -} - -func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - amount := types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration)) - if err := c.node.EnsureFunds(ctx, p.Client, storagemarket.TokenAmount(amount)); err != nil { - return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) - } - - commP, pieceSize, err := c.commP(ctx, p.Data) - if err != nil { - return cid.Undef, xerrors.Errorf("computing commP failed: %w", err) - } - - dealProposal := &actors.StorageDealProposal{ - PieceRef: commP, - PieceSize: uint64(pieceSize), - Client: p.Client, - Provider: p.ProviderAddress, - ProposalExpiration: p.ProposalExpiration, - Duration: p.Duration, - StoragePricePerEpoch: p.PricePerEpoch, - StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc - } - - if err := c.node.SignProposal(ctx, p.Client, dealProposal); err != nil { - return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) - } - - proposalNd, err := cborutil.AsIpld(dealProposal) - if err != nil { - return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) - } - - s, err := c.h.NewStream(ctx, p.MinerID, storagemarket.DealProtocolID) - if err != nil { - return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) - } - - proposal := &Proposal{ - DealProposal: dealProposal, - Piece: p.Data, - } - - if err := cborutil.WriteCborRPC(s, proposal); err != nil { - s.Reset() - return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) - } - - deal := &ClientDeal{ - ClientDeal: storagemarket.ClientDeal{ - ProposalCid: proposalNd.Cid(), - Proposal: *dealProposal, - State: api.DealUnknown, - Miner: p.MinerID, - MinerWorker: p.MinerWorker, - PayloadCid: p.Data, - }, - - s: s, - } - - c.incoming <- deal - - return deal.ProposalCid, c.discovery.AddPeer(p.Data, retrievalmarket.RetrievalPeer{ - Address: dealProposal.Provider, - ID: deal.Miner, - }) -} - -func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) { - s, err := c.h.NewStream(ctx, p, storagemarket.AskProtocolID) - if err != nil { - return nil, xerrors.Errorf("failed to open stream to miner: %w", err) - } - - req := &AskRequest{ - Miner: a, - } - if err := cborutil.WriteCborRPC(s, req); err != nil { - return nil, xerrors.Errorf("failed to send ask request: %w", err) - } - - var out AskResponse - if err := cborutil.ReadCborRPC(s, &out); err != nil { - return nil, xerrors.Errorf("failed to read ask response: %w", err) - } - - if out.Ask == nil { - return nil, xerrors.Errorf("got no ask back") - } - - if out.Ask.Ask.Miner != a { - return nil, xerrors.Errorf("got back ask for wrong miner") - } - - if err := c.checkAskSignature(out.Ask); err != nil { - return nil, xerrors.Errorf("ask was not properly signed") - } - - return out.Ask, nil -} - -func (c *Client) List() ([]ClientDeal, error) { - var out []ClientDeal - if err := c.deals.List(&out); err != nil { - return nil, err - } - return out, nil -} - -func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) { - var out ClientDeal - if err := c.deals.Get(d, &out); err != nil { - return nil, err - } - return &out, nil -} - -func (c *Client) Stop() { - close(c.stop) - <-c.stopped -} diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go deleted file mode 100644 index e82beb126..000000000 --- a/chain/deals/client_states.go +++ /dev/null @@ -1,97 +0,0 @@ -package deals - -import ( - "context" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" -) - -type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) - -func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) { - go func() { - mut, err := cb(ctx, deal) - if err != nil { - next = api.DealError - } - - if err == nil && next == api.DealNoUpdate { - return - } - - select { - case c.updated <- clientDealUpdate{ - newState: next, - id: deal.ProposalCid, - err: err, - mut: mut, - }: - case <-c.stop: - } - }() -} - -func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - resp, err := c.readStorageDealResp(deal) - if err != nil { - return nil, err - } - - // TODO: verify StorageDealSubmission - - if err := c.disconnect(deal); err != nil { - return nil, err - } - - /* data transfer happens */ - if resp.State != api.DealAccepted { - return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) - } - - return func(info *ClientDeal) { - info.PublishMessage = resp.PublishMessage - }, nil -} - -func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - log.Infow("DEAL ACCEPTED!") - - dealId, err := c.node.ValidatePublishedDeal(ctx, deal.ClientDeal) - if err != nil { - return nil, err - } - - return func(info *ClientDeal) { - info.DealID = dealId - }, nil -} - -func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - // TODO: Maybe wait for pre-commit - - return nil, nil -} - -func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - cb := func(err error) { - select { - case c.updated <- clientDealUpdate{ - newState: api.DealComplete, - id: deal.ProposalCid, - err: err, - }: - case <-c.stop: - } - } - - err := c.node.OnDealSectorCommitted(ctx, deal.Proposal.Provider, deal.DealID, cb) - - return nil, err -} - -func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { - return c.node.ValidateAskSignature(ask) -} diff --git a/chain/deals/client_storagemarket.go b/chain/deals/client_storagemarket.go deleted file mode 100644 index 08dacc7a3..000000000 --- a/chain/deals/client_storagemarket.go +++ /dev/null @@ -1,118 +0,0 @@ -package deals - -// this file implements storagemarket.StorageClient - -import ( - "context" - - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -func (c *Client) ListProviders(ctx context.Context) (<-chan storagemarket.StorageProviderInfo, error) { - providers, err := c.node.ListStorageProviders(ctx) - if err != nil { - return nil, err - } - - out := make(chan storagemarket.StorageProviderInfo) - - go func() { - for _, p := range providers { - select { - case out <- *p: - case <-ctx.Done(): - return - } - - } - }() - - return out, nil -} - -func (c *Client) ListDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) { - return c.node.ListClientDeals(ctx, addr) -} - -func (c *Client) ListInProgressDeals(ctx context.Context) ([]storagemarket.ClientDeal, error) { - deals, err := c.List() - if err != nil { - return nil, err - } - - out := make([]storagemarket.ClientDeal, len(deals)) - for k, v := range deals { - out[k] = storagemarket.ClientDeal{ - ProposalCid: v.ProposalCid, - Proposal: v.Proposal, - State: v.State, - Miner: v.Miner, - MinerWorker: v.MinerWorker, - DealID: v.DealID, - PublishMessage: v.PublishMessage, - } - } - - return out, nil -} - -func (c *Client) GetInProgressDeal(ctx context.Context, cid cid.Cid) (storagemarket.ClientDeal, error) { - deals, err := c.ListInProgressDeals(ctx) - if err != nil { - return storagemarket.ClientDeal{}, err - } - - for _, deal := range deals { - if deal.ProposalCid == cid { - return deal, nil - } - } - - return storagemarket.ClientDeal{}, xerrors.Errorf("couldn't find client deal") -} - -func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderInfo) (*storagemarket.StorageAsk, error) { - return c.QueryAsk(ctx, info.PeerID, info.Address) -} - -func (c *Client) ProposeStorageDeal(ctx context.Context, addr address.Address, info *storagemarket.StorageProviderInfo, payloadCid cid.Cid, proposalExpiration storagemarket.Epoch, duration storagemarket.Epoch, price storagemarket.TokenAmount, collateral storagemarket.TokenAmount) (*storagemarket.ProposeStorageDealResult, error) { - - proposal := ClientDealProposal{ - Data: payloadCid, - PricePerEpoch: types.BigInt(price), - ProposalExpiration: uint64(proposalExpiration), - Duration: uint64(duration), - Client: addr, - ProviderAddress: info.Address, - MinerWorker: info.Worker, - MinerID: info.PeerID, - } - - proposalCid, err := c.Start(ctx, proposal) - - result := &storagemarket.ProposeStorageDealResult{ - ProposalCid: proposalCid, - } - - return result, err -} - -func (c *Client) GetPaymentEscrow(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - - balance, err := c.node.GetBalance(ctx, addr) - - return balance, err -} - -func (c *Client) AddPaymentEscrow(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { - - return c.node.AddFunds(ctx, addr, amount) -} - -var _ storagemarket.StorageClient = &Client{} diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go deleted file mode 100644 index f2d150a23..000000000 --- a/chain/deals/client_utils.go +++ /dev/null @@ -1,170 +0,0 @@ -package deals - -import ( - "context" - "runtime" - - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-statestore" - - "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -func (c *Client) failDeal(id cid.Cid, cerr error) { - if cerr == nil { - _, f, l, _ := runtime.Caller(1) - cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) - } - - s, ok := c.conns[id] - if ok { - _ = s.Reset() - delete(c.conns, id) - } - - // TODO: store in some sort of audit log - log.Errorf("deal %s failed: %+v", id, cerr) -} - -func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) { - root, err := c.dag.Get(ctx, data) - if err != nil { - log.Errorf("failed to get file root for deal: %s", err) - return nil, 0, err - } - - n, err := unixfile.NewUnixfsFile(ctx, c.dag, root) - if err != nil { - log.Errorf("cannot open unixfs file: %s", err) - return nil, 0, err - } - - uf, ok := n.(files.File) - if !ok { - // TODO: we probably got directory, how should we handle this in unixfs mode? - return nil, 0, xerrors.New("unsupported unixfs type") - } - - s, err := uf.Size() - if err != nil { - return nil, 0, err - } - - pr, psize := padreader.New(uf, uint64(s)) - - commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize) - if err != nil { - return nil, 0, xerrors.Errorf("generating CommP: %w", err) - } - - return commp[:], psize, nil -} - -func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { - s, ok := c.conns[deal.ProposalCid] - if !ok { - // TODO: Try to re-establish the connection using query protocol - return nil, xerrors.Errorf("no connection to miner") - } - - var resp SignedResponse - if err := cborutil.ReadCborRPC(s, &resp); err != nil { - log.Errorw("failed to read Response message", "error", err) - return nil, err - } - - if err := resp.Verify(deal.MinerWorker); err != nil { - return nil, xerrors.Errorf("verifying response signature failed", err) - } - - if resp.Response.Proposal != deal.ProposalCid { - return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid) - } - - return &resp.Response, nil -} - -func (c *Client) disconnect(deal ClientDeal) error { - s, ok := c.conns[deal.ProposalCid] - if !ok { - return nil - } - - err := s.Close() - delete(c.conns, deal.ProposalCid) - return err -} - -var _ datatransfer.RequestValidator = &ClientRequestValidator{} - -// ClientRequestValidator validates data transfer requests for the client -// in a storage market -type ClientRequestValidator struct { - deals *statestore.StateStore -} - -// NewClientRequestValidator returns a new client request validator for the -// given datastore -func NewClientRequestValidator(deals dtypes.ClientDealStore) *ClientRequestValidator { - crv := &ClientRequestValidator{ - deals: deals, - } - return crv -} - -// ValidatePush validates a push request received from the peer that will send data -// Will always error because clients should not accept push requests from a provider -// in a storage deal (i.e. send data to client). -func (c *ClientRequestValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - return ErrNoPushAccepted -} - -// ValidatePull validates a pull request received from the peer that will receive data -// Will succeed only if: -// - voucher has correct type -// - voucher references an active deal -// - referenced deal matches the receiver (miner) -// - referenced deal matches the given base CID -// - referenced deal is in an acceptable state -func (c *ClientRequestValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - dealVoucher, ok := voucher.(*StorageDataTransferVoucher) - if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) - } - - var deal ClientDeal - err := c.deals.Get(dealVoucher.Proposal, &deal) - if err != nil { - return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) - } - if deal.Miner != receiver { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) - } - if !deal.PayloadCid.Equals(baseCid) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) - } - for _, state := range DataTransferStates { - if deal.State == state { - return nil - } - } - return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState) -} diff --git a/chain/deals/provider.go b/chain/deals/provider.go deleted file mode 100644 index 1a020de71..000000000 --- a/chain/deals/provider.go +++ /dev/null @@ -1,293 +0,0 @@ -package deals - -import ( - "context" - "errors" - "sync" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - "github.com/libp2p/go-libp2p-core/host" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storagemarket" -) - -var ProviderDsPrefix = "/deals/provider" - -type MinerDeal struct { - Client peer.ID - Proposal actors.StorageDealProposal - ProposalCid cid.Cid - State api.DealState - - Ref cid.Cid - - DealID uint64 - SectorID uint64 // Set when State >= DealStaged - - s inet.Stream -} - -type Provider struct { - pricePerByteBlock types.BigInt // how much we want for storing one byte for one block - minPieceSize uint64 - - ask *types.SignedStorageAsk - askLk sync.Mutex - - spn storagemarket.StorageProviderNode - - // TODO: This will go away once storage market module + CAR - // is implemented - dag dtypes.StagingDAG - - // dataTransfer is the manager of data transfers used by this storage provider - dataTransfer dtypes.ProviderDataTransfer - - deals *statestore.StateStore - ds dtypes.MetadataDS - - conns map[cid.Cid]inet.Stream - - actor address.Address - - incoming chan MinerDeal - updated chan minerDealUpdate - stop chan struct{} - stopped chan struct{} -} - -type minerDealUpdate struct { - newState api.DealState - id cid.Cid - err error - mut func(*MinerDeal) -} - -var ( - // ErrDataTransferFailed means a data transfer for a deal failed - ErrDataTransferFailed = errors.New("deal data transfer failed") -) - -func NewProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { - addr, err := ds.Get(datastore.NewKey("miner-address")) - if err != nil { - return nil, err - } - minerAddress, err := address.NewFromBytes(addr) - if err != nil { - return nil, err - } - - h := &Provider{ - dag: dag, - dataTransfer: dataTransfer, - spn: spn, - - pricePerByteBlock: types.NewInt(3), // TODO: allow setting - minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) - - conns: map[cid.Cid]inet.Stream{}, - - incoming: make(chan MinerDeal), - updated: make(chan minerDealUpdate), - stop: make(chan struct{}), - stopped: make(chan struct{}), - - actor: minerAddress, - - deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))), - ds: ds, - } - - if err := h.tryLoadAsk(); err != nil { - return nil, err - } - - if h.ask == nil { - // TODO: we should be fine with this state, and just say it means 'not actively accepting deals' - // for now... lets just set a price - if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil { - return nil, xerrors.Errorf("failed setting a default price: %w", err) - } - } - - // register a data transfer event handler -- this will move deals from - // accepted to staged - h.dataTransfer.SubscribeToEvents(h.onDataTransferEvent) - - return h, nil -} - -func (p *Provider) Run(ctx context.Context, host host.Host) { - // TODO: restore state - - host.SetStreamHandler(storagemarket.DealProtocolID, p.HandleStream) - host.SetStreamHandler(storagemarket.AskProtocolID, p.HandleAskStream) - - go func() { - defer log.Warn("quitting deal provider loop") - defer close(p.stopped) - - for { - select { - case deal := <-p.incoming: // DealAccepted - p.onIncoming(deal) - case update := <-p.updated: // DealStaged - p.onUpdated(ctx, update) - case <-p.stop: - return - } - } - }() -} - -func (p *Provider) onIncoming(deal MinerDeal) { - log.Info("incoming deal") - - p.conns[deal.ProposalCid] = deal.s - - if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil { - // This can happen when client re-sends proposal - p.failDeal(context.TODO(), deal.ProposalCid, err) - log.Errorf("deal tracking failed: %s", err) - return - } - - go func() { - p.updated <- minerDealUpdate{ - newState: api.DealAccepted, - id: deal.ProposalCid, - err: nil, - } - }() -} - -func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { - log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState]) - if update.err != nil { - log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err) - p.failDeal(ctx, update.id, update.err) - return - } - var deal MinerDeal - err := p.deals.Mutate(update.id, func(d *MinerDeal) error { - d.State = update.newState - if update.mut != nil { - update.mut(d) - } - deal = *d - return nil - }) - if err != nil { - p.failDeal(ctx, update.id, err) - return - } - - switch update.newState { - case api.DealAccepted: - p.handle(ctx, deal, p.accept, api.DealNoUpdate) - case api.DealStaged: - p.handle(ctx, deal, p.staged, api.DealSealing) - case api.DealSealing: - p.handle(ctx, deal, p.sealing, api.DealComplete) - case api.DealComplete: - p.handle(ctx, deal, p.complete, api.DealNoUpdate) - } -} - -// onDataTransferEvent is the function called when an event occurs in a data -// transfer -- it reads the voucher to verify this even occurred in a storage -// market deal, then, based on the data transfer event that occurred, it generates -// and update message for the deal -- either moving to staged for a completion -// event or moving to error if a data transfer error occurs -func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState datatransfer.ChannelState) { - voucher, ok := channelState.Voucher().(*StorageDataTransferVoucher) - // if this event is for a transfer not related to storage, ignore - if !ok { - return - } - - // data transfer events for opening and progress do not affect deal state - var next api.DealState - var err error - var mut func(*MinerDeal) - switch event.Code { - case datatransfer.Complete: - next = api.DealStaged - mut = func(deal *MinerDeal) { - deal.DealID = voucher.DealID - } - case datatransfer.Error: - next = api.DealFailed - err = ErrDataTransferFailed - default: - // the only events we care about are complete and error - return - } - - select { - case p.updated <- minerDealUpdate{ - newState: next, - id: voucher.Proposal, - err: err, - mut: mut, - }: - case <-p.stop: - } -} - -func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) { - proposalNd, err := cborutil.AsIpld(proposal.DealProposal) - if err != nil { - return MinerDeal{}, err - } - - return MinerDeal{ - Client: s.Conn().RemotePeer(), - Proposal: *proposal.DealProposal, - ProposalCid: proposalNd.Cid(), - State: api.DealUnknown, - - Ref: proposal.Piece, - - s: s, - }, nil -} - -func (p *Provider) HandleStream(s inet.Stream) { - log.Info("Handling storage deal proposal!") - - proposal, err := p.readProposal(s) - if err != nil { - log.Error(err) - s.Close() - return - } - - deal, err := p.newDeal(s, proposal) - if err != nil { - log.Errorf("%+v", err) - s.Close() - return - } - - p.incoming <- deal -} - -func (p *Provider) Stop() { - close(p.stop) - <-p.stopped -} diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go deleted file mode 100644 index a0cdd9c71..000000000 --- a/chain/deals/provider_asks.go +++ /dev/null @@ -1,143 +0,0 @@ -package deals - -import ( - "bytes" - "context" - "time" - - "github.com/ipfs/go-datastore" - inet "github.com/libp2p/go-libp2p-core/network" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/chain/types" -) - -func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error { - p.askLk.Lock() - defer p.askLk.Unlock() - - var seqno uint64 - if p.ask != nil { - seqno = p.ask.Ask.SeqNo + 1 - } - - now := time.Now().Unix() - ask := &types.StorageAsk{ - Price: price, - Timestamp: uint64(now), - Expiry: uint64(now + ttlsecs), - Miner: p.actor, - SeqNo: seqno, - MinPieceSize: p.minPieceSize, - } - - ssa, err := p.signAsk(ask) - if err != nil { - return err - } - - return p.saveAsk(ssa) -} - -func (p *Provider) GetAsk(m address.Address) *types.SignedStorageAsk { - p.askLk.Lock() - defer p.askLk.Unlock() - if m != p.actor { - return nil - } - - return p.ask -} - -func (p *Provider) HandleAskStream(s inet.Stream) { - defer s.Close() - var ar AskRequest - if err := cborutil.ReadCborRPC(s, &ar); err != nil { - log.Errorf("failed to read AskRequest from incoming stream: %s", err) - return - } - - resp := p.processAskRequest(&ar) - - if err := cborutil.WriteCborRPC(s, resp); err != nil { - log.Errorf("failed to write ask response: %s", err) - return - } -} - -func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse { - return &AskResponse{ - Ask: p.GetAsk(ar.Miner), - } -} - -var bestAskKey = datastore.NewKey("latest-ask") - -func (p *Provider) tryLoadAsk() error { - p.askLk.Lock() - defer p.askLk.Unlock() - - err := p.loadAsk() - if err != nil { - if xerrors.Is(err, datastore.ErrNotFound) { - log.Warn("no previous ask found, miner will not accept deals until a price is set") - return nil - } - return err - } - - return nil -} - -func (p *Provider) loadAsk() error { - askb, err := p.ds.Get(datastore.NewKey("latest-ask")) - if err != nil { - return xerrors.Errorf("failed to load most recent ask from disk: %w", err) - } - - var ssa types.SignedStorageAsk - if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { - return err - } - - p.ask = &ssa - return nil -} - -func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { - b, err := cborutil.Dump(a) - if err != nil { - return nil, err - } - - worker, err := p.spn.GetMinerWorker(context.TODO(), p.actor) - if err != nil { - return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err) - } - - sig, err := p.spn.SignBytes(context.TODO(), worker, b) - if err != nil { - return nil, err - } - - return &types.SignedStorageAsk{ - Ask: a, - Signature: sig, - }, nil -} - -func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { - b, err := cborutil.Dump(a) - if err != nil { - return err - } - - if err := p.ds.Put(bestAskKey, b); err != nil { - return err - } - - p.ask = a - return nil -} diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go deleted file mode 100644 index 76fa9a99b..000000000 --- a/chain/deals/provider_states.go +++ /dev/null @@ -1,172 +0,0 @@ -package deals - -import ( - "context" - - ipldfree "github.com/ipld/go-ipld-prime/impl/free" - "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) - -func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) { - go func() { - mut, err := cb(ctx, deal) - - if err == nil && next == api.DealNoUpdate { - return - } - - select { - case p.updated <- minerDealUpdate{ - newState: next, - id: deal.ProposalCid, - err: err, - mut: mut, - }: - case <-p.stop: - } - }() -} - -// ACCEPTED -func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - - head, err := p.spn.MostRecentStateId(ctx) - if err != nil { - return nil, err - } - if head.Height() >= deal.Proposal.ProposalExpiration { - return nil, xerrors.Errorf("deal proposal already expired") - } - - // TODO: check StorageCollateral - - minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30)) - if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { - return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) - } - - if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize { - return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize) - } - - // check market funds - clientMarketBalance, err := p.spn.GetBalance(ctx, deal.Proposal.Client) - if err != nil { - return nil, xerrors.Errorf("getting client market balance failed: %w", err) - } - - // This doesn't guarantee that the client won't withdraw / lock those funds - // but it's a decent first filter - if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) { - return nil, xerrors.New("clientMarketBalance.Available too small") - } - - waddr, err := p.spn.GetMinerWorker(ctx, deal.Proposal.Provider) - if err != nil { - return nil, err - } - - // TODO: check StorageCollateral (may be too large (or too small)) - if err := p.spn.EnsureFunds(ctx, waddr, storagemarket.TokenAmount(deal.Proposal.StorageCollateral)); err != nil { - return nil, err - } - - smDeal := storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - SectorID: deal.SectorID, - } - - dealId, mcid, err := p.spn.PublishDeals(ctx, smDeal) - if err != nil { - return nil, err - } - - log.Infof("fetching data for a deal %d", dealId) - err = p.sendSignedResponse(ctx, &Response{ - State: api.DealAccepted, - - Proposal: deal.ProposalCid, - PublishMessage: &mcid, - }) - if err != nil { - return nil, err - } - - if err := p.disconnect(deal); err != nil { - log.Warnf("closing client connection: %+v", err) - } - - ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) - - // this is the selector for "get the whole DAG" - // TODO: support storage deals with custom payload selectors - allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - - // initiate a pull data transfer. This will complete asynchronously and the - // completion of the data transfer will trigger a change in deal state - // (see onDataTransferEvent) - _, err = p.dataTransfer.OpenPullDataChannel(ctx, - deal.Client, - &StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: uint64(dealId)}, - deal.Ref, - allSelector, - ) - if err != nil { - return nil, xerrors.Errorf("failed to open pull data channel: %w", err) - } - - return nil, nil -} - -// STAGED - -func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - sectorID, err := p.spn.OnDealComplete( - ctx, - storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - DealID: deal.DealID, - }, - "", - ) - - if err != nil { - return nil, err - } - - return func(deal *MinerDeal) { - deal.SectorID = sectorID - }, nil -} - -// SEALING - -func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: consider waiting for seal to happen - - return nil, nil -} - -func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: observe sector lifecycle, status, expiration.. - - return nil, nil -} diff --git a/chain/deals/provider_storagemarket.go b/chain/deals/provider_storagemarket.go deleted file mode 100644 index 4d9863bd9..000000000 --- a/chain/deals/provider_storagemarket.go +++ /dev/null @@ -1,65 +0,0 @@ -package deals - -// this file implements storagemarket.StorageClient - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -func (p *Provider) AddAsk(price storagemarket.TokenAmount, ttlsecs int64) error { - return p.SetPrice(types.BigInt(price), ttlsecs) -} - -func (p *Provider) ListAsks(addr address.Address) []*types.SignedStorageAsk { - ask := p.GetAsk(addr) - - if ask != nil { - return []*types.SignedStorageAsk{ask} - } - - return nil -} - -func (p *Provider) ListDeals(ctx context.Context) ([]actors.OnChainDeal, error) { - return p.spn.ListProviderDeals(ctx, p.actor) -} - -func (p *Provider) AddStorageCollateral(ctx context.Context, amount storagemarket.TokenAmount) error { - return p.spn.AddFunds(ctx, p.actor, amount) -} - -func (p *Provider) GetStorageCollateral(ctx context.Context) (storagemarket.Balance, error) { - balance, err := p.spn.GetBalance(ctx, p.actor) - - return balance, err -} - -func (p *Provider) ListIncompleteDeals() ([]storagemarket.MinerDeal, error) { - var out []storagemarket.MinerDeal - - var deals []MinerDeal - if err := p.deals.List(&deals); err != nil { - return nil, err - } - - for _, deal := range deals { - out = append(out, storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - DealID: deal.DealID, - SectorID: deal.SectorID, - }) - } - - return out, nil -} - -var _ storagemarket.StorageProvider = &Provider{} diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go deleted file mode 100644 index 54b838a43..000000000 --- a/chain/deals/provider_utils.go +++ /dev/null @@ -1,180 +0,0 @@ -package deals - -import ( - "context" - "runtime" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-statestore" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -func (p *Provider) failDeal(ctx context.Context, id cid.Cid, cerr error) { - if err := p.deals.End(id); err != nil { - log.Warnf("deals.End: %s", err) - } - - if cerr == nil { - _, f, l, _ := runtime.Caller(1) - cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) - } - - log.Warnf("deal %s failed: %s", id, cerr) - - err := p.sendSignedResponse(ctx, &Response{ - State: api.DealFailed, - Message: cerr.Error(), - Proposal: id, - }) - - s, ok := p.conns[id] - if ok { - _ = s.Reset() - delete(p.conns, id) - } - - if err != nil { - log.Warnf("notifying client about deal failure: %s", err) - } -} - -func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { - if err := cborutil.ReadCborRPC(s, &proposal); err != nil { - log.Errorw("failed to read proposal message", "error", err) - return proposal, err - } - - if proposal.DealProposal.ProposerSignature == nil { - return proposal, xerrors.Errorf("incoming deal proposal has no signature") - } - - if err := proposal.DealProposal.Verify(address.Undef); err != nil { - return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) - } - - if proposal.DealProposal.Provider != p.actor { - log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider) - return proposal, err - } - - return -} - -func (p *Provider) sendSignedResponse(ctx context.Context, resp *Response) error { - s, ok := p.conns[resp.Proposal] - if !ok { - return xerrors.New("couldn't send response: not connected") - } - - msg, err := cborutil.Dump(resp) - if err != nil { - return xerrors.Errorf("serializing response: %w", err) - } - - worker, err := p.spn.GetMinerWorker(ctx, p.actor) - if err != nil { - return err - } - - sig, err := p.spn.SignBytes(ctx, worker, msg) - if err != nil { - return xerrors.Errorf("failed to sign response message: %w", err) - } - - signedResponse := &SignedResponse{ - Response: *resp, - Signature: sig, - } - - err = cborutil.WriteCborRPC(s, signedResponse) - if err != nil { - // Assume client disconnected - s.Close() - delete(p.conns, resp.Proposal) - } - return err -} - -func (p *Provider) disconnect(deal MinerDeal) error { - s, ok := p.conns[deal.ProposalCid] - if !ok { - return nil - } - - err := s.Close() - delete(p.conns, deal.ProposalCid) - return err -} - -var _ datatransfer.RequestValidator = &ProviderRequestValidator{} - -// ProviderRequestValidator validates data transfer requests for the provider -// in a storage market -type ProviderRequestValidator struct { - deals *statestore.StateStore -} - -// NewProviderRequestValidator returns a new client request validator for the -// given datastore -func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *ProviderRequestValidator { - return &ProviderRequestValidator{ - deals: deals, - } -} - -// ValidatePush validates a push request received from the peer that will send data -// Will succeed only if: -// - voucher has correct type -// - voucher references an active deal -// - referenced deal matches the client -// - referenced deal matches the given base CID -// - referenced deal is in an acceptable state -func (m *ProviderRequestValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - dealVoucher, ok := voucher.(*StorageDataTransferVoucher) - if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) - } - - var deal MinerDeal - err := m.deals.Get(dealVoucher.Proposal, &deal) - if err != nil { - return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) - } - if deal.Client != sender { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer) - } - - if !deal.Ref.Equals(baseCid) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) - } - for _, state := range DataTransferStates { - if deal.State == state { - return nil - } - } - return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState) -} - -// ValidatePull validates a pull request received from the peer that will receive data. -// Will always error because providers should not accept pull requests from a client -// in a storage deal (i.e. send data to client). -func (m *ProviderRequestValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - return ErrNoPullAccepted -} diff --git a/chain/deals/request_validation_test.go b/chain/deals/request_validation_test.go deleted file mode 100644 index 7607eacb5..000000000 --- a/chain/deals/request_validation_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package deals_test - -import ( - "fmt" - "math/rand" - "testing" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - dss "github.com/ipfs/go-datastore/sync" - blocksutil "github.com/ipfs/go-ipfs-blocksutil" - "github.com/libp2p/go-libp2p-core/peer" - xerrors "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/deals" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -var blockGenerator = blocksutil.NewBlockGenerator() - -type wrongDTType struct { -} - -func (wrongDTType) ToBytes() ([]byte, error) { - return []byte{}, nil -} - -func (wrongDTType) FromBytes([]byte) error { - return fmt.Errorf("not implemented") -} - -func (wrongDTType) Type() string { - return "WrongDTTYPE" -} - -func uniqueStorageDealProposal() (actors.StorageDealProposal, error) { - clientAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return actors.StorageDealProposal{}, err - } - providerAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return actors.StorageDealProposal{}, err - } - return actors.StorageDealProposal{ - PieceRef: blockGenerator.Next().Cid().Bytes(), - Client: clientAddr, - Provider: providerAddr, - ProposerSignature: &types.Signature{ - Data: []byte("foo bar cat dog"), - Type: types.KTBLS, - }, - }, nil -} - -func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, error) { - newProposal, err := uniqueStorageDealProposal() - if err != nil { - return deals.ClientDeal{}, err - } - proposalNd, err := cborutil.AsIpld(&newProposal) - if err != nil { - return deals.ClientDeal{}, err - } - minerAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return deals.ClientDeal{}, err - } - - return deals.ClientDeal{ - ClientDeal: storagemarket.ClientDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - PayloadCid: blockGenerator.Next().Cid(), - Miner: minerID, - MinerWorker: minerAddr, - State: state, - }, - }, nil -} - -func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error) { - newProposal, err := uniqueStorageDealProposal() - if err != nil { - return deals.MinerDeal{}, err - } - proposalNd, err := cborutil.AsIpld(&newProposal) - if err != nil { - return deals.MinerDeal{}, err - } - ref := blockGenerator.Next().Cid() - - return deals.MinerDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - Client: clientID, - State: state, - Ref: ref, - }, nil -} - -func TestClientRequestValidation(t *testing.T) { - ds := dss.MutexWrap(datastore.NewMapDatastore()) - state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) - - crv := deals.NewClientRequestValidator(state) - minerID := peer.ID("fakepeerid") - block := blockGenerator.Next() - t.Run("ValidatePush fails", func(t *testing.T) { - if !xerrors.Is(crv.ValidatePush(minerID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPushAccepted) { - t.Fatal("Push should fail for the client request validator for storage deals") - } - }) - t.Run("ValidatePull fails deal not found", func(t *testing.T) { - proposal, err := uniqueStorageDealProposal() - if err != nil { - t.Fatal("error creating proposal") - } - proposalNd, err := cborutil.AsIpld(&proposal) - if err != nil { - t.Fatal("error serializing proposal") - } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) { - t.Fatal("Pull should fail if there is no deal stored") - } - }) - t.Run("ValidatePull fails wrong client", func(t *testing.T) { - otherMiner := peer.ID("otherminer") - clientDeal, err := newClientDeal(otherMiner, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - payloadCid := clientDeal.PayloadCid - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrWrongPeer) { - t.Fatal("Pull should fail if miner address is incorrect") - } - }) - t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) { - t.Fatal("Pull should fail if piece ref is incorrect") - } - }) - t.Run("ValidatePull fails wrong deal state", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealComplete) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - payloadCid := clientDeal.PayloadCid - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrInacceptableDealState) { - t.Fatal("Pull should fail if deal is in a state that cannot be data transferred") - } - }) - t.Run("ValidatePull succeeds", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - payloadCid := clientDeal.PayloadCid - if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil) != nil { - t.Fatal("Pull should should succeed when all parameters are correct") - } - }) -} - -func TestProviderRequestValidation(t *testing.T) { - ds := dss.MutexWrap(datastore.NewMapDatastore()) - state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) - - mrv := deals.NewProviderRequestValidator(state) - clientID := peer.ID("fakepeerid") - block := blockGenerator.Next() - t.Run("ValidatePull fails", func(t *testing.T) { - if !xerrors.Is(mrv.ValidatePull(clientID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPullAccepted) { - t.Fatal("Pull should fail for the provider request validator for storage deals") - } - }) - - t.Run("ValidatePush fails deal not found", func(t *testing.T) { - proposal, err := uniqueStorageDealProposal() - if err != nil { - t.Fatal("error creating proposal") - } - proposalNd, err := cborutil.AsIpld(&proposal) - if err != nil { - t.Fatal("error serializing proposal") - } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) { - t.Fatal("Push should fail if there is no deal stored") - } - }) - t.Run("ValidatePush fails wrong miner", func(t *testing.T) { - otherClient := peer.ID("otherclient") - minerDeal, err := newMinerDeal(otherClient, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - ref := minerDeal.Ref - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrWrongPeer) { - t.Fatal("Push should fail if miner address is incorrect") - } - }) - t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) { - t.Fatal("Push should fail if piece ref is incorrect") - } - }) - t.Run("ValidatePush fails wrong deal state", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealComplete) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - ref := minerDeal.Ref - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrInacceptableDealState) { - t.Fatal("Push should fail if deal is in a state that cannot be data transferred") - } - }) - t.Run("ValidatePush succeeds", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - ref := minerDeal.Ref - if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil) != nil { - t.Fatal("Push should should succeed when all parameters are correct") - } - }) -} diff --git a/chain/deals/types.go b/chain/deals/types.go deleted file mode 100644 index a35f537ab..000000000 --- a/chain/deals/types.go +++ /dev/null @@ -1,112 +0,0 @@ -package deals - -import ( - "bytes" - "errors" - - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" -) - -var ( - // ErrWrongVoucherType means the voucher was not the correct type can validate against - ErrWrongVoucherType = errors.New("cannot validate voucher type.") - - // ErrNoPushAccepted just means clients do not accept pushes for storage deals - ErrNoPushAccepted = errors.New("client should not receive data for a storage deal.") - - // ErrNoPullAccepted just means providers do not accept pulls for storage deals - ErrNoPullAccepted = errors.New("provider should not send data for a storage deal.") - - // ErrNoDeal means no active deal was found for this vouchers proposal cid - ErrNoDeal = errors.New("no deal found for this proposal.") - - // ErrWrongPeer means that the other peer for this data transfer request does not match - // the other peer for the deal - ErrWrongPeer = errors.New("data Transfer peer id and Deal peer id do not match.") - - // ErrWrongPiece means that the pieceref for this data transfer request does not match - // the one specified in the deal - ErrWrongPiece = errors.New("base CID for deal does not match CID for piece.") - - // ErrInacceptableDealState means the deal for this transfer is not in a deal state - // where transfer can be performed - ErrInacceptableDealState = errors.New("deal is not a in a state where deals are accepted.") - - // DataTransferStates are the states in which it would make sense to actually start a data transfer - DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown} -) - -type Proposal struct { - DealProposal *actors.StorageDealProposal - - Piece cid.Cid // Used for retrieving from the client -} - -type Response struct { - State api.DealState - - // DealProposalRejected - Message string - Proposal cid.Cid - - // DealAccepted - PublishMessage *cid.Cid -} - -// TODO: Do we actually need this to be signed? -type SignedResponse struct { - Response Response - - Signature *types.Signature -} - -func (r *SignedResponse) Verify(addr address.Address) error { - b, err := cborutil.Dump(&r.Response) - if err != nil { - return err - } - - return r.Signature.Verify(addr, b) -} - -type AskRequest struct { - Miner address.Address -} - -type AskResponse struct { - Ask *types.SignedStorageAsk -} - -// StorageDataTransferVoucher is the voucher type for data transfers -// used by the storage market -type StorageDataTransferVoucher struct { - Proposal cid.Cid - DealID uint64 -} - -// ToBytes converts the StorageDataTransferVoucher to raw bytes -func (dv *StorageDataTransferVoucher) ToBytes() ([]byte, error) { - var buf bytes.Buffer - err := dv.MarshalCBOR(&buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// FromBytes converts the StorageDataTransferVoucher to raw bytes -func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error { - r := bytes.NewReader(raw) - return dv.UnmarshalCBOR(r) -} - -// Type is the unique string identifier for a StorageDataTransferVoucher -func (dv *StorageDataTransferVoucher) Type() string { - return "StorageDataTransferVoucher" -} diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index ba547d65e..9704eca09 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/json" "fmt" + "github.com/filecoin-project/lotus/storagemarketadapter" "io/ioutil" "os" "path/filepath" @@ -23,10 +24,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/genesis" @@ -281,15 +283,21 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) + proposal, err := storagemarketadapter.ToSharedStorageDealProposal(§or.Deal) + if err != nil { + return err + } deal := &deals.MinerDeal{ - Proposal: sector.Deal, - ProposalCid: proposalCid, - State: lapi.DealComplete, - Ref: proposalCid, // TODO: This is super wrong, but there - // are no params for CommP CIDs, we can't recover unixfs cid easily, - // and this isn't even used after the deal enters Complete state - DealID: dealID, - SectorID: sector.SectorID, + MinerDeal: storagemarket.MinerDeal{ + Proposal: *proposal, + ProposalCid: proposalCid, + State: lapi.DealComplete, + Ref: proposalCid, // TODO: This is super wrong, but there + // are no params for CommP CIDs, we can't recover unixfs cid easily, + // and this isn't even used after the deal enters Complete state + DealID: dealID, + SectorID: sector.SectorID, + }, } b, err = cborutil.Dump(deal) diff --git a/gen/main.go b/gen/main.go index 0cb569148..7778135ad 100644 --- a/gen/main.go +++ b/gen/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "github.com/filecoin-project/lotus/storagemarket" "os" gen "github.com/whyrusleeping/cbor-gen" @@ -10,7 +9,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/storage" @@ -123,31 +121,6 @@ func main() { os.Exit(1) } - err = gen.WriteTupleEncodersToFile("./storagemarket/cbor_gen.go", "storagemarket", - storagemarket.ClientDeal{}, - storagemarket.MinerDeal{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals", - deals.AskRequest{}, - deals.AskResponse{}, - deals.Proposal{}, - deals.Response{}, - deals.SignedResponse{}, - deals.ClientDealProposal{}, - deals.ClientDeal{}, - deals.MinerDeal{}, - deals.StorageDataTransferVoucher{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - err = gen.WriteMapEncodersToFile("./storage/cbor_gen.go", "storage", storage.SealTicket{}, storage.SealSeed{}, diff --git a/go.mod b/go.mod index 1bcb04ec4..ee48d7fe5 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,6 @@ require ( github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 github.com/ipfs/go-ipfs-blockstore v0.1.1 - github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 diff --git a/lib/sharedutils/converters.go b/lib/sharedutils/converters.go new file mode 100644 index 000000000..58a5b44f9 --- /dev/null +++ b/lib/sharedutils/converters.go @@ -0,0 +1,101 @@ +package sharedutils + +import ( + "bytes" + sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/lotus/chain/types" +) + + +func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { + return types.BigInt{Int: in.Int} +} + +func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { + return sharedamount.TokenAmount{Int: in.Int} +} + +func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/node/builder.go b/node/builder.go index ee7286908..9a656563c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -21,10 +21,11 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/messagepool" @@ -47,7 +48,6 @@ import ( "github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarket" "github.com/filecoin-project/lotus/storagemarketadapter" ) @@ -228,8 +228,8 @@ func Online() Option { Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), Override(new(dtypes.ClientDealStore), modules.NewClientDealStore), Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), - Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator), - Override(new(storagemarket.StorageClient), deals.NewClient), + Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator), + Override(new(storagemarket.StorageClient), modules.StorageClient), Override(new(storagemarket.StorageClientNode), storagemarketadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), Override(RunDealClientKey, modules.RunDealClient), @@ -252,8 +252,8 @@ func Online() Option { Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), - Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator), - Override(new(storagemarket.StorageProvider), deals.NewProvider), + Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator), + Override(new(storagemarket.StorageProvider), modules.StorageProvider), Override(new(storagemarket.StorageProviderNode), storagemarketadapter.NewProviderNodeAdapter), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 09bd5d9ee..70aff2b0c 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -26,15 +26,18 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/retrievaladapter" - "github.com/filecoin-project/lotus/storagemarket" + "github.com/filecoin-project/lotus/storagemarketadapter" ) type API struct { @@ -73,20 +76,16 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad if err != nil { return nil, xerrors.Errorf("failed getting miner worker: %w", err) } - providerInfo := storagemarket.StorageProviderInfo{ - Address: miner, - Worker: mw, - PeerID: pid, - } + providerInfo := storagemarketadapter.NewStorageProviderInfo(miner, mw, 0, pid) result, err := a.SMDealClient.ProposeStorageDeal( ctx, - addr, + sharedutils.ToSharedAddress(addr), &providerInfo, data, storagemarket.Epoch(math.MaxUint64), storagemarket.Epoch(blocksDuration), - storagemarket.TokenAmount(epochPrice), - storagemarket.TokenAmount(storagemarket.EmptyInt)) + sharedutils.ToSharedTokenAmount(epochPrice), + tokenamount.Empty) if err != nil { return nil, xerrors.Errorf("failed to start deal: %w", err) @@ -106,12 +105,12 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { out[k] = api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: v.Proposal.Provider, + Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: v.Proposal.StoragePricePerEpoch, + PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, } } @@ -128,10 +127,10 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, return &api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: v.Proposal.Provider, + Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: v.Proposal.StoragePricePerEpoch, + PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, }, nil } @@ -160,13 +159,13 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe for k, p := range peers { queryResponse, err := a.Retrieval.Query(ctx, p, root.Bytes(), retrievalmarket.QueryParams{}) if err != nil { - out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + out[k] = api.QueryOffer{Err: err.Error(), Miner: retrievaladapter.FromSharedAddress(p.Address), MinerPeerID: p.ID} } else { out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, MinPrice: retrievaladapter.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), - Miner: p.Address, // TODO: check + Miner: retrievaladapter.FromSharedAddress(p.Address), // TODO: check MinerPeerID: p.ID, } } @@ -296,10 +295,10 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path ctx, order.Root.Bytes(), retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), - retrievaladapter.ToSharedTokenAmount(order.Total), + sharedutils.ToSharedTokenAmount(order.Total), order.MinerPeerID, - order.Client, - order.Miner) + retrievaladapter.ToSharedAddress(order.Client), + retrievaladapter.ToSharedAddress(order.Miner)) select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") @@ -323,6 +322,10 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path } func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { - info := storagemarket.StorageProviderInfo{Address: miner, PeerID: p} - return a.SMDealClient.GetAsk(ctx, info) + info := storagemarketadapter.NewStorageProviderInfo(miner, address.Undef, 0, p) + signedAsk, err := a.SMDealClient.GetAsk(ctx, info) + if err != nil { + return nil, err + } + return sharedutils.FromSignedStorageAsk(signedAsk) } diff --git a/node/modules/client.go b/node/modules/client.go index a03ea3691..313672c6b 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -2,13 +2,18 @@ package modules import ( "context" - "github.com/filecoin-project/lotus/retrievaladapter" "path/filepath" "reflect" + "github.com/filecoin-project/lotus/retrievaladapter" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" + + "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/paych" @@ -30,7 +35,8 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-data-transfer/impl/graphsync" - "github.com/filecoin-project/lotus/chain/deals" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" @@ -104,6 +110,14 @@ func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Client return gs } +func NewClientRequestValidator(deals dtypes.ClientDealStore) *storageimpl.ClientRequestValidator { + return storageimpl.NewClientRequestValidator(deals) +} + +func StorageClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) storagemarket.StorageClient { + return storageimpl.NewClient(h, dag, dataTransfer, discovery, deals, scn) +} + // RetrievalClient creates a new retrieval client attached to the client blockstore func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClient { adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi) diff --git a/node/modules/services.go b/node/modules/services.go index bf1985041..e12b9a310 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" - "github.com/filecoin-project/lotus/storagemarket" + "github.com/filecoin-project/go-fil-components/storagemarket" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1197870d5..1ddbb433a 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -27,21 +27,26 @@ import ( dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/go-fil-components/storagemarket" "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarket" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -264,6 +269,14 @@ func SealTicketGen(api api.FullNode) storage.TicketFn { } } +func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *storageimpl.ProviderRequestValidator { + return storageimpl.NewProviderRequestValidator(deals) +} + +func StorageProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { + return storageimpl.NewProvider(ds, dag, dataTransfer, spn) +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider { adapter := retrievaladapter.NewRetrievalProviderNode(sblks, full) diff --git a/retrievaladapter/client.go b/retrievaladapter/client.go index 8f0289921..cd5f819e5 100644 --- a/retrievaladapter/client.go +++ b/retrievaladapter/client.go @@ -2,6 +2,7 @@ package retrievaladapter import ( "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -26,7 +27,7 @@ func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retriev // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // between a client and a miner and insures the client has the given amount of funds available in the channel func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) { - paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, FromSharedTokenAmount(clientFundsAvailable)) + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, sharedutils.FromSharedTokenAmount(clientFundsAvailable)) return paych, err } @@ -41,9 +42,9 @@ func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (ui // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) { - voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, FromSharedTokenAmount(amount), lane) + voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, sharedutils.FromSharedTokenAmount(amount), lane) if err != nil { return nil, err } - return ToSharedSignedVoucher(voucher) + return sharedutils.ToSharedSignedVoucher(voucher) } diff --git a/retrievaladapter/converters.go b/retrievaladapter/converters.go deleted file mode 100644 index 7ea3ffec0..000000000 --- a/retrievaladapter/converters.go +++ /dev/null @@ -1,45 +0,0 @@ -package retrievaladapter - -import ( - "bytes" - sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" - "github.com/filecoin-project/lotus/chain/types" -) - - -func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { - return types.BigInt{Int: in.Int} -} - -func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { - return sharedamount.TokenAmount{Int: in.Int} -} - -func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go index c5b238eda..d2499757b 100644 --- a/retrievaladapter/provider.go +++ b/retrievaladapter/provider.go @@ -2,6 +2,7 @@ package retrievaladapter import ( "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -37,10 +38,10 @@ func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) b } func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) { - localVoucher, err := FromSharedSignedVoucher(voucher) + localVoucher, err := sharedutils.FromSharedSignedVoucher(voucher) if err != nil { return retrievaltoken.FromInt(0), err } - added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, FromSharedTokenAmount(expectedAmount)) - return ToSharedTokenAmount(added), err + added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, sharedutils.FromSharedTokenAmount(expectedAmount)) + return sharedutils.ToSharedTokenAmount(added), err } diff --git a/storagemarket/bigint.go b/storagemarket/bigint.go deleted file mode 100644 index f36e242b0..000000000 --- a/storagemarket/bigint.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copied from lotus until this can be extracted into shared types - -package storagemarket - -import ( - "encoding/json" - "fmt" - "io" - "math/big" - - "github.com/filecoin-project/lotus/build" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/polydawn/refmt/obj/atlas" - - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" -) - -const BigIntMaxSerializedLen = 128 // is this big enough? or too big? - -var TotalFilecoinInt = FromFil(build.TotalFilecoin) - -func init() { - cbor.RegisterCborType(atlas.BuildEntry(BigInt{}).Transform(). - TransformMarshal(atlas.MakeMarshalTransformFunc( - func(i BigInt) ([]byte, error) { - return i.cborBytes(), nil - })). - TransformUnmarshal(atlas.MakeUnmarshalTransformFunc( - func(x []byte) (BigInt, error) { - return fromCborBytes(x) - })). - Complete()) -} - -var EmptyInt = BigInt{} - -type BigInt struct { - *big.Int -} - -func NewInt(i uint64) BigInt { - return BigInt{big.NewInt(0).SetUint64(i)} -} - -func FromFil(i uint64) BigInt { - return BigMul(NewInt(i), NewInt(build.FilecoinPrecision)) -} - -func BigFromBytes(b []byte) BigInt { - i := big.NewInt(0).SetBytes(b) - return BigInt{i} -} - -func BigFromString(s string) (BigInt, error) { - v, ok := big.NewInt(0).SetString(s, 10) - if !ok { - return BigInt{}, fmt.Errorf("failed to parse string as a big int") - } - - return BigInt{v}, nil -} - -func BigMul(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Mul(a.Int, b.Int)} -} - -func BigDiv(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Div(a.Int, b.Int)} -} - -func BigMod(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Mod(a.Int, b.Int)} -} - -func BigAdd(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Add(a.Int, b.Int)} -} - -func BigSub(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Sub(a.Int, b.Int)} -} - -func BigCmp(a, b BigInt) int { - return a.Int.Cmp(b.Int) -} - -func (bi BigInt) Nil() bool { - return bi.Int == nil -} - -// LessThan returns true if bi < o -func (bi BigInt) LessThan(o BigInt) bool { - return BigCmp(bi, o) < 0 -} - -// GreaterThan returns true if bi > o -func (bi BigInt) GreaterThan(o BigInt) bool { - return BigCmp(bi, o) > 0 -} - -// Equals returns true if bi == o -func (bi BigInt) Equals(o BigInt) bool { - return BigCmp(bi, o) == 0 -} - -func (bi *BigInt) MarshalJSON() ([]byte, error) { - return json.Marshal(bi.String()) -} - -func (bi *BigInt) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - - i, ok := big.NewInt(0).SetString(s, 10) - if !ok { - if string(s) == "" { - return nil - } - return xerrors.Errorf("failed to parse bigint string: '%s'", string(b)) - } - - bi.Int = i - return nil -} - -func (bi *BigInt) Scan(value interface{}) error { - switch value := value.(type) { - case string: - i, ok := big.NewInt(0).SetString(value, 10) - if !ok { - if value == "" { - return nil - } - return xerrors.Errorf("failed to parse bigint string: '%s'", value) - } - - bi.Int = i - - return nil - case int64: - bi.Int = big.NewInt(value) - return nil - default: - return xerrors.Errorf("non-string types unsupported: %T", value) - } -} - -func (bi *BigInt) cborBytes() []byte { - if bi.Int == nil { - return []byte{} - } - - switch { - case bi.Sign() > 0: - return append([]byte{0}, bi.Bytes()...) - case bi.Sign() < 0: - return append([]byte{1}, bi.Bytes()...) - default: // bi.Sign() == 0: - return []byte{} - } -} - -func fromCborBytes(buf []byte) (BigInt, error) { - if len(buf) == 0 { - return NewInt(0), nil - } - - var negative bool - switch buf[0] { - case 0: - negative = false - case 1: - negative = true - default: - return EmptyInt, fmt.Errorf("big int prefix should be either 0 or 1, got %d", buf[0]) - } - - i := big.NewInt(0).SetBytes(buf[1:]) - if negative { - i.Neg(i) - } - - return BigInt{i}, nil -} - -func (bi *BigInt) MarshalCBOR(w io.Writer) error { - if bi.Int == nil { - zero := NewInt(0) - return zero.MarshalCBOR(w) - } - - enc := bi.cborBytes() - - header := cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(enc))) - if _, err := w.Write(header); err != nil { - return err - } - - if _, err := w.Write(enc); err != nil { - return err - } - - return nil -} - -func (bi *BigInt) UnmarshalCBOR(br io.Reader) error { - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - - if maj != cbg.MajByteString { - return fmt.Errorf("cbor input for fil big int was not a byte string (%x)", maj) - } - - if extra == 0 { - bi.Int = big.NewInt(0) - return nil - } - - if extra > BigIntMaxSerializedLen { - return fmt.Errorf("big integer byte array too long") - } - - buf := make([]byte, extra) - if _, err := io.ReadFull(br, buf); err != nil { - return err - } - - i, err := fromCborBytes(buf) - if err != nil { - return err - } - - *bi = i - - return nil -} diff --git a/storagemarket/cbor_gen.go b/storagemarket/cbor_gen.go deleted file mode 100644 index e4a9efcfa..000000000 --- a/storagemarket/cbor_gen.go +++ /dev/null @@ -1,352 +0,0 @@ -package storagemarket - -import ( - "fmt" - "io" - - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *ClientDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{136}); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Miner (peer.ID) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Miner)); err != nil { - return err - } - - // t.MinerWorker (address.Address) (struct) - if err := t.MinerWorker.MarshalCBOR(w); err != nil { - return err - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.PayloadCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.PayloadCid); err != nil { - return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err) - } - - // t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } - } - - return nil -} - -func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 8 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Miner (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Miner = peer.ID(sval) - } - // t.MinerWorker (address.Address) (struct) - - { - - if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.PayloadCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err) - } - - t.PayloadCid = c - - } - // t.PublishMessage (cid.Cid) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) - } - - t.PublishMessage = &c - } - - } - return nil -} - -func (t *MinerDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{136}); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.Miner (peer.ID) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Miner)); err != nil { - return err - } - - // t.Client (peer.ID) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Client)); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.SectorID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { - return err - } - return nil -} - -func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 8 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Miner (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Miner = peer.ID(sval) - } - // t.Client (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Client = peer.ID(sval) - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.SectorID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = uint64(extra) - return nil -} diff --git a/storagemarket/types.go b/storagemarket/types.go deleted file mode 100644 index 1497063c4..000000000 --- a/storagemarket/types.go +++ /dev/null @@ -1,199 +0,0 @@ -package storagemarket - -import ( - "context" - - "github.com/ipfs/go-cid" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/peer" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" -) - -const DealProtocolID = "/fil/storage/mk/1.0.1" -const AskProtocolID = "/fil/storage/ask/1.0.1" - -// type shims - used during migration into separate module -type Balance = actors.StorageParticipantBalance -type DealID uint64 -type Signature = types.Signature -type StorageDeal = actors.OnChainDeal -type StorageAsk = types.SignedStorageAsk -type StateKey = *types.TipSet -type Epoch uint64 -type TokenAmount BigInt - -// Duplicated from deals package for now -type MinerDeal struct { - ProposalCid cid.Cid - Proposal actors.StorageDealProposal - Miner peer.ID - Client peer.ID - State api.DealState - - Ref cid.Cid - - DealID uint64 - SectorID uint64 // Set when sm >= DealStaged -} - -type ClientDeal struct { - ProposalCid cid.Cid - Proposal actors.StorageDealProposal - State api.DealState - Miner peer.ID - MinerWorker address.Address - DealID uint64 - PayloadCid cid.Cid - PublishMessage *cid.Cid -} - -// The interface provided for storage providers -type StorageProvider interface { - Run(ctx context.Context, host host.Host) - - Stop() - - AddAsk(price TokenAmount, ttlsecs int64) error - - // ListAsks lists current asks - ListAsks(addr address.Address) []*StorageAsk - - // ListDeals lists on-chain deals associated with this provider - ListDeals(ctx context.Context) ([]StorageDeal, error) - - // ListIncompleteDeals lists deals that are in progress or rejected - ListIncompleteDeals() ([]MinerDeal, error) - - // AddStorageCollateral adds storage collateral - AddStorageCollateral(ctx context.Context, amount TokenAmount) error - - // GetStorageCollateral returns the current collateral balance - GetStorageCollateral(ctx context.Context) (Balance, error) -} - -// Node dependencies for a StorageProvider -type StorageProviderNode interface { - MostRecentStateId(ctx context.Context) (StateKey, error) - - // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. - AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - // Ensures that a storage market participant has a certain amount of available funds - EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. - GetBalance(ctx context.Context, addr address.Address) (Balance, error) - - // Publishes deal on chain - PublishDeals(ctx context.Context, deal MinerDeal) (DealID, cid.Cid, error) - - // ListProviderDeals lists all deals associated with a storage provider - ListProviderDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) - - // Called when a deal is complete and on chain, and data has been transferred and is ready to be added to a sector - // returns sector id - OnDealComplete(ctx context.Context, deal MinerDeal, piecePath string) (uint64, error) - - // returns the worker address associated with a miner - GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) - - // Signs bytes - SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) -} - -type DealSectorCommittedCallback func(error) - -// Node dependencies for a StorageClient -type StorageClientNode interface { - MostRecentStateId(ctx context.Context) (StateKey, error) - - // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. - AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. - GetBalance(ctx context.Context, addr address.Address) (Balance, error) - - //// ListClientDeals lists all on-chain deals associated with a storage client - ListClientDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) - - // GetProviderInfo returns information about a single storage provider - //GetProviderInfo(stateId StateID, addr Address) *StorageProviderInfo - - // GetStorageProviders returns information about known miners - ListStorageProviders(ctx context.Context) ([]*StorageProviderInfo, error) - - // Subscribes to storage market actor state changes for a given address. - // TODO: Should there be a timeout option for this? In the case that we are waiting for funds to be deposited and it never happens? - //SubscribeStorageMarketEvents(addr Address, handler StorageMarketEventHandler) (SubID, error) - - // Cancels a subscription - //UnsubscribeStorageMarketEvents(subId SubID) - ValidatePublishedDeal(ctx context.Context, deal ClientDeal) (uint64, error) - - // SignProposal signs a proposal - SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error - - GetDefaultWalletAddress(ctx context.Context) (address.Address, error) - - OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb DealSectorCommittedCallback) error - - ValidateAskSignature(ask *StorageAsk) error -} - -type StorageClientProofs interface { - //GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (CommP, error) -} - -// Closely follows the MinerInfo struct in the spec -type StorageProviderInfo struct { - Address address.Address // actor address - Owner address.Address - Worker address.Address // signs messages - SectorSize uint64 - PeerID peer.ID - // probably more like how much storage power, available collateral etc -} - -type ProposeStorageDealResult struct { - ProposalCid cid.Cid -} - -// The interface provided by the module to the outside world for storage clients. -type StorageClient interface { - Run(ctx context.Context) - - Stop() - - // ListProviders queries chain state and returns active storage providers - ListProviders(ctx context.Context) (<-chan StorageProviderInfo, error) - - // ListDeals lists on-chain deals associated with this provider - ListDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) - - // ListInProgressDeals lists deals that are in progress or rejected - ListInProgressDeals(ctx context.Context) ([]ClientDeal, error) - - // ListInProgressDeals lists deals that are in progress or rejected - GetInProgressDeal(ctx context.Context, cid cid.Cid) (ClientDeal, error) - - // GetAsk returns the current ask for a storage provider - GetAsk(ctx context.Context, info StorageProviderInfo) (*StorageAsk, error) - - //// FindStorageOffers lists providers and queries them to find offers that satisfy some criteria based on price, duration, etc. - //FindStorageOffers(criteria AskCriteria, limit uint) []*StorageOffer - - // ProposeStorageDeal initiates deal negotiation with a Storage Provider - ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, payloadCid cid.Cid, proposalExpiration Epoch, duration Epoch, price TokenAmount, collateral TokenAmount) (*ProposeStorageDealResult, error) - - // GetPaymentEscrow returns the current funds available for deal payment - GetPaymentEscrow(ctx context.Context, addr address.Address) (Balance, error) - - // AddStorageCollateral adds storage collateral - AddPaymentEscrow(ctx context.Context, addr address.Address, amount TokenAmount) error -} diff --git a/storagemarketadapter/client_adapter.go b/storagemarketadapter/client_adapter.go index 9e23771f5..90e764da9 100644 --- a/storagemarketadapter/client_adapter.go +++ b/storagemarketadapter/client_adapter.go @@ -5,11 +5,14 @@ package storagemarketadapter import ( "bytes" "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -19,7 +22,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" - "github.com/filecoin-project/lotus/storagemarket" ) type ClientNodeAdapter struct { @@ -79,13 +81,8 @@ func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storag if err != nil { return nil, err } - - out = append(out, &storagemarket.StorageProviderInfo{ - Address: addr, - Worker: workerAddr, - SectorSize: sectorSize, - PeerID: peerId, - }) + storageProviderInfo := NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId) + out = append(out, &storageProviderInfo) } return out, nil @@ -97,11 +94,12 @@ func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Ad return nil, err } - var out []actors.OnChainDeal + var out []storagemarket.StorageDeal for _, deal := range allDeals { - if deal.Client == addr { - out = append(out, deal) + storageDeal := FromOnChainDeal(deal) + if storageDeal.Client == addr { + out = append(out, storageDeal) } } @@ -113,12 +111,12 @@ func (n *ClientNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarke } // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. -func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { +func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: addr, - Value: types.BigInt(amount), + From: sharedutils.FromSharedAddress(addr), + Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -139,17 +137,17 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, return nil } -func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { - return n.fm.EnsureAvailable(ctx, addr, types.BigInt(amount)) +func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { + return n.fm.EnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amount)) } func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, addr, nil) + bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) if err != nil { return storagemarket.Balance{}, err } - return bal, nil + return ToSharedBalance(bal), nil } // ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal @@ -162,7 +160,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("getting deal pubsish message: %w", err) } - pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) + pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, sharedutils.FromSharedAddress(deal.Proposal.Provider)) if err != nil { return 0, xerrors.Errorf("getting miner worker failed: %w", err) } @@ -270,7 +268,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } matchEvent := func(msg *types.Message) (bool, error) { - if msg.To != provider { + if msg.To != sharedutils.FromSharedAddress(provider) { return false, nil } @@ -301,18 +299,23 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider return nil } -func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error { - return api.SignWith(ctx, n.Wallet.Sign, signer, proposal) +func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error { + localProposal, err := FromSharedStorageDealProposal(proposal) + if err != nil { + return err + } + return api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) } func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { - return n.Wallet.GetDefault() + addr, err := n.Wallet.GetDefault() + return sharedutils.ToSharedAddress(addr), err } -func (n *ClientNodeAdapter) ValidateAskSignature(ask *types.SignedStorageAsk) error { +func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error { tss := n.cs.GetHeaviestTipSet().ParentState() - w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner) + w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, sharedutils.FromSharedAddress(ask.Ask.Miner)) if err != nil { return xerrors.Errorf("failed to get worker for miner in ask", err) } @@ -322,7 +325,7 @@ func (n *ClientNodeAdapter) ValidateAskSignature(ask *types.SignedStorageAsk) er return xerrors.Errorf("failed to re-serialize ask") } - return ask.Signature.Verify(w, sigb) + return ask.Signature.Verify(sharedutils.ToSharedAddress(w), sigb) } var _ storagemarket.StorageClientNode = &ClientNodeAdapter{} diff --git a/storagemarketadapter/provider_adapter.go b/storagemarketadapter/provider_adapter.go index 0691bee36..0db1fde5e 100644 --- a/storagemarketadapter/provider_adapter.go +++ b/storagemarketadapter/provider_adapter.go @@ -6,19 +6,23 @@ import ( "bytes" "context" + "github.com/filecoin-project/lotus/lib/sharedutils" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarket" ) var log = logging.Logger("provideradapter") @@ -43,13 +47,17 @@ func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBloc func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { log.Info("publishing deal") - worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil) + worker, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(deal.Proposal.Provider), nil) if err != nil { return 0, cid.Undef, err } + localProposal, err := FromSharedStorageDealProposal(&deal.Proposal) + if err != nil { + return 0, cid.Undef, err + } params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ - Deals: []actors.StorageDealProposal{deal.Proposal}, + Deals: []actors.StorageDealProposal{*localProposal}, }) if err != nil { @@ -124,17 +132,18 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema return sectorID, nil } -func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) { +func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) { allDeals, err := n.StateMarketDeals(ctx, nil) if err != nil { return nil, err } - var out []actors.OnChainDeal + var out []storagemarket.StorageDeal for _, deal := range allDeals { - if deal.Provider == addr { - out = append(out, deal) + sharedDeal := FromOnChainDeal(deal) + if sharedDeal.Provider == addr { + out = append(out, sharedDeal) } } @@ -142,15 +151,20 @@ func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr addres } func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { - return n.StateMinerWorker(ctx, miner, nil) + addr, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(miner), nil) + return sharedutils.ToSharedAddress(addr), err } -func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) { - return n.WalletSign(ctx, signer, b) +func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) { + localSignature, err := n.WalletSign(ctx, sharedutils.FromSharedAddress(signer), b) + if err != nil { + return nil, err + } + return sharedutils.ToSharedSignature(localSignature) } -func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt storagemarket.TokenAmount) error { - return n.MarketEnsureAvailable(ctx, addr, types.BigInt(amt)) +func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error { + return n.MarketEnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amt)) } func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { @@ -158,12 +172,12 @@ func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemar } // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. -func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { +func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: addr, - Value: types.BigInt(amount), + From: sharedutils.FromSharedAddress(addr), + Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -185,12 +199,12 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address } func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, addr, nil) + bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) if err != nil { return storagemarket.Balance{}, err } - return bal, nil + return ToSharedBalance(bal), nil } var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{} diff --git a/storagemarketadapter/utilities.go b/storagemarketadapter/utilities.go new file mode 100644 index 000000000..9b815180f --- /dev/null +++ b/storagemarketadapter/utilities.go @@ -0,0 +1,79 @@ +package storagemarketadapter + +import ( + "bytes" + + "github.com/filecoin-project/go-fil-components/storagemarket" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/lib/sharedutils" + peer "github.com/libp2p/go-libp2p-peer" +) + +func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { + return storagemarket.StorageProviderInfo{ + Address: sharedutils.ToSharedAddress(address), + Worker: sharedutils.ToSharedAddress(miner), + SectorSize: sectorSize, + PeerID: peer, + } +} + +func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { + return storagemarket.StorageDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: sharedutils.ToSharedAddress(deal.Client), + Provider: sharedutils.ToSharedAddress(deal.Provider), + StoragePricePerEpoch: sharedutils.ToSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: sharedutils.ToSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { + return actors.OnChainDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: sharedutils.FromSharedAddress(deal.Client), + Provider: sharedutils.FromSharedAddress(deal.Provider), + StoragePricePerEpoch: sharedutils.FromSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: sharedutils.FromSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance { + return storagemarket.Balance{ + Locked: sharedutils.ToSharedTokenAmount(balance.Locked), + Available: sharedutils.ToSharedTokenAmount(balance.Available), + } +} + +func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out storagemarket.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out actors.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} From 353f93fddf1ad95ab4170a9a5324028495d93dfe Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 17 Dec 2019 09:44:26 -0800 Subject: [PATCH 42/61] fix(storagemarketadapter): minor fix for test --- storagemarketadapter/client_adapter.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/storagemarketadapter/client_adapter.go b/storagemarketadapter/client_adapter.go index 90e764da9..8984dd882 100644 --- a/storagemarketadapter/client_adapter.go +++ b/storagemarketadapter/client_adapter.go @@ -304,7 +304,16 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add if err != nil { return err } - return api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) + err = api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) + if err != nil { + return err + } + signature, err := sharedutils.ToSharedSignature(localProposal.ProposerSignature) + if err != nil { + return err + } + proposal.ProposerSignature = signature + return nil } func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { From 83605dfd95b1d9930ef1c026ee2e3bcbca5497fc Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 10 Jan 2020 09:57:58 -0800 Subject: [PATCH 43/61] check for duplicate candidates in block headers --- chain/sync.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chain/sync.go b/chain/sync.go index 16fb6278b..e5a79133c 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -545,7 +545,13 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("no candidates") } + wins := make(map[uint64]bool) for _, t := range h.EPostProof.Candidates { + if wins[t.ChallengeIndex] { + return xerrors.Errorf("block had duplicate epost candidates") + } + wins[t.ChallengeIndex] = true + if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) { return xerrors.Errorf("miner created a block but was not a winner") } From 56653eeca6d4619ddaaf4cf111b2aeb05144da8d Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 10 Jan 2020 10:01:48 -0800 Subject: [PATCH 44/61] fix(storagemarket): fix compile errors --- go.sum | 1 + node/impl/client/client.go | 17 ++++++++--------- node/modules/services.go | 2 +- node/modules/storageminer.go | 6 +----- storagemarketadapter/client_adapter.go | 22 ++++++++++++---------- storagemarketadapter/provider_adapter.go | 14 +++++++------- storagemarketadapter/utilities.go | 17 +++++++++-------- 7 files changed, 39 insertions(+), 40 deletions(-) diff --git a/go.sum b/go.sum index 0e14ed7b1..b6e36c0bd 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,7 @@ github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= +github.com/filecoin-project/go-fil-markets v0.0.0-20200110172620-7ac5a8af146a h1:mLz0GBjx8nlztlo2sZ6w1JbuyN7EX7zoCYQBblrgoBM= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 70aff2b0c..12eaf117c 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -36,7 +36,6 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storagemarketadapter" ) @@ -79,7 +78,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad providerInfo := storagemarketadapter.NewStorageProviderInfo(miner, mw, 0, pid) result, err := a.SMDealClient.ProposeStorageDeal( ctx, - sharedutils.ToSharedAddress(addr), + addr, &providerInfo, data, storagemarket.Epoch(math.MaxUint64), @@ -105,7 +104,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { out[k] = api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), + Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, @@ -127,7 +126,7 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, return &api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), + Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), @@ -159,13 +158,13 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe for k, p := range peers { queryResponse, err := a.Retrieval.Query(ctx, p, root.Bytes(), retrievalmarket.QueryParams{}) if err != nil { - out[k] = api.QueryOffer{Err: err.Error(), Miner: retrievaladapter.FromSharedAddress(p.Address), MinerPeerID: p.ID} + out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } else { out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, - MinPrice: retrievaladapter.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), - Miner: retrievaladapter.FromSharedAddress(p.Address), // TODO: check + MinPrice: sharedutils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), + Miner: p.Address, // TODO: check MinerPeerID: p.ID, } } @@ -297,8 +296,8 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), sharedutils.ToSharedTokenAmount(order.Total), order.MinerPeerID, - retrievaladapter.ToSharedAddress(order.Client), - retrievaladapter.ToSharedAddress(order.Miner)) + order.Client, + order.Miner) select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") diff --git a/node/modules/services.go b/node/modules/services.go index e12b9a310..33f2733b5 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/messagepool" @@ -18,7 +19,6 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" - "github.com/filecoin-project/go-fil-components/storagemarket" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1ddbb433a..4909a8e67 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -29,21 +29,17 @@ import ( retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/address" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/go-fil-components/storagemarket" "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" diff --git a/storagemarketadapter/client_adapter.go b/storagemarketadapter/client_adapter.go index 8984dd882..d517c726b 100644 --- a/storagemarketadapter/client_adapter.go +++ b/storagemarketadapter/client_adapter.go @@ -5,12 +5,14 @@ package storagemarketadapter import ( "bytes" "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" @@ -115,7 +117,7 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: sharedutils.FromSharedAddress(addr), + From: addr, Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), @@ -138,11 +140,11 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, } func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { - return n.fm.EnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amount)) + return n.fm.EnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amount)) } func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) + bal, err := n.StateMarketBalance(ctx, addr, nil) if err != nil { return storagemarket.Balance{}, err } @@ -160,7 +162,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("getting deal pubsish message: %w", err) } - pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, sharedutils.FromSharedAddress(deal.Proposal.Provider)) + pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) if err != nil { return 0, xerrors.Errorf("getting miner worker failed: %w", err) } @@ -268,7 +270,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } matchEvent := func(msg *types.Message) (bool, error) { - if msg.To != sharedutils.FromSharedAddress(provider) { + if msg.To != provider { return false, nil } @@ -304,7 +306,7 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add if err != nil { return err } - err = api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) + err = api.SignWith(ctx, n.Wallet.Sign, signer, localProposal) if err != nil { return err } @@ -318,13 +320,13 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { addr, err := n.Wallet.GetDefault() - return sharedutils.ToSharedAddress(addr), err + return addr, err } func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error { tss := n.cs.GetHeaviestTipSet().ParentState() - w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, sharedutils.FromSharedAddress(ask.Ask.Miner)) + w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner) if err != nil { return xerrors.Errorf("failed to get worker for miner in ask", err) } @@ -334,7 +336,7 @@ func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageA return xerrors.Errorf("failed to re-serialize ask") } - return ask.Signature.Verify(sharedutils.ToSharedAddress(w), sigb) + return ask.Signature.Verify(w, sigb) } var _ storagemarket.StorageClientNode = &ClientNodeAdapter{} diff --git a/storagemarketadapter/provider_adapter.go b/storagemarketadapter/provider_adapter.go index 0db1fde5e..a0df86096 100644 --- a/storagemarketadapter/provider_adapter.go +++ b/storagemarketadapter/provider_adapter.go @@ -47,7 +47,7 @@ func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBloc func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { log.Info("publishing deal") - worker, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(deal.Proposal.Provider), nil) + worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil) if err != nil { return 0, cid.Undef, err } @@ -151,12 +151,12 @@ func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr addres } func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { - addr, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(miner), nil) - return sharedutils.ToSharedAddress(addr), err + addr, err := n.StateMinerWorker(ctx, miner, nil) + return addr, err } func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) { - localSignature, err := n.WalletSign(ctx, sharedutils.FromSharedAddress(signer), b) + localSignature, err := n.WalletSign(ctx, signer, b) if err != nil { return nil, err } @@ -164,7 +164,7 @@ func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Addr } func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error { - return n.MarketEnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amt)) + return n.MarketEnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amt)) } func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { @@ -176,7 +176,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: sharedutils.FromSharedAddress(addr), + From: addr, Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), @@ -199,7 +199,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address } func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) + bal, err := n.StateMarketBalance(ctx, addr, nil) if err != nil { return storagemarket.Balance{}, err } diff --git a/storagemarketadapter/utilities.go b/storagemarketadapter/utilities.go index 9b815180f..2922f9d1a 100644 --- a/storagemarketadapter/utilities.go +++ b/storagemarketadapter/utilities.go @@ -3,17 +3,18 @@ package storagemarketadapter import ( "bytes" - "github.com/filecoin-project/go-fil-components/storagemarket" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sharedutils" peer "github.com/libp2p/go-libp2p-peer" ) func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { return storagemarket.StorageProviderInfo{ - Address: sharedutils.ToSharedAddress(address), - Worker: sharedutils.ToSharedAddress(miner), + Address: address, + Worker: miner, SectorSize: sectorSize, PeerID: peer, } @@ -23,8 +24,8 @@ func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { return storagemarket.StorageDeal{ PieceRef: deal.PieceRef, PieceSize: deal.PieceSize, - Client: sharedutils.ToSharedAddress(deal.Client), - Provider: sharedutils.ToSharedAddress(deal.Provider), + Client: deal.Client, + Provider: deal.Provider, StoragePricePerEpoch: sharedutils.ToSharedTokenAmount(deal.StoragePricePerEpoch), StorageCollateral: sharedutils.ToSharedTokenAmount(deal.StorageCollateral), ActivationEpoch: deal.ActivationEpoch, @@ -35,8 +36,8 @@ func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { return actors.OnChainDeal{ PieceRef: deal.PieceRef, PieceSize: deal.PieceSize, - Client: sharedutils.FromSharedAddress(deal.Client), - Provider: sharedutils.FromSharedAddress(deal.Provider), + Client: deal.Client, + Provider: deal.Provider, StoragePricePerEpoch: sharedutils.FromSharedTokenAmount(deal.StoragePricePerEpoch), StorageCollateral: sharedutils.FromSharedTokenAmount(deal.StorageCollateral), ActivationEpoch: deal.ActivationEpoch, From 0ce1bf706b817f25d9673f45ff3c5f41211ed46a Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 10 Jan 2020 10:21:46 -0800 Subject: [PATCH 45/61] refactor(markets): reorg file structure --- cmd/lotus-storage-miner/init.go | 8 +- go.mod | 1 - go.sum | 2 +- lib/sharedutils/converters.go | 101 ---------- .../retrievaladapter}/client.go | 9 +- .../retrievaladapter}/provider.go | 13 +- .../storageadapter/client.go | 19 +- .../storageadapter/provider.go | 17 +- markets/utils/converters.go | 174 ++++++++++++++++++ node/builder.go | 6 +- node/impl/client/client.go | 19 +- node/modules/client.go | 2 +- node/modules/storageminer.go | 2 +- storagemarketadapter/utilities.go | 80 -------- 14 files changed, 221 insertions(+), 232 deletions(-) delete mode 100644 lib/sharedutils/converters.go rename {retrievaladapter => markets/retrievaladapter}/client.go (90%) rename {retrievaladapter => markets/retrievaladapter}/provider.go (86%) rename storagemarketadapter/client_adapter.go => markets/storageadapter/client.go (94%) rename storagemarketadapter/provider_adapter.go => markets/storageadapter/provider.go (93%) create mode 100644 markets/utils/converters.go delete mode 100644 storagemarketadapter/utilities.go diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9704eca09..4aefb004e 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -6,7 +6,6 @@ import ( "crypto/rand" "encoding/json" "fmt" - "github.com/filecoin-project/lotus/storagemarketadapter" "io/ioutil" "os" "path/filepath" @@ -22,16 +21,17 @@ import ( "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-sectorbuilder" + cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-fil-markets/storagemarket" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/go-sectorbuilder" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -283,7 +283,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) - proposal, err := storagemarketadapter.ToSharedStorageDealProposal(§or.Deal) + proposal, err := utils.ToSharedStorageDealProposal(§or.Deal) if err != nil { return err } diff --git a/go.mod b/go.mod index ee48d7fe5..b2a08de0d 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,6 @@ require ( github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb - github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 github.com/libp2p/go-libp2p v0.4.2 github.com/libp2p/go-libp2p-circuit v0.1.4 github.com/libp2p/go-libp2p-connmgr v0.1.0 diff --git a/go.sum b/go.sum index b6e36c0bd..a15c42b49 100644 --- a/go.sum +++ b/go.sum @@ -113,7 +113,6 @@ github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= -github.com/filecoin-project/go-fil-markets v0.0.0-20200110172620-7ac5a8af146a h1:mLz0GBjx8nlztlo2sZ6w1JbuyN7EX7zoCYQBblrgoBM= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -700,6 +699,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/lib/sharedutils/converters.go b/lib/sharedutils/converters.go deleted file mode 100644 index 58a5b44f9..000000000 --- a/lib/sharedutils/converters.go +++ /dev/null @@ -1,101 +0,0 @@ -package sharedutils - -import ( - "bytes" - sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" - "github.com/filecoin-project/lotus/chain/types" -) - - -func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { - return types.BigInt{Int: in.Int} -} - -func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { - return sharedamount.TokenAmount{Int: in.Int} -} - -func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.Signature - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.Signature - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.SignedStorageAsk - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.SignedStorageAsk - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} diff --git a/retrievaladapter/client.go b/markets/retrievaladapter/client.go similarity index 90% rename from retrievaladapter/client.go rename to markets/retrievaladapter/client.go index cd5f819e5..676398014 100644 --- a/retrievaladapter/client.go +++ b/markets/retrievaladapter/client.go @@ -2,13 +2,12 @@ package retrievaladapter import ( "context" - "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" - + "github.com/filecoin-project/lotus/markets/utils" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" ) @@ -27,7 +26,7 @@ func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retriev // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // between a client and a miner and insures the client has the given amount of funds available in the channel func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) { - paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, sharedutils.FromSharedTokenAmount(clientFundsAvailable)) + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, utils.FromSharedTokenAmount(clientFundsAvailable)) return paych, err } @@ -42,9 +41,9 @@ func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (ui // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) { - voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, sharedutils.FromSharedTokenAmount(amount), lane) + voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, utils.FromSharedTokenAmount(amount), lane) if err != nil { return nil, err } - return sharedutils.ToSharedSignedVoucher(voucher) + return utils.ToSharedSignedVoucher(voucher) } diff --git a/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go similarity index 86% rename from retrievaladapter/provider.go rename to markets/retrievaladapter/provider.go index d2499757b..82c765123 100644 --- a/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -2,16 +2,17 @@ package retrievaladapter import ( "context" - "github.com/filecoin-project/lotus/lib/sharedutils" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" ) type retrievalProviderNode struct { @@ -38,10 +39,10 @@ func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) b } func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) { - localVoucher, err := sharedutils.FromSharedSignedVoucher(voucher) + localVoucher, err := utils.FromSharedSignedVoucher(voucher) if err != nil { return retrievaltoken.FromInt(0), err } - added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, sharedutils.FromSharedTokenAmount(expectedAmount)) - return sharedutils.ToSharedTokenAmount(added), err + added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, utils.FromSharedTokenAmount(expectedAmount)) + return utils.ToSharedTokenAmount(added), err } diff --git a/storagemarketadapter/client_adapter.go b/markets/storageadapter/client.go similarity index 94% rename from storagemarketadapter/client_adapter.go rename to markets/storageadapter/client.go index d517c726b..519e12e57 100644 --- a/storagemarketadapter/client_adapter.go +++ b/markets/storageadapter/client.go @@ -1,4 +1,4 @@ -package storagemarketadapter +package storageadapter // this file implements storagemarket.StorageClientNode @@ -6,8 +6,6 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/lib/sharedutils" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -23,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" ) @@ -83,7 +82,7 @@ func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storag if err != nil { return nil, err } - storageProviderInfo := NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId) + storageProviderInfo := utils.NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId) out = append(out, &storageProviderInfo) } @@ -99,7 +98,7 @@ func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Ad var out []storagemarket.StorageDeal for _, deal := range allDeals { - storageDeal := FromOnChainDeal(deal) + storageDeal := utils.FromOnChainDeal(deal) if storageDeal.Client == addr { out = append(out, storageDeal) } @@ -118,7 +117,7 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, From: addr, - Value: sharedutils.FromSharedTokenAmount(amount), + Value: utils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -140,7 +139,7 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, } func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { - return n.fm.EnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amount)) + return n.fm.EnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amount)) } func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { @@ -149,7 +148,7 @@ func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address return storagemarket.Balance{}, err } - return ToSharedBalance(bal), nil + return utils.ToSharedBalance(bal), nil } // ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal @@ -302,7 +301,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error { - localProposal, err := FromSharedStorageDealProposal(proposal) + localProposal, err := utils.FromSharedStorageDealProposal(proposal) if err != nil { return err } @@ -310,7 +309,7 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add if err != nil { return err } - signature, err := sharedutils.ToSharedSignature(localProposal.ProposerSignature) + signature, err := utils.ToSharedSignature(localProposal.ProposerSignature) if err != nil { return err } diff --git a/storagemarketadapter/provider_adapter.go b/markets/storageadapter/provider.go similarity index 93% rename from storagemarketadapter/provider_adapter.go rename to markets/storageadapter/provider.go index a0df86096..abe7f66eb 100644 --- a/storagemarketadapter/provider_adapter.go +++ b/markets/storageadapter/provider.go @@ -1,4 +1,4 @@ -package storagemarketadapter +package storageadapter // this file implements storagemarket.StorageProviderNode @@ -6,8 +6,6 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/lib/sharedutils" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" unixfile "github.com/ipfs/go-unixfs/file" @@ -21,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/padreader" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -52,7 +51,7 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark return 0, cid.Undef, err } - localProposal, err := FromSharedStorageDealProposal(&deal.Proposal) + localProposal, err := utils.FromSharedStorageDealProposal(&deal.Proposal) if err != nil { return 0, cid.Undef, err } @@ -141,7 +140,7 @@ func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr addres var out []storagemarket.StorageDeal for _, deal := range allDeals { - sharedDeal := FromOnChainDeal(deal) + sharedDeal := utils.FromOnChainDeal(deal) if sharedDeal.Provider == addr { out = append(out, sharedDeal) } @@ -160,11 +159,11 @@ func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Addr if err != nil { return nil, err } - return sharedutils.ToSharedSignature(localSignature) + return utils.ToSharedSignature(localSignature) } func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error { - return n.MarketEnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amt)) + return n.MarketEnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amt)) } func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { @@ -177,7 +176,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, From: addr, - Value: sharedutils.FromSharedTokenAmount(amount), + Value: utils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -204,7 +203,7 @@ func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Addre return storagemarket.Balance{}, err } - return ToSharedBalance(bal), nil + return utils.ToSharedBalance(bal), nil } var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{} diff --git a/markets/utils/converters.go b/markets/utils/converters.go new file mode 100644 index 000000000..58080896f --- /dev/null +++ b/markets/utils/converters.go @@ -0,0 +1,174 @@ +package utils + +import ( + "bytes" + + peer "github.com/libp2p/go-libp2p-peer" + + "github.com/filecoin-project/go-address" + sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { + return types.BigInt{Int: in.Int} +} + +func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { + return sharedamount.TokenAmount{Int: in.Int} +} + +func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { + return storagemarket.StorageProviderInfo{ + Address: address, + Worker: miner, + SectorSize: sectorSize, + PeerID: peer, + } +} + +func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { + return storagemarket.StorageDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: deal.Client, + Provider: deal.Provider, + StoragePricePerEpoch: ToSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: ToSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { + return actors.OnChainDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: deal.Client, + Provider: deal.Provider, + StoragePricePerEpoch: FromSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: FromSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance { + return storagemarket.Balance{ + Locked: ToSharedTokenAmount(balance.Locked), + Available: ToSharedTokenAmount(balance.Available), + } +} + +func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out storagemarket.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out actors.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/node/builder.go b/node/builder.go index 9a656563c..178f973e5 100644 --- a/node/builder.go +++ b/node/builder.go @@ -34,6 +34,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/hello" @@ -48,7 +49,6 @@ import ( "github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarketadapter" ) // special is a type used to give keys to modules which @@ -230,7 +230,7 @@ func Online() Option { Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator), Override(new(storagemarket.StorageClient), modules.StorageClient), - Override(new(storagemarket.StorageClientNode), storagemarketadapter.NewClientNodeAdapter), + Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), Override(RunDealClientKey, modules.RunDealClient), @@ -254,7 +254,7 @@ func Online() Option { Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator), Override(new(storagemarket.StorageProvider), modules.StorageProvider), - Override(new(storagemarket.StorageProviderNode), storagemarketadapter.NewProviderNodeAdapter), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(GetParamsKey, modules.GetParams), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 12eaf117c..5623f4ae1 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -32,11 +32,10 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sharedutils" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storagemarketadapter" ) type API struct { @@ -75,7 +74,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad if err != nil { return nil, xerrors.Errorf("failed getting miner worker: %w", err) } - providerInfo := storagemarketadapter.NewStorageProviderInfo(miner, mw, 0, pid) + providerInfo := utils.NewStorageProviderInfo(miner, mw, 0, pid) result, err := a.SMDealClient.ProposeStorageDeal( ctx, addr, @@ -83,7 +82,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad data, storagemarket.Epoch(math.MaxUint64), storagemarket.Epoch(blocksDuration), - sharedutils.ToSharedTokenAmount(epochPrice), + utils.ToSharedTokenAmount(epochPrice), tokenamount.Empty) if err != nil { @@ -109,7 +108,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), + PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, } } @@ -129,7 +128,7 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), + PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, }, nil } @@ -163,7 +162,7 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, - MinPrice: sharedutils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), + MinPrice: utils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), Miner: p.Address, // TODO: check MinerPeerID: p.ID, } @@ -294,7 +293,7 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path ctx, order.Root.Bytes(), retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), - sharedutils.ToSharedTokenAmount(order.Total), + utils.ToSharedTokenAmount(order.Total), order.MinerPeerID, order.Client, order.Miner) @@ -321,10 +320,10 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path } func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { - info := storagemarketadapter.NewStorageProviderInfo(miner, address.Undef, 0, p) + info := utils.NewStorageProviderInfo(miner, address.Undef, 0, p) signedAsk, err := a.SMDealClient.GetAsk(ctx, info) if err != nil { return nil, err } - return sharedutils.FromSignedStorageAsk(signedAsk) + return utils.FromSignedStorageAsk(signedAsk) } diff --git a/node/modules/client.go b/node/modules/client.go index 313672c6b..91796634d 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -5,7 +5,7 @@ import ( "path/filepath" "reflect" - "github.com/filecoin-project/lotus/retrievaladapter" + "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4909a8e67..d74595012 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -40,7 +40,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/retrievaladapter" + "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) diff --git a/storagemarketadapter/utilities.go b/storagemarketadapter/utilities.go deleted file mode 100644 index 2922f9d1a..000000000 --- a/storagemarketadapter/utilities.go +++ /dev/null @@ -1,80 +0,0 @@ -package storagemarketadapter - -import ( - "bytes" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/storagemarket" - - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/lib/sharedutils" - peer "github.com/libp2p/go-libp2p-peer" -) - -func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { - return storagemarket.StorageProviderInfo{ - Address: address, - Worker: miner, - SectorSize: sectorSize, - PeerID: peer, - } -} - -func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { - return storagemarket.StorageDeal{ - PieceRef: deal.PieceRef, - PieceSize: deal.PieceSize, - Client: deal.Client, - Provider: deal.Provider, - StoragePricePerEpoch: sharedutils.ToSharedTokenAmount(deal.StoragePricePerEpoch), - StorageCollateral: sharedutils.ToSharedTokenAmount(deal.StorageCollateral), - ActivationEpoch: deal.ActivationEpoch, - } -} - -func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { - return actors.OnChainDeal{ - PieceRef: deal.PieceRef, - PieceSize: deal.PieceSize, - Client: deal.Client, - Provider: deal.Provider, - StoragePricePerEpoch: sharedutils.FromSharedTokenAmount(deal.StoragePricePerEpoch), - StorageCollateral: sharedutils.FromSharedTokenAmount(deal.StorageCollateral), - ActivationEpoch: deal.ActivationEpoch, - } -} - -func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance { - return storagemarket.Balance{ - Locked: sharedutils.ToSharedTokenAmount(balance.Locked), - Available: sharedutils.ToSharedTokenAmount(balance.Available), - } -} - -func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) { - var encoded bytes.Buffer - err := proposal.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out storagemarket.StorageDealProposal - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) { - var encoded bytes.Buffer - err := proposal.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out actors.StorageDealProposal - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} From a896b30039ecdc85f8abdcea9efb9900260701b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 10 Jan 2020 20:55:58 +0100 Subject: [PATCH 46/61] gofmt --- node/modules/client.go | 3 +-- node/modules/storageminer.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/node/modules/client.go b/node/modules/client.go index 91796634d..3405bae91 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -11,9 +11,8 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" - - "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/paych" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index c5766c89f..822b70baa 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -27,9 +27,9 @@ import ( dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" + "github.com/filecoin-project/go-fil-markets/storagemarket" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" - "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" From 2ed89f496fe53142c6573fdeb7c10c192116eb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 13 Jan 2020 21:54:28 +0100 Subject: [PATCH 47/61] Revesve some sector states --- api/api_storage.go | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 8d0c39038..3fa2ab6d0 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -13,7 +13,8 @@ type SectorState = uint64 const ( UndefinedSectorState SectorState = iota - Empty // TODO: Is this useful + // happy path + Empty Packing // sector not in sealStore, and not on chain Unsealed // sealing / queued @@ -22,13 +23,32 @@ const ( Committing CommitWait // waiting for message to land on chain Proving + _ // reserved + _ + _ + _ + + // recovery handling + // Reseal + _ + _ + _ + _ + _ + _ + _ + + // error modes + FailedUnrecoverable SealFailed PreCommitFailed SealCommitFailed CommitFailed - - FailedUnrecoverable + _ + _ + _ + _ Faulty // sector is corrupted or gone for some reason FaultReported // sector has been declared as a fault on chain From 0402abe36b9f80575f7f73a4cd5ef03e0bff6f60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 13 Jan 2020 21:51:34 +0100 Subject: [PATCH 48/61] Fix tests --- cmd/lotus-townhall/main.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/lotus-townhall/main.go b/cmd/lotus-townhall/main.go index 8e7e78008..a52fdcf66 100644 --- a/cmd/lotus-townhall/main.go +++ b/cmd/lotus-townhall/main.go @@ -27,6 +27,11 @@ var topic = "/fil/headnotifs/" func init() { genBytes := build.MaybeGenesis() + if len(genBytes) == 0 { + topic = "" + return + } + bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) c, err := car.LoadCar(bs, bytes.NewReader(genBytes)) @@ -49,6 +54,11 @@ var upgrader = websocket.Upgrader{ } func main() { + if topic == "" { + fmt.Println("FATAL: No genesis found") + return + } + ctx := context.Background() protec, err := pnet.NewProtector(strings.NewReader(lp2p.LotusKey)) From 9298d183bf975c78285b00c7ead4174ccb406785 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Mon, 13 Jan 2020 22:08:11 +0000 Subject: [PATCH 49/61] Add bootstrap peers and genesis --- build/bootstrap/bootstrappers.pi | 6 ++++++ build/genesis/devnet.car | Bin 0 -> 36191 bytes 2 files changed, 6 insertions(+) create mode 100644 build/bootstrap/bootstrappers.pi create mode 100644 build/genesis/devnet.car diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi new file mode 100644 index 000000000..e646ccf3f --- /dev/null +++ b/build/bootstrap/bootstrappers.pi @@ -0,0 +1,6 @@ +/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWLZs8BWtEzRTYET4yR4jzDtPamaA1YsyPQJq6cf2RfxBD +/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWGvrgjWw4Yqo4AFWqYp4g37FpUvUCQBkNWudZVSwR9tY1 +/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWSfNcrD1cs5Cj5eSHbK6nHCqJLffAuPqvRMBRgvUdqQhX +/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWNkXyVPspUnrHUiSC3VJPMcXvHuNdy3BTCLTPPnDgwwTT +/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWSgJWJZK8LTRtCWzPa5FQheCFJjHpficVYgEQWeimcqCu +/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWFPaC4dyGpbNXCpVHjZucdJnDwmv4ng9tponPx5GrzJkT diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car new file mode 100644 index 0000000000000000000000000000000000000000..162461991b0c337799e6e6b56bdb71833ab2660e GIT binary patch literal 36191 zcmcJ&1z1$u8$LR8clSuQ5+WiUB8?z0l+s;-NOy;Tv`7llAkq?20)m8qbO?e7(jg7r z@f^oHXYTL0`=0;d&f{~=c+NNPyY_n5T6^s;))swX?(FFJ$R%Kc6T${UdYHY040hFZ zo~NF^IXunWvbkq?!lNae?ujDl}+ldP&W^HUWjT?VvXmKd`5ENL-VAx;R1ZRrWAMZEQ?;(8B zz%S1K4OjLaO=QrInMAs{P;>Jwy&Xk3(RBIqy@sZu(rF(ch%I{(E+aEautK^;DL30 zZ(5UgpQyxym)7!P!pWS|zdcCDu2^bo1xfyeV+v|QfdZ-j0>b$l$llt)+!+q(ZJ)=& zdNTEbD38-n30aoSSE{^syP7K`<>N-?TFsu_NJ=0iBg4$qP&Psk=y&5RpqLuJF!BDz zG&VJLbai+HhdN@zN|o=da2s`jrds~Ppa$RSZUaj@`BS%z9;s&aGCL?LN#9HM!*_#D ze^5aPmu&4|4G*az$8 z-cT?|UHAXpeWswfU%>xwN>eC}N{k^}9Sn9?y%6~3f|MvpM_V+3Yw0QYbBP>^ND_0%T-xxs4R%ABZ zYJ`=3H?X)2wP4xAl8jV}?XHy^PC0(z7yjnc+?H&^0Pz2rX{}H;yS(rBhR=l=gPD`N zWwzn#O)SIx{^hHbA#!Ze?NT6+JP7Hp^?e@%s`%Y-ptb~|QnCk0VFbao$xUbLhe|Lg zhAtq)(MWSU3uJHn4VfGyoY<;MloS|N3TMj3h_*%G)JKh$ek%826|HT;w^jJo)66#s zq!83cwtL}lrk@?{(~Yk_BcM2s*2X+4(Te zqvy{CWAF;0_Ow~Ul(r+I;naXq=XO;@grYdg0~(6uoT?TNS2$CAooi_VMM4I`1lEtT z0=eI;bACVTkln;$tCHb6)lndWGu>?9sr%eH6{N+B!;L2!5&C(NZ~EI;9scj-b`9^y zZDQd}x9-iTB3D#h&8|#&pQ1&&+qrw}^Y+szr*;~8kLP~QYk%#(hY(OhgQSq)H%<;$ z^o1D+sKC3#k%RiyRM@ZL6olb){pT=6?S<2k?jbeATD6F zX@@XPw%{-bX-Ycz7xZcwr?H&)bnyuL>v2!U4I$vHnRf{RemvJyqDi?gEnU(LrB-+a7WGpgC`e76P#wLHVmOJuU0zCy4R}vN`0V{z zGRtG1A(b*FKV=Ab5*_kA^1AWoTK3UV?9+MTkGI@5Kx+?KOPcy?p*Nb0c zsaL|9UV5ce|66%@nWL@`755*xfec}CTX}uF(?Ije<{?{7B7=8CU1~D9vGiO&u5m{B z!y)@PbO`vR-{y|@%#~xW&3Tp2y<@pPY_-gtJTD#cY;E5D@b`U=I$HO1b2EJW5V4g+>QK!W%zaTxt*<|xNOi`Sx(P$TCKheRtbr!U^S2nT zZ#ekR5Zmo#X=c7yPPPRPcWGb8hk_&$;0e7H?AZB+Dm=Qw%{;{Wa*->V`$&tUEnYNQ z?KvR?%z9Vh-1?`NHkhP)A0Eeh* zB79k+T%M?nqN^Hzz>9%ym%n4egT6RdwAyV6Xc=sNQVqySm=`Yhq$Z z?fI9N62!z6^JNDKXL{*Ky1hXzFy~ohyr_$ED5Hmfqt2qYafV`hZ9oo@imK&5xHoQ| zIakcE9#fZYU^pw%KtWLYI#0mqRVb@W$0c#^Csz2%BFud8Hd{=SH@|yYc_Bla`LVwumd#rI>o38vW7|uI__gikDMIS+z8S|}Nos{B-s|8=$y^M7} z>4lNpvwj|v&_Ro#I4#wX1(O@d4gq$mu<7Dn>NRain=_RrtT$AeC|dcW#k?p?77G}L zGjo-H^ z0Sk#d;RP88IBba;r1bDqDe0k2h3?fQEBae5@8e>1Lp`rNXTFh^fD8rkb+#dzp+l)u zsvuZ!RSuJP$o#l#H<5=|s@ZV^!-&BN0=`ABg1zsYd2{@0M``u=GaJW%L+;v>991z6 z8%7A5ARiP&{7&lo`wULB;)1$&eLn3B>)cJ;aBLZ1@2>e)cgFUE2Lk?j#Uwxc4z7Us z_AE^k2!H>1$N~wE$!RpwqG%;kF*-JEQv$TnLD-b&TjiJ)C^~%b8sG>B9IPyVR2DMa z;+=V2B%s&wUQ*r?-1fn>J>uEDH;bhgHRrV^Ni6734d3RBe7fD*cB={;A)v$#XWC4! zC@W;tKw$JjfeQ~y>zIgP-pl{aPh^VTv7Q}MNhCPauHYBCE9 z@Cq0?-}r>qd_~JQ5jNm}8j3LBQYhYJ5HWo!=ro9YX{;}G{ZT$yVNIgwV5h5`wITI0 z9~dl<9Rlp?Tk(+@GRaU`2g78=eOATpvAxh!>=R2*Qbr7&YoZu`#xal~Y#ghVu0ER0 zXjaLdkIC6*U+NZqB8l9VO(8GSgXZCeISGQ^f{FsTlJD1sXBD`|3-ed&oMLDTl+uuO z4?@@0_qN3>_3%%3Zi5KkQ+MB1ci0FV?V&GIy?#{)0;VINHzRdW zy`dl`x%>qc>G7#2cS~U|`0k3zBLTLxXfY^=qdUFEW4GYJT~ch5$c-#gjUGM5yAEd_ zrF2sJXD3yH5b#NQ?9&-448Nx-Sr@k!Qzo)_x|o;mx26NQz{e=Ap3cV2owBlk|O7yZ(!-BXk-atK(VY9p%VVj~#W$hH!R>rLZk)4=-EdB^1)M)}0~x~Pj!Z9FnO^Fs9Da2x z^}TQ{-rTmW_M-z7Yx(6dpN?{n90Yvv=7Os?47q$_@5=s%K2W%E@~q9&#My*br1)-~ zV;?27la+j*4rAHg-6_S`@Fu=Vd84e@;qb=nfzO8YxHEN{9> ztfjWEG)skRV6|oz1qsjPT026(t|fZdY*Pa24uK7X(}b9Z_g2L5$-U(sH#v(NJ(bbt z_$@ckf`On*pv+$JfUllwMM0pCMNK)b64X1cV+4QX1~P=nZ4p?qd7qbfC##`9OzIx5?6_$z zP0nqWtVvYaC$X<)bs=E-pV)#~6NTapU1bM!%xA^EHX}v6A-K4k3Cu%c!-;gzPDZ=F zR?|@b1(|;%xknIuZ*K3gJa!;~W@aopItKIUo)!db|8@%hBBRKjgc?1NooJl$C(aQ4 z3bMw`$k!{kYBvV2K|w}y+e>Q91`Y2#sR@)A`Tzes!SR?GIh@c>$184N9IOvk?&Xt)bZ0+km)idi7 ztqCWesZ9ot>F=OHz(*ap`4?^Nh2WTmFP&N_A|y^stvqVqf(Gs+u=A$~QT~=2XwgT| z#hnMjAE-1RO=gUFuvN{vW^kl<|3^u!^Zm= zq~2`vkP`~>_!D544vV^QYzz;{ch300C-sNtZ=BGTEHtv94~MUJ<~nFy9?5oS zLFiTW&-7ByZ(O~`h<7_Jtj}3($W0gulJI~WNo&v>%=fL+(B_7zTAGZ>lb*>k6D^H+ zOA?eVN$BXMC)j&P#;Q}sJXgPLD-m8A=d5qx_*(TnslASq6CQuSZ%+x3OC?AOEf8)` zAkY)2uDrU+-+*g<>XCnV(<{z-dDPnC{YqbtG?C44JDe$g1&y53r2m-rmn~wa&U0Z^ zotaJxquQTRuLtxNX;#@_LjwpBVQ8Qji{`wp4yR%qXmR7_HPb1Qb0DwiWZkRfcu)gqN1 zoILRJI>0ayvl8~&b2RXJ_)}Uuvrkj?ZD&vkBXn)}@V$V-!rbH87m85)q^nj^t351z zuWN_zY9=+ZVxwR|b)Fl~M|~D@Cv?1K(0q=wyq%&4crozKe|56@=1O-8wJ25brs}IsD3ku$$pw-Y}O+4*d z%J^zY#gmdxNH9c=EV|Bu!3Vlud@D6QNQ@_YspvLEzADD9=@Sq^$(zJs7 z=_?a=UR7T&O^ifI%~pH=3jC!BR~P@r8tvL$=tdd2uaw(MHP$}j1JiYZ#s{Zyy-~Jy z8jNup4?kiiI|=<>V1R}Nf-XwMn373B9g&JRC;L5F*-rHiADvX**!Y<~PiP4)5g~@j z4P=J^>y-Yj5v>Olc?GTg!zlK74_D;cI$D?ux5AqWeRCoL^*?e08N%d_>nxy|6?Wpz zs~x9aQyITWV7!RTzp2l>XX%G}*FlsF>W*O34_hCS%@}MFxZ+ca-yVdL@vJst>s&3K zCnG0Q?$@orXlaE&HFMV$B5p62O{7@rBzjP^&%2{=Yfpkrvb~TB0+zuNZh30ppZ&@D z;zM~U<^8q(CdHgb4)}B)<|2j0C3>os%oI{Z+bB}oZ z{4NF;1T3&!top^nqBZ3DN}Ui9sbcO2EtJ6AtCNnZMS^pIj=$t)V_Bt47Ed4#MiM=x znRvQ>s8|W=z}GP%5M?iqz%PT|Psm1=zqvpI32+h(wnMr!g8tKW{h4UN}AIvX$ zk2CxC0t4i74u!vgfzw+2=*ECAc_UBG^tv#FI)sQ2MPvQfG4Ap_R2{PY` zvr{Ui&}ziM5ltx$~oIf`WoH(C8)RuLvgn|05dskI>+6&p%LDFdHs(@GJcH zH^#4TT7Ua&fr5bW%WL)PrCio`_0j3vFUwlRlS*F4OG`WS-etQ1*)P6pFc43LvJ&&# zw+dt$2%+)^{pHF3b*+DW2vL833H~c6>duFfzj4^xOox8hGQkPDk~MBJJ>WGL9}zNL zn(L)J{#v8#M`})^i4HOTEOv-$>&+fpN^{sR|E*+{W`F(C6`51vdn|bh#cMe&yd038 z`^vui+%Bax3^qG#Db|e_GHm|dl6&!0wGX+8^I+Y@4)}1G6w6HO5_j?^dIUs|;Has#bYOc!7BsjvZ&UCiZ9S^K zAwZcvq~HyClO~GZsi`eo6HZ@SM2WrW97r>Lzt!Y>#QDfZnxo1QfZ6K06M7yb_W#2oMDwaX;nH5FY`)ax65~@KOse$%hn!Jzk zRsBvxAM7NgnQRTKc_+i}kw6VaxJNF88aaJrUHUmv#d>$Z_vg@BUf6qT-}@uW!;JOU zk4bLBV1euqVD(!Hb9z4q;ALwS@!z)89aW*ncvhLoLa$hLPHi#lw*BYU3uFkp^=_0C z%rVCFeA?r!IKgu0jHdKvw^8exO`DF`tm`Zdm{abB+XkgL2z@@p>GKL`m3m(6qkw{RY$gq)xhHFxo~x)ly!QO+rbqjo zI#4-SYtAT+qx~Z-^Z{Y=DVVmDHa^I5La~L=_{-w<64LiI8J*4AK}}bz1KF_yH>~`D^c6PPE8M# z6$VV91z&H4x9e2DcUnM$4MCSz7f5-??SmsOeHxlpA8Yst%T6W~DLvEo_Lp-;Q_WUj zas$~Rz+S_(rnuTowHW5~pf^`^>Cq?Qn0}2z&rhmvD9gV;wn6JQNK zDb4PQs1;(>Ubr^wBm`C6v*SLQBYd&PXAA+)%(wdBVzA7!u8Yz7uhhFhE_Opjf`89X(y~3>z3!hoB{4||v@--n~^~0h5&-xoj z(nl_lx<^vkxQE0=0xNx#!sHwcWl6;xP>`<-o11SB8N!8-=CdR%mcFZ`3;91eGB$I1 z#X@f~D*_EC%#!LW^d$t)MU_1r&N5V~Jb$FXhkCw^`nZN}!@4dL{#IK6= z^^920z@FTJ77PSkatf8{o+(Fu#QMQp{}QKji~Q9i-Xb@)}*Bb*fv|(}s*&)D6 zV3>}a3AJynE-^f3nUnSIz)1Z`Aw#h0jW%-vz8NX_M{XcPnA`=6jz&4eQpN_eF+Z~F z%hh~n?j}?nkF${EaYal#vb_QUOKUldm1oVyRZG$`rw@KXHyzb1Q_Wx%Yv+(*PIDU) zgmyA}B=wmv%WO(vp&|{0m5exSvTV~8&q5)EzPg93ii{BgzAedt`rbY4v|lDP`aV8A z8C8->9fx^*-}d7a8q~e@Uva)cs-Gh111<93--dJC#TG_lq6wicbyrJlp})XiXg7m~ zEy;b)I(3}itCEJTx0a-R?iV9GeckMIXe3;>n_sM``70*z(lmFouIn_wET$3tbyO+(hA_Fc%$^$?q-kRAYzlEMC=|>d`OWxP z2S_zP#Zxre%5j1E<%Dz~N>m1~Rwl}8Ao5{%l?QgUb*l0)_xsuEjW0#IFX?l?ehe3fhAlV9Xw%!Hzp)i~& zhO0WMh&W5DM*;)3mcEFsy91AMRmNBBSylxJmoEofumK0uP=o>Z{paw6tZ_*(iOrU% z-^>fL&Gr_TCNvFZ_WOP-2g<2YFjycv1lUxCJ7wV*M}_yla`PsfEb=Kdlm0tpIMMH7i}|0u&aUg0W@B{DpO4S zqx*5fxkvZ+?-baX8?Oo<=6_CAme}GJ8;W^J1sy#Ym{^%Erw=?H^89>qZ%S>%v!Ag_ z!3m7FN-&6nV-X??0k>K1-E_rQTkVm$CEA#*xhyT zy{KDD0%IA?Bm{0eQh1tl!(YdFxnl1Xf0K(-fO->4g-S-Hv7cw`f_95bbLVOg)ef!| z-o#Ch3jL{>2s|8MPP7+%8&g{=uJ06p3jTu zbg{XhQ@T=kZ~5M%a7CJ6+Ah*DyAkc<4-VYFmo1>dhM-F8qPTA>kSsmy2XzN%>kJ#BuTV72#@o* zK^^4!L1rvgFx;&@6eksX$e5IiZG;!v$wX@kpZrU^u)8I5DzAAQ9_MIKn?SlyY3tCW z8S0+&L*;%EM?7>vIFD@H|DhB2MVQhtFMiIU8&@({Jq9GFvinyEs%RL!HCARTEuqT) z8Ab-tpy0^Mkf)4vKV0PuJ_Mf}T0(=~EO^-^G&xPF^y17!rS{iVkR$MW^6H7Li zl~M4^@2JFiGjVup6@nIJxWD`5lt-hhfaBBpqT9aUy6Zmkh7AO~)!YAcjkqDu={1G7 z*`S-MKjve$>$~l^lYM&0R=AIdf6EQDU?Av1s>GXbS?wO~^%II=(yi)^y-xijH;^Gr zZh_r}?ysT(Q6Ssl6||cY?}=t(Xy@;cn|bMEr=_h;u|dENyGO_Cbl2WDPm~`#H_GJf%!@LngO-ZiauCO`VH=EQp>p#PlO%mhZUGa2mP? z{k*CM=1r9?9Pk0uqm#b8m4>x%0@}dxtfV4BR*C=arL)(91Dd&Jug^>(IxEA61_i;( zI5^c3L{)!%eEVy_pxv`<`b-HLF?UX*jTmff-d5-%hxtg81Qllv^0n5+c_t0-W_4Ww zY{THQyovW#gYV=f)EfF8~duS~q&Hm_BWm6A51nkUJGormSiD5btfX z(m)fV)h#3UYfh=?TGCEthxCB<5z5$<95-TYszu_nYN3#*E`;MJoh*qp~4m3 z*3L7heb%&j4Z1|!T2s~`qEl}4E#Nb0{q*J#ecgRqRi1<8!WG1g-}Y-Um$0p*V{{_g z?lR@<6q%}e$v+|362G3}Qg)%^oPW{}jbrH>hG#`pFlAaHZ*`B5r#&{(2RknL}kxmp~F`? z-p6HQ6SX&W>#tYzmasMP(*=0KV1euqV6D`aO7r&UDyECuOlmS5-)cC4(hQ2CN(A&t zI`aihV*ZRcAVb)QV-(#VBK>gm1Y{h2H}@UM8UMr7gkA@%B(*SHkstkW&~30i*P_7s z#GX$kA=4zWWx9>ro}CNP@IzA7jYE>5x$|Gs+DpxKfBCIF7vJI1h^$W2R5FGI(L~%* zQ%s32EonsnFElYR`8aDtPk;q?_+oT5V{FXw>@=5Ik19f5fB&(BXCxR3qV20+C#@g7 z+o}K4E#sqZjI-<+akj7|#dy|X%8SVfXc|ndR?eaIN72F;Jr=(D0{dOa0w~hKTFdX! zWQo)nahC|8ATA1TOd>~fJU_QN=8B=re!eOCgB<@PK0>C*<>!^|Rp@$2SYO5&WwR+Ro-XzUIfh&vKydm(x4;)=72T zzk-FdT}x)s&y%I;5=z973pG5Qd?*CWI7{_FPg}D^V!VE3#M;l^r!4pSP_2g*OJ%ey z_rM4Iot`YfUq>-l4nIzS8tT&k1!Rr7yl_SS;`l-_av}e*QnvBUs!4sGKMgzN!Pr z=MT34{tHtxlXI7njO1ZZ|8l2m2O%{tdLR0D7GZYad`Ye%k3rS%xC5+Dfy{sP>A#J` z{2%<}cb^If2>j_&TEtqY-&Hh!)kXRJ9X0`~iGr^}p}er^GtSMm#l0b_C=!#e6u`CH z`&`3cJucn-6Q))qE?jprPmi6A1Wf;&cCOWY;I~K-~dJ{%$c4W{}iR_#zuy zJjW8ADc|AxScoxl&EUEfgnTy#{SGO5geZTJv@d)iW>Ub`ro6wx%}q*FqETaCvm0&R zl1Esyg4SDP#HZ;+N^q_1Rq?1VT@Fr>n{P{ax!~m?)lQMSq0*YBu+K|$?L^fF&XnJ{ zghh4^rHlO9aB$+3^c_ug#WWRs{VAf=jRCjH&@edDsO7V$ncAoM9Z&2ebiTa3d+fZX zFu*WC5IETqYM#r3yS(Svon|7itTzhU&dQ!r%=)=lOHLS7!aM2(2Qx z&uNbBlGA!MMpdK20~N-H9)d|H5cL8oXt_)-pt_gKxil`=WCU8bV3QH3E(N|} z&zPciuYBEMxd)l4AJzIQ(Sm!T-O#~q6VvU)yjs)~(nlo8(GtKs)|X7Bqs2NdQpc;zH%bO4gHg*LEQ41ylranOs1%5|_yZR8nziT(HRq^mGfGj6j71@C}QeOVtk$v(QRW z?65AcXtLL4XwTxyWTKclF^$8hCvZ|WQKF=>5Vilxofa0_y3B(pE>OL}WpV+PC0r&K zQ1!xPasgEWz;h)C2MTwb(RoVuA6mt*Q{>Ts-BwRw-P!A=QfzIt!PP#y;HO5 z$0z)jf(;P$0xB4|OfI0hg3IIrDnGb1F4$xQ@*lw_BT%j>e8c{&=UN7OVivpu2Ph@= zGP!^UM3e%>gi>20cEIO8W(Ib0(od*lMyI26uw~%NGJ&o z$hp^j#x&o~$HvO}aHw31ZE>ZH3TAzayVF&GW?#o0P{wYz;49(_tvt*?6c;Fi^fI}C z5=}3Y3n(A;GP!_qJK?$Vo2OUVB_7B2KMviuOFP=!oRkT?WhAsysgjDZUa54i4|@wY z`sZ@9e7Exu^0ufDqFz8LpqI%7lqq_dTtEq>m&WyHG6KXvV3QFj;Ss)J#e*Kv9^ZW{ z!WySnP@0JkmoGMZa3X6juJGhnZ|WDfkavHXUPjmPifAv;@-Z71MHCk(FY+?EfYK)~ zlM5)z@-n%AG8W;vDlPq)7!3EknQyQqmM4GC-pW|KV#Wx;#N`}a&*Ru`XVjYehTFw9 z`^LabAfdsD3Q;eh#Kz0y0?L29OfH}_$xGv!_+1tX6crAV@`Dec28`YfSVv`!E1|4m zMBWQh4Z&^+Zcumff8>)s(uhat0vGDc(__1X`#n%WdC_N#9{JNJSAF9Px~ALjY)Xk% zbA)W*Po%6C@vVROB7`wc%tyz^$f@4qJDV+%Vn4=I+qNv9tRe{<>W9xa7k4pgd2u1` zIl?0^i@8OQbUa0^aE)Zp^@u=^@!?FhL}!oQgL;P1I3jO);f8PL%guDXFmm6T^nR6f z^MVIHjDA{XIbr&gw$9ck$ad|CXKmVuzA?*Vy&rY&W{{Oi&+o&bVko?t$v?a8cd8}C zBUsGt$EWZSi;0axYA!P5_Hle`{+|dgP*=g0IiSo>c*%2L;MR0usrQg~Fb=+CUAy)8 zkkcyNP4=8a(#R{5V_>;Dt*tD)?$*6bs{~=3`%_7X(*`I(^fI}C@<=a}3n<<6GP!_K zH{rQzZS_Wvl~Ls@I$?7)CP^W9|i>%S7yt6%Ez;Bttq%U?z_tt|0tam&pZ`TX|_*u*nE?yN6B2=env2m9aaCj3!oiT2fEyOHfd^P+Nzl%3GR) zNxW)rz&GsQCLqdy`jO#XW226qq@LrL5IgS$x`*EeCR-J$x|G=CV>VjXwQ#fUUL@N;4O|R(-nZlDAZW0yo%j8o3ra6Bft>)FSRZ$BBeO=m) ze|bwJa(vgiYGibT(t-Fgxv*C)D{OlUT|!B4ERzk@0vipC8bk?|jw?&bHs(F+k>R<@ zOn2+H@W}Gch$%?)>g_OeG?uPxqE9ruGHtdpLc5|WnWVe>Rf=zkYJ=NV1$Rjcc-9KgPKun(l{mv64bfjF7ex?tAb5D; z+1{8i`NY(%@hp{Vk&+*``AAL;Tbily@Zh;pSH_sq2<02QFF1wY1*YWTsL|UrzT4?m zRe5j!#{E@g8XeMEa5uKFSKf`{heu6phE<_! zptdkQXLnk=?|q!`Hnid&gElr=sSq<`xSzTCm?aNym?FgLAVmWEw7|~jqY_#VO_!S0 z@ZD)$k{L<#`d22t7fy)czPq8i6dlfHdqZY@jRGf8_crPK=*XM2CB|P_{of%`)4+Ga zf8|<KIV~1r$%+|Q2GN=+78oJ@GO4| zDTN>Noqt?c7`HyFaG}-h-tQodo!8o7`B7fk7PwjAuOCcViOo3E3xIGO8c{txyYFrH zGYd6tA?meePwqebZqJLRnnf^FLQL;gGKlg^!Qk$@;bd7qMOPB|>-w);Ol#NkD;%dX zU$#7bJ2ccGSdmV^dP?|xnA~)a2#Y0<=rXw=_Hi<3%U7wv2T}PJvPo+jxM(@Is4o0{ z8KN>px|ne;jSDszft*6H$(U)b9jyI>`6zz-js7W(H958z374J9*Iv1gPDkpNVwms^ zd$ljdQNaK6@@Hn*gUVt=XM_tM)!=Tf{){!iaoi}FOw^U_F0ziwJS1I$hw^o=&Li5uh$^+T-SVe-eN9lHHA%t2vUuJOoH5Ik2^oS2Bj;@gJOJkfh9U{#P#BBql#46kE{BS*j!3i#7#1zYit5N{+#|qBDK?y59X` zk@>G&PhTp;VBT`xOC;>qkf%y7*E>KDzSn|O?U_82v`w+|kHzypals}dklPhD856D% z7=)H=hZ=2Yi8|Np*papdwZGzWRy7@8uE@gpjsY)j>friD83m7+&K zZcrR}pn;|Os+T`oGop#86X_@tGgg^KCXDbv-ueCg-6 zQPuS)SJ zj@}oR+l78&zF$EuS6({ih^SXr?&6E#5t=AvT@gxI%N?wDGB?tL8u|@rCrgj1)gAt^ z$oy9>YkEdaZdLw$E7B3y?#{EK_nsnNmbqjGM1%KPCrLaRFY~&jnBL#voG#BdW0B>! z#gECDx0+hE9P)f!zD?kQmmcdM&!_+Cy5L?~_Sd`J{w}xt>ucEW8b+XZyoJDpQkvWQ zv@bAcr8Z_(a9AWFUNFB|$$d&+W@N2%t6y?-gBZ?KaP;LZMS3EqTh)%tUz|_g$f{1u z7OLQV&2#3LA+QziK!Lu845bxX<_5C(yM7J`I}k$e|H~Zq@rQl+>)+771^sozs427r z4(a!|IV9M(eq5Z`M=#Hj^;M#$h#_1I6 z`&PG4N1i$Tz5pPze{lf^e=Y#_tsnUSc1|eJtMvX+j|`#wf~AlHO(k?d&%yD1``m~U z=kN^svkCZlwS+;q?+ZA^{)H(}f${%7ClrX_f1DGVNJ95a+(%m6rC8?E#)yow=)>u? zfL`|?>rdSW#^_xsaLw<$kZ}ztnZsdkJz|aO{~+;`V8xJtfXeSKNzojke=?6Z(@2Y_^oQc*7T~x(Md%a1^E%2qA}Jd)=fi8L%ov!)NdtE2hW6!%hjI zfx9q8hlV@WS4aJG-sj!*ZT~p_x+%W!O651R!_U`usgWJwPwz^*esVMDsHtax&cFZ- z^ofzlAturI*yN3x_Ko1l!l$RO(-43fif|f&dB#8%x_bhOG{LchZA}+Mhb8E!MfFI} z;fJw|NQr?l3>L@^0d~BEYybM!(T4sZ(ob^lR?ew;D&M8?OJ!z^$~7w)#Z>o+&T$El%5 zSOWNpf1jxVG}sVy(M`e!H=5p1?)lMqK)+50=6fjT)8t6lb0;djXhe!A8zwi99RjTO zZI9t%DxV$3)UAw$!+!AUM+?n+w6}CMo0uMDznpOXbEXE6Axv)XP<(C|&ZVH27s}5* z+#Sp$94GHS$02(y5_WK97*=esX_p4DP5a;aNms9sVE!m4NGKj-hi?R^w1sv zk2DnINR%W~Grvm3LRje7wpkAMt_e$P&NhrxJu+h#fj z<+0dAu+&Er5+N5ayvcqmXiET`&%fmcTJ#ZgxpOpF=5C=`D!5}n*!NCJ6m(He!z}Zv z9`w-IN7hyt4JJ2`9Rlnl5)}CvOQTuF914Z}yX*3z?hkZ+C~8*^l5|Q|Pvay1ksHVm zCU>*qislcpkjzz70J`706)@?POUh!DRQgPF!X+ zq~q>^On0T|q0ZrXpvu+d~_>ACr3>?dzV5`w&*Pvyp$Q*?$TB$55YX@$p@m=fjM|uHb zR7t#VIIY_3vNsGBghw<5q6@Y<$3qJSMknj9eYLyg=p9#lLyse?yvm?&c@{0d&b?)B3Oh2h>o60hhNq zmV-?DEwmF^uUwjqu)*yk=%cRT9X_{2MU{{wbYvJTkR1Z-tafi~PeHB8;_x9ciKp#M zgAm3KX20?JAA8dc3fyF;f5tJ8A#5BsG6hiFw3_kAy@(VDmeGfFZ_M+~mM~8~>AhcV z+xhMx^yRn!u>t7l;Yt&2c9(cP7m9Aw`e7wcXrOoIsaZwr-Yfi$af)FD=wtwL`a?mc z%(Aj@_g|=_dNvgy-$q`Ur>8MaR|xr77sIBQr0^Nq5+H9r6hx-Vp@^AfCrsu0t+ofI z1xFTNizkLv{pK6nimdq;JMp1y1#r{1~isPH6-IUSy5?xA8{=L)i?*GhbbjcIi!YRf*4LC*;V z^78+d8)(5m(8a=Sa<<$r`9(j-yt^twwt`%$>QFSN879MOY^s5chOA>#+{zq;gLzvu1qF4rJ4kq>zq>0nrH^*-O zq);Ftre=vV`aA&;Y=CyMwRtzoxLWi+(}5j{{NamlSh1^US) zT)9~-G8XGqDD?mjBS|K%KGg2LzdT-weA65w6N=4Mq?nvRI##j>4n0eDC8eQi{NHy=9)nrGh5(O7OS8P(DY z3T-u&+z>13&xhU-fWig=zrBb+E)|4xLhA{GlZ}|J%$8QF%N#W=8muWm*sPrhdo zdxqnLz6iH{xWz=INLJV8Q}vpUi7>`^e~_$vjSF z@V3J~_0wNy$+JE74SU9sKGKPhjhKUkGhKzqb`gM!Sl(bDW;$R5T_ zjW90saBqMB3RGwDMnM9{1OMPW-tSY$F5fT-2peB=o8HMBvH`cQ-;q;=zPk({G6Dsu zPdb%u=$4v@)7EjLnS1;U$7nHKb`?8NQh1kijJqCsX9Eg(KtUST9ad1a+oi1Skj1g6 z3LPI1yd!^~WPG3Z^VSpm<6ny1H2IjSZ`Ltdf`Y{sz% z{#`C<#^~?17AmJ;as$~Rz~&H^ZlM+8SKj~1D`K(MujyAw8EWeajVY6eaP%21SN)M2 z$PgxXSPz#W|Lfdr)LgT-{1XF_9~0Y#7B_FU%k=CO&jpJ@Ulsxse}Q(g_4_e_bv--A z^M!4nAY!ZXsk#(AWd`}JIP$HoW}5HL_+1qxz8QLLFIbaIk$W!$8-dqaJua#qvS zxzE`AD62(Yx19(|0}wHRf`CCH!Te6kKZwr4vU|knlgpK5`iy1Fwi3j|UMCE^fj&9` zg+gGba5~Xbx0rJ_CTkrpShE{E91#mcdYbZ0l?FoiWc(rlr{o`A?|*T$=D#onI;!;d zDV#ok2O6F@;$zLr35C=LSS9sq7i?Y~Faoh|GL_02x^IKqdL`kjS^h;MfK2}{sD+VE literal 0 HcmV?d00001 From 0414e99fe7708541b36ff059ccb9a5ce23d3cde0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 13 Jan 2020 15:18:26 -0800 Subject: [PATCH 50/61] remove support for 1gb sectors --- build/params_testnet.go | 1 - 1 file changed, 1 deletion(-) diff --git a/build/params_testnet.go b/build/params_testnet.go index aa75ebcd7..016109770 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -3,7 +3,6 @@ package build var SectorSizes = []uint64{ - 1 << 30, 32 << 30, } From fefd21ad9c5469bc8f6b74c97a86e142cc4d7409 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 13 Jan 2020 15:29:06 -0800 Subject: [PATCH 51/61] remove private network and put dht into a sub namespace --- node/builder.go | 2 -- node/modules/lp2p/host.go | 1 + node/modules/lp2p/pnet.go | 63 --------------------------------------- 3 files changed, 1 insertion(+), 65 deletions(-) delete mode 100644 node/modules/lp2p/pnet.go diff --git a/node/builder.go b/node/builder.go index 178f973e5..dd0af277d 100644 --- a/node/builder.go +++ b/node/builder.go @@ -58,7 +58,6 @@ type special struct{ id int } //nolint:golint var ( DefaultTransportsKey = special{0} // Libp2p option - PNetKey = special{1} // Option + multiret DiscoveryHandlerKey = special{2} // Private type AddrsFactoryKey = special{3} // Libp2p option SmuxTransportKey = special{4} // Libp2p option @@ -142,7 +141,6 @@ func libp2p() Option { Override(new(peerstore.Peerstore), pstoremem.NewPeerstore), Override(DefaultTransportsKey, lp2p.DefaultTransports), - Override(PNetKey, lp2p.PNet), Override(new(lp2p.RawHost), lp2p.Host), Override(new(host.Host), lp2p.RoutedHost), diff --git a/node/modules/lp2p/host.go b/node/modules/lp2p/host.go index ccb579763..6809d4a5c 100644 --- a/node/modules/lp2p/host.go +++ b/node/modules/lp2p/host.go @@ -79,6 +79,7 @@ func DHTRouting(client bool) interface{} { dhtopts.Client(client), dhtopts.Datastore(dstore), dhtopts.Validator(validator), + dhtopts.Protocols("/lotus/kad/1.0.0"), ) if err != nil { diff --git a/node/modules/lp2p/pnet.go b/node/modules/lp2p/pnet.go deleted file mode 100644 index f182be125..000000000 --- a/node/modules/lp2p/pnet.go +++ /dev/null @@ -1,63 +0,0 @@ -package lp2p - -import ( - "fmt" - "strings" - - "github.com/libp2p/go-libp2p" - pnet "github.com/libp2p/go-libp2p-pnet" -) - -var LotusKey = "/key/swarm/psk/1.0.0/\n/base16/\n20c72388e6299c7bbc1b501fdcc8abe4f89f798e9b93b2d2bc02e3c29b6a088e" - -type PNetFingerprint []byte - -func PNet() (opts Libp2pOpts, fp PNetFingerprint, err error) { - protec, err := pnet.NewProtector(strings.NewReader(LotusKey)) - if err != nil { - return opts, nil, fmt.Errorf("failed to configure private network: %s", err) - } - fp = protec.Fingerprint() - - opts.Opts = append(opts.Opts, libp2p.PrivateNetwork(protec)) - return opts, fp, nil -} - -/* -func PNetChecker(repo repo.Repo, ph host.Host, lc fx.Lifecycle) error { - // TODO: better check? - swarmkey, err := repo.SwarmKey() - if err != nil || swarmkey == nil { - return err - } - - done := make(chan struct{}) - lc.Append(fx.Hook{ - OnStart: func(_ context.Context) error { - go func() { - t := time.NewTicker(30 * time.Second) - defer t.Stop() - - <-t.C // swallow one tick - for { - select { - case <-t.C: - if len(ph.Network().Peers()) == 0 { - log.Warn("We are in private network and have no peers.") - log.Warn("This might be configuration mistake.") - } - case <-done: - return - } - } - }() - return nil - }, - OnStop: func(_ context.Context) error { - close(done) - return nil - }, - }) - return nil -} -*/ From 0bf41bcd109b9c9b6e860ea25454fd661f4a9bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 00:36:05 +0100 Subject: [PATCH 52/61] Drop pnet from townhall --- cmd/lotus-townhall/main.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cmd/lotus-townhall/main.go b/cmd/lotus-townhall/main.go index a52fdcf66..a2cc8bcef 100644 --- a/cmd/lotus-townhall/main.go +++ b/cmd/lotus-townhall/main.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "net/http" - "strings" "time" rice "github.com/GeertJohan/go.rice" @@ -16,11 +15,9 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/peer" - pnet "github.com/libp2p/go-libp2p-pnet" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/node/modules/lp2p" ) var topic = "/fil/headnotifs/" @@ -61,15 +58,9 @@ func main() { ctx := context.Background() - protec, err := pnet.NewProtector(strings.NewReader(lp2p.LotusKey)) - if err != nil { - panic(err) - } - host, err := libp2p.New( ctx, libp2p.Defaults, - libp2p.PrivateNetwork(protec), ) if err != nil { panic(err) From 85f0f0bf81da3b8737f360c8d83e9e68479760ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 00:38:08 +0100 Subject: [PATCH 53/61] mod tidy --- go.mod | 1 - go.sum | 4 ---- 2 files changed, 5 deletions(-) diff --git a/go.mod b/go.mod index 234dac187..afbe0c0a9 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,6 @@ require ( github.com/libp2p/go-libp2p-mplex v0.2.1 github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.1.4 - github.com/libp2p/go-libp2p-pnet v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.2.3 github.com/libp2p/go-libp2p-quic-transport v0.1.1 github.com/libp2p/go-libp2p-record v0.1.1 diff --git a/go.sum b/go.sum index 887c09eda..a615db69c 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,6 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0= -github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= @@ -453,8 +451,6 @@ github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+ github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k= github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= -github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= -github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.2.3 h1:qJRnRnM7Z4xnHb4i6EBb3DKQXRPgtFWlKP4AmfJudLQ= From 4d30ab77be2435ca3218f002b0bdededeb9cc244 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 14 Jan 2020 00:50:48 +0000 Subject: [PATCH 54/61] Add rice calls to stats and lotus-shed --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 083a15d70..b81840767 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,7 @@ BINS+=lotus-seal-worker lotus-shed: $(BUILD_DEPS) rm -f lotus-shed go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed + go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build .PHONY: lotus-seal-worker BINS+=lotus-seal-worker @@ -144,6 +145,7 @@ BINS+=bench stats: rm -f stats go build -o stats ./tools/stats + go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build .PHONY: stats BINS+=stats From a2bcc1fec29f3e81bae84b6110ea3052921d2e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 13 Jan 2020 21:47:27 +0100 Subject: [PATCH 55/61] Mostly functional mock sectorbuilder --- api/test/deals.go | 2 +- chain/actors/actor_miner.go | 14 +--- chain/actors/actor_miner_test.go | 3 +- chain/actors/actor_storagemarket.go | 2 +- chain/actors/actors_test.go | 4 +- chain/actors/harness2_test.go | 5 +- chain/gen/gen.go | 7 +- chain/gen/utils.go | 6 +- chain/stmgr/call.go | 2 +- chain/stmgr/stmgr.go | 2 +- chain/store/store.go | 10 ++- chain/store/store_test.go | 2 +- chain/store/weight.go | 2 +- chain/sync.go | 2 +- chain/types/vmcontext.go | 13 +++- chain/validation/applier.go | 3 +- chain/vm/syscalls.go | 20 ++++- chain/vm/vm.go | 6 +- cmd/lotus-bench/main.go | 6 +- go.mod | 4 + node/builder.go | 6 +- node/impl/storminer.go | 2 +- node/modules/chain.go | 4 +- node/modules/storageminer.go | 2 +- node/modules/testing/genesis.go | 12 +-- node/node_test.go | 108 +++++++++++++++++++++++++- storage/fpost_sched.go | 4 +- storage/garbage.go | 2 +- storage/miner.go | 28 ++----- storage/sbmock/preseal.go | 52 +++++++++++++ storage/sbmock/sbmock.go | 116 ++++++++++++++++++++++++---- storage/sbmock/util.go | 28 +++++++ storage/sectorblocks/blocks.go | 6 +- 33 files changed, 390 insertions(+), 95 deletions(-) create mode 100644 storage/sbmock/preseal.go create mode 100644 storage/sbmock/util.go diff --git a/api/test/deals.go b/api/test/deals.go index 76a739605..30faee4eb 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -64,7 +64,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { go func() { defer close(done) for mine { - time.Sleep(time.Second) + time.Sleep(100 * time.Millisecond) fmt.Println("mining a block now") if err := sn[0].MineOne(ctx); err != nil { t.Error(err) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index c6bb7ae6d..62bfd3ab0 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -19,7 +19,6 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - "go.opencensus.io/trace" "golang.org/x/xerrors" ) @@ -526,7 +525,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM }) } - if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize, + if ok, lerr := vmctx.Sys().VerifyFallbackPost(vmctx.Context(), mi.SectorSize, sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil { if lerr != nil { // TODO: study PoST errors @@ -641,17 +640,6 @@ func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids [ return ncid, nil } -func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) { - _, span := trace.StartSpan(ctx, "ValidatePoRep") - defer span.End() - ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof) - if err != nil { - return false, aerrors.Absorb(err, 25, "verify seal failed") - } - - return ok, nil -} - func CollateralForPower(power types.BigInt) types.BigInt { return types.BigMul(power, types.NewInt(10)) /* TODO: this diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go index f76509373..410dfcdc3 100644 --- a/chain/actors/actor_miner_test.go +++ b/chain/actors/actor_miner_test.go @@ -254,7 +254,7 @@ func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Bl func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Address, size uint64) *actors.StorageDealProposal { data := make([]byte, size) rand.Read(data) - commP, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(data), size) + commP, err := (§orbuilder.SectorBuilder{}).GeneratePieceCommitment(bytes.NewReader(data), size) if err != nil { t.Fatal(err) } @@ -262,7 +262,6 @@ func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Addre prop := actors.StorageDealProposal{ PieceRef: commP[:], PieceSize: size, - //PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid Client: client, Provider: miner, diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index a3fb269bb..022baa1b8 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -649,7 +649,7 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type }) } - commd, err := sectorbuilder.GenerateDataCommitment(params.SectorSize, pieces) + commd, err := vmctx.Sys().GenerateDataCommitment(params.SectorSize, pieces) if err != nil { return nil, aerrors.Absorb(err, 6, "failed to generate data commitment from pieces") } diff --git a/chain/actors/actors_test.go b/chain/actors/actors_test.go index 64207b422..d0f8fba86 100644 --- a/chain/actors/actors_test.go +++ b/chain/actors/actors_test.go @@ -50,10 +50,10 @@ func setupVMTestEnv(t *testing.T) (*vm.VM, []address.Address, bstore.Blockstore) t.Fatal(err) } - cs := store.NewChainStore(bs, nil) + cs := store.NewChainStore(bs, nil, nil) // TODO: should probabaly mock out the randomness bit, nil works for now - vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore()) + vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore(), cs.VMSys()) if err != nil { t.Fatal(err) } diff --git a/chain/actors/harness2_test.go b/chain/actors/harness2_test.go index 2739d0cc8..213d1d48e 100644 --- a/chain/actors/harness2_test.go +++ b/chain/actors/harness2_test.go @@ -3,6 +3,7 @@ package actors_test import ( "bytes" "context" + "github.com/filecoin-project/go-sectorbuilder" "math/rand" "testing" @@ -194,8 +195,8 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { t.Fatal(err) } - h.cs = store.NewChainStore(h.bs, nil) - h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore()) + h.cs = store.NewChainStore(h.bs, nil, vm.Syscalls(sectorbuilder.ProofVerifier)) + h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore(), h.cs.VMSys()) if err != nil { t.Fatal(err) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 0b396ad06..bf0c7a2b9 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + "github.com/filecoin-project/lotus/chain/vm" "io/ioutil" "sync/atomic" @@ -170,7 +171,9 @@ func NewGenerator() (*ChainGen, error) { MinerAddrs: []address.Address{maddr1, maddr2}, } - genb, err := MakeGenesisBlock(bs, map[address.Address]types.BigInt{ + sys := vm.Syscalls(sectorbuilder.ProofVerifier) + + genb, err := MakeGenesisBlock(bs, sys, map[address.Address]types.BigInt{ mk1: types.FromFil(40000), mk2: types.FromFil(40000), banker: types.FromFil(50000), @@ -179,7 +182,7 @@ func NewGenerator() (*ChainGen, error) { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds) + cs := store.NewChainStore(bs, ds, sys) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 3f528075e..8b53e7c7a 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -275,7 +275,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte { } func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) { - vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore()) + vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore(), cs.VMSys()) if err != nil { return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err) } @@ -555,7 +555,7 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value return ret.Return, nil } -func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) { +func MakeGenesisBlock(bs bstore.Blockstore, sys *types.VMSyscalls, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) { ctx := context.Background() state, err := MakeInitialStateTree(bs, balances) @@ -569,7 +569,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore()) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys) stateroot, deals, err := SetupStorageMiners(ctx, cs, stateroot, gmcfg) if err != nil { return nil, xerrors.Errorf("setup storage miners failed: %w", err) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index cfbe729c0..bc4a2f9fc 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -18,7 +18,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw") defer span.End() - vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore()) + vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore(), sm.cs.VMSys()) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 34204fb3e..310f0a9e9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -139,7 +139,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl r := store.NewChainRand(sm.cs, cids, blks[0].Height) - vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore()) + vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore(), sm.cs.VMSys()) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err) } diff --git a/chain/store/store.go b/chain/store/store.go index 07d1c692c..c75f1ffa0 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -51,9 +51,11 @@ type ChainStore struct { mmCache *lru.ARCCache tsCache *lru.ARCCache + + vmcalls *types.VMSyscalls } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls *types.VMSyscalls) *ChainStore { c, _ := lru.NewARC(2048) tsc, _ := lru.NewARC(4096) cs := &ChainStore{ @@ -63,6 +65,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore { tipsets: make(map[uint64][]cid.Cid), mmCache: c, tsCache: tsc, + vmcalls: vmcalls, } cs.reorgCh = cs.reorgWorker(context.TODO()) @@ -793,6 +796,11 @@ func (cs *ChainStore) Blockstore() bstore.Blockstore { return cs.bs } +func (cs *ChainStore) VMSys() *types.VMSyscalls { + return cs.vmcalls +} + + func (cs *ChainStore) TryFillTipSet(ts *types.TipSet) (*FullTipSet, error) { var out []*types.FullBlock diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 4d6d11c1b..c3c7a2ef4 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -55,7 +55,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds) + cs := store.NewChainStore(bs, mds, nil) b.ResetTimer() diff --git a/chain/store/weight.go b/chain/store/weight.go index 901356728..49ef9ef26 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -60,7 +60,7 @@ func (cs *ChainStore) call(ctx context.Context, msg *types.Message, ts *types.Ti r := NewChainRand(cs, ts.Cids(), ts.Height()) - vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs) + vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs, cs.vmcalls) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/sync.go b/chain/sync.go index e5a79133c..7c8e13b36 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -690,7 +690,7 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc } hvrf := sha256.Sum256(h.EPostProof.PostRand) - ok, err := sectorbuilder.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner) + ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner) if err != nil { return xerrors.Errorf("failed to verify election post: %w", err) } diff --git a/chain/types/vmcontext.go b/chain/types/vmcontext.go index 1af2aa3da..609044d3c 100644 --- a/chain/types/vmcontext.go +++ b/chain/types/vmcontext.go @@ -2,6 +2,7 @@ package types import ( "context" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-amt-ipld" @@ -46,7 +47,17 @@ type VMContext interface { } type VMSyscalls struct { - ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError) + ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError) + VerifyFallbackPost func(ctx context.Context, + sectorSize uint64, + sectorInfo sectorbuilder.SortedPublicSectorInfo, + challengeSeed []byte, + proof []byte, + candidates []sectorbuilder.EPostCandidate, + proverID address.Address, + faults uint64) (bool, error) + + GenerateDataCommitment func(ssize uint64, pieces []sectorbuilder.PublicPieceInfo) ([sectorbuilder.CommLen]byte, error) } type storageWrapper struct { diff --git a/chain/validation/applier.go b/chain/validation/applier.go index 4e21f1228..8caf1b2dd 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -2,6 +2,7 @@ package validation import ( "context" + "github.com/filecoin-project/go-sectorbuilder" vchain "github.com/filecoin-project/chain-validation/pkg/chain" vstate "github.com/filecoin-project/chain-validation/pkg/state" @@ -32,7 +33,7 @@ func (a *Applier) ApplyMessage(eCtx *vchain.ExecutionContext, state vstate.Wrapp if err != nil { return vchain.MessageReceipt{}, err } - lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs) + lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs, vm.Syscalls(sectorbuilder.ProofVerifier)) if err != nil { return vchain.MessageReceipt{}, err } diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 92f9ea269..5c83c654f 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -1,14 +1,30 @@ package vm import ( + "context" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" + "go.opencensus.io/trace" ) // Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there -func DefaultSyscalls() *types.VMSyscalls { +func Syscalls(verifier sectorbuilder.Verifier) *types.VMSyscalls { return &types.VMSyscalls{ - ValidatePoRep: actors.ValidatePoRep, + ValidatePoRep: func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, actors.ActorError) { + _, span := trace.StartSpan(ctx, "ValidatePoRep") + defer span.End() + ok, err := verifier.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof) + if err != nil { + return false, aerrors.Absorb(err, 25, "verify seal failed") + } + + return ok, nil + }, + VerifyFallbackPost: verifier.VerifyFallbackPost, + GenerateDataCommitment: verifier.GenerateDataCommitment, } } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index a038211b1..a0ab66999 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -312,7 +312,7 @@ type VM struct { Syscalls *types.VMSyscalls } -func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) { +func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore, syscalls *types.VMSyscalls) (*VM, error) { buf := bufbstore.NewBufferedBstore(cbs) cst := hamt.CSTFromBstore(buf) state, err := state.LoadStateTree(cst, base) @@ -328,8 +328,8 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block blockHeight: height, blockMiner: maddr, inv: newInvoker(), - rand: r, - Syscalls: DefaultSyscalls(), + rand: r, // TODO: Probably should be a syscall + Syscalls: syscalls, }, nil } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 6bd212c78..2cd059a2c 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -213,7 +213,7 @@ func main() { sealcommit := time.Now() commD := pi.CommP - ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof) + ok, err := sectorbuilder.ProofVerifier.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof) if err != nil { return err } @@ -307,7 +307,7 @@ func main() { log.Warn("separate epost calls returned different proof values (this might be bad)") } - ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr) + ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr) if err != nil { return err } @@ -317,7 +317,7 @@ func main() { verifypost1 := time.Now() - ok, err = sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr) + ok, err = sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr) if err != nil { return err } diff --git a/go.mod b/go.mod index afbe0c0a9..55657aff6 100644 --- a/go.mod +++ b/go.mod @@ -109,3 +109,7 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi + +replace github.com/filecoin-project/go-sectorbuilder => /home/magik6k/gohack/github.com/filecoin-project/go-sectorbuilder + +replace github.com/filecoin-project/go-fil-markets => /home/magik6k/gohack/github.com/filecoin-project/go-fil-markets diff --git a/node/builder.go b/node/builder.go index dd0af277d..e2563c81e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -23,6 +23,7 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/go-fil-markets/storagemarket" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" @@ -33,6 +34,7 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" @@ -193,6 +195,8 @@ func Online() Option { Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), + Override(new(sectorbuilder.Verifier), sectorbuilder.ProofVerifier), + Override(new(*types.VMSyscalls), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), Override(new(*wallet.Wallet), wallet.NewWallet), @@ -239,7 +243,7 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, - Override(new(storage.SectorBuilder), modules.SectorBuilder), + Override(new(sectorbuilder.Interface), modules.SectorBuilder), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index ac149b64b..9bd5b4c73 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -26,7 +26,7 @@ type StorageMinerAPI struct { CommonAPI SectorBuilderConfig *sectorbuilder.Config - SectorBuilder storage.SectorBuilder + SectorBuilder sectorbuilder.Interface SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner diff --git a/node/modules/chain.go b/node/modules/chain.go index bc0ff3ce2..b70cb6892 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -73,8 +73,8 @@ func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtyp return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS) *store.ChainStore { - chain := store.NewChainStore(bs, ds) +func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls *types.VMSyscalls) *store.ChainStore { + chain := store.NewChainStore(bs, ds, syscalls) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 822b70baa..ae9780560 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -98,7 +98,7 @@ func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit } } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb storage.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb sectorbuilder.Interface, tktFn storage.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index bfb6e26c5..a5cc2e68b 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -30,8 +30,8 @@ import ( var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { - return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { +func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { + return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") defk, err := w.GenerateKey(types.KTBLS) @@ -51,7 +51,7 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo alloc[waddr] = types.FromFil(10000) } - b, err := gen.MakeGenesisBlock(bs, alloc, gmc, 100000) + b, err := gen.MakeGenesisBlock(bs, syscalls, alloc, gmc, 100000) if err != nil { return nil, err } @@ -68,8 +68,8 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo } } -func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { - return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { +func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { + return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") presealInfo, err := homedir.Expand(presealInfo) @@ -130,7 +130,7 @@ func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlo ts = uint64(t.Unix()) } - b, err := gen.MakeGenesisBlock(bs, addrs, gmc, ts) + b, err := gen.MakeGenesisBlock(bs, syscalls, addrs, gmc, ts) if err != nil { return nil, err } diff --git a/node/node_test.go b/node/node_test.go index eb063ff58..e813c39b5 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -36,6 +36,7 @@ import ( "github.com/filecoin-project/lotus/node/modules" modtest "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sbmock" ) func init() { @@ -102,6 +103,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a node.Test(), node.MockHost(mn), + node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])), node.Override(new(api.FullNode), tnd), node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)), @@ -247,6 +249,110 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te return fulls, storers } +func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) { + ctx := context.Background() + mn := mocknet.New(ctx) + + fulls := make([]test.TestNode, nFull) + storers := make([]test.TestStorageNode, len(storage)) + + pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + minerPid, err := peer.IDFromPrivateKey(pk) + require.NoError(t, err) + + var genbuf bytes.Buffer + + if len(storage) > 1 { + panic("need more peer IDs") + } + // PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE + // TODO: would be great if there was a better way to fake the preseals + gmc := &gen.GenMinerCfg{ + PeerIDs: []peer.ID{minerPid}, // TODO: if we have more miners, need more peer IDs + PreSeals: map[string]genesis.GenesisMiner{}, + } + + var presealDirs []string + for i := 0; i < len(storage); i++ { + maddr, err := address.NewIDAddress(300 + uint64(i)) + if err != nil { + t.Fatal(err) + } + tdir, err := ioutil.TempDir("", "preseal-memgen") + if err != nil { + t.Fatal(err) + } + genm, err := sbmock.PreSeal(1024, maddr, 1) + if err != nil { + t.Fatal(err) + } + + presealDirs = append(presealDirs, tdir) + gmc.MinerAddrs = append(gmc.MinerAddrs, maddr) + gmc.PreSeals[maddr.String()] = *genm + } + + // END PRESEAL SECTION + + for i := 0; i < nFull; i++ { + var genesis node.Option + if i == 0 { + genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, gmc)) + } else { + genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) + } + + var err error + // TODO: Don't ignore stop + _, err = node.New(ctx, + node.FullAPI(&fulls[i].FullNode), + node.Online(), + node.Repo(repo.NewMemory(nil)), + node.MockHost(mn), + node.Test(), + + node.Override(new(sectorbuilder.Verifier), sbmock.MockVerifier), + + genesis, + ) + if err != nil { + t.Fatal(err) + } + + } + + for i, full := range storage { + // TODO: support non-bootstrap miners + if i != 0 { + t.Fatal("only one storage node supported") + } + if full != 0 { + t.Fatal("storage nodes only supported on the first full node") + } + + f := fulls[full] + + genMiner := gmc.MinerAddrs[i] + wa := gmc.PreSeals[genMiner.String()].Worker + + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn) + + /*if err := sma.SectorBuilder.ImportFrom(osb, false); err != nil { + t.Fatal(err) + }*/ + + } + + if err := mn.LinkAll(); err != nil { + t.Fatal(err) + } + + return fulls, storers +} + + func TestAPI(t *testing.T) { test.TestApis(t, builder) } @@ -292,5 +398,5 @@ func TestAPIDealFlow(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } - test.TestDealFlow(t, builder) + test.TestDealFlow(t, mockSbBuilder) } diff --git a/storage/fpost_sched.go b/storage/fpost_sched.go index 30dadbd56..cafb212b6 100644 --- a/storage/fpost_sched.go +++ b/storage/fpost_sched.go @@ -8,6 +8,8 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -19,7 +21,7 @@ const StartConfidence = 4 // TODO: config type fpostScheduler struct { api storageMinerApi - sb SectorBuilder + sb sectorbuilder.Interface actor address.Address worker address.Address diff --git a/storage/garbage.go b/storage/garbage.go index a5470cdaa..5da1771b2 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -22,7 +22,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece deals := make([]actors.StorageDealProposal, len(sizes)) for i, size := range sizes { release := m.sb.RateLimit() - commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) + commP, err := m.sb.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) release() if err != nil { diff --git a/storage/miner.go b/storage/miner.go index 90c4d9fac..02022d03f 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,7 +3,6 @@ package storage import ( "context" "errors" - "io" "time" "github.com/ipfs/go-cid" @@ -37,7 +36,7 @@ type Miner struct { worker address.Address // Sealing - sb SectorBuilder + sb sectorbuilder.Interface sectors *statestore.StateStore tktFn TicketFn @@ -72,24 +71,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -type SectorBuilder interface { - RateLimit() func() - AddPiece(uint64, uint64, io.Reader, []uint64) (sectorbuilder.PublicPieceInfo, error) - SectorSize() uint64 - AcquireSectorId() (uint64, error) - Scrub(sectorbuilder.SortedPublicSectorInfo) []*sectorbuilder.Fault - GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error) - SealPreCommit(context.Context, uint64, sectorbuilder.SealTicket, []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) - SealCommit(context.Context, uint64, sectorbuilder.SealTicket, sectorbuilder.SealSeed, []sectorbuilder.PublicPieceInfo, sectorbuilder.RawSealPreCommitOutput) ([]byte, error) - - // Not so sure about these being on the interface - GetPath(string, string) (string, error) - WorkerStats() sectorbuilder.WorkerStats - AddWorker(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) - TaskDone(context.Context, uint64, sectorbuilder.SealRes) error -} - -func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb SectorBuilder, tktFn TicketFn) (*Miner, error) { +func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb sectorbuilder.Interface, tktFn TicketFn) (*Miner, error) { return &Miner{ api: api, @@ -162,11 +144,11 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { } type SectorBuilderEpp struct { - sb *sectorbuilder.SectorBuilder + sb sectorbuilder.Interface } -func NewElectionPoStProver(sb SectorBuilder) *SectorBuilderEpp { - return &SectorBuilderEpp{sb.(*sectorbuilder.SectorBuilder)} +func NewElectionPoStProver(sb sectorbuilder.Interface) *SectorBuilderEpp { + return &SectorBuilderEpp{sb} } var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil) diff --git a/storage/sbmock/preseal.go b/storage/sbmock/preseal.go new file mode 100644 index 000000000..ca498a134 --- /dev/null +++ b/storage/sbmock/preseal.go @@ -0,0 +1,52 @@ +package sbmock + +import ( + "math" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/genesis" +) + +func PreSeal(ssize uint64, maddr address.Address, sectors int) (*genesis.GenesisMiner, error) { + k, err := wallet.GenerateKey(types.KTBLS) + if err != nil { + return nil, err + } + + genm := &genesis.GenesisMiner{ + Owner: k.Address, + Worker: k.Address, + SectorSize: ssize, + Sectors: make([]*genesis.PreSeal, sectors), + Key: k.KeyInfo, + } + + for i := range genm.Sectors { + preseal := &genesis.PreSeal{} + sdata := randB(sectorbuilder.UserBytesForSectorSize(ssize)) + + preseal.CommD = commD(sdata) + preseal.CommR = commDR(preseal.CommD[:]) + preseal.SectorID = uint64(i + 1) + preseal.Deal = actors.StorageDealProposal{ + PieceRef: preseal.CommD[:], + PieceSize: sectorbuilder.UserBytesForSectorSize(ssize), + Client: maddr, + Provider: maddr, + ProposalExpiration: math.MaxUint64, + Duration: math.MaxUint64, + StoragePricePerEpoch: types.NewInt(0), + StorageCollateral: types.NewInt(0), + ProposerSignature: nil, + } + + genm.Sectors[i] = preseal + } + + return genm, nil +} \ No newline at end of file diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index 73905db28..191751c2d 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -6,19 +6,16 @@ import ( "fmt" "io" "io/ioutil" + "math/big" "math/rand" "sync" + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" ) -func randComm() [sectorbuilder.CommLen]byte { - var out [sectorbuilder.CommLen]byte - rand.Read(out[:]) - return out -} - type SBMock struct { sectors map[uint64]*sectorState sectorSize uint64 @@ -28,11 +25,13 @@ type SBMock struct { lk sync.Mutex } +type mockVerif struct {} + func NewMockSectorBuilder(threads int, ssize uint64) *SBMock { return &SBMock{ sectors: make(map[uint64]*sectorState), sectorSize: ssize, - nextSectorID: 0, + nextSectorID: 5, rateLimit: make(chan struct{}, threads), } } @@ -82,7 +81,7 @@ func (sb *SBMock) AddPiece(size uint64, sectorId uint64, r io.Reader, existingPi ss.pieces = append(ss.pieces, b) return sectorbuilder.PublicPieceInfo{ Size: size, - // TODO: should we compute a commP? maybe do it when we need it + CommP: commD(b), }, nil } @@ -158,9 +157,22 @@ func (sb *SBMock) SealPreCommit(ctx context.Context, sid uint64, ticket sectorbu ss.state = statePreCommit + pis := make([]ffi.PublicPieceInfo, len(ss.pieces)) + for i, piece := range ss.pieces { + pis[i] = ffi.PublicPieceInfo{ + Size: uint64(len(piece)), + CommP: commD(piece), + } + } + + commd, err := MockVerifier.GenerateDataCommitment(sb.sectorSize, pis) + if err != nil { + return sectorbuilder.RawSealPreCommitOutput{}, err + } + return sectorbuilder.RawSealPreCommitOutput{ - CommD: randComm(), - CommR: randComm(), + CommD: commd, + CommR: commDR(commd[:]), }, nil } @@ -184,9 +196,11 @@ func (sb *SBMock) SealCommit(ctx context.Context, sid uint64, ticket sectorbuild opFinishWait(ctx) - buf := make([]byte, 32) - rand.Read(buf) - return buf, nil + var out [32]byte + for i := range out { + out[i] = precommit.CommD[i] + precommit.CommR[31 - i] - ticket.TicketBytes[i] * seed.TicketBytes[i] + } + return out[:], nil } func (sb *SBMock) GetPath(string, string) (string, error) { @@ -235,6 +249,45 @@ func AddOpFinish(ctx context.Context) (context.Context, func()) { } } +func (sb *SBMock) ComputeElectionPoSt(sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, winners []sectorbuilder.EPostCandidate) ([]byte, error) { + panic("implement me") +} + +func (sb *SBMock) GenerateEPostCandidates(sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed [sectorbuilder.CommLen]byte, faults []uint64) ([]sectorbuilder.EPostCandidate, error) { + if len(faults) > 0 { + panic("todo") + } + + n := sectorbuilder.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), uint64(len(faults))) + if n > uint64(len(sectorInfo.Values())) { + n = uint64(len(sectorInfo.Values())) + } + + out := make([]sectorbuilder.EPostCandidate, len(sectorInfo.Values())) + + seed := big.NewInt(0).SetBytes(challengeSeed[:]) + start := seed.Mod(seed, big.NewInt(int64(len(sectorInfo.Values())))).Int64() + + for i := 0; uint64(i) < n; i++ { + out[i] = sectorbuilder.EPostCandidate{ + SectorID: uint64((int(start) + i) % len(sectorInfo.Values())), + PartialTicket: challengeSeed, + Ticket: commDR(challengeSeed[:]), + SectorChallengeIndex: 0, + } + } + + return out, nil +} + +func (sb *SBMock) GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [sectorbuilder.CommLen]byte, err error) { + panic("implement me") +} + +func (sb *SBMock) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) { + panic("implement me") +} + func (sb *SBMock) StageFakeData() (uint64, []sectorbuilder.PublicPieceInfo, error) { usize := sectorbuilder.UserBytesForSectorSize(sb.sectorSize) sid, err := sb.AcquireSectorId() @@ -252,3 +305,40 @@ func (sb *SBMock) StageFakeData() (uint64, []sectorbuilder.PublicPieceInfo, erro return sid, []sectorbuilder.PublicPieceInfo{pi}, nil } + +func (m mockVerif) VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []sectorbuilder.EPostCandidate, proverID address.Address) (bool, error) { + panic("implement me") +} + +func (m mockVerif) VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []sectorbuilder.EPostCandidate, proverID address.Address, faults uint64) (bool, error) { + panic("implement me") +} + +func (m mockVerif) VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) { + if len(proof) != 32 { // Real ones are longer, but this should be fine + return false, nil + } + + for i, b := range proof { + if b != commD[i] + commR[31 - i] - ticket[i] * seed[i] { + return false, nil + } + } + + return true, nil +} + +func (m mockVerif) GenerateDataCommitment(ssize uint64, pieces []ffi.PublicPieceInfo) ([sectorbuilder.CommLen]byte, error) { + if len(pieces) != 1 { + panic("todo") + } + if pieces[0].Size != sectorbuilder.UserBytesForSectorSize(ssize) { + panic("todo") + } + return pieces[0].CommP, nil +} + +var MockVerifier = mockVerif{} + +var _ sectorbuilder.Verifier = MockVerifier +var _ sectorbuilder.Interface = &SBMock{} diff --git a/storage/sbmock/util.go b/storage/sbmock/util.go new file mode 100644 index 000000000..30a60ea32 --- /dev/null +++ b/storage/sbmock/util.go @@ -0,0 +1,28 @@ +package sbmock + +import ( + "crypto/rand" + "crypto/sha256" + "io" + "io/ioutil" +) + +func randB(n uint64) []byte { + b, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(n))) + if err != nil { + panic(err) + } + return b +} + +func commDR(in []byte) (out [32]byte) { + for i, b := range in { + out[i] = ^b + } + + return out +} + +func commD(b []byte) [32]byte { + return sha256.Sum256(b) +} diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 4b80d0fb6..ed3a718df 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -39,7 +39,7 @@ var ErrNotFound = errors.New("not found") type SectorBlocks struct { *storage.Miner - sb *sectorbuilder.SectorBuilder + sb sectorbuilder.Interface intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore @@ -47,10 +47,10 @@ type SectorBlocks struct { keyLk sync.Mutex } -func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb storage.SectorBuilder) *SectorBlocks { +func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb sectorbuilder.Interface) *SectorBlocks { sbc := &SectorBlocks{ Miner: miner, - sb: sb.(*sectorbuilder.SectorBuilder), + sb: sb, intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)), From 2ca0c152ba382ccd1604c2457b627eccc70c4c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 02:24:04 +0100 Subject: [PATCH 56/61] Deal tests on mock sectorbuilder --- chain/actors/actor_storagemarket.go | 2 +- chain/types/vmcontext.go | 2 -- chain/vm/syscalls.go | 1 - node/modules/client.go | 28 ++++++++++++---------------- storage/garbage.go | 2 +- storage/sbmock/sbmock.go | 13 ++++++------- storage/sbmock/util.go | 10 ++++++++-- 7 files changed, 28 insertions(+), 30 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 022baa1b8..a3fb269bb 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -649,7 +649,7 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type }) } - commd, err := vmctx.Sys().GenerateDataCommitment(params.SectorSize, pieces) + commd, err := sectorbuilder.GenerateDataCommitment(params.SectorSize, pieces) if err != nil { return nil, aerrors.Absorb(err, 6, "failed to generate data commitment from pieces") } diff --git a/chain/types/vmcontext.go b/chain/types/vmcontext.go index 609044d3c..b6eac6ca2 100644 --- a/chain/types/vmcontext.go +++ b/chain/types/vmcontext.go @@ -56,8 +56,6 @@ type VMSyscalls struct { candidates []sectorbuilder.EPostCandidate, proverID address.Address, faults uint64) (bool, error) - - GenerateDataCommitment func(ssize uint64, pieces []sectorbuilder.PublicPieceInfo) ([sectorbuilder.CommLen]byte, error) } type storageWrapper struct { diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 5c83c654f..39e599130 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -25,6 +25,5 @@ func Syscalls(verifier sectorbuilder.Verifier) *types.VMSyscalls { return ok, nil }, VerifyFallbackPost: verifier.VerifyFallbackPost, - GenerateDataCommitment: verifier.GenerateDataCommitment, } } diff --git a/node/modules/client.go b/node/modules/client.go index 3405bae91..b113a5968 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -5,40 +5,36 @@ import ( "path/filepath" "reflect" - "github.com/filecoin-project/lotus/markets/retrievaladapter" - + "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" - "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/node/modules/helpers" - "github.com/filecoin-project/lotus/paych" - "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" - graphsync "github.com/ipfs/go-graphsync/impl" - "github.com/ipfs/go-graphsync/ipldbridge" - gsnet "github.com/ipfs/go-graphsync/network" - "github.com/ipfs/go-graphsync/storeutil" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/routing" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-filestore" + graphsync "github.com/ipfs/go-graphsync/impl" + "github.com/ipfs/go-graphsync/ipldbridge" + gsnet "github.com/ipfs/go-graphsync/network" + "github.com/ipfs/go-graphsync/storeutil" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" "go.uber.org/fx" - "github.com/filecoin-project/go-data-transfer/impl/graphsync" - deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" - storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/lotus/markets/retrievaladapter" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/paych" ) func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) { diff --git a/storage/garbage.go b/storage/garbage.go index 5da1771b2..a5470cdaa 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -22,7 +22,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece deals := make([]actors.StorageDealProposal, len(sizes)) for i, size := range sizes { release := m.sb.RateLimit() - commP, err := m.sb.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) + commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) release() if err != nil { diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index 191751c2d..195289f71 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -263,12 +263,12 @@ func (sb *SBMock) GenerateEPostCandidates(sectorInfo sectorbuilder.SortedPublicS n = uint64(len(sectorInfo.Values())) } - out := make([]sectorbuilder.EPostCandidate, len(sectorInfo.Values())) + out := make([]sectorbuilder.EPostCandidate, n) seed := big.NewInt(0).SetBytes(challengeSeed[:]) start := seed.Mod(seed, big.NewInt(int64(len(sectorInfo.Values())))).Int64() - for i := 0; uint64(i) < n; i++ { + for i := range out { out[i] = sectorbuilder.EPostCandidate{ SectorID: uint64((int(start) + i) % len(sectorInfo.Values())), PartialTicket: challengeSeed, @@ -280,12 +280,11 @@ func (sb *SBMock) GenerateEPostCandidates(sectorInfo sectorbuilder.SortedPublicS return out, nil } -func (sb *SBMock) GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [sectorbuilder.CommLen]byte, err error) { - panic("implement me") -} - func (sb *SBMock) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) { - panic("implement me") + if len(sb.sectors[sectorID].pieces) > 1 { + panic("implme") + } + return ioutil.NopCloser(io.LimitReader(bytes.NewReader(sb.sectors[sectorID].pieces[0][offset:]), int64(size))), nil } func (sb *SBMock) StageFakeData() (uint64, []sectorbuilder.PublicPieceInfo, error) { diff --git a/storage/sbmock/util.go b/storage/sbmock/util.go index 30a60ea32..1fb5206e7 100644 --- a/storage/sbmock/util.go +++ b/storage/sbmock/util.go @@ -1,10 +1,12 @@ package sbmock import ( + "bytes" "crypto/rand" - "crypto/sha256" "io" "io/ioutil" + + "github.com/filecoin-project/go-sectorbuilder" ) func randB(n uint64) []byte { @@ -24,5 +26,9 @@ func commDR(in []byte) (out [32]byte) { } func commD(b []byte) [32]byte { - return sha256.Sum256(b) + c, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(b), uint64(len(b))) + if err != nil { + panic(err) + } + return c } From 94a21e3aebc2139dc45c2ee912602030ccfc663e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 03:04:33 +0100 Subject: [PATCH 57/61] Update deps --- api/test/deals.go | 2 +- chain/actors/actor_miner_test.go | 2 +- chain/store/store.go | 1 - chain/vm/syscalls.go | 2 +- go.mod | 10 +++------- go.sum | 6 ++++++ node/node_test.go | 23 ++++++++++++----------- storage/sbmock/preseal.go | 2 +- storage/sbmock/sbmock.go | 10 +++++----- storage/sector_utils.go | 2 +- storage/sectors.go | 2 +- 11 files changed, 32 insertions(+), 30 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 30faee4eb..4d4b2a2a3 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -64,7 +64,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { go func() { defer close(done) for mine { - time.Sleep(100 * time.Millisecond) + time.Sleep(10 * time.Millisecond) fmt.Println("mining a block now") if err := sn[0].MineOne(ctx); err != nil { t.Error(err) diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go index 410dfcdc3..e08342248 100644 --- a/chain/actors/actor_miner_test.go +++ b/chain/actors/actor_miner_test.go @@ -254,7 +254,7 @@ func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Bl func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Address, size uint64) *actors.StorageDealProposal { data := make([]byte, size) rand.Read(data) - commP, err := (§orbuilder.SectorBuilder{}).GeneratePieceCommitment(bytes.NewReader(data), size) + commP, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(data), size) if err != nil { t.Fatal(err) } diff --git a/chain/store/store.go b/chain/store/store.go index c75f1ffa0..5bd1bfdf5 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -800,7 +800,6 @@ func (cs *ChainStore) VMSys() *types.VMSyscalls { return cs.vmcalls } - func (cs *ChainStore) TryFillTipSet(ts *types.TipSet) (*FullTipSet, error) { var out []*types.FullBlock diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 39e599130..fa3ea18c9 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -24,6 +24,6 @@ func Syscalls(verifier sectorbuilder.Verifier) *types.VMSyscalls { return ok, nil }, - VerifyFallbackPost: verifier.VerifyFallbackPost, + VerifyFallbackPost: verifier.VerifyFallbackPost, } } diff --git a/go.mod b/go.mod index 55657aff6..a46e9a9b2 100644 --- a/go.mod +++ b/go.mod @@ -16,10 +16,10 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce - github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 + github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 github.com/filecoin-project/go-paramfetch v0.0.1 - github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 - github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 + github.com/filecoin-project/go-sectorbuilder v0.0.1 + github.com/filecoin-project/go-statestore v0.1.0 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gorilla/mux v1.7.3 @@ -109,7 +109,3 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi - -replace github.com/filecoin-project/go-sectorbuilder => /home/magik6k/gohack/github.com/filecoin-project/go-sectorbuilder - -replace github.com/filecoin-project/go-fil-markets => /home/magik6k/gohack/github.com/filecoin-project/go-fil-markets diff --git a/go.sum b/go.sum index a615db69c..642078758 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= +github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 h1:g3oodvSz+Ou+ObwcVBB2wyt8SHdWpwzMiNJ19U1zZNA= +github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8/go.mod h1:c8NTjvFVy1Ud02mmGDjOiMeawY2t6ALfrrdvAB01FQc= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -120,9 +122,13 @@ github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyC github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs= +github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= +github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= +github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= diff --git a/node/node_test.go b/node/node_test.go index e813c39b5..5bc80649b 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -46,7 +46,7 @@ func init() { build.MinimumMinerPower = 1024 } -func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode { +func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet, opts node.Option) test.TestStorageNode { r := repo.NewMemory(nil) lr, err := r.Lock(repo.StorageMiner) @@ -103,10 +103,11 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a node.Test(), node.MockHost(mn), - node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])), node.Override(new(api.FullNode), tnd), node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)), + + opts, ) if err != nil { t.Fatalf("failed to construct node: %v", err) @@ -216,7 +217,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te genMiner := gmc.MinerAddrs[i] wa := gmc.PreSeals[genMiner.String()].Worker - storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn) + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options()) sma := storers[i].StorageMiner.(*impl.StorageMinerAPI) @@ -337,12 +338,9 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t genMiner := gmc.MinerAddrs[i] wa := gmc.PreSeals[genMiner.String()].Worker - storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn) - - /*if err := sma.SectorBuilder.ImportFrom(osb, false); err != nil { - t.Fatal(err) - }*/ - + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options( + node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])), + )) } if err := mn.LinkAll(); err != nil { @@ -352,7 +350,6 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t return fulls, storers } - func TestAPI(t *testing.T) { test.TestApis(t, builder) } @@ -395,8 +392,12 @@ func TestAPIRPC(t *testing.T) { } func TestAPIDealFlow(t *testing.T) { + test.TestDealFlow(t, mockSbBuilder) +} + +func TestAPIDealFlowReal(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } - test.TestDealFlow(t, mockSbBuilder) + test.TestDealFlow(t, builder) } diff --git a/storage/sbmock/preseal.go b/storage/sbmock/preseal.go index ca498a134..ad4fc4f63 100644 --- a/storage/sbmock/preseal.go +++ b/storage/sbmock/preseal.go @@ -49,4 +49,4 @@ func PreSeal(ssize uint64, maddr address.Address, sectors int) (*genesis.Genesis } return genm, nil -} \ No newline at end of file +} diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index 195289f71..b8ca9b69e 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -25,7 +25,7 @@ type SBMock struct { lk sync.Mutex } -type mockVerif struct {} +type mockVerif struct{} func NewMockSectorBuilder(threads int, ssize uint64) *SBMock { return &SBMock{ @@ -80,7 +80,7 @@ func (sb *SBMock) AddPiece(size uint64, sectorId uint64, r io.Reader, existingPi ss.pieces = append(ss.pieces, b) return sectorbuilder.PublicPieceInfo{ - Size: size, + Size: size, CommP: commD(b), }, nil } @@ -160,7 +160,7 @@ func (sb *SBMock) SealPreCommit(ctx context.Context, sid uint64, ticket sectorbu pis := make([]ffi.PublicPieceInfo, len(ss.pieces)) for i, piece := range ss.pieces { pis[i] = ffi.PublicPieceInfo{ - Size: uint64(len(piece)), + Size: uint64(len(piece)), CommP: commD(piece), } } @@ -198,7 +198,7 @@ func (sb *SBMock) SealCommit(ctx context.Context, sid uint64, ticket sectorbuild var out [32]byte for i := range out { - out[i] = precommit.CommD[i] + precommit.CommR[31 - i] - ticket.TicketBytes[i] * seed.TicketBytes[i] + out[i] = precommit.CommD[i] + precommit.CommR[31-i] - ticket.TicketBytes[i]*seed.TicketBytes[i] } return out[:], nil } @@ -319,7 +319,7 @@ func (m mockVerif) VerifySeal(sectorSize uint64, commR, commD []byte, proverID a } for i, b := range proof { - if b != commD[i] + commR[31 - i] - ticket[i] * seed[i] { + if b != commD[i]+commR[31-i]-ticket[i]*seed[i] { return false, nil } } diff --git a/storage/sector_utils.go b/storage/sector_utils.go index 664223a20..96fcbf891 100644 --- a/storage/sector_utils.go +++ b/storage/sector_utils.go @@ -52,6 +52,6 @@ func (m *Miner) ListSectors() ([]SectorInfo, error) { func (m *Miner) GetSectorInfo(sid uint64) (SectorInfo, error) { var out SectorInfo - err := m.sectors.Get(sid, &out) + err := m.sectors.Get(sid).Get(&out) return out, err } diff --git a/storage/sectors.go b/storage/sectors.go index e8894749e..bc07902b9 100644 --- a/storage/sectors.go +++ b/storage/sectors.go @@ -161,7 +161,7 @@ func (m *Miner) onSectorIncoming(sector *SectorInfo) { func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { log.Infof("Sector %d updated state to %s", update.id, api.SectorStates[update.newState]) var sector SectorInfo - err := m.sectors.Mutate(update.id, func(s *SectorInfo) error { + err := m.sectors.Get(update.id).Mutate(func(s *SectorInfo) error { if update.nonce < s.Nonce { return xerrors.Errorf("update nonce too low, ignoring (%d < %d)", update.nonce, s.Nonce) } From 3d3e85c52ddfa1b0f5cacc8a8875917de42966da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 03:04:40 +0100 Subject: [PATCH 58/61] mod tidy --- go.sum | 7 ------- 1 file changed, 7 deletions(-) diff --git a/go.sum b/go.sum index 642078758..b95fdb314 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,6 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= -github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= -github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 h1:g3oodvSz+Ou+ObwcVBB2wyt8SHdWpwzMiNJ19U1zZNA= github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8/go.mod h1:c8NTjvFVy1Ud02mmGDjOiMeawY2t6ALfrrdvAB01FQc= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= @@ -120,13 +118,8 @@ github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go. github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE= github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs= github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= -github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= -github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= -github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= From 69752b419dd99dd18f65e640e11cebb3b30de6ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 03:09:34 +0100 Subject: [PATCH 59/61] Update sectorbuilder --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a46e9a9b2..ecdaea8aa 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 github.com/filecoin-project/go-paramfetch v0.0.1 - github.com/filecoin-project/go-sectorbuilder v0.0.1 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689 github.com/filecoin-project/go-statestore v0.1.0 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index b95fdb314..a0dfb52ec 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyC github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs= github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689 h1:2cT5bhm/5I0RY+HBIPdRRrtjCwLj33Qx6DHRs9TCslY= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= From a0f2bb07556dc355dec93fd569a4bf8aa1895878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 03:51:14 +0100 Subject: [PATCH 60/61] Increase block delay for TestAPIDealFlowReal --- api/test/deals.go | 4 ++-- node/node_test.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 4d4b2a2a3..4c2fb7439 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -24,7 +24,7 @@ func init() { build.InsecurePoStValidation = true } -func TestDealFlow(t *testing.T, b APIBuilder) { +func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() @@ -64,7 +64,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { go func() { defer close(done) for mine { - time.Sleep(10 * time.Millisecond) + time.Sleep(blocktime) fmt.Println("mining a block now") if err := sn[0].MineOne(ctx); err != nil { t.Error(err) diff --git a/node/node_test.go b/node/node_test.go index 5bc80649b..ea92a92b6 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "path/filepath" "testing" + "time" "github.com/filecoin-project/go-address" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" @@ -392,12 +393,12 @@ func TestAPIRPC(t *testing.T) { } func TestAPIDealFlow(t *testing.T) { - test.TestDealFlow(t, mockSbBuilder) + test.TestDealFlow(t, mockSbBuilder, 10 * time.Millisecond) } func TestAPIDealFlowReal(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } - test.TestDealFlow(t, builder) + test.TestDealFlow(t, builder, time.Second) } From 488a156e6fb517f5f955b1a55b86830f0d3f04ec Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 14 Jan 2020 03:10:27 +0000 Subject: [PATCH 61/61] New testnet2 genesis --- build/genesis/devnet.car | Bin 36191 -> 36191 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index 162461991b0c337799e6e6b56bdb71833ab2660e..ef2fe2dababbe6141ad983a3d021719e509be87f 100644 GIT binary patch delta 845 zcmcaVi|PI>rU@c}dS`R`eP+~u7d_FT>7uae#=_XG?0ZCc8YO0T=V*59Kb2mVT2!2w zpU1e0iLr3O6D9>h8YZTuDa0gC_nNWy{quX-rknq0^i*WoFBQ0)Y11ZqJmYcu)dy!L z|FbZgtjK6+FEk;TDJdsEIa{GPGd(Y{q_ij%2pD>}Aa)TlXY)ix8%Bj}1M}Jd|1Wd-7W*&BC4fy`thpVsp=~=Pn9XMISUY2`O!gkZfRZ zU|^UWDDTIZGI@c#C1cv;Tk>Iysgw14Hcfu4pf{P_V$o(n#cyJh4HY&`{%fg1&}Wk! zt;`kbA8d5F-21_qy{7iowwVhf*sILC8$t~x@Yek0w&~ATn>@qHq@IwXW+sV>W=CdZ zzb_Mbf35oL`heZDZ*X2;9o=+!r-+r~idTO%=gmsLwYEnbDVSE9&5OQ%WTVEiSF2Q+ zboX!jnL2M;WQ_bLouEg{lK5=?H|R7oLjshLq0Nr}J0|sk?YLlBSt0u3s*fjZJPaObPu7x20 delta 866 zcmcaVi|PI>rU@c}rIAGsq;6YYx~;PR-RIa}x3HeJYiBGq+w!en!*hK8*g4L*|j&v&SJ{^x9|5{PrNU`KX3Wz=B73Bv1hmM znf%YfY_cMwp*>4OFjG=aesZ=#ab|j6Vo7OHDiARAa6#-MWX|S^j5dr40j|$(^?q>j zHS%(1-4o_Ld+SZfncZznPmg^1BO(86bNA%8Oq!FESWGs1F-tH`_F~hW+|N>{@O;I> zC41iAU}9nC%?>zIb}2czLc;FZDYK<{UH_YV7F%rAWtC-|EXJ(6xtuLpqMnc?O*gb6 z)EOYo-m>w@VznbXdgkaXU3gtV?oSgDb1*)4zd|4G3b8eC(^NT)cXcAJ|6d~Eb z;K0BzIZ)n@F?I3+c}vEW$+zUgnA0{ePL5$wn|#rtYw}|SJ*Mr9lQ~$_Hs4k}CpKA7 zanWS`o=uZ`yBAH4wp1bL)yeZM%@tH`U;A^p@%I1QMcabo>mN(EEStW#Jo{Tu@!fw3 zWi0YMlfPM-)Du$F%p_6K?8uB97;^xbfckSAD#0P`ud#?sw&tic`Kb zy^2K&H=Dhoe^0VimimVNN@~+Oa9GIwrSGEY5g-3BpPc+CNnuBWPBSwkgb5ki?D)T9 zvP9%{R