From 9691211df1726b1f5423567bb060538f999251e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 26 Aug 2019 10:02:26 +0200 Subject: [PATCH 01/30] Sealed blockstore --- chain/deals/client.go | 2 +- chain/deals/handler.go | 7 +- chain/deals/handler_states.go | 14 +- chain/deals/types.go | 2 +- go.mod | 48 +++--- go.sum | 213 +++++++++++++++++---------- node/impl/storminer.go | 2 +- retrieval/types.go | 31 ++++ storage/sealedbstore/sealedbstore.go | 120 +++++++++++++++ storage/sector/store.go | 2 +- 10 files changed, 320 insertions(+), 121 deletions(-) create mode 100644 retrieval/types.go create mode 100644 storage/sealedbstore/sealedbstore.go diff --git a/chain/deals/client.go b/chain/deals/client.go index 1789d04c4..58d2ef7e4 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -212,7 +212,7 @@ func (c *Client) VerifyParams(ctx context.Context, data cid.Cid) (*actors.PieceI func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.PieceInclVoucherData) (cid.Cid, error) { // TODO: use data proposal := StorageDealProposal{ - PieceRef: p.Data.String(), + PieceRef: p.Data, SerializationMode: SerializationUnixFs, CommP: vd.CommP[:], Size: vd.PieceSize.Uint64(), diff --git a/chain/deals/handler.go b/chain/deals/handler.go index 94f64a96b..6bf8713f1 100644 --- a/chain/deals/handler.go +++ b/chain/deals/handler.go @@ -2,13 +2,12 @@ package deals import ( "context" + "github.com/filecoin-project/go-lotus/storage/sealedbstore" "math" "github.com/filecoin-project/go-lotus/api" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/node/modules/dtypes" - "github.com/filecoin-project/go-lotus/storage/sector" - "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -35,7 +34,7 @@ type MinerDeal struct { } type Handler struct { - secst *sector.Store + secst *sealedbstore.Sealedbstore full api.FullNode // TODO: Use a custom protocol or graphsync in the future @@ -60,7 +59,7 @@ type dealUpdate struct { mut func(*MinerDeal) } -func NewHandler(ds dtypes.MetadataDS, secst *sector.Store, dag dtypes.StagingDAG, fullNode api.FullNode) (*Handler, error) { +func NewHandler(ds dtypes.MetadataDS, secst *sealedbstore.Sealedbstore, dag dtypes.StagingDAG, fullNode api.FullNode) (*Handler, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err diff --git a/chain/deals/handler_states.go b/chain/deals/handler_states.go index 486af2c83..1693d5a95 100644 --- a/chain/deals/handler_states.go +++ b/chain/deals/handler_states.go @@ -2,12 +2,13 @@ package deals import ( "context" - "github.com/filecoin-project/go-lotus/lib/sectorbuilder" - files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/lib/sectorbuilder" + "github.com/filecoin-project/go-lotus/storage/sealedbstore" ) type handlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) @@ -75,18 +76,13 @@ func (h *Handler) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), return nil, xerrors.Errorf("cannot open unixfs file: %s", err) } - uf, ok := n.(files.File) + uf, ok := n.(sealedbstore.UnixfsReader) if !ok { // we probably got directory, unsupported for now return nil, xerrors.Errorf("unsupported unixfs type") } - size, err := uf.Size() - if err != nil { - return nil, xerrors.Errorf("failed to get file size: %s", err) - } - - sectorID, err := h.secst.AddPiece(deal.Proposal.PieceRef, uint64(size), uf, deal.Proposal.Duration) + sectorID, err := h.secst.AddUnixfsPiece(deal.Proposal.PieceRef, uf, deal.Proposal.Duration) if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } diff --git a/chain/deals/types.go b/chain/deals/types.go index f03688cab..f17fb1cac 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -41,7 +41,7 @@ const ( ) type StorageDealProposal struct { - PieceRef string + PieceRef cid.Cid // TODO: port to spec SerializationMode SerializationMode CommP []byte diff --git a/go.mod b/go.mod index 7e2508c41..cf385f063 100644 --- a/go.mod +++ b/go.mod @@ -5,67 +5,66 @@ go 1.12 require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 github.com/BurntSushi/toml v0.3.1 + github.com/dgraph-io/badger v1.6.0 // indirect github.com/filecoin-project/go-bls-sigs v0.0.0-20190718224239-4bc4b8a7bbf8 github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 github.com/filecoin-project/go-sectorbuilder v0.0.0-00010101000000-000000000000 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 + github.com/golang/mock v1.3.1 // indirect github.com/gorilla/websocket v1.4.0 - github.com/ipfs/go-bitswap v0.1.6-0.20190808170517-167327fc3c5e + github.com/ipfs/go-bitswap v0.1.7-0.20190808170517-167327fc3c5e github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.2 - github.com/ipfs/go-car v0.0.1 + github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4 github.com/ipfs/go-cid v0.0.3 - github.com/ipfs/go-datastore v0.0.5 + github.com/ipfs/go-datastore v0.1.0 github.com/ipfs/go-ds-badger v0.0.5 github.com/ipfs/go-filestore v0.0.2 github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf - github.com/ipfs/go-ipfs-blockstore v0.0.1 + github.com/ipfs/go-ipfs-blockstore v0.0.2 github.com/ipfs/go-ipfs-chunker v0.0.1 + github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 - github.com/ipfs/go-ipfs-files v0.0.3 + github.com/ipfs/go-ipfs-files v0.0.4 github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipld-cbor v0.0.3 github.com/ipfs/go-ipld-format v0.0.2 github.com/ipfs/go-log v0.0.2-0.20190708183747-9c9fd6111bea - github.com/ipfs/go-merkledag v0.1.0 - github.com/ipfs/go-unixfs v0.1.0 + github.com/ipfs/go-merkledag v0.2.3 + github.com/ipfs/go-unixfs v0.2.1 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 - github.com/libp2p/go-eventbus v0.0.3 // indirect - github.com/libp2p/go-libp2p v0.2.0 - github.com/libp2p/go-libp2p-circuit v0.1.0 + github.com/libp2p/go-libp2p v0.3.0 + github.com/libp2p/go-libp2p-circuit v0.1.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 - github.com/libp2p/go-libp2p-core v0.0.6 + github.com/libp2p/go-libp2p-core v0.2.2 github.com/libp2p/go-libp2p-discovery v0.1.0 github.com/libp2p/go-libp2p-kad-dht v0.1.1 github.com/libp2p/go-libp2p-mplex v0.2.1 github.com/libp2p/go-libp2p-peer v0.2.0 - github.com/libp2p/go-libp2p-peerstore v0.1.2-0.20190621130618-cfa9bb890c1a + github.com/libp2p/go-libp2p-peerstore v0.1.3 github.com/libp2p/go-libp2p-pnet v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.1.0 github.com/libp2p/go-libp2p-quic-transport v0.1.1 - github.com/libp2p/go-libp2p-record v0.1.0 + github.com/libp2p/go-libp2p-record v0.1.1 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 - github.com/libp2p/go-libp2p-secio v0.1.0 - github.com/libp2p/go-libp2p-swarm v0.1.1 // indirect + github.com/libp2p/go-libp2p-secio v0.2.0 github.com/libp2p/go-libp2p-tls v0.1.0 github.com/libp2p/go-libp2p-yamux v0.2.1 github.com/libp2p/go-maddr-filter v0.0.5 + github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.0.4 github.com/multiformats/go-multiaddr-dns v0.0.3 github.com/multiformats/go-multiaddr-net v0.0.1 - github.com/multiformats/go-multihash v0.0.6 + github.com/multiformats/go-multihash v0.0.7 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a - github.com/smartystreets/assertions v1.0.1 // indirect - github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect - github.com/stretchr/objx v0.1.1 // indirect - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d @@ -75,11 +74,8 @@ require ( go.uber.org/fx v1.9.0 go.uber.org/goleak v0.10.0 // indirect go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect - golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect - golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect - golang.org/x/sys v0.0.0-20190730183949-1393eb018365 // indirect golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 + google.golang.org/api v0.9.0 // indirect gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) @@ -87,3 +83,7 @@ require ( replace github.com/filecoin-project/go-bls-sigs => ./extern/go-bls-sigs replace github.com/filecoin-project/go-sectorbuilder => ./extern/go-sectorbuilder + +replace github.com/ipfs/go-car => /home/magik6k/gohack/github.com/ipfs/go-car + +replace github.com/ipfs/go-unixfs => /home/magik6k/gohack/github.com/ipfs/go-unixfs diff --git a/go.sum b/go.sum index 10669286e..a578cb228 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= @@ -21,10 +22,14 @@ github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50 h1:4i3KsuVA0o0KoBxAC5x+MY7RbteiMK1V7gf/G08NGIQ= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a h1:We35J+0yvVFrEXbtViYUW8H/wNOhqjIF3PsrW4yYmGw= +github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -53,8 +58,8 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQY github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1 h1:JphPpoBZJ3WHha133BGYlQqltSGIhV+VsEID0++nN9A= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger/v2 v2.0.0-20190620211019-41d170b5158f/go.mod h1:jUaIjOV835xZ/mCLG/8P/38ZxiT4bG/K1khDNZJxuwU= -github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -63,7 +68,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 h1:aMJGfgqe1QDhAVwxRg5fjCRF533xHidiKsugk7Vvzug= github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543/go.mod h1:mjrHv1cDGJWDlGmC0eDc1E5VJr8DmL9XMUcaFwiuKg8= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -74,6 +78,7 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJY github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -83,26 +88,39 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -110,31 +128,30 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.1 h1:s7KkiBPfxCeDVo47KySjK0ACPc5GJRUxFpdyWEuDjhw= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= -github.com/ipfs/go-bitswap v0.0.1/go.mod h1:z+tP3h+HTJ810n1R5yMy2ccKFffJ2F6Vqm/5Bf7vs2c= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= -github.com/ipfs/go-bitswap v0.1.6-0.20190808170517-167327fc3c5e h1:LiCecZPwRrr6m91+HfyXiawmAT1/t7h9OXwoNjS6bjY= -github.com/ipfs/go-bitswap v0.1.6-0.20190808170517-167327fc3c5e/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= +github.com/ipfs/go-bitswap v0.1.6 h1:3jj6/69bsqAFmNViEXU8MWUDE8iE1mrqVPaKaIChu7k= +github.com/ipfs/go-bitswap v0.1.6/go.mod h1:oRNdV7SkA9glUUMHd6O2ztSwimBDLFdIF0fYIuDEzVo= +github.com/ipfs/go-bitswap v0.1.7-0.20190808170517-167327fc3c5e h1:O5Qmnj6VLa89My1nYwOto+DPJ/ZxH98PKDbIVVnahFs= +github.com/ipfs/go-bitswap v0.1.7-0.20190808170517-167327fc3c5e/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-blockservice v0.0.1/go.mod h1:2Ao89U7jV1KIqqNk5EdhSTBG/Pgc1vMFr0bhkx376j4= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc= github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= -github.com/ipfs/go-car v0.0.1 h1:Nn3RjJbysnDud4wILAybzOAvzsaycrCoKM4BMIK0Y04= -github.com/ipfs/go-car v0.0.1/go.mod h1:pUz3tUIpudsTch0ZQrEPOvNUBT1LufCXg8aZ4KOrEMM= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= @@ -142,10 +159,11 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5 h1:q3OfiOZV5rlsK1H5V8benjeUApRfMGs4Mrhmr6NriQo= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= -github.com/ipfs/go-ds-badger v0.0.4/go.mod h1:UIu++7eal30eVc+njb9LyGgBoJ3F+Y5cBpvD/dwn5VQ= github.com/ipfs/go-ds-badger v0.0.5 h1:dxKuqw5T1Jm8OuV+lchA76H9QZFyPKZeLuT6bN42hJQ= github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= @@ -157,6 +175,8 @@ github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf h1:P9Kkd8YCG4 github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf/go.mod h1:gaK14QN1GOlYGgq+o+t5+WTExZZogkMt0k0IIBNjXsM= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= +github.com/ipfs/go-ipfs-blockstore v0.0.2 h1:C32j/vtBmSS/UV3pb5/6JvPGgiMMzviGYL/QdGD0KKQ= +github.com/ipfs/go-ipfs-blockstore v0.0.2/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= @@ -172,17 +192,16 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= github.com/ipfs/go-ipfs-files v0.0.3 h1:ME+QnC3uOyla1ciRPezDW0ynQYK2ikOh9OCKAEg4uUA= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= +github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE= +github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1 h1:zgUotX8dcAB/w/HidJh1zzc1yFq6Vm8J7T2F4itj/RU= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= -github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs= github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRDI/HQQ= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipld-cbor v0.0.1/go.mod h1:RXHr8s4k0NE0TKhnrxqZC9M888QfsBN9rhS5NjfKzY8= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= @@ -192,9 +211,10 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v0.0.2-0.20190708183747-9c9fd6111bea h1:T5Xz37N8fqK3u0aLbWXCoTOO+S62T+TvOLRZh8q8vX0= github.com/ipfs/go-log v0.0.2-0.20190708183747-9c9fd6111bea/go.mod h1:YTiqro5xwLoGra88hB8tMBlN+7ByaT3Kdaa0UqwCmI0= -github.com/ipfs/go-merkledag v0.0.1/go.mod h1:CRdtHMROECqaehAGeJ0Wd9TtlmWv/ta5cUnvbTnniEI= github.com/ipfs/go-merkledag v0.1.0 h1:CAEXjRFEDPvealQj3TgEjV1IJckwjvmxAqtq5QSXJrg= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= +github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -202,13 +222,10 @@ github.com/ipfs/go-peertaskqueue v0.1.1 h1:+gPjbI+V3NktXZOqJA1kzbms2pYmhjgQQal0M github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw= github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= -github.com/ipfs/go-unixfs v0.1.0 h1:KkjcfqObdNwUN8heMtt5OdrgrRKYTIWEvpGl1bDYIho= -github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= -github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -222,11 +239,16 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= @@ -245,30 +267,26 @@ github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpz github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= -github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= -github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= -github.com/libp2p/go-eventbus v0.0.3 h1:4sB0NrwnWr6qGeq2RWUp/JG1wNajf6gyILInId72hrw= -github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= +github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ= +github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= -github.com/libp2p/go-libp2p v0.0.1/go.mod h1:bmRs8I0vwn6iRaVssZnJx/epY6WPSKiLoK1vyle4EX0= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= -github.com/libp2p/go-libp2p v0.2.0 h1:hYJgMZYdcwHzDHKb/nLePrtuSP3LqkGIFOQ2aIbKOCM= -github.com/libp2p/go-libp2p v0.2.0/go.mod h1:5nXHmf4Hs+NmkaMsmWcFJgUHTbYNpCfxr20lwus0p1c= -github.com/libp2p/go-libp2p-autonat v0.0.1/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= +github.com/libp2p/go-libp2p v0.3.0 h1:XhYEJKmIdi4U4Zbie/ym9k6lqgg3PHM2stGS/cOUDWk= +github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= -github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3 h1:0KycuXvPDhmehw0ASsg+s1o3IfXgCUDqfzAl94KEBOg= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= -github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE= github.com/libp2p/go-libp2p-circuit v0.1.0 h1:eniLL3Y9aq/sryfyV1IAHj5rlvuyj3b7iz8tSiZpdhY= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.1 h1:eopfG9fAg6rEHWQO1TSrLosXDgYbbbu/RTva/tBANus= +github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-connmgr v0.1.0 h1:vp0t0F0EuT3rrlTtnMnIyyzCnly7nIlRoEbhJpgp0qU= github.com/libp2p/go-libp2p-connmgr v0.1.0/go.mod h1:wZxh8veAmU5qdrfJ0ZBLcU8oJe9L82ciVP/fl1VHjXk= github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= @@ -277,106 +295,90 @@ github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvx github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= github.com/libp2p/go-libp2p-core v0.0.6 h1:SsYhfWJ47vLP1Rd9/0hqEm/W/PlFbC/3YLZyLCcvo1w= github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= -github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= +github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= +github.com/libp2p/go-libp2p-core v0.2.2 h1:Sv1ggdoMx9c7v7FOFkR7agraHCnAgqYsXrU1ARSRUMs= +github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= -github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= -github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= -github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= github.com/libp2p/go-libp2p-kad-dht v0.1.1 h1:IH6NQuoUv5w5e1O8Jc3KyVDtr0rNd0G9aaADpLI1xVo= github.com/libp2p/go-libp2p-kad-dht v0.1.1/go.mod h1:1kj2Rk5pX3/0RwqMm9AMNCT7DzcMHYhgDN5VTi+cY0M= github.com/libp2p/go-libp2p-kbucket v0.2.0 h1:FB2a0VkOTNGTP5gu/I444u4WabNM9V1zCkQcWb7zajI= github.com/libp2p/go-libp2p-kbucket v0.2.0/go.mod h1:JNymBToym3QXKBMKGy3m29+xprg0EVr/GJFHxFEdgh8= -github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= -github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= -github.com/libp2p/go-libp2p-nat v0.0.1/go.mod h1:4L6ajyUIlJvx1Cbh5pc6Ma6vMDpKXf3GgLO5u7W0oQ4= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= -github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= -github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q= github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= -github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= -github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-peerstore v0.1.1/go.mod h1:ojEWnwG7JpJLkJ9REWYXQslyu9ZLrPWPEcCdiZzEbSM= -github.com/libp2p/go-libp2p-peerstore v0.1.2-0.20190621130618-cfa9bb890c1a h1:xW2Q7yiWAQnBpxe6m5Y094bYxxBCaoNruxc1sDlVxs0= -github.com/libp2p/go-libp2p-peerstore v0.1.2-0.20190621130618-cfa9bb890c1a/go.mod h1:DAchSrPUuksotuxrqPcvk5jvifXlxC3oH/65iHFmBns= +github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= +github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= -github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-pubsub v0.1.0 h1:SmQeMa7IUv5vadh0fYgYsafWCBA1sCy5d/68kIYqGcU= github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= -github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= -github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= +github.com/libp2p/go-libp2p-record v0.1.1 h1:ZJK2bHXYUBqObHX+rHLSNrM3M8fmJUlUHrodDPPATmY= +github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU= github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8= github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= -github.com/libp2p/go-libp2p-secio v0.0.1/go.mod h1:IdG6iQybdcYmbTzxp4J5dwtUEDTOvZrT0opIDVNPrJs= github.com/libp2p/go-libp2p-secio v0.1.0 h1:NNP5KLxuP97sE5Bu3iuwOWyT/dKEGMN5zSLMWdB7GTQ= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= -github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs= +github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= +github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= -github.com/libp2p/go-libp2p-swarm v0.1.1 h1:QW7pjyTRIxt9yyBid52YmMRGtkFXUE/rbEVWsQ0ae+w= -github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwLx73ln1FTefR91Q= +github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= +github.com/libp2p/go-libp2p-swarm v0.2.1 h1:9A8oQqPIZvbaRyrjViHeDYS7fE7fNtP7BRWdJrBHbe8= +github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.4 h1:Qev57UR47GcLPXWjrunv5aLIQGO4n9mhI/8/EIrEEFc= github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= +github.com/libp2p/go-libp2p-testing v0.1.0 h1:WaFRj/t3HdMZGNZqnU2pS7pDRBmMeoDx7/HDNpeyT9U= +github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-tls v0.1.0 h1:o4bjjAdnUjNgJoPoDd0wUaZH7K+EenlNWJpgyXB3ulA= github.com/libp2p/go-libp2p-tls v0.1.0/go.mod h1:VZdoSWQDeNpIIAFJFv+6uqTqpnIIDHcqZQSTC/A1TT0= -github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= -github.com/libp2p/go-libp2p-transport v0.0.4/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= -github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= github.com/libp2p/go-libp2p-yamux v0.2.1 h1:Q3XYNiKCC2vIxrvUJL+Jg1kiyeEaIDNKLjgEjo3VQdI= github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= -github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= -github.com/libp2p/go-mplex v0.0.1/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= -github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-nat v0.0.3 h1:l6fKV+p0Xa354EqQOQP+d8CivdLM4kl5GxC1hSc/UeI= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= +github.com/libp2p/go-openssl v0.0.2 h1:9pP2d3Ubaxkv7ZisLjx9BFwgOGnQdQYnfcH29HNY3ls= +github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= -github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= -github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6mK0en8rlpA6BBTs= github.com/libp2p/go-tcp-transport v0.1.0 h1:IGhowvEqyMFknOar4FWCKSWE0zL36UFKQtiRQD60/8o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= -github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= -github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= github.com/libp2p/go-ws-transport v0.1.0 h1:F+0OvvdmPTDsVc4AjPHjV7L7Pk1B7D5QwtDcKE2oag4= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -385,18 +387,24 @@ github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.16 h1:iMEQ/IVHxPTtx2Q07JP/k4CKRvSjiAZjZ0hnhgYEDmE= +github.com/miekg/dns v1.1.16/go.mod h1:YNV562EiewvSmpCB6/W4c6yqjK7Z+M/aIS1JHsIVeg8= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -406,6 +414,10 @@ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= @@ -428,19 +440,24 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= -github.com/multiformats/go-multihash v0.0.6 h1:cAVKO4epVd+SSpYJQD6d3vbdqQJvsrtGbTGzsp+V094= -github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= -github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-multihash v0.0.7 h1:uoqoE03rGJdlQEPq2EAc6UeSbo4L7mZyeAAoqNalf54= +github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c/go.mod h1:eTBvSIlRgLo+CNFFQRQTwUGTZOEdvXIKeZS/xG+D2yU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= +github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= +github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -455,28 +472,41 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14 h1:2m16U/rLwVaRdz7ANkHtHTodP3zTP3N451MADg64x5k= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 h1:N8Bg45zpk/UcpNGnfJt2y/3lRWASHNTUET8owPYCgYI= -github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81/go.mod h1:xIvvI5FiHLxhv8prbzVpaMHaaGPFPFQSuTcxC91ryOo= +github.com/sourcegraph/go-langserver v2.0.0+incompatible/go.mod h1:bBMjfpzEHd6ijPRoQ7f+knFfw+e8R+W158/MsqAy77c= +github.com/sourcegraph/jsonrpc2 v0.0.0-20190106185902-35a74f039c6a/go.mod h1:eESpbCslcLDs8j2D7IEdGVgul7xuk9odqDTaor30IUU= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a h1:/eS3yfGjQKG+9kayBkj0ip1BGhq6zJ3eaVksphxAaek= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= @@ -486,16 +516,18 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -504,7 +536,6 @@ github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= -github.com/whyrusleeping/cbor-gen v0.0.0-20190822002707-4e02357de5c1 h1:wDIXmhgPVpV1gWj68IIj1zQNyp7QzBvr5d5UOvMqRNw= github.com/whyrusleeping/cbor-gen v0.0.0-20190822002707-4e02357de5c1/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804 h1:wK83hcnZgKf1AyH8804pc4zVZOB2ND+d2cagsuhQmZc= github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= @@ -515,9 +546,6 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= -github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= -github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= -github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30 h1:nMCC9Pwz1pxfC1Y6mYncdk+kq8d5aLx0Q+/gyZGE44M= @@ -530,7 +558,7 @@ github.com/whyrusleeping/sharray v0.0.0-20190718051354-e41931821e33 h1:7Bsg3GZnF github.com/whyrusleeping/sharray v0.0.0-20190718051354-e41931821e33/go.mod h1:c1pwhNePDPlcYJZinQlfLTOKwTmVf45nfdTg73yOOcA= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -553,6 +581,7 @@ go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWA go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -565,18 +594,20 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -584,14 +615,17 @@ golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -601,10 +635,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -612,35 +646,50 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190730183949-1393eb018365 h1:SaXEMXhWzMJThc05vu6uh61Q245r4KaWMrsTedk0FDc= -golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190813142322-97f12d73768f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -651,6 +700,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= @@ -660,5 +711,7 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 2c3b6e552..dab62407d 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -31,7 +31,7 @@ func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) size := uint64(1016) // this is the most data we can fit in a 1024 byte sector name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), 1016), 0) + sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), 1016)) if err != nil { return 0, err } diff --git a/retrieval/types.go b/retrieval/types.go new file mode 100644 index 000000000..bc19f6468 --- /dev/null +++ b/retrieval/types.go @@ -0,0 +1,31 @@ +package retrieval + +import ( + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-lotus/chain/types" +) + +type QueryResponse int +const ( + Available QueryResponse = iota + Unavailable +) + +type RetDealProposal struct { + Piece cid.Cid + Price types.BigInt + Payment types.SignedVoucher +} + +type RetQuery struct { + Piece cid.Cid +} + +type RetQueryResponse struct { + Status QueryResponse + + MinPricePerMiB types.BigInt // TODO: check units used for sector size +} + + diff --git a/storage/sealedbstore/sealedbstore.go b/storage/sealedbstore/sealedbstore.go new file mode 100644 index 000000000..d6472b9a4 --- /dev/null +++ b/storage/sealedbstore/sealedbstore.go @@ -0,0 +1,120 @@ +package sealedbstore + +import ( + "sync" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + files "github.com/ipfs/go-ipfs-files" + cbor "github.com/ipfs/go-ipld-cbor" + + "github.com/filecoin-project/go-lotus/storage/sector" +) + +type SealSerialization uint8 + +const ( + SerializationUnixfs0 SealSerialization = 'u' +) + +type SealedRef struct { + Serialization SealSerialization + + Piece cid.Cid + Offset uint64 + Size uint32 +} + +type Sealedbstore struct { + *sector.Store + + keys datastore.Batching + keyLk sync.Mutex +} + +type UnixfsReader interface { + files.File + + // ReadBlock reads data from a single unixfs block. Data is nil + // for intermediate nodes + ReadBlock() (data []byte, offset uint64, cid cid.Cid, err error) +} + +type refStorer struct { + blockReader UnixfsReader + writeRef func(cid cid.Cid, offset uint64, size uint32) error + + pieceRef string + remaining []byte +} + +func (st *Sealedbstore) writeRef(cid cid.Cid, offset uint64, size uint32) error { + st.keyLk.Lock() // TODO: make this multithreaded + defer st.keyLk.Unlock() + + v, err := st.keys.Get(dshelp.CidToDsKey(cid)) + if err == datastore.ErrNotFound { + err = nil + } + if err != nil { + return err + } + + var refs []SealedRef + if len(v) > 0 { + if err := cbor.DecodeInto(v, &refs); err != nil { + return err + } + } + + refs = append(refs, SealedRef{ + Serialization: SerializationUnixfs0, + Piece: cid, + Offset: offset, + Size: size, + }) + + newRef, err := cbor.DumpObject(&refs) + if err != nil { + return err + } + return st.keys.Put(dshelp.CidToDsKey(cid), newRef) // TODO: batch somehow +} + +func (r *refStorer) Read(p []byte) (n int, err error) { + offset := 0 + if len(r.remaining) > 0 { + offset += len(r.remaining) + read := copy(p, r.remaining) + if read == len(r.remaining) { + r.remaining = nil + } else { + r.remaining = r.remaining[read:] + } + return read, nil + } + + for { + data, offset, cid, err := r.blockReader.ReadBlock() + if err != nil { + return 0, err + } + + if err := r.writeRef(cid, offset, uint32(len(data))); err != nil { + return 0, err + } + + } +} + +func (st *Sealedbstore) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast uint64) (sectorID uint64, err error) { + size, err := r.Size() + if err != nil { + return 0, err + } + + refst := &refStorer{blockReader: r, pieceRef: string(SerializationUnixfs0)+ref.String(), writeRef: st.writeRef} + + return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) +} diff --git a/storage/sector/store.go b/storage/sector/store.go index 2f366cbe3..3d18e77ad 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -99,7 +99,7 @@ func (s *Store) service() { } } -func (s *Store) AddPiece(ref string, size uint64, r io.Reader, keepAtLeast uint64) (sectorID uint64, err error) { +func (s *Store) AddPiece(ref string, size uint64, r io.Reader) (sectorID uint64, err error) { err = withTemp(r, func(f string) (err error) { sectorID, err = s.sb.AddPiece(ref, size, f) return err From e0dc17bc1a5ac74e0761af04c6dd8123a3832def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 26 Aug 2019 11:08:39 +0200 Subject: [PATCH 02/30] Correctly construct sealbstore --- node/builder.go | 2 ++ retrieval/types.go | 7 +++---- storage/sealedbstore/sealedbstore.go | 23 +++++++++++++++++------ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/node/builder.go b/node/builder.go index 5ef4b5b3e..b9ca72c12 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,6 +3,7 @@ package node import ( "context" "errors" + "github.com/filecoin-project/go-lotus/storage/sealedbstore" "reflect" "time" @@ -232,6 +233,7 @@ func Online() Option { ApplyIf(func(s *Settings) bool { return s.nodeType == nodeStorageMiner }, Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), Override(new(*sector.Store), sector.NewStore), + Override(new(*sealedbstore.Sealedbstore), sealedbstore.NewSealedbstore), Override(new(*storage.Miner), modules.StorageMiner), Override(new(dtypes.StagingDAG), modules.StagingDAG), diff --git a/retrieval/types.go b/retrieval/types.go index bc19f6468..1c55990e5 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -7,14 +7,15 @@ import ( ) type QueryResponse int + const ( Available QueryResponse = iota Unavailable ) type RetDealProposal struct { - Piece cid.Cid - Price types.BigInt + Piece cid.Cid + Price types.BigInt Payment types.SignedVoucher } @@ -27,5 +28,3 @@ type RetQueryResponse struct { MinPricePerMiB types.BigInt // TODO: check units used for sector size } - - diff --git a/storage/sealedbstore/sealedbstore.go b/storage/sealedbstore/sealedbstore.go index d6472b9a4..0c9d4df4b 100644 --- a/storage/sealedbstore/sealedbstore.go +++ b/storage/sealedbstore/sealedbstore.go @@ -1,6 +1,8 @@ package sealedbstore import ( + "github.com/filecoin-project/go-lotus/node/modules/dtypes" + "github.com/ipfs/go-datastore/namespace" "sync" "github.com/ipfs/go-cid" @@ -18,21 +20,30 @@ const ( SerializationUnixfs0 SealSerialization = 'u' ) +var dsPrefix = datastore.NewKey("/sealedblocks") + type SealedRef struct { Serialization SealSerialization - Piece cid.Cid + Piece cid.Cid Offset uint64 - Size uint32 + Size uint32 } type Sealedbstore struct { *sector.Store - keys datastore.Batching + keys datastore.Batching keyLk sync.Mutex } +func NewSealedbstore(sectst *sector.Store, ds dtypes.MetadataDS) *Sealedbstore { + return &Sealedbstore{ + Store: sectst, + keys: namespace.Wrap(ds, dsPrefix), + } +} + type UnixfsReader interface { files.File @@ -43,9 +54,9 @@ type UnixfsReader interface { type refStorer struct { blockReader UnixfsReader - writeRef func(cid cid.Cid, offset uint64, size uint32) error + writeRef func(cid cid.Cid, offset uint64, size uint32) error - pieceRef string + pieceRef string remaining []byte } @@ -114,7 +125,7 @@ func (st *Sealedbstore) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast return 0, err } - refst := &refStorer{blockReader: r, pieceRef: string(SerializationUnixfs0)+ref.String(), writeRef: st.writeRef} + refst := &refStorer{blockReader: r, pieceRef: string(SerializationUnixfs0) + ref.String(), writeRef: st.writeRef} return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) } From cad3efb9babb37c51647a3f9b892478cc93b1c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 26 Aug 2019 12:04:57 +0200 Subject: [PATCH 03/30] Command to list sealed blocks --- api/api.go | 18 ++++- api/struct.go | 6 ++ chain/deals/handler.go | 6 +- chain/deals/handler_states.go | 6 +- cmd/lotus-storage-miner/sectors.go | 26 +++++++ lib/jsonrpc/client.go | 1 + node/builder.go | 4 +- node/impl/storminer.go | 18 +++++ .../blockstore.go} | 73 +++++++++++++------ 9 files changed, 125 insertions(+), 33 deletions(-) rename storage/{sealedbstore/sealedbstore.go => sectorblocks/blockstore.go} (61%) diff --git a/api/api.go b/api/api.go index 8850a9d03..39ef937f8 100644 --- a/api/api.go +++ b/api/api.go @@ -3,6 +3,9 @@ package api import ( "context" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-filestore" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -10,12 +13,13 @@ import ( "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/store" "github.com/filecoin-project/go-lotus/chain/types" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-filestore" ) +func init() { + cbor.RegisterCborType(SealedRef{}) +} + type Common interface { // Auth AuthVerify(ctx context.Context, token string) ([]string, error) @@ -129,6 +133,8 @@ type StorageMiner interface { // Seal all staged sectors SectorsStagedSeal(context.Context) error + + SectorsRefs(context.Context) (map[string][]SealedRef, error) } // Version provides various build-time information @@ -178,3 +184,9 @@ type MinerPower struct { MinerPower types.BigInt TotalPower types.BigInt } + +type SealedRef struct { + Piece string + Offset uint64 + Size uint32 +} diff --git a/api/struct.go b/api/struct.go index 84fdbf1fc..73a3db4f1 100644 --- a/api/struct.go +++ b/api/struct.go @@ -101,6 +101,8 @@ type StorageMinerStruct struct { SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` SectorsStagedList func(context.Context) ([]sectorbuilder.StagedSectorMetadata, error) `perm:"read"` SectorsStagedSeal func(context.Context) error `perm:"write"` + + SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` } } @@ -329,6 +331,10 @@ func (c *StorageMinerStruct) SectorsStagedSeal(ctx context.Context) error { return c.Internal.SectorsStagedSeal(ctx) } +func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]SealedRef, error) { + return c.Internal.SectorsRefs(ctx) +} + var _ Common = &CommonStruct{} var _ FullNode = &FullNodeStruct{} var _ StorageMiner = &StorageMinerStruct{} diff --git a/chain/deals/handler.go b/chain/deals/handler.go index 6bf8713f1..f4e33c6b6 100644 --- a/chain/deals/handler.go +++ b/chain/deals/handler.go @@ -2,7 +2,7 @@ package deals import ( "context" - "github.com/filecoin-project/go-lotus/storage/sealedbstore" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" "math" "github.com/filecoin-project/go-lotus/api" @@ -34,7 +34,7 @@ type MinerDeal struct { } type Handler struct { - secst *sealedbstore.Sealedbstore + secst *sectorblocks.SectorBlocks full api.FullNode // TODO: Use a custom protocol or graphsync in the future @@ -59,7 +59,7 @@ type dealUpdate struct { mut func(*MinerDeal) } -func NewHandler(ds dtypes.MetadataDS, secst *sealedbstore.Sealedbstore, dag dtypes.StagingDAG, fullNode api.FullNode) (*Handler, error) { +func NewHandler(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, fullNode api.FullNode) (*Handler, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err diff --git a/chain/deals/handler_states.go b/chain/deals/handler_states.go index 1693d5a95..c84320fb1 100644 --- a/chain/deals/handler_states.go +++ b/chain/deals/handler_states.go @@ -8,7 +8,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" - "github.com/filecoin-project/go-lotus/storage/sealedbstore" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" ) type handlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) @@ -76,10 +76,10 @@ func (h *Handler) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), return nil, xerrors.Errorf("cannot open unixfs file: %s", err) } - uf, ok := n.(sealedbstore.UnixfsReader) + uf, ok := n.(sectorblocks.UnixfsReader) if !ok { // we probably got directory, unsupported for now - return nil, xerrors.Errorf("unsupported unixfs type") + return nil, xerrors.Errorf("unsupported unixfs file type") } sectorID, err := h.secst.AddUnixfsPiece(deal.Proposal.PieceRef, uf, deal.Proposal.Duration) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 3fd5aab07..7aba38b0b 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -36,6 +36,7 @@ var sectorsCmd = &cli.Command{ sectorsStatusCmd, sectorsStagedListCmd, sectorsStagedSealCmd, + sectorsRefsCmd, }, } @@ -110,3 +111,28 @@ var sectorsStagedSealCmd = &cli.Command{ return nodeApi.SectorsStagedSeal(ctx) }, } + +var sectorsRefsCmd = &cli.Command{ + Name: "refs", + Usage: "List References to sectors", + Action: func(cctx *cli.Context) error { + nodeApi, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + ctx := lcli.ReqContext(cctx) + + refs, err := nodeApi.SectorsRefs(ctx) + if err != nil { + return err + } + + for name, refs := range refs { + fmt.Printf("Block %s:\n", name) + for _, ref := range refs { + fmt.Printf("\t%s+%d %d bytes\n", ref.Piece, ref.Offset, ref.Size) + } + } + return nil + }, +} diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index dd95f9c8f..73c09e0ca 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -288,6 +288,7 @@ func (fn *rpcFunc) handleRpcCall(args []reflect.Value) (results []reflect.Value) if resp.Result != nil { log.Debugw("rpc result", "type", fn.ftyp.Out(fn.valOut)) if err := json.Unmarshal(resp.Result, val.Interface()); err != nil { + log.Warnw("unmarshaling failed", "message", string(resp.Result)) return fn.processError(xerrors.Errorf("unmarshaling result: %w", err)) } } diff --git a/node/builder.go b/node/builder.go index b9ca72c12..06d90227d 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,7 +3,7 @@ package node import ( "context" "errors" - "github.com/filecoin-project/go-lotus/storage/sealedbstore" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" "reflect" "time" @@ -233,7 +233,7 @@ func Online() Option { ApplyIf(func(s *Settings) bool { return s.nodeType == nodeStorageMiner }, Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), Override(new(*sector.Store), sector.NewStore), - Override(new(*sealedbstore.Sealedbstore), sealedbstore.NewSealedbstore), + Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(*storage.Miner), modules.StorageMiner), Override(new(dtypes.StagingDAG), modules.StagingDAG), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index dab62407d..6c2856735 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,6 +3,7 @@ package impl import ( "context" "fmt" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" "io" "math/rand" @@ -19,6 +20,7 @@ type StorageMinerAPI struct { SectorBuilderConfig *sectorbuilder.SectorBuilderConfig SectorBuilder *sectorbuilder.SectorBuilder Sectors *sector.Store + SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner } @@ -53,4 +55,20 @@ func (sm *StorageMinerAPI) SectorsStagedSeal(context.Context) error { return sm.SectorBuilder.SealAllStagedSectors() } +func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) { + // json can't handle cids as map keys + out := map[string][]api.SealedRef{} + + refs, err := sm.SectorBlocks.List() + if err != nil { + return nil, err + } + + for k, v := range refs { + out[k.String()] = v + } + + return out, nil +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/storage/sealedbstore/sealedbstore.go b/storage/sectorblocks/blockstore.go similarity index 61% rename from storage/sealedbstore/sealedbstore.go rename to storage/sectorblocks/blockstore.go index 0c9d4df4b..970e67925 100644 --- a/storage/sealedbstore/sealedbstore.go +++ b/storage/sectorblocks/blockstore.go @@ -1,8 +1,11 @@ -package sealedbstore +package sectorblocks import ( + "context" + "github.com/filecoin-project/go-lotus/api" "github.com/filecoin-project/go-lotus/node/modules/dtypes" "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" "sync" "github.com/ipfs/go-cid" @@ -22,23 +25,15 @@ const ( var dsPrefix = datastore.NewKey("/sealedblocks") -type SealedRef struct { - Serialization SealSerialization - - Piece cid.Cid - Offset uint64 - Size uint32 -} - -type Sealedbstore struct { +type SectorBlocks struct { *sector.Store keys datastore.Batching keyLk sync.Mutex } -func NewSealedbstore(sectst *sector.Store, ds dtypes.MetadataDS) *Sealedbstore { - return &Sealedbstore{ +func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS) *SectorBlocks { + return &SectorBlocks{ Store: sectst, keys: namespace.Wrap(ds, dsPrefix), } @@ -49,7 +44,7 @@ type UnixfsReader interface { // ReadBlock reads data from a single unixfs block. Data is nil // for intermediate nodes - ReadBlock() (data []byte, offset uint64, cid cid.Cid, err error) + ReadBlock(context.Context) (data []byte, offset uint64, cid cid.Cid, err error) } type refStorer struct { @@ -60,7 +55,7 @@ type refStorer struct { remaining []byte } -func (st *Sealedbstore) writeRef(cid cid.Cid, offset uint64, size uint32) error { +func (st *SectorBlocks) writeRef(cid cid.Cid, offset uint64, size uint32) error { st.keyLk.Lock() // TODO: make this multithreaded defer st.keyLk.Unlock() @@ -72,18 +67,17 @@ func (st *Sealedbstore) writeRef(cid cid.Cid, offset uint64, size uint32) error return err } - var refs []SealedRef + var refs []api.SealedRef if len(v) > 0 { if err := cbor.DecodeInto(v, &refs); err != nil { return err } } - refs = append(refs, SealedRef{ - Serialization: SerializationUnixfs0, - Piece: cid, - Offset: offset, - Size: size, + refs = append(refs, api.SealedRef{ + Piece: string(SerializationUnixfs0) + cid.String(), + Offset: offset, + Size: size, }) newRef, err := cbor.DumpObject(&refs) @@ -107,7 +101,7 @@ func (r *refStorer) Read(p []byte) (n int, err error) { } for { - data, offset, cid, err := r.blockReader.ReadBlock() + data, offset, cid, err := r.blockReader.ReadBlock(context.TODO()) if err != nil { return 0, err } @@ -116,10 +110,16 @@ func (r *refStorer) Read(p []byte) (n int, err error) { return 0, err } + read := copy(p, data) + if read < len(data) { + r.remaining = data[read:] + } + // TODO: read multiple + return read, nil } } -func (st *Sealedbstore) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast uint64) (sectorID uint64, err error) { +func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast uint64) (sectorID uint64, err error) { size, err := r.Size() if err != nil { return 0, err @@ -129,3 +129,32 @@ func (st *Sealedbstore) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) } + +func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { + res, err := st.keys.Query(query.Query{}) + if err != nil { + return nil, err + } + + ents, err := res.Rest() + if err != nil { + return nil, err + } + + out := map[cid.Cid][]api.SealedRef{} + for _, ent := range ents { + refCid, err := dshelp.DsKeyToCid(datastore.RawKey(ent.Key)) + if err != nil { + return nil, err + } + + var refs []api.SealedRef + if err := cbor.DecodeInto(ent.Value, &refs); err != nil { + return nil, err + } + + out[refCid] = refs + } + + return out, nil +} From 28d3eb38ebd04e42879b10be0d98b2a48e3af4a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 26 Aug 2019 15:45:36 +0200 Subject: [PATCH 04/30] basic retrieval content discovery --- api/api.go | 12 ++++ api/struct.go | 12 +++- chain/deals/client.go | 31 ++++++---- cli/client.go | 98 ++++++++++++++++++++++++++++++ node/builder.go | 12 +++- node/impl/full/client.go | 42 ++++++++++++- node/modules/client.go | 6 +- node/modules/dtypes/storage.go | 1 + node/modules/services.go | 5 ++ node/modules/storageminer.go | 10 +++ retrieval/client.go | 62 +++++++++++++++++++ retrieval/discovery/discovery.go | 25 ++++++++ retrieval/discovery/local.go | 53 ++++++++++++++++ retrieval/miner.go | 51 ++++++++++++++++ retrieval/types.go | 16 ++++- storage/sectorblocks/blockstore.go | 18 ++++++ 16 files changed, 434 insertions(+), 20 deletions(-) create mode 100644 retrieval/client.go create mode 100644 retrieval/discovery/discovery.go create mode 100644 retrieval/discovery/local.go create mode 100644 retrieval/miner.go diff --git a/api/api.go b/api/api.go index 39ef937f8..9651ee45e 100644 --- a/api/api.go +++ b/api/api.go @@ -91,6 +91,8 @@ type FullNode interface { // ClientImport imports file under the specified path into filestore ClientImport(ctx context.Context, path string) (cid.Cid, error) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) + ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) + ClientFindData(ctx context.Context, root cid.Cid) ([]RetrievalOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) // ClientUnimport removes references to the specified file from filestore //ClientUnimport(path string) @@ -190,3 +192,13 @@ type SealedRef struct { Offset uint64 Size uint32 } + +type RetrievalOffer struct { + Err string + + Size uint64 + MinPrice types.BigInt + + Miner address.Address + MinerPeerID peer.ID +} diff --git a/api/struct.go b/api/struct.go index 73a3db4f1..0b275e032 100644 --- a/api/struct.go +++ b/api/struct.go @@ -69,7 +69,9 @@ type FullNodeStruct struct { MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"write"` - ClientListImports func(ctx context.Context) ([]Import, error) `perm:"read"` + ClientListImports func(ctx context.Context) ([]Import, error) `perm:"write"` + ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` + ClientFindData func(ctx context.Context, root cid.Cid) ([]RetrievalOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` @@ -152,6 +154,14 @@ func (c *FullNodeStruct) ClientImport(ctx context.Context, path string) (cid.Cid return c.Internal.ClientImport(ctx, path) } +func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { + return c.Internal.ClientHasLocal(ctx, root) +} + +func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]RetrievalOffer, error) { + return c.Internal.ClientFindData(ctx, root) +} + func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } diff --git a/chain/deals/client.go b/chain/deals/client.go index 58d2ef7e4..839698406 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -4,8 +4,6 @@ import ( "context" "math" - "github.com/filecoin-project/go-lotus/chain/actors" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -19,12 +17,14 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + "github.com/filecoin-project/go-lotus/chain/actors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/store" "github.com/filecoin-project/go-lotus/chain/types" "github.com/filecoin-project/go-lotus/chain/wallet" "github.com/filecoin-project/go-lotus/lib/cborrpc" "github.com/filecoin-project/go-lotus/node/modules/dtypes" + "github.com/filecoin-project/go-lotus/retrieval/discovery" ) func init() { @@ -48,10 +48,11 @@ type ClientDeal struct { } type Client struct { - cs *store.ChainStore - h host.Host - w *wallet.Wallet - dag dtypes.ClientDAG + cs *store.ChainStore + h host.Host + w *wallet.Wallet + dag dtypes.ClientDAG + discovery *discovery.Local deals StateStore @@ -61,12 +62,13 @@ type Client struct { stopped chan struct{} } -func NewClient(cs *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG) *Client { +func NewClient(cs *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local) *Client { c := &Client{ - cs: cs, - h: h, - w: w, - dag: dag, + cs: cs, + h: h, + w: w, + dag: dag, + discovery: discovery, deals: StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}, @@ -242,7 +244,12 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.Pie // TODO: actually care about what happens with the deal after it was accepted //c.incoming <- deal - return deal.ProposalCid, nil + + // TODO: start tracking after the deal is sealed + return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{ + Address: proposal.MinerAddress, + ID: deal.Miner, + }) } func (c *Client) Stop() { diff --git a/cli/client.go b/cli/client.go index 61c58678c..20293bfb3 100644 --- a/cli/client.go +++ b/cli/client.go @@ -19,6 +19,8 @@ var clientCmd = &cli.Command{ clientImportCmd, clientLocalCmd, clientDealCmd, + clientFindCmd, + clientRetrieveCmd, }, } @@ -108,3 +110,99 @@ var clientDealCmd = &cli.Command{ return nil }, } + +var clientFindCmd = &cli.Command{ + Name: "find", + Usage: "find data in the network", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + fmt.Println("Usage: retrieve [CID]") + return nil + } + + file, err := cid.Parse(cctx.Args().First()) + if err != nil { + return err + } + + api, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + ctx := ReqContext(cctx) + + // Check if we already have this data locally + + has, err := api.ClientHasLocal(ctx, file) + if err != nil { + return err + } + + if has { + fmt.Println("LOCAL") + } + + offers, err := api.ClientFindData(ctx, file) + if err != nil { + return err + } + + for _, offer := range offers { + if offer.Err != "" { + fmt.Printf("ERR %s@%s: %s\n", offer.Miner, offer.MinerPeerID, offer.Err) + continue + } + fmt.Printf("RETRIEVAL %s@%s-%sfil-%db\n", offer.Miner, offer.MinerPeerID, offer.MinPrice, offer.Size) + } + + return nil + }, +} + +var clientRetrieveCmd = &cli.Command{ + Name: "retrieve", + Usage: "retrieve data from network", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + fmt.Println("Usage: retrieve [CID]") + return nil + } + + file, err := cid.Parse(cctx.Args().First()) + if err != nil { + return err + } + + api, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + ctx := ReqContext(cctx) + + // Check if we already have this data locally + + has, err := api.ClientHasLocal(ctx, file) + if err != nil { + return err + } + + if has { + fmt.Println("Success: Already in local storage") + return nil + } + + _, err = api.ClientFindData(ctx, file) + if err != nil { + return err + } + + // Find miner which may have this data + + // Get merkle proofs (intermediate nodes) + + // if acceptable, make retrieval deals to get data + // done + + panic("TODO") + }, +} diff --git a/node/builder.go b/node/builder.go index 06d90227d..538a15f9d 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,7 +3,7 @@ package node import ( "context" "errors" - "github.com/filecoin-project/go-lotus/storage/sectorblocks" + "github.com/filecoin-project/go-lotus/retrieval" "reflect" "time" @@ -36,8 +36,10 @@ import ( "github.com/filecoin-project/go-lotus/node/modules/testing" "github.com/filecoin-project/go-lotus/node/repo" "github.com/filecoin-project/go-lotus/paych" + "github.com/filecoin-project/go-lotus/retrieval/discovery" "github.com/filecoin-project/go-lotus/storage" "github.com/filecoin-project/go-lotus/storage/sector" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" ) // special is a type used to give keys to modules which @@ -80,6 +82,7 @@ const ( // storage miner HandleDealsKey + HandleRetrievalKey RunSectorServiceKey RegisterMinerKey @@ -220,6 +223,10 @@ func Online() Option { Override(RunBlockSyncKey, modules.RunBlockSync), Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), + Override(new(*discovery.Local), discovery.NewLocal), + Override(new(discovery.PeerResolver), modules.RetrievalResolver), + + Override(new(*retrieval.Client), retrieval.NewClient), Override(new(*deals.Client), deals.NewClient), Override(RunDealClientKey, modules.RunDealClient), @@ -238,7 +245,9 @@ func Online() Option { Override(new(dtypes.StagingDAG), modules.StagingDAG), + Override(new(*retrieval.Miner), retrieval.NewMiner), Override(new(*deals.Handler), deals.NewHandler), + Override(HandleRetrievalKey, modules.HandleRetrieval), Override(HandleDealsKey, modules.HandleDeals), Override(RunSectorServiceKey, modules.RunSectorService), Override(RegisterMinerKey, modules.RegisterMiner), @@ -302,6 +311,7 @@ func Repo(r repo.Repo) Option { Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore), Override(new(dtypes.ClientFilestore), modules.ClientFstore), + Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), Override(new(dtypes.ClientDAG), modules.ClientDAG), Override(new(ci.PrivKey), pk), diff --git a/node/impl/full/client.go b/node/impl/full/client.go index 49bf4b0aa..fd909f723 100644 --- a/node/impl/full/client.go +++ b/node/impl/full/client.go @@ -3,6 +3,11 @@ package full import ( "context" "errors" + "github.com/filecoin-project/go-lotus/retrieval" + "github.com/filecoin-project/go-lotus/retrieval/discovery" + "github.com/ipfs/go-blockservice" + offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-merkledag" "os" "github.com/filecoin-project/go-lotus/api" @@ -31,10 +36,13 @@ type ClientAPI struct { WalletAPI PaychAPI - DealClient *deals.Client + DealClient *deals.Client + RetDiscovery discovery.PeerResolver + Retrieval *retrieval.Client - LocalDAG dtypes.ClientDAG - Filestore dtypes.ClientFilestore `optional:"true"` + LocalDAG dtypes.ClientDAG + Blockstore dtypes.ClientBlockstore + Filestore dtypes.ClientFilestore `optional:"true"` } func (a *ClientAPI) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { @@ -116,6 +124,34 @@ func (a *ClientAPI) ClientStartDeal(ctx context.Context, data cid.Cid, miner add return &c, err } +func (a *ClientAPI) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { + // TODO: check if we have the ENTIRE dag + + offExch := merkledag.NewDAGService(blockservice.New(a.Blockstore, offline.Exchange(a.Blockstore))) + _, err := offExch.Get(ctx, root) + if err == ipld.ErrNotFound { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (a *ClientAPI) ClientFindData(ctx context.Context, root cid.Cid) ([]api.RetrievalOffer, error) { + peers, err := a.RetDiscovery.GetPeers(root) + if err != nil { + return nil, err + } + + out := make([]api.RetrievalOffer, len(peers)) + for k, p := range peers { + out[k] = a.Retrieval.Query(ctx, p, root) + } + + return out, nil +} + func (a *ClientAPI) ClientImport(ctx context.Context, path string) (cid.Cid, error) { f, err := os.Open(path) if err != nil { diff --git a/node/modules/client.go b/node/modules/client.go index 0ddd5b7d2..3fa87ad00 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -36,9 +36,11 @@ func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) { return filestore.NewFilestore(bs, fm), nil } -func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore, rt routing.Routing, h host.Host) dtypes.ClientDAG { - ibs := blockstore.NewIdStore((*filestore.Filestore)(fstore)) +func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore { + return blockstore.NewIdStore((*filestore.Filestore)(fstore)) +} +func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG { bitswapNetwork := network.NewFromIpfsHost(h, rt) exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index a33f68c89..5e062cd68 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -21,6 +21,7 @@ type ChainExchange exchange.Interface type ChainBlockService bserv.BlockService type ClientFilestore *filestore.Filestore +type ClientBlockstore blockstore.Blockstore type ClientDAG ipld.DAGService type StagingDAG ipld.DAGService diff --git a/node/modules/services.go b/node/modules/services.go index babee96d9..07528c0e4 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -2,6 +2,7 @@ package modules import ( "context" + "github.com/filecoin-project/go-lotus/retrieval/discovery" "github.com/filecoin-project/go-lotus/storage/sector" "github.com/libp2p/go-libp2p-core/host" @@ -83,3 +84,7 @@ func RunSectorService(lc fx.Lifecycle, secst *sector.Store) { }, }) } + +func RetrievalResolver(l *discovery.Local) discovery.PeerResolver { + return discovery.Multi(l) +} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 773e488cc..ca4b3f262 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,6 +2,7 @@ package modules import ( "context" + "github.com/filecoin-project/go-lotus/retrieval" "github.com/filecoin-project/go-lotus/storage/sector" "path/filepath" @@ -86,6 +87,15 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h return sm, nil } +func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) { + lc.Append(fx.Hook{ + OnStart: func(context.Context) error { + host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleStream) + return nil + }, + }) +} + func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Handler) { ctx := helpers.LifecycleCtx(mctx, lc) diff --git a/retrieval/client.go b/retrieval/client.go new file mode 100644 index 000000000..3c6f2bf95 --- /dev/null +++ b/retrieval/client.go @@ -0,0 +1,62 @@ +package retrieval + +import ( + "context" + "github.com/filecoin-project/go-lotus/lib/cborrpc" + "io/ioutil" + + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p-core/host" + + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/retrieval/discovery" +) + +var log = logging.Logger("retrieval") + +type Client struct { + h host.Host +} + +func NewClient(h host.Host) *Client { + return &Client{h: h} +} + +func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid.Cid) api.RetrievalOffer { + s, err := c.h.NewStream(ctx, p.ID, QueryProtocolID) + if err != nil { + log.Warn(err) + return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + defer s.Close() + + err = cborrpc.WriteCborRPC(s, RetQuery{ + Piece: data, + }) + if err != nil { + log.Warn(err) + return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + + // TODO: read deadline + rawResp, err := ioutil.ReadAll(s) + if err != nil { + log.Warn(err) + return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + + var resp RetQueryResponse + if err := cbor.DecodeInto(rawResp, &resp); err != nil { + log.Warn(err) + return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + + return api.RetrievalOffer{ + Size: resp.Size, + MinPrice: resp.MinPrice, + Miner: p.Address, // TODO: check + MinerPeerID: p.ID, + } +} diff --git a/retrieval/discovery/discovery.go b/retrieval/discovery/discovery.go new file mode 100644 index 000000000..21a555fc6 --- /dev/null +++ b/retrieval/discovery/discovery.go @@ -0,0 +1,25 @@ +package discovery + +import ( + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/libp2p/go-libp2p-core/peer" +) + +func init() { + cbor.RegisterCborType(RetrievalPeer{}) +} + +type RetrievalPeer struct { + Address address.Address + ID peer.ID // optional +} + +type PeerResolver interface { + GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel +} + +func Multi(r PeerResolver) PeerResolver { // TODO: actually support multiple mechanisms + return r +} diff --git a/retrieval/discovery/local.go b/retrieval/discovery/local.go new file mode 100644 index 000000000..512ea12a5 --- /dev/null +++ b/retrieval/discovery/local.go @@ -0,0 +1,53 @@ +package discovery + +import ( + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + dshelp "github.com/ipfs/go-ipfs-ds-help" + cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + + "github.com/filecoin-project/go-lotus/node/modules/dtypes" +) + +var log = logging.Logger("ret-discovery") + +type Local struct { + ds datastore.Datastore +} + +func NewLocal(ds dtypes.MetadataDS) *Local { + return &Local{ds: namespace.Wrap(ds, datastore.NewKey("/deals/local"))} +} + +func (l *Local) AddPeer(cid cid.Cid, peer RetrievalPeer) error { + // TODO: allow multiple peers here + // (implement an util for tracking map[thing][]otherThing, use in sectorBlockstore too) + + log.Warn("Tracking multiple retrieval peers not implemented") + + entry, err := cbor.DumpObject(peer) + if err != nil { + return err + } + + return l.ds.Put(dshelp.CidToDsKey(cid), entry) +} + +func (l *Local) GetPeers(data cid.Cid) ([]RetrievalPeer, error) { + entry, err := l.ds.Get(dshelp.CidToDsKey(data)) + if err == datastore.ErrNotFound { + return []RetrievalPeer{}, nil + } + if err != nil { + return nil, err + } + var peer RetrievalPeer + if err := cbor.DecodeInto(entry, &peer); err != nil { + return nil, err + } + return []RetrievalPeer{peer}, nil +} + +var _ PeerResolver = &Local{} diff --git a/retrieval/miner.go b/retrieval/miner.go new file mode 100644 index 000000000..0ca2439b3 --- /dev/null +++ b/retrieval/miner.go @@ -0,0 +1,51 @@ +package retrieval + +import ( + "github.com/libp2p/go-libp2p-core/network" + + "github.com/filecoin-project/go-lotus/chain/types" + "github.com/filecoin-project/go-lotus/lib/cborrpc" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" +) + +type Miner struct { + sectorBlocks *sectorblocks.SectorBlocks +} + +func NewMiner(sblks *sectorblocks.SectorBlocks) *Miner { + return &Miner{ + sectorBlocks: sblks, + } +} + +func (m *Miner) HandleStream(stream network.Stream) { + defer stream.Close() + + var query RetQuery + if err := cborrpc.ReadCborRPC(stream, &query); err != nil { + log.Errorf("Retrieval query: ReadCborRPC: %s", err) + return + } + + refs, err := m.sectorBlocks.GetRefs(query.Piece) + if err != nil { + log.Errorf("Retrieval query: GetRefs: %s", err) + return + } + + answer := RetQueryResponse{ + Status: Unavailable, + } + if len(refs) > 0 { + answer.Status = Available + + // TODO: get price, look for already unsealed ref to reduce work + answer.MinPrice = types.NewInt(uint64(refs[0].Size)) // TODO: Get this from somewhere + answer.Size = uint64(refs[0].Size) + } + + if err := cborrpc.WriteCborRPC(stream, answer); err != nil { + log.Errorf("Retrieval query: WriteCborRPC: %s", err) + return + } +} diff --git a/retrieval/types.go b/retrieval/types.go index 1c55990e5..764a9ae6a 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -2,10 +2,14 @@ package retrieval import ( "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/go-lotus/chain/types" ) +const ProtocolID = "/fil/retrieval/-1.0.0" // TODO: spec +const QueryProtocolID = "/fil/retrieval/qry/-1.0.0" // TODO: spec + type QueryResponse int const ( @@ -13,6 +17,13 @@ const ( Unavailable ) +func init() { + cbor.RegisterCborType(RetDealProposal{}) + + cbor.RegisterCborType(RetQuery{}) + cbor.RegisterCborType(RetQueryResponse{}) +} + type RetDealProposal struct { Piece cid.Cid Price types.BigInt @@ -26,5 +37,8 @@ type RetQuery struct { type RetQueryResponse struct { Status QueryResponse - MinPricePerMiB types.BigInt // TODO: check units used for sector size + Size uint64 // TODO: spec + // TODO: unseal price (+spec) + // TODO: address to send money for the deal? + MinPrice types.BigInt } diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index 970e67925..9cecf9a85 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -106,6 +106,10 @@ func (r *refStorer) Read(p []byte) (n int, err error) { return 0, err } + if len(data) == 0 { + panic("Handle intermediate nodes") // TODO: ! + } + if err := r.writeRef(cid, offset, uint32(len(data))); err != nil { return 0, err } @@ -158,3 +162,17 @@ func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { return out, nil } + +func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: track unsealed sectors + ent, err := st.keys.Get(dshelp.CidToDsKey(k)) + if err != nil { + return nil, err + } + + var refs []api.SealedRef + if err := cbor.DecodeInto(ent, &refs); err != nil { + return nil, err + } + + return refs, nil +} From 60eedb699ec58f5a57645f70c622374ef2bf48c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 26 Aug 2019 20:23:11 +0200 Subject: [PATCH 05/30] retrieval: Client impl --- api/api.go | 4 +- api/struct.go | 4 +- build/params.go | 7 ++ go.mod | 1 + node/impl/full/client.go | 7 +- retrieval/client.go | 167 ++++++++++++++++++++++++++++++++++++--- retrieval/miner.go | 8 +- retrieval/types.go | 46 +++++++---- retrieval/verify.go | 19 +++++ 9 files changed, 229 insertions(+), 34 deletions(-) create mode 100644 build/params.go create mode 100644 retrieval/verify.go diff --git a/api/api.go b/api/api.go index 9651ee45e..17f633d64 100644 --- a/api/api.go +++ b/api/api.go @@ -92,7 +92,7 @@ type FullNode interface { ClientImport(ctx context.Context, path string) (cid.Cid, error) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) - ClientFindData(ctx context.Context, root cid.Cid) ([]RetrievalOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) + ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) // ClientUnimport removes references to the specified file from filestore //ClientUnimport(path string) @@ -193,7 +193,7 @@ type SealedRef struct { Size uint32 } -type RetrievalOffer struct { +type QueryOffer struct { Err string Size uint64 diff --git a/api/struct.go b/api/struct.go index 0b275e032..f3e6d449c 100644 --- a/api/struct.go +++ b/api/struct.go @@ -71,7 +71,7 @@ type FullNodeStruct struct { ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"write"` ClientListImports func(ctx context.Context) ([]Import, error) `perm:"write"` ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` - ClientFindData func(ctx context.Context, root cid.Cid) ([]RetrievalOffer, error) `perm:"read"` + ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` @@ -158,7 +158,7 @@ func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool return c.Internal.ClientHasLocal(ctx, root) } -func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]RetrievalOffer, error) { +func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) { return c.Internal.ClientFindData(ctx, root) } diff --git a/build/params.go b/build/params.go new file mode 100644 index 000000000..b3b71c563 --- /dev/null +++ b/build/params.go @@ -0,0 +1,7 @@ +package build + +// Core network constants + +const UnixfsChunkSize uint64 = 1 << 20 + +// TODO: Move other important consts here diff --git a/go.mod b/go.mod index cf385f063..10f2bf6e6 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/libp2p/go-libp2p-tls v0.1.0 github.com/libp2p/go-libp2p-yamux v0.2.1 github.com/libp2p/go-maddr-filter v0.0.5 + github.com/libp2p/go-msgio v0.0.4 github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/mitchellh/go-homedir v1.1.0 diff --git a/node/impl/full/client.go b/node/impl/full/client.go index fd909f723..4c5a4e89a 100644 --- a/node/impl/full/client.go +++ b/node/impl/full/client.go @@ -3,6 +3,7 @@ package full import ( "context" "errors" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/retrieval" "github.com/filecoin-project/go-lotus/retrieval/discovery" "github.com/ipfs/go-blockservice" @@ -138,13 +139,13 @@ func (a *ClientAPI) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, err return true, nil } -func (a *ClientAPI) ClientFindData(ctx context.Context, root cid.Cid) ([]api.RetrievalOffer, error) { +func (a *ClientAPI) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) { peers, err := a.RetDiscovery.GetPeers(root) if err != nil { return nil, err } - out := make([]api.RetrievalOffer, len(peers)) + out := make([]api.QueryOffer, len(peers)) for k, p := range peers { out[k] = a.Retrieval.Query(ctx, p, root) } @@ -177,7 +178,7 @@ func (a *ClientAPI) ClientImport(ctx context.Context, path string) (cid.Cid, err NoCopy: true, } - db, err := params.New(chunker.DefaultSplitter(file)) + db, err := params.New(chunker.NewSizeSplitter(file, build.UnixfsChunkSize)) if err != nil { return cid.Undef, err } diff --git a/retrieval/client.go b/retrieval/client.go index 3c6f2bf95..b151bd728 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -2,15 +2,23 @@ package retrieval import ( "context" - "github.com/filecoin-project/go-lotus/lib/cborrpc" "io/ioutil" + blocks "github.com/ipfs/go-block-format" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-msgio" + "golang.org/x/xerrors" + pb "github.com/ipfs/go-bitswap/message/pb" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/filecoin-project/go-lotus/lib/cborrpc" "github.com/filecoin-project/go-lotus/retrieval/discovery" ) @@ -24,39 +32,180 @@ func NewClient(h host.Host) *Client { return &Client{h: h} } -func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid.Cid) api.RetrievalOffer { +func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid.Cid) api.QueryOffer { s, err := c.h.NewStream(ctx, p.ID, QueryProtocolID) if err != nil { log.Warn(err) - return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } defer s.Close() - err = cborrpc.WriteCborRPC(s, RetQuery{ + err = cborrpc.WriteCborRPC(s, Query{ Piece: data, }) if err != nil { log.Warn(err) - return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } // TODO: read deadline rawResp, err := ioutil.ReadAll(s) if err != nil { log.Warn(err) - return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } - var resp RetQueryResponse + var resp QueryResponse if err := cbor.DecodeInto(rawResp, &resp); err != nil { log.Warn(err) - return api.RetrievalOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } - return api.RetrievalOffer{ + return api.QueryOffer{ Size: resp.Size, MinPrice: resp.MinPrice, Miner: p.Address, // TODO: check MinerPeerID: p.ID, } } + +type clientStream struct { + stream network.Stream + + root cid.Cid + offset uint64 + + windowSize uint64 // how much we "trust" the peer + verifier BlockVerifier +} + +// C > S +// +// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size +// +// > Deal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} +// < Resp{Accept} +// < ..(Intermediate Block) +// < ..Blocks +// < ..(Intermediate Block) +// < ..Blocks +// > Deal(...) +// < ... +func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, miner peer.ID, minerAddr address.Address) error { + s, err := c.h.NewStream(ctx, miner, ProtocolID) + if err != nil { + return err + } + defer s.Close() + + cst := clientStream{ + stream: s, + + root: root, + offset: 0, // TODO: check how much data we have locally + + windowSize: build.UnixfsChunkSize, + verifier: &OptimisticVerifier{}, // TODO: Use a real verifier + } + + for { + toFetch := cst.windowSize + if toFetch+cst.offset > size { + toFetch = size - cst.offset + } + + err := cst.doOneExchange(toFetch) + if err != nil { + return err + } + } +} + +func (cst *clientStream) doOneExchange(toFetch uint64) error { + deal := Deal{Unixfs0: &Unixfs0Offer{ + Root: cst.root, + Offset: cst.offset, + Size: toFetch, + }} + + if err := cborrpc.WriteCborRPC(cst.stream, deal); err != nil { + return err + } + + var resp DealResponse + if err := cborrpc.ReadCborRPC(cst.stream, &resp); err != nil { + return err + } + + if resp.AcceptedResponse == nil { + cst.windowSize = build.UnixfsChunkSize + // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) + + if resp.ErrorResponse != nil { + return xerrors.Errorf("storage deal error: %s", resp.ErrorResponse.Message) + } + if resp.RejectedResponse != nil { + return xerrors.Errorf("storage deal rejected: %s", resp.RejectedResponse.Message) + } + return xerrors.New("storage deal response had no Accepted section") + } + + return cst.fetchBlocks(toFetch) + + // TODO: maybe increase miner window size after success +} + +func (cst *clientStream) fetchBlocks(toFetch uint64) error { + blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize + + // TODO: put msgio into spec + reader := msgio.NewVarintReaderSize(cst.stream, network.MessageSizeMax) + + for i := uint64(0); i < blocksToFetch; { + msg, err := reader.ReadMsg() + if err != nil { + return err + } + + var pb pb.Message_Block + if err := pb.Unmarshal(msg); err != nil { + return err + } + + dataBlocks, err := cst.consumeBlockMessage(pb) + if err != nil { + return err + } + + i += dataBlocks + + reader.ReleaseMsg(msg) + } + + return nil +} + +func (cst *clientStream) consumeBlockMessage(pb pb.Message_Block) (uint64, error) { + prefix, err := cid.PrefixFromBytes(pb.GetPrefix()) + if err != nil { + return 0, err + } + cid, err := prefix.Sum(pb.GetData()) + + blk, err := blocks.NewBlockWithCid(pb.GetData(), cid) + if err != nil { + return 0, err + } + + internal, err := cst.verifier.Verify(blk) + if err != nil { + return 0, err + } + + // TODO: Persist block + + if internal { + return 0, nil + } + return 1, nil +} diff --git a/retrieval/miner.go b/retrieval/miner.go index 0ca2439b3..5e2525d76 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -21,7 +21,7 @@ func NewMiner(sblks *sectorblocks.SectorBlocks) *Miner { func (m *Miner) HandleStream(stream network.Stream) { defer stream.Close() - var query RetQuery + var query Query if err := cborrpc.ReadCborRPC(stream, &query); err != nil { log.Errorf("Retrieval query: ReadCborRPC: %s", err) return @@ -33,15 +33,15 @@ func (m *Miner) HandleStream(stream network.Stream) { return } - answer := RetQueryResponse{ + answer := QueryResponse{ Status: Unavailable, } if len(refs) > 0 { answer.Status = Available // TODO: get price, look for already unsealed ref to reduce work - answer.MinPrice = types.NewInt(uint64(refs[0].Size)) // TODO: Get this from somewhere - answer.Size = uint64(refs[0].Size) + answer.MinPrice = types.NewInt(uint64(refs[0].Size) * 2) // TODO: Get this from somewhere + answer.Size = uint64(refs[0].Size) // TODO: verify on intermediate } if err := cborrpc.WriteCborRPC(stream, answer); err != nil { diff --git a/retrieval/types.go b/retrieval/types.go index 764a9ae6a..31dc07119 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -10,35 +10,53 @@ import ( const ProtocolID = "/fil/retrieval/-1.0.0" // TODO: spec const QueryProtocolID = "/fil/retrieval/qry/-1.0.0" // TODO: spec -type QueryResponse int +type QueryResponseStatus int const ( - Available QueryResponse = iota + Available QueryResponseStatus = iota Unavailable ) func init() { - cbor.RegisterCborType(RetDealProposal{}) + cbor.RegisterCborType(Deal{}) - cbor.RegisterCborType(RetQuery{}) - cbor.RegisterCborType(RetQueryResponse{}) + cbor.RegisterCborType(Query{}) + cbor.RegisterCborType(QueryResponse{}) } -type RetDealProposal struct { - Piece cid.Cid - Price types.BigInt - Payment types.SignedVoucher -} - -type RetQuery struct { +type Query struct { Piece cid.Cid + // TODO: payment } -type RetQueryResponse struct { - Status QueryResponse +type QueryResponse struct { + Status QueryResponseStatus Size uint64 // TODO: spec // TODO: unseal price (+spec) + // TODO: sectors to unseal // TODO: address to send money for the deal? MinPrice types.BigInt } + +type Unixfs0Offer struct { + Root cid.Cid + Offset uint64 + Size uint64 +} + +type Deal struct { + Unixfs0 *Unixfs0Offer +} + +type AcceptedResponse struct{} +type RejectedResponse struct { + Message string +} +type ErrorResponse RejectedResponse + +type DealResponse struct { + *AcceptedResponse + *RejectedResponse + *ErrorResponse +} diff --git a/retrieval/verify.go b/retrieval/verify.go new file mode 100644 index 000000000..0c0dca227 --- /dev/null +++ b/retrieval/verify.go @@ -0,0 +1,19 @@ +package retrieval + +import blocks "github.com/ipfs/go-block-format" + +type BlockVerifier interface { + Verify(blocks.Block) (internal bool, err error) +} + +// TODO: BalancedUnixFs0Verifier + +type OptimisticVerifier struct { +} + +func (o *OptimisticVerifier) Verify(blocks.Block) (bool, error) { + // It's probably fine + return false, nil +} + +var _ BlockVerifier = &OptimisticVerifier{} From be30bc79a5f3311c51a78fc32138c5a002aadff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 27 Aug 2019 20:45:21 +0200 Subject: [PATCH 06/30] Retrieval works! --- api/api.go | 22 ++++ api/struct.go | 5 + chain/sub/incoming.go | 4 +- chain/sync.go | 2 +- cli/client.go | 20 +-- go.sum | 1 + miner/miner.go | 2 +- node/impl/full/client.go | 6 +- node/modules/storageminer.go | 3 +- retrieval/client.go | 32 +++-- retrieval/miner.go | 152 ++++++++++++++++++++- retrieval/types.go | 20 +-- storage/sectorblocks/blocks.go | 204 +++++++++++++++++++++++++++++ storage/sectorblocks/blockstore.go | 190 +++++++-------------------- storage/sectorblocks/unsealed.go | 98 ++++++++++++++ 15 files changed, 576 insertions(+), 185 deletions(-) create mode 100644 storage/sectorblocks/blocks.go create mode 100644 storage/sectorblocks/unsealed.go diff --git a/api/api.go b/api/api.go index 17f633d64..8130df873 100644 --- a/api/api.go +++ b/api/api.go @@ -93,6 +93,7 @@ type FullNode interface { ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) + ClientRetrieve(ctx context.Context, order RetrievalOrder) error // TODO: maybe just allow putting this straight into some file // ClientUnimport removes references to the specified file from filestore //ClientUnimport(path string) @@ -196,9 +197,30 @@ type SealedRef struct { type QueryOffer struct { Err string + Root cid.Cid + Size uint64 MinPrice types.BigInt Miner address.Address MinerPeerID peer.ID } + +func (o *QueryOffer) Order() RetrievalOrder { + return RetrievalOrder{ + Root: o.Root, + Size: o.Size, + Miner: o.Miner, + MinerPeerID: o.MinerPeerID, + } +} + +type RetrievalOrder struct { + // TODO: make this loss unixfs specific + Root cid.Cid + Size uint64 + // TODO: support offset + + Miner address.Address + MinerPeerID peer.ID +} diff --git a/api/struct.go b/api/struct.go index f3e6d449c..00424bfbe 100644 --- a/api/struct.go +++ b/api/struct.go @@ -73,6 +73,7 @@ type FullNodeStruct struct { ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` + ClientRetrieve func(ctx context.Context, order RetrievalOrder) error `perm:"admin"` StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` StateMinerProvingSet func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` @@ -166,6 +167,10 @@ func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, mine return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } +func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order RetrievalOrder) error { + return c.Internal.ClientRetrieve(ctx, order) +} + func (c *FullNodeStruct) MpoolPending(ctx context.Context, ts *types.TipSet) ([]*types.SignedMessage, error) { return c.Internal.MpoolPending(ctx, ts) } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 06898eb70..8ff51804c 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -28,7 +28,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha } go func() { - log.Info("about to fetch messages for block from pubsub") + log.Debug("about to fetch messages for block from pubsub") bmsgs, err := s.Bsync.FetchMessagesByCids(context.TODO(), blk.BlsMessages) if err != nil { log.Errorf("failed to fetch all bls messages for block received over pubusb: %s", err) @@ -41,7 +41,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha return } - log.Info("inform new block over pubsub") + log.Debug("inform new block over pubsub") s.InformNewBlock(msg.GetFrom(), &types.FullBlock{ Header: blk.Header, BlsMessages: bmsgs, diff --git a/chain/sync.go b/chain/sync.go index c0581f28b..33650e258 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -88,7 +88,7 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { } if from == syncer.self { // TODO: this is kindof a hack... - log.Infof("got block from ourselves") + log.Debug("got block from ourselves") if err := syncer.Sync(fts); err != nil { log.Errorf("failed to sync our own block: %s", err) diff --git a/cli/client.go b/cli/client.go index 20293bfb3..4d7f92476 100644 --- a/cli/client.go +++ b/cli/client.go @@ -181,7 +181,7 @@ var clientRetrieveCmd = &cli.Command{ // Check if we already have this data locally - has, err := api.ClientHasLocal(ctx, file) + /*has, err := api.ClientHasLocal(ctx, file) if err != nil { return err } @@ -189,20 +189,20 @@ var clientRetrieveCmd = &cli.Command{ if has { fmt.Println("Success: Already in local storage") return nil - } + }*/ // TODO: uncomment before merge - _, err = api.ClientFindData(ctx, file) + offers, err := api.ClientFindData(ctx, file) if err != nil { return err } - // Find miner which may have this data + // TODO: parse offer strings from `client find`, make this smarter - // Get merkle proofs (intermediate nodes) - - // if acceptable, make retrieval deals to get data - // done - - panic("TODO") + order := offers[0].Order() + err = api.ClientRetrieve(ctx, order) + if err == nil { + fmt.Println("Success") + } + return err }, } diff --git a/go.sum b/go.sum index a578cb228..4bc79d4e9 100644 --- a/go.sum +++ b/go.sum @@ -485,6 +485,7 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= diff --git a/miner/miner.go b/miner/miner.go index 26e1d4650..e0ff2fd37 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -193,7 +193,7 @@ func (m *Miner) GetBestMiningCandidate() (*MiningBase, error) { } func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*chain.BlockMsg, error) { - log.Info("attempting to mine a block on:", base.ts.Cids()) + log.Debug("attempting to mine a block on:", base.ts.Cids()) ticket, err := m.scratchTicket(ctx, base) if err != nil { return nil, errors.Wrap(err, "scratching ticket failed") diff --git a/node/impl/full/client.go b/node/impl/full/client.go index 4c5a4e89a..f64202965 100644 --- a/node/impl/full/client.go +++ b/node/impl/full/client.go @@ -178,7 +178,7 @@ func (a *ClientAPI) ClientImport(ctx context.Context, path string) (cid.Cid, err NoCopy: true, } - db, err := params.New(chunker.NewSizeSplitter(file, build.UnixfsChunkSize)) + db, err := params.New(chunker.NewSizeSplitter(file, int64(build.UnixfsChunkSize))) if err != nil { return cid.Undef, err } @@ -218,3 +218,7 @@ func (a *ClientAPI) ClientListImports(ctx context.Context) ([]api.Import, error) }) } } + +func (a *ClientAPI) ClientRetrieve(ctx context.Context, order api.RetrievalOrder) error { + return a.Retrieval.RetrieveUnixfs(ctx, order.Root, order.Size, order.MinerPeerID, order.Miner) +} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index ca4b3f262..8a4bc77bf 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -90,7 +90,8 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) { lc.Append(fx.Hook{ OnStart: func(context.Context) error { - host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleStream) + host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleQueryStream) + host.SetStreamHandler(retrieval.ProtocolID, m.HandleDealStream) return nil }, }) diff --git a/retrieval/client.go b/retrieval/client.go index b151bd728..143ccb5e3 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -4,16 +4,16 @@ import ( "context" "io/ioutil" - blocks "github.com/ipfs/go-block-format" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-msgio" - "golang.org/x/xerrors" pb "github.com/ipfs/go-bitswap/message/pb" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-msgio" + "golang.org/x/xerrors" "github.com/filecoin-project/go-lotus/api" "github.com/filecoin-project/go-lotus/build" @@ -62,6 +62,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. } return api.QueryOffer{ + Root: data, Size: resp.Size, MinPrice: resp.MinPrice, Miner: p.Address, // TODO: check @@ -102,13 +103,15 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, stream: s, root: root, - offset: 0, // TODO: check how much data we have locally + offset: 0, // TODO: Check how much data we have locally + // TODO: Support in handler + // TODO: Allow client to specify this windowSize: build.UnixfsChunkSize, verifier: &OptimisticVerifier{}, // TODO: Use a real verifier } - for { + for cst.offset != size { toFetch := cst.windowSize if toFetch+cst.offset > size { toFetch = size - cst.offset @@ -118,7 +121,11 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, if err != nil { return err } + + cst.offset += toFetch } + log.Info("RETRIEVE SUCCESSFUL") + return nil } func (cst *clientStream) doOneExchange(toFetch uint64) error { @@ -134,18 +141,19 @@ func (cst *clientStream) doOneExchange(toFetch uint64) error { var resp DealResponse if err := cborrpc.ReadCborRPC(cst.stream, &resp); err != nil { + log.Error(err) return err } - if resp.AcceptedResponse == nil { + if resp.Status != Accepted { cst.windowSize = build.UnixfsChunkSize // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) - if resp.ErrorResponse != nil { - return xerrors.Errorf("storage deal error: %s", resp.ErrorResponse.Message) + if resp.Status == Error { + return xerrors.Errorf("storage deal error: %s", resp.Message) } - if resp.RejectedResponse != nil { - return xerrors.Errorf("storage deal rejected: %s", resp.RejectedResponse.Message) + if resp.Status == Rejected { + return xerrors.Errorf("storage deal rejected: %s", resp.Message) } return xerrors.New("storage deal response had no Accepted section") } diff --git a/retrieval/miner.go b/retrieval/miner.go index 5e2525d76..d36f52342 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -1,24 +1,37 @@ package retrieval import ( - "github.com/libp2p/go-libp2p-core/network" + "context" + "github.com/filecoin-project/go-lotus/build" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-msgio" + "golang.org/x/xerrors" "github.com/filecoin-project/go-lotus/chain/types" "github.com/filecoin-project/go-lotus/lib/cborrpc" "github.com/filecoin-project/go-lotus/storage/sectorblocks" + pb "github.com/ipfs/go-bitswap/message/pb" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-merkledag" + unixfile "github.com/ipfs/go-unixfs/file" + "github.com/libp2p/go-libp2p-core/network" ) type Miner struct { sectorBlocks *sectorblocks.SectorBlocks + + pricePerByte types.BigInt + // TODO: Unseal price } func NewMiner(sblks *sectorblocks.SectorBlocks) *Miner { return &Miner{ sectorBlocks: sblks, + pricePerByte: types.NewInt(2), // TODO: allow setting } } -func (m *Miner) HandleStream(stream network.Stream) { +func (m *Miner) HandleQueryStream(stream network.Stream) { defer stream.Close() var query Query @@ -40,7 +53,7 @@ func (m *Miner) HandleStream(stream network.Stream) { answer.Status = Available // TODO: get price, look for already unsealed ref to reduce work - answer.MinPrice = types.NewInt(uint64(refs[0].Size) * 2) // TODO: Get this from somewhere + answer.MinPrice = types.BigMul(types.NewInt(uint64(refs[0].Size)), m.pricePerByte) answer.Size = uint64(refs[0].Size) // TODO: verify on intermediate } @@ -49,3 +62,136 @@ func (m *Miner) HandleStream(stream network.Stream) { return } } + +func writeErr(stream network.Stream, err error) { + log.Errorf("Retrieval deal error: %s", err) + _ = cborrpc.WriteCborRPC(stream, DealResponse{ + Status: Error, + Message: err.Error(), + }) +} + +func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we block in stream handlers + defer stream.Close() + + var ufsr sectorblocks.UnixfsReader + var open cid.Cid + var at uint64 + var size uint64 + + for { + var deal Deal + if err := cborrpc.ReadCborRPC(stream, &deal); err != nil { + return + } + + if deal.Unixfs0 == nil { + writeErr(stream, xerrors.New("unknown deal type")) + return + } + + // TODO: Verify payment, check how much we can send based on that + // Or reject (possibly returning the payment to retain reputation with the client) + + bstore := m.sectorBlocks.SealedBlockstore(func() error { + return nil // TODO: approve unsealing based on amount paid + }) + + if open != deal.Unixfs0.Root || at != deal.Unixfs0.Offset { + if deal.Unixfs0.Offset != 0 { + // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs + writeErr(stream, xerrors.New("sending merkle proofs for nonzero offset not supported yet")) + return + } + at = deal.Unixfs0.Offset + + ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) + rootNd, err := ds.Get(context.TODO(), deal.Unixfs0.Root) + if err != nil { + writeErr(stream, err) + return + } + + fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd) + if err != nil { + writeErr(stream, err) + return + } + + var ok bool + ufsr, ok = fsr.(sectorblocks.UnixfsReader) + if !ok { + writeErr(stream, xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Unixfs0.Root)) + return + } + + isize, err := ufsr.Size() + if err != nil { + writeErr(stream, err) + return + } + size = uint64(isize) + } + + if deal.Unixfs0.Offset + deal.Unixfs0.Size > size { + writeErr(stream, xerrors.Errorf("tried to read too much %d+%d > %d", deal.Unixfs0.Offset, deal.Unixfs0.Size, size)) + return + } + + resp := DealResponse{ + Status: Accepted, + } + if err := cborrpc.WriteCborRPC(stream, resp); err != nil { + log.Errorf("Retrieval query: Write Accepted resp: %s", err) + return + } + + buf := make([]byte, network.MessageSizeMax) + msgw := msgio.NewVarintWriter(stream) + + blocksToSend := (deal.Unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize + for i := uint64(0); i < blocksToSend; { + data, offset, nd, err := ufsr.ReadBlock(context.TODO()) + if err != nil { + writeErr(stream, err) + return + } + + log.Infof("sending block for a deal: %s", nd.Cid()) + + if offset != deal.Unixfs0.Offset { + writeErr(stream, xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", deal.Unixfs0.Offset, offset)) + return + } + + /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) + writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data))) + return + }*/ + + block := pb.Message_Block{ + Prefix: nd.Cid().Prefix().Bytes(), + Data: nd.RawData(), + } + + n, err := block.MarshalTo(buf) + if err != nil { + writeErr(stream, err) + return + } + + if err := msgw.WriteMsg(buf[:n]); err != nil { + log.Error(err) + return + } + + if len(data) > 0 { // don't count internal nodes + i++ + } + } + + // TODO: set `at` + + } + +} diff --git a/retrieval/types.go b/retrieval/types.go index 31dc07119..bd1a51810 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -17,11 +17,20 @@ const ( Unavailable ) +const ( + Accepted = iota + Error + Rejected +) + func init() { cbor.RegisterCborType(Deal{}) cbor.RegisterCborType(Query{}) cbor.RegisterCborType(QueryResponse{}) + cbor.RegisterCborType(Unixfs0Offer{}) + + cbor.RegisterCborType(DealResponse{}) } type Query struct { @@ -49,14 +58,7 @@ type Deal struct { Unixfs0 *Unixfs0Offer } -type AcceptedResponse struct{} -type RejectedResponse struct { +type DealResponse struct { + Status int // TODO: make this more spec complainant Message string } -type ErrorResponse RejectedResponse - -type DealResponse struct { - *AcceptedResponse - *RejectedResponse - *ErrorResponse -} diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go new file mode 100644 index 000000000..324309aef --- /dev/null +++ b/storage/sectorblocks/blocks.go @@ -0,0 +1,204 @@ +package sectorblocks + +import ( + "context" + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/lib/sectorbuilder" + "github.com/filecoin-project/go-lotus/node/modules/dtypes" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" + ipld "github.com/ipfs/go-ipld-format" + "sync" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + files "github.com/ipfs/go-ipfs-files" + cbor "github.com/ipfs/go-ipld-cbor" + + "github.com/filecoin-project/go-lotus/storage/sector" +) + +type SealSerialization uint8 + +const ( + SerializationUnixfs0 SealSerialization = 'u' +) + +var dsPrefix = datastore.NewKey("/sealedblocks") + +type SectorBlocks struct { + *sector.Store + + unsealed *unsealedBlocks + keys datastore.Batching + keyLk sync.Mutex +} + +func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { + sbc := &SectorBlocks{ + Store: sectst, + keys: namespace.Wrap(ds, dsPrefix), + } + + unsealed := &unsealedBlocks{ // TODO: untangle this + sb: sb, + unsealed: map[string][]byte{}, + unsealing: map[string]chan struct{}{}, + } + + sbc.unsealed = unsealed + return sbc +} + +type UnixfsReader interface { + files.File + + // ReadBlock reads data from a single unixfs block. Data is nil + // for intermediate nodes + ReadBlock(context.Context) (data []byte, offset uint64, nd ipld.Node, err error) +} + +type refStorer struct { + blockReader UnixfsReader + writeRef func(cid cid.Cid, offset uint64, size uint32) error + + pieceRef string + remaining []byte +} + +func (st *SectorBlocks) writeRef(cid cid.Cid, offset uint64, size uint32) error { + st.keyLk.Lock() // TODO: make this multithreaded + defer st.keyLk.Unlock() + + v, err := st.keys.Get(dshelp.CidToDsKey(cid)) + if err == datastore.ErrNotFound { + err = nil + } + if err != nil { + return err + } + + var refs []api.SealedRef + if len(v) > 0 { + if err := cbor.DecodeInto(v, &refs); err != nil { + return err + } + } + + refs = append(refs, api.SealedRef{ + Piece: string(SerializationUnixfs0) + cid.String(), + Offset: offset, + Size: size, + }) + + newRef, err := cbor.DumpObject(&refs) + if err != nil { + return err + } + return st.keys.Put(dshelp.CidToDsKey(cid), newRef) // TODO: batch somehow +} + +func (r *refStorer) Read(p []byte) (n int, err error) { + offset := 0 + if len(r.remaining) > 0 { + offset += len(r.remaining) + read := copy(p, r.remaining) + if read == len(r.remaining) { + r.remaining = nil + } else { + r.remaining = r.remaining[read:] + } + return read, nil + } + + for { + data, offset, nd, err := r.blockReader.ReadBlock(context.TODO()) + if err != nil { + return 0, err + } + + if len(data) == 0 { + panic("Handle intermediate nodes") // TODO: ! + } + + if err := r.writeRef(nd.Cid(), offset, uint32(len(data))); err != nil { + return 0, err + } + + read := copy(p, data) + if read < len(data) { + r.remaining = data[read:] + } + // TODO: read multiple + return read, nil + } +} + +func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast uint64) (sectorID uint64, err error) { + size, err := r.Size() + if err != nil { + return 0, err + } + + refst := &refStorer{blockReader: r, pieceRef: string(SerializationUnixfs0) + ref.String(), writeRef: st.writeRef} + + return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) +} + +func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { + res, err := st.keys.Query(query.Query{}) + if err != nil { + return nil, err + } + + ents, err := res.Rest() + if err != nil { + return nil, err + } + + out := map[cid.Cid][]api.SealedRef{} + for _, ent := range ents { + refCid, err := dshelp.DsKeyToCid(datastore.RawKey(ent.Key)) + if err != nil { + return nil, err + } + + var refs []api.SealedRef + if err := cbor.DecodeInto(ent.Value, &refs); err != nil { + return nil, err + } + + out[refCid] = refs + } + + return out, nil +} + +func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: track local sectors + ent, err := st.keys.Get(dshelp.CidToDsKey(k)) + if err != nil { + return nil, err + } + + var refs []api.SealedRef + if err := cbor.DecodeInto(ent, &refs); err != nil { + return nil, err + } + + return refs, nil +} + +func (st *SectorBlocks) Has(k cid.Cid) (bool, error) { + // TODO: ensure sector is still there + return st.keys.Has(dshelp.CidToDsKey(k)) +} + +func (st *SectorBlocks) SealedBlockstore(approveUnseal func() error) *SectorBlockStore { + return &SectorBlockStore{ + //local: nil, // TODO: Pass staging + sectorBlocks: st, + approveUnseal: approveUnseal, + } +} + diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index 9cecf9a85..e69c16e62 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -2,177 +2,77 @@ package sectorblocks import ( "context" - "github.com/filecoin-project/go-lotus/api" - "github.com/filecoin-project/go-lotus/node/modules/dtypes" - "github.com/ipfs/go-datastore/namespace" - "github.com/ipfs/go-datastore/query" - "sync" - + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - dshelp "github.com/ipfs/go-ipfs-ds-help" - files "github.com/ipfs/go-ipfs-files" - cbor "github.com/ipfs/go-ipld-cbor" - - "github.com/filecoin-project/go-lotus/storage/sector" + blockstore "github.com/ipfs/go-ipfs-blockstore" ) -type SealSerialization uint8 +type SectorBlockStore struct { + // local blockstore.Blockstore // staging before GC // TODO: Pass staging + sectorBlocks *SectorBlocks -const ( - SerializationUnixfs0 SealSerialization = 'u' -) - -var dsPrefix = datastore.NewKey("/sealedblocks") - -type SectorBlocks struct { - *sector.Store - - keys datastore.Batching - keyLk sync.Mutex + approveUnseal func() error } -func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS) *SectorBlocks { - return &SectorBlocks{ - Store: sectst, - keys: namespace.Wrap(ds, dsPrefix), - } +func (s *SectorBlockStore) DeleteBlock(cid.Cid) error { + panic("not supported") +} +func (s *SectorBlockStore) GetSize(cid.Cid) (int, error) { + panic("not supported") } -type UnixfsReader interface { - files.File - - // ReadBlock reads data from a single unixfs block. Data is nil - // for intermediate nodes - ReadBlock(context.Context) (data []byte, offset uint64, cid cid.Cid, err error) +func (s *SectorBlockStore) Put(blocks.Block) error { + panic("not supported") } -type refStorer struct { - blockReader UnixfsReader - writeRef func(cid cid.Cid, offset uint64, size uint32) error - - pieceRef string - remaining []byte +func (s *SectorBlockStore) PutMany([]blocks.Block) error { + panic("not supported") } -func (st *SectorBlocks) writeRef(cid cid.Cid, offset uint64, size uint32) error { - st.keyLk.Lock() // TODO: make this multithreaded - defer st.keyLk.Unlock() +func (s *SectorBlockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + panic("not supported") +} - v, err := st.keys.Get(dshelp.CidToDsKey(cid)) - if err == datastore.ErrNotFound { - err = nil - } +func (s *SectorBlockStore) HashOnRead(enabled bool) { + panic("not supported") +} + +func (s *SectorBlockStore) Has(c cid.Cid) (bool, error) { + /*has, err := s.local.Has(c) // TODO: Pass staging if err != nil { - return err + return false, err } + if has { + return true, nil + }*/ - var refs []api.SealedRef - if len(v) > 0 { - if err := cbor.DecodeInto(v, &refs); err != nil { - return err - } + return s.sectorBlocks.Has(c) +} + +func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) { + /*val, err := s.local.Get(c) // TODO: Pass staging + if err == nil { + return val, nil } + if err != blockstore.ErrNotFound { + return nil, err + }*/ - refs = append(refs, api.SealedRef{ - Piece: string(SerializationUnixfs0) + cid.String(), - Offset: offset, - Size: size, - }) - - newRef, err := cbor.DumpObject(&refs) + refs, err := s.sectorBlocks.GetRefs(c) if err != nil { - return err + return nil, err } - return st.keys.Put(dshelp.CidToDsKey(cid), newRef) // TODO: batch somehow -} - -func (r *refStorer) Read(p []byte) (n int, err error) { - offset := 0 - if len(r.remaining) > 0 { - offset += len(r.remaining) - read := copy(p, r.remaining) - if read == len(r.remaining) { - r.remaining = nil - } else { - r.remaining = r.remaining[read:] - } - return read, nil + if len(refs) == 0 { + return nil, blockstore.ErrNotFound } - for { - data, offset, cid, err := r.blockReader.ReadBlock(context.TODO()) - if err != nil { - return 0, err - } - - if len(data) == 0 { - panic("Handle intermediate nodes") // TODO: ! - } - - if err := r.writeRef(cid, offset, uint32(len(data))); err != nil { - return 0, err - } - - read := copy(p, data) - if read < len(data) { - r.remaining = data[read:] - } - // TODO: read multiple - return read, nil - } -} - -func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast uint64) (sectorID uint64, err error) { - size, err := r.Size() - if err != nil { - return 0, err - } - - refst := &refStorer{blockReader: r, pieceRef: string(SerializationUnixfs0) + ref.String(), writeRef: st.writeRef} - - return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) -} - -func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { - res, err := st.keys.Query(query.Query{}) + data, err := s.sectorBlocks.unsealed.getRef(context.TODO(), refs, s.approveUnseal) if err != nil { return nil, err } - ents, err := res.Rest() - if err != nil { - return nil, err - } - - out := map[cid.Cid][]api.SealedRef{} - for _, ent := range ents { - refCid, err := dshelp.DsKeyToCid(datastore.RawKey(ent.Key)) - if err != nil { - return nil, err - } - - var refs []api.SealedRef - if err := cbor.DecodeInto(ent.Value, &refs); err != nil { - return nil, err - } - - out[refCid] = refs - } - - return out, nil + return blocks.NewBlockWithCid(data, c) } -func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: track unsealed sectors - ent, err := st.keys.Get(dshelp.CidToDsKey(k)) - if err != nil { - return nil, err - } - var refs []api.SealedRef - if err := cbor.DecodeInto(ent, &refs); err != nil { - return nil, err - } - - return refs, nil -} +var _ blockstore.Blockstore = &SectorBlockStore{} diff --git a/storage/sectorblocks/unsealed.go b/storage/sectorblocks/unsealed.go new file mode 100644 index 000000000..d2cbfa820 --- /dev/null +++ b/storage/sectorblocks/unsealed.go @@ -0,0 +1,98 @@ +package sectorblocks + +import ( + "context" + "sync" + + logging "github.com/ipfs/go-log" + + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/lib/sectorbuilder" +) + +var log = logging.Logger("sectorblocks") + +type unsealedBlocks struct { + lk sync.Mutex + sb *sectorbuilder.SectorBuilder + + // TODO: Treat this as some sort of cache, one with rather aggressive GC + // TODO: This REALLY, REALLY needs to be on-disk + unsealed map[string][]byte + + unsealing map[string]chan struct{} +} + +func (ub *unsealedBlocks) getRef(ctx context.Context, refs []api.SealedRef, approveUnseal func() error) ([]byte, error) { + var best api.SealedRef + + ub.lk.Lock() + for _, ref := range refs { + b, ok := ub.unsealed[ref.Piece] + if ok { + ub.lk.Unlock() + return b[ref.Offset:ref.Offset + uint64(ref.Size)], nil // TODO: check slice math + } + // TODO: pick unsealing based on how long it's running (or just select all relevant, usually it'll be just one) + _, ok = ub.unsealing[ref.Piece] + if ok { + best = ref + break + } + best = ref + } + ub.lk.Unlock() + + b, err := ub.maybeUnseal(ctx, best.Piece, approveUnseal) + if err != nil { + return nil, err + } + + return b[best.Offset:best.Offset + uint64(best.Size)], nil // TODO: check slice math +} + +func (ub *unsealedBlocks) maybeUnseal(ctx context.Context, pieceKey string, approveUnseal func() error) ([]byte, error) { + ub.lk.Lock() + defer ub.lk.Unlock() + + out, ok := ub.unsealed[pieceKey] + if ok { + return out, nil + } + + wait, ok := ub.unsealing[pieceKey] + if ok { + ub.lk.Unlock() + select { + case <-wait: + ub.lk.Lock() + // TODO: make sure this is not racy with gc when it's implemented + return ub.unsealed[pieceKey], nil + case <-ctx.Done(): + ub.lk.Lock() + return nil, ctx.Err() + } + } + + // TODO: doing this under a lock is suboptimal.. but simpler + err := approveUnseal() + if err != nil { + return nil, err + } + + ub.unsealing[pieceKey] = make(chan struct{}) + ub.lk.Unlock() + + log.Infof("Unsealing piece '%s'", pieceKey) + data, err := ub.sb.ReadPieceFromSealedSector(pieceKey) + if err != nil { + return nil, err + } + + ub.lk.Lock() + ub.unsealed[pieceKey] = data + close(ub.unsealing[pieceKey]) + return data, nil +} + + From 6714240653c5bb510122b7721e844a5f42df7162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 27 Aug 2019 21:54:14 +0200 Subject: [PATCH 07/30] SectorSize const --- .gitignore | 1 + build/params.go | 1 + chain/actors/actor_storagemarket.go | 5 ++--- chain/gen/utils.go | 3 ++- node/impl/storminer.go | 3 ++- node/modules/storageminer.go | 3 ++- storage/miner.go | 3 ++- 7 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 73837f5eb..25f132ace 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /lotus-storage-miner /pond /lotuspond/front/node_modules +/lotuspond/front/build **/*.h **/*.a **/*.pc diff --git a/build/params.go b/build/params.go index b3b71c563..1be30939f 100644 --- a/build/params.go +++ b/build/params.go @@ -3,5 +3,6 @@ package build // Core network constants const UnixfsChunkSize uint64 = 1 << 20 +const SectorSize = 1024 // TODO: Move other important consts here diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 2fd0ad592..68632c35e 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -1,6 +1,7 @@ package actors import ( + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/chain/actors/aerrors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/types" @@ -9,8 +10,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -const SectorSize = 1024 - func init() { cbor.RegisterCborType(StorageMarketState{}) cbor.RegisterCborType(CreateStorageMinerParams{}) @@ -104,7 +103,7 @@ func (sma StorageMarketActor) CreateStorageMiner(act *types.Actor, vmctx types.V } func SupportedSectorSize(ssize types.BigInt) bool { - if ssize.Uint64() == SectorSize { + if ssize.Uint64() == build.SectorSize { return true } return false diff --git a/chain/gen/utils.go b/chain/gen/utils.go index ce5e4607e..d94958d45 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -3,6 +3,7 @@ package gen import ( "context" "fmt" + "github.com/filecoin-project/go-lotus/build" actors "github.com/filecoin-project/go-lotus/chain/actors" "github.com/filecoin-project/go-lotus/chain/address" @@ -168,7 +169,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid params := mustEnc(actors.CreateStorageMinerParams{ Owner: gmcfg.Owner, Worker: gmcfg.Worker, - SectorSize: types.NewInt(1024), + SectorSize: types.NewInt(build.SectorSize), PeerID: gmcfg.PeerID, }) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 6c2856735..11bfe39f2 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,6 +3,7 @@ package impl import ( "context" "fmt" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/storage/sectorblocks" "io" "math/rand" @@ -30,7 +31,7 @@ func (sm *StorageMinerAPI) ActorAddresses(context.Context) ([]address.Address, e } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) { - size := uint64(1016) // this is the most data we can fit in a 1024 byte sector + size := uint64(build.SectorSize - 8) // this is the most data we can fit in a sector. TODO: check if correct name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), 1016)) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 8a4bc77bf..c3b4ef5b0 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,6 +2,7 @@ package modules import ( "context" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/retrieval" "github.com/filecoin-project/go-lotus/storage/sector" "path/filepath" @@ -55,7 +56,7 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS) (*sectorbui sb := §orbuilder.SectorBuilderConfig{ Miner: minerAddr, - SectorSize: 1024, + SectorSize: build.SectorSize, MetadataDir: metadata, SealedDir: sealed, StagedDir: staging, diff --git a/storage/miner.go b/storage/miner.go index 81e1a35b9..51e63faf8 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,6 +3,7 @@ package storage import ( "context" "fmt" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/storage/sector" "github.com/filecoin-project/go-lotus/api" @@ -102,7 +103,7 @@ func (m *Miner) handlePostingSealedSectors(ctx context.Context) { func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error { log.Info("committing sector") - ok, err := sectorbuilder.VerifySeal(1024, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof) + ok, err := sectorbuilder.VerifySeal(build.SectorSize, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof) if err != nil { log.Error("failed to verify seal we just created: ", err) } From 433550e9a4f19135a67e36170ab7ad664838ec19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 27 Aug 2019 21:54:39 +0200 Subject: [PATCH 08/30] gofmt --- api/api.go | 2 +- api/struct.go | 2 +- retrieval/client.go | 6 +++--- retrieval/miner.go | 10 +++++----- retrieval/types.go | 2 +- storage/miner.go | 14 +++++++------- storage/sectorblocks/blocks.go | 7 +++---- storage/sectorblocks/blockstore.go | 1 - storage/sectorblocks/unsealed.go | 6 ++---- 9 files changed, 23 insertions(+), 27 deletions(-) diff --git a/api/api.go b/api/api.go index 8130df873..466bb6229 100644 --- a/api/api.go +++ b/api/api.go @@ -93,7 +93,7 @@ type FullNode interface { ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) - ClientRetrieve(ctx context.Context, order RetrievalOrder) error // TODO: maybe just allow putting this straight into some file + ClientRetrieve(ctx context.Context, order RetrievalOrder) error // TODO: maybe just allow putting this straight into some file // ClientUnimport removes references to the specified file from filestore //ClientUnimport(path string) diff --git a/api/struct.go b/api/struct.go index 00424bfbe..53551010d 100644 --- a/api/struct.go +++ b/api/struct.go @@ -73,7 +73,7 @@ type FullNodeStruct struct { ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` - ClientRetrieve func(ctx context.Context, order RetrievalOrder) error `perm:"admin"` + ClientRetrieve func(ctx context.Context, order RetrievalOrder) error `perm:"admin"` StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` StateMinerProvingSet func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` diff --git a/retrieval/client.go b/retrieval/client.go index 143ccb5e3..9d052ec30 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -62,7 +62,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. } return api.QueryOffer{ - Root: data, + Root: data, Size: resp.Size, MinPrice: resp.MinPrice, Miner: p.Address, // TODO: check @@ -104,8 +104,8 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, root: root, offset: 0, // TODO: Check how much data we have locally - // TODO: Support in handler - // TODO: Allow client to specify this + // TODO: Support in handler + // TODO: Allow client to specify this windowSize: build.UnixfsChunkSize, verifier: &OptimisticVerifier{}, // TODO: Use a real verifier diff --git a/retrieval/miner.go b/retrieval/miner.go index d36f52342..c8d8a2ccf 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -54,7 +54,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { // TODO: get price, look for already unsealed ref to reduce work answer.MinPrice = types.BigMul(types.NewInt(uint64(refs[0].Size)), m.pricePerByte) - answer.Size = uint64(refs[0].Size) // TODO: verify on intermediate + answer.Size = uint64(refs[0].Size) // TODO: verify on intermediate } if err := cborrpc.WriteCborRPC(stream, answer); err != nil { @@ -66,7 +66,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { func writeErr(stream network.Stream, err error) { log.Errorf("Retrieval deal error: %s", err) _ = cborrpc.WriteCborRPC(stream, DealResponse{ - Status: Error, + Status: Error, Message: err.Error(), }) } @@ -82,9 +82,9 @@ func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we blo for { var deal Deal if err := cborrpc.ReadCborRPC(stream, &deal); err != nil { - return + return } - + if deal.Unixfs0 == nil { writeErr(stream, xerrors.New("unknown deal type")) return @@ -133,7 +133,7 @@ func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we blo size = uint64(isize) } - if deal.Unixfs0.Offset + deal.Unixfs0.Size > size { + if deal.Unixfs0.Offset+deal.Unixfs0.Size > size { writeErr(stream, xerrors.Errorf("tried to read too much %d+%d > %d", deal.Unixfs0.Offset, deal.Unixfs0.Size, size)) return } diff --git a/retrieval/types.go b/retrieval/types.go index bd1a51810..765c583af 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -59,6 +59,6 @@ type Deal struct { } type DealResponse struct { - Status int // TODO: make this more spec complainant + Status int // TODO: make this more spec complainant Message string } diff --git a/storage/miner.go b/storage/miner.go index 51e63faf8..e46910879 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,21 +3,21 @@ package storage import ( "context" "fmt" - "github.com/filecoin-project/go-lotus/build" - "github.com/filecoin-project/go-lotus/storage/sector" - "github.com/filecoin-project/go-lotus/api" - "github.com/filecoin-project/go-lotus/chain/actors" - "github.com/filecoin-project/go-lotus/chain/address" - "github.com/filecoin-project/go-lotus/chain/store" - "github.com/filecoin-project/go-lotus/chain/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" host "github.com/libp2p/go-libp2p-core/host" "github.com/pkg/errors" + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/chain/actors" + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/filecoin-project/go-lotus/chain/store" + "github.com/filecoin-project/go-lotus/chain/types" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" + "github.com/filecoin-project/go-lotus/storage/sector" ) var log = logging.Logger("storageminer") diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 324309aef..76b0fe045 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -31,8 +31,8 @@ type SectorBlocks struct { *sector.Store unsealed *unsealedBlocks - keys datastore.Batching - keyLk sync.Mutex + keys datastore.Batching + keyLk sync.Mutex } func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { @@ -43,7 +43,7 @@ func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuild unsealed := &unsealedBlocks{ // TODO: untangle this sb: sb, - unsealed: map[string][]byte{}, + unsealed: map[string][]byte{}, unsealing: map[string]chan struct{}{}, } @@ -201,4 +201,3 @@ func (st *SectorBlocks) SealedBlockstore(approveUnseal func() error) *SectorBloc approveUnseal: approveUnseal, } } - diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index e69c16e62..2087b9b6d 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -74,5 +74,4 @@ func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) { return blocks.NewBlockWithCid(data, c) } - var _ blockstore.Blockstore = &SectorBlockStore{} diff --git a/storage/sectorblocks/unsealed.go b/storage/sectorblocks/unsealed.go index d2cbfa820..d211910d3 100644 --- a/storage/sectorblocks/unsealed.go +++ b/storage/sectorblocks/unsealed.go @@ -31,7 +31,7 @@ func (ub *unsealedBlocks) getRef(ctx context.Context, refs []api.SealedRef, appr b, ok := ub.unsealed[ref.Piece] if ok { ub.lk.Unlock() - return b[ref.Offset:ref.Offset + uint64(ref.Size)], nil // TODO: check slice math + return b[ref.Offset : ref.Offset+uint64(ref.Size)], nil // TODO: check slice math } // TODO: pick unsealing based on how long it's running (or just select all relevant, usually it'll be just one) _, ok = ub.unsealing[ref.Piece] @@ -48,7 +48,7 @@ func (ub *unsealedBlocks) getRef(ctx context.Context, refs []api.SealedRef, appr return nil, err } - return b[best.Offset:best.Offset + uint64(best.Size)], nil // TODO: check slice math + return b[best.Offset : best.Offset+uint64(best.Size)], nil // TODO: check slice math } func (ub *unsealedBlocks) maybeUnseal(ctx context.Context, pieceKey string, approveUnseal func() error) ([]byte, error) { @@ -94,5 +94,3 @@ func (ub *unsealedBlocks) maybeUnseal(ctx context.Context, pieceKey string, appr close(ub.unsealing[pieceKey]) return data, nil } - - From c0566399c6498ba538acc4ccb43eebb856894563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 00:10:23 +0200 Subject: [PATCH 09/30] retrieval: Way to get the data --- api/api.go | 2 +- api/struct.go | 6 +- build/params.go | 2 + cli/client.go | 8 +- cmd/lotus-storage-miner/init.go | 2 +- node/client/import.go | 1 - node/impl/full/client.go | 17 ++++- retrieval/client.go | 27 ++++--- retrieval/verify.go | 131 ++++++++++++++++++++++++++++++-- 9 files changed, 168 insertions(+), 28 deletions(-) delete mode 100644 node/client/import.go diff --git a/api/api.go b/api/api.go index 466bb6229..902fe90d2 100644 --- a/api/api.go +++ b/api/api.go @@ -93,7 +93,7 @@ type FullNode interface { ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) - ClientRetrieve(ctx context.Context, order RetrievalOrder) error // TODO: maybe just allow putting this straight into some file + ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error // ClientUnimport removes references to the specified file from filestore //ClientUnimport(path string) diff --git a/api/struct.go b/api/struct.go index 53551010d..aeb1857d9 100644 --- a/api/struct.go +++ b/api/struct.go @@ -73,7 +73,7 @@ type FullNodeStruct struct { ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` - ClientRetrieve func(ctx context.Context, order RetrievalOrder) error `perm:"admin"` + ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` StateMinerProvingSet func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` @@ -167,8 +167,8 @@ func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, mine return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } -func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order RetrievalOrder) error { - return c.Internal.ClientRetrieve(ctx, order) +func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error { + return c.Internal.ClientRetrieve(ctx, order, path) } func (c *FullNodeStruct) MpoolPending(ctx context.Context, ts *types.TipSet) ([]*types.SignedMessage, error) { diff --git a/build/params.go b/build/params.go index 1be30939f..0901bebc7 100644 --- a/build/params.go +++ b/build/params.go @@ -3,6 +3,8 @@ package build // Core network constants const UnixfsChunkSize uint64 = 1 << 20 +const UnixfsLinksPerLevel = 1024 + const SectorSize = 1024 // TODO: Move other important consts here diff --git a/cli/client.go b/cli/client.go index 4d7f92476..8a3525e46 100644 --- a/cli/client.go +++ b/cli/client.go @@ -163,8 +163,8 @@ var clientRetrieveCmd = &cli.Command{ Name: "retrieve", Usage: "retrieve data from network", Action: func(cctx *cli.Context) error { - if !cctx.Args().Present() { - fmt.Println("Usage: retrieve [CID]") + if cctx.NArg() != 2 { + fmt.Println("Usage: retrieve [CID] [outfile]") return nil } @@ -189,7 +189,7 @@ var clientRetrieveCmd = &cli.Command{ if has { fmt.Println("Success: Already in local storage") return nil - }*/ // TODO: uncomment before merge + }*/ // TODO: fix offers, err := api.ClientFindData(ctx, file) if err != nil { @@ -199,7 +199,7 @@ var clientRetrieveCmd = &cli.Command{ // TODO: parse offer strings from `client find`, make this smarter order := offers[0].Order() - err = api.ClientRetrieve(ctx, order) + err = api.ClientRetrieve(ctx, order, cctx.Args().Get(1)) if err == nil { fmt.Println("Success") } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 0567f51b0..ee49589fb 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -227,7 +227,7 @@ func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID) ( params, err := actors.SerializeParams(actors.CreateStorageMinerParams{ Owner: defOwner, Worker: k, - SectorSize: types.NewInt(actors.SectorSize), + SectorSize: types.NewInt(build.SectorSize), PeerID: peerid, }) if err != nil { diff --git a/node/client/import.go b/node/client/import.go deleted file mode 100644 index da13c8ef3..000000000 --- a/node/client/import.go +++ /dev/null @@ -1 +0,0 @@ -package client diff --git a/node/impl/full/client.go b/node/impl/full/client.go index f64202965..f70fa21ad 100644 --- a/node/impl/full/client.go +++ b/node/impl/full/client.go @@ -171,7 +171,7 @@ func (a *ClientAPI) ClientImport(ctx context.Context, path string) (cid.Cid, err bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG) params := ihelper.DagBuilderParams{ - Maxlinks: ihelper.DefaultLinksPerBlock, + Maxlinks: build.UnixfsLinksPerLevel, RawLeaves: true, CidBuilder: nil, Dagserv: bufferedDS, @@ -219,6 +219,17 @@ func (a *ClientAPI) ClientListImports(ctx context.Context) ([]api.Import, error) } } -func (a *ClientAPI) ClientRetrieve(ctx context.Context, order api.RetrievalOrder) error { - return a.Retrieval.RetrieveUnixfs(ctx, order.Root, order.Size, order.MinerPeerID, order.Miner) +func (a *ClientAPI) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error { + outFile, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) + if err != nil { + return err + } + + err = a.Retrieval.RetrieveUnixfs(ctx, order.Root, order.Size, order.MinerPeerID, order.Miner, outFile) + if err != nil { + _ = outFile.Close() + return err + } + + return outFile.Close() } diff --git a/retrieval/client.go b/retrieval/client.go index 9d052ec30..baaca9d5d 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -2,6 +2,7 @@ package retrieval import ( "context" + "io" "io/ioutil" pb "github.com/ipfs/go-bitswap/message/pb" @@ -92,7 +93,7 @@ type clientStream struct { // < ..Blocks // > Deal(...) // < ... -func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, miner peer.ID, minerAddr address.Address) error { +func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, miner peer.ID, minerAddr address.Address, out io.Writer) error { s, err := c.h.NewStream(ctx, miner, ProtocolID) if err != nil { return err @@ -108,7 +109,7 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, // TODO: Allow client to specify this windowSize: build.UnixfsChunkSize, - verifier: &OptimisticVerifier{}, // TODO: Use a real verifier + verifier: &UnixFs0Verifier{Root: root}, } for cst.offset != size { @@ -116,8 +117,9 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, if toFetch+cst.offset > size { toFetch = size - cst.offset } + log.Infof("Retrieve %dB @%d", toFetch, cst.offset) - err := cst.doOneExchange(toFetch) + err := cst.doOneExchange(toFetch, out) if err != nil { return err } @@ -128,7 +130,7 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, return nil } -func (cst *clientStream) doOneExchange(toFetch uint64) error { +func (cst *clientStream) doOneExchange(toFetch uint64, out io.Writer) error { deal := Deal{Unixfs0: &Unixfs0Offer{ Root: cst.root, Offset: cst.offset, @@ -158,18 +160,21 @@ func (cst *clientStream) doOneExchange(toFetch uint64) error { return xerrors.New("storage deal response had no Accepted section") } - return cst.fetchBlocks(toFetch) + log.Info("Retrieval accepted, fetching blocks") + + return cst.fetchBlocks(toFetch, out) // TODO: maybe increase miner window size after success } -func (cst *clientStream) fetchBlocks(toFetch uint64) error { +func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize // TODO: put msgio into spec reader := msgio.NewVarintReaderSize(cst.stream, network.MessageSizeMax) for i := uint64(0); i < blocksToFetch; { + log.Infof("block %d of %d", i+1, blocksToFetch) msg, err := reader.ReadMsg() if err != nil { return err @@ -180,7 +185,7 @@ func (cst *clientStream) fetchBlocks(toFetch uint64) error { return err } - dataBlocks, err := cst.consumeBlockMessage(pb) + dataBlocks, err := cst.consumeBlockMessage(pb, out) if err != nil { return err } @@ -193,7 +198,7 @@ func (cst *clientStream) fetchBlocks(toFetch uint64) error { return nil } -func (cst *clientStream) consumeBlockMessage(pb pb.Message_Block) (uint64, error) { +func (cst *clientStream) consumeBlockMessage(pb pb.Message_Block, out io.Writer) (uint64, error) { prefix, err := cid.PrefixFromBytes(pb.GetPrefix()) if err != nil { return 0, err @@ -205,15 +210,17 @@ func (cst *clientStream) consumeBlockMessage(pb pb.Message_Block) (uint64, error return 0, err } - internal, err := cst.verifier.Verify(blk) + internal, err := cst.verifier.Verify(context.TODO(), blk, out) if err != nil { return 0, err } - // TODO: Persist block + // TODO: Smarter out, maybe add to filestore automagically + // (Also, persist intermediate nodes) if internal { return 0, nil } + return 1, nil } diff --git a/retrieval/verify.go b/retrieval/verify.go index 0c0dca227..e232a06ac 100644 --- a/retrieval/verify.go +++ b/retrieval/verify.go @@ -1,19 +1,140 @@ package retrieval -import blocks "github.com/ipfs/go-block-format" +import ( + "context" + "io" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-unixfs" + pb "github.com/ipfs/go-unixfs/pb" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/build" +) type BlockVerifier interface { - Verify(blocks.Block) (internal bool, err error) + Verify(context.Context, blocks.Block, io.Writer) (internal bool, err error) } -// TODO: BalancedUnixFs0Verifier - type OptimisticVerifier struct { } -func (o *OptimisticVerifier) Verify(blocks.Block) (bool, error) { +func (o *OptimisticVerifier) Verify(context.Context, blocks.Block, io.Writer) (bool, error) { // It's probably fine return false, nil } +type UnixFs0Verifier struct { + Root cid.Cid + rootBlk blocks.Block + + expect int + seen int + + sub *UnixFs0Verifier +} + +func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.Writer) (last bool, internal bool, err error) { + if b.sub != nil { + subLast, internal, err := b.sub.verify(ctx, blk, out) + if err != nil { + return false, false, err + } + if subLast { + b.sub = nil + b.seen++ + } + + return b.seen == b.expect, internal, nil + } + + if b.seen >= b.expect { // this is probably impossible + return false, false, xerrors.New("unixfs verifier: too many nodes in level") + } + + links, err := b.checkInternal(blk, out) + if err != nil { + return false, false, err + } + + if links > 0 { // TODO: check if all links are intermediate (or all aren't) + if links > build.UnixfsLinksPerLevel { + return false, false, xerrors.New("unixfs verifier: too many links in intermediate node") + } + + if b.seen+1 == b.expect && links != build.UnixfsLinksPerLevel { + return false, false, xerrors.New("unixfs verifier: too few nodes in level") + } + + b.sub = &UnixFs0Verifier{ + Root: blk.Cid(), + rootBlk: blk, + expect: links, + } + + // don't mark as seen yet + return false, true, nil + } + + b.seen++ + return b.seen == b.expect, false, nil +} + +func (b *UnixFs0Verifier) checkInternal(blk blocks.Block, out io.Writer) (int, error) { + nd, err := ipld.Decode(blk) + if err != nil { + return 0, err + } + + // TODO: check size + switch nd.(type) { + case *merkledag.ProtoNode: + fsn, err := unixfs.FSNodeFromBytes(nd.RawData()) + if err != nil { + return 0, err + } + if fsn.Type() != pb.Data_File { + return 0, xerrors.New("internal nodes must be a file") + } + if len(fsn.Data()) > 0 { + return 0, xerrors.New("internal node with data") + } + if len(nd.Links()) == 0 { + return 0, xerrors.New("internal node with no links") + } + return len(nd.Links()), nil + + case *merkledag.RawNode: + // TODO: do we check the hash before writing? + _, err := out.Write(nd.RawData()) + return 0, err + default: + return 0, xerrors.New("verifier: unknown node type") + } +} + +func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block, w io.Writer) (bool, error) { + // root is special + if b.rootBlk == nil { + if !b.Root.Equals(blk.Cid()) { + return false, xerrors.Errorf("unixfs verifier: root block CID didn't match: valid %s, got %s", b.Root, blk.Cid()) + } + b.rootBlk = blk + links, err := b.checkInternal(blk, w) + if err != nil { + return false, err + } + + b.expect = links + return links != 0, nil + } + + _, internal, err := b.verify(ctx, blk, w) + return internal, err +} + var _ BlockVerifier = &OptimisticVerifier{} +var _ BlockVerifier = &UnixFs0Verifier{} From a5d3122ba57114180eaa1a75b36d3afb800931e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 01:03:22 +0200 Subject: [PATCH 10/30] Make deals work with chunked files --- build/params.go | 2 +- node/impl/storminer.go | 2 +- storage/sectorblocks/blocks.go | 33 +++++++++++++++++++++++------- storage/sectorblocks/blockstore.go | 10 ++++----- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/build/params.go b/build/params.go index 0901bebc7..21c102e9f 100644 --- a/build/params.go +++ b/build/params.go @@ -5,6 +5,6 @@ package build const UnixfsChunkSize uint64 = 1 << 20 const UnixfsLinksPerLevel = 1024 -const SectorSize = 1024 +const SectorSize = 16 << 20 // TODO: Move other important consts here diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 11bfe39f2..41371fa66 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -31,7 +31,7 @@ func (sm *StorageMinerAPI) ActorAddresses(context.Context) ([]address.Address, e } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) { - size := uint64(build.SectorSize - 8) // this is the most data we can fit in a sector. TODO: check if correct + size := sectorbuilder.UserBytesForSectorSize(build.SectorSize) name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), 1016)) diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 76b0fe045..cebb8132c 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-lotus/node/modules/dtypes" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-datastore/query" + blockstore "github.com/ipfs/go-ipfs-blockstore" ipld "github.com/ipfs/go-ipld-format" "sync" @@ -26,10 +27,13 @@ const ( ) var dsPrefix = datastore.NewKey("/sealedblocks") +var imBlocksPrefix = datastore.NewKey("/intermediate") type SectorBlocks struct { *sector.Store + intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore + unsealed *unsealedBlocks keys datastore.Batching keyLk sync.Mutex @@ -38,11 +42,15 @@ type SectorBlocks struct { func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { sbc := &SectorBlocks{ Store: sectst, - keys: namespace.Wrap(ds, dsPrefix), + + intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)), + + keys: namespace.Wrap(ds, dsPrefix), } unsealed := &unsealedBlocks{ // TODO: untangle this - sb: sb, + sb: sb, + unsealed: map[string][]byte{}, unsealing: map[string]chan struct{}{}, } @@ -60,8 +68,9 @@ type UnixfsReader interface { } type refStorer struct { - blockReader UnixfsReader - writeRef func(cid cid.Cid, offset uint64, size uint32) error + blockReader UnixfsReader + writeRef func(cid cid.Cid, offset uint64, size uint32) error + intermediate blockstore.Blockstore pieceRef string remaining []byte @@ -119,7 +128,12 @@ func (r *refStorer) Read(p []byte) (n int, err error) { } if len(data) == 0 { - panic("Handle intermediate nodes") // TODO: ! + // TODO: batch + // TODO: GC + if err := r.intermediate.Put(nd); err != nil { + return 0, err + } + continue } if err := r.writeRef(nd.Cid(), offset, uint32(len(data))); err != nil { @@ -141,7 +155,12 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast return 0, err } - refst := &refStorer{blockReader: r, pieceRef: string(SerializationUnixfs0) + ref.String(), writeRef: st.writeRef} + refst := &refStorer{ + blockReader: r, + pieceRef: string(SerializationUnixfs0) + ref.String(), + writeRef: st.writeRef, + intermediate: st.intermediate, + } return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) } @@ -196,7 +215,7 @@ func (st *SectorBlocks) Has(k cid.Cid) (bool, error) { func (st *SectorBlocks) SealedBlockstore(approveUnseal func() error) *SectorBlockStore { return &SectorBlockStore{ - //local: nil, // TODO: Pass staging + intermediate: st.intermediate, sectorBlocks: st, approveUnseal: approveUnseal, } diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index 2087b9b6d..e28e750d0 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -8,7 +8,7 @@ import ( ) type SectorBlockStore struct { - // local blockstore.Blockstore // staging before GC // TODO: Pass staging + intermediate blockstore.Blockstore sectorBlocks *SectorBlocks approveUnseal func() error @@ -38,25 +38,25 @@ func (s *SectorBlockStore) HashOnRead(enabled bool) { } func (s *SectorBlockStore) Has(c cid.Cid) (bool, error) { - /*has, err := s.local.Has(c) // TODO: Pass staging + has, err := s.intermediate.Has(c) if err != nil { return false, err } if has { return true, nil - }*/ + } return s.sectorBlocks.Has(c) } func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) { - /*val, err := s.local.Get(c) // TODO: Pass staging + val, err := s.intermediate.Get(c) if err == nil { return val, nil } if err != blockstore.ErrNotFound { return nil, err - }*/ + } refs, err := s.sectorBlocks.GetRefs(c) if err != nil { From d7d42416d7796d3c1826e67d495c718b89e3ff8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 01:40:14 +0200 Subject: [PATCH 11/30] retrieval: Almost working chunked files --- retrieval/miner.go | 28 ++++++++++----------- storage/sectorblocks/blocks.go | 42 +++++++++++++++++++++++++++++--- storage/sectorblocks/unsealed.go | 5 +++- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/retrieval/miner.go b/retrieval/miner.go index c8d8a2ccf..1555836ef 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -31,17 +31,25 @@ func NewMiner(sblks *sectorblocks.SectorBlocks) *Miner { } } +func writeErr(stream network.Stream, err error) { + log.Errorf("Retrieval deal error: %s", err) + _ = cborrpc.WriteCborRPC(stream, DealResponse{ + Status: Error, + Message: err.Error(), + }) +} + func (m *Miner) HandleQueryStream(stream network.Stream) { defer stream.Close() var query Query if err := cborrpc.ReadCborRPC(stream, &query); err != nil { - log.Errorf("Retrieval query: ReadCborRPC: %s", err) + writeErr(stream, err) return } - refs, err := m.sectorBlocks.GetRefs(query.Piece) - if err != nil { + size, err := m.sectorBlocks.GetSize(query.Piece) + if err != nil && err != sectorblocks.ErrNotFound { log.Errorf("Retrieval query: GetRefs: %s", err) return } @@ -49,12 +57,12 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { answer := QueryResponse{ Status: Unavailable, } - if len(refs) > 0 { + if err == nil { answer.Status = Available // TODO: get price, look for already unsealed ref to reduce work - answer.MinPrice = types.BigMul(types.NewInt(uint64(refs[0].Size)), m.pricePerByte) - answer.Size = uint64(refs[0].Size) // TODO: verify on intermediate + answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), m.pricePerByte) + answer.Size = uint64(size) // TODO: verify on intermediate } if err := cborrpc.WriteCborRPC(stream, answer); err != nil { @@ -63,14 +71,6 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { } } -func writeErr(stream network.Stream, err error) { - log.Errorf("Retrieval deal error: %s", err) - _ = cborrpc.WriteCborRPC(stream, DealResponse{ - Status: Error, - Message: err.Error(), - }) -} - func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we block in stream handlers defer stream.Close() diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index cebb8132c..90d691660 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -2,6 +2,7 @@ package sectorblocks import ( "context" + "errors" "github.com/filecoin-project/go-lotus/api" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" "github.com/filecoin-project/go-lotus/node/modules/dtypes" @@ -9,6 +10,7 @@ import ( "github.com/ipfs/go-datastore/query" blockstore "github.com/ipfs/go-ipfs-blockstore" ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-unixfs" "sync" "github.com/ipfs/go-cid" @@ -29,6 +31,8 @@ const ( var dsPrefix = datastore.NewKey("/sealedblocks") var imBlocksPrefix = datastore.NewKey("/intermediate") +var ErrNotFound = errors.New("not found") + type SectorBlocks struct { *sector.Store @@ -69,14 +73,14 @@ type UnixfsReader interface { type refStorer struct { blockReader UnixfsReader - writeRef func(cid cid.Cid, offset uint64, size uint32) error + writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint32) error intermediate blockstore.Blockstore pieceRef string remaining []byte } -func (st *SectorBlocks) writeRef(cid cid.Cid, offset uint64, size uint32) error { +func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint32) error { st.keyLk.Lock() // TODO: make this multithreaded defer st.keyLk.Unlock() @@ -96,7 +100,7 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, offset uint64, size uint32) error } refs = append(refs, api.SealedRef{ - Piece: string(SerializationUnixfs0) + cid.String(), + Piece: pieceRef, Offset: offset, Size: size, }) @@ -136,7 +140,7 @@ func (r *refStorer) Read(p []byte) (n int, err error) { continue } - if err := r.writeRef(nd.Cid(), offset, uint32(len(data))); err != nil { + if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint32(len(data))); err != nil { return 0, err } @@ -196,6 +200,9 @@ func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: track local sectors ent, err := st.keys.Get(dshelp.CidToDsKey(k)) + if err == datastore.ErrNotFound { + err = ErrNotFound + } if err != nil { return nil, err } @@ -208,6 +215,33 @@ func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: t return refs, nil } +func (st *SectorBlocks) GetSize(k cid.Cid) (uint64, error) { + blk, err := st.intermediate.Get(k) + if err == blockstore.ErrNotFound { + refs, err := st.GetRefs(k) + if err != nil { + return 0, err + } + + return uint64(refs[0].Size), nil + } + if err != nil { + return 0, err + } + + nd, err := ipld.Decode(blk) + if err != nil { + return 0, err + } + + fsn, err := unixfs.ExtractFSNode(nd) + if err != nil { + return 0, err + } + + return fsn.FileSize(), nil +} + func (st *SectorBlocks) Has(k cid.Cid) (bool, error) { // TODO: ensure sector is still there return st.keys.Has(dshelp.CidToDsKey(k)) diff --git a/storage/sectorblocks/unsealed.go b/storage/sectorblocks/unsealed.go index d211910d3..d532b82ab 100644 --- a/storage/sectorblocks/unsealed.go +++ b/storage/sectorblocks/unsealed.go @@ -85,11 +85,14 @@ func (ub *unsealedBlocks) maybeUnseal(ctx context.Context, pieceKey string, appr log.Infof("Unsealing piece '%s'", pieceKey) data, err := ub.sb.ReadPieceFromSealedSector(pieceKey) + ub.lk.Lock() + if err != nil { + // TODO: tell subs + log.Error(err) return nil, err } - ub.lk.Lock() ub.unsealed[pieceKey] = data close(ub.unsealing[pieceKey]) return data, nil From 1c4296f7acc8da93c938bafe69e7c4180e2a9ae7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 16:22:27 +0200 Subject: [PATCH 12/30] update sectorbuilder --- extern/go-bls-sigs | 2 +- extern/go-sectorbuilder | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/go-bls-sigs b/extern/go-bls-sigs index 4bc4b8a7b..03705e06e 160000 --- a/extern/go-bls-sigs +++ b/extern/go-bls-sigs @@ -1 +1 @@ -Subproject commit 4bc4b8a7bbf8118b8629084ff9207cfbf146df5f +Subproject commit 03705e06e83ac0d4c98695dacd0f20a350cc93d7 diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index e75bc9b0a..186d3b6f5 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit e75bc9b0aaeab4e1c8ea7eda42d63a39ae80728f +Subproject commit 186d3b6f5a78625979f4cec1a6b72e91a2291308 From cd6aa9ad77fc5ffe22c923bfc32eecf38858347c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 18:14:32 +0200 Subject: [PATCH 13/30] Import proofs for paramfetch --- .gitmodules | 3 ++ Makefile | 21 +++++++++ extern/proofs/Makefile | 14 ++++++ extern/proofs/bin/.keep | 0 extern/proofs/install-proofs | 31 +++++++++++++ extern/proofs/install-shared.sh | 82 +++++++++++++++++++++++++++++++++ extern/proofs/misc/.keep | 0 extern/proofs/rust-fil-proofs | 1 + 8 files changed, 152 insertions(+) create mode 100644 extern/proofs/Makefile create mode 100644 extern/proofs/bin/.keep create mode 100644 extern/proofs/install-proofs create mode 100644 extern/proofs/install-shared.sh create mode 100644 extern/proofs/misc/.keep create mode 160000 extern/proofs/rust-fil-proofs diff --git a/.gitmodules b/.gitmodules index d8e0ccfa6..a5162bd71 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "extern/go-sectorbuilder"] path = extern/go-sectorbuilder url = https://github.com/filecoin-project/go-sectorbuilder +[submodule "extern/proofs/rust-fil-proofs"] + path = extern/proofs/rust-fil-proofs + url = git@github.com:filecoin-project/rust-fil-proofs.git diff --git a/Makefile b/Makefile index f26794f9b..5a087c129 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ MODULES:= CLEAN:= +## BLS + BLS_PATH:=extern/go-bls-sigs/ BLS_DEPS:=libbls_signatures.a libbls_signatures.pc libbls_signatures.h BLS_DEPS:=$(addprefix $(BLS_PATH),$(BLS_DEPS)) @@ -21,6 +23,8 @@ MODULES+=$(BLS_PATH) BUILD_DEPS+=build/.bls-install CLEAN+=build/.bls-install +## SECTOR BUILDER + SECTOR_BUILDER_PATH:=extern/go-sectorbuilder/ SECTOR_BUILDER_DEPS:=libsector_builder_ffi.a sector_builder_ffi.pc sector_builder_ffi.h SECTOR_BUILDER_DEPS:=$(addprefix $(SECTOR_BUILDER_PATH),$(SECTOR_BUILDER_DEPS)) @@ -35,6 +39,23 @@ MODULES+=$(SECTOR_BUILDER_PATH) BUILD_DEPS+=build/.sector-builder-install CLEAN+=build/.sector-builder-install +## PROOFS + +PROOFS_PATH:=extern/proofs/ +PROOFS_DEPS:=bin/paramcache bin/paramfetch misc/parameters.json +PROOFS_DEPS:=$(addprefix $(SECTOR_BUILDER_PATH),$(SECTOR_BUILDER_DEPS)) + +$(PROOFS_DEPS): build/.proofs-install ; + +build/.proofs-install: $(PROOFS_PATH) + $(MAKE) -C $(PROOFS_PATH) $(PROOFS_DEPS:$(PROOFS_PATH)%=%) + @touch $@ + +MODULES+=$(PROOFS_PATH) +BUILD_DEPS+=build/.proofs-install +CLEAN+=build/.proofs-install + +# end git modules $(MODULES): build/.update-modules ; diff --git a/extern/proofs/Makefile b/extern/proofs/Makefile new file mode 100644 index 000000000..33bc829f7 --- /dev/null +++ b/extern/proofs/Makefile @@ -0,0 +1,14 @@ +DEPS:=bin/paramcache bin/paramfetch misc/parameters.json + +all: $(DEPS) +.PHONY: all + +$(DEPS): .install-fil-proofs ; + +.install-fil-proofs: bls-signatures + ./install-proofs + @touch $@ + +clean: + rm -rf $(DEPS) .install-bls +.PHONY: clean diff --git a/extern/proofs/bin/.keep b/extern/proofs/bin/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/extern/proofs/install-proofs b/extern/proofs/install-proofs new file mode 100644 index 000000000..f1de838ad --- /dev/null +++ b/extern/proofs/install-proofs @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -Eeo pipefail + +# shellcheck source=install-shared.bash +source "$(dirname "${BASH_SOURCE[0]}")/install-shared.bash" + +subm_dir="rust-fil-proofs" + +git submodule update --init --recursive ${subm_dir} + +if download_release_tarball tarball_path "${subm_dir}"; then + tmp_dir=$(mktemp -d) + tar -C "$tmp_dir" -xzf "$tarball_path" + + mkdir -p bin + mkdir -p misc + + cp -R "${tmp_dir}/bin/*" bin + cp -R "${tmp_dir}/misc/*" misc +else + echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build" + build_from_source "${subm_dir}" + + mkdir -p bin + mkdir -p misc + + find "${subm_dir}" -type f -name parameters.json -exec mv -- "{}" ./misc/ \; + find "${subm_dir}" -type f -name paramcache -exec cp -- "{}" ./bin/ \; + find "${subm_dir}" -type f -name paramfetch -exec cp -- "{}" ./bin/ \; +fi \ No newline at end of file diff --git a/extern/proofs/install-shared.sh b/extern/proofs/install-shared.sh new file mode 100644 index 000000000..3c026bc81 --- /dev/null +++ b/extern/proofs/install-shared.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +download_release_tarball() { + __resultvar=$1 + __submodule_path=$2 + __repo_name=$(echo "$__submodule_path" | cut -d '/' -f 2) + __release_name="${__repo_name}-$(uname)" + __release_sha1=$(git rev-parse @:"${__submodule_path}") + __release_tag="${__release_sha1:0:16}" + __release_tag_url="https://api.github.com/repos/filecoin-project/${__repo_name}/releases/tags/${__release_tag}" + + echo "acquiring release @ ${__release_tag}" + + __release_response=$(curl \ + --retry 3 \ + --location "$__release_tag_url") + + __release_url=$(echo "$__release_response" | jq -r ".assets[] | select(.name | contains(\"${__release_name}\")) | .url") + + if [[ -z "$__release_url" ]]; then + (>&2 echo "failed to download release (tag URL: ${__release_tag_url}, response: ${__release_response})") + return 1 + fi + + __tar_path="/tmp/${__release_name}_$(basename "${__release_url}").tar.gz" + + if [[ ! -f "${__tar_path}" ]]; then + __asset_url=$(curl \ + --head \ + --retry 3 \ + --header "Accept:application/octet-stream" \ + --location \ + --output /dev/null \ + -w "%{url_effective}" \ + "$__release_url") + + curl --retry 3 --output "${__tar_path}" "$__asset_url" + if [[ $? -ne "0" ]]; then + (>&2 echo "failed to download release asset (tag URL: ${__release_tag_url}, asset URL: ${__asset_url})") + return 1 + fi + fi + + eval $__resultvar="'$__tar_path'" +} + +build_from_source() { + __submodule_path=$1 + __submodule_sha1=$(git rev-parse @:"${__submodule_path}") + __submodule_sha1_truncated="${__submodule_sha1:0:16}" + + echo "building from source @ ${__submodule_sha1_truncated}" + + if ! [ -x "$(command -v cargo)" ]; then + (>&2 echo 'Error: cargo is not installed.') + (>&2 echo 'Install Rust toolchain to resolve this problem.') + exit 1 + fi + + if ! [ -x "$(command -v rustup)" ]; then + (>&2 echo 'Error: rustup is not installed.') + (>&2 echo 'Install Rust toolchain installer to resolve this problem.') + exit 1 + fi + + if ! pushd "$__submodule_path"; then + (>&2 echo 'Error: could not cd into submodule') + exit 1 + fi + + + cargo --version + cargo update + + if [[ -f "./build-release.sh" ]]; then + ./build-release.sh "$(cat rust-toolchain)" + else + cargo build --release --all + fi + + popd || exit 1 +} diff --git a/extern/proofs/misc/.keep b/extern/proofs/misc/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/extern/proofs/rust-fil-proofs b/extern/proofs/rust-fil-proofs new file mode 160000 index 000000000..8ef99ff86 --- /dev/null +++ b/extern/proofs/rust-fil-proofs @@ -0,0 +1 @@ +Subproject commit 8ef99ff867bd0c60acbaae6b4fe4214066936c78 From c87d9dc4b4b2e54217c58b422015d73359073f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 18:42:22 +0200 Subject: [PATCH 14/30] Extract go-fil-proofs --- .gitmodules | 6 +-- Makefile | 16 ++++++- build/params.go | 2 +- extern/go-fil-proofs | 1 + extern/proofs/Makefile | 14 ------ extern/proofs/bin/.keep | 0 extern/proofs/install-proofs | 31 ------------- extern/proofs/install-shared.sh | 82 --------------------------------- extern/proofs/misc/.keep | 0 extern/proofs/rust-fil-proofs | 1 - 10 files changed, 19 insertions(+), 134 deletions(-) create mode 160000 extern/go-fil-proofs delete mode 100644 extern/proofs/Makefile delete mode 100644 extern/proofs/bin/.keep delete mode 100644 extern/proofs/install-proofs delete mode 100644 extern/proofs/install-shared.sh delete mode 100644 extern/proofs/misc/.keep delete mode 160000 extern/proofs/rust-fil-proofs diff --git a/.gitmodules b/.gitmodules index a5162bd71..c05da4f38 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,6 +4,6 @@ [submodule "extern/go-sectorbuilder"] path = extern/go-sectorbuilder url = https://github.com/filecoin-project/go-sectorbuilder -[submodule "extern/proofs/rust-fil-proofs"] - path = extern/proofs/rust-fil-proofs - url = git@github.com:filecoin-project/rust-fil-proofs.git +[submodule "extern/go-fil-proofs"] + path = extern/go-fil-proofs + url = git@github.com:filecoin-project/go-fil-proofs.git diff --git a/Makefile b/Makefile index 5a087c129..ddf325607 100644 --- a/Makefile +++ b/Makefile @@ -41,9 +41,9 @@ CLEAN+=build/.sector-builder-install ## PROOFS -PROOFS_PATH:=extern/proofs/ +PROOFS_PATH:=extern/go-fil-proofs/ PROOFS_DEPS:=bin/paramcache bin/paramfetch misc/parameters.json -PROOFS_DEPS:=$(addprefix $(SECTOR_BUILDER_PATH),$(SECTOR_BUILDER_DEPS)) +PROOFS_DEPS:=$(addprefix $(PROOFS_PATH),$(PROOFS_DEPS)) $(PROOFS_DEPS): build/.proofs-install ; @@ -55,6 +55,16 @@ MODULES+=$(PROOFS_PATH) BUILD_DEPS+=build/.proofs-install CLEAN+=build/.proofs-install +PARAM_SECTOR_SIZES:=1024 268435456 +PARAM_SECTOR_SIZES:=$(addprefix params-,$(PARAM_SECTOR_SIZES)) + +$(PARAM_SECTOR_SIZES): extern/go-fil-proofs/bin/paramfetch extern/go-fil-proofs/misc/parameters.json + ./extern/go-fil-proofs/bin/paramfetch -z $(subst params-,,$@) --verbose --json=./extern/go-fil-proofs/misc/parameters.json +.PHONY: $(PARAM_SECTOR_SIZES) + +paramcache: extern/go-fil-proofs/bin/paramcache + RUST_LOG=info ./extern/go-fil-proofs/bin/paramcache + # end git modules $(MODULES): build/.update-modules ; @@ -82,6 +92,8 @@ pond: build clean: rm -rf $(CLEAN) -$(MAKE) -C $(BLS_PATH) clean + -$(MAKE) -C $(SECTOR_BUILDER_PATH) clean + -$(MAKE) -C $(PROOFS_PATH) clean .PHONY: clean dist-clean: diff --git a/build/params.go b/build/params.go index 21c102e9f..5160b2725 100644 --- a/build/params.go +++ b/build/params.go @@ -5,6 +5,6 @@ package build const UnixfsChunkSize uint64 = 1 << 20 const UnixfsLinksPerLevel = 1024 -const SectorSize = 16 << 20 +const SectorSize = 268435456 // TODO: Move other important consts here diff --git a/extern/go-fil-proofs b/extern/go-fil-proofs new file mode 160000 index 000000000..5201fa9a4 --- /dev/null +++ b/extern/go-fil-proofs @@ -0,0 +1 @@ +Subproject commit 5201fa9a47494a673da4322dffdf20b7bc335acb diff --git a/extern/proofs/Makefile b/extern/proofs/Makefile deleted file mode 100644 index 33bc829f7..000000000 --- a/extern/proofs/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -DEPS:=bin/paramcache bin/paramfetch misc/parameters.json - -all: $(DEPS) -.PHONY: all - -$(DEPS): .install-fil-proofs ; - -.install-fil-proofs: bls-signatures - ./install-proofs - @touch $@ - -clean: - rm -rf $(DEPS) .install-bls -.PHONY: clean diff --git a/extern/proofs/bin/.keep b/extern/proofs/bin/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/extern/proofs/install-proofs b/extern/proofs/install-proofs deleted file mode 100644 index f1de838ad..000000000 --- a/extern/proofs/install-proofs +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -Eeo pipefail - -# shellcheck source=install-shared.bash -source "$(dirname "${BASH_SOURCE[0]}")/install-shared.bash" - -subm_dir="rust-fil-proofs" - -git submodule update --init --recursive ${subm_dir} - -if download_release_tarball tarball_path "${subm_dir}"; then - tmp_dir=$(mktemp -d) - tar -C "$tmp_dir" -xzf "$tarball_path" - - mkdir -p bin - mkdir -p misc - - cp -R "${tmp_dir}/bin/*" bin - cp -R "${tmp_dir}/misc/*" misc -else - echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build" - build_from_source "${subm_dir}" - - mkdir -p bin - mkdir -p misc - - find "${subm_dir}" -type f -name parameters.json -exec mv -- "{}" ./misc/ \; - find "${subm_dir}" -type f -name paramcache -exec cp -- "{}" ./bin/ \; - find "${subm_dir}" -type f -name paramfetch -exec cp -- "{}" ./bin/ \; -fi \ No newline at end of file diff --git a/extern/proofs/install-shared.sh b/extern/proofs/install-shared.sh deleted file mode 100644 index 3c026bc81..000000000 --- a/extern/proofs/install-shared.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -download_release_tarball() { - __resultvar=$1 - __submodule_path=$2 - __repo_name=$(echo "$__submodule_path" | cut -d '/' -f 2) - __release_name="${__repo_name}-$(uname)" - __release_sha1=$(git rev-parse @:"${__submodule_path}") - __release_tag="${__release_sha1:0:16}" - __release_tag_url="https://api.github.com/repos/filecoin-project/${__repo_name}/releases/tags/${__release_tag}" - - echo "acquiring release @ ${__release_tag}" - - __release_response=$(curl \ - --retry 3 \ - --location "$__release_tag_url") - - __release_url=$(echo "$__release_response" | jq -r ".assets[] | select(.name | contains(\"${__release_name}\")) | .url") - - if [[ -z "$__release_url" ]]; then - (>&2 echo "failed to download release (tag URL: ${__release_tag_url}, response: ${__release_response})") - return 1 - fi - - __tar_path="/tmp/${__release_name}_$(basename "${__release_url}").tar.gz" - - if [[ ! -f "${__tar_path}" ]]; then - __asset_url=$(curl \ - --head \ - --retry 3 \ - --header "Accept:application/octet-stream" \ - --location \ - --output /dev/null \ - -w "%{url_effective}" \ - "$__release_url") - - curl --retry 3 --output "${__tar_path}" "$__asset_url" - if [[ $? -ne "0" ]]; then - (>&2 echo "failed to download release asset (tag URL: ${__release_tag_url}, asset URL: ${__asset_url})") - return 1 - fi - fi - - eval $__resultvar="'$__tar_path'" -} - -build_from_source() { - __submodule_path=$1 - __submodule_sha1=$(git rev-parse @:"${__submodule_path}") - __submodule_sha1_truncated="${__submodule_sha1:0:16}" - - echo "building from source @ ${__submodule_sha1_truncated}" - - if ! [ -x "$(command -v cargo)" ]; then - (>&2 echo 'Error: cargo is not installed.') - (>&2 echo 'Install Rust toolchain to resolve this problem.') - exit 1 - fi - - if ! [ -x "$(command -v rustup)" ]; then - (>&2 echo 'Error: rustup is not installed.') - (>&2 echo 'Install Rust toolchain installer to resolve this problem.') - exit 1 - fi - - if ! pushd "$__submodule_path"; then - (>&2 echo 'Error: could not cd into submodule') - exit 1 - fi - - - cargo --version - cargo update - - if [[ -f "./build-release.sh" ]]; then - ./build-release.sh "$(cat rust-toolchain)" - else - cargo build --release --all - fi - - popd || exit 1 -} diff --git a/extern/proofs/misc/.keep b/extern/proofs/misc/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/extern/proofs/rust-fil-proofs b/extern/proofs/rust-fil-proofs deleted file mode 160000 index 8ef99ff86..000000000 --- a/extern/proofs/rust-fil-proofs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8ef99ff867bd0c60acbaae6b4fe4214066936c78 From e28c0c8d802c4d1513816052886d64e3e091e874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 22:20:03 +0200 Subject: [PATCH 15/30] Fix sectorbuilder poRepProofPartitions --- lib/sectorbuilder/sectorbuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index ec8714506..e9d7e4047 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -34,7 +34,7 @@ type SectorBuilderConfig struct { func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { proverId := addressToProverID(cfg.Miner) - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 1, 1, 1, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 1, 1, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) if err != nil { return nil, err } From a6914309a60671f8688aee55403fe233a7ec34f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Aug 2019 23:11:29 +0200 Subject: [PATCH 16/30] retrieval: 'fix' GetPIP in handler --- chain/deals/handler_states.go | 3 ++- retrieval/client.go | 1 + retrieval/verify.go | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/chain/deals/handler_states.go b/chain/deals/handler_states.go index c84320fb1..8e5e07a26 100644 --- a/chain/deals/handler_states.go +++ b/chain/deals/handler_states.go @@ -134,7 +134,8 @@ func (h *Handler) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, err } - ip, err := getInclusionProof(deal.Ref.String(), status) + // TODO: don't hardcode unixfs + ip, err := getInclusionProof(string(sectorblocks.SerializationUnixfs0) + deal.Ref.String(), status) if err != nil { return nil, err } diff --git a/retrieval/client.go b/retrieval/client.go index baaca9d5d..1e51aaa00 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -212,6 +212,7 @@ func (cst *clientStream) consumeBlockMessage(pb pb.Message_Block, out io.Writer) internal, err := cst.verifier.Verify(context.TODO(), blk, out) if err != nil { + log.Warnf("block verify failed: %s", err) return 0, err } diff --git a/retrieval/verify.go b/retrieval/verify.go index e232a06ac..f17f58a55 100644 --- a/retrieval/verify.go +++ b/retrieval/verify.go @@ -86,14 +86,16 @@ func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.W func (b *UnixFs0Verifier) checkInternal(blk blocks.Block, out io.Writer) (int, error) { nd, err := ipld.Decode(blk) if err != nil { + log.Warnf("IPLD Decode failed: %s", err) return 0, err } // TODO: check size - switch nd.(type) { + switch nd := nd.(type) { case *merkledag.ProtoNode: - fsn, err := unixfs.FSNodeFromBytes(nd.RawData()) + fsn, err := unixfs.FSNodeFromBytes(nd.Data()) if err != nil { + log.Warnf("unixfs.FSNodeFromBytes failed: %s", err) return 0, err } if fsn.Type() != pb.Data_File { From 483f06b329766171e12e6e13824b60497f7c66cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 00:12:52 +0200 Subject: [PATCH 17/30] retrieval: make handler work with chunked files --- retrieval/miner.go | 252 +++++++++++++++++++++++++-------------------- 1 file changed, 138 insertions(+), 114 deletions(-) diff --git a/retrieval/miner.go b/retrieval/miner.go index 1555836ef..ec04c0dc8 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -71,127 +71,151 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { } } +type handlerDeal struct { + m *Miner + stream network.Stream + + ufsr sectorblocks.UnixfsReader + open cid.Cid + at uint64 + size uint64 +} + func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we block in stream handlers defer stream.Close() - var ufsr sectorblocks.UnixfsReader - var open cid.Cid - var at uint64 - var size uint64 + hnd := &handlerDeal{ + m: m, + + stream: stream, + } for { - var deal Deal - if err := cborrpc.ReadCborRPC(stream, &deal); err != nil { + err := hnd.handleNext() // TODO: 'more' bool + if err != nil { + writeErr(stream, err) return } - - if deal.Unixfs0 == nil { - writeErr(stream, xerrors.New("unknown deal type")) - return - } - - // TODO: Verify payment, check how much we can send based on that - // Or reject (possibly returning the payment to retain reputation with the client) - - bstore := m.sectorBlocks.SealedBlockstore(func() error { - return nil // TODO: approve unsealing based on amount paid - }) - - if open != deal.Unixfs0.Root || at != deal.Unixfs0.Offset { - if deal.Unixfs0.Offset != 0 { - // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs - writeErr(stream, xerrors.New("sending merkle proofs for nonzero offset not supported yet")) - return - } - at = deal.Unixfs0.Offset - - ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) - rootNd, err := ds.Get(context.TODO(), deal.Unixfs0.Root) - if err != nil { - writeErr(stream, err) - return - } - - fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd) - if err != nil { - writeErr(stream, err) - return - } - - var ok bool - ufsr, ok = fsr.(sectorblocks.UnixfsReader) - if !ok { - writeErr(stream, xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Unixfs0.Root)) - return - } - - isize, err := ufsr.Size() - if err != nil { - writeErr(stream, err) - return - } - size = uint64(isize) - } - - if deal.Unixfs0.Offset+deal.Unixfs0.Size > size { - writeErr(stream, xerrors.Errorf("tried to read too much %d+%d > %d", deal.Unixfs0.Offset, deal.Unixfs0.Size, size)) - return - } - - resp := DealResponse{ - Status: Accepted, - } - if err := cborrpc.WriteCborRPC(stream, resp); err != nil { - log.Errorf("Retrieval query: Write Accepted resp: %s", err) - return - } - - buf := make([]byte, network.MessageSizeMax) - msgw := msgio.NewVarintWriter(stream) - - blocksToSend := (deal.Unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - for i := uint64(0); i < blocksToSend; { - data, offset, nd, err := ufsr.ReadBlock(context.TODO()) - if err != nil { - writeErr(stream, err) - return - } - - log.Infof("sending block for a deal: %s", nd.Cid()) - - if offset != deal.Unixfs0.Offset { - writeErr(stream, xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", deal.Unixfs0.Offset, offset)) - return - } - - /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) - writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data))) - return - }*/ - - block := pb.Message_Block{ - Prefix: nd.Cid().Prefix().Bytes(), - Data: nd.RawData(), - } - - n, err := block.MarshalTo(buf) - if err != nil { - writeErr(stream, err) - return - } - - if err := msgw.WriteMsg(buf[:n]); err != nil { - log.Error(err) - return - } - - if len(data) > 0 { // don't count internal nodes - i++ - } - } - - // TODO: set `at` - } } + +func (hnd *handlerDeal) handleNext() error { + var deal Deal + if err := cborrpc.ReadCborRPC(hnd.stream, &deal); err != nil { + return err + } + + if deal.Unixfs0 == nil { + return xerrors.New("unknown deal type") + } + + // TODO: Verify payment, check how much we can send based on that + // Or reject (possibly returning the payment to retain reputation with the client) + + if hnd.open != deal.Unixfs0.Root || hnd.at != deal.Unixfs0.Offset { + log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, deal.Unixfs0.Offset) + if err := hnd.openFile(deal); err != nil { + return err + } + } + + if deal.Unixfs0.Offset+deal.Unixfs0.Size > hnd.size { + return xerrors.Errorf("tried to read too much %d+%d > %d", deal.Unixfs0.Offset, deal.Unixfs0.Size, hnd.size) + } + + return hnd.accept(deal) +} + +func (hnd *handlerDeal) openFile(deal Deal) error { + if deal.Unixfs0.Offset != 0 { + // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs + return xerrors.New("sending merkle proofs for nonzero offset not supported yet") + } + hnd.at = deal.Unixfs0.Offset + + bstore := hnd.m.sectorBlocks.SealedBlockstore(func() error { + return nil // TODO: approve unsealing based on amount paid + }) + + ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) + rootNd, err := ds.Get(context.TODO(), deal.Unixfs0.Root) + if err != nil { + return err + } + + fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd) + if err != nil { + return err + } + + var ok bool + hnd.ufsr, ok = fsr.(sectorblocks.UnixfsReader) + if !ok { + return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Unixfs0.Root) + } + + isize, err := hnd.ufsr.Size() + if err != nil { + return err + } + hnd.size = uint64(isize) + + hnd.open = deal.Unixfs0.Root + + return nil +} + +func (hnd *handlerDeal) accept(deal Deal) error { + resp := DealResponse{ + Status: Accepted, + } + if err := cborrpc.WriteCborRPC(hnd.stream, resp); err != nil { + log.Errorf("Retrieval query: Write Accepted resp: %s", err) + return err + } + + buf := make([]byte, network.MessageSizeMax) + msgw := msgio.NewVarintWriter(hnd.stream) + + blocksToSend := (deal.Unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize + for i := uint64(0); i < blocksToSend; { + data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) + if err != nil { + return err + } + + log.Infof("sending block for a deal: %s", nd.Cid()) + + if offset != deal.Unixfs0.Offset { + return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", deal.Unixfs0.Offset, offset) + } + + /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) + writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data))) + return + }*/ + + block := pb.Message_Block{ + Prefix: nd.Cid().Prefix().Bytes(), + Data: nd.RawData(), + } + + n, err := block.MarshalTo(buf) + if err != nil { + return err + } + + if err := msgw.WriteMsg(buf[:n]); err != nil { + log.Error(err) + return err + } + + if len(data) > 0 { // don't count internal nodes + hnd.at += uint64(len(data)) + i++ + } + } + + return nil +} From 985ca7d43940d0d0d555b7a06748e55d4c6574d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 01:01:12 +0200 Subject: [PATCH 18/30] remove gohacks from gomod --- go.mod | 6 +----- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 10f2bf6e6..db54da84a 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/ipfs/go-ipld-format v0.0.2 github.com/ipfs/go-log v0.0.2-0.20190708183747-9c9fd6111bea github.com/ipfs/go-merkledag v0.2.3 - github.com/ipfs/go-unixfs v0.2.1 + github.com/ipfs/go-unixfs v0.2.2-0.20190823181532-868af2e9e5cb63f114a445a00d595c028026c28c github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 github.com/libp2p/go-libp2p v0.3.0 github.com/libp2p/go-libp2p-circuit v0.1.1 @@ -84,7 +84,3 @@ require ( replace github.com/filecoin-project/go-bls-sigs => ./extern/go-bls-sigs replace github.com/filecoin-project/go-sectorbuilder => ./extern/go-sectorbuilder - -replace github.com/ipfs/go-car => /home/magik6k/gohack/github.com/ipfs/go-car - -replace github.com/ipfs/go-unixfs => /home/magik6k/gohack/github.com/ipfs/go-unixfs diff --git a/go.sum b/go.sum index 4bc79d4e9..9865a229f 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,8 @@ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJ github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc= github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= +github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4 h1:qYLz/x/d1SOiiFGS8dwCBCFJ5Oh64Y8HMBrS+MbaU8c= +github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4/go.mod h1:NSSM0pxlhej9rSFXQmB/lDru7TYNoDjKgmvqzlsd06Y= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= @@ -222,6 +224,8 @@ github.com/ipfs/go-peertaskqueue v0.1.1 h1:+gPjbI+V3NktXZOqJA1kzbms2pYmhjgQQal0M github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw= github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= +github.com/ipfs/go-unixfs v0.2.2-0.20190823181532-868af2e9e5cb63f114a445a00d595c028026c28c h1:SaTWPkLp9OWnd+Wm6oyBQ1NODnvSmRFz9bKNo4PheVU= +github.com/ipfs/go-unixfs v0.2.2-0.20190823181532-868af2e9e5cb63f114a445a00d595c028026c28c/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= From d8bcb37f9f13abefd40e7cb91502ddbf26537a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 01:01:28 +0200 Subject: [PATCH 19/30] retrieval: Address review --- cli/client.go | 2 +- retrieval/client.go | 10 ++++++---- retrieval/miner.go | 27 +++++++++++++++++++-------- retrieval/verify.go | 3 ++- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cli/client.go b/cli/client.go index 8a3525e46..6838ef0bf 100644 --- a/cli/client.go +++ b/cli/client.go @@ -116,7 +116,7 @@ var clientFindCmd = &cli.Command{ Usage: "find data in the network", Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { - fmt.Println("Usage: retrieve [CID]") + fmt.Println("Usage: find [CID]") return nil } diff --git a/retrieval/client.go b/retrieval/client.go index 1e51aaa00..da219a019 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -100,19 +100,21 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, } defer s.Close() + initialOffset := uint64(0) // TODO: Check how much data we have locally + // TODO: Support in handler + // TODO: Allow client to specify this + cst := clientStream{ stream: s, root: root, - offset: 0, // TODO: Check how much data we have locally - // TODO: Support in handler - // TODO: Allow client to specify this + offset: initialOffset, windowSize: build.UnixfsChunkSize, verifier: &UnixFs0Verifier{Root: root}, } - for cst.offset != size { + for cst.offset != size + initialOffset { toFetch := cst.windowSize if toFetch+cst.offset > size { toFetch = size - cst.offset diff --git a/retrieval/miner.go b/retrieval/miner.go index ec04c0dc8..1be4b41f6 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-msgio" "golang.org/x/xerrors" + "io" "github.com/filecoin-project/go-lotus/chain/types" "github.com/filecoin-project/go-lotus/lib/cborrpc" @@ -90,8 +91,11 @@ func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we blo stream: stream, } - for { - err := hnd.handleNext() // TODO: 'more' bool + var err error + more := true + + for more { + more, err = hnd.handleNext() // TODO: 'more' bool if err != nil { writeErr(stream, err) return @@ -100,14 +104,17 @@ func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we blo } -func (hnd *handlerDeal) handleNext() error { +func (hnd *handlerDeal) handleNext() (bool, error) { var deal Deal if err := cborrpc.ReadCborRPC(hnd.stream, &deal); err != nil { - return err + if err == io.EOF { // client sent all deals + err = nil + } + return false, err } if deal.Unixfs0 == nil { - return xerrors.New("unknown deal type") + return false, xerrors.New("unknown deal type") } // TODO: Verify payment, check how much we can send based on that @@ -116,15 +123,19 @@ func (hnd *handlerDeal) handleNext() error { if hnd.open != deal.Unixfs0.Root || hnd.at != deal.Unixfs0.Offset { log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, deal.Unixfs0.Offset) if err := hnd.openFile(deal); err != nil { - return err + return false, err } } if deal.Unixfs0.Offset+deal.Unixfs0.Size > hnd.size { - return xerrors.Errorf("tried to read too much %d+%d > %d", deal.Unixfs0.Offset, deal.Unixfs0.Size, hnd.size) + return false, xerrors.Errorf("tried to read too much %d+%d > %d", deal.Unixfs0.Offset, deal.Unixfs0.Size, hnd.size) } - return hnd.accept(deal) + err := hnd.accept(deal) + if err != nil { + return false, err + } + return true, nil } func (hnd *handlerDeal) openFile(deal Deal) error { diff --git a/retrieval/verify.go b/retrieval/verify.go index f17f58a55..b8cd8d400 100644 --- a/retrieval/verify.go +++ b/retrieval/verify.go @@ -39,6 +39,8 @@ type UnixFs0Verifier struct { func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.Writer) (last bool, internal bool, err error) { if b.sub != nil { + // TODO: check links here (iff b.sub.sub == nil) + subLast, internal, err := b.sub.verify(ctx, blk, out) if err != nil { return false, false, err @@ -110,7 +112,6 @@ func (b *UnixFs0Verifier) checkInternal(blk blocks.Block, out io.Writer) (int, e return len(nd.Links()), nil case *merkledag.RawNode: - // TODO: do we check the hash before writing? _, err := out.Write(nd.RawData()) return 0, err default: From c437b6884db402d40d210006b8c1769576e781ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 01:01:38 +0200 Subject: [PATCH 20/30] retrieval: gofmt --- chain/deals/handler_states.go | 2 +- retrieval/client.go | 2 +- retrieval/miner.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/deals/handler_states.go b/chain/deals/handler_states.go index 8e5e07a26..12ca2d002 100644 --- a/chain/deals/handler_states.go +++ b/chain/deals/handler_states.go @@ -135,7 +135,7 @@ func (h *Handler) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal) } // TODO: don't hardcode unixfs - ip, err := getInclusionProof(string(sectorblocks.SerializationUnixfs0) + deal.Ref.String(), status) + ip, err := getInclusionProof(string(sectorblocks.SerializationUnixfs0)+deal.Ref.String(), status) if err != nil { return nil, err } diff --git a/retrieval/client.go b/retrieval/client.go index da219a019..1ba21f46a 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -114,7 +114,7 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, verifier: &UnixFs0Verifier{Root: root}, } - for cst.offset != size + initialOffset { + for cst.offset != size+initialOffset { toFetch := cst.windowSize if toFetch+cst.offset > size { toFetch = size - cst.offset diff --git a/retrieval/miner.go b/retrieval/miner.go index 1be4b41f6..92f27aa8b 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -73,12 +73,12 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { } type handlerDeal struct { - m *Miner + m *Miner stream network.Stream ufsr sectorblocks.UnixfsReader open cid.Cid - at uint64 + at uint64 size uint64 } From a43890dc760bedde256ab16a959fb58a6e96dda2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 13:31:25 +0200 Subject: [PATCH 21/30] retrieval: send blocks as cbor --- retrieval/client.go | 28 +++++++++------------------- retrieval/miner.go | 28 +++++++++------------------- retrieval/types.go | 6 ++++++ 3 files changed, 24 insertions(+), 38 deletions(-) diff --git a/retrieval/client.go b/retrieval/client.go index 1ba21f46a..8b21078d3 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -5,7 +5,6 @@ import ( "io" "io/ioutil" - pb "github.com/ipfs/go-bitswap/message/pb" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -13,7 +12,6 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-msgio" "golang.org/x/xerrors" "github.com/filecoin-project/go-lotus/api" @@ -172,42 +170,34 @@ func (cst *clientStream) doOneExchange(toFetch uint64, out io.Writer) error { func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - // TODO: put msgio into spec - reader := msgio.NewVarintReaderSize(cst.stream, network.MessageSizeMax) - for i := uint64(0); i < blocksToFetch; { log.Infof("block %d of %d", i+1, blocksToFetch) - msg, err := reader.ReadMsg() - if err != nil { + + var block Block + if err := cborrpc.ReadCborRPC(cst.stream, &block); err != nil { return err } - var pb pb.Message_Block - if err := pb.Unmarshal(msg); err != nil { - return err - } - - dataBlocks, err := cst.consumeBlockMessage(pb, out) + dataBlocks, err := cst.consumeBlockMessage(block, out) if err != nil { return err } i += dataBlocks - - reader.ReleaseMsg(msg) } return nil } -func (cst *clientStream) consumeBlockMessage(pb pb.Message_Block, out io.Writer) (uint64, error) { - prefix, err := cid.PrefixFromBytes(pb.GetPrefix()) +func (cst *clientStream) consumeBlockMessage(block Block, out io.Writer) (uint64, error) { + prefix, err := cid.PrefixFromBytes(block.Prefix) if err != nil { return 0, err } - cid, err := prefix.Sum(pb.GetData()) - blk, err := blocks.NewBlockWithCid(pb.GetData(), cid) + cid, err := prefix.Sum(block.Data) + + blk, err := blocks.NewBlockWithCid(block.Data, cid) if err != nil { return 0, err } diff --git a/retrieval/miner.go b/retrieval/miner.go index 92f27aa8b..88fba6a46 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -2,20 +2,19 @@ package retrieval import ( "context" - "github.com/filecoin-project/go-lotus/build" - "github.com/ipfs/go-cid" - "github.com/libp2p/go-msgio" - "golang.org/x/xerrors" "io" - "github.com/filecoin-project/go-lotus/chain/types" - "github.com/filecoin-project/go-lotus/lib/cborrpc" - "github.com/filecoin-project/go-lotus/storage/sectorblocks" - pb "github.com/ipfs/go-bitswap/message/pb" "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" "github.com/libp2p/go-libp2p-core/network" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/chain/types" + "github.com/filecoin-project/go-lotus/lib/cborrpc" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" ) type Miner struct { @@ -186,9 +185,6 @@ func (hnd *handlerDeal) accept(deal Deal) error { return err } - buf := make([]byte, network.MessageSizeMax) - msgw := msgio.NewVarintWriter(hnd.stream) - blocksToSend := (deal.Unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize for i := uint64(0); i < blocksToSend; { data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) @@ -207,18 +203,12 @@ func (hnd *handlerDeal) accept(deal Deal) error { return }*/ - block := pb.Message_Block{ + block := Block{ Prefix: nd.Cid().Prefix().Bytes(), Data: nd.RawData(), } - n, err := block.MarshalTo(buf) - if err != nil { - return err - } - - if err := msgw.WriteMsg(buf[:n]); err != nil { - log.Error(err) + if err := cborrpc.WriteCborRPC(hnd.stream, block); err != nil { return err } diff --git a/retrieval/types.go b/retrieval/types.go index 765c583af..bef07c697 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -31,6 +31,7 @@ func init() { cbor.RegisterCborType(Unixfs0Offer{}) cbor.RegisterCborType(DealResponse{}) + cbor.RegisterCborType(Block{}) } type Query struct { @@ -62,3 +63,8 @@ type DealResponse struct { Status int // TODO: make this more spec complainant Message string } + +type Block struct { // TODO: put in spec + Prefix []byte // TODO: fix cid.Prefix marshaling somehow + Data []byte +} From fe6d5ff3a8f19fa4959c22647256c0c1f2c9d9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 17:09:34 +0200 Subject: [PATCH 22/30] retrieval: resolve some TODOs --- api/api.go | 2 +- chain/deals/client.go | 1 - chain/gen/utils.go | 2 +- node/builder.go | 2 +- node/impl/storminer.go | 4 ++-- node/modules/services.go | 4 ++-- node/modules/storageminer.go | 9 ++++----- retrieval/miner.go | 2 +- retrieval/types.go | 2 +- 9 files changed, 13 insertions(+), 15 deletions(-) diff --git a/api/api.go b/api/api.go index 902fe90d2..6d460f0b2 100644 --- a/api/api.go +++ b/api/api.go @@ -216,7 +216,7 @@ func (o *QueryOffer) Order() RetrievalOrder { } type RetrievalOrder struct { - // TODO: make this loss unixfs specific + // TODO: make this less unixfs specific Root cid.Cid Size uint64 // TODO: support offset diff --git a/chain/deals/client.go b/chain/deals/client.go index 839698406..427565890 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -212,7 +212,6 @@ func (c *Client) VerifyParams(ctx context.Context, data cid.Cid) (*actors.PieceI } func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.PieceInclVoucherData) (cid.Cid, error) { - // TODO: use data proposal := StorageDealProposal{ PieceRef: p.Data, SerializationMode: SerializationUnixFs, diff --git a/chain/gen/utils.go b/chain/gen/utils.go index d94958d45..61d913a85 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -3,8 +3,8 @@ package gen import ( "context" "fmt" - "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/build" actors "github.com/filecoin-project/go-lotus/chain/actors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/state" diff --git a/node/builder.go b/node/builder.go index 538a15f9d..0eb594fad 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,7 +3,6 @@ package node import ( "context" "errors" - "github.com/filecoin-project/go-lotus/retrieval" "reflect" "time" @@ -36,6 +35,7 @@ import ( "github.com/filecoin-project/go-lotus/node/modules/testing" "github.com/filecoin-project/go-lotus/node/repo" "github.com/filecoin-project/go-lotus/paych" + "github.com/filecoin-project/go-lotus/retrieval" "github.com/filecoin-project/go-lotus/retrieval/discovery" "github.com/filecoin-project/go-lotus/storage" "github.com/filecoin-project/go-lotus/storage/sector" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 41371fa66..8a58f7fd1 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,16 +3,16 @@ package impl import ( "context" "fmt" - "github.com/filecoin-project/go-lotus/build" - "github.com/filecoin-project/go-lotus/storage/sectorblocks" "io" "math/rand" "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" "github.com/filecoin-project/go-lotus/storage" "github.com/filecoin-project/go-lotus/storage/sector" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" ) type StorageMinerAPI struct { diff --git a/node/modules/services.go b/node/modules/services.go index 07528c0e4..658ac5fd8 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -2,8 +2,6 @@ package modules import ( "context" - "github.com/filecoin-project/go-lotus/retrieval/discovery" - "github.com/filecoin-project/go-lotus/storage/sector" "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" @@ -15,6 +13,8 @@ import ( "github.com/filecoin-project/go-lotus/chain/sub" "github.com/filecoin-project/go-lotus/node/hello" "github.com/filecoin-project/go-lotus/node/modules/helpers" + "github.com/filecoin-project/go-lotus/retrieval/discovery" + "github.com/filecoin-project/go-lotus/storage/sector" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index c3b4ef5b0..5ca2b3a71 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,31 +2,30 @@ package modules import ( "context" - "github.com/filecoin-project/go-lotus/build" - "github.com/filecoin-project/go-lotus/retrieval" - "github.com/filecoin-project/go-lotus/storage/sector" "path/filepath" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" - "github.com/libp2p/go-libp2p-core/routing" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" "github.com/mitchellh/go-homedir" "go.uber.org/fx" "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/deals" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" "github.com/filecoin-project/go-lotus/node/modules/dtypes" "github.com/filecoin-project/go-lotus/node/modules/helpers" "github.com/filecoin-project/go-lotus/node/repo" + "github.com/filecoin-project/go-lotus/retrieval" "github.com/filecoin-project/go-lotus/storage" + "github.com/filecoin-project/go-lotus/storage/sector" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { diff --git a/retrieval/miner.go b/retrieval/miner.go index 88fba6a46..9bb3f429d 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -81,7 +81,7 @@ type handlerDeal struct { size uint64 } -func (m *Miner) HandleDealStream(stream network.Stream) { // TODO: should we block in stream handlers +func (m *Miner) HandleDealStream(stream network.Stream) { defer stream.Close() hnd := &handlerDeal{ diff --git a/retrieval/types.go b/retrieval/types.go index bef07c697..1392a815f 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -49,7 +49,7 @@ type QueryResponse struct { MinPrice types.BigInt } -type Unixfs0Offer struct { +type Unixfs0Offer struct { // UNBORK Root cid.Cid Offset uint64 Size uint64 From f79b755c5843b6df2e6347a2f93156d6bc207cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 17:43:45 +0200 Subject: [PATCH 23/30] retrieval: Make types more spec complaiant --- go.mod | 1 - lib/cborrpc/rpc.go | 17 ++++++++++++++++- retrieval/client.go | 18 +++++++++++------- retrieval/miner.go | 38 ++++++++++++++++++++++---------------- retrieval/types.go | 14 ++++++++++---- 5 files changed, 59 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index db54da84a..7aa974af9 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,6 @@ require ( github.com/libp2p/go-libp2p-tls v0.1.0 github.com/libp2p/go-libp2p-yamux v0.2.1 github.com/libp2p/go-maddr-filter v0.0.5 - github.com/libp2p/go-msgio v0.0.4 github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/mitchellh/go-homedir v1.1.0 diff --git a/lib/cborrpc/rpc.go b/lib/cborrpc/rpc.go index db3553212..c02fc5298 100644 --- a/lib/cborrpc/rpc.go +++ b/lib/cborrpc/rpc.go @@ -1,16 +1,27 @@ package cborrpc import ( + "encoding/hex" "io" cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" cbg "github.com/whyrusleeping/cbor-gen" ) -const MessageSizeLimit = 1 << 20 +var log = logging.Logger("cborrrpc") + +const Debug = false + +func init() { + if Debug { + log.Warn("CBOR-RPC Debugging enabled") + } +} func WriteCborRPC(w io.Writer, obj interface{}) error { if m, ok := obj.(cbg.CBORMarshaler); ok { + // TODO: impl debug return m.MarshalCBOR(w) } data, err := cbor.DumpObject(obj) @@ -18,6 +29,10 @@ func WriteCborRPC(w io.Writer, obj interface{}) error { return err } + if Debug { + log.Infof("> %s", hex.EncodeToString(data)) + } + _, err = w.Write(data) return err } diff --git a/retrieval/client.go b/retrieval/client.go index 8b21078d3..d46d9620c 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -83,13 +83,13 @@ type clientStream struct { // // Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size // -// > Deal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} +// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} // < Resp{Accept} // < ..(Intermediate Block) // < ..Blocks // < ..(Intermediate Block) // < ..Blocks -// > Deal(...) +// > DealProposal(...) // < ... func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, miner peer.ID, minerAddr address.Address, out io.Writer) error { s, err := c.h.NewStream(ctx, miner, ProtocolID) @@ -131,11 +131,15 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, } func (cst *clientStream) doOneExchange(toFetch uint64, out io.Writer) error { - deal := Deal{Unixfs0: &Unixfs0Offer{ - Root: cst.root, - Offset: cst.offset, - Size: toFetch, - }} + deal := DealProposal{ + Ref: cst.root, + Params: RetParams{ + Unixfs0: &Unixfs0Offer{ + Offset: cst.offset, + Size: toFetch, + }, + }, + } if err := cborrpc.WriteCborRPC(cst.stream, deal); err != nil { return err diff --git a/retrieval/miner.go b/retrieval/miner.go index 9bb3f429d..bc1a95886 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -104,7 +104,7 @@ func (m *Miner) HandleDealStream(stream network.Stream) { } func (hnd *handlerDeal) handleNext() (bool, error) { - var deal Deal + var deal DealProposal if err := cborrpc.ReadCborRPC(hnd.stream, &deal); err != nil { if err == io.EOF { // client sent all deals err = nil @@ -112,22 +112,24 @@ func (hnd *handlerDeal) handleNext() (bool, error) { return false, err } - if deal.Unixfs0 == nil { + if deal.Params.Unixfs0 == nil { return false, xerrors.New("unknown deal type") } + unixfs0 := deal.Params.Unixfs0 + // TODO: Verify payment, check how much we can send based on that // Or reject (possibly returning the payment to retain reputation with the client) - if hnd.open != deal.Unixfs0.Root || hnd.at != deal.Unixfs0.Offset { - log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, deal.Unixfs0.Offset) + if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { + log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, unixfs0.Offset) if err := hnd.openFile(deal); err != nil { return false, err } } - if deal.Unixfs0.Offset+deal.Unixfs0.Size > hnd.size { - return false, xerrors.Errorf("tried to read too much %d+%d > %d", deal.Unixfs0.Offset, deal.Unixfs0.Size, hnd.size) + if unixfs0.Offset+unixfs0.Size > hnd.size { + return false, xerrors.Errorf("tried to read too much %d+%d > %d", unixfs0.Offset, unixfs0.Size, hnd.size) } err := hnd.accept(deal) @@ -137,19 +139,21 @@ func (hnd *handlerDeal) handleNext() (bool, error) { return true, nil } -func (hnd *handlerDeal) openFile(deal Deal) error { - if deal.Unixfs0.Offset != 0 { +func (hnd *handlerDeal) openFile(deal DealProposal) error { + unixfs0 := deal.Params.Unixfs0 + + if unixfs0.Offset != 0 { // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs return xerrors.New("sending merkle proofs for nonzero offset not supported yet") } - hnd.at = deal.Unixfs0.Offset + hnd.at = unixfs0.Offset bstore := hnd.m.sectorBlocks.SealedBlockstore(func() error { return nil // TODO: approve unsealing based on amount paid }) ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) - rootNd, err := ds.Get(context.TODO(), deal.Unixfs0.Root) + rootNd, err := ds.Get(context.TODO(), deal.Ref) if err != nil { return err } @@ -162,7 +166,7 @@ func (hnd *handlerDeal) openFile(deal Deal) error { var ok bool hnd.ufsr, ok = fsr.(sectorblocks.UnixfsReader) if !ok { - return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Unixfs0.Root) + return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Ref) } isize, err := hnd.ufsr.Size() @@ -171,12 +175,14 @@ func (hnd *handlerDeal) openFile(deal Deal) error { } hnd.size = uint64(isize) - hnd.open = deal.Unixfs0.Root + hnd.open = deal.Ref return nil } -func (hnd *handlerDeal) accept(deal Deal) error { +func (hnd *handlerDeal) accept(deal DealProposal) error { + unixfs0 := deal.Params.Unixfs0 + resp := DealResponse{ Status: Accepted, } @@ -185,7 +191,7 @@ func (hnd *handlerDeal) accept(deal Deal) error { return err } - blocksToSend := (deal.Unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize + blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize for i := uint64(0); i < blocksToSend; { data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) if err != nil { @@ -194,8 +200,8 @@ func (hnd *handlerDeal) accept(deal Deal) error { log.Infof("sending block for a deal: %s", nd.Cid()) - if offset != deal.Unixfs0.Offset { - return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", deal.Unixfs0.Offset, offset) + if offset != unixfs0.Offset { + return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", unixfs0.Offset, offset) } /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) diff --git a/retrieval/types.go b/retrieval/types.go index 1392a815f..968c5eaad 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -21,10 +21,11 @@ const ( Accepted = iota Error Rejected + Unsealing ) func init() { - cbor.RegisterCborType(Deal{}) + cbor.RegisterCborType(DealProposal{}) cbor.RegisterCborType(Query{}) cbor.RegisterCborType(QueryResponse{}) @@ -50,17 +51,22 @@ type QueryResponse struct { } type Unixfs0Offer struct { // UNBORK - Root cid.Cid Offset uint64 Size uint64 } -type Deal struct { +type RetParams struct { Unixfs0 *Unixfs0Offer } +type DealProposal struct { + Ref cid.Cid + Params RetParams + // TODO: payment +} + type DealResponse struct { - Status int // TODO: make this more spec complainant + Status int Message string } From 83748b40b616acee66503e870584a097b0f83620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 20:55:20 +0200 Subject: [PATCH 24/30] retrieval: address some review comments --- retrieval/miner.go | 2 ++ retrieval/types.go | 5 +++-- storage/sectorblocks/unsealed.go | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/retrieval/miner.go b/retrieval/miner.go index bc1a95886..434ccfab3 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -121,6 +121,8 @@ func (hnd *handlerDeal) handleNext() (bool, error) { // TODO: Verify payment, check how much we can send based on that // Or reject (possibly returning the payment to retain reputation with the client) + // If the file isn't open (new deal stream), isn't the right file, or isn't + // at the right offset, (re)open it if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, unixfs0.Offset) if err := hnd.openFile(deal); err != nil { diff --git a/retrieval/types.go b/retrieval/types.go index 968c5eaad..145b06732 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -25,12 +25,13 @@ const ( ) func init() { - cbor.RegisterCborType(DealProposal{}) + cbor.RegisterCborType(RetParams{}) cbor.RegisterCborType(Query{}) cbor.RegisterCborType(QueryResponse{}) cbor.RegisterCborType(Unixfs0Offer{}) + cbor.RegisterCborType(DealProposal{}) cbor.RegisterCborType(DealResponse{}) cbor.RegisterCborType(Block{}) } @@ -50,7 +51,7 @@ type QueryResponse struct { MinPrice types.BigInt } -type Unixfs0Offer struct { // UNBORK +type Unixfs0Offer struct { Offset uint64 Size uint64 } diff --git a/storage/sectorblocks/unsealed.go b/storage/sectorblocks/unsealed.go index d532b82ab..042e2c931 100644 --- a/storage/sectorblocks/unsealed.go +++ b/storage/sectorblocks/unsealed.go @@ -31,7 +31,7 @@ func (ub *unsealedBlocks) getRef(ctx context.Context, refs []api.SealedRef, appr b, ok := ub.unsealed[ref.Piece] if ok { ub.lk.Unlock() - return b[ref.Offset : ref.Offset+uint64(ref.Size)], nil // TODO: check slice math + return b[ref.Offset : ref.Offset+uint64(ref.Size)], nil } // TODO: pick unsealing based on how long it's running (or just select all relevant, usually it'll be just one) _, ok = ub.unsealing[ref.Piece] @@ -48,7 +48,7 @@ func (ub *unsealedBlocks) getRef(ctx context.Context, refs []api.SealedRef, appr return nil, err } - return b[best.Offset : best.Offset+uint64(best.Size)], nil // TODO: check slice math + return b[best.Offset : best.Offset+uint64(best.Size)], nil } func (ub *unsealedBlocks) maybeUnseal(ctx context.Context, pieceKey string, approveUnseal func() error) ([]byte, error) { From eedd098295eb9c0934f406e1ab3998fccc8a170c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 21:00:12 +0200 Subject: [PATCH 25/30] retrieval: set params back to 1024 --- build/params.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/params.go b/build/params.go index 5160b2725..0901bebc7 100644 --- a/build/params.go +++ b/build/params.go @@ -5,6 +5,6 @@ package build const UnixfsChunkSize uint64 = 1 << 20 const UnixfsLinksPerLevel = 1024 -const SectorSize = 268435456 +const SectorSize = 1024 // TODO: Move other important consts here From 6b80b98c64d2887dae4d4a69fe56a73af7a07d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 21:01:29 +0200 Subject: [PATCH 26/30] retrieval: fix actor tests --- chain/actors/actor_storagemarket_test.go | 3 ++- chain/actors/actors_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/chain/actors/actor_storagemarket_test.go b/chain/actors/actor_storagemarket_test.go index f4e6d7d3c..f368592ea 100644 --- a/chain/actors/actor_storagemarket_test.go +++ b/chain/actors/actor_storagemarket_test.go @@ -1,6 +1,7 @@ package actors_test import ( + "github.com/filecoin-project/go-lotus/build" "testing" . "github.com/filecoin-project/go-lotus/chain/actors" @@ -32,7 +33,7 @@ func TestStorageMarketCreateMiner(t *testing.T) { CreateStorageMinerParams{ Owner: ownerAddr, Worker: workerAddr, - SectorSize: types.NewInt(SectorSize), + SectorSize: types.NewInt(build.SectorSize), PeerID: "fakepeerid", }) ApplyOK(t, ret) diff --git a/chain/actors/actors_test.go b/chain/actors/actors_test.go index 26f0e6d69..9d3419355 100644 --- a/chain/actors/actors_test.go +++ b/chain/actors/actors_test.go @@ -3,6 +3,7 @@ package actors_test import ( "context" "encoding/binary" + "github.com/filecoin-project/go-lotus/build" "testing" . "github.com/filecoin-project/go-lotus/chain/actors" @@ -111,7 +112,7 @@ func TestStorageMarketActorCreateMiner(t *testing.T) { params := &StorageMinerConstructorParams{ Worker: maddr, - SectorSize: types.NewInt(SectorSize), + SectorSize: types.NewInt(build.SectorSize), PeerID: "fakepeerid", } enc, err := cbor.DumpObject(params) From 1ec0bf5e7b7c9058a1e62903e3433f306e311d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 21:05:55 +0200 Subject: [PATCH 27/30] mk: paramfetch in deps --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index ddf325607..d1c1f4584 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,8 @@ $(PARAM_SECTOR_SIZES): extern/go-fil-proofs/bin/paramfetch extern/go-fil-proofs/ ./extern/go-fil-proofs/bin/paramfetch -z $(subst params-,,$@) --verbose --json=./extern/go-fil-proofs/misc/parameters.json .PHONY: $(PARAM_SECTOR_SIZES) +BUILD_DEPS+=params-1024 + paramcache: extern/go-fil-proofs/bin/paramcache RUST_LOG=info ./extern/go-fil-proofs/bin/paramcache From 598ea1f23e041e9a9aad9266ffa42782b3997a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Aug 2019 21:08:12 +0200 Subject: [PATCH 28/30] Try to fix Circle --- .circleci/config.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b15393509..4cc49d88d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,7 @@ commands: steps: - go/install-ssh - go/install: {package: git} + - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y prepare: steps: - checkout @@ -62,7 +63,10 @@ jobs: - install-deps - prepare - go/mod-download - - run: make deps + - run: echo 'export PATH=$HOME/.cargo/bin:$PATH' >> $BASH_ENV + - run: + command: make deps + no_output_timeout: 30m - go/install-gotestsum: gobin: $HOME/.local/bin - run: @@ -114,7 +118,9 @@ jobs: - install-deps - prepare - go/mod-download - - run: make deps + - run: + command: make deps + no_output_timeout: 30mps - go/install-golangci-lint: gobin: $HOME/.local/bin version: << parameters.golangci-lint-version >> From 2b6c0686553be0512f90ce7a4b8246ef348f0a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 30 Aug 2019 13:26:20 +0200 Subject: [PATCH 29/30] Fix circleci again --- .circleci/config.yml | 4 ++-- Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4cc49d88d..4062f665b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,8 +119,8 @@ jobs: - prepare - go/mod-download - run: - command: make deps - no_output_timeout: 30mps + command: make deps + no_output_timeout: 30m - go/install-golangci-lint: gobin: $HOME/.local/bin version: << parameters.golangci-lint-version >> diff --git a/Makefile b/Makefile index d1c1f4584..85a23d1bb 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ PARAM_SECTOR_SIZES:=1024 268435456 PARAM_SECTOR_SIZES:=$(addprefix params-,$(PARAM_SECTOR_SIZES)) $(PARAM_SECTOR_SIZES): extern/go-fil-proofs/bin/paramfetch extern/go-fil-proofs/misc/parameters.json - ./extern/go-fil-proofs/bin/paramfetch -z $(subst params-,,$@) --verbose --json=./extern/go-fil-proofs/misc/parameters.json + IPFS_LOGGING=info ./extern/go-fil-proofs/bin/paramfetch -z $(subst params-,,$@) --verbose --json=./extern/go-fil-proofs/misc/parameters.json --ipget-version=v0.4.0 --ipget-args="--node spawn" .PHONY: $(PARAM_SECTOR_SIZES) BUILD_DEPS+=params-1024 From 640ea80eef0ec3d950b4cb3fb697fa712df6845d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 30 Aug 2019 14:08:41 -0700 Subject: [PATCH 30/30] Add in addresses of cluster peers to fetch from --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 85a23d1bb..9525e4572 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ PARAM_SECTOR_SIZES:=1024 268435456 PARAM_SECTOR_SIZES:=$(addprefix params-,$(PARAM_SECTOR_SIZES)) $(PARAM_SECTOR_SIZES): extern/go-fil-proofs/bin/paramfetch extern/go-fil-proofs/misc/parameters.json - IPFS_LOGGING=info ./extern/go-fil-proofs/bin/paramfetch -z $(subst params-,,$@) --verbose --json=./extern/go-fil-proofs/misc/parameters.json --ipget-version=v0.4.0 --ipget-args="--node spawn" + IPFS_LOGGING=info ./extern/go-fil-proofs/bin/paramfetch -z $(subst params-,,$@) --verbose --json=./extern/go-fil-proofs/misc/parameters.json --ipget-args="--node spawn -p /ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p /ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p /ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p /ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p /ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i" --ipget-version=v0.4.0 .PHONY: $(PARAM_SECTOR_SIZES) BUILD_DEPS+=params-1024