diff --git a/.circleci/config.yml b/.circleci/config.yml index b15393509..4062f665b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,7 @@ commands: steps: - go/install-ssh - go/install: {package: git} + - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y prepare: steps: - checkout @@ -62,7 +63,10 @@ jobs: - install-deps - prepare - go/mod-download - - run: make deps + - run: echo 'export PATH=$HOME/.cargo/bin:$PATH' >> $BASH_ENV + - run: + command: make deps + no_output_timeout: 30m - go/install-gotestsum: gobin: $HOME/.local/bin - run: @@ -114,7 +118,9 @@ jobs: - install-deps - prepare - go/mod-download - - run: make deps + - run: + command: make deps + no_output_timeout: 30m - go/install-golangci-lint: gobin: $HOME/.local/bin version: << parameters.golangci-lint-version >> diff --git a/.gitignore b/.gitignore index 73837f5eb..25f132ace 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /lotus-storage-miner /pond /lotuspond/front/node_modules +/lotuspond/front/build **/*.h **/*.a **/*.pc diff --git a/.gitmodules b/.gitmodules index d8e0ccfa6..c05da4f38 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "extern/go-sectorbuilder"] path = extern/go-sectorbuilder url = https://github.com/filecoin-project/go-sectorbuilder +[submodule "extern/go-fil-proofs"] + path = extern/go-fil-proofs + url = git@github.com:filecoin-project/go-fil-proofs.git diff --git a/Makefile b/Makefile index f26794f9b..9525e4572 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ MODULES:= CLEAN:= +## BLS + BLS_PATH:=extern/go-bls-sigs/ BLS_DEPS:=libbls_signatures.a libbls_signatures.pc libbls_signatures.h BLS_DEPS:=$(addprefix $(BLS_PATH),$(BLS_DEPS)) @@ -21,6 +23,8 @@ MODULES+=$(BLS_PATH) BUILD_DEPS+=build/.bls-install CLEAN+=build/.bls-install +## SECTOR BUILDER + SECTOR_BUILDER_PATH:=extern/go-sectorbuilder/ SECTOR_BUILDER_DEPS:=libsector_builder_ffi.a sector_builder_ffi.pc sector_builder_ffi.h SECTOR_BUILDER_DEPS:=$(addprefix $(SECTOR_BUILDER_PATH),$(SECTOR_BUILDER_DEPS)) @@ -35,6 +39,35 @@ MODULES+=$(SECTOR_BUILDER_PATH) BUILD_DEPS+=build/.sector-builder-install CLEAN+=build/.sector-builder-install +## PROOFS + +PROOFS_PATH:=extern/go-fil-proofs/ +PROOFS_DEPS:=bin/paramcache bin/paramfetch misc/parameters.json +PROOFS_DEPS:=$(addprefix $(PROOFS_PATH),$(PROOFS_DEPS)) + +$(PROOFS_DEPS): build/.proofs-install ; + +build/.proofs-install: $(PROOFS_PATH) + $(MAKE) -C $(PROOFS_PATH) $(PROOFS_DEPS:$(PROOFS_PATH)%=%) + @touch $@ + +MODULES+=$(PROOFS_PATH) +BUILD_DEPS+=build/.proofs-install +CLEAN+=build/.proofs-install + +PARAM_SECTOR_SIZES:=1024 268435456 +PARAM_SECTOR_SIZES:=$(addprefix params-,$(PARAM_SECTOR_SIZES)) + +$(PARAM_SECTOR_SIZES): extern/go-fil-proofs/bin/paramfetch extern/go-fil-proofs/misc/parameters.json + IPFS_LOGGING=info ./extern/go-fil-proofs/bin/paramfetch -z $(subst params-,,$@) --verbose --json=./extern/go-fil-proofs/misc/parameters.json --ipget-args="--node spawn -p /ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p /ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p /ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p /ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p /ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i" --ipget-version=v0.4.0 +.PHONY: $(PARAM_SECTOR_SIZES) + +BUILD_DEPS+=params-1024 + +paramcache: extern/go-fil-proofs/bin/paramcache + RUST_LOG=info ./extern/go-fil-proofs/bin/paramcache + +# end git modules $(MODULES): build/.update-modules ; @@ -61,6 +94,8 @@ pond: build clean: rm -rf $(CLEAN) -$(MAKE) -C $(BLS_PATH) clean + -$(MAKE) -C $(SECTOR_BUILDER_PATH) clean + -$(MAKE) -C $(PROOFS_PATH) clean .PHONY: clean dist-clean: diff --git a/api/api.go b/api/api.go index 8850a9d03..6d460f0b2 100644 --- a/api/api.go +++ b/api/api.go @@ -3,6 +3,9 @@ package api import ( "context" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-filestore" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -10,12 +13,13 @@ import ( "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/store" "github.com/filecoin-project/go-lotus/chain/types" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-filestore" ) +func init() { + cbor.RegisterCborType(SealedRef{}) +} + type Common interface { // Auth AuthVerify(ctx context.Context, token string) ([]string, error) @@ -87,6 +91,9 @@ type FullNode interface { // ClientImport imports file under the specified path into filestore ClientImport(ctx context.Context, path string) (cid.Cid, error) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) + ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) + ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) + ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error // ClientUnimport removes references to the specified file from filestore //ClientUnimport(path string) @@ -129,6 +136,8 @@ type StorageMiner interface { // Seal all staged sectors SectorsStagedSeal(context.Context) error + + SectorsRefs(context.Context) (map[string][]SealedRef, error) } // Version provides various build-time information @@ -178,3 +187,40 @@ type MinerPower struct { MinerPower types.BigInt TotalPower types.BigInt } + +type SealedRef struct { + Piece string + Offset uint64 + Size uint32 +} + +type QueryOffer struct { + Err string + + Root cid.Cid + + Size uint64 + MinPrice types.BigInt + + Miner address.Address + MinerPeerID peer.ID +} + +func (o *QueryOffer) Order() RetrievalOrder { + return RetrievalOrder{ + Root: o.Root, + Size: o.Size, + Miner: o.Miner, + MinerPeerID: o.MinerPeerID, + } +} + +type RetrievalOrder struct { + // TODO: make this less unixfs specific + Root cid.Cid + Size uint64 + // TODO: support offset + + Miner address.Address + MinerPeerID peer.ID +} diff --git a/api/struct.go b/api/struct.go index 84fdbf1fc..aeb1857d9 100644 --- a/api/struct.go +++ b/api/struct.go @@ -69,8 +69,11 @@ type FullNodeStruct struct { MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"write"` - ClientListImports func(ctx context.Context) ([]Import, error) `perm:"read"` + ClientListImports func(ctx context.Context) ([]Import, error) `perm:"write"` + ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` + ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` + ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` StateMinerProvingSet func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` @@ -101,6 +104,8 @@ type StorageMinerStruct struct { SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` SectorsStagedList func(context.Context) ([]sectorbuilder.StagedSectorMetadata, error) `perm:"read"` SectorsStagedSeal func(context.Context) error `perm:"write"` + + SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` } } @@ -150,10 +155,22 @@ func (c *FullNodeStruct) ClientImport(ctx context.Context, path string) (cid.Cid return c.Internal.ClientImport(ctx, path) } +func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { + return c.Internal.ClientHasLocal(ctx, root) +} + +func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) { + return c.Internal.ClientFindData(ctx, root) +} + func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } +func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error { + return c.Internal.ClientRetrieve(ctx, order, path) +} + func (c *FullNodeStruct) MpoolPending(ctx context.Context, ts *types.TipSet) ([]*types.SignedMessage, error) { return c.Internal.MpoolPending(ctx, ts) } @@ -329,6 +346,10 @@ func (c *StorageMinerStruct) SectorsStagedSeal(ctx context.Context) error { return c.Internal.SectorsStagedSeal(ctx) } +func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]SealedRef, error) { + return c.Internal.SectorsRefs(ctx) +} + var _ Common = &CommonStruct{} var _ FullNode = &FullNodeStruct{} var _ StorageMiner = &StorageMinerStruct{} diff --git a/build/params.go b/build/params.go new file mode 100644 index 000000000..0901bebc7 --- /dev/null +++ b/build/params.go @@ -0,0 +1,10 @@ +package build + +// Core network constants + +const UnixfsChunkSize uint64 = 1 << 20 +const UnixfsLinksPerLevel = 1024 + +const SectorSize = 1024 + +// TODO: Move other important consts here diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 2fd0ad592..68632c35e 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -1,6 +1,7 @@ package actors import ( + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/chain/actors/aerrors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/types" @@ -9,8 +10,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -const SectorSize = 1024 - func init() { cbor.RegisterCborType(StorageMarketState{}) cbor.RegisterCborType(CreateStorageMinerParams{}) @@ -104,7 +103,7 @@ func (sma StorageMarketActor) CreateStorageMiner(act *types.Actor, vmctx types.V } func SupportedSectorSize(ssize types.BigInt) bool { - if ssize.Uint64() == SectorSize { + if ssize.Uint64() == build.SectorSize { return true } return false diff --git a/chain/actors/actor_storagemarket_test.go b/chain/actors/actor_storagemarket_test.go index 7da4a7e89..8ba9897df 100644 --- a/chain/actors/actor_storagemarket_test.go +++ b/chain/actors/actor_storagemarket_test.go @@ -1,6 +1,7 @@ package actors_test import ( + "github.com/filecoin-project/go-lotus/build" "testing" . "github.com/filecoin-project/go-lotus/chain/actors" @@ -32,7 +33,7 @@ func TestStorageMarketCreateMiner(t *testing.T) { CreateStorageMinerParams{ Owner: ownerAddr, Worker: workerAddr, - SectorSize: types.NewInt(SectorSize), + SectorSize: types.NewInt(build.SectorSize), PeerID: "fakepeerid", }) ApplyOK(t, ret) diff --git a/chain/actors/actors_test.go b/chain/actors/actors_test.go index 9de35eb52..7fce5eef2 100644 --- a/chain/actors/actors_test.go +++ b/chain/actors/actors_test.go @@ -3,6 +3,7 @@ package actors_test import ( "context" "encoding/binary" + "github.com/filecoin-project/go-lotus/build" "testing" . "github.com/filecoin-project/go-lotus/chain/actors" @@ -111,7 +112,7 @@ func TestStorageMarketActorCreateMiner(t *testing.T) { params := &StorageMinerConstructorParams{ Worker: maddr, - SectorSize: types.NewInt(SectorSize), + SectorSize: types.NewInt(build.SectorSize), PeerID: "fakepeerid", } enc, err := cbor.DumpObject(params) diff --git a/chain/deals/client.go b/chain/deals/client.go index 1789d04c4..427565890 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -4,8 +4,6 @@ import ( "context" "math" - "github.com/filecoin-project/go-lotus/chain/actors" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -19,12 +17,14 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + "github.com/filecoin-project/go-lotus/chain/actors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/store" "github.com/filecoin-project/go-lotus/chain/types" "github.com/filecoin-project/go-lotus/chain/wallet" "github.com/filecoin-project/go-lotus/lib/cborrpc" "github.com/filecoin-project/go-lotus/node/modules/dtypes" + "github.com/filecoin-project/go-lotus/retrieval/discovery" ) func init() { @@ -48,10 +48,11 @@ type ClientDeal struct { } type Client struct { - cs *store.ChainStore - h host.Host - w *wallet.Wallet - dag dtypes.ClientDAG + cs *store.ChainStore + h host.Host + w *wallet.Wallet + dag dtypes.ClientDAG + discovery *discovery.Local deals StateStore @@ -61,12 +62,13 @@ type Client struct { stopped chan struct{} } -func NewClient(cs *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG) *Client { +func NewClient(cs *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local) *Client { c := &Client{ - cs: cs, - h: h, - w: w, - dag: dag, + cs: cs, + h: h, + w: w, + dag: dag, + discovery: discovery, deals: StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}, @@ -210,9 +212,8 @@ func (c *Client) VerifyParams(ctx context.Context, data cid.Cid) (*actors.PieceI } func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.PieceInclVoucherData) (cid.Cid, error) { - // TODO: use data proposal := StorageDealProposal{ - PieceRef: p.Data.String(), + PieceRef: p.Data, SerializationMode: SerializationUnixFs, CommP: vd.CommP[:], Size: vd.PieceSize.Uint64(), @@ -242,7 +243,12 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.Pie // TODO: actually care about what happens with the deal after it was accepted //c.incoming <- deal - return deal.ProposalCid, nil + + // TODO: start tracking after the deal is sealed + return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{ + Address: proposal.MinerAddress, + ID: deal.Miner, + }) } func (c *Client) Stop() { diff --git a/chain/deals/handler.go b/chain/deals/handler.go index 94f64a96b..f4e33c6b6 100644 --- a/chain/deals/handler.go +++ b/chain/deals/handler.go @@ -2,13 +2,12 @@ package deals import ( "context" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" "math" "github.com/filecoin-project/go-lotus/api" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/node/modules/dtypes" - "github.com/filecoin-project/go-lotus/storage/sector" - "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -35,7 +34,7 @@ type MinerDeal struct { } type Handler struct { - secst *sector.Store + secst *sectorblocks.SectorBlocks full api.FullNode // TODO: Use a custom protocol or graphsync in the future @@ -60,7 +59,7 @@ type dealUpdate struct { mut func(*MinerDeal) } -func NewHandler(ds dtypes.MetadataDS, secst *sector.Store, dag dtypes.StagingDAG, fullNode api.FullNode) (*Handler, error) { +func NewHandler(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, fullNode api.FullNode) (*Handler, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err diff --git a/chain/deals/handler_states.go b/chain/deals/handler_states.go index 486af2c83..12ca2d002 100644 --- a/chain/deals/handler_states.go +++ b/chain/deals/handler_states.go @@ -2,12 +2,13 @@ package deals import ( "context" - "github.com/filecoin-project/go-lotus/lib/sectorbuilder" - files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/lib/sectorbuilder" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" ) type handlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) @@ -75,18 +76,13 @@ func (h *Handler) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), return nil, xerrors.Errorf("cannot open unixfs file: %s", err) } - uf, ok := n.(files.File) + uf, ok := n.(sectorblocks.UnixfsReader) if !ok { // we probably got directory, unsupported for now - return nil, xerrors.Errorf("unsupported unixfs type") + return nil, xerrors.Errorf("unsupported unixfs file type") } - size, err := uf.Size() - if err != nil { - return nil, xerrors.Errorf("failed to get file size: %s", err) - } - - sectorID, err := h.secst.AddPiece(deal.Proposal.PieceRef, uint64(size), uf, deal.Proposal.Duration) + sectorID, err := h.secst.AddUnixfsPiece(deal.Proposal.PieceRef, uf, deal.Proposal.Duration) if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } @@ -138,7 +134,8 @@ func (h *Handler) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, err } - ip, err := getInclusionProof(deal.Ref.String(), status) + // TODO: don't hardcode unixfs + ip, err := getInclusionProof(string(sectorblocks.SerializationUnixfs0)+deal.Ref.String(), status) if err != nil { return nil, err } diff --git a/chain/deals/types.go b/chain/deals/types.go index f03688cab..f17fb1cac 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -41,7 +41,7 @@ const ( ) type StorageDealProposal struct { - PieceRef string + PieceRef cid.Cid // TODO: port to spec SerializationMode SerializationMode CommP []byte diff --git a/chain/gen/utils.go b/chain/gen/utils.go index ff31a1c7c..471e4ea25 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/filecoin-project/go-lotus/build" actors "github.com/filecoin-project/go-lotus/chain/actors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/state" @@ -168,7 +169,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid params := mustEnc(actors.CreateStorageMinerParams{ Owner: gmcfg.Owner, Worker: gmcfg.Worker, - SectorSize: types.NewInt(1024), + SectorSize: types.NewInt(build.SectorSize), PeerID: gmcfg.PeerID, }) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 06898eb70..8ff51804c 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -28,7 +28,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha } go func() { - log.Info("about to fetch messages for block from pubsub") + log.Debug("about to fetch messages for block from pubsub") bmsgs, err := s.Bsync.FetchMessagesByCids(context.TODO(), blk.BlsMessages) if err != nil { log.Errorf("failed to fetch all bls messages for block received over pubusb: %s", err) @@ -41,7 +41,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha return } - log.Info("inform new block over pubsub") + log.Debug("inform new block over pubsub") s.InformNewBlock(msg.GetFrom(), &types.FullBlock{ Header: blk.Header, BlsMessages: bmsgs, diff --git a/chain/sync.go b/chain/sync.go index 64fbd7f05..3ddb594d2 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -90,7 +90,7 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { } if from == syncer.self { // TODO: this is kindof a hack... - log.Infof("got block from ourselves") + log.Debug("got block from ourselves") if err := syncer.Sync(fts); err != nil { log.Errorf("failed to sync our own block: %s", err) diff --git a/cli/client.go b/cli/client.go index 61c58678c..6838ef0bf 100644 --- a/cli/client.go +++ b/cli/client.go @@ -19,6 +19,8 @@ var clientCmd = &cli.Command{ clientImportCmd, clientLocalCmd, clientDealCmd, + clientFindCmd, + clientRetrieveCmd, }, } @@ -108,3 +110,99 @@ var clientDealCmd = &cli.Command{ return nil }, } + +var clientFindCmd = &cli.Command{ + Name: "find", + Usage: "find data in the network", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + fmt.Println("Usage: find [CID]") + return nil + } + + file, err := cid.Parse(cctx.Args().First()) + if err != nil { + return err + } + + api, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + ctx := ReqContext(cctx) + + // Check if we already have this data locally + + has, err := api.ClientHasLocal(ctx, file) + if err != nil { + return err + } + + if has { + fmt.Println("LOCAL") + } + + offers, err := api.ClientFindData(ctx, file) + if err != nil { + return err + } + + for _, offer := range offers { + if offer.Err != "" { + fmt.Printf("ERR %s@%s: %s\n", offer.Miner, offer.MinerPeerID, offer.Err) + continue + } + fmt.Printf("RETRIEVAL %s@%s-%sfil-%db\n", offer.Miner, offer.MinerPeerID, offer.MinPrice, offer.Size) + } + + return nil + }, +} + +var clientRetrieveCmd = &cli.Command{ + Name: "retrieve", + Usage: "retrieve data from network", + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 2 { + fmt.Println("Usage: retrieve [CID] [outfile]") + return nil + } + + file, err := cid.Parse(cctx.Args().First()) + if err != nil { + return err + } + + api, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + ctx := ReqContext(cctx) + + // Check if we already have this data locally + + /*has, err := api.ClientHasLocal(ctx, file) + if err != nil { + return err + } + + if has { + fmt.Println("Success: Already in local storage") + return nil + }*/ // TODO: fix + + offers, err := api.ClientFindData(ctx, file) + if err != nil { + return err + } + + // TODO: parse offer strings from `client find`, make this smarter + + order := offers[0].Order() + err = api.ClientRetrieve(ctx, order, cctx.Args().Get(1)) + if err == nil { + fmt.Println("Success") + } + return err + }, +} diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 0567f51b0..ee49589fb 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -227,7 +227,7 @@ func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID) ( params, err := actors.SerializeParams(actors.CreateStorageMinerParams{ Owner: defOwner, Worker: k, - SectorSize: types.NewInt(actors.SectorSize), + SectorSize: types.NewInt(build.SectorSize), PeerID: peerid, }) if err != nil { diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 3fd5aab07..7aba38b0b 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -36,6 +36,7 @@ var sectorsCmd = &cli.Command{ sectorsStatusCmd, sectorsStagedListCmd, sectorsStagedSealCmd, + sectorsRefsCmd, }, } @@ -110,3 +111,28 @@ var sectorsStagedSealCmd = &cli.Command{ return nodeApi.SectorsStagedSeal(ctx) }, } + +var sectorsRefsCmd = &cli.Command{ + Name: "refs", + Usage: "List References to sectors", + Action: func(cctx *cli.Context) error { + nodeApi, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + ctx := lcli.ReqContext(cctx) + + refs, err := nodeApi.SectorsRefs(ctx) + if err != nil { + return err + } + + for name, refs := range refs { + fmt.Printf("Block %s:\n", name) + for _, ref := range refs { + fmt.Printf("\t%s+%d %d bytes\n", ref.Piece, ref.Offset, ref.Size) + } + } + return nil + }, +} diff --git a/extern/go-bls-sigs b/extern/go-bls-sigs index 4bc4b8a7b..03705e06e 160000 --- a/extern/go-bls-sigs +++ b/extern/go-bls-sigs @@ -1 +1 @@ -Subproject commit 4bc4b8a7bbf8118b8629084ff9207cfbf146df5f +Subproject commit 03705e06e83ac0d4c98695dacd0f20a350cc93d7 diff --git a/extern/go-fil-proofs b/extern/go-fil-proofs new file mode 160000 index 000000000..5201fa9a4 --- /dev/null +++ b/extern/go-fil-proofs @@ -0,0 +1 @@ +Subproject commit 5201fa9a47494a673da4322dffdf20b7bc335acb diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index e75bc9b0a..186d3b6f5 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit e75bc9b0aaeab4e1c8ea7eda42d63a39ae80728f +Subproject commit 186d3b6f5a78625979f4cec1a6b72e91a2291308 diff --git a/go.mod b/go.mod index cdad5dcf6..8db58f6f6 100644 --- a/go.mod +++ b/go.mod @@ -5,16 +5,18 @@ go 1.12 require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 github.com/BurntSushi/toml v0.3.1 + github.com/dgraph-io/badger v1.6.0 // indirect github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985 github.com/filecoin-project/go-bls-sigs v0.0.0-20190718224239-4bc4b8a7bbf8 github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 github.com/filecoin-project/go-sectorbuilder v0.0.0-00010101000000-000000000000 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 + github.com/golang/mock v1.3.1 // indirect github.com/gorilla/websocket v1.4.0 - github.com/ipfs/go-bitswap v0.1.6-0.20190808170517-167327fc3c5e + github.com/ipfs/go-bitswap v0.1.7-0.20190808170517-167327fc3c5e github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.2 - github.com/ipfs/go-car v0.0.1 + github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4 github.com/ipfs/go-cid v0.0.3 github.com/ipfs/go-datastore v0.1.0 github.com/ipfs/go-ds-badger v0.0.5 @@ -23,49 +25,47 @@ require ( github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf github.com/ipfs/go-ipfs-blockstore v0.1.0 github.com/ipfs/go-ipfs-chunker v0.0.1 + github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 - github.com/ipfs/go-ipfs-files v0.0.3 + github.com/ipfs/go-ipfs-files v0.0.4 github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipld-cbor v0.0.3 github.com/ipfs/go-ipld-format v0.0.2 github.com/ipfs/go-log v0.0.2-0.20190708183747-9c9fd6111bea - github.com/ipfs/go-merkledag v0.1.0 - github.com/ipfs/go-unixfs v0.1.0 + github.com/ipfs/go-merkledag v0.2.3 + github.com/ipfs/go-unixfs v0.2.2-0.20190823181532-868af2e9e5cb63f114a445a00d595c028026c28c github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 - github.com/libp2p/go-eventbus v0.0.3 // indirect - github.com/libp2p/go-libp2p v0.2.0 - github.com/libp2p/go-libp2p-circuit v0.1.0 + github.com/libp2p/go-libp2p v0.3.0 + github.com/libp2p/go-libp2p-circuit v0.1.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 - github.com/libp2p/go-libp2p-core v0.0.6 + github.com/libp2p/go-libp2p-core v0.2.2 github.com/libp2p/go-libp2p-discovery v0.1.0 github.com/libp2p/go-libp2p-kad-dht v0.1.1 github.com/libp2p/go-libp2p-mplex v0.2.1 github.com/libp2p/go-libp2p-peer v0.2.0 - github.com/libp2p/go-libp2p-peerstore v0.1.2-0.20190621130618-cfa9bb890c1a + github.com/libp2p/go-libp2p-peerstore v0.1.3 github.com/libp2p/go-libp2p-pnet v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.1.0 github.com/libp2p/go-libp2p-quic-transport v0.1.1 - github.com/libp2p/go-libp2p-record v0.1.0 + github.com/libp2p/go-libp2p-record v0.1.1 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 - github.com/libp2p/go-libp2p-secio v0.1.0 - github.com/libp2p/go-libp2p-swarm v0.1.1 // indirect + github.com/libp2p/go-libp2p-secio v0.2.0 github.com/libp2p/go-libp2p-tls v0.1.0 github.com/libp2p/go-libp2p-yamux v0.2.1 github.com/libp2p/go-maddr-filter v0.0.5 + github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.0.4 github.com/multiformats/go-multiaddr-dns v0.0.3 github.com/multiformats/go-multiaddr-net v0.0.1 - github.com/multiformats/go-multihash v0.0.6 + github.com/multiformats/go-multihash v0.0.7 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a - github.com/smartystreets/assertions v1.0.1 // indirect - github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/cbor-gen v0.0.0-20190822231004-8db835b09a5a github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d @@ -75,11 +75,8 @@ require ( go.uber.org/fx v1.9.0 go.uber.org/goleak v0.10.0 // indirect go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect - golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect - golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect - golang.org/x/sys v0.0.0-20190730183949-1393eb018365 // indirect golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 + google.golang.org/api v0.9.0 // indirect gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) diff --git a/go.sum b/go.sum index 70b7effae..ba1b24ea1 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= @@ -21,10 +22,14 @@ github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50 h1:4i3KsuVA0o0KoBxAC5x+MY7RbteiMK1V7gf/G08NGIQ= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a h1:We35J+0yvVFrEXbtViYUW8H/wNOhqjIF3PsrW4yYmGw= +github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -53,8 +58,8 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQY github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1 h1:JphPpoBZJ3WHha133BGYlQqltSGIhV+VsEID0++nN9A= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger/v2 v2.0.0-20190620211019-41d170b5158f/go.mod h1:jUaIjOV835xZ/mCLG/8P/38ZxiT4bG/K1khDNZJxuwU= -github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -63,7 +68,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985 h1:rpid5Xgp6GnDACqZvugxWvJ8L1HpyttUkzpUk/4BPXk= github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985/go.mod h1:muo8IeR187EUiX5AcMmIb0XSgpSU3qrszgh+pGWf3rY= github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 h1:aMJGfgqe1QDhAVwxRg5fjCRF533xHidiKsugk7Vvzug= @@ -76,6 +80,7 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJY github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -85,26 +90,39 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -112,10 +130,11 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -124,21 +143,20 @@ github.com/ipfs/bbloom v0.0.1 h1:s7KkiBPfxCeDVo47KySjK0ACPc5GJRUxFpdyWEuDjhw= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitswap v0.0.1/go.mod h1:z+tP3h+HTJ810n1R5yMy2ccKFffJ2F6Vqm/5Bf7vs2c= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= -github.com/ipfs/go-bitswap v0.1.6-0.20190808170517-167327fc3c5e h1:LiCecZPwRrr6m91+HfyXiawmAT1/t7h9OXwoNjS6bjY= -github.com/ipfs/go-bitswap v0.1.6-0.20190808170517-167327fc3c5e/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= +github.com/ipfs/go-bitswap v0.1.6/go.mod h1:oRNdV7SkA9glUUMHd6O2ztSwimBDLFdIF0fYIuDEzVo= +github.com/ipfs/go-bitswap v0.1.7-0.20190808170517-167327fc3c5e h1:O5Qmnj6VLa89My1nYwOto+DPJ/ZxH98PKDbIVVnahFs= +github.com/ipfs/go-bitswap v0.1.7-0.20190808170517-167327fc3c5e/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-blockservice v0.0.1/go.mod h1:2Ao89U7jV1KIqqNk5EdhSTBG/Pgc1vMFr0bhkx376j4= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc= github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= -github.com/ipfs/go-car v0.0.1 h1:Nn3RjJbysnDud4wILAybzOAvzsaycrCoKM4BMIK0Y04= -github.com/ipfs/go-car v0.0.1/go.mod h1:pUz3tUIpudsTch0ZQrEPOvNUBT1LufCXg8aZ4KOrEMM= +github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4 h1:qYLz/x/d1SOiiFGS8dwCBCFJ5Oh64Y8HMBrS+MbaU8c= +github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4/go.mod h1:NSSM0pxlhej9rSFXQmB/lDru7TYNoDjKgmvqzlsd06Y= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= @@ -151,7 +169,6 @@ github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= -github.com/ipfs/go-ds-badger v0.0.4/go.mod h1:UIu++7eal30eVc+njb9LyGgBoJ3F+Y5cBpvD/dwn5VQ= github.com/ipfs/go-ds-badger v0.0.5 h1:dxKuqw5T1Jm8OuV+lchA76H9QZFyPKZeLuT6bN42hJQ= github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= @@ -163,6 +180,7 @@ github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf h1:P9Kkd8YCG4 github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf/go.mod h1:gaK14QN1GOlYGgq+o+t5+WTExZZogkMt0k0IIBNjXsM= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= +github.com/ipfs/go-ipfs-blockstore v0.0.2/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= @@ -180,17 +198,16 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= github.com/ipfs/go-ipfs-files v0.0.3 h1:ME+QnC3uOyla1ciRPezDW0ynQYK2ikOh9OCKAEg4uUA= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= +github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE= +github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1 h1:zgUotX8dcAB/w/HidJh1zzc1yFq6Vm8J7T2F4itj/RU= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= -github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs= github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRDI/HQQ= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipld-cbor v0.0.1/go.mod h1:RXHr8s4k0NE0TKhnrxqZC9M888QfsBN9rhS5NjfKzY8= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= @@ -200,9 +217,10 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v0.0.2-0.20190708183747-9c9fd6111bea h1:T5Xz37N8fqK3u0aLbWXCoTOO+S62T+TvOLRZh8q8vX0= github.com/ipfs/go-log v0.0.2-0.20190708183747-9c9fd6111bea/go.mod h1:YTiqro5xwLoGra88hB8tMBlN+7ByaT3Kdaa0UqwCmI0= -github.com/ipfs/go-merkledag v0.0.1/go.mod h1:CRdtHMROECqaehAGeJ0Wd9TtlmWv/ta5cUnvbTnniEI= github.com/ipfs/go-merkledag v0.1.0 h1:CAEXjRFEDPvealQj3TgEjV1IJckwjvmxAqtq5QSXJrg= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= +github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -210,13 +228,12 @@ github.com/ipfs/go-peertaskqueue v0.1.1 h1:+gPjbI+V3NktXZOqJA1kzbms2pYmhjgQQal0M github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw= github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= -github.com/ipfs/go-unixfs v0.1.0 h1:KkjcfqObdNwUN8heMtt5OdrgrRKYTIWEvpGl1bDYIho= -github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= +github.com/ipfs/go-unixfs v0.2.2-0.20190823181532-868af2e9e5cb63f114a445a00d595c028026c28c h1:SaTWPkLp9OWnd+Wm6oyBQ1NODnvSmRFz9bKNo4PheVU= +github.com/ipfs/go-unixfs v0.2.2-0.20190823181532-868af2e9e5cb63f114a445a00d595c028026c28c/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= -github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -230,11 +247,16 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= @@ -253,30 +275,26 @@ github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpz github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= -github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= -github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= -github.com/libp2p/go-eventbus v0.0.3 h1:4sB0NrwnWr6qGeq2RWUp/JG1wNajf6gyILInId72hrw= -github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= +github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ= +github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= -github.com/libp2p/go-libp2p v0.0.1/go.mod h1:bmRs8I0vwn6iRaVssZnJx/epY6WPSKiLoK1vyle4EX0= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= -github.com/libp2p/go-libp2p v0.2.0 h1:hYJgMZYdcwHzDHKb/nLePrtuSP3LqkGIFOQ2aIbKOCM= -github.com/libp2p/go-libp2p v0.2.0/go.mod h1:5nXHmf4Hs+NmkaMsmWcFJgUHTbYNpCfxr20lwus0p1c= -github.com/libp2p/go-libp2p-autonat v0.0.1/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= +github.com/libp2p/go-libp2p v0.3.0 h1:XhYEJKmIdi4U4Zbie/ym9k6lqgg3PHM2stGS/cOUDWk= +github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= -github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3 h1:0KycuXvPDhmehw0ASsg+s1o3IfXgCUDqfzAl94KEBOg= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= -github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE= github.com/libp2p/go-libp2p-circuit v0.1.0 h1:eniLL3Y9aq/sryfyV1IAHj5rlvuyj3b7iz8tSiZpdhY= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.1 h1:eopfG9fAg6rEHWQO1TSrLosXDgYbbbu/RTva/tBANus= +github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-connmgr v0.1.0 h1:vp0t0F0EuT3rrlTtnMnIyyzCnly7nIlRoEbhJpgp0qU= github.com/libp2p/go-libp2p-connmgr v0.1.0/go.mod h1:wZxh8veAmU5qdrfJ0ZBLcU8oJe9L82ciVP/fl1VHjXk= github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= @@ -285,106 +303,90 @@ github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvx github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= github.com/libp2p/go-libp2p-core v0.0.6 h1:SsYhfWJ47vLP1Rd9/0hqEm/W/PlFbC/3YLZyLCcvo1w= github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= -github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= +github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= +github.com/libp2p/go-libp2p-core v0.2.2 h1:Sv1ggdoMx9c7v7FOFkR7agraHCnAgqYsXrU1ARSRUMs= +github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= -github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= -github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= -github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= github.com/libp2p/go-libp2p-kad-dht v0.1.1 h1:IH6NQuoUv5w5e1O8Jc3KyVDtr0rNd0G9aaADpLI1xVo= github.com/libp2p/go-libp2p-kad-dht v0.1.1/go.mod h1:1kj2Rk5pX3/0RwqMm9AMNCT7DzcMHYhgDN5VTi+cY0M= github.com/libp2p/go-libp2p-kbucket v0.2.0 h1:FB2a0VkOTNGTP5gu/I444u4WabNM9V1zCkQcWb7zajI= github.com/libp2p/go-libp2p-kbucket v0.2.0/go.mod h1:JNymBToym3QXKBMKGy3m29+xprg0EVr/GJFHxFEdgh8= -github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= -github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= -github.com/libp2p/go-libp2p-nat v0.0.1/go.mod h1:4L6ajyUIlJvx1Cbh5pc6Ma6vMDpKXf3GgLO5u7W0oQ4= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= -github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= -github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q= github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= -github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= -github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-peerstore v0.1.1/go.mod h1:ojEWnwG7JpJLkJ9REWYXQslyu9ZLrPWPEcCdiZzEbSM= -github.com/libp2p/go-libp2p-peerstore v0.1.2-0.20190621130618-cfa9bb890c1a h1:xW2Q7yiWAQnBpxe6m5Y094bYxxBCaoNruxc1sDlVxs0= -github.com/libp2p/go-libp2p-peerstore v0.1.2-0.20190621130618-cfa9bb890c1a/go.mod h1:DAchSrPUuksotuxrqPcvk5jvifXlxC3oH/65iHFmBns= +github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= +github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= -github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-pubsub v0.1.0 h1:SmQeMa7IUv5vadh0fYgYsafWCBA1sCy5d/68kIYqGcU= github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= -github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= -github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= +github.com/libp2p/go-libp2p-record v0.1.1 h1:ZJK2bHXYUBqObHX+rHLSNrM3M8fmJUlUHrodDPPATmY= +github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU= github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8= github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= -github.com/libp2p/go-libp2p-secio v0.0.1/go.mod h1:IdG6iQybdcYmbTzxp4J5dwtUEDTOvZrT0opIDVNPrJs= github.com/libp2p/go-libp2p-secio v0.1.0 h1:NNP5KLxuP97sE5Bu3iuwOWyT/dKEGMN5zSLMWdB7GTQ= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= -github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs= +github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= +github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= -github.com/libp2p/go-libp2p-swarm v0.1.1 h1:QW7pjyTRIxt9yyBid52YmMRGtkFXUE/rbEVWsQ0ae+w= -github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwLx73ln1FTefR91Q= +github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= +github.com/libp2p/go-libp2p-swarm v0.2.1 h1:9A8oQqPIZvbaRyrjViHeDYS7fE7fNtP7BRWdJrBHbe8= +github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.4 h1:Qev57UR47GcLPXWjrunv5aLIQGO4n9mhI/8/EIrEEFc= github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= +github.com/libp2p/go-libp2p-testing v0.1.0 h1:WaFRj/t3HdMZGNZqnU2pS7pDRBmMeoDx7/HDNpeyT9U= +github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-tls v0.1.0 h1:o4bjjAdnUjNgJoPoDd0wUaZH7K+EenlNWJpgyXB3ulA= github.com/libp2p/go-libp2p-tls v0.1.0/go.mod h1:VZdoSWQDeNpIIAFJFv+6uqTqpnIIDHcqZQSTC/A1TT0= -github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= -github.com/libp2p/go-libp2p-transport v0.0.4/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= -github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= github.com/libp2p/go-libp2p-yamux v0.2.1 h1:Q3XYNiKCC2vIxrvUJL+Jg1kiyeEaIDNKLjgEjo3VQdI= github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= -github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= -github.com/libp2p/go-mplex v0.0.1/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= -github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-nat v0.0.3 h1:l6fKV+p0Xa354EqQOQP+d8CivdLM4kl5GxC1hSc/UeI= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= +github.com/libp2p/go-openssl v0.0.2 h1:9pP2d3Ubaxkv7ZisLjx9BFwgOGnQdQYnfcH29HNY3ls= +github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= -github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= -github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6mK0en8rlpA6BBTs= github.com/libp2p/go-tcp-transport v0.1.0 h1:IGhowvEqyMFknOar4FWCKSWE0zL36UFKQtiRQD60/8o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= -github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= -github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= github.com/libp2p/go-ws-transport v0.1.0 h1:F+0OvvdmPTDsVc4AjPHjV7L7Pk1B7D5QwtDcKE2oag4= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -393,18 +395,24 @@ github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.16 h1:iMEQ/IVHxPTtx2Q07JP/k4CKRvSjiAZjZ0hnhgYEDmE= +github.com/miekg/dns v1.1.16/go.mod h1:YNV562EiewvSmpCB6/W4c6yqjK7Z+M/aIS1JHsIVeg8= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -414,6 +422,10 @@ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= @@ -436,19 +448,24 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= -github.com/multiformats/go-multihash v0.0.6 h1:cAVKO4epVd+SSpYJQD6d3vbdqQJvsrtGbTGzsp+V094= -github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= -github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-multihash v0.0.7 h1:uoqoE03rGJdlQEPq2EAc6UeSbo4L7mZyeAAoqNalf54= +github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c/go.mod h1:eTBvSIlRgLo+CNFFQRQTwUGTZOEdvXIKeZS/xG+D2yU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= +github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= +github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -463,28 +480,41 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14 h1:2m16U/rLwVaRdz7ANkHtHTodP3zTP3N451MADg64x5k= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 h1:N8Bg45zpk/UcpNGnfJt2y/3lRWASHNTUET8owPYCgYI= -github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81/go.mod h1:xIvvI5FiHLxhv8prbzVpaMHaaGPFPFQSuTcxC91ryOo= +github.com/sourcegraph/go-langserver v2.0.0+incompatible/go.mod h1:bBMjfpzEHd6ijPRoQ7f+knFfw+e8R+W158/MsqAy77c= +github.com/sourcegraph/jsonrpc2 v0.0.0-20190106185902-35a74f039c6a/go.mod h1:eESpbCslcLDs8j2D7IEdGVgul7xuk9odqDTaor30IUU= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a h1:/eS3yfGjQKG+9kayBkj0ip1BGhq6zJ3eaVksphxAaek= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= @@ -494,16 +524,18 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -523,9 +555,6 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= -github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= -github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= -github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30 h1:nMCC9Pwz1pxfC1Y6mYncdk+kq8d5aLx0Q+/gyZGE44M= @@ -538,7 +567,7 @@ github.com/whyrusleeping/sharray v0.0.0-20190718051354-e41931821e33 h1:7Bsg3GZnF github.com/whyrusleeping/sharray v0.0.0-20190718051354-e41931821e33/go.mod h1:c1pwhNePDPlcYJZinQlfLTOKwTmVf45nfdTg73yOOcA= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -561,6 +590,7 @@ go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWA go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -573,18 +603,20 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -592,14 +624,17 @@ golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -609,10 +644,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -620,35 +655,50 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190730183949-1393eb018365 h1:SaXEMXhWzMJThc05vu6uh61Q245r4KaWMrsTedk0FDc= -golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190813142322-97f12d73768f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -659,6 +709,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= @@ -668,5 +720,7 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= diff --git a/lib/cborrpc/rpc.go b/lib/cborrpc/rpc.go index db3553212..c02fc5298 100644 --- a/lib/cborrpc/rpc.go +++ b/lib/cborrpc/rpc.go @@ -1,16 +1,27 @@ package cborrpc import ( + "encoding/hex" "io" cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" cbg "github.com/whyrusleeping/cbor-gen" ) -const MessageSizeLimit = 1 << 20 +var log = logging.Logger("cborrrpc") + +const Debug = false + +func init() { + if Debug { + log.Warn("CBOR-RPC Debugging enabled") + } +} func WriteCborRPC(w io.Writer, obj interface{}) error { if m, ok := obj.(cbg.CBORMarshaler); ok { + // TODO: impl debug return m.MarshalCBOR(w) } data, err := cbor.DumpObject(obj) @@ -18,6 +29,10 @@ func WriteCborRPC(w io.Writer, obj interface{}) error { return err } + if Debug { + log.Infof("> %s", hex.EncodeToString(data)) + } + _, err = w.Write(data) return err } diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index dd95f9c8f..73c09e0ca 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -288,6 +288,7 @@ func (fn *rpcFunc) handleRpcCall(args []reflect.Value) (results []reflect.Value) if resp.Result != nil { log.Debugw("rpc result", "type", fn.ftyp.Out(fn.valOut)) if err := json.Unmarshal(resp.Result, val.Interface()); err != nil { + log.Warnw("unmarshaling failed", "message", string(resp.Result)) return fn.processError(xerrors.Errorf("unmarshaling result: %w", err)) } } diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index ec8714506..e9d7e4047 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -34,7 +34,7 @@ type SectorBuilderConfig struct { func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { proverId := addressToProverID(cfg.Miner) - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 1, 1, 1, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 1, 1, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) if err != nil { return nil, err } diff --git a/miner/miner.go b/miner/miner.go index 4464e3654..67396d469 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -194,7 +194,7 @@ func (m *Miner) GetBestMiningCandidate() (*MiningBase, error) { } func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*chain.BlockMsg, error) { - log.Info("attempting to mine a block on:", base.ts.Cids()) + log.Debug("attempting to mine a block on:", base.ts.Cids()) ticket, err := m.scratchTicket(ctx, base) if err != nil { return nil, errors.Wrap(err, "scratching ticket failed") diff --git a/node/builder.go b/node/builder.go index 5ef4b5b3e..0eb594fad 100644 --- a/node/builder.go +++ b/node/builder.go @@ -35,8 +35,11 @@ import ( "github.com/filecoin-project/go-lotus/node/modules/testing" "github.com/filecoin-project/go-lotus/node/repo" "github.com/filecoin-project/go-lotus/paych" + "github.com/filecoin-project/go-lotus/retrieval" + "github.com/filecoin-project/go-lotus/retrieval/discovery" "github.com/filecoin-project/go-lotus/storage" "github.com/filecoin-project/go-lotus/storage/sector" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" ) // special is a type used to give keys to modules which @@ -79,6 +82,7 @@ const ( // storage miner HandleDealsKey + HandleRetrievalKey RunSectorServiceKey RegisterMinerKey @@ -219,6 +223,10 @@ func Online() Option { Override(RunBlockSyncKey, modules.RunBlockSync), Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), + Override(new(*discovery.Local), discovery.NewLocal), + Override(new(discovery.PeerResolver), modules.RetrievalResolver), + + Override(new(*retrieval.Client), retrieval.NewClient), Override(new(*deals.Client), deals.NewClient), Override(RunDealClientKey, modules.RunDealClient), @@ -232,11 +240,14 @@ func Online() Option { ApplyIf(func(s *Settings) bool { return s.nodeType == nodeStorageMiner }, Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), Override(new(*sector.Store), sector.NewStore), + Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(*storage.Miner), modules.StorageMiner), Override(new(dtypes.StagingDAG), modules.StagingDAG), + Override(new(*retrieval.Miner), retrieval.NewMiner), Override(new(*deals.Handler), deals.NewHandler), + Override(HandleRetrievalKey, modules.HandleRetrieval), Override(HandleDealsKey, modules.HandleDeals), Override(RunSectorServiceKey, modules.RunSectorService), Override(RegisterMinerKey, modules.RegisterMiner), @@ -300,6 +311,7 @@ func Repo(r repo.Repo) Option { Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore), Override(new(dtypes.ClientFilestore), modules.ClientFstore), + Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), Override(new(dtypes.ClientDAG), modules.ClientDAG), Override(new(ci.PrivKey), pk), diff --git a/node/client/import.go b/node/client/import.go deleted file mode 100644 index da13c8ef3..000000000 --- a/node/client/import.go +++ /dev/null @@ -1 +0,0 @@ -package client diff --git a/node/impl/full/client.go b/node/impl/full/client.go index 49bf4b0aa..f70fa21ad 100644 --- a/node/impl/full/client.go +++ b/node/impl/full/client.go @@ -3,6 +3,12 @@ package full import ( "context" "errors" + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/retrieval" + "github.com/filecoin-project/go-lotus/retrieval/discovery" + "github.com/ipfs/go-blockservice" + offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-merkledag" "os" "github.com/filecoin-project/go-lotus/api" @@ -31,10 +37,13 @@ type ClientAPI struct { WalletAPI PaychAPI - DealClient *deals.Client + DealClient *deals.Client + RetDiscovery discovery.PeerResolver + Retrieval *retrieval.Client - LocalDAG dtypes.ClientDAG - Filestore dtypes.ClientFilestore `optional:"true"` + LocalDAG dtypes.ClientDAG + Blockstore dtypes.ClientBlockstore + Filestore dtypes.ClientFilestore `optional:"true"` } func (a *ClientAPI) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { @@ -116,6 +125,34 @@ func (a *ClientAPI) ClientStartDeal(ctx context.Context, data cid.Cid, miner add return &c, err } +func (a *ClientAPI) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { + // TODO: check if we have the ENTIRE dag + + offExch := merkledag.NewDAGService(blockservice.New(a.Blockstore, offline.Exchange(a.Blockstore))) + _, err := offExch.Get(ctx, root) + if err == ipld.ErrNotFound { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (a *ClientAPI) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) { + peers, err := a.RetDiscovery.GetPeers(root) + if err != nil { + return nil, err + } + + out := make([]api.QueryOffer, len(peers)) + for k, p := range peers { + out[k] = a.Retrieval.Query(ctx, p, root) + } + + return out, nil +} + func (a *ClientAPI) ClientImport(ctx context.Context, path string) (cid.Cid, error) { f, err := os.Open(path) if err != nil { @@ -134,14 +171,14 @@ func (a *ClientAPI) ClientImport(ctx context.Context, path string) (cid.Cid, err bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG) params := ihelper.DagBuilderParams{ - Maxlinks: ihelper.DefaultLinksPerBlock, + Maxlinks: build.UnixfsLinksPerLevel, RawLeaves: true, CidBuilder: nil, Dagserv: bufferedDS, NoCopy: true, } - db, err := params.New(chunker.DefaultSplitter(file)) + db, err := params.New(chunker.NewSizeSplitter(file, int64(build.UnixfsChunkSize))) if err != nil { return cid.Undef, err } @@ -181,3 +218,18 @@ func (a *ClientAPI) ClientListImports(ctx context.Context) ([]api.Import, error) }) } } + +func (a *ClientAPI) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error { + outFile, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) + if err != nil { + return err + } + + err = a.Retrieval.RetrieveUnixfs(ctx, order.Root, order.Size, order.MinerPeerID, order.Miner, outFile) + if err != nil { + _ = outFile.Close() + return err + } + + return outFile.Close() +} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 2c3b6e552..8a58f7fd1 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -7,10 +7,12 @@ import ( "math/rand" "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" "github.com/filecoin-project/go-lotus/storage" "github.com/filecoin-project/go-lotus/storage/sector" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" ) type StorageMinerAPI struct { @@ -19,6 +21,7 @@ type StorageMinerAPI struct { SectorBuilderConfig *sectorbuilder.SectorBuilderConfig SectorBuilder *sectorbuilder.SectorBuilder Sectors *sector.Store + SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner } @@ -28,10 +31,10 @@ func (sm *StorageMinerAPI) ActorAddresses(context.Context) ([]address.Address, e } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) { - size := uint64(1016) // this is the most data we can fit in a 1024 byte sector + size := sectorbuilder.UserBytesForSectorSize(build.SectorSize) name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), 1016), 0) + sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), 1016)) if err != nil { return 0, err } @@ -53,4 +56,20 @@ func (sm *StorageMinerAPI) SectorsStagedSeal(context.Context) error { return sm.SectorBuilder.SealAllStagedSectors() } +func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) { + // json can't handle cids as map keys + out := map[string][]api.SealedRef{} + + refs, err := sm.SectorBlocks.List() + if err != nil { + return nil, err + } + + for k, v := range refs { + out[k.String()] = v + } + + return out, nil +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/client.go b/node/modules/client.go index 0ddd5b7d2..3fa87ad00 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -36,9 +36,11 @@ func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) { return filestore.NewFilestore(bs, fm), nil } -func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore, rt routing.Routing, h host.Host) dtypes.ClientDAG { - ibs := blockstore.NewIdStore((*filestore.Filestore)(fstore)) +func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore { + return blockstore.NewIdStore((*filestore.Filestore)(fstore)) +} +func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG { bitswapNetwork := network.NewFromIpfsHost(h, rt) exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index a33f68c89..5e062cd68 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -21,6 +21,7 @@ type ChainExchange exchange.Interface type ChainBlockService bserv.BlockService type ClientFilestore *filestore.Filestore +type ClientBlockstore blockstore.Blockstore type ClientDAG ipld.DAGService type StagingDAG ipld.DAGService diff --git a/node/modules/services.go b/node/modules/services.go index babee96d9..658ac5fd8 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -2,7 +2,6 @@ package modules import ( "context" - "github.com/filecoin-project/go-lotus/storage/sector" "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" @@ -14,6 +13,8 @@ import ( "github.com/filecoin-project/go-lotus/chain/sub" "github.com/filecoin-project/go-lotus/node/hello" "github.com/filecoin-project/go-lotus/node/modules/helpers" + "github.com/filecoin-project/go-lotus/retrieval/discovery" + "github.com/filecoin-project/go-lotus/storage/sector" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { @@ -83,3 +84,7 @@ func RunSectorService(lc fx.Lifecycle, secst *sector.Store) { }, }) } + +func RetrievalResolver(l *discovery.Local) discovery.PeerResolver { + return discovery.Multi(l) +} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 773e488cc..5ca2b3a71 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,29 +2,30 @@ package modules import ( "context" - "github.com/filecoin-project/go-lotus/storage/sector" "path/filepath" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" - "github.com/libp2p/go-libp2p-core/routing" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" "github.com/mitchellh/go-homedir" "go.uber.org/fx" "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/deals" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" "github.com/filecoin-project/go-lotus/node/modules/dtypes" "github.com/filecoin-project/go-lotus/node/modules/helpers" "github.com/filecoin-project/go-lotus/node/repo" + "github.com/filecoin-project/go-lotus/retrieval" "github.com/filecoin-project/go-lotus/storage" + "github.com/filecoin-project/go-lotus/storage/sector" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -54,7 +55,7 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS) (*sectorbui sb := §orbuilder.SectorBuilderConfig{ Miner: minerAddr, - SectorSize: 1024, + SectorSize: build.SectorSize, MetadataDir: metadata, SealedDir: sealed, StagedDir: staging, @@ -86,6 +87,16 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h return sm, nil } +func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) { + lc.Append(fx.Hook{ + OnStart: func(context.Context) error { + host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleQueryStream) + host.SetStreamHandler(retrieval.ProtocolID, m.HandleDealStream) + return nil + }, + }) +} + func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Handler) { ctx := helpers.LifecycleCtx(mctx, lc) diff --git a/retrieval/client.go b/retrieval/client.go new file mode 100644 index 000000000..d46d9620c --- /dev/null +++ b/retrieval/client.go @@ -0,0 +1,223 @@ +package retrieval + +import ( + "context" + "io" + "io/ioutil" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/filecoin-project/go-lotus/lib/cborrpc" + "github.com/filecoin-project/go-lotus/retrieval/discovery" +) + +var log = logging.Logger("retrieval") + +type Client struct { + h host.Host +} + +func NewClient(h host.Host) *Client { + return &Client{h: h} +} + +func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid.Cid) api.QueryOffer { + s, err := c.h.NewStream(ctx, p.ID, QueryProtocolID) + if err != nil { + log.Warn(err) + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + defer s.Close() + + err = cborrpc.WriteCborRPC(s, Query{ + Piece: data, + }) + if err != nil { + log.Warn(err) + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + + // TODO: read deadline + rawResp, err := ioutil.ReadAll(s) + if err != nil { + log.Warn(err) + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + + var resp QueryResponse + if err := cbor.DecodeInto(rawResp, &resp); err != nil { + log.Warn(err) + return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } + + return api.QueryOffer{ + Root: data, + Size: resp.Size, + MinPrice: resp.MinPrice, + Miner: p.Address, // TODO: check + MinerPeerID: p.ID, + } +} + +type clientStream struct { + stream network.Stream + + root cid.Cid + offset uint64 + + windowSize uint64 // how much we "trust" the peer + verifier BlockVerifier +} + +// C > S +// +// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size +// +// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} +// < Resp{Accept} +// < ..(Intermediate Block) +// < ..Blocks +// < ..(Intermediate Block) +// < ..Blocks +// > DealProposal(...) +// < ... +func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, miner peer.ID, minerAddr address.Address, out io.Writer) error { + s, err := c.h.NewStream(ctx, miner, ProtocolID) + if err != nil { + return err + } + defer s.Close() + + initialOffset := uint64(0) // TODO: Check how much data we have locally + // TODO: Support in handler + // TODO: Allow client to specify this + + cst := clientStream{ + stream: s, + + root: root, + offset: initialOffset, + + windowSize: build.UnixfsChunkSize, + verifier: &UnixFs0Verifier{Root: root}, + } + + for cst.offset != size+initialOffset { + toFetch := cst.windowSize + if toFetch+cst.offset > size { + toFetch = size - cst.offset + } + log.Infof("Retrieve %dB @%d", toFetch, cst.offset) + + err := cst.doOneExchange(toFetch, out) + if err != nil { + return err + } + + cst.offset += toFetch + } + log.Info("RETRIEVE SUCCESSFUL") + return nil +} + +func (cst *clientStream) doOneExchange(toFetch uint64, out io.Writer) error { + deal := DealProposal{ + Ref: cst.root, + Params: RetParams{ + Unixfs0: &Unixfs0Offer{ + Offset: cst.offset, + Size: toFetch, + }, + }, + } + + if err := cborrpc.WriteCborRPC(cst.stream, deal); err != nil { + return err + } + + var resp DealResponse + if err := cborrpc.ReadCborRPC(cst.stream, &resp); err != nil { + log.Error(err) + return err + } + + if resp.Status != Accepted { + cst.windowSize = build.UnixfsChunkSize + // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) + + if resp.Status == Error { + return xerrors.Errorf("storage deal error: %s", resp.Message) + } + if resp.Status == Rejected { + return xerrors.Errorf("storage deal rejected: %s", resp.Message) + } + return xerrors.New("storage deal response had no Accepted section") + } + + log.Info("Retrieval accepted, fetching blocks") + + return cst.fetchBlocks(toFetch, out) + + // TODO: maybe increase miner window size after success +} + +func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { + blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize + + for i := uint64(0); i < blocksToFetch; { + log.Infof("block %d of %d", i+1, blocksToFetch) + + var block Block + if err := cborrpc.ReadCborRPC(cst.stream, &block); err != nil { + return err + } + + dataBlocks, err := cst.consumeBlockMessage(block, out) + if err != nil { + return err + } + + i += dataBlocks + } + + return nil +} + +func (cst *clientStream) consumeBlockMessage(block Block, out io.Writer) (uint64, error) { + prefix, err := cid.PrefixFromBytes(block.Prefix) + if err != nil { + return 0, err + } + + cid, err := prefix.Sum(block.Data) + + blk, err := blocks.NewBlockWithCid(block.Data, cid) + if err != nil { + return 0, err + } + + internal, err := cst.verifier.Verify(context.TODO(), blk, out) + if err != nil { + log.Warnf("block verify failed: %s", err) + return 0, err + } + + // TODO: Smarter out, maybe add to filestore automagically + // (Also, persist intermediate nodes) + + if internal { + return 0, nil + } + + return 1, nil +} diff --git a/retrieval/discovery/discovery.go b/retrieval/discovery/discovery.go new file mode 100644 index 000000000..21a555fc6 --- /dev/null +++ b/retrieval/discovery/discovery.go @@ -0,0 +1,25 @@ +package discovery + +import ( + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/libp2p/go-libp2p-core/peer" +) + +func init() { + cbor.RegisterCborType(RetrievalPeer{}) +} + +type RetrievalPeer struct { + Address address.Address + ID peer.ID // optional +} + +type PeerResolver interface { + GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel +} + +func Multi(r PeerResolver) PeerResolver { // TODO: actually support multiple mechanisms + return r +} diff --git a/retrieval/discovery/local.go b/retrieval/discovery/local.go new file mode 100644 index 000000000..512ea12a5 --- /dev/null +++ b/retrieval/discovery/local.go @@ -0,0 +1,53 @@ +package discovery + +import ( + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + dshelp "github.com/ipfs/go-ipfs-ds-help" + cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + + "github.com/filecoin-project/go-lotus/node/modules/dtypes" +) + +var log = logging.Logger("ret-discovery") + +type Local struct { + ds datastore.Datastore +} + +func NewLocal(ds dtypes.MetadataDS) *Local { + return &Local{ds: namespace.Wrap(ds, datastore.NewKey("/deals/local"))} +} + +func (l *Local) AddPeer(cid cid.Cid, peer RetrievalPeer) error { + // TODO: allow multiple peers here + // (implement an util for tracking map[thing][]otherThing, use in sectorBlockstore too) + + log.Warn("Tracking multiple retrieval peers not implemented") + + entry, err := cbor.DumpObject(peer) + if err != nil { + return err + } + + return l.ds.Put(dshelp.CidToDsKey(cid), entry) +} + +func (l *Local) GetPeers(data cid.Cid) ([]RetrievalPeer, error) { + entry, err := l.ds.Get(dshelp.CidToDsKey(data)) + if err == datastore.ErrNotFound { + return []RetrievalPeer{}, nil + } + if err != nil { + return nil, err + } + var peer RetrievalPeer + if err := cbor.DecodeInto(entry, &peer); err != nil { + return nil, err + } + return []RetrievalPeer{peer}, nil +} + +var _ PeerResolver = &Local{} diff --git a/retrieval/miner.go b/retrieval/miner.go new file mode 100644 index 000000000..434ccfab3 --- /dev/null +++ b/retrieval/miner.go @@ -0,0 +1,230 @@ +package retrieval + +import ( + "context" + "io" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-merkledag" + unixfile "github.com/ipfs/go-unixfs/file" + "github.com/libp2p/go-libp2p-core/network" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/chain/types" + "github.com/filecoin-project/go-lotus/lib/cborrpc" + "github.com/filecoin-project/go-lotus/storage/sectorblocks" +) + +type Miner struct { + sectorBlocks *sectorblocks.SectorBlocks + + pricePerByte types.BigInt + // TODO: Unseal price +} + +func NewMiner(sblks *sectorblocks.SectorBlocks) *Miner { + return &Miner{ + sectorBlocks: sblks, + pricePerByte: types.NewInt(2), // TODO: allow setting + } +} + +func writeErr(stream network.Stream, err error) { + log.Errorf("Retrieval deal error: %s", err) + _ = cborrpc.WriteCborRPC(stream, DealResponse{ + Status: Error, + Message: err.Error(), + }) +} + +func (m *Miner) HandleQueryStream(stream network.Stream) { + defer stream.Close() + + var query Query + if err := cborrpc.ReadCborRPC(stream, &query); err != nil { + writeErr(stream, err) + return + } + + size, err := m.sectorBlocks.GetSize(query.Piece) + if err != nil && err != sectorblocks.ErrNotFound { + log.Errorf("Retrieval query: GetRefs: %s", err) + return + } + + answer := QueryResponse{ + Status: Unavailable, + } + if err == nil { + answer.Status = Available + + // TODO: get price, look for already unsealed ref to reduce work + answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), m.pricePerByte) + answer.Size = uint64(size) // TODO: verify on intermediate + } + + if err := cborrpc.WriteCborRPC(stream, answer); err != nil { + log.Errorf("Retrieval query: WriteCborRPC: %s", err) + return + } +} + +type handlerDeal struct { + m *Miner + stream network.Stream + + ufsr sectorblocks.UnixfsReader + open cid.Cid + at uint64 + size uint64 +} + +func (m *Miner) HandleDealStream(stream network.Stream) { + defer stream.Close() + + hnd := &handlerDeal{ + m: m, + + stream: stream, + } + + var err error + more := true + + for more { + more, err = hnd.handleNext() // TODO: 'more' bool + if err != nil { + writeErr(stream, err) + return + } + } + +} + +func (hnd *handlerDeal) handleNext() (bool, error) { + var deal DealProposal + if err := cborrpc.ReadCborRPC(hnd.stream, &deal); err != nil { + if err == io.EOF { // client sent all deals + err = nil + } + return false, err + } + + if deal.Params.Unixfs0 == nil { + return false, xerrors.New("unknown deal type") + } + + unixfs0 := deal.Params.Unixfs0 + + // TODO: Verify payment, check how much we can send based on that + // Or reject (possibly returning the payment to retain reputation with the client) + + // If the file isn't open (new deal stream), isn't the right file, or isn't + // at the right offset, (re)open it + if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { + log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, unixfs0.Offset) + if err := hnd.openFile(deal); err != nil { + return false, err + } + } + + if unixfs0.Offset+unixfs0.Size > hnd.size { + return false, xerrors.Errorf("tried to read too much %d+%d > %d", unixfs0.Offset, unixfs0.Size, hnd.size) + } + + err := hnd.accept(deal) + if err != nil { + return false, err + } + return true, nil +} + +func (hnd *handlerDeal) openFile(deal DealProposal) error { + unixfs0 := deal.Params.Unixfs0 + + if unixfs0.Offset != 0 { + // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs + return xerrors.New("sending merkle proofs for nonzero offset not supported yet") + } + hnd.at = unixfs0.Offset + + bstore := hnd.m.sectorBlocks.SealedBlockstore(func() error { + return nil // TODO: approve unsealing based on amount paid + }) + + ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) + rootNd, err := ds.Get(context.TODO(), deal.Ref) + if err != nil { + return err + } + + fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd) + if err != nil { + return err + } + + var ok bool + hnd.ufsr, ok = fsr.(sectorblocks.UnixfsReader) + if !ok { + return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Ref) + } + + isize, err := hnd.ufsr.Size() + if err != nil { + return err + } + hnd.size = uint64(isize) + + hnd.open = deal.Ref + + return nil +} + +func (hnd *handlerDeal) accept(deal DealProposal) error { + unixfs0 := deal.Params.Unixfs0 + + resp := DealResponse{ + Status: Accepted, + } + if err := cborrpc.WriteCborRPC(hnd.stream, resp); err != nil { + log.Errorf("Retrieval query: Write Accepted resp: %s", err) + return err + } + + blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize + for i := uint64(0); i < blocksToSend; { + data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) + if err != nil { + return err + } + + log.Infof("sending block for a deal: %s", nd.Cid()) + + if offset != unixfs0.Offset { + return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", unixfs0.Offset, offset) + } + + /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) + writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data))) + return + }*/ + + block := Block{ + Prefix: nd.Cid().Prefix().Bytes(), + Data: nd.RawData(), + } + + if err := cborrpc.WriteCborRPC(hnd.stream, block); err != nil { + return err + } + + if len(data) > 0 { // don't count internal nodes + hnd.at += uint64(len(data)) + i++ + } + } + + return nil +} diff --git a/retrieval/types.go b/retrieval/types.go new file mode 100644 index 000000000..145b06732 --- /dev/null +++ b/retrieval/types.go @@ -0,0 +1,77 @@ +package retrieval + +import ( + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + + "github.com/filecoin-project/go-lotus/chain/types" +) + +const ProtocolID = "/fil/retrieval/-1.0.0" // TODO: spec +const QueryProtocolID = "/fil/retrieval/qry/-1.0.0" // TODO: spec + +type QueryResponseStatus int + +const ( + Available QueryResponseStatus = iota + Unavailable +) + +const ( + Accepted = iota + Error + Rejected + Unsealing +) + +func init() { + cbor.RegisterCborType(RetParams{}) + + cbor.RegisterCborType(Query{}) + cbor.RegisterCborType(QueryResponse{}) + cbor.RegisterCborType(Unixfs0Offer{}) + + cbor.RegisterCborType(DealProposal{}) + cbor.RegisterCborType(DealResponse{}) + cbor.RegisterCborType(Block{}) +} + +type Query struct { + Piece cid.Cid + // TODO: payment +} + +type QueryResponse struct { + Status QueryResponseStatus + + Size uint64 // TODO: spec + // TODO: unseal price (+spec) + // TODO: sectors to unseal + // TODO: address to send money for the deal? + MinPrice types.BigInt +} + +type Unixfs0Offer struct { + Offset uint64 + Size uint64 +} + +type RetParams struct { + Unixfs0 *Unixfs0Offer +} + +type DealProposal struct { + Ref cid.Cid + Params RetParams + // TODO: payment +} + +type DealResponse struct { + Status int + Message string +} + +type Block struct { // TODO: put in spec + Prefix []byte // TODO: fix cid.Prefix marshaling somehow + Data []byte +} diff --git a/retrieval/verify.go b/retrieval/verify.go new file mode 100644 index 000000000..b8cd8d400 --- /dev/null +++ b/retrieval/verify.go @@ -0,0 +1,143 @@ +package retrieval + +import ( + "context" + "io" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-unixfs" + pb "github.com/ipfs/go-unixfs/pb" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/build" +) + +type BlockVerifier interface { + Verify(context.Context, blocks.Block, io.Writer) (internal bool, err error) +} + +type OptimisticVerifier struct { +} + +func (o *OptimisticVerifier) Verify(context.Context, blocks.Block, io.Writer) (bool, error) { + // It's probably fine + return false, nil +} + +type UnixFs0Verifier struct { + Root cid.Cid + rootBlk blocks.Block + + expect int + seen int + + sub *UnixFs0Verifier +} + +func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.Writer) (last bool, internal bool, err error) { + if b.sub != nil { + // TODO: check links here (iff b.sub.sub == nil) + + subLast, internal, err := b.sub.verify(ctx, blk, out) + if err != nil { + return false, false, err + } + if subLast { + b.sub = nil + b.seen++ + } + + return b.seen == b.expect, internal, nil + } + + if b.seen >= b.expect { // this is probably impossible + return false, false, xerrors.New("unixfs verifier: too many nodes in level") + } + + links, err := b.checkInternal(blk, out) + if err != nil { + return false, false, err + } + + if links > 0 { // TODO: check if all links are intermediate (or all aren't) + if links > build.UnixfsLinksPerLevel { + return false, false, xerrors.New("unixfs verifier: too many links in intermediate node") + } + + if b.seen+1 == b.expect && links != build.UnixfsLinksPerLevel { + return false, false, xerrors.New("unixfs verifier: too few nodes in level") + } + + b.sub = &UnixFs0Verifier{ + Root: blk.Cid(), + rootBlk: blk, + expect: links, + } + + // don't mark as seen yet + return false, true, nil + } + + b.seen++ + return b.seen == b.expect, false, nil +} + +func (b *UnixFs0Verifier) checkInternal(blk blocks.Block, out io.Writer) (int, error) { + nd, err := ipld.Decode(blk) + if err != nil { + log.Warnf("IPLD Decode failed: %s", err) + return 0, err + } + + // TODO: check size + switch nd := nd.(type) { + case *merkledag.ProtoNode: + fsn, err := unixfs.FSNodeFromBytes(nd.Data()) + if err != nil { + log.Warnf("unixfs.FSNodeFromBytes failed: %s", err) + return 0, err + } + if fsn.Type() != pb.Data_File { + return 0, xerrors.New("internal nodes must be a file") + } + if len(fsn.Data()) > 0 { + return 0, xerrors.New("internal node with data") + } + if len(nd.Links()) == 0 { + return 0, xerrors.New("internal node with no links") + } + return len(nd.Links()), nil + + case *merkledag.RawNode: + _, err := out.Write(nd.RawData()) + return 0, err + default: + return 0, xerrors.New("verifier: unknown node type") + } +} + +func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block, w io.Writer) (bool, error) { + // root is special + if b.rootBlk == nil { + if !b.Root.Equals(blk.Cid()) { + return false, xerrors.Errorf("unixfs verifier: root block CID didn't match: valid %s, got %s", b.Root, blk.Cid()) + } + b.rootBlk = blk + links, err := b.checkInternal(blk, w) + if err != nil { + return false, err + } + + b.expect = links + return links != 0, nil + } + + _, internal, err := b.verify(ctx, blk, w) + return internal, err +} + +var _ BlockVerifier = &OptimisticVerifier{} +var _ BlockVerifier = &UnixFs0Verifier{} diff --git a/storage/miner.go b/storage/miner.go index 6a4945c57..f81a8ccf9 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -4,20 +4,20 @@ import ( "context" "fmt" - "github.com/filecoin-project/go-lotus/storage/sector" - - "github.com/filecoin-project/go-lotus/api" - "github.com/filecoin-project/go-lotus/chain/actors" - "github.com/filecoin-project/go-lotus/chain/address" - "github.com/filecoin-project/go-lotus/chain/store" - "github.com/filecoin-project/go-lotus/chain/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" host "github.com/libp2p/go-libp2p-core/host" "github.com/pkg/errors" + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/chain/actors" + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/filecoin-project/go-lotus/chain/store" + "github.com/filecoin-project/go-lotus/chain/types" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" + "github.com/filecoin-project/go-lotus/storage/sector" ) var log = logging.Logger("storageminer") @@ -103,7 +103,7 @@ func (m *Miner) handlePostingSealedSectors(ctx context.Context) { func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error { log.Info("committing sector") - ok, err := sectorbuilder.VerifySeal(1024, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof) + ok, err := sectorbuilder.VerifySeal(build.SectorSize, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof) if err != nil { log.Error("failed to verify seal we just created: ", err) } diff --git a/storage/sector/store.go b/storage/sector/store.go index 2f366cbe3..3d18e77ad 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -99,7 +99,7 @@ func (s *Store) service() { } } -func (s *Store) AddPiece(ref string, size uint64, r io.Reader, keepAtLeast uint64) (sectorID uint64, err error) { +func (s *Store) AddPiece(ref string, size uint64, r io.Reader) (sectorID uint64, err error) { err = withTemp(r, func(f string) (err error) { sectorID, err = s.sb.AddPiece(ref, size, f) return err diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go new file mode 100644 index 000000000..90d691660 --- /dev/null +++ b/storage/sectorblocks/blocks.go @@ -0,0 +1,256 @@ +package sectorblocks + +import ( + "context" + "errors" + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/lib/sectorbuilder" + "github.com/filecoin-project/go-lotus/node/modules/dtypes" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" + blockstore "github.com/ipfs/go-ipfs-blockstore" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-unixfs" + "sync" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + files "github.com/ipfs/go-ipfs-files" + cbor "github.com/ipfs/go-ipld-cbor" + + "github.com/filecoin-project/go-lotus/storage/sector" +) + +type SealSerialization uint8 + +const ( + SerializationUnixfs0 SealSerialization = 'u' +) + +var dsPrefix = datastore.NewKey("/sealedblocks") +var imBlocksPrefix = datastore.NewKey("/intermediate") + +var ErrNotFound = errors.New("not found") + +type SectorBlocks struct { + *sector.Store + + intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore + + unsealed *unsealedBlocks + keys datastore.Batching + keyLk sync.Mutex +} + +func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { + sbc := &SectorBlocks{ + Store: sectst, + + intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)), + + keys: namespace.Wrap(ds, dsPrefix), + } + + unsealed := &unsealedBlocks{ // TODO: untangle this + sb: sb, + + unsealed: map[string][]byte{}, + unsealing: map[string]chan struct{}{}, + } + + sbc.unsealed = unsealed + return sbc +} + +type UnixfsReader interface { + files.File + + // ReadBlock reads data from a single unixfs block. Data is nil + // for intermediate nodes + ReadBlock(context.Context) (data []byte, offset uint64, nd ipld.Node, err error) +} + +type refStorer struct { + blockReader UnixfsReader + writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint32) error + intermediate blockstore.Blockstore + + pieceRef string + remaining []byte +} + +func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint32) error { + st.keyLk.Lock() // TODO: make this multithreaded + defer st.keyLk.Unlock() + + v, err := st.keys.Get(dshelp.CidToDsKey(cid)) + if err == datastore.ErrNotFound { + err = nil + } + if err != nil { + return err + } + + var refs []api.SealedRef + if len(v) > 0 { + if err := cbor.DecodeInto(v, &refs); err != nil { + return err + } + } + + refs = append(refs, api.SealedRef{ + Piece: pieceRef, + Offset: offset, + Size: size, + }) + + newRef, err := cbor.DumpObject(&refs) + if err != nil { + return err + } + return st.keys.Put(dshelp.CidToDsKey(cid), newRef) // TODO: batch somehow +} + +func (r *refStorer) Read(p []byte) (n int, err error) { + offset := 0 + if len(r.remaining) > 0 { + offset += len(r.remaining) + read := copy(p, r.remaining) + if read == len(r.remaining) { + r.remaining = nil + } else { + r.remaining = r.remaining[read:] + } + return read, nil + } + + for { + data, offset, nd, err := r.blockReader.ReadBlock(context.TODO()) + if err != nil { + return 0, err + } + + if len(data) == 0 { + // TODO: batch + // TODO: GC + if err := r.intermediate.Put(nd); err != nil { + return 0, err + } + continue + } + + if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint32(len(data))); err != nil { + return 0, err + } + + read := copy(p, data) + if read < len(data) { + r.remaining = data[read:] + } + // TODO: read multiple + return read, nil + } +} + +func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast uint64) (sectorID uint64, err error) { + size, err := r.Size() + if err != nil { + return 0, err + } + + refst := &refStorer{ + blockReader: r, + pieceRef: string(SerializationUnixfs0) + ref.String(), + writeRef: st.writeRef, + intermediate: st.intermediate, + } + + return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) +} + +func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { + res, err := st.keys.Query(query.Query{}) + if err != nil { + return nil, err + } + + ents, err := res.Rest() + if err != nil { + return nil, err + } + + out := map[cid.Cid][]api.SealedRef{} + for _, ent := range ents { + refCid, err := dshelp.DsKeyToCid(datastore.RawKey(ent.Key)) + if err != nil { + return nil, err + } + + var refs []api.SealedRef + if err := cbor.DecodeInto(ent.Value, &refs); err != nil { + return nil, err + } + + out[refCid] = refs + } + + return out, nil +} + +func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: track local sectors + ent, err := st.keys.Get(dshelp.CidToDsKey(k)) + if err == datastore.ErrNotFound { + err = ErrNotFound + } + if err != nil { + return nil, err + } + + var refs []api.SealedRef + if err := cbor.DecodeInto(ent, &refs); err != nil { + return nil, err + } + + return refs, nil +} + +func (st *SectorBlocks) GetSize(k cid.Cid) (uint64, error) { + blk, err := st.intermediate.Get(k) + if err == blockstore.ErrNotFound { + refs, err := st.GetRefs(k) + if err != nil { + return 0, err + } + + return uint64(refs[0].Size), nil + } + if err != nil { + return 0, err + } + + nd, err := ipld.Decode(blk) + if err != nil { + return 0, err + } + + fsn, err := unixfs.ExtractFSNode(nd) + if err != nil { + return 0, err + } + + return fsn.FileSize(), nil +} + +func (st *SectorBlocks) Has(k cid.Cid) (bool, error) { + // TODO: ensure sector is still there + return st.keys.Has(dshelp.CidToDsKey(k)) +} + +func (st *SectorBlocks) SealedBlockstore(approveUnseal func() error) *SectorBlockStore { + return &SectorBlockStore{ + intermediate: st.intermediate, + sectorBlocks: st, + approveUnseal: approveUnseal, + } +} diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go new file mode 100644 index 000000000..e28e750d0 --- /dev/null +++ b/storage/sectorblocks/blockstore.go @@ -0,0 +1,77 @@ +package sectorblocks + +import ( + "context" + "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" +) + +type SectorBlockStore struct { + intermediate blockstore.Blockstore + sectorBlocks *SectorBlocks + + approveUnseal func() error +} + +func (s *SectorBlockStore) DeleteBlock(cid.Cid) error { + panic("not supported") +} +func (s *SectorBlockStore) GetSize(cid.Cid) (int, error) { + panic("not supported") +} + +func (s *SectorBlockStore) Put(blocks.Block) error { + panic("not supported") +} + +func (s *SectorBlockStore) PutMany([]blocks.Block) error { + panic("not supported") +} + +func (s *SectorBlockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + panic("not supported") +} + +func (s *SectorBlockStore) HashOnRead(enabled bool) { + panic("not supported") +} + +func (s *SectorBlockStore) Has(c cid.Cid) (bool, error) { + has, err := s.intermediate.Has(c) + if err != nil { + return false, err + } + if has { + return true, nil + } + + return s.sectorBlocks.Has(c) +} + +func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) { + val, err := s.intermediate.Get(c) + if err == nil { + return val, nil + } + if err != blockstore.ErrNotFound { + return nil, err + } + + refs, err := s.sectorBlocks.GetRefs(c) + if err != nil { + return nil, err + } + if len(refs) == 0 { + return nil, blockstore.ErrNotFound + } + + data, err := s.sectorBlocks.unsealed.getRef(context.TODO(), refs, s.approveUnseal) + if err != nil { + return nil, err + } + + return blocks.NewBlockWithCid(data, c) +} + +var _ blockstore.Blockstore = &SectorBlockStore{} diff --git a/storage/sectorblocks/unsealed.go b/storage/sectorblocks/unsealed.go new file mode 100644 index 000000000..042e2c931 --- /dev/null +++ b/storage/sectorblocks/unsealed.go @@ -0,0 +1,99 @@ +package sectorblocks + +import ( + "context" + "sync" + + logging "github.com/ipfs/go-log" + + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/lib/sectorbuilder" +) + +var log = logging.Logger("sectorblocks") + +type unsealedBlocks struct { + lk sync.Mutex + sb *sectorbuilder.SectorBuilder + + // TODO: Treat this as some sort of cache, one with rather aggressive GC + // TODO: This REALLY, REALLY needs to be on-disk + unsealed map[string][]byte + + unsealing map[string]chan struct{} +} + +func (ub *unsealedBlocks) getRef(ctx context.Context, refs []api.SealedRef, approveUnseal func() error) ([]byte, error) { + var best api.SealedRef + + ub.lk.Lock() + for _, ref := range refs { + b, ok := ub.unsealed[ref.Piece] + if ok { + ub.lk.Unlock() + return b[ref.Offset : ref.Offset+uint64(ref.Size)], nil + } + // TODO: pick unsealing based on how long it's running (or just select all relevant, usually it'll be just one) + _, ok = ub.unsealing[ref.Piece] + if ok { + best = ref + break + } + best = ref + } + ub.lk.Unlock() + + b, err := ub.maybeUnseal(ctx, best.Piece, approveUnseal) + if err != nil { + return nil, err + } + + return b[best.Offset : best.Offset+uint64(best.Size)], nil +} + +func (ub *unsealedBlocks) maybeUnseal(ctx context.Context, pieceKey string, approveUnseal func() error) ([]byte, error) { + ub.lk.Lock() + defer ub.lk.Unlock() + + out, ok := ub.unsealed[pieceKey] + if ok { + return out, nil + } + + wait, ok := ub.unsealing[pieceKey] + if ok { + ub.lk.Unlock() + select { + case <-wait: + ub.lk.Lock() + // TODO: make sure this is not racy with gc when it's implemented + return ub.unsealed[pieceKey], nil + case <-ctx.Done(): + ub.lk.Lock() + return nil, ctx.Err() + } + } + + // TODO: doing this under a lock is suboptimal.. but simpler + err := approveUnseal() + if err != nil { + return nil, err + } + + ub.unsealing[pieceKey] = make(chan struct{}) + ub.lk.Unlock() + + log.Infof("Unsealing piece '%s'", pieceKey) + data, err := ub.sb.ReadPieceFromSealedSector(pieceKey) + ub.lk.Lock() + + if err != nil { + // TODO: tell subs + log.Error(err) + return nil, err + } + + ub.unsealed[pieceKey] = data + close(ub.unsealing[pieceKey]) + return data, nil +}