basic retrieval content discovery

node/builder.go

@@ -3,7 +3,7 @@ package node
 import (
 	"context"
 	"errors"
-	"github.com/filecoin-project/go-lotus/storage/sectorblocks"
+	"github.com/filecoin-project/go-lotus/retrieval"
 	"reflect"
 	"time"
 
@@ -36,8 +36,10 @@ import (
 	"github.com/filecoin-project/go-lotus/node/modules/testing"
 	"github.com/filecoin-project/go-lotus/node/repo"
 	"github.com/filecoin-project/go-lotus/paych"
+	"github.com/filecoin-project/go-lotus/retrieval/discovery"
 	"github.com/filecoin-project/go-lotus/storage"
 	"github.com/filecoin-project/go-lotus/storage/sector"
+	"github.com/filecoin-project/go-lotus/storage/sectorblocks"
 )
 
 // special is a type used to give keys to modules which
@@ -80,6 +82,7 @@ const (
 
 	// storage miner
 	HandleDealsKey
+	HandleRetrievalKey
 	RunSectorServiceKey
 	RegisterMinerKey
 
@@ -220,6 +223,10 @@ func Online() Option {
 			Override(RunBlockSyncKey, modules.RunBlockSync),
 			Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
 
+			Override(new(*discovery.Local), discovery.NewLocal),
+			Override(new(discovery.PeerResolver), modules.RetrievalResolver),
+
+			Override(new(*retrieval.Client), retrieval.NewClient),
 			Override(new(*deals.Client), deals.NewClient),
 			Override(RunDealClientKey, modules.RunDealClient),
 
@@ -238,7 +245,9 @@ func Online() Option {
 
 			Override(new(dtypes.StagingDAG), modules.StagingDAG),
 
+			Override(new(*retrieval.Miner), retrieval.NewMiner),
 			Override(new(*deals.Handler), deals.NewHandler),
+			Override(HandleRetrievalKey, modules.HandleRetrieval),
 			Override(HandleDealsKey, modules.HandleDeals),
 			Override(RunSectorServiceKey, modules.RunSectorService),
 			Override(RegisterMinerKey, modules.RegisterMiner),
@@ -302,6 +311,7 @@ func Repo(r repo.Repo) Option {
 		Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore),
 
 		Override(new(dtypes.ClientFilestore), modules.ClientFstore),
+		Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore),
 		Override(new(dtypes.ClientDAG), modules.ClientDAG),
 
 		Override(new(ci.PrivKey), pk),

node/impl/full/client.go

@@ -3,6 +3,11 @@ package full
 import (
 	"context"
 	"errors"
+	"github.com/filecoin-project/go-lotus/retrieval"
+	"github.com/filecoin-project/go-lotus/retrieval/discovery"
+	"github.com/ipfs/go-blockservice"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	"github.com/ipfs/go-merkledag"
 	"os"
 
 	"github.com/filecoin-project/go-lotus/api"
@@ -31,10 +36,13 @@ type ClientAPI struct {
 	WalletAPI
 	PaychAPI
 
-	DealClient *deals.Client
+	DealClient   *deals.Client
+	RetDiscovery discovery.PeerResolver
+	Retrieval    *retrieval.Client
 
-	LocalDAG  dtypes.ClientDAG
-	Filestore dtypes.ClientFilestore `optional:"true"`
+	LocalDAG   dtypes.ClientDAG
+	Blockstore dtypes.ClientBlockstore
+	Filestore  dtypes.ClientFilestore `optional:"true"`
 }
 
 func (a *ClientAPI) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) {
@@ -116,6 +124,34 @@ func (a *ClientAPI) ClientStartDeal(ctx context.Context, data cid.Cid, miner add
 	return &c, err
 }
 
+func (a *ClientAPI) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
+	// TODO: check if we have the ENTIRE dag
+
+	offExch := merkledag.NewDAGService(blockservice.New(a.Blockstore, offline.Exchange(a.Blockstore)))
+	_, err := offExch.Get(ctx, root)
+	if err == ipld.ErrNotFound {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func (a *ClientAPI) ClientFindData(ctx context.Context, root cid.Cid) ([]api.RetrievalOffer, error) {
+	peers, err := a.RetDiscovery.GetPeers(root)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make([]api.RetrievalOffer, len(peers))
+	for k, p := range peers {
+		out[k] = a.Retrieval.Query(ctx, p, root)
+	}
+
+	return out, nil
+}
+
 func (a *ClientAPI) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
 	f, err := os.Open(path)
 	if err != nil {

node/modules/client.go

@@ -36,9 +36,11 @@ func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) {
 	return filestore.NewFilestore(bs, fm), nil
 }
 
-func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore, rt routing.Routing, h host.Host) dtypes.ClientDAG {
-	ibs := blockstore.NewIdStore((*filestore.Filestore)(fstore))
+func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore {
+	return blockstore.NewIdStore((*filestore.Filestore)(fstore))
+}
 
+func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG {
 	bitswapNetwork := network.NewFromIpfsHost(h, rt)
 	exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs)
 

node/modules/dtypes/storage.go

@@ -21,6 +21,7 @@ type ChainExchange exchange.Interface
 type ChainBlockService bserv.BlockService
 
 type ClientFilestore *filestore.Filestore
+type ClientBlockstore blockstore.Blockstore
 type ClientDAG ipld.DAGService
 
 type StagingDAG ipld.DAGService

node/modules/services.go

@@ -2,6 +2,7 @@ package modules
 
 import (
 	"context"
+	"github.com/filecoin-project/go-lotus/retrieval/discovery"
 	"github.com/filecoin-project/go-lotus/storage/sector"
 
 	"github.com/libp2p/go-libp2p-core/host"
@@ -83,3 +84,7 @@ func RunSectorService(lc fx.Lifecycle, secst *sector.Store) {
 		},
 	})
 }
+
+func RetrievalResolver(l *discovery.Local) discovery.PeerResolver {
+	return discovery.Multi(l)
+}

node/modules/storageminer.go

@@ -2,6 +2,7 @@ package modules
 
 import (
 	"context"
+	"github.com/filecoin-project/go-lotus/retrieval"
 	"github.com/filecoin-project/go-lotus/storage/sector"
 	"path/filepath"
 
@@ -86,6 +87,15 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h
 	return sm, nil
 }
 
+func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) {
+	lc.Append(fx.Hook{
+		OnStart: func(context.Context) error {
+			host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleStream)
+			return nil
+		},
+	})
+}
+
 func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Handler) {
 	ctx := helpers.LifecycleCtx(mctx, lc)