Merge pull request #533 from filecoin-project/test/deal-flow

Tests for interactive PoRep branch

api/api_full.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-filestore"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -73,6 +74,7 @@ type FullNode interface {
 	// ClientImport imports file under the specified path into filestore
 	ClientImport(ctx context.Context, path string) (cid.Cid, error)
 	ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, epochPrice types.BigInt, blocksDuration uint64) (*cid.Cid, error)
+	ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
 	ClientListDeals(ctx context.Context) ([]DealInfo, error)
 	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
 	ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now)

api/struct.go

@@ -82,6 +82,7 @@ type FullNodeStruct struct {
 		ClientHasLocal    func(ctx context.Context, root cid.Cid) (bool, error)                                                                       `perm:"write"`
 		ClientFindData    func(ctx context.Context, root cid.Cid) ([]QueryOffer, error)                                                               `perm:"read"`
 		ClientStartDeal   func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"`
+		ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error)                                                                           `perm:"read"`
 		ClientListDeals   func(ctx context.Context) ([]DealInfo, error)                                                                               `perm:"write"`
 		ClientRetrieve    func(ctx context.Context, order RetrievalOrder, path string) error                                                          `perm:"admin"`
 		ClientQueryAsk    func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error)                                `perm:"read"`
@@ -193,6 +194,9 @@ func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]Qu
 func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) {
 	return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration)
 }
+func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) {
+	return c.Internal.ClientGetDealInfo(ctx, deal)
+}
 
 func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) {
 	return c.Internal.ClientListDeals(ctx)

api/test/deals.go

@@ -0,0 +1,96 @@
+package test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	logging "github.com/ipfs/go-log"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/address"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl"
+)
+
+func TestDealFlow(t *testing.T, b APIBuilder) {
+	os.Setenv("BELLMAN_NO_GPU", "1")
+
+	logging.SetAllLoggers(logging.LevelInfo)
+	ctx := context.TODO()
+	n, sn := b(t, 1, []int{0})
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	r := io.LimitReader(rand.New(rand.NewSource(17)), 350)
+	fcid, err := client.ClientImportLocal(ctx, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	maddr, err := address.NewFromString("t0101")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println("FILE CID: ", fcid)
+
+	mine := true
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(time.Second)
+			fmt.Println("mining a block now")
+			if err := n[0].MineOne(ctx); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}()
+	deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(400), 100)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	time.Sleep(time.Second)
+	loop:
+	for {
+		di, err := client.ClientGetDealInfo(ctx, *deal)
+		if err != nil {
+			t.Fatal(err)
+		}
+		switch di.State {
+		case api.DealRejected:
+			t.Fatal("deal rejected")
+		case api.DealFailed:
+			t.Fatal("deal failed")
+		case api.DealError:
+			t.Fatal("deal errored")
+		case api.DealComplete:
+			fmt.Println("COMPLETE", di)
+			break loop
+		}
+		fmt.Println("Deal state: ", api.DealStates[di.State])
+		time.Sleep(time.Second / 2)
+	}
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
+}

api/types.go

@@ -25,6 +25,17 @@ const (
 	DealNoUpdate = DealUnknown
 )
 
+var DealStates = []string{
+	"DealUnknown",
+	"DealRejected",
+	"DealAccepted",
+	"DealStaged",
+	"DealSealing",
+	"DealFailed",
+	"DealComplete",
+	"DealError",
+}
+
 // TODO: check if this exists anywhere else
 type MultiaddrSlice []ma.Multiaddr
 

chain/deals/client.go

@@ -2,6 +2,7 @@ package deals
 
 import (
 	"context"
+
 	"github.com/filecoin-project/lotus/lib/statestore"
 	"github.com/filecoin-project/lotus/node/impl/full"
 
@@ -226,7 +227,6 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro
 
 	s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID)
 	if err != nil {
-		s.Reset()
 		return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err)
 	}
 
@@ -298,6 +298,14 @@ func (c *Client) List() ([]ClientDeal, error) {
 	return out, nil
 }
 
+func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) {
+	var out ClientDeal
+	if err := c.deals.Get(d, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
 func (c *Client) Stop() {
 	close(c.stop)
 	<-c.stopped

chain/deals/provider.go

@@ -193,7 +193,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
 }
 
 func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) {
-	proposalNd, err := cborrpc.AsIpld(&proposal)
+	proposalNd, err := cborrpc.AsIpld(proposal.DealProposal)
 	if err != nil {
 		return MinerDeal{}, err
 	}
@@ -222,7 +222,7 @@ func (p *Provider) HandleStream(s inet.Stream) {
 
 	deal, err := p.newDeal(s, proposal)
 	if err != nil {
-		log.Error(err)
+		log.Errorf("%+v", err)
 		s.Close()
 		return
 	}

chain/deals/provider_states.go

@@ -256,9 +256,6 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal
 	if err != nil {
 		return nil, err
 	}*/
-	// TODO: Spec doesn't say anything about inclusion proofs anywhere
-	//  Not sure what mechanisms prevents miner from storing data that isn't
-	//  clients' data
 
 	return nil, nil
 }

chain/sub/incoming.go

@@ -44,7 +44,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 				return
 			}
 
-			log.Infow("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom())
+			log.Debugw("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom())
 			s.InformNewBlock(msg.GetFrom(), &types.FullBlock{
 				Header:        blk.Header,
 				BlsMessages:   bmsgs,

chain/sync.go

@@ -102,7 +102,7 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) {
 
 	if from == syncer.self {
 		// TODO: this is kindof a hack...
-		log.Info("got block from ourselves")
+		log.Debug("got block from ourselves")
 
 		if err := syncer.Sync(ctx, fts.TipSet()); err != nil {
 			log.Errorf("failed to sync our own block %s: %+v", fts.TipSet().Cids(), err)
@@ -907,7 +907,7 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error
 	}
 
 	syncer.syncState.SetStage(api.StageSyncComplete)
-	log.Infow("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids()))
+	log.Debugw("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids()))
 
 	return nil
 }

cmd/lotus-storage-miner/run.go

@@ -8,13 +8,12 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/filecoin-project/lotus/build"
-
 	"github.com/multiformats/go-multiaddr"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/lib/auth"
 	"github.com/filecoin-project/lotus/lib/jsonrpc"

lib/statestore/store.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/query"
+	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/lib/cborrpc"
@@ -110,6 +111,19 @@ func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)
 	return st.ds.Put(k, mutated)
 }
 
+func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error {
+	k := toKey(i)
+	val, err := st.ds.Get(k)
+	if err != nil {
+		if xerrors.Is(err, datastore.ErrNotFound) {
+			return xerrors.Errorf("No state for %s", i)
+		}
+		return err
+	}
+
+	return out.UnmarshalCBOR(bytes.NewReader(val))
+}
+
 // out: *[]T
 func (st *StateStore) List(out interface{}) error {
 	res, err := st.ds.Query(query.Query{})

miner/miner.go

@@ -230,7 +230,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error)
 }
 
 func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, error) {
-	log.Infow("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids()))
+	log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids()))
 	ticket, err := m.scratchTicket(ctx, base)
 	if err != nil {
 		return nil, errors.Wrap(err, "scratching ticket failed")

node/impl/client/client.go

@@ -58,7 +58,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
 	// TODO: make this a param
 	self, err := a.WalletDefaultAddress(ctx)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("failed to get default address: %w", err)
 	}
 
 	// get miner peerID
@@ -70,11 +70,15 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
 
 	r, err := a.StateCall(ctx, msg, nil)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("failed getting peer ID: %w", err)
 	}
+	if r.ExitCode != 0 {
+		return nil, xerrors.Errorf("call to get peer ID for miner failed: exit code %d", r.ExitCode)
+	}
+
 	pid, err := peer.IDFromBytes(r.Return)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("parsing peer ID wrong: %w", err)
 	}
 
 	proposal := deals.ClientDealProposal{
@@ -88,7 +92,11 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
 	}
 
 	c, err := a.DealClient.Start(ctx, proposal)
-	return &c, err
+	if err != nil {
+		return nil, xerrors.Errorf("failed to start deal: %w", err)
+	}
+
+	return &c, nil
 }
 
 func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
@@ -115,6 +123,22 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
 	return out, nil
 }
 
+func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) {
+	v, err := a.DealClient.GetDeal(d)
+	if err != nil {
+		return nil, err
+	}
+	return &api.DealInfo{
+		ProposalCid:   v.ProposalCid,
+		State:         v.State,
+		Provider:      v.Proposal.Provider,
+		PieceRef:      v.Proposal.PieceRef,
+		Size:          v.Proposal.PieceSize,
+		PricePerEpoch: v.Proposal.StoragePricePerEpoch,
+		Duration:      v.Proposal.Duration,
+	}, nil
+}
+
 func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
 	// TODO: check if we have the ENTIRE dag
 
@@ -177,7 +201,11 @@ func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
 		return cid.Undef, err
 	}
 
-	return nd.Cid(), bufferedDS.Commit()
+	if err := bufferedDS.Commit(); err != nil {
+		return cid.Undef, err
+	}
+
+	return nd.Cid(), nil
 }
 
 func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) {

node/modules/testing/genesis.go

@@ -25,7 +25,7 @@ import (
 
 var glog = logging.Logger("genesis")
 
-func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
+func MakeGenesisMem(out io.Writer, minerPid peer.ID) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
 	return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
 		return func() (*types.BlockHeader, error) {
 			glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
@@ -38,7 +38,7 @@ func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wal
 			gmc := &gen.GenMinerCfg{
 				Owners:  []address.Address{w},
 				Workers: []address.Address{w},
-				PeerIDs: []peer.ID{"peerID 1"},
+				PeerIDs: []peer.ID{minerPid},
 			}
 			alloc := map[address.Address]types.BigInt{
 				w: types.FromFil(10000),

node/node_test.go

@@ -31,15 +31,12 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
-func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, tnd test.TestNode) test.TestStorageNode {
+func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode {
 	r := repo.NewMemory(nil)
 
 	lr, err := r.Lock(repo.RepoStorageMiner)
 	require.NoError(t, err)
 
-	pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
-	require.NoError(t, err)
-
 	ks, err := lr.KeyStore()
 	require.NoError(t, err)
 
@@ -93,6 +90,8 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
 		node.Repo(r),
 		node.Test(),
 
+		node.MockHost(mn),
+
 		node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath, 2)),
 		node.Override(new(api.FullNode), tnd),
 	)
@@ -115,12 +114,18 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 	fulls := make([]test.TestNode, nFull)
 	storers := make([]test.TestStorageNode, len(storage))
 
+	pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
+	require.NoError(t, err)
+
+	minerPid, err := peer.IDFromPrivateKey(pk)
+	require.NoError(t, err)
+
 	var genbuf bytes.Buffer
 
 	for i := 0; i < nFull; i++ {
 		var genesis node.Option
 		if i == 0 {
-			genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf))
+			genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, minerPid))
 		} else {
 			genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
 		}
@@ -171,7 +176,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 		genMiner, err := address.NewFromString("t0101")
 		require.NoError(t, err)
 
-		storers[i] = testStorageNode(ctx, t, wa, genMiner, f)
+		storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
 	}
 
 	if err := mn.LinkAll(); err != nil {
@@ -221,3 +226,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
 func TestAPIRPC(t *testing.T) {
 	test.TestApis(t, rpcBuilder)
 }
+
+func TestAPIDealFlow(t *testing.T) {
+	test.TestDealFlow(t, builder)
+}

storage/miner.go

@@ -2,9 +2,10 @@ package storage
 
 import (
 	"context"
+	"sync"
+
 	"github.com/filecoin-project/lotus/lib/statestore"
 	"github.com/ipfs/go-datastore/namespace"
-	"sync"
 
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"

storage/post.go

@@ -2,9 +2,10 @@ package storage
 
 import (
 	"context"
+	"time"
+
 	"github.com/ipfs/go-cid"
 	"go.opencensus.io/trace"
-	"time"
 
 	"golang.org/x/xerrors"
 

storage/sector/store.go

@@ -4,15 +4,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"math/bits"
-	"sync"
-
 	"github.com/filecoin-project/go-sectorbuilder/sealing_state"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	logging "github.com/ipfs/go-log"
 	"golang.org/x/xerrors"
+	"io"
+	"math/bits"
+	"sync"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/lib/cborrpc"

storage/sector/store_test.go

@@ -1,11 +1,18 @@
 package sector
 
 import (
+	"context"
+	"fmt"
+	"github.com/filecoin-project/lotus/lib/padreader"
+	"io"
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/lib/sectorbuilder"
+	"github.com/ipfs/go-datastore"
 )
 
 func testFill(t *testing.T, n uint64, exp []uint64) {
@@ -44,3 +51,35 @@ func TestFillersFromRem(t *testing.T) {
 	}
 
 }
+
+func TestSectorStore(t *testing.T) {
+	if err := build.GetParams(true); err != nil {
+		t.Fatal(err)
+	}
+
+	sb, cleanup, err := sectorbuilder.TempSectorbuilder(1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	tktFn := func(context.Context) (*sectorbuilder.SealTicket, error) {
+		return &sectorbuilder.SealTicket{
+			BlockHeight: 17,
+			TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2},
+		}, nil
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	store := NewStore(sb, ds, tktFn)
+
+	pr := io.LimitReader(rand.New(rand.NewSource(17)), 300)
+	pr, n := padreader.New(pr, 300)
+
+	sid, err := store.AddPiece("a", n, pr, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(sid)
+}