cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #533 from filecoin-project/test/deal-flow

Browse Source 
Tests for interactive PoRep branch
This commit is contained in:
Whyrusleeping 2019-11-06 17:01:42 -08:00 committed by GitHub
commit 90faa63f17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 239 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/api_full.go
View File

@ -2,6 +2,7 @@ package api
import (
"context"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-filestore"
"github.com/libp2p/go-libp2p-core/peer"
@ -73,6 +74,7 @@ type FullNode interface {
// ClientImport imports file under the specified path into filestore
ClientImport(ctx context.Context, path string) (cid.Cid, error)
ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, epochPrice types.BigInt, blocksDuration uint64) (*cid.Cid, error)
ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
ClientListDeals(ctx context.Context) ([]DealInfo, error)
ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now)

4
api/struct.go
View File

@ -82,6 +82,7 @@ type FullNodeStruct struct {
ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"`
ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"`
ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"`
ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error) `perm:"read"`
ClientListDeals func(ctx context.Context) ([]DealInfo, error) `perm:"write"`
ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"`
ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"`
@ -193,6 +194,9 @@ func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]Qu
func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) {
return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration)
}
func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) {
return c.Internal.ClientGetDealInfo(ctx, deal)
}
func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) {
return c.Internal.ClientListDeals(ctx)

96
api/test/deals.go Normal file
View File

@ -0,0 +1,96 @@
package test
import (
"context"
"fmt"
"io"
"math/rand"
"os"
"testing"
"time"
logging "github.com/ipfs/go-log"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl"
)
func TestDealFlow(t *testing.T, b APIBuilder) {
os.Setenv("BELLMAN_NO_GPU", "1")
logging.SetAllLoggers(logging.LevelInfo)
ctx := context.TODO()
n, sn := b(t, 1, []int{0})
client := n[0].FullNode.(*impl.FullNodeAPI)
miner := sn[0]
addrinfo, err := client.NetAddrsListen(ctx)
if err != nil {
t.Fatal(err)
}
if err := miner.NetConnect(ctx, addrinfo); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
r := io.LimitReader(rand.New(rand.NewSource(17)), 350)
fcid, err := client.ClientImportLocal(ctx, r)
if err != nil {
t.Fatal(err)
}
maddr, err := address.NewFromString("t0101")
if err != nil {
t.Fatal(err)
}
fmt.Println("FILE CID: ", fcid)
mine := true
done := make(chan struct{})
go func() {
defer close(done)
for mine {
time.Sleep(time.Second)
fmt.Println("mining a block now")
if err := n[0].MineOne(ctx); err != nil {
t.Fatal(err)
}
}
}()
deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(400), 100)
if err != nil {
t.Fatal(err)
}
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)
loop:
for {
di, err := client.ClientGetDealInfo(ctx, *deal)
if err != nil {
t.Fatal(err)
}
switch di.State {
case api.DealRejected:
t.Fatal("deal rejected")
case api.DealFailed:
t.Fatal("deal failed")
case api.DealError:
t.Fatal("deal errored")
case api.DealComplete:
fmt.Println("COMPLETE", di)
break loop
}
fmt.Println("Deal state: ", api.DealStates[di.State])
time.Sleep(time.Second / 2)
}
mine = false
fmt.Println("shutting down mining")
<-done
}

11
api/types.go
View File

@ -25,6 +25,17 @@ const (
DealNoUpdate = DealUnknown
)
var DealStates = []string{
"DealUnknown",
"DealRejected",
"DealAccepted",
"DealStaged",
"DealSealing",
"DealFailed",
"DealComplete",
"DealError",
}
// TODO: check if this exists anywhere else
type MultiaddrSlice []ma.Multiaddr

10
chain/deals/client.go
View File

@ -2,6 +2,7 @@ package deals
import (
"context"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/impl/full"
@ -226,7 +227,6 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro
s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID)
if err != nil {
s.Reset()
return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err)
}
@ -298,6 +298,14 @@ func (c *Client) List() ([]ClientDeal, error) {
return out, nil
}
func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) {
var out ClientDeal
if err := c.deals.Get(d, &out); err != nil {
return nil, err
}
return &out, nil
}
func (c *Client) Stop() {
close(c.stop)
<-c.stopped

4
chain/deals/provider.go
View File

@ -193,7 +193,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
}
func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) {
proposalNd, err := cborrpc.AsIpld(&proposal)
proposalNd, err := cborrpc.AsIpld(proposal.DealProposal)
if err != nil {
return MinerDeal{}, err
}
@ -222,7 +222,7 @@ func (p *Provider) HandleStream(s inet.Stream) {
deal, err := p.newDeal(s, proposal)
if err != nil {
log.Error(err)
log.Errorf("%+v", err)
s.Close()
return
}

3
chain/deals/provider_states.go
View File

@ -256,9 +256,6 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal
if err != nil {
return nil, err
}*/
// TODO: Spec doesn't say anything about inclusion proofs anywhere
// Not sure what mechanisms prevents miner from storing data that isn't
// clients' data
return nil, nil
}

2
chain/sub/incoming.go
View File

@ -44,7 +44,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
return
}
log.Infow("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom())
log.Debugw("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom())
s.InformNewBlock(msg.GetFrom(), &types.FullBlock{
Header: blk.Header,
BlsMessages: bmsgs,

4
chain/sync.go
View File

@ -102,7 +102,7 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) {
if from == syncer.self {
// TODO: this is kindof a hack...
log.Info("got block from ourselves")
log.Debug("got block from ourselves")
if err := syncer.Sync(ctx, fts.TipSet()); err != nil {
log.Errorf("failed to sync our own block %s: %+v", fts.TipSet().Cids(), err)
@ -907,7 +907,7 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error
}
syncer.syncState.SetStage(api.StageSyncComplete)
log.Infow("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids()))
log.Debugw("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids()))
return nil
}

3
cmd/lotus-storage-miner/run.go
View File

@ -8,13 +8,12 @@ import (
"os/signal"
"syscall"
"github.com/filecoin-project/lotus/build"
"github.com/multiformats/go-multiaddr"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/auth"
"github.com/filecoin-project/lotus/lib/jsonrpc"

14
lib/statestore/store.go
View File

@ -7,6 +7,7 @@ import (
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/cborrpc"
@ -110,6 +111,19 @@ func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)
return st.ds.Put(k, mutated)
}
func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error {
k := toKey(i)
val, err := st.ds.Get(k)
if err != nil {
if xerrors.Is(err, datastore.ErrNotFound) {
return xerrors.Errorf("No state for %s", i)
}
return err
}
return out.UnmarshalCBOR(bytes.NewReader(val))
}
// out: *[]T
func (st *StateStore) List(out interface{}) error {
res, err := st.ds.Query(query.Query{})

2
miner/miner.go
View File

@ -230,7 +230,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error)
}
func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, error) {
log.Infow("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids()))
log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids()))
ticket, err := m.scratchTicket(ctx, base)
if err != nil {
return nil, errors.Wrap(err, "scratching ticket failed")

38
node/impl/client/client.go
View File

@ -58,7 +58,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
// TODO: make this a param
self, err := a.WalletDefaultAddress(ctx)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed to get default address: %w", err)
}
// get miner peerID
@ -70,11 +70,15 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
r, err := a.StateCall(ctx, msg, nil)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed getting peer ID: %w", err)
}
if r.ExitCode != 0 {
return nil, xerrors.Errorf("call to get peer ID for miner failed: exit code %d", r.ExitCode)
}
pid, err := peer.IDFromBytes(r.Return)
if err != nil {
return nil, err
return nil, xerrors.Errorf("parsing peer ID wrong: %w", err)
}
proposal := deals.ClientDealProposal{
@ -88,7 +92,11 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
}
c, err := a.DealClient.Start(ctx, proposal)
return &c, err
if err != nil {
return nil, xerrors.Errorf("failed to start deal: %w", err)
}
return &c, nil
}
func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
@ -115,6 +123,22 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
return out, nil
}
func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) {
v, err := a.DealClient.GetDeal(d)
if err != nil {
return nil, err
}
return &api.DealInfo{
ProposalCid: v.ProposalCid,
State: v.State,
Provider: v.Proposal.Provider,
PieceRef: v.Proposal.PieceRef,
Size: v.Proposal.PieceSize,
PricePerEpoch: v.Proposal.StoragePricePerEpoch,
Duration: v.Proposal.Duration,
}, nil
}
func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
// TODO: check if we have the ENTIRE dag
@ -177,7 +201,11 @@ func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
return cid.Undef, err
}
return nd.Cid(), bufferedDS.Commit()
if err := bufferedDS.Commit(); err != nil {
return cid.Undef, err
}
return nd.Cid(), nil
}
func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) {

4
node/modules/testing/genesis.go
View File

@ -25,7 +25,7 @@ import (
var glog = logging.Logger("genesis")
func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
func MakeGenesisMem(out io.Writer, minerPid peer.ID) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func() (*types.BlockHeader, error) {
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
@ -38,7 +38,7 @@ func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wal
gmc := &gen.GenMinerCfg{
Owners: []address.Address{w},
Workers: []address.Address{w},
PeerIDs: []peer.ID{"peerID 1"},
PeerIDs: []peer.ID{minerPid},
}
alloc := map[address.Address]types.BigInt{
w: types.FromFil(10000),

21
node/node_test.go
View File

@ -31,15 +31,12 @@ import (
"github.com/filecoin-project/lotus/node/repo"
)
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, tnd test.TestNode) test.TestStorageNode {
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode {
r := repo.NewMemory(nil)
lr, err := r.Lock(repo.RepoStorageMiner)
require.NoError(t, err)
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
ks, err := lr.KeyStore()
require.NoError(t, err)
@ -93,6 +90,8 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
node.Repo(r),
node.Test(),
node.MockHost(mn),
node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath, 2)),
node.Override(new(api.FullNode), tnd),
)
@ -115,12 +114,18 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
fulls := make([]test.TestNode, nFull)
storers := make([]test.TestStorageNode, len(storage))
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
minerPid, err := peer.IDFromPrivateKey(pk)
require.NoError(t, err)
var genbuf bytes.Buffer
for i := 0; i < nFull; i++ {
var genesis node.Option
if i == 0 {
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf))
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, minerPid))
} else {
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
}
@ -171,7 +176,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
genMiner, err := address.NewFromString("t0101")
require.NoError(t, err)
storers[i] = testStorageNode(ctx, t, wa, genMiner, f)
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
}
if err := mn.LinkAll(); err != nil {
@ -221,3 +226,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
func TestAPIRPC(t *testing.T) {
test.TestApis(t, rpcBuilder)
}
func TestAPIDealFlow(t *testing.T) {
test.TestDealFlow(t, builder)
}

3
storage/miner.go
View File

@ -2,9 +2,10 @@ package storage
import (
"context"
"sync"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/ipfs/go-datastore/namespace"
"sync"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"

3
storage/post.go
View File

@ -2,9 +2,10 @@ package storage
import (
"context"
"time"
"github.com/ipfs/go-cid"
"go.opencensus.io/trace"
"time"
"golang.org/x/xerrors"

7
storage/sector/store.go
View File

@ -4,15 +4,14 @@ import (
"bytes"
"context"
"fmt"
"io"
"math/bits"
"sync"
"github.com/filecoin-project/go-sectorbuilder/sealing_state"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"io"
"math/bits"
"sync"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/cborrpc"

39
storage/sector/store_test.go
View File

@ -1,11 +1,18 @@
package sector
import (
"context"
"fmt"
"github.com/filecoin-project/lotus/lib/padreader"
"io"
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/ipfs/go-datastore"
)
func testFill(t *testing.T, n uint64, exp []uint64) {
@ -44,3 +51,35 @@ func TestFillersFromRem(t *testing.T) {
}
}
func TestSectorStore(t *testing.T) {
if err := build.GetParams(true); err != nil {
t.Fatal(err)
}
sb, cleanup, err := sectorbuilder.TempSectorbuilder(1024)
if err != nil {
t.Fatal(err)
}
defer cleanup()
tktFn := func(context.Context) (*sectorbuilder.SealTicket, error) {
return &sectorbuilder.SealTicket{
BlockHeight: 17,
TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2},
}, nil
}
ds := datastore.NewMapDatastore()
store := NewStore(sb, ds, tktFn)
pr := io.LimitReader(rand.New(rand.NewSource(17)), 300)
pr, n := padreader.New(pr, 300)
sid, err := store.AddPiece("a", n, pr, 1)
if err != nil {
t.Fatal(err)
}
fmt.Println(sid)
}