cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Rework miner test setups to fix tests

Browse Source
This commit is contained in:
whyrusleeping 2019-11-30 17:17:50 -06:00
parent d5f78d6659
commit 59d9c681ad
14 changed files with 164 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api/test/deals.go
View File

@ -12,15 +12,19 @@ import (
logging "github.com/ipfs/go-log"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl"
)
func init() {
logging.SetAllLoggers(logging.LevelInfo)
build.InsecurePoStValidation = true
}
func TestDealFlow(t *testing.T, b APIBuilder) {
os.Setenv("BELLMAN_NO_GPU", "1")
logging.SetAllLoggers(logging.LevelInfo)
ctx := context.Background()
n, sn := b(t, 1, []int{0})
client := n[0].FullNode.(*impl.FullNodeAPI)
@ -42,7 +46,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
t.Fatal(err)
}
maddr, err := address.NewFromString("t0102")
maddr, err := miner.ActorAddress(ctx)
if err != nil {
t.Fatal(err)
}
@ -57,7 +61,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
for mine {
time.Sleep(time.Second)
fmt.Println("mining a block now")
if err := n[0].MineOne(ctx); err != nil {
if err := sn[0].MineOne(ctx); err != nil {
t.Fatal(err)
}
}

4
api/test/mining.go
View File

@ -9,7 +9,7 @@ import (
func (ts *testSuite) testMining(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 1, []int{0})
apis, sn := ts.makeNodes(t, 1, []int{0})
api := apis[0]
h1, err := api.ChainHead(ctx)
@ -20,7 +20,7 @@ func (ts *testSuite) testMining(t *testing.T) {
require.NoError(t, err)
<-newHeads
err = api.MineOne(ctx)
err = sn[0].MineOne(ctx)
require.NoError(t, err)
<-newHeads

10
api/test/test.go
View File

@ -11,12 +11,12 @@ import (
type TestNode struct {
api.FullNode
MineOne func(context.Context) error
}
type TestStorageNode struct {
api.StorageMiner
MineOne func(context.Context) error
}
// APIBuilder is a function which is invoked in test suite to provide
@ -43,7 +43,7 @@ func TestApis(t *testing.T, b APIBuilder) {
func (ts *testSuite) testVersion(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 1, []int{})
apis, _ := ts.makeNodes(t, 1, []int{0})
api := apis[0]
v, err := api.Version(ctx)
@ -57,7 +57,7 @@ func (ts *testSuite) testVersion(t *testing.T) {
func (ts *testSuite) testID(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 1, []int{})
apis, _ := ts.makeNodes(t, 1, []int{0})
api := apis[0]
id, err := api.ID(ctx)
@ -69,7 +69,7 @@ func (ts *testSuite) testID(t *testing.T) {
func (ts *testSuite) testConnectTwo(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 2, []int{})
apis, _ := ts.makeNodes(t, 2, []int{0})
p, err := apis[0].NetPeers(ctx)
if err != nil {

3
build/paramfetch.go
View File

@ -54,6 +54,9 @@ func GetParams(storage bool, tests bool) error {
ft := &fetch{}
for name, info := range params {
if info.SectorSize != 1024 {
continue
}
if !(SupportedSectorSize(info.SectorSize) || (tests && info.SectorSize == 1<<10)) {
continue
}

1
chain/gen/gen.go
View File

@ -614,7 +614,6 @@ func ComputeVRF(ctx context.Context, sign SignFunc, worker, miner address.Addres
if err != nil {
return nil, err
}
log.Warnf("making ticket: %x %s %s %x %x", sig.Data, worker, miner, input, sigInput)
if sig.Type != types.KTBLS {
return nil, fmt.Errorf("miner worker address was not a BLS key")

4
cmd/lotus-seed/main.go
View File

@ -61,7 +61,7 @@ var preSealCmd = &cli.Command{
Value: "lotus is fire",
Usage: "set the ticket preimage for sealing randomness",
},
&cli.Uint64Flag{
&cli.IntFlag{
Name: "num-sectors",
Value: 1,
Usage: "select number of sectors to pre-seal",
@ -79,7 +79,7 @@ var preSealCmd = &cli.Command{
return err
}
gm, err := seed.PreSeal(maddr, c.Uint64("sector-size"), c.Uint64("num-sectors"), sbroot, []byte(c.String("ticket-preimage")))
gm, err := seed.PreSeal(maddr, c.Uint64("sector-size"), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")))
if err != nil {
return err
}

8
cmd/lotus-seed/seed/seed.go
View File

@ -11,6 +11,7 @@ import (
"path/filepath"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
@ -23,7 +24,9 @@ import (
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func PreSeal(maddr address.Address, ssize uint64, sectors uint64, sbroot string, preimage []byte) (*genesis.GenesisMiner, error) {
var log = logging.Logger("preseal")
func PreSeal(maddr address.Address, ssize uint64, sectors int, sbroot string, preimage []byte) (*genesis.GenesisMiner, error) {
cfg := &sectorbuilder.Config{
Miner: maddr,
SectorSize: ssize,
@ -58,7 +61,7 @@ func PreSeal(maddr address.Address, ssize uint64, sectors uint64, sbroot string,
size := sectorbuilder.UserBytesForSectorSize(ssize)
var sealedSectors []*genesis.PreSeal
for i := uint64(1); i <= sectors; i++ {
for i := 0; i < sectors; i++ {
sid, err := sb.AcquireSectorId()
if err != nil {
return nil, err
@ -81,6 +84,7 @@ func PreSeal(maddr address.Address, ssize uint64, sectors uint64, sbroot string,
return nil, xerrors.Errorf("commit: %w", err)
}
log.Warn("PreCommitOutput: ", sid, pco)
sealedSectors = append(sealedSectors, &genesis.PreSeal{
CommR: pco.CommR,
CommD: pco.CommD,

68
cmd/lotus-storage-miner/init.go
View File

@ -133,7 +133,28 @@ var initCmd = &cli.Command{
return err
}
if err := migratePreSealedSectors(pssb, repoPath, mds); err != nil {
oldmds, err := badger.NewDatastore(filepath.Join(pssb, "badger"), nil)
if err != nil {
return err
}
oldsb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: 1024,
SealedDir: filepath.Join(pssb, "sealed"),
CacheDir: filepath.Join(pssb, "cache"),
StagedDir: filepath.Join(pssb, "staging"),
MetadataDir: filepath.Join(pssb, "meta"),
}, oldmds)
nsb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: 1024,
SealedDir: filepath.Join(lr.Path(), "sealed"),
CacheDir: filepath.Join(lr.Path(), "cache"),
StagedDir: filepath.Join(lr.Path(), "staging"),
MetadataDir: filepath.Join(lr.Path(), "meta"),
}, mds)
if err := nsb.ImportFrom(oldsb); err != nil {
return err
}
if err := lr.Close(); err != nil {
@ -161,51 +182,6 @@ var initCmd = &cli.Command{
},
}
// TODO: this method should be a lot more robust for mainnet. For testnet, its
// fine if we mess things up a few times
// Also probably makes sense for this method to be in the sectorbuilder package
func migratePreSealedSectors(presealsb string, repoPath string, mds dtypes.MetadataDS) error {
pspath, err := homedir.Expand(presealsb)
if err != nil {
return err
}
srcds, err := badger.NewDatastore(filepath.Join(pspath, "badger"), nil)
if err != nil {
return xerrors.Errorf("openning presealed sectors datastore: %w", err)
}
expRepo, err := homedir.Expand(repoPath)
if err != nil {
return err
}
if stat, err := os.Stat(pspath); err != nil {
return xerrors.Errorf("failed to stat presealed sectors directory: %w", err)
} else if !stat.IsDir() {
return xerrors.Errorf("given presealed sectors path was not a directory: %w", err)
}
for _, dir := range []string{"meta", "sealed", "staging", "cache"} {
from := filepath.Join(pspath, dir)
to := filepath.Join(expRepo, dir)
if err := os.Rename(from, to); err != nil {
return err
}
}
val, err := srcds.Get(sectorbuilder.LastSectorIdKey)
if err != nil {
return xerrors.Errorf("getting source last sector ID: %w", err)
}
if err := mds.Put(sectorbuilder.LastSectorIdKey, val); err != nil {
return xerrors.Errorf("failed to write last sector ID key to target datastore: %w", err)
}
return nil
}
func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir string, maddr address.Address, mds dtypes.MetadataDS) error {
b, err := ioutil.ReadFile(filepath.Join(presealDir, "pre-seal-"+maddr.String()+".json"))
if err != nil {

51
lib/sectorbuilder/sectorbuilder.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"sync"
@ -100,6 +101,7 @@ func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
default:
return nil, err
}
log.Warn("STARTING UP SECTOR BULIDER, LAST ID: ", lastUsedID)
sb := &SectorBuilder{
ds: ds,
@ -290,6 +292,9 @@ func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo,
}
proverID := addressToProverID(sb.Miner)
log.Info("GENERATING ELECTION POST")
defer log.Info("DONE GENERATING ELECTION POST")
return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners)
}
@ -422,3 +427,49 @@ func fallbackPostChallengeCount(sectors uint64) uint64 {
}
return challengeCount
}
func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder) error {
if err := moveAllFiles(osb.cacheDir, sb.cacheDir); err != nil {
return err
}
if err := moveAllFiles(osb.sealedDir, sb.sealedDir); err != nil {
return err
}
if err := moveAllFiles(osb.stagedDir, sb.stagedDir); err != nil {
return err
}
val, err := osb.ds.Get(LastSectorIdKey)
if err != nil {
return err
}
if err := sb.ds.Put(LastSectorIdKey, val); err != nil {
return err
}
sb.lastID = osb.lastID
return nil
}
func moveAllFiles(from, to string) error {
dir, err := os.Open(from)
if err != nil {
return err
}
names, err := dir.Readdirnames(0)
if err != nil {
return xerrors.Errorf("failed to list items in dir: %w", err)
}
for _, n := range names {
if err := os.Rename(filepath.Join(from, n), filepath.Join(to, n)); err != nil {
return xerrors.Errorf("moving file failed: %w", err)
}
}
return nil
}

1
miner/miner.go
View File

@ -119,6 +119,7 @@ func (m *Miner) Unregister(ctx context.Context, addr address.Address) error {
}
func (m *Miner) mine(ctx context.Context) {
log.Warn("Starting mining!!")
ctx, span := trace.StartSpan(ctx, "/mine")
defer span.End()

14
miner/testminer.go
View File

@ -4,14 +4,22 @@ import (
"context"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/gen"
)
func NewTestMiner(nextCh <-chan struct{}) func(api api.FullNode) *Miner {
return func(api api.FullNode) *Miner {
return &Miner{
func NewTestMiner(nextCh <-chan struct{}, addr address.Address) func(api.FullNode, gen.ElectionPoStProver) *Miner {
return func(api api.FullNode, epp gen.ElectionPoStProver) *Miner {
m := &Miner{
api: api,
waitFunc: chanWaiter(nextCh),
epp: epp,
}
if err := m.Register(addr); err != nil {
panic(err)
}
return m
}
}

64
node/node_test.go
View File

@ -6,11 +6,13 @@ import (
"crypto/rand"
"io/ioutil"
"net/http/httptest"
"path/filepath"
"testing"
"github.com/libp2p/go-libp2p-core/crypto"
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
"github.com/libp2p/go-libp2p-core/peer"
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
@ -26,8 +28,10 @@ import (
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/jsonrpc"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/modules"
modtest "github.com/filecoin-project/lotus/node/modules/testing"
"github.com/filecoin-project/lotus/node/repo"
@ -85,6 +89,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
// start node
var minerapi api.StorageMiner
mineBlock := make(chan struct{})
// TODO: use stop
_, err = node.New(ctx,
node.StorageMiner(&minerapi),
@ -95,6 +100,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
node.MockHost(mn),
node.Override(new(api.FullNode), tnd),
node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)),
)
if err != nil {
t.Fatalf("failed to construct node: %v", err)
@ -106,8 +112,16 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
err = minerapi.NetConnect(ctx, remoteAddrs)
require.NoError(t, err)*/
mineOne := func(ctx context.Context) error {
select {
case mineBlock <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
return test.TestStorageNode{minerapi}
return test.TestStorageNode{minerapi, mineOne}
}
func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
@ -134,6 +148,8 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
PeerIDs: []peer.ID{minerPid}, // TODO: if we have more miners, need more peer IDs
PreSeals: map[string]genesis.GenesisMiner{},
}
var presealDirs []string
for i := 0; i < len(storage); i++ {
maddr, err := address.NewIDAddress(300 + uint64(i))
if err != nil {
@ -147,6 +163,8 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
if err != nil {
t.Fatal(err)
}
presealDirs = append(presealDirs, tdir)
gmc.MinerAddrs = append(gmc.MinerAddrs, maddr)
gmc.PreSeals[maddr.String()] = *genm
}
@ -161,8 +179,6 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
}
mineBlock := make(chan struct{})
var err error
// TODO: Don't ignore stop
_, err = node.New(ctx,
@ -172,22 +188,12 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
node.MockHost(mn),
node.Test(),
node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock)),
genesis,
)
if err != nil {
t.Fatal(err)
}
fulls[i].MineOne = func(ctx context.Context) error {
select {
case mineBlock <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
}
for i, full := range storage {
@ -201,12 +207,36 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
f := fulls[full]
wa, err := f.WalletDefaultAddress(ctx)
require.NoError(t, err)
genMiner := gmc.MinerAddrs[i]
wa := gmc.PreSeals[genMiner.String()].Worker
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
psd := presealDirs[i]
mds, err := badger.NewDatastore(filepath.Join(psd, "badger"), nil)
if err != nil {
t.Fatal(err)
}
osb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: 1024,
WorkerThreads: 2,
Miner: genMiner,
CacheDir: filepath.Join(psd, "cache"),
StagedDir: filepath.Join(psd, "staging"),
SealedDir: filepath.Join(psd, "sealed"),
MetadataDir: filepath.Join(psd, "meta"),
}, mds)
if err != nil {
t.Fatal(err)
}
if err := sma.SectorBuilder.ImportFrom(osb); err != nil {
t.Fatal(err)
}
}
if err := mn.LinkAll(); err != nil {
@ -235,7 +265,6 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
if err != nil {
t.Fatal(err)
}
fulls[i].MineOne = a.MineOne
}
for i, a := range storaApis {
@ -248,6 +277,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
if err != nil {
t.Fatal(err)
}
storers[i].MineOne = a.MineOne
}
return fulls, storers

2
node/repo/interface.go
View File

@ -47,6 +47,6 @@ type LockedRepo interface {
// KeyStore returns store of private keys for Filecoin transactions
KeyStore() (types.KeyStore, error)
// Path returns absolute path of the repo (or empty string if in-memory)
// Path returns absolute path of the repo
Path() string
}

5
storage/miner.go
View File

@ -13,6 +13,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/gen"
@ -161,5 +162,9 @@ func (epp *sectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi sectorb
}
func (epp *sectorBuilderEpp) ComputeProof(ctx context.Context, ssi sectorbuilder.SortedPublicSectorInfo, rand []byte, winners []sectorbuilder.EPostCandidate) ([]byte, error) {
if build.InsecurePoStValidation {
log.Warn("Generating fake EPost proof! You should only see this while running tests!")
return []byte("valid proof"), nil
}
return epp.sb.ComputeElectionPoSt(ssi, rand, winners)
}