Rework miner test setups to fix tests

api/test/deals.go

@@ -12,15 +12,19 @@ import (
 	logging "github.com/ipfs/go-log"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/address"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/impl"
 )
 
+func init() {
+	logging.SetAllLoggers(logging.LevelInfo)
+	build.InsecurePoStValidation = true
+}
+
 func TestDealFlow(t *testing.T, b APIBuilder) {
 	os.Setenv("BELLMAN_NO_GPU", "1")
 
-	logging.SetAllLoggers(logging.LevelInfo)
 	ctx := context.Background()
 	n, sn := b(t, 1, []int{0})
 	client := n[0].FullNode.(*impl.FullNodeAPI)
@@ -42,7 +46,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
 		t.Fatal(err)
 	}
 
-	maddr, err := address.NewFromString("t0102")
+	maddr, err := miner.ActorAddress(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -57,7 +61,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
 		for mine {
 			time.Sleep(time.Second)
 			fmt.Println("mining a block now")
-			if err := n[0].MineOne(ctx); err != nil {
+			if err := sn[0].MineOne(ctx); err != nil {
 				t.Fatal(err)
 			}
 		}

api/test/mining.go

@@ -9,7 +9,7 @@ import (
 
 func (ts *testSuite) testMining(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, []int{0})
+	apis, sn := ts.makeNodes(t, 1, []int{0})
 	api := apis[0]
 
 	h1, err := api.ChainHead(ctx)
@@ -20,7 +20,7 @@ func (ts *testSuite) testMining(t *testing.T) {
 	require.NoError(t, err)
 	<-newHeads
 
-	err = api.MineOne(ctx)
+	err = sn[0].MineOne(ctx)
 	require.NoError(t, err)
 
 	<-newHeads

api/test/test.go

@@ -11,12 +11,12 @@ import (
 
 type TestNode struct {
 	api.FullNode
-
-	MineOne func(context.Context) error
 }
 
 type TestStorageNode struct {
 	api.StorageMiner
+
+	MineOne func(context.Context) error
 }
 
 // APIBuilder is a function which is invoked in test suite to provide
@@ -43,7 +43,7 @@ func TestApis(t *testing.T, b APIBuilder) {
 
 func (ts *testSuite) testVersion(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, []int{})
+	apis, _ := ts.makeNodes(t, 1, []int{0})
 	api := apis[0]
 
 	v, err := api.Version(ctx)
@@ -57,7 +57,7 @@ func (ts *testSuite) testVersion(t *testing.T) {
 
 func (ts *testSuite) testID(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, []int{})
+	apis, _ := ts.makeNodes(t, 1, []int{0})
 	api := apis[0]
 
 	id, err := api.ID(ctx)
@@ -69,7 +69,7 @@ func (ts *testSuite) testID(t *testing.T) {
 
 func (ts *testSuite) testConnectTwo(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 2, []int{})
+	apis, _ := ts.makeNodes(t, 2, []int{0})
 
 	p, err := apis[0].NetPeers(ctx)
 	if err != nil {

build/paramfetch.go

@@ -54,6 +54,9 @@ func GetParams(storage bool, tests bool) error {
 	ft := &fetch{}
 
 	for name, info := range params {
+		if info.SectorSize != 1024 {
+			continue
+		}
 		if !(SupportedSectorSize(info.SectorSize) || (tests && info.SectorSize == 1<<10)) {
 			continue
 		}

chain/gen/gen.go

@@ -614,7 +614,6 @@ func ComputeVRF(ctx context.Context, sign SignFunc, worker, miner address.Addres
 	if err != nil {
 		return nil, err
 	}
-	log.Warnf("making ticket: %x %s %s %x %x", sig.Data, worker, miner, input, sigInput)
 
 	if sig.Type != types.KTBLS {
 		return nil, fmt.Errorf("miner worker address was not a BLS key")

cmd/lotus-seed/main.go

@@ -61,7 +61,7 @@ var preSealCmd = &cli.Command{
 			Value: "lotus is fire",
 			Usage: "set the ticket preimage for sealing randomness",
 		},
-		&cli.Uint64Flag{
+		&cli.IntFlag{
 			Name:  "num-sectors",
 			Value: 1,
 			Usage: "select number of sectors to pre-seal",
@@ -79,7 +79,7 @@ var preSealCmd = &cli.Command{
 			return err
 		}
 
-		gm, err := seed.PreSeal(maddr, c.Uint64("sector-size"), c.Uint64("num-sectors"), sbroot, []byte(c.String("ticket-preimage")))
+		gm, err := seed.PreSeal(maddr, c.Uint64("sector-size"), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")))
 		if err != nil {
 			return err
 		}

cmd/lotus-seed/seed/seed.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 
 	badger "github.com/ipfs/go-ds-badger"
+	logging "github.com/ipfs/go-log"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
@@ -23,7 +24,9 @@ import (
 	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
-func PreSeal(maddr address.Address, ssize uint64, sectors uint64, sbroot string, preimage []byte) (*genesis.GenesisMiner, error) {
+var log = logging.Logger("preseal")
+
+func PreSeal(maddr address.Address, ssize uint64, sectors int, sbroot string, preimage []byte) (*genesis.GenesisMiner, error) {
 	cfg := &sectorbuilder.Config{
 		Miner:         maddr,
 		SectorSize:    ssize,
@@ -58,7 +61,7 @@ func PreSeal(maddr address.Address, ssize uint64, sectors uint64, sbroot string,
 	size := sectorbuilder.UserBytesForSectorSize(ssize)
 
 	var sealedSectors []*genesis.PreSeal
-	for i := uint64(1); i <= sectors; i++ {
+	for i := 0; i < sectors; i++ {
 		sid, err := sb.AcquireSectorId()
 		if err != nil {
 			return nil, err
@@ -81,6 +84,7 @@ func PreSeal(maddr address.Address, ssize uint64, sectors uint64, sbroot string,
 			return nil, xerrors.Errorf("commit: %w", err)
 		}
 
+		log.Warn("PreCommitOutput: ", sid, pco)
 		sealedSectors = append(sealedSectors, &genesis.PreSeal{
 			CommR:    pco.CommR,
 			CommD:    pco.CommD,

cmd/lotus-storage-miner/init.go

@@ -133,7 +133,28 @@ var initCmd = &cli.Command{
 				return err
 			}
 
-			if err := migratePreSealedSectors(pssb, repoPath, mds); err != nil {
+			oldmds, err := badger.NewDatastore(filepath.Join(pssb, "badger"), nil)
+			if err != nil {
+				return err
+			}
+
+			oldsb, err := sectorbuilder.New(&sectorbuilder.Config{
+				SectorSize:  1024,
+				SealedDir:   filepath.Join(pssb, "sealed"),
+				CacheDir:    filepath.Join(pssb, "cache"),
+				StagedDir:   filepath.Join(pssb, "staging"),
+				MetadataDir: filepath.Join(pssb, "meta"),
+			}, oldmds)
+
+			nsb, err := sectorbuilder.New(&sectorbuilder.Config{
+				SectorSize:  1024,
+				SealedDir:   filepath.Join(lr.Path(), "sealed"),
+				CacheDir:    filepath.Join(lr.Path(), "cache"),
+				StagedDir:   filepath.Join(lr.Path(), "staging"),
+				MetadataDir: filepath.Join(lr.Path(), "meta"),
+			}, mds)
+
+			if err := nsb.ImportFrom(oldsb); err != nil {
 				return err
 			}
 			if err := lr.Close(); err != nil {
@@ -161,51 +182,6 @@ var initCmd = &cli.Command{
 	},
 }
 
-// TODO: this method should be a lot more robust for mainnet. For testnet, its
-// fine if we mess things up a few times
-// Also probably makes sense for this method to be in the sectorbuilder package
-func migratePreSealedSectors(presealsb string, repoPath string, mds dtypes.MetadataDS) error {
-	pspath, err := homedir.Expand(presealsb)
-	if err != nil {
-		return err
-	}
-
-	srcds, err := badger.NewDatastore(filepath.Join(pspath, "badger"), nil)
-	if err != nil {
-		return xerrors.Errorf("openning presealed sectors datastore: %w", err)
-	}
-
-	expRepo, err := homedir.Expand(repoPath)
-	if err != nil {
-		return err
-	}
-
-	if stat, err := os.Stat(pspath); err != nil {
-		return xerrors.Errorf("failed to stat presealed sectors directory: %w", err)
-	} else if !stat.IsDir() {
-		return xerrors.Errorf("given presealed sectors path was not a directory: %w", err)
-	}
-
-	for _, dir := range []string{"meta", "sealed", "staging", "cache"} {
-		from := filepath.Join(pspath, dir)
-		to := filepath.Join(expRepo, dir)
-		if err := os.Rename(from, to); err != nil {
-			return err
-		}
-	}
-
-	val, err := srcds.Get(sectorbuilder.LastSectorIdKey)
-	if err != nil {
-		return xerrors.Errorf("getting source last sector ID: %w", err)
-	}
-
-	if err := mds.Put(sectorbuilder.LastSectorIdKey, val); err != nil {
-		return xerrors.Errorf("failed to write last sector ID key to target datastore: %w", err)
-	}
-
-	return nil
-}
-
 func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir string, maddr address.Address, mds dtypes.MetadataDS) error {
 	b, err := ioutil.ReadFile(filepath.Join(presealDir, "pre-seal-"+maddr.String()+".json"))
 	if err != nil {

lib/sectorbuilder/sectorbuilder.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strconv"
 	"sync"
 
@@ -100,6 +101,7 @@ func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
 	default:
 		return nil, err
 	}
+	log.Warn("STARTING UP SECTOR BULIDER, LAST ID: ", lastUsedID)
 
 	sb := &SectorBuilder{
 		ds: ds,
@@ -290,6 +292,9 @@ func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo,
 	}
 
 	proverID := addressToProverID(sb.Miner)
+	log.Info("GENERATING ELECTION POST")
+	defer log.Info("DONE GENERATING ELECTION POST")
+
 	return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners)
 }
 
@@ -422,3 +427,49 @@ func fallbackPostChallengeCount(sectors uint64) uint64 {
 	}
 	return challengeCount
 }
+
+func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder) error {
+	if err := moveAllFiles(osb.cacheDir, sb.cacheDir); err != nil {
+		return err
+	}
+
+	if err := moveAllFiles(osb.sealedDir, sb.sealedDir); err != nil {
+		return err
+	}
+
+	if err := moveAllFiles(osb.stagedDir, sb.stagedDir); err != nil {
+		return err
+	}
+
+	val, err := osb.ds.Get(LastSectorIdKey)
+	if err != nil {
+		return err
+	}
+
+	if err := sb.ds.Put(LastSectorIdKey, val); err != nil {
+		return err
+	}
+
+	sb.lastID = osb.lastID
+
+	return nil
+}
+
+func moveAllFiles(from, to string) error {
+	dir, err := os.Open(from)
+	if err != nil {
+		return err
+	}
+
+	names, err := dir.Readdirnames(0)
+	if err != nil {
+		return xerrors.Errorf("failed to list items in dir: %w", err)
+	}
+	for _, n := range names {
+		if err := os.Rename(filepath.Join(from, n), filepath.Join(to, n)); err != nil {
+			return xerrors.Errorf("moving file failed: %w", err)
+		}
+	}
+
+	return nil
+}

miner/miner.go

@@ -119,6 +119,7 @@ func (m *Miner) Unregister(ctx context.Context, addr address.Address) error {
 }
 
 func (m *Miner) mine(ctx context.Context) {
+	log.Warn("Starting mining!!")
 	ctx, span := trace.StartSpan(ctx, "/mine")
 	defer span.End()
 

miner/testminer.go

@@ -4,14 +4,22 @@ import (
 	"context"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/address"
+	"github.com/filecoin-project/lotus/chain/gen"
 )
 
-func NewTestMiner(nextCh <-chan struct{}) func(api api.FullNode) *Miner {
+func NewTestMiner(nextCh <-chan struct{}, addr address.Address) func(api.FullNode, gen.ElectionPoStProver) *Miner {
-	return func(api api.FullNode) *Miner {
+	return func(api api.FullNode, epp gen.ElectionPoStProver) *Miner {
-		return &Miner{
+		m := &Miner{
 			api:      api,
 			waitFunc: chanWaiter(nextCh),
+			epp:      epp,
 		}
+
+		if err := m.Register(addr); err != nil {
+			panic(err)
+		}
+		return m
 	}
 }
 

node/node_test.go

@@ -6,11 +6,13 @@ import (
 	"crypto/rand"
 	"io/ioutil"
 	"net/http/httptest"
+	"path/filepath"
 	"testing"
 
 	"github.com/libp2p/go-libp2p-core/crypto"
 
 	"github.com/ipfs/go-datastore"
+	badger "github.com/ipfs/go-ds-badger"
 	logging "github.com/ipfs/go-log"
 	"github.com/libp2p/go-libp2p-core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
@@ -26,8 +28,10 @@ import (
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
 	"github.com/filecoin-project/lotus/genesis"
 	"github.com/filecoin-project/lotus/lib/jsonrpc"
+	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 	"github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/impl"
 	"github.com/filecoin-project/lotus/node/modules"
 	modtest "github.com/filecoin-project/lotus/node/modules/testing"
 	"github.com/filecoin-project/lotus/node/repo"
@@ -85,6 +89,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
 	// start node
 	var minerapi api.StorageMiner
 
+	mineBlock := make(chan struct{})
 	// TODO: use stop
 	_, err = node.New(ctx,
 		node.StorageMiner(&minerapi),
@@ -95,6 +100,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
 		node.MockHost(mn),
 
 		node.Override(new(api.FullNode), tnd),
+		node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)),
 	)
 	if err != nil {
 		t.Fatalf("failed to construct node: %v", err)
@@ -106,8 +112,16 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
 
 	err = minerapi.NetConnect(ctx, remoteAddrs)
 	require.NoError(t, err)*/
+	mineOne := func(ctx context.Context) error {
+		select {
+		case mineBlock <- struct{}{}:
+			return nil
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
 
-	return test.TestStorageNode{minerapi}
+	return test.TestStorageNode{minerapi, mineOne}
 }
 
 func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
@@ -134,6 +148,8 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 		PeerIDs:  []peer.ID{minerPid}, // TODO: if we have more miners, need more peer IDs
 		PreSeals: map[string]genesis.GenesisMiner{},
 	}
+
+	var presealDirs []string
 	for i := 0; i < len(storage); i++ {
 		maddr, err := address.NewIDAddress(300 + uint64(i))
 		if err != nil {
@@ -147,6 +163,8 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 		if err != nil {
 			t.Fatal(err)
 		}
+
+		presealDirs = append(presealDirs, tdir)
 		gmc.MinerAddrs = append(gmc.MinerAddrs, maddr)
 		gmc.PreSeals[maddr.String()] = *genm
 	}
@@ -161,8 +179,6 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 			genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
 		}
 
-		mineBlock := make(chan struct{})
-
 		var err error
 		// TODO: Don't ignore stop
 		_, err = node.New(ctx,
@@ -172,22 +188,12 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 			node.MockHost(mn),
 			node.Test(),
 
-			node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock)),
-
 			genesis,
 		)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		fulls[i].MineOne = func(ctx context.Context) error {
-			select {
-			case mineBlock <- struct{}{}:
-				return nil
-			case <-ctx.Done():
-				return ctx.Err()
-			}
-		}
 	}
 
 	for i, full := range storage {
@@ -201,12 +207,36 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 
 		f := fulls[full]
 
-		wa, err := f.WalletDefaultAddress(ctx)
-		require.NoError(t, err)
-
 		genMiner := gmc.MinerAddrs[i]
+		wa := gmc.PreSeals[genMiner.String()].Worker
 
 		storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
+
+		sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
+
+		psd := presealDirs[i]
+		mds, err := badger.NewDatastore(filepath.Join(psd, "badger"), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		osb, err := sectorbuilder.New(&sectorbuilder.Config{
+			SectorSize:    1024,
+			WorkerThreads: 2,
+			Miner:         genMiner,
+			CacheDir:      filepath.Join(psd, "cache"),
+			StagedDir:     filepath.Join(psd, "staging"),
+			SealedDir:     filepath.Join(psd, "sealed"),
+			MetadataDir:   filepath.Join(psd, "meta"),
+		}, mds)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err := sma.SectorBuilder.ImportFrom(osb); err != nil {
+			t.Fatal(err)
+		}
+
 	}
 
 	if err := mn.LinkAll(); err != nil {
@@ -235,7 +265,6 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
 		if err != nil {
 			t.Fatal(err)
 		}
-		fulls[i].MineOne = a.MineOne
 	}
 
 	for i, a := range storaApis {
@@ -248,6 +277,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
 		if err != nil {
 			t.Fatal(err)
 		}
+		storers[i].MineOne = a.MineOne
 	}
 
 	return fulls, storers

node/repo/interface.go

@@ -47,6 +47,6 @@ type LockedRepo interface {
 	// KeyStore returns store of private keys for Filecoin transactions
 	KeyStore() (types.KeyStore, error)
 
-	// Path returns absolute path of the repo (or empty string if in-memory)
+	// Path returns absolute path of the repo
 	Path() string
 }

storage/miner.go

@@ -13,6 +13,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/address"
 	"github.com/filecoin-project/lotus/chain/events"
 	"github.com/filecoin-project/lotus/chain/gen"
@@ -161,5 +162,9 @@ func (epp *sectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi sectorb
 }
 
 func (epp *sectorBuilderEpp) ComputeProof(ctx context.Context, ssi sectorbuilder.SortedPublicSectorInfo, rand []byte, winners []sectorbuilder.EPostCandidate) ([]byte, error) {
+	if build.InsecurePoStValidation {
+		log.Warn("Generating fake EPost proof! You should only see this while running tests!")
+		return []byte("valid proof"), nil
+	}
 	return epp.sb.ComputeElectionPoSt(ssi, rand, winners)
 }