Merge branch 'master' into feat/chainwatch-pg

Makefile

@@ -14,6 +14,7 @@ MODULES:=
 
 CLEAN:=
 BINS:=
+GOFLAGS+=-ldflags="-X "github.com/filecoin-project/lotus/build".CurrentCommit=+git$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))"
 
 ## FFI
 
@@ -47,7 +48,7 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
-debug: GOFLAGS=-tags=debug
+debug: GOFLAGS+=-tags=debug
 debug: lotus lotus-storage-miner lotus-seal-worker lotus-seed
 
 lotus: $(BUILD_DEPS)

api/api_full.go

@@ -113,6 +113,7 @@ type FullNode interface {
 	StateLookupID(context.Context, address.Address, *types.TipSet) (address.Address, error)
 	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)
 	StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)
+	StateMinerSectorCount(context.Context, address.Address, *types.TipSet) (MinerSectors, error)
 
 	MarketEnsureAvailable(context.Context, address.Address, types.BigInt) error
 	// MarketFreeBalance
@@ -131,6 +132,11 @@ type FullNode interface {
 	PaychVoucherSubmit(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error)
 }
 
+type MinerSectors struct {
+	Pset uint64
+	Sset uint64
+}
+
 type Import struct {
 	Status   filestore.Status
 	Key      cid.Cid

api/apistruct/struct.go

@@ -109,6 +109,7 @@ type FullNodeStruct struct {
 		StateLookupID                 func(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error)      `perm:"read"`
 		StateChangedActors            func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)                         `perm:"read"`
 		StateGetReceipt               func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)                    `perm:"read"`
+		StateMinerSectorCount         func(context.Context, address.Address, *types.TipSet) (api.MinerSectors, error)                 `perm:"read"`
 
 		MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"`
 
@@ -128,6 +129,10 @@ type FullNodeStruct struct {
 	}
 }
 
+func (c *FullNodeStruct) StateMinerSectorCount(ctx context.Context, addr address.Address, ts *types.TipSet) (api.MinerSectors, error) {
+	return c.Internal.StateMinerSectorCount(ctx, addr, ts)
+}
+
 type StorageMinerStruct struct {
 	CommonStruct
 

build/forks.go

@@ -0,0 +1,3 @@
+package build
+
+const ForkCCM = 1750

build/version.go

@@ -1,7 +1,11 @@
 package build
 
+var CurrentCommit string
+
 // Version is the local build version, set by build system
-const Version = "0.1.0"
+const Version = "0.1.1"
+
+var UserVersion = Version + CurrentCommit
 
 // APIVersion is a hex semver version of the rpc api exposed
 //
@@ -12,7 +16,7 @@ const Version = "0.1.0"
 //                   R R  H
 //                   |\vv/|
 //                   vv  vv
-const APIVersion = 0x000100
+const APIVersion = 0x000101
 
 const (
 	MajorMask = 0xff0000

chain/stmgr/utils.go

@@ -160,6 +160,30 @@ func GetMinerElectionPeriodStart(ctx context.Context, sm *StateManager, ts *type
 	return mas.ElectionPeriodStart, nil
 }
 
+func SectorSetSizes(ctx context.Context, sm *StateManager, maddr address.Address, ts *types.TipSet) (api.MinerSectors, error) {
+	var mas actors.StorageMinerActorState
+	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
+	if err != nil {
+		return api.MinerSectors{}, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
+	}
+
+	blks := amt.WrapBlockstore(sm.ChainStore().Blockstore())
+	ss, err := amt.LoadAMT(blks, mas.Sectors)
+	if err != nil {
+		return api.MinerSectors{}, err
+	}
+
+	ps, err := amt.LoadAMT(blks, mas.ProvingSet)
+	if err != nil {
+		return api.MinerSectors{}, err
+	}
+
+	return api.MinerSectors{
+		Pset: ps.Count,
+		Sset: ss.Count,
+	}, nil
+}
+
 func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) {
 	var mas actors.StorageMinerActorState
 	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)

chain/sync.go

@@ -513,6 +513,14 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
 
 		snum := types.BigDiv(mpow, types.NewInt(ssize))
 
+		// FORK START
+		if h.Height > build.ForkCCM {
+			if len(h.EPostProof.Candidates) == 0 {
+				return xerrors.Errorf("no candidates")
+			}
+		}
+		// FORK END
+
 		for _, t := range h.EPostProof.Candidates {
 			if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) {
 				return xerrors.Errorf("miner created a block but was not a winner")
@@ -634,6 +642,13 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc
 			SectorChallengeIndex: t.ChallengeIndex,
 		})
 	}
+	// FORK START
+	if h.Height > build.ForkCCM {
+		if len(winners) == 0 {
+			return xerrors.Errorf("no candidates")
+		}
+	}
+	// FORK END
 
 	sectorInfo, err := stmgr.GetSectorsForElectionPost(ctx, syncer.sm, baseTs, h.Miner)
 	if err != nil {
@@ -647,6 +662,7 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc
 		return xerrors.Errorf("[TESTING] election post was invalid")
 	}
 	hvrf := sha256.Sum256(h.EPostProof.PostRand)
+
 	ok, err := sectorbuilder.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
 	if err != nil {
 		return xerrors.Errorf("failed to verify election post: %w", err)

cli/state.go

@@ -24,6 +24,7 @@ var stateCmd = &cli.Command{
 		stateGetActorCmd,
 		stateLookupIDCmd,
 		stateReplaySetCmd,
+		stateSectorSizeCmd,
 	},
 }
 
@@ -334,3 +335,35 @@ var stateLookupIDCmd = &cli.Command{
 		return nil
 	},
 }
+
+var stateSectorSizeCmd = &cli.Command{
+	Name:  "sector-size",
+	Usage: "Look up miners sector size",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass address of actor to get")
+		}
+
+		addr, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		ssize, err := api.StateMinerSectorSize(ctx, addr, nil)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("%d\n", ssize)
+
+		return nil
+	},
+}

cmd/lotus-bench/main.go

@@ -59,7 +59,7 @@ func main() {
 	app := &cli.App{
 		Name:    "lotus-bench",
 		Usage:   "Benchmark performance of lotus on your hardware",
-		Version: build.Version,
+		Version: build.UserVersion,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:  "storage-dir",

cmd/lotus-chainwatch/dot.go

@@ -50,6 +50,7 @@ var dotCmd = &cli.Command{
 
 			hasstr := ""
 			if !has {
+				col = 0xffffffff
 				hasstr = " UNSYNCED"
 			}
 

cmd/lotus-chainwatch/main.go

@@ -28,7 +28,7 @@ func main() {
 	app := &cli.App{
 		Name:    "lotus-chainwatch",
 		Usage:   "Devnet token distribution utility",
-		Version: build.Version,
+		Version: build.UserVersion,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    "repo",

cmd/lotus-chainwatch/storage.go

@@ -66,7 +66,14 @@ create table if not exists blocks
 	parentStateRoot text not null,
 	height int not null,
 	miner text not null,
-	timestamp int not null
+	timestamp int not null,
+	vrfproof bytea,
+	tickets int not null,
+	eprof bytea,
+	prand bytea,
+	ep0partial bytea,
+	ep0sector int not null,
+	ep0challangei int not null
 );
 
 create unique index if not exists block_cid_uindex
@@ -177,7 +184,9 @@ create table if not exists miner_heads
 	addr text not null,
 	stateroot text not null,
 	sectorset text not null,
+	setsize int not null,
 	provingset text not null,
+	provingsize int not null,
 	owner text not null,
 	worker text not null,
 	peerid text not null,
@@ -256,7 +265,7 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into miner_heads (head, addr, stateroot, sectorset, provingset, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) on conflict do nothing`)
+	stmt, err := tx.Prepare(`insert into miner_heads (head, addr, stateroot, sectorset, setsize, provingset, provingsize, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) on conflict do nothing`)
 	if err != nil {
 		return err
 	}
@@ -267,7 +276,9 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
 			k.addr.String(),
 			k.stateroot.String(),
 			i.state.Sectors.String(),
+			i.ssize,
 			i.state.ProvingSet.String(),
+			i.psize,
 			i.info.Owner.String(),
 			i.info.Worker.String(),
 			i.info.PeerID.String(),
@@ -284,73 +295,75 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
 	return tx.Commit()
 }
 
-func (st *storage) batch(n int) chan *sql.Tx {
-	out := make(chan *sql.Tx, n)
-	for i := 0; i < n; i++ {
-		tx, err := st.db.Begin()
-		if err != nil {
-			log.Error(err)
-		}
-
-		out <- tx
-	}
-
-	return out
-}
-
-func endbatch(b chan *sql.Tx) {
-	n := len(b)
-	for i := 0; i < n; i++ {
-		tx := <- b
-		if err := tx.Commit(); err != nil {
-			log.Error(err)
-		}
-	}
-}
-
 func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) error {
 	st.headerLk.Lock()
 	defer st.headerLk.Unlock()
 
-	tb := st.batch(50)
+	tx, err := st.db.Begin()
-
-	par(100, maparr(bhs), func(bh *types.BlockHeader) {
-		for _, parent := range bh.Parents {
-			log.Info("bps ", bh.Cid())
-			tx := <-tb
-			stmt2, err := tx.Prepare(`insert into block_parents (block, parent) values ($1, $2) on conflict do nothing`)
 	if err != nil {
 		return err
 	}
-			if _, err := tx.Exec(`insert into block_parents (block, parent) values ($1, $2) on conflict do nothing`, bh.Cid().String(), parent.String()); err != nil {
+
+	stmt, err := tx.Prepare(`insert into block_parents (block, parent) values ($1, $2) on conflict do nothing`)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	for _, bh := range bhs {
+		for _, parent := range bh.Parents {
+			if _, err := stmt.Exec(bh.Cid().String(), parent.String()); err != nil {
 				log.Error(err)
 			}
-			tb <- tx
 		}
-	})
+	}
 
 	if sync {
 		now := time.Now().Unix()
 
-		par(50, maparr(bhs), func(bh *types.BlockHeader) {
+		stmt, err := tx.Prepare(`insert into blocks_synced (cid, add_ts) values ($1, $2) on conflict do nothing`)
-			tx := <-tb
+		if err != nil {
-			if _, err := tx.Exec(`insert into blocks_synced (cid, add_ts) values ($1, $2) on conflict do nothing`, bh.Cid().String(), now); err != nil {
+			return err
+		}
+		defer stmt.Close()
+
+		for _, bh := range bhs {
+			if _, err := tx.Exec(bh.Cid().String(), now); err != nil {
 				log.Error(err)
 			}
-			tb <- tx
+		}
-		})
 	}
 
-	par(50, maparr(bhs), func(bh *types.BlockHeader) {
+	stmt2, err := tx.Prepare(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) values ($1, $2, $3, $4, $5, $6,$7,$8,$9,$10,$11,$12,$13) on conflict do nothing`)
-		log.Info("bh", bh.Cid())
+	if err != nil {
-		tx := <-tb
+		return err
-		if _, err := tx.Exec(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp") values ($1, $2, $3, $4, $5, $6) on conflict do nothing`, bh.Cid().String(), bh.ParentWeight.String(), bh.ParentStateRoot.String(), bh.Height, bh.Miner.String(), bh.Timestamp); err != nil {
+	}
+	defer stmt2.Close()
+
+	for _, bh := range bhs {
+		l := len(bh.EPostProof.Candidates)
+		if len(bh.EPostProof.Candidates) == 0 {
+			bh.EPostProof.Candidates = append(bh.EPostProof.Candidates, types.EPostTicket{})
+		}
+
+		if _, err := stmt2.Exec(
+			bh.Cid().String(),
+			bh.ParentWeight.String(),
+			bh.ParentStateRoot.String(),
+			bh.Height,
+			bh.Miner.String(),
+			bh.Timestamp,
+			bh.Ticket.VRFProof,
+			l,
+			bh.EPostProof.Proof,
+			bh.EPostProof.PostRand,
+			bh.EPostProof.Candidates[0].Partial,
+			bh.EPostProof.Candidates[0].SectorID,
+			bh.EPostProof.Candidates[0].ChallengeIndex); err != nil {
 			log.Error(err)
 		}
-		tb <- tx
+	}
-	})
 
-	endbatch(tb)
 	return nil
 }
 

cmd/lotus-chainwatch/sync.go

@@ -46,6 +46,9 @@ type minerKey struct {
 type minerInfo struct {
 	state actors2.StorageMinerActorState
 	info  actors2.MinerInfo
+
+	ssize uint64
+	psize uint64
 }
 
 func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipSet) {
@@ -196,6 +199,14 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 	par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) {
 		k, info := it()
 
+		sszs, err := api.StateMinerSectorCount(ctx, k.addr, nil)
+		if err != nil {
+			log.Error(err)
+			return
+		}
+		info.psize = sszs.Pset
+		info.ssize = sszs.Sset
+
 		astb, err := api.ChainReadObj(ctx, k.act.Head)
 		if err != nil {
 			log.Error(err)

cmd/lotus-fountain/main.go

@@ -40,7 +40,7 @@ func main() {
 	app := &cli.App{
 		Name:    "lotus-fountain",
 		Usage:   "Devnet token distribution utility",
-		Version: build.Version,
+		Version: build.UserVersion,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    "repo",

cmd/lotus-seal-worker/main.go

@@ -30,7 +30,7 @@ func main() {
 	app := &cli.App{
 		Name:    "lotus-seal-worker",
 		Usage:   "Remote storage miner worker",
-		Version: build.Version,
+		Version: build.UserVersion,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    "repo",

cmd/lotus-seed/main.go

@@ -36,7 +36,7 @@ func main() {
 	app := &cli.App{
 		Name:    "lotus-seed",
 		Usage:   "Seal sectors for genesis miner",
-		Version: build.Version,
+		Version: build.UserVersion,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:  "sectorbuilder-dir",

cmd/lotus-storage-miner/info.go

@@ -53,6 +53,13 @@ var infoCmd = &cli.Command{
 		percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000)), pow.TotalPower)
 		fmt.Printf("Power: %s / %s (%0.4f%%)\n", lcli.SizeStr(pow.MinerPower), lcli.SizeStr(pow.TotalPower), float64(percI.Int64())/100000*10000)
 
+		secCounts, err := api.StateMinerSectorCount(ctx, maddr, nil)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("\tCommitted: %s\n", lcli.SizeStr(types.BigMul(types.NewInt(secCounts.Sset), types.NewInt(sizeByte))))
+		fmt.Printf("\tProving: %s\n", lcli.SizeStr(types.BigMul(types.NewInt(secCounts.Pset), types.NewInt(sizeByte))))
+
 		// TODO: indicate whether the post worker is in use
 		wstat, err := nodeApi.WorkerStats(ctx)
 		if err != nil {

cmd/lotus-storage-miner/main.go

@@ -52,7 +52,7 @@ func main() {
 	app := &cli.App{
 		Name:    "lotus-storage-miner",
 		Usage:   "Filecoin decentralized storage network storage miner",
-		Version: build.Version,
+		Version: build.UserVersion,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    "repo",

cmd/lotus/main.go

@@ -50,7 +50,7 @@ func main() {
 	app := &cli.App{
 		Name:    "lotus",
 		Usage:   "Filecoin decentralized storage network client",
-		Version: build.Version,
+		Version: build.UserVersion,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    "repo",

miner/miner.go

@@ -270,7 +270,7 @@ func (m *Miner) mineOne(ctx context.Context, addr address.Address, base *MiningB
 		return nil, nil
 	}
 
-	log.Infof("Time delta between now and our mining base: %ds", uint64(time.Now().Unix())-base.ts.MinTimestamp())
+	log.Infof("Time delta between now and our mining base: %ds (nulls: %d)", uint64(time.Now().Unix())-base.ts.MinTimestamp(), base.nullRounds)
 
 	ticket, err := m.computeTicket(ctx, addr, base)
 	if err != nil {

node/hello/hello.go

@@ -60,7 +60,6 @@ func NewHelloService(h host.Host, cs *store.ChainStore, syncer *chain.Syncer, pm
 }
 
 func (hs *Service) HandleStream(s inet.Stream) {
-	defer s.Close()
 
 	var hmsg Message
 	if err := cborutil.ReadCborRPC(s, &hmsg); err != nil {
@@ -81,6 +80,8 @@ func (hs *Service) HandleStream(s inet.Stream) {
 		return
 	}
 	go func() {
+		defer s.Close()
+
 		sent := time.Now()
 		msg := &Message{
 			TArrial: arrived.UnixNano(),
@@ -110,7 +111,6 @@ func (hs *Service) SayHello(ctx context.Context, pid peer.ID) error {
 	if err != nil {
 		return err
 	}
-	defer s.Close()
 
 	hts := hs.cs.GetHeaviestTipSet()
 	weight, err := hs.cs.Weight(ctx, hts)
@@ -136,9 +136,11 @@ func (hs *Service) SayHello(ctx context.Context, pid peer.ID) error {
 	}
 
 	go func() {
+		defer s.Close()
+
 		hmsg = &Message{}
 		s.SetReadDeadline(time.Now().Add(10 * time.Second))
-		err := cborutil.ReadCborRPC(s, hmsg) // ignore error
+		err := cborutil.ReadCborRPC(s, hmsg)
 		ok := err == nil
 
 		t3 := time.Now()

node/impl/full/state.go

@@ -358,3 +358,7 @@ func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid.
 
 	return out, nil
 }
+
+func (a *StateAPI) StateMinerSectorCount(ctx context.Context, addr address.Address, ts *types.TipSet) (api.MinerSectors, error) {
+	return stmgr.SectorSetSizes(ctx, a.StateManager, addr, ts)
+}