Merge pull request #878 from filecoin-project/feat/cw-sset-counts

chainwatch: Collect sector set sizes

api/api_full.go

@@ -113,6 +113,7 @@ type FullNode interface {
 	StateLookupID(context.Context, address.Address, *types.TipSet) (address.Address, error)
 	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)
 	StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)
+	StateMinerSectorCount(context.Context, address.Address, *types.TipSet) (MinerSectors, error)
 
 	MarketEnsureAvailable(context.Context, address.Address, types.BigInt) error
 	// MarketFreeBalance
@@ -131,6 +132,11 @@ type FullNode interface {
 	PaychVoucherSubmit(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error)
 }
 
+type MinerSectors struct {
+	Pset uint64
+	Sset uint64
+}
+
 type Import struct {
 	Status   filestore.Status
 	Key      cid.Cid

api/apistruct/struct.go

@@ -109,6 +109,7 @@ type FullNodeStruct struct {
 		StateLookupID                 func(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error)      `perm:"read"`
 		StateChangedActors            func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)                         `perm:"read"`
 		StateGetReceipt               func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)                    `perm:"read"`
+		StateMinerSectorCount         func(context.Context, address.Address, *types.TipSet) (api.MinerSectors, error)                 `perm:"read"`
 
 		MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"`
 
@@ -128,6 +129,10 @@ type FullNodeStruct struct {
 	}
 }
 
+func (c *FullNodeStruct) StateMinerSectorCount(ctx context.Context, addr address.Address, ts *types.TipSet) (api.MinerSectors, error) {
+	return c.Internal.StateMinerSectorCount(ctx, addr, ts)
+}
+
 type StorageMinerStruct struct {
 	CommonStruct
 

chain/stmgr/utils.go

@@ -160,6 +160,30 @@ func GetMinerElectionPeriodStart(ctx context.Context, sm *StateManager, ts *type
 	return mas.ElectionPeriodStart, nil
 }
 
+func SectorSetSizes(ctx context.Context, sm *StateManager, maddr address.Address, ts *types.TipSet) (api.MinerSectors, error) {
+	var mas actors.StorageMinerActorState
+	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
+	if err != nil {
+		return api.MinerSectors{}, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
+	}
+
+	blks := amt.WrapBlockstore(sm.ChainStore().Blockstore())
+	ss, err := amt.LoadAMT(blks, mas.Sectors)
+	if err != nil {
+		return api.MinerSectors{}, err
+	}
+
+	ps, err := amt.LoadAMT(blks, mas.ProvingSet)
+	if err != nil {
+		return api.MinerSectors{}, err
+	}
+
+	return api.MinerSectors{
+		Pset: ps.Count,
+		Sset: ss.Count,
+	}, nil
+}
+
 func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) {
 	var mas actors.StorageMinerActorState
 	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)

cmd/lotus-chainwatch/dot.go

@@ -48,6 +48,7 @@ var dotCmd = &cli.Command{
 
 			hasstr := ""
 			if !has {
+				col = 0xffffffff
 				hasstr = " UNSYNCED"
 			}
 

cmd/lotus-chainwatch/storage.go

@@ -101,7 +101,14 @@ create table if not exists blocks
 	miner text not null
 		constraint blocks_id_address_map_miner_fk
 			references id_address_map (address),
-	timestamp int not null
+	timestamp int not null,
+	vrfproof blob,
+	tickets int not null,
+	eprof blob,
+	prand blob,
+	ep0partial blob,
+	ep0sector int not null,
+	ep0challangei int not null
 );
 
 create unique index if not exists block_cid_uindex
@@ -193,7 +200,9 @@ create table if not exists miner_heads
 		constraint miner_heads_blocks_stateroot_fk
 			references blocks (parentStateRoot),
 	sectorset text not null,
+	setsize int not null,
 	provingset text not null,
+	provingsize int not null,
 	owner text not null,
 	worker text not null,
 	peerid text not null,
@@ -255,7 +264,7 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into miner_heads (head, addr, stateroot, sectorset, provingset, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
+	stmt, err := tx.Prepare(`insert into miner_heads (head, addr, stateroot, sectorset, setsize, provingset, provingsize, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
 	if err != nil {
 		return err
 	}
@@ -266,7 +275,9 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
 			k.addr.String(),
 			k.stateroot.String(),
 			i.state.Sectors.String(),
+			i.ssize,
 			i.state.ProvingSet.String(),
+			i.psize,
 			i.info.Owner.String(),
 			i.info.Worker.String(),
 			i.info.PeerID.String(),
@@ -292,13 +303,31 @@ func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) e
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp") values (?, ?, ?, ?, ?, ?) on conflict do nothing`)
+	stmt, err := tx.Prepare(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
 	if err != nil {
 		return err
 	}
 	defer stmt.Close()
 	for _, bh := range bhs {
-		if _, err := stmt.Exec(bh.Cid().String(), bh.ParentWeight.String(), bh.ParentStateRoot.String(), bh.Height, bh.Miner.String(), bh.Timestamp); err != nil {
+		l := len(bh.EPostProof.Candidates)
+		if len(bh.EPostProof.Candidates) == 0 {
+			bh.EPostProof.Candidates = append(bh.EPostProof.Candidates, types.EPostTicket{})
+		}
+
+		if _, err := stmt.Exec(bh.Cid().String(),
+			bh.ParentWeight.String(),
+			bh.ParentStateRoot.String(),
+			bh.Height,
+			bh.Miner.String(),
+			bh.Timestamp,
+			bh.Ticket.VRFProof,
+			l,
+			bh.EPostProof.Proof,
+			bh.EPostProof.PostRand,
+			bh.EPostProof.Candidates[0].Partial,
+			bh.EPostProof.Candidates[0].SectorID,
+			bh.EPostProof.Candidates[0].ChallengeIndex,
+	); err != nil {
 			return err
 		}
 	}

cmd/lotus-chainwatch/sync.go

@@ -46,6 +46,9 @@ type minerKey struct {
 type minerInfo struct {
 	state actors2.StorageMinerActorState
 	info  actors2.MinerInfo
+
+	ssize uint64
+	psize uint64
 }
 
 func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipSet) {
@@ -185,6 +188,14 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 	par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) {
 		k, info := it()
 
+		sszs, err := api.StateMinerSectorCount(ctx, k.addr, nil)
+		if err != nil {
+			log.Error(err)
+			return
+		}
+		info.psize = sszs.Pset
+		info.ssize = sszs.Sset
+
 		astb, err := api.ChainReadObj(ctx, k.act.Head)
 		if err != nil {
 			log.Error(err)

cmd/lotus-storage-miner/info.go

@@ -53,6 +53,13 @@ var infoCmd = &cli.Command{
 		percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000)), pow.TotalPower)
 		fmt.Printf("Power: %s / %s (%0.4f%%)\n", lcli.SizeStr(pow.MinerPower), lcli.SizeStr(pow.TotalPower), float64(percI.Int64())/100000*10000)
 
+		secCounts, err := api.StateMinerSectorCount(ctx, maddr, nil)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("\tCommitted: %s\n", lcli.SizeStr(types.BigMul(types.NewInt(secCounts.Sset), types.NewInt(sizeByte))))
+		fmt.Printf("\tProving: %s\n", lcli.SizeStr(types.BigMul(types.NewInt(secCounts.Pset), types.NewInt(sizeByte))))
+
 		// TODO: indicate whether the post worker is in use
 		wstat, err := nodeApi.WorkerStats(ctx)
 		if err != nil {

node/impl/full/state.go

@@ -358,3 +358,7 @@ func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid.
 
 	return out, nil
 }
+
+func (a *StateAPI) StateMinerSectorCount(ctx context.Context, addr address.Address, ts *types.TipSet) (api.MinerSectors, error) {
+	return stmgr.SectorSetSizes(ctx, a.StateManager, addr, ts)
+}