Merge pull request #7941 from filecoin-project/chore/more_detailed_network-wide_shed_stats

chore: shed: storage stats 2.0

cmd/lotus-shed/storage-stats.go

@@ -4,10 +4,12 @@ import (
 	"encoding/json"
 	corebig "math/big"
 	"os"
+	"strconv"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	filbig "github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
@@ -22,29 +24,50 @@ type networkTotalsOutput struct {
 	Payload  networkTotals `json:"payload"`
 }
 
-type networkTotals struct {
+type providerMeta struct {
-	QaNetworkPower       filbig.Int `json:"total_qa_power"`
+	nonidentifiable bool
-	RawNetworkPower      filbig.Int `json:"total_raw_capacity"`
+}
-	CapacityCarryingData float64    `json:"capacity_fraction_carrying_data"`
+
-	UniqueCids           int        `json:"total_unique_cids"`
+// force formatting as decimal to aid human readers
-	UniqueProviders      int        `json:"total_unique_providers"`
+type humanFloat float64
-	UniqueClients        int        `json:"total_unique_clients"`
+
+func (f humanFloat) MarshalJSON() ([]byte, error) {
+	// 'f' uses decimal digits without exponents.
+	// The bit size of 32 ensures we don't use too many decimal places.
+	return []byte(strconv.FormatFloat(float64(f), 'f', -1, 32)), nil
+}
+
+type Totals struct {
 	TotalDeals           int        `json:"total_num_deals"`
 	TotalBytes           int64      `json:"total_stored_data_size"`
-	FilplusTotalDeals    int        `json:"filplus_total_num_deals"`
+	PrivateTotalDeals    int        `json:"private_total_num_deals"`
-	FilplusTotalBytes    int64      `json:"filplus_total_stored_data_size"`
+	PrivateTotalBytes    int64      `json:"private_total_stored_data_size"`
+	CapacityCarryingData humanFloat `json:"capacity_fraction_carrying_data"`
+}
 
-	seenClient   map[address.Address]bool
+type networkTotals struct {
-	seenProvider map[address.Address]bool
+	QaNetworkPower         filbig.Int `json:"total_qa_power"`
-	seenPieceCid map[cid.Cid]bool
+	RawNetworkPower        filbig.Int `json:"total_raw_capacity"`
+	UniqueCids             int        `json:"total_unique_cids"`
+	UniqueBytes            int64      `json:"total_unique_data_size"`
+	UniqueClients          int        `json:"total_unique_clients"`
+	UniqueProviders        int        `json:"total_unique_providers"`
+	UniquePrivateProviders int        `json:"total_unique_private_providers"`
+	Totals
+	FilPlus Totals `json:"filecoin_plus_subset"`
+
+	pieces    map[cid.Cid]struct{}
+	clients   map[address.Address]struct{}
+	providers map[address.Address]providerMeta
 }
 
 var storageStatsCmd = &cli.Command{
 	Name:  "storage-stats",
 	Usage: "Translates current lotus state into a json summary suitable for driving https://storage.filecoin.io/",
 	Flags: []cli.Flag{
-		&cli.Int64Flag{
+		&cli.StringFlag{
-			Name: "height",
+			Name:  "tipset",
+			Usage: "Comma separated array of cids, or @height",
 		},
 	},
 	Action: func(cctx *cli.Context) error {
@@ -56,22 +79,24 @@ var storageStatsCmd = &cli.Command{
 		}
 		defer apiCloser()
 
-		head, err := api.ChainHead(ctx)
+		var ts *types.TipSet
-		if err != nil {
+		if cctx.String("tipset") == "" {
-			return err
+			ts, err = api.ChainHead(ctx)
-		}
+			if err != nil {
-
+				return err
-		requestedHeight := cctx.Int64("height")
+			}
-		if requestedHeight > 0 {
+			ts, err = api.ChainGetTipSetByHeight(ctx, ts.Height()-defaultEpochLookback, ts.Key())
-			head, err = api.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(requestedHeight), head.Key())
+			if err != nil {
+				return err
+			}
 		} else {
-			head, err = api.ChainGetTipSetByHeight(ctx, head.Height()-defaultEpochLookback, head.Key())
+			ts, err = lcli.ParseTipSetRef(ctx, api, cctx.String("tipset"))
-		}
+			if err != nil {
-		if err != nil {
+				return err
-			return err
+			}
 		}
 
-		power, err := api.StateMinerPower(ctx, address.Address{}, head.Key())
+		power, err := api.StateMinerPower(ctx, address.Address{}, ts.Key())
 		if err != nil {
 			return err
 		}
@@ -79,12 +104,12 @@ var storageStatsCmd = &cli.Command{
 		netTotals := networkTotals{
 			QaNetworkPower:  power.TotalPower.QualityAdjPower,
 			RawNetworkPower: power.TotalPower.RawBytePower,
-			seenClient:      make(map[address.Address]bool),
+			pieces:          make(map[cid.Cid]struct{}),
-			seenProvider:    make(map[address.Address]bool),
+			clients:         make(map[address.Address]struct{}),
-			seenPieceCid:    make(map[cid.Cid]bool),
+			providers:       make(map[address.Address]providerMeta),
 		}
 
-		deals, err := api.StateMarketDeals(ctx, head.Key())
+		deals, err := api.StateMarketDeals(ctx, ts.Key())
 		if err != nil {
 			return err
 		}
@@ -94,35 +119,76 @@ var storageStatsCmd = &cli.Command{
 			// Only count deals that have properly started, not past/future ones
 			// https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
 			// Bail on 0 as well in case SectorStartEpoch is uninitialized due to some bug
+			//
+			// Additionally if the SlashEpoch is set this means the underlying sector is
+			// terminated for whatever reason ( not just slashed ), and the deal record
+			// will soon be removed from the state entirely
 			if dealInfo.State.SectorStartEpoch <= 0 ||
-				dealInfo.State.SectorStartEpoch > head.Height() {
+				dealInfo.State.SectorStartEpoch > ts.Height() ||
+				dealInfo.State.SlashEpoch > -1 {
 				continue
 			}
 
-			netTotals.seenClient[dealInfo.Proposal.Client] = true
+			netTotals.clients[dealInfo.Proposal.Client] = struct{}{}
+
+			if _, seen := netTotals.providers[dealInfo.Proposal.Provider]; !seen {
+				pm := providerMeta{}
+
+				mi, err := api.StateMinerInfo(ctx, dealInfo.Proposal.Provider, ts.Key())
+				if err != nil {
+					return err
+				}
+
+				if mi.PeerId == nil || *mi.PeerId == "" {
+					log.Infof("private provider %s", dealInfo.Proposal.Provider)
+					pm.nonidentifiable = true
+					netTotals.UniquePrivateProviders++
+				}
+
+				netTotals.providers[dealInfo.Proposal.Provider] = pm
+				netTotals.UniqueProviders++
+			}
+
+			if _, seen := netTotals.pieces[dealInfo.Proposal.PieceCID]; !seen {
+				netTotals.pieces[dealInfo.Proposal.PieceCID] = struct{}{}
+				netTotals.UniqueBytes += int64(dealInfo.Proposal.PieceSize)
+				netTotals.UniqueCids++
+			}
+
 			netTotals.TotalBytes += int64(dealInfo.Proposal.PieceSize)
-			netTotals.seenProvider[dealInfo.Proposal.Provider] = true
-			netTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true
 			netTotals.TotalDeals++
+			if netTotals.providers[dealInfo.Proposal.Provider].nonidentifiable {
+				netTotals.PrivateTotalBytes += int64(dealInfo.Proposal.PieceSize)
+				netTotals.PrivateTotalDeals++
+			}
 
 			if dealInfo.Proposal.VerifiedDeal {
-				netTotals.FilplusTotalDeals++
+				netTotals.FilPlus.TotalBytes += int64(dealInfo.Proposal.PieceSize)
-				netTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize)
+				netTotals.FilPlus.TotalDeals++
+				if netTotals.providers[dealInfo.Proposal.Provider].nonidentifiable {
+					netTotals.FilPlus.PrivateTotalBytes += int64(dealInfo.Proposal.PieceSize)
+					netTotals.FilPlus.PrivateTotalDeals++
+				}
 			}
 		}
 
-		netTotals.UniqueCids = len(netTotals.seenPieceCid)
+		netTotals.UniqueClients = len(netTotals.clients)
-		netTotals.UniqueClients = len(netTotals.seenClient)
-		netTotals.UniqueProviders = len(netTotals.seenProvider)
 
-		netTotals.CapacityCarryingData, _ = new(corebig.Rat).SetFrac(
+		ccd, _ := new(corebig.Rat).SetFrac(
 			corebig.NewInt(netTotals.TotalBytes),
 			netTotals.RawNetworkPower.Int,
 		).Float64()
+		netTotals.CapacityCarryingData = humanFloat(ccd)
+
+		ccdfp, _ := new(corebig.Rat).SetFrac(
+			corebig.NewInt(netTotals.FilPlus.TotalBytes),
+			netTotals.RawNetworkPower.Int,
+		).Float64()
+		netTotals.FilPlus.CapacityCarryingData = humanFloat(ccdfp)
 
 		return json.NewEncoder(os.Stdout).Encode(
 			networkTotalsOutput{
-				Epoch:    int64(head.Height()),
+				Epoch:    int64(ts.Height()),
 				Endpoint: "NETWORK_WIDE_TOTALS",
 				Payload:  netTotals,
 			},