package main import ( "encoding/json" corebig "math/big" "os" "strconv" "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" filbig "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) // How many epochs back to look at for dealstats var defaultEpochLookback = abi.ChainEpoch(10) type networkTotalsOutput struct { Epoch int64 `json:"epoch"` Endpoint string `json:"endpoint"` Payload networkTotals `json:"payload"` } type providerMeta struct { nonidentifiable bool } // force formatting as decimal to aid human readers type humanFloat float64 func (f humanFloat) MarshalJSON() ([]byte, error) { // 'f' uses decimal digits without exponents. // The bit size of 32 ensures we don't use too many decimal places. return []byte(strconv.FormatFloat(float64(f), 'f', -1, 32)), nil } type Totals struct { TotalDeals int `json:"total_num_deals"` TotalBytes int64 `json:"total_stored_data_size"` PrivateTotalDeals int `json:"private_total_num_deals"` PrivateTotalBytes int64 `json:"private_total_stored_data_size"` CapacityCarryingData humanFloat `json:"capacity_fraction_carrying_data"` } type networkTotals struct { QaNetworkPower filbig.Int `json:"total_qa_power"` RawNetworkPower filbig.Int `json:"total_raw_capacity"` UniqueCids int `json:"total_unique_cids"` UniqueBytes int64 `json:"total_unique_data_size"` UniqueClients int `json:"total_unique_clients"` UniqueProviders int `json:"total_unique_providers"` UniquePrivateProviders int `json:"total_unique_private_providers"` Totals FilPlus Totals `json:"filecoin_plus_subset"` pieces map[cid.Cid]struct{} clients map[address.Address]struct{} providers map[address.Address]providerMeta } var storageStatsCmd = &cli.Command{ Name: "storage-stats", Usage: "Translates current lotus state into a json summary suitable for driving https://storage.filecoin.io/", Flags: []cli.Flag{ &cli.StringFlag{ Name: "tipset", Usage: "Comma separated array of cids, or @height", }, }, Action: func(cctx *cli.Context) error { ctx := lcli.ReqContext(cctx) api, apiCloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer apiCloser() var ts *types.TipSet if cctx.String("tipset") == "" { ts, err = api.ChainHead(ctx) if err != nil { return err } ts, err = api.ChainGetTipSetByHeight(ctx, ts.Height()-defaultEpochLookback, ts.Key()) if err != nil { return err } } else { ts, err = lcli.ParseTipSetRef(ctx, api, cctx.String("tipset")) if err != nil { return err } } power, err := api.StateMinerPower(ctx, address.Address{}, ts.Key()) if err != nil { return err } netTotals := networkTotals{ QaNetworkPower: power.TotalPower.QualityAdjPower, RawNetworkPower: power.TotalPower.RawBytePower, pieces: make(map[cid.Cid]struct{}), clients: make(map[address.Address]struct{}), providers: make(map[address.Address]providerMeta), } deals, err := api.StateMarketDeals(ctx, ts.Key()) if err != nil { return err } for _, dealInfo := range deals { // Only count deals that have properly started, not past/future ones // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85 // Bail on 0 as well in case SectorStartEpoch is uninitialized due to some bug // // Additionally if the SlashEpoch is set this means the underlying sector is // terminated for whatever reason ( not just slashed ), and the deal record // will soon be removed from the state entirely if dealInfo.State.SectorStartEpoch <= 0 || dealInfo.State.SectorStartEpoch > ts.Height() || dealInfo.State.SlashEpoch > -1 { continue } netTotals.clients[dealInfo.Proposal.Client] = struct{}{} if _, seen := netTotals.providers[dealInfo.Proposal.Provider]; !seen { pm := providerMeta{} mi, err := api.StateMinerInfo(ctx, dealInfo.Proposal.Provider, ts.Key()) if err != nil { return err } if mi.PeerId == nil || *mi.PeerId == "" { log.Infof("private provider %s", dealInfo.Proposal.Provider) pm.nonidentifiable = true netTotals.UniquePrivateProviders++ } netTotals.providers[dealInfo.Proposal.Provider] = pm netTotals.UniqueProviders++ } if _, seen := netTotals.pieces[dealInfo.Proposal.PieceCID]; !seen { netTotals.pieces[dealInfo.Proposal.PieceCID] = struct{}{} netTotals.UniqueBytes += int64(dealInfo.Proposal.PieceSize) netTotals.UniqueCids++ } netTotals.TotalBytes += int64(dealInfo.Proposal.PieceSize) netTotals.TotalDeals++ if netTotals.providers[dealInfo.Proposal.Provider].nonidentifiable { netTotals.PrivateTotalBytes += int64(dealInfo.Proposal.PieceSize) netTotals.PrivateTotalDeals++ } if dealInfo.Proposal.VerifiedDeal { netTotals.FilPlus.TotalBytes += int64(dealInfo.Proposal.PieceSize) netTotals.FilPlus.TotalDeals++ if netTotals.providers[dealInfo.Proposal.Provider].nonidentifiable { netTotals.FilPlus.PrivateTotalBytes += int64(dealInfo.Proposal.PieceSize) netTotals.FilPlus.PrivateTotalDeals++ } } } netTotals.UniqueClients = len(netTotals.clients) ccd, _ := new(corebig.Rat).SetFrac( corebig.NewInt(netTotals.TotalBytes), netTotals.RawNetworkPower.Int, ).Float64() netTotals.CapacityCarryingData = humanFloat(ccd) ccdfp, _ := new(corebig.Rat).SetFrac( corebig.NewInt(netTotals.FilPlus.TotalBytes), netTotals.RawNetworkPower.Int, ).Float64() netTotals.FilPlus.CapacityCarryingData = humanFloat(ccdfp) return json.NewEncoder(os.Stdout).Encode( networkTotalsOutput{ Epoch: int64(ts.Height()), Endpoint: "NETWORK_WIDE_TOTALS", Payload: netTotals, }, ) }, }