Remove the SR2 stats, leave just the network totals

The SR2 aggregator now lives at https://github.com/filecoin-project/slingshot-stats
This commit is contained in:
Peter Rabbitson 2020-12-19 22:08:34 +00:00
parent 9f721bfde5
commit bdafdf8060
5 changed files with 115 additions and 459 deletions

View File

@ -40,7 +40,7 @@ func main() {
mpoolStatsCmd,
exportChainCmd,
consensusCmd,
rollupDealStatsCmd,
storageStatsCmd,
syncCmd,
stateTreePruneCmd,
datastoreCmd,

View File

@ -1,455 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strings"
"github.com/Jeffail/gabs"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
)
// Requested by @jbenet
// How many epochs back to look at for dealstats
var epochLookback = abi.ChainEpoch(10)
var resolvedWallets = map[address.Address]address.Address{}
var knownAddrMap = map[address.Address]string{}
//
// contents of basic_stats.json
type competitionTotalOutput struct {
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload competitionTotal `json:"payload"`
}
type competitionTotal struct {
UniqueCids int `json:"total_unique_cids"`
UniqueProviders int `json:"total_unique_providers"`
UniqueProjects int `json:"total_unique_projects"`
UniqueClients int `json:"total_unique_clients"`
TotalDeals int `json:"total_num_deals"`
TotalBytes int64 `json:"total_stored_data_size"`
FilplusTotalDeals int `json:"filplus_total_num_deals"`
FilplusTotalBytes int64 `json:"filplus_total_stored_data_size"`
seenProject map[string]bool
seenClient map[address.Address]bool
seenProvider map[address.Address]bool
seenPieceCid map[cid.Cid]bool
}
//
// contents of client_stats.json
type projectAggregateStatsOutput struct {
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload map[string]*projectAggregateStats `json:"payload"`
}
type projectAggregateStats struct {
ProjectID string `json:"project_id"`
DataSizeMaxProvider int64 `json:"max_data_size_stored_with_single_provider"`
HighestCidDealCount int `json:"max_same_cid_deals"`
DataSize int64 `json:"total_data_size"`
NumCids int `json:"total_num_cids"`
NumDeals int `json:"total_num_deals"`
NumProviders int `json:"total_num_providers"`
ClientStats map[string]*clientAggregateStats `json:"clients"`
dataPerProvider map[address.Address]int64
cidDeals map[cid.Cid]int
}
type clientAggregateStats struct {
Client string `json:"client"`
DataSize int64 `json:"total_data_size"`
NumCids int `json:"total_num_cids"`
NumDeals int `json:"total_num_deals"`
NumProviders int `json:"total_num_providers"`
providers map[address.Address]bool
cids map[cid.Cid]bool
}
//
// contents of deals_list_{{projid}}.json
type dealListOutput struct {
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload []*individualDeal `json:"payload"`
}
type individualDeal struct {
ProjectID string `json:"project_id"`
Client string `json:"client"`
DealID string `json:"deal_id"`
DealStartEpoch int64 `json:"deal_start_epoch"`
MinerID string `json:"miner_id"`
PayloadCID string `json:"payload_cid"`
PaddedSize int64 `json:"data_size"`
}
var rollupDealStatsCmd = &cli.Command{
Name: "rollup-deal-stats",
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 2 || cctx.Args().Get(0) == "" || cctx.Args().Get(1) == "" {
return errors.New("must supply 2 arguments: a nonexistent target directory to write results to and a source of currently active projects")
}
outDirName := cctx.Args().Get(0)
if _, err := os.Stat(outDirName); err == nil {
return fmt.Errorf("unable to proceed: supplied stat target '%s' already exists", outDirName)
}
if err := os.MkdirAll(outDirName, 0755); err != nil {
return fmt.Errorf("creation of destination '%s' failed: %s", outDirName, err)
}
ctx := lcli.ReqContext(cctx)
projListName := cctx.Args().Get(1)
var projListFh *os.File
{
// Parses JSON input in the form:
// {
// "payload": [
// {
// "project": "5fb5f5b3ad3275e236287ce3",
// "address": "f3w3r2c6iukyh3u6f6kx62s5g6n2gf54aqp33ukqrqhje2y6xhf7k55przg4xqgahpcdal6laljz6zonma5pka"
// },
// {
// "project": "5fb608c4ad3275e236287ced",
// "address": "f3rs2khurnubol6ent27lpggidxxujqo2lg5aap5d5bmtam6yjb5wfla5cxxdgj45tqoaawgpzt5lofc3vpzfq"
// },
// ...
// ]
// }
if strings.HasPrefix(projListName, "http://") || strings.HasPrefix(projListName, "https://") {
req, err := http.NewRequestWithContext(ctx, "GET", projListName, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close() //nolint:errcheck
if resp.StatusCode != http.StatusOK {
return xerrors.Errorf("non-200 response: %d", resp.StatusCode)
}
projListFh, err = os.Create(outDirName + "/client_list.json")
if err != nil {
return err
}
_, err = io.Copy(projListFh, resp.Body)
if err != nil {
return err
}
} else {
return errors.New("file inputs not yet supported")
}
if _, err := projListFh.Seek(0, 0); err != nil {
return err
}
defer projListFh.Close() //nolint:errcheck
projList, err := gabs.ParseJSONBuffer(projListFh)
if err != nil {
return err
}
proj, err := projList.Search("payload").Children()
if err != nil {
return err
}
for _, p := range proj {
a, err := address.NewFromString(p.S("address").Data().(string))
if err != nil {
return err
}
knownAddrMap[a] = p.S("project").Data().(string)
}
if len(knownAddrMap) == 0 {
return fmt.Errorf("no active projects/clients found in '%s': unable to continue", projListName)
}
}
outClientStatsFd, err := os.Create(outDirName + "/client_stats.json")
if err != nil {
return err
}
defer outClientStatsFd.Close() //nolint:errcheck
outBasicStatsFd, err := os.Create(outDirName + "/basic_stats.json")
if err != nil {
return err
}
defer outBasicStatsFd.Close() //nolint:errcheck
outUnfilteredStatsFd, err := os.Create(outDirName + "/unfiltered_basic_stats.json")
if err != nil {
return err
}
defer outUnfilteredStatsFd.Close() //nolint:errcheck
api, apiCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer apiCloser()
head, err := api.ChainHead(ctx)
if err != nil {
return err
}
head, err = api.ChainGetTipSetByHeight(ctx, head.Height()-epochLookback, head.Key())
if err != nil {
return err
}
grandTotals := competitionTotal{
seenProject: make(map[string]bool),
seenClient: make(map[address.Address]bool),
seenProvider: make(map[address.Address]bool),
seenPieceCid: make(map[cid.Cid]bool),
}
unfilteredGrandTotals := competitionTotal{
seenClient: make(map[address.Address]bool),
seenProvider: make(map[address.Address]bool),
seenPieceCid: make(map[cid.Cid]bool),
}
projStats := make(map[string]*projectAggregateStats)
projDealLists := make(map[string][]*individualDeal)
deals, err := api.StateMarketDeals(ctx, head.Key())
if err != nil {
return err
}
for dealID, dealInfo := range deals {
// Only count deals that have properly started, not past/future ones
// https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
// Bail on 0 as well in case SectorStartEpoch is uninitialized due to some bug
if dealInfo.State.SectorStartEpoch <= 0 ||
dealInfo.State.SectorStartEpoch > head.Height() {
continue
}
clientAddr, found := resolvedWallets[dealInfo.Proposal.Client]
if !found {
var err error
clientAddr, err = api.StateAccountKey(ctx, dealInfo.Proposal.Client, head.Key())
if err != nil {
log.Warnf("failed to resolve id '%s' to wallet address: %s", dealInfo.Proposal.Client, err)
continue
}
resolvedWallets[dealInfo.Proposal.Client] = clientAddr
}
unfilteredGrandTotals.seenClient[clientAddr] = true
unfilteredGrandTotals.TotalBytes += int64(dealInfo.Proposal.PieceSize)
unfilteredGrandTotals.seenProvider[dealInfo.Proposal.Provider] = true
unfilteredGrandTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true
unfilteredGrandTotals.TotalDeals++
if dealInfo.Proposal.VerifiedDeal {
unfilteredGrandTotals.FilplusTotalDeals++
unfilteredGrandTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize)
}
// perl -E 'say scalar gmtime ( 166560 * 30 + 1598306400 )'
// Wed Oct 21 18:00:00 2020
if dealInfo.Proposal.StartEpoch <= 166560 {
continue
}
projID, projKnown := knownAddrMap[clientAddr]
if !projKnown {
continue
}
grandTotals.seenProject[projID] = true
projStatEntry, ok := projStats[projID]
if !ok {
projStatEntry = &projectAggregateStats{
ProjectID: projID,
ClientStats: make(map[string]*clientAggregateStats),
cidDeals: make(map[cid.Cid]int),
dataPerProvider: make(map[address.Address]int64),
}
projStats[projID] = projStatEntry
}
if projStatEntry.cidDeals[dealInfo.Proposal.PieceCID] >= 10 {
continue
}
grandTotals.seenClient[clientAddr] = true
clientStatEntry, ok := projStatEntry.ClientStats[clientAddr.String()]
if !ok {
clientStatEntry = &clientAggregateStats{
Client: clientAddr.String(),
cids: make(map[cid.Cid]bool),
providers: make(map[address.Address]bool),
}
projStatEntry.ClientStats[clientAddr.String()] = clientStatEntry
}
grandTotals.TotalBytes += int64(dealInfo.Proposal.PieceSize)
projStatEntry.DataSize += int64(dealInfo.Proposal.PieceSize)
clientStatEntry.DataSize += int64(dealInfo.Proposal.PieceSize)
grandTotals.seenProvider[dealInfo.Proposal.Provider] = true
projStatEntry.dataPerProvider[dealInfo.Proposal.Provider] += int64(dealInfo.Proposal.PieceSize)
clientStatEntry.providers[dealInfo.Proposal.Provider] = true
grandTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true
projStatEntry.cidDeals[dealInfo.Proposal.PieceCID]++
clientStatEntry.cids[dealInfo.Proposal.PieceCID] = true
grandTotals.TotalDeals++
projStatEntry.NumDeals++
clientStatEntry.NumDeals++
if dealInfo.Proposal.VerifiedDeal {
grandTotals.FilplusTotalDeals++
grandTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize)
}
payloadCid := "unknown"
if c, err := cid.Parse(dealInfo.Proposal.Label); err == nil {
payloadCid = c.String()
}
projDealLists[projID] = append(projDealLists[projID], &individualDeal{
DealID: dealID,
ProjectID: projID,
Client: clientAddr.String(),
MinerID: dealInfo.Proposal.Provider.String(),
PayloadCID: payloadCid,
PaddedSize: int64(dealInfo.Proposal.PieceSize),
DealStartEpoch: int64(dealInfo.State.SectorStartEpoch),
})
}
//
// Write out per-project deal lists
for proj, dl := range projDealLists {
err := func() error {
outListFd, err := os.Create(fmt.Sprintf(outDirName+"/deals_list_%s.json", proj))
if err != nil {
return err
}
defer outListFd.Close() //nolint:errcheck
ridiculousLintMandatedRebind := dl
sort.Slice(dl, func(i, j int) bool {
return ridiculousLintMandatedRebind[j].PaddedSize < ridiculousLintMandatedRebind[i].PaddedSize
})
if err := json.NewEncoder(outListFd).Encode(
dealListOutput{
Epoch: int64(head.Height()),
Endpoint: "DEAL_LIST",
Payload: dl,
},
); err != nil {
return err
}
return nil
}()
if err != nil {
return err
}
}
//
// write out basic_stats.json and unfiltered_basic_stats.json
for _, st := range []*competitionTotal{&grandTotals, &unfilteredGrandTotals} {
st.UniqueCids = len(st.seenPieceCid)
st.UniqueClients = len(st.seenClient)
st.UniqueProviders = len(st.seenProvider)
if st.seenProject != nil {
st.UniqueProjects = len(st.seenProject)
}
}
if err := json.NewEncoder(outBasicStatsFd).Encode(
competitionTotalOutput{
Epoch: int64(head.Height()),
Endpoint: "COMPETITION_TOTALS",
Payload: grandTotals,
},
); err != nil {
return err
}
if err := json.NewEncoder(outUnfilteredStatsFd).Encode(
competitionTotalOutput{
Epoch: int64(head.Height()),
Endpoint: "NETWORK_WIDE_TOTALS",
Payload: unfilteredGrandTotals,
},
); err != nil {
return err
}
//
// write out client_stats.json
for _, ps := range projStats {
ps.NumCids = len(ps.cidDeals)
ps.NumProviders = len(ps.dataPerProvider)
for _, dealsForCid := range ps.cidDeals {
if ps.HighestCidDealCount < dealsForCid {
ps.HighestCidDealCount = dealsForCid
}
}
for _, dataForProvider := range ps.dataPerProvider {
if ps.DataSizeMaxProvider < dataForProvider {
ps.DataSizeMaxProvider = dataForProvider
}
}
for _, cs := range ps.ClientStats {
cs.NumCids = len(cs.cids)
cs.NumProviders = len(cs.providers)
}
}
if err := json.NewEncoder(outClientStatsFd).Encode(
projectAggregateStatsOutput{
Epoch: int64(head.Height()),
Endpoint: "PROJECT_DEAL_STATS",
Payload: projStats,
},
); err != nil {
return err
}
return nil
},
}

View File

@ -0,0 +1,114 @@
package main
import (
"encoding/json"
"os"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
)
// How many epochs back to look at for dealstats
var defaultEpochLookback = abi.ChainEpoch(10)
type networkTotalsOutput struct {
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload networkTotals `json:"payload"`
}
type networkTotals struct {
UniqueCids int `json:"total_unique_cids"`
UniqueProviders int `json:"total_unique_providers"`
UniqueClients int `json:"total_unique_clients"`
TotalDeals int `json:"total_num_deals"`
TotalBytes int64 `json:"total_stored_data_size"`
FilplusTotalDeals int `json:"filplus_total_num_deals"`
FilplusTotalBytes int64 `json:"filplus_total_stored_data_size"`
seenClient map[address.Address]bool
seenProvider map[address.Address]bool
seenPieceCid map[cid.Cid]bool
}
var storageStatsCmd = &cli.Command{
Name: "storage-stats",
Usage: "Translates current lotus state into a json summary suitable for driving https://storage.filecoin.io/",
Flags: []cli.Flag{
&cli.Int64Flag{
Name: "height",
},
},
Action: func(cctx *cli.Context) error {
ctx := lcli.ReqContext(cctx)
api, apiCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer apiCloser()
head, err := api.ChainHead(ctx)
if err != nil {
return err
}
requestedHeight := cctx.Int64("height")
if requestedHeight > 0 {
head, err = api.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(requestedHeight), head.Key())
} else {
head, err = api.ChainGetTipSetByHeight(ctx, head.Height()-defaultEpochLookback, head.Key())
}
if err != nil {
return err
}
netTotals := networkTotals{
seenClient: make(map[address.Address]bool),
seenProvider: make(map[address.Address]bool),
seenPieceCid: make(map[cid.Cid]bool),
}
deals, err := api.StateMarketDeals(ctx, head.Key())
if err != nil {
return err
}
for _, dealInfo := range deals {
// Only count deals that have properly started, not past/future ones
// https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
// Bail on 0 as well in case SectorStartEpoch is uninitialized due to some bug
if dealInfo.State.SectorStartEpoch <= 0 ||
dealInfo.State.SectorStartEpoch > head.Height() {
continue
}
netTotals.seenClient[dealInfo.Proposal.Client] = true
netTotals.TotalBytes += int64(dealInfo.Proposal.PieceSize)
netTotals.seenProvider[dealInfo.Proposal.Provider] = true
netTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true
netTotals.TotalDeals++
if dealInfo.Proposal.VerifiedDeal {
netTotals.FilplusTotalDeals++
netTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize)
}
}
netTotals.UniqueCids = len(netTotals.seenPieceCid)
netTotals.UniqueClients = len(netTotals.seenClient)
netTotals.UniqueProviders = len(netTotals.seenProvider)
return json.NewEncoder(os.Stdout).Encode(
networkTotalsOutput{
Epoch: int64(head.Height()),
Endpoint: "NETWORK_WIDE_TOTALS",
Payload: netTotals,
},
)
},
}

1
go.mod
View File

@ -8,7 +8,6 @@ require (
github.com/BurntSushi/toml v0.3.1
github.com/GeertJohan/go.rice v1.0.0
github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
github.com/Jeffail/gabs v1.4.0
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129

2
go.sum
View File

@ -40,8 +40,6 @@ github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voi
github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K172oDhSKU0dJ/miJramo9NITOMyZQ=
github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY=
github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=