From 1e6aa4b1297ea06e36592ec3cda731f055215311 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Thu, 3 Dec 2020 19:11:31 +0000 Subject: [PATCH 1/4] Only count phase2 deals ( start epoch 166560 ) --- cmd/lotus-shed/sr2-dealstats-rollup.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/lotus-shed/sr2-dealstats-rollup.go b/cmd/lotus-shed/sr2-dealstats-rollup.go index e0673013e..d4d8a63c9 100644 --- a/cmd/lotus-shed/sr2-dealstats-rollup.go +++ b/cmd/lotus-shed/sr2-dealstats-rollup.go @@ -269,6 +269,12 @@ var rollupDealStatsCmd = &cli.Command{ unfilteredGrandTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true unfilteredGrandTotals.TotalDeals++ + // perl -E 'say scalar gmtime ( 166560 * 30 + 1598306400 )' + // Wed Oct 21 18:00:00 2020 + if dealInfo.Proposal.StartEpoch <= 166560 { + continue + } + projID, projKnown := knownAddrMap[clientAddr] if !projKnown { continue From 88bd7570249bb7bbf32c012fe7b05d4f830d170a Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Thu, 3 Dec 2020 19:11:59 +0000 Subject: [PATCH 2/4] Only count the first 10 copies of the same piece per project --- cmd/lotus-shed/sr2-dealstats-rollup.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/sr2-dealstats-rollup.go b/cmd/lotus-shed/sr2-dealstats-rollup.go index d4d8a63c9..bba1d71c6 100644 --- a/cmd/lotus-shed/sr2-dealstats-rollup.go +++ b/cmd/lotus-shed/sr2-dealstats-rollup.go @@ -281,8 +281,6 @@ var rollupDealStatsCmd = &cli.Command{ } grandTotals.seenProject[projID] = true - grandTotals.seenClient[clientAddr] = true - projStatEntry, ok := projStats[projID] if !ok { projStatEntry = &projectAggregateStats{ @@ -294,6 +292,11 @@ var rollupDealStatsCmd = &cli.Command{ projStats[projID] = projStatEntry } + if projStatEntry.cidDeals[dealInfo.Proposal.PieceCID] >= 10 { + continue + } + + grandTotals.seenClient[clientAddr] = true clientStatEntry, ok := projStatEntry.ClientStats[clientAddr.String()] if !ok { clientStatEntry = &clientAggregateStats{ From 2514c96cffc982ec1f46b85c83d2cbef4d08a8fa Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Thu, 3 Dec 2020 19:15:55 +0000 Subject: [PATCH 3/4] Count deals from sealed sectors only --- cmd/lotus-shed/sr2-dealstats-rollup.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/sr2-dealstats-rollup.go b/cmd/lotus-shed/sr2-dealstats-rollup.go index bba1d71c6..9959fb1e0 100644 --- a/cmd/lotus-shed/sr2-dealstats-rollup.go +++ b/cmd/lotus-shed/sr2-dealstats-rollup.go @@ -245,11 +245,13 @@ var rollupDealStatsCmd = &cli.Command{ for dealID, dealInfo := range deals { - // Counting no-longer-active deals as per Pooja's request - // // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85 - // if d.State.SectorStartEpoch < 0 { - // continue - // } + // Only count deals that have properly started, not past/future ones + // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85 + // Bail on 0 as well in case SectorStartEpoch is uninitialized due to some bug + if dealInfo.State.SectorStartEpoch <= 0 || + dealInfo.State.SectorStartEpoch > head.Height() { + continue + } clientAddr, found := resolvedWallets[dealInfo.Proposal.Client] if !found { From d9e3cc03749f1145f66878f704aedaa746e06fd5 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Fri, 4 Dec 2020 17:59:47 +0000 Subject: [PATCH 4/4] Add an extra filplus counter --- cmd/lotus-shed/sr2-dealstats-rollup.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-shed/sr2-dealstats-rollup.go b/cmd/lotus-shed/sr2-dealstats-rollup.go index 9959fb1e0..a2c6d03d9 100644 --- a/cmd/lotus-shed/sr2-dealstats-rollup.go +++ b/cmd/lotus-shed/sr2-dealstats-rollup.go @@ -34,12 +34,14 @@ type competitionTotalOutput struct { Payload competitionTotal `json:"payload"` } type competitionTotal struct { - UniqueCids int `json:"total_unique_cids"` - UniqueProviders int `json:"total_unique_providers"` - UniqueProjects int `json:"total_unique_projects"` - UniqueClients int `json:"total_unique_clients"` - TotalDeals int `json:"total_num_deals"` - TotalBytes int64 `json:"total_stored_data_size"` + UniqueCids int `json:"total_unique_cids"` + UniqueProviders int `json:"total_unique_providers"` + UniqueProjects int `json:"total_unique_projects"` + UniqueClients int `json:"total_unique_clients"` + TotalDeals int `json:"total_num_deals"` + TotalBytes int64 `json:"total_stored_data_size"` + FilplusTotalDeals int `json:"filplus_total_num_deals"` + FilplusTotalBytes int64 `json:"filplus_total_stored_data_size"` seenProject map[string]bool seenClient map[address.Address]bool @@ -271,6 +273,11 @@ var rollupDealStatsCmd = &cli.Command{ unfilteredGrandTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true unfilteredGrandTotals.TotalDeals++ + if dealInfo.Proposal.VerifiedDeal { + unfilteredGrandTotals.FilplusTotalDeals++ + unfilteredGrandTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize) + } + // perl -E 'say scalar gmtime ( 166560 * 30 + 1598306400 )' // Wed Oct 21 18:00:00 2020 if dealInfo.Proposal.StartEpoch <= 166560 { @@ -325,6 +332,11 @@ var rollupDealStatsCmd = &cli.Command{ projStatEntry.NumDeals++ clientStatEntry.NumDeals++ + if dealInfo.Proposal.VerifiedDeal { + grandTotals.FilplusTotalDeals++ + grandTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize) + } + payloadCid := "unknown" if c, err := cid.Parse(dealInfo.Proposal.Label); err == nil { payloadCid = c.String()