From a3e26233b2ea4f5534ce34236c67193ad84b60f6 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 12 Oct 2020 23:00:29 -0400 Subject: [PATCH 01/11] Add postfind --- cmd/lotus-shed/postfind.go | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 cmd/lotus-shed/postfind.go diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go new file mode 100644 index 000000000..91377b734 --- /dev/null +++ b/cmd/lotus-shed/postfind.go @@ -0,0 +1,76 @@ +package main + +import ( + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/urfave/cli/v2" +) + +var postFindCmd = &cli.Command{ + Name: "post-find", + Description: "return addresses of all miners who have posted in the last day", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tipset", + Usage: "specify tipset state to search on", + }, + }, + Action: func(c *cli.Context) error { + api, acloser, err := lcli.GetFullNodeAPI(c) + if err != nil { + return err + } + defer acloser() + ctx := lcli.ReqContext(c) + + ts, err := lcli.LoadTipSet(ctx, c, api) + if err != nil { + return err + } + if ts == nil { + ts, err = api.ChainHead(ctx) + if err != nil { + return err + } + } + oneDayAgo := ts.Height() - abi.ChainEpoch(2880) + + mAddrs, err := api.StateListMiners(ctx, ts.Key()) + if err != nil { + return err + } + + for _, mAddr := range mAddrs { + // if they have no power ignore. This filters out 14k inactive miners + // so we can do 100x fewer expensive message queries + power, err := api.StateMinerPower(ctx, mAddr, ts.Key()) + if err != nil { + return err + } + if !power.HasMinPower { + continue + } + query := &types.Message{To: mAddr} + mCids, err := api.StateListMessages(ctx, query, ts.Key(), oneDayAgo) + if err != nil { + return err + } + for _, mCid := range mCids { + msg, err := api.ChainGetMessage(ctx, mCid) + if err != nil { + return err + } + if msg.Method == builtin.MethodsMiner.SubmitWindowedPoSt { + fmt.Printf("%s\n", mAddr) + break // go to next mAddr + } + } + } + + return nil + }, +} From 79d94c477183ef3e5ebbf347487b9072737c2e08 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 12 Oct 2020 23:09:40 -0400 Subject: [PATCH 02/11] Hook up cmd --- cmd/lotus-shed/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 29a105355..991cccf71 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -28,6 +28,7 @@ func main() { importObjectCmd, commpToCidCmd, fetchParamCmd, + postFindCmd, proofsCmd, verifRegCmd, miscCmd, From 768d2db114e52cc8cc55c3a351a62695b6d4b563 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 12 Oct 2020 23:53:14 -0400 Subject: [PATCH 03/11] Tipset traversal for speed --- cmd/lotus-shed/postfind.go | 60 ++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index 91377b734..f4d9967a0 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -3,7 +3,10 @@ package main import ( "fmt" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/specs-actors/v2/actors/builtin" @@ -12,7 +15,7 @@ import ( var postFindCmd = &cli.Command{ Name: "post-find", - Description: "return addresses of all miners who have posted in the last day", + Description: "return addresses of all miners who have over zero power and have posted in the last day", Flags: []cli.Flag{ &cli.StringFlag{ Name: "tipset", @@ -39,11 +42,30 @@ var postFindCmd = &cli.Command{ } oneDayAgo := ts.Height() - abi.ChainEpoch(2880) + // Get all messages over the last day + msgs := make([]*types.Message, 0) + for ts.Height() > oneDayAgo { + // Get messages on ts parent + next, err := api.ChainGetParentMessages(ctx, ts.Cids()[0]) + if err != nil { + return err + } + msgs = append(msgs, messagesFromAPIMessages(next)...) + + // Next ts + ts, err = api.ChainGetTipSet(ctx, ts.Parents()) + if err != nil { + return err + } + } + fmt.Printf("Loaded messages to height %d\n", ts.Height()) + mAddrs, err := api.StateListMiners(ctx, ts.Key()) if err != nil { return err } + minersWithPower := make(map[address.Address]struct{}) for _, mAddr := range mAddrs { // if they have no power ignore. This filters out 14k inactive miners // so we can do 100x fewer expensive message queries @@ -51,26 +73,32 @@ var postFindCmd = &cli.Command{ if err != nil { return err } - if !power.HasMinPower { - continue + if power.MinerPower.RawBytePower.GreaterThan(big.Zero()) { + minersWithPower[mAddr] = struct{}{} } - query := &types.Message{To: mAddr} - mCids, err := api.StateListMessages(ctx, query, ts.Key(), oneDayAgo) - if err != nil { - return err - } - for _, mCid := range mCids { - msg, err := api.ChainGetMessage(ctx, mCid) - if err != nil { - return err - } + } + fmt.Printf("Loaded %d miners with power\n", len(minersWithPower)) + + postedMiners := make(map[address.Address]struct{}) + for _, msg := range msgs { + _, hasPower := minersWithPower[msg.To] + _, seenBefore := postedMiners[msg.To] + + if hasPower && !seenBefore { if msg.Method == builtin.MethodsMiner.SubmitWindowedPoSt { - fmt.Printf("%s\n", mAddr) - break // go to next mAddr + fmt.Printf("%s\n", msg.To) + postedMiners[msg.To] = struct{}{} } } } - return nil }, } + +func messagesFromAPIMessages(apiMessages []lapi.Message) []*types.Message { + messages := make([]*types.Message, len(apiMessages)) + for i, apiMessage := range apiMessages { + messages[i] = apiMessage.Message + } + return messages +} From c9955c06351d6cb2cb88de86285d93e5ca796c11 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 12 Oct 2020 23:59:30 -0400 Subject: [PATCH 04/11] verbose flag --- cmd/lotus-shed/postfind.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index f4d9967a0..6fea6a69e 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -21,6 +21,10 @@ var postFindCmd = &cli.Command{ Name: "tipset", Usage: "specify tipset state to search on", }, + &cli.BoolFlag{ + Name: "verbose", + Usage: "get more frequent print updates", + }, }, Action: func(c *cli.Context) error { api, acloser, err := lcli.GetFullNodeAPI(c) @@ -29,6 +33,7 @@ var postFindCmd = &cli.Command{ } defer acloser() ctx := lcli.ReqContext(c) + verbose := c.Bool("verbose") ts, err := lcli.LoadTipSet(ctx, c, api) if err != nil { @@ -41,7 +46,9 @@ var postFindCmd = &cli.Command{ } } oneDayAgo := ts.Height() - abi.ChainEpoch(2880) - + if verbose { + fmt.Printf("Collecting messages between %d and %d\n", ts.Height, oneDayAgo) + } // Get all messages over the last day msgs := make([]*types.Message, 0) for ts.Height() > oneDayAgo { @@ -57,6 +64,9 @@ var postFindCmd = &cli.Command{ if err != nil { return err } + if verbose && int64(ts.Height())%100 == 0 { + fmt.Printf("Collected messages back to height %d\n", ts.Height()) + } } fmt.Printf("Loaded messages to height %d\n", ts.Height()) @@ -80,7 +90,7 @@ var postFindCmd = &cli.Command{ fmt.Printf("Loaded %d miners with power\n", len(minersWithPower)) postedMiners := make(map[address.Address]struct{}) - for _, msg := range msgs { + for i, msg := range msgs { _, hasPower := minersWithPower[msg.To] _, seenBefore := postedMiners[msg.To] @@ -90,6 +100,9 @@ var postFindCmd = &cli.Command{ postedMiners[msg.To] = struct{}{} } } + if verbose && int64(i)%1000 == 0 { + fmt.Printf("Searched through message %d of %d\n", i, len(msgs)) + } } return nil }, From 448ca6e43c9252af6e06a9516a8f845a85b4e7c8 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 13 Oct 2020 00:16:51 -0400 Subject: [PATCH 05/11] less verbose --- cmd/lotus-shed/postfind.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index 6fea6a69e..2ac5af730 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -90,7 +90,7 @@ var postFindCmd = &cli.Command{ fmt.Printf("Loaded %d miners with power\n", len(minersWithPower)) postedMiners := make(map[address.Address]struct{}) - for i, msg := range msgs { + for _, msg := range msgs { _, hasPower := minersWithPower[msg.To] _, seenBefore := postedMiners[msg.To] @@ -100,9 +100,6 @@ var postFindCmd = &cli.Command{ postedMiners[msg.To] = struct{}{} } } - if verbose && int64(i)%1000 == 0 { - fmt.Printf("Searched through message %d of %d\n", i, len(msgs)) - } } return nil }, From 0b9499a00ffc67107c1197b6d5397fadef88a9c6 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 13 Oct 2020 00:29:43 -0400 Subject: [PATCH 06/11] withpower flag --- cmd/lotus-shed/postfind.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index 2ac5af730..d252166b7 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -25,6 +25,10 @@ var postFindCmd = &cli.Command{ Name: "verbose", Usage: "get more frequent print updates", }, + &cli.BoolFlag{ + Name: "withpower", + Usage: "only print addrs of miners with more than zero power", + }, }, Action: func(c *cli.Context) error { api, acloser, err := lcli.GetFullNodeAPI(c) @@ -34,6 +38,7 @@ var postFindCmd = &cli.Command{ defer acloser() ctx := lcli.ReqContext(c) verbose := c.Bool("verbose") + withpower := c.Bool("withpower") ts, err := lcli.LoadTipSet(ctx, c, api) if err != nil { @@ -47,7 +52,7 @@ var postFindCmd = &cli.Command{ } oneDayAgo := ts.Height() - abi.ChainEpoch(2880) if verbose { - fmt.Printf("Collecting messages between %d and %d\n", ts.Height, oneDayAgo) + fmt.Printf("Collecting messages between %d and %d\n", ts.Height(), oneDayAgo) } // Get all messages over the last day msgs := make([]*types.Message, 0) @@ -75,29 +80,33 @@ var postFindCmd = &cli.Command{ return err } - minersWithPower := make(map[address.Address]struct{}) + minersToCheck := make(map[address.Address]struct{}) for _, mAddr := range mAddrs { // if they have no power ignore. This filters out 14k inactive miners // so we can do 100x fewer expensive message queries - power, err := api.StateMinerPower(ctx, mAddr, ts.Key()) - if err != nil { - return err - } - if power.MinerPower.RawBytePower.GreaterThan(big.Zero()) { - minersWithPower[mAddr] = struct{}{} + if withpower { + power, err := api.StateMinerPower(ctx, mAddr, ts.Key()) + if err != nil { + return err + } + if power.MinerPower.RawBytePower.GreaterThan(big.Zero()) { + minersToCheck[mAddr] = struct{}{} + } + } else { + minersToCheck[mAddr] = struct{}{} } } - fmt.Printf("Loaded %d miners with power\n", len(minersWithPower)) + fmt.Printf("Loaded %d miners with power\n", len(minersToCheck)) postedMiners := make(map[address.Address]struct{}) for _, msg := range msgs { - _, hasPower := minersWithPower[msg.To] + _, hasPower := minersToCheck[msg.To] _, seenBefore := postedMiners[msg.To] if hasPower && !seenBefore { if msg.Method == builtin.MethodsMiner.SubmitWindowedPoSt { fmt.Printf("%s\n", msg.To) - postedMiners[msg.To] = struct{}{} + minersToCheck[msg.To] = struct{}{} } } } From 546614455f8ff43729d053de0ebb074576325a1b Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 13 Oct 2020 00:38:15 -0400 Subject: [PATCH 07/11] Fix print --- cmd/lotus-shed/postfind.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index d252166b7..fd309445d 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -96,7 +96,7 @@ var postFindCmd = &cli.Command{ minersToCheck[mAddr] = struct{}{} } } - fmt.Printf("Loaded %d miners with power\n", len(minersToCheck)) + fmt.Printf("Loaded %d miners to check\n", len(minersToCheck)) postedMiners := make(map[address.Address]struct{}) for _, msg := range msgs { From 0e88f32d6628010c42bc7f96355523cd5eb769b4 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 13 Oct 2020 00:47:43 -0400 Subject: [PATCH 08/11] Fix bug and add flag --- cmd/lotus-shed/postfind.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index fd309445d..ffdfb4946 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -29,6 +29,11 @@ var postFindCmd = &cli.Command{ Name: "withpower", Usage: "only print addrs of miners with more than zero power", }, + &cli.IntFlag{ + Name: "lookback", + Usage: "number of past epochs to search for post", + Value: 2880, //default 1 day + } }, Action: func(c *cli.Context) error { api, acloser, err := lcli.GetFullNodeAPI(c) @@ -100,13 +105,13 @@ var postFindCmd = &cli.Command{ postedMiners := make(map[address.Address]struct{}) for _, msg := range msgs { - _, hasPower := minersToCheck[msg.To] + _, shouldCheck := minersToCheck[msg.To] _, seenBefore := postedMiners[msg.To] - if hasPower && !seenBefore { + if shouldCheck && !seenBefore { if msg.Method == builtin.MethodsMiner.SubmitWindowedPoSt { fmt.Printf("%s\n", msg.To) - minersToCheck[msg.To] = struct{}{} + postedMiners[msg.To] = struct{}{} } } } From 619ee1e6f9b91e9df3e6f303d40a62b9f442c58f Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 13 Oct 2020 00:49:32 -0400 Subject: [PATCH 09/11] add comma --- cmd/lotus-shed/postfind.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index ffdfb4946..af6b6517d 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -30,10 +30,10 @@ var postFindCmd = &cli.Command{ Usage: "only print addrs of miners with more than zero power", }, &cli.IntFlag{ - Name: "lookback", + Name: "lookback", Usage: "number of past epochs to search for post", Value: 2880, //default 1 day - } + }, }, Action: func(c *cli.Context) error { api, acloser, err := lcli.GetFullNodeAPI(c) From 20c57fcd0e95034e94bd24d080bab884cebb8d04 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 13 Oct 2020 19:00:37 +0000 Subject: [PATCH 10/11] lotus-shed: post-find use lookback --- cmd/lotus-shed/postfind.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index af6b6517d..57a163c42 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -55,7 +55,7 @@ var postFindCmd = &cli.Command{ return err } } - oneDayAgo := ts.Height() - abi.ChainEpoch(2880) + oneDayAgo := ts.Height() - abi.ChainEpoch(c.Int("lookback")) if verbose { fmt.Printf("Collecting messages between %d and %d\n", ts.Height(), oneDayAgo) } From 2613e837a9411f0c2bfca6f133acd8cdad89d60f Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 14 Oct 2020 03:01:41 -0400 Subject: [PATCH 11/11] Bugfix: Eligible miners should be computed based on start tipset, not end tipset --- cmd/lotus-shed/postfind.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/lotus-shed/postfind.go b/cmd/lotus-shed/postfind.go index 57a163c42..83006fd09 100644 --- a/cmd/lotus-shed/postfind.go +++ b/cmd/lotus-shed/postfind.go @@ -45,23 +45,24 @@ var postFindCmd = &cli.Command{ verbose := c.Bool("verbose") withpower := c.Bool("withpower") - ts, err := lcli.LoadTipSet(ctx, c, api) + startTs, err := lcli.LoadTipSet(ctx, c, api) if err != nil { return err } - if ts == nil { - ts, err = api.ChainHead(ctx) + if startTs == nil { + startTs, err = api.ChainHead(ctx) if err != nil { return err } } - oneDayAgo := ts.Height() - abi.ChainEpoch(c.Int("lookback")) + stopEpoch := startTs.Height() - abi.ChainEpoch(c.Int("lookback")) if verbose { - fmt.Printf("Collecting messages between %d and %d\n", ts.Height(), oneDayAgo) + fmt.Printf("Collecting messages between %d and %d\n", startTs.Height(), stopEpoch) } // Get all messages over the last day + ts := startTs msgs := make([]*types.Message, 0) - for ts.Height() > oneDayAgo { + for ts.Height() > stopEpoch { // Get messages on ts parent next, err := api.ChainGetParentMessages(ctx, ts.Cids()[0]) if err != nil { @@ -80,7 +81,7 @@ var postFindCmd = &cli.Command{ } fmt.Printf("Loaded messages to height %d\n", ts.Height()) - mAddrs, err := api.StateListMiners(ctx, ts.Key()) + mAddrs, err := api.StateListMiners(ctx, startTs.Key()) if err != nil { return err } @@ -90,7 +91,7 @@ var postFindCmd = &cli.Command{ // if they have no power ignore. This filters out 14k inactive miners // so we can do 100x fewer expensive message queries if withpower { - power, err := api.StateMinerPower(ctx, mAddr, ts.Key()) + power, err := api.StateMinerPower(ctx, mAddr, startTs.Key()) if err != nil { return err }