From eea9144ebe157f765ade1e3cb096e2a104b0e91c Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 19 Dec 2019 15:58:26 +0100 Subject: [PATCH 1/7] Use golog in stats tool, and better sync control --- tools/stats/main.go | 15 ++++---- tools/stats/metrics.go | 11 +++--- tools/stats/rpc.go | 77 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 80 insertions(+), 23 deletions(-) diff --git a/tools/stats/main.go b/tools/stats/main.go index c8d60de0c..7c0f12e97 100644 --- a/tools/stats/main.go +++ b/tools/stats/main.go @@ -3,11 +3,14 @@ package main import ( "context" "flag" - "log" "os" "time" + + logging "github.com/ipfs/go-log" ) +var log = logging.Logger("stats") + const ( INFLUX_ADDR = "INFLUX_ADDR" INFLUX_USER = "INFLUX_USER" @@ -47,7 +50,7 @@ func main() { if !reset && height == 0 { h, err := GetLastRecordedHeight(influx, database) if err != nil { - log.Print(err) + log.Info(err) } height = h @@ -76,17 +79,17 @@ func main() { height := tipset.Height() if err := RecordTipsetPoints(ctx, api, pl, tipset); err != nil { - log.Printf("Failed to record tipset at height %d: %w", height, err) + log.Warnw("Failed to record tipset", "height", height, "error", err) continue } if err := RecordTipsetMessagesPoints(ctx, api, pl, tipset); err != nil { - log.Printf("Failed to record messages at height %d: %w", height, err) + log.Warnw("Failed to record messages", "height", height, "error", err) continue } if err := RecordTipsetStatePoints(ctx, api, pl, tipset); err != nil { - log.Printf("Failed to record state at height %d: %w", height, err) + log.Warnw("Failed to record state", "height", height, "error", err) continue } @@ -108,7 +111,7 @@ func main() { nb.SetDatabase(database) - log.Printf("Writing %d points for height %d", len(nb.Points()), tipset.Height()) + log.Infow("Adding points", "count", len(nb.Points()), "height", tipset.Height()) wq.AddBatch(nb) } diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index d1e1b1e1c..25f7a440a 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "log" "math/big" "strings" "time" @@ -57,7 +56,7 @@ func NewInfluxWriteQueue(ctx context.Context, influx client.Client) *InfluxWrite case batch := <-ch: for i := 0; i < maxRetries; i++ { if err := influx.Write(batch); err != nil { - log.Printf("Failed to write batch: %w", err) + log.Warnw("Failed to write batch", "error", err) time.Sleep(time.Second * 15) continue } @@ -65,7 +64,7 @@ func NewInfluxWriteQueue(ctx context.Context, influx client.Client) *InfluxWrite continue main } - log.Printf("Dropping batch due to failure to write") + log.Error("Dropping batch due to failure to write") } } }() @@ -259,14 +258,14 @@ func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *Point } func ResetDatabase(influx client.Client, database string) error { - log.Print("Resetting database") + log.Info("Resetting database") q := client.NewQuery(fmt.Sprintf(`DROP DATABASE "%s"; CREATE DATABASE "%s";`, database, database), "", "") _, err := influx.Query(q) return err } func GetLastRecordedHeight(influx client.Client, database string) (int64, error) { - log.Print("Retrieving last record height") + log.Info("Retrieving last record height") q := client.NewQuery(`SELECT "value" FROM "chain.height" ORDER BY time DESC LIMIT 1`, database, "") res, err := influx.Query(q) if err != nil { @@ -286,7 +285,7 @@ func GetLastRecordedHeight(influx client.Client, database string) (int64, error) return 0, err } - log.Printf("Last record height %d", height) + log.Infow("Last record height", "height", height) return height, nil } diff --git a/tools/stats/rpc.go b/tools/stats/rpc.go index 12afd56e4..860ffc95f 100644 --- a/tools/stats/rpc.go +++ b/tools/stats/rpc.go @@ -2,7 +2,6 @@ package main import ( "context" - "log" "net/http" "time" @@ -13,6 +12,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/jsonrpc" @@ -36,7 +36,7 @@ func getAPI(path string) (string, http.Header, error) { var headers http.Header token, err := r.APIToken() if err != nil { - log.Printf("Couldn't load CLI token, capabilities may be limited: %w", err) + log.Warnw("Couldn't load CLI token, capabilities may be limited", "error", err) } else { headers = http.Header{} headers.Add("Authorization", "Bearer "+string(token)) @@ -46,19 +46,74 @@ func getAPI(path string) (string, http.Header, error) { } func WaitForSyncComplete(ctx context.Context, napi api.FullNode) error { +sync_complete: for { select { case <-ctx.Done(): return ctx.Err() - case <-time.After(3 * time.Second): + case <-time.After(5 * time.Second): + state, err := napi.SyncState(ctx) + if err != nil { + return err + } + + for i, w := range state.ActiveSyncs { + if w.Target == nil { + continue + } + + if w.Stage == api.StageSyncErrored { + log.Errorw( + "Syncing", + "worker", i, + "base", w.Base.Key(), + "target", w.Target.Key(), + "target_height", w.Target.Height(), + "height", w.Height, + "error", w.Message, + "stage", chain.SyncStageString(w.Stage), + ) + } else { + log.Infow( + "Syncing", + "worker", i, + "base", w.Base.Key(), + "target", w.Target.Key(), + "target_height", w.Target.Height(), + "height", w.Height, + "stage", chain.SyncStageString(w.Stage), + ) + } + + if w.Stage == api.StageSyncComplete { + break sync_complete + } + } + } + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(5 * time.Second): head, err := napi.ChainHead(ctx) if err != nil { return err } - log.Printf("Height %d", head.Height()) + timestampDelta := time.Now().Unix() - int64(head.MinTimestamp()) - if time.Now().Unix()-int64(head.MinTimestamp()) < build.BlockDelay { + log.Infow( + "Waiting for reasonable head height", + "height", head.Height(), + "timestamp_delta", timestampDelta, + ) + + // If we get within 20 blocks of the current exected block height we + // consider sync complete. Block propagation is not always great but we still + // want to be recording stats as soon as we can + if timestampDelta < build.BlockDelay*20 { return nil } } @@ -82,13 +137,13 @@ func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan * select { case changes := <-notif: for _, change := range changes { - log.Printf("Head event { height:%d; type: %s }", change.Val.Height(), change.Type) + log.Infow("Head event", "height", change.Val.Height(), "type", change.Type) switch change.Type { case store.HCCurrent: tipsets, err := loadTipsets(ctx, api, change.Val, lastHeight) if err != nil { - log.Print(err) + log.Info(err) return } @@ -100,17 +155,17 @@ func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan * } } case <-ping: - log.Print("Running health check") + log.Info("Running health check") cctx, cancel := context.WithTimeout(ctx, 5*time.Second) if _, err := api.ID(cctx); err != nil { - log.Print("Health check failed") + log.Error("Health check failed") return } cancel() - log.Print("Node online") + log.Info("Node online") case <-ctx.Done(): return } @@ -131,7 +186,7 @@ func loadTipsets(ctx context.Context, api api.FullNode, curr *types.TipSet, lowe break } - log.Printf("Walking back { height:%d }", curr.Height()) + log.Infow("Walking back", "height", curr.Height()) tipsets = append(tipsets, curr) tsk := curr.Parents() From a11e8093e57acb86ca1bc05a425cbde4079f7c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 7 Jan 2020 17:44:55 +0100 Subject: [PATCH 2/7] mpool: Return CID in MpoolPush --- api/api_full.go | 2 +- api/apistruct/struct.go | 4 ++-- build/version.go | 2 +- chain/messagepool/messagepool.go | 10 +++++----- node/impl/full/mpool.go | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 6e6796476..5325a1973 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -44,7 +44,7 @@ type FullNode interface { // messages MpoolPending(context.Context, *types.TipSet) ([]*types.SignedMessage, error) - MpoolPush(context.Context, *types.SignedMessage) error // TODO: remove + MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error) MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error) // get nonce, sign, push MpoolGetNonce(context.Context, address.Address) (uint64, error) MpoolSub(context.Context) (<-chan MpoolUpdate, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7a95f318c..ec160d1e9 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -60,7 +60,7 @@ type FullNodeStruct struct { SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"` MpoolPending func(context.Context, *types.TipSet) ([]*types.SignedMessage, error) `perm:"read"` - MpoolPush func(context.Context, *types.SignedMessage) error `perm:"write"` + MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"` MpoolPushMessage func(context.Context, *types.Message) (*types.SignedMessage, error) `perm:"sign"` MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"` @@ -232,7 +232,7 @@ func (c *FullNodeStruct) MpoolPending(ctx context.Context, ts *types.TipSet) ([] return c.Internal.MpoolPending(ctx, ts) } -func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) error { +func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { return c.Internal.MpoolPush(ctx, smsg) } diff --git a/build/version.go b/build/version.go index c93598684..99590904e 100644 --- a/build/version.go +++ b/build/version.go @@ -31,7 +31,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool { } // APIVersion is a semver version of the rpc api exposed -var APIVersion Version = newVer(0, 1, 5) +var APIVersion Version = newVer(0, 1, 6) const ( majorMask = 0xff0000 diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 94f495d0a..8bbf3823a 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -263,24 +263,24 @@ func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error { return nil } -func (mp *MessagePool) Push(m *types.SignedMessage) error { +func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) { msgb, err := m.Serialize() if err != nil { - return err + return cid.Undef, err } if err := mp.Add(m); err != nil { - return err + return cid.Undef, err } mp.lk.Lock() if err := mp.addLocal(m, msgb); err != nil { mp.lk.Unlock() - return err + return cid.Undef, err } mp.lk.Unlock() - return mp.api.PubSubPublish(msgTopic, msgb) + return m.Cid(), mp.api.PubSubPublish(msgTopic, msgb) } func (mp *MessagePool) Add(m *types.SignedMessage) error { diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 9f1f02e9a..cedb1785e 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -78,7 +78,7 @@ func (a *MpoolAPI) MpoolPending(ctx context.Context, ts *types.TipSet) ([]*types } } -func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) error { +func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { return a.Mpool.Push(smsg) } From 62db7a6ddad36bc1e653e066b518e4d410266de4 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 7 Jan 2020 11:52:17 -0800 Subject: [PATCH 3/7] fix build --- cmd/lotus-shed/nonce-fix.go | 2 +- go.sum | 11 ----------- node/impl/paych/paych.go | 4 ++-- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/cmd/lotus-shed/nonce-fix.go b/cmd/lotus-shed/nonce-fix.go index 67b34d7b7..4450c3c79 100644 --- a/cmd/lotus-shed/nonce-fix.go +++ b/cmd/lotus-shed/nonce-fix.go @@ -54,7 +54,7 @@ var noncefix = &cli.Command{ return err } - err = api.MpoolPush(ctx, smsg) + _, err = api.MpoolPush(ctx, smsg) if err != nil { return err } diff --git a/go.sum b/go.sum index 34d5667dc..3bd19a52e 100644 --- a/go.sum +++ b/go.sum @@ -7,7 +7,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -21,7 +20,6 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -281,7 +279,6 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -542,12 +539,10 @@ github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wS github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= -github.com/multiformats/go-varint v0.0.1 h1:TR/0rdQtnNxuN2IhiB639xC3tWM4IUi7DkTBVTdGW/M= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -624,9 +619,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= @@ -664,11 +657,9 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -678,7 +669,6 @@ go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY= go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -734,7 +724,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index d3811ec0c..92ffdd9f5 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -129,7 +129,7 @@ func (a *PaychAPI) PaychClose(ctx context.Context, addr address.Address) (cid.Ci return cid.Undef, err } - if err := a.MpoolPush(ctx, smsg); err != nil { + if _, err := a.MpoolPush(ctx, smsg); err != nil { return cid.Undef, err } @@ -244,7 +244,7 @@ func (a *PaychAPI) PaychVoucherSubmit(ctx context.Context, ch address.Address, s return cid.Undef, err } - if err := a.MpoolPush(ctx, smsg); err != nil { + if _, err := a.MpoolPush(ctx, smsg); err != nil { return cid.Undef, err } From 897cf7293320fcc2f502646e3a943f02ee6737a0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 20 Dec 2019 22:10:40 -0800 Subject: [PATCH 4/7] Add command to mark given blocks as bad --- api/api_full.go | 1 + api/apistruct/struct.go | 5 +++++ chain/sync.go | 4 ++++ cli/sync.go | 25 +++++++++++++++++++++++++ node/impl/full/sync.go | 8 ++++++++ 5 files changed, 43 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 6e6796476..6b3dfa6fd 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -41,6 +41,7 @@ type FullNode interface { SyncState(context.Context) (*SyncState, error) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) + SyncMarkBad(ctx context.Context, bcid cid.Cid) error // messages MpoolPending(context.Context, *types.TipSet) ([]*types.SignedMessage, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7a95f318c..1f7c97f95 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -58,6 +58,7 @@ type FullNodeStruct struct { SyncState func(context.Context) (*api.SyncState, error) `perm:"read"` SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"` + SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"write"` MpoolPending func(context.Context, *types.TipSet) ([]*types.SignedMessage, error) `perm:"read"` MpoolPush func(context.Context, *types.SignedMessage) error `perm:"write"` @@ -360,6 +361,10 @@ func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types. return c.Internal.SyncIncomingBlocks(ctx) } +func (c *FullNodeStruct) SyncMarkBad(ctx context.Context, bcid cid.Cid) error { + return c.Internal.SyncMarkBad(ctx, bcid) +} + func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, ts) } diff --git a/chain/sync.go b/chain/sync.go index 19d83df27..9065bc099 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1181,3 +1181,7 @@ func (syncer *Syncer) State() []SyncerState { } return out } + +func (syncer *Syncer) MarkBad(blk cid.Cid) { + syncer.bad.Add(blk) +} diff --git a/cli/sync.go b/cli/sync.go index 1b051769a..d3d425967 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -19,6 +19,7 @@ var syncCmd = &cli.Command{ Subcommands: []*cli.Command{ syncStatusCmd, syncWaitCmd, + syncMarkBadCmd, }, } @@ -90,6 +91,30 @@ var syncWaitCmd = &cli.Command{ }, } +var syncMarkBadCmd = &cli.Command{ + Name: "mark-bad", + Usage: "Mark the given block as bad, will prevent syncing to a chain that contains it", + Action: func(cctx *cli.Context) error { + napi, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if !cctx.Args().Present() { + return fmt.Errorf("must specify block cid to mark") + } + + bcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("failed to decode input as a cid: %s", err) + } + + return napi.SyncMarkBad(ctx, bcid) + }, +} + func SyncWait(ctx context.Context, napi api.FullNode) error { for { state, err := napi.SyncState(ctx) diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 0ebdad536..250837ab3 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -6,6 +6,8 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + "github.com/prometheus/common/log" pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" @@ -80,3 +82,9 @@ func (a *SyncAPI) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) erro func (a *SyncAPI) SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) { return a.Syncer.IncomingBlocks(ctx) } + +func (a *SyncAPI) SyncMarkBad(ctx context.Context, bcid cid.Cid) error { + log.Warnf("Marking block %s as bad", bcid) + a.Syncer.MarkBad(bcid) + return nil +} From 80c19df4bb948c3c45c36a61c4f87eb2076f91d9 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 7 Jan 2020 21:42:07 -0800 Subject: [PATCH 5/7] go mod tidy --- go.mod | 1 + go.sum | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3283f456b..e9e85962d 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a + github.com/prometheus/common v0.2.0 github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 diff --git a/go.sum b/go.sum index 34d5667dc..2a5f3c8fa 100644 --- a/go.sum +++ b/go.sum @@ -23,7 +23,9 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -293,6 +295,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= @@ -542,7 +545,6 @@ github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wS github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= -github.com/multiformats/go-varint v0.0.1 h1:TR/0rdQtnNxuN2IhiB639xC3tWM4IUi7DkTBVTdGW/M= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -584,6 +586,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -592,6 +595,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -664,11 +668,9 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -678,7 +680,6 @@ go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY= go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -734,7 +735,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -797,6 +797,7 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 4de01190a1c14aedde2bf418ccdefbd084b60500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jan 2020 13:34:53 +0100 Subject: [PATCH 6/7] api: Require admin perm for SyncMarkBad --- api/apistruct/struct.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1f7c97f95..09e686731 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -58,7 +58,7 @@ type FullNodeStruct struct { SyncState func(context.Context) (*api.SyncState, error) `perm:"read"` SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"` - SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"write"` + SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` MpoolPending func(context.Context, *types.TipSet) ([]*types.SignedMessage, error) `perm:"read"` MpoolPush func(context.Context, *types.SignedMessage) error `perm:"write"` From 95a89ad19253307789f94ad0761105c8641fb5ea Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 7 Jan 2020 11:03:11 -0800 Subject: [PATCH 7/7] implement list-messages command --- api/api_full.go | 2 ++ api/apistruct/struct.go | 56 +++++++++++++++++++------------- cli/state.go | 72 +++++++++++++++++++++++++++++++++++++++++ go.sum | 4 +++ node/impl/full/chain.go | 9 ++++++ node/impl/full/state.go | 49 ++++++++++++++++++++++++++++ 6 files changed, 169 insertions(+), 23 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 6b3dfa6fd..6532ca672 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -36,6 +36,7 @@ type FullNode interface { ChainGetGenesis(context.Context) (*types.TipSet, error) ChainTipSetWeight(context.Context, *types.TipSet) (types.BigInt, error) ChainGetNode(ctx context.Context, p string) (interface{}, error) + ChainGetMessage(context.Context, cid.Cid) (*types.Message, error) // syncer SyncState(context.Context) (*SyncState, error) @@ -96,6 +97,7 @@ type FullNode interface { StateReplay(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error) + StateListMessages(ctx context.Context, match *types.Message, ts *types.TipSet, toht uint64) ([]cid.Cid, error) StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 09e686731..188eec94b 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -54,6 +54,7 @@ type FullNodeStruct struct { ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"` ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` ChainGetNode func(ctx context.Context, p string) (interface{}, error) `perm:"read"` + ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"` SyncState func(context.Context) (*api.SyncState, error) `perm:"read"` SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` @@ -89,29 +90,30 @@ type FullNodeStruct struct { ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` - StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` - StateMinerPower func(context.Context, address.Address, *types.TipSet) (api.MinerPower, error) `perm:"read"` - StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` - StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` - StateMinerElectionPeriodStart func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"` - StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"` - StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` - StateReplay func(context.Context, *types.TipSet, cid.Cid) (*api.ReplayResults, error) `perm:"read"` - StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"` - StateReadState func(context.Context, *types.Actor, *types.TipSet) (*api.ActorState, error) `perm:"read"` - StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` - StateWaitMsg func(context.Context, cid.Cid) (*api.MsgWait, error) `perm:"read"` - StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` - StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` - StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` - StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"` - StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"` - StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"` - StateLookupID func(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) `perm:"read"` - StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` - StateGetReceipt func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` - StateMinerSectorCount func(context.Context, address.Address, *types.TipSet) (api.MinerSectors, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` + StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` + StateMinerPower func(context.Context, address.Address, *types.TipSet) (api.MinerPower, error) `perm:"read"` + StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` + StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` + StateMinerElectionPeriodStart func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"` + StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"` + StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` + StateReplay func(context.Context, *types.TipSet, cid.Cid) (*api.ReplayResults, error) `perm:"read"` + StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"` + StateReadState func(context.Context, *types.Actor, *types.TipSet) (*api.ActorState, error) `perm:"read"` + StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` + StateWaitMsg func(context.Context, cid.Cid) (*api.MsgWait, error) `perm:"read"` + StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` + StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` + StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` + StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"` + StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"` + StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"` + StateLookupID func(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) `perm:"read"` + StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` + StateGetReceipt func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` + StateMinerSectorCount func(context.Context, address.Address, *types.TipSet) (api.MinerSectors, error) `perm:"read"` + StateListMessages func(ctx context.Context, match *types.Message, ts *types.TipSet, toht uint64) ([]cid.Cid, error) `perm:"read"` MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"` @@ -349,6 +351,10 @@ func (c *FullNodeStruct) ChainGetNode(ctx context.Context, p string) (interface{ return c.Internal.ChainGetNode(ctx, p) } +func (c *FullNodeStruct) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { + return c.Internal.ChainGetMessage(ctx, mc) +} + func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) { return c.Internal.SyncState(ctx) } @@ -452,6 +458,10 @@ func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, ts *t return c.Internal.StateGetReceipt(ctx, msg, ts) } +func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Message, ts *types.TipSet, toht uint64) ([]cid.Cid, error) { + return c.Internal.StateListMessages(ctx, match, ts, toht) +} + func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { return c.Internal.MarketEnsureAvailable(ctx, addr, amt) } diff --git a/cli/state.go b/cli/state.go index 2f13fc3f1..36c9ffa1b 100644 --- a/cli/state.go +++ b/cli/state.go @@ -36,6 +36,7 @@ var stateCmd = &cli.Command{ stateReplaySetCmd, stateSectorSizeCmd, stateReadStateCmd, + stateListMessagesCmd, }, } @@ -508,3 +509,74 @@ var stateReadStateCmd = &cli.Command{ return nil }, } + +var stateListMessagesCmd = &cli.Command{ + Name: "list-messages", + Usage: "list messages on chain matching given criteria", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "to", + Usage: "return messages to a given address", + }, + &cli.StringFlag{ + Name: "from", + Usage: "return messages from a given address", + }, + &cli.Uint64Flag{ + Name: "toheight", + Usage: "don't look before given block height", + }, + }, + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + var toa, froma address.Address + if tos := cctx.String("to"); tos != "" { + a, err := address.NewFromString(tos) + if err != nil { + return fmt.Errorf("given 'to' address %q was invalid: %w", tos, err) + } + toa = a + } + + if froms := cctx.String("from"); froms != "" { + a, err := address.NewFromString(froms) + if err != nil { + return fmt.Errorf("given 'from' address %q was invalid: %w", froms, err) + } + froma = a + } + + toh := cctx.Uint64("toheight") + + ts, err := loadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + msgs, err := api.StateListMessages(ctx, &types.Message{To: toa, From: froma}, ts, toh) + if err != nil { + return err + } + + for _, c := range msgs { + m, err := api.ChainGetMessage(ctx, c) + if err != nil { + return err + } + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + } + + return nil + }, +} diff --git a/go.sum b/go.sum index 2a5f3c8fa..10ab358fb 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,10 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 702f0a32a..b8622421b 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -299,3 +299,12 @@ func (a *ChainAPI) ChainGetNode(ctx context.Context, p string) (interface{}, err return node, nil } + +func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { + cm, err := a.Chain.GetCMessage(mc) + if err != nil { + return nil, err + } + + return cm.VMMessage(), nil +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 0a5f915e0..2d0835f68 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -351,3 +351,52 @@ func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid. func (a *StateAPI) StateMinerSectorCount(ctx context.Context, addr address.Address, ts *types.TipSet) (api.MinerSectors, error) { return stmgr.SectorSetSizes(ctx, a.StateManager, addr, ts) } + +func (a *StateAPI) StateListMessages(ctx context.Context, match *types.Message, ts *types.TipSet, toheight uint64) ([]cid.Cid, error) { + if ts == nil { + ts = a.Chain.GetHeaviestTipSet() + } + + if match.To == address.Undef && match.From == address.Undef { + return nil, xerrors.Errorf("must specify at least To or From in message filter") + } + + matchFunc := func(msg *types.Message) bool { + if match.From != address.Undef && match.From != msg.From { + return false + } + + if match.To != address.Undef && match.To != msg.To { + return false + } + + return true + } + + var out []cid.Cid + for ts.Height() >= toheight { + msgs, err := a.Chain.MessagesForTipset(ts) + if err != nil { + return nil, xerrors.Errorf("failed to get messages for tipset (%s): %w", ts.Key(), err) + } + + for _, msg := range msgs { + if matchFunc(msg.VMMessage()) { + out = append(out, msg.Cid()) + } + } + + if ts.Height() == 0 { + break + } + + next, err := a.Chain.LoadTipSet(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading next tipset: %w", err) + } + + ts = next + } + + return out, nil +}