From 7595b1d8d40157365f288a1db92df024df092e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 17 Aug 2020 11:48:12 +0200 Subject: [PATCH 1/2] cli: pprof goroutines command --- cli/cmd.go | 2 + cli/pprof.go | 59 +++++++++++++++++++++++++++++ cmd/lotus-storage-miner/info_all.go | 5 +++ 3 files changed, 66 insertions(+) create mode 100644 cli/pprof.go diff --git a/cli/cmd.go b/cli/cmd.go index 59a0d6c68..a201af2e9 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -238,6 +238,7 @@ var CommonCommands = []*cli.Command{ logCmd, waitApiCmd, fetchParamCmd, + pprofCmd, VersionCmd, } @@ -256,6 +257,7 @@ var Commands = []*cli.Command{ WithCategory("developer", fetchParamCmd), WithCategory("network", netCmd), WithCategory("network", syncCmd), + pprofCmd, VersionCmd, } diff --git a/cli/pprof.go b/cli/pprof.go new file mode 100644 index 000000000..dff089e7f --- /dev/null +++ b/cli/pprof.go @@ -0,0 +1,59 @@ +package cli + +import ( + "io" + "net/http" + "os" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/node/repo" + manet "github.com/multiformats/go-multiaddr-net" +) + +var pprofCmd = &cli.Command{ + Name: "pprof", + Hidden: true, + Subcommands: []*cli.Command{ + PprofGoroutines, + }, +} + +var PprofGoroutines = &cli.Command{ + Name: "goroutines", + Usage: "Get goroutine stacks", + Action: func(cctx *cli.Context) error { + ti, ok := cctx.App.Metadata["repoType"] + if !ok { + log.Errorf("unknown repo type, are you sure you want to use GetAPI?") + ti = repo.FullNode + } + t, ok := ti.(repo.RepoType) + if !ok { + log.Errorf("repoType type does not match the type of repo.RepoType") + } + ainfo, err := GetAPIInfo(cctx, t) + if err != nil { + return xerrors.Errorf("could not get API info: %w", err) + } + _, addr, err := manet.DialArgs(ainfo.Addr) + if err != nil { + return err + } + + addr = "http://" + addr + "/debug/pprof/goroutine?debug=2" + + r, err := http.Get(addr) + if err != nil { + return err + } + + if _, err := io.Copy(os.Stdout, r.Body); err != nil { + return err + } + + return r.Body.Close() + }, +} + diff --git a/cmd/lotus-storage-miner/info_all.go b/cmd/lotus-storage-miner/info_all.go index cc0cadbac..78b7cb0d5 100644 --- a/cmd/lotus-storage-miner/info_all.go +++ b/cmd/lotus-storage-miner/info_all.go @@ -148,6 +148,11 @@ var infoAllCmd = &cli.Command{ } } + fmt.Println("\n#: Goroutines") + if err := lcli.PprofGoroutines.Action(cctx); err != nil { + return err + } + return nil }, } From f31473c4cf667ff517421bb0d3cdef49b12d2499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 17 Aug 2020 17:34:56 +0200 Subject: [PATCH 2/2] Update jsonrpc to fix sealing sched hangs --- api/client/client.go | 5 +++-- cli/cmd.go | 4 ++-- cmd/lotus-seal-worker/main.go | 4 +++- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/api/client/client.go b/api/client/client.go index 02fb7c775..b55d6bca4 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -39,7 +39,7 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr } // NewStorageMinerRPC creates a new http jsonrpc client for miner -func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) { +func NewStorageMinerRPC(addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) { var res apistruct.StorageMinerStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ @@ -47,6 +47,7 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMine &res.Internal, }, requestHeader, + opts..., ) return &res, closer, err @@ -75,7 +76,7 @@ func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrp requestHeader, rpcenc.ReaderParamEncoder(u.String()), jsonrpc.WithNoReconnect(), - jsonrpc.WithWriteTimeout(30*time.Second), + jsonrpc.WithTimeout(30*time.Second), ) return &res, closer, err diff --git a/cli/cmd.go b/cli/cmd.go index a201af2e9..22b95089d 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -198,13 +198,13 @@ func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error return client.NewFullNodeRPC(addr, headers) } -func GetStorageMinerAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, error) { +func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) { addr, headers, err := GetRawAPI(ctx, repo.StorageMiner) if err != nil { return nil, nil, err } - return client.NewStorageMinerRPC(addr, headers) + return client.NewStorageMinerRPC(addr, headers, opts...) } func DaemonContext(cctx *cli.Context) context.Context { diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 128862e0d..2e6833921 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -164,7 +164,9 @@ var runCmd = &cli.Command{ var closer func() var err error for { - nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx) + nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, + jsonrpc.WithNoReconnect(), + jsonrpc.WithTimeout(30*time.Second)) if err == nil { break } diff --git a/go.mod b/go.mod index 8f4e9fc16..f0689f5d3 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/filecoin-project/go-data-transfer v0.6.1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f github.com/filecoin-project/go-fil-markets v0.5.6 - github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c + github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 diff --git a/go.sum b/go.sum index 2eab0cce0..4ce33885e 100644 --- a/go.sum +++ b/go.sum @@ -242,8 +242,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v0.5.6 h1:WmBbV0qBU4NvLJ64xROpzrKUbkZxZqszZiEiCGmCEIY= github.com/filecoin-project/go-fil-markets v0.5.6/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c h1:4HGPNCqGbTdhAdu/9yCbrTVpokWHWHATPBlML7Uhh98= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0 h1:/GT3V+3f+H5w5odb7LcCWJ1zPw8H8m9TsGQcU0cGSHo= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=