cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #3112 from filecoin-project/fix/sched-hang

Browse Source 
Fix sealing sched hands on unclean worker disconnect
This commit is contained in:
Łukasz Magiera 2020-08-17 18:43:25 +02:00 committed by GitHub
commit 29db7047a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 77 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/client/client.go
View File

@ -39,7 +39,7 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr
}
// NewStorageMinerRPC creates a new http jsonrpc client for miner
func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
func NewStorageMinerRPC(addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
var res apistruct.StorageMinerStruct
closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
[]interface{}{
@ -47,6 +47,7 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMine
&res.Internal,
},
requestHeader,
opts...,
)
return &res, closer, err
@ -75,7 +76,7 @@ func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrp
requestHeader,
rpcenc.ReaderParamEncoder(u.String()),
jsonrpc.WithNoReconnect(),
jsonrpc.WithWriteTimeout(30*time.Second),
jsonrpc.WithTimeout(30*time.Second),
)
return &res, closer, err

6
cli/cmd.go
View File

@ -198,13 +198,13 @@ func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error
return client.NewFullNodeRPC(addr, headers)
}
func GetStorageMinerAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, error) {
func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
if err != nil {
return nil, nil, err
}
return client.NewStorageMinerRPC(addr, headers)
return client.NewStorageMinerRPC(addr, headers, opts...)
}
func DaemonContext(cctx *cli.Context) context.Context {
@ -238,6 +238,7 @@ var CommonCommands = []*cli.Command{
logCmd,
waitApiCmd,
fetchParamCmd,
pprofCmd,
VersionCmd,
}
@ -256,6 +257,7 @@ var Commands = []*cli.Command{
WithCategory("developer", fetchParamCmd),
WithCategory("network", netCmd),
WithCategory("network", syncCmd),
pprofCmd,
VersionCmd,
}

59
cli/pprof.go Normal file
View File

@ -0,0 +1,59 @@
package cli
import (
"io"
"net/http"
"os"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/node/repo"
manet "github.com/multiformats/go-multiaddr-net"
)
var pprofCmd = &cli.Command{
Name: "pprof",
Hidden: true,
Subcommands: []*cli.Command{
PprofGoroutines,
},
}
var PprofGoroutines = &cli.Command{
Name: "goroutines",
Usage: "Get goroutine stacks",
Action: func(cctx *cli.Context) error {
ti, ok := cctx.App.Metadata["repoType"]
if !ok {
log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
ti = repo.FullNode
}
t, ok := ti.(repo.RepoType)
if !ok {
log.Errorf("repoType type does not match the type of repo.RepoType")
}
ainfo, err := GetAPIInfo(cctx, t)
if err != nil {
return xerrors.Errorf("could not get API info: %w", err)
}
_, addr, err := manet.DialArgs(ainfo.Addr)
if err != nil {
return err
}
addr = "http://" + addr + "/debug/pprof/goroutine?debug=2"
r, err := http.Get(addr)
if err != nil {
return err
}
if _, err := io.Copy(os.Stdout, r.Body); err != nil {
return err
}
return r.Body.Close()
},
}

4
cmd/lotus-seal-worker/main.go
View File

@ -164,7 +164,9 @@ var runCmd = &cli.Command{
var closer func()
var err error
for {
nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx)
nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx,
jsonrpc.WithNoReconnect(),
jsonrpc.WithTimeout(30*time.Second))
if err == nil {
break
}

5
cmd/lotus-storage-miner/info_all.go
View File

@ -148,6 +148,11 @@ var infoAllCmd = &cli.Command{
}
}
fmt.Println("\n#: Goroutines")
if err := lcli.PprofGoroutines.Action(cctx); err != nil {
return err
}
return nil
},
}

2
go.mod
View File

@ -30,7 +30,7 @@ require (
github.com/filecoin-project/go-data-transfer v0.6.1
github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
github.com/filecoin-project/go-fil-markets v0.5.6
github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c
github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0
github.com/filecoin-project/go-multistore v0.0.3
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261

4
go.sum
View File

@ -245,8 +245,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1
github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
github.com/filecoin-project/go-fil-markets v0.5.6 h1:WmBbV0qBU4NvLJ64xROpzrKUbkZxZqszZiEiCGmCEIY=
github.com/filecoin-project/go-fil-markets v0.5.6/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E=
github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c h1:4HGPNCqGbTdhAdu/9yCbrTVpokWHWHATPBlML7Uhh98=
github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0 h1:/GT3V+3f+H5w5odb7LcCWJ1zPw8H8m9TsGQcU0cGSHo=
github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI=
github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ=
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=