Merge pull request #3112 from filecoin-project/fix/sched-hang

Fix sealing sched hands on unclean worker disconnect

api/client/client.go

@@ -39,7 +39,7 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr
 }
 
 // NewStorageMinerRPC creates a new http jsonrpc client for miner
-func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+func NewStorageMinerRPC(addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
 	var res apistruct.StorageMinerStruct
 	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
@@ -47,6 +47,7 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMine
 			&res.Internal,
 		},
 		requestHeader,
+		opts...,
 	)
 
 	return &res, closer, err
@@ -75,7 +76,7 @@ func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrp
 		requestHeader,
 		rpcenc.ReaderParamEncoder(u.String()),
 		jsonrpc.WithNoReconnect(),
-		jsonrpc.WithWriteTimeout(30*time.Second),
+		jsonrpc.WithTimeout(30*time.Second),
 	)
 
 	return &res, closer, err

cli/cmd.go

@@ -198,13 +198,13 @@ func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error
 	return client.NewFullNodeRPC(addr, headers)
 }
 
-func GetStorageMinerAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
 	addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	return client.NewStorageMinerRPC(addr, headers)
+	return client.NewStorageMinerRPC(addr, headers, opts...)
 }
 
 func DaemonContext(cctx *cli.Context) context.Context {
@@ -238,6 +238,7 @@ var CommonCommands = []*cli.Command{
 	logCmd,
 	waitApiCmd,
 	fetchParamCmd,
+	pprofCmd,
 	VersionCmd,
 }
 
@@ -256,6 +257,7 @@ var Commands = []*cli.Command{
 	WithCategory("developer", fetchParamCmd),
 	WithCategory("network", netCmd),
 	WithCategory("network", syncCmd),
+	pprofCmd,
 	VersionCmd,
 }
 

cli/pprof.go

@@ -0,0 +1,59 @@
+package cli
+
+import (
+	"io"
+	"net/http"
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/node/repo"
+	manet "github.com/multiformats/go-multiaddr-net"
+)
+
+var pprofCmd = &cli.Command{
+	Name:  "pprof",
+	Hidden: true,
+	Subcommands: []*cli.Command{
+		PprofGoroutines,
+	},
+}
+
+var PprofGoroutines = &cli.Command{
+	Name:  "goroutines",
+	Usage: "Get goroutine stacks",
+	Action: func(cctx *cli.Context) error {
+		ti, ok := cctx.App.Metadata["repoType"]
+		if !ok {
+			log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
+			ti = repo.FullNode
+		}
+		t, ok := ti.(repo.RepoType)
+		if !ok {
+			log.Errorf("repoType type does not match the type of repo.RepoType")
+		}
+		ainfo, err := GetAPIInfo(cctx, t)
+		if err != nil {
+			return xerrors.Errorf("could not get API info: %w", err)
+		}
+		_, addr, err := manet.DialArgs(ainfo.Addr)
+		if err != nil {
+			return err
+		}
+
+		 addr = "http://" + addr + "/debug/pprof/goroutine?debug=2"
+
+		r, err := http.Get(addr)
+		if err != nil {
+			return err
+		}
+
+		if _, err := io.Copy(os.Stdout, r.Body); err != nil {
+			return err
+		}
+
+		return r.Body.Close()
+	},
+}
+

cmd/lotus-seal-worker/main.go

@@ -164,7 +164,9 @@ var runCmd = &cli.Command{
 		var closer func()
 		var err error
 		for {
-			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx)
+			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx,
+				jsonrpc.WithNoReconnect(),
+				jsonrpc.WithTimeout(30*time.Second))
 			if err == nil {
 				break
 			}

cmd/lotus-storage-miner/info_all.go

@@ -148,6 +148,11 @@ var infoAllCmd = &cli.Command{
 			}
 		}
 
+		fmt.Println("\n#: Goroutines")
+		if err := lcli.PprofGoroutines.Action(cctx); err != nil {
+			return err
+		}
+
 		return nil
 	},
 }

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/filecoin-project/go-data-transfer v0.6.1
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
 	github.com/filecoin-project/go-fil-markets v0.5.6
-	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c
+	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261

go.sum

@@ -245,8 +245,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-markets v0.5.6 h1:WmBbV0qBU4NvLJ64xROpzrKUbkZxZqszZiEiCGmCEIY=
 github.com/filecoin-project/go-fil-markets v0.5.6/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E=
-github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c h1:4HGPNCqGbTdhAdu/9yCbrTVpokWHWHATPBlML7Uhh98=
+github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0 h1:/GT3V+3f+H5w5odb7LcCWJ1zPw8H8m9TsGQcU0cGSHo=
-github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200814233340-494a301dc59c/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
+github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
 github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI=
 github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ=
 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=