Merge pull request #9177 from filecoin-project/9171-add-retries-to-mpool-push-message

feat: message: Add retries to mpool push message from lotus miner

.circleci/config.yml

@@ -5,7 +5,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.17.9
+      - image: cimg/go:1.18.1
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -52,7 +52,7 @@ commands:
             - run:
                 name: Install Go
                 command: |
-                  curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                  curl https://dl.google.com/go/go1.18.1.darwin-amd64.pkg -o /tmp/go.pkg && \
                   sudo installer -pkg /tmp/go.pkg -target /
             - run:
                 name: Export Go

.circleci/template.yml

@@ -5,7 +5,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.17.9
+      - image: cimg/go:1.18.1
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -52,7 +52,7 @@ commands:
             - run:
                 name: Install Go
                 command: |
-                  curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                  curl https://dl.google.com/go/go1.18.1.darwin-amd64.pkg -o /tmp/go.pkg && \
                   sudo installer -pkg /tmp/go.pkg -target /
             - run:
                 name: Export Go

.github/workflows/codeql-analysis.yml

@@ -37,7 +37,7 @@ jobs:
 
     - uses: actions/setup-go@v1
       with:
-        go-version: '1.17.9'
+        go-version: '1.18.1'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

CHANGELOG.md

@@ -368,7 +368,7 @@ This is an optional release of lotus that include new APIs, some improvements an
 
 This is a highly recommended feature lotus release v1.15.2. This feature release introduces many new features and  for SPs, including PoSt workers, sealing scheduler, snap deal queue and so on.
 
-Note: You need to be using go v1.17.9&up from this release onwards.
+Note: You need to be using go v1.18.1&up from this release onwards.
 
 ## Highlights
 ### ❣️❣️❣️ PoSt Workers ❣️❣️❣️

Dockerfile.lotus

@@ -1,4 +1,4 @@
-FROM golang:1.17.9-buster AS builder-deps
+FROM golang:1.18.1-buster AS builder-deps
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev

Makefile

@@ -8,9 +8,9 @@ unexport GOFLAGS
 GOCC?=go
 
 GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
-ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
+ifeq ($(shell expr $(GOVERSION) \< 1018001), 1)
 $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
-$(error Update Golang to version to at least 1.17.9)
+$(error Update Golang to version to at least 1.18.1)
 endif
 
 # git modules that need to be loaded

README.md

@@ -71,10 +71,10 @@ For other distributions you can find the required dependencies [here.](https://d
 
 #### Go
 
-To build Lotus, you need a working installation of [Go 1.17.9 or higher](https://golang.org/dl/):
+To build Lotus, you need a working installation of [Go 1.18.1 or higher](https://golang.org/dl/):
 
 ```bash
-wget -c https://golang.org/dl/go1.17.9.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
+wget -c https://golang.org/dl/go1.18.1.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
 ```
 
 **TIP:**

go.mod

@@ -1,6 +1,6 @@
 module github.com/filecoin-project/lotus
 
-go 1.17
+go 1.18
 
 retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead.
 
@@ -38,7 +38,7 @@ require (
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
 	github.com/filecoin-project/go-fil-markets v1.24.0
-	github.com/filecoin-project/go-jsonrpc v0.1.5
+	github.com/filecoin-project/go-jsonrpc v0.1.7
 	github.com/filecoin-project/go-legs v0.4.4
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4

lib/retry/retry.go

@@ -0,0 +1,37 @@
+package retry
+
+import (
+	"errors"
+	"reflect"
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var log = logging.Logger("retry")
+
+func ErrorIsIn(err error, errorTypes []error) bool {
+	for _, etype := range errorTypes {
+		tmp := reflect.New(reflect.PointerTo(reflect.ValueOf(etype).Elem().Type())).Interface()
+		if errors.As(err, tmp) {
+			return true
+		}
+	}
+	return false
+}
+
+func Retry[T any](attempts int, initialBackoff time.Duration, errorTypes []error, f func() (T, error)) (result T, err error) {
+	for i := 0; i < attempts; i++ {
+		if i > 0 {
+			log.Info("Retrying after error:", err)
+			time.Sleep(initialBackoff)
+			initialBackoff *= 2
+		}
+		result, err = f()
+		if err == nil || !ErrorIsIn(err, errorTypes) {
+			return result, err
+		}
+	}
+	log.Errorf("Failed after %d attempts, last error: %s", attempts, err)
+	return result, err
+}

lib/retry/retry_test.go

@@ -0,0 +1,25 @@
+package retry
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-jsonrpc"
+)
+
+func TestRetryErrorIsInTrue(t *testing.T) {
+	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
+	require.True(t, ErrorIsIn(&jsonrpc.RPCConnectionError{}, errorsToRetry))
+}
+
+func TestRetryErrorIsInFalse(t *testing.T) {
+	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
+	require.False(t, ErrorIsIn(xerrors.Errorf("random error"), errorsToRetry))
+}
+
+func TestRetryWrappedErrorIsInTrue(t *testing.T) {
+	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
+	require.True(t, ErrorIsIn(xerrors.Errorf("wrapped: %w", &jsonrpc.RPCConnectionError{}), errorsToRetry))
+}

node/modules/storageminer.go

@@ -37,6 +37,7 @@ import (
 	storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
 	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
 	smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
+	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-paramfetch"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -55,6 +56,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/gen/slashfilter"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/lib/retry"
 	"github.com/filecoin-project/lotus/markets"
 	"github.com/filecoin-project/lotus/markets/dagstore"
 	"github.com/filecoin-project/lotus/markets/idxprov"
@@ -87,7 +89,12 @@ func (a *UuidWrapper) MpoolPushMessage(ctx context.Context, msg *types.Message,
 		spec = new(api.MessageSendSpec)
 	}
 	spec.MsgUuid = uuid.New()
-	return a.FullNode.MpoolPushMessage(ctx, msg, spec)
+	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
+	initialBackoff, err := time.ParseDuration("1s")
+	if err != nil {
+		return nil, err
+	}
+	return retry.Retry(5, initialBackoff, errorsToRetry, func() (*types.SignedMessage, error) { return a.FullNode.MpoolPushMessage(ctx, msg, spec) })
 }
 
 func MakeUuidWrapper(a v1api.RawFullNodeAPI) v1api.FullNode {

testplans/composer/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.9-buster as tg-build
+FROM golang:1.18.1-buster as tg-build
 
 ARG TESTGROUND_REF="oni"
 WORKDIR /usr/src

testplans/docker-images/Dockerfile.oni-buildbase

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.17.9
+ARG GO_VERSION=1.18.1
 
 FROM golang:${GO_VERSION}-buster
 

testplans/docker-images/Dockerfile.oni-runtime

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.17.9
+ARG GO_VERSION=1.18.1
 
 FROM golang:${GO_VERSION}-buster as downloader
 

testplans/docker-images/Dockerfile.oni-runtime-debug

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.17.9
+ARG GO_VERSION=1.18.1
 
 FROM golang:${GO_VERSION}-buster as downloader
 

testplans/lotus-soup/go.mod

@@ -1,6 +1,6 @@
 module github.com/filecoin-project/lotus/testplans/lotus-soup
 
-go 1.17
+go 1.18
 
 require (
 	contrib.go.opencensus.io/exporter/prometheus v0.4.0

testplans/lotus-soup/go.sum

@@ -2055,7 +2055,7 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/Op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI=
 github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=