Merge branch 'master' into sbansal/lotus-worker-stop-cmd

.circleci/config.yml

@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,6 +106,16 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -335,9 +374,13 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
       xcode: "12.5.0"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
@@ -345,48 +388,28 @@ jobs:
       - prepare:
           linux: false
           darwin: true
-      - run:
+      - install_ipfs_macos
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
+      - when:
-          command: make build
+          condition: << parameters.publish >>
-          no_output_timeout: 30m
+          steps:
-      - run:
+            - run: goreleaser release --rm-dist
-          name: check tag and version output match
+            - run: ./scripts/generate-checksums.sh
-          command: ./scripts/version-check.sh ./lotus
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
+          path: dist
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -523,10 +546,6 @@ jobs:
         default: false
         description: publish linux binaries?
         type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
       appimage:
         default: false
         description: publish appimage binaries?
@@ -546,11 +565,6 @@ jobs:
           steps:
             - run: ./scripts/build-arch-bundle.sh linux
             - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
       - when:
           condition: << parameters.appimage >>
           steps:
@@ -976,9 +990,9 @@ workflows:
           target: "./itests/verifreg_test.go"
       
       - test:
-          name: test-itest-wdpost_check_config
+          name: test-itest-wdpost_config
-          suite: itest-wdpost_check_config
+          suite: itest-wdpost_config
-          target: "./itests/wdpost_check_config_test.go"
+          target: "./itests/wdpost_config_test.go"
       
       - test:
           name: test-itest-wdpost_dispute
@@ -1057,10 +1071,20 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-appimage:
           filters:
             branches:
@@ -1069,18 +1093,6 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
           name: publish-linux
           linux: true
@@ -1106,12 +1118,14 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
+          name: build-and-push/lotus-test
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test

.circleci/template.yml

@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,6 +106,16 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -335,9 +374,13 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
       xcode: "12.5.0"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
@@ -345,48 +388,28 @@ jobs:
       - prepare:
           linux: false
           darwin: true
-      - run:
+      - install_ipfs_macos
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
+      - when:
-          command: make build
+          condition: << parameters.publish >>
-          no_output_timeout: 30m
+          steps:
-      - run:
+            - run: goreleaser release --rm-dist
-          name: check tag and version output match
+            - run: ./scripts/generate-checksums.sh
-          command: ./scripts/version-check.sh ./lotus
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
+          path: dist
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -523,10 +546,6 @@ jobs:
         default: false
         description: publish linux binaries?
         type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
       appimage:
         default: false
         description: publish appimage binaries?
@@ -546,11 +565,6 @@ jobs:
           steps:
             - run: ./scripts/build-arch-bundle.sh linux
             - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
       - when:
           condition: << parameters.appimage >>
           steps:
@@ -817,10 +831,20 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-appimage:
           filters:
             branches:
@@ -829,18 +853,6 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
           name: publish-linux
           linux: true
@@ -866,12 +878,14 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
+          name: build-and-push/lotus-test
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test

.gitignore

@@ -51,3 +51,5 @@ scratchpad
 
 build/builtin-actors/v*
 build/builtin-actors/*.car
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,166 @@
+project_name: lotus
+before:
+  hooks:
+    - go mod tidy
+    - make deps
+
+universal_binaries:
+  - id: lotus
+    replace: true
+    name_template: lotus
+    ids:
+      - lotus_darwin_amd64
+      - lotus_darwin_arm64
+  - id: lotus-miner
+    replace: true
+    name_template: lotus-miner
+    ids:
+      - lotus-miner_darwin_amd64
+      - lotus-miner_darwin_arm64
+  - id: lotus-worker
+    replace: true
+    name_template: lotus-worker
+    ids:
+      - lotus-worker_darwin_amd64
+      - lotus-worker_darwin_arm64
+
+builds:
+  - id: lotus_darwin_amd64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_amd64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_amd64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus_darwin_arm64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_arm64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_arm64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+#  - id: linux
+#    main: ./cmd/lotus
+#    binary: lotus
+#    goos:
+#      - linux
+#    goarch:
+#      - amd64
+#    env:
+#      - CGO_ENABLED=1
+#    ldflags:
+#      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+
+archives:
+  - id: primary
+    format: tar.gz
+    wrap_in_directory: true
+    files:
+      # this is a dumb but required hack so it doesn't include the default files
+      # https://github.com/goreleaser/goreleaser/issues/602
+      - _n_o_n_e_*
+
+release:
+  github:
+    owner: filecoin-project
+    name: lotus
+  prerelease: auto
+  mode: append
+  name_template: "Release v{{.Version}}"
+
+
+brews:
+  - tap:
+      owner: filecoin-project
+      name: homebrew-lotus
+      branch: master
+    ids:
+      - primary
+    install: |
+      bin.install "lotus"
+      bin.install "lotus-miner"
+      bin.install "lotus-worker"
+    test: |
+      system "#{bin}/lotus --version"
+      system "#{bin}/lotus-miner --version"
+      system "#{bin}/lotus-worker --version"
+    folder: Formula
+    homepage: "https://filecoin.io"
+    description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
+    license: MIT
+    dependencies:
+      - name: pkg-config
+      - name: jq
+      - name: bzr
+      - name: hwloc
+
+# produced manually so we can include cid checksums
+checksum:
+  disable: true
+
+snapshot:
+  name_template: "{{ .Tag }}"

Makefile

@@ -302,11 +302,14 @@ type-gen: api-gen
 method-gen: api-gen
 	(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)
 
-actors-gen:
+actors-code-gen:
 	$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...
 
+actors-gen: actors-code-gen fiximports
+.PHONY: actors-gen
+
 bundle-gen:
 	$(GOCC) run ./gen/bundle
 	$(GOCC) fmt ./build/...
@@ -364,7 +367,7 @@ docsgen-openrpc-gateway: docsgen-openrpc-bin
 fiximports:
 	./scripts/fiximports
 
-gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen fiximports
+gen: actors-code-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen fiximports
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen
 

api/api_storage.go

@@ -48,6 +48,11 @@ type StorageMiner interface {
 	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
 	ActorAddressConfig(ctx context.Context) (AddressConfig, error)            //perm:read
 
+	// WithdrawBalance allows to withdraw balance from miner actor to owner address
+	// Specify amount as "0" to withdraw full balance. This method returns a message CID
+	// and does not wait for message execution
+	ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) //perm:admin
+
 	MiningBase(context.Context) (*types.TipSet, error) //perm:read
 
 	ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin

api/proxy_gen.go

@@ -656,6 +656,8 @@ type StorageMinerStruct struct {
 
 		ActorSectorSize func(p0 context.Context, p1 address.Address) (abi.SectorSize, error) `perm:"read"`
 
+		ActorWithdrawBalance func(p0 context.Context, p1 abi.TokenAmount) (cid.Cid, error) `perm:"admin"`
+
 		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storiface.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
 
 		ComputeDataCid func(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storiface.Data) (abi.PieceInfo, error) `perm:"admin"`
@@ -3956,6 +3958,17 @@ func (s *StorageMinerStub) ActorSectorSize(p0 context.Context, p1 address.Addres
 	return *new(abi.SectorSize), ErrNotSupported
 }
 
+func (s *StorageMinerStruct) ActorWithdrawBalance(p0 context.Context, p1 abi.TokenAmount) (cid.Cid, error) {
+	if s.Internal.ActorWithdrawBalance == nil {
+		return *new(cid.Cid), ErrNotSupported
+	}
+	return s.Internal.ActorWithdrawBalance(p0, p1)
+}
+
+func (s *StorageMinerStub) ActorWithdrawBalance(p0 context.Context, p1 abi.TokenAmount) (cid.Cid, error) {
+	return *new(cid.Cid), ErrNotSupported
+}
+
 func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storiface.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
 	if s.Internal.CheckProvable == nil {
 		return *new(map[abi.SectorNumber]string), ErrNotSupported

chain/types/message.go

@@ -196,7 +196,7 @@ func (m *Message) ValidForBlockInclusion(minGas int64, version network.Version)
 	}
 
 	if m.GasLimit > build.BlockGasLimit {
-		return xerrors.New("'GasLimit' field cannot be greater than a block's gas limit")
+		return xerrors.Errorf("'GasLimit' field cannot be greater than a block's gas limit (%d > %d)", m.GasLimit, build.BlockGasLimit)
 	}
 
 	// since prices might vary with time, this is technically semantic validation

cmd/lotus-miner/actor.go

@@ -218,6 +218,17 @@ var actorWithdrawCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		amount := abi.NewTokenAmount(0)
+
+		if cctx.Args().Present() {
+			f, err := types.ParseFIL(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("parsing 'amount' argument: %w", err)
+			}
+
+			amount = abi.TokenAmount(f)
+		}
+
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
 		if err != nil {
 			return err
@@ -232,67 +243,19 @@ var actorWithdrawCmd = &cli.Command{
 
 		ctx := lcli.ReqContext(cctx)
 
-		maddr, err := nodeApi.ActorAddress(ctx)
+		res, err := nodeApi.ActorWithdrawBalance(ctx, amount)
 		if err != nil {
 			return err
 		}
 
-		mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
-		if err != nil {
-			return err
-		}
-
-		available, err := api.StateMinerAvailableBalance(ctx, maddr, types.EmptyTSK)
-		if err != nil {
-			return err
-		}
-
-		amount := available
-		if cctx.Args().Present() {
-			f, err := types.ParseFIL(cctx.Args().First())
-			if err != nil {
-				return xerrors.Errorf("parsing 'amount' argument: %w", err)
-			}
-
-			amount = abi.TokenAmount(f)
-
-			if amount.GreaterThan(available) {
-				return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amount), types.FIL(available))
-			}
-		}
-
-		params, err := actors.SerializeParams(&miner.WithdrawBalanceParams{
-			AmountRequested: amount, // Default to attempting to withdraw all the extra funds in the miner actor
-		})
-		if err != nil {
-			return err
-		}
-
-		smsg, err := api.MpoolPushMessage(ctx, &types.Message{
-			To:     maddr,
-			From:   mi.Owner,
-			Value:  types.NewInt(0),
-			Method: builtin.MethodsMiner.WithdrawBalance,
-			Params: params,
-		}, nil)
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Requested rewards withdrawal in message %s\n", smsg.Cid())
-
 		// wait for it to get mined into a block
-		fmt.Printf("waiting for %d epochs for confirmation..\n", uint64(cctx.Int("confidence")))
+		wait, err := api.StateWaitMsg(ctx, res, uint64(cctx.Int("confidence")))
-
-		wait, err := api.StateWaitMsg(ctx, smsg.Cid(), uint64(cctx.Int("confidence")))
 		if err != nil {
-			return err
+			return xerrors.Errorf("Timeout waiting for withdrawal message %s", wait.Message)
 		}
 
-		// check it executed successfully
 		if wait.Receipt.ExitCode != 0 {
-			fmt.Println(cctx.App.Writer, "withdrawal failed!")
+			return xerrors.Errorf("Failed to execute withdrawal message %s: %w", wait.Message, wait.Receipt.ExitCode.Error())
-			return err
 		}
 
 		nv, err := api.StateNetworkVersion(ctx, wait.TipSet)

documentation/en/api-v0-methods-miner.md

@@ -9,6 +9,7 @@
   * [ActorAddress](#ActorAddress)
   * [ActorAddressConfig](#ActorAddressConfig)
   * [ActorSectorSize](#ActorSectorSize)
+  * [ActorWithdrawBalance](#ActorWithdrawBalance)
 * [Auth](#Auth)
   * [AuthNew](#AuthNew)
   * [AuthVerify](#AuthVerify)
@@ -292,6 +293,28 @@ Inputs:
 
 Response: `34359738368`
 
+### ActorWithdrawBalance
+WithdrawBalance allows to withdraw balance from miner actor to owner address
+Specify amount as "0" to withdraw full balance. This method returns a message CID
+and does not wait for message execution
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  "0"
+]
+```
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
 ## Auth
 
 

itests/wdpost_config_test.go

@@ -186,3 +186,117 @@ func TestWindowPostNoPreChecks(t *testing.T) {
 	sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
 	require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged
 }
+
+func TestWindowPostMaxSectorsRecoveryConfig(t *testing.T) {
+	oldVal := wdpost.RecoveringSectorLimit
+	defer func() {
+		wdpost.RecoveringSectorLimit = oldVal
+	}()
+	wdpost.RecoveringSectorLimit = 1
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	client, miner, ens := kit.EnsembleMinimal(t,
+		kit.LatestActorsAt(-1),
+		kit.MockProofs())
+	ens.InterconnectAll().BeginMining(2 * time.Millisecond)
+
+	nSectors := 10
+
+	miner.PledgeSectors(ctx, nSectors, 0, nil)
+
+	maddr, err := miner.ActorAddress(ctx)
+	require.NoError(t, err)
+	di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	mid, err := address.IDFromAddress(maddr)
+	require.NoError(t, err)
+
+	t.Log("Running one proving period")
+	waitUntil := di.Open + di.WPoStProvingPeriod
+	t.Logf("End for head.Height > %d", waitUntil)
+
+	ts := client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil))
+	t.Logf("Now head.Height = %d", ts.Height())
+
+	p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	ssz, err := miner.ActorSectorSize(ctx, maddr)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+kit.DefaultPresealsPerBootstrapMiner)))
+
+	t.Log("Drop some sectors")
+
+	// Drop 2 sectors from deadline 2 partition 0 (full partition / deadline)
+	parts, err := client.StateMinerPartitions(ctx, maddr, 2, types.EmptyTSK)
+	require.NoError(t, err)
+	require.Greater(t, len(parts), 0)
+
+	secs := parts[0].AllSectors
+	n, err := secs.Count()
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), n)
+
+	// Drop the partition
+	err = secs.ForEach(func(sid uint64) error {
+		return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(storiface.SectorRef{
+			ID: abi.SectorID{
+				Miner:  abi.ActorID(mid),
+				Number: abi.SectorNumber(sid),
+			},
+		}, true)
+	})
+	require.NoError(t, err)
+
+	di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	t.Log("Go through another PP, wait for sectors to become faulty")
+	waitUntil = di.Open + di.WPoStProvingPeriod
+	t.Logf("End for head.Height > %d", waitUntil)
+
+	ts = client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil))
+	t.Logf("Now head.Height = %d", ts.Height())
+
+	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+
+	sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
+	require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-2, int(sectors)) // -2 just removed sectors
+
+	t.Log("Make the sectors recoverable")
+
+	err = secs.ForEach(func(sid uint64) error {
+		return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(storiface.SectorRef{
+			ID: abi.SectorID{
+				Miner:  abi.ActorID(mid),
+				Number: abi.SectorNumber(sid),
+			},
+		}, false)
+	})
+	require.NoError(t, err)
+
+	di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	waitUntil = di.Open + di.WPoStProvingPeriod + 200
+	t.Logf("End for head.Height > %d", waitUntil)
+
+	ts = client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil))
+	t.Logf("Now head.Height = %d", ts.Height())
+
+	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+
+	sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
+	require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-1, int(sectors)) // -1 not recovered sector
+}

node/impl/storminer.go

@@ -32,12 +32,15 @@ import (
 	filmktsstore "github.com/filecoin-project/go-fil-markets/stores"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	builtintypes "github.com/filecoin-project/go-state-types/builtin"
 	minertypes "github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -1279,3 +1282,43 @@ func (sm *StorageMinerAPI) ComputeProof(ctx context.Context, ssi []builtin.Exten
 func (sm *StorageMinerAPI) RuntimeSubsystems(context.Context) (res api.MinerSubsystems, err error) {
 	return sm.EnabledSubsystems, nil
 }
+
+func (sm *StorageMinerAPI) ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) {
+	available, err := sm.Full.StateMinerAvailableBalance(ctx, sm.Miner.Address(), types.EmptyTSK)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("Error getting miner balance: %w", err)
+	}
+
+	if amount.GreaterThan(available) {
+		return cid.Undef, xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amount), types.FIL(available))
+	}
+
+	if amount.Equals(big.Zero()) {
+		amount = available
+	}
+
+	params, err := actors.SerializeParams(&minertypes.WithdrawBalanceParams{
+		AmountRequested: amount,
+	})
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	mi, err := sm.Full.StateMinerInfo(ctx, sm.Miner.Address(), types.EmptyTSK)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("Error getting miner's owner address: %w", err)
+	}
+
+	smsg, err := sm.Full.MpoolPushMessage(ctx, &types.Message{
+		To:     sm.Miner.Address(),
+		From:   mi.Owner,
+		Value:  types.NewInt(0),
+		Method: builtintypes.MethodsMiner.WithdrawBalance,
+		Params: params,
+	}, nil)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	return smsg.Cid(), nil
+}

scripts/generate-checksums.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -exo
+
+REQUIRED=(
+    "sha512sum"
+    "ipfs"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+# start ipfs
+export IPFS_PATH=`mktemp -d`
+ipfs init
+ipfs daemon &
+PID="$!"
+trap "kill -9 ${PID}" EXIT
+
+# generate checksums
+for FILE in dist/*.tar.gz
+do
+  sha512sum "${FILE}" > "${FILE}.sha512"
+  until ipfs add -q "${FILE}" > "${FILE}.cid"
+  do
+    echo "Waiting for ipfs daemon to start..."
+    sleep 2
+  done
+done

scripts/publish-checksums.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -exo
+
+pushd dist
+
+# make sure we have a token set, api requests won't work otherwise
+if [ -z "${GITHUB_TOKEN}" ]; then
+  echo "\${GITHUB_TOKEN} not set, publish failed"
+  exit 1
+fi
+
+REQUIRED=(
+    "jq"
+    "curl"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+#see if the release already exists by tag
+RELEASE_RESPONSE=`
+  curl \
+    --fail \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}"
+`
+RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'`
+
+if [ "${RELEASE_ID}" = "null" ]; then
+  echo "creating release"
+
+  COND_CREATE_DISCUSSION=""
+  PRERELEASE=true
+  if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\","
+    PRERELEASE=false
+  fi
+
+  RELEASE_DATA="{
+    \"tag_name\": \"${CIRCLE_TAG}\",
+    \"target_commitish\": \"${CIRCLE_SHA1}\",
+    ${COND_CREATE_DISCUSSION}
+    \"name\": \"${CIRCLE_TAG}\",
+    \"body\": \"\",
+    \"prerelease\": ${PRERELEASE}
+  }"
+
+  # create it if it doesn't exist yet
+  RELEASE_RESPONSE=`
+    curl \
+        --fail \
+        --request POST \
+        --header "Authorization: token ${GITHUB_TOKEN}" \
+        --header "Content-Type: application/json" \
+        --data "${RELEASE_DATA}" \
+        "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases"
+  `
+else
+  echo "release already exists"
+fi
+
+RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1`
+echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}"
+
+for CHECKSUM_FILE in *.{cid,sha512}
+do
+  echo "Uploading ${CHECKSUM_FILE}..."
+  curl \
+    --fail \
+    --request POST \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    --header "Content-Type: application/octet-stream" \
+    --data-binary "@${CHECKSUM_FILE}" \
+    "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")"
+
+  echo "Uploaded ${CHECKSUM_FILE}"
+done
+
+popd

storage/wdpost/wdpost_run_faults.go

@@ -2,6 +2,9 @@ package wdpost
 
 import (
 	"context"
+	"math"
+	"os"
+	"strconv"
 
 	"github.com/ipfs/go-cid"
 	"go.opencensus.io/trace"
@@ -19,6 +22,18 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+var RecoveringSectorLimit uint64 = 0
+
+func init() {
+	if rcl := os.Getenv("LOTUS_RECOVERING_SECTOR_LIMIT"); rcl != "" {
+		var err error
+		RecoveringSectorLimit, err = strconv.ParseUint(rcl, 10, 64)
+		if err != nil {
+			log.Errorw("parsing LOTUS_RECOVERING_SECTOR_LIMIT", "error", err)
+		}
+	}
+}
+
 // declareRecoveries identifies sectors that were previously marked as faulty
 // for our miner, but are now recovered (i.e. are now provable again) and
 // still not reported as such.
@@ -44,7 +59,7 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6
 
 	var batchedRecoveryDecls [][]miner.RecoveryDeclaration
 	batchedRecoveryDecls = append(batchedRecoveryDecls, []miner.RecoveryDeclaration{})
-	totalRecoveries := 0
+	totalSectorsToRecover := uint64(0)
 
 	for partIdx, partition := range partitions {
 		unrecovered, err := bitfield.SubtractBitField(partition.FaultySectors, partition.RecoveringSectors)
@@ -78,6 +93,31 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6
 			continue
 		}
 
+		// rules to follow if we have indicated that we don't want to recover more than X sectors in a deadline
+		if RecoveringSectorLimit > 0 {
+			// something weird happened, break because we can't recover any more
+			if RecoveringSectorLimit < totalSectorsToRecover {
+				log.Warnf("accepted more recoveries (%d) than RecoveringSectorLimit (%d)", totalSectorsToRecover, RecoveringSectorLimit)
+				break
+			}
+
+			maxNewRecoverable := RecoveringSectorLimit - totalSectorsToRecover
+
+			// we need to trim the recover bitfield
+			if recoveredCount > maxNewRecoverable {
+				recoverySlice, err := recovered.All(math.MaxUint64)
+				if err != nil {
+					log.Errorw("failed to slice recovery bitfield, breaking out of recovery loop", err)
+					break
+				}
+
+				log.Warnf("only adding %d sectors to respect RecoveringSectorLimit %d", maxNewRecoverable, RecoveringSectorLimit)
+
+				recovered = bitfield.NewFromSet(recoverySlice[:maxNewRecoverable])
+				recoveredCount = maxNewRecoverable
+			}
+		}
+
 		// respect user config if set
 		if s.maxPartitionsPerRecoveryMessage > 0 &&
 			len(batchedRecoveryDecls[len(batchedRecoveryDecls)-1]) >= s.maxPartitionsPerRecoveryMessage {
@@ -90,10 +130,17 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6
 			Sectors:   recovered,
 		})
 
-		totalRecoveries++
+		totalSectorsToRecover += recoveredCount
+
+		if RecoveringSectorLimit > 0 && totalSectorsToRecover >= RecoveringSectorLimit {
+			log.Errorf("reached recovering sector limit %d, only marking %d sectors for recovery now",
+				RecoveringSectorLimit,
+				totalSectorsToRecover)
+			break
+		}
 	}
 
-	if totalRecoveries == 0 {
+	if totalSectorsToRecover == 0 {
 		if faulty != 0 {
 			log.Warnw("No recoveries to declare", "deadline", dlIdx, "faulty", faulty)
 		}
@@ -101,6 +148,7 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6
 		return nil, nil, nil
 	}
 
+	log.Infof("attempting recovery declarations for %d sectors", totalSectorsToRecover)
 	var msgs []*types.SignedMessage
 	for _, recovery := range batchedRecoveryDecls {
 		params := &miner.DeclareFaultsRecoveredParams{
@@ -122,7 +170,6 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6
 		if err := s.prepareMessage(ctx, msg, spec); err != nil {
 			return nil, nil, err
 		}
-
 		sm, err := s.api.MpoolPushMessage(ctx, msg, &api.MessageSendSpec{MaxFee: abi.TokenAmount(s.feeCfg.MaxWindowPoStGasFee)})
 		if err != nil {
 			return nil, nil, xerrors.Errorf("pushing message to mpool: %w", err)