Merge pull request #8421 from filecoin-project/jen/v1515

build: release: v1.15.1-rc5

.circleci/config.yml

@@ -2,7 +2,6 @@ version: 2.1
 orbs:
   go: gotest/tools@0.0.13
   aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
@@ -12,6 +11,16 @@ executors:
   ubuntu:
     docker:
       - image: ubuntu:20.04
+  packer:
+    description: |
+      The HashiCorp provided Packer container
+    parameters:
+      packer-version:
+        type: string
+        default: "1.8"
+    docker:
+      - image: hashicorp/packer:<< parameters.packer-version >>
+
 
 commands:
   install-deps:
@@ -75,6 +84,26 @@ commands:
           command: |
             git fetch --all
 
+  packer_build:
+    description: "Run a packer build"
+    parameters:
+      template:
+        description: |
+          The name of the packer template file
+        type: string
+        default: packer.json
+      args:
+        description: |
+          Arguments to pass to the packer build command
+        type: string
+        default: ""
+
+    steps:
+      - run:
+          name: "Run a packer build"
+          command: packer build << parameters.args >> << parameters.template >>
+          no_output_timeout: 30m
+
 jobs:
   mod-tidy-check:
     executor: golang
@@ -628,6 +657,11 @@ jobs:
         default: "latest"
         description: A comma-separated string containing docker image tags to build and push (default = latest)
 
+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
     steps:
       - run:
           name: Confirm that environment variables are set
@@ -667,6 +701,7 @@ jobs:
 
             docker build \
               <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
               -f <<parameters.path>>/<<parameters.dockerfile>> \
               $docker_tag_args \
               <<parameters.path>>
@@ -682,49 +717,45 @@ jobs:
   publish-packer-mainnet:
     description: build and push AWS IAM and DigitalOcean droplet.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
   publish-packer-calibrationnet:
     description: build and push AWS IAM and DigitalOcean droplet.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
   publish-packer-butterflynet:
     description: build and push AWS IAM and DigitalOcean droplet.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
   publish-packer-snap:
     description: build packer image with snap. mainnet only.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
     description: publish to dockerhub
@@ -1037,6 +1068,13 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
       - publish-packer-mainnet:
           requires:
             - build-all

.circleci/template.yml

@@ -2,7 +2,6 @@ version: 2.1
 orbs:
   go: gotest/tools@0.0.13
   aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
@@ -12,6 +11,16 @@ executors:
   ubuntu:
     docker:
       - image: ubuntu:20.04
+  packer:
+    description: |
+      The HashiCorp provided Packer container
+    parameters:
+      packer-version:
+        type: string
+        default: "1.8"
+    docker:
+      - image: hashicorp/packer:<< parameters.packer-version >>
+
 
 commands:
   install-deps:
@@ -75,6 +84,26 @@ commands:
           command: |
             git fetch --all
 
+  packer_build:
+    description: "Run a packer build"
+    parameters:
+      template:
+        description: |
+          The name of the packer template file
+        type: string
+        default: packer.json
+      args:
+        description: |
+          Arguments to pass to the packer build command
+        type: string
+        default: ""
+
+    steps:
+      - run:
+          name: "Run a packer build"
+          command: packer build << parameters.args >> << parameters.template >>
+          no_output_timeout: 30m
+
 jobs:
   mod-tidy-check:
     executor: golang
@@ -628,6 +657,11 @@ jobs:
         default: "latest"
         description: A comma-separated string containing docker image tags to build and push (default = latest)
 
+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
     steps:
       - run:
           name: Confirm that environment variables are set
@@ -667,6 +701,7 @@ jobs:
 
             docker build \
               <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
               -f <<parameters.path>>/<<parameters.dockerfile>> \
               $docker_tag_args \
               <<parameters.path>>
@@ -682,49 +717,45 @@ jobs:
   publish-packer-mainnet:
     description: build and push AWS IAM and DigitalOcean droplet.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
   publish-packer-calibrationnet:
     description: build and push AWS IAM and DigitalOcean droplet.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
   publish-packer-butterflynet:
     description: build and push AWS IAM and DigitalOcean droplet.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
   publish-packer-snap:
     description: build packer image with snap. mainnet only.
     executor:
-      name: packer/default
+      name: packer
-      packer-version: 1.6.6
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
+      - packer_build:
           template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
     description: publish to dockerhub
@@ -862,6 +893,13 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
       - publish-packer-mainnet:
           requires:
             - build-all

CHANGELOG.md

@@ -1,8 +1,8 @@
 # Lotus changelog
 
-# 1.15.1-rc4 / 2022-03-29
+# 1.15.1-rc5 / 2022-03-31
 
-This is the fourth release candidate of the next *highly recommended* feature release v1.15.1. This feature release introduces Index Provider, GraphSync v2, and many other latest functionalities, improvements and bug fixes. 
+This is the fifth release candidate of the next *highly recommended* feature release v1.15.1. This feature release introduces Index Provider, GraphSync v2, and many other latest functionalities, improvements and bug fixes. 
 More importantly, node operator can now enable the FVM(experimental) to sync mainnet!!
 
 ## Highlights
@@ -70,6 +70,8 @@ Follow the instruction [here](https://lotus.filecoin.io/storage-providers/operat
 - feat: config: Move MakeNewSectorForDeals config into the Sealing section([filecoin-project/lotus#8382](https://github.com/filecoin-project/lotus/pull/8382))
 
 ## Bug Fixes
+- fix: sealing: Fix PR1 worker selection (#8421)
+- fix: miner: dead loop on removing sector (#8421)
 - fix: sealing: Remove sector copies from workers after snapdeals ([filecoin-project/lotus#8331](https://github.com/filecoin-project/lotus/pull/8331))
 - fix: storagefsm: Fix error loop on bad event ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339))
 - fix: sealing: FinalizeSector doesn't need sealed replica access ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339))

Dockerfile.lotus

@@ -28,6 +28,14 @@ WORKDIR /opt/filecoin
 RUN make clean deps
 
 
+FROM builder-local AS builder-test
+MAINTAINER Lotus Development Team
+
+WORKDIR /opt/filecoin
+
+RUN make debug
+
+
 FROM builder-local AS builder
 MAINTAINER Lotus Development Team
 
@@ -197,6 +205,43 @@ RUN chown fc: /var/lib/lotus-worker
 RUN chown fc: /var/lib/lotus-wallet
 
 
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777
+
+###
+from base as lotus-test
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+COPY --from=builder-test /opt/filecoin/lotus      /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
 VOLUME /var/tmp/filecoin-proof-parameters
 VOLUME /var/lib/lotus
 VOLUME /var/lib/lotus-miner

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.15.1-rc4"
+const BuildVersion = "1.15.1-rc5"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

cmd/lotus-sim/info_commit.go

@@ -6,12 +6,11 @@ import (
 	"os"
 	"syscall"
 
-	"github.com/streadway/quantile"
+	"github.com/ipfs/go-cid"
+	"github.com/koalacxr/quantile"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/go-state-types/exitcode"
-	"github.com/ipfs/go-cid"
-
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"

cmd/lotus-sim/info_message.go

@@ -4,13 +4,14 @@ import (
 	"fmt"
 	"syscall"
 
+	"github.com/ipfs/go-cid"
+	"github.com/koalacxr/quantile"
+	"github.com/urfave/cli/v2"
+
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
 	"github.com/filecoin-project/lotus/lib/stati"
-	"github.com/ipfs/go-cid"
-	"github.com/streadway/quantile"
-	"github.com/urfave/cli/v2"
 )
 
 var infoMessageSizeSimCommand = &cli.Command{

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.1-rc4
+   1.15.1-rc5
 
 COMMANDS:
    init     Initialize a lotus miner repo

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.1-rc4
+   1.15.1-rc5
 
 COMMANDS:
    run         Start lotus worker

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.1-rc4
+   1.15.1-rc5
 
 COMMANDS:
    daemon   Start a lotus daemon process

extern/filecoin-ffi

@@ -1 +1 @@
-Subproject commit dd1c85ababbbc48d6a0458d8378c16a5d30fddb7
+Subproject commit 5d00bb4365a97890bd038edfceecaa63ebbb3e2d

extern/sector-storage/manager.go

@@ -851,7 +851,7 @@ func (m *Manager) ProveReplicaUpdate1(ctx context.Context, sector storage.Sector
 	// NOTE: We set allowFetch to false in so that we always execute on a worker
 	// with direct access to the data. We want to do that because this step is
 	// generally very cheap / fast, and transferring data is not worth the effort
-	selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTSealed|storiface.FTCache, false)
+	selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, false)
 
 	err = m.sched.Schedule(ctx, sector, sealtasks.TTProveReplicaUpdate1, selector, m.schedFetch(sector, storiface.FTSealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error {
 

extern/storage-sealing/fsm.go

@@ -320,7 +320,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	FaultReported: final, // not really supported right now
 
 	FaultedFinal: final,
-	Removed:      final,
+	Removed:      finalRemoved,
 
 	FailedUnrecoverable: final,
 }
@@ -694,6 +694,12 @@ func (m *Sealing) ForceSectorState(ctx context.Context, id abi.SectorNumber, sta
 	return m.sectors.Send(id, SectorForceState{state})
 }
 
+// as sector has been removed, it's no needs to care about later events,
+// just returns length of events as `processed` is ok.
+func finalRemoved(events []statemachine.Event, state *SectorInfo) (uint64, error) {
+	return uint64(len(events)), nil
+}
+
 func final(events []statemachine.Event, state *SectorInfo) (uint64, error) {
 	if len(events) > 0 {
 		if gm, ok := events[0].User.(globalMutator); ok {

go.mod

@@ -110,6 +110,7 @@ require (
 	github.com/ipld/go-ipld-selector-text-lite v0.0.1
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/libp2p/go-eventbus v0.2.1
 	github.com/libp2p/go-libp2p v0.18.0
@@ -143,7 +144,6 @@ require (
 	github.com/prometheus/client_golang v1.11.0
 	github.com/raulk/clock v1.1.0
 	github.com/raulk/go-watchdog v1.2.0
-	github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect

go.sum

@@ -1010,6 +1010,8 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/koalacxr/quantile v0.0.1 h1:wAW+SQ286Erny9wOjVww96t8ws+x5Zj6AKHDULUK+o0=
+github.com/koalacxr/quantile v0.0.1/go.mod h1:bGN/mCZLZ4lrSDHRQ6Lglj9chowGux8sGUIND+DQeD0=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
@@ -1865,8 +1867,6 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3
 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
-github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A=
-github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=