diff --git a/.circleci/config.yml b/.circleci/config.yml index 478c84500..a3d9e793d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,6 @@ version: 2.1 orbs: go: gotest/tools@0.0.13 aws-cli: circleci/aws-cli@1.3.2 - packer: salaxander/packer@0.0.3 executors: golang: @@ -12,6 +11,16 @@ executors: ubuntu: docker: - image: ubuntu:20.04 + packer: + description: | + The HashiCorp provided Packer container + parameters: + packer-version: + type: string + default: "1.8" + docker: + - image: hashicorp/packer:<< parameters.packer-version >> + commands: install-deps: @@ -75,6 +84,26 @@ commands: command: | git fetch --all + packer_build: + description: "Run a packer build" + parameters: + template: + description: | + The name of the packer template file + type: string + default: packer.json + args: + description: | + Arguments to pass to the packer build command + type: string + default: "" + + steps: + - run: + name: "Run a packer build" + command: packer build << parameters.args >> << parameters.template >> + no_output_timeout: 30m + jobs: mod-tidy-check: executor: golang @@ -628,6 +657,11 @@ jobs: default: "latest" description: A comma-separated string containing docker image tags to build and push (default = latest) + target: + type: string + default: "lotus-all-in-one" + description: Docker target to build + steps: - run: name: Confirm that environment variables are set @@ -667,6 +701,7 @@ jobs: docker build \ <<#parameters.extra-build-args>><><> \ + --target <> \ -f <>/<> \ $docker_tag_args \ <> @@ -682,49 +717,45 @@ jobs: publish-packer-mainnet: description: build and push AWS IAM and DigitalOcean droplet. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG" publish-packer-calibrationnet: description: build and push AWS IAM and DigitalOcean droplet. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" publish-packer-butterflynet: description: build and push AWS IAM and DigitalOcean droplet. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" publish-packer-snap: description: build packer image with snap. mainnet only. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus-snap.pkr.hcl publish-dockerhub: description: publish to dockerhub @@ -1037,6 +1068,13 @@ workflows: path: . repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}' + target: lotus-all-in-one + - build-and-push-image: + dockerfile: Dockerfile.lotus + path: . + repo: lotus-test + tag: '${CIRCLE_SHA1:0:8}' + target: lotus-test - publish-packer-mainnet: requires: - build-all @@ -1112,4 +1150,4 @@ workflows: only: - master jobs: - - publish-packer-snap + - publish-packer-snap \ No newline at end of file diff --git a/.circleci/template.yml b/.circleci/template.yml index c8643e3ac..f2cb2ccf5 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -2,7 +2,6 @@ version: 2.1 orbs: go: gotest/tools@0.0.13 aws-cli: circleci/aws-cli@1.3.2 - packer: salaxander/packer@0.0.3 executors: golang: @@ -12,6 +11,16 @@ executors: ubuntu: docker: - image: ubuntu:20.04 + packer: + description: | + The HashiCorp provided Packer container + parameters: + packer-version: + type: string + default: "1.8" + docker: + - image: hashicorp/packer:<< parameters.packer-version >> + commands: install-deps: @@ -75,6 +84,26 @@ commands: command: | git fetch --all + packer_build: + description: "Run a packer build" + parameters: + template: + description: | + The name of the packer template file + type: string + default: packer.json + args: + description: | + Arguments to pass to the packer build command + type: string + default: "" + + steps: + - run: + name: "Run a packer build" + command: packer build << parameters.args >> << parameters.template >> + no_output_timeout: 30m + jobs: mod-tidy-check: executor: golang @@ -628,6 +657,11 @@ jobs: default: "latest" description: A comma-separated string containing docker image tags to build and push (default = latest) + target: + type: string + default: "lotus-all-in-one" + description: Docker target to build + steps: - run: name: Confirm that environment variables are set @@ -667,6 +701,7 @@ jobs: docker build \ <<#parameters.extra-build-args>><><> \ + --target <> \ -f <>/<> \ $docker_tag_args \ <> @@ -682,49 +717,45 @@ jobs: publish-packer-mainnet: description: build and push AWS IAM and DigitalOcean droplet. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG" publish-packer-calibrationnet: description: build and push AWS IAM and DigitalOcean droplet. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" publish-packer-butterflynet: description: build and push AWS IAM and DigitalOcean droplet. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" publish-packer-snap: description: build packer image with snap. mainnet only. executor: - name: packer/default - packer-version: 1.6.6 + name: packer steps: - checkout - attach_workspace: at: "." - - packer/build: + - packer_build: template: tools/packer/lotus-snap.pkr.hcl publish-dockerhub: description: publish to dockerhub @@ -862,6 +893,13 @@ workflows: path: . repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}' + target: lotus-all-in-one + - build-and-push-image: + dockerfile: Dockerfile.lotus + path: . + repo: lotus-test + tag: '${CIRCLE_SHA1:0:8}' + target: lotus-test - publish-packer-mainnet: requires: - build-all @@ -937,4 +975,4 @@ workflows: only: - master jobs: - - publish-packer-snap + - publish-packer-snap \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4180aab28..78eccc510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Lotus changelog -# 1.15.1-rc4 / 2022-03-29 +# 1.15.1-rc5 / 2022-03-31 -This is the fourth release candidate of the next *highly recommended* feature release v1.15.1. This feature release introduces Index Provider, GraphSync v2, and many other latest functionalities, improvements and bug fixes. +This is the fifth release candidate of the next *highly recommended* feature release v1.15.1. This feature release introduces Index Provider, GraphSync v2, and many other latest functionalities, improvements and bug fixes. More importantly, node operator can now enable the FVM(experimental) to sync mainnet!! ## Highlights @@ -70,6 +70,8 @@ Follow the instruction [here](https://lotus.filecoin.io/storage-providers/operat - feat: config: Move MakeNewSectorForDeals config into the Sealing section([filecoin-project/lotus#8382](https://github.com/filecoin-project/lotus/pull/8382)) ## Bug Fixes +- fix: sealing: Fix PR1 worker selection (#8421) +- fix: miner: dead loop on removing sector (#8421) - fix: sealing: Remove sector copies from workers after snapdeals ([filecoin-project/lotus#8331](https://github.com/filecoin-project/lotus/pull/8331)) - fix: storagefsm: Fix error loop on bad event ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339)) - fix: sealing: FinalizeSector doesn't need sealed replica access ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339)) diff --git a/Dockerfile.lotus b/Dockerfile.lotus index 812ad9f61..ba3266824 100644 --- a/Dockerfile.lotus +++ b/Dockerfile.lotus @@ -28,6 +28,14 @@ WORKDIR /opt/filecoin RUN make clean deps +FROM builder-local AS builder-test +MAINTAINER Lotus Development Team + +WORKDIR /opt/filecoin + +RUN make debug + + FROM builder-local AS builder MAINTAINER Lotus Development Team @@ -207,3 +215,40 @@ EXPOSE 1234 EXPOSE 2345 EXPOSE 3456 EXPOSE 1777 + +### +from base as lotus-test + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV LOTUS_MINER_PATH /var/lib/lotus-miner +ENV LOTUS_PATH /var/lib/lotus +ENV LOTUS_WORKER_PATH /var/lib/lotus-worker +ENV WALLET_PATH /var/lib/lotus-wallet + +COPY --from=builder-test /opt/filecoin/lotus /usr/local/bin/ +COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/ +COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/ +COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/ + +RUN mkdir /var/tmp/filecoin-proof-parameters +RUN mkdir /var/lib/lotus +RUN mkdir /var/lib/lotus-miner +RUN mkdir /var/lib/lotus-worker +RUN mkdir /var/lib/lotus-wallet +RUN chown fc: /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus +RUN chown fc: /var/lib/lotus-miner +RUN chown fc: /var/lib/lotus-worker +RUN chown fc: /var/lib/lotus-wallet + + +VOLUME /var/tmp/filecoin-proof-parameters +VOLUME /var/lib/lotus +VOLUME /var/lib/lotus-miner +VOLUME /var/lib/lotus-worker +VOLUME /var/lib/lotus-wallet + +EXPOSE 1234 +EXPOSE 2345 +EXPOSE 3456 +EXPOSE 1777 \ No newline at end of file diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 71922543b..8a83321a5 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 3c9cb8ec0..7499e1f20 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index bf803844c..e85e46280 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/build/version.go b/build/version.go index 9942c10bd..1c08a7e6a 100644 --- a/build/version.go +++ b/build/version.go @@ -37,7 +37,7 @@ func BuildTypeString() string { } // BuildVersion is the local build version -const BuildVersion = "1.15.1-rc4" +const BuildVersion = "1.15.1-rc5" func UserVersion() string { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/cmd/lotus-sim/info_commit.go b/cmd/lotus-sim/info_commit.go index 738fcde95..7a0daa332 100644 --- a/cmd/lotus-sim/info_commit.go +++ b/cmd/lotus-sim/info_commit.go @@ -6,12 +6,11 @@ import ( "os" "syscall" - "github.com/streadway/quantile" + "github.com/ipfs/go-cid" + "github.com/koalacxr/quantile" "github.com/urfave/cli/v2" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/ipfs/go-cid" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" diff --git a/cmd/lotus-sim/info_message.go b/cmd/lotus-sim/info_message.go index 33c45e728..afdd20c4e 100644 --- a/cmd/lotus-sim/info_message.go +++ b/cmd/lotus-sim/info_message.go @@ -4,13 +4,14 @@ import ( "fmt" "syscall" + "github.com/ipfs/go-cid" + "github.com/koalacxr/quantile" + "github.com/urfave/cli/v2" + "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cmd/lotus-sim/simulation" "github.com/filecoin-project/lotus/lib/stati" - "github.com/ipfs/go-cid" - "github.com/streadway/quantile" - "github.com/urfave/cli/v2" ) var infoMessageSizeSimCommand = &cli.Command{ diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 16fd041b4..aad430703 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -7,7 +7,7 @@ USAGE: lotus-miner [global options] command [command options] [arguments...] VERSION: - 1.15.1-rc4 + 1.15.1-rc5 COMMANDS: init Initialize a lotus miner repo diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index b7266ccfc..0e3f8b58e 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -7,7 +7,7 @@ USAGE: lotus-worker [global options] command [command options] [arguments...] VERSION: - 1.15.1-rc4 + 1.15.1-rc5 COMMANDS: run Start lotus worker diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 369121fc8..ec41ed92c 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.15.1-rc4 + 1.15.1-rc5 COMMANDS: daemon Start a lotus daemon process diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index dd1c85aba..5d00bb436 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit dd1c85ababbbc48d6a0458d8378c16a5d30fddb7 +Subproject commit 5d00bb4365a97890bd038edfceecaa63ebbb3e2d diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 70195d333..1cf9f60bd 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -851,7 +851,7 @@ func (m *Manager) ProveReplicaUpdate1(ctx context.Context, sector storage.Sector // NOTE: We set allowFetch to false in so that we always execute on a worker // with direct access to the data. We want to do that because this step is // generally very cheap / fast, and transferring data is not worth the effort - selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTSealed|storiface.FTCache, false) + selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, false) err = m.sched.Schedule(ctx, sector, sealtasks.TTProveReplicaUpdate1, selector, m.schedFetch(sector, storiface.FTSealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error { diff --git a/extern/storage-sealing/fsm.go b/extern/storage-sealing/fsm.go index a02666135..034433a59 100644 --- a/extern/storage-sealing/fsm.go +++ b/extern/storage-sealing/fsm.go @@ -320,7 +320,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto FaultReported: final, // not really supported right now FaultedFinal: final, - Removed: final, + Removed: finalRemoved, FailedUnrecoverable: final, } @@ -694,6 +694,12 @@ func (m *Sealing) ForceSectorState(ctx context.Context, id abi.SectorNumber, sta return m.sectors.Send(id, SectorForceState{state}) } +// as sector has been removed, it's no needs to care about later events, +// just returns length of events as `processed` is ok. +func finalRemoved(events []statemachine.Event, state *SectorInfo) (uint64, error) { + return uint64(len(events)), nil +} + func final(events []statemachine.Event, state *SectorInfo) (uint64, error) { if len(events) > 0 { if gm, ok := events[0].User.(globalMutator); ok { diff --git a/go.mod b/go.mod index 33857abcd..a665fd4c1 100644 --- a/go.mod +++ b/go.mod @@ -110,6 +110,7 @@ require ( github.com/ipld/go-ipld-selector-text-lite v0.0.1 github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kelseyhightower/envconfig v1.4.0 + github.com/koalacxr/quantile v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-eventbus v0.2.1 github.com/libp2p/go-libp2p v0.18.0 @@ -143,7 +144,6 @@ require ( github.com/prometheus/client_golang v1.11.0 github.com/raulk/clock v1.1.0 github.com/raulk/go-watchdog v1.2.0 - github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect diff --git a/go.sum b/go.sum index 49dddb84c..993cdc582 100644 --- a/go.sum +++ b/go.sum @@ -1010,6 +1010,8 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/koalacxr/quantile v0.0.1 h1:wAW+SQ286Erny9wOjVww96t8ws+x5Zj6AKHDULUK+o0= +github.com/koalacxr/quantile v0.0.1/go.mod h1:bGN/mCZLZ4lrSDHRQ6Lglj9chowGux8sGUIND+DQeD0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -1865,8 +1867,6 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= -github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A= -github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=