cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge tag 'v1.11.0-rc1' into feat/docker-compose

Browse Source
This commit is contained in:
Cory Schwartz 2021-06-30 15:40:54 -07:00
commit d3ff5a2a4f
159 changed files with 9789 additions and 3441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

250
.circleci/config.yml
View File

@ -112,7 +112,7 @@ jobs:
- run:
command: make debug
test: &test
test:
description: |
Run tests with gotestsum.
parameters: &test-params
@ -123,26 +123,20 @@ jobs:
type: string
default: "-timeout 30m"
description: Flags passed to go test.
packages:
target:
type: string
default: "./..."
description: Import paths of packages to be tested.
winpost-test:
type: string
default: "0"
deadline-test:
type: string
default: "0"
proofs-log-test:
type: string
default: "0"
test-suite-name:
suite:
type: string
default: unit
description: Test suite name to report to CircleCI.
gotestsum-format:
type: string
default: pkgname-and-test-fails
default: standard-verbose
description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
coverage:
type: string
@ -150,7 +144,7 @@ jobs:
description: Coverage flag. Set to the empty string to disable.
codecov-upload:
type: boolean
default: false
default: true
description: |
Upload coverage report to https://codecov.io/. Requires the codecov API token to be
set as an environment variable for private projects.
@ -168,26 +162,24 @@ jobs:
- run:
name: go test
environment:
LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
LOTUS_TEST_DEADLINE_TOGGLING: << parameters.deadline-test >>
TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
SKIP_CONFORMANCE: "1"
command: |
mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
gotestsum \
--format << parameters.gotestsum-format >> \
--junitfile /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml \
--jsonfile /tmp/test-artifacts/<< parameters.test-suite-name >>.json \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
--jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-- \
<< parameters.coverage >> \
<< parameters.go-test-flags >> \
<< parameters.packages >>
<< parameters.target >>
no_output_timeout: 30m
- store_test_results:
path: /tmp/test-reports
- store_artifacts:
path: /tmp/test-artifacts/<< parameters.test-suite-name >>.json
path: /tmp/test-artifacts/<< parameters.suite >>.json
- when:
condition: << parameters.codecov-upload >>
steps:
@ -198,26 +190,6 @@ jobs:
command: |
bash <(curl -s https://codecov.io/bash)
test-chain:
<<: *test
test-node:
<<: *test
test-storage:
<<: *test
test-cli:
<<: *test
test-short:
<<: *test
test-window-post:
<<: *test
test-window-post-dispute:
<<: *test
test-deadline-toggling:
<<: *test
test-terminate:
<<: *test
check-proofs-multicore-sdr:
<<: *test
test-conformance:
description: |
Run tests using a corpus of interoperable test vectors for Filecoin
@ -462,7 +434,7 @@ jobs:
name: prepare workspace
command: |
mkdir appimage
mv Lotus-latest-x86_64.AppImage appimage
mv Lotus-*.AppImage appimage
- persist_to_workspace:
root: "."
paths:
@ -598,7 +570,6 @@ jobs:
name: publish snap
command: snapcraft push *.snap --release << parameters.channel >>
build-and-push-image:
description: build and push docker images to public AWS ECR registry
executor: aws-cli/default
@ -776,64 +747,145 @@ workflows:
- gen-check
- docs-check
- test:
codecov-upload: true
test-suite-name: full
- test-chain:
codecov-upload: true
test-suite-name: chain
packages: "./chain/..."
- test-node:
codecov-upload: true
test-suite-name: node
packages: "./node/..."
- test-storage:
codecov-upload: true
test-suite-name: storage
packages: "./storage/... ./extern/..."
- test-cli:
codecov-upload: true
test-suite-name: cli
packages: "./cli/... ./cmd/... ./api/..."
- test-window-post:
codecov-upload: true
go-test-flags: "-run=TestWindowedPost"
winpost-test: "1"
test-suite-name: window-post
- test-window-post-dispute:
codecov-upload: true
go-test-flags: "-run=TestWindowPostDispute"
winpost-test: "1"
test-suite-name: window-post-dispute
- test-terminate:
codecov-upload: true
go-test-flags: "-run=TestTerminate"
winpost-test: "1"
test-suite-name: terminate
- test-deadline-toggling:
codecov-upload: true
go-test-flags: "-run=TestDeadlineToggling"
deadline-test: "1"
test-suite-name: deadline-toggling
- test-short:
go-test-flags: "--timeout 10m --short"
test-suite-name: short
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- check-proofs-multicore-sdr:
codecov-upload: true
name: test-itest-api
suite: itest-api
target: "./itests/api_test.go"
- test:
name: test-itest-batch_deal
suite: itest-batch_deal
target: "./itests/batch_deal_test.go"
- test:
name: test-itest-ccupgrade
suite: itest-ccupgrade
target: "./itests/ccupgrade_test.go"
- test:
name: test-itest-cli
suite: itest-cli
target: "./itests/cli_test.go"
- test:
name: test-itest-deadlines
suite: itest-deadlines
target: "./itests/deadlines_test.go"
- test:
name: test-itest-deals_concurrent
suite: itest-deals_concurrent
target: "./itests/deals_concurrent_test.go"
- test:
name: test-itest-deals_offline
suite: itest-deals_offline
target: "./itests/deals_offline_test.go"
- test:
name: test-itest-deals_power
suite: itest-deals_power
target: "./itests/deals_power_test.go"
- test:
name: test-itest-deals_pricing
suite: itest-deals_pricing
target: "./itests/deals_pricing_test.go"
- test:
name: test-itest-deals_publish
suite: itest-deals_publish
target: "./itests/deals_publish_test.go"
- test:
name: test-itest-deals
suite: itest-deals
target: "./itests/deals_test.go"
- test:
name: test-itest-gateway
suite: itest-gateway
target: "./itests/gateway_test.go"
- test:
name: test-itest-multisig
suite: itest-multisig
target: "./itests/multisig_test.go"
- test:
name: test-itest-paych_api
suite: itest-paych_api
target: "./itests/paych_api_test.go"
- test:
name: test-itest-paych_cli
suite: itest-paych_cli
target: "./itests/paych_cli_test.go"
- test:
name: test-itest-sdr_upgrade
suite: itest-sdr_upgrade
target: "./itests/sdr_upgrade_test.go"
- test:
name: test-itest-sector_pledge
suite: itest-sector_pledge
target: "./itests/sector_pledge_test.go"
- test:
name: test-itest-sector_terminate
suite: itest-sector_terminate
target: "./itests/sector_terminate_test.go"
- test:
name: test-itest-tape
suite: itest-tape
target: "./itests/tape_test.go"
- test:
name: test-itest-verifreg
suite: itest-verifreg
target: "./itests/verifreg_test.go"
- test:
name: test-itest-wdpost_dispute
suite: itest-wdpost_dispute
target: "./itests/wdpost_dispute_test.go"
- test:
name: test-itest-wdpost
suite: itest-wdpost
target: "./itests/wdpost_test.go"
- test:
name: test-unit-cli
suite: utest-unit-cli
target: "./cli/... ./cmd/... ./api/..."
- test:
name: test-unit-node
suite: utest-unit-node
target: "./node/..."
- test:
name: test-unit-rest
suite: utest-unit-rest
target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..."
- test:
name: test-unit-storage
suite: utest-unit-storage
target: "./storage/... ./extern/..."
- test:
go-test-flags: "-run=TestMulticoreSDR"
test-suite-name: multicore-sdr-check
packages: "./extern/sector-storage/ffiwrapper"
suite: multicore-sdr-check
target: "./extern/sector-storage/ffiwrapper"
proofs-log-test: "1"
- test-conformance:
test-suite-name: conformance
packages: "./conformance"
suite: conformance
codecov-upload: false
target: "./conformance"
- test-conformance:
name: test-conformance-bleeding-edge
test-suite-name: conformance-bleeding-edge
packages: "./conformance"
codecov-upload: false
suite: conformance-bleeding-edge
target: "./conformance"
vectors-branch: master
- trigger-testplans:
filters:
@ -842,37 +894,27 @@ workflows:
- master
- build-debug
- build-all:
requires:
- test-short
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-calibration:
requires:
- test-short
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-butterfly:
requires:
- test-short
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-nerpa:
requires:
- test-short
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-lotus-soup
- build-macos:
requires:
- test-short
filters:
branches:
ignore:
@ -881,8 +923,6 @@ workflows:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-appimage:
requires:
- test-short
filters:
branches:
ignore:

136
.circleci/gen.go Normal file
View File

@ -0,0 +1,136 @@
package main
import (
"embed"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"text/template"
)
//go:generate go run ./gen.go ..
//go:embed template.yml
var templateFile embed.FS
type (
dirs = []string
suite = string
)
// groupedUnitTests maps suite names to top-level directories that should be
// included in that suite. The program adds an implicit group "rest" that
// includes all other top-level directories.
var groupedUnitTests = map[suite]dirs{
"unit-node": {"node"},
"unit-storage": {"storage", "extern"},
"unit-cli": {"cli", "cmd", "api"},
}
func main() {
if len(os.Args) != 2 {
panic("expected path to repo as argument")
}
repo := os.Args[1]
tmpl := template.New("template.yml")
tmpl.Delims("[[", "]]")
tmpl.Funcs(template.FuncMap{
"stripSuffix": func(in string) string {
return strings.TrimSuffix(in, "_test.go")
},
})
tmpl = template.Must(tmpl.ParseFS(templateFile, "*"))
// list all itests.
itests, err := filepath.Glob(filepath.Join(repo, "./itests/*_test.go"))
if err != nil {
panic(err)
}
// strip the dir from all entries.
for i, f := range itests {
itests[i] = filepath.Base(f)
}
// calculate the exclusion set of unit test directories to exclude because
// they are already included in a grouped suite.
var excluded = map[string]struct{}{}
for _, ss := range groupedUnitTests {
for _, s := range ss {
e, err := filepath.Abs(filepath.Join(repo, s))
if err != nil {
panic(err)
}
excluded[e] = struct{}{}
}
}
// all unit tests top-level dirs that are not itests, nor included in other suites.
var rest = map[string]struct{}{}
err = filepath.Walk(repo, func(path string, f os.FileInfo, err error) error {
// include all tests that aren't in the itests directory.
if strings.Contains(path, "itests") {
return filepath.SkipDir
}
// exclude all tests included in other suites
if f.IsDir() {
if _, ok := excluded[path]; ok {
return filepath.SkipDir
}
}
if strings.HasSuffix(path, "_test.go") {
rel, err := filepath.Rel(repo, path)
if err != nil {
panic(err)
}
// take the first directory
rest[strings.Split(rel, string(os.PathSeparator))[0]] = struct{}{}
}
return err
})
if err != nil {
panic(err)
}
// add other directories to a 'rest' suite.
for k := range rest {
groupedUnitTests["unit-rest"] = append(groupedUnitTests["unit-rest"], k)
}
// map iteration guarantees no order, so sort the array in-place.
sort.Strings(groupedUnitTests["unit-rest"])
// form the input data.
type data struct {
ItestFiles []string
UnitSuites map[string]string
}
in := data{
ItestFiles: itests,
UnitSuites: func() map[string]string {
ret := make(map[string]string)
for name, dirs := range groupedUnitTests {
for i, d := range dirs {
dirs[i] = fmt.Sprintf("./%s/...", d) // turn into package
}
ret[name] = strings.Join(dirs, " ")
}
return ret
}(),
}
out, err := os.Create("./config.yml")
if err != nil {
panic(err)
}
defer out.Close()
// execute the template.
if err := tmpl.Execute(out, in); err != nil {
panic(err)
}
}

902
.circleci/template.yml Normal file
View File

@ -0,0 +1,902 @@
version: 2.1
orbs:
go: gotest/tools@0.0.13
aws-cli: circleci/aws-cli@1.3.2
packer: salaxander/packer@0.0.3
executors:
golang:
docker:
- image: circleci/golang:1.16.4
resource_class: 2xlarge
ubuntu:
docker:
- image: ubuntu:20.04
commands:
install-deps:
steps:
- go/install-ssh
- go/install: {package: git}
prepare:
parameters:
linux:
default: true
description: is a linux build environment?
type: boolean
darwin:
default: false
description: is a darwin build environment?
type: boolean
steps:
- checkout
- git_fetch_all_tags
- checkout
- when:
condition: << parameters.linux >>
steps:
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
- run: git submodule sync
- run: git submodule update --init
download-params:
steps:
- restore_cache:
name: Restore parameters cache
keys:
- 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
key: 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
install_ipfs:
steps:
- run: |
apt update
apt install -y wget
wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz
wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512
if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ]
then
echo "ipfs failed checksum check"
exit 1
fi
tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz
mv go-ipfs/ipfs /usr/local/bin/ipfs
chmod +x /usr/local/bin/ipfs
git_fetch_all_tags:
steps:
- run:
name: fetch all tags
command: |
git fetch --all
jobs:
mod-tidy-check:
executor: golang
steps:
- install-deps
- prepare
- go/mod-tidy-check
build-all:
executor: golang
steps:
- install-deps
- prepare
- run: sudo apt-get update
- run: sudo apt-get install npm
- run:
command: make buildall
- store_artifacts:
path: lotus
- store_artifacts:
path: lotus-miner
- store_artifacts:
path: lotus-worker
- run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
- persist_to_workspace:
root: "."
paths:
- linux
build-debug:
executor: golang
steps:
- install-deps
- prepare
- run:
command: make debug
test:
description: |
Run tests with gotestsum.
parameters: &test-params
executor:
type: executor
default: golang
go-test-flags:
type: string
default: "-timeout 30m"
description: Flags passed to go test.
target:
type: string
default: "./..."
description: Import paths of packages to be tested.
proofs-log-test:
type: string
default: "0"
suite:
type: string
default: unit
description: Test suite name to report to CircleCI.
gotestsum-format:
type: string
default: standard-verbose
description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
coverage:
type: string
default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
description: Coverage flag. Set to the empty string to disable.
codecov-upload:
type: boolean
default: true
description: |
Upload coverage report to https://codecov.io/. Requires the codecov API token to be
set as an environment variable for private projects.
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run:
command: make deps lotus
no_output_timeout: 30m
- download-params
- go/install-gotestsum:
gobin: $HOME/.local/bin
version: 0.5.2
- run:
name: go test
environment:
TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
SKIP_CONFORMANCE: "1"
command: |
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
gotestsum \
--format << parameters.gotestsum-format >> \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
--jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-- \
<< parameters.coverage >> \
<< parameters.go-test-flags >> \
<< parameters.target >>
no_output_timeout: 30m
- store_test_results:
path: /tmp/test-reports
- store_artifacts:
path: /tmp/test-artifacts/<< parameters.suite >>.json
- when:
condition: << parameters.codecov-upload >>
steps:
- go/install: {package: bash}
- go/install: {package: curl}
- run:
shell: /bin/bash -eo pipefail
command: |
bash <(curl -s https://codecov.io/bash)
test-conformance:
description: |
Run tests using a corpus of interoperable test vectors for Filecoin
implementations to test their correctness and compliance with the Filecoin
specifications.
parameters:
<<: *test-params
vectors-branch:
type: string
default: ""
description: |
Branch on github.com/filecoin-project/test-vectors to checkout and
test with. If empty (the default) the commit defined by the git
submodule is used.
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run:
command: make deps lotus
no_output_timeout: 30m
- download-params
- when:
condition:
not:
equal: [ "", << parameters.vectors-branch >> ]
steps:
- run:
name: checkout vectors branch
command: |
cd extern/test-vectors
git fetch
git checkout origin/<< parameters.vectors-branch >>
- go/install-gotestsum:
gobin: $HOME/.local/bin
version: 0.5.2
- run:
name: install statediff globally
command: |
## statediff is optional; we succeed even if compilation fails.
mkdir -p /tmp/statediff
git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
cd /tmp/statediff
go install ./cmd/statediff || exit 0
- run:
name: go test
environment:
SKIP_CONFORMANCE: "0"
command: |
mkdir -p /tmp/test-reports
mkdir -p /tmp/test-artifacts
gotestsum \
--format pkgname-and-test-fails \
--junitfile /tmp/test-reports/junit.xml \
-- \
-v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
no_output_timeout: 30m
- store_test_results:
path: /tmp/test-reports
- store_artifacts:
path: /tmp/test-artifacts/conformance-coverage.html
build-ntwk-calibration:
description: |
Compile lotus binaries for the calibration network
parameters:
<<: *test-params
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run: make calibnet
- run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
- persist_to_workspace:
root: "."
paths:
- linux-calibrationnet
build-ntwk-butterfly:
description: |
Compile lotus binaries for the butterfly network
parameters:
<<: *test-params
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run: make butterflynet
- run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
- persist_to_workspace:
root: "."
paths:
- linux-butterflynet
build-ntwk-nerpa:
description: |
Compile lotus binaries for the nerpa network
parameters:
<<: *test-params
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run: make nerpanet
- run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet
- persist_to_workspace:
root: "."
paths:
- linux-nerpanet
build-lotus-soup:
description: |
Compile `lotus-soup` Testground test plan
parameters:
<<: *test-params
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run: cd extern/filecoin-ffi && make
- run:
name: "go get lotus@master"
command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy
- run:
name: "build lotus-soup testplan"
command: pushd testplans/lotus-soup && go build -tags=testground .
trigger-testplans:
description: |
Trigger `lotus-soup` test cases on TaaS
parameters:
<<: *test-params
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run:
name: "download testground"
command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli
- run:
name: "prepare .env.toml"
command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
- run:
name: "prepare testground home dir and link test plans"
command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
- run:
name: "go get lotus@master"
command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
- run:
name: "trigger deals baseline testplan on taas"
command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
- run:
name: "trigger payment channel stress testplan on taas"
command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
- run:
name: "trigger graphsync testplan on taas"
command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
build-macos:
description: build darwin lotus binary
macos:
xcode: "10.0.0"
working_directory: ~/go/src/github.com/filecoin-project/lotus
steps:
- prepare:
linux: false
darwin: true
- run:
name: Install go
command: |
curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \
sudo installer -pkg go1.16.4.darwin-amd64.pkg -target /
- run:
name: Install pkg-config
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
- run: go version
- run:
name: Install Rust
command: |
curl https://sh.rustup.rs -sSf | sh -s -- -y
- run:
name: Install jq
command: |
curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
chmod +x /usr/local/bin/jq
- run:
name: Install hwloc
command: |
mkdir ~/hwloc
curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
cd ~/hwloc
tar -xvzpf hwloc-2.4.1.tar.gz
cd hwloc-2.4.1
./configure && make && sudo make install
- restore_cache:
name: restore cargo cache
key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
- install-deps
- run:
command: make build
no_output_timeout: 30m
- store_artifacts:
path: lotus
- store_artifacts:
path: lotus-miner
- store_artifacts:
path: lotus-worker
- run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
- persist_to_workspace:
root: "."
paths:
- darwin
- save_cache:
name: save cargo cache
key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
paths:
- "~/.rustup"
- "~/.cargo"
build-appimage:
machine:
image: ubuntu-2004:202104-01
steps:
- checkout
- attach_workspace:
at: "."
- run:
name: install appimage-builder
command: |
# docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
sudo apt update
sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
sudo chmod +x /usr/local/bin/appimagetool
sudo pip3 install appimage-builder
- run:
name: install lotus dependencies
command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
- run:
name: build appimage
command: |
sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
make appimage
- run:
name: prepare workspace
command: |
mkdir appimage
mv Lotus-*.AppImage appimage
- persist_to_workspace:
root: "."
paths:
- appimage
gofmt:
executor: golang
steps:
- install-deps
- prepare
- run:
command: "! go fmt ./... 2>&1 | read"
gen-check:
executor: golang
steps:
- install-deps
- prepare
- run: make deps
- run: go install golang.org/x/tools/cmd/goimports
- run: go install github.com/hannahhoward/cbor-gen-for
- run: make gen
- run: git --no-pager diff
- run: git --no-pager diff --quiet
- run: make docsgen-cli
- run: git --no-pager diff
- run: git --no-pager diff --quiet
docs-check:
executor: golang
steps:
- install-deps
- prepare
- run: go install golang.org/x/tools/cmd/goimports
- run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
- run: make deps
- run: make docsgen
- run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
- run: git --no-pager diff
- run: diff ../pre-openrpc-full ../post-openrpc-full
- run: diff ../pre-openrpc-miner ../post-openrpc-miner
- run: diff ../pre-openrpc-worker ../post-openrpc-worker
- run: git --no-pager diff --quiet
lint: &lint
description: |
Run golangci-lint.
parameters:
executor:
type: executor
default: golang
golangci-lint-version:
type: string
default: 1.27.0
concurrency:
type: string
default: '2'
description: |
Concurrency used to run linters. Defaults to 2 because NumCPU is not
aware of container CPU limits.
args:
type: string
default: ''
description: |
Arguments to pass to golangci-lint
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run:
command: make deps
no_output_timeout: 30m
- go/install-golangci-lint:
gobin: $HOME/.local/bin
version: << parameters.golangci-lint-version >>
- run:
name: Lint
command: |
$HOME/.local/bin/golangci-lint run -v --timeout 2m \
--concurrency << parameters.concurrency >> << parameters.args >>
lint-all:
<<: *lint
publish:
description: publish binary artifacts
executor: ubuntu
steps:
- run:
name: Install git jq curl
command: apt update && apt install -y git jq curl
- checkout
- git_fetch_all_tags
- checkout
- install_ipfs
- attach_workspace:
at: "."
- run:
name: Create bundles
command: ./scripts/build-bundle.sh
- run:
name: Publish release
command: ./scripts/publish-release.sh
publish-snapcraft:
description: build and push snapcraft
machine:
image: ubuntu-2004:202104-01
resource_class: 2xlarge
parameters:
channel:
type: string
default: "edge"
description: snapcraft channel
steps:
- checkout
- run:
name: install snapcraft
command: sudo snap install snapcraft --classic
- run:
name: create snapcraft config file
command: |
mkdir -p ~/.config/snapcraft
echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
- run:
name: build snap
command: snapcraft --use-lxd
- run:
name: publish snap
command: snapcraft push *.snap --release << parameters.channel >>
build-and-push-image:
description: build and push docker images to public AWS ECR registry
executor: aws-cli/default
parameters:
profile-name:
type: string
default: "default"
description: AWS profile name to be configured.
aws-access-key-id:
type: env_var_name
default: AWS_ACCESS_KEY_ID
description: >
AWS access key id for IAM role. Set this to the name of
the environment variable you will set to hold this
value, i.e. AWS_ACCESS_KEY.
aws-secret-access-key:
type: env_var_name
default: AWS_SECRET_ACCESS_KEY
description: >
AWS secret key for IAM role. Set this to the name of
the environment variable you will set to hold this
value, i.e. AWS_SECRET_ACCESS_KEY.
region:
type: env_var_name
default: AWS_REGION
description: >
Name of env var storing your AWS region information,
defaults to AWS_REGION
account-url:
type: env_var_name
default: AWS_ECR_ACCOUNT_URL
description: >
Env var storing Amazon ECR account URL that maps to an AWS account,
e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
defaults to AWS_ECR_ACCOUNT_URL
dockerfile:
type: string
default: Dockerfile
description: Name of dockerfile to use. Defaults to Dockerfile.
path:
type: string
default: .
description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
extra-build-args:
type: string
default: ""
description: >
Extra flags to pass to docker build. For examples, see
https://docs.docker.com/engine/reference/commandline/build
repo:
type: string
description: Name of an Amazon ECR repository
tag:
type: string
default: "latest"
description: A comma-separated string containing docker image tags to build and push (default = latest)
steps:
- run:
name: Confirm that environment variables are set
command: |
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
circleci-agent step halt
fi
- aws-cli/setup:
profile-name: <<parameters.profile-name>>
aws-access-key-id: <<parameters.aws-access-key-id>>
aws-secret-access-key: <<parameters.aws-secret-access-key>>
aws-region: <<parameters.region>>
- run:
name: Log into Amazon ECR
command: |
aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
- checkout
- setup_remote_docker:
version: 19.03.13
docker_layer_caching: false
- run:
name: Build docker image
command: |
registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
docker_tag_args=""
IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
for tag in "${DOCKER_TAGS[@]}"; do
docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
done
docker build \
<<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
-f <<parameters.path>>/<<parameters.dockerfile>> \
$docker_tag_args \
<<parameters.path>>
- run:
name: Push image to Amazon ECR
command: |
IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
for tag in "${DOCKER_TAGS[@]}"; do
docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
done
publish-packer-mainnet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
publish-packer-calibrationnet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
publish-packer-butterflynet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
publish-packer-nerpanet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
workflows:
version: 2.1
ci:
jobs:
- lint-all:
concurrency: "16" # expend all docker 2xlarge CPUs.
- mod-tidy-check
- gofmt
- gen-check
- docs-check
[[- range $file := .ItestFiles -]]
[[ with $name := $file | stripSuffix ]]
- test:
name: test-itest-[[ $name ]]
suite: itest-[[ $name ]]
target: "./itests/[[ $file ]]"
[[ end ]]
[[- end -]]
[[range $suite, $pkgs := .UnitSuites]]
- test:
name: test-[[ $suite ]]
suite: utest-[[ $suite ]]
target: "[[ $pkgs ]]"
[[- end]]
- test:
go-test-flags: "-run=TestMulticoreSDR"
suite: multicore-sdr-check
target: "./extern/sector-storage/ffiwrapper"
proofs-log-test: "1"
- test-conformance:
suite: conformance
codecov-upload: false
target: "./conformance"
- test-conformance:
name: test-conformance-bleeding-edge
codecov-upload: false
suite: conformance-bleeding-edge
target: "./conformance"
vectors-branch: master
- trigger-testplans:
filters:
branches:
only:
- master
- build-debug
- build-all:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-calibration:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-butterfly:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-nerpa:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-lotus-soup
- build-macos:
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-appimage:
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish:
requires:
- build-all
- build-macos
- build-appimage
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-and-push-image:
dockerfile: Dockerfile.lotus
path: .
repo: lotus-dev
tag: '${CIRCLE_SHA1:0:8}'
- publish-packer-mainnet:
requires:
- build-all
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-calibrationnet:
requires:
- build-ntwk-calibration
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-butterflynet:
requires:
- build-ntwk-butterfly
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-nerpanet:
requires:
- build-ntwk-nerpa
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-snapcraft:
name: publish-snapcraft-stable
channel: stable
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master
jobs:
- publish-snapcraft:
name: publish-snapcraft-nightly
channel: edge

1
.gitignore vendored
View File

@ -8,6 +8,7 @@
/lotus-health
/lotus-chainwatch
/lotus-shed
/lotus-sim
/lotus-pond
/lotus-townhall
/lotus-fountain

358
CHANGELOG.md
View File

@ -1,5 +1,363 @@
# Lotus changelog
# 1.11.0-rc1 / 2021-06-28
This is the first release candidate for the optional Lotus v1.11.0 release that introduces several months of bugfixes and feature development.
- github.com/filecoin-project/lotus:
- Lotus version 1.11.0
- gateway: Add support for Version method ([filecoin-project/lotus#6618](https://github.com/filecoin-project/lotus/pull/6618))
- Miner SimultaneousTransfers config ([filecoin-project/lotus#6612](https://github.com/filecoin-project/lotus/pull/6612))
- revamped integration test kit (aka. Operation Sparks Joy) ([filecoin-project/lotus#6329](https://github.com/filecoin-project/lotus/pull/6329))
- downgrade libp2p/go-libp2p-yamux to v0.5.1. ([filecoin-project/lotus#6605](https://github.com/filecoin-project/lotus/pull/6605))
- Fix wallet error messages ([filecoin-project/lotus#6594](https://github.com/filecoin-project/lotus/pull/6594))
- Fix CircleCI gen ([filecoin-project/lotus#6589](https://github.com/filecoin-project/lotus/pull/6589))
- Make query-ask CLI more graceful ([filecoin-project/lotus#6590](https://github.com/filecoin-project/lotus/pull/6590))
- ([filecoin-project/lotus#6406](https://github.com/filecoin-project/lotus/pull/6406))
- move with changed name ([filecoin-project/lotus#6587](https://github.com/filecoin-project/lotus/pull/6587))
- scale up sector expiration to avoid sector expire in batch-pre-commit waitting ([filecoin-project/lotus#6566](https://github.com/filecoin-project/lotus/pull/6566))
- Merge release branch into master ([filecoin-project/lotus#6583](https://github.com/filecoin-project/lotus/pull/6583))
- ([filecoin-project/lotus#6582](https://github.com/filecoin-project/lotus/pull/6582))
- fix circleci being out of sync. ([filecoin-project/lotus#6573](https://github.com/filecoin-project/lotus/pull/6573))
- dynamic circleci config for streamlining test execution ([filecoin-project/lotus#6561](https://github.com/filecoin-project/lotus/pull/6561))
- Merge 1.10 branch into master ([filecoin-project/lotus#6571](https://github.com/filecoin-project/lotus/pull/6571))
- Fix helptext ([filecoin-project/lotus#6560](https://github.com/filecoin-project/lotus/pull/6560))
- extern/storage: add ability to ignore worker resources when scheduling. ([filecoin-project/lotus#6542](https://github.com/filecoin-project/lotus/pull/6542))
- Merge 1.10 branch into master ([filecoin-project/lotus#6540](https://github.com/filecoin-project/lotus/pull/6540))
- Initial draft: basic build instructions on Readme ([filecoin-project/lotus#6498](https://github.com/filecoin-project/lotus/pull/6498))
- fix commit finalize failed ([filecoin-project/lotus#6521](https://github.com/filecoin-project/lotus/pull/6521))
- Dynamic Retrieval pricing ([filecoin-project/lotus#6175](https://github.com/filecoin-project/lotus/pull/6175))
- Fix soup ([filecoin-project/lotus#6501](https://github.com/filecoin-project/lotus/pull/6501))
- fix: pick the correct partitions-per-post limit ([filecoin-project/lotus#6502](https://github.com/filecoin-project/lotus/pull/6502))
- Fix the build
- Adjust various CLI display ratios to arbitrary precision ([filecoin-project/lotus#6309](https://github.com/filecoin-project/lotus/pull/6309))
- Add utils to use multisigs as miner owners ([filecoin-project/lotus#6490](https://github.com/filecoin-project/lotus/pull/6490))
- Test multicore SDR support ([filecoin-project/lotus#6479](https://github.com/filecoin-project/lotus/pull/6479))
- sealing: Fix restartSectors race ([filecoin-project/lotus#6495](https://github.com/filecoin-project/lotus/pull/6495))
- Merge 1.10 into master ([filecoin-project/lotus#6487](https://github.com/filecoin-project/lotus/pull/6487))
- Unit tests for sector batchers ([filecoin-project/lotus#6432](https://github.com/filecoin-project/lotus/pull/6432))
- Merge 1.10 changes into master ([filecoin-project/lotus#6466](https://github.com/filecoin-project/lotus/pull/6466))
- Update chain list with correct help instructions ([filecoin-project/lotus#6465](https://github.com/filecoin-project/lotus/pull/6465))
- clean failed sectors in batch commit ([filecoin-project/lotus#6451](https://github.com/filecoin-project/lotus/pull/6451))
- itests/kit: add guard to ensure imports from tests only. ([filecoin-project/lotus#6445](https://github.com/filecoin-project/lotus/pull/6445))
- consolidate integration tests into `itests` package; create test kit; cleanup ([filecoin-project/lotus#6311](https://github.com/filecoin-project/lotus/pull/6311))
- Remove rc changelog, compile the new changelog for final release only ([filecoin-project/lotus#6444](https://github.com/filecoin-project/lotus/pull/6444))
- updated configuration comments for docs ([filecoin-project/lotus#6440](https://github.com/filecoin-project/lotus/pull/6440))
- Set ntwk v13 HyperDrive Calibration upgrade epoch ([filecoin-project/lotus#6441](https://github.com/filecoin-project/lotus/pull/6441))
- Merge release/v1.10.10 into master ([filecoin-project/lotus#6439](https://github.com/filecoin-project/lotus/pull/6439))
- implement a command to export a car ([filecoin-project/lotus#6405](https://github.com/filecoin-project/lotus/pull/6405))
- Merge v1.10 release branch into master ([filecoin-project/lotus#6435](https://github.com/filecoin-project/lotus/pull/6435))
- Fee config for sector batching ([filecoin-project/lotus#6420](https://github.com/filecoin-project/lotus/pull/6420))
- Fix: correct the change of message size limit ([filecoin-project/lotus#6430](https://github.com/filecoin-project/lotus/pull/6430))
- UX: lotus state power CLI should fail if called with a not-miner ([filecoin-project/lotus#6425](https://github.com/filecoin-project/lotus/pull/6425))
- network reset friday
- Increase message size limit ([filecoin-project/lotus#6419](https://github.com/filecoin-project/lotus/pull/6419))
- polish(stmgr): define ExecMonitor for message application callback ([filecoin-project/lotus#6389](https://github.com/filecoin-project/lotus/pull/6389))
- upgrade testground action version ([filecoin-project/lotus#6403](https://github.com/filecoin-project/lotus/pull/6403))
- Fix logging of stringified CIDs double-encoded in hex ([filecoin-project/lotus#6413](https://github.com/filecoin-project/lotus/pull/6413))
- Update libp2p to 0.14.2 ([filecoin-project/lotus#6404](https://github.com/filecoin-project/lotus/pull/6404))
- Bypass task scheduler for reading unsealed pieces ([filecoin-project/lotus#6280](https://github.com/filecoin-project/lotus/pull/6280))
- testplans: lotus-soup: use default WPoStChallengeWindow ([filecoin-project/lotus#6400](https://github.com/filecoin-project/lotus/pull/6400))
- build snapcraft ([filecoin-project/lotus#6388](https://github.com/filecoin-project/lotus/pull/6388))
- Fix the doc errors of the sealing config funcs ([filecoin-project/lotus#6399](https://github.com/filecoin-project/lotus/pull/6399))
- Integration tests for offline deals ([filecoin-project/lotus#6081](https://github.com/filecoin-project/lotus/pull/6081))
- Fix success handling in Retreival ([filecoin-project/lotus#5921](https://github.com/filecoin-project/lotus/pull/5921))
- Fix some flaky tests ([filecoin-project/lotus#6397](https://github.com/filecoin-project/lotus/pull/6397))
- build appimage in CI ([filecoin-project/lotus#6384](https://github.com/filecoin-project/lotus/pull/6384))
- Add doc on gas balancing ([filecoin-project/lotus#6392](https://github.com/filecoin-project/lotus/pull/6392))
- Add a command to list retrievals ([filecoin-project/lotus#6337](https://github.com/filecoin-project/lotus/pull/6337))
- Add interop network ([filecoin-project/lotus#6387](https://github.com/filecoin-project/lotus/pull/6387))
- Network version 13 (v1.11) ([filecoin-project/lotus#6342](https://github.com/filecoin-project/lotus/pull/6342))
- Generate AppImage ([filecoin-project/lotus#6208](https://github.com/filecoin-project/lotus/pull/6208))
- lotus-gateway: add check command ([filecoin-project/lotus#6373](https://github.com/filecoin-project/lotus/pull/6373))
- Add a warning to the release issue template ([filecoin-project/lotus#6374](https://github.com/filecoin-project/lotus/pull/6374))
- update to markets-v1.4.0 ([filecoin-project/lotus#6369](https://github.com/filecoin-project/lotus/pull/6369))
- Add test for AddVerifiedClient ([filecoin-project/lotus#6317](https://github.com/filecoin-project/lotus/pull/6317))
- Typo fix in error message: "pubusb" -> "pubsub" ([filecoin-project/lotus#6365](https://github.com/filecoin-project/lotus/pull/6365))
- Improve the cli state call command ([filecoin-project/lotus#6226](https://github.com/filecoin-project/lotus/pull/6226))
- Upscale mineOne message to a WARN on unexpected ineligibility ([filecoin-project/lotus#6358](https://github.com/filecoin-project/lotus/pull/6358))
- storagefsm: Fix batch deal packing behavior ([filecoin-project/lotus#6041](https://github.com/filecoin-project/lotus/pull/6041))
- Remove few useless variable assignments ([filecoin-project/lotus#6359](https://github.com/filecoin-project/lotus/pull/6359))
- lotus-wallet: JWT Support ([filecoin-project/lotus#6360](https://github.com/filecoin-project/lotus/pull/6360))
- Reduce noise from 'peer has different genesis' messages ([filecoin-project/lotus#6357](https://github.com/filecoin-project/lotus/pull/6357))
- events: Fix handling of multiple matched events per epoch ([filecoin-project/lotus#6355](https://github.com/filecoin-project/lotus/pull/6355))
- Update RELEASE_ISSUE_TEMPLATE.md ([filecoin-project/lotus#6236](https://github.com/filecoin-project/lotus/pull/6236))
- Get current seal proof when necessary ([filecoin-project/lotus#6339](https://github.com/filecoin-project/lotus/pull/6339))
- Allow starting networks from arbitrary actor versions ([filecoin-project/lotus#6333](https://github.com/filecoin-project/lotus/pull/6333))
- Remove log line when tracing is not configured ([filecoin-project/lotus#6334](https://github.com/filecoin-project/lotus/pull/6334))
- Revert "Allow starting networks from arbitrary actor versions" ([filecoin-project/lotus#6330](https://github.com/filecoin-project/lotus/pull/6330))
- separate tracing environment variables ([filecoin-project/lotus#6323](https://github.com/filecoin-project/lotus/pull/6323))
- Allow starting networks from arbitrary actor versions ([filecoin-project/lotus#6305](https://github.com/filecoin-project/lotus/pull/6305))
- feat: log dispute rate ([filecoin-project/lotus#6322](https://github.com/filecoin-project/lotus/pull/6322))
- Use new actor tags ([filecoin-project/lotus#6291](https://github.com/filecoin-project/lotus/pull/6291))
- Fix logging around mineOne ([filecoin-project/lotus#6310](https://github.com/filecoin-project/lotus/pull/6310))
- Fix shell completions ([filecoin-project/lotus#6316](https://github.com/filecoin-project/lotus/pull/6316))
- Allow 8MB sectors in devnet ([filecoin-project/lotus#6312](https://github.com/filecoin-project/lotus/pull/6312))
- fix ticket expired ([filecoin-project/lotus#6304](https://github.com/filecoin-project/lotus/pull/6304))
- oh, snap! ([filecoin-project/lotus#6202](https://github.com/filecoin-project/lotus/pull/6202))
- Move verifreg shed utils to CLI ([filecoin-project/lotus#6135](https://github.com/filecoin-project/lotus/pull/6135))
- consider storiface.PathStorage when calculating storage requirements ([filecoin-project/lotus#6233](https://github.com/filecoin-project/lotus/pull/6233))
- `storage` module: add go docs and minor code quality refactors ([filecoin-project/lotus#6259](https://github.com/filecoin-project/lotus/pull/6259))
- Revert "chore: update go-libp2p" ([filecoin-project/lotus#6306](https://github.com/filecoin-project/lotus/pull/6306))
- Increase data transfer timeouts ([filecoin-project/lotus#6300](https://github.com/filecoin-project/lotus/pull/6300))
- gateway: spin off from cmd to package ([filecoin-project/lotus#6294](https://github.com/filecoin-project/lotus/pull/6294))
- Update to markets 1.3 ([filecoin-project/lotus#6149](https://github.com/filecoin-project/lotus/pull/6149))
- Add a shed util to count 64 GiB miner stats ([filecoin-project/lotus#6290](https://github.com/filecoin-project/lotus/pull/6290))
- Delete CODEOWNERS ([filecoin-project/lotus#6289](https://github.com/filecoin-project/lotus/pull/6289))
- Merge v1.9.0 to master ([filecoin-project/lotus#6275](https://github.com/filecoin-project/lotus/pull/6275))
- Backport 6200 to master ([filecoin-project/lotus#6272](https://github.com/filecoin-project/lotus/pull/6272))
- Introduce stateless offline dealflow, bypassing the FSM/deallists ([filecoin-project/lotus#5961](https://github.com/filecoin-project/lotus/pull/5961))
- chore: update go-libp2p ([filecoin-project/lotus#6231](https://github.com/filecoin-project/lotus/pull/6231))
- fix: wait-api should use GetAPI to acquire binary specific API ([filecoin-project/lotus#6246](https://github.com/filecoin-project/lotus/pull/6246))
- Update RELEASE_ISSUE_TEMPLATE.md
- fix(ci): Updates to lotus CI build process ([filecoin-project/lotus#6256](https://github.com/filecoin-project/lotus/pull/6256))
- add flags to control gateway lookback parameters ([filecoin-project/lotus#6247](https://github.com/filecoin-project/lotus/pull/6247))
- Feat/nerpa v4 ([filecoin-project/lotus#6248](https://github.com/filecoin-project/lotus/pull/6248))
- chore(ci): Enable build on RC tags ([filecoin-project/lotus#6238](https://github.com/filecoin-project/lotus/pull/6238))
- Transplant some useful commands to lotus-shed actor ([filecoin-project/lotus#5913](https://github.com/filecoin-project/lotus/pull/5913))
- wip actor wrapper codegen ([filecoin-project/lotus#6108](https://github.com/filecoin-project/lotus/pull/6108))
- Robust message management ([filecoin-project/lotus#5822](https://github.com/filecoin-project/lotus/pull/5822))
- Add a shed util to count miners by post type ([filecoin-project/lotus#6169](https://github.com/filecoin-project/lotus/pull/6169))
- Introduce a release issue template ([filecoin-project/lotus#5826](https://github.com/filecoin-project/lotus/pull/5826))
- cron-wc ([filecoin-project/lotus#6178](https://github.com/filecoin-project/lotus/pull/6178))
- This is a 1:1 forward-port of PR#6183 from 1.9.x to master ([filecoin-project/lotus#6196](https://github.com/filecoin-project/lotus/pull/6196))
- Allow creation of state tree v3s ([filecoin-project/lotus#6167](https://github.com/filecoin-project/lotus/pull/6167))
- drand: fix beacon cache ([filecoin-project/lotus#6164](https://github.com/filecoin-project/lotus/pull/6164))
- Update cli gen ([filecoin-project/lotus#6155](https://github.com/filecoin-project/lotus/pull/6155))
- mpool: Cleanup pre-nv12 selection logic ([filecoin-project/lotus#6148](https://github.com/filecoin-project/lotus/pull/6148))
- Update ffi to proofs v7 ([filecoin-project/lotus#6150](https://github.com/filecoin-project/lotus/pull/6150))
- Generate CLI docs ([filecoin-project/lotus#6145](https://github.com/filecoin-project/lotus/pull/6145))
- feat: allow checkpointing to forks ([filecoin-project/lotus#6107](https://github.com/filecoin-project/lotus/pull/6107))
- attempt to do better padding on pieces being written into sectors ([filecoin-project/lotus#5988](https://github.com/filecoin-project/lotus/pull/5988))
- remove duplicate ask and calculate ping before lock ([filecoin-project/lotus#5968](https://github.com/filecoin-project/lotus/pull/5968))
- Add a command to get the fees of a deal ([filecoin-project/lotus#5307](https://github.com/filecoin-project/lotus/pull/5307))
- flaky tests improvement: separate TestBatchDealInput from TestAPIDealFlow ([filecoin-project/lotus#6141](https://github.com/filecoin-project/lotus/pull/6141))
- Testground checks on push ([filecoin-project/lotus#5887](https://github.com/filecoin-project/lotus/pull/5887))
- Add a CLI tool for miner proving deadline ([filecoin-project/lotus#6132](https://github.com/filecoin-project/lotus/pull/6132))
- Use EmptyTSK where appropriate ([filecoin-project/lotus#6134](https://github.com/filecoin-project/lotus/pull/6134))
- fix: use a consistent tipset in commands ([filecoin-project/lotus#6142](https://github.com/filecoin-project/lotus/pull/6142))
- go mod tidy for lotus-soup testplans ([filecoin-project/lotus#6124](https://github.com/filecoin-project/lotus/pull/6124))
- fix testground payment channel tests: use 1 miner ([filecoin-project/lotus#6126](https://github.com/filecoin-project/lotus/pull/6126))
- fix: use the parent state when listing actors ([filecoin-project/lotus#6143](https://github.com/filecoin-project/lotus/pull/6143))
- Speed up StateListMessages in some cases ([filecoin-project/lotus#6007](https://github.com/filecoin-project/lotus/pull/6007))
- Return total power when GetPowerRaw doesn't find miner claim ([filecoin-project/lotus#4938](https://github.com/filecoin-project/lotus/pull/4938))
- fix(splitstore): fix a panic on revert-only head changes ([filecoin-project/lotus#6133](https://github.com/filecoin-project/lotus/pull/6133))
- shed: command to list duplicate messages in tipsets (steb) ([filecoin-project/lotus#5847](https://github.com/filecoin-project/lotus/pull/5847))
- upgrade `lotus-soup` testplans and reduce deals concurrency to a single miner ([filecoin-project/lotus#6122](https://github.com/filecoin-project/lotus/pull/6122))
- Merge releases (1.8.0) into master ([filecoin-project/lotus#6118](https://github.com/filecoin-project/lotus/pull/6118))
- github.com/filecoin-project/go-commp-utils (v0.1.0 -> v0.1.1-0.20210427191551-70bf140d31c7):
- add a padding helper function ([filecoin-project/go-commp-utils#3](https://github.com/filecoin-project/go-commp-utils/pull/3))
- github.com/filecoin-project/go-data-transfer (v1.4.3 -> v1.6.0):
- release: v1.6.0
- fix: option to disable accept and complete timeouts
- fix: disable restart ack timeout
- release: v1.5.0
- Add isRestart param to validators (#197) ([filecoin-project/go-data-transfer#197](https://github.com/filecoin-project/go-data-transfer/pull/197))
- fix: flaky TestChannelMonitorAutoRestart (#198) ([filecoin-project/go-data-transfer#198](https://github.com/filecoin-project/go-data-transfer/pull/198))
- Channel monitor watches for errors instead of measuring data rate (#190) ([filecoin-project/go-data-transfer#190](https://github.com/filecoin-project/go-data-transfer/pull/190))
- fix: prevent concurrent restarts for same channel (#195) ([filecoin-project/go-data-transfer#195](https://github.com/filecoin-project/go-data-transfer/pull/195))
- fix: channel state machine event handling (#194) ([filecoin-project/go-data-transfer#194](https://github.com/filecoin-project/go-data-transfer/pull/194))
- Dont double count data sent (#185) ([filecoin-project/go-data-transfer#185](https://github.com/filecoin-project/go-data-transfer/pull/185))
- release: v1.4.3 (#189) ([filecoin-project/go-data-transfer#189](https://github.com/filecoin-project/go-data-transfer/pull/189))
- github.com/filecoin-project/go-fil-markets (v1.2.5 -> v1.5.0):
- release: v1.5.0
- Dynamic Retrieval Pricing (#542) ([filecoin-project/go-fil-markets#542](https://github.com/filecoin-project/go-fil-markets/pull/542))
- release: v1.4.0 (#551) ([filecoin-project/go-fil-markets#551](https://github.com/filecoin-project/go-fil-markets/pull/551))
- Update to go data transfer v1.6.0 (#550) ([filecoin-project/go-fil-markets#550](https://github.com/filecoin-project/go-fil-markets/pull/550))
- fix first make error (#548) ([filecoin-project/go-fil-markets#548](https://github.com/filecoin-project/go-fil-markets/pull/548))
- release: v1.3.0 (#544) ([filecoin-project/go-fil-markets#544](https://github.com/filecoin-project/go-fil-markets/pull/544))
- fix restarts during data transfer for a retrieval deal (#540) ([filecoin-project/go-fil-markets#540](https://github.com/filecoin-project/go-fil-markets/pull/540))
- Test Retrieval for offline deals (#541) ([filecoin-project/go-fil-markets#541](https://github.com/filecoin-project/go-fil-markets/pull/541))
- Allow anonymous submodule checkout (#535) ([filecoin-project/go-fil-markets#535](https://github.com/filecoin-project/go-fil-markets/pull/535))
- github.com/filecoin-project/specs-actors (v0.9.13 -> v0.9.14):
- Set ConsensusMinerMinPower to 10 TiB (#1427) ([filecoin-project/specs-actors#1427](https://github.com/filecoin-project/specs-actors/pull/1427))
- github.com/filecoin-project/specs-actors/v2 (v2.3.5-0.20210114162132-5b58b773f4fb -> v2.3.5):
- Set ConsensusMinerMinPower to 10 TiB (#1428) ([filecoin-project/specs-actors#1428](https://github.com/filecoin-project/specs-actors/pull/1428))
- v2 VM satisfies SimVM interface (#1355) ([filecoin-project/specs-actors#1355](https://github.com/filecoin-project/specs-actors/pull/1355))
- github.com/filecoin-project/specs-actors/v3 (v3.1.0 -> v3.1.1):
- Set ConsensusMinerMinPower to 10 TiB for all PoStProofPolicies (#1429) ([filecoin-project/specs-actors#1429](https://github.com/filecoin-project/specs-actors/pull/1429))
- github.com/filecoin-project/specs-actors/v4 (v4.0.0 -> v4.0.1):
- Set ConsensusMinerMinPower to 10 TiB for all PoStProofPolicies (#1430) ([filecoin-project/specs-actors#1430](https://github.com/filecoin-project/specs-actors/pull/1430))
Contributors
| Contributor | Commits | Lines ± | Files Changed |
|-------------|---------|---------|---------------|
| Raúl Kripalani | 118 | +11972/-10860 | 472 |
| Łukasz Magiera | 65 | +10824/-4158 | 353 |
| aarshkshah1992 | 59 | +8057/-3355 | 224 |
| Aayush Rajasekaran | 41 | +8786/-1691 | 331 |
| Steven Allen | 106 | +7653/-2718 | 273 |
| dirkmc | 11 | +2580/-1371 | 77 |
| Dirk McCormick | 39 | +1865/-1194 | 79 |
| Jakub Sztandera | 19 | +1973/-485 | 81 |
| vyzo | 4 | +1748/-330 | 50 |
| Aarsh Shah | 5 | +1462/-213 | 27 |
| Cory Schwartz | 35 | +568/-206 | 59 |
| chadwick2143 | 3 | +739/-1 | 4 |
| Peter Rabbitson | 21 | +487/-164 | 36 |
| hannahhoward | 5 | +544/-5 | 19 |
| Jennifer Wang | 8 | +206/-172 | 17 |
| frrist | 1 | +137/-88 | 7 |
| Travis Person | 3 | +175/-6 | 7 |
| Alex Wade | 1 | +48/-129 | 1 |
| whyrusleeping | 8 | +161/-13 | 11 |
| lotus | 1 | +114/-46 | 1 |
| Anton Evangelatov | 8 | +107/-53 | 20 |
| Rjan | 4 | +115/-33 | 4 |
| ZenGround0 | 3 | +114/-1 | 4 |
| Aloxaf | 1 | +43/-61 | 7 |
| yaohcn | 4 | +89/-9 | 5 |
| mitchellsoo | 1 | +51/-0 | 1 |
| Mike Greenberg | 3 | +28/-18 | 4 |
| Jennifer | 6 | +9/-14 | 6 |
| Frank | 2 | +11/-10 | 2 |
| wangchao | 3 | +5/-4 | 4 |
| Steve Loeppky | 1 | +7/-1 | 1 |
| Lion | 1 | +4/-2 | 1 |
| Mimir | 1 | +2/-2 | 1 |
| raulk | 1 | +1/-1 | 1 |
| Jack Yao | 1 | +1/-1 | 1 |
| IPFSUnion | 1 | +1/-1 | 1 |
# 1.10.0 / 2021-06-23
This is a mandatory release of Lotus that introduces Filecoin network v13, codenamed the HyperDrive upgrade. The
Filecoin mainnet will upgrade, which is epoch 892800, on 2021-06-30T22:00:00Z. The network upgrade introduces the
following FIPs:
- [FIP-0008](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0008.md): Add miner batched sector pre-commit method
- [FIP-0011](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0011.md): Remove reward auction from reporting consensus faults
- [FIP-0012](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0012.md): DataCap Top up for FIL+ Client Addresses
- [FIP-0013](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md): Add ProveCommitSectorAggregated method to reduce on-chain congestion
- [FIP-0015](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0015.md): Revert FIP-0009(Exempt Window PoSts from BaseFee burn)
Note that this release is built on top of Lotus v1.9.0. Enterprising users can use the `master` branch of Lotus to get the latest functionality, including all changes in this release candidate.
## Proof batching and aggregation
FIPs [0008](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0008.md) and [0013](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md) combine to allow for a significant increase in the rate of onboarding storage on the Filecoin network. This aims to lead to more useful data being stored on the network, reduced network congestion, and lower network base fee.
**Check out the documentation [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#precommitsectorsbatch) for details on the new Lotus miner sealing config options, [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#fees-section) for fee config options, and explanations of the new features.**
Note:
- We recommend to keep `PreCommitSectorsBatch` as 1.
- We recommend miners to set `PreCommitBatchWait` lower than 30 hours.
- We recommend miners to set a longer `CommitBatchSlack` and `PreCommitBatchSlack` to prevent message failures
due to expirations.
### Projected state tree growth
In order to validate the Hyperdrive changes, we wrote a simulation to seal as many sectors as quickly as possible, assuming the same number and mix of 32GiB and 64GiB miners as the current network.
Given these assumptions:
- We'd expect a network storage growth rate of around 530PiB per day. 😳 🎉 🥳 😅
- We'd expect network bandwidth dedicated to `SubmitWindowedPoSt` to grow by about 0.02% per day.
- We'd expect the [state-tree](https://spec.filecoin.io/#section-systems.filecoin_vm.state_tree) (and therefore [snapshot](https://docs.filecoin.io/get-started/lotus/chain/#lightweight-snapshot)) size to grow by 1.16GiB per day.
- Nearly all of the state-tree growth is expected to come from new sector metadata.
- We'd expect the daily lotus datastore growth rate to increase by about 10-15% (from current ~21GiB/day).
- Most "growth" of the lotus datastore is due to "churn", historical data that's no longer referenced by the latest state-tree.
### Future improvements
Various Lotus improvements are planned moving forward to mitigate the effects of the growing state tree size. The primary improvement is the [Lotus splitstore](https://github.com/filecoin-project/lotus/discussions/5788), which will soon be enabled by default. The feature allows for [online garbage collection](https://github.com/filecoin-project/lotus/issues/6577) for nodes that do not seek to maintain full chain and state history, thus eliminating the need for users to delete their datastores and sync from snapshots.
Other improvements including better compressed snapshots, faster pre-migrations, and improved chain exports are in the roadmap.
## WindowPost base fee burn
Included in the HyperDrive upgrade is [FIP-0015](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0015.md) which eliminates the special-case gas treatment of `SubmitWindowedPoSt` messages that was introduced in [FIP-0009](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0009.md). Although `SubmitWindowedPoSt` messages will be relatively cheap, thanks to the introduction of optimistic acceptance of these proofs in [FIP-0010](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0010.md), storage providers should pay attention to their `MaxWindowPoStGasFee` config option: too low and PoSts may not land on chain; too high and they may cost an exorbitant amount!
## Changelog
### New Features
- Implement FIP-0015 ([filecoin-project/lotus#6361](https://github.com/filecoin-project/lotus/pull/6361))
- Integrate FIP0013 and FIP0008 ([filecoin-project/lotus#6235](https://github.com/filecoin-project/lotus/pull/6235))
- [Configuration docs and cli examples](https://docs.filecoin.io/mine/lotus/miner-configuration/#precommitsectorsbatch)
- [cli docs](https://github.com/filecoin-project/lotus/blob/master/documentation/en/cli-lotus-miner.md#lotus-miner-sectors-batching)
- Introduce gas prices for aggregate verifications ([filecoin-project/lotus#6347](https://github.com/filecoin-project/lotus/pull/6347))
- Introduce v5 actors ([filecoin-project/lotus#6195](https://github.com/filecoin-project/lotus/pull/6195))
- Robustify commit batcher ([filecoin-project/lotus#6367](https://github.com/filecoin-project/lotus/pull/6367))
- Always flush when timer goes off ([filecoin-project/lotus#6563](https://github.com/filecoin-project/lotus/pull/6563))
- Update default fees for aggregates ([filecoin-project/lotus#6548](https://github.com/filecoin-project/lotus/pull/6548))
- sealing: Early finalization option ([filecoin-project/lotus#6452](https://github.com/filecoin-project/lotus/pull/6452))
- `./lotus-miner/config.toml/[Sealing.FinalizeEarly]`: default to false. Enable if you want to FinalizeSector before commiting
- Add filplus utils to CLI ([filecoin-project/lotus#6351](https://github.com/filecoin-project/lotus/pull/6351))
- cli doc can be found [here](https://github.com/filecoin-project/lotus/blob/master/documentation/en/cli-lotus.md#lotus-filplus)
- Add miner-side MaxDealStartDelay config ([filecoin-project/lotus#6576](https://github.com/filecoin-project/lotus/pull/6576))
### Bug Fixes
- chainstore: Don't take heaviestLk with backlogged reorgCh ([filecoin-project/lotus#6526](https://github.com/filecoin-project/lotus/pull/6526))
- Backport #6041 - storagefsm: Fix batch deal packing behavior ([filecoin-project/lotus#6519](https://github.com/filecoin-project/lotus/pull/6519))
- backport: pick the correct partitions-per-post limit ([filecoin-project/lotus#6503](https://github.com/filecoin-project/lotus/pull/6503))
- failed sectors should be added into res correctly ([filecoin-project/lotus#6472](https://github.com/filecoin-project/lotus/pull/6472))
- sealing: Fix restartSectors race ([filecoin-project/lotus#6491](https://github.com/filecoin-project/lotus/pull/6491))
- Fund miners with the aggregate fee when ProveCommitting ([filecoin-project/lotus#6428](https://github.com/filecoin-project/lotus/pull/6428))
- Commit and Precommit batcher cannot share a getSectorDeadline method ([filecoin-project/lotus#6416](https://github.com/filecoin-project/lotus/pull/6416))
- Fix supported proof type manipulations for v5 actors ([filecoin-project/lotus#6366](https://github.com/filecoin-project/lotus/pull/6366))
- events: Fix handling of multiple matched events per epoch ([filecoin-project/lotus#6362](https://github.com/filecoin-project/lotus/pull/6362))
- Fix randomness fetching around null blocks ([filecoin-project/lotus#6240](https://github.com/filecoin-project/lotus/pull/6240))
### Improvements
- Appimage v1.10.0 rc3 ([filecoin-project/lotus#6492](https://github.com/filecoin-project/lotus/pull/6492))
- Expand on Drand change testing ([filecoin-project/lotus#6500](https://github.com/filecoin-project/lotus/pull/6500))
- Backport Fix logging around mineOne ([filecoin-project/lotus#6499](https://github.com/filecoin-project/lotus/pull/6499))
- mpool: Add more metrics ([filecoin-project/lotus#6453](https://github.com/filecoin-project/lotus/pull/6453))
- Merge backported PRs into v1.10 release branch ([filecoin-project/lotus#6436](https://github.com/filecoin-project/lotus/pull/6436))
- Fix tests ([filecoin-project/lotus#6371](https://github.com/filecoin-project/lotus/pull/6371))
- Extend the default deal start epoch delay ([filecoin-project/lotus#6350](https://github.com/filecoin-project/lotus/pull/6350))
- sealing: Wire up context to batchers ([filecoin-project/lotus#6497](https://github.com/filecoin-project/lotus/pull/6497))
- Improve address resolution for messages ([filecoin-project/lotus#6364](https://github.com/filecoin-project/lotus/pull/6364))
### Dependency Updates
- Proofs v8.0.2 ([filecoin-project/lotus#6524](https://github.com/filecoin-project/lotus/pull/6524))
- Update to fixed Bellperson ([filecoin-project/lotus#6480](https://github.com/filecoin-project/lotus/pull/6480))
- Update to go-praamfetch with fslocks ([filecoin-project/lotus#6473](https://github.com/filecoin-project/lotus/pull/6473))
- Update ffi with fixed multicore sdr support ([filecoin-project/lotus#6471](https://github.com/filecoin-project/lotus/pull/6471))
- github.com/filecoin-project/go-paramfetch (v0.0.2-0.20200701152213-3e0f0afdc261 -> v0.0.2-0.20210614165157-25a6c7769498)
- github.com/filecoin-project/specs-actors/v5 (v5.0.0-20210512015452-4fe3889fff57 -> v5.0.0)
- github.com/filecoin-project/go-hamt-ipld/v3 (v3.0.1 -> v3.1.0)
- github.com/ipfs/go-log/v2 (v2.1.2-0.20200626104915-0016c0b4b3e4 -> v2.1.3)
- github.com/filecoin-project/go-amt-ipld/v3 (v3.0.0 -> v3.1.0)
### Network Version v13 HyperDrive Upgrade
- Set HyperDrive upgrade epoch ([filecoin-project/lotus#6565](https://github.com/filecoin-project/lotus/pull/6565))
- version bump to lotus v1.10.0-rc6 ([filecoin-project/lotus#6529](https://github.com/filecoin-project/lotus/pull/6529))
- Upgrade epochs for calibration reset ([filecoin-project/lotus#6528](https://github.com/filecoin-project/lotus/pull/6528))
- Lotus version 1.10.0-rc5 ([filecoin-project/lotus#6504](https://github.com/filecoin-project/lotus/pull/6504))
- Merge releases into v1.10 release ([filecoin-project/lotus#6494](https://github.com/filecoin-project/lotus/pull/6494))
- update lotus to v1.10.0-rc3 ([filecoin-project/lotus#6481](https://github.com/filecoin-project/lotus/pull/6481))
- updated configuration comments for docs
- Lotus version 1.10.0-rc2 ([filecoin-project/lotus#6443](https://github.com/filecoin-project/lotus/pull/6443))
- Set ntwk v13 HyperDrive Calibration upgrade epoch ([filecoin-project/lotus#6442](https://github.com/filecoin-project/lotus/pull/6442))
## Contributors
💙Thank you to all the contributors!
| Contributor | Commits | Lines ± | Files Changed |
|--------------------|---------|-------------|---------------|
| @magik6k | 81 | +9606/-1536 | 361 |
| @arajasek | 41 | +6543/-679 | 189 |
| @ZenGround0 | 11 | +4074/-727 | 110 |
| @anorth | 10 | +2035/-1177 | 55 |
| @iand | 1 | +779/-12 | 5 |
| @frrist | 2 | +722/-6 | 6 |
| @Stebalien | 6 | +368/-24 | 15 |
| @jennijuju | 11 | +204/-111 | 19 |
| @vyzo | 6 | +155/-66 | 13 |
| @coryschwartz | 10 | +171/-27 | 14 |
| @Kubuxu | 4 | +177/-13 | 7 |
| @ribasushi | 4 | +65/-42 | 5 |
| @travisperson | 2 | +11/-11 | 4 |
| @kirk-baird | 1 | +1/-5 | 1 |
| @wangchao | 2 | +3/-2 | 2 |
# 1.9.0 / 2021-05-17
This is an optional Lotus release that introduces various improvements to the sealing, mining, and deal-making processes.

13
Makefile
View File

@ -42,7 +42,7 @@ BUILD_DEPS+=build/.filecoin-install
CLEAN+=build/.filecoin-install
ffi-version-check:
@[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 2 ]] || (echo "FFI version mismatch, update submodules"; exit 1)
@[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 3 ]] || (echo "FFI version mismatch, update submodules"; exit 1)
BUILD_DEPS+=ffi-version-check
.PHONY: ffi-version-check
@ -234,6 +234,12 @@ BINS+=tvx
install-chainwatch: lotus-chainwatch
install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
lotus-sim: $(BUILD_DEPS)
rm -f lotus-sim
go build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim
.PHONY: lotus-sim
BINS+=lotus-sim
# SYSTEMD
install-daemon-service: install-daemon
@ -364,7 +370,7 @@ docsgen-openrpc-worker: docsgen-openrpc-bin
.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
gen: actors-gen type-gen method-gen docsgen api-gen
gen: actors-gen type-gen method-gen docsgen api-gen circleci
@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
.PHONY: gen
@ -379,3 +385,6 @@ docsgen-cli: lotus lotus-miner lotus-worker
print-%:
@echo $*=$($*)
circleci:
go generate -x ./.circleci

84
README.md
View File

@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
## Building & Documentation
For instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/).
For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
## Reporting a Vulnerability
@ -50,6 +50,88 @@ When implementing a change:
7. Title the PR in a meaningful way and describe the rationale and the thought process in the PR description.
8. Write clean, thoughtful, and detailed [commit messages](https://chris.beams.io/posts/git-commit/). This is even more important than the PR description, because commit messages are stored _inside_ the Git history. One good rule is: if you are happy posting the commit message as the PR description, then it's a good commit message.
## Basic Build Instructions
**System-specific Software Dependencies**:
Building Lotus requires some system dependencies, usually provided by your distribution.
Ubuntu/Debian:
```
sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
```
Fedora:
```
sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
```
For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
#### Go
To build Lotus, you need a working installation of [Go 1.16.4 or higher](https://golang.org/dl/):
```bash
wget -c https://golang.org/dl/go1.16.4.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
```
**TIP:**
You'll need to add `/usr/local/go/bin` to your path. For most Linux distributions you can run something like:
```shell
echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.bashrc && source ~/.bashrc
```
See the [official Golang installation instructions](https://golang.org/doc/install) if you get stuck.
### Build and install Lotus
Once all the dependencies are installed, you can build and install the Lotus suite (`lotus`, `lotus-miner`, and `lotus-worker`).
1. Clone the repository:
```sh
git clone https://github.com/filecoin-project/lotus.git
cd lotus/
```
Note: The default branch `master` is the dev branch where the latest new features, bug fixes and improvement are in. However, if you want to run lotus on Filecoin mainnet and want to run a production-ready lotus, get the latest release[ here](https://github.com/filecoin-project/lotus/releases).
2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases).
If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding.
For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below.
```sh
git checkout <tag_or_branch>
# For example:
git checkout <vX.X.X> # tag for a release
```
Currently, the latest code on the _master_ branch corresponds to mainnet.
3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)".
4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed.
5. Build and install Lotus:
```sh
make clean all #mainnet
# Or to join a testnet or devnet:
make clean calibnet # Calibration with min 32GiB sectors
make clean nerpanet # Nerpa with min 512MiB sectors
sudo make install
```
This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`.
`lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder.
6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain).
## License
Dual-licensed under [MIT](https://github.com/filecoin-project/lotus/blob/master/LICENSE-MIT) + [Apache 2.0](https://github.com/filecoin-project/lotus/blob/master/LICENSE-APACHE)

1
api/api_gateway.go
View File

@ -58,4 +58,5 @@ type Gateway interface {
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
WalletBalance(context.Context, address.Address) (types.BigInt, error)
Version(context.Context) (APIVersion, error)
}

10
api/proxy_gen.go
View File

@ -531,6 +531,8 @@ type GatewayStruct struct {
StateWaitMsg func(p0 context.Context, p1 cid.Cid, p2 uint64, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) ``
Version func(p0 context.Context) (APIVersion, error) ``
WalletBalance func(p0 context.Context, p1 address.Address) (types.BigInt, error) ``
}
}
@ -2679,6 +2681,14 @@ func (s *GatewayStub) StateWaitMsg(p0 context.Context, p1 cid.Cid, p2 uint64, p3
return nil, xerrors.New("method not supported")
}
func (s *GatewayStruct) Version(p0 context.Context) (APIVersion, error) {
return s.Internal.Version(p0)
}
func (s *GatewayStub) Version(p0 context.Context) (APIVersion, error) {
return *new(APIVersion), xerrors.New("method not supported")
}
func (s *GatewayStruct) WalletBalance(p0 context.Context, p1 address.Address) (types.BigInt, error) {
return s.Internal.WalletBalance(p0, p1)
}

1
api/v0api/gateway.go
View File

@ -63,6 +63,7 @@ type Gateway interface {
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error)
WalletBalance(context.Context, address.Address) (types.BigInt, error)
Version(context.Context) (api.APIVersion, error)
}
var _ Gateway = *new(FullNode)

10
api/v0api/proxy_gen.go
View File

@ -449,6 +449,8 @@ type GatewayStruct struct {
StateWaitMsg func(p0 context.Context, p1 cid.Cid, p2 uint64) (*api.MsgLookup, error) ``
Version func(p0 context.Context) (api.APIVersion, error) ``
WalletBalance func(p0 context.Context, p1 address.Address) (types.BigInt, error) ``
}
}
@ -2096,6 +2098,14 @@ func (s *GatewayStub) StateWaitMsg(p0 context.Context, p1 cid.Cid, p2 uint64) (*
return nil, xerrors.New("method not supported")
}
func (s *GatewayStruct) Version(p0 context.Context) (api.APIVersion, error) {
return s.Internal.Version(p0)
}
func (s *GatewayStub) Version(p0 context.Context) (api.APIVersion, error) {
return *new(api.APIVersion), xerrors.New("method not supported")
}
func (s *GatewayStruct) WalletBalance(p0 context.Context, p1 address.Address) (types.BigInt, error) {
return s.Internal.WalletBalance(p0, p1)
}

21
api/wrap.go
View File

@ -26,6 +26,27 @@ func Wrap(proxyT, wrapperT, impl interface{}) interface{} {
}))
}
for i := 0; i < proxy.Elem().NumField(); i++ {
if proxy.Elem().Type().Field(i).Name == "Internal" {
continue
}
subProxy := proxy.Elem().Field(i).FieldByName("Internal")
for i := 0; i < ri.NumMethod(); i++ {
mt := ri.Type().Method(i)
if subProxy.FieldByName(mt.Name).Kind() == reflect.Invalid {
continue
}
fn := ri.Method(i)
of := subProxy.FieldByName(mt.Name)
subProxy.FieldByName(mt.Name).Set(reflect.MakeFunc(of.Type(), func(args []reflect.Value) (results []reflect.Value) {
return fn.Call(args)
}))
}
}
wp := reflect.New(reflect.TypeOf(wrapperT).Elem())
wp.Elem().Field(0).Set(proxy)
return wp.Interface()

8
build/bootstrap/calibnet.pi
View File

@ -1,4 +1,4 @@
/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWRLZAseMo9h7fRD6ojn6YYDXHsBSavX5YmjBZ9ngtAEec
/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWJFtDXgZEQMEkjJPSrbfdvh2xfjVKrXeNFG1t8ioJXAzv
/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWP1uB9Lo7yCA3S17TD4Y5wStP5Nk7Vqh53m8GsFjkyujD
/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWLrPM4WPK1YRGPCUwndWcDX8GCYgms3DiuofUmxwvhMCn
/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWJkikQQkxS58spo76BYzFt4fotaT5NpV2zngvrqm4u5ow
/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWLce5FDHR4EX4CrYavphA5xS3uDsX6aoowXh5tzDUxJav
/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWA9hFfQG9GjP6bHeuQQbMD3FDtZLdW1NayxKXUT26PQZu
/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWMHDi3LVTFG8Szqogt7RkNXvonbQYqSazxBx41A5aeuVz

BIN
build/genesis/calibnet.car
View File

Binary file not shown.

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

14
build/params_calibnet.go
View File

@ -33,20 +33,20 @@ const UpgradeLiftoffHeight = -5
const UpgradeKumquatHeight = 90
const UpgradeCalicoHeight = 100
const UpgradeCalicoHeight = 120
const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1)
const UpgradeClausHeight = 250
const UpgradeClausHeight = 270
const UpgradeOrangeHeight = 300
const UpgradeTrustHeight = 600
const UpgradeNorwegianHeight = 114000
const UpgradeTrustHeight = 330
const UpgradeTurboHeight = 193789
const UpgradeNorwegianHeight = 360
// 2021-06-11T14:30:00Z
const UpgradeHyperdriveHeight = 321519
const UpgradeTurboHeight = 390
const UpgradeHyperdriveHeight = 420
func init() {
policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))

4
build/params_mainnet.go
View File

@ -62,8 +62,8 @@ const UpgradeNorwegianHeight = 665280
// 2021-04-29T06:00:00Z
const UpgradeTurboHeight = 712320
// ???
var UpgradeHyperdriveHeight = abi.ChainEpoch(9999999)
// 2021-06-30T22:00:00Z
var UpgradeHyperdriveHeight = abi.ChainEpoch(892800)
func init() {
if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {

2
build/version.go
View File

@ -34,7 +34,7 @@ func buildType() string {
}
// BuildVersion is the local build version, set by build system
const BuildVersion = "1.11.0-dev"
const BuildVersion = "1.11.0-rc1"
func UserVersion() string {
if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

6
chain/actors/builtin/miner/actor.go.template
View File

@ -22,6 +22,7 @@ import (
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
miner3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner"
miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
{{range .versions}}
builtin{{.}} "github.com/filecoin-project/specs-actors{{import .}}actors/builtin"
{{end}}
@ -97,9 +98,13 @@ type State interface {
FindSector(abi.SectorNumber) (*SectorLocation, error)
GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error)
GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error)
ForEachPrecommittedSector(func(SectorPreCommitOnChainInfo) error) error
LoadSectors(sectorNos *bitfield.BitField) ([]*SectorOnChainInfo, error)
NumLiveSectors() (uint64, error)
IsAllocated(abi.SectorNumber) (bool, error)
// UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
// count if there aren't enough).
UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
GetProvingPeriodStart() (abi.ChainEpoch, error)
@ -176,6 +181,7 @@ type DeclareFaultsRecoveredParams = miner0.DeclareFaultsRecoveredParams
type SubmitWindowedPoStParams = miner0.SubmitWindowedPoStParams
type ProveCommitSectorParams = miner0.ProveCommitSectorParams
type DisputeWindowedPoStParams = miner3.DisputeWindowedPoStParams
type ProveCommitAggregateParams = miner5.ProveCommitAggregateParams
func PreferredSealProofTypeFromWindowPoStType(nver network.Version, proof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
// We added support for the new proofs in network version 7, and removed support for the old

6
chain/actors/builtin/miner/miner.go
View File

@ -22,6 +22,7 @@ import (
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
miner3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner"
miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
@ -156,9 +157,13 @@ type State interface {
FindSector(abi.SectorNumber) (*SectorLocation, error)
GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error)
GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error)
ForEachPrecommittedSector(func(SectorPreCommitOnChainInfo) error) error
LoadSectors(sectorNos *bitfield.BitField) ([]*SectorOnChainInfo, error)
NumLiveSectors() (uint64, error)
IsAllocated(abi.SectorNumber) (bool, error)
// UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
// count if there aren't enough).
UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
GetProvingPeriodStart() (abi.ChainEpoch, error)
@ -235,6 +240,7 @@ type DeclareFaultsRecoveredParams = miner0.DeclareFaultsRecoveredParams
type SubmitWindowedPoStParams = miner0.SubmitWindowedPoStParams
type ProveCommitSectorParams = miner0.ProveCommitSectorParams
type DisputeWindowedPoStParams = miner3.DisputeWindowedPoStParams
type ProveCommitAggregateParams = miner5.ProveCommitAggregateParams
func PreferredSealProofTypeFromWindowPoStType(nver network.Version, proof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
// We added support for the new proofs in network version 7, and removed support for the old

67
chain/actors/builtin/miner/state.go.template
View File

@ -8,6 +8,7 @@ import (
{{end}}
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
rle "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/dline"
"github.com/ipfs/go-cid"
@ -209,6 +210,26 @@ func (s *state{{.v}}) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCom
return &ret, nil
}
func (s *state{{.v}}) ForEachPrecommittedSector(cb func(SectorPreCommitOnChainInfo) error) error {
{{if (ge .v 3) -}}
precommitted, err := adt{{.v}}.AsMap(s.store, s.State.PreCommittedSectors, builtin{{.v}}.DefaultHamtBitwidth)
{{- else -}}
precommitted, err := adt{{.v}}.AsMap(s.store, s.State.PreCommittedSectors)
{{- end}}
if err != nil {
return err
}
var info miner{{.v}}.SectorPreCommitOnChainInfo
if err := precommitted.ForEach(&info, func(_ string) error {
return cb(fromV{{.v}}SectorPreCommitOnChainInfo(info))
}); err != nil {
return err
}
return nil
}
func (s *state{{.v}}) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) {
sectors, err := miner{{.v}}.LoadSectors(s.store, s.State.Sectors)
if err != nil {
@ -242,9 +263,15 @@ func (s *state{{.v}}) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo
return infos, nil
}
func (s *state{{.v}}) IsAllocated(num abi.SectorNumber) (bool, error) {
func (s *state{{.v}}) loadAllocatedSectorNumbers() (bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors)
return allocatedSectors, err
}
func (s *state{{.v}}) IsAllocated(num abi.SectorNumber) (bool, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return false, err
}
@ -255,6 +282,42 @@ func (s *state{{.v}}) GetProvingPeriodStart() (abi.ChainEpoch, error) {
return s.State.ProvingPeriodStart, nil
}
func (s *state{{.v}}) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return nil, err
}
allocatedRuns, err := allocatedSectors.RunIterator()
if err != nil {
return nil, err
}
unallocatedRuns, err := rle.Subtract(
&rle.RunSliceIterator{Runs: []rle.Run{ {Val: true, Len: abi.MaxSectorNumber} }},
allocatedRuns,
)
if err != nil {
return nil, err
}
iter, err := rle.BitsFromRuns(unallocatedRuns)
if err != nil {
return nil, err
}
sectors := make([]abi.SectorNumber, 0, count)
for iter.HasNext() && len(sectors) < count {
nextNo, err := iter.Next()
if err != nil {
return nil, err
}
sectors = append(sectors, abi.SectorNumber(nextNo))
}
return sectors, nil
}
func (s *state{{.v}}) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

63
chain/actors/builtin/miner/v0.go
View File

@ -8,6 +8,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
rle "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/dline"
"github.com/ipfs/go-cid"
@ -206,6 +207,22 @@ func (s *state0) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn
return &ret, nil
}
func (s *state0) ForEachPrecommittedSector(cb func(SectorPreCommitOnChainInfo) error) error {
precommitted, err := adt0.AsMap(s.store, s.State.PreCommittedSectors)
if err != nil {
return err
}
var info miner0.SectorPreCommitOnChainInfo
if err := precommitted.ForEach(&info, func(_ string) error {
return cb(fromV0SectorPreCommitOnChainInfo(info))
}); err != nil {
return err
}
return nil
}
func (s *state0) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) {
sectors, err := miner0.LoadSectors(s.store, s.State.Sectors)
if err != nil {
@ -239,9 +256,15 @@ func (s *state0) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err
return infos, nil
}
func (s *state0) IsAllocated(num abi.SectorNumber) (bool, error) {
func (s *state0) loadAllocatedSectorNumbers() (bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors)
return allocatedSectors, err
}
func (s *state0) IsAllocated(num abi.SectorNumber) (bool, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return false, err
}
@ -252,6 +275,42 @@ func (s *state0) GetProvingPeriodStart() (abi.ChainEpoch, error) {
return s.State.ProvingPeriodStart, nil
}
func (s *state0) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return nil, err
}
allocatedRuns, err := allocatedSectors.RunIterator()
if err != nil {
return nil, err
}
unallocatedRuns, err := rle.Subtract(
&rle.RunSliceIterator{Runs: []rle.Run{{Val: true, Len: abi.MaxSectorNumber}}},
allocatedRuns,
)
if err != nil {
return nil, err
}
iter, err := rle.BitsFromRuns(unallocatedRuns)
if err != nil {
return nil, err
}
sectors := make([]abi.SectorNumber, 0, count)
for iter.HasNext() && len(sectors) < count {
nextNo, err := iter.Next()
if err != nil {
return nil, err
}
sectors = append(sectors, abi.SectorNumber(nextNo))
}
return sectors, nil
}
func (s *state0) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

63
chain/actors/builtin/miner/v2.go
View File

@ -6,6 +6,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
rle "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/dline"
"github.com/ipfs/go-cid"
@ -204,6 +205,22 @@ func (s *state2) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn
return &ret, nil
}
func (s *state2) ForEachPrecommittedSector(cb func(SectorPreCommitOnChainInfo) error) error {
precommitted, err := adt2.AsMap(s.store, s.State.PreCommittedSectors)
if err != nil {
return err
}
var info miner2.SectorPreCommitOnChainInfo
if err := precommitted.ForEach(&info, func(_ string) error {
return cb(fromV2SectorPreCommitOnChainInfo(info))
}); err != nil {
return err
}
return nil
}
func (s *state2) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) {
sectors, err := miner2.LoadSectors(s.store, s.State.Sectors)
if err != nil {
@ -237,9 +254,15 @@ func (s *state2) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err
return infos, nil
}
func (s *state2) IsAllocated(num abi.SectorNumber) (bool, error) {
func (s *state2) loadAllocatedSectorNumbers() (bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors)
return allocatedSectors, err
}
func (s *state2) IsAllocated(num abi.SectorNumber) (bool, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return false, err
}
@ -250,6 +273,42 @@ func (s *state2) GetProvingPeriodStart() (abi.ChainEpoch, error) {
return s.State.ProvingPeriodStart, nil
}
func (s *state2) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return nil, err
}
allocatedRuns, err := allocatedSectors.RunIterator()
if err != nil {
return nil, err
}
unallocatedRuns, err := rle.Subtract(
&rle.RunSliceIterator{Runs: []rle.Run{{Val: true, Len: abi.MaxSectorNumber}}},
allocatedRuns,
)
if err != nil {
return nil, err
}
iter, err := rle.BitsFromRuns(unallocatedRuns)
if err != nil {
return nil, err
}
sectors := make([]abi.SectorNumber, 0, count)
for iter.HasNext() && len(sectors) < count {
nextNo, err := iter.Next()
if err != nil {
return nil, err
}
sectors = append(sectors, abi.SectorNumber(nextNo))
}
return sectors, nil
}
func (s *state2) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

63
chain/actors/builtin/miner/v3.go
View File

@ -6,6 +6,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
rle "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/dline"
"github.com/ipfs/go-cid"
@ -206,6 +207,22 @@ func (s *state3) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn
return &ret, nil
}
func (s *state3) ForEachPrecommittedSector(cb func(SectorPreCommitOnChainInfo) error) error {
precommitted, err := adt3.AsMap(s.store, s.State.PreCommittedSectors, builtin3.DefaultHamtBitwidth)
if err != nil {
return err
}
var info miner3.SectorPreCommitOnChainInfo
if err := precommitted.ForEach(&info, func(_ string) error {
return cb(fromV3SectorPreCommitOnChainInfo(info))
}); err != nil {
return err
}
return nil
}
func (s *state3) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) {
sectors, err := miner3.LoadSectors(s.store, s.State.Sectors)
if err != nil {
@ -239,9 +256,15 @@ func (s *state3) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err
return infos, nil
}
func (s *state3) IsAllocated(num abi.SectorNumber) (bool, error) {
func (s *state3) loadAllocatedSectorNumbers() (bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors)
return allocatedSectors, err
}
func (s *state3) IsAllocated(num abi.SectorNumber) (bool, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return false, err
}
@ -252,6 +275,42 @@ func (s *state3) GetProvingPeriodStart() (abi.ChainEpoch, error) {
return s.State.ProvingPeriodStart, nil
}
func (s *state3) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return nil, err
}
allocatedRuns, err := allocatedSectors.RunIterator()
if err != nil {
return nil, err
}
unallocatedRuns, err := rle.Subtract(
&rle.RunSliceIterator{Runs: []rle.Run{{Val: true, Len: abi.MaxSectorNumber}}},
allocatedRuns,
)
if err != nil {
return nil, err
}
iter, err := rle.BitsFromRuns(unallocatedRuns)
if err != nil {
return nil, err
}
sectors := make([]abi.SectorNumber, 0, count)
for iter.HasNext() && len(sectors) < count {
nextNo, err := iter.Next()
if err != nil {
return nil, err
}
sectors = append(sectors, abi.SectorNumber(nextNo))
}
return sectors, nil
}
func (s *state3) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

63
chain/actors/builtin/miner/v4.go
View File

@ -6,6 +6,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
rle "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/dline"
"github.com/ipfs/go-cid"
@ -206,6 +207,22 @@ func (s *state4) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn
return &ret, nil
}
func (s *state4) ForEachPrecommittedSector(cb func(SectorPreCommitOnChainInfo) error) error {
precommitted, err := adt4.AsMap(s.store, s.State.PreCommittedSectors, builtin4.DefaultHamtBitwidth)
if err != nil {
return err
}
var info miner4.SectorPreCommitOnChainInfo
if err := precommitted.ForEach(&info, func(_ string) error {
return cb(fromV4SectorPreCommitOnChainInfo(info))
}); err != nil {
return err
}
return nil
}
func (s *state4) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) {
sectors, err := miner4.LoadSectors(s.store, s.State.Sectors)
if err != nil {
@ -239,9 +256,15 @@ func (s *state4) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err
return infos, nil
}
func (s *state4) IsAllocated(num abi.SectorNumber) (bool, error) {
func (s *state4) loadAllocatedSectorNumbers() (bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors)
return allocatedSectors, err
}
func (s *state4) IsAllocated(num abi.SectorNumber) (bool, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return false, err
}
@ -252,6 +275,42 @@ func (s *state4) GetProvingPeriodStart() (abi.ChainEpoch, error) {
return s.State.ProvingPeriodStart, nil
}
func (s *state4) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return nil, err
}
allocatedRuns, err := allocatedSectors.RunIterator()
if err != nil {
return nil, err
}
unallocatedRuns, err := rle.Subtract(
&rle.RunSliceIterator{Runs: []rle.Run{{Val: true, Len: abi.MaxSectorNumber}}},
allocatedRuns,
)
if err != nil {
return nil, err
}
iter, err := rle.BitsFromRuns(unallocatedRuns)
if err != nil {
return nil, err
}
sectors := make([]abi.SectorNumber, 0, count)
for iter.HasNext() && len(sectors) < count {
nextNo, err := iter.Next()
if err != nil {
return nil, err
}
sectors = append(sectors, abi.SectorNumber(nextNo))
}
return sectors, nil
}
func (s *state4) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

63
chain/actors/builtin/miner/v5.go
View File

@ -6,6 +6,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
rle "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/dline"
"github.com/ipfs/go-cid"
@ -206,6 +207,22 @@ func (s *state5) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn
return &ret, nil
}
func (s *state5) ForEachPrecommittedSector(cb func(SectorPreCommitOnChainInfo) error) error {
precommitted, err := adt5.AsMap(s.store, s.State.PreCommittedSectors, builtin5.DefaultHamtBitwidth)
if err != nil {
return err
}
var info miner5.SectorPreCommitOnChainInfo
if err := precommitted.ForEach(&info, func(_ string) error {
return cb(fromV5SectorPreCommitOnChainInfo(info))
}); err != nil {
return err
}
return nil
}
func (s *state5) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) {
sectors, err := miner5.LoadSectors(s.store, s.State.Sectors)
if err != nil {
@ -239,9 +256,15 @@ func (s *state5) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err
return infos, nil
}
func (s *state5) IsAllocated(num abi.SectorNumber) (bool, error) {
func (s *state5) loadAllocatedSectorNumbers() (bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors)
return allocatedSectors, err
}
func (s *state5) IsAllocated(num abi.SectorNumber) (bool, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return false, err
}
@ -252,6 +275,42 @@ func (s *state5) GetProvingPeriodStart() (abi.ChainEpoch, error) {
return s.State.ProvingPeriodStart, nil
}
func (s *state5) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error) {
allocatedSectors, err := s.loadAllocatedSectorNumbers()
if err != nil {
return nil, err
}
allocatedRuns, err := allocatedSectors.RunIterator()
if err != nil {
return nil, err
}
unallocatedRuns, err := rle.Subtract(
&rle.RunSliceIterator{Runs: []rle.Run{{Val: true, Len: abi.MaxSectorNumber}}},
allocatedRuns,
)
if err != nil {
return nil, err
}
iter, err := rle.BitsFromRuns(unallocatedRuns)
if err != nil {
return nil, err
}
sectors := make([]abi.SectorNumber, 0, count)
for iter.HasNext() && len(sectors) < count {
nextNo, err := iter.Next()
if err != nil {
return nil, err
}
sectors = append(sectors, abi.SectorNumber(nextNo))
}
return sectors, nil
}
func (s *state5) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

1
chain/actors/builtin/multisig/actor.go.template
View File

@ -115,6 +115,7 @@ type MessageBuilder interface {
type ProposalHashData = msig{{.latestVersion}}.ProposalHashData
type ProposeReturn = msig{{.latestVersion}}.ProposeReturn
type ProposeParams = msig{{.latestVersion}}.ProposeParams
type ApproveReturn = msig{{.latestVersion}}.ApproveReturn
func txnParams(id uint64, data *ProposalHashData) ([]byte, error) {
params := msig{{.latestVersion}}.TxnIDParams{ID: msig{{.latestVersion}}.TxnID(id)}

1
chain/actors/builtin/multisig/multisig.go
View File

@ -186,6 +186,7 @@ type MessageBuilder interface {
type ProposalHashData = msig5.ProposalHashData
type ProposeReturn = msig5.ProposeReturn
type ProposeParams = msig5.ProposeParams
type ApproveReturn = msig5.ApproveReturn
func txnParams(id uint64, data *ProposalHashData) ([]byte, error) {
params := msig5.TxnIDParams{ID: msig5.TxnID(id)}

6
chain/gen/genesis/f00_system.go
View File

@ -3,6 +3,7 @@ package genesis
import (
"context"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin/system"
@ -32,8 +33,9 @@ func SetupSystemActor(ctx context.Context, bs bstore.Blockstore, av actors.Versi
}
act := &types.Actor{
Code: actcid,
Head: statecid,
Code: actcid,
Head: statecid,
Balance: big.Zero(),
}
return act, nil

6
chain/gen/genesis/f01_init.go
View File

@ -11,6 +11,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/specs-actors/actors/util/adt"
@ -181,8 +182,9 @@ func SetupInitActor(ctx context.Context, bs bstore.Blockstore, netname string, i
}
act := &types.Actor{
Code: actcid,
Head: statecid,
Code: actcid,
Head: statecid,
Balance: big.Zero(),
}
return counter, act, keyToId, nil

6
chain/gen/genesis/f03_cron.go
View File

@ -3,6 +3,7 @@ package genesis
import (
"context"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin/cron"
@ -31,8 +32,9 @@ func SetupCronActor(ctx context.Context, bs bstore.Blockstore, av actors.Version
}
act := &types.Actor{
Code: actcid,
Head: statecid,
Code: actcid,
Head: statecid,
Balance: big.Zero(),
}
return act, nil

6
chain/gen/genesis/f04_power.go
View File

@ -3,6 +3,7 @@ package genesis
import (
"context"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/actors"
@ -33,8 +34,9 @@ func SetupStoragePowerActor(ctx context.Context, bs bstore.Blockstore, av actors
}
act := &types.Actor{
Code: actcid,
Head: statecid,
Code: actcid,
Head: statecid,
Balance: big.Zero(),
}
return act, nil

6
chain/gen/genesis/f05_market.go
View File

@ -3,6 +3,7 @@ package genesis
import (
"context"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
@ -31,8 +32,9 @@ func SetupStorageMarketActor(ctx context.Context, bs bstore.Blockstore, av actor
}
act := &types.Actor{
Code: actcid,
Head: statecid,
Code: actcid,
Head: statecid,
Balance: big.Zero(),
}
return act, nil

6
chain/gen/genesis/f06_vreg.go
View File

@ -3,6 +3,7 @@ package genesis
import (
"context"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
"github.com/filecoin-project/lotus/chain/actors"
@ -46,8 +47,9 @@ func SetupVerifiedRegistryActor(ctx context.Context, bs bstore.Blockstore, av ac
}
act := &types.Actor{
Code: actcid,
Head: statecid,
Code: actcid,
Head: statecid,
Balance: big.Zero(),
}
return act, nil

7
chain/gen/genesis/genesis.go
View File

@ -313,11 +313,10 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
totalFilAllocated := big.Zero()
// flush as ForEach works on the HAMT
if _, err := state.Flush(ctx); err != nil {
return nil, nil, err
}
err = state.ForEach(func(addr address.Address, act *types.Actor) error {
if act.Balance.Nil() {
panic(fmt.Sprintf("actor %s (%s) has nil balance", addr, builtin.ActorNameByCode(act.Code)))
}
totalFilAllocated = big.Add(totalFilAllocated, act.Balance)
return nil
})

26
chain/state/statetree.go
View File

@ -504,6 +504,26 @@ func (st *StateTree) MutateActor(addr address.Address, f func(*types.Actor) erro
}
func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error {
// Walk through layers, if any.
seen := make(map[address.Address]struct{})
for i := len(st.snaps.layers) - 1; i >= 0; i-- {
for addr, op := range st.snaps.layers[i].actors {
if _, ok := seen[addr]; ok {
continue
}
seen[addr] = struct{}{}
if op.Delete {
continue
}
act := op.Act // copy
if err := f(addr, &act); err != nil {
return err
}
}
}
// Now walk through the saved actors.
var act types.Actor
return st.root.ForEach(&act, func(k string) error {
act := act // copy
@ -512,6 +532,12 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error
return xerrors.Errorf("invalid address (%x) found in state tree key: %w", []byte(k), err)
}
// no need to record anything here, there are no duplicates in the actors HAMT
// iself.
if _, ok := seen[addr]; ok {
return nil
}
return f(addr, &act)
})
}

20
chain/store/store.go
View File

@ -11,6 +11,7 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/filecoin-project/lotus/chain/state"
@ -404,7 +405,20 @@ func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error {
// internal state as our new head, if and only if it is heavier than the current
// head and does not exceed the maximum fork length.
func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipSet) error {
cs.heaviestLk.Lock()
for {
cs.heaviestLk.Lock()
if len(cs.reorgCh) < reorgChBuf/2 {
break
}
cs.heaviestLk.Unlock()
log.Errorf("reorg channel is heavily backlogged, waiting a bit before trying to take process new tipsets")
select {
case <-time.After(time.Second / 2):
case <-ctx.Done():
return ctx.Err()
}
}
defer cs.heaviestLk.Unlock()
w, err := cs.Weight(ctx, ts)
if err != nil {
@ -529,8 +543,10 @@ type reorg struct {
new *types.TipSet
}
const reorgChBuf = 32
func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNotifee) chan<- reorg {
out := make(chan reorg, 32)
out := make(chan reorg, reorgChBuf)
notifees := make([]ReorgNotifee, len(initialNotifees))
copy(notifees, initialNotifees)

52
chain/sync_test.go
View File

@ -85,6 +85,7 @@ type syncTestUtil struct {
blocks []*store.FullTipSet
nds []api.FullNode
us stmgr.UpgradeSchedule
}
func prepSyncTest(t testing.TB, h int) *syncTestUtil {
@ -104,9 +105,10 @@ func prepSyncTest(t testing.TB, h int) *syncTestUtil {
mn: mocknet.New(ctx),
g: g,
us: stmgr.DefaultUpgradeSchedule(),
}
tu.addSourceNode(stmgr.DefaultUpgradeSchedule(), h)
tu.addSourceNode(h)
//tu.checkHeight("source", source, h)
@ -119,7 +121,7 @@ func prepSyncTest(t testing.TB, h int) *syncTestUtil {
func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syncTestUtil {
logging.SetLogLevel("*", "INFO")
us := stmgr.UpgradeSchedule{{
sched := stmgr.UpgradeSchedule{{
// prepare for upgrade.
Network: network.Version9,
Height: 1,
@ -138,7 +140,7 @@ func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syn
Migration: stmgr.UpgradeActorsV5,
}}
g, err := gen.NewGeneratorWithUpgradeSchedule(us)
g, err := gen.NewGeneratorWithUpgradeSchedule(sched)
if err != nil {
t.Fatalf("%+v", err)
@ -153,9 +155,10 @@ func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syn
mn: mocknet.New(ctx),
g: g,
us: sched,
}
tu.addSourceNode(us, h)
tu.addSourceNode(h)
//tu.checkHeight("source", source, h)
// separate logs
@ -266,7 +269,7 @@ func (tu *syncTestUtil) mineNewBlock(src int, miners []int) {
tu.g.CurTipset = mts
}
func (tu *syncTestUtil) addSourceNode(us stmgr.UpgradeSchedule, gen int) {
func (tu *syncTestUtil) addSourceNode(gen int) {
if tu.genesis != nil {
tu.t.Fatal("source node already exists")
}
@ -282,7 +285,7 @@ func (tu *syncTestUtil) addSourceNode(us stmgr.UpgradeSchedule, gen int) {
node.Test(),
node.Override(new(modules.Genesis), modules.LoadGenesis(genesis)),
node.Override(new(stmgr.UpgradeSchedule), us),
node.Override(new(stmgr.UpgradeSchedule), tu.us),
)
require.NoError(tu.t, err)
tu.t.Cleanup(func() { _ = stop(context.Background()) })
@ -315,6 +318,7 @@ func (tu *syncTestUtil) addClientNode() int {
node.Test(),
node.Override(new(modules.Genesis), modules.LoadGenesis(tu.genesis)),
node.Override(new(stmgr.UpgradeSchedule), tu.us),
)
require.NoError(tu.t, err)
tu.t.Cleanup(func() { _ = stop(context.Background()) })
@ -1032,18 +1036,24 @@ func TestDrandNull(t *testing.T) {
build.UpgradeHyperdriveHeight = v5h
tu := prepSyncTestWithV5Height(t, H, v5h)
p0 := tu.addClientNode()
p1 := tu.addClientNode()
tu.loadChainToNode(p0)
tu.loadChainToNode(p1)
entropy := []byte{0, 2, 3, 4}
// arbitrarily chosen
pers := crypto.DomainSeparationTag_WinningPoStChallengeSeed
beforeNull := tu.g.CurTipset
afterNull := tu.mineOnBlock(beforeNull, 0, nil, false, false, nil, 2)
afterNull := tu.mineOnBlock(beforeNull, p0, nil, false, false, nil, 2)
nullHeight := beforeNull.TipSet().Height() + 1
if afterNull.TipSet().Height() == nullHeight {
t.Fatal("didn't inject nulls as expected")
}
rand, err := tu.nds[0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy)
rand, err := tu.nds[p0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy)
require.NoError(t, err)
// calculate the expected randomness based on the beacon BEFORE the null
@ -1054,20 +1064,20 @@ func TestDrandNull(t *testing.T) {
require.Equal(t, []byte(rand), expectedRand)
// zoom zoom to past the v5 upgrade by injecting many many nulls
postUpgrade := tu.mineOnBlock(afterNull, 0, nil, false, false, nil, v5h)
nv, err := tu.nds[0].StateNetworkVersion(tu.ctx, types.EmptyTSK)
postUpgrade := tu.mineOnBlock(afterNull, p0, nil, false, false, nil, v5h)
nv, err := tu.nds[p0].StateNetworkVersion(tu.ctx, postUpgrade.TipSet().Key())
require.NoError(t, err)
if nv != network.Version13 {
t.Fatal("expect to be v13 by now")
}
afterNull = tu.mineOnBlock(postUpgrade, 0, nil, false, false, nil, 2)
afterNull = tu.mineOnBlock(postUpgrade, p0, nil, false, false, nil, 2)
nullHeight = postUpgrade.TipSet().Height() + 1
if afterNull.TipSet().Height() == nullHeight {
t.Fatal("didn't inject nulls as expected")
}
rand, err = tu.nds[0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy)
rand0, err := tu.nds[p0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy)
require.NoError(t, err)
// calculate the expected randomness based on the beacon AFTER the null
@ -1075,7 +1085,21 @@ func TestDrandNull(t *testing.T) {
expectedRand, err = store.DrawRandomness(expectedBE[len(expectedBE)-1].Data, pers, nullHeight, entropy)
require.NoError(t, err)
require.Equal(t, []byte(rand), expectedRand)
build.UpgradeHyperdriveHeight = ov5h
require.Equal(t, []byte(rand0), expectedRand)
// Introduce p1 to friendly p0 who has all the blocks
require.NoError(t, tu.mn.LinkAll())
tu.connect(p0, p1)
tu.waitUntilNodeHasTs(p1, afterNull.TipSet().Key())
p1Head := tu.getHead(p1)
// Yes, p1 syncs well to p0's chain
require.Equal(tu.t, p1Head.Key(), afterNull.TipSet().Key())
// Yes, p1 sources the same randomness as p0
rand1, err := tu.nds[p1].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy)
require.NoError(t, err)
require.Equal(t, rand0, rand1)
build.UpgradeHyperdriveHeight = ov5h
}

8
chain/vm/vm.go
View File

@ -439,6 +439,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet,
},
GasCosts: &gasOutputs,
Duration: time.Since(start),
ActorErr: aerrors.Newf(exitcode.SysErrOutOfGas,
"message gas limit does not cover on-chain gas costs"),
}, nil
}
@ -669,6 +671,12 @@ func (vm *VM) Flush(ctx context.Context) (cid.Cid, error) {
return root, nil
}
// Get the buffered blockstore associated with the VM. This includes any temporary blocks produced
// during this VM's execution.
func (vm *VM) ActorStore(ctx context.Context) adt.Store {
return adt.WrapStore(ctx, vm.cst)
}
func linksForObj(blk block.Block, cb func(cid.Cid)) error {
switch blk.Cid().Prefix().Codec {
case cid.DagCBOR:

11
chain/wallet/multi.go
View File

@ -4,6 +4,7 @@ import (
"context"
"go.uber.org/fx"
"go.uber.org/multierr"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
@ -56,18 +57,18 @@ func nonNil(wallets ...getif) []api.Wallet {
func (m MultiWallet) find(ctx context.Context, address address.Address, wallets ...getif) (api.Wallet, error) {
ws := nonNil(wallets...)
var merr error
for _, w := range ws {
have, err := w.WalletHas(ctx, address)
if err != nil {
return nil, err
}
merr = multierr.Append(merr, err)
if have {
if err == nil && have {
return w, nil
}
}
return nil, nil
return nil, merr
}
func (m MultiWallet) WalletNew(ctx context.Context, keyType types.KeyType) (address.Address, error) {

6
cli/client.go
View File

@ -308,8 +308,8 @@ var clientDealCmd = &cli.Command{
Description: `Make a deal with a miner.
dataCid comes from running 'lotus client import'.
miner is the address of the miner you wish to make a deal with.
price is measured in FIL/GB/Epoch. Miners usually don't accept a bid
lower than their advertised ask. You can check a miners listed price
price is measured in FIL/Epoch. Miners usually don't accept a bid
lower than their advertised ask (which is in FIL/GiB/Epoch). You can check a miners listed price
with 'lotus client query-ask <miner address>'.
duration is how long the miner should store the data for, in blocks.
The minimum value is 518400 (6 months).`,
@ -1759,7 +1759,7 @@ var clientQueryAskCmd = &cli.Command{
return xerrors.Errorf("failed to get peerID for miner: %w", err)
}
if *mi.PeerId == peer.ID("SETME") {
if mi.PeerId == nil || *mi.PeerId == peer.ID("SETME") {
return fmt.Errorf("the miner hasn't initialized yet")
}

2
cli/multisig.go
View File

@ -1428,7 +1428,7 @@ var msigLockCancelCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 6 {
if cctx.Args().Len() != 5 {
return ShowHelp(cctx, fmt.Errorf("must pass multisig address, tx id, start epoch, unlock duration, and amount"))
}

28
cmd/lotus-sim/copy.go Normal file
View File

@ -0,0 +1,28 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
)
var copySimCommand = &cli.Command{
Name: "copy",
ArgsUsage: "<new-name>",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
if cctx.NArg() != 1 {
return fmt.Errorf("expected 1 argument")
}
name := cctx.Args().First()
return node.CopySim(cctx.Context, cctx.String("simulation"), name)
},
}

49
cmd/lotus-sim/create.go Normal file
View File

@ -0,0 +1,49 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
)
var createSimCommand = &cli.Command{
Name: "create",
ArgsUsage: "[tipset]",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
var ts *types.TipSet
switch cctx.NArg() {
case 0:
if err := node.Chainstore.Load(); err != nil {
return err
}
ts = node.Chainstore.GetHeaviestTipSet()
case 1:
cids, err := lcli.ParseTipSetString(cctx.Args().Get(1))
if err != nil {
return err
}
tsk := types.NewTipSetKey(cids...)
ts, err = node.Chainstore.LoadTipSet(tsk)
if err != nil {
return err
}
default:
return fmt.Errorf("expected 0 or 1 arguments")
}
_, err = node.CreateSim(cctx.Context, cctx.String("simulation"), ts)
return err
},
}

22
cmd/lotus-sim/delete.go Normal file
View File

@ -0,0 +1,22 @@
package main
import (
"github.com/urfave/cli/v2"
)
var deleteSimCommand = &cli.Command{
Name: "delete",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
return node.DeleteSim(cctx.Context, cctx.String("simulation"))
},
}

110
cmd/lotus-sim/info.go Normal file
View File

@ -0,0 +1,110 @@
package main
import (
"context"
"fmt"
"io"
"text/tabwriter"
"time"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
)
func getTotalPower(ctx context.Context, sm *stmgr.StateManager, ts *types.TipSet) (power.Claim, error) {
actor, err := sm.LoadActor(ctx, power.Address, ts)
if err != nil {
return power.Claim{}, err
}
state, err := power.Load(sm.ChainStore().ActorStore(ctx), actor)
if err != nil {
return power.Claim{}, err
}
return state.TotalPower()
}
func printInfo(ctx context.Context, sim *simulation.Simulation, out io.Writer) error {
head := sim.GetHead()
start := sim.GetStart()
powerNow, err := getTotalPower(ctx, sim.StateManager, head)
if err != nil {
return err
}
powerLookbackEpoch := head.Height() - builtin.EpochsInDay*2
if powerLookbackEpoch < start.Height() {
powerLookbackEpoch = start.Height()
}
lookbackTs, err := sim.Node.Chainstore.GetTipsetByHeight(ctx, powerLookbackEpoch, head, false)
if err != nil {
return err
}
powerLookback, err := getTotalPower(ctx, sim.StateManager, lookbackTs)
if err != nil {
return err
}
// growth rate in size/day
growthRate := big.Div(
big.Mul(big.Sub(powerNow.RawBytePower, powerLookback.RawBytePower),
big.NewInt(builtin.EpochsInDay)),
big.NewInt(int64(head.Height()-lookbackTs.Height())),
)
tw := tabwriter.NewWriter(out, 8, 8, 1, ' ', 0)
headEpoch := head.Height()
firstEpoch := start.Height() + 1
headTime := time.Unix(int64(head.MinTimestamp()), 0)
startTime := time.Unix(int64(start.MinTimestamp()), 0)
duration := headTime.Sub(startTime)
fmt.Fprintf(tw, "Name:\t%s\n", sim.Name())
fmt.Fprintf(tw, "Head:\t%s\n", head)
fmt.Fprintf(tw, "Start Epoch:\t%d\n", firstEpoch)
fmt.Fprintf(tw, "End Epoch:\t%d\n", headEpoch)
fmt.Fprintf(tw, "Length:\t%d\n", headEpoch-firstEpoch)
fmt.Fprintf(tw, "Start Date:\t%s\n", startTime)
fmt.Fprintf(tw, "End Date:\t%s\n", headTime)
fmt.Fprintf(tw, "Duration:\t%.2f day(s)\n", duration.Hours()/24)
fmt.Fprintf(tw, "Capacity:\t%s\n", types.SizeStr(powerNow.RawBytePower))
fmt.Fprintf(tw, "Daily Capacity Growth:\t%s/day\n", types.SizeStr(growthRate))
fmt.Fprintf(tw, "Network Version:\t%d\n", sim.GetNetworkVersion())
return tw.Flush()
}
var infoSimCommand = &cli.Command{
Name: "info",
Description: "Output information about the simulation.",
Subcommands: []*cli.Command{
infoCommitGasSimCommand,
infoMessageSizeSimCommand,
infoWindowPostBandwidthSimCommand,
infoCapacityGrowthSimCommand,
infoStateGrowthSimCommand,
},
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
return printInfo(cctx.Context, sim, cctx.App.Writer)
},
}

67
cmd/lotus-sim/info_capacity.go Normal file
View File

@ -0,0 +1,67 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/types"
)
var infoCapacityGrowthSimCommand = &cli.Command{
Name: "capacity-growth",
Description: "List daily capacity growth over the course of the simulation starting at the end.",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
firstEpoch := sim.GetStart().Height()
ts := sim.GetHead()
lastPower, err := getTotalPower(cctx.Context, sim.StateManager, ts)
if err != nil {
return err
}
lastHeight := ts.Height()
for ts.Height() > firstEpoch && cctx.Err() == nil {
ts, err = sim.Node.Chainstore.LoadTipSet(ts.Parents())
if err != nil {
return err
}
newEpoch := ts.Height()
if newEpoch != firstEpoch && newEpoch+builtin.EpochsInDay > lastHeight {
continue
}
newPower, err := getTotalPower(cctx.Context, sim.StateManager, ts)
if err != nil {
return err
}
growthRate := big.Div(
big.Mul(big.Sub(lastPower.RawBytePower, newPower.RawBytePower),
big.NewInt(builtin.EpochsInDay)),
big.NewInt(int64(lastHeight-newEpoch)),
)
lastPower = newPower
lastHeight = newEpoch
fmt.Fprintf(cctx.App.Writer, "%s/day\n", types.SizeStr(growthRate))
}
return cctx.Err()
},
}

148
cmd/lotus-sim/info_commit.go Normal file
View File

@ -0,0 +1,148 @@
package main
import (
"bytes"
"fmt"
"os"
"syscall"
"github.com/streadway/quantile"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
"github.com/filecoin-project/lotus/lib/stati"
)
var infoCommitGasSimCommand = &cli.Command{
Name: "commit-gas",
Description: "Output information about the gas for commits",
Flags: []cli.Flag{
&cli.Int64Flag{
Name: "lookback",
Value: 0,
},
},
Action: func(cctx *cli.Context) (err error) {
log := func(f string, i ...interface{}) {
fmt.Fprintf(os.Stderr, f, i...)
}
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
go profileOnSignal(cctx, syscall.SIGUSR2)
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
var gasAgg, proofsAgg uint64
var gasAggMax, proofsAggMax uint64
var gasSingle, proofsSingle uint64
qpoints := []struct{ q, tol float64 }{
{0.01, 0.0005},
{0.05, 0.001},
{0.20, 0.01},
{0.25, 0.01},
{0.30, 0.01},
{0.40, 0.01},
{0.45, 0.01},
{0.50, 0.01},
{0.60, 0.01},
{0.80, 0.01},
{0.95, 0.001},
{0.99, 0.0005},
}
estims := make([]quantile.Estimate, len(qpoints))
for i, p := range qpoints {
estims[i] = quantile.Known(p.q, p.tol)
}
qua := quantile.New(estims...)
hist, err := stati.NewHistogram([]float64{
1, 3, 5, 7, 15, 30, 50, 100, 200, 400, 600, 700, 819})
if err != nil {
return err
}
err = sim.Walk(cctx.Context, cctx.Int64("lookback"), func(
sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid,
messages []*simulation.AppliedMessage,
) error {
for _, m := range messages {
if m.ExitCode != exitcode.Ok {
continue
}
if m.Method == miner.Methods.ProveCommitAggregate {
param := miner.ProveCommitAggregateParams{}
err := param.UnmarshalCBOR(bytes.NewReader(m.Params))
if err != nil {
log("failed to decode params: %+v", err)
return nil
}
c, err := param.SectorNumbers.Count()
if err != nil {
log("failed to count sectors")
return nil
}
gasAgg += uint64(m.GasUsed)
proofsAgg += c
if c == 819 {
gasAggMax += uint64(m.GasUsed)
proofsAggMax += c
}
for i := uint64(0); i < c; i++ {
qua.Add(float64(c))
}
hist.Observe(float64(c))
}
if m.Method == miner.Methods.ProveCommitSector {
gasSingle += uint64(m.GasUsed)
proofsSingle++
qua.Add(1)
hist.Observe(1)
}
}
return nil
})
if err != nil {
return err
}
idealGassUsed := float64(gasAggMax) / float64(proofsAggMax) * float64(proofsAgg+proofsSingle)
fmt.Printf("Gas usage efficiency in comparison to all 819: %f%%\n", 100*idealGassUsed/float64(gasAgg+gasSingle))
fmt.Printf("Proofs in singles: %d\n", proofsSingle)
fmt.Printf("Proofs in Aggs: %d\n", proofsAgg)
fmt.Printf("Proofs in Aggs(819): %d\n", proofsAggMax)
fmt.Println()
fmt.Println("Quantiles of proofs in given aggregate size:")
for _, p := range qpoints {
fmt.Printf("%.0f%%\t%.0f\n", p.q*100, qua.Get(p.q))
}
fmt.Println()
fmt.Println("Histogram of messages:")
fmt.Printf("Total\t%d\n", hist.Total())
for i, b := range hist.Buckets[1:] {
fmt.Printf("%.0f\t%d\n", b, hist.Get(i))
}
return nil
},
}

95
cmd/lotus-sim/info_message.go Normal file
View File

@ -0,0 +1,95 @@
package main
import (
"fmt"
"syscall"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
"github.com/filecoin-project/lotus/lib/stati"
"github.com/ipfs/go-cid"
"github.com/streadway/quantile"
"github.com/urfave/cli/v2"
)
var infoMessageSizeSimCommand = &cli.Command{
Name: "message-size",
Description: "Output information about message size distribution",
Flags: []cli.Flag{
&cli.Int64Flag{
Name: "lookback",
Value: 0,
},
},
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
go profileOnSignal(cctx, syscall.SIGUSR2)
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
qpoints := []struct{ q, tol float64 }{
{0.30, 0.01},
{0.40, 0.01},
{0.60, 0.01},
{0.70, 0.01},
{0.80, 0.01},
{0.85, 0.01},
{0.90, 0.01},
{0.95, 0.001},
{0.99, 0.0005},
{0.999, 0.0001},
}
estims := make([]quantile.Estimate, len(qpoints))
for i, p := range qpoints {
estims[i] = quantile.Known(p.q, p.tol)
}
qua := quantile.New(estims...)
hist, err := stati.NewHistogram([]float64{
1 << 8, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
})
if err != nil {
return err
}
err = sim.Walk(cctx.Context, cctx.Int64("lookback"), func(
sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid,
messages []*simulation.AppliedMessage,
) error {
for _, m := range messages {
msgSize := float64(m.ChainLength())
qua.Add(msgSize)
hist.Observe(msgSize)
}
return nil
})
if err != nil {
return err
}
fmt.Println("Quantiles of message sizes:")
for _, p := range qpoints {
fmt.Printf("%.1f%%\t%.0f\n", p.q*100, qua.Get(p.q))
}
fmt.Println()
fmt.Println("Histogram of message sizes:")
fmt.Printf("Total\t%d\n", hist.Total())
for i, b := range hist.Buckets[1:] {
fmt.Printf("%.0f\t%d\t%.1f%%\n", b, hist.Get(i), 100*hist.GetRatio(i))
}
return nil
},
}

141
cmd/lotus-sim/info_state.go Normal file
View File

@ -0,0 +1,141 @@
package main
import (
"bytes"
"context"
"fmt"
"math"
"sync"
"sync/atomic"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/types"
)
var infoStateGrowthSimCommand = &cli.Command{
Name: "state-size",
Description: "List daily state size over the course of the simulation starting at the end.",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
// NOTE: This code is entirely read-bound.
store := node.Chainstore.StateBlockstore()
stateSize := func(ctx context.Context, c cid.Cid) (uint64, error) {
seen := cid.NewSet()
sema := make(chan struct{}, 40)
var lock sync.Mutex
var recSize func(cid.Cid) (uint64, error)
recSize = func(c cid.Cid) (uint64, error) {
// Not a part of the chain state.
if err := ctx.Err(); err != nil {
return 0, err
}
lock.Lock()
visit := seen.Visit(c)
lock.Unlock()
// Already seen?
if !visit {
return 0, nil
}
var links []cid.Cid
var totalSize uint64
if err := store.View(c, func(data []byte) error {
totalSize += uint64(len(data))
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
if c.Prefix().Codec != cid.DagCBOR {
return
}
links = append(links, c)
})
}); err != nil {
return 0, err
}
var wg sync.WaitGroup
errCh := make(chan error, 1)
cb := func(c cid.Cid) {
size, err := recSize(c)
if err != nil {
select {
case errCh <- err:
default:
}
return
}
atomic.AddUint64(&totalSize, size)
}
asyncCb := func(c cid.Cid) {
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sema }()
cb(c)
}()
}
for _, link := range links {
select {
case sema <- struct{}{}:
asyncCb(link)
default:
cb(link)
}
}
wg.Wait()
select {
case err := <-errCh:
return 0, err
default:
}
return totalSize, nil
}
return recSize(c)
}
firstEpoch := sim.GetStart().Height()
ts := sim.GetHead()
lastHeight := abi.ChainEpoch(math.MaxInt64)
for ts.Height() > firstEpoch && cctx.Err() == nil {
if ts.Height()+builtin.EpochsInDay <= lastHeight {
lastHeight = ts.Height()
parentStateSize, err := stateSize(cctx.Context, ts.ParentState())
if err != nil {
return err
}
fmt.Fprintf(cctx.App.Writer, "%d: %s\n", ts.Height(), types.SizeStr(types.NewInt(parentStateSize)))
}
ts, err = sim.Node.Chainstore.LoadTipSet(ts.Parents())
if err != nil {
return err
}
}
return cctx.Err()
},
}

69
cmd/lotus-sim/info_wdpost.go Normal file
View File

@ -0,0 +1,69 @@
package main
import (
"fmt"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
)
var infoWindowPostBandwidthSimCommand = &cli.Command{
Name: "post-bandwidth",
Description: "List average chain bandwidth used by window posts for each day of the simulation.",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
var postGas, totalGas int64
printStats := func() {
fmt.Fprintf(cctx.App.Writer, "%.4f%%\n", float64(100*postGas)/float64(totalGas))
}
idx := 0
err = sim.Walk(cctx.Context, 0, func(
sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid,
messages []*simulation.AppliedMessage,
) error {
for _, m := range messages {
totalGas += m.GasUsed
if m.ExitCode != exitcode.Ok {
continue
}
if m.Method == miner.Methods.SubmitWindowedPoSt {
postGas += m.GasUsed
}
}
idx++
idx %= builtin.EpochsInDay
if idx == 0 {
printStats()
postGas = 0
totalGas = 0
}
return nil
})
if idx > 0 {
printStats()
}
return err
},
}

38
cmd/lotus-sim/list.go Normal file
View File

@ -0,0 +1,38 @@
package main
import (
"fmt"
"text/tabwriter"
"github.com/urfave/cli/v2"
)
var listSimCommand = &cli.Command{
Name: "list",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
list, err := node.ListSims(cctx.Context)
if err != nil {
return err
}
tw := tabwriter.NewWriter(cctx.App.Writer, 8, 8, 0, ' ', 0)
for _, name := range list {
sim, err := node.LoadSim(cctx.Context, name)
if err != nil {
return err
}
head := sim.GetHead()
fmt.Fprintf(tw, "%s\t%s\t%s\n", name, head.Height(), head.Key())
}
return tw.Flush()
},
}

63
cmd/lotus-sim/main.go Normal file
View File

@ -0,0 +1,63 @@
package main
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"github.com/urfave/cli/v2"
logging "github.com/ipfs/go-log/v2"
)
var root []*cli.Command = []*cli.Command{
createSimCommand,
deleteSimCommand,
copySimCommand,
renameSimCommand,
listSimCommand,
runSimCommand,
infoSimCommand,
upgradeCommand,
}
func main() {
if _, set := os.LookupEnv("GOLOG_LOG_LEVEL"); !set {
_ = logging.SetLogLevel("simulation", "DEBUG")
_ = logging.SetLogLevel("simulation-mock", "DEBUG")
}
app := &cli.App{
Name: "lotus-sim",
Usage: "A tool to simulate a network.",
Commands: root,
Writer: os.Stdout,
ErrWriter: os.Stderr,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "repo",
EnvVars: []string{"LOTUS_PATH"},
Hidden: true,
Value: "~/.lotus",
},
&cli.StringFlag{
Name: "simulation",
Aliases: []string{"sim"},
EnvVars: []string{"LOTUS_SIMULATION"},
Value: "default",
},
},
}
ctx, cancel := signal.NotifyContext(context.Background(),
syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
defer cancel()
if err := app.RunContext(ctx, os.Args); err != nil {
fmt.Fprintf(os.Stderr, "Error: %s\n", err)
os.Exit(1)
return
}
}

94
cmd/lotus-sim/profile.go Normal file
View File

@ -0,0 +1,94 @@
package main
import (
"context"
"fmt"
"os"
"os/signal"
"path/filepath"
"runtime/pprof"
"time"
"github.com/urfave/cli/v2"
)
func takeProfiles(ctx context.Context) (fname string, _err error) {
dir, err := os.MkdirTemp(".", ".profiles-temp*")
if err != nil {
return "", err
}
if err := writeProfiles(ctx, dir); err != nil {
_ = os.RemoveAll(dir)
return "", err
}
fname = fmt.Sprintf("pprof-simulation-%s", time.Now().Format(time.RFC3339))
if err := os.Rename(dir, fname); err != nil {
_ = os.RemoveAll(dir)
return "", err
}
return fname, nil
}
func writeProfiles(ctx context.Context, dir string) error {
for _, profile := range pprof.Profiles() {
file, err := os.Create(filepath.Join(dir, profile.Name()+".pprof.gz"))
if err != nil {
return err
}
if err := profile.WriteTo(file, 0); err != nil {
_ = file.Close()
return err
}
if err := file.Close(); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
}
file, err := os.Create(filepath.Join(dir, "cpu.pprof.gz"))
if err != nil {
return err
}
if err := pprof.StartCPUProfile(file); err != nil {
_ = file.Close()
return err
}
select {
case <-time.After(30 * time.Second):
case <-ctx.Done():
}
pprof.StopCPUProfile()
err = file.Close()
if err := ctx.Err(); err != nil {
return err
}
return err
}
func profileOnSignal(cctx *cli.Context, signals ...os.Signal) {
ch := make(chan os.Signal, 1)
signal.Notify(ch, signals...)
defer signal.Stop(ch)
for {
select {
case <-ch:
fname, err := takeProfiles(cctx.Context)
switch err {
case context.Canceled:
return
case nil:
fmt.Fprintf(cctx.App.ErrWriter, "Wrote profile to %q\n", fname)
default:
fmt.Fprintf(cctx.App.ErrWriter, "ERROR: failed to write profile: %s\n", err)
}
case <-cctx.Done():
return
}
}
}

29
cmd/lotus-sim/rename.go Normal file
View File

@ -0,0 +1,29 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
)
var renameSimCommand = &cli.Command{
Name: "rename",
ArgsUsage: "<new-name>",
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
if cctx.NArg() != 1 {
return fmt.Errorf("expected 1 argument")
}
name := cctx.Args().First()
return node.RenameSim(cctx.Context, cctx.String("simulation"), name)
},
}

72
cmd/lotus-sim/run.go Normal file
View File

@ -0,0 +1,72 @@
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
"github.com/urfave/cli/v2"
)
var runSimCommand = &cli.Command{
Name: "run",
Description: `Run the simulation.
Signals:
- SIGUSR1: Print information about the current simulation (equivalent to 'lotus-sim info').
- SIGUSR2: Write pprof profiles to ./pprof-simulation-$DATE/`,
Flags: []cli.Flag{
&cli.IntFlag{
Name: "epochs",
Usage: "Advance the given number of epochs then stop.",
},
},
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
go profileOnSignal(cctx, syscall.SIGUSR2)
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
targetEpochs := cctx.Int("epochs")
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGUSR1)
defer signal.Stop(ch)
for i := 0; targetEpochs == 0 || i < targetEpochs; i++ {
ts, err := sim.Step(cctx.Context)
if err != nil {
return err
}
fmt.Fprintf(cctx.App.Writer, "advanced to %d %s\n", ts.Height(), ts.Key())
// Print
select {
case <-ch:
fmt.Fprintln(cctx.App.Writer, "---------------------")
if err := printInfo(cctx.Context, sim, cctx.App.Writer); err != nil {
fmt.Fprintf(cctx.App.ErrWriter, "ERROR: failed to print info: %s\n", err)
}
fmt.Fprintln(cctx.App.Writer, "---------------------")
case <-cctx.Context.Done():
return cctx.Err()
default:
}
}
fmt.Fprintln(cctx.App.Writer, "simulation done")
return err
},
}

93
cmd/lotus-sim/simulation/block.go Normal file
View File

@ -0,0 +1,93 @@
package simulation
import (
"context"
"crypto/sha256"
"encoding/binary"
"time"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
)
const beaconPrefix = "mockbeacon:"
// nextBeaconEntries returns a fake beacon entries for the next block.
func (sim *Simulation) nextBeaconEntries() []types.BeaconEntry {
parentBeacons := sim.head.Blocks()[0].BeaconEntries
lastBeacon := parentBeacons[len(parentBeacons)-1]
beaconRound := lastBeacon.Round + 1
buf := make([]byte, len(beaconPrefix)+8)
copy(buf, beaconPrefix)
binary.BigEndian.PutUint64(buf[len(beaconPrefix):], beaconRound)
beaconRand := sha256.Sum256(buf)
return []types.BeaconEntry{{
Round: beaconRound,
Data: beaconRand[:],
}}
}
// nextTicket returns a fake ticket for the next block.
func (sim *Simulation) nextTicket() *types.Ticket {
newProof := sha256.Sum256(sim.head.MinTicket().VRFProof)
return &types.Ticket{
VRFProof: newProof[:],
}
}
// makeTipSet generates and executes the next tipset from the given messages. This method:
//
// 1. Stores the given messages in the Chainstore.
// 2. Creates and persists a single block mined by the same miner as the parent.
// 3. Creates a tipset from this block and executes it.
// 4. Returns the resulting tipset.
//
// This method does _not_ mutate local state (although it does add blocks to the datastore).
func (sim *Simulation) makeTipSet(ctx context.Context, messages []*types.Message) (*types.TipSet, error) {
parentTs := sim.head
parentState, parentRec, err := sim.StateManager.TipSetState(ctx, parentTs)
if err != nil {
return nil, xerrors.Errorf("failed to compute parent tipset: %w", err)
}
msgsCid, err := sim.storeMessages(ctx, messages)
if err != nil {
return nil, xerrors.Errorf("failed to store block messages: %w", err)
}
uts := parentTs.MinTimestamp() + build.BlockDelaySecs
blks := []*types.BlockHeader{{
Miner: parentTs.MinTicketBlock().Miner, // keep reusing the same miner.
Ticket: sim.nextTicket(),
BeaconEntries: sim.nextBeaconEntries(),
Parents: parentTs.Cids(),
Height: parentTs.Height() + 1,
ParentStateRoot: parentState,
ParentMessageReceipts: parentRec,
Messages: msgsCid,
ParentBaseFee: abi.NewTokenAmount(0),
Timestamp: uts,
ElectionProof: &types.ElectionProof{WinCount: 1},
}}
err = sim.Node.Chainstore.PersistBlockHeaders(blks...)
if err != nil {
return nil, xerrors.Errorf("failed to persist block headers: %w", err)
}
newTipSet, err := types.NewTipSet(blks)
if err != nil {
return nil, xerrors.Errorf("failed to create new tipset: %w", err)
}
now := time.Now()
_, _, err = sim.StateManager.TipSetState(ctx, newTipSet)
if err != nil {
return nil, xerrors.Errorf("failed to compute new tipset: %w", err)
}
duration := time.Since(now)
log.Infow("computed tipset", "duration", duration, "height", newTipSet.Height())
return newTipSet, nil
}

280
cmd/lotus-sim/simulation/blockbuilder/blockbuilder.go Normal file
View File

@ -0,0 +1,280 @@
package blockbuilder
import (
"context"
"math"
"go.uber.org/zap"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/account"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
)
const (
// 0.25 is the default, but the number below is from the network.
gasOverestimation = 1.0 / 0.808
// The number of expected blocks in a tipset. We use this to determine how much gas a tipset
// has.
// 5 per tipset, but we effectively get 4 blocks worth of messages.
expectedBlocks = 4
// TODO: This will produce invalid blocks but it will accurately model the amount of gas
// we're willing to use per-tipset.
// A more correct approach would be to produce 5 blocks. We can do that later.
targetGas = build.BlockGasTarget * expectedBlocks
)
type BlockBuilder struct {
ctx context.Context
logger *zap.SugaredLogger
parentTs *types.TipSet
parentSt *state.StateTree
vm *vm.VM
sm *stmgr.StateManager
gasTotal int64
messages []*types.Message
}
// NewBlockBuilder constructs a new block builder from the parent state. Use this to pack a block
// with messages.
//
// NOTE: The context applies to the life of the block builder itself (but does not need to be canceled).
func NewBlockBuilder(ctx context.Context, logger *zap.SugaredLogger, sm *stmgr.StateManager, parentTs *types.TipSet) (*BlockBuilder, error) {
parentState, _, err := sm.TipSetState(ctx, parentTs)
if err != nil {
return nil, err
}
parentSt, err := sm.StateTree(parentState)
if err != nil {
return nil, err
}
bb := &BlockBuilder{
ctx: ctx,
logger: logger.With("epoch", parentTs.Height()+1),
sm: sm,
parentTs: parentTs,
parentSt: parentSt,
}
// Then we construct a VM to execute messages for gas estimation.
//
// Most parts of this VM are "real" except:
// 1. We don't charge a fee.
// 2. The runtime has "fake" proof logic.
// 3. We don't actually save any of the results.
r := store.NewChainRand(sm.ChainStore(), parentTs.Cids())
vmopt := &vm.VMOpts{
StateBase: parentState,
Epoch: parentTs.Height() + 1,
Rand: r,
Bstore: sm.ChainStore().StateBlockstore(),
Syscalls: sm.ChainStore().VMSys(),
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,
BaseFee: abi.NewTokenAmount(0),
LookbackState: stmgr.LookbackStateGetterForTipset(sm, parentTs),
}
bb.vm, err = vm.NewVM(bb.ctx, vmopt)
if err != nil {
return nil, err
}
return bb, nil
}
// PushMessages tries to push the specified message into the block.
//
// 1. All messages will be executed in-order.
// 2. Gas computation & nonce selection will be handled internally.
// 3. The base-fee is 0 so the sender does not need funds.
// 4. As usual, the sender must be an account (any account).
// 5. If the message fails to execute, this method will fail.
//
// Returns ErrOutOfGas when out of gas. Check BlockBuilder.GasRemaining and try pushing a cheaper
// message.
func (bb *BlockBuilder) PushMessage(msg *types.Message) (*types.MessageReceipt, error) {
if bb.gasTotal >= targetGas {
return nil, new(ErrOutOfGas)
}
st := bb.StateTree()
store := bb.ActorStore()
// Copy the message before we start mutating it.
msgCpy := *msg
msg = &msgCpy
actor, err := st.GetActor(msg.From)
if err != nil {
return nil, err
}
if !builtin.IsAccountActor(actor.Code) {
return nil, xerrors.Errorf(
"messags may only be sent from account actors, got message from %s (%s)",
msg.From, builtin.ActorNameByCode(actor.Code),
)
}
msg.Nonce = actor.Nonce
if msg.From.Protocol() == address.ID {
state, err := account.Load(store, actor)
if err != nil {
return nil, err
}
msg.From, err = state.PubkeyAddress()
if err != nil {
return nil, err
}
}
// TODO: Our gas estimation is broken for payment channels due to horrible hacks in
// gasEstimateGasLimit.
if msg.Value == types.EmptyInt {
msg.Value = abi.NewTokenAmount(0)
}
msg.GasPremium = abi.NewTokenAmount(0)
msg.GasFeeCap = abi.NewTokenAmount(0)
msg.GasLimit = build.BlockGasTarget
// We manually snapshot so we can revert nonce changes, etc. on failure.
err = st.Snapshot(bb.ctx)
if err != nil {
return nil, xerrors.Errorf("failed to take a snapshot while estimating message gas: %w", err)
}
defer st.ClearSnapshot()
ret, err := bb.vm.ApplyMessage(bb.ctx, msg)
if err != nil {
_ = st.Revert()
return nil, err
}
if ret.ActorErr != nil {
_ = st.Revert()
return nil, ret.ActorErr
}
// Sometimes there are bugs. Let's catch them.
if ret.GasUsed == 0 {
_ = st.Revert()
return nil, xerrors.Errorf("used no gas %v -> %v", msg, ret)
}
// Update the gas limit taking overestimation into account.
msg.GasLimit = int64(math.Ceil(float64(ret.GasUsed) * gasOverestimation))
// Did we go over? Yes, revert.
newTotal := bb.gasTotal + msg.GasLimit
if newTotal > targetGas {
_ = st.Revert()
return nil, &ErrOutOfGas{Available: targetGas - bb.gasTotal, Required: msg.GasLimit}
}
bb.gasTotal = newTotal
bb.messages = append(bb.messages, msg)
return &ret.MessageReceipt, nil
}
// ActorStore returns the VM's current (pending) blockstore.
func (bb *BlockBuilder) ActorStore() adt.Store {
return bb.vm.ActorStore(bb.ctx)
}
// StateTree returns the VM's current (pending) state-tree. This includes any changes made by
// successfully pushed messages.
//
// You probably want ParentStateTree
func (bb *BlockBuilder) StateTree() *state.StateTree {
return bb.vm.StateTree().(*state.StateTree)
}
// ParentStateTree returns the parent state-tree (not the paren't tipset's parent state-tree).
func (bb *BlockBuilder) ParentStateTree() *state.StateTree {
return bb.parentSt
}
// StateTreeByHeight will return a state-tree up through and including the current in-progress
// epoch.
//
// NOTE: This will return the state after the given epoch, not the parent state for the epoch.
func (bb *BlockBuilder) StateTreeByHeight(epoch abi.ChainEpoch) (*state.StateTree, error) {
now := bb.Height()
if epoch > now {
return nil, xerrors.Errorf(
"cannot load state-tree from future: %d > %d", epoch, bb.Height(),
)
} else if epoch <= 0 {
return nil, xerrors.Errorf(
"cannot load state-tree: epoch %d <= 0", epoch,
)
}
// Manually handle "now" and "previous".
switch epoch {
case now:
return bb.StateTree(), nil
case now - 1:
return bb.ParentStateTree(), nil
}
// Get the tipset of the block _after_ the target epoch so we can use its parent state.
targetTs, err := bb.sm.ChainStore().GetTipsetByHeight(bb.ctx, epoch+1, bb.parentTs, false)
if err != nil {
return nil, err
}
return bb.sm.StateTree(targetTs.ParentState())
}
// Messages returns all messages currently packed into the next block.
// 1. DO NOT modify the slice, copy it.
// 2. DO NOT retain the slice, copy it.
func (bb *BlockBuilder) Messages() []*types.Message {
return bb.messages
}
// GasRemaining returns the amount of remaining gas in the next block.
func (bb *BlockBuilder) GasRemaining() int64 {
return targetGas - bb.gasTotal
}
// ParentTipSet returns the parent tipset.
func (bb *BlockBuilder) ParentTipSet() *types.TipSet {
return bb.parentTs
}
// Height returns the epoch for the target block.
func (bb *BlockBuilder) Height() abi.ChainEpoch {
return bb.parentTs.Height() + 1
}
// NetworkVersion returns the network version for the target block.
func (bb *BlockBuilder) NetworkVersion() network.Version {
return bb.sm.GetNtwkVersion(bb.ctx, bb.Height())
}
// StateManager returns the stmgr.StateManager.
func (bb *BlockBuilder) StateManager() *stmgr.StateManager {
return bb.sm
}
// ActorsVersion returns the actors version for the target block.
func (bb *BlockBuilder) ActorsVersion() actors.Version {
return actors.VersionForNetwork(bb.NetworkVersion())
}
func (bb *BlockBuilder) L() *zap.SugaredLogger {
return bb.logger
}

25
cmd/lotus-sim/simulation/blockbuilder/errors.go Normal file
View File

@ -0,0 +1,25 @@
package blockbuilder
import (
"errors"
"fmt"
)
// ErrOutOfGas is returned from BlockBuilder.PushMessage when the block does not have enough gas to
// fit the given message.
type ErrOutOfGas struct {
Available, Required int64
}
func (e *ErrOutOfGas) Error() string {
if e.Available == 0 {
return "out of gas: block full"
}
return fmt.Sprintf("out of gas: %d < %d", e.Required, e.Available)
}
// IsOutOfGas returns true if the error is an "out of gas" error.
func IsOutOfGas(err error) bool {
var oog *ErrOutOfGas
return errors.As(err, &oog)
}

58
cmd/lotus-sim/simulation/messages.go Normal file
View File

@ -0,0 +1,58 @@
package simulation
import (
"context"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/chain/types"
)
// toArray converts the given set of CIDs to an AMT. This is usually used to pack messages into blocks.
func toArray(store blockadt.Store, cids []cid.Cid) (cid.Cid, error) {
arr := blockadt.MakeEmptyArray(store)
for i, c := range cids {
oc := cbg.CborCid(c)
if err := arr.Set(uint64(i), &oc); err != nil {
return cid.Undef, err
}
}
return arr.Root()
}
// storeMessages packs a set of messages into a types.MsgMeta and returns the resulting CID. The
// resulting CID is valid for the BlocKHeader's Messages field.
func (sim *Simulation) storeMessages(ctx context.Context, messages []*types.Message) (cid.Cid, error) {
// We store all messages as "bls" messages so they're executed in-order. This ensures
// accurate gas accounting. It also ensures we don't, e.g., try to fund a miner after we
// fail a pre-commit...
var msgCids []cid.Cid
for _, msg := range messages {
c, err := sim.Node.Chainstore.PutMessage(msg)
if err != nil {
return cid.Undef, err
}
msgCids = append(msgCids, c)
}
adtStore := sim.Node.Chainstore.ActorStore(ctx)
blsMsgArr, err := toArray(adtStore, msgCids)
if err != nil {
return cid.Undef, err
}
sekpMsgArr, err := toArray(adtStore, nil)
if err != nil {
return cid.Undef, err
}
msgsCid, err := adtStore.Put(adtStore.Context(), &types.MsgMeta{
BlsMessages: blsMsgArr,
SecpkMessages: sekpMsgArr,
})
if err != nil {
return cid.Undef, err
}
return msgsCid, nil
}

179
cmd/lotus-sim/simulation/mock/mock.go Normal file
View File

@ -0,0 +1,179 @@
package mock
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
tutils "github.com/filecoin-project/specs-actors/v5/support/testing"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
)
// Ideally, we'd use extern/sector-storage/mock. Unfortunately, those mocks are a bit _too_ accurate
// and would force us to load sector info for window post proofs.
const (
mockSealProofPrefix = "valid seal proof:"
mockAggregateSealProofPrefix = "valid aggregate seal proof:"
mockPoStProofPrefix = "valid post proof:"
)
var log = logging.Logger("simulation-mock")
// mockVerifier is a simple mock for verifying "fake" proofs.
type mockVerifier struct{}
var Verifier ffiwrapper.Verifier = mockVerifier{}
func (mockVerifier) VerifySeal(proof proof5.SealVerifyInfo) (bool, error) {
addr, err := address.NewIDAddress(uint64(proof.Miner))
if err != nil {
return false, err
}
mockProof, err := MockSealProof(proof.SealProof, addr)
if err != nil {
return false, err
}
if bytes.Equal(proof.Proof, mockProof) {
return true, nil
}
log.Debugw("invalid seal proof", "expected", mockProof, "actual", proof.Proof, "miner", addr)
return false, nil
}
func (mockVerifier) VerifyAggregateSeals(aggregate proof5.AggregateSealVerifyProofAndInfos) (bool, error) {
addr, err := address.NewIDAddress(uint64(aggregate.Miner))
if err != nil {
return false, err
}
mockProof, err := MockAggregateSealProof(aggregate.SealProof, addr, len(aggregate.Infos))
if err != nil {
return false, err
}
if bytes.Equal(aggregate.Proof, mockProof) {
return true, nil
}
log.Debugw("invalid aggregate seal proof",
"expected", mockProof,
"actual", aggregate.Proof,
"count", len(aggregate.Infos),
"miner", addr,
)
return false, nil
}
func (mockVerifier) VerifyWinningPoSt(ctx context.Context, info proof5.WinningPoStVerifyInfo) (bool, error) {
panic("should not be called")
}
func (mockVerifier) VerifyWindowPoSt(ctx context.Context, info proof5.WindowPoStVerifyInfo) (bool, error) {
if len(info.Proofs) != 1 {
return false, fmt.Errorf("expected exactly one proof")
}
proof := info.Proofs[0]
addr, err := address.NewIDAddress(uint64(info.Prover))
if err != nil {
return false, err
}
mockProof, err := MockWindowPoStProof(proof.PoStProof, addr)
if err != nil {
return false, err
}
if bytes.Equal(proof.ProofBytes, mockProof) {
return true, nil
}
log.Debugw("invalid window post proof",
"expected", mockProof,
"actual", info.Proofs[0],
"miner", addr,
)
return false, nil
}
func (mockVerifier) GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error) {
panic("should not be called")
}
// MockSealProof generates a mock "seal" proof tied to the specified proof type and the given miner.
func MockSealProof(proofType abi.RegisteredSealProof, minerAddr address.Address) ([]byte, error) {
plen, err := proofType.ProofSize()
if err != nil {
return nil, err
}
proof := make([]byte, plen)
i := copy(proof, mockSealProofPrefix)
binary.BigEndian.PutUint64(proof[i:], uint64(proofType))
i += 8
i += copy(proof[i:], minerAddr.Bytes())
return proof, nil
}
// MockAggregateSealProof generates a mock "seal" aggregate proof tied to the specified proof type,
// the given miner, and the number of proven sectors.
func MockAggregateSealProof(proofType abi.RegisteredSealProof, minerAddr address.Address, count int) ([]byte, error) {
proof := make([]byte, aggProofLen(count))
i := copy(proof, mockAggregateSealProofPrefix)
binary.BigEndian.PutUint64(proof[i:], uint64(proofType))
i += 8
binary.BigEndian.PutUint64(proof[i:], uint64(count))
i += 8
i += copy(proof[i:], minerAddr.Bytes())
return proof, nil
}
// MockWindowPoStProof generates a mock "window post" proof tied to the specified proof type, and the
// given miner.
func MockWindowPoStProof(proofType abi.RegisteredPoStProof, minerAddr address.Address) ([]byte, error) {
plen, err := proofType.ProofSize()
if err != nil {
return nil, err
}
proof := make([]byte, plen)
i := copy(proof, mockPoStProofPrefix)
i += copy(proof[i:], minerAddr.Bytes())
return proof, nil
}
// makeCommR generates a "fake" but valid CommR for a sector. It is unique for the given sector/miner.
func MockCommR(minerAddr address.Address, sno abi.SectorNumber) cid.Cid {
return tutils.MakeCID(fmt.Sprintf("%s:%d", minerAddr, sno), &miner5.SealedCIDPrefix)
}
// TODO: dedup
func aggProofLen(nproofs int) int {
switch {
case nproofs <= 8:
return 11220
case nproofs <= 16:
return 14196
case nproofs <= 32:
return 17172
case nproofs <= 64:
return 20148
case nproofs <= 128:
return 23124
case nproofs <= 256:
return 26100
case nproofs <= 512:
return 29076
case nproofs <= 1024:
return 32052
case nproofs <= 2048:
return 35028
case nproofs <= 4096:
return 38004
case nproofs <= 8192:
return 40980
default:
panic("too many proofs")
}
}

241
cmd/lotus-sim/simulation/node.go Normal file
View File

@ -0,0 +1,241 @@
package simulation
import (
"context"
"strings"
"go.uber.org/multierr"
"golang.org/x/xerrors"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/stages"
"github.com/filecoin-project/lotus/node/repo"
)
// Node represents the local lotus node, or at least the part of it we care about.
type Node struct {
repo repo.LockedRepo
Blockstore blockstore.Blockstore
MetadataDS datastore.Batching
Chainstore *store.ChainStore
}
// OpenNode opens the local lotus node for writing. This will fail if the node is online.
func OpenNode(ctx context.Context, path string) (*Node, error) {
r, err := repo.NewFS(path)
if err != nil {
return nil, err
}
return NewNode(ctx, r)
}
// NewNode constructs a new node from the given repo.
func NewNode(ctx context.Context, r repo.Repo) (nd *Node, _err error) {
lr, err := r.Lock(repo.FullNode)
if err != nil {
return nil, err
}
defer func() {
if _err != nil {
_ = lr.Close()
}
}()
bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
if err != nil {
return nil, err
}
ds, err := lr.Datastore(ctx, "/metadata")
if err != nil {
return nil, err
}
return &Node{
repo: lr,
Chainstore: store.NewChainStore(bs, bs, ds, vm.Syscalls(mock.Verifier), nil),
MetadataDS: ds,
Blockstore: bs,
}, err
}
// Close cleanly close the repo. Please call this on shutdown to make sure everything is flushed.
func (nd *Node) Close() error {
if nd.repo != nil {
return nd.repo.Close()
}
return nil
}
// LoadSim loads
func (nd *Node) LoadSim(ctx context.Context, name string) (*Simulation, error) {
stages, err := stages.DefaultPipeline()
if err != nil {
return nil, err
}
sim := &Simulation{
Node: nd,
name: name,
stages: stages,
}
sim.head, err = sim.loadNamedTipSet("head")
if err != nil {
return nil, err
}
sim.start, err = sim.loadNamedTipSet("start")
if err != nil {
return nil, err
}
err = sim.loadConfig()
if err != nil {
return nil, xerrors.Errorf("failed to load config for simulation %s: %w", name, err)
}
us, err := sim.config.upgradeSchedule()
if err != nil {
return nil, xerrors.Errorf("failed to create upgrade schedule for simulation %s: %w", name, err)
}
sim.StateManager, err = stmgr.NewStateManagerWithUpgradeSchedule(nd.Chainstore, us)
if err != nil {
return nil, xerrors.Errorf("failed to create state manager for simulation %s: %w", name, err)
}
return sim, nil
}
// Create creates a new simulation.
//
// - This will fail if a simulation already exists with the given name.
// - Name must not contain a '/'.
func (nd *Node) CreateSim(ctx context.Context, name string, head *types.TipSet) (*Simulation, error) {
if strings.Contains(name, "/") {
return nil, xerrors.Errorf("simulation name %q cannot contain a '/'", name)
}
stages, err := stages.DefaultPipeline()
if err != nil {
return nil, err
}
sim := &Simulation{
name: name,
Node: nd,
StateManager: stmgr.NewStateManager(nd.Chainstore),
stages: stages,
}
if has, err := nd.MetadataDS.Has(sim.key("head")); err != nil {
return nil, err
} else if has {
return nil, xerrors.Errorf("simulation named %s already exists", name)
}
if err := sim.storeNamedTipSet("start", head); err != nil {
return nil, xerrors.Errorf("failed to set simulation start: %w", err)
}
if err := sim.SetHead(head); err != nil {
return nil, err
}
return sim, nil
}
// ListSims lists all simulations.
func (nd *Node) ListSims(ctx context.Context) ([]string, error) {
prefix := simulationPrefix.ChildString("head").String()
items, err := nd.MetadataDS.Query(query.Query{
Prefix: prefix,
KeysOnly: true,
Orders: []query.Order{query.OrderByKey{}},
})
if err != nil {
return nil, xerrors.Errorf("failed to list simulations: %w", err)
}
defer func() { _ = items.Close() }()
var names []string
for {
select {
case result, ok := <-items.Next():
if !ok {
return names, nil
}
if result.Error != nil {
return nil, xerrors.Errorf("failed to retrieve next simulation: %w", result.Error)
}
names = append(names, strings.TrimPrefix(result.Key, prefix+"/"))
case <-ctx.Done():
return nil, ctx.Err()
}
}
}
var simFields = []string{"head", "start", "config"}
// DeleteSim deletes a simulation and all related metadata.
//
// NOTE: This function does not delete associated messages, blocks, or chain state.
func (nd *Node) DeleteSim(ctx context.Context, name string) error {
var err error
for _, field := range simFields {
key := simulationPrefix.ChildString(field).ChildString(name)
err = multierr.Append(err, nd.MetadataDS.Delete(key))
}
return err
}
// CopySim copies a simulation.
func (nd *Node) CopySim(ctx context.Context, oldName, newName string) error {
if strings.Contains(newName, "/") {
return xerrors.Errorf("simulation name %q cannot contain a '/'", newName)
}
if strings.Contains(oldName, "/") {
return xerrors.Errorf("simulation name %q cannot contain a '/'", oldName)
}
values := make(map[string][]byte)
for _, field := range simFields {
key := simulationPrefix.ChildString(field).ChildString(oldName)
value, err := nd.MetadataDS.Get(key)
if err == datastore.ErrNotFound {
continue
} else if err != nil {
return err
}
values[field] = value
}
if _, ok := values["head"]; !ok {
return xerrors.Errorf("simulation named %s not found", oldName)
}
for _, field := range simFields {
key := simulationPrefix.ChildString(field).ChildString(newName)
var err error
if value, ok := values[field]; ok {
err = nd.MetadataDS.Put(key, value)
} else {
err = nd.MetadataDS.Delete(key)
}
if err != nil {
return err
}
}
return nil
}
// RenameSim renames a simulation.
func (nd *Node) RenameSim(ctx context.Context, oldName, newName string) error {
if err := nd.CopySim(ctx, oldName, newName); err != nil {
return err
}
return nd.DeleteSim(ctx, oldName)
}

408
cmd/lotus-sim/simulation/simulation.go Normal file
View File

@ -0,0 +1,408 @@
package simulation
import (
"context"
"encoding/json"
"runtime"
"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/stages"
)
var log = logging.Logger("simulation")
// config is the simulation's config, persisted to the local metadata store and loaded on start.
//
// See Simulation.loadConfig and Simulation.saveConfig.
type config struct {
Upgrades map[network.Version]abi.ChainEpoch
}
// upgradeSchedule constructs an stmgr.StateManager upgrade schedule, overriding any network upgrade
// epochs as specified in the config.
func (c *config) upgradeSchedule() (stmgr.UpgradeSchedule, error) {
upgradeSchedule := stmgr.DefaultUpgradeSchedule()
expected := make(map[network.Version]struct{}, len(c.Upgrades))
for nv := range c.Upgrades {
expected[nv] = struct{}{}
}
// Update network upgrade epochs.
newUpgradeSchedule := upgradeSchedule[:0]
for _, upgrade := range upgradeSchedule {
if height, ok := c.Upgrades[upgrade.Network]; ok {
delete(expected, upgrade.Network)
if height < 0 {
continue
}
upgrade.Height = height
}
newUpgradeSchedule = append(newUpgradeSchedule, upgrade)
}
// Make sure we didn't try to configure an unknown network version.
if len(expected) > 0 {
missing := make([]network.Version, 0, len(expected))
for nv := range expected {
missing = append(missing, nv)
}
return nil, xerrors.Errorf("unknown network versions %v in config", missing)
}
// Finally, validate it. This ensures we don't change the order of the upgrade or anything
// like that.
if err := newUpgradeSchedule.Validate(); err != nil {
return nil, err
}
return newUpgradeSchedule, nil
}
// Simulation specifies a lotus-sim simulation.
type Simulation struct {
Node *Node
StateManager *stmgr.StateManager
name string
config config
start *types.TipSet
// head
head *types.TipSet
stages []stages.Stage
}
// loadConfig loads a simulation's config from the datastore. This must be called on startup and may
// be called to restore the config from-disk.
func (sim *Simulation) loadConfig() error {
configBytes, err := sim.Node.MetadataDS.Get(sim.key("config"))
if err == nil {
err = json.Unmarshal(configBytes, &sim.config)
}
switch err {
case nil:
case datastore.ErrNotFound:
sim.config = config{}
default:
return xerrors.Errorf("failed to load config: %w", err)
}
return nil
}
// saveConfig saves the current config to the datastore. This must be called whenever the config is
// changed.
func (sim *Simulation) saveConfig() error {
buf, err := json.Marshal(sim.config)
if err != nil {
return err
}
return sim.Node.MetadataDS.Put(sim.key("config"), buf)
}
var simulationPrefix = datastore.NewKey("/simulation")
// key returns the the key in the form /simulation/<subkey>/<simulation-name>. For example,
// /simulation/head/default.
func (sim *Simulation) key(subkey string) datastore.Key {
return simulationPrefix.ChildString(subkey).ChildString(sim.name)
}
// loadNamedTipSet the tipset with the given name (for this simulation)
func (sim *Simulation) loadNamedTipSet(name string) (*types.TipSet, error) {
tskBytes, err := sim.Node.MetadataDS.Get(sim.key(name))
if err != nil {
return nil, xerrors.Errorf("failed to load tipset %s/%s: %w", sim.name, name, err)
}
tsk, err := types.TipSetKeyFromBytes(tskBytes)
if err != nil {
return nil, xerrors.Errorf("failed to parse tipste %v (%s/%s): %w", tskBytes, sim.name, name, err)
}
ts, err := sim.Node.Chainstore.LoadTipSet(tsk)
if err != nil {
return nil, xerrors.Errorf("failed to load tipset %s (%s/%s): %w", tsk, sim.name, name, err)
}
return ts, nil
}
// storeNamedTipSet stores the tipset at name (relative to the simulation).
func (sim *Simulation) storeNamedTipSet(name string, ts *types.TipSet) error {
if err := sim.Node.MetadataDS.Put(sim.key(name), ts.Key().Bytes()); err != nil {
return xerrors.Errorf("failed to store tipset (%s/%s): %w", sim.name, name, err)
}
return nil
}
// GetHead returns the current simulation head.
func (sim *Simulation) GetHead() *types.TipSet {
return sim.head
}
// GetStart returns simulation's parent tipset.
func (sim *Simulation) GetStart() *types.TipSet {
return sim.start
}
// GetNetworkVersion returns the current network version for the simulation.
func (sim *Simulation) GetNetworkVersion() network.Version {
return sim.StateManager.GetNtwkVersion(context.TODO(), sim.head.Height())
}
// SetHead updates the current head of the simulation and stores it in the metadata store. This is
// called for every Simulation.Step.
func (sim *Simulation) SetHead(head *types.TipSet) error {
if err := sim.storeNamedTipSet("head", head); err != nil {
return err
}
sim.head = head
return nil
}
// Name returns the simulation's name.
func (sim *Simulation) Name() string {
return sim.name
}
// SetUpgradeHeight sets the height of the given network version change (and saves the config).
//
// This fails if the specified epoch has already passed or the new upgrade schedule is invalid.
func (sim *Simulation) SetUpgradeHeight(nv network.Version, epoch abi.ChainEpoch) (_err error) {
if epoch <= sim.head.Height() {
return xerrors.Errorf("cannot set upgrade height in the past (%d <= %d)", epoch, sim.head.Height())
}
if sim.config.Upgrades == nil {
sim.config.Upgrades = make(map[network.Version]abi.ChainEpoch, 1)
}
sim.config.Upgrades[nv] = epoch
defer func() {
if _err != nil {
// try to restore the old config on error.
_ = sim.loadConfig()
}
}()
newUpgradeSchedule, err := sim.config.upgradeSchedule()
if err != nil {
return err
}
sm, err := stmgr.NewStateManagerWithUpgradeSchedule(sim.Node.Chainstore, newUpgradeSchedule)
if err != nil {
return err
}
err = sim.saveConfig()
if err != nil {
return err
}
sim.StateManager = sm
return nil
}
// ListUpgrades returns any future network upgrades.
func (sim *Simulation) ListUpgrades() (stmgr.UpgradeSchedule, error) {
upgrades, err := sim.config.upgradeSchedule()
if err != nil {
return nil, err
}
var pending stmgr.UpgradeSchedule
for _, upgrade := range upgrades {
if upgrade.Height < sim.head.Height() {
continue
}
pending = append(pending, upgrade)
}
return pending, nil
}
type AppliedMessage struct {
types.Message
types.MessageReceipt
}
// Walk walks the simulation's chain from the current head back to the first tipset.
func (sim *Simulation) Walk(
ctx context.Context,
lookback int64,
cb func(sm *stmgr.StateManager,
ts *types.TipSet,
stCid cid.Cid,
messages []*AppliedMessage) error,
) error {
store := sim.Node.Chainstore.ActorStore(ctx)
minEpoch := sim.start.Height()
if lookback != 0 {
minEpoch = sim.head.Height() - abi.ChainEpoch(lookback)
}
// Given tha loading messages and receipts can be a little bit slow, we do this in parallel.
//
// 1. We spin up some number of workers.
// 2. We hand tipsets to workers in round-robin order.
// 3. We pull "resolved" tipsets in the same round-robin order.
// 4. We serially call the callback in reverse-chain order.
//
// We have a buffer of size 1 for both resolved tipsets and unresolved tipsets. This should
// ensure that we never block unecessarily.
type work struct {
ts *types.TipSet
stCid cid.Cid
recCid cid.Cid
}
type result struct {
ts *types.TipSet
stCid cid.Cid
messages []*AppliedMessage
}
// This is more disk bound than CPU bound, but eh...
workerCount := runtime.NumCPU() * 2
workQs := make([]chan *work, workerCount)
resultQs := make([]chan *result, workerCount)
for i := range workQs {
workQs[i] = make(chan *work, 1)
}
for i := range resultQs {
resultQs[i] = make(chan *result, 1)
}
grp, ctx := errgroup.WithContext(ctx)
// Walk the chain and fire off work items.
grp.Go(func() error {
ts := sim.head
stCid, recCid, err := sim.StateManager.TipSetState(ctx, ts)
if err != nil {
return err
}
i := 0
for ts.Height() > minEpoch {
if err := ctx.Err(); err != nil {
return ctx.Err()
}
select {
case workQs[i] <- &work{ts, stCid, recCid}:
case <-ctx.Done():
return ctx.Err()
}
stCid = ts.MinTicketBlock().ParentStateRoot
recCid = ts.MinTicketBlock().ParentMessageReceipts
ts, err = sim.Node.Chainstore.LoadTipSet(ts.Parents())
if err != nil {
return xerrors.Errorf("loading parent: %w", err)
}
i = (i + 1) % workerCount
}
for _, q := range workQs {
close(q)
}
return nil
})
// Spin up one worker per queue pair.
for i := 0; i < workerCount; i++ {
workQ := workQs[i]
resultQ := resultQs[i]
grp.Go(func() error {
for {
if err := ctx.Err(); err != nil {
return ctx.Err()
}
var job *work
var ok bool
select {
case job, ok = <-workQ:
case <-ctx.Done():
return ctx.Err()
}
if !ok {
break
}
msgs, err := sim.Node.Chainstore.MessagesForTipset(job.ts)
if err != nil {
return err
}
recs, err := blockadt.AsArray(store, job.recCid)
if err != nil {
return xerrors.Errorf("amt load: %w", err)
}
applied := make([]*AppliedMessage, len(msgs))
var rec types.MessageReceipt
err = recs.ForEach(&rec, func(i int64) error {
applied[i] = &AppliedMessage{
Message: *msgs[i].VMMessage(),
MessageReceipt: rec,
}
return nil
})
if err != nil {
return err
}
select {
case resultQ <- &result{
ts: job.ts,
stCid: job.stCid,
messages: applied,
}:
case <-ctx.Done():
return ctx.Err()
}
}
close(resultQ)
return nil
})
}
// Process results in the same order we enqueued them.
grp.Go(func() error {
qs := resultQs
for len(qs) > 0 {
newQs := qs[:0]
for _, q := range qs {
if err := ctx.Err(); err != nil {
return ctx.Err()
}
select {
case r, ok := <-q:
if !ok {
continue
}
err := cb(sim.StateManager, r.ts, r.stCid, r.messages)
if err != nil {
return err
}
case <-ctx.Done():
return ctx.Err()
}
newQs = append(newQs, q)
}
qs = newQs
}
return nil
})
// Wait for everything to finish.
return grp.Wait()
}

38
cmd/lotus-sim/simulation/stages/actor_iter.go Normal file
View File

@ -0,0 +1,38 @@
package stages
import (
"math/rand"
"github.com/filecoin-project/go-address"
)
// actorIter is a simple persistent iterator that loops over a set of actors.
type actorIter struct {
actors []address.Address
offset int
}
// shuffle randomly permutes the set of actors.
func (p *actorIter) shuffle() {
rand.Shuffle(len(p.actors), func(i, j int) {
p.actors[i], p.actors[j] = p.actors[j], p.actors[i]
})
}
// next returns the next actor's address and advances the iterator.
func (p *actorIter) next() address.Address {
next := p.actors[p.offset]
p.offset++
p.offset %= len(p.actors)
return next
}
// add adds a new actor to the iterator.
func (p *actorIter) add(addr address.Address) {
p.actors = append(p.actors, addr)
}
// len returns the number of actors in the iterator.
func (p *actorIter) len() int {
return len(p.actors)
}

200
cmd/lotus-sim/simulation/stages/commit_queue.go Normal file
View File

@ -0,0 +1,200 @@
package stages
import (
"sort"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/policy"
)
// pendingCommitTracker tracks pending commits per-miner for a single epoch.
type pendingCommitTracker map[address.Address]minerPendingCommits
// minerPendingCommits tracks a miner's pending commits during a single epoch (grouped by seal proof type).
type minerPendingCommits map[abi.RegisteredSealProof][]abi.SectorNumber
// finish marks count sectors of the given proof type as "prove-committed".
func (m minerPendingCommits) finish(proof abi.RegisteredSealProof, count int) {
snos := m[proof]
if len(snos) < count {
panic("not enough sector numbers to finish")
} else if len(snos) == count {
delete(m, proof)
} else {
m[proof] = snos[count:]
}
}
// empty returns true if there are no pending commits.
func (m minerPendingCommits) empty() bool {
return len(m) == 0
}
// count returns the number of pending commits.
func (m minerPendingCommits) count() int {
count := 0
for _, snos := range m {
count += len(snos)
}
return count
}
// commitQueue is used to track pending prove-commits.
//
// Miners are processed in round-robin where _all_ commits from a given miner are finished before
// moving on to the next. This is designed to maximize batching.
type commitQueue struct {
minerQueue []address.Address
queue []pendingCommitTracker
offset abi.ChainEpoch
}
// ready returns the number of prove-commits ready to be proven at the current epoch. Useful for logging.
func (q *commitQueue) ready() int {
if len(q.queue) == 0 {
return 0
}
count := 0
for _, pending := range q.queue[0] {
count += pending.count()
}
return count
}
// nextMiner returns the next miner to be proved and the set of pending prove commits for that
// miner. When some number of sectors have successfully been proven, call "finish" so we don't try
// to prove them again.
func (q *commitQueue) nextMiner() (address.Address, minerPendingCommits, bool) {
if len(q.queue) == 0 {
return address.Undef, nil, false
}
next := q.queue[0]
// Go through the queue and find the first non-empty batch.
for len(q.minerQueue) > 0 {
addr := q.minerQueue[0]
q.minerQueue = q.minerQueue[1:]
pending := next[addr]
if !pending.empty() {
return addr, pending, true
}
delete(next, addr)
}
return address.Undef, nil, false
}
// advanceEpoch will advance to the next epoch. If some sectors were left unproven in the current
// epoch, they will be "prepended" into the next epochs sector set.
func (q *commitQueue) advanceEpoch(epoch abi.ChainEpoch) {
if epoch < q.offset {
panic("cannot roll epoch backwards")
}
// Now we "roll forwards", merging each epoch we advance over with the next.
for len(q.queue) > 1 && q.offset < epoch {
curr := q.queue[0]
q.queue[0] = nil
q.queue = q.queue[1:]
q.offset++
next := q.queue[0]
// Cleanup empty entries.
for addr, pending := range curr {
if pending.empty() {
delete(curr, addr)
}
}
// If the entire level is actually empty, just skip to the next one.
if len(curr) == 0 {
continue
}
// Otherwise, merge the next into the current.
for addr, nextPending := range next {
currPending := curr[addr]
if currPending.empty() {
curr[addr] = nextPending
continue
}
for ty, nextSnos := range nextPending {
currSnos := currPending[ty]
if len(currSnos) == 0 {
currPending[ty] = nextSnos
continue
}
currPending[ty] = append(currSnos, nextSnos...)
}
}
// Now replace next with the merged curr.
q.queue[0] = curr
}
q.offset = epoch
if len(q.queue) == 0 {
return
}
next := q.queue[0]
seenMiners := make(map[address.Address]struct{}, len(q.minerQueue))
for _, addr := range q.minerQueue {
seenMiners[addr] = struct{}{}
}
// Find the new miners not already in the queue.
offset := len(q.minerQueue)
for addr, pending := range next {
if pending.empty() {
delete(next, addr)
continue
}
if _, ok := seenMiners[addr]; ok {
continue
}
q.minerQueue = append(q.minerQueue, addr)
}
// Sort the new miners only.
newMiners := q.minerQueue[offset:]
sort.Slice(newMiners, func(i, j int) bool {
// eh, escape analysis should be fine here...
return string(newMiners[i].Bytes()) < string(newMiners[j].Bytes())
})
}
// enquueProveCommit enqueues prove-commit for the given pre-commit for the given miner.
func (q *commitQueue) enqueueProveCommit(addr address.Address, preCommitEpoch abi.ChainEpoch, info miner.SectorPreCommitInfo) error {
// Compute the epoch at which we can start trying to commit.
preCommitDelay := policy.GetPreCommitChallengeDelay()
minCommitEpoch := preCommitEpoch + preCommitDelay + 1
// Figure out the offset in the queue.
i := int(minCommitEpoch - q.offset)
if i < 0 {
i = 0
}
// Expand capacity and insert.
if cap(q.queue) <= i {
pc := make([]pendingCommitTracker, i+1, preCommitDelay*2)
copy(pc, q.queue)
q.queue = pc
} else if len(q.queue) <= i {
q.queue = q.queue[:i+1]
}
tracker := q.queue[i]
if tracker == nil {
tracker = make(pendingCommitTracker)
q.queue[i] = tracker
}
minerPending := tracker[addr]
if minerPending == nil {
minerPending = make(minerPendingCommits)
tracker[addr] = minerPending
}
minerPending[info.SealProof] = append(minerPending[info.SealProof], info.SectorNumber)
return nil
}

128
cmd/lotus-sim/simulation/stages/commit_queue_test.go Normal file
View File

@ -0,0 +1,128 @@
package stages
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/policy"
)
func TestCommitQueue(t *testing.T) {
var q commitQueue
addr1, err := address.NewIDAddress(1000)
require.NoError(t, err)
proofType := abi.RegisteredSealProof_StackedDrg64GiBV1_1
require.NoError(t, q.enqueueProveCommit(addr1, 0, miner.SectorPreCommitInfo{
SealProof: proofType,
SectorNumber: 0,
}))
require.NoError(t, q.enqueueProveCommit(addr1, 0, miner.SectorPreCommitInfo{
SealProof: proofType,
SectorNumber: 1,
}))
require.NoError(t, q.enqueueProveCommit(addr1, 1, miner.SectorPreCommitInfo{
SealProof: proofType,
SectorNumber: 2,
}))
require.NoError(t, q.enqueueProveCommit(addr1, 1, miner.SectorPreCommitInfo{
SealProof: proofType,
SectorNumber: 3,
}))
require.NoError(t, q.enqueueProveCommit(addr1, 3, miner.SectorPreCommitInfo{
SealProof: proofType,
SectorNumber: 4,
}))
require.NoError(t, q.enqueueProveCommit(addr1, 4, miner.SectorPreCommitInfo{
SealProof: proofType,
SectorNumber: 5,
}))
require.NoError(t, q.enqueueProveCommit(addr1, 6, miner.SectorPreCommitInfo{
SealProof: proofType,
SectorNumber: 6,
}))
epoch := abi.ChainEpoch(0)
q.advanceEpoch(epoch)
_, _, ok := q.nextMiner()
require.False(t, ok)
epoch += policy.GetPreCommitChallengeDelay()
q.advanceEpoch(epoch)
_, _, ok = q.nextMiner()
require.False(t, ok)
// 0 : empty + non-empty
epoch++
q.advanceEpoch(epoch)
addr, sectors, ok := q.nextMiner()
require.True(t, ok)
require.Equal(t, sectors.count(), 2)
require.Equal(t, addr, addr1)
sectors.finish(proofType, 1)
require.Equal(t, sectors.count(), 1)
require.EqualValues(t, []abi.SectorNumber{1}, sectors[proofType])
// 1 : non-empty + non-empty
epoch++
q.advanceEpoch(epoch)
addr, sectors, ok = q.nextMiner()
require.True(t, ok)
require.Equal(t, addr, addr1)
require.Equal(t, sectors.count(), 3)
require.EqualValues(t, []abi.SectorNumber{1, 2, 3}, sectors[proofType])
sectors.finish(proofType, 3)
require.Equal(t, sectors.count(), 0)
// 2 : empty + empty
epoch++
q.advanceEpoch(epoch)
_, _, ok = q.nextMiner()
require.False(t, ok)
// 3 : empty + non-empty
epoch++
q.advanceEpoch(epoch)
_, sectors, ok = q.nextMiner()
require.True(t, ok)
require.Equal(t, sectors.count(), 1)
require.EqualValues(t, []abi.SectorNumber{4}, sectors[proofType])
// 4 : non-empty + non-empty
epoch++
q.advanceEpoch(epoch)
_, sectors, ok = q.nextMiner()
require.True(t, ok)
require.Equal(t, sectors.count(), 2)
require.EqualValues(t, []abi.SectorNumber{4, 5}, sectors[proofType])
// 5 : empty + non-empty
epoch++
q.advanceEpoch(epoch)
_, sectors, ok = q.nextMiner()
require.True(t, ok)
require.Equal(t, sectors.count(), 2)
require.EqualValues(t, []abi.SectorNumber{4, 5}, sectors[proofType])
sectors.finish(proofType, 1)
require.EqualValues(t, []abi.SectorNumber{5}, sectors[proofType])
// 6
epoch++
q.advanceEpoch(epoch)
_, sectors, ok = q.nextMiner()
require.True(t, ok)
require.Equal(t, sectors.count(), 2)
require.EqualValues(t, []abi.SectorNumber{5, 6}, sectors[proofType])
// 8
epoch += 2
q.advanceEpoch(epoch)
_, sectors, ok = q.nextMiner()
require.True(t, ok)
require.Equal(t, sectors.count(), 2)
require.EqualValues(t, []abi.SectorNumber{5, 6}, sectors[proofType])
}

318
cmd/lotus-sim/simulation/stages/funding_stage.go Normal file
View File

@ -0,0 +1,318 @@
package stages
import (
"bytes"
"context"
"sort"
"time"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
)
var (
TargetFunds = abi.TokenAmount(types.MustParseFIL("1000FIL"))
MinimumFunds = abi.TokenAmount(types.MustParseFIL("100FIL"))
)
type FundingStage struct {
fundAccount address.Address
taxMin abi.TokenAmount
minFunds, maxFunds abi.TokenAmount
}
func NewFundingStage() (*FundingStage, error) {
// TODO: make all this configurable.
addr, err := address.NewIDAddress(100)
if err != nil {
return nil, err
}
return &FundingStage{
fundAccount: addr,
taxMin: abi.TokenAmount(types.MustParseFIL("1000FIL")),
minFunds: abi.TokenAmount(types.MustParseFIL("1000000FIL")),
maxFunds: abi.TokenAmount(types.MustParseFIL("100000000FIL")),
}, nil
}
func (*FundingStage) Name() string {
return "funding"
}
func (fs *FundingStage) Fund(bb *blockbuilder.BlockBuilder, target address.Address) error {
return fs.fund(bb, target, 0)
}
// sendAndFund "packs" the given message, funding the actor if necessary. It:
//
// 1. Tries to send the given message.
// 2. If that fails, it checks to see if the exit code was ErrInsufficientFunds.
// 3. If so, it sends 1K FIL from the "burnt funds actor" (because we need to send it from
// somewhere) and re-tries the message.0
func (fs *FundingStage) SendAndFund(bb *blockbuilder.BlockBuilder, msg *types.Message) (res *types.MessageReceipt, err error) {
for i := 0; i < 10; i++ {
res, err = bb.PushMessage(msg)
if err == nil {
return res, nil
}
aerr, ok := err.(aerrors.ActorError)
if !ok || aerr.RetCode() != exitcode.ErrInsufficientFunds {
return nil, err
}
// Ok, insufficient funds. Let's fund this miner and try again.
if err := fs.fund(bb, msg.To, i); err != nil {
if !blockbuilder.IsOutOfGas(err) {
err = xerrors.Errorf("failed to fund %s: %w", msg.To, err)
}
return nil, err
}
}
return res, err
}
// fund funds the target actor with 'TargetFunds << shift' FIL. The "shift" parameter allows us to
// keep doubling the amount until the intended operation succeeds.
func (fs *FundingStage) fund(bb *blockbuilder.BlockBuilder, target address.Address, shift int) error {
amt := TargetFunds
if shift > 0 {
if shift >= 8 {
shift = 8 // cap
}
amt = big.Lsh(amt, uint(shift))
}
_, err := bb.PushMessage(&types.Message{
From: fs.fundAccount,
To: target,
Value: amt,
Method: builtin.MethodSend,
})
return err
}
func (fs *FundingStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
st := bb.StateTree()
fundAccActor, err := st.GetActor(fs.fundAccount)
if err != nil {
return err
}
if fs.minFunds.LessThan(fundAccActor.Balance) {
return nil
}
// Ok, we're going to go fund this thing.
start := time.Now()
type actor struct {
types.Actor
Address address.Address
}
var targets []*actor
err = st.ForEach(func(addr address.Address, act *types.Actor) error {
// Don't steal from ourselves!
if addr == fs.fundAccount {
return nil
}
if act.Balance.LessThan(fs.taxMin) {
return nil
}
if !(builtin.IsAccountActor(act.Code) || builtin.IsMultisigActor(act.Code)) {
return nil
}
targets = append(targets, &actor{*act, addr})
return nil
})
if err != nil {
return err
}
balance := fundAccActor.Balance.Copy()
sort.Slice(targets, func(i, j int) bool {
return targets[i].Balance.GreaterThan(targets[j].Balance)
})
store := bb.ActorStore()
epoch := bb.Height()
actorsVersion := bb.ActorsVersion()
var accounts, multisigs int
defer func() {
if _err != nil {
return
}
bb.L().Infow("finished funding the simulation",
"duration", time.Since(start),
"targets", len(targets),
"epoch", epoch,
"new-balance", types.FIL(balance),
"old-balance", types.FIL(fundAccActor.Balance),
"multisigs", multisigs,
"accounts", accounts,
)
}()
for _, actor := range targets {
switch {
case builtin.IsAccountActor(actor.Code):
if _, err := bb.PushMessage(&types.Message{
From: actor.Address,
To: fs.fundAccount,
Value: actor.Balance,
}); blockbuilder.IsOutOfGas(err) {
return nil
} else if err != nil {
return err
}
accounts++
case builtin.IsMultisigActor(actor.Code):
msigState, err := multisig.Load(store, &actor.Actor)
if err != nil {
return err
}
threshold, err := msigState.Threshold()
if err != nil {
return err
}
if threshold > 16 {
bb.L().Debugw("ignoring multisig with high threshold",
"multisig", actor.Address,
"threshold", threshold,
"max", 16,
)
continue
}
locked, err := msigState.LockedBalance(epoch)
if err != nil {
return err
}
if locked.LessThan(fs.taxMin) {
continue // not worth it.
}
allSigners, err := msigState.Signers()
if err != nil {
return err
}
signers := make([]address.Address, 0, threshold)
for _, signer := range allSigners {
actor, err := st.GetActor(signer)
if err != nil {
return err
}
if !builtin.IsAccountActor(actor.Code) {
// I am so not dealing with this mess.
continue
}
if uint64(len(signers)) >= threshold {
break
}
}
// Ok, we're not dealing with this one.
if uint64(len(signers)) < threshold {
continue
}
available := big.Sub(actor.Balance, locked)
var txnId uint64
{
msg, err := multisig.Message(actorsVersion, signers[0]).Propose(
actor.Address, fs.fundAccount, available,
builtin.MethodSend, nil,
)
if err != nil {
return err
}
res, err := bb.PushMessage(msg)
if err != nil {
if blockbuilder.IsOutOfGas(err) {
err = nil
}
return err
}
var ret multisig.ProposeReturn
err = ret.UnmarshalCBOR(bytes.NewReader(res.Return))
if err != nil {
return err
}
if ret.Applied {
if !ret.Code.IsSuccess() {
bb.L().Errorw("failed to tax multisig",
"multisig", actor.Address,
"exitcode", ret.Code,
)
}
break
}
txnId = uint64(ret.TxnID)
}
var ret multisig.ProposeReturn
for _, signer := range signers[1:] {
msg, err := multisig.Message(actorsVersion, signer).Approve(actor.Address, txnId, nil)
if err != nil {
return err
}
res, err := bb.PushMessage(msg)
if err != nil {
if blockbuilder.IsOutOfGas(err) {
err = nil
}
return err
}
var ret multisig.ProposeReturn
err = ret.UnmarshalCBOR(bytes.NewReader(res.Return))
if err != nil {
return err
}
// A bit redundant, but nice.
if ret.Applied {
break
}
}
if !ret.Applied {
bb.L().Errorw("failed to apply multisig transaction",
"multisig", actor.Address,
"txnid", txnId,
"signers", len(signers),
"threshold", threshold,
)
continue
}
if !ret.Code.IsSuccess() {
bb.L().Errorw("failed to tax multisig",
"multisig", actor.Address,
"txnid", txnId,
"exitcode", ret.Code,
)
} else {
multisigs++
}
default:
panic("impossible case")
}
balance = big.Int{Int: balance.Add(balance.Int, actor.Balance.Int)}
if balance.GreaterThanEqual(fs.maxFunds) {
// There's no need to get greedy.
// Well, really, we're trying to avoid messing with state _too_ much.
return nil
}
}
return nil
}

27
cmd/lotus-sim/simulation/stages/interface.go Normal file
View File

@ -0,0 +1,27 @@
package stages
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
)
// Stage is a stage of the simulation. It's asked to pack messages for every block.
type Stage interface {
Name() string
PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) error
}
type Funding interface {
SendAndFund(*blockbuilder.BlockBuilder, *types.Message) (*types.MessageReceipt, error)
Fund(*blockbuilder.BlockBuilder, address.Address) error
}
type Committer interface {
EnqueueProveCommit(addr address.Address, preCommitEpoch abi.ChainEpoch, info miner.SectorPreCommitInfo) error
}

31
cmd/lotus-sim/simulation/stages/pipeline.go Normal file
View File

@ -0,0 +1,31 @@
package stages
// DefaultPipeline returns the default stage pipeline. This pipeline.
//
// 1. Funds a "funding" actor, if necessary.
// 2. Submits any ready window posts.
// 3. Submits any ready prove commits.
// 4. Submits pre-commits with the remaining gas.
func DefaultPipeline() ([]Stage, error) {
// TODO: make this configurable. E.g., through DI?
// Ideally, we'd also be able to change priority, limit throughput (by limiting gas in the
// block builder, etc.
funding, err := NewFundingStage()
if err != nil {
return nil, err
}
wdpost, err := NewWindowPoStStage()
if err != nil {
return nil, err
}
provecommit, err := NewProveCommitStage(funding)
if err != nil {
return nil, err
}
precommit, err := NewPreCommitStage(funding, provecommit)
if err != nil {
return nil, err
}
return []Stage{funding, wdpost, provecommit, precommit}, nil
}

347
cmd/lotus-sim/simulation/stages/precommit_stage.go Normal file
View File

@ -0,0 +1,347 @@
package stages
import (
"context"
"sort"
"time"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/network"
miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
)
const (
minPreCommitBatchSize = 1
maxPreCommitBatchSize = miner5.PreCommitSectorBatchMaxSize
)
type PreCommitStage struct {
funding Funding
committer Committer
// The tiers represent the top 1%, top 10%, and everyone else. When sealing sectors, we seal
// a group of sectors for the top 1%, a group (half that size) for the top 10%, and one
// sector for everyone else. We determine these rates by looking at two power tables.
// TODO Ideally we'd "learn" this distribution from the network. But this is good enough for
// now.
top1, top10, rest actorIter
initialized bool
}
func NewPreCommitStage(funding Funding, committer Committer) (*PreCommitStage, error) {
return &PreCommitStage{
funding: funding,
committer: committer,
}, nil
}
func (*PreCommitStage) Name() string {
return "pre-commit"
}
// packPreCommits packs pre-commit messages until the block is full.
func (stage *PreCommitStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
if !stage.initialized {
if err := stage.load(ctx, bb); err != nil {
return err
}
}
var (
full bool
top1Count, top10Count, restCount int
)
start := time.Now()
defer func() {
if _err != nil {
return
}
bb.L().Debugw("packed pre commits",
"done", top1Count+top10Count+restCount,
"top1", top1Count,
"top10", top10Count,
"rest", restCount,
"filled-block", full,
"duration", time.Since(start),
)
}()
var top1Miners, top10Miners, restMiners int
for i := 0; ; i++ {
var (
minerAddr address.Address
count *int
)
// We pre-commit for the top 1%, 10%, and the of the network 1/3rd of the time each.
// This won't yield the most accurate distribution... but it'll give us a good
// enough distribution.
switch {
case (i%3) <= 0 && top1Miners < stage.top1.len():
count = &top1Count
minerAddr = stage.top1.next()
top1Miners++
case (i%3) <= 1 && top10Miners < stage.top10.len():
count = &top10Count
minerAddr = stage.top10.next()
top10Miners++
case (i%3) <= 2 && restMiners < stage.rest.len():
count = &restCount
minerAddr = stage.rest.next()
restMiners++
default:
// Well, we've run through all miners.
return nil
}
var (
added int
err error
)
added, full, err = stage.packMiner(ctx, bb, minerAddr, maxProveCommitBatchSize)
if err != nil {
return xerrors.Errorf("failed to pack precommits for miner %s: %w", minerAddr, err)
}
*count += added
if full {
return nil
}
}
}
// packPreCommitsMiner packs count pre-commits for the given miner.
func (stage *PreCommitStage) packMiner(
ctx context.Context, bb *blockbuilder.BlockBuilder,
minerAddr address.Address, count int,
) (int, bool, error) {
log := bb.L().With("miner", minerAddr)
epoch := bb.Height()
nv := bb.NetworkVersion()
minerActor, err := bb.StateTree().GetActor(minerAddr)
if err != nil {
return 0, false, err
}
minerState, err := miner.Load(bb.ActorStore(), minerActor)
if err != nil {
return 0, false, err
}
minerInfo, err := minerState.Info()
if err != nil {
return 0, false, err
}
// Make sure the miner is funded.
minerBalance, err := minerState.AvailableBalance(minerActor.Balance)
if err != nil {
return 0, false, err
}
if big.Cmp(minerBalance, MinimumFunds) < 0 {
err := stage.funding.Fund(bb, minerAddr)
if err != nil {
if blockbuilder.IsOutOfGas(err) {
return 0, true, nil
}
return 0, false, err
}
}
// Generate pre-commits.
sealType, err := miner.PreferredSealProofTypeFromWindowPoStType(
nv, minerInfo.WindowPoStProofType,
)
if err != nil {
return 0, false, err
}
sectorNos, err := minerState.UnallocatedSectorNumbers(count)
if err != nil {
return 0, false, err
}
expiration := epoch + policy.GetMaxSectorExpirationExtension()
infos := make([]miner.SectorPreCommitInfo, len(sectorNos))
for i, sno := range sectorNos {
infos[i] = miner.SectorPreCommitInfo{
SealProof: sealType,
SectorNumber: sno,
SealedCID: mock.MockCommR(minerAddr, sno),
SealRandEpoch: epoch - 1,
Expiration: expiration,
}
}
// Commit the pre-commits.
added := 0
if nv >= network.Version13 {
targetBatchSize := maxPreCommitBatchSize
for targetBatchSize >= minPreCommitBatchSize && len(infos) >= minPreCommitBatchSize {
batch := infos
if len(batch) > targetBatchSize {
batch = batch[:targetBatchSize]
}
params := miner5.PreCommitSectorBatchParams{
Sectors: batch,
}
enc, err := actors.SerializeParams(&params)
if err != nil {
return added, false, err
}
// NOTE: just in-case, sendAndFund will "fund" and re-try for any message
// that fails due to "insufficient funds".
if _, err := stage.funding.SendAndFund(bb, &types.Message{
To: minerAddr,
From: minerInfo.Worker,
Value: abi.NewTokenAmount(0),
Method: miner.Methods.PreCommitSectorBatch,
Params: enc,
}); blockbuilder.IsOutOfGas(err) {
// try again with a smaller batch.
targetBatchSize /= 2
continue
} else if aerr, ok := err.(aerrors.ActorError); ok && !aerr.IsFatal() {
// Log the error and move on. No reason to stop.
log.Errorw("failed to pre-commit for unknown reasons",
"error", aerr,
"sectors", batch,
)
return added, false, nil
} else if err != nil {
return added, false, err
}
for _, info := range batch {
if err := stage.committer.EnqueueProveCommit(minerAddr, epoch, info); err != nil {
return added, false, err
}
added++
}
infos = infos[len(batch):]
}
}
for _, info := range infos {
enc, err := actors.SerializeParams(&info) //nolint
if err != nil {
return 0, false, err
}
if _, err := stage.funding.SendAndFund(bb, &types.Message{
To: minerAddr,
From: minerInfo.Worker,
Value: abi.NewTokenAmount(0),
Method: miner.Methods.PreCommitSector,
Params: enc,
}); blockbuilder.IsOutOfGas(err) {
return added, true, nil
} else if err != nil {
return added, false, err
}
if err := stage.committer.EnqueueProveCommit(minerAddr, epoch, info); err != nil {
return added, false, err
}
added++
}
return added, false, nil
}
func (stage *PreCommitStage) load(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
bb.L().Infow("loading miner power for pre-commits")
start := time.Now()
defer func() {
if _err != nil {
return
}
bb.L().Infow("loaded miner power for pre-commits",
"duration", time.Since(start),
"top1", stage.top1.len(),
"top10", stage.top10.len(),
"rest", stage.rest.len(),
)
}()
store := bb.ActorStore()
st := bb.ParentStateTree()
powerState, err := loadPower(store, st)
if err != nil {
return xerrors.Errorf("failed to power actor: %w", err)
}
type onboardingInfo struct {
addr address.Address
sectorCount uint64
}
var sealList []onboardingInfo
err = powerState.ForEachClaim(func(addr address.Address, claim power.Claim) error {
if claim.RawBytePower.IsZero() {
return nil
}
minerState, err := loadMiner(store, st, addr)
if err != nil {
return err
}
info, err := minerState.Info()
if err != nil {
return err
}
sectorCount := sectorsFromClaim(info.SectorSize, claim)
if sectorCount > 0 {
sealList = append(sealList, onboardingInfo{addr, uint64(sectorCount)})
}
return nil
})
if err != nil {
return err
}
if len(sealList) == 0 {
return xerrors.Errorf("simulation has no miners")
}
// Now that we have a list of sealing miners, sort them into percentiles.
sort.Slice(sealList, func(i, j int) bool {
return sealList[i].sectorCount < sealList[j].sectorCount
})
// reset, just in case.
stage.top1 = actorIter{}
stage.top10 = actorIter{}
stage.rest = actorIter{}
for i, oi := range sealList {
var dist *actorIter
if i < len(sealList)/100 {
dist = &stage.top1
} else if i < len(sealList)/10 {
dist = &stage.top10
} else {
dist = &stage.rest
}
dist.add(oi.addr)
}
stage.top1.shuffle()
stage.top10.shuffle()
stage.rest.shuffle()
stage.initialized = true
return nil
}

372
cmd/lotus-sim/simulation/stages/provecommit_stage.go Normal file
View File

@ -0,0 +1,372 @@
package stages
import (
"context"
"time"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/go-state-types/network"
miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
power5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
)
const (
minProveCommitBatchSize = 4
maxProveCommitBatchSize = miner5.MaxAggregatedSectors
)
type ProveCommitStage struct {
funding Funding
// We track the set of pending commits. On simulation load, and when a new pre-commit is
// added to the chain, we put the commit in this queue. advanceEpoch(currentEpoch) should be
// called on this queue at every epoch before using it.
commitQueue commitQueue
initialized bool
}
func NewProveCommitStage(funding Funding) (*ProveCommitStage, error) {
return &ProveCommitStage{
funding: funding,
}, nil
}
func (*ProveCommitStage) Name() string {
return "prove-commit"
}
func (stage *ProveCommitStage) EnqueueProveCommit(
minerAddr address.Address, preCommitEpoch abi.ChainEpoch, info miner.SectorPreCommitInfo,
) error {
return stage.commitQueue.enqueueProveCommit(minerAddr, preCommitEpoch, info)
}
// packProveCommits packs all prove-commits for all "ready to be proven" sectors until it fills the
// block or runs out.
func (stage *ProveCommitStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
if !stage.initialized {
if err := stage.load(ctx, bb); err != nil {
return err
}
}
// Roll the commitQueue forward.
stage.commitQueue.advanceEpoch(bb.Height())
start := time.Now()
var failed, done, unbatched, count int
defer func() {
if _err != nil {
return
}
remaining := stage.commitQueue.ready()
bb.L().Debugw("packed prove commits",
"remaining", remaining,
"done", done,
"failed", failed,
"unbatched", unbatched,
"miners-processed", count,
"duration", time.Since(start),
)
}()
for {
addr, pending, ok := stage.commitQueue.nextMiner()
if !ok {
return nil
}
res, err := stage.packProveCommitsMiner(ctx, bb, addr, pending)
if err != nil {
return err
}
failed += res.failed
done += res.done
unbatched += res.unbatched
count++
if res.full {
return nil
}
}
}
type proveCommitResult struct {
done, failed, unbatched int
full bool
}
// packProveCommitsMiner enqueues a prove commits from the given miner until it runs out of
// available prove-commits, batching as much as possible.
//
// This function will fund as necessary from the "burnt funds actor" (look, it's convenient).
func (stage *ProveCommitStage) packProveCommitsMiner(
ctx context.Context, bb *blockbuilder.BlockBuilder, minerAddr address.Address,
pending minerPendingCommits,
) (res proveCommitResult, _err error) {
minerActor, err := bb.StateTree().GetActor(minerAddr)
if err != nil {
return res, err
}
minerState, err := miner.Load(bb.ActorStore(), minerActor)
if err != nil {
return res, err
}
info, err := minerState.Info()
if err != nil {
return res, err
}
log := bb.L().With("miner", minerAddr)
nv := bb.NetworkVersion()
for sealType, snos := range pending {
if nv >= network.Version13 {
for len(snos) > minProveCommitBatchSize {
batchSize := maxProveCommitBatchSize
if len(snos) < batchSize {
batchSize = len(snos)
}
batch := snos[:batchSize]
proof, err := mock.MockAggregateSealProof(sealType, minerAddr, batchSize)
if err != nil {
return res, err
}
params := miner5.ProveCommitAggregateParams{
SectorNumbers: bitfield.New(),
AggregateProof: proof,
}
for _, sno := range batch {
params.SectorNumbers.Set(uint64(sno))
}
enc, err := actors.SerializeParams(&params)
if err != nil {
return res, err
}
if _, err := stage.funding.SendAndFund(bb, &types.Message{
From: info.Worker,
To: minerAddr,
Value: abi.NewTokenAmount(0),
Method: miner.Methods.ProveCommitAggregate,
Params: enc,
}); err == nil {
res.done += len(batch)
} else if blockbuilder.IsOutOfGas(err) {
res.full = true
return res, nil
} else if aerr, ok := err.(aerrors.ActorError); !ok || aerr.IsFatal() {
// If we get a random error, or a fatal actor error, bail.
return res, err
} else if aerr.RetCode() == exitcode.ErrNotFound || aerr.RetCode() == exitcode.ErrIllegalArgument {
// If we get a "not-found" or illegal argument error, try to
// remove any missing prove-commits and continue. This can
// happen either because:
//
// 1. The pre-commit failed on execution (but not when
// packing). This shouldn't happen, but we might as well
// gracefully handle it.
// 2. The pre-commit has expired. We'd have to be really
// backloged to hit this case, but we might as well handle
// it.
// First, split into "good" and "missing"
good, err := stage.filterProveCommits(ctx, bb, minerAddr, batch)
if err != nil {
log.Errorw("failed to filter prove commits", "error", err)
// fail with the original error.
return res, aerr
}
removed := len(batch) - len(good)
if removed == 0 {
log.Errorw("failed to prove-commit for unknown reasons",
"error", aerr,
"sectors", batch,
)
res.failed += len(batch)
} else if len(good) == 0 {
log.Errorw("failed to prove commit missing pre-commits",
"error", aerr,
"discarded", removed,
)
res.failed += len(batch)
} else {
// update the pending sector numbers in-place to remove the expired ones.
snos = snos[removed:]
copy(snos, good)
pending.finish(sealType, removed)
log.Errorw("failed to prove commit expired/missing pre-commits",
"error", aerr,
"discarded", removed,
"kept", len(good),
)
res.failed += removed
// Then try again.
continue
}
} else {
log.Errorw("failed to prove commit sector(s)",
"error", err,
"sectors", batch,
)
res.failed += len(batch)
}
pending.finish(sealType, len(batch))
snos = snos[len(batch):]
}
}
for len(snos) > 0 && res.unbatched < power5.MaxMinerProveCommitsPerEpoch {
sno := snos[0]
snos = snos[1:]
proof, err := mock.MockSealProof(sealType, minerAddr)
if err != nil {
return res, err
}
params := miner.ProveCommitSectorParams{
SectorNumber: sno,
Proof: proof,
}
enc, err := actors.SerializeParams(&params)
if err != nil {
return res, err
}
if _, err := stage.funding.SendAndFund(bb, &types.Message{
From: info.Worker,
To: minerAddr,
Value: abi.NewTokenAmount(0),
Method: miner.Methods.ProveCommitSector,
Params: enc,
}); err == nil {
res.unbatched++
res.done++
} else if blockbuilder.IsOutOfGas(err) {
res.full = true
return res, nil
} else if aerr, ok := err.(aerrors.ActorError); !ok || aerr.IsFatal() {
return res, err
} else {
log.Errorw("failed to prove commit sector(s)",
"error", err,
"sectors", []abi.SectorNumber{sno},
)
res.failed++
}
// mark it as "finished" regardless so we skip it.
pending.finish(sealType, 1)
}
// if we get here, we can't pre-commit anything more.
}
return res, nil
}
// loadMiner enqueue all pending prove-commits for the given miner. This is called on load to
// populate the commitQueue and should not need to be called later.
//
// It will drop any pre-commits that have already expired.
func (stage *ProveCommitStage) loadMiner(ctx context.Context, bb *blockbuilder.BlockBuilder, addr address.Address) error {
epoch := bb.Height()
av := bb.ActorsVersion()
minerState, err := loadMiner(bb.ActorStore(), bb.ParentStateTree(), addr)
if err != nil {
return err
}
// Find all pending prove commits and group by proof type. Really, there should never
// (except during upgrades be more than one type.
var total, dropped int
err = minerState.ForEachPrecommittedSector(func(info miner.SectorPreCommitOnChainInfo) error {
total++
msd := policy.GetMaxProveCommitDuration(av, info.Info.SealProof)
if epoch > info.PreCommitEpoch+msd {
dropped++
return nil
}
return stage.commitQueue.enqueueProveCommit(addr, info.PreCommitEpoch, info.Info)
})
if err != nil {
return err
}
if dropped > 0 {
bb.L().Warnw("dropped expired pre-commits on load",
"miner", addr,
"total", total,
"expired", dropped,
)
}
return nil
}
// filterProveCommits filters out expired and/or missing pre-commits.
func (stage *ProveCommitStage) filterProveCommits(
ctx context.Context, bb *blockbuilder.BlockBuilder,
minerAddr address.Address, snos []abi.SectorNumber,
) ([]abi.SectorNumber, error) {
act, err := bb.StateTree().GetActor(minerAddr)
if err != nil {
return nil, err
}
minerState, err := miner.Load(bb.ActorStore(), act)
if err != nil {
return nil, err
}
nextEpoch := bb.Height()
av := bb.ActorsVersion()
good := make([]abi.SectorNumber, 0, len(snos))
for _, sno := range snos {
info, err := minerState.GetPrecommittedSector(sno)
if err != nil {
return nil, err
}
if info == nil {
continue
}
msd := policy.GetMaxProveCommitDuration(av, info.Info.SealProof)
if nextEpoch > info.PreCommitEpoch+msd {
continue
}
good = append(good, sno)
}
return good, nil
}
func (stage *ProveCommitStage) load(ctx context.Context, bb *blockbuilder.BlockBuilder) error {
stage.initialized = false // in case something failes while we're doing this.
stage.commitQueue = commitQueue{offset: bb.Height()}
powerState, err := loadPower(bb.ActorStore(), bb.ParentStateTree())
if err != nil {
return err
}
err = powerState.ForEachClaim(func(minerAddr address.Address, claim power.Claim) error {
// TODO: If we want to finish pre-commits for "new" miners, we'll need to change
// this.
if claim.RawBytePower.IsZero() {
return nil
}
return stage.loadMiner(ctx, bb, minerAddr)
})
if err != nil {
return err
}
stage.initialized = true
return nil
}

51
cmd/lotus-sim/simulation/stages/util.go Normal file
View File

@ -0,0 +1,51 @@
package stages
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
)
func loadMiner(store adt.Store, st types.StateTree, addr address.Address) (miner.State, error) {
minerActor, err := st.GetActor(addr)
if err != nil {
return nil, err
}
return miner.Load(store, minerActor)
}
func loadPower(store adt.Store, st types.StateTree) (power.State, error) {
powerActor, err := st.GetActor(power.Address)
if err != nil {
return nil, err
}
return power.Load(store, powerActor)
}
// Compute the number of sectors a miner has from their power claim.
func sectorsFromClaim(sectorSize abi.SectorSize, c power.Claim) int64 {
if c.RawBytePower.Int == nil {
return 0
}
sectorCount := big.Div(c.RawBytePower, big.NewIntUnsigned(uint64(sectorSize)))
if !sectorCount.IsInt64() {
panic("impossible number of sectors")
}
return sectorCount.Int64()
}
func postChainCommitInfo(ctx context.Context, bb *blockbuilder.BlockBuilder, epoch abi.ChainEpoch) (abi.Randomness, error) {
cs := bb.StateManager().ChainStore()
ts := bb.ParentTipSet()
commitRand, err := cs.GetChainRandomness(ctx, ts.Cids(), crypto.DomainSeparationTag_PoStChainCommit, epoch, nil, true)
return commitRand, err
}

317
cmd/lotus-sim/simulation/stages/windowpost_stage.go Normal file
View File

@ -0,0 +1,317 @@
package stages
import (
"context"
"math"
"time"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
)
type WindowPoStStage struct {
// We track the window post periods per miner and assume that no new miners are ever added.
// We record all pending window post messages, and the epoch up through which we've
// generated window post messages.
pendingWposts []*types.Message
wpostPeriods [][]address.Address // (epoch % (epochs in a deadline)) -> miner
nextWpostEpoch abi.ChainEpoch
}
func NewWindowPoStStage() (*WindowPoStStage, error) {
return new(WindowPoStStage), nil
}
func (*WindowPoStStage) Name() string {
return "window-post"
}
// packWindowPoSts packs window posts until either the block is full or all healty sectors
// have been proven. It does not recover sectors.
func (stage *WindowPoStStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
// Push any new window posts into the queue.
if err := stage.tick(ctx, bb); err != nil {
return err
}
done := 0
failed := 0
defer func() {
if _err != nil {
return
}
bb.L().Debugw("packed window posts",
"done", done,
"failed", failed,
"remaining", len(stage.pendingWposts),
)
}()
// Then pack as many as we can.
for len(stage.pendingWposts) > 0 {
next := stage.pendingWposts[0]
if _, err := bb.PushMessage(next); err != nil {
if blockbuilder.IsOutOfGas(err) {
return nil
}
if aerr, ok := err.(aerrors.ActorError); !ok || aerr.IsFatal() {
return err
}
bb.L().Errorw("failed to submit windowed post",
"error", err,
"miner", next.To,
)
failed++
} else {
done++
}
stage.pendingWposts = stage.pendingWposts[1:]
}
stage.pendingWposts = nil
return nil
}
// stepWindowPoStsMiner enqueues all missing window posts for the current epoch for the given miner.
func (stage *WindowPoStStage) queueMiner(
ctx context.Context, bb *blockbuilder.BlockBuilder,
addr address.Address, minerState miner.State,
commitEpoch abi.ChainEpoch, commitRand abi.Randomness,
) error {
if active, err := minerState.DeadlineCronActive(); err != nil {
return err
} else if !active {
return nil
}
minerInfo, err := minerState.Info()
if err != nil {
return err
}
di, err := minerState.DeadlineInfo(bb.Height())
if err != nil {
return err
}
di = di.NextNotElapsed()
dl, err := minerState.LoadDeadline(di.Index)
if err != nil {
return err
}
provenBf, err := dl.PartitionsPoSted()
if err != nil {
return err
}
proven, err := provenBf.AllMap(math.MaxUint64)
if err != nil {
return err
}
poStBatchSize, err := policy.GetMaxPoStPartitions(bb.NetworkVersion(), minerInfo.WindowPoStProofType)
if err != nil {
return err
}
var (
partitions []miner.PoStPartition
partitionGroups [][]miner.PoStPartition
)
// Only prove partitions with live sectors.
err = dl.ForEachPartition(func(idx uint64, part miner.Partition) error {
if proven[idx] {
return nil
}
// NOTE: We're mimicing the behavior of wdpost_run.go here.
if len(partitions) > 0 && idx%uint64(poStBatchSize) == 0 {
partitionGroups = append(partitionGroups, partitions)
partitions = nil
}
live, err := part.LiveSectors()
if err != nil {
return err
}
liveCount, err := live.Count()
if err != nil {
return err
}
faulty, err := part.FaultySectors()
if err != nil {
return err
}
faultyCount, err := faulty.Count()
if err != nil {
return err
}
if liveCount-faultyCount > 0 {
partitions = append(partitions, miner.PoStPartition{Index: idx})
}
return nil
})
if err != nil {
return err
}
if len(partitions) > 0 {
partitionGroups = append(partitionGroups, partitions)
partitions = nil
}
proof, err := mock.MockWindowPoStProof(minerInfo.WindowPoStProofType, addr)
if err != nil {
return err
}
for _, group := range partitionGroups {
params := miner.SubmitWindowedPoStParams{
Deadline: di.Index,
Partitions: group,
Proofs: []proof5.PoStProof{{
PoStProof: minerInfo.WindowPoStProofType,
ProofBytes: proof,
}},
ChainCommitEpoch: commitEpoch,
ChainCommitRand: commitRand,
}
enc, aerr := actors.SerializeParams(&params)
if aerr != nil {
return xerrors.Errorf("could not serialize submit window post parameters: %w", aerr)
}
msg := &types.Message{
To: addr,
From: minerInfo.Worker,
Method: miner.Methods.SubmitWindowedPoSt,
Params: enc,
Value: types.NewInt(0),
}
stage.pendingWposts = append(stage.pendingWposts, msg)
}
return nil
}
func (stage *WindowPoStStage) load(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
bb.L().Info("loading window post info")
start := time.Now()
defer func() {
if _err != nil {
return
}
bb.L().Infow("loaded window post info", "duration", time.Since(start))
}()
// reset
stage.wpostPeriods = make([][]address.Address, miner.WPoStChallengeWindow)
stage.pendingWposts = nil
stage.nextWpostEpoch = bb.Height() + 1
st := bb.ParentStateTree()
store := bb.ActorStore()
powerState, err := loadPower(store, st)
if err != nil {
return err
}
commitEpoch := bb.ParentTipSet().Height()
commitRand, err := postChainCommitInfo(ctx, bb, commitEpoch)
if err != nil {
return err
}
return powerState.ForEachClaim(func(minerAddr address.Address, claim power.Claim) error {
// TODO: If we start recovering power, we'll need to change this.
if claim.RawBytePower.IsZero() {
return nil
}
minerState, err := loadMiner(store, st, minerAddr)
if err != nil {
return err
}
// Shouldn't be necessary if the miner has power, but we might as well be safe.
if active, err := minerState.DeadlineCronActive(); err != nil {
return err
} else if !active {
return nil
}
// Record when we need to prove for this miner.
dinfo, err := minerState.DeadlineInfo(bb.Height())
if err != nil {
return err
}
dinfo = dinfo.NextNotElapsed()
ppOffset := int(dinfo.PeriodStart % miner.WPoStChallengeWindow)
stage.wpostPeriods[ppOffset] = append(stage.wpostPeriods[ppOffset], minerAddr)
return stage.queueMiner(ctx, bb, minerAddr, minerState, commitEpoch, commitRand)
})
}
func (stage *WindowPoStStage) tick(ctx context.Context, bb *blockbuilder.BlockBuilder) error {
// If this is our first time, load from scratch.
if stage.wpostPeriods == nil {
return stage.load(ctx, bb)
}
targetHeight := bb.Height()
now := time.Now()
was := len(stage.pendingWposts)
count := 0
defer func() {
bb.L().Debugw("computed window posts",
"miners", count,
"count", len(stage.pendingWposts)-was,
"duration", time.Since(now),
)
}()
st := bb.ParentStateTree()
store := bb.ActorStore()
// Perform a bit of catch up. This lets us do things like skip blocks at upgrades then catch
// up to make the simulation easier.
for ; stage.nextWpostEpoch <= targetHeight; stage.nextWpostEpoch++ {
if stage.nextWpostEpoch+miner.WPoStChallengeWindow < targetHeight {
bb.L().Warnw("skipping old window post", "deadline-open", stage.nextWpostEpoch)
continue
}
commitEpoch := stage.nextWpostEpoch - 1
commitRand, err := postChainCommitInfo(ctx, bb, commitEpoch)
if err != nil {
return err
}
for _, addr := range stage.wpostPeriods[int(stage.nextWpostEpoch%miner.WPoStChallengeWindow)] {
minerState, err := loadMiner(store, st, addr)
if err != nil {
return err
}
if err := stage.queueMiner(ctx, bb, addr, minerState, commitEpoch, commitRand); err != nil {
return err
}
count++
}
}
return nil
}

71
cmd/lotus-sim/simulation/step.go Normal file
View File

@ -0,0 +1,71 @@
package simulation
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
)
// Step steps the simulation forward one step. This may move forward by more than one epoch.
func (sim *Simulation) Step(ctx context.Context) (*types.TipSet, error) {
log.Infow("step", "epoch", sim.head.Height()+1)
messages, err := sim.popNextMessages(ctx)
if err != nil {
return nil, xerrors.Errorf("failed to select messages for block: %w", err)
}
head, err := sim.makeTipSet(ctx, messages)
if err != nil {
return nil, xerrors.Errorf("failed to make tipset: %w", err)
}
if err := sim.SetHead(head); err != nil {
return nil, xerrors.Errorf("failed to update head: %w", err)
}
return head, nil
}
// popNextMessages generates/picks a set of messages to be included in the next block.
//
// - This function is destructive and should only be called once per epoch.
// - This function does not store anything in the repo.
// - This function handles all gas estimation. The returned messages should all fit in a single
// block.
func (sim *Simulation) popNextMessages(ctx context.Context) ([]*types.Message, error) {
parentTs := sim.head
// First we make sure we don't have an upgrade at this epoch. If we do, we return no
// messages so we can just create an empty block at that epoch.
//
// This isn't what the network does, but it makes things easier. Otherwise, we'd need to run
// migrations before this epoch and I'd rather not deal with that.
nextHeight := parentTs.Height() + 1
prevVer := sim.StateManager.GetNtwkVersion(ctx, nextHeight-1)
nextVer := sim.StateManager.GetNtwkVersion(ctx, nextHeight)
if nextVer != prevVer {
log.Warnw("packing no messages for version upgrade block",
"old", prevVer,
"new", nextVer,
"epoch", nextHeight,
)
return nil, nil
}
bb, err := blockbuilder.NewBlockBuilder(
ctx, log.With("simulation", sim.name),
sim.StateManager, parentTs,
)
if err != nil {
return nil, err
}
for _, stage := range sim.stages {
// We're intentionally ignoring the "full" signal so we can try to pack a few more
// messages.
if err := stage.PackMessages(ctx, bb); err != nil && !blockbuilder.IsOutOfGas(err) {
return nil, xerrors.Errorf("when packing messages with %s: %w", stage.Name(), err)
}
}
return bb.Messages(), nil
}

109
cmd/lotus-sim/upgrade.go Normal file
View File

@ -0,0 +1,109 @@
package main
import (
"fmt"
"strconv"
"strings"
"text/tabwriter"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
)
var upgradeCommand = &cli.Command{
Name: "upgrade",
Description: "Modifies network upgrade heights.",
Subcommands: []*cli.Command{
upgradeSetCommand,
upgradeList,
},
}
var upgradeList = &cli.Command{
Name: "list",
Description: "Lists all pending upgrades.",
Subcommands: []*cli.Command{
upgradeSetCommand,
},
Action: func(cctx *cli.Context) (err error) {
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
upgrades, err := sim.ListUpgrades()
if err != nil {
return err
}
tw := tabwriter.NewWriter(cctx.App.Writer, 8, 8, 0, ' ', 0)
fmt.Fprintf(tw, "version\theight\tepochs\tmigration\texpensive")
epoch := sim.GetHead().Height()
for _, upgrade := range upgrades {
fmt.Fprintf(
tw, "%d\t%d\t%+d\t%t\t%t",
upgrade.Network, upgrade.Height, upgrade.Height-epoch,
upgrade.Migration != nil,
upgrade.Expensive,
)
}
return nil
},
}
var upgradeSetCommand = &cli.Command{
Name: "set",
ArgsUsage: "<network-version> [+]<epochs>",
Description: "Set a network upgrade height. Prefix with '+' to set it relative to the last epoch.",
Action: func(cctx *cli.Context) (err error) {
args := cctx.Args()
if args.Len() != 2 {
return fmt.Errorf("expected 2 arguments")
}
nvString := args.Get(0)
networkVersion, err := strconv.ParseUint(nvString, 10, 32)
if err != nil {
return fmt.Errorf("failed to parse network version %q: %w", nvString, err)
}
heightString := args.Get(1)
relative := false
if strings.HasPrefix(heightString, "+") {
heightString = heightString[1:]
relative = true
}
height, err := strconv.ParseInt(heightString, 10, 64)
if err != nil {
return fmt.Errorf("failed to parse height version %q: %w", heightString, err)
}
node, err := open(cctx)
if err != nil {
return err
}
defer func() {
if cerr := node.Close(); err == nil {
err = cerr
}
}()
sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
if err != nil {
return err
}
if relative {
height += int64(sim.GetHead().Height())
}
return sim.SetUpgradeHeight(network.Version(networkVersion), abi.ChainEpoch(height))
},
}

18
cmd/lotus-sim/util.go Normal file
View File

@ -0,0 +1,18 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
"github.com/filecoin-project/lotus/lib/ulimit"
)
func open(cctx *cli.Context) (*simulation.Node, error) {
_, _, err := ulimit.ManageFdLimit()
if err != nil {
fmt.Fprintf(cctx.App.ErrWriter, "ERROR: failed to raise ulimit: %s\n", err)
}
return simulation.OpenNode(cctx.Context, cctx.String("repo"))
}

62
cmd/lotus-storage-miner/actor_test.go
View File

@ -10,14 +10,13 @@ import (
"testing"
"time"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/go-state-types/network"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/itests/kit"
@ -32,36 +31,21 @@ func TestWorkerKeyChange(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = logging.SetLogLevel("*", "INFO")
policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
kit.QuietMiningLogs()
blocktime := 1 * time.Millisecond
clients, miners := kit.MockMinerBuilder(t,
[]kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(-1), kit.FullNodeWithLatestActorsAt(-1)},
kit.OneMiner)
client1 := clients[0]
client2 := clients[1]
// Connect the nodes.
addrinfo, err := client1.NetAddrsListen(ctx)
require.NoError(t, err)
err = client2.NetConnect(ctx, addrinfo)
require.NoError(t, err)
client1, client2, miner, ens := kit.EnsembleTwoOne(t, kit.MockProofs(),
kit.ConstructorOpts(kit.InstantaneousNetworkVersion(network.Version13)),
)
ens.InterconnectAll().BeginMining(blocktime)
output := bytes.NewBuffer(nil)
run := func(cmd *cli.Command, args ...string) error {
app := cli.NewApp()
app.Metadata = map[string]interface{}{
"repoType": repo.StorageMiner,
"testnode-full": clients[0],
"testnode-storage": miners[0],
"testnode-full": client1,
"testnode-storage": miner,
}
app.Writer = output
api.RunningNodeType = api.NodeMiner
@ -78,9 +62,6 @@ func TestWorkerKeyChange(t *testing.T) {
return cmd.Action(cctx)
}
// start mining
kit.ConnectAndStartMining(t, blocktime, miners[0], client1, client2)
newKey, err := client1.WalletNew(ctx, types.KTBLS)
require.NoError(t, err)
@ -105,14 +86,8 @@ func TestWorkerKeyChange(t *testing.T) {
require.Error(t, run(actorConfirmChangeWorker, "--really-do-it", newKey.String()))
output.Reset()
for {
head, err := client1.ChainHead(ctx)
require.NoError(t, err)
if head.Height() >= abi.ChainEpoch(targetEpoch) {
break
}
build.Clock.Sleep(10 * blocktime)
}
client1.WaitTillChain(ctx, kit.HeightAtLeast(abi.ChainEpoch(targetEpoch)))
require.NoError(t, run(actorConfirmChangeWorker, "--really-do-it", newKey.String()))
output.Reset()
@ -121,23 +96,8 @@ func TestWorkerKeyChange(t *testing.T) {
// Wait for finality (worker key switch).
targetHeight := head.Height() + policy.ChainFinality
for {
head, err := client1.ChainHead(ctx)
require.NoError(t, err)
if head.Height() >= targetHeight {
break
}
build.Clock.Sleep(10 * blocktime)
}
client1.WaitTillChain(ctx, kit.HeightAtLeast(targetHeight))
// Make sure the other node can catch up.
for i := 0; i < 20; i++ {
head, err := client2.ChainHead(ctx)
require.NoError(t, err)
if head.Height() >= targetHeight {
return
}
build.Clock.Sleep(10 * blocktime)
}
t.Fatal("failed to reach target epoch on the second miner")
client2.WaitTillChain(ctx, kit.HeightAtLeast(targetHeight))
}

30
cmd/lotus-storage-miner/allinfo_test.go
View File

@ -7,13 +7,9 @@ import (
"time"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/node/impl"
logging "github.com/ipfs/go-log/v2"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/node/repo"
@ -24,12 +20,6 @@ func TestMinerAllInfo(t *testing.T) {
t.Skip("skipping test in short mode")
}
_ = logging.SetLogLevel("*", "INFO")
policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
_test = true
kit.QuietMiningLogs()
@ -40,16 +30,15 @@ func TestMinerAllInfo(t *testing.T) {
policy.SetPreCommitChallengeDelay(oldDelay)
})
n, sn := kit.Builder(t, kit.OneFull, kit.OneMiner)
client, miner := n[0].FullNode, sn[0]
kit.ConnectAndStartMining(t, time.Second, miner, client.(*impl.FullNodeAPI))
client, miner, ens := kit.EnsembleMinimal(t)
ens.InterconnectAll().BeginMining(time.Second)
run := func(t *testing.T) {
app := cli.NewApp()
app.Metadata = map[string]interface{}{
"repoType": repo.StorageMiner,
"testnode-full": n[0],
"testnode-storage": sn[0],
"testnode-full": client,
"testnode-storage": miner,
}
api.RunningNodeType = api.NodeMiner
@ -61,14 +50,9 @@ func TestMinerAllInfo(t *testing.T) {
t.Run("pre-info-all", run)
dh := kit.NewDealHarness(t, client, miner)
_, _, _ = dh.MakeFullDeal(kit.MakeFullDealParams{
Ctx: context.Background(),
Rseed: 6,
CarExport: false,
FastRet: false,
StartEpoch: 0,
DoRetrieval: true,
})
deal, res, inPath := dh.MakeOnlineDeal(context.Background(), kit.MakeFullDealParams{Rseed: 6})
outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, false)
kit.AssertFilesEqual(t, inPath, outPath)
t.Run("post-info-all", run)
}

1
documentation/en/api-v0-methods-miner.md
View File

@ -2205,6 +2205,7 @@ Response:
"ef8d99a2-6865-4189-8ffa-9fef0f806eee": {
"Info": {
"Hostname": "host",
"IgnoreResources": false,
"Resources": {
"MemPhysical": 274877906944,
"MemSwap": 128849018880,

1
documentation/en/api-v0-methods-worker.md
View File

@ -89,6 +89,7 @@ Response:
```json
{
"Hostname": "string value",
"IgnoreResources": true,
"Resources": {
"MemPhysical": 42,
"MemSwap": 42,

4
documentation/en/cli-lotus.md
View File

@ -558,8 +558,8 @@ DESCRIPTION:
Make a deal with a miner.
dataCid comes from running 'lotus client import'.
miner is the address of the miner you wish to make a deal with.
price is measured in FIL/GB/Epoch. Miners usually don't accept a bid
lower than their advertised ask. You can check a miners listed price
price is measured in FIL/Epoch. Miners usually don't accept a bid
lower than their advertised ask (which is in FIL/GiB/Epoch). You can check a miners listed price
with 'lotus client query-ask <miner address>'.
duration is how long the miner should store the data for, in blocks.
The minimum value is 518400 (6 months).

2
extern/filecoin-ffi vendored

@ -1 +1 @@
Subproject commit d2e3aa7d61501d69bed6e898de13d1312b021e62
Subproject commit d60fc680aa8abeafba698f738fed5b94c9bda33d

1
extern/sector-storage/ffiwrapper/sealer_test.go vendored
View File

@ -545,6 +545,7 @@ func TestSealAndVerifyAggregate(t *testing.T) {
avi.Proof, err = ProofProver.AggregateSealProofs(avi, toAggregate)
require.NoError(t, err)
require.Len(t, avi.Proof, 11188)
aggDone := time.Now()

29
extern/sector-storage/manager.go vendored
View File

@ -87,6 +87,20 @@ type result struct {
err error
}
// ResourceFilteringStrategy is an enum indicating the kinds of resource
// filtering strategies that can be configured for workers.
type ResourceFilteringStrategy string
const (
// ResourceFilteringHardware specifies that available hardware resources
// should be evaluated when scheduling a task against the worker.
ResourceFilteringHardware = ResourceFilteringStrategy("hardware")
// ResourceFilteringDisabled disables resource filtering against this
// worker. The scheduler may assign any task to this worker.
ResourceFilteringDisabled = ResourceFilteringStrategy("disabled")
)
type SealerConfig struct {
ParallelFetchLimit int
@ -96,6 +110,11 @@ type SealerConfig struct {
AllowPreCommit2 bool
AllowCommit bool
AllowUnseal bool
// ResourceFiltering instructs the system which resource filtering strategy
// to use when evaluating tasks against this worker. An empty value defaults
// to "hardware".
ResourceFiltering ResourceFilteringStrategy
}
type StorageAuth http.Header
@ -104,7 +123,6 @@ type WorkerStateStore *statestore.StateStore
type ManagerStateStore *statestore.StateStore
func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls stores.LocalStorage, si stores.SectorIndex, sc SealerConfig, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) {
prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si})
if err != nil {
return nil, xerrors.Errorf("creating prover instance: %w", err)
@ -151,9 +169,12 @@ func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls store
localTasks = append(localTasks, sealtasks.TTUnseal)
}
err = m.AddWorker(ctx, NewLocalWorker(WorkerConfig{
TaskTypes: localTasks,
}, stor, lstor, si, m, wss))
wcfg := WorkerConfig{
IgnoreResourceFiltering: sc.ResourceFiltering == ResourceFilteringDisabled,
TaskTypes: localTasks,
}
worker := NewLocalWorker(wcfg, stor, lstor, si, m, wss)
err = m.AddWorker(ctx, worker)
if err != nil {
return nil, xerrors.Errorf("adding local worker: %w", err)
}

24
extern/sector-storage/sched.go vendored
View File

@ -349,24 +349,24 @@ func (sh *scheduler) trySched() {
defer sh.workersLk.RUnlock()
windowsLen := len(sh.openWindows)
queuneLen := sh.schedQueue.Len()
queueLen := sh.schedQueue.Len()
log.Debugf("SCHED %d queued; %d open windows", queuneLen, windowsLen)
log.Debugf("SCHED %d queued; %d open windows", queueLen, windowsLen)
if windowsLen == 0 || queuneLen == 0 {
if windowsLen == 0 || queueLen == 0 {
// nothing to schedule on
return
}
windows := make([]schedWindow, windowsLen)
acceptableWindows := make([][]int, queuneLen)
acceptableWindows := make([][]int, queueLen)
// Step 1
throttle := make(chan struct{}, windowsLen)
var wg sync.WaitGroup
wg.Add(queuneLen)
for i := 0; i < queuneLen; i++ {
wg.Add(queueLen)
for i := 0; i < queueLen; i++ {
throttle <- struct{}{}
go func(sqi int) {
@ -393,7 +393,7 @@ func (sh *scheduler) trySched() {
}
// TODO: allow bigger windows
if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, "schedAcceptable", worker.info.Resources) {
if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, "schedAcceptable", worker.info) {
continue
}
@ -451,27 +451,27 @@ func (sh *scheduler) trySched() {
// Step 2
scheduled := 0
rmQueue := make([]int, 0, queuneLen)
rmQueue := make([]int, 0, queueLen)
for sqi := 0; sqi < queuneLen; sqi++ {
for sqi := 0; sqi < queueLen; sqi++ {
task := (*sh.schedQueue)[sqi]
needRes := ResourceTable[task.taskType][task.sector.ProofType]
selectedWindow := -1
for _, wnd := range acceptableWindows[task.indexHeap] {
wid := sh.openWindows[wnd].worker
wr := sh.workers[wid].info.Resources
info := sh.workers[wid].info
log.Debugf("SCHED try assign sqi:%d sector %d to window %d", sqi, task.sector.ID.Number, wnd)
// TODO: allow bigger windows
if !windows[wnd].allocated.canHandleRequest(needRes, wid, "schedAssign", wr) {
if !windows[wnd].allocated.canHandleRequest(needRes, wid, "schedAssign", info) {
continue
}
log.Debugf("SCHED ASSIGNED sqi:%d sector %d task %s to window %d", sqi, task.sector.ID.Number, task.taskType, wnd)
windows[wnd].allocated.add(wr, needRes)
windows[wnd].allocated.add(info.Resources, needRes)
// TODO: We probably want to re-sort acceptableWindows here based on new
// workerHandle.utilization + windows[wnd].allocated.utilization (workerHandle.utilization is used in all
// task selectors, but not in the same way, so need to figure out how to do that in a non-O(n^2 way), and

15
extern/sector-storage/sched_resources.go vendored
View File

@ -6,7 +6,7 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
)
func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error {
func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerInfo, r Resources, locker sync.Locker, cb func() error) error {
for !a.canHandleRequest(r, id, "withResources", wr) {
if a.cond == nil {
a.cond = sync.NewCond(locker)
@ -14,11 +14,11 @@ func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResource
a.cond.Wait()
}
a.add(wr, r)
a.add(wr.Resources, r)
err := cb()
a.free(wr, r)
a.free(wr.Resources, r)
if a.cond != nil {
a.cond.Broadcast()
}
@ -44,8 +44,15 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) {
a.memUsedMax -= r.MaxMemory
}
func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, caller string, res storiface.WorkerResources) bool {
// canHandleRequest evaluates if the worker has enough available resources to
// handle the request.
func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, caller string, info storiface.WorkerInfo) bool {
if info.IgnoreResources {
// shortcircuit; if this worker is ignoring resources, it can always handle the request.
return true
}
res := info.Resources
// TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running)
minNeedMem := res.MemReserved + a.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory
if minNeedMem > res.MemPhysical {

71
extern/sector-storage/sched_test.go vendored
View File

@ -38,6 +38,20 @@ func TestWithPriority(t *testing.T) {
require.Equal(t, 2222, getPriority(ctx))
}
var decentWorkerResources = storiface.WorkerResources{
MemPhysical: 128 << 30,
MemSwap: 200 << 30,
MemReserved: 2 << 30,
CPUs: 32,
GPUs: []string{"a GPU"},
}
var constrainedWorkerResources = storiface.WorkerResources{
MemPhysical: 1 << 30,
MemReserved: 2 << 30,
CPUs: 1,
}
type schedTestWorker struct {
name string
taskTypes map[sealtasks.TaskType]struct{}
@ -45,6 +59,9 @@ type schedTestWorker struct {
closed bool
session uuid.UUID
resources storiface.WorkerResources
ignoreResources bool
}
func (s *schedTestWorker) SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) {
@ -107,18 +124,11 @@ func (s *schedTestWorker) Paths(ctx context.Context) ([]stores.StoragePath, erro
return s.paths, nil
}
var decentWorkerResources = storiface.WorkerResources{
MemPhysical: 128 << 30,
MemSwap: 200 << 30,
MemReserved: 2 << 30,
CPUs: 32,
GPUs: []string{"a GPU"},
}
func (s *schedTestWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) {
return storiface.WorkerInfo{
Hostname: s.name,
Resources: decentWorkerResources,
Hostname: s.name,
IgnoreResources: s.ignoreResources,
Resources: s.resources,
}, nil
}
@ -137,13 +147,16 @@ func (s *schedTestWorker) Close() error {
var _ Worker = &schedTestWorker{}
func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name string, taskTypes map[sealtasks.TaskType]struct{}) {
func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name string, taskTypes map[sealtasks.TaskType]struct{}, resources storiface.WorkerResources, ignoreResources bool) {
w := &schedTestWorker{
name: name,
taskTypes: taskTypes,
paths: []stores.StoragePath{{ID: "bb-8", Weight: 2, LocalPath: "<octopus>food</octopus>", CanSeal: true, CanStore: true}},
session: uuid.New(),
resources: resources,
ignoreResources: ignoreResources,
}
for _, path := range w.paths {
@ -169,7 +182,7 @@ func TestSchedStartStop(t *testing.T) {
sched := newScheduler()
go sched.runSched()
addTestWorker(t, sched, stores.NewIndex(), "fred", nil)
addTestWorker(t, sched, stores.NewIndex(), "fred", nil, decentWorkerResources, false)
require.NoError(t, sched.Close(context.TODO()))
}
@ -183,6 +196,9 @@ func TestSched(t *testing.T) {
type workerSpec struct {
name string
taskTypes map[sealtasks.TaskType]struct{}
resources storiface.WorkerResources
ignoreResources bool
}
noopAction := func(ctx context.Context, w Worker) error {
@ -295,7 +311,7 @@ func TestSched(t *testing.T) {
go sched.runSched()
for _, worker := range workers {
addTestWorker(t, sched, index, worker.name, worker.taskTypes)
addTestWorker(t, sched, index, worker.name, worker.taskTypes, worker.resources, worker.ignoreResources)
}
rm := runMeta{
@ -322,31 +338,42 @@ func TestSched(t *testing.T) {
}
}
// checks behaviour with workers with constrained resources
// the first one is not ignoring resource constraints, so we assign to the second worker, who is
t.Run("constrained-resources", testFunc([]workerSpec{
{name: "fred1", resources: constrainedWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
{name: "fred2", resources: constrainedWorkerResources, ignoreResources: true, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
}, []task{
sched("pc1-1", "fred2", 8, sealtasks.TTPreCommit1),
taskStarted("pc1-1"),
taskDone("pc1-1"),
}))
t.Run("one-pc1", testFunc([]workerSpec{
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
{name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
}, []task{
sched("pc1-1", "fred", 8, sealtasks.TTPreCommit1),
taskDone("pc1-1"),
}))
t.Run("pc1-2workers-1", testFunc([]workerSpec{
{name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}},
{name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
{name: "fred2", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}},
{name: "fred1", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
}, []task{
sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1),
taskDone("pc1-1"),
}))
t.Run("pc1-2workers-2", testFunc([]workerSpec{
{name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
{name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}},
{name: "fred1", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
{name: "fred2", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}},
}, []task{
sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1),
taskDone("pc1-1"),
}))
t.Run("pc1-block-pc2", testFunc([]workerSpec{
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
{name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
}, []task{
sched("pc1", "fred", 8, sealtasks.TTPreCommit1),
taskStarted("pc1"),
@ -359,7 +386,7 @@ func TestSched(t *testing.T) {
}))
t.Run("pc2-block-pc1", testFunc([]workerSpec{
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
{name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
}, []task{
sched("pc2", "fred", 8, sealtasks.TTPreCommit2),
taskStarted("pc2"),
@ -372,7 +399,7 @@ func TestSched(t *testing.T) {
}))
t.Run("pc1-batching", testFunc([]workerSpec{
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
{name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
}, []task{
sched("t1", "fred", 8, sealtasks.TTPreCommit1),
taskStarted("t1"),
@ -459,7 +486,7 @@ func TestSched(t *testing.T) {
// run this one a bunch of times, it had a very annoying tendency to fail randomly
for i := 0; i < 40; i++ {
t.Run("pc1-pc2-prio", testFunc([]workerSpec{
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
{name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
}, []task{
// fill queues
twoPC1("w0", 0, taskStarted),

6
extern/sector-storage/sched_worker.go vendored
View File

@ -296,7 +296,7 @@ func (sw *schedWorker) workerCompactWindows() {
for ti, todo := range window.todo {
needRes := ResourceTable[todo.taskType][todo.sector.ProofType]
if !lower.allocated.canHandleRequest(needRes, sw.wid, "compactWindows", worker.info.Resources) {
if !lower.allocated.canHandleRequest(needRes, sw.wid, "compactWindows", worker.info) {
continue
}
@ -352,7 +352,7 @@ assignLoop:
worker.lk.Lock()
for t, todo := range firstWindow.todo {
needRes := ResourceTable[todo.taskType][todo.sector.ProofType]
if worker.preparing.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info.Resources) {
if worker.preparing.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info) {
tidx = t
break
}
@ -424,7 +424,7 @@ func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRe
}
// wait (if needed) for resources in the 'active' window
err = w.active.withResources(sw.wid, w.info.Resources, needRes, &sh.workersLk, func() error {
err = w.active.withResources(sw.wid, w.info, needRes, &sh.workersLk, func() error {
w.lk.Lock()
w.preparing.free(w.info.Resources, needRes)
w.lk.Unlock()

7
extern/sector-storage/storiface/worker.go vendored
View File

@ -18,7 +18,12 @@ import (
type WorkerInfo struct {
Hostname string
Resources WorkerResources
// IgnoreResources indicates whether the worker's available resources should
// be used ignored (true) or used (false) for the purposes of scheduling and
// task assignment. Only supported on local workers. Used for testing.
// Default should be false (zero value, i.e. resources taken into account).
IgnoreResources bool
Resources WorkerResources
}
type WorkerResources struct {

25
extern/sector-storage/worker_local.go vendored
View File

@ -20,7 +20,7 @@ import (
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-statestore"
storage "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
@ -33,6 +33,11 @@ var pathTypes = []storiface.SectorFileType{storiface.FTUnsealed, storiface.FTSea
type WorkerConfig struct {
TaskTypes []sealtasks.TaskType
NoSwap bool
// IgnoreResourceFiltering enables task distribution to happen on this
// worker regardless of its currently available resources. Used in testing
// with the local worker.
IgnoreResourceFiltering bool
}
// used do provide custom proofs impl (mostly used in testing)
@ -46,6 +51,9 @@ type LocalWorker struct {
executor ExecutorFunc
noSwap bool
// see equivalent field on WorkerConfig.
ignoreResources bool
ct *workerCallTracker
acceptTasks map[sealtasks.TaskType]struct{}
running sync.WaitGroup
@ -71,12 +79,12 @@ func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, store stores.Store
ct: &workerCallTracker{
st: cst,
},
acceptTasks: acceptTasks,
executor: executor,
noSwap: wcfg.NoSwap,
session: uuid.New(),
closing: make(chan struct{}),
acceptTasks: acceptTasks,
executor: executor,
noSwap: wcfg.NoSwap,
ignoreResources: wcfg.IgnoreResourceFiltering,
session: uuid.New(),
closing: make(chan struct{}),
}
if w.executor == nil {
@ -501,7 +509,8 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) {
}
return storiface.WorkerInfo{
Hostname: hostname,
Hostname: hostname,
IgnoreResources: l.ignoreResources,
Resources: storiface.WorkerResources{
MemPhysical: mem.Total,
MemSwap: memSwap,

13
extern/storage-sealing/commit_batch.go vendored
View File

@ -110,7 +110,8 @@ func (b *CommitBatcher) run() {
}
lastMsg = nil
var sendAboveMax, sendAboveMin bool
// indicates whether we should only start a batch if we have reached or exceeded cfg.MaxCommitBatch
var sendAboveMax bool
select {
case <-b.stop:
close(b.stopped)
@ -118,13 +119,13 @@ func (b *CommitBatcher) run() {
case <-b.notify:
sendAboveMax = true
case <-b.batchWait(cfg.CommitBatchWait, cfg.CommitBatchSlack):
sendAboveMin = true
// do nothing
case fr := <-b.force: // user triggered
forceRes = fr
}
var err error
lastMsg, err = b.maybeStartBatch(sendAboveMax, sendAboveMin)
lastMsg, err = b.maybeStartBatch(sendAboveMax)
if err != nil {
log.Warnw("CommitBatcher processBatch error", "error", err)
}
@ -172,7 +173,7 @@ func (b *CommitBatcher) batchWait(maxWait, slack time.Duration) <-chan time.Time
return time.After(wait)
}
func (b *CommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.CommitBatchRes, error) {
func (b *CommitBatcher) maybeStartBatch(notif bool) ([]sealiface.CommitBatchRes, error) {
b.lk.Lock()
defer b.lk.Unlock()
@ -190,10 +191,6 @@ func (b *CommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.CommitBa
return nil, nil
}
if after && total < cfg.MinCommitBatch {
return nil, nil
}
var res []sealiface.CommitBatchRes
if total < cfg.MinCommitBatch || total < miner5.MinAggregatedSectors {

1
extern/storage-sealing/commit_batch_test.go vendored
View File

@ -48,7 +48,6 @@ func TestCommitBatcher(t *testing.T) {
AlwaysKeepUnsealedCopy: true,
BatchPreCommits: true,
MinPreCommitBatch: 1,
MaxPreCommitBatch: miner5.PreCommitSectorBatchMaxSize,
PreCommitBatchWait: 24 * time.Hour,
PreCommitBatchSlack: 3 * time.Hour,

3
extern/storage-sealing/input.go vendored
View File

@ -395,6 +395,7 @@ func (m *Sealing) updateInput(ctx context.Context, sp abi.RegisteredSealProof) e
func (m *Sealing) tryCreateDealSector(ctx context.Context, sp abi.RegisteredSealProof) error {
m.startupWait.Wait()
if m.creating != nil {
return nil // new sector is being created right now
}
@ -448,8 +449,8 @@ func (m *Sealing) createSector(ctx context.Context, cfg sealiface.Config, sp abi
func (m *Sealing) StartPacking(sid abi.SectorNumber) error {
m.startupWait.Wait()
log.Infow("starting to seal deal sector", "sector", sid, "trigger", "user")
log.Infow("starting to seal deal sector", "sector", sid, "trigger", "user")
return m.sectors.Send(uint64(sid), SectorStartPacking{})
}

12
extern/storage-sealing/precommit_batch.go vendored
View File

@ -95,7 +95,7 @@ func (b *PreCommitBatcher) run() {
}
lastRes = nil
var sendAboveMax, sendAboveMin bool
var sendAboveMax bool
select {
case <-b.stop:
close(b.stopped)
@ -103,13 +103,13 @@ func (b *PreCommitBatcher) run() {
case <-b.notify:
sendAboveMax = true
case <-b.batchWait(cfg.PreCommitBatchWait, cfg.PreCommitBatchSlack):
sendAboveMin = true
// do nothing
case fr := <-b.force: // user triggered
forceRes = fr
}
var err error
lastRes, err = b.maybeStartBatch(sendAboveMax, sendAboveMin)
lastRes, err = b.maybeStartBatch(sendAboveMax)
if err != nil {
log.Warnw("PreCommitBatcher processBatch error", "error", err)
}
@ -157,7 +157,7 @@ func (b *PreCommitBatcher) batchWait(maxWait, slack time.Duration) <-chan time.T
return time.After(wait)
}
func (b *PreCommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.PreCommitBatchRes, error) {
func (b *PreCommitBatcher) maybeStartBatch(notif bool) ([]sealiface.PreCommitBatchRes, error) {
b.lk.Lock()
defer b.lk.Unlock()
@ -175,10 +175,6 @@ func (b *PreCommitBatcher) maybeStartBatch(notif, after bool) ([]sealiface.PreCo
return nil, nil
}
if after && total < cfg.MinPreCommitBatch {
return nil, nil
}
// todo support multiple batches
res, err := b.processBatch(cfg)
if err != nil && len(res) == 0 {

1
extern/storage-sealing/precommit_batch_test.go vendored
View File

@ -55,7 +55,6 @@ func TestPrecommitBatcher(t *testing.T) {
AlwaysKeepUnsealedCopy: true,
BatchPreCommits: true,
MinPreCommitBatch: 1,
MaxPreCommitBatch: maxBatch,
PreCommitBatchWait: 24 * time.Hour,
PreCommitBatchSlack: 3 * time.Hour,

1
extern/storage-sealing/sealiface/config.go vendored
View File

@ -22,7 +22,6 @@ type Config struct {
BatchPreCommits bool
MaxPreCommitBatch int
MinPreCommitBatch int
PreCommitBatchWait time.Duration
PreCommitBatchSlack time.Duration

8
extern/storage-sealing/sealing.go vendored
View File

@ -132,7 +132,7 @@ type pendingPiece struct {
accepted func(abi.SectorNumber, abi.UnpaddedPieceSize, error)
}
func New(api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing {
func New(mctx context.Context, api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing {
s := &Sealing{
api: api,
feeCfg: fc,
@ -153,9 +153,9 @@ func New(api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.
notifee: notifee,
addrSel: as,
terminator: NewTerminationBatcher(context.TODO(), maddr, api, as, fc, gc),
precommiter: NewPreCommitBatcher(context.TODO(), maddr, api, as, fc, gc),
commiter: NewCommitBatcher(context.TODO(), maddr, api, as, fc, gc, prov),
terminator: NewTerminationBatcher(mctx, maddr, api, as, fc, gc),
precommiter: NewPreCommitBatcher(mctx, maddr, api, as, fc, gc),
commiter: NewCommitBatcher(mctx, maddr, api, as, fc, gc, prov),
getConfig: gc,
dealInfo: &CurrentDealInfoManager{api},

2
extern/storage-sealing/states_sealing.go vendored
View File

@ -274,7 +274,7 @@ func (m *Sealing) preCommitParams(ctx statemachine.Context, sector SectorInfo) (
msd := policy.GetMaxProveCommitDuration(actors.VersionForNetwork(nv), sector.SectorType)
if minExpiration := height + msd + miner.MinSectorExpiration + 10; expiration < minExpiration {
if minExpiration := sector.TicketEpoch + policy.MaxPreCommitRandomnessLookback + msd + miner.MinSectorExpiration; expiration < minExpiration {
expiration = minExpiration
}
// TODO: enforce a reasonable _maximum_ sector lifetime?

Some files were not shown because too many files have changed in this diff Show More