cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #2145 from filecoin-project/next

Browse Source 
Next Testnet Staging
This commit is contained in:
Łukasz Magiera 2020-08-19 02:14:30 +02:00 committed by GitHub
commit 80e6e56a82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
444 changed files with 40142 additions and 7542 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.circleci/config.yml
View File

@ -5,7 +5,7 @@ orbs:
executors:
golang:
docker:
- image: circleci/golang:1.14.2
- image: circleci/golang:1.14.6
resource_class: 2xlarge
ubuntu:
docker:
@ -93,10 +93,10 @@ jobs:
- store_artifacts:
path: lotus
- store_artifacts:
path: lotus-storage-miner
path: lotus-miner
- store_artifacts:
path: lotus-seal-worker
- run: mkdir linux && mv lotus lotus-storage-miner lotus-seal-worker linux/
path: lotus-worker
- run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
- persist_to_workspace:
root: "."
paths:
@ -134,7 +134,7 @@ jobs:
description: Test suite name to report to CircleCI.
gotestsum-format:
type: string
default: short
default: pkgname-and-test-fails
description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
coverage:
type: string
@ -156,21 +156,27 @@ jobs:
- download-params
- go/install-gotestsum:
gobin: $HOME/.local/bin
version: 0.5.2
- run:
name: go test
environment:
GOTESTSUM_JUNITFILE: /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml
GOTESTSUM_FORMAT: << parameters.gotestsum-format >>
LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
command: |
mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
gotestsum -- \
mkdir -p /tmp/test-artifacts
gotestsum \
--format << parameters.gotestsum-format >> \
--junitfile /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml \
--jsonfile /tmp/test-artifacts/<< parameters.test-suite-name >>.json \
-- \
<< parameters.coverage >> \
<< parameters.go-test-flags >> \
<< parameters.packages >>
no_output_timeout: 30m
- store_test_results:
path: /tmp/test-reports
- store_artifacts:
path: /tmp/test-artifacts/<< parameters.test-suite-name >>.json
- when:
condition: << parameters.codecov-upload >>
steps:
@ -223,10 +229,10 @@ jobs:
- store_artifacts:
path: lotus
- store_artifacts:
path: lotus-storage-miner
path: lotus-miner
- store_artifacts:
path: lotus-seal-worker
- run: mkdir darwin && mv lotus lotus-storage-miner lotus-seal-worker darwin/
path: lotus-worker
- run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
- persist_to_workspace:
root: "."
paths:
@ -314,16 +320,19 @@ workflows:
ci:
jobs:
- lint-changes:
args: "--new-from-rev origin/master"
args: "--new-from-rev origin/next"
- mod-tidy-check
- gofmt
- test:
codecov-upload: true
test-suite-name: full
- test-window-post:
go-test-flags: "-run=TestWindowedPost"
winpost-test: "1"
test-suite-name: window-post
- test-short:
go-test-flags: "--timeout 10m --short"
test-suite-name: short
filters:
tags:
only:

17
.gitignore vendored
View File

@ -1,19 +1,21 @@
/lotus
/lotus-storage-miner
/lotus-seal-worker
/lotus-miner
/lotus-worker
/lotus-seed
/lotus-health
/lotus-chainwatch
/lotus-shed
/pond
/townhall
/fountain
/stats
/bench
/lotus-pond
/lotus-townhall
/lotus-fountain
/lotus-stats
/lotus-bench
/bench.json
/lotuspond/front/node_modules
/lotuspond/front/build
/cmd/lotus-townhall/townhall/node_modules
/cmd/lotus-townhall/townhall/build
/cmd/lotus-townhall/townhall/package-lock.json
extern/filecoin-ffi/rust/target
**/*.a
**/*.pc
@ -24,7 +26,6 @@ build/paramfetch.sh
/vendor
/blocks.dot
/blocks.svg
/chainwatch
/chainwatch.db
/bundle
/darwin

156
Makefile
View File

@ -58,10 +58,10 @@ deps: $(BUILD_DEPS)
.PHONY: deps
debug: GOFLAGS+=-tags=debug
debug: lotus lotus-storage-miner lotus-seal-worker lotus-seed
debug: lotus lotus-miner lotus-worker lotus-seed
2k: GOFLAGS+=-tags=2k
2k: lotus lotus-storage-miner lotus-seal-worker lotus-seed
2k: lotus lotus-miner lotus-worker lotus-seed
lotus: $(BUILD_DEPS)
rm -f lotus
@ -71,19 +71,19 @@ lotus: $(BUILD_DEPS)
.PHONY: lotus
BINS+=lotus
lotus-storage-miner: $(BUILD_DEPS)
rm -f lotus-storage-miner
go build $(GOFLAGS) -o lotus-storage-miner ./cmd/lotus-storage-miner
go run github.com/GeertJohan/go.rice/rice append --exec lotus-storage-miner -i ./build
.PHONY: lotus-storage-miner
BINS+=lotus-storage-miner
lotus-miner: $(BUILD_DEPS)
rm -f lotus-miner
go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-storage-miner
go run github.com/GeertJohan/go.rice/rice append --exec lotus-miner -i ./build
.PHONY: lotus-miner
BINS+=lotus-miner
lotus-seal-worker: $(BUILD_DEPS)
rm -f lotus-seal-worker
go build $(GOFLAGS) -o lotus-seal-worker ./cmd/lotus-seal-worker
go run github.com/GeertJohan/go.rice/rice append --exec lotus-seal-worker -i ./build
.PHONY: lotus-seal-worker
BINS+=lotus-seal-worker
lotus-worker: $(BUILD_DEPS)
rm -f lotus-worker
go build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
go run github.com/GeertJohan/go.rice/rice append --exec lotus-worker -i ./build
.PHONY: lotus-worker
BINS+=lotus-worker
lotus-shed: $(BUILD_DEPS)
rm -f lotus-shed
@ -92,7 +92,7 @@ lotus-shed: $(BUILD_DEPS)
.PHONY: lotus-shed
BINS+=lotus-shed
build: lotus lotus-storage-miner lotus-seal-worker
build: lotus lotus-miner lotus-worker
@[[ $$(type -P "lotus") ]] && echo "Caution: you have \
an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true
@ -102,13 +102,13 @@ install: install-daemon install-miner install-worker
install-daemon:
install -C ./lotus /usr/local/bin/lotus
install-miner:
install -C ./lotus-storage-miner /usr/local/bin/lotus-storage-miner
install-worker:
install -C ./lotus-seal-worker /usr/local/bin/lotus-seal-worker
install-miner:
install -C ./lotus-miner /usr/local/bin/lotus-miner
install-worker:
install -C ./lotus-worker /usr/local/bin/lotus-worker
# TOOLS
lotus-seed: $(BUILD_DEPS)
@ -125,63 +125,68 @@ benchmarks:
@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
.PHONY: benchmarks
pond: 2k
go build -o pond ./lotuspond
lotus-pond: 2k
go build -o lotus-pond ./lotuspond
(cd lotuspond/front && npm i && CI=false npm run build)
.PHONY: pond
BINS+=pond
.PHONY: lotus-pond
BINS+=lotus-pond
townhall:
rm -f townhall
go build -o townhall ./cmd/lotus-townhall
lotus-townhall:
rm -f lotus-townhall
go build -o lotus-townhall ./cmd/lotus-townhall
(cd ./cmd/lotus-townhall/townhall && npm i && npm run build)
go run github.com/GeertJohan/go.rice/rice append --exec townhall -i ./cmd/lotus-townhall -i ./build
.PHONY: townhall
BINS+=townhall
go run github.com/GeertJohan/go.rice/rice append --exec lotus-townhall -i ./cmd/lotus-townhall -i ./build
.PHONY: lotus-townhall
BINS+=lotus-townhall
fountain:
rm -f fountain
go build -o fountain ./cmd/lotus-fountain
go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain -i ./build
.PHONY: fountain
BINS+=fountain
lotus-fountain:
rm -f lotus-fountain
go build -o lotus-fountain ./cmd/lotus-fountain
go run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
.PHONY: lotus-fountain
BINS+=lotus-fountain
chainwatch:
rm -f chainwatch
go build -o chainwatch ./cmd/lotus-chainwatch
.PHONY: chainwatch
BINS+=chainwatch
lotus-chainwatch:
rm -f lotus-chainwatch
go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
.PHONY: lotus-chainwatch
BINS+=lotus-chainwatch
bench:
rm -f bench
go build -o bench ./cmd/lotus-bench
go run github.com/GeertJohan/go.rice/rice append --exec bench -i ./build
.PHONY: bench
BINS+=bench
lotus-bench:
rm -f lotus-bench
go build -o lotus-bench ./cmd/lotus-bench
go run github.com/GeertJohan/go.rice/rice append --exec lotus-bench -i ./build
.PHONY: lotus-bench
BINS+=lotus-bench
stats:
rm -f stats
go build -o stats ./tools/stats
go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build
.PHONY: stats
BINS+=stats
lotus-stats:
rm -f lotus-stats
go build -o lotus-stats ./cmd/lotus-stats
go run github.com/GeertJohan/go.rice/rice append --exec lotus-stats -i ./build
.PHONY: lotus-stats
BINS+=lotus-stats
health:
lotus-pcr:
rm -f lotus-pcr
go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
go run github.com/GeertJohan/go.rice/rice append --exec lotus-pcr -i ./build
.PHONY: lotus-pcr
BINS+=lotus-pcr
lotus-health:
rm -f lotus-health
go build -o lotus-health ./cmd/lotus-health
go run github.com/GeertJohan/go.rice/rice append --exec lotus-health -i ./build
.PHONY: health
BINS+=health
.PHONY: lotus-health
BINS+=lotus-health
testground:
go build -tags testground -o /dev/null ./cmd/lotus
.PHONY: testground
BINS+=testground
install-chainwatch: chainwatch
install -C ./chainwatch /usr/local/bin/chainwatch
install-chainwatch: lotus-chainwatch
install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
# SYSTEMD
@ -208,34 +213,34 @@ install-chainwatch-service: install-chainwatch install-daemon-service
systemctl daemon-reload
@echo
@echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup."
install-main-services: install-miner-service
install-all-services: install-main-services install-chainwatch-service
install-services: install-main-services
clean-daemon-service: clean-miner-service clean-chainwatch-service
clean-daemon-service: clean-miner-service clean-chainwatch-service
-systemctl stop lotus-daemon
-systemctl disable lotus-daemon
rm -f /etc/systemd/system/lotus-daemon.service
systemctl daemon-reload
clean-miner-service:
clean-miner-service:
-systemctl stop lotus-miner
-systemctl disable lotus-miner
rm -f /etc/systemd/system/lotus-miner.service
systemctl daemon-reload
clean-chainwatch-service:
-systemctl stop chainwatch
-systemctl disable chainwatch
-systemctl stop lotus-chainwatch
-systemctl disable lotus-chainwatch
rm -f /etc/systemd/system/lotus-chainwatch.service
systemctl daemon-reload
systemctl daemon-reload
clean-main-services: clean-daemon-service
clean-all-services: clean-main-services
clean-all-services: clean-main-services
clean-services: clean-all-services
@ -245,15 +250,15 @@ buildall: $(BINS)
completions:
./scripts/make-completions.sh lotus
./scripts/make-completions.sh lotus-storage-miner
./scripts/make-completions.sh lotus-miner
.PHONY: completions
install-completions:
mkdir -p /usr/share/bash-completion/completions /usr/local/share/zsh/site-functions/
install -C ./scripts/bash-completion/lotus /usr/share/bash-completion/completions/lotus
install -C ./scripts/bash-completion/lotus-storage-miner /usr/share/bash-completion/completions/lotus-storage-miner
install -C ./scripts/bash-completion/lotus-miner /usr/share/bash-completion/completions/lotus-miner
install -C ./scripts/zsh-completion/lotus /usr/local/share/zsh/site-functions/_lotus
install -C ./scripts/zsh-completion/lotus-storage-miner /usr/local/share/zsh/site-functions/_lotus-storage-miner
install -C ./scripts/zsh-completion/lotus-miner /usr/local/share/zsh/site-functions/_lotus-miner
clean:
rm -rf $(CLEAN) $(BINS)
@ -267,6 +272,7 @@ dist-clean:
type-gen:
go run ./gen/main.go
go generate ./...
method-gen:
(cd ./lotuspond/front/src/chain && go run ./methodgen.go)

9
api/api_common.go
View File

@ -4,11 +4,10 @@ import (
"context"
"fmt"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/build"
)
@ -28,6 +27,7 @@ type Common interface {
NetDisconnect(context.Context, peer.ID) error
NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
NetPubsubScores(context.Context) ([]PubsubScore, error)
NetAutoNatStatus(context.Context) (NatInfo, error)
// MethodGroup: Common
@ -65,3 +65,8 @@ type Version struct {
func (v Version) String() string {
return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
}
type NatInfo struct {
Reachability network.Reachability
PublicAddr string
}

209
api/api_full.go
View File

@ -5,20 +5,23 @@ import (
"time"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-filestore"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-multistore"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/builtin/paych"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
@ -37,8 +40,11 @@ type FullNode interface {
// ChainHead returns the current head of the chain.
ChainHead(context.Context) (*types.TipSet, error)
// ChainGetRandomness is used to sample the chain for randomness.
ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
// ChainGetRandomnessFromTickets is used to sample the chain for randomness.
ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
// ChainGetRandomnessFromBeacon is used to sample the beacon for randomness.
ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
// ChainGetBlock returns the block specified by the given CID.
ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
@ -67,7 +73,11 @@ type FullNode interface {
// ChainHasObj checks if a given CID exists in the chain blockstore.
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainStatObj(context.Context, cid.Cid, cid.Cid) (ObjStat, error)
// ChainStatObj returns statistics about the graph referenced by 'obj'.
// If 'base' is also specified, then the returned stat will be a diff
// between the two objects.
ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (ObjStat, error)
// ChainSetHead forcefully sets current chain head. Use with caution.
ChainSetHead(context.Context, types.TipSetKey) error
@ -101,6 +111,27 @@ type FullNode interface {
// ChainExport returns a stream of bytes with CAR dump of chain data.
ChainExport(context.Context, types.TipSetKey) (<-chan []byte, error)
// MethodGroup: Beacon
// The Beacon method group contains methods for interacting with the random beacon (DRAND)
// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
// the entry has not yet been produced, the call will block until the entry
// becomes available
BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error)
// GasEstimateFeeCap estimates gas fee cap
GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error)
// GasEstimateGasLimit estimates gas used by the message and returns it.
// It fails if message fails to execute.
GasEstimateGasLimit(context.Context, *types.Message, types.TipSetKey) (int64, error)
// GasEstimateGasPremium estimates what gas price should be used for a
// message to have high likelihood of inclusion in `nblocksincl` epochs.
GasEstimateGasPremium(_ context.Context, nblocksincl uint64,
sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error)
// MethodGroup: Sync
// The Sync method group contains methods for interacting with and
// observing the lotus sync service.
@ -131,21 +162,29 @@ type FullNode interface {
// MpoolPending returns pending mempool messages.
MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)
// MpoolSelect returns a list of pending messages for inclusion in the next block
MpoolSelect(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error)
// MpoolPush pushes a signed message to mempool.
MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error)
// MpoolPushMessage atomically assigns a nonce, signs, and pushes a message
// to mempool.
MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error)
// maxFee is only used when GasFeeCap/GasPremium fields aren't specified
//
// When maxFee is set to 0, MpoolPushMessage will guess appropriate fee
// based on current chain conditions
MpoolPushMessage(ctx context.Context, msg *types.Message, spec *MessageSendSpec) (*types.SignedMessage, error)
// MpoolGetNonce gets next nonce for the specified sender.
// Note that this method may not be atomic. Use MpoolPushMessage instead.
MpoolGetNonce(context.Context, address.Address) (uint64, error)
MpoolSub(context.Context) (<-chan MpoolUpdate, error)
// MpoolEstimateGasPrice estimates what gas price should be used for a
// message to have high likelihood of inclusion in `nblocksincl` epochs.
MpoolEstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error)
// MpoolGetConfig returns (a copy of) the current mpool config
MpoolGetConfig(context.Context) (*types.MpoolConfig, error)
// MpoolSetConfig sets the mpool config to (a copy of) the supplied config
MpoolSetConfig(context.Context, *types.MpoolConfig) error
// MethodGroup: Miner
@ -189,7 +228,9 @@ type FullNode interface {
// retrieval markets as a client
// ClientImport imports file under the specified path into filestore.
ClientImport(ctx context.Context, ref FileRef) (cid.Cid, error)
ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error)
// ClientRemoveImport removes file import
ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error
// ClientStartDeal proposes a deal with a miner.
ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error)
// ClientGetDealInfo returns the latest information about a given deal.
@ -199,17 +240,22 @@ type FullNode interface {
// ClientHasLocal indicates whether a certain CID is locally stored.
ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
// ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).
ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error)
ClientFindData(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]QueryOffer, error)
// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
ClientMinerQueryOffer(ctx context.Context, root cid.Cid, miner address.Address) (QueryOffer, error)
ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error)
// ClientRetrieve initiates the retrieval of a file, as specified in the order.
ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error
// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
// of status updates.
ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error)
// ClientQueryAsk returns a signed StorageAsk from the specified miner.
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)
// ClientCalcCommP calculates the CommP for a specified file, based on the sector size of the provided miner.
ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*CommPRet, error)
// ClientCalcCommP calculates the CommP for a specified file
ClientCalcCommP(ctx context.Context, inpath string) (*CommPRet, error)
// ClientGenCar generates a CAR file for the specified file.
ClientGenCar(ctx context.Context, ref FileRef, outpath string) error
// ClientDealSize calculates real deal data size
ClientDealSize(ctx context.Context, root cid.Cid) (DataSize, error)
// ClientUnimport removes references to the specified file from filestore
//ClientUnimport(path string)
@ -241,8 +287,8 @@ type FullNode interface {
// If the filterOut boolean is set to true, any sectors in the filter are excluded.
// If false, only those sectors in the filter are included.
StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*ChainSectorInfo, error)
// StateMinerProvingSet returns info about those sectors that a given miner is actively proving.
StateMinerProvingSet(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error)
// StateMinerActiveSectors returns info about sectors that a given miner is actively proving.
StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error)
// StateMinerProvingDeadline calculates the deadline at some epoch for a proving period
// and returns the deadline-related calculations.
StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error)
@ -251,22 +297,31 @@ type FullNode interface {
// StateMinerInfo returns info about the indicated miner
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error)
// StateMinerDeadlines returns all the proving deadlines for the given miner
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error)
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error)
// StateMinerPartitions loads miner partitions for the specified miner/deadline
StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)
// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
StateMinerFaults(context.Context, address.Address, types.TipSetKey) (abi.BitField, error)
// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, ts types.TipSetKey) ([]*Fault, error)
// StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner
StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (abi.BitField, error)
// StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector
StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
// StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector
StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error)
StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
// StateSectorGetInfo returns the on-chain info for the specified miner's sector
// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
// expiration epoch
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
StatePledgeCollateral(context.Context, types.TipSetKey) (types.BigInt, error)
// StateSectorExpiration returns epoch at which given sector will expire
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*SectorExpiration, error)
// StateSectorPartition finds deadline/partition with the specified sector
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error)
// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error)
// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
@ -298,6 +353,16 @@ type FullNode interface {
// StateCompute is a flexible command that applies the given messages on the given tipset.
// The messages are run as though the VM were at the provided height.
StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*ComputeStateOutput, error)
// StateVerifiedClientStatus returns the data cap for the given address.
// Returns nil if there is no entry in the data cap table for the
// address.
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*verifreg.DataCap, error)
// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
// can issue. It takes the deal size and verified status as parameters.
StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error)
// StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset
StateCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error)
// MethodGroup: Msig
// The Msig methods are used to interact with multisig wallets on the
@ -308,7 +373,7 @@ type FullNode interface {
// MsigCreate creates a multisig wallet
// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
//<initial balance>, <sender address of the create msg>, <gas price>
MsigCreate(context.Context, int64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error)
MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error)
// MsigPropose proposes a multisig message
// It takes the following params: <multisig address>, <recipient address>, <value to transfer>,
// <sender address of the propose msg>, <method to call in the proposed message>, <params to include in the proposed message>
@ -340,10 +405,12 @@ type FullNode interface {
// MethodGroup: Paych
// The Paych methods are for interacting with and managing payment channels
PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error)
PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)
PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)
PaychList(context.Context) ([]address.Address, error)
PaychStatus(context.Context, address.Address) (*PaychStatus, error)
PaychClose(context.Context, address.Address) (cid.Cid, error)
PaychSettle(context.Context, address.Address) (cid.Cid, error)
PaychCollect(context.Context, address.Address) (cid.Cid, error)
PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error)
PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error)
PaychVoucherCheckValid(context.Context, address.Address, *paych.SignedVoucher) error
@ -360,15 +427,35 @@ type FileRef struct {
}
type MinerSectors struct {
Sset uint64
Pset uint64
Sectors uint64
Active uint64
}
type SectorExpiration struct {
OnTime abi.ChainEpoch
// non-zero if sector is faulty, epoch at which it will be permanently
// removed if it doesn't recover
Early abi.ChainEpoch
}
type SectorLocation struct {
Deadline uint64
Partition uint64
}
type ImportRes struct {
Root cid.Cid
ImportID multistore.StoreID
}
type Import struct {
Status filestore.Status
Key cid.Cid
Key multistore.StoreID
Err string
Root *cid.Cid
Source string
FilePath string
Size uint64
}
type DealInfo struct {
@ -377,6 +464,7 @@ type DealInfo struct {
Message string // more information about deal state, particularly errors
Provider address.Address
DataRef *storagemarket.DataRef
PieceCID cid.Cid
Size uint64
@ -387,6 +475,7 @@ type DealInfo struct {
}
type MsgLookup struct {
Message cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed
Receipt types.MessageReceipt
ReturnDec interface{}
TipSet types.TipSetKey
@ -429,14 +518,14 @@ type PaychStatus struct {
}
type ChannelInfo struct {
Channel address.Address
ChannelMessage cid.Cid
Channel address.Address
WaitSentinel cid.Cid
}
type PaymentInfo struct {
Channel address.Address
ChannelMessage *cid.Cid
Vouchers []*paych.SignedVoucher
Channel address.Address
WaitSentinel cid.Cid
Vouchers []*paych.SignedVoucher
}
type VoucherSpec struct {
@ -456,27 +545,31 @@ type MinerPower struct {
type QueryOffer struct {
Err string
Root cid.Cid
Root cid.Cid
Piece *cid.Cid
Size uint64
MinPrice types.BigInt
UnsealPrice types.BigInt
PaymentInterval uint64
PaymentIntervalIncrease uint64
Miner address.Address
MinerPeerID peer.ID
MinerPeer retrievalmarket.RetrievalPeer
}
func (o *QueryOffer) Order(client address.Address) RetrievalOrder {
return RetrievalOrder{
Root: o.Root,
Piece: o.Piece,
Size: o.Size,
Total: o.MinPrice,
UnsealPrice: o.UnsealPrice,
PaymentInterval: o.PaymentInterval,
PaymentIntervalIncrease: o.PaymentIntervalIncrease,
Client: client,
Miner: o.Miner,
MinerPeerID: o.MinerPeerID,
Miner: o.Miner,
MinerPeer: o.MinerPeer,
}
}
@ -492,15 +585,17 @@ type MarketDeal struct {
type RetrievalOrder struct {
// TODO: make this less unixfs specific
Root cid.Cid
Size uint64
Root cid.Cid
Piece *cid.Cid
Size uint64
// TODO: support offset
Total types.BigInt
UnsealPrice types.BigInt
PaymentInterval uint64
PaymentIntervalIncrease uint64
Client address.Address
Miner address.Address
MinerPeerID peer.ID
MinerPeer retrievalmarket.RetrievalPeer
}
type InvocResult struct {
@ -517,12 +612,15 @@ type MethodCall struct {
}
type StartDealParams struct {
Data *storagemarket.DataRef
Wallet address.Address
Miner address.Address
EpochPrice types.BigInt
MinBlocksDuration uint64
DealStartEpoch abi.ChainEpoch
Data *storagemarket.DataRef
Wallet address.Address
Miner address.Address
EpochPrice types.BigInt
MinBlocksDuration uint64
ProviderCollateral big.Int
DealStartEpoch abi.ChainEpoch
FastRetrieval bool
VerifiedDeal bool
}
type IpldObject struct {
@ -574,6 +672,19 @@ type ComputeStateOutput struct {
Trace []*InvocResult
}
type DealCollateralBounds struct {
Min abi.TokenAmount
Max abi.TokenAmount
}
type CirculatingSupply struct {
FilVested abi.TokenAmount
FilMined abi.TokenAmount
FilBurnt abi.TokenAmount
FilLocked abi.TokenAmount
FilCirculating abi.TokenAmount
}
type MiningBaseInfo struct {
MinerPower types.BigInt
NetworkPower types.BigInt
@ -582,6 +693,7 @@ type MiningBaseInfo struct {
SectorSize abi.SectorSize
PrevBeaconEntry types.BeaconEntry
BeaconEntries []types.BeaconEntry
HasMinPower bool
}
type BlockTemplate struct {
@ -596,6 +708,11 @@ type BlockTemplate struct {
WinningPoStProof []abi.PoStProof
}
type DataSize struct {
PayloadSize int64
PieceSize abi.PaddedPieceSize
}
type CommPRet struct {
Root cid.Cid
Size abi.UnpaddedPieceSize

56
api/api_storage.go
View File

@ -3,14 +3,18 @@ package api
import (
"bytes"
"context"
"time"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/piecestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/specs-actors/actors/abi"
)
@ -28,31 +32,53 @@ type StorageMiner interface {
PledgeSector(context.Context) error
// Get the status of a given sector by ID
SectorsStatus(context.Context, abi.SectorNumber) (SectorInfo, error)
SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
// List all staged sectors
SectorsList(context.Context) ([]abi.SectorNumber, error)
SectorsRefs(context.Context) (map[string][]SealedRef, error)
// SectorStartSealing can be called on sectors in Empty or WaitDeals states
// to trigger sealing early
SectorStartSealing(context.Context, abi.SectorNumber) error
// SectorSetSealDelay sets the time that a newly-created sector
// waits for more deals before it starts sealing
SectorSetSealDelay(context.Context, time.Duration) error
// SectorGetSealDelay gets the time that a newly-created sector
// waits for more deals before it starts sealing
SectorGetSealDelay(context.Context) (time.Duration, error)
// SectorSetExpectedSealDuration sets the expected time for a sector to seal
SectorSetExpectedSealDuration(context.Context, time.Duration) error
// SectorGetExpectedSealDuration gets the expected time for a sector to seal
SectorGetExpectedSealDuration(context.Context) (time.Duration, error)
SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error
SectorRemove(context.Context, abi.SectorNumber) error
SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error
StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)
StorageLocal(ctx context.Context) (map[stores.ID]string, error)
StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error)
StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error)
// WorkerConnect tells the node to connect to workers RPC
WorkerConnect(context.Context, string) error
WorkerStats(context.Context) (map[uint64]storiface.WorkerStats, error)
WorkerJobs(context.Context) (map[uint64][]storiface.WorkerJob, error)
// SealingSchedDiag dumps internal sealing scheduler state
SealingSchedDiag(context.Context) (interface{}, error)
stores.SectorIndex
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error)
MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)
MarketGetDealUpdates(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error)
MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)
MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error
MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)
DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
@ -68,6 +94,11 @@ type StorageMiner interface {
DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
StorageAddLocal(ctx context.Context, path string) error
PiecesListPieces(ctx context.Context) ([]cid.Cid, error)
PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)
PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error)
PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)
}
type SealRes struct {
@ -100,11 +131,24 @@ type SectorInfo struct {
LastErr string
Log []SectorLog
// On Chain Info
SealProof abi.RegisteredSealProof // The seal proof type implies the PoSt proof/s
Activation abi.ChainEpoch // Epoch during which the sector proof was accepted
Expiration abi.ChainEpoch // Epoch during which the sector expires
DealWeight abi.DealWeight // Integral of active deals over sector lifetime
VerifiedDealWeight abi.DealWeight // Integral of active verified deals over sector lifetime
InitialPledge abi.TokenAmount // Pledge collected to commit this sector
// Expiration Info
OnTime abi.ChainEpoch
// non-zero if sector is faulty, epoch at which it will be permanently
// removed if it doesn't recover
Early abi.ChainEpoch
}
type SealedRef struct {
SectorID abi.SectorNumber
Offset uint64
Offset abi.PaddedPieceSize
Size abi.UnpaddedPieceSize
}

10
api/api_worker.go
View File

@ -6,9 +6,9 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/sector-storage/sealtasks"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
@ -23,12 +23,14 @@ type WorkerAPI interface {
Paths(context.Context) ([]stores.StoragePath, error)
Info(context.Context) (storiface.WorkerInfo, error)
AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)
storage.Sealer
MoveStorage(ctx context.Context, sector abi.SectorID) error
UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error
ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error
ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error)
Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error

3
api/apibstore/apibstore.go
View File

@ -5,8 +5,9 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/blockstore"
)
type ChainIO interface {

460
api/apistruct/struct.go
View File

@ -3,22 +3,28 @@ package apistruct
import (
"context"
"io"
"time"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/piecestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/sector-storage/sealtasks"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/sector-storage/storiface"
"github.com/filecoin-project/go-multistore"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/builtin/paych"
"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-storage/storage"
@ -43,6 +49,7 @@ type CommonStruct struct {
NetDisconnect func(context.Context, peer.ID) error `perm:"write"`
NetFindPeer func(context.Context, peer.ID) (peer.AddrInfo, error) `perm:"read"`
NetPubsubScores func(context.Context) ([]api.PubsubScore, error) `perm:"read"`
NetAutoNatStatus func(context.Context) (api.NatInfo, error) `perm:"read"`
ID func(context.Context) (peer.ID, error) `perm:"read"`
Version func(context.Context) (api.Version, error) `perm:"read"`
@ -60,25 +67,32 @@ type FullNodeStruct struct {
CommonStruct
Internal struct {
ChainNotify func(context.Context) (<-chan []*api.HeadChange, error) `perm:"read"`
ChainHead func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainGetRandomness func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"`
ChainGetTipSet func(context.Context, types.TipSetKey) (*types.TipSet, error) `perm:"read"`
ChainGetBlockMessages func(context.Context, cid.Cid) (*api.BlockMessages, error) `perm:"read"`
ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"`
ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error) `perm:"read"`
ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) `perm:"read"`
ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"`
ChainHasObj func(context.Context, cid.Cid) (bool, error) `perm:"read"`
ChainStatObj func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error) `perm:"read"`
ChainSetHead func(context.Context, types.TipSetKey) error `perm:"admin"`
ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainTipSetWeight func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"`
ChainGetNode func(ctx context.Context, p string) (*api.IpldObject, error) `perm:"read"`
ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"`
ChainGetPath func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error) `perm:"read"`
ChainExport func(context.Context, types.TipSetKey) (<-chan []byte, error) `perm:"read"`
ChainNotify func(context.Context) (<-chan []*api.HeadChange, error) `perm:"read"`
ChainHead func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainGetRandomnessFromTickets func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
ChainGetRandomnessFromBeacon func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"`
ChainGetTipSet func(context.Context, types.TipSetKey) (*types.TipSet, error) `perm:"read"`
ChainGetBlockMessages func(context.Context, cid.Cid) (*api.BlockMessages, error) `perm:"read"`
ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"`
ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error) `perm:"read"`
ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) `perm:"read"`
ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"`
ChainHasObj func(context.Context, cid.Cid) (bool, error) `perm:"read"`
ChainStatObj func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error) `perm:"read"`
ChainSetHead func(context.Context, types.TipSetKey) error `perm:"admin"`
ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainTipSetWeight func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"`
ChainGetNode func(ctx context.Context, p string) (*api.IpldObject, error) `perm:"read"`
ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"`
ChainGetPath func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error) `perm:"read"`
ChainExport func(context.Context, types.TipSetKey) (<-chan []byte, error) `perm:"read"`
BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"`
GasEstimateGasPremium func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"`
GasEstimateGasLimit func(context.Context, *types.Message, types.TipSetKey) (int64, error) `perm:"read"`
GasEstimateFeeCap func(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"`
SyncState func(context.Context) (*api.SyncState, error) `perm:"read"`
SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"`
@ -86,12 +100,16 @@ type FullNodeStruct struct {
SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"`
SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"`
MpoolPending func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) `perm:"read"`
MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"`
MpoolPushMessage func(context.Context, *types.Message) (*types.SignedMessage, error) `perm:"sign"`
MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"`
MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"`
MpoolEstimateGasPrice func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"`
MpoolGetConfig func(context.Context) (*types.MpoolConfig, error) `perm:"read"`
MpoolSetConfig func(context.Context, *types.MpoolConfig) error `perm:"write"`
MpoolSelect func(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error) `perm:"read"`
MpoolPending func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) `perm:"read"`
MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"`
MpoolPushMessage func(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) `perm:"sign"`
MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"`
MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"`
MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"`
MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error) `perm:"write"`
@ -109,56 +127,65 @@ type FullNodeStruct struct {
WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"`
WalletDelete func(context.Context, address.Address) error `perm:"write"`
ClientImport func(ctx context.Context, ref api.FileRef) (cid.Cid, error) `perm:"admin"`
ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"`
ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"`
ClientFindData func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) `perm:"read"`
ClientMinerQueryOffer func(ctx context.Context, root cid.Cid, miner address.Address) (api.QueryOffer, error) `perm:"read"`
ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"`
ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"`
ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"`
ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"`
ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
ClientCalcCommP func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) `perm:"read"`
ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"`
ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"`
ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"`
ClientRemoveImport func(ctx context.Context, importID multistore.StoreID) error `perm:"admin"`
ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"`
ClientFindData func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) `perm:"read"`
ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"`
ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"`
ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"`
ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"`
ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"`
ClientRetrieveWithEvents func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
ClientCalcCommP func(ctx context.Context, inpath string) (*api.CommPRet, error) `perm:"read"`
ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"`
ClientDealSize func(ctx context.Context, root cid.Cid) (api.DataSize, error) `perm:"read"`
StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"`
StateMinerProvingSet func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"`
StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"`
StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"`
StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"`
StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error) `perm:"read"`
StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"`
StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"`
StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"`
StateMinerInitialPledgeCollateral func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) `perm:"read"`
StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"`
StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"`
StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"`
StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"`
StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"`
StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"`
StatePledgeCollateral func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"`
StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"`
StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"`
StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"`
StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"`
StateMarketBalance func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) `perm:"read"`
StateMarketParticipants func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"`
StateMarketDeals func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"`
StateMarketStorageDeal func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) `perm:"read"`
StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) `perm:"read"`
StateAccountKey func(context.Context, address.Address, types.TipSetKey) (address.Address, error) `perm:"read"`
StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"`
StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"`
StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"`
StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"`
StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"`
StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"`
StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"`
StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"`
StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"`
StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"`
StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) `perm:"read"`
StateMinerPartitions func(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error) `perm:"read"`
StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (abi.BitField, error) `perm:"read"`
StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"`
StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (abi.BitField, error) `perm:"read"`
StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"`
StateMinerInitialPledgeCollateral func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"`
StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"`
StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"`
StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorExpiration, error) `perm:"read"`
StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorLocation, error) `perm:"read"`
StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"`
StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"`
StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"`
StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"`
StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"`
StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"`
StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"`
StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"`
StateMarketBalance func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) `perm:"read"`
StateMarketParticipants func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"`
StateMarketDeals func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"`
StateMarketStorageDeal func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) `perm:"read"`
StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) `perm:"read"`
StateAccountKey func(context.Context, address.Address, types.TipSetKey) (address.Address, error) `perm:"read"`
StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"`
StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"`
StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"`
StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"`
StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"`
StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error) `perm:"read"`
StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"`
StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"`
MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"`
MsigCreate func(context.Context, int64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
MsigCreate func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
MsigPropose func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
MsigApprove func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
MsigCancel func(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
@ -168,10 +195,12 @@ type FullNodeStruct struct {
MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) `perm:"sign"`
PaychGet func(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) `perm:"sign"`
PaychGetWaitReady func(context.Context, cid.Cid) (address.Address, error) `perm:"sign"`
PaychList func(context.Context) ([]address.Address, error) `perm:"read"`
PaychStatus func(context.Context, address.Address) (*api.PaychStatus, error) `perm:"read"`
PaychClose func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"`
PaychSettle func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"`
PaychCollect func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"`
PaychAllocateLane func(context.Context, address.Address) (uint64, error) `perm:"sign"`
PaychNewPayment func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
PaychVoucherCheck func(context.Context, *paych.SignedVoucher) error `perm:"read"`
@ -197,30 +226,43 @@ type StorageMinerStruct struct {
MiningBase func(context.Context) (*types.TipSet, error) `perm:"read"`
MarketImportDealData func(context.Context, cid.Cid, string) error `perm:"write"`
MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"`
MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"`
MarketSetAsk func(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"`
MarketGetAsk func(ctx context.Context) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
MarketImportDealData func(context.Context, cid.Cid, string) error `perm:"write"`
MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"`
MarketListRetrievalDeals func(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) `perm:"read"`
MarketGetDealUpdates func(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error) `perm:"read"`
MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"`
MarketSetAsk func(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"`
MarketGetAsk func(ctx context.Context) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
MarketSetRetrievalAsk func(ctx context.Context, rask *retrievalmarket.Ask) error `perm:"admin"`
MarketGetRetrievalAsk func(ctx context.Context) (*retrievalmarket.Ask, error) `perm:"read"`
PledgeSector func(context.Context) error `perm:"write"`
SectorsStatus func(context.Context, abi.SectorNumber) (api.SectorInfo, error) `perm:"read"`
SectorsList func(context.Context) ([]abi.SectorNumber, error) `perm:"read"`
SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"`
SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"`
SectorRemove func(context.Context, abi.SectorNumber) error `perm:"admin"`
SectorsStatus func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
SectorsList func(context.Context) ([]abi.SectorNumber, error) `perm:"read"`
SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"`
SectorStartSealing func(context.Context, abi.SectorNumber) error `perm:"write"`
SectorSetSealDelay func(context.Context, time.Duration) error `perm:"write"`
SectorGetSealDelay func(context.Context) (time.Duration, error) `perm:"read"`
SectorSetExpectedSealDuration func(context.Context, time.Duration) error `perm:"write"`
SectorGetExpectedSealDuration func(context.Context) (time.Duration, error) `perm:"read"`
SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"admin"`
SectorRemove func(context.Context, abi.SectorNumber) error `perm:"admin"`
SectorMarkForUpgrade func(ctx context.Context, id abi.SectorNumber) error `perm:"admin"`
WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm
WorkerStats func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"`
WorkerJobs func(context.Context) (map[uint64][]storiface.WorkerJob, error) `perm:"admin"`
SealingSchedDiag func(context.Context) (interface{}, error) `perm:"admin"`
StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"`
StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"`
StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"`
StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"`
StorageStat func(context.Context, stores.ID) (fsutil.FsStat, error) `perm:"admin"`
StorageAttach func(context.Context, stores.StorageInfo, fsutil.FsStat) error `perm:"admin"`
StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"`
StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"`
StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"`
StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, abi.RegisteredSealProof, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"`
StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"`
StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"`
StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"`
@ -241,6 +283,11 @@ type StorageMinerStruct struct {
DealsSetPieceCidBlocklist func(context.Context, []cid.Cid) error `perm:"admin"`
StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
PiecesListPieces func(ctx context.Context) ([]cid.Cid, error) `perm:"read"`
PiecesListCidInfos func(ctx context.Context) ([]cid.Cid, error) `perm:"read"`
PiecesGetPieceInfo func(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"`
PiecesGetCIDInfo func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"`
}
}
@ -254,6 +301,7 @@ type WorkerStruct struct {
Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"`
Info func(context.Context) (storiface.WorkerInfo, error) `perm:"admin"`
AddPiece func(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) `perm:"admin"`
SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"`
SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"`
SealCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"`
@ -264,7 +312,7 @@ type WorkerStruct struct {
MoveStorage func(ctx context.Context, sector abi.SectorID) error `perm:"admin"`
UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"`
ReadPiece func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error `perm:"admin"`
ReadPiece func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error) `perm:"admin"`
Fetch func(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error `perm:"admin"`
@ -285,6 +333,7 @@ func (c *CommonStruct) AuthNew(ctx context.Context, perms []auth.Permission) ([]
func (c *CommonStruct) NetPubsubScores(ctx context.Context) ([]api.PubsubScore, error) {
return c.Internal.NetPubsubScores(ctx)
}
func (c *CommonStruct) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) {
return c.Internal.NetConnectedness(ctx, pid)
}
@ -309,6 +358,10 @@ func (c *CommonStruct) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInf
return c.Internal.NetFindPeer(ctx, p)
}
func (c *CommonStruct) NetAutoNatStatus(ctx context.Context) (api.NatInfo, error) {
return c.Internal.NetAutoNatStatus(ctx)
}
// ID implements API.ID
func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
return c.Internal.ID(ctx)
@ -341,7 +394,11 @@ func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, e
return c.Internal.ClientListImports(ctx)
}
func (c *FullNodeStruct) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
func (c *FullNodeStruct) ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error {
return c.Internal.ClientRemoveImport(ctx, importID)
}
func (c *FullNodeStruct) ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) {
return c.Internal.ClientImport(ctx, ref)
}
@ -349,17 +406,18 @@ func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool
return c.Internal.ClientHasLocal(ctx, root)
}
func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) {
return c.Internal.ClientFindData(ctx, root)
func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) {
return c.Internal.ClientFindData(ctx, root, piece)
}
func (c *FullNodeStruct) ClientMinerQueryOffer(ctx context.Context, root cid.Cid, miner address.Address) (api.QueryOffer, error) {
return c.Internal.ClientMinerQueryOffer(ctx, root, miner)
func (c *FullNodeStruct) ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) {
return c.Internal.ClientMinerQueryOffer(ctx, miner, root, piece)
}
func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) {
return c.Internal.ClientStartDeal(ctx, params)
}
func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*api.DealInfo, error) {
return c.Internal.ClientGetDealInfo(ctx, deal)
}
@ -372,17 +430,51 @@ func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.Retrieval
return c.Internal.ClientRetrieve(ctx, order, ref)
}
func (c *FullNodeStruct) ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
return c.Internal.ClientRetrieveWithEvents(ctx, order, ref)
}
func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {
return c.Internal.ClientQueryAsk(ctx, p, miner)
}
func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) {
return c.Internal.ClientCalcCommP(ctx, inpath, miner)
func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) {
return c.Internal.ClientCalcCommP(ctx, inpath)
}
func (c *FullNodeStruct) ClientGenCar(ctx context.Context, ref api.FileRef, outpath string) error {
return c.Internal.ClientGenCar(ctx, ref, outpath)
}
func (c *FullNodeStruct) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) {
return c.Internal.ClientDealSize(ctx, root)
}
func (c *FullNodeStruct) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64,
sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.GasEstimateGasPremium(ctx, nblocksincl, sender, gaslimit, tsk)
}
func (c *FullNodeStruct) GasEstimateFeeCap(ctx context.Context, msg *types.Message,
maxqueueblks int64, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.GasEstimateFeeCap(ctx, msg, maxqueueblks, tsk)
}
func (c *FullNodeStruct) GasEstimateGasLimit(ctx context.Context, msg *types.Message,
tsk types.TipSetKey) (int64, error) {
return c.Internal.GasEstimateGasLimit(ctx, msg, tsk)
}
func (c *FullNodeStruct) MpoolGetConfig(ctx context.Context) (*types.MpoolConfig, error) {
return c.Internal.MpoolGetConfig(ctx)
}
func (c *FullNodeStruct) MpoolSetConfig(ctx context.Context, cfg *types.MpoolConfig) error {
return c.Internal.MpoolSetConfig(ctx, cfg)
}
func (c *FullNodeStruct) MpoolSelect(ctx context.Context, tsk types.TipSetKey, tq float64) ([]*types.SignedMessage, error) {
return c.Internal.MpoolSelect(ctx, tsk, tq)
}
func (c *FullNodeStruct) MpoolPending(ctx context.Context, tsk types.TipSetKey) ([]*types.SignedMessage, error) {
return c.Internal.MpoolPending(ctx, tsk)
}
@ -391,18 +483,14 @@ func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessag
return c.Internal.MpoolPush(ctx, smsg)
}
func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) {
return c.Internal.MpoolPushMessage(ctx, msg)
func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
return c.Internal.MpoolPushMessage(ctx, msg, spec)
}
func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) {
return c.Internal.MpoolSub(ctx)
}
func (c *FullNodeStruct) MpoolEstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, limit int64, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.MpoolEstimateGasPrice(ctx, nblocksincl, sender, limit, tsk)
}
func (c *FullNodeStruct) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoch abi.ChainEpoch, tsk types.TipSetKey) (*api.MiningBaseInfo, error) {
return c.Internal.MinerGetBaseInfo(ctx, maddr, epoch, tsk)
}
@ -415,8 +503,12 @@ func (c *FullNodeStruct) ChainHead(ctx context.Context) (*types.TipSet, error) {
return c.Internal.ChainHead(ctx)
}
func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
return c.Internal.ChainGetRandomness(ctx, tsk, personalization, randEpoch, entropy)
func (c *FullNodeStruct) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
return c.Internal.ChainGetRandomnessFromTickets(ctx, tsk, personalization, randEpoch, entropy)
}
func (c *FullNodeStruct) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
return c.Internal.ChainGetRandomnessFromBeacon(ctx, tsk, personalization, randEpoch, entropy)
}
func (c *FullNodeStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
@ -539,6 +631,10 @@ func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (
return c.Internal.ChainExport(ctx, tsk)
}
func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
return c.Internal.BeaconGetEntry(ctx, epoch)
}
func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
return c.Internal.SyncState(ctx)
}
@ -567,8 +663,8 @@ func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Add
return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk)
}
func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
return c.Internal.StateMinerProvingSet(ctx, addr, tsk)
func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
return c.Internal.StateMinerActiveSectors(ctx, addr, tsk)
}
func (c *FullNodeStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) {
@ -583,11 +679,15 @@ func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Addre
return c.Internal.StateMinerInfo(ctx, actor, tsk)
}
func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) (*miner.Deadlines, error) {
func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) {
return c.Internal.StateMinerDeadlines(ctx, m, tsk)
}
func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
func (c *FullNodeStruct) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]*miner.Partition, error) {
return c.Internal.StateMinerPartitions(ctx, m, dlIdx, tsk)
}
func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (abi.BitField, error) {
return c.Internal.StateMinerFaults(ctx, actor, tsk)
}
@ -595,12 +695,16 @@ func (c *FullNodeStruct) StateAllMinerFaults(ctx context.Context, cutoff abi.Cha
return c.Internal.StateAllMinerFaults(ctx, cutoff, endTsk)
}
func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (abi.BitField, error) {
return c.Internal.StateMinerRecoveries(ctx, actor, tsk)
}
func (c *FullNodeStruct) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, snum abi.SectorNumber, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.StateMinerInitialPledgeCollateral(ctx, maddr, snum, tsk)
func (c *FullNodeStruct) StateMinerPreCommitDepositForPower(ctx context.Context, maddr address.Address, pci miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.StateMinerPreCommitDepositForPower(ctx, maddr, pci, tsk)
}
func (c *FullNodeStruct) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, pci miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.StateMinerInitialPledgeCollateral(ctx, maddr, pci, tsk)
}
func (c *FullNodeStruct) StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tsk types.TipSetKey) (types.BigInt, error) {
@ -615,6 +719,14 @@ func (c *FullNodeStruct) StateSectorGetInfo(ctx context.Context, maddr address.A
return c.Internal.StateSectorGetInfo(ctx, maddr, n, tsk)
}
func (c *FullNodeStruct) StateSectorExpiration(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*api.SectorExpiration, error) {
return c.Internal.StateSectorExpiration(ctx, maddr, n, tsk)
}
func (c *FullNodeStruct) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*api.SectorLocation, error) {
return c.Internal.StateSectorPartition(ctx, maddr, sectorNumber, tok)
}
func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) {
return c.Internal.StateCall(ctx, msg, tsk)
}
@ -631,10 +743,6 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres
return c.Internal.StateReadState(ctx, addr, tsk)
}
func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.StatePledgeCollateral(ctx, tsk)
}
func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) {
return c.Internal.StateWaitMsg(ctx, msgc, confidence)
}
@ -691,11 +799,23 @@ func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch
return c.Internal.StateCompute(ctx, height, msgs, tsk)
}
func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*verifreg.DataCap, error) {
return c.Internal.StateVerifiedClientStatus(ctx, addr, tsk)
}
func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) {
return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk)
}
func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) {
return c.Internal.StateCirculatingSupply(ctx, tsk)
}
func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address.Address, tsk types.TipSetKey) (types.BigInt, error) {
return c.Internal.MsigGetAvailableBalance(ctx, a, tsk)
}
func (c *FullNodeStruct) MsigCreate(ctx context.Context, req int64, addrs []address.Address, duration abi.ChainEpoch, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) {
func (c *FullNodeStruct) MsigCreate(ctx context.Context, req uint64, addrs []address.Address, duration abi.ChainEpoch, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) {
return c.Internal.MsigCreate(ctx, req, addrs, duration, val, src, gp)
}
@ -727,8 +847,12 @@ func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet
return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt)
}
func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) {
return c.Internal.PaychGet(ctx, from, to, ensureFunds)
func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
return c.Internal.PaychGet(ctx, from, to, amt)
}
func (c *FullNodeStruct) PaychGetWaitReady(ctx context.Context, sentinel cid.Cid) (address.Address, error) {
return c.Internal.PaychGetWaitReady(ctx, sentinel)
}
func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, error) {
@ -759,8 +883,12 @@ func (c *FullNodeStruct) PaychVoucherList(ctx context.Context, pch address.Addre
return c.Internal.PaychVoucherList(ctx, pch)
}
func (c *FullNodeStruct) PaychClose(ctx context.Context, a address.Address) (cid.Cid, error) {
return c.Internal.PaychClose(ctx, a)
func (c *FullNodeStruct) PaychSettle(ctx context.Context, a address.Address) (cid.Cid, error) {
return c.Internal.PaychSettle(ctx, a)
}
func (c *FullNodeStruct) PaychCollect(ctx context.Context, a address.Address) (cid.Cid, error) {
return c.Internal.PaychCollect(ctx, a)
}
func (c *FullNodeStruct) PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error) {
@ -794,8 +922,8 @@ func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
}
// Get the status of a given sector by ID
func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber) (api.SectorInfo, error) {
return c.Internal.SectorsStatus(ctx, sid)
func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) {
return c.Internal.SectorsStatus(ctx, sid, showOnChainInfo)
}
// List all staged sectors
@ -807,6 +935,26 @@ func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]api.
return c.Internal.SectorsRefs(ctx)
}
func (c *StorageMinerStruct) SectorStartSealing(ctx context.Context, number abi.SectorNumber) error {
return c.Internal.SectorStartSealing(ctx, number)
}
func (c *StorageMinerStruct) SectorSetSealDelay(ctx context.Context, delay time.Duration) error {
return c.Internal.SectorSetSealDelay(ctx, delay)
}
func (c *StorageMinerStruct) SectorGetSealDelay(ctx context.Context) (time.Duration, error) {
return c.Internal.SectorGetSealDelay(ctx)
}
func (c *StorageMinerStruct) SectorSetExpectedSealDuration(ctx context.Context, delay time.Duration) error {
return c.Internal.SectorSetExpectedSealDuration(ctx, delay)
}
func (c *StorageMinerStruct) SectorGetExpectedSealDuration(ctx context.Context) (time.Duration, error) {
return c.Internal.SectorGetExpectedSealDuration(ctx)
}
func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id abi.SectorNumber, state api.SectorState) error {
return c.Internal.SectorsUpdate(ctx, id, state)
}
@ -815,6 +963,10 @@ func (c *StorageMinerStruct) SectorRemove(ctx context.Context, number abi.Sector
return c.Internal.SectorRemove(ctx, number)
}
func (c *StorageMinerStruct) SectorMarkForUpgrade(ctx context.Context, number abi.SectorNumber) error {
return c.Internal.SectorMarkForUpgrade(ctx, number)
}
func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) error {
return c.Internal.WorkerConnect(ctx, url)
}
@ -823,7 +975,15 @@ func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (map[uint64]storif
return c.Internal.WorkerStats(ctx)
}
func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st stores.FsStat) error {
func (c *StorageMinerStruct) WorkerJobs(ctx context.Context) (map[uint64][]storiface.WorkerJob, error) {
return c.Internal.WorkerJobs(ctx)
}
func (c *StorageMinerStruct) SealingSchedDiag(ctx context.Context) (interface{}, error) {
return c.Internal.SealingSchedDiag(ctx)
}
func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st fsutil.FsStat) error {
return c.Internal.StorageAttach(ctx, si, st)
}
@ -835,8 +995,8 @@ func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId st
return c.Internal.StorageDropSector(ctx, storageId, s, ft)
}
func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, allowFetch bool) ([]stores.SectorStorageInfo, error) {
return c.Internal.StorageFindSector(ctx, si, types, allowFetch)
func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]stores.SectorStorageInfo, error) {
return c.Internal.StorageFindSector(ctx, si, types, spt, allowFetch)
}
func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) {
@ -847,7 +1007,7 @@ func (c *StorageMinerStruct) StorageLocal(ctx context.Context) (map[stores.ID]st
return c.Internal.StorageLocal(ctx)
}
func (c *StorageMinerStruct) StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error) {
func (c *StorageMinerStruct) StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) {
return c.Internal.StorageStat(ctx, id)
}
@ -879,18 +1039,34 @@ func (c *StorageMinerStruct) MarketListDeals(ctx context.Context) ([]storagemark
return c.Internal.MarketListDeals(ctx)
}
func (c *StorageMinerStruct) MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) {
return c.Internal.MarketListRetrievalDeals(ctx)
}
func (c *StorageMinerStruct) MarketGetDealUpdates(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error) {
return c.Internal.MarketGetDealUpdates(ctx, d)
}
func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) {
return c.Internal.MarketListIncompleteDeals(ctx)
}
func (c *StorageMinerStruct) MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error {
return c.Internal.MarketSetAsk(ctx, price, duration, minPieceSize, maxPieceSize)
func (c *StorageMinerStruct) MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error {
return c.Internal.MarketSetAsk(ctx, price, verifiedPrice, duration, minPieceSize, maxPieceSize)
}
func (c *StorageMinerStruct) MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error) {
return c.Internal.MarketGetAsk(ctx)
}
func (c *StorageMinerStruct) MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error {
return c.Internal.MarketSetRetrievalAsk(ctx, rask)
}
func (c *StorageMinerStruct) MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error) {
return c.Internal.MarketGetRetrievalAsk(ctx)
}
func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error {
return c.Internal.DealsImportData(ctx, dealPropCid, file)
}
@ -943,6 +1119,22 @@ func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) e
return c.Internal.StorageAddLocal(ctx, path)
}
func (c *StorageMinerStruct) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
return c.Internal.PiecesListPieces(ctx)
}
func (c *StorageMinerStruct) PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) {
return c.Internal.PiecesListCidInfos(ctx)
}
func (c *StorageMinerStruct) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) {
return c.Internal.PiecesGetPieceInfo(ctx, pieceCid)
}
func (c *StorageMinerStruct) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) {
return c.Internal.PiecesGetCIDInfo(ctx, payloadCid)
}
// WorkerStruct
func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
@ -961,6 +1153,10 @@ func (w *WorkerStruct) Info(ctx context.Context) (storiface.WorkerInfo, error) {
return w.Internal.Info(ctx)
}
func (w *WorkerStruct) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
return w.Internal.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData)
}
func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces)
}
@ -997,7 +1193,7 @@ func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index s
return w.Internal.UnsealPiece(ctx, id, index, size, randomness, c)
}
func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error {
func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) {
return w.Internal.ReadPiece(ctx, writer, id, index, size)
}

98
api/cbor_gen.go
View File

@ -33,7 +33,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Channel"); err != nil {
if _, err := io.WriteString(w, string("Channel")); err != nil {
return err
}
@ -41,26 +41,20 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
return err
}
// t.ChannelMessage (cid.Cid) (struct)
if len("ChannelMessage") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"ChannelMessage\" was too long")
// t.WaitSentinel (cid.Cid) (struct)
if len("WaitSentinel") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelMessage"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
return err
}
if _, err := io.WriteString(w, "ChannelMessage"); err != nil {
if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
return err
}
if t.ChannelMessage == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCidBuf(scratch, w, *t.ChannelMessage); err != nil {
return xerrors.Errorf("failed to write cid field t.ChannelMessage: %w", err)
}
if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
}
// t.Vouchers ([]*paych.SignedVoucher) (slice)
@ -71,7 +65,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Vouchers"); err != nil {
if _, err := io.WriteString(w, string("Vouchers")); err != nil {
return err
}
@ -91,6 +85,8 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
}
func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
*t = PaymentInfo{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -131,29 +127,17 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
}
}
// t.ChannelMessage (cid.Cid) (struct)
case "ChannelMessage":
// t.WaitSentinel (cid.Cid) (struct)
case "WaitSentinel":
{
pb, err := br.PeekByte()
c, err := cbg.ReadCid(br)
if err != nil {
return err
return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err)
}
t.ChannelMessage = &c
}
t.WaitSentinel = c
}
// t.Vouchers ([]*paych.SignedVoucher) (slice)
@ -212,7 +196,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
return err
}
if _, err := io.WriteString(w, "SectorID"); err != nil {
if _, err := io.WriteString(w, string("SectorID")); err != nil {
return err
}
@ -220,7 +204,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
return err
}
// t.Offset (uint64) (uint64)
// t.Offset (abi.PaddedPieceSize) (uint64)
if len("Offset") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Offset\" was too long")
}
@ -228,7 +212,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Offset"); err != nil {
if _, err := io.WriteString(w, string("Offset")); err != nil {
return err
}
@ -244,7 +228,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Size"); err != nil {
if _, err := io.WriteString(w, string("Size")); err != nil {
return err
}
@ -256,6 +240,8 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
}
func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
*t = SealedRef{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -301,7 +287,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
t.SectorID = abi.SectorNumber(extra)
}
// t.Offset (uint64) (uint64)
// t.Offset (abi.PaddedPieceSize) (uint64)
case "Offset":
{
@ -313,7 +299,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Offset = uint64(extra)
t.Offset = abi.PaddedPieceSize(extra)
}
// t.Size (abi.UnpaddedPieceSize) (uint64)
@ -358,7 +344,7 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Refs"); err != nil {
if _, err := io.WriteString(w, string("Refs")); err != nil {
return err
}
@ -378,6 +364,8 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
}
func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
*t = SealedRefs{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -464,7 +452,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Value"); err != nil {
if _, err := io.WriteString(w, string("Value")); err != nil {
return err
}
@ -476,7 +464,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(t.Value); err != nil {
if _, err := w.Write(t.Value[:]); err != nil {
return err
}
@ -488,7 +476,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Epoch"); err != nil {
if _, err := io.WriteString(w, string("Epoch")); err != nil {
return err
}
@ -505,6 +493,8 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
}
func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
*t = SealTicket{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -549,8 +539,12 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Value = make([]byte, extra)
if _, err := io.ReadFull(br, t.Value); err != nil {
if extra > 0 {
t.Value = make([]uint8, extra)
}
if _, err := io.ReadFull(br, t.Value[:]); err != nil {
return err
}
// t.Epoch (abi.ChainEpoch) (int64)
@ -606,7 +600,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Value"); err != nil {
if _, err := io.WriteString(w, string("Value")); err != nil {
return err
}
@ -618,7 +612,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(t.Value); err != nil {
if _, err := w.Write(t.Value[:]); err != nil {
return err
}
@ -630,7 +624,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, "Epoch"); err != nil {
if _, err := io.WriteString(w, string("Epoch")); err != nil {
return err
}
@ -647,6 +641,8 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
}
func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
*t = SealSeed{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -691,8 +687,12 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Value = make([]byte, extra)
if _, err := io.ReadFull(br, t.Value); err != nil {
if extra > 0 {
t.Value = make([]uint8, extra)
}
if _, err := io.ReadFull(br, t.Value[:]); err != nil {
return err
}
// t.Epoch (abi.ChainEpoch) (int64)

26
api/client/client.go
View File

@ -2,11 +2,15 @@ package client
import (
"net/http"
"net/url"
"path"
"time"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apistruct"
"github.com/filecoin-project/lotus/lib/rpcenc"
)
// NewCommonRPC creates a new http jsonrpc client.
@ -34,8 +38,8 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr
return &res, closer, err
}
// NewStorageMinerRPC creates a new http jsonrpc client for storage miner
func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
// NewStorageMinerRPC creates a new http jsonrpc client for miner
func NewStorageMinerRPC(addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
var res apistruct.StorageMinerStruct
closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
[]interface{}{
@ -43,18 +47,36 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMine
&res.Internal,
},
requestHeader,
opts...,
)
return &res, closer, err
}
func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
u, err := url.Parse(addr)
if err != nil {
return nil, nil, err
}
switch u.Scheme {
case "ws":
u.Scheme = "http"
case "wss":
u.Scheme = "https"
}
///rpc/v0 -> /rpc/streams/v0/push
u.Path = path.Join(u.Path, "../streams/v0/push")
var res apistruct.WorkerStruct
closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
[]interface{}{
&res.Internal,
},
requestHeader,
rpcenc.ReaderParamEncoder(u.String()),
jsonrpc.WithNoReconnect(),
jsonrpc.WithTimeout(30*time.Second),
)
return &res, closer, err

99
api/test/ccupgrade.go Normal file
View File

@ -0,0 +1,99 @@
package test
import (
"context"
"fmt"
"os"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl"
)
func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
_ = os.Setenv("BELLMAN_NO_GPU", "1")
ctx := context.Background()
n, sn := b(t, 1, oneMiner)
client := n[0].FullNode.(*impl.FullNodeAPI)
miner := sn[0]
addrinfo, err := client.NetAddrsListen(ctx)
if err != nil {
t.Fatal(err)
}
if err := miner.NetConnect(ctx, addrinfo); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
mine := int64(1)
done := make(chan struct{})
go func() {
defer close(done)
for atomic.LoadInt64(&mine) == 1 {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, MineNext); err != nil {
t.Error(err)
}
}
}()
maddr, err := miner.ActorAddress(ctx)
if err != nil {
t.Fatal(err)
}
CC := abi.SectorNumber(GenesisPreseals + 1)
Upgraded := CC + 1
pledgeSectors(t, ctx, miner, 1, 0, nil)
sl, err := miner.SectorsList(ctx)
if err != nil {
t.Fatal(err)
}
if len(sl) != 1 {
t.Fatal("expected 1 sector")
}
if sl[0] != CC {
t.Fatal("bad")
}
{
si, err := client.StateSectorGetInfo(ctx, maddr, CC, types.EmptyTSK)
require.NoError(t, err)
require.Less(t, 50000, int(si.Expiration))
}
if err := miner.SectorMarkForUpgrade(ctx, sl[0]); err != nil {
t.Fatal(err)
}
makeDeal(t, ctx, 6, client, miner, false, false)
// Validate upgrade
{
exp, err := client.StateSectorExpiration(ctx, maddr, CC, types.EmptyTSK)
require.NoError(t, err)
require.Greater(t, 50000, int(exp.OnTime))
}
{
exp, err := client.StateSectorExpiration(ctx, maddr, Upgraded, types.EmptyTSK)
require.NoError(t, err)
require.Less(t, 50000, int(exp.OnTime))
}
fmt.Println("shutting down mining")
atomic.AddInt64(&mine, -1)
<-done
}

235
api/test/deals.go
View File

@ -12,8 +12,9 @@ import (
"testing"
"time"
"github.com/ipfs/go-cid"
"github.com/stretchr/testify/require"
"github.com/ipfs/go-cid"
files "github.com/ipfs/go-ipfs-files"
logging "github.com/ipfs/go-log/v2"
"github.com/ipld/go-car"
@ -21,6 +22,8 @@ import (
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/miner"
dag "github.com/ipfs/go-merkledag"
dstest "github.com/ipfs/go-merkledag/test"
unixfile "github.com/ipfs/go-unixfs/file"
@ -30,12 +33,17 @@ import (
ipld "github.com/ipfs/go-ipld-format"
)
var MineNext = miner.MineReq{
InjectNulls: 0,
Done: func(bool, error) {},
}
func init() {
logging.SetAllLoggers(logging.LevelInfo)
build.InsecurePoStValidation = true
}
func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport, fastRet bool) {
_ = os.Setenv("BELLMAN_NO_GPU", "1")
ctx := context.Background()
@ -59,13 +67,13 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
defer close(done)
for atomic.LoadInt64(&mine) == 1 {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
if err := sn[0].MineOne(ctx, MineNext); err != nil {
t.Error(err)
}
}
}()
makeDeal(t, ctx, 6, client, miner, carExport)
makeDeal(t, ctx, 6, client, miner, carExport, fastRet)
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
@ -97,21 +105,21 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
defer close(done)
for atomic.LoadInt64(&mine) == 1 {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
if err := sn[0].MineOne(ctx, MineNext); err != nil {
t.Error(err)
}
}
}()
makeDeal(t, ctx, 6, client, miner, false)
makeDeal(t, ctx, 7, client, miner, false)
makeDeal(t, ctx, 6, client, miner, false, false)
makeDeal(t, ctx, 7, client, miner, false, false)
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
<-done
}
func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport bool) {
func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport, fastRet bool) {
data := make([]byte, 1600)
rand.New(rand.NewSource(int64(rseed))).Read(data)
@ -123,18 +131,151 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
fmt.Println("FILE CID: ", fcid)
deal := startDeal(t, ctx, miner, client, fcid)
deal := startDeal(t, ctx, miner, client, fcid, fastRet)
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)
waitDealSealed(t, ctx, client, deal)
waitDealSealed(t, ctx, miner, client, deal, false)
// Retrieval
info, err := client.ClientGetDealInfo(ctx, *deal)
require.NoError(t, err)
testRetrieval(t, ctx, err, client, fcid, carExport, data)
testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, carExport, data)
}
func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid {
func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
_ = os.Setenv("BELLMAN_NO_GPU", "1")
ctx := context.Background()
n, sn := b(t, 1, oneMiner)
client := n[0].FullNode.(*impl.FullNodeAPI)
miner := sn[0]
addrinfo, err := client.NetAddrsListen(ctx)
if err != nil {
t.Fatal(err)
}
if err := miner.NetConnect(ctx, addrinfo); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
mine := int64(1)
done := make(chan struct{})
go func() {
defer close(done)
for atomic.LoadInt64(&mine) == 1 {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, MineNext); err != nil {
t.Error(err)
}
}
}()
data := make([]byte, 1600)
rand.New(rand.NewSource(int64(8))).Read(data)
r := bytes.NewReader(data)
fcid, err := client.ClientImportLocal(ctx, r)
if err != nil {
t.Fatal(err)
}
fmt.Println("FILE CID: ", fcid)
deal := startDeal(t, ctx, miner, client, fcid, true)
waitDealPublished(t, ctx, miner, deal)
fmt.Println("deal published, retrieving")
// Retrieval
info, err := client.ClientGetDealInfo(ctx, *deal)
require.NoError(t, err)
testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, false, data)
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
<-done
}
func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration) {
_ = os.Setenv("BELLMAN_NO_GPU", "1")
ctx := context.Background()
n, sn := b(t, 1, oneMiner)
client := n[0].FullNode.(*impl.FullNodeAPI)
miner := sn[0]
addrinfo, err := client.NetAddrsListen(ctx)
if err != nil {
t.Fatal(err)
}
if err := miner.NetConnect(ctx, addrinfo); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
mine := int64(1)
done := make(chan struct{})
go func() {
defer close(done)
for atomic.LoadInt64(&mine) == 1 {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, MineNext); err != nil {
t.Error(err)
}
}
}()
{
data1 := make([]byte, 800)
rand.New(rand.NewSource(int64(3))).Read(data1)
r := bytes.NewReader(data1)
fcid1, err := client.ClientImportLocal(ctx, r)
if err != nil {
t.Fatal(err)
}
data2 := make([]byte, 800)
rand.New(rand.NewSource(int64(9))).Read(data2)
r2 := bytes.NewReader(data2)
fcid2, err := client.ClientImportLocal(ctx, r2)
if err != nil {
t.Fatal(err)
}
deal1 := startDeal(t, ctx, miner, client, fcid1, true)
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)
waitDealSealed(t, ctx, miner, client, deal1, true)
deal2 := startDeal(t, ctx, miner, client, fcid2, true)
time.Sleep(time.Second)
waitDealSealed(t, ctx, miner, client, deal2, false)
// Retrieval
info, err := client.ClientGetDealInfo(ctx, *deal2)
require.NoError(t, err)
rf, _ := miner.SectorsRefs(ctx)
fmt.Printf("refs: %+v\n", rf)
testRetrieval(t, ctx, err, client, fcid2, &info.PieceCID, false, data2)
}
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
<-done
}
func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid, fastRet bool) *cid.Cid {
maddr, err := miner.ActorAddress(ctx)
if err != nil {
t.Fatal(err)
@ -145,11 +286,15 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
t.Fatal(err)
}
deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{
Data: &storagemarket.DataRef{Root: fcid},
Data: &storagemarket.DataRef{
TransferType: storagemarket.TTGraphsync,
Root: fcid,
},
Wallet: addr,
Miner: maddr,
EpochPrice: types.NewInt(1000000),
MinBlocksDuration: 100,
MinBlocksDuration: uint64(build.MinDealDuration),
FastRetrieval: fastRet,
})
if err != nil {
t.Fatalf("%+v", err)
@ -157,7 +302,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
return deal
}
func waitDealSealed(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, deal *cid.Cid) {
func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, deal *cid.Cid, noseal bool) {
loop:
for {
di, err := client.ClientGetDealInfo(ctx, *deal)
@ -165,6 +310,11 @@ loop:
t.Fatal(err)
}
switch di.State {
case storagemarket.StorageDealSealing:
if noseal {
return
}
startSealingWaiting(t, ctx, miner)
case storagemarket.StorageDealProposalRejected:
t.Fatal("deal rejected")
case storagemarket.StorageDealFailing:
@ -180,8 +330,51 @@ loop:
}
}
func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.FullNodeAPI, fcid cid.Cid, carExport bool, data []byte) {
offers, err := client.ClientFindData(ctx, fcid)
func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, deal *cid.Cid) {
subCtx, cancel := context.WithCancel(ctx)
defer cancel()
updates, err := miner.MarketGetDealUpdates(subCtx, *deal)
if err != nil {
t.Fatal(err)
}
for {
select {
case <-ctx.Done():
t.Fatal("context timeout")
case di := <-updates:
switch di.State {
case storagemarket.StorageDealProposalRejected:
t.Fatal("deal rejected")
case storagemarket.StorageDealFailing:
t.Fatal("deal failed")
case storagemarket.StorageDealError:
t.Fatal("deal errored", di.Message)
case storagemarket.StorageDealFinalizing, storagemarket.StorageDealSealing, storagemarket.StorageDealActive:
fmt.Println("COMPLETE", di)
return
}
fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
}
}
}
func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNode) {
snums, err := miner.SectorsList(ctx)
require.NoError(t, err)
for _, snum := range snums {
si, err := miner.SectorsStatus(ctx, snum, false)
require.NoError(t, err)
t.Logf("Sector state: %s", si.State)
if si.State == api.SectorState(sealing.WaitDeals) {
require.NoError(t, miner.SectorStartSealing(ctx, snum))
}
}
}
func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.FullNodeAPI, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) {
offers, err := client.ClientFindData(ctx, fcid, piece)
if err != nil {
t.Fatal(err)
}
@ -205,9 +398,11 @@ func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.Fu
Path: filepath.Join(rpath, "ret"),
IsCAR: carExport,
}
err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
if err != nil {
t.Fatalf("%+v", err)
updates, err := client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref)
for update := range updates {
if update.Err != "" {
t.Fatalf("%v", err)
}
}
rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret"))

118
api/test/mining.go
View File

@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/impl"
)
@ -26,22 +27,25 @@ func (ts *testSuite) testMining(t *testing.T) {
apis, sn := ts.makeNodes(t, 1, oneMiner)
api := apis[0]
h1, err := api.ChainHead(ctx)
require.NoError(t, err)
require.Equal(t, abi.ChainEpoch(0), h1.Height())
newHeads, err := api.ChainNotify(ctx)
require.NoError(t, err)
<-newHeads
initHead := (<-newHeads)[0]
if initHead.Val.Height() != 2 {
<-newHeads
}
err = sn[0].MineOne(ctx, func(bool, error) {})
h1, err := api.ChainHead(ctx)
require.NoError(t, err)
require.Equal(t, abi.ChainEpoch(2), h1.Height())
err = sn[0].MineOne(ctx, MineNext)
require.NoError(t, err)
<-newHeads
h2, err := api.ChainHead(ctx)
require.NoError(t, err)
require.Equal(t, abi.ChainEpoch(1), h2.Height())
require.Equal(t, abi.ChainEpoch(3), h2.Height())
}
func (ts *testSuite) testMiningReal(t *testing.T) {
@ -54,31 +58,34 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
apis, sn := ts.makeNodes(t, 1, oneMiner)
api := apis[0]
h1, err := api.ChainHead(ctx)
require.NoError(t, err)
require.Equal(t, abi.ChainEpoch(0), h1.Height())
newHeads, err := api.ChainNotify(ctx)
require.NoError(t, err)
<-newHeads
initHead := (<-newHeads)[0]
if initHead.Val.Height() != 2 {
<-newHeads
}
err = sn[0].MineOne(ctx, func(bool, error) {})
h1, err := api.ChainHead(ctx)
require.NoError(t, err)
require.Equal(t, abi.ChainEpoch(2), h1.Height())
err = sn[0].MineOne(ctx, MineNext)
require.NoError(t, err)
<-newHeads
h2, err := api.ChainHead(ctx)
require.NoError(t, err)
require.Equal(t, abi.ChainEpoch(1), h2.Height())
require.Equal(t, abi.ChainEpoch(3), h2.Height())
err = sn[0].MineOne(ctx, func(bool, error) {})
err = sn[0].MineOne(ctx, MineNext)
require.NoError(t, err)
<-newHeads
h2, err = api.ChainHead(ctx)
require.NoError(t, err)
require.Equal(t, abi.ChainEpoch(2), h2.Height())
require.Equal(t, abi.ChainEpoch(4), h2.Height())
}
func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
@ -89,7 +96,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
ctx := context.Background()
n, sn := b(t, 1, []StorageMiner{
{Full: 0, Preseal: PresealGenesis},
{Full: 0, Preseal: 0}, // TODO: Add support for storage miners on non-first full node
{Full: 0, Preseal: 0}, // TODO: Add support for miners on non-first full node
})
client := n[0].FullNode.(*impl.FullNodeAPI)
provider := sn[1]
@ -125,28 +132,30 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
done := make(chan struct{})
minedTwo := make(chan struct{})
m2addr, err := sn[1].ActorAddress(context.TODO())
if err != nil {
t.Fatal(err)
}
go func() {
doneMinedTwo := false
defer close(done)
prevExpect := 0
complChan := minedTwo
for atomic.LoadInt32(&mine) != 0 {
wait := make(chan int, 2)
wait := make(chan int)
mdone := func(mined bool, err error) {
go func() {
n := 0
if mined {
n = 1
}
wait <- n
}()
n := 0
if mined {
n = 1
}
wait <- n
}
if err := sn[0].MineOne(ctx, mdone); err != nil {
if err := sn[0].MineOne(ctx, miner.MineReq{Done: mdone}); err != nil {
t.Error(err)
}
if err := sn[1].MineOne(ctx, mdone); err != nil {
if err := sn[1].MineOne(ctx, miner.MineReq{Done: mdone}); err != nil {
t.Error(err)
}
@ -154,43 +163,42 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
expect += <-wait
time.Sleep(blocktime)
for {
n := 0
for i, node := range sn {
mb, err := node.MiningBase(ctx)
if err != nil {
t.Error(err)
return
}
if len(mb.Cids()) != expect {
log.Warnf("node %d mining base not complete (%d, want %d)", i, len(mb.Cids()), expect)
continue
}
n++
}
if n == len(sn) {
break
}
time.Sleep(blocktime)
if expect == 0 {
// null block
continue
}
if prevExpect == 2 && expect == 2 && !doneMinedTwo {
close(minedTwo)
doneMinedTwo = true
var nodeOneMined bool
for _, node := range sn {
mb, err := node.MiningBase(ctx)
if err != nil {
t.Error(err)
return
}
for _, b := range mb.Blocks() {
if b.Miner == m2addr {
nodeOneMined = true
break
}
}
}
if nodeOneMined && complChan != nil {
close(complChan)
complChan = nil
}
prevExpect = expect
}
}()
deal := startDeal(t, ctx, provider, client, fcid)
deal := startDeal(t, ctx, provider, client, fcid, false)
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)
waitDealSealed(t, ctx, client, deal)
waitDealSealed(t, ctx, provider, client, deal, false)
<-minedTwo

321
api/test/paych.go Normal file
View File

@ -0,0 +1,321 @@
package test
import (
"context"
"fmt"
"os"
"sync/atomic"
"testing"
"time"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin/paych"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/events/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/miner"
)
func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
_ = os.Setenv("BELLMAN_NO_GPU", "1")
ctx := context.Background()
n, sn := b(t, 2, oneMiner)
paymentCreator := n[0]
paymentReceiver := n[1]
miner := sn[0]
// get everyone connected
addrs, err := paymentCreator.NetAddrsListen(ctx)
if err != nil {
t.Fatal(err)
}
if err := paymentReceiver.NetConnect(ctx, addrs); err != nil {
t.Fatal(err)
}
if err := miner.NetConnect(ctx, addrs); err != nil {
t.Fatal(err)
}
// start mining blocks
bm := newBlockMiner(ctx, t, miner, blocktime)
bm.mineBlocks()
// send some funds to register the receiver
receiverAddr, err := paymentReceiver.WalletNew(ctx, wallet.ActSigType("secp256k1"))
if err != nil {
t.Fatal(err)
}
sendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18))
// setup the payment channel
createrAddr, err := paymentCreator.WalletDefaultAddress(ctx)
if err != nil {
t.Fatal(err)
}
channelAmt := int64(100000)
channelInfo, err := paymentCreator.PaychGet(ctx, createrAddr, receiverAddr, abi.NewTokenAmount(channelAmt))
if err != nil {
t.Fatal(err)
}
channel, err := paymentCreator.PaychGetWaitReady(ctx, channelInfo.WaitSentinel)
if err != nil {
t.Fatal(err)
}
// allocate three lanes
var lanes []uint64
for i := 0; i < 3; i++ {
lane, err := paymentCreator.PaychAllocateLane(ctx, channel)
if err != nil {
t.Fatal(err)
}
lanes = append(lanes, lane)
}
// Make two vouchers each for each lane, then save on the other side
// Note that the voucher with a value of 2000 has a higher nonce, so it
// supersedes the voucher with a value of 1000
for _, lane := range lanes {
vouch1, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(1000), lane)
if err != nil {
t.Fatal(err)
}
vouch2, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(2000), lane)
if err != nil {
t.Fatal(err)
}
delta1, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch1, nil, abi.NewTokenAmount(1000))
if err != nil {
t.Fatal(err)
}
if !delta1.Equals(abi.NewTokenAmount(1000)) {
t.Fatal("voucher didn't have the right amount")
}
delta2, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch2, nil, abi.NewTokenAmount(1000))
if err != nil {
t.Fatal(err)
}
if !delta2.Equals(abi.NewTokenAmount(1000)) {
t.Fatal("voucher didn't have the right amount")
}
}
// settle the payment channel
settleMsgCid, err := paymentCreator.PaychSettle(ctx, channel)
if err != nil {
t.Fatal(err)
}
res := waitForMessage(ctx, t, paymentCreator, settleMsgCid, time.Second*10, "settle")
if res.Receipt.ExitCode != 0 {
t.Fatal("Unable to settle payment channel")
}
// wait for the receiver to submit their vouchers
ev := events.NewEvents(ctx, paymentCreator)
preds := state.NewStatePredicates(paymentCreator)
finished := make(chan struct{})
err = ev.StateChanged(func(ts *types.TipSet) (done bool, more bool, err error) {
act, err := paymentCreator.StateReadState(ctx, channel, ts.Key())
if err != nil {
return false, false, err
}
state := act.State.(paych.State)
if state.ToSend.GreaterThanEqual(abi.NewTokenAmount(6000)) {
return true, false, nil
}
return false, true, nil
}, func(oldTs, newTs *types.TipSet, states events.StateChange, curH abi.ChainEpoch) (more bool, err error) {
toSendChange := states.(*state.PayChToSendChange)
if toSendChange.NewToSend.GreaterThanEqual(abi.NewTokenAmount(6000)) {
close(finished)
return false, nil
}
return true, nil
}, func(ctx context.Context, ts *types.TipSet) error {
return nil
}, int(build.MessageConfidence)+1, build.SealRandomnessLookbackLimit, func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
return preds.OnPaymentChannelActorChanged(channel, preds.OnToSendAmountChanges())(ctx, oldTs.Key(), newTs.Key())
})
select {
case <-finished:
case <-time.After(time.Second):
t.Fatal("Timed out waiting for receiver to submit vouchers")
}
// wait for the settlement period to pass before collecting
waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych.SettleDelay)
creatorPreCollectBalance, err := paymentCreator.WalletBalance(ctx, createrAddr)
if err != nil {
t.Fatal(err)
}
// collect funds (from receiver, though either party can do it)
collectMsg, err := paymentReceiver.PaychCollect(ctx, channel)
if err != nil {
t.Fatal(err)
}
res, err = paymentReceiver.StateWaitMsg(ctx, collectMsg, 3)
if err != nil {
t.Fatal(err)
}
if res.Receipt.ExitCode != 0 {
t.Fatal("unable to collect on payment channel")
}
// Finally, check the balance for the creator
currentCreatorBalance, err := paymentCreator.WalletBalance(ctx, createrAddr)
if err != nil {
t.Fatal(err)
}
// The highest nonce voucher that the creator sent on each lane is 2000
totalVouchers := int64(len(lanes) * 2000)
// When receiver submits the tokens to the chain, creator should get a
// refund on the remaining balance, which is
// channel amount - total voucher value
expectedRefund := channelAmt - totalVouchers
delta := big.Sub(currentCreatorBalance, creatorPreCollectBalance)
if !delta.Equals(abi.NewTokenAmount(expectedRefund)) {
t.Fatalf("did not send correct funds from creator: expected %d, got %d", expectedRefund, delta)
}
// shut down mining
bm.stop()
}
func waitForBlocks(ctx context.Context, t *testing.T, bm *blockMiner, paymentReceiver TestNode, receiverAddr address.Address, count int) {
// We need to add null blocks in batches, if we add too many the chain can't sync
batchSize := 60
for i := 0; i < count; i += batchSize {
size := batchSize
if i > count {
size = count - i
}
// Add a batch of null blocks
atomic.StoreInt64(&bm.nulls, int64(size-1))
// Add a real block
m, err := paymentReceiver.MpoolPushMessage(ctx, &types.Message{
To: builtin.BurntFundsActorAddr,
From: receiverAddr,
Value: types.NewInt(0),
}, nil)
if err != nil {
t.Fatal(err)
}
_, err = paymentReceiver.StateWaitMsg(ctx, m.Cid(), 1)
if err != nil {
t.Fatal(err)
}
}
}
func waitForMessage(ctx context.Context, t *testing.T, paymentCreator TestNode, msgCid cid.Cid, duration time.Duration, desc string) *api.MsgLookup {
ctx, cancel := context.WithTimeout(ctx, duration)
defer cancel()
fmt.Println("Waiting for", desc)
res, err := paymentCreator.StateWaitMsg(ctx, msgCid, 1)
if err != nil {
fmt.Println("Error waiting for", desc, err)
t.Fatal(err)
}
if res.Receipt.ExitCode != 0 {
t.Fatalf("did not successfully send %s", desc)
}
fmt.Println("Confirmed", desc)
return res
}
type blockMiner struct {
ctx context.Context
t *testing.T
miner TestStorageNode
blocktime time.Duration
mine int64
nulls int64
done chan struct{}
}
func newBlockMiner(ctx context.Context, t *testing.T, miner TestStorageNode, blocktime time.Duration) *blockMiner {
return &blockMiner{
ctx: ctx,
t: t,
miner: miner,
blocktime: blocktime,
mine: int64(1),
done: make(chan struct{}),
}
}
func (bm *blockMiner) mineBlocks() {
time.Sleep(time.Second)
go func() {
defer close(bm.done)
for atomic.LoadInt64(&bm.mine) == 1 {
time.Sleep(bm.blocktime)
nulls := atomic.SwapInt64(&bm.nulls, 0)
if err := bm.miner.MineOne(bm.ctx, miner.MineReq{
InjectNulls: abi.ChainEpoch(nulls),
Done: func(bool, error) {},
}); err != nil {
bm.t.Error(err)
}
}
}()
}
func (bm *blockMiner) stop() {
atomic.AddInt64(&bm.mine, -1)
fmt.Println("shutting down mining")
<-bm.done
}
func sendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) {
senderAddr, err := sender.WalletDefaultAddress(ctx)
if err != nil {
t.Fatal(err)
}
msg := &types.Message{
From: senderAddr,
To: addr,
Value: amount,
}
sm, err := sender.MpoolPushMessage(ctx, msg, nil)
if err != nil {
t.Fatal(err)
}
res, err := sender.StateWaitMsg(ctx, sm.Cid(), 1)
if err != nil {
t.Fatal(err)
}
if res.Receipt.ExitCode != 0 {
t.Fatal("did not successfully send money")
}
}

3
api/test/test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/miner"
)
type TestNode struct {
@ -18,7 +19,7 @@ type TestNode struct {
type TestStorageNode struct {
api.StorageMiner
MineOne func(context.Context, func(bool, error)) error
MineOne func(context.Context, miner.MineReq) error
}
var PresealGenesis = -1

181
api/test/window_post.go
View File

@ -3,7 +3,7 @@ package test
import (
"context"
"fmt"
"github.com/filecoin-project/lotus/api"
"os"
"strings"
"testing"
@ -11,11 +11,16 @@ import (
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/extern/sector-storage/mock"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/specs-actors/actors/abi"
miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
sealing "github.com/filecoin-project/storage-fsm"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
bminer "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/impl"
)
@ -35,41 +40,46 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect
if err := miner.NetConnect(ctx, addrinfo); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
build.Clock.Sleep(time.Second)
mine := true
done := make(chan struct{})
go func() {
defer close(done)
for mine {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
build.Clock.Sleep(blocktime)
if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, error) {
}}); err != nil {
t.Error(err)
}
}
}()
pledgeSectors(t, ctx, miner, nSectors)
pledgeSectors(t, ctx, miner, nSectors, 0, nil)
mine = false
<-done
}
func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n int) {
func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n, existing int, blockNotif <-chan struct{}) {
for i := 0; i < n; i++ {
err := miner.PledgeSector(ctx)
require.NoError(t, err)
if i%3 == 0 && blockNotif != nil {
<-blockNotif
}
}
for {
s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
require.NoError(t, err)
fmt.Printf("Sectors: %d\n", len(s))
if len(s) >= n {
if len(s) >= n+existing {
break
}
time.Sleep(100 * time.Millisecond)
build.Clock.Sleep(100 * time.Millisecond)
}
fmt.Printf("All sectors is fsm\n")
@ -84,7 +94,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n i
for len(toCheck) > 0 {
for n := range toCheck {
st, err := miner.SectorsStatus(ctx, n)
st, err := miner.SectorsStatus(ctx, n, false)
require.NoError(t, err)
if st.State == api.SectorState(sealing.Proving) {
delete(toCheck, n)
@ -94,7 +104,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n i
}
}
time.Sleep(100 * time.Millisecond)
build.Clock.Sleep(100 * time.Millisecond)
fmt.Printf("WaitSeal: %d\n", len(s))
}
}
@ -115,21 +125,21 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
if err := miner.NetConnect(ctx, addrinfo); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
build.Clock.Sleep(time.Second)
mine := true
done := make(chan struct{})
go func() {
defer close(done)
for mine {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
build.Clock.Sleep(blocktime)
if err := sn[0].MineOne(ctx, MineNext); err != nil {
t.Error(err)
}
}
}()
pledgeSectors(t, ctx, miner, nSectors)
pledgeSectors(t, ctx, miner, nSectors, 0, nil)
maddr, err := miner.ActorAddress(ctx)
require.NoError(t, err)
@ -137,7 +147,10 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
fmt.Printf("Running one proving periods\n")
mid, err := address.IDFromAddress(maddr)
require.NoError(t, err)
fmt.Printf("Running one proving period\n")
for {
head, err := client.ChainHead(ctx)
@ -150,7 +163,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
if head.Height()%100 == 0 {
fmt.Printf("@%d\n", head.Height())
}
time.Sleep(blocktime)
build.Clock.Sleep(blocktime)
}
p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK)
@ -162,7 +175,139 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
require.Equal(t, p.MinerPower, p.TotalPower)
require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals)))
// TODO: Inject faults here
fmt.Printf("Drop some sectors\n")
// Drop 2 sectors from deadline 2 partition 0 (full partition / deadline)
{
parts, err := client.StateMinerPartitions(ctx, maddr, 2, types.EmptyTSK)
require.NoError(t, err)
require.Greater(t, len(parts), 0)
n, err := parts[0].Sectors.Count()
require.NoError(t, err)
require.Equal(t, uint64(2), n)
// Drop the partition
err = parts[0].Sectors.ForEach(func(sid uint64) error {
return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(abi.SectorID{
Miner: abi.ActorID(mid),
Number: abi.SectorNumber(sid),
}, true)
})
require.NoError(t, err)
}
var s abi.SectorID
// Drop 1 sectors from deadline 3 partition 0
{
parts, err := client.StateMinerPartitions(ctx, maddr, 3, types.EmptyTSK)
require.NoError(t, err)
require.Greater(t, len(parts), 0)
n, err := parts[0].Sectors.Count()
require.NoError(t, err)
require.Equal(t, uint64(2), n)
// Drop the sector
sn, err := parts[0].Sectors.First()
require.NoError(t, err)
s = abi.SectorID{
Miner: abi.ActorID(mid),
Number: abi.SectorNumber(sn),
}
err = miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(s, true)
require.NoError(t, err)
}
di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
fmt.Printf("Go through another PP, wait for sectors to become faulty\n")
for {
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 {
break
}
if head.Height()%100 == 0 {
fmt.Printf("@%d\n", head.Height())
}
build.Clock.Sleep(blocktime)
}
p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
require.Equal(t, p.MinerPower, p.TotalPower)
sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
require.Equal(t, nSectors+GenesisPreseals-3, int(sectors)) // -3 just removed sectors
fmt.Printf("Recover one sector\n")
err = miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(s, false)
require.NoError(t, err)
di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
for {
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 {
break
}
if head.Height()%100 == 0 {
fmt.Printf("@%d\n", head.Height())
}
build.Clock.Sleep(blocktime)
}
p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
require.Equal(t, p.MinerPower, p.TotalPower)
sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
require.Equal(t, nSectors+GenesisPreseals-2, int(sectors)) // -2 not recovered sectors
// pledge a sector after recovery
pledgeSectors(t, ctx, miner, 1, nSectors, nil)
{
// wait a bit more
head, err := client.ChainHead(ctx)
require.NoError(t, err)
waitUntil := head.Height() + 10
for {
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > waitUntil {
break
}
}
}
p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
require.Equal(t, p.MinerPower, p.TotalPower)
sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
require.Equal(t, nSectors+GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged
mine = false
<-done

28
api/types.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/libp2p/go-libp2p-core/peer"
@ -49,20 +50,25 @@ type MinerInfo struct {
Worker address.Address // Must be an ID-address.
NewWorker address.Address // Must be an ID-address.
WorkerChangeEpoch abi.ChainEpoch
PeerId peer.ID
PeerId *peer.ID
Multiaddrs []abi.Multiaddrs
SealProofType abi.RegisteredSealProof
SectorSize abi.SectorSize
WindowPoStPartitionSectors uint64
}
func NewApiMinerInfo(info miner.MinerInfo) MinerInfo {
func NewApiMinerInfo(info *miner.MinerInfo) MinerInfo {
var pid *peer.ID
if peerID, err := peer.IDFromBytes(info.PeerId); err == nil {
pid = &peerID
}
mi := MinerInfo{
Owner: info.Owner,
Worker: info.Worker,
NewWorker: address.Undef,
WorkerChangeEpoch: -1,
PeerId: peer.ID(info.PeerId),
PeerId: pid,
Multiaddrs: info.Multiaddrs,
SealProofType: info.SealProofType,
SectorSize: info.SectorSize,
@ -76,3 +82,19 @@ func NewApiMinerInfo(info miner.MinerInfo) MinerInfo {
return mi
}
type MessageSendSpec struct {
MaxFee abi.TokenAmount
}
var DefaultMessageSendSpec = MessageSendSpec{
MaxFee: big.Zero(),
}
func (ms *MessageSendSpec) Get() MessageSendSpec {
if ms == nil {
return DefaultMessageSendSpec
}
return *ms
}

9
build/bootstrap.go
View File

@ -38,12 +38,3 @@ func BuiltinBootstrap() ([]peer.AddrInfo, error) {
})
return out, err
}
func DrandBootstrap() ([]peer.AddrInfo, error) {
addrs := []string{
"/dnsaddr/pl-eu.testnet.drand.sh/",
"/dnsaddr/pl-us.testnet.drand.sh/",
"/dnsaddr/pl-sin.testnet.drand.sh/",
}
return addrutil.ParseAddresses(context.TODO(), addrs)
}

10
build/clock.go Normal file
View File

@ -0,0 +1,10 @@
package build
import "github.com/raulk/clock"
// Clock is the global clock for the system. In standard builds,
// we use a real-time clock, which maps to the `time` package.
//
// Tests that need control of time can replace this variable with
// clock.NewMock(). Always use real time for socket/stream deadlines.
var Clock = clock.New()

58
build/drand.go Normal file
View File

@ -0,0 +1,58 @@
package build
import "github.com/filecoin-project/lotus/node/modules/dtypes"
var DrandNetwork = DrandMainnet
func DrandConfig() dtypes.DrandConfig {
return DrandConfigs[DrandNetwork]
}
type DrandEnum int
const (
DrandMainnet DrandEnum = iota + 1
DrandTestnet
DrandDevnet
DrandLocalnet
)
var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{
DrandMainnet: {
Servers: []string{
"https://api.drand.sh",
"https://api2.drand.sh",
"https://api3.drand.sh",
},
Relays: []string{
"/dnsaddr/api.drand.sh/",
"/dnsaddr/api2.drand.sh/",
"/dnsaddr/api3.drand.sh/",
},
ChainInfoJSON: `{"public_key":"868f005eb8e6e4ca0a47c8a77ceaa5309a47978a7c71bc5cce96366b5d7a569937c529eeda66c7293784a9402801af31","period":30,"genesis_time":1595431050,"hash":"8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce","groupHash":"176f93498eac9ca337150b46d21dd58673ea4e3581185f869672e59fa4cb390a"}`,
},
DrandTestnet: {
Servers: []string{
"https://pl-eu.testnet.drand.sh",
"https://pl-us.testnet.drand.sh",
"https://pl-sin.testnet.drand.sh",
},
Relays: []string{
"/dnsaddr/pl-eu.testnet.drand.sh/",
"/dnsaddr/pl-us.testnet.drand.sh/",
"/dnsaddr/pl-sin.testnet.drand.sh/",
},
ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"84b2234fb34e835dccd048255d7ad3194b81af7d978c3bf157e3469592ae4e02","groupHash":"4dd408e5fdff9323c76a9b6f087ba8fdc5a6da907bd9217d9d10f2287d081957"}`,
},
DrandDevnet: {
Servers: []string{
"https://dev1.drand.sh",
"https://dev2.drand.sh",
},
Relays: []string{
"/dnsaddr/dev1.drand.sh/",
"/dnsaddr/dev2.drand.sh/",
},
ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`,
},
}

4
build/params_2k.go
View File

@ -20,9 +20,9 @@ func init() {
BuildType |= Build2k
}
const BlockDelaySecs = uint64(2)
const BlockDelaySecs = uint64(4)
const PropagationDelaySecs = uint64(3)
const PropagationDelaySecs = uint64(1)
// SlashablePowerDelay is the number of epochs after ElectionPeriodStart, after
// which the miner is slashed

32
build/params_shared_vals.go
View File

@ -8,8 +8,6 @@ import (
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
// /////
@ -30,7 +28,7 @@ const ForkLengthThreshold = Finality
var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch)
// Epochs
const Finality = miner.ChainFinalityish
const Finality = miner.ChainFinality
const MessageConfidence = uint64(5)
// constants for Weight calculation
@ -61,8 +59,8 @@ const WinningPoStSectorSetLookback = abi.ChainEpoch(10)
// /////
// Devnet settings
const TotalFilecoin = uint64(2_000_000_000)
const MiningRewardTotal = uint64(1_400_000_000)
const FilBase = uint64(2_000_000_000)
const FilAllocStorageMining = uint64(1_100_000_000)
const FilecoinPrecision = uint64(1_000_000_000_000_000_000)
@ -71,7 +69,7 @@ var InitialRewardBalance *big.Int
// TODO: Move other important consts here
func init() {
InitialRewardBalance = big.NewInt(int64(MiningRewardTotal))
InitialRewardBalance = big.NewInt(int64(FilAllocStorageMining))
InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision)))
}
@ -90,14 +88,16 @@ const VerifSigCacheSize = 32000
// Limits
// TODO: If this is gonna stay, it should move to specs-actors
const BlockMessageLimit = 512
const BlockGasLimit = 100_000_000_000
const BlockMessageLimit = 10000
var DrandConfig = dtypes.DrandConfig{
Servers: []string{
"https://pl-eu.testnet.drand.sh",
"https://pl-us.testnet.drand.sh",
"https://pl-sin.testnet.drand.sh",
},
ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}`,
}
const BlockGasLimit = 10_000_000_000
const BlockGasTarget = BlockGasLimit / 2
const BaseFeeMaxChangeDenom = 8 // 12.5%
const InitialBaseFee = 100e6
const MinimumBaseFee = 100
const PackingEfficiencyNum = 4
const PackingEfficiencyDenom = 5
// Actor consts
// TODO: Pull from actors when its made not private
var MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay)

37
build/params_testground.go
View File

@ -10,8 +10,6 @@ package build
import (
"math/big"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
@ -21,15 +19,19 @@ var (
UnixfsChunkSize = uint64(1 << 20)
UnixfsLinksPerLevel = 1024
BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch)
BlockMessageLimit = 512
BlockGasLimit = int64(100_000_000_000)
BlockDelaySecs = uint64(builtin.EpochDurationSeconds)
PropagationDelaySecs = uint64(6)
BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch)
BlockMessageLimit = 512
BlockGasLimit = int64(100_000_000_000)
BlockGasTarget = int64(BlockGasLimit / 2)
BaseFeeMaxChangeDenom = int64(8) // 12.5%
InitialBaseFee = int64(100e6)
MinimumBaseFee = int64(100)
BlockDelaySecs = uint64(builtin.EpochDurationSeconds)
PropagationDelaySecs = uint64(6)
AllowableClockDriftSecs = uint64(1)
Finality = miner.ChainFinalityish
Finality = miner.ChainFinality
ForkLengthThreshold = Finality
SlashablePowerDelay = 20
@ -51,23 +53,20 @@ var (
TicketRandomnessLookback = abi.ChainEpoch(1)
WinningPoStSectorSetLookback = abi.ChainEpoch(10)
TotalFilecoin uint64 = 2_000_000_000
MiningRewardTotal uint64 = 1_400_000_000
FilBase uint64 = 2_000_000_000
FilAllocStorageMining uint64 = 1_400_000_000
FilecoinPrecision uint64 = 1_000_000_000_000_000_000
InitialRewardBalance = func() *big.Int {
v := big.NewInt(int64(MiningRewardTotal))
v := big.NewInt(int64(FilAllocStorageMining))
v = v.Mul(v, big.NewInt(int64(FilecoinPrecision)))
return v
}()
// Actor consts
// TODO: Pull from actors when its made not private
MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay)
DrandConfig = dtypes.DrandConfig{
Servers: []string{
"https://pl-eu.testnet.drand.sh",
"https://pl-us.testnet.drand.sh",
"https://pl-sin.testnet.drand.sh",
},
ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}`,
}
PackingEfficiencyNum int64 = 4
PackingEfficiencyDenom int64 = 5
)

182
build/proof-params/parameters.json
View File

@ -1,152 +1,152 @@
{
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
"cid": "QmeDRyxek34F1H6xJY6AkFdWvPsy5F6dKTrebV3ZtWT4ky",
"digest": "f5827f2d8801c62c831e0f972f6dc8bb",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
"cid": "QmVxjFRyhmyQaZEtCh7nk2abc7LhFkzhnRX4rcHqCCpikR",
"digest": "7610b9f82bfc88405b7a832b651ce2f6",
"sector_size": 2048
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": {
"cid": "QmUw1ZmG4BBbX19MsbH3zAEGKUc42iFJc5ZAyomDHeJTsA",
"digest": "398fecdb4b2de445125852bc3c080b35",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": {
"cid": "QmcS5JZs8X3TdtkEBpHAdUYjdNDqcL7fWQFtQz69mpnu2X",
"digest": "0e0958009936b9d5e515ec97b8cb792d",
"sector_size": 2048
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": {
"cid": "QmUeNKp9YZpiAFm81RV5KuxH1FDGJx2DuwcbU2XNSZLLSv",
"digest": "2b6d2972ac9e862e8134d98fb695b0c5",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": {
"cid": "QmUiRx71uxfmUE8V3H9sWAsAXoM88KR4eo1ByvvcFNeTLR",
"digest": "1a7d4a9c8a502a497ed92a54366af33f",
"sector_size": 536870912
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": {
"cid": "QmQaQmTXX995Akd66ggtJY5bNx6Gkxk8P34JTdMMq8393G",
"digest": "3688c9eb256b7b17f411dad78d5ef74a",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": {
"cid": "QmfCeddjFpWtavzfEzZpJfzSajGNwfL4RjFXWAvA9TSnTV",
"digest": "4dae975de4f011f101f5a2f86d1daaba",
"sector_size": 536870912
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": {
"cid": "QmfEYTMSkwGJTumQx26iKXGNKiYh3mmAC4SkdybZpJCj5p",
"digest": "09bff16aed893349d94485cfae366a9c",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": {
"cid": "QmcSTqDcFVLGGVYz1njhUZ7B6fkKtBumsLUwx4nkh22TzS",
"digest": "82c88066be968bb550a05e30ff6c2413",
"sector_size": 2048
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": {
"cid": "QmP4ThPieSUJyRanjibWpT5R5cCMzMAU4j8Y7kBn7CSW1Q",
"digest": "142f2f7e8f1b1779290315cabfd2c803",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": {
"cid": "QmSTCXF2ipGA3f6muVo6kHc2URSx6PzZxGUqu7uykaH5KU",
"digest": "ffd79788d614d27919ae5bd2d94eacb6",
"sector_size": 2048
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": {
"cid": "QmcAixrHsz29DgvtZiMc2kQjvPRvWxYUp36QYmRDZbmREm",
"digest": "8f987f64d434365562180b96ec12e299",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": {
"cid": "QmU9SBzJNrcjRFDiFc4GcApqdApN6z9X7MpUr66mJ2kAJP",
"digest": "700171ecf7334e3199437c930676af82",
"sector_size": 8388608
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": {
"cid": "QmT4iFnbL6r4txS5PXsiV7NTzbhCxHy54PvdkJJGV2VFXb",
"digest": "94b6c24ac01924f4feeecedd16b5d77d",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": {
"cid": "QmbmUMa3TbbW3X5kFhExs6WgC4KeWT18YivaVmXDkB6ANG",
"digest": "79ebb55f56fda427743e35053edad8fc",
"sector_size": 8388608
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": {
"cid": "QmbjFst6SFCK1KsTQrfwPdxf3VTNa1raed574tEZZ9PoyQ",
"digest": "2c245fe8179839dd6c6cdea207c67ae8",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": {
"cid": "QmdNEL2RtqL52GQNuj8uz6mVj5Z34NVnbaJ1yMyh1oXtBx",
"digest": "c49499bb76a0762884896f9683403f55",
"sector_size": 8388608
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": {
"cid": "QmQJKmvZN1a5cQ1Nw6CDyXs3nuRPzvyU5NvCFMUL2BfcZC",
"digest": "56ae47bfda53bb8d22981ed8d8d27d72",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": {
"cid": "QmUiVYCQUgr6Y13pZFr8acWpSM4xvTXUdcvGmxyuHbKhsc",
"digest": "34d4feeacd9abf788d69ef1bb4d8fd00",
"sector_size": 8388608
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": {
"cid": "QmQCABxeTpdvXTyjDyk7nPBxkQzCh7MXfGztWnSXEPKMLW",
"digest": "7e6b2eb5ecbb11ac651ad66ebbb2075a",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": {
"cid": "QmVgCsJFRXKLuuUhT3aMYwKVGNA9rDeR6DCrs7cAe8riBT",
"digest": "827359440349fe8f5a016e7598993b79",
"sector_size": 536870912
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": {
"cid": "QmPBweyugh5Sx4umk8ULhgEGbjY8xmWLfU6M7EMpc8Mad6",
"digest": "94a8d9e25a9ab9674d339833664eba25",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": {
"cid": "QmfA31fbCWojSmhSGvvfxmxaYCpMoXP95zEQ9sLvBGHNaN",
"digest": "bd2cd62f65c1ab84f19ca27e97b7c731",
"sector_size": 536870912
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": {
"cid": "QmY5yax1E9KymBnCeHksE9Zi8NieZbmwcpoDGoabkeeb9h",
"digest": "c909ea9e3fe25ab9b391a64593afdbba",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": {
"cid": "QmaUmfcJt6pozn8ndq1JVBzLRjRJdHMTPd4foa8iw5sjBZ",
"digest": "2cf49eb26f1fee94c85781a390ddb4c8",
"sector_size": 34359738368
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": {
"cid": "QmXnPo4yH5mwMguwrvqgRfduSttbmPrXtbBfbwU21wQWHt",
"digest": "caf900461e988bbf86dbcaca087b7864",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": {
"cid": "QmR9i9KL3vhhAqTBGj1bPPC7LvkptxrH9RvxJxLN1vvsBE",
"digest": "0f8ec542485568fa3468c066e9fed82b",
"sector_size": 34359738368
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": {
"cid": "QmZtzzPWwmZEgR7MSMvXRbt9KVK8k4XZ5RLWHybHJW9SdE",
"digest": "a2844f0703f186d143a06146a04577d8",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": {
"cid": "Qmdtczp7p4wrbDofmHdGhiixn9irAcN77mV9AEHZBaTt1i",
"digest": "d84f79a16fe40e9e25a36e2107bb1ba0",
"sector_size": 34359738368
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": {
"cid": "QmWxEA7EdQCUJTzjNpxg5XTF45D2uVyYnN1QRUb5TRYU8M",
"digest": "2306247a1e616dbe07f01b88196c2044",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": {
"cid": "QmZCvxKcKP97vDAk8Nxs9R1fWtqpjQrAhhfXPoCi1nkDoF",
"digest": "fc02943678dd119e69e7fab8420e8819",
"sector_size": 34359738368
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": {
"cid": "QmP676KwuvyF9Y64uJnXvLtvD1xcuWQ6wD23RzYtQ6dd4f",
"digest": "215b1c667a4f46a1d0178338df568615",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": {
"cid": "QmeAN4vuANhXsF8xP2Lx5j2L6yMSdogLzpcvqCJThRGK1V",
"digest": "3810b7780ac0e299b22ae70f1f94c9bc",
"sector_size": 68719476736
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": {
"cid": "QmPvPwbJtcSGyqB1rQJhSF5yvFbX9ZBSsHVej5F8JUyHUJ",
"digest": "0c9c423b28b1455fcbc329a1045fd4dd",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": {
"cid": "QmWV8rqZLxs1oQN9jxNWmnT1YdgLwCcscv94VARrhHf1T7",
"digest": "59d2bf1857adc59a4f08fcf2afaa916b",
"sector_size": 68719476736
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": {
"cid": "QmUxPQfvckzm1t6MFRdDZ1fDK5UJzAjK7pTZ97cwyachdr",
"digest": "965132f51ae445b0e6d32692b7561995",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": {
"cid": "QmVkrXc1SLcpgcudK5J25HH93QvR9tNsVhVTYHm5UymXAz",
"digest": "2170a91ad5bae22ea61f2ea766630322",
"sector_size": 68719476736
},
"v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": {
"cid": "QmTxq2EBnQWb5R8tS4MHdchj4vNfLYGoSXxwJFvs5xgW4K",
"digest": "fc8c3d26e0e56373ad96cb41520d55a6",
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": {
"cid": "QmbfQjPD7EpzjhWGmvWAsyN2mAZ4PcYhsf3ujuhU9CSuBm",
"digest": "6d3789148fb6466d07ee1e24d6292fd6",
"sector_size": 68719476736
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": {
"cid": "QmRjgZHERgqGoRagR788Kh6ybi26csVYa8mqbqhmZm57Jx",
"digest": "cfc7b0897d1eee48c586f7beb89e67f7",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": {
"cid": "QmWceMgnWYLopMuM4AoGMvGEau7tNe5UK83XFjH5V9B17h",
"digest": "434fb1338ecfaf0f59256f30dde4968f",
"sector_size": 2048
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": {
"cid": "QmNjvnvFP7KgovHUddULoB19fBHT81iz7NcUbzEHZUUPsm",
"digest": "fb59bd061c987eac7068008c44de346b",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": {
"cid": "QmamahpFCstMUqHi2qGtVoDnRrsXhid86qsfvoyCTKJqHr",
"digest": "dc1ade9929ade1708238f155343044ac",
"sector_size": 2048
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": {
"cid": "QmTpRPBA4dt8fgGpcVzi4L1KA1U2eBHCE8WVmS2GUygMvT",
"digest": "36d465915b0afbf96bd08e7915e00952",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": {
"cid": "QmYBpTt7LWNAWr1JXThV5VxX7wsQFLd1PHrGYVbrU1EZjC",
"digest": "6c77597eb91ab936c1cef4cf19eba1b3",
"sector_size": 536870912
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": {
"cid": "QmRzDyVfQCLsxspoVsed5bcQRsG6KiktngJfcNBL3TJPZe",
"digest": "99d16df0eb6a7e227a4f4570c4f6b6f1",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": {
"cid": "QmWionkqH2B6TXivzBSQeSyBxojaiAFbzhjtwYRrfwd8nH",
"digest": "065179da19fbe515507267677f02823e",
"sector_size": 536870912
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": {
"cid": "QmV8ZjTSGzDUWmFvsq9NSyPBR7eDDUcvCPNgj2yE7HMAFu",
"digest": "34f3ddf1d1c9f41c0cd73b91e8b4bc27",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": {
"cid": "QmPXAPPuQtuQz7Zz3MHMAMEtsYwqM1o9H1csPLeiMUQwZH",
"digest": "09e612e4eeb7a0eb95679a88404f960c",
"sector_size": 8388608
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": {
"cid": "QmTa3VbjTiqJWU6r4WKayaQrUaaBsrpp5UDqYvPDd2C5hs",
"digest": "ec62d59651daa5631d3d1e9c782dd940",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": {
"cid": "QmYCuipFyvVW1GojdMrjK1JnMobXtT4zRCZs1CGxjizs99",
"digest": "b687beb9adbd9dabe265a7e3620813e4",
"sector_size": 8388608
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": {
"cid": "Qmf8ngfArxrv9tFWDqBcNegdBMymvuakwyHKd1pbW3pbsb",
"digest": "a16d6f4c6424fb280236739f84b24f97",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": {
"cid": "QmengpM684XLQfG8754ToonszgEg2bQeAGUan5uXTHUQzJ",
"digest": "6a388072a518cf46ebd661f5cc46900a",
"sector_size": 34359738368
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": {
"cid": "QmfQgVFerArJ6Jupwyc9tKjLD9n1J9ajLHBdpY465tRM7M",
"digest": "7a139d82b8a02e35279d657e197f5c1f",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": {
"cid": "Qmf93EMrADXAK6CyiSfE8xx45fkMfR3uzKEPCvZC1n2kzb",
"digest": "0c7b4aac1c40fdb7eb82bc355b41addf",
"sector_size": 34359738368
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": {
"cid": "QmfDha8271nXJn14Aq3qQeghjMBWbs6HNSGa6VuzCVk4TW",
"digest": "5d3cd3f107a3bea8a96d1189efd2965c",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": {
"cid": "QmS7ye6Ri2MfFzCkcUJ7FQ6zxDKuJ6J6B8k5PN7wzSR9sX",
"digest": "1801f8a6e1b00bceb00cc27314bb5ce3",
"sector_size": 68719476736
},
"v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": {
"cid": "QmRVtTtiFzHJTHurYzaCvetGAchux9cktixT4aGHthN6Zt",
"digest": "62c366405404e60f171e661492740b1c",
"v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": {
"cid": "QmehSmC6BhrgRZakPDta2ewoH9nosNzdjCqQRXsNFNUkLN",
"digest": "a89884252c04c298d0b3c81bfd884164",
"sector_size": 68719476736
}
}
}

4
build/version.go
View File

@ -25,7 +25,7 @@ func buildType() string {
}
// BuildVersion is the local build version, set by build system
const BuildVersion = "0.4.1"
const BuildVersion = "0.4.6"
func UserVersion() string {
return BuildVersion + buildType() + CurrentCommit
@ -53,7 +53,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool {
}
// APIVersion is a semver version of the rpc api exposed
var APIVersion Version = newVer(0, 6, 0)
var APIVersion Version = newVer(0, 11, 0)
//nolint:varcheck,deadcode
const (

9
chain/beacon/beacon.go
View File

@ -2,12 +2,13 @@ package beacon
import (
"context"
"time"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
)
var log = logging.Logger("beacon")
@ -52,7 +53,7 @@ func ValidateBlockValues(b RandomBeacon, h *types.BlockHeader, prevEntry types.B
}
func BeaconEntriesForBlock(ctx context.Context, beacon RandomBeacon, round abi.ChainEpoch, prev types.BeaconEntry) ([]types.BeaconEntry, error) {
start := time.Now()
start := build.Clock.Now()
maxRound := beacon.MaxBeaconRoundForEpoch(round, prev)
if maxRound == prev.Round {
@ -81,7 +82,7 @@ func BeaconEntriesForBlock(ctx context.Context, beacon RandomBeacon, round abi.C
}
}
log.Debugw("fetching beacon entries", "took", time.Since(start), "numEntries", len(out))
log.Debugw("fetching beacon entries", "took", build.Clock.Since(start), "numEntries", len(out))
reverse(out)
return out, nil
}

9
chain/beacon/drand/drand.go
View File

@ -21,6 +21,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/beacon"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@ -131,7 +132,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
}
go func() {
start := time.Now()
start := build.Clock.Now()
log.Infow("start fetching randomness", "round", round)
resp, err := db.client.Get(ctx, round)
@ -142,7 +143,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
br.Entry.Round = resp.Round()
br.Entry.Data = resp.Signature()
}
log.Infow("done fetching randomness", "round", round, "took", time.Since(start))
log.Infow("done fetching randomness", "round", round, "took", build.Clock.Since(start))
out <- br
close(out)
}()
@ -170,6 +171,10 @@ func (db *DrandBeacon) VerifyEntry(curr types.BeaconEntry, prev types.BeaconEntr
// TODO handle genesis better
return nil
}
if be := db.getCachedValue(curr.Round); be != nil {
// return no error if the value is in the cache already
return nil
}
b := &dchain.Beacon{
PreviousSig: prev.Data,
Round: curr.Round,

2
chain/beacon/drand/drand_test.go
View File

@ -12,7 +12,7 @@ import (
)
func TestPrintGroupInfo(t *testing.T) {
server := build.DrandConfig.Servers[0]
server := build.DrandConfig().Servers[0]
c, err := hclient.New(server, nil, nil)
assert.NoError(t, err)
cg := c.(interface {

5
chain/block_receipt_tracker.go
View File

@ -5,6 +5,7 @@ import (
"sync"
"time"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/hashicorp/golang-lru"
peer "github.com/libp2p/go-libp2p-core/peer"
@ -37,14 +38,14 @@ func (brt *blockReceiptTracker) Add(p peer.ID, ts *types.TipSet) {
if !ok {
pset := &peerSet{
peers: map[peer.ID]time.Time{
p: time.Now(),
p: build.Clock.Now(),
},
}
brt.cache.Add(ts.Key(), pset)
return
}
val.(*peerSet).peers[p] = time.Now()
val.(*peerSet).peers[p] = build.Clock.Now()
}
func (brt *blockReceiptTracker) GetPeers(ts *types.TipSet) []peer.ID {

276
chain/blocksync/blocksync.go
View File

@ -1,276 +0,0 @@
package blocksync
import (
"bufio"
"context"
"time"
"github.com/libp2p/go-libp2p-core/protocol"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
)
var log = logging.Logger("blocksync")
type NewStreamFunc func(context.Context, peer.ID, ...protocol.ID) (inet.Stream, error)
const BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
const BlockSyncMaxRequestLength = 800
// BlockSyncService is the component that services BlockSync requests from
// peers.
//
// BlockSync is the basic chain synchronization protocol of Filecoin. BlockSync
// is an RPC-oriented protocol, with a single operation to request blocks.
//
// A request contains a start anchor block (referred to with a CID), and a
// amount of blocks requested beyond the anchor (including the anchor itself).
//
// A client can also pass options, encoded as a 64-bit bitfield. Lotus supports
// two options at the moment:
//
// - include block contents
// - include block messages
//
// The response will include a status code, an optional message, and the
// response payload in case of success. The payload is a slice of serialized
// tipsets.
type BlockSyncService struct {
cs *store.ChainStore
}
type BlockSyncRequest struct {
Start []cid.Cid
RequestLength uint64
Options uint64
}
type BSOptions struct {
IncludeBlocks bool
IncludeMessages bool
}
func ParseBSOptions(optfield uint64) *BSOptions {
return &BSOptions{
IncludeBlocks: optfield&(BSOptBlocks) != 0,
IncludeMessages: optfield&(BSOptMessages) != 0,
}
}
const (
BSOptBlocks = 1 << iota
BSOptMessages
)
const (
StatusOK = uint64(0)
StatusPartial = uint64(101)
StatusNotFound = uint64(201)
StatusGoAway = uint64(202)
StatusInternalError = uint64(203)
StatusBadRequest = uint64(204)
)
type BlockSyncResponse struct {
Chain []*BSTipSet
Status uint64
Message string
}
type BSTipSet struct {
Blocks []*types.BlockHeader
BlsMessages []*types.Message
BlsMsgIncludes [][]uint64
SecpkMessages []*types.SignedMessage
SecpkMsgIncludes [][]uint64
}
func NewBlockSyncService(cs *store.ChainStore) *BlockSyncService {
return &BlockSyncService{
cs: cs,
}
}
func (bss *BlockSyncService) HandleStream(s inet.Stream) {
ctx, span := trace.StartSpan(context.Background(), "blocksync.HandleStream")
defer span.End()
defer s.Close() //nolint:errcheck
var req BlockSyncRequest
if err := cborutil.ReadCborRPC(bufio.NewReader(s), &req); err != nil {
log.Warnf("failed to read block sync request: %s", err)
return
}
log.Infow("block sync request", "start", req.Start, "len", req.RequestLength)
resp, err := bss.processRequest(ctx, s.Conn().RemotePeer(), &req)
if err != nil {
log.Warn("failed to process block sync request: ", err)
return
}
writeDeadline := 60 * time.Second
_ = s.SetDeadline(time.Now().Add(writeDeadline))
if err := cborutil.WriteCborRPC(s, resp); err != nil {
log.Warnw("failed to write back response for handle stream", "err", err, "peer", s.Conn().RemotePeer())
return
}
}
func (bss *BlockSyncService) processRequest(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) {
_, span := trace.StartSpan(ctx, "blocksync.ProcessRequest")
defer span.End()
opts := ParseBSOptions(req.Options)
if len(req.Start) == 0 {
return &BlockSyncResponse{
Status: StatusBadRequest,
Message: "no cids given in blocksync request",
}, nil
}
span.AddAttributes(
trace.BoolAttribute("blocks", opts.IncludeBlocks),
trace.BoolAttribute("messages", opts.IncludeMessages),
trace.Int64Attribute("reqlen", int64(req.RequestLength)),
)
reqlen := req.RequestLength
if reqlen > BlockSyncMaxRequestLength {
log.Warnw("limiting blocksync request length", "orig", req.RequestLength, "peer", p)
reqlen = BlockSyncMaxRequestLength
}
chain, err := collectChainSegment(bss.cs, types.NewTipSetKey(req.Start...), reqlen, opts)
if err != nil {
log.Warn("encountered error while responding to block sync request: ", err)
return &BlockSyncResponse{
Status: StatusInternalError,
Message: err.Error(),
}, nil
}
status := StatusOK
if reqlen < req.RequestLength {
status = StatusPartial
}
return &BlockSyncResponse{
Chain: chain,
Status: status,
}, nil
}
func collectChainSegment(cs *store.ChainStore, start types.TipSetKey, length uint64, opts *BSOptions) ([]*BSTipSet, error) {
var bstips []*BSTipSet
cur := start
for {
var bst BSTipSet
ts, err := cs.LoadTipSet(cur)
if err != nil {
return nil, xerrors.Errorf("failed loading tipset %s: %w", cur, err)
}
if opts.IncludeMessages {
bmsgs, bmincl, smsgs, smincl, err := gatherMessages(cs, ts)
if err != nil {
return nil, xerrors.Errorf("gather messages failed: %w", err)
}
bst.BlsMessages = bmsgs
bst.BlsMsgIncludes = bmincl
bst.SecpkMessages = smsgs
bst.SecpkMsgIncludes = smincl
}
if opts.IncludeBlocks {
bst.Blocks = ts.Blocks()
}
bstips = append(bstips, &bst)
if uint64(len(bstips)) >= length || ts.Height() == 0 {
return bstips, nil
}
cur = ts.Parents()
}
}
func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
blsmsgmap := make(map[cid.Cid]uint64)
secpkmsgmap := make(map[cid.Cid]uint64)
var secpkmsgs []*types.SignedMessage
var blsmsgs []*types.Message
var secpkincl, blsincl [][]uint64
for _, b := range ts.Blocks() {
bmsgs, smsgs, err := cs.MessagesForBlock(b)
if err != nil {
return nil, nil, nil, nil, err
}
bmi := make([]uint64, 0, len(bmsgs))
for _, m := range bmsgs {
i, ok := blsmsgmap[m.Cid()]
if !ok {
i = uint64(len(blsmsgs))
blsmsgs = append(blsmsgs, m)
blsmsgmap[m.Cid()] = i
}
bmi = append(bmi, i)
}
blsincl = append(blsincl, bmi)
smi := make([]uint64, 0, len(smsgs))
for _, m := range smsgs {
i, ok := secpkmsgmap[m.Cid()]
if !ok {
i = uint64(len(secpkmsgs))
secpkmsgs = append(secpkmsgs, m)
secpkmsgmap[m.Cid()] = i
}
smi = append(smi, i)
}
secpkincl = append(secpkincl, smi)
}
return blsmsgs, blsincl, secpkmsgs, secpkincl, nil
}
func bstsToFullTipSet(bts *BSTipSet) (*store.FullTipSet, error) {
fts := &store.FullTipSet{}
for i, b := range bts.Blocks {
fb := &types.FullBlock{
Header: b,
}
for _, mi := range bts.BlsMsgIncludes[i] {
fb.BlsMessages = append(fb.BlsMessages, bts.BlsMessages[mi])
}
for _, mi := range bts.SecpkMsgIncludes[i] {
fb.SecpkMessages = append(fb.SecpkMessages, bts.SecpkMessages[mi])
}
fts.Blocks = append(fts.Blocks, fb)
}
return fts, nil
}

602
chain/blocksync/blocksync_client.go
View File

@ -1,602 +0,0 @@
package blocksync
import (
"bufio"
"context"
"fmt"
"math/rand"
"sort"
"sync"
"time"
blocks "github.com/ipfs/go-block-format"
bserv "github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
graphsync "github.com/ipfs/go-graphsync"
gsnet "github.com/ipfs/go-graphsync/network"
host "github.com/libp2p/go-libp2p-core/host"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
incrt "github.com/filecoin-project/lotus/lib/increadtimeout"
"github.com/filecoin-project/lotus/lib/peermgr"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
type BlockSync struct {
bserv bserv.BlockService
gsync graphsync.GraphExchange
host host.Host
syncPeers *bsPeerTracker
peerMgr *peermgr.PeerMgr
}
func NewBlockSyncClient(bserv dtypes.ChainBlockService, h host.Host, pmgr peermgr.MaybePeerMgr, gs dtypes.Graphsync) *BlockSync {
return &BlockSync{
bserv: bserv,
host: h,
syncPeers: newPeerTracker(pmgr.Mgr),
peerMgr: pmgr.Mgr,
gsync: gs,
}
}
func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) error {
switch res.Status {
case StatusPartial: // Partial Response
return xerrors.Errorf("not handling partial blocksync responses yet")
case StatusNotFound: // req.Start not found
return xerrors.Errorf("not found")
case StatusGoAway: // Go Away
return xerrors.Errorf("not handling 'go away' blocksync responses yet")
case StatusInternalError: // Internal Error
return xerrors.Errorf("block sync peer errored: %s", res.Message)
case StatusBadRequest:
return xerrors.Errorf("block sync request invalid: %s", res.Message)
default:
return xerrors.Errorf("unrecognized response code: %d", res.Status)
}
}
// GetBlocks fetches count blocks from the network, from the provided tipset
// *backwards*, returning as many tipsets as count.
//
// {hint/usage}: This is used by the Syncer during normal chain syncing and when
// resolving forks.
func (bs *BlockSync) GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error) {
ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks")
defer span.End()
if span.IsRecordingEvents() {
span.AddAttributes(
trace.StringAttribute("tipset", fmt.Sprint(tsk.Cids())),
trace.Int64Attribute("count", int64(count)),
)
}
req := &BlockSyncRequest{
Start: tsk.Cids(),
RequestLength: uint64(count),
Options: BSOptBlocks,
}
// this peerset is sorted by latency and failure counting.
peers := bs.getPeers()
// randomize the first few peers so we don't always pick the same peer
shufflePrefix(peers)
start := time.Now()
var oerr error
for _, p := range peers {
// TODO: doing this synchronously isnt great, but fetching in parallel
// may not be a good idea either. think about this more
select {
case <-ctx.Done():
return nil, xerrors.Errorf("blocksync getblocks failed: %w", ctx.Err())
default:
}
res, err := bs.sendRequestToPeer(ctx, p, req)
if err != nil {
oerr = err
if !xerrors.Is(err, inet.ErrNoConn) {
log.Warnf("BlockSync request failed for peer %s: %s", p.String(), err)
}
continue
}
if res.Status == StatusOK || res.Status == StatusPartial {
resp, err := bs.processBlocksResponse(req, res)
if err != nil {
return nil, xerrors.Errorf("success response from peer failed to process: %w", err)
}
bs.syncPeers.logGlobalSuccess(time.Since(start))
bs.host.ConnManager().TagPeer(p, "bsync", 25)
return resp, nil
}
oerr = bs.processStatus(req, res)
if oerr != nil {
log.Warnf("BlockSync peer %s response was an error: %s", p.String(), oerr)
}
}
return nil, xerrors.Errorf("GetBlocks failed with all peers: %w", oerr)
}
func (bs *BlockSync) GetFullTipSet(ctx context.Context, p peer.ID, tsk types.TipSetKey) (*store.FullTipSet, error) {
// TODO: round robin through these peers on error
req := &BlockSyncRequest{
Start: tsk.Cids(),
RequestLength: 1,
Options: BSOptBlocks | BSOptMessages,
}
res, err := bs.sendRequestToPeer(ctx, p, req)
if err != nil {
return nil, err
}
switch res.Status {
case 0: // Success
if len(res.Chain) == 0 {
return nil, fmt.Errorf("got zero length chain response")
}
bts := res.Chain[0]
return bstsToFullTipSet(bts)
case 101: // Partial Response
return nil, xerrors.Errorf("partial responses are not handled for single tipset fetching")
case 201: // req.Start not found
return nil, fmt.Errorf("not found")
case 202: // Go Away
return nil, xerrors.Errorf("received 'go away' response peer")
case 203: // Internal Error
return nil, fmt.Errorf("block sync peer errored: %q", res.Message)
case 204: // Invalid Request
return nil, fmt.Errorf("block sync request invalid: %q", res.Message)
default:
return nil, fmt.Errorf("unrecognized response code")
}
}
func shufflePrefix(peers []peer.ID) {
pref := 5
if len(peers) < pref {
pref = len(peers)
}
buf := make([]peer.ID, pref)
perm := rand.Perm(pref)
for i, v := range perm {
buf[i] = peers[v]
}
copy(peers, buf)
}
func (bs *BlockSync) GetChainMessages(ctx context.Context, h *types.TipSet, count uint64) ([]*BSTipSet, error) {
ctx, span := trace.StartSpan(ctx, "GetChainMessages")
defer span.End()
peers := bs.getPeers()
// randomize the first few peers so we don't always pick the same peer
shufflePrefix(peers)
req := &BlockSyncRequest{
Start: h.Cids(),
RequestLength: count,
Options: BSOptMessages,
}
var err error
start := time.Now()
for _, p := range peers {
res, rerr := bs.sendRequestToPeer(ctx, p, req)
if rerr != nil {
err = rerr
log.Warnf("BlockSync request failed for peer %s: %s", p.String(), err)
continue
}
if res.Status == StatusOK {
bs.syncPeers.logGlobalSuccess(time.Since(start))
return res.Chain, nil
}
if res.Status == StatusPartial {
// TODO: track partial response sizes to ensure we don't overrequest too often
return res.Chain, nil
}
err = bs.processStatus(req, res)
if err != nil {
log.Warnf("BlockSync peer %s response was an error: %s", p.String(), err)
}
}
if err == nil {
return nil, xerrors.Errorf("GetChainMessages failed, no peers connected")
}
// TODO: What if we have no peers (and err is nil)?
return nil, xerrors.Errorf("GetChainMessages failed with all peers(%d): %w", len(peers), err)
}
func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *BlockSyncRequest) (_ *BlockSyncResponse, err error) {
ctx, span := trace.StartSpan(ctx, "sendRequestToPeer")
defer span.End()
defer func() {
if err != nil {
if span.IsRecordingEvents() {
span.SetStatus(trace.Status{
Code: 5,
Message: err.Error(),
})
}
}
}()
if span.IsRecordingEvents() {
span.AddAttributes(
trace.StringAttribute("peer", p.Pretty()),
)
}
gsproto := string(gsnet.ProtocolGraphsync)
supp, err := bs.host.Peerstore().SupportsProtocols(p, BlockSyncProtocolID, gsproto)
if err != nil {
return nil, xerrors.Errorf("failed to get protocols for peer: %w", err)
}
if len(supp) == 0 {
return nil, xerrors.Errorf("peer %s supports no known sync protocols", p)
}
switch supp[0] {
case BlockSyncProtocolID:
res, err := bs.fetchBlocksBlockSync(ctx, p, req)
if err != nil {
return nil, xerrors.Errorf("blocksync req failed: %w", err)
}
return res, nil
case gsproto:
res, err := bs.fetchBlocksGraphSync(ctx, p, req)
if err != nil {
return nil, xerrors.Errorf("graphsync req failed: %w", err)
}
return res, nil
default:
return nil, xerrors.Errorf("peerstore somehow returned unexpected protocols: %v", supp)
}
}
func (bs *BlockSync) fetchBlocksBlockSync(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) {
ctx, span := trace.StartSpan(ctx, "blockSyncFetch")
defer span.End()
start := time.Now()
s, err := bs.host.NewStream(inet.WithNoDial(ctx, "should already have connection"), p, BlockSyncProtocolID)
if err != nil {
bs.RemovePeer(p)
return nil, xerrors.Errorf("failed to open stream to peer: %w", err)
}
_ = s.SetWriteDeadline(time.Now().Add(5 * time.Second))
if err := cborutil.WriteCborRPC(s, req); err != nil {
_ = s.SetWriteDeadline(time.Time{})
bs.syncPeers.logFailure(p, time.Since(start))
return nil, err
}
_ = s.SetWriteDeadline(time.Time{})
var res BlockSyncResponse
r := incrt.New(s, 50<<10, 5*time.Second)
if err := cborutil.ReadCborRPC(bufio.NewReader(r), &res); err != nil {
bs.syncPeers.logFailure(p, time.Since(start))
return nil, err
}
if span.IsRecordingEvents() {
span.AddAttributes(
trace.Int64Attribute("resp_status", int64(res.Status)),
trace.StringAttribute("msg", res.Message),
trace.Int64Attribute("chain_len", int64(len(res.Chain))),
)
}
bs.syncPeers.logSuccess(p, time.Since(start))
return &res, nil
}
func (bs *BlockSync) processBlocksResponse(req *BlockSyncRequest, res *BlockSyncResponse) ([]*types.TipSet, error) {
if len(res.Chain) == 0 {
return nil, xerrors.Errorf("got no blocks in successful blocksync response")
}
cur, err := types.NewTipSet(res.Chain[0].Blocks)
if err != nil {
return nil, err
}
out := []*types.TipSet{cur}
for bi := 1; bi < len(res.Chain); bi++ {
next := res.Chain[bi].Blocks
nts, err := types.NewTipSet(next)
if err != nil {
return nil, err
}
if !types.CidArrsEqual(cur.Parents().Cids(), nts.Cids()) {
return nil, fmt.Errorf("parents of tipset[%d] were not tipset[%d]", bi-1, bi)
}
out = append(out, nts)
cur = nts
}
return out, nil
}
func (bs *BlockSync) GetBlock(ctx context.Context, c cid.Cid) (*types.BlockHeader, error) {
sb, err := bs.bserv.GetBlock(ctx, c)
if err != nil {
return nil, err
}
return types.DecodeBlock(sb.RawData())
}
func (bs *BlockSync) AddPeer(p peer.ID) {
bs.syncPeers.addPeer(p)
}
func (bs *BlockSync) RemovePeer(p peer.ID) {
bs.syncPeers.removePeer(p)
}
// getPeers returns a preference-sorted set of peers to query.
func (bs *BlockSync) getPeers() []peer.ID {
return bs.syncPeers.prefSortedPeers()
}
func (bs *BlockSync) FetchMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.Message, error) {
out := make([]*types.Message, len(cids))
err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error {
msg, err := types.DecodeMessage(b.RawData())
if err != nil {
return err
}
if out[i] != nil {
return fmt.Errorf("received duplicate message")
}
out[i] = msg
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
func (bs *BlockSync) FetchSignedMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.SignedMessage, error) {
out := make([]*types.SignedMessage, len(cids))
err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error {
smsg, err := types.DecodeSignedMessage(b.RawData())
if err != nil {
return err
}
if out[i] != nil {
return fmt.Errorf("received duplicate message")
}
out[i] = smsg
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
func (bs *BlockSync) fetchCids(ctx context.Context, cids []cid.Cid, cb func(int, blocks.Block) error) error {
resp := bs.bserv.GetBlocks(context.TODO(), cids)
m := make(map[cid.Cid]int)
for i, c := range cids {
m[c] = i
}
for i := 0; i < len(cids); i++ {
select {
case v, ok := <-resp:
if !ok {
if i == len(cids)-1 {
break
}
return fmt.Errorf("failed to fetch all messages")
}
ix, ok := m[v.Cid()]
if !ok {
return fmt.Errorf("received message we didnt ask for")
}
if err := cb(ix, v); err != nil {
return err
}
}
}
return nil
}
type peerStats struct {
successes int
failures int
firstSeen time.Time
averageTime time.Duration
}
type bsPeerTracker struct {
lk sync.Mutex
peers map[peer.ID]*peerStats
avgGlobalTime time.Duration
pmgr *peermgr.PeerMgr
}
func newPeerTracker(pmgr *peermgr.PeerMgr) *bsPeerTracker {
return &bsPeerTracker{
peers: make(map[peer.ID]*peerStats),
pmgr: pmgr,
}
}
func (bpt *bsPeerTracker) addPeer(p peer.ID) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
if _, ok := bpt.peers[p]; ok {
return
}
bpt.peers[p] = &peerStats{
firstSeen: time.Now(),
}
}
const (
// newPeerMul is how much better than average is the new peer assumed to be
// less than one to encourouge trying new peers
newPeerMul = 0.9
)
func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
// TODO: this could probably be cached, but as long as its not too many peers, fine for now
bpt.lk.Lock()
defer bpt.lk.Unlock()
out := make([]peer.ID, 0, len(bpt.peers))
for p := range bpt.peers {
out = append(out, p)
}
// sort by 'expected cost' of requesting data from that peer
// additionally handle edge cases where not enough data is available
sort.Slice(out, func(i, j int) bool {
pi := bpt.peers[out[i]]
pj := bpt.peers[out[j]]
var costI, costJ float64
getPeerInitLat := func(p peer.ID) float64 {
var res float64
if bpt.pmgr != nil {
if lat, ok := bpt.pmgr.GetPeerLatency(p); ok {
res = float64(lat)
}
}
if res == 0 {
res = float64(bpt.avgGlobalTime)
}
return res * newPeerMul
}
if pi.successes+pi.failures > 0 {
failRateI := float64(pi.failures) / float64(pi.failures+pi.successes)
costI = float64(pi.averageTime) + failRateI*float64(bpt.avgGlobalTime)
} else {
costI = getPeerInitLat(out[i])
}
if pj.successes+pj.failures > 0 {
failRateJ := float64(pj.failures) / float64(pj.failures+pj.successes)
costJ = float64(pj.averageTime) + failRateJ*float64(bpt.avgGlobalTime)
} else {
costJ = getPeerInitLat(out[j])
}
return costI < costJ
})
return out
}
const (
// xInvAlpha = (N+1)/2
localInvAlpha = 5 // 86% of the value is the last 9
globalInvAlpha = 20 // 86% of the value is the last 39
)
func (bpt *bsPeerTracker) logGlobalSuccess(dur time.Duration) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
if bpt.avgGlobalTime == 0 {
bpt.avgGlobalTime = dur
return
}
delta := (dur - bpt.avgGlobalTime) / globalInvAlpha
bpt.avgGlobalTime += delta
}
func logTime(pi *peerStats, dur time.Duration) {
if pi.averageTime == 0 {
pi.averageTime = dur
return
}
delta := (dur - pi.averageTime) / localInvAlpha
pi.averageTime += delta
}
func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
var pi *peerStats
var ok bool
if pi, ok = bpt.peers[p]; !ok {
log.Warnw("log success called on peer not in tracker", "peerid", p.String())
return
}
pi.successes++
logTime(pi, dur)
}
func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
var pi *peerStats
var ok bool
if pi, ok = bpt.peers[p]; !ok {
log.Warn("log failure called on peer not in tracker", "peerid", p.String())
return
}
pi.failures++
logTime(pi, dur)
}
func (bpt *bsPeerTracker) removePeer(p peer.ID) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
delete(bpt.peers, p)
}

658
chain/blocksync/cbor_gen.go
View File

@ -14,36 +14,36 @@ import (
var _ = xerrors.Errorf
var lengthBufBlockSyncRequest = []byte{131}
var lengthBufRequest = []byte{131}
func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error {
func (t *Request) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write(lengthBufBlockSyncRequest); err != nil {
if _, err := w.Write(lengthBufRequest); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Start ([]cid.Cid) (slice)
if len(t.Start) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.Start was too long")
// t.Head ([]cid.Cid) (slice)
if len(t.Head) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.Head was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Start))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Head))); err != nil {
return err
}
for _, v := range t.Start {
for _, v := range t.Head {
if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
return xerrors.Errorf("failed writing cid field t.Start: %w", err)
return xerrors.Errorf("failed writing cid field t.Head: %w", err)
}
}
// t.RequestLength (uint64) (uint64)
// t.Length (uint64) (uint64)
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.RequestLength)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Length)); err != nil {
return err
}
@ -56,7 +56,9 @@ func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error {
return nil
}
func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
func (t *Request) UnmarshalCBOR(r io.Reader) error {
*t = Request{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -72,7 +74,7 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Start ([]cid.Cid) (slice)
// t.Head ([]cid.Cid) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
@ -80,7 +82,7 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.Start: array too large (%d)", extra)
return fmt.Errorf("t.Head: array too large (%d)", extra)
}
if maj != cbg.MajArray {
@ -88,19 +90,19 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
}
if extra > 0 {
t.Start = make([]cid.Cid, extra)
t.Head = make([]cid.Cid, extra)
}
for i := 0; i < int(extra); i++ {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("reading cid field t.Start failed: %w", err)
return xerrors.Errorf("reading cid field t.Head failed: %w", err)
}
t.Start[i] = c
t.Head[i] = c
}
// t.RequestLength (uint64) (uint64)
// t.Length (uint64) (uint64)
{
@ -111,7 +113,7 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.RequestLength = uint64(extra)
t.Length = uint64(extra)
}
// t.Options (uint64) (uint64)
@ -131,19 +133,37 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
return nil
}
var lengthBufBlockSyncResponse = []byte{131}
var lengthBufResponse = []byte{131}
func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error {
func (t *Response) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write(lengthBufBlockSyncResponse); err != nil {
if _, err := w.Write(lengthBufResponse); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Status (blocksync.status) (uint64)
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil {
return err
}
// t.ErrorMessage (string) (string)
if len(t.ErrorMessage) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.ErrorMessage was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ErrorMessage))); err != nil {
return err
}
if _, err := io.WriteString(w, string(t.ErrorMessage)); err != nil {
return err
}
// t.Chain ([]*blocksync.BSTipSet) (slice)
if len(t.Chain) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.Chain was too long")
@ -157,28 +177,12 @@ func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error {
return err
}
}
// t.Status (uint64) (uint64)
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil {
return err
}
// t.Message (string) (string)
if len(t.Message) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Message was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Message))); err != nil {
return err
}
if _, err := io.WriteString(w, t.Message); err != nil {
return err
}
return nil
}
func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error {
func (t *Response) UnmarshalCBOR(r io.Reader) error {
*t = Response{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -194,6 +198,30 @@ func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Status (blocksync.status) (uint64)
{
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Status = status(extra)
}
// t.ErrorMessage (string) (string)
{
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
t.ErrorMessage = string(sval)
}
// t.Chain ([]*blocksync.BSTipSet) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
@ -223,34 +251,296 @@ func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error {
t.Chain[i] = &v
}
// t.Status (uint64) (uint64)
return nil
}
{
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Status = uint64(extra)
var lengthBufCompactedMessages = []byte{132}
func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write(lengthBufCompactedMessages); err != nil {
return err
}
// t.Message (string) (string)
{
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
scratch := make([]byte, 9)
// t.Bls ([]*types.Message) (slice)
if len(t.Bls) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.Bls was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Bls))); err != nil {
return err
}
for _, v := range t.Bls {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
t.Message = string(sval)
// t.BlsIncludes ([][]uint64) (slice)
if len(t.BlsIncludes) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.BlsIncludes was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsIncludes))); err != nil {
return err
}
for _, v := range t.BlsIncludes {
if len(v) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field v was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
return err
}
for _, v := range v {
if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
return err
}
}
}
// t.Secpk ([]*types.SignedMessage) (slice)
if len(t.Secpk) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.Secpk was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Secpk))); err != nil {
return err
}
for _, v := range t.Secpk {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
// t.SecpkIncludes ([][]uint64) (slice)
if len(t.SecpkIncludes) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkIncludes))); err != nil {
return err
}
for _, v := range t.SecpkIncludes {
if len(v) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field v was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
return err
}
for _, v := range v {
if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
return err
}
}
}
return nil
}
var lengthBufBSTipSet = []byte{133}
func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
*t = CompactedMessages{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 4 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Bls ([]*types.Message) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.Bls: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Bls = make([]*types.Message, extra)
}
for i := 0; i < int(extra); i++ {
var v types.Message
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.Bls[i] = &v
}
// t.BlsIncludes ([][]uint64) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.BlsIncludes: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.BlsIncludes = make([][]uint64, extra)
}
for i := 0; i < int(extra); i++ {
{
var maj byte
var extra uint64
var err error
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.BlsIncludes[i]: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.BlsIncludes[i] = make([]uint64, extra)
}
for j := 0; j < int(extra); j++ {
maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return xerrors.Errorf("failed to read uint64 for t.BlsIncludes[i] slice: %w", err)
}
if maj != cbg.MajUnsignedInt {
return xerrors.Errorf("value read for array t.BlsIncludes[i] was not a uint, instead got %d", maj)
}
t.BlsIncludes[i][j] = uint64(val)
}
}
}
// t.Secpk ([]*types.SignedMessage) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.Secpk: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Secpk = make([]*types.SignedMessage, extra)
}
for i := 0; i < int(extra); i++ {
var v types.SignedMessage
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.Secpk[i] = &v
}
// t.SecpkIncludes ([][]uint64) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.SecpkIncludes: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.SecpkIncludes = make([][]uint64, extra)
}
for i := 0; i < int(extra); i++ {
{
var maj byte
var extra uint64
var err error
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.SecpkIncludes[i]: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.SecpkIncludes[i] = make([]uint64, extra)
}
for j := 0; j < int(extra); j++ {
maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return xerrors.Errorf("failed to read uint64 for t.SecpkIncludes[i] slice: %w", err)
}
if maj != cbg.MajUnsignedInt {
return xerrors.Errorf("value read for array t.SecpkIncludes[i] was not a uint, instead got %d", maj)
}
t.SecpkIncludes[i][j] = uint64(val)
}
}
}
return nil
}
var lengthBufBSTipSet = []byte{130}
func (t *BSTipSet) MarshalCBOR(w io.Writer) error {
if t == nil {
@ -277,83 +567,16 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error {
}
}
// t.BlsMessages ([]*types.Message) (slice)
if len(t.BlsMessages) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.BlsMessages was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsMessages))); err != nil {
// t.Messages (blocksync.CompactedMessages) (struct)
if err := t.Messages.MarshalCBOR(w); err != nil {
return err
}
for _, v := range t.BlsMessages {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
// t.BlsMsgIncludes ([][]uint64) (slice)
if len(t.BlsMsgIncludes) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.BlsMsgIncludes was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsMsgIncludes))); err != nil {
return err
}
for _, v := range t.BlsMsgIncludes {
if len(v) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field v was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
return err
}
for _, v := range v {
if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
return err
}
}
}
// t.SecpkMessages ([]*types.SignedMessage) (slice)
if len(t.SecpkMessages) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.SecpkMessages was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkMessages))); err != nil {
return err
}
for _, v := range t.SecpkMessages {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
// t.SecpkMsgIncludes ([][]uint64) (slice)
if len(t.SecpkMsgIncludes) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.SecpkMsgIncludes was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkMsgIncludes))); err != nil {
return err
}
for _, v := range t.SecpkMsgIncludes {
if len(v) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field v was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
return err
}
for _, v := range v {
if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
return err
}
}
}
return nil
}
func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
*t = BSTipSet{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -365,7 +588,7 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 5 {
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -398,181 +621,24 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
t.Blocks[i] = &v
}
// t.BlsMessages ([]*types.Message) (slice)
// t.Messages (blocksync.CompactedMessages) (struct)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
{
if extra > cbg.MaxLength {
return fmt.Errorf("t.BlsMessages: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.BlsMessages = make([]*types.Message, extra)
}
for i := 0; i < int(extra); i++ {
var v types.Message
if err := v.UnmarshalCBOR(br); err != nil {
b, err := br.ReadByte()
if err != nil {
return err
}
t.BlsMessages[i] = &v
}
// t.BlsMsgIncludes ([][]uint64) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.BlsMsgIncludes: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.BlsMsgIncludes = make([][]uint64, extra)
}
for i := 0; i < int(extra); i++ {
{
var maj byte
var extra uint64
var err error
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.BlsMsgIncludes[i]: array too large (%d)", extra)
t.Messages = new(CompactedMessages)
if err := t.Messages.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.Messages pointer: %w", err)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.BlsMsgIncludes[i] = make([]uint64, extra)
}
for j := 0; j < int(extra); j++ {
maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return xerrors.Errorf("failed to read uint64 for t.BlsMsgIncludes[i] slice: %w", err)
}
if maj != cbg.MajUnsignedInt {
return xerrors.Errorf("value read for array t.BlsMsgIncludes[i] was not a uint, instead got %d", maj)
}
t.BlsMsgIncludes[i][j] = uint64(val)
}
}
}
// t.SecpkMessages ([]*types.SignedMessage) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.SecpkMessages: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.SecpkMessages = make([]*types.SignedMessage, extra)
}
for i := 0; i < int(extra); i++ {
var v types.SignedMessage
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.SecpkMessages[i] = &v
}
// t.SecpkMsgIncludes ([][]uint64) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.SecpkMsgIncludes: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.SecpkMsgIncludes = make([][]uint64, extra)
}
for i := 0; i < int(extra); i++ {
{
var maj byte
var extra uint64
var err error
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.SecpkMsgIncludes[i]: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.SecpkMsgIncludes[i] = make([]uint64, extra)
}
for j := 0; j < int(extra); j++ {
maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return xerrors.Errorf("failed to read uint64 for t.SecpkMsgIncludes[i] slice: %w", err)
}
if maj != cbg.MajUnsignedInt {
return xerrors.Errorf("value read for array t.SecpkMsgIncludes[i] was not a uint, instead got %d", maj)
}
t.SecpkMsgIncludes[i][j] = uint64(val)
}
}
}
return nil
}

446
chain/blocksync/client.go Normal file
View File

@ -0,0 +1,446 @@
package blocksync
import (
"bufio"
"context"
"fmt"
"math/rand"
"time"
host "github.com/libp2p/go-libp2p-core/host"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
incrt "github.com/filecoin-project/lotus/lib/increadtimeout"
"github.com/filecoin-project/lotus/lib/peermgr"
)
// Protocol client.
// FIXME: Rename to just `Client`. Not done at the moment to avoid
// disrupting too much of the consumer code, should be done along
// https://github.com/filecoin-project/lotus/issues/2612.
type BlockSync struct {
// Connection manager used to contact the server.
// FIXME: We should have a reduced interface here, initialized
// just with our protocol ID, we shouldn't be able to open *any*
// connection.
host host.Host
peerTracker *bsPeerTracker
}
func NewClient(
host host.Host,
pmgr peermgr.MaybePeerMgr,
) *BlockSync {
return &BlockSync{
host: host,
peerTracker: newPeerTracker(pmgr.Mgr),
}
}
// Main logic of the client request service. The provided `Request`
// is sent to the `singlePeer` if one is indicated or to all available
// ones otherwise. The response is processed and validated according
// to the `Request` options. Either a `validatedResponse` is returned
// (which can be safely accessed), or an `error` that may represent
// either a response error status, a failed validation or an internal
// error.
//
// This is the internal single point of entry for all external-facing
// APIs, currently we have 3 very heterogeneous services exposed:
// * GetBlocks: Headers
// * GetFullTipSet: Headers | Messages
// * GetChainMessages: Messages
// This function handles all the different combinations of the available
// request options without disrupting external calls. In the future the
// consumers should be forced to use a more standardized service and
// adhere to a single API derived from this function.
func (client *BlockSync) doRequest(
ctx context.Context,
req *Request,
singlePeer *peer.ID,
) (*validatedResponse, error) {
// Validate request.
if req.Length == 0 {
return nil, xerrors.Errorf("invalid request of length 0")
}
if req.Length > MaxRequestLength {
return nil, xerrors.Errorf("request length (%d) above maximum (%d)",
req.Length, MaxRequestLength)
}
if req.Options == 0 {
return nil, xerrors.Errorf("request with no options set")
}
// Generate the list of peers to be queried, either the
// `singlePeer` indicated or all peers available (sorted
// by an internal peer tracker with some randomness injected).
var peers []peer.ID
if singlePeer != nil {
peers = []peer.ID{*singlePeer}
} else {
peers = client.getShuffledPeers()
if len(peers) == 0 {
return nil, xerrors.Errorf("no peers available")
}
}
// Try the request for each peer in the list,
// return on the first successful response.
// FIXME: Doing this serially isn't great, but fetching in parallel
// may not be a good idea either. Think about this more.
globalTime := build.Clock.Now()
// Global time used to track what is the expected time we will need to get
// a response if a client fails us.
for _, peer := range peers {
select {
case <-ctx.Done():
return nil, xerrors.Errorf("context cancelled: %w", ctx.Err())
default:
}
// Send request, read response.
res, err := client.sendRequestToPeer(ctx, peer, req)
if err != nil {
if !xerrors.Is(err, inet.ErrNoConn) {
log.Warnf("could not connect to peer %s: %s",
peer.String(), err)
}
continue
}
// Process and validate response.
validRes, err := client.processResponse(req, res)
if err != nil {
log.Warnf("processing peer %s response failed: %s",
peer.String(), err)
continue
}
client.peerTracker.logGlobalSuccess(build.Clock.Since(globalTime))
client.host.ConnManager().TagPeer(peer, "bsync", SUCCESS_PEER_TAG_VALUE)
return validRes, nil
}
errString := "doRequest failed for all peers"
if singlePeer != nil {
errString = fmt.Sprintf("doRequest failed for single peer %s", *singlePeer)
}
return nil, xerrors.Errorf(errString)
}
// Process and validate response. Check the status, the integrity of the
// information returned, and that it matches the request. Extract the information
// into a `validatedResponse` for the external-facing APIs to select what they
// need.
//
// We are conflating in the single error returned both status and validation
// errors. Peer penalization should happen here then, before returning, so
// we can apply the correct penalties depending on the cause of the error.
func (client *BlockSync) processResponse(
req *Request,
res *Response,
// FIXME: Add the `peer` as argument once we implement penalties.
) (*validatedResponse, error) {
err := res.statusToError()
if err != nil {
return nil, xerrors.Errorf("status error: %s", err)
}
options := parseOptions(req.Options)
if options.noOptionsSet() {
// Safety check: this shouldn't have been sent, and even if it did
// it should have been caught by the peer in its error status.
return nil, xerrors.Errorf("nothing was requested")
}
// Verify that the chain segment returned is in the valid range.
// Note that the returned length might be less than requested.
resLength := len(res.Chain)
if resLength == 0 {
return nil, xerrors.Errorf("got no chain in successful response")
}
if resLength > int(req.Length) {
return nil, xerrors.Errorf("got longer response (%d) than requested (%d)",
resLength, req.Length)
}
if resLength < int(req.Length) && res.Status != Partial {
return nil, xerrors.Errorf("got less than requested without a proper status: %s", res.Status)
}
validRes := &validatedResponse{}
if options.IncludeHeaders {
// Check for valid block sets and extract them into `TipSet`s.
validRes.tipsets = make([]*types.TipSet, resLength)
for i := 0; i < resLength; i++ {
validRes.tipsets[i], err = types.NewTipSet(res.Chain[i].Blocks)
if err != nil {
return nil, xerrors.Errorf("invalid tipset blocks at height (head - %d): %w", i, err)
}
}
// Check that the returned head matches the one requested.
if !types.CidArrsEqual(validRes.tipsets[0].Cids(), req.Head) {
return nil, xerrors.Errorf("returned chain head does not match request")
}
// Check `TipSet`s are connected (valid chain).
for i := 0; i < len(validRes.tipsets)-1; i++ {
if validRes.tipsets[i].IsChildOf(validRes.tipsets[i+1]) == false {
return nil, fmt.Errorf("tipsets are not connected at height (head - %d)/(head - %d)",
i, i+1)
// FIXME: Maybe give more information here, like CIDs.
}
}
}
if options.IncludeMessages {
validRes.messages = make([]*CompactedMessages, resLength)
for i := 0; i < resLength; i++ {
if res.Chain[i].Messages == nil {
return nil, xerrors.Errorf("no messages included for tipset at height (head - %d): %w", i)
}
validRes.messages[i] = res.Chain[i].Messages
}
if options.IncludeHeaders {
// If the headers were also returned check that the compression
// indexes are valid before `toFullTipSets()` is called by the
// consumer.
for tipsetIdx := 0; tipsetIdx < resLength; tipsetIdx++ {
msgs := res.Chain[tipsetIdx].Messages
blocksNum := len(res.Chain[tipsetIdx].Blocks)
if len(msgs.BlsIncludes) != blocksNum {
return nil, xerrors.Errorf("BlsIncludes (%d) does not match number of blocks (%d)",
len(msgs.BlsIncludes), blocksNum)
}
if len(msgs.SecpkIncludes) != blocksNum {
return nil, xerrors.Errorf("SecpkIncludes (%d) does not match number of blocks (%d)",
len(msgs.SecpkIncludes), blocksNum)
}
for blockIdx := 0; blockIdx < blocksNum; blockIdx++ {
for _, mi := range msgs.BlsIncludes[blockIdx] {
if int(mi) >= len(msgs.Bls) {
return nil, xerrors.Errorf("index in BlsIncludes (%d) exceeds number of messages (%d)",
mi, len(msgs.Bls))
}
}
for _, mi := range msgs.SecpkIncludes[blockIdx] {
if int(mi) >= len(msgs.Secpk) {
return nil, xerrors.Errorf("index in SecpkIncludes (%d) exceeds number of messages (%d)",
mi, len(msgs.Secpk))
}
}
}
}
}
}
return validRes, nil
}
// GetBlocks fetches count blocks from the network, from the provided tipset
// *backwards*, returning as many tipsets as count.
//
// {hint/usage}: This is used by the Syncer during normal chain syncing and when
// resolving forks.
func (client *BlockSync) GetBlocks(
ctx context.Context,
tsk types.TipSetKey,
count int,
) ([]*types.TipSet, error) {
ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks")
defer span.End()
if span.IsRecordingEvents() {
span.AddAttributes(
trace.StringAttribute("tipset", fmt.Sprint(tsk.Cids())),
trace.Int64Attribute("count", int64(count)),
)
}
req := &Request{
Head: tsk.Cids(),
Length: uint64(count),
Options: Headers,
}
validRes, err := client.doRequest(ctx, req, nil)
if err != nil {
return nil, err
}
return validRes.tipsets, nil
}
func (client *BlockSync) GetFullTipSet(
ctx context.Context,
peer peer.ID,
tsk types.TipSetKey,
) (*store.FullTipSet, error) {
// TODO: round robin through these peers on error
req := &Request{
Head: tsk.Cids(),
Length: 1,
Options: Headers | Messages,
}
validRes, err := client.doRequest(ctx, req, &peer)
if err != nil {
return nil, err
}
return validRes.toFullTipSets()[0], nil
// If `doRequest` didn't fail we are guaranteed to have at least
// *one* tipset here, so it's safe to index directly.
}
func (client *BlockSync) GetChainMessages(
ctx context.Context,
head *types.TipSet,
length uint64,
) ([]*CompactedMessages, error) {
ctx, span := trace.StartSpan(ctx, "GetChainMessages")
defer span.End()
req := &Request{
Head: head.Cids(),
Length: length,
Options: Messages,
}
validRes, err := client.doRequest(ctx, req, nil)
if err != nil {
return nil, err
}
return validRes.messages, nil
}
// Send a request to a peer. Write request in the stream and read the
// response back. We do not do any processing of the request/response
// here.
func (client *BlockSync) sendRequestToPeer(
ctx context.Context,
peer peer.ID,
req *Request,
) (_ *Response, err error) {
// Trace code.
ctx, span := trace.StartSpan(ctx, "sendRequestToPeer")
defer span.End()
if span.IsRecordingEvents() {
span.AddAttributes(
trace.StringAttribute("peer", peer.Pretty()),
)
}
defer func() {
if err != nil {
if span.IsRecordingEvents() {
span.SetStatus(trace.Status{
Code: 5,
Message: err.Error(),
})
}
}
}()
// -- TRACE --
supported, err := client.host.Peerstore().SupportsProtocols(peer, BlockSyncProtocolID)
if err != nil {
return nil, xerrors.Errorf("failed to get protocols for peer: %w", err)
}
if len(supported) == 0 || supported[0] != BlockSyncProtocolID {
return nil, xerrors.Errorf("peer %s does not support protocol %s",
peer, BlockSyncProtocolID)
// FIXME: `ProtoBook` should support a *single* protocol check that returns
// a bool instead of a list.
}
connectionStart := build.Clock.Now()
// Open stream to peer.
stream, err := client.host.NewStream(
inet.WithNoDial(ctx, "should already have connection"),
peer,
BlockSyncProtocolID)
if err != nil {
client.RemovePeer(peer)
return nil, xerrors.Errorf("failed to open stream to peer: %w", err)
}
// Write request.
_ = stream.SetWriteDeadline(time.Now().Add(WRITE_REQ_DEADLINE))
if err := cborutil.WriteCborRPC(stream, req); err != nil {
_ = stream.SetWriteDeadline(time.Time{})
client.peerTracker.logFailure(peer, build.Clock.Since(connectionStart))
// FIXME: Should we also remove peer here?
return nil, err
}
_ = stream.SetWriteDeadline(time.Time{}) // clear deadline // FIXME: Needs
// its own API (https://github.com/libp2p/go-libp2p-core/issues/162).
// Read response.
var res Response
err = cborutil.ReadCborRPC(
bufio.NewReader(incrt.New(stream, READ_RES_MIN_SPEED, READ_RES_DEADLINE)),
&res)
if err != nil {
client.peerTracker.logFailure(peer, build.Clock.Since(connectionStart))
return nil, xerrors.Errorf("failed to read blocksync response: %w", err)
}
// FIXME: Move all this together at the top using a defer as done elsewhere.
// Maybe we need to declare `res` in the signature.
if span.IsRecordingEvents() {
span.AddAttributes(
trace.Int64Attribute("resp_status", int64(res.Status)),
trace.StringAttribute("msg", res.ErrorMessage),
trace.Int64Attribute("chain_len", int64(len(res.Chain))),
)
}
client.peerTracker.logSuccess(peer, build.Clock.Since(connectionStart))
// FIXME: We should really log a success only after we validate the response.
// It might be a bit hard to do.
return &res, nil
}
func (client *BlockSync) AddPeer(p peer.ID) {
client.peerTracker.addPeer(p)
}
func (client *BlockSync) RemovePeer(p peer.ID) {
client.peerTracker.removePeer(p)
}
// getShuffledPeers returns a preference-sorted set of peers (by latency
// and failure counting), shuffling the first few peers so we don't always
// pick the same peer.
// FIXME: Consider merging with `shufflePrefix()s`.
func (client *BlockSync) getShuffledPeers() []peer.ID {
peers := client.peerTracker.prefSortedPeers()
shufflePrefix(peers)
return peers
}
func shufflePrefix(peers []peer.ID) {
prefix := SHUFFLE_PEERS_PREFIX
if len(peers) < prefix {
prefix = len(peers)
}
buf := make([]peer.ID, prefix)
perm := rand.Perm(prefix)
for i, v := range perm {
buf[i] = peers[v]
}
copy(peers, buf)
}

151
chain/blocksync/graphsync_client.go
View File

@ -1,151 +0,0 @@
package blocksync
import (
"context"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-graphsync"
"github.com/ipld/go-ipld-prime"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
store "github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
basicnode "github.com/ipld/go-ipld-prime/node/basic"
ipldselector "github.com/ipld/go-ipld-prime/traversal/selector"
selectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder"
)
const (
// AMT selector recursion. An AMT has arity of 8 so this gives allows
// us to retrieve trees with 8^10 (1,073,741,824) elements.
amtRecursionDepth = uint32(10)
// some constants for looking up tuple encoded struct fields
// field index of Parents field in a block header
blockIndexParentsField = 5
// field index of Messages field in a block header
blockIndexMessagesField = 10
// field index of AMT node in AMT head
amtHeadNodeFieldIndex = 2
// field index of links array AMT node
amtNodeLinksFieldIndex = 1
// field index of values array AMT node
amtNodeValuesFieldIndex = 2
// maximum depth per traversal
maxRequestLength = 50
)
var amtSelector selectorbuilder.SelectorSpec
func init() {
// builer for selectors
ssb := selectorbuilder.NewSelectorSpecBuilder(basicnode.Style.Any)
// amt selector -- needed to selector through a messages AMT
amtSelector = ssb.ExploreIndex(amtHeadNodeFieldIndex,
ssb.ExploreRecursive(ipldselector.RecursionLimitDepth(int(amtRecursionDepth)),
ssb.ExploreUnion(
ssb.ExploreIndex(amtNodeLinksFieldIndex,
ssb.ExploreAll(ssb.ExploreRecursiveEdge())),
ssb.ExploreIndex(amtNodeValuesFieldIndex,
ssb.ExploreAll(ssb.Matcher())))))
}
func selectorForRequest(req *BlockSyncRequest) ipld.Node {
// builer for selectors
ssb := selectorbuilder.NewSelectorSpecBuilder(basicnode.Style.Any)
bso := ParseBSOptions(req.Options)
if bso.IncludeMessages {
return ssb.ExploreRecursive(ipldselector.RecursionLimitDepth(int(req.RequestLength)),
ssb.ExploreIndex(blockIndexParentsField,
ssb.ExploreUnion(
ssb.ExploreAll(
ssb.ExploreIndex(blockIndexMessagesField,
ssb.ExploreRange(0, 2, amtSelector),
)),
ssb.ExploreIndex(0, ssb.ExploreRecursiveEdge()),
))).Node()
}
return ssb.ExploreRecursive(ipldselector.RecursionLimitDepth(int(req.RequestLength)), ssb.ExploreIndex(blockIndexParentsField,
ssb.ExploreUnion(
ssb.ExploreAll(
ssb.Matcher(),
),
ssb.ExploreIndex(0, ssb.ExploreRecursiveEdge()),
))).Node()
}
func firstTipsetSelector(req *BlockSyncRequest) ipld.Node {
// builer for selectors
ssb := selectorbuilder.NewSelectorSpecBuilder(basicnode.Style.Any)
bso := ParseBSOptions(req.Options)
if bso.IncludeMessages {
return ssb.ExploreIndex(blockIndexMessagesField,
ssb.ExploreRange(0, 2, amtSelector),
).Node()
}
return ssb.Matcher().Node()
}
func (bs *BlockSync) executeGsyncSelector(ctx context.Context, p peer.ID, root cid.Cid, sel ipld.Node) error {
extension := graphsync.ExtensionData{
Name: "chainsync",
Data: nil,
}
_, errs := bs.gsync.Request(ctx, p, cidlink.Link{Cid: root}, sel, extension)
for err := range errs {
return xerrors.Errorf("failed to complete graphsync request: %w", err)
}
return nil
}
// Fallback for interacting with other non-lotus nodes
func (bs *BlockSync) fetchBlocksGraphSync(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
immediateTsSelector := firstTipsetSelector(req)
// Do this because we can only request one root at a time
for _, r := range req.Start {
if err := bs.executeGsyncSelector(ctx, p, r, immediateTsSelector); err != nil {
return nil, err
}
}
if req.RequestLength > maxRequestLength {
req.RequestLength = maxRequestLength
}
sel := selectorForRequest(req)
// execute the selector forreal
if err := bs.executeGsyncSelector(ctx, p, req.Start[0], sel); err != nil {
return nil, err
}
// Now pull the data we fetched out of the chainstore (where it should now be persisted)
tempcs := store.NewChainStore(bs.bserv.Blockstore(), datastore.NewMapDatastore(), nil)
opts := ParseBSOptions(req.Options)
tsk := types.NewTipSetKey(req.Start...)
chain, err := collectChainSegment(tempcs, tsk, req.RequestLength, opts)
if err != nil {
return nil, xerrors.Errorf("failed to load chain data from chainstore after successful graphsync response (start = %v): %w", req.Start, err)
}
return &BlockSyncResponse{Chain: chain}, nil
}

170
chain/blocksync/peer_tracker.go Normal file
View File

@ -0,0 +1,170 @@
package blocksync
// FIXME: This needs to be reviewed.
import (
"sort"
"sync"
"time"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/lib/peermgr"
)
type peerStats struct {
successes int
failures int
firstSeen time.Time
averageTime time.Duration
}
type bsPeerTracker struct {
lk sync.Mutex
peers map[peer.ID]*peerStats
avgGlobalTime time.Duration
pmgr *peermgr.PeerMgr
}
func newPeerTracker(pmgr *peermgr.PeerMgr) *bsPeerTracker {
return &bsPeerTracker{
peers: make(map[peer.ID]*peerStats),
pmgr: pmgr,
}
}
func (bpt *bsPeerTracker) addPeer(p peer.ID) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
if _, ok := bpt.peers[p]; ok {
return
}
bpt.peers[p] = &peerStats{
firstSeen: build.Clock.Now(),
}
}
const (
// newPeerMul is how much better than average is the new peer assumed to be
// less than one to encourouge trying new peers
newPeerMul = 0.9
)
func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
// TODO: this could probably be cached, but as long as its not too many peers, fine for now
bpt.lk.Lock()
defer bpt.lk.Unlock()
out := make([]peer.ID, 0, len(bpt.peers))
for p := range bpt.peers {
out = append(out, p)
}
// sort by 'expected cost' of requesting data from that peer
// additionally handle edge cases where not enough data is available
sort.Slice(out, func(i, j int) bool {
pi := bpt.peers[out[i]]
pj := bpt.peers[out[j]]
var costI, costJ float64
getPeerInitLat := func(p peer.ID) float64 {
var res float64
if bpt.pmgr != nil {
if lat, ok := bpt.pmgr.GetPeerLatency(p); ok {
res = float64(lat)
}
}
if res == 0 {
res = float64(bpt.avgGlobalTime)
}
return res * newPeerMul
}
if pi.successes+pi.failures > 0 {
failRateI := float64(pi.failures) / float64(pi.failures+pi.successes)
costI = float64(pi.averageTime) + failRateI*float64(bpt.avgGlobalTime)
} else {
costI = getPeerInitLat(out[i])
}
if pj.successes+pj.failures > 0 {
failRateJ := float64(pj.failures) / float64(pj.failures+pj.successes)
costJ = float64(pj.averageTime) + failRateJ*float64(bpt.avgGlobalTime)
} else {
costJ = getPeerInitLat(out[j])
}
return costI < costJ
})
return out
}
const (
// xInvAlpha = (N+1)/2
localInvAlpha = 5 // 86% of the value is the last 9
globalInvAlpha = 20 // 86% of the value is the last 39
)
func (bpt *bsPeerTracker) logGlobalSuccess(dur time.Duration) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
if bpt.avgGlobalTime == 0 {
bpt.avgGlobalTime = dur
return
}
delta := (dur - bpt.avgGlobalTime) / globalInvAlpha
bpt.avgGlobalTime += delta
}
func logTime(pi *peerStats, dur time.Duration) {
if pi.averageTime == 0 {
pi.averageTime = dur
return
}
delta := (dur - pi.averageTime) / localInvAlpha
pi.averageTime += delta
}
func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
var pi *peerStats
var ok bool
if pi, ok = bpt.peers[p]; !ok {
log.Warnw("log success called on peer not in tracker", "peerid", p.String())
return
}
pi.successes++
logTime(pi, dur)
}
func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
var pi *peerStats
var ok bool
if pi, ok = bpt.peers[p]; !ok {
log.Warn("log failure called on peer not in tracker", "peerid", p.String())
return
}
pi.failures++
logTime(pi, dur)
}
func (bpt *bsPeerTracker) removePeer(p peer.ID) {
bpt.lk.Lock()
defer bpt.lk.Unlock()
delete(bpt.peers, p)
}

194
chain/blocksync/protocol.go Normal file
View File

@ -0,0 +1,194 @@
package blocksync
import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store"
"time"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/types"
)
var log = logging.Logger("blocksync")
const BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
// FIXME: Bumped from original 800 to this to accommodate `syncFork()`
// use of `GetBlocks()`. It seems the expectation of that API is to
// fetch any amount of blocks leaving it to the internal logic here
// to partition and reassemble the requests if they go above the maximum.
// (Also as a consequence of this temporarily removing the `const`
// qualifier to avoid "const initializer [...] is not a constant" error.)
var MaxRequestLength = uint64(build.ForkLengthThreshold)
// Extracted constants from the code.
// FIXME: Should be reviewed and confirmed.
const SUCCESS_PEER_TAG_VALUE = 25
const WRITE_REQ_DEADLINE = 5 * time.Second
const READ_RES_DEADLINE = WRITE_REQ_DEADLINE
const READ_RES_MIN_SPEED = 50 << 10
const SHUFFLE_PEERS_PREFIX = 5
const WRITE_RES_DEADLINE = 60 * time.Second
// FIXME: Rename. Make private.
type Request struct {
// List of ordered CIDs comprising a `TipSetKey` from where to start
// fetching backwards.
// FIXME: Consider using `TipSetKey` now (introduced after the creation
// of this protocol) instead of converting back and forth.
Head []cid.Cid
// Number of block sets to fetch from `Head` (inclusive, should always
// be in the range `[1, MaxRequestLength]`).
Length uint64
// Request options, see `Options` type for more details. Compressed
// in a single `uint64` to save space.
Options uint64
}
// `Request` processed and validated to query the tipsets needed.
type validatedRequest struct {
head types.TipSetKey
length uint64
options *parsedOptions
}
// Request options. When fetching the chain segment we can fetch
// either block headers, messages, or both.
const (
Headers = 1 << iota
Messages
)
// Decompressed options into separate struct members for easy access
// during internal processing..
type parsedOptions struct {
IncludeHeaders bool
IncludeMessages bool
}
func (options *parsedOptions) noOptionsSet() bool {
return options.IncludeHeaders == false &&
options.IncludeMessages == false
}
func parseOptions(optfield uint64) *parsedOptions {
return &parsedOptions{
IncludeHeaders: optfield&(uint64(Headers)) != 0,
IncludeMessages: optfield&(uint64(Messages)) != 0,
}
}
// FIXME: Rename. Make private.
type Response struct {
Status status
// String that complements the error status when converting to an
// internal error (see `statusToError()`).
ErrorMessage string
Chain []*BSTipSet
}
type status uint64
const (
Ok status = 0
// We could not fetch all blocks requested (but at least we returned
// the `Head` requested). Not considered an error.
Partial = 101
// Errors
NotFound = 201
GoAway = 202
InternalError = 203
BadRequest = 204
)
// Convert status to internal error.
func (res *Response) statusToError() error {
switch res.Status {
case Ok, Partial:
return nil
// FIXME: Consider if we want to not process `Partial` responses
// and return an error instead.
case NotFound:
return xerrors.Errorf("not found")
case GoAway:
return xerrors.Errorf("not handling 'go away' blocksync responses yet")
case InternalError:
return xerrors.Errorf("block sync peer errored: %s", res.ErrorMessage)
case BadRequest:
return xerrors.Errorf("block sync request invalid: %s", res.ErrorMessage)
default:
return xerrors.Errorf("unrecognized response code: %d", res.Status)
}
}
// FIXME: Rename.
type BSTipSet struct {
Blocks []*types.BlockHeader
Messages *CompactedMessages
}
// All messages of a single tipset compacted together instead
// of grouped by block to save space, since there are normally
// many repeated messages per tipset in different blocks.
//
// `BlsIncludes`/`SecpkIncludes` matches `Bls`/`Secpk` messages
// to blocks in the tipsets with the format:
// `BlsIncludes[BI][MI]`
// * BI: block index in the tipset.
// * MI: message index in `Bls` list
//
// FIXME: The logic to decompress this structure should belong
// to itself, not to the consumer.
type CompactedMessages struct {
Bls []*types.Message
BlsIncludes [][]uint64
Secpk []*types.SignedMessage
SecpkIncludes [][]uint64
}
// Response that has been validated according to the protocol
// and can be safely accessed.
type validatedResponse struct {
tipsets []*types.TipSet
// List of all messages per tipset (grouped by tipset,
// not by block, hence a single index like `tipsets`).
messages []*CompactedMessages
}
// Decompress messages and form full tipsets with them. The headers
// need to have been requested as well.
func (res *validatedResponse) toFullTipSets() []*store.FullTipSet {
if len(res.tipsets) == 0 || len(res.tipsets) != len(res.messages) {
// This decompression can only be done if both headers and
// messages are returned in the response. (The second check
// is already implied by the guarantees of `validatedResponse`,
// added here just for completeness.)
return nil
}
ftsList := make([]*store.FullTipSet, len(res.tipsets))
for tipsetIdx := range res.tipsets {
fts := &store.FullTipSet{} // FIXME: We should use the `NewFullTipSet` API.
msgs := res.messages[tipsetIdx]
for blockIdx, b := range res.tipsets[tipsetIdx].Blocks() {
fb := &types.FullBlock{
Header: b,
}
for _, mi := range msgs.BlsIncludes[blockIdx] {
fb.BlsMessages = append(fb.BlsMessages, msgs.Bls[mi])
}
for _, mi := range msgs.SecpkIncludes[blockIdx] {
fb.SecpkMessages = append(fb.SecpkMessages, msgs.Secpk[mi])
}
fts.Blocks = append(fts.Blocks, fb)
}
ftsList[tipsetIdx] = fts
}
return ftsList
}

263
chain/blocksync/server.go Normal file
View File

@ -0,0 +1,263 @@
package blocksync
import (
"bufio"
"context"
"fmt"
"time"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
inet "github.com/libp2p/go-libp2p-core/network"
)
// BlockSyncService is the component that services BlockSync requests from
// peers.
//
// BlockSync is the basic chain synchronization protocol of Filecoin. BlockSync
// is an RPC-oriented protocol, with a single operation to request blocks.
//
// A request contains a start anchor block (referred to with a CID), and a
// amount of blocks requested beyond the anchor (including the anchor itself).
//
// A client can also pass options, encoded as a 64-bit bitfield. Lotus supports
// two options at the moment:
//
// - include block contents
// - include block messages
//
// The response will include a status code, an optional message, and the
// response payload in case of success. The payload is a slice of serialized
// tipsets.
// FIXME: Rename to just `Server` (will be done later, see note on `BlockSync`).
type BlockSyncService struct {
cs *store.ChainStore
}
func NewBlockSyncService(cs *store.ChainStore) *BlockSyncService {
return &BlockSyncService{
cs: cs,
}
}
// Entry point of the service, handles `Request`s.
func (server *BlockSyncService) HandleStream(stream inet.Stream) {
ctx, span := trace.StartSpan(context.Background(), "blocksync.HandleStream")
defer span.End()
defer stream.Close() //nolint:errcheck
var req Request
if err := cborutil.ReadCborRPC(bufio.NewReader(stream), &req); err != nil {
log.Warnf("failed to read block sync request: %s", err)
return
}
log.Infow("block sync request",
"start", req.Head, "len", req.Length)
resp, err := server.processRequest(ctx, &req)
if err != nil {
log.Warn("failed to process request: ", err)
return
}
_ = stream.SetDeadline(time.Now().Add(WRITE_RES_DEADLINE))
if err := cborutil.WriteCborRPC(stream, resp); err != nil {
_ = stream.SetDeadline(time.Time{})
log.Warnw("failed to write back response for handle stream",
"err", err, "peer", stream.Conn().RemotePeer())
return
}
_ = stream.SetDeadline(time.Time{})
}
// Validate and service the request. We return either a protocol
// response or an internal error.
func (server *BlockSyncService) processRequest(
ctx context.Context,
req *Request,
) (*Response, error) {
validReq, errResponse := validateRequest(ctx, req)
if errResponse != nil {
// The request did not pass validation, return the response
// indicating it.
return errResponse, nil
}
return server.serviceRequest(ctx, validReq)
}
// Validate request. We either return a `validatedRequest`, or an error
// `Response` indicating why we can't process it. We do not return any
// internal errors here, we just signal protocol ones.
func validateRequest(
ctx context.Context,
req *Request,
) (*validatedRequest, *Response) {
_, span := trace.StartSpan(ctx, "blocksync.ValidateRequest")
defer span.End()
validReq := validatedRequest{}
validReq.options = parseOptions(req.Options)
if validReq.options.noOptionsSet() {
return nil, &Response{
Status: BadRequest,
ErrorMessage: "no options set",
}
}
validReq.length = req.Length
if validReq.length > MaxRequestLength {
return nil, &Response{
Status: BadRequest,
ErrorMessage: fmt.Sprintf("request length over maximum allowed (%d)",
MaxRequestLength),
}
}
if validReq.length == 0 {
return nil, &Response{
Status: BadRequest,
ErrorMessage: "invalid request length of zero",
}
}
if len(req.Head) == 0 {
return nil, &Response{
Status: BadRequest,
ErrorMessage: "no cids in request",
}
}
validReq.head = types.NewTipSetKey(req.Head...)
// FIXME: Add as a defer at the start.
span.AddAttributes(
trace.BoolAttribute("blocks", validReq.options.IncludeHeaders),
trace.BoolAttribute("messages", validReq.options.IncludeMessages),
trace.Int64Attribute("reqlen", int64(validReq.length)),
)
return &validReq, nil
}
func (server *BlockSyncService) serviceRequest(
ctx context.Context,
req *validatedRequest,
) (*Response, error) {
_, span := trace.StartSpan(ctx, "blocksync.ServiceRequest")
defer span.End()
chain, err := collectChainSegment(server.cs, req)
if err != nil {
log.Warn("block sync request: collectChainSegment failed: ", err)
return &Response{
Status: InternalError,
ErrorMessage: err.Error(),
}, nil
}
status := Ok
if len(chain) < int(req.length) {
status = Partial
}
return &Response{
Chain: chain,
Status: status,
}, nil
}
func collectChainSegment(
cs *store.ChainStore,
req *validatedRequest,
) ([]*BSTipSet, error) {
var bstips []*BSTipSet
cur := req.head
for {
var bst BSTipSet
ts, err := cs.LoadTipSet(cur)
if err != nil {
return nil, xerrors.Errorf("failed loading tipset %s: %w", cur, err)
}
if req.options.IncludeHeaders {
bst.Blocks = ts.Blocks()
}
if req.options.IncludeMessages {
bmsgs, bmincl, smsgs, smincl, err := gatherMessages(cs, ts)
if err != nil {
return nil, xerrors.Errorf("gather messages failed: %w", err)
}
// FIXME: Pass the response to `gatherMessages()` and set all this there.
bst.Messages = &CompactedMessages{}
bst.Messages.Bls = bmsgs
bst.Messages.BlsIncludes = bmincl
bst.Messages.Secpk = smsgs
bst.Messages.SecpkIncludes = smincl
}
bstips = append(bstips, &bst)
// If we collected the length requested or if we reached the
// start (genesis), then stop.
if uint64(len(bstips)) >= req.length || ts.Height() == 0 {
return bstips, nil
}
cur = ts.Parents()
}
}
func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
blsmsgmap := make(map[cid.Cid]uint64)
secpkmsgmap := make(map[cid.Cid]uint64)
var secpkmsgs []*types.SignedMessage
var blsmsgs []*types.Message
var secpkincl, blsincl [][]uint64
for _, block := range ts.Blocks() {
bmsgs, smsgs, err := cs.MessagesForBlock(block)
if err != nil {
return nil, nil, nil, nil, err
}
// FIXME: DRY. Use `chain.Message` interface.
bmi := make([]uint64, 0, len(bmsgs))
for _, m := range bmsgs {
i, ok := blsmsgmap[m.Cid()]
if !ok {
i = uint64(len(blsmsgs))
blsmsgs = append(blsmsgs, m)
blsmsgmap[m.Cid()] = i
}
bmi = append(bmi, i)
}
blsincl = append(blsincl, bmi)
smi := make([]uint64, 0, len(smsgs))
for _, m := range smsgs {
i, ok := secpkmsgmap[m.Cid()]
if !ok {
i = uint64(len(secpkmsgs))
secpkmsgs = append(secpkmsgs, m)
secpkmsgmap[m.Cid()] = i
}
smi = append(smi, i)
}
secpkincl = append(secpkincl, smi)
}
return blsmsgs, blsincl, secpkmsgs, secpkincl, nil
}

2
chain/events/events.go
View File

@ -99,7 +99,7 @@ func (e *Events) listenHeadChanges(ctx context.Context) {
log.Warnf("not restarting listenHeadChanges: context error: %s", ctx.Err())
return
}
time.Sleep(time.Second)
build.Clock.Sleep(time.Second)
log.Info("restarting listenHeadChanges")
}
}

10
chain/events/events_called.go
View File

@ -485,13 +485,15 @@ func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID]eventDat
for tid, matchFns := range me.matchers {
var matched bool
var once bool
for _, matchFn := range matchFns {
ok, err := matchFn(msg)
matchOne, ok, err := matchFn(msg)
if err != nil {
log.Errorf("event matcher failed: %s", err)
continue
}
matched = ok
once = matchOne
if matched {
break
@ -500,7 +502,9 @@ func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID]eventDat
if matched {
res[tid] = msg
break
if once {
break
}
}
}
})
@ -548,7 +552,7 @@ func (me *messageEvents) messagesForTs(ts *types.TipSet, consume func(*types.Mes
// `curH`-`ts.Height` = `confidence`
type MsgHandler func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error)
type MsgMatchFunc func(msg *types.Message) (bool, error)
type MsgMatchFunc func(msg *types.Message) (matchOnce bool, matched bool, err error)
// Called registers a callback which is triggered when a specified method is
// called on an actor, or a timeout is reached.

6
chain/events/events_test.go
View File

@ -561,9 +561,9 @@ func TestAtChainedConfidenceNull(t *testing.T) {
require.Equal(t, false, reverted)
}
func matchAddrMethod(to address.Address, m abi.MethodNum) func(msg *types.Message) (bool, error) {
return func(msg *types.Message) (bool, error) {
return to == msg.To && m == msg.Method, nil
func matchAddrMethod(to address.Address, m abi.MethodNum) func(msg *types.Message) (matchOnce bool, matched bool, err error) {
return func(msg *types.Message) (matchOnce bool, matched bool, err error) {
return true, to == msg.To && m == msg.Method, nil
}
}

15
chain/events/state/diff_adt.go
View File

@ -1,6 +1,7 @@
package state
import (
"bytes"
"github.com/filecoin-project/specs-actors/actors/util/adt"
typegen "github.com/whyrusleeping/cbor-gen"
)
@ -39,8 +40,11 @@ func DiffAdtArray(preArr, curArr *adt.Array, out AdtArrayDiff) error {
return nil
}
if err := out.Modify(uint64(i), prevVal, curVal); err != nil {
return err
// no modification
if !bytes.Equal(prevVal.Raw, curVal.Raw) {
if err := out.Modify(uint64(i), prevVal, curVal); err != nil {
return err
}
}
return curArr.Delete(uint64(i))
@ -90,8 +94,11 @@ func DiffAdtMap(preMap, curMap *adt.Map, out AdtMapDiff) error {
return nil
}
if err := out.Modify(key, prevVal, curVal); err != nil {
return err
// no modification
if !bytes.Equal(prevVal.Raw, curVal.Raw) {
if err := out.Modify(key, prevVal, curVal); err != nil {
return err
}
}
return curMap.Delete(k)

195
chain/events/state/diff_adt_test.go
View File

@ -8,14 +8,13 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
ds "github.com/ipfs/go-datastore"
ds_sync "github.com/ipfs/go-datastore/sync"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbornode "github.com/ipfs/go-ipld-cbor"
typegen "github.com/whyrusleeping/cbor-gen"
"github.com/filecoin-project/specs-actors/actors/runtime"
"github.com/filecoin-project/specs-actors/actors/util/adt"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func TestDiffAdtArray(t *testing.T) {
@ -41,7 +40,7 @@ func TestDiffAdtArray(t *testing.T) {
require.NoError(t, arrB.Set(5, runtime.CBORBytes{8})) // add
require.NoError(t, arrB.Set(6, runtime.CBORBytes{9})) // add
changes := new(TestAdtDiff)
changes := new(TestDiffArray)
assert.NoError(t, DiffAdtArray(arrA, arrB, changes))
assert.NotNil(t, changes)
@ -72,38 +71,186 @@ func TestDiffAdtArray(t *testing.T) {
assert.EqualValues(t, []byte{1}, changes.Removed[1].val)
}
type adtDiffResult struct {
key uint64
val runtime.CBORBytes
func TestDiffAdtMap(t *testing.T) {
ctxstoreA := newContextStore()
ctxstoreB := newContextStore()
mapA := adt.MakeEmptyMap(ctxstoreA)
mapB := adt.MakeEmptyMap(ctxstoreB)
require.NoError(t, mapA.Put(adt.UIntKey(0), runtime.CBORBytes([]byte{0}))) // delete
require.NoError(t, mapA.Put(adt.UIntKey(1), runtime.CBORBytes([]byte{0}))) // modify
require.NoError(t, mapB.Put(adt.UIntKey(1), runtime.CBORBytes([]byte{1})))
require.NoError(t, mapA.Put(adt.UIntKey(2), runtime.CBORBytes([]byte{1}))) // delete
require.NoError(t, mapA.Put(adt.UIntKey(3), runtime.CBORBytes([]byte{0}))) // noop
require.NoError(t, mapB.Put(adt.UIntKey(3), runtime.CBORBytes([]byte{0})))
require.NoError(t, mapA.Put(adt.UIntKey(4), runtime.CBORBytes([]byte{0}))) // modify
require.NoError(t, mapB.Put(adt.UIntKey(4), runtime.CBORBytes([]byte{6})))
require.NoError(t, mapB.Put(adt.UIntKey(5), runtime.CBORBytes{8})) // add
require.NoError(t, mapB.Put(adt.UIntKey(6), runtime.CBORBytes{9})) // add
changes := new(TestDiffMap)
assert.NoError(t, DiffAdtMap(mapA, mapB, changes))
assert.NotNil(t, changes)
assert.Equal(t, 2, len(changes.Added))
// keys 5 and 6 were added
assert.EqualValues(t, uint64(6), changes.Added[0].key)
assert.EqualValues(t, []byte{9}, changes.Added[0].val)
assert.EqualValues(t, uint64(5), changes.Added[1].key)
assert.EqualValues(t, []byte{8}, changes.Added[1].val)
assert.Equal(t, 2, len(changes.Modified))
// keys 1 and 4 were modified
assert.EqualValues(t, uint64(1), changes.Modified[0].From.key)
assert.EqualValues(t, []byte{0}, changes.Modified[0].From.val)
assert.EqualValues(t, uint64(1), changes.Modified[0].To.key)
assert.EqualValues(t, []byte{1}, changes.Modified[0].To.val)
assert.EqualValues(t, uint64(4), changes.Modified[1].From.key)
assert.EqualValues(t, []byte{0}, changes.Modified[1].From.val)
assert.EqualValues(t, uint64(4), changes.Modified[1].To.key)
assert.EqualValues(t, []byte{6}, changes.Modified[1].To.val)
assert.Equal(t, 2, len(changes.Removed))
// keys 0 and 2 were deleted
assert.EqualValues(t, uint64(0), changes.Removed[0].key)
assert.EqualValues(t, []byte{0}, changes.Removed[0].val)
assert.EqualValues(t, uint64(2), changes.Removed[1].key)
assert.EqualValues(t, []byte{1}, changes.Removed[1].val)
}
type TestAdtDiff struct {
Added []adtDiffResult
Modified []TestAdtDiffModified
Removed []adtDiffResult
type TestDiffMap struct {
Added []adtMapDiffResult
Modified []TestAdtMapDiffModified
Removed []adtMapDiffResult
}
var _ AdtArrayDiff = &TestAdtDiff{}
var _ AdtMapDiff = &TestDiffMap{}
type TestAdtDiffModified struct {
From adtDiffResult
To adtDiffResult
func (t *TestDiffMap) AsKey(key string) (adt.Keyer, error) {
k, err := adt.ParseUIntKey(key)
if err != nil {
return nil, err
}
return adt.UIntKey(k), nil
}
func (t *TestAdtDiff) Add(key uint64, val *typegen.Deferred) error {
func (t *TestDiffMap) Add(key string, val *typegen.Deferred) error {
v := new(runtime.CBORBytes)
err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
if err != nil {
return err
}
t.Added = append(t.Added, adtDiffResult{
k, err := adt.ParseUIntKey(key)
if err != nil {
return err
}
t.Added = append(t.Added, adtMapDiffResult{
key: k,
val: *v,
})
return nil
}
func (t *TestDiffMap) Modify(key string, from, to *typegen.Deferred) error {
vFrom := new(runtime.CBORBytes)
err := vFrom.UnmarshalCBOR(bytes.NewReader(from.Raw))
if err != nil {
return err
}
vTo := new(runtime.CBORBytes)
err = vTo.UnmarshalCBOR(bytes.NewReader(to.Raw))
if err != nil {
return err
}
k, err := adt.ParseUIntKey(key)
if err != nil {
return err
}
if !bytes.Equal(*vFrom, *vTo) {
t.Modified = append(t.Modified, TestAdtMapDiffModified{
From: adtMapDiffResult{
key: k,
val: *vFrom,
},
To: adtMapDiffResult{
key: k,
val: *vTo,
},
})
}
return nil
}
func (t *TestDiffMap) Remove(key string, val *typegen.Deferred) error {
v := new(runtime.CBORBytes)
err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
if err != nil {
return err
}
k, err := adt.ParseUIntKey(key)
if err != nil {
return err
}
t.Removed = append(t.Removed, adtMapDiffResult{
key: k,
val: *v,
})
return nil
}
type adtMapDiffResult struct {
key uint64
val runtime.CBORBytes
}
type TestAdtMapDiffModified struct {
From adtMapDiffResult
To adtMapDiffResult
}
type adtArrayDiffResult struct {
key uint64
val runtime.CBORBytes
}
type TestDiffArray struct {
Added []adtArrayDiffResult
Modified []TestAdtArrayDiffModified
Removed []adtArrayDiffResult
}
var _ AdtArrayDiff = &TestDiffArray{}
type TestAdtArrayDiffModified struct {
From adtArrayDiffResult
To adtArrayDiffResult
}
func (t *TestDiffArray) Add(key uint64, val *typegen.Deferred) error {
v := new(runtime.CBORBytes)
err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
if err != nil {
return err
}
t.Added = append(t.Added, adtArrayDiffResult{
key: key,
val: *v,
})
return nil
}
func (t *TestAdtDiff) Modify(key uint64, from, to *typegen.Deferred) error {
func (t *TestDiffArray) Modify(key uint64, from, to *typegen.Deferred) error {
vFrom := new(runtime.CBORBytes)
err := vFrom.UnmarshalCBOR(bytes.NewReader(from.Raw))
if err != nil {
@ -117,12 +264,12 @@ func (t *TestAdtDiff) Modify(key uint64, from, to *typegen.Deferred) error {
}
if !bytes.Equal(*vFrom, *vTo) {
t.Modified = append(t.Modified, TestAdtDiffModified{
From: adtDiffResult{
t.Modified = append(t.Modified, TestAdtArrayDiffModified{
From: adtArrayDiffResult{
key: key,
val: *vFrom,
},
To: adtDiffResult{
To: adtArrayDiffResult{
key: key,
val: *vTo,
},
@ -131,13 +278,13 @@ func (t *TestAdtDiff) Modify(key uint64, from, to *typegen.Deferred) error {
return nil
}
func (t *TestAdtDiff) Remove(key uint64, val *typegen.Deferred) error {
func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error {
v := new(runtime.CBORBytes)
err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
if err != nil {
return err
}
t.Removed = append(t.Removed, adtDiffResult{
t.Removed = append(t.Removed, adtArrayDiffResult{
key: key,
val: *v,
})
@ -146,7 +293,7 @@ func (t *TestAdtDiff) Remove(key uint64, val *typegen.Deferred) error {
func newContextStore() *contextStore {
ctx := context.Background()
bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
bs := bstore.NewTemporarySync()
store := cbornode.NewCborStore(bs)
return &contextStore{
ctx: ctx,

318
chain/events/state/predicates.go
View File

@ -4,16 +4,18 @@ import (
"bytes"
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/builtin/paych"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
typegen "github.com/whyrusleeping/cbor-gen"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/chain/types"
)
@ -84,6 +86,50 @@ func (sp *StatePredicates) OnStorageMarketActorChanged(diffStorageMarketState Di
})
}
type BalanceTables struct {
EscrowTable *adt.BalanceTable
LockedTable *adt.BalanceTable
}
// DiffBalanceTablesFunc compares two balance tables
type DiffBalanceTablesFunc func(ctx context.Context, oldBalanceTable, newBalanceTable BalanceTables) (changed bool, user UserData, err error)
// OnBalanceChanged runs when the escrow table for available balances changes
func (sp *StatePredicates) OnBalanceChanged(diffBalances DiffBalanceTablesFunc) DiffStorageMarketStateFunc {
return func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error) {
if oldState.EscrowTable.Equals(newState.EscrowTable) && oldState.LockedTable.Equals(newState.LockedTable) {
return false, nil, nil
}
ctxStore := &contextStore{
ctx: ctx,
cst: sp.cst,
}
oldEscrowRoot, err := adt.AsBalanceTable(ctxStore, oldState.EscrowTable)
if err != nil {
return false, nil, err
}
oldLockedRoot, err := adt.AsBalanceTable(ctxStore, oldState.LockedTable)
if err != nil {
return false, nil, err
}
newEscrowRoot, err := adt.AsBalanceTable(ctxStore, newState.EscrowTable)
if err != nil {
return false, nil, err
}
newLockedRoot, err := adt.AsBalanceTable(ctxStore, newState.LockedTable)
if err != nil {
return false, nil, err
}
return diffBalances(ctx, BalanceTables{oldEscrowRoot, oldLockedRoot}, BalanceTables{newEscrowRoot, newLockedRoot})
}
}
type DiffAdtArraysFunc func(ctx context.Context, oldDealStateRoot, newDealStateRoot *adt.Array) (changed bool, user UserData, err error)
// OnDealStateChanged calls diffDealStates when the market deal state changes
@ -279,17 +325,22 @@ func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffAdtA
var oldDealPtr, newDealPtr *market.DealState
var oldDeal, newDeal market.DealState
_, err := oldDealStateArray.Get(uint64(dealID), &oldDeal)
// If the deal has been removed, we just set it to nil
found, err := oldDealStateArray.Get(uint64(dealID), &oldDeal)
if err != nil {
return false, nil, err
}
oldDealPtr = &oldDeal
if found {
oldDealPtr = &oldDeal
}
_, err = newDealStateArray.Get(uint64(dealID), &newDeal)
found, err = newDealStateArray.Get(uint64(dealID), &newDeal)
if err != nil {
return false, nil, err
}
newDealPtr = &newDeal
if found {
newDealPtr = &newDeal
}
if oldDeal != newDeal {
changedDeals[dealID] = DealStateChange{dealID, oldDealPtr, newDealPtr}
@ -302,8 +353,74 @@ func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffAdtA
}
}
// ChangedBalances is a set of changes to deal state
type ChangedBalances map[address.Address]BalanceChange
// BalanceChange is a change in balance from -> to
type BalanceChange struct {
From abi.TokenAmount
To abi.TokenAmount
}
// AvailableBalanceChangedForAddresses detects changes in the escrow table for the given addresses
func (sp *StatePredicates) AvailableBalanceChangedForAddresses(getAddrs func() []address.Address) DiffBalanceTablesFunc {
return func(ctx context.Context, oldBalances, newBalances BalanceTables) (changed bool, user UserData, err error) {
changedBalances := make(ChangedBalances)
addrs := getAddrs()
for _, addr := range addrs {
// If the deal has been removed, we just set it to nil
oldEscrowBalance, err := oldBalances.EscrowTable.Get(addr)
if err != nil {
return false, nil, err
}
oldLockedBalance, err := oldBalances.LockedTable.Get(addr)
if err != nil {
return false, nil, err
}
oldBalance := big.Sub(oldEscrowBalance, oldLockedBalance)
newEscrowBalance, err := newBalances.EscrowTable.Get(addr)
if err != nil {
return false, nil, err
}
newLockedBalance, err := newBalances.LockedTable.Get(addr)
if err != nil {
return false, nil, err
}
newBalance := big.Sub(newEscrowBalance, newLockedBalance)
if !oldBalance.Equals(newBalance) {
changedBalances[addr] = BalanceChange{oldBalance, newBalance}
}
}
if len(changedBalances) > 0 {
return true, changedBalances, nil
}
return false, nil, nil
}
}
type DiffMinerActorStateFunc func(ctx context.Context, oldState *miner.State, newState *miner.State) (changed bool, user UserData, err error)
func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorStateFunc) DiffTipSetKeyFunc {
return sp.OnActorStateChanged(builtin.InitActorAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) {
var oldState init_.State
if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil {
return false, nil, err
}
var newState init_.State
if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil {
return false, nil, err
}
return diffInitActorState(ctx, &oldState, &newState)
})
}
func (sp *StatePredicates) OnMinerActorChange(minerAddr address.Address, diffMinerActorState DiffMinerActorStateFunc) DiffTipSetKeyFunc {
return sp.OnActorStateChanged(minerAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) {
var oldState miner.State
@ -354,7 +471,7 @@ func (m *MinerSectorChanges) Modify(key uint64, from, to *typegen.Deferred) erro
return err
}
if siFrom.Info.Expiration != siTo.Info.Expiration {
if siFrom.Expiration != siTo.Expiration {
m.Extended = append(m.Extended, SectorExtensions{
From: *siFrom,
To: *siTo,
@ -488,3 +605,182 @@ func (sp *StatePredicates) OnMinerPreCommitChange() DiffMinerActorStateFunc {
return true, precommitChanges, nil
}
}
// DiffPaymentChannelStateFunc is function that compares two states for the payment channel
type DiffPaymentChannelStateFunc func(ctx context.Context, oldState *paych.State, newState *paych.State) (changed bool, user UserData, err error)
// OnPaymentChannelActorChanged calls diffPaymentChannelState when the state changes for the the payment channel actor
func (sp *StatePredicates) OnPaymentChannelActorChanged(paychAddr address.Address, diffPaymentChannelState DiffPaymentChannelStateFunc) DiffTipSetKeyFunc {
return sp.OnActorStateChanged(paychAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) {
var oldState paych.State
if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil {
return false, nil, err
}
var newState paych.State
if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil {
return false, nil, err
}
return diffPaymentChannelState(ctx, &oldState, &newState)
})
}
// PayChToSendChange is a difference in the amount to send on a payment channel when the money is collected
type PayChToSendChange struct {
OldToSend abi.TokenAmount
NewToSend abi.TokenAmount
}
// OnToSendAmountChanges monitors changes on the total amount to send from one party to the other on a payment channel
func (sp *StatePredicates) OnToSendAmountChanges() DiffPaymentChannelStateFunc {
return func(ctx context.Context, oldState *paych.State, newState *paych.State) (changed bool, user UserData, err error) {
if oldState.ToSend.Equals(newState.ToSend) {
return false, nil, nil
}
return true, &PayChToSendChange{
OldToSend: oldState.ToSend,
NewToSend: newState.ToSend,
}, nil
}
}
type AddressPair struct {
ID address.Address
PK address.Address
}
type InitActorAddressChanges struct {
Added []AddressPair
Modified []AddressChange
Removed []AddressPair
}
type AddressChange struct {
From AddressPair
To AddressPair
}
type DiffInitActorStateFunc func(ctx context.Context, oldState *init_.State, newState *init_.State) (changed bool, user UserData, err error)
func (i *InitActorAddressChanges) AsKey(key string) (adt.Keyer, error) {
addr, err := address.NewFromBytes([]byte(key))
if err != nil {
return nil, err
}
return adt.AddrKey(addr), nil
}
func (i *InitActorAddressChanges) Add(key string, val *typegen.Deferred) error {
pkAddr, err := address.NewFromBytes([]byte(key))
if err != nil {
return err
}
id := new(typegen.CborInt)
if err := id.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
return err
}
idAddr, err := address.NewIDAddress(uint64(*id))
if err != nil {
return err
}
i.Added = append(i.Added, AddressPair{
ID: idAddr,
PK: pkAddr,
})
return nil
}
func (i *InitActorAddressChanges) Modify(key string, from, to *typegen.Deferred) error {
pkAddr, err := address.NewFromBytes([]byte(key))
if err != nil {
return err
}
fromID := new(typegen.CborInt)
if err := fromID.UnmarshalCBOR(bytes.NewReader(from.Raw)); err != nil {
return err
}
fromIDAddr, err := address.NewIDAddress(uint64(*fromID))
if err != nil {
return err
}
toID := new(typegen.CborInt)
if err := toID.UnmarshalCBOR(bytes.NewReader(to.Raw)); err != nil {
return err
}
toIDAddr, err := address.NewIDAddress(uint64(*toID))
if err != nil {
return err
}
i.Modified = append(i.Modified, AddressChange{
From: AddressPair{
ID: fromIDAddr,
PK: pkAddr,
},
To: AddressPair{
ID: toIDAddr,
PK: pkAddr,
},
})
return nil
}
func (i *InitActorAddressChanges) Remove(key string, val *typegen.Deferred) error {
pkAddr, err := address.NewFromBytes([]byte(key))
if err != nil {
return err
}
id := new(typegen.CborInt)
if err := id.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
return err
}
idAddr, err := address.NewIDAddress(uint64(*id))
if err != nil {
return err
}
i.Removed = append(i.Removed, AddressPair{
ID: idAddr,
PK: pkAddr,
})
return nil
}
func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc {
return func(ctx context.Context, oldState, newState *init_.State) (changed bool, user UserData, err error) {
ctxStore := &contextStore{
ctx: ctx,
cst: sp.cst,
}
addressChanges := &InitActorAddressChanges{
Added: []AddressPair{},
Modified: []AddressChange{},
Removed: []AddressPair{},
}
if oldState.AddressMap.Equals(newState.AddressMap) {
return false, nil, nil
}
oldAddrs, err := adt.AsMap(ctxStore, oldState.AddressMap)
if err != nil {
return false, nil, err
}
newAddrs, err := adt.AsMap(ctxStore, newState.AddressMap)
if err != nil {
return false, nil, err
}
if err := DiffAdtMap(oldAddrs, newAddrs, addressChanges); err != nil {
return false, nil, err
}
if len(addressChanges.Added)+len(addressChanges.Removed)+len(addressChanges.Modified) == 0 {
return false, nil, nil
}
return true, addressChanges, nil
}
}

238
chain/events/state/predicates_test.go
View File

@ -4,18 +4,15 @@ import (
"context"
"testing"
"github.com/filecoin-project/go-bitfield"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
ds_sync "github.com/ipfs/go-datastore/sync"
"github.com/ipfs/go-hamt-ipld"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbornode "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
@ -25,6 +22,7 @@ import (
tutils "github.com/filecoin-project/specs-actors/support/testing"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
var dummyCid cid.Cid
@ -68,8 +66,8 @@ func (m mockAPI) setActor(tsk types.TipSetKey, act *types.Actor) {
func TestMarketPredicates(t *testing.T) {
ctx := context.Background()
bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
store := cbornode.NewCborStore(bs)
bs := bstore.NewTemporarySync()
store := adt.WrapStore(ctx, cbornode.NewCborStore(bs))
oldDeal1 := &market.DealState{
SectorStartEpoch: 1,
@ -115,18 +113,23 @@ func TestMarketPredicates(t *testing.T) {
abi.DealID(2): oldProp2,
}
oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps)
oldBalances := map[address.Address]balance{
tutils.NewIDAddr(t, 1): balance{abi.NewTokenAmount(1000), abi.NewTokenAmount(1000)},
tutils.NewIDAddr(t, 2): balance{abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
tutils.NewIDAddr(t, 3): balance{abi.NewTokenAmount(3000), abi.NewTokenAmount(2000)},
tutils.NewIDAddr(t, 5): balance{abi.NewTokenAmount(3000), abi.NewTokenAmount(1000)},
}
oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps, oldBalances)
newDeal1 := &market.DealState{
SectorStartEpoch: 1,
LastUpdatedEpoch: 3,
SlashEpoch: 0,
}
newDeal2 := &market.DealState{
SectorStartEpoch: 4,
LastUpdatedEpoch: 6,
SlashEpoch: 6,
}
// deal 2 removed
// added
newDeal3 := &market.DealState{
SectorStartEpoch: 1,
@ -135,7 +138,7 @@ func TestMarketPredicates(t *testing.T) {
}
newDeals := map[abi.DealID]*market.DealState{
abi.DealID(1): newDeal1,
abi.DealID(2): newDeal2,
// deal 2 was removed
abi.DealID(3): newDeal3,
}
@ -158,7 +161,14 @@ func TestMarketPredicates(t *testing.T) {
abi.DealID(3): newProp3, // new
// NB: DealProposals cannot be modified, so don't test that case.
}
newStateC := createMarketState(ctx, t, store, newDeals, newProps)
newBalances := map[address.Address]balance{
tutils.NewIDAddr(t, 1): balance{abi.NewTokenAmount(3000), abi.NewTokenAmount(0)},
tutils.NewIDAddr(t, 2): balance{abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
tutils.NewIDAddr(t, 4): balance{abi.NewTokenAmount(5000), abi.NewTokenAmount(0)},
tutils.NewIDAddr(t, 5): balance{abi.NewTokenAmount(1000), abi.NewTokenAmount(3000)},
}
newStateC := createMarketState(ctx, t, store, newDeals, newProps, newBalances)
minerAddr, err := address.NewFromString("t00")
require.NoError(t, err)
@ -197,10 +207,17 @@ func TestMarketPredicates(t *testing.T) {
t.Fatal("Unexpected change to LastUpdatedEpoch")
}
deal2 := changedDealIDs[abi.DealID(2)]
if deal2.From.SlashEpoch != 0 || deal2.To.SlashEpoch != 6 {
t.Fatal("Unexpected change to SlashEpoch")
if deal2.From.LastUpdatedEpoch != 5 || deal2.To != nil {
t.Fatal("Expected To to be nil")
}
// Diff with non-existent deal.
noDeal := []abi.DealID{4}
diffNoDealFn := preds.OnStorageMarketActorChanged(preds.OnDealStateChanged(preds.DealStateChangedForIDs(noDeal)))
changed, _, err = diffNoDealFn(ctx, oldState.Key(), newState.Key())
require.NoError(t, err)
require.False(t, changed)
// Test that OnActorStateChanged does not call the callback if the state has not changed
mockAddr, err := address.NewFromString("t01")
require.NoError(t, err)
@ -241,16 +258,14 @@ func TestMarketPredicates(t *testing.T) {
require.Equal(t, abi.DealID(3), changedDeals.Added[0].ID)
require.Equal(t, *newDeal3, changedDeals.Added[0].Deal)
require.Len(t, changedDeals.Removed, 0)
require.Len(t, changedDeals.Removed, 1)
require.Len(t, changedDeals.Modified, 2)
require.Len(t, changedDeals.Modified, 1)
require.Equal(t, abi.DealID(1), changedDeals.Modified[0].ID)
require.Equal(t, newDeal1, changedDeals.Modified[0].To)
require.Equal(t, oldDeal1, changedDeals.Modified[0].From)
require.Equal(t, abi.DealID(2), changedDeals.Modified[1].ID)
require.Equal(t, oldDeal2, changedDeals.Modified[1].From)
require.Equal(t, newDeal2, changedDeals.Modified[1].To)
require.Equal(t, abi.DealID(2), changedDeals.Removed[0].ID)
})
t.Run("deal proposal array predicate", func(t *testing.T) {
@ -276,12 +291,69 @@ func TestMarketPredicates(t *testing.T) {
require.Equal(t, abi.DealID(2), changedProps.Removed[0].ID)
require.Equal(t, *oldProp2, changedProps.Removed[0].Proposal)
})
t.Run("balances predicate", func(t *testing.T) {
preds := NewStatePredicates(api)
getAddresses := func() []address.Address {
return []address.Address{tutils.NewIDAddr(t, 1), tutils.NewIDAddr(t, 2), tutils.NewIDAddr(t, 3), tutils.NewIDAddr(t, 4)}
}
diffBalancesFn := preds.OnStorageMarketActorChanged(preds.OnBalanceChanged(preds.AvailableBalanceChangedForAddresses(getAddresses)))
// Diff a state against itself: expect no change
changed, _, err := diffBalancesFn(ctx, oldState.Key(), oldState.Key())
require.NoError(t, err)
require.False(t, changed)
// Diff old state against new state
changed, valIDs, err := diffBalancesFn(ctx, oldState.Key(), newState.Key())
require.NoError(t, err)
require.True(t, changed)
changedBalances, ok := valIDs.(ChangedBalances)
require.True(t, ok)
require.Len(t, changedBalances, 3)
require.Contains(t, changedBalances, tutils.NewIDAddr(t, 1))
require.Contains(t, changedBalances, tutils.NewIDAddr(t, 3))
require.Contains(t, changedBalances, tutils.NewIDAddr(t, 4))
balance1 := changedBalances[tutils.NewIDAddr(t, 1)]
if !balance1.From.Equals(abi.NewTokenAmount(1000)) || !balance1.To.Equals(abi.NewTokenAmount(3000)) {
t.Fatal("Unexpected change to balance")
}
balance3 := changedBalances[tutils.NewIDAddr(t, 3)]
if !balance3.From.Equals(abi.NewTokenAmount(3000)) || !balance3.To.Equals(abi.NewTokenAmount(0)) {
t.Fatal("Unexpected change to balance")
}
balance4 := changedBalances[tutils.NewIDAddr(t, 4)]
if !balance4.From.Equals(abi.NewTokenAmount(0)) || !balance4.To.Equals(abi.NewTokenAmount(5000)) {
t.Fatal("Unexpected change to balance")
}
// Diff with non-existent address.
getNoAddress := func() []address.Address { return []address.Address{tutils.NewIDAddr(t, 6)} }
diffNoAddressFn := preds.OnStorageMarketActorChanged(preds.OnBalanceChanged(preds.AvailableBalanceChangedForAddresses(getNoAddress)))
changed, _, err = diffNoAddressFn(ctx, oldState.Key(), newState.Key())
require.NoError(t, err)
require.False(t, changed)
// Test that OnBalanceChanged does not call the callback if the state has not changed
diffDealBalancesFn := preds.OnBalanceChanged(func(context.Context, BalanceTables, BalanceTables) (bool, UserData, error) {
t.Fatal("No state change so this should not be called")
return false, nil, nil
})
marketState := createEmptyMarketState(t, store)
changed, _, err = diffDealBalancesFn(ctx, marketState, marketState)
require.NoError(t, err)
require.False(t, changed)
})
}
func TestMinerSectorChange(t *testing.T) {
ctx := context.Background()
bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
store := cbornode.NewCborStore(bs)
bs := bstore.NewTemporarySync()
store := adt.WrapStore(ctx, cbornode.NewCborStore(bs))
nextID := uint64(0)
nextIDAddrF := func() address.Address {
@ -290,18 +362,18 @@ func TestMinerSectorChange(t *testing.T) {
}
owner, worker := nextIDAddrF(), nextIDAddrF()
si0 := newSectorOnChainInfo(0, tutils.MakeCID("0"), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10))
si1 := newSectorOnChainInfo(1, tutils.MakeCID("1"), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11))
si2 := newSectorOnChainInfo(2, tutils.MakeCID("2"), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11))
si0 := newSectorOnChainInfo(0, tutils.MakeCID("0", &miner.SealedCIDPrefix), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10))
si1 := newSectorOnChainInfo(1, tutils.MakeCID("1", &miner.SealedCIDPrefix), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11))
si2 := newSectorOnChainInfo(2, tutils.MakeCID("2", &miner.SealedCIDPrefix), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11))
oldMinerC := createMinerState(ctx, t, store, owner, worker, []miner.SectorOnChainInfo{si0, si1, si2})
si3 := newSectorOnChainInfo(3, tutils.MakeCID("3"), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12))
si3 := newSectorOnChainInfo(3, tutils.MakeCID("3", &miner.SealedCIDPrefix), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12))
// 0 delete
// 1 extend
// 2 same
// 3 added
si1Ext := si1
si1Ext.Info.Expiration++
si1Ext.Expiration++
newMinerC := createMinerState(ctx, t, store, owner, worker, []miner.SectorOnChainInfo{si1Ext, si2, si3})
minerAddr := nextIDAddrF()
@ -373,50 +445,82 @@ func mockTipset(minerAddr address.Address, timestamp uint64) (*types.TipSet, err
}})
}
func createMarketState(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, deals map[abi.DealID]*market.DealState, props map[abi.DealID]*market.DealProposal) cid.Cid {
type balance struct {
available abi.TokenAmount
locked abi.TokenAmount
}
func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market.DealState, props map[abi.DealID]*market.DealProposal, balances map[address.Address]balance) cid.Cid {
dealRootCid := createDealAMT(ctx, t, store, deals)
propRootCid := createProposalAMT(ctx, t, store, props)
balancesCids := createBalanceTable(ctx, t, store, balances)
state := createEmptyMarketState(t, store)
state.States = dealRootCid
state.Proposals = propRootCid
state.EscrowTable = balancesCids[0]
state.LockedTable = balancesCids[1]
stateC, err := store.Put(ctx, state)
require.NoError(t, err)
return stateC
}
func createEmptyMarketState(t *testing.T, store *cbornode.BasicIpldStore) *market.State {
emptyArrayCid, err := amt.NewAMT(store).Flush(context.TODO())
func createEmptyMarketState(t *testing.T, store adt.Store) *market.State {
emptyArrayCid, err := adt.MakeEmptyArray(store).Root()
require.NoError(t, err)
emptyMap, err := store.Put(context.TODO(), hamt.NewNode(store, hamt.UseTreeBitWidth(5)))
emptyMap, err := adt.MakeEmptyMap(store).Root()
require.NoError(t, err)
return market.ConstructState(emptyArrayCid, emptyMap, emptyMap)
}
func createDealAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, deals map[abi.DealID]*market.DealState) cid.Cid {
root := amt.NewAMT(store)
func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market.DealState) cid.Cid {
root := adt.MakeEmptyArray(store)
for dealID, dealState := range deals {
err := root.Set(ctx, uint64(dealID), dealState)
err := root.Set(uint64(dealID), dealState)
require.NoError(t, err)
}
rootCid, err := root.Flush(ctx)
rootCid, err := root.Root()
require.NoError(t, err)
return rootCid
}
func createProposalAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, props map[abi.DealID]*market.DealProposal) cid.Cid {
root := amt.NewAMT(store)
func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props map[abi.DealID]*market.DealProposal) cid.Cid {
root := adt.MakeEmptyArray(store)
for dealID, prop := range props {
err := root.Set(ctx, uint64(dealID), prop)
err := root.Set(uint64(dealID), prop)
require.NoError(t, err)
}
rootCid, err := root.Flush(ctx)
rootCid, err := root.Root()
require.NoError(t, err)
return rootCid
}
func createMinerState(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, owner, worker address.Address, sectors []miner.SectorOnChainInfo) cid.Cid {
func createBalanceTable(ctx context.Context, t *testing.T, store adt.Store, balances map[address.Address]balance) [2]cid.Cid {
escrowMapRoot := adt.MakeEmptyMap(store)
escrowMapRootCid, err := escrowMapRoot.Root()
require.NoError(t, err)
escrowRoot, err := adt.AsBalanceTable(store, escrowMapRootCid)
require.NoError(t, err)
lockedMapRoot := adt.MakeEmptyMap(store)
lockedMapRootCid, err := lockedMapRoot.Root()
require.NoError(t, err)
lockedRoot, err := adt.AsBalanceTable(store, lockedMapRootCid)
for addr, balance := range balances {
err := escrowRoot.Add(addr, big.Add(balance.available, balance.locked))
require.NoError(t, err)
err = lockedRoot.Add(addr, balance.locked)
require.NoError(t, err)
}
escrowRootCid, err := escrowRoot.Root()
require.NoError(t, err)
lockedRootCid, err := lockedRoot.Root()
require.NoError(t, err)
return [2]cid.Cid{escrowRootCid, lockedRootCid}
}
func createMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address, sectors []miner.SectorOnChainInfo) cid.Cid {
rootCid := createSectorsAMT(ctx, t, store, sectors)
state := createEmptyMinerState(ctx, t, store, owner, worker)
@ -427,30 +531,42 @@ func createMinerState(ctx context.Context, t *testing.T, store *cbornode.BasicIp
return stateC
}
func createEmptyMinerState(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, owner, worker address.Address) *miner.State {
emptyArrayCid, err := amt.NewAMT(store).Flush(context.TODO())
func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address) *miner.State {
emptyArrayCid, err := adt.MakeEmptyArray(store).Root()
require.NoError(t, err)
emptyMap, err := store.Put(context.TODO(), hamt.NewNode(store, hamt.UseTreeBitWidth(5)))
emptyMap, err := adt.MakeEmptyMap(store).Root()
require.NoError(t, err)
emptyDeadlines := miner.ConstructDeadlines()
emptyDeadlinesCid, err := store.Put(context.Background(), emptyDeadlines)
emptyDeadline, err := store.Put(store.Context(), miner.ConstructDeadline(emptyArrayCid))
require.NoError(t, err)
state, err := miner.ConstructState(emptyArrayCid, emptyMap, emptyDeadlinesCid, owner, worker, abi.PeerID{'1'}, nil, abi.RegisteredSealProof_StackedDrg64GiBV1, 0)
emptyVestingFunds := miner.ConstructVestingFunds()
emptyVestingFundsCid, err := store.Put(store.Context(), emptyVestingFunds)
emptyDeadlines := miner.ConstructDeadlines(emptyDeadline)
emptyDeadlinesCid, err := store.Put(store.Context(), emptyDeadlines)
require.NoError(t, err)
minerInfo := emptyMap
emptyBitfield := bitfield.NewFromSet(nil)
emptyBitfieldCid, err := store.Put(store.Context(), emptyBitfield)
require.NoError(t, err)
state, err := miner.ConstructState(minerInfo, 123, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid)
require.NoError(t, err)
return state
}
func createSectorsAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, sectors []miner.SectorOnChainInfo) cid.Cid {
root := amt.NewAMT(store)
func createSectorsAMT(ctx context.Context, t *testing.T, store adt.Store, sectors []miner.SectorOnChainInfo) cid.Cid {
root := adt.MakeEmptyArray(store)
for _, sector := range sectors {
sector := sector
err := root.Set(ctx, uint64(sector.Info.SectorNumber), &sector)
err := root.Set(uint64(sector.SectorNumber), &sector)
require.NoError(t, err)
}
rootCid, err := root.Flush(ctx)
rootCid, err := root.Root()
require.NoError(t, err)
return rootCid
}
@ -459,10 +575,18 @@ func createSectorsAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIp
func newSectorOnChainInfo(sectorNo abi.SectorNumber, sealed cid.Cid, weight big.Int, activation, expiration abi.ChainEpoch) miner.SectorOnChainInfo {
info := newSectorPreCommitInfo(sectorNo, sealed, expiration)
return miner.SectorOnChainInfo{
Info: *info,
ActivationEpoch: activation,
DealWeight: weight,
VerifiedDealWeight: weight,
SectorNumber: info.SectorNumber,
SealProof: info.SealProof,
SealedCID: info.SealedCID,
DealIDs: info.DealIDs,
Expiration: info.Expiration,
Activation: activation,
DealWeight: weight,
VerifiedDealWeight: weight,
InitialPledge: big.Zero(),
ExpectedDayReward: big.Zero(),
ExpectedStoragePledge: big.Zero(),
}
}

6
chain/events/utils.go
View File

@ -34,11 +34,11 @@ func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd
}
func (me *messageEvents) MatchMsg(inmsg *types.Message) MsgMatchFunc {
return func(msg *types.Message) (bool, error) {
return func(msg *types.Message) (matchOnce bool, matched bool, err error) {
if msg.From == inmsg.From && msg.Nonce == inmsg.Nonce && !inmsg.Equals(msg) {
return false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %d", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce)
return true, false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %d", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce)
}
return inmsg.Equals(msg), nil
return true, inmsg.Equals(msg), nil
}
}

121
chain/gen/gen.go
View File

@ -9,14 +9,13 @@ import (
"time"
"github.com/filecoin-project/go-address"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/crypto"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
offline "github.com/ipfs/go-ipfs-exchange-offline"
format "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
@ -35,10 +34,11 @@ import (
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/sector-storage/ffiwrapper"
)
var log = logging.Logger("gen")
@ -92,6 +92,32 @@ func (m mybs) Get(c cid.Cid) (block.Block, error) {
return b, nil
}
var rootkey, _ = address.NewIDAddress(80)
var rootkeyMultisig = genesis.MultisigMeta{
Signers: []address.Address{rootkey},
Threshold: 1,
VestingDuration: 0,
VestingStart: 0,
}
var DefaultVerifregRootkeyActor = genesis.Actor{
Type: genesis.TMultisig,
Balance: big.NewInt(0),
Meta: rootkeyMultisig.ActorMeta(),
}
var remAccTestKey, _ = address.NewFromString("t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy")
var remAccMeta = genesis.AccountMeta{
Owner: remAccTestKey,
}
var DefaultRemainderAccountActor = genesis.Actor{
Type: genesis.TAccount,
Balance: big.NewInt(0),
Meta: remAccMeta.ActorMeta(),
}
func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
abi.RegisteredSealProof_StackedDrg2KiBV1: {},
@ -113,7 +139,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
return nil, xerrors.Errorf("failed to get blocks datastore: %w", err)
}
bs := mybs{blockstore.NewIdStore(blockstore.NewBlockstore(bds))}
bs := mybs{blockstore.NewBlockstore(bds)}
ks, err := lr.KeyStore()
if err != nil {
@ -177,12 +203,12 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
Accounts: []genesis.Actor{
{
Type: genesis.TAccount,
Balance: types.FromFil(40000),
Balance: types.FromFil(20_000_000),
Meta: (&genesis.AccountMeta{Owner: mk1}).ActorMeta(),
},
{
Type: genesis.TAccount,
Balance: types.FromFil(40000),
Balance: types.FromFil(20_000_000),
Meta: (&genesis.AccountMeta{Owner: mk2}).ActorMeta(),
},
{
@ -195,8 +221,10 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
*genm1,
*genm2,
},
NetworkName: "",
Timestamp: uint64(time.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
VerifregRootKey: DefaultVerifregRootkeyActor,
RemainderAccount: DefaultRemainderAccountActor,
NetworkName: "",
Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
}
genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl)
@ -256,6 +284,10 @@ func NewGenerator() (*ChainGen, error) {
return NewGeneratorWithSectors(1)
}
func (cg *ChainGen) StateManager() *stmgr.StateManager {
return cg.sm
}
func (cg *ChainGen) SetStateManager(sm *stmgr.StateManager) {
cg.sm = sm
}
@ -284,7 +316,8 @@ func (cg *ChainGen) GenesisCar() ([]byte, error) {
func CarWalkFunc(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if link.Cid.Prefix().MhType == uint64(commcid.FC_SEALED_V1) || link.Cid.Prefix().MhType == uint64(commcid.FC_UNSEALED_V1) {
pref := link.Cid.Prefix()
if pref.Codec == cid.FilCommitmentSealed || pref.Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
@ -362,16 +395,37 @@ func (cg *ChainGen) NextTipSet() (*MinedTipSet, error) {
return mts, nil
}
func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Address) (*MinedTipSet, error) {
var blks []*types.FullBlock
func (cg *ChainGen) SetWinningPoStProver(m address.Address, wpp WinningPoStProver) {
cg.eppProvs[m] = wpp
}
msgs, err := cg.GetMessages(cg)
func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Address) (*MinedTipSet, error) {
ms, err := cg.GetMessages(cg)
if err != nil {
return nil, xerrors.Errorf("get random messages: %w", err)
}
msgs := make([][]*types.SignedMessage, len(miners))
for i := range msgs {
msgs[i] = ms
}
fts, err := cg.NextTipSetFromMinersWithMessages(base, miners, msgs)
if err != nil {
return nil, err
}
return &MinedTipSet{
TipSet: fts,
Messages: ms,
}, nil
}
func (cg *ChainGen) NextTipSetFromMinersWithMessages(base *types.TipSet, miners []address.Address, msgs [][]*types.SignedMessage) (*store.FullTipSet, error) {
var blks []*types.FullBlock
for round := base.Height() + 1; len(blks) == 0; round++ {
for _, m := range miners {
for mi, m := range miners {
bvals, et, ticket, err := cg.nextBlockProof(context.TODO(), base, m, round)
if err != nil {
return nil, xerrors.Errorf("next block proof: %w", err)
@ -384,7 +438,7 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad
return nil, err
}
fblk, err := cg.makeBlock(base, m, ticket, et, bvals, round, wpost, msgs)
fblk, err := cg.makeBlock(base, m, ticket, et, bvals, round, wpost, msgs[mi])
if err != nil {
return nil, xerrors.Errorf("making a block for next tipset failed: %w", err)
}
@ -398,12 +452,7 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad
}
}
fts := store.NewFullTipSet(blks)
return &MinedTipSet{
TipSet: fts,
Messages: msgs,
}, nil
return store.NewFullTipSet(blks), nil
}
func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticket *types.Ticket,
@ -438,7 +487,8 @@ func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticke
// ResyncBankerNonce is used for dealing with messages made when
// simulating forks
func (cg *ChainGen) ResyncBankerNonce(ts *types.TipSet) error {
act, err := cg.sm.GetActor(cg.banker, ts)
var act types.Actor
err := cg.sm.WithParentState(ts, cg.sm.WithActor(cg.banker, stmgr.GetActor(&act)))
if err != nil {
return err
}
@ -468,8 +518,9 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) {
Method: 0,
GasLimit: 10000,
GasPrice: types.NewInt(0),
GasLimit: 100_000_000,
GasFeeCap: types.NewInt(0),
GasPremium: types.NewInt(0),
}
sig, err := cg.w.Sign(context.TODO(), cg.banker, msg.Cid().Bytes())
@ -494,7 +545,8 @@ func (cg *ChainGen) YieldRepo() (repo.Repo, error) {
}
type MiningCheckAPI interface {
ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error)
@ -508,13 +560,22 @@ type mca struct {
bcn beacon.RandomBeacon
}
func (mca mca) ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
func (mca mca) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
pts, err := mca.sm.ChainStore().LoadTipSet(tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset key: %w", err)
}
return mca.sm.ChainStore().GetRandomness(ctx, pts.Cids(), personalization, randEpoch, entropy)
return mca.sm.ChainStore().GetChainRandomness(ctx, pts.Cids(), personalization, randEpoch, entropy)
}
func (mca mca) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
pts, err := mca.sm.ChainStore().LoadTipSet(tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset key: %w", err)
}
return mca.sm.ChainStore().GetBeaconRandomness(ctx, pts.Cids(), personalization, randEpoch, entropy)
}
func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoch abi.ChainEpoch, tsk types.TipSetKey) (*api.MiningBaseInfo, error) {
@ -558,12 +619,14 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, round abi.ChainEpoch,
return nil, xerrors.Errorf("failed to compute VRF: %w", err)
}
// TODO: wire in real power
if !types.IsTicketWinner(vrfout, mbi.MinerPower, mbi.NetworkPower) {
ep := &types.ElectionProof{VRFProof: vrfout}
j := ep.ComputeWinCount(mbi.MinerPower, mbi.NetworkPower)
ep.WinCount = j
if j < 1 {
return nil, nil
}
return &types.ElectionProof{VRFProof: vrfout}, nil
return ep, nil
}
type SignFunc func(context.Context, address.Address, []byte) (*crypto.Signature, error)

4
chain/gen/gen_test.go
View File

@ -39,8 +39,8 @@ func testGeneration(t testing.TB, n int, msgs int, sectors int) {
}
func TestChainGeneration(t *testing.T) {
testGeneration(t, 10, 20, 1)
testGeneration(t, 10, 20, 25)
t.Run("10-20-1", func(t *testing.T) { testGeneration(t, 10, 20, 1) })
t.Run("10-20-25", func(t *testing.T) { testGeneration(t, 10, 20, 25) })
}
func BenchmarkChainGeneration(b *testing.B) {

41
chain/gen/genesis/genblock.go Normal file
View File

@ -0,0 +1,41 @@
package genesis
import (
"encoding/hex"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
const genesisMultihashString = "1220107d821c25dc0735200249df94a8bebc9c8e489744f86a4ca8919e81f19dcd72"
const genesisBlockHex = "a5684461746574696d6573323031372d30352d30352030313a32373a3531674e6574776f726b6846696c65636f696e65546f6b656e6846696c65636f696e6c546f6b656e416d6f756e7473a36b546f74616c537570706c796d322c3030302c3030302c303030664d696e6572736d312c3430302c3030302c3030306c50726f746f636f6c4c616273a36b446576656c6f706d656e746b3330302c3030302c3030306b46756e6472616973696e676b3230302c3030302c3030306a466f756e646174696f6e6b3130302c3030302c303030674d657373616765784854686973206973207468652047656e6573697320426c6f636b206f66207468652046696c65636f696e20446563656e7472616c697a65642053746f72616765204e6574776f726b2e"
var cidBuilder = cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.SHA2_256}
func expectedCid() cid.Cid {
mh, err := multihash.FromHexString(genesisMultihashString)
if err != nil {
panic(err)
}
return cid.NewCidV1(cidBuilder.Codec, mh)
}
func getGenesisBlock() (blocks.Block, error) {
genesisBlockData, err := hex.DecodeString(genesisBlockHex)
if err != nil {
return nil, err
}
genesisCid, err := cidBuilder.Sum(genesisBlockData)
if err != nil {
return nil, err
}
block, err := blocks.NewBlockWithCid(genesisBlockData, genesisCid)
if err != nil {
return nil, err
}
return block, nil
}

328
chain/gen/genesis/genesis.go
View File

@ -3,27 +3,33 @@ package genesis
import (
"context"
"encoding/json"
"fmt"
"github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/account"
"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
"github.com/filecoin-project/specs-actors/actors/runtime"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/account"
"github.com/filecoin-project/specs-actors/actors/builtin/multisig"
"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/genesis"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/sigs"
)
const AccountStart = 100
@ -59,8 +65,15 @@ The process:
- power.CreateMiner, set msg value to PowerBalance
- market.AddFunds with correct value
- market.PublishDeals for related sectors
- Set precommits
- Commit presealed sectors
- Set network power in the power actor to what we'll have after genesis creation
- Recreate reward actor state with the right power
- For each precommitted sector
- Get deal weight
- Calculate QA Power
- Remove fake power from the power actor
- Calculate pledge
- Precommit
- Confirm valid
Data Types:
@ -91,90 +104,91 @@ Genesis: {
*/
func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template genesis.Template) (*state.StateTree, error) {
func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template genesis.Template) (*state.StateTree, map[address.Address]address.Address, error) {
// Create empty state tree
cst := cbor.NewCborStore(bs)
_, err := cst.Put(context.TODO(), []struct{}{})
if err != nil {
return nil, xerrors.Errorf("putting empty object: %w", err)
return nil, nil, xerrors.Errorf("putting empty object: %w", err)
}
state, err := state.NewStateTree(cst)
if err != nil {
return nil, xerrors.Errorf("making new state tree: %w", err)
return nil, nil, xerrors.Errorf("making new state tree: %w", err)
}
emptyobject, err := cst.Put(context.TODO(), []struct{}{})
if err != nil {
return nil, xerrors.Errorf("failed putting empty object: %w", err)
return nil, nil, xerrors.Errorf("failed putting empty object: %w", err)
}
// Create system actor
sysact, err := SetupSystemActor(bs)
if err != nil {
return nil, xerrors.Errorf("setup init actor: %w", err)
return nil, nil, xerrors.Errorf("setup init actor: %w", err)
}
if err := state.SetActor(builtin.SystemActorAddr, sysact); err != nil {
return nil, xerrors.Errorf("set init actor: %w", err)
return nil, nil, xerrors.Errorf("set init actor: %w", err)
}
// Create init actor
initact, err := SetupInitActor(bs, template.NetworkName, template.Accounts)
initact, keyIDs, err := SetupInitActor(bs, template.NetworkName, template.Accounts, template.VerifregRootKey)
if err != nil {
return nil, xerrors.Errorf("setup init actor: %w", err)
return nil, nil, xerrors.Errorf("setup init actor: %w", err)
}
if err := state.SetActor(builtin.InitActorAddr, initact); err != nil {
return nil, xerrors.Errorf("set init actor: %w", err)
return nil, nil, xerrors.Errorf("set init actor: %w", err)
}
// Setup reward
rewact, err := SetupRewardActor(bs)
// RewardActor's state is overrwritten by SetupStorageMiners
rewact, err := SetupRewardActor(bs, big.Zero())
if err != nil {
return nil, xerrors.Errorf("setup init actor: %w", err)
return nil, nil, xerrors.Errorf("setup init actor: %w", err)
}
err = state.SetActor(builtin.RewardActorAddr, rewact)
if err != nil {
return nil, xerrors.Errorf("set network account actor: %w", err)
return nil, nil, xerrors.Errorf("set network account actor: %w", err)
}
// Setup cron
cronact, err := SetupCronActor(bs)
if err != nil {
return nil, xerrors.Errorf("setup cron actor: %w", err)
return nil, nil, xerrors.Errorf("setup cron actor: %w", err)
}
if err := state.SetActor(builtin.CronActorAddr, cronact); err != nil {
return nil, xerrors.Errorf("set cron actor: %w", err)
return nil, nil, xerrors.Errorf("set cron actor: %w", err)
}
// Create empty power actor
spact, err := SetupStoragePowerActor(bs)
if err != nil {
return nil, xerrors.Errorf("setup storage market actor: %w", err)
return nil, nil, xerrors.Errorf("setup storage market actor: %w", err)
}
if err := state.SetActor(builtin.StoragePowerActorAddr, spact); err != nil {
return nil, xerrors.Errorf("set storage market actor: %w", err)
return nil, nil, xerrors.Errorf("set storage market actor: %w", err)
}
// Create empty market actor
marketact, err := SetupStorageMarketActor(bs)
if err != nil {
return nil, xerrors.Errorf("setup storage market actor: %w", err)
return nil, nil, xerrors.Errorf("setup storage market actor: %w", err)
}
if err := state.SetActor(builtin.StorageMarketActorAddr, marketact); err != nil {
return nil, xerrors.Errorf("set market actor: %w", err)
return nil, nil, xerrors.Errorf("set market actor: %w", err)
}
// Create verified registry
verifact, err := SetupVerifiedRegistryActor(bs)
if err != nil {
return nil, xerrors.Errorf("setup storage market actor: %w", err)
return nil, nil, xerrors.Errorf("setup storage market actor: %w", err)
}
if err := state.SetActor(builtin.VerifiedRegistryActorAddr, verifact); err != nil {
return nil, xerrors.Errorf("set market actor: %w", err)
return nil, nil, xerrors.Errorf("set market actor: %w", err)
}
// Setup burnt-funds
@ -184,91 +198,212 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
Head: emptyobject,
})
if err != nil {
return nil, xerrors.Errorf("set burnt funds account actor: %w", err)
return nil, nil, xerrors.Errorf("set burnt funds account actor: %w", err)
}
// Create accounts
for id, info := range template.Accounts {
if info.Type != genesis.TAccount {
return nil, xerrors.New("unsupported account type") // TODO: msigs
if info.Type != genesis.TAccount && info.Type != genesis.TMultisig {
return nil, nil, xerrors.New("unsupported account type")
}
ida, err := address.NewIDAddress(uint64(AccountStart + id))
if err != nil {
return nil, err
return nil, nil, err
}
if err = createAccount(ctx, bs, cst, state, ida, info); err != nil {
return nil, nil, err
}
}
vregroot, err := address.NewIDAddress(80)
if err != nil {
return nil, nil, err
}
if err = createAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey); err != nil {
return nil, nil, err
}
// Setup the first verifier as ID-address 81
// TODO: remove this
skBytes, err := sigs.Generate(crypto.SigTypeBLS)
if err != nil {
return nil, nil, xerrors.Errorf("creating random verifier secret key: %w", err)
}
verifierPk, err := sigs.ToPublic(crypto.SigTypeBLS, skBytes)
if err != nil {
return nil, nil, xerrors.Errorf("creating random verifier public key: %w", err)
}
verifierAd, err := address.NewBLSAddress(verifierPk)
if err != nil {
return nil, nil, xerrors.Errorf("creating random verifier address: %w", err)
}
verifierId, err := address.NewIDAddress(81)
if err != nil {
return nil, nil, err
}
verifierState, err := cst.Put(ctx, &account.State{Address: verifierAd})
if err != nil {
return nil, nil, err
}
err = state.SetActor(verifierId, &types.Actor{
Code: builtin.AccountActorCodeID,
Balance: types.NewInt(0),
Head: verifierState,
})
if err != nil {
return nil, nil, xerrors.Errorf("setting account from actmap: %w", err)
}
totalFilAllocated := big.Zero()
// flush as ForEach works on the HAMT
if _, err := state.Flush(ctx); err != nil {
return nil, nil, err
}
err = state.ForEach(func(addr address.Address, act *types.Actor) error {
totalFilAllocated = big.Add(totalFilAllocated, act.Balance)
return nil
})
if err != nil {
return nil, nil, xerrors.Errorf("summing account balances in state tree: %w", err)
}
totalFil := big.Mul(big.NewInt(int64(build.FilBase)), big.NewInt(int64(build.FilecoinPrecision)))
remainingFil := big.Sub(totalFil, totalFilAllocated)
if remainingFil.Sign() < 0 {
return nil, nil, xerrors.Errorf("somehow overallocated filecoin (allocated = %s)", types.FIL(totalFilAllocated))
}
remAccKey, err := address.NewIDAddress(90)
if err != nil {
return nil, nil, err
}
if err := createAccount(ctx, bs, cst, state, remAccKey, template.RemainderAccount); err != nil {
return nil, nil, err
}
err = state.SetActor(remAccKey, &types.Actor{
Code: builtin.AccountActorCodeID,
Balance: remainingFil,
Head: emptyobject,
})
if err != nil {
return nil, nil, xerrors.Errorf("set burnt funds account actor: %w", err)
}
return state, keyIDs, nil
}
func createAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.IpldStore, state *state.StateTree, ida address.Address, info genesis.Actor) error {
if info.Type == genesis.TAccount {
var ainfo genesis.AccountMeta
if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
return xerrors.Errorf("unmarshaling account meta: %w", err)
}
st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner})
if err != nil {
return nil, err
return err
}
err = state.SetActor(ida, &types.Actor{
Code: builtin.AccountActorCodeID,
Balance: info.Balance,
Head: st,
})
if err != nil {
return nil, xerrors.Errorf("setting account from actmap: %w", err)
return xerrors.Errorf("setting account from actmap: %w", err)
}
return nil
} else if info.Type == genesis.TMultisig {
var ainfo genesis.MultisigMeta
if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
return xerrors.Errorf("unmarshaling account meta: %w", err)
}
pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
if err != nil {
return xerrors.Errorf("failed to create empty map: %v", err)
}
st, err := cst.Put(ctx, &multisig.State{
Signers: ainfo.Signers,
NumApprovalsThreshold: uint64(ainfo.Threshold),
StartEpoch: abi.ChainEpoch(ainfo.VestingStart),
UnlockDuration: abi.ChainEpoch(ainfo.VestingDuration),
PendingTxns: pending,
InitialBalance: info.Balance,
})
if err != nil {
return err
}
err = state.SetActor(ida, &types.Actor{
Code: builtin.MultisigActorCodeID,
Balance: info.Balance,
Head: st,
})
if err != nil {
return xerrors.Errorf("setting account from actmap: %w", err)
}
return nil
}
vregroot, err := address.NewIDAddress(80)
if err != nil {
return nil, err
}
vrst, err := cst.Put(ctx, &account.State{Address: RootVerifierAddr})
if err != nil {
return nil, err
}
err = state.SetActor(vregroot, &types.Actor{
Code: builtin.AccountActorCodeID,
Balance: types.NewInt(0),
Head: vrst,
})
if err != nil {
return nil, xerrors.Errorf("setting account from actmap: %w", err)
}
return state, nil
return fmt.Errorf("failed to create account")
}
func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot cid.Cid, template genesis.Template) (cid.Cid, error) {
func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot cid.Cid, template genesis.Template, keyIDs map[address.Address]address.Address) (cid.Cid, error) {
verifNeeds := make(map[address.Address]abi.PaddedPieceSize)
var sum abi.PaddedPieceSize
for _, m := range template.Miners {
for _, s := range m.Sectors {
amt := s.Deal.PieceSize
verifNeeds[s.Deal.Client] += amt
sum += amt
}
}
verifier, err := address.NewIDAddress(80)
if err != nil {
return cid.Undef, err
vmopt := vm.VMOpts{
StateBase: stateroot,
Epoch: 0,
Rand: &fakeRand{},
Bstore: cs.Blockstore(),
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
CircSupplyCalc: nil,
BaseFee: types.NewInt(0),
}
vm, err := vm.NewVM(stateroot, 0, &fakeRand{}, cs.Blockstore(), &fakedSigSyscalls{cs.VMSys()})
vm, err := vm.NewVM(&vmopt)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
}
_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, RootVerifierAddr, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
for _, m := range template.Miners {
// Add the miner to the market actor's balance table
_, err = doExec(ctx, vm, builtin.StorageMarketActorAddr, m.Owner, builtin.MethodsMarket.AddBalance, mustEnc(adt.Empty))
for _, s := range m.Sectors {
amt := s.Deal.PieceSize
verifNeeds[keyIDs[s.Deal.Client]] += amt
sum += amt
}
}
verifregRoot, err := address.NewIDAddress(80)
if err != nil {
return cid.Undef, err
}
verifier, err := address.NewIDAddress(81)
if err != nil {
return cid.Undef, err
}
_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, verifregRoot, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
Address: verifier,
Allowance: abi.NewStoragePower(int64(sum)), // eh, close enough
}))
if err != nil {
return cid.Undef, xerrors.Errorf("failed to failed to create verifier: %w", err)
return cid.Undef, xerrors.Errorf("failed to create verifier: %w", err)
}
for c, amt := range verifNeeds {
@ -281,11 +416,16 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
}
}
return vm.Flush(ctx)
st, err := vm.Flush(ctx)
if err != nil {
return cid.Cid{}, xerrors.Errorf("vm flush: %w", err)
}
return st, nil
}
func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys runtime.Syscalls, template genesis.Template) (*GenesisBootstrap, error) {
st, err := MakeInitialStateTree(ctx, bs, template)
func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) {
st, keyIDs, err := MakeInitialStateTree(ctx, bs, template)
if err != nil {
return nil, xerrors.Errorf("make initial state tree failed: %w", err)
}
@ -299,19 +439,18 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys runtime.Sys
cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys)
// Verify PreSealed Data
stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template)
stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs)
if err != nil {
return nil, xerrors.Errorf("failed to verify presealed data: %w", err)
}
stateroot, err = SetupStorageMiners(ctx, cs, stateroot, template.Miners)
if err != nil {
return nil, xerrors.Errorf("setup storage miners failed: %w", err)
return nil, xerrors.Errorf("setup miners failed: %w", err)
}
cst := cbor.NewCborStore(bs)
emptyroot, err := amt.FromArray(ctx, cst, nil)
store := adt.WrapStore(ctx, cbor.NewCborStore(bs))
emptyroot, err := adt.MakeEmptyArray(store).Root()
if err != nil {
return nil, xerrors.Errorf("amt build failed: %w", err)
}
@ -334,10 +473,32 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys runtime.Sys
VRFProof: []byte("vrf proof0000000vrf proof0000000"),
}
filecoinGenesisCid, err := cid.Decode("bafyreiaqpwbbyjo4a42saasj36kkrpv4tsherf2e7bvezkert2a7dhonoi")
if err != nil {
return nil, xerrors.Errorf("failed to decode filecoin genesis block CID: %w", err)
}
if !expectedCid().Equals(filecoinGenesisCid) {
return nil, xerrors.Errorf("expectedCid != filecoinGenesisCid")
}
gblk, err := getGenesisBlock()
if err != nil {
return nil, xerrors.Errorf("failed to construct filecoin genesis block: %w", err)
}
if !filecoinGenesisCid.Equals(gblk.Cid()) {
return nil, xerrors.Errorf("filecoinGenesisCid != gblk.Cid")
}
if err := bs.Put(gblk); err != nil {
return nil, xerrors.Errorf("failed writing filecoin genesis block to blockstore: %w", err)
}
b := &types.BlockHeader{
Miner: builtin.SystemActorAddr,
Ticket: genesisticket,
Parents: []cid.Cid{},
Parents: []cid.Cid{filecoinGenesisCid},
Height: 0,
ParentWeight: types.NewInt(0),
ParentStateRoot: stateroot,
@ -353,6 +514,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys runtime.Sys
Data: make([]byte, 32),
},
},
ParentBaseFee: abi.NewTokenAmount(build.InitialBaseFee),
}
sb, err := b.ToStorageBlock()

318
chain/gen/genesis/miners.go
View File

@ -6,23 +6,25 @@ import (
"fmt"
"math/rand"
"github.com/filecoin-project/lotus/chain/state"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
"github.com/filecoin-project/specs-actors/actors/builtin/reward"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/runtime"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
@ -46,8 +48,30 @@ func (fss *fakedSigSyscalls) VerifySignature(signature crypto.Signature, signer
return nil
}
func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder {
return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls {
return &fakedSigSyscalls{
base(ctx, cstate, cst),
}
}
}
func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, miners []genesis.Miner) (cid.Cid, error) {
vm, err := vm.NewVM(sroot, 0, &fakeRand{}, cs.Blockstore(), &fakedSigSyscalls{cs.VMSys()})
csc := func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error) {
return big.Zero(), nil
}
vmopt := &vm.VMOpts{
StateBase: sroot,
Epoch: 0,
Rand: &fakeRand{},
Bstore: cs.Blockstore(),
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
CircSupplyCalc: csc,
BaseFee: types.NewInt(0),
}
vm, err := vm.NewVM(vmopt)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
}
@ -56,6 +80,14 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
return cid.Undef, xerrors.New("no genesis miners")
}
minerInfos := make([]struct {
maddr address.Address
presealExp abi.ChainEpoch
dealIDs []abi.DealID
}, len(miners))
for i, m := range miners {
// Create miner through power actor
i := i
@ -66,7 +98,6 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
return cid.Undef, err
}
var maddr address.Address
{
constructorParams := &power.CreateMinerParams{
Owner: m.Worker,
@ -90,29 +121,28 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
if ma.IDAddress != expma {
return cid.Undef, xerrors.Errorf("miner assigned wrong address: %s != %s", ma.IDAddress, expma)
}
maddr = ma.IDAddress
minerInfos[i].maddr = ma.IDAddress
err = vm.MutateState(ctx, minerInfos[i].maddr, func(cst cbor.IpldStore, st *miner.State) error {
maxPeriods := miner.MaxSectorExpirationExtension / miner.WPoStProvingPeriod
minerInfos[i].presealExp = (maxPeriods-1)*miner.WPoStProvingPeriod + st.ProvingPeriodStart - 1
return nil
})
}
// Add market funds
{
params := mustEnc(&maddr)
if m.MarketBalance.GreaterThan(big.Zero()) {
params := mustEnc(&minerInfos[i].maddr)
_, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, m.MarketBalance, builtin.MethodsMarket.AddBalance, params)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err)
}
}
{
params := mustEnc(&m.Worker)
_, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, big.Zero(), builtin.MethodsMarket.AddBalance, params)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err)
}
}
// Publish preseal deals
var dealIDs []abi.DealID
{
publish := func(params *market.PublishStorageDealsParams) error {
fmt.Printf("publishing %d storage deals on miner %s with worker %s\n", len(params.Deals), params.Deals[0].Proposal.Provider, m.Worker)
@ -126,13 +156,14 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
return xerrors.Errorf("unmarsahling publishStorageDeals result: %w", err)
}
dealIDs = append(dealIDs, ids.IDs...)
minerInfos[i].dealIDs = append(minerInfos[i].dealIDs, ids.IDs...)
return nil
}
params := &market.PublishStorageDealsParams{}
for _, preseal := range m.Sectors {
preseal.Deal.VerifiedDeal = true
preseal.Deal.EndEpoch = minerInfos[i].presealExp
params.Deals = append(params.Deals, market.ClientDealProposal{
Proposal: preseal.Deal,
ClientSignature: crypto.Signature{Type: crypto.SigTypeBLS}, // TODO: do we want to sign these? Or do we want to fake signatures for genesis setup?
@ -153,99 +184,134 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
}
}
}
}
// Commit sectors
for pi, preseal := range m.Sectors {
preseal := preseal
// TODO: Maybe check seal (Can just be snark inputs, doesn't go into the genesis file)
// adjust total network power for equal pledge per sector
rawPow, qaPow := big.NewInt(0), big.NewInt(0)
{
for i, m := range miners {
for pi := range m.Sectors {
rawPow = types.BigAdd(rawPow, types.NewInt(uint64(m.SectorSize)))
// check deals, get dealWeight
var dealWeight market.VerifyDealsOnSectorProveCommitReturn
{
params := &market.VerifyDealsOnSectorProveCommitParams{
DealIDs: []abi.DealID{dealIDs[pi]},
SectorExpiry: preseal.Deal.EndEpoch,
}
ret, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, maddr, big.Zero(), builtin.MethodsMarket.VerifyDealsOnSectorProveCommit, mustEnc(params))
dweight, err := dealWeight(ctx, vm, minerInfos[i].maddr, []abi.DealID{minerInfos[i].dealIDs[pi]}, 0, minerInfos[i].presealExp)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to verify preseal deals miner: %w", err)
}
if err := dealWeight.UnmarshalCBOR(bytes.NewReader(ret)); err != nil {
return cid.Undef, xerrors.Errorf("unmarshaling market onProveCommit result: %w", err)
}
}
// update power claims
{
err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error {
weight := &power.SectorStorageWeightDesc{
SectorSize: m.SectorSize,
Duration: preseal.Deal.Duration(),
DealWeight: dealWeight.DealWeight,
VerifiedDealWeight: dealWeight.VerifiedDealWeight,
}
qapower := power.QAPowerForWeight(weight)
err := st.AddToClaim(&state.AdtStore{cst}, maddr, types.NewInt(uint64(weight.SectorSize)), qapower)
if err != nil {
return xerrors.Errorf("add to claim: %w", err)
}
fmt.Println("Added weight to claim: ", st.TotalRawBytePower, st.TotalQualityAdjPower)
return nil
})
if err != nil {
return cid.Undef, xerrors.Errorf("register power claim in power actor: %w", err)
}
}
// Put sectors to miner sector sets
{
newSectorInfo := &miner.SectorOnChainInfo{
Info: miner.SectorPreCommitInfo{
SealProof: preseal.ProofType,
SectorNumber: preseal.SectorID,
SealedCID: preseal.CommR,
SealRandEpoch: 0,
DealIDs: []abi.DealID{dealIDs[pi]},
Expiration: preseal.Deal.EndEpoch,
},
ActivationEpoch: 0,
DealWeight: dealWeight.DealWeight,
VerifiedDealWeight: dealWeight.VerifiedDealWeight,
return cid.Undef, xerrors.Errorf("getting deal weight: %w", err)
}
err = vm.MutateState(ctx, maddr, func(cst cbor.IpldStore, st *miner.State) error {
store := &state.AdtStore{cst}
sectorWeight := miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight)
if err = st.PutSector(store, newSectorInfo); err != nil {
return xerrors.Errorf("failed to put sector: %v", err)
}
if err := st.AddNewSectors(newSectorInfo.Info.SectorNumber); err != nil {
return xerrors.Errorf("failed to add NewSector: %w", err)
}
return nil
})
if err != nil {
return cid.Cid{}, xerrors.Errorf("put to sset: %w", err)
}
qaPow = types.BigAdd(qaPow, sectorWeight)
}
}
err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error {
st.TotalQualityAdjPower = qaPow
st.TotalRawBytePower = rawPow
st.ThisEpochQualityAdjPower = qaPow
st.ThisEpochRawBytePower = rawPow
return nil
})
if err != nil {
return cid.Undef, xerrors.Errorf("mutating state: %w", err)
}
err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *reward.State) error {
st = reward.ConstructState(qaPow)
return nil
})
}
// TODO: to avoid division by zero, we set the initial power actor power to 1, this adjusts that back down so the accounting is accurate.
for i, m := range miners {
// Commit sectors
{
for pi, preseal := range m.Sectors {
params := &miner.SectorPreCommitInfo{
SealProof: preseal.ProofType,
SectorNumber: preseal.SectorID,
SealedCID: preseal.CommR,
SealRandEpoch: -1,
DealIDs: []abi.DealID{minerInfos[i].dealIDs[pi]},
Expiration: minerInfos[i].presealExp, // TODO: Allow setting externally!
}
dweight, err := dealWeight(ctx, vm, minerInfos[i].maddr, params.DealIDs, 0, minerInfos[i].presealExp)
if err != nil {
return cid.Undef, xerrors.Errorf("getting deal weight: %w", err)
}
sectorWeight := miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight)
// we've added fake power for this sector above, remove it now
err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error {
st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight)
st.TotalRawBytePower = types.BigSub(st.TotalRawBytePower, types.NewInt(uint64(m.SectorSize)))
return nil
})
if err != nil {
return cid.Undef, xerrors.Errorf("removing fake power: %w", err)
}
epochReward, err := currentEpochBlockReward(ctx, vm, minerInfos[i].maddr)
if err != nil {
return cid.Undef, xerrors.Errorf("getting current epoch reward: %w", err)
}
tpow, err := currentTotalPower(ctx, vm, minerInfos[i].maddr)
if err != nil {
return cid.Undef, xerrors.Errorf("getting current total power: %w", err)
}
pcd := miner.PreCommitDepositForPower(epochReward.ThisEpochRewardSmoothed, tpow.QualityAdjPowerSmoothed, sectorWeight)
pledge := miner.InitialPledgeForPower(
sectorWeight,
epochReward.ThisEpochBaselinePower,
tpow.PledgeCollateral,
epochReward.ThisEpochRewardSmoothed,
tpow.QualityAdjPowerSmoothed,
circSupply(ctx, vm, minerInfos[i].maddr),
)
pledge = big.Add(pcd, pledge)
fmt.Println(types.FIL(pledge))
_, err = doExecValue(ctx, vm, minerInfos[i].maddr, m.Worker, pledge, builtin.MethodsMiner.PreCommitSector, mustEnc(params))
if err != nil {
return cid.Undef, xerrors.Errorf("failed to confirm presealed sectors: %w", err)
}
// Commit one-by-one, otherwise pledge math tends to explode
confirmParams := &builtin.ConfirmSectorProofsParams{
Sectors: []abi.SectorNumber{preseal.SectorID},
}
_, err = doExecValue(ctx, vm, minerInfos[i].maddr, builtin.StoragePowerActorAddr, big.Zero(), builtin.MethodsMiner.ConfirmSectorProofsValid, mustEnc(confirmParams))
if err != nil {
return cid.Undef, xerrors.Errorf("failed to confirm presealed sectors: %w", err)
}
}
}
}
// Sanity-check total network power
err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error {
st.TotalQualityAdjPower = big.Sub(st.TotalQualityAdjPower, big.NewInt(1))
if !st.TotalRawBytePower.Equals(rawPow) {
return xerrors.Errorf("st.TotalRawBytePower doesn't match previously calculated rawPow")
}
if !st.TotalQualityAdjPower.Equals(qaPow) {
return xerrors.Errorf("st.TotalQualityAdjPower doesn't match previously calculated qaPow")
}
return nil
})
if err != nil {
return cid.Undef, xerrors.Errorf("mutating state: %w", err)
}
// TODO: Should we re-ConstructState for the reward actor using rawPow as currRealizedPower here?
c, err := vm.Flush(ctx)
if err != nil {
return cid.Undef, xerrors.Errorf("flushing vm: %w", err)
@ -256,8 +322,76 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
// TODO: copied from actors test harness, deduplicate or remove from here
type fakeRand struct{}
func (fr *fakeRand) GetRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
func (fr *fakeRand) GetChainRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
out := make([]byte, 32)
_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out)
return out, nil
}
func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
out := make([]byte, 32)
_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out)
return out, nil
}
func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*power.CurrentTotalPowerReturn, error) {
pwret, err := doExecValue(ctx, vm, builtin.StoragePowerActorAddr, maddr, big.Zero(), builtin.MethodsPower.CurrentTotalPower, nil)
if err != nil {
return nil, err
}
var pwr power.CurrentTotalPowerReturn
if err := pwr.UnmarshalCBOR(bytes.NewReader(pwret)); err != nil {
return nil, err
}
return &pwr, nil
}
func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs []abi.DealID, sectorStart, sectorExpiry abi.ChainEpoch) (market.VerifyDealsForActivationReturn, error) {
params := &market.VerifyDealsForActivationParams{
DealIDs: dealIDs,
SectorStart: sectorStart,
SectorExpiry: sectorExpiry,
}
var dealWeights market.VerifyDealsForActivationReturn
ret, err := doExecValue(ctx, vm,
builtin.StorageMarketActorAddr,
maddr,
abi.NewTokenAmount(0),
builtin.MethodsMarket.VerifyDealsForActivation,
mustEnc(params),
)
if err != nil {
return market.VerifyDealsForActivationReturn{}, err
}
if err := dealWeights.UnmarshalCBOR(bytes.NewReader(ret)); err != nil {
return market.VerifyDealsForActivationReturn{}, err
}
return dealWeights, nil
}
func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address) (*reward.ThisEpochRewardReturn, error) {
rwret, err := doExecValue(ctx, vm, builtin.RewardActorAddr, maddr, big.Zero(), builtin.MethodsReward.ThisEpochReward, nil)
if err != nil {
return nil, err
}
var epochReward reward.ThisEpochRewardReturn
if err := epochReward.UnmarshalCBOR(bytes.NewReader(rwret)); err != nil {
return nil, err
}
return &epochReward, nil
}
func circSupply(ctx context.Context, vmi *vm.VM, maddr address.Address) abi.TokenAmount {
unsafeVM := &vm.UnsafeVM{VM: vmi}
rt := unsafeVM.MakeRuntime(ctx, &types.Message{
GasLimit: 1_000_000_000,
From: maddr,
}, maddr, 0, 0, 0)
return rt.TotalFilCircSupply()
}

6
chain/gen/genesis/t00_system.go
View File

@ -2,12 +2,14 @@ package genesis
import (
"context"
"github.com/filecoin-project/specs-actors/actors/builtin/system"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/builtin"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupSystemActor(bs bstore.Blockstore) (*types.Actor, error) {

65
chain/gen/genesis/t01_init.go
View File

@ -5,64 +5,83 @@ import (
"encoding/json"
"fmt"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/util/adt"
init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/ipfs/go-hamt-ipld"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/genesis"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor) (*types.Actor, error) {
func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor) (*types.Actor, map[address.Address]address.Address, error) {
if len(initialActors) > MaxAccounts {
return nil, xerrors.New("too many initial actors")
return nil, nil, xerrors.New("too many initial actors")
}
var ias init_.State
ias.NextID = MinerStart
ias.NetworkName = netname
cst := cbor.NewCborStore(bs)
amap := hamt.NewNode(cst, hamt.UseTreeBitWidth(5)) // TODO: use spec adt map
store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
amap := adt.MakeEmptyMap(store)
keyToId := map[address.Address]address.Address{}
for i, a := range initialActors {
if a.Type == genesis.TMultisig {
continue
}
if a.Type != genesis.TAccount {
return nil, xerrors.Errorf("unsupported account type: %s", a.Type) // TODO: Support msig (skip here)
return nil, nil, xerrors.Errorf("unsupported account type: %s", a.Type) // TODO: Support msig (skip here)
}
var ainfo genesis.AccountMeta
if err := json.Unmarshal(a.Meta, &ainfo); err != nil {
return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
}
fmt.Printf("init set %s t0%d\n", ainfo.Owner, AccountStart+uint64(i))
fmt.Printf("init set %s t0%d\n", ainfo.Owner, AccountStart+int64(i))
if err := amap.Set(context.TODO(), string(ainfo.Owner.Bytes()), AccountStart+uint64(i)); err != nil {
return nil, err
value := cbg.CborInt(AccountStart + int64(i))
if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil {
return nil, nil, err
}
var err error
keyToId[ainfo.Owner], err = address.NewIDAddress(uint64(value))
if err != nil {
return nil, nil, err
}
}
if err := amap.Set(context.TODO(), string(RootVerifierAddr.Bytes()), 80); err != nil {
return nil, err
if rootVerifier.Type == genesis.TAccount {
var ainfo genesis.AccountMeta
if err := json.Unmarshal(rootVerifier.Meta, &ainfo); err != nil {
return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
}
value := cbg.CborInt(80)
if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil {
return nil, nil, err
}
}
if err := amap.Flush(context.TODO()); err != nil {
return nil, err
}
amapcid, err := cst.Put(context.TODO(), amap)
amapaddr, err := amap.Root()
if err != nil {
return nil, err
return nil, nil, err
}
ias.AddressMap = amapaddr
ias.AddressMap = amapcid
statecid, err := cst.Put(context.TODO(), &ias)
statecid, err := store.Put(store.Context(), &ias)
if err != nil {
return nil, err
return nil, nil, err
}
act := &types.Actor{
@ -70,5 +89,5 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi
Head: statecid,
}
return act, nil
return act, keyToId, nil
}

14
chain/gen/genesis/t02_reward.go
View File

@ -3,19 +3,21 @@ package genesis
import (
"context"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/reward"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupRewardActor(bs bstore.Blockstore) (*types.Actor, error) {
func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) {
cst := cbor.NewCborStore(bs)
st := reward.ConstructState()
st.LastPerEpochReward = types.FromFil(100)
st := reward.ConstructState(qaPower)
hcid, err := cst.Put(context.TODO(), st)
if err != nil {

2
chain/gen/genesis/t03_cron.go
View File

@ -5,10 +5,10 @@ import (
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/cron"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupCronActor(bs bstore.Blockstore) (*types.Actor, error) {

35
chain/gen/genesis/t04_power.go
View File

@ -2,38 +2,37 @@ package genesis
import (
"context"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
"github.com/ipfs/go-hamt-ipld"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {
ctx := context.TODO()
cst := cbor.NewCborStore(bs)
nd := hamt.NewNode(cst, hamt.UseTreeBitWidth(5))
emptyhamt, err := cst.Put(ctx, nd)
store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
emptyMap, err := adt.MakeEmptyMap(store).Root()
if err != nil {
return nil, err
}
sms := &power.State{
TotalRawBytePower: big.NewInt(0),
TotalQualityAdjPower: big.NewInt(1), // TODO: has to be 1 initially to avoid div by zero. Kinda annoying, should find a way to fix
TotalPledgeCollateral: big.NewInt(0),
MinerCount: 0,
CronEventQueue: emptyhamt,
LastEpochTick: 0,
Claims: emptyhamt,
NumMinersMeetingMinPower: 0,
multiMap, err := adt.AsMultimap(store, emptyMap)
if err != nil {
return nil, err
}
stcid, err := cst.Put(ctx, sms)
emptyMultiMap, err := multiMap.Root()
if err != nil {
return nil, err
}
sms := power.ConstructState(emptyMap, emptyMultiMap)
stcid, err := store.Put(store.Context(), sms)
if err != nil {
return nil, err
}

13
chain/gen/genesis/t05_market.go
View File

@ -2,32 +2,31 @@ package genesis
import (
"context"
"github.com/ipfs/go-hamt-ipld"
"github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
bstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/filecoin-project/specs-actors/actors/util/adt"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) {
cst := cbor.NewCborStore(bs)
store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
a, err := amt.NewAMT(cst).Flush(context.TODO())
a, err := adt.MakeEmptyArray(store).Root()
if err != nil {
return nil, err
}
h, err := cst.Put(context.TODO(), hamt.NewNode(cst, hamt.UseTreeBitWidth(5)))
h, err := adt.MakeEmptyMap(store).Root()
if err != nil {
return nil, err
}
sms := market.ConstructState(a, h, h)
stcid, err := cst.Put(context.TODO(), sms)
stcid, err := store.Put(store.Context(), sms)
if err != nil {
return nil, err
}

18
chain/gen/genesis/t06_vreg.go
View File

@ -4,27 +4,19 @@ import (
"context"
"github.com/filecoin-project/go-address"
"github.com/ipfs/go-hamt-ipld"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
var RootVerifierAddr address.Address
var RootVerifierID address.Address
func init() {
k, err := address.NewFromString("t3qfoulel6fy6gn3hjmbhpdpf6fs5aqjb5fkurhtwvgssizq4jey5nw4ptq5up6h7jk7frdvvobv52qzmgjinq")
if err != nil {
panic(err)
}
RootVerifierAddr = k
idk, err := address.NewFromString("t080")
if err != nil {
@ -35,16 +27,16 @@ func init() {
}
func SetupVerifiedRegistryActor(bs bstore.Blockstore) (*types.Actor, error) {
cst := cbor.NewCborStore(bs)
store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
h, err := cst.Put(context.TODO(), hamt.NewNode(cst, hamt.UseTreeBitWidth(5)))
h, err := adt.MakeEmptyMap(store).Root()
if err != nil {
return nil, err
}
sms := verifreg.ConstructState(h, RootVerifierID)
stcid, err := cst.Put(context.TODO(), sms)
stcid, err := store.Put(store.Context(), sms)
if err != nil {
return nil, err
}

1
chain/gen/genesis/util.go
View File

@ -37,7 +37,6 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value
Method: method,
Params: params,
GasLimit: 1_000_000_000_000_000,
GasPrice: types.NewInt(0),
Value: value,
Nonce: act.Nonce,
})

28
chain/gen/mining.go
View File

@ -3,8 +3,8 @@ package gen
import (
"context"
amt "github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/util/adt"
cid "github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
cbg "github.com/whyrusleeping/cbor-gen"
@ -78,17 +78,17 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal
}
}
bs := cbor.NewCborStore(sm.ChainStore().Blockstore())
blsmsgroot, err := amt.FromArray(ctx, bs, toIfArr(blsMsgCids))
store := sm.ChainStore().Store(ctx)
blsmsgroot, err := toArray(store, blsMsgCids)
if err != nil {
return nil, xerrors.Errorf("building bls amt: %w", err)
}
secpkmsgroot, err := amt.FromArray(ctx, bs, toIfArr(secpkMsgCids))
secpkmsgroot, err := toArray(store, secpkMsgCids)
if err != nil {
return nil, xerrors.Errorf("building secpk amt: %w", err)
}
mmcid, err := bs.Put(ctx, &types.MsgMeta{
mmcid, err := store.Put(store.Context(), &types.MsgMeta{
BlsMessages: blsmsgroot,
SecpkMessages: secpkmsgroot,
})
@ -109,6 +109,12 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal
}
next.ParentWeight = pweight
baseFee, err := sm.ChainStore().ComputeBaseFee(ctx, pts)
if err != nil {
return nil, xerrors.Errorf("computing base fee: %w", err)
}
next.ParentBaseFee = baseFee
cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
tree, err := state.LoadStateTree(cst, st)
if err != nil {
@ -174,11 +180,13 @@ func aggregateSignatures(sigs []crypto.Signature) (*crypto.Signature, error) {
}, nil
}
func toIfArr(cids []cid.Cid) []cbg.CBORMarshaler {
out := make([]cbg.CBORMarshaler, 0, len(cids))
for _, c := range cids {
func toArray(store adt.Store, cids []cid.Cid) (cid.Cid, error) {
arr := adt.MakeEmptyArray(store)
for i, c := range cids {
oc := cbg.CborCid(c)
out = append(out, &oc)
if err := arr.Set(uint64(i), &oc); err != nil {
return cid.Undef, err
}
}
return out
return arr.Root()
}

112
chain/gen/slashfilter/slashfilter.go Normal file
View File

@ -0,0 +1,112 @@
package slashfilter
import (
"fmt"
"golang.org/x/xerrors"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi"
)
type SlashFilter struct {
byEpoch ds.Datastore // double-fork mining faults, parent-grinding fault
byParents ds.Datastore // time-offset mining faults
}
func New(dstore ds.Batching) *SlashFilter {
return &SlashFilter{
byEpoch: namespace.Wrap(dstore, ds.NewKey("/slashfilter/epoch")),
byParents: namespace.Wrap(dstore, ds.NewKey("/slashfilter/parents")),
}
}
func (f *SlashFilter) MinedBlock(bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error {
epochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height))
{
// double-fork mining (2 blocks at one epoch)
if err := checkFault(f.byEpoch, epochKey, bh, "double-fork mining faults"); err != nil {
return err
}
}
parentsKey := ds.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, types.NewTipSetKey(bh.Parents...).Bytes()))
{
// time-offset mining faults (2 blocks with the same parents)
if err := checkFault(f.byParents, parentsKey, bh, "time-offset mining faults"); err != nil {
return err
}
}
{
// parent-grinding fault (didn't mine on top of our own block)
// First check if we have mined a block on the parent epoch
parentEpochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, parentEpoch))
have, err := f.byEpoch.Has(parentEpochKey)
if err != nil {
return err
}
if have {
// If we had, make sure it's in our parent tipset
cidb, err := f.byEpoch.Get(parentEpochKey)
if err != nil {
return xerrors.Errorf("getting other block cid: %w", err)
}
_, parent, err := cid.CidFromBytes(cidb)
if err != nil {
return err
}
var found bool
for _, c := range bh.Parents {
if c.Equals(parent) {
found = true
}
}
if !found {
return xerrors.Errorf("produced block would trigger 'parent-grinding fault' consensus fault; miner: %s; bh: %s, expected parent: %s", bh.Miner, bh.Cid(), parent)
}
}
}
if err := f.byParents.Put(parentsKey, bh.Cid().Bytes()); err != nil {
return xerrors.Errorf("putting byEpoch entry: %w", err)
}
if err := f.byEpoch.Put(epochKey, bh.Cid().Bytes()); err != nil {
return xerrors.Errorf("putting byEpoch entry: %w", err)
}
return nil
}
func checkFault(t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) error {
fault, err := t.Has(key)
if err != nil {
return err
}
if fault {
cidb, err := t.Get(key)
if err != nil {
return xerrors.Errorf("getting other block cid: %w", err)
}
_, other, err := cid.CidFromBytes(cidb)
if err != nil {
return err
}
return xerrors.Errorf("produced block would trigger '%s' consensus fault; miner: %s; bh: %s, other: %s", faultType, bh.Miner, bh.Cid(), other)
}
return nil
}

155
chain/market/fundmgr.go
View File

@ -4,76 +4,159 @@ import (
"context"
"sync"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"go.uber.org/fx"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/events/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl/full"
)
var log = logging.Logger("market_adapter")
type FundMgr struct {
sm *stmgr.StateManager
mpool full.MpoolAPI
// API is the dependencies need to run a fund manager
type API struct {
fx.In
lk sync.Mutex
full.ChainAPI
full.StateAPI
full.MpoolAPI
}
// FundMgr monitors available balances and adds funds when EnsureAvailable is called
type FundMgr struct {
api fundMgrAPI
lk sync.RWMutex
available map[address.Address]types.BigInt
}
func NewFundMgr(sm *stmgr.StateManager, mpool full.MpoolAPI) *FundMgr {
return &FundMgr{
sm: sm,
mpool: mpool,
// StartFundManager creates a new fund manager and sets up event hooks to manage state changes
func StartFundManager(lc fx.Lifecycle, api API) *FundMgr {
fm := newFundMgr(&api)
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
ev := events.NewEvents(ctx, &api)
preds := state.NewStatePredicates(&api)
dealDiffFn := preds.OnStorageMarketActorChanged(preds.OnBalanceChanged(preds.AvailableBalanceChangedForAddresses(fm.getAddresses)))
match := func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
return dealDiffFn(ctx, oldTs.Key(), newTs.Key())
}
return ev.StateChanged(fm.checkFunc, fm.stateChanged, fm.revert, 0, events.NoTimeout, match)
},
})
return fm
}
type fundMgrAPI interface {
StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)
MpoolPushMessage(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error)
StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error)
}
func newFundMgr(api fundMgrAPI) *FundMgr {
return &FundMgr{
api: api,
available: map[address.Address]types.BigInt{},
}
}
func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
fm.lk.Lock()
avail, ok := fm.available[addr]
// checkFunc tells the events api to simply proceed (we always want to watch)
func (fm *FundMgr) checkFunc(ts *types.TipSet) (done bool, more bool, err error) {
return false, true, nil
}
// revert handles reverts to balances
func (fm *FundMgr) revert(ctx context.Context, ts *types.TipSet) error {
// TODO: Is it ok to just ignore this?
log.Warn("balance change reverted; TODO: actually handle this!")
return nil
}
// stateChanged handles balance changes monitored on the chain from one tipset to the next
func (fm *FundMgr) stateChanged(ts *types.TipSet, ts2 *types.TipSet, states events.StateChange, h abi.ChainEpoch) (more bool, err error) {
changedBalances, ok := states.(state.ChangedBalances)
if !ok {
bal, err := fm.sm.MarketBalance(ctx, addr, nil)
if err != nil {
fm.lk.Unlock()
return cid.Undef, err
panic("Expected state.ChangedBalances")
}
// overwrite our in memory cache with new values from chain (chain is canonical)
fm.lk.Lock()
for addr, balanceChange := range changedBalances {
if fm.available[addr].Int != nil {
log.Infof("State balance change recorded, prev: %s, new: %s", fm.available[addr].String(), balanceChange.To.String())
}
avail = types.BigSub(bal.Escrow, bal.Locked)
fm.available[addr] = balanceChange.To
}
fm.lk.Unlock()
return true, nil
}
func (fm *FundMgr) getAddresses() []address.Address {
fm.lk.RLock()
defer fm.lk.RUnlock()
addrs := make([]address.Address, 0, len(fm.available))
for addr := range fm.available {
addrs = append(addrs, addr)
}
return addrs
}
// EnsureAvailable looks at the available balance in escrow for a given
// address, and if less than the passed in amount, adds the difference
func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
idAddr, err := fm.api.StateLookupID(ctx, addr, types.EmptyTSK)
if err != nil {
return cid.Undef, err
}
fm.lk.Lock()
bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK)
if err != nil {
fm.lk.Unlock()
return cid.Undef, err
}
toAdd := types.NewInt(0)
avail = types.BigSub(avail, amt)
if avail.LessThan(types.NewInt(0)) {
// TODO: some rules around adding more to avoid doing stuff on-chain
// all the time
toAdd = types.BigSub(toAdd, avail)
avail = types.NewInt(0)
}
fm.available[addr] = avail
stateAvail := types.BigSub(bal.Escrow, bal.Locked)
avail, ok := fm.available[idAddr]
if !ok {
avail = stateAvail
}
toAdd := types.BigSub(amt, avail)
if toAdd.LessThan(types.NewInt(0)) {
toAdd = types.NewInt(0)
}
fm.available[idAddr] = big.Add(avail, toAdd)
fm.lk.Unlock()
var err error
log.Infof("Funds operation w/ Expected Balance: %s, In State: %s, Requested: %s, Adding: %s", avail.String(), stateAvail.String(), amt.String(), toAdd.String())
if toAdd.LessThanEqual(big.Zero()) {
return cid.Undef, nil
}
params, err := actors.SerializeParams(&addr)
if err != nil {
return cid.Undef, err
}
smsg, err := fm.mpool.MpoolPushMessage(ctx, &types.Message{
To: builtin.StorageMarketActorAddr,
From: wallet,
Value: toAdd,
GasPrice: types.NewInt(0),
GasLimit: 1000000,
Method: builtin.MethodsMarket.AddBalance,
Params: params,
})
smsg, err := fm.api.MpoolPushMessage(ctx, &types.Message{
To: builtin.StorageMarketActorAddr,
From: wallet,
Value: toAdd,
Method: builtin.MethodsMarket.AddBalance,
Params: params,
}, nil)
if err != nil {
return cid.Undef, err
}

187
chain/market/fundmgr_test.go Normal file
View File

@ -0,0 +1,187 @@
package market
import (
"context"
"errors"
"math/rand"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/crypto"
tutils "github.com/filecoin-project/specs-actors/support/testing"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
)
type fakeAPI struct {
returnedBalance api.MarketBalance
returnedBalanceErr error
signature crypto.Signature
receivedMessage *types.Message
pushMessageErr error
lookupIDErr error
}
func (fapi *fakeAPI) StateLookupID(_ context.Context, addr address.Address, _ types.TipSetKey) (address.Address, error) {
return addr, fapi.lookupIDErr
}
func (fapi *fakeAPI) StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) {
return fapi.returnedBalance, fapi.returnedBalanceErr
}
func (fapi *fakeAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
fapi.receivedMessage = msg
return &types.SignedMessage{
Message: *msg,
Signature: fapi.signature,
}, fapi.pushMessageErr
}
func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Address) *types.Message {
params, _ := actors.SerializeParams(&addr)
return &types.Message{
To: builtin.StorageMarketActorAddr,
From: wallet,
Value: toAdd,
Method: builtin.MethodsMarket.AddBalance,
Params: params,
}
}
type expectedResult struct {
addAmt abi.TokenAmount
shouldAdd bool
err error
}
func TestAddFunds(t *testing.T) {
ctx := context.Background()
testCases := map[string]struct {
returnedBalanceErr error
returnedBalance api.MarketBalance
addAmounts []abi.TokenAmount
pushMessageErr error
expectedResults []expectedResult
lookupIDErr error
}{
"succeeds, trivial case": {
returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(0), Locked: abi.NewTokenAmount(0)},
addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)},
expectedResults: []expectedResult{
{
addAmt: abi.NewTokenAmount(100),
shouldAdd: true,
err: nil,
},
},
},
"succeeds, money already present": {
returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(150), Locked: abi.NewTokenAmount(50)},
addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)},
expectedResults: []expectedResult{
{
shouldAdd: false,
err: nil,
},
},
},
"succeeds, multiple adds": {
returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(150), Locked: abi.NewTokenAmount(50)},
addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100), abi.NewTokenAmount(200), abi.NewTokenAmount(250), abi.NewTokenAmount(250)},
expectedResults: []expectedResult{
{
shouldAdd: false,
err: nil,
},
{
addAmt: abi.NewTokenAmount(100),
shouldAdd: true,
err: nil,
},
{
addAmt: abi.NewTokenAmount(50),
shouldAdd: true,
err: nil,
},
{
shouldAdd: false,
err: nil,
},
},
},
"error on market balance": {
returnedBalanceErr: errors.New("something went wrong"),
addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)},
expectedResults: []expectedResult{
{
err: errors.New("something went wrong"),
},
},
},
"error on push message": {
returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(0), Locked: abi.NewTokenAmount(0)},
pushMessageErr: errors.New("something went wrong"),
addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)},
expectedResults: []expectedResult{
{
err: errors.New("something went wrong"),
},
},
},
"error looking up address": {
lookupIDErr: errors.New("something went wrong"),
addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)},
expectedResults: []expectedResult{
{
err: errors.New("something went wrong"),
},
},
},
}
for testCase, data := range testCases {
t.Run(testCase, func(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
sig := make([]byte, 100)
_, err := rand.Read(sig)
require.NoError(t, err)
fapi := &fakeAPI{
returnedBalance: data.returnedBalance,
returnedBalanceErr: data.returnedBalanceErr,
signature: crypto.Signature{
Type: crypto.SigTypeUnknown,
Data: sig,
},
pushMessageErr: data.pushMessageErr,
lookupIDErr: data.lookupIDErr,
}
fundMgr := newFundMgr(fapi)
addr := tutils.NewIDAddr(t, uint64(rand.Uint32()))
wallet := tutils.NewIDAddr(t, uint64(rand.Uint32()))
for i, amount := range data.addAmounts {
fapi.receivedMessage = nil
_, err := fundMgr.EnsureAvailable(ctx, addr, wallet, amount)
expected := data.expectedResults[i]
if expected.err == nil {
require.NoError(t, err)
if expected.shouldAdd {
expectedMessage := addFundsMsg(expected.addAmt, addr, wallet)
require.Equal(t, expectedMessage, fapi.receivedMessage)
} else {
require.Nil(t, fapi.receivedMessage)
}
} else {
require.EqualError(t, err, expected.err.Error())
}
}
})
}
}

102
chain/messagepool/block_proba.go Normal file
View File

@ -0,0 +1,102 @@
package messagepool
import (
"math"
"sync"
)
var noWinnersProbCache []float64
var noWinnersProbOnce sync.Once
func noWinnersProb() []float64 {
noWinnersProbOnce.Do(func() {
poissPdf := func(x float64) float64 {
const Mu = 5
lg, _ := math.Lgamma(x + 1)
result := math.Exp((math.Log(Mu) * x) - lg - Mu)
return result
}
out := make([]float64, 0, MaxBlocks)
for i := 0; i < MaxBlocks; i++ {
out = append(out, poissPdf(float64(i)))
}
noWinnersProbCache = out
})
return noWinnersProbCache
}
var noWinnersProbAssumingCache []float64
var noWinnersProbAssumingOnce sync.Once
func noWinnersProbAssumingMoreThanOne() []float64 {
noWinnersProbAssumingOnce.Do(func() {
cond := math.Log(-1 + math.Exp(5))
poissPdf := func(x float64) float64 {
const Mu = 5
lg, _ := math.Lgamma(x + 1)
result := math.Exp((math.Log(Mu) * x) - lg - cond)
return result
}
out := make([]float64, 0, MaxBlocks)
for i := 0; i < MaxBlocks; i++ {
out = append(out, poissPdf(float64(i+1)))
}
noWinnersProbAssumingCache = out
})
return noWinnersProbAssumingCache
}
func binomialCoefficient(n, k float64) float64 {
if k > n {
return math.NaN()
}
r := 1.0
for d := 1.0; d <= k; d++ {
r *= n
r /= d
n -= 1
}
return r
}
func (mp *MessagePool) blockProbabilities(tq float64) []float64 {
noWinners := noWinnersProbAssumingMoreThanOne()
p := 1 - tq
binoPdf := func(x, trials float64) float64 {
// based on https://github.com/atgjack/prob
if x > trials {
return 0
}
if p == 0 {
if x == 0 {
return 1.0
}
return 0.0
}
if p == 1 {
if x == trials {
return 1.0
}
return 0.0
}
coef := binomialCoefficient(trials, x)
pow := math.Pow(p, x) * math.Pow(1-p, trials-x)
if math.IsInf(coef, 0) {
return 0
}
return coef * pow
}
out := make([]float64, 0, MaxBlocks)
for place := 0; place < MaxBlocks; place++ {
var pPlace float64
for otherWinners, pCase := range noWinners {
pPlace += pCase * binoPdf(float64(place), float64(otherWinners))
}
out = append(out, pPlace)
}
return out
}

43
chain/messagepool/block_proba_test.go Normal file
View File

@ -0,0 +1,43 @@
package messagepool
import (
"math"
"math/rand"
"testing"
"time"
)
func TestBlockProbability(t *testing.T) {
mp := &MessagePool{}
bp := mp.blockProbabilities(1 - 0.15)
t.Logf("%+v\n", bp)
for i := 0; i < len(bp)-1; i++ {
if bp[i] < bp[i+1] {
t.Fatalf("expected decreasing block probabilities for this quality: %d %f %f",
i, bp[i], bp[i+1])
}
}
}
func TestWinnerProba(t *testing.T) {
rand.Seed(time.Now().UnixNano())
const N = 1000000
winnerProba := noWinnersProb()
sum := 0
for i := 0; i < N; i++ {
minersRand := rand.Float64()
j := 0
for ; j < MaxBlocks; j++ {
minersRand -= winnerProba[j]
if minersRand < 0 {
break
}
}
sum += j
}
if avg := float64(sum) / N; math.Abs(avg-5) > 0.01 {
t.Fatalf("avg too far off: %f", avg)
}
}

92
chain/messagepool/config.go Normal file
View File

@ -0,0 +1,92 @@
package messagepool
import (
"encoding/json"
"fmt"
"time"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/ipfs/go-datastore"
)
var (
ReplaceByFeeRatioDefault = 1.25
MemPoolSizeLimitHiDefault = 30000
MemPoolSizeLimitLoDefault = 20000
PruneCooldownDefault = time.Minute
GasLimitOverestimation = 1.25
ConfigKey = datastore.NewKey("/mpool/config")
)
func loadConfig(ds dtypes.MetadataDS) (*types.MpoolConfig, error) {
haveCfg, err := ds.Has(ConfigKey)
if err != nil {
return nil, err
}
if !haveCfg {
return DefaultConfig(), nil
}
cfgBytes, err := ds.Get(ConfigKey)
if err != nil {
return nil, err
}
cfg := new(types.MpoolConfig)
err = json.Unmarshal(cfgBytes, cfg)
return cfg, err
}
func saveConfig(cfg *types.MpoolConfig, ds dtypes.MetadataDS) error {
cfgBytes, err := json.Marshal(cfg)
if err != nil {
return err
}
return ds.Put(ConfigKey, cfgBytes)
}
func (mp *MessagePool) GetConfig() *types.MpoolConfig {
mp.cfgLk.Lock()
defer mp.cfgLk.Unlock()
return mp.cfg.Clone()
}
func validateConfg(cfg *types.MpoolConfig) error {
if cfg.ReplaceByFeeRatio < ReplaceByFeeRatioDefault {
return fmt.Errorf("'ReplaceByFeeRatio' is less than required %f < %f",
cfg.ReplaceByFeeRatio, ReplaceByFeeRatioDefault)
}
if cfg.GasLimitOverestimation < 1 {
return fmt.Errorf("'GasLimitOverestimation' cannot be less than 1")
}
return nil
}
func (mp *MessagePool) SetConfig(cfg *types.MpoolConfig) error {
if err := validateConfg(cfg); err != nil {
return err
}
cfg = cfg.Clone()
mp.cfgLk.Lock()
mp.cfg = cfg
err := saveConfig(cfg, mp.ds)
if err != nil {
log.Warnf("error persisting mpool config: %s", err)
}
mp.cfgLk.Unlock()
return nil
}
func DefaultConfig() *types.MpoolConfig {
return &types.MpoolConfig{
SizeLimitHigh: MemPoolSizeLimitHiDefault,
SizeLimitLow: MemPoolSizeLimitLoDefault,
ReplaceByFeeRatio: ReplaceByFeeRatioDefault,
PruneCooldown: PruneCooldownDefault,
GasLimitOverestimation: GasLimitOverestimation,
}
}

79
chain/messagepool/gasguess/guessgas.go Normal file
View File

@ -0,0 +1,79 @@
package gasguess
import (
"context"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
)
type ActorLookup func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error)
const failedGasGuessRatio = 0.5
const failedGasGuessMax = 25_000_000
const MinGas = 1298450
const MaxGas = 1600271356
type CostKey struct {
Code cid.Cid
M abi.MethodNum
}
var Costs = map[CostKey]int64{
{builtin.InitActorCodeID, 2}: 8916753,
{builtin.StorageMarketActorCodeID, 2}: 6955002,
{builtin.StorageMarketActorCodeID, 4}: 245436108,
{builtin.StorageMinerActorCodeID, 4}: 2315133,
{builtin.StorageMinerActorCodeID, 5}: 1600271356,
{builtin.StorageMinerActorCodeID, 6}: 22864493,
{builtin.StorageMinerActorCodeID, 7}: 142002419,
{builtin.StorageMinerActorCodeID, 10}: 23008274,
{builtin.StorageMinerActorCodeID, 11}: 19303178,
{builtin.StorageMinerActorCodeID, 14}: 566356835,
{builtin.StorageMinerActorCodeID, 16}: 5325185,
{builtin.StorageMinerActorCodeID, 18}: 2328637,
{builtin.StoragePowerActorCodeID, 2}: 23600956,
}
func failedGuess(msg *types.SignedMessage) int64 {
guess := int64(float64(msg.Message.GasLimit) * failedGasGuessRatio)
if guess > failedGasGuessMax {
guess = failedGasGuessMax
}
return guess
}
func GuessGasUsed(ctx context.Context, tsk types.TipSetKey, msg *types.SignedMessage, al ActorLookup) (int64, error) {
if msg.Message.Method == builtin.MethodSend {
switch msg.Message.From.Protocol() {
case address.BLS:
return 1298450, nil
case address.SECP256K1:
return 1385999, nil
default:
// who knows?
return 1298450, nil
}
}
to, err := al(ctx, msg.Message.To, tsk)
if err != nil {
return failedGuess(msg), xerrors.Errorf("could not lookup actor: %w", err)
}
guess, ok := Costs[CostKey{to.Code, msg.Message.Method}]
if !ok {
return failedGuess(msg), xerrors.Errorf("unknown code-method combo")
}
if guess > msg.Message.GasLimit {
guess = msg.Message.GasLimit
}
return guess, nil
}

376
chain/messagepool/messagepool.go
View File

@ -10,6 +10,7 @@ import (
"sync"
"time"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
lru "github.com/hashicorp/golang-lru"
"github.com/ipfs/go-cid"
@ -19,28 +20,30 @@ import (
logging "github.com/ipfs/go-log/v2"
pubsub "github.com/libp2p/go-libp2p-pubsub"
lps "github.com/whyrusleeping/pubsub"
"go.uber.org/multierr"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/raulk/clock"
)
var log = logging.Logger("messagepool")
const futureDebug = false
const ReplaceByFeeRatio = 1.25
var rbfNumBig = types.NewInt(uint64((ReplaceByFeeRatioDefault - 1) * RbfDenom))
var rbfDenomBig = types.NewInt(RbfDenom)
var (
rbfNum = types.NewInt(uint64((ReplaceByFeeRatio - 1) * 256))
rbfDenom = types.NewInt(256)
)
const RbfDenom = 256
var RepublishInterval = pubsub.TimeCacheDuration + time.Duration(5*build.BlockDelaySecs+build.PropagationDelaySecs)*time.Second
var (
ErrMessageTooBig = errors.New("message too big")
@ -53,7 +56,10 @@ var (
ErrInvalidToAddr = errors.New("message had invalid to address")
ErrBroadcastAnyway = errors.New("broadcasting message despite validation fail")
ErrBroadcastAnyway = errors.New("broadcasting message despite validation fail")
ErrRBFTooLowPremium = errors.New("replace by fee has too low GasPremium")
ErrTryAgain = errors.New("state inconsistency while pushing message; please try again")
)
const (
@ -65,8 +71,16 @@ const (
type MessagePool struct {
lk sync.Mutex
closer chan struct{}
repubTk *time.Ticker
ds dtypes.MetadataDS
addSema chan struct{}
closer chan struct{}
repubTk *clock.Ticker
repubTrigger chan struct{}
republished map[cid.Cid]struct{}
localAddrs map[address.Address]struct{}
@ -75,11 +89,20 @@ type MessagePool struct {
curTsLk sync.Mutex // DO NOT LOCK INSIDE lk
curTs *types.TipSet
cfgLk sync.Mutex
cfg *types.MpoolConfig
api Provider
minGasPrice types.BigInt
maxTxPoolSize int
currentSize int
// pruneTrigger is a channel used to trigger a mempool pruning
pruneTrigger chan struct{}
// pruneCooldown is a channel used to allow a cooldown time between prunes
pruneCooldown chan struct{}
blsSigCache *lru.TwoQueueCache
@ -103,7 +126,7 @@ func newMsgSet() *msgSet {
}
}
func (ms *msgSet) add(m *types.SignedMessage) error {
func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool) (bool, error) {
if len(ms.msgs) == 0 || m.Message.Nonce >= ms.nextNonce {
ms.nextNonce = m.Message.Nonce + 1
}
@ -111,100 +134,65 @@ func (ms *msgSet) add(m *types.SignedMessage) error {
if has {
if m.Cid() != exms.Cid() {
// check if RBF passes
minPrice := exms.Message.GasPrice
minPrice = types.BigAdd(minPrice, types.BigDiv(types.BigMul(minPrice, rbfNum), rbfDenom))
minPrice := exms.Message.GasPremium
minPrice = types.BigAdd(minPrice, types.BigDiv(types.BigMul(minPrice, rbfNumBig), rbfDenomBig))
minPrice = types.BigAdd(minPrice, types.NewInt(1))
if types.BigCmp(m.Message.GasPrice, minPrice) > 0 {
log.Infow("add with RBF", "oldprice", exms.Message.GasPrice,
"newprice", m.Message.GasPrice, "addr", m.Message.From, "nonce", m.Message.Nonce)
if types.BigCmp(m.Message.GasPremium, minPrice) >= 0 {
log.Infow("add with RBF", "oldpremium", exms.Message.GasPremium,
"newpremium", m.Message.GasPremium, "addr", m.Message.From, "nonce", m.Message.Nonce)
} else {
log.Info("add with duplicate nonce")
return xerrors.Errorf("message to %s with nonce %d already in mpool", m.Message.To, m.Message.Nonce)
return false, xerrors.Errorf("message from %s with nonce %d already in mpool,"+
" increase GasPremium to %s from %s to trigger replace by fee: %w",
m.Message.From, m.Message.Nonce, minPrice, m.Message.GasPremium,
ErrRBFTooLowPremium)
}
}
}
ms.msgs[m.Message.Nonce] = m
return nil
}
type Provider interface {
SubscribeHeadChanges(func(rev, app []*types.TipSet) error) *types.TipSet
PutMessage(m types.ChainMsg) (cid.Cid, error)
PubSubPublish(string, []byte) error
StateGetActor(address.Address, *types.TipSet) (*types.Actor, error)
StateAccountKey(context.Context, address.Address, *types.TipSet) (address.Address, error)
MessagesForBlock(*types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error)
MessagesForTipset(*types.TipSet) ([]types.ChainMsg, error)
LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error)
}
type mpoolProvider struct {
sm *stmgr.StateManager
ps *pubsub.PubSub
}
func NewProvider(sm *stmgr.StateManager, ps *pubsub.PubSub) Provider {
return &mpoolProvider{sm, ps}
}
func (mpp *mpoolProvider) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet {
mpp.sm.ChainStore().SubscribeHeadChanges(cb)
return mpp.sm.ChainStore().GetHeaviestTipSet()
}
func (mpp *mpoolProvider) PutMessage(m types.ChainMsg) (cid.Cid, error) {
return mpp.sm.ChainStore().PutMessage(m)
}
func (mpp *mpoolProvider) PubSubPublish(k string, v []byte) error {
return mpp.ps.Publish(k, v)
}
func (mpp *mpoolProvider) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
return mpp.sm.GetActor(addr, ts)
}
func (mpp *mpoolProvider) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
return mpp.sm.ResolveToKeyAddress(ctx, addr, ts)
}
func (mpp *mpoolProvider) MessagesForBlock(h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
return mpp.sm.ChainStore().MessagesForBlock(h)
}
func (mpp *mpoolProvider) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) {
return mpp.sm.ChainStore().MessagesForTipset(ts)
}
func (mpp *mpoolProvider) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
return mpp.sm.ChainStore().LoadTipSet(tsk)
return !has, nil
}
func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) {
cache, _ := lru.New2Q(build.BlsSignatureCacheSize)
verifcache, _ := lru.New2Q(build.VerifSigCacheSize)
cfg, err := loadConfig(ds)
if err != nil {
if err != nil {
return nil, xerrors.Errorf("error loading mpool config: %w", err)
}
}
mp := &MessagePool{
ds: ds,
addSema: make(chan struct{}, 1),
closer: make(chan struct{}),
repubTk: time.NewTicker(time.Duration(build.BlockDelaySecs) * 10 * time.Second),
repubTk: build.Clock.Ticker(RepublishInterval),
repubTrigger: make(chan struct{}, 1),
localAddrs: make(map[address.Address]struct{}),
pending: make(map[address.Address]*msgSet),
minGasPrice: types.NewInt(0),
maxTxPoolSize: 5000,
pruneTrigger: make(chan struct{}, 1),
pruneCooldown: make(chan struct{}, 1),
blsSigCache: cache,
sigValCache: verifcache,
changes: lps.New(50),
localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)),
api: api,
netName: netName,
cfg: cfg,
}
// enable initial prunes
mp.pruneCooldown <- struct{}{}
if err := mp.loadLocal(); err != nil {
log.Errorf("loading local messages: %+v", err)
}
go mp.repubLocal()
go mp.runLoop()
mp.curTs = api.SubscribeHeadChanges(func(rev, app []*types.TipSet) error {
err := mp.HeadChange(rev, app)
@ -222,70 +210,38 @@ func (mp *MessagePool) Close() error {
return nil
}
func (mp *MessagePool) repubLocal() {
func (mp *MessagePool) Prune() {
// this magic incantation of triggering prune thrice is here to make the Prune method
// synchronous:
// so, its a single slot buffered channel. The first send fills the channel,
// the second send goes through when the pruning starts,
// and the third send goes through (and noops) after the pruning finishes
// and goes through the loop again
mp.pruneTrigger <- struct{}{}
mp.pruneTrigger <- struct{}{}
mp.pruneTrigger <- struct{}{}
}
func (mp *MessagePool) runLoop() {
for {
select {
case <-mp.repubTk.C:
mp.lk.Lock()
msgsForAddr := make(map[address.Address][]*types.SignedMessage)
for a := range mp.localAddrs {
msgsForAddr[a] = mp.pendingFor(a)
if err := mp.republishPendingMessages(); err != nil {
log.Errorf("error while republishing messages: %s", err)
}
mp.lk.Unlock()
var errout error
outputMsgs := []*types.SignedMessage{}
for a, msgs := range msgsForAddr {
a, err := mp.api.StateGetActor(a, nil)
if err != nil {
errout = multierr.Append(errout, xerrors.Errorf("could not get actor state: %w", err))
continue
}
curNonce := a.Nonce
for _, m := range msgs {
if m.Message.Nonce < curNonce {
continue
}
if m.Message.Nonce != curNonce {
break
}
outputMsgs = append(outputMsgs, m)
curNonce++
}
case <-mp.repubTrigger:
if err := mp.republishPendingMessages(); err != nil {
log.Errorf("error while republishing messages: %s", err)
}
if len(outputMsgs) != 0 {
log.Infow("republishing local messages", "n", len(outputMsgs))
}
for _, msg := range outputMsgs {
msgb, err := msg.Serialize()
if err != nil {
errout = multierr.Append(errout, xerrors.Errorf("could not serialize: %w", err))
continue
}
err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
if err != nil {
errout = multierr.Append(errout, xerrors.Errorf("could not publish: %w", err))
continue
}
}
if errout != nil {
log.Errorf("errors while republishing: %+v", errout)
case <-mp.pruneTrigger:
if err := mp.pruneExcessMessages(); err != nil {
log.Errorf("failed to prune excess messages from mempool: %s", err)
}
case <-mp.closer:
mp.repubTk.Stop()
return
}
}
}
func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error {
@ -298,15 +254,51 @@ func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error {
return nil
}
func (mp *MessagePool) verifyMsgBeforePush(m *types.SignedMessage, epoch abi.ChainEpoch) error {
minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
if err := m.VMMessage().ValidForBlockInclusion(minGas.Total()); err != nil {
return xerrors.Errorf("message will not be included in a block: %w", err)
}
return nil
}
func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) {
err := mp.checkMessage(m)
if err != nil {
return cid.Undef, err
}
// serialize push access to reduce lock contention
mp.addSema <- struct{}{}
defer func() {
<-mp.addSema
}()
mp.curTsLk.Lock()
curTs := mp.curTs
epoch := curTs.Height()
mp.curTsLk.Unlock()
if err := mp.verifyMsgBeforePush(m, epoch); err != nil {
return cid.Undef, err
}
msgb, err := m.Serialize()
if err != nil {
return cid.Undef, err
}
if err := mp.Add(m); err != nil {
mp.curTsLk.Lock()
if mp.curTs != curTs {
mp.curTsLk.Unlock()
return cid.Undef, ErrTryAgain
}
if err := mp.addTs(m, mp.curTs); err != nil {
mp.curTsLk.Unlock()
return cid.Undef, err
}
mp.curTsLk.Unlock()
mp.lk.Lock()
if err := mp.addLocal(m, msgb); err != nil {
@ -318,7 +310,7 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) {
return m.Cid(), mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
}
func (mp *MessagePool) Add(m *types.SignedMessage) error {
func (mp *MessagePool) checkMessage(m *types.SignedMessage) error {
// big messages are bad, anti DOS
if m.Size() > 32*1024 {
return xerrors.Errorf("mpool message too large (%dB): %w", m.Size(), ErrMessageTooBig)
@ -337,6 +329,21 @@ func (mp *MessagePool) Add(m *types.SignedMessage) error {
return err
}
return nil
}
func (mp *MessagePool) Add(m *types.SignedMessage) error {
err := mp.checkMessage(m)
if err != nil {
return err
}
// serialize push access to reduce lock contention
mp.addSema <- struct{}{}
defer func() {
<-mp.addSema
}()
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
return mp.addTs(m, mp.curTs)
@ -436,8 +443,21 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error {
mp.pending[m.Message.From] = mset
}
if err := mset.add(m); err != nil {
incr, err := mset.add(m, mp)
if err != nil {
log.Info(err)
return err
}
if incr {
mp.currentSize++
if mp.currentSize > mp.cfg.SizeLimitHigh {
// send signal to prune messages if it hasnt already been sent
select {
case mp.pruneTrigger <- struct{}{}:
default:
}
}
}
mp.changes.Pub(api.MpoolUpdate{
@ -522,31 +542,68 @@ func (mp *MessagePool) getStateBalance(addr address.Address, ts *types.TipSet) (
}
func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, cb func(address.Address, uint64) (*types.SignedMessage, error)) (*types.SignedMessage, error) {
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
// serialize push access to reduce lock contention
mp.addSema <- struct{}{}
defer func() {
<-mp.addSema
}()
mp.curTsLk.Lock()
mp.lk.Lock()
defer mp.lk.Unlock()
curTs := mp.curTs
fromKey := addr
if fromKey.Protocol() == address.ID {
var err error
fromKey, err = mp.api.StateAccountKey(ctx, fromKey, mp.curTs)
if err != nil {
mp.lk.Unlock()
mp.curTsLk.Unlock()
return nil, xerrors.Errorf("resolving sender key: %w", err)
}
}
nonce, err := mp.getNonceLocked(fromKey, mp.curTs)
if err != nil {
mp.lk.Unlock()
mp.curTsLk.Unlock()
return nil, xerrors.Errorf("get nonce locked failed: %w", err)
}
// release the locks for signing
mp.lk.Unlock()
mp.curTsLk.Unlock()
msg, err := cb(fromKey, nonce)
if err != nil {
return nil, err
}
// reacquire the locks and check state for consistency
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
if mp.curTs != curTs {
return nil, ErrTryAgain
}
mp.lk.Lock()
defer mp.lk.Unlock()
nonce2, err := mp.getNonceLocked(fromKey, mp.curTs)
if err != nil {
return nil, xerrors.Errorf("get nonce locked failed: %w", err)
}
if nonce2 != nonce {
return nil, ErrTryAgain
}
if err := mp.verifyMsgBeforePush(msg, mp.curTs.Height()); err != nil {
return nil, err
}
msgb, err := msg.Serialize()
if err != nil {
return nil, err
@ -566,6 +623,10 @@ func (mp *MessagePool) Remove(from address.Address, nonce uint64) {
mp.lk.Lock()
defer mp.lk.Unlock()
mp.remove(from, nonce)
}
func (mp *MessagePool) remove(from address.Address, nonce uint64) {
mset, ok := mp.pending[from]
if !ok {
return
@ -576,6 +637,8 @@ func (mp *MessagePool) Remove(from address.Address, nonce uint64) {
Type: api.MpoolRemove,
Message: m,
}, localUpdates)
mp.currentSize--
}
// NB: This deletes any message with the given nonce. This makes sense
@ -613,6 +676,14 @@ func (mp *MessagePool) Pending() ([]*types.SignedMessage, *types.TipSet) {
return out, mp.curTs
}
func (mp *MessagePool) PendingFor(a address.Address) ([]*types.SignedMessage, *types.TipSet) {
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
mp.lk.Lock()
defer mp.lk.Unlock()
return mp.pendingFor(a), mp.curTs
}
func (mp *MessagePool) pendingFor(a address.Address) []*types.SignedMessage {
mset := mp.pending[a]
@ -637,6 +708,7 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
repubTrigger := false
rmsgs := make(map[address.Address]map[uint64]*types.SignedMessage)
add := func(m *types.SignedMessage) {
s, ok := rmsgs[m.Message.From]
@ -661,6 +733,17 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
mp.Remove(from, nonce)
}
maybeRepub := func(cid cid.Cid) {
if !repubTrigger {
mp.lk.Lock()
_, republished := mp.republished[cid]
mp.lk.Unlock()
if republished {
repubTrigger = true
}
}
}
for _, ts := range revert {
pts, err := mp.api.LoadTipSet(ts.Parents())
if err != nil {
@ -687,16 +770,25 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
}
for _, msg := range smsgs {
rm(msg.Message.From, msg.Message.Nonce)
maybeRepub(msg.Cid())
}
for _, msg := range bmsgs {
rm(msg.From, msg.Nonce)
maybeRepub(msg.Cid())
}
}
mp.curTs = ts
}
if repubTrigger {
select {
case mp.repubTrigger <- struct{}{}:
default:
}
}
for _, s := range rmsgs {
for _, msg := range s {
if err := mp.addSkipChecks(msg); err != nil {
@ -864,21 +956,9 @@ func (mp *MessagePool) loadLocal() error {
log.Errorf("adding local message: %+v", err)
}
mp.localAddrs[sm.Message.From] = struct{}{}
}
return nil
}
const MinGasPrice = 0
func (mp *MessagePool) EstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) {
// TODO: something smarter obviously
switch nblocksincl {
case 0:
return types.NewInt(MinGasPrice + 2), nil
case 1:
return types.NewInt(MinGasPrice + 1), nil
default:
return types.NewInt(MinGasPrice), nil
}
}

77
chain/messagepool/messagepool_test.go
View File

@ -11,16 +11,23 @@ import (
"github.com/filecoin-project/lotus/chain/wallet"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
)
func init() {
_ = logging.SetLogLevel("*", "INFO")
}
type testMpoolAPI struct {
cb func(rev, app []*types.TipSet) error
bmsgs map[cid.Cid][]*types.SignedMessage
statenonce map[address.Address]uint64
balance map[address.Address]types.BigInt
tipsets []*types.TipSet
}
@ -29,6 +36,7 @@ func newTestMpoolAPI() *testMpoolAPI {
return &testMpoolAPI{
bmsgs: make(map[cid.Cid][]*types.SignedMessage),
statenonce: make(map[address.Address]uint64),
balance: make(map[address.Address]types.BigInt),
}
}
@ -50,6 +58,14 @@ func (tma *testMpoolAPI) setStateNonce(addr address.Address, v uint64) {
tma.statenonce[addr] = v
}
func (tma *testMpoolAPI) setBalance(addr address.Address, v uint64) {
tma.balance[addr] = types.FromFil(v)
}
func (tma *testMpoolAPI) setBalanceRaw(addr address.Address, v types.BigInt) {
tma.balance[addr] = v
}
func (tma *testMpoolAPI) setBlockMessages(h *types.BlockHeader, msgs ...*types.SignedMessage) {
tma.bmsgs[h.Cid()] = msgs
tma.tipsets = append(tma.tipsets, mock.TipSet(h))
@ -69,9 +85,15 @@ func (tma *testMpoolAPI) PubSubPublish(string, []byte) error {
}
func (tma *testMpoolAPI) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
balance, ok := tma.balance[addr]
if !ok {
balance = types.NewInt(1000e6)
tma.balance[addr] = balance
}
return &types.Actor{
Code: builtin.StorageMarketActorCodeID,
Nonce: tma.statenonce[addr],
Balance: types.NewInt(90000000),
Balance: balance,
}, nil
}
@ -118,6 +140,10 @@ func (tma *testMpoolAPI) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error)
return nil, fmt.Errorf("tipset not found")
}
func (tma *testMpoolAPI) ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {
return types.NewInt(100), nil
}
func assertNonce(t *testing.T, mp *MessagePool, addr address.Address, val uint64) {
t.Helper()
n, err := mp.GetNonce(addr)
@ -233,3 +259,52 @@ func TestRevertMessages(t *testing.T) {
}
}
func TestPruningSimple(t *testing.T) {
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
if err != nil {
t.Fatal(err)
}
ds := datastore.NewMapDatastore()
mp, err := New(tma, ds, "mptest")
if err != nil {
t.Fatal(err)
}
a := mock.MkBlock(nil, 1, 1)
tma.applyBlock(t, a)
sender, err := w.GenerateKey(crypto.SigTypeBLS)
if err != nil {
t.Fatal(err)
}
target := mock.Address(1001)
for i := 0; i < 5; i++ {
smsg := mock.MkMessage(sender, target, uint64(i), w)
if err := mp.Add(smsg); err != nil {
t.Fatal(err)
}
}
for i := 10; i < 50; i++ {
smsg := mock.MkMessage(sender, target, uint64(i), w)
if err := mp.Add(smsg); err != nil {
t.Fatal(err)
}
}
mp.cfg.SizeLimitHigh = 40
mp.cfg.SizeLimitLow = 10
mp.Prune()
msgs, _ := mp.Pending()
if len(msgs) != 5 {
t.Fatal("expected only 5 messages in pool, got: ", len(msgs))
}
}

76
chain/messagepool/provider.go Normal file
View File

@ -0,0 +1,76 @@
package messagepool
import (
"context"
"github.com/ipfs/go-cid"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
)
type Provider interface {
SubscribeHeadChanges(func(rev, app []*types.TipSet) error) *types.TipSet
PutMessage(m types.ChainMsg) (cid.Cid, error)
PubSubPublish(string, []byte) error
StateGetActor(address.Address, *types.TipSet) (*types.Actor, error)
StateAccountKey(context.Context, address.Address, *types.TipSet) (address.Address, error)
MessagesForBlock(*types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error)
MessagesForTipset(*types.TipSet) ([]types.ChainMsg, error)
LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error)
ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error)
}
type mpoolProvider struct {
sm *stmgr.StateManager
ps *pubsub.PubSub
}
func NewProvider(sm *stmgr.StateManager, ps *pubsub.PubSub) Provider {
return &mpoolProvider{sm: sm, ps: ps}
}
func (mpp *mpoolProvider) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet {
mpp.sm.ChainStore().SubscribeHeadChanges(cb)
return mpp.sm.ChainStore().GetHeaviestTipSet()
}
func (mpp *mpoolProvider) PutMessage(m types.ChainMsg) (cid.Cid, error) {
return mpp.sm.ChainStore().PutMessage(m)
}
func (mpp *mpoolProvider) PubSubPublish(k string, v []byte) error {
return mpp.ps.Publish(k, v)
}
func (mpp *mpoolProvider) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
var act types.Actor
return &act, mpp.sm.WithParentState(ts, mpp.sm.WithActor(addr, stmgr.GetActor(&act)))
}
func (mpp *mpoolProvider) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
return mpp.sm.ResolveToKeyAddress(ctx, addr, ts)
}
func (mpp *mpoolProvider) MessagesForBlock(h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
return mpp.sm.ChainStore().MessagesForBlock(h)
}
func (mpp *mpoolProvider) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) {
return mpp.sm.ChainStore().MessagesForTipset(ts)
}
func (mpp *mpoolProvider) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
return mpp.sm.ChainStore().LoadTipSet(tsk)
}
func (mpp *mpoolProvider) ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {
baseFee, err := mpp.sm.ChainStore().ComputeBaseFee(ctx, ts)
if err != nil {
return types.NewInt(0), xerrors.Errorf("computing base fee at %s: %w", ts, err)
}
return baseFee, nil
}

105
chain/messagepool/pruning.go Normal file
View File

@ -0,0 +1,105 @@
package messagepool
import (
"context"
"sort"
"time"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)
func (mp *MessagePool) pruneExcessMessages() error {
mp.curTsLk.Lock()
ts := mp.curTs
mp.curTsLk.Unlock()
mp.lk.Lock()
defer mp.lk.Unlock()
if mp.currentSize < mp.cfg.SizeLimitHigh {
return nil
}
select {
case <-mp.pruneCooldown:
err := mp.pruneMessages(context.TODO(), ts)
go func() {
time.Sleep(mp.cfg.PruneCooldown)
mp.pruneCooldown <- struct{}{}
}()
return err
default:
return xerrors.New("cannot prune before cooldown")
}
}
func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) error {
start := time.Now()
defer func() {
log.Infof("message pruning took %s", time.Since(start))
}()
baseFee, err := mp.api.ChainComputeBaseFee(ctx, ts)
if err != nil {
return xerrors.Errorf("computing basefee: %w", err)
}
pending, _ := mp.getPendingMessages(ts, ts)
// priority actors -- not pruned
priority := make(map[address.Address]struct{})
for _, actor := range mp.cfg.PriorityAddrs {
priority[actor] = struct{}{}
}
// Collect all messages to track which ones to remove and create chains for block inclusion
pruneMsgs := make(map[cid.Cid]*types.SignedMessage, mp.currentSize)
keepCount := 0
var chains []*msgChain
for actor, mset := range pending {
// we never prune priority actors
_, keep := priority[actor]
if keep {
keepCount += len(mset)
continue
}
// not a priority actor, track the messages and create chains
for _, m := range mset {
pruneMsgs[m.Message.Cid()] = m
}
actorChains := mp.createMessageChains(actor, mset, baseFee, ts)
chains = append(chains, actorChains...)
}
// Sort the chains
sort.Slice(chains, func(i, j int) bool {
return chains[i].Before(chains[j])
})
// Keep messages (remove them from pruneMsgs) from chains while we are under the low water mark
loWaterMark := mp.cfg.SizeLimitLow
keepLoop:
for _, chain := range chains {
for _, m := range chain.msgs {
if keepCount < loWaterMark {
delete(pruneMsgs, m.Message.Cid())
keepCount++
} else {
break keepLoop
}
}
}
// and remove all messages that are still in pruneMsgs after processing the chains
log.Infof("Pruning %d messages", len(pruneMsgs))
for _, m := range pruneMsgs {
mp.remove(m.Message.From, m.Message.Nonce)
}
return nil
}

148
chain/messagepool/repub.go Normal file
View File

@ -0,0 +1,148 @@
package messagepool
import (
"context"
"sort"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
)
const repubMsgLimit = 30
func (mp *MessagePool) republishPendingMessages() error {
mp.curTsLk.Lock()
ts := mp.curTs
baseFee, err := mp.api.ChainComputeBaseFee(context.TODO(), ts)
if err != nil {
mp.curTsLk.Unlock()
return xerrors.Errorf("computing basefee: %w", err)
}
pending := make(map[address.Address]map[uint64]*types.SignedMessage)
mp.lk.Lock()
mp.republished = nil // clear this to avoid races triggering an early republish
for actor := range mp.localAddrs {
mset, ok := mp.pending[actor]
if !ok {
continue
}
if len(mset.msgs) == 0 {
continue
}
// we need to copy this while holding the lock to avoid races with concurrent modification
pend := make(map[uint64]*types.SignedMessage, len(mset.msgs))
for nonce, m := range mset.msgs {
pend[nonce] = m
}
pending[actor] = pend
}
mp.lk.Unlock()
mp.curTsLk.Unlock()
if len(pending) == 0 {
return nil
}
var chains []*msgChain
for actor, mset := range pending {
next := mp.createMessageChains(actor, mset, baseFee, ts)
chains = append(chains, next...)
}
if len(chains) == 0 {
return nil
}
sort.Slice(chains, func(i, j int) bool {
return chains[i].Before(chains[j])
})
// we don't republish negative performing chains; this is an error that will be screamed
// at the user
if chains[0].gasPerf < 0 {
return xerrors.Errorf("skipping republish: all message chains have negative gas performance; best gas performance: %f", chains[0].gasPerf)
}
gasLimit := int64(build.BlockGasLimit)
minGas := int64(gasguess.MinGas)
var msgs []*types.SignedMessage
for i := 0; i < len(chains); {
chain := chains[i]
// we can exceed this if we have picked (some) longer chain already
if len(msgs) > repubMsgLimit {
break
}
// there is not enough gas for any message
if gasLimit <= minGas {
break
}
// we don't republish negative performing chains, as they won't be included in
// a block anyway
if chain.gasPerf < 0 {
break
}
// has the chain been invalidated?
if !chain.valid {
i++
continue
}
// does it fit in a block?
if chain.gasLimit <= gasLimit {
gasLimit -= chain.gasLimit
msgs = append(msgs, chain.msgs...)
i++
continue
}
// we can't fit the current chain but there is gas to spare
// trim it and push it down
chain.Trim(gasLimit, mp, baseFee, ts, false)
for j := i; j < len(chains)-1; j++ {
if chains[j].Before(chains[j+1]) {
break
}
chains[j], chains[j+1] = chains[j+1], chains[j]
}
}
count := 0
log.Infof("republishing %d messages", len(msgs))
for _, m := range msgs {
mb, err := m.Serialize()
if err != nil {
return xerrors.Errorf("cannot serialize message: %w", err)
}
err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), mb)
if err != nil {
return xerrors.Errorf("cannot publish: %w", err)
}
count++
}
// track most recently republished messages
republished := make(map[cid.Cid]struct{})
for _, m := range msgs[:count] {
republished[m.Cid()] = struct{}{}
}
mp.lk.Lock()
// update the republished set so that we can trigger early republish from head changes
mp.republished = republished
mp.lk.Unlock()
return nil
}

889
chain/messagepool/selection.go Normal file
View File

@ -0,0 +1,889 @@
package messagepool
import (
"context"
"math/big"
"sort"
"time"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
abig "github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/ipfs/go-cid"
)
var bigBlockGasLimit = big.NewInt(build.BlockGasLimit)
const MaxBlocks = 15
type msgChain struct {
msgs []*types.SignedMessage
gasReward *big.Int
gasLimit int64
gasPerf float64
effPerf float64
bp float64
parentOffset float64
valid bool
merged bool
next *msgChain
prev *msgChain
}
func (mp *MessagePool) SelectMessages(ts *types.TipSet, tq float64) ([]*types.SignedMessage, error) {
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
mp.lk.Lock()
defer mp.lk.Unlock()
// if the ticket quality is high enough that the first block has higher probability
// than any other block, then we don't bother with optimal selection because the
// first block will always have higher effective performance
if tq > 0.84 {
return mp.selectMessagesGreedy(mp.curTs, ts)
}
return mp.selectMessagesOptimal(mp.curTs, ts, tq)
}
func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64) ([]*types.SignedMessage, error) {
start := time.Now()
baseFee, err := mp.api.ChainComputeBaseFee(context.TODO(), ts)
if err != nil {
return nil, xerrors.Errorf("computing basefee: %w", err)
}
// 0. Load messages from the target tipset; if it is the same as the current tipset in
// the mpool, then this is just the pending messages
pending, err := mp.getPendingMessages(curTs, ts)
if err != nil {
return nil, err
}
if len(pending) == 0 {
return nil, nil
}
// defer only here so if we have no pending messages we don't spam
defer func() {
log.Infow("message selection done", "took", time.Since(start))
}()
// 0b. Select all priority messages that fit in the block
minGas := int64(gasguess.MinGas)
result, gasLimit := mp.selectPriorityMessages(pending, baseFee, ts)
// have we filled the block?
if gasLimit < minGas {
return result, nil
}
// 1. Create a list of dependent message chains with maximal gas reward per limit consumed
startChains := time.Now()
var chains []*msgChain
for actor, mset := range pending {
next := mp.createMessageChains(actor, mset, baseFee, ts)
chains = append(chains, next...)
}
if dt := time.Since(startChains); dt > time.Millisecond {
log.Infow("create message chains done", "took", dt)
}
// 2. Sort the chains
sort.Slice(chains, func(i, j int) bool {
return chains[i].Before(chains[j])
})
if len(chains) != 0 && chains[0].gasPerf < 0 {
log.Warnw("all messages in mpool have non-positive gas performance", "bestGasPerf", chains[0].gasPerf)
return nil, nil
}
// 3. Parition chains into blocks (without trimming)
// we use the full blockGasLimit (as opposed to the residual gas limit from the
// priority message selection) as we have to account for what other miners are doing
nextChain := 0
partitions := make([][]*msgChain, MaxBlocks)
for i := 0; i < MaxBlocks && nextChain < len(chains); i++ {
gasLimit := int64(build.BlockGasLimit)
for nextChain < len(chains) {
chain := chains[nextChain]
nextChain++
partitions[i] = append(partitions[i], chain)
gasLimit -= chain.gasLimit
if gasLimit < minGas {
break
}
}
}
// 4. Compute effective performance for each chain, based on the partition they fall into
// The effective performance is the gasPerf of the chain * block probability
blockProb := mp.blockProbabilities(tq)
effChains := 0
for i := 0; i < MaxBlocks; i++ {
for _, chain := range partitions[i] {
chain.SetEffectivePerf(blockProb[i])
}
effChains += len(partitions[i])
}
// nullify the effective performance of chains that don't fit in any partition
for _, chain := range chains[effChains:] {
chain.SetNullEffectivePerf()
}
// 5. Resort the chains based on effective performance
sort.Slice(chains, func(i, j int) bool {
return chains[i].BeforeEffective(chains[j])
})
// 6. Merge the head chains to produce the list of messages selected for inclusion
// subject to the residual gas limit
// When a chain is merged in, all its previous dependent chains *must* also be
// merged in or we'll have a broken block
startMerge := time.Now()
last := len(chains)
for i, chain := range chains {
// did we run out of performing chains?
if chain.gasPerf < 0 {
break
}
// has it already been merged?
if chain.merged {
continue
}
// compute the dependencies that must be merged and the gas limit including deps
chainGasLimit := chain.gasLimit
var chainDeps []*msgChain
for curChain := chain.prev; curChain != nil && !curChain.merged; curChain = curChain.prev {
chainDeps = append(chainDeps, curChain)
chainGasLimit += curChain.gasLimit
}
// does it all fit in the block?
if chainGasLimit <= gasLimit {
// include it together with all dependencies
for i := len(chainDeps) - 1; i >= 0; i-- {
curChain := chainDeps[i]
curChain.merged = true
result = append(result, curChain.msgs...)
}
chain.merged = true
// adjust the effective pefromance for all subsequent chains
if next := chain.next; next != nil && next.effPerf > 0 {
next.effPerf += next.parentOffset
for next = next.next; next != nil && next.effPerf > 0; next = next.next {
next.setEffPerf()
}
}
result = append(result, chain.msgs...)
gasLimit -= chainGasLimit
// resort to account for already merged chains and effective performance adjustments
sort.Slice(chains[i+1:], func(i, j int) bool {
return chains[i].BeforeEffective(chains[j])
})
continue
}
// we can't fit this chain and its dependencies because of block gasLimit -- we are
// at the edge
last = i
break
}
if dt := time.Since(startMerge); dt > time.Millisecond {
log.Infow("merge message chains done", "took", dt)
}
// 7. We have reached the edge of what can fit wholesale; if we still hae available
// gasLimit to pack some more chains, then trim the last chain and push it down.
// Trimming invalidaates subsequent dependent chains so that they can't be selected
// as their dependency cannot be (fully) included.
// We do this in a loop because the blocker might have been inordinately large and
// we might have to do it multiple times to satisfy tail packing
startTail := time.Now()
tailLoop:
for gasLimit >= minGas && last < len(chains) {
// trim if necessary
if chains[last].gasLimit > gasLimit {
chains[last].Trim(gasLimit, mp, baseFee, ts, false)
}
// push down if it hasn't been invalidated
if chains[last].valid {
for i := last; i < len(chains)-1; i++ {
if chains[i].BeforeEffective(chains[i+1]) {
break
}
chains[i], chains[i+1] = chains[i+1], chains[i]
}
}
// select the next (valid and fitting) chain and its dependencies for inclusion
for i, chain := range chains[last:] {
// has the chain been invalidated?
if !chain.valid {
continue
}
// has it already been merged?
if chain.merged {
continue
}
// if gasPerf < 0 we have no more profitable chains
if chain.gasPerf < 0 {
break tailLoop
}
// compute the dependencies that must be merged and the gas limit including deps
chainGasLimit := chain.gasLimit
depGasLimit := int64(0)
var chainDeps []*msgChain
for curChain := chain.prev; curChain != nil && !curChain.merged; curChain = curChain.prev {
chainDeps = append(chainDeps, curChain)
chainGasLimit += curChain.gasLimit
depGasLimit += curChain.gasLimit
}
// does it all fit in the bock
if chainGasLimit <= gasLimit {
// include it together with all dependencies
for i := len(chainDeps) - 1; i >= 0; i-- {
curChain := chainDeps[i]
curChain.merged = true
result = append(result, curChain.msgs...)
}
chain.merged = true
result = append(result, chain.msgs...)
gasLimit -= chainGasLimit
continue
}
// it doesn't all fit; now we have to take into account the dependent chains before
// making a decision about trimming or invalidating.
// if the dependencies exceed the gas limit, then we must invalidate the chain
// as it can never be included.
// Otherwise we can just trim and continue
if depGasLimit > gasLimit {
chain.Invalidate()
last += i + 1
continue tailLoop
}
// dependencies fit, just trim it
chain.Trim(gasLimit-depGasLimit, mp, baseFee, ts, false)
last += i
continue tailLoop
}
// the merge loop ended after processing all the chains and we we probably have still
// gas to spare; end the loop.
break
}
if dt := time.Since(startTail); dt > time.Millisecond {
log.Infow("pack tail chains done", "took", dt)
}
return result, nil
}
func (mp *MessagePool) selectMessagesGreedy(curTs, ts *types.TipSet) ([]*types.SignedMessage, error) {
start := time.Now()
baseFee, err := mp.api.ChainComputeBaseFee(context.TODO(), ts)
if err != nil {
return nil, xerrors.Errorf("computing basefee: %w", err)
}
// 0. Load messages for the target tipset; if it is the same as the current tipset in the mpool
// then this is just the pending messages
pending, err := mp.getPendingMessages(curTs, ts)
if err != nil {
return nil, err
}
if len(pending) == 0 {
return nil, nil
}
// defer only here so if we have no pending messages we don't spam
defer func() {
log.Infow("message selection done", "took", time.Since(start))
}()
// 0b. Select all priority messages that fit in the block
minGas := int64(gasguess.MinGas)
result, gasLimit := mp.selectPriorityMessages(pending, baseFee, ts)
// have we filled the block?
if gasLimit < minGas {
return result, nil
}
// 1. Create a list of dependent message chains with maximal gas reward per limit consumed
startChains := time.Now()
var chains []*msgChain
for actor, mset := range pending {
next := mp.createMessageChains(actor, mset, baseFee, ts)
chains = append(chains, next...)
}
if dt := time.Since(startChains); dt > time.Millisecond {
log.Infow("create message chains done", "took", dt)
}
// 2. Sort the chains
sort.Slice(chains, func(i, j int) bool {
return chains[i].Before(chains[j])
})
if len(chains) != 0 && chains[0].gasPerf < 0 {
log.Warnw("all messages in mpool have non-positive gas performance", "bestGasPerf", chains[0].gasPerf)
return nil, nil
}
// 3. Merge the head chains to produce the list of messages selected for inclusion, subject to
// the block gas limit.
startMerge := time.Now()
last := len(chains)
for i, chain := range chains {
// did we run out of performing chains?
if chain.gasPerf < 0 {
break
}
// does it fit in the block?
if chain.gasLimit <= gasLimit {
gasLimit -= chain.gasLimit
result = append(result, chain.msgs...)
continue
}
// we can't fit this chain because of block gasLimit -- we are at the edge
last = i
break
}
if dt := time.Since(startMerge); dt > time.Millisecond {
log.Infow("merge message chains done", "took", dt)
}
// 4. We have reached the edge of what we can fit wholesale; if we still have available gasLimit
// to pack some more chains, then trim the last chain and push it down.
// Trimming invalidates subsequent dependent chains so that they can't be selected as their
// dependency cannot be (fully) included.
// We do this in a loop because the blocker might have been inordinately large and we might
// have to do it multiple times to satisfy tail packing.
startTail := time.Now()
tailLoop:
for gasLimit >= minGas && last < len(chains) {
// trim
chains[last].Trim(gasLimit, mp, baseFee, ts, false)
// push down if it hasn't been invalidated
if chains[last].valid {
for i := last; i < len(chains)-1; i++ {
if chains[i].Before(chains[i+1]) {
break
}
chains[i], chains[i+1] = chains[i+1], chains[i]
}
}
// select the next (valid and fitting) chain for inclusion
for i, chain := range chains[last:] {
// has the chain been invalidated?
if !chain.valid {
continue
}
// if gasPerf < 0 we have no more profitable chains
if chain.gasPerf < 0 {
break tailLoop
}
// does it fit in the bock?
if chain.gasLimit <= gasLimit {
gasLimit -= chain.gasLimit
result = append(result, chain.msgs...)
continue
}
// this chain needs to be trimmed
last += i
continue tailLoop
}
// the merge loop ended after processing all the chains and we probably still have
// gas to spare; end the loop
break
}
if dt := time.Since(startTail); dt > time.Millisecond {
log.Infow("pack tail chains done", "took", dt)
}
return result, nil
}
func (mp *MessagePool) selectPriorityMessages(pending map[address.Address]map[uint64]*types.SignedMessage, baseFee types.BigInt, ts *types.TipSet) ([]*types.SignedMessage, int64) {
start := time.Now()
defer func() {
if dt := time.Since(start); dt > time.Millisecond {
log.Infow("select priority messages done", "took", dt)
}
}()
result := make([]*types.SignedMessage, 0, mp.cfg.SizeLimitLow)
gasLimit := int64(build.BlockGasLimit)
minGas := int64(gasguess.MinGas)
// 1. Get priority actor chains
var chains []*msgChain
priority := mp.cfg.PriorityAddrs
for _, actor := range priority {
mset, ok := pending[actor]
if ok {
// remove actor from pending set as we are already processed these messages
delete(pending, actor)
// create chains for the priority actor
next := mp.createMessageChains(actor, mset, baseFee, ts)
chains = append(chains, next...)
}
}
// 2. Sort the chains
sort.Slice(chains, func(i, j int) bool {
return chains[i].Before(chains[j])
})
// 3. Merge chains until the block limit; we are willing to include negative performing chains
// as these are messages from our own miners
last := len(chains)
for i, chain := range chains {
if chain.gasLimit <= gasLimit {
gasLimit -= chain.gasLimit
result = append(result, chain.msgs...)
continue
}
// we can't fit this chain because of block gasLimit -- we are at the edge
last = i
break
}
tailLoop:
for gasLimit >= minGas && last < len(chains) {
// trim, without discarding negative performing messages
chains[last].Trim(gasLimit, mp, baseFee, ts, true)
// push down if it hasn't been invalidated
if chains[last].valid {
for i := last; i < len(chains)-1; i++ {
if chains[i].Before(chains[i+1]) {
break
}
chains[i], chains[i+1] = chains[i+1], chains[i]
}
}
// select the next (valid and fitting) chain for inclusion
for i, chain := range chains[last:] {
// has the chain been invalidated
if !chain.valid {
continue
}
// does it fit in the bock?
if chain.gasLimit <= gasLimit {
gasLimit -= chain.gasLimit
result = append(result, chain.msgs...)
continue
}
// this chain needs to be trimmed
last += i
continue tailLoop
}
// the merge loop ended after processing all the chains and we probably still have gas to spare
// -- mark the end.
last = len(chains)
}
return result, gasLimit
}
func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.Address]map[uint64]*types.SignedMessage, error) {
start := time.Now()
result := make(map[address.Address]map[uint64]*types.SignedMessage)
haveCids := make(map[cid.Cid]struct{})
defer func() {
if dt := time.Since(start); dt > time.Millisecond {
log.Infow("get pending messages done", "took", dt)
}
}()
// are we in sync?
inSync := false
if curTs.Height() == ts.Height() && curTs.Equals(ts) {
inSync = true
}
// first add our current pending messages
for a, mset := range mp.pending {
if inSync {
// no need to copy the map
result[a] = mset.msgs
} else {
// we need to copy the map to avoid clobbering it as we load more messages
msetCopy := make(map[uint64]*types.SignedMessage, len(mset.msgs))
for nonce, m := range mset.msgs {
msetCopy[nonce] = m
}
result[a] = msetCopy
// mark the messages as seen
for _, m := range mset.msgs {
haveCids[m.Cid()] = struct{}{}
}
}
}
// we are in sync, that's the happy path
if inSync {
return result, nil
}
// nope, we need to sync the tipsets
for {
if curTs.Height() == ts.Height() {
if curTs.Equals(ts) {
return result, nil
}
// different blocks in tipsets -- we mark them as seen so that they are not included in
// in the message set we return, but *neither me (vyzo) nor why understand why*
// this code is also probably completely untested in production, so I am adding a big fat
// warning to revisit this case and sanity check this decision.
log.Warnf("mpool tipset has same height as target tipset but it's not equal; beware of dragons!")
have, err := mp.MessagesForBlocks(ts.Blocks())
if err != nil {
return nil, xerrors.Errorf("error retrieving messages for tipset: %w", err)
}
for _, m := range have {
haveCids[m.Cid()] = struct{}{}
}
}
msgs, err := mp.MessagesForBlocks(ts.Blocks())
if err != nil {
return nil, xerrors.Errorf("error retrieving messages for tipset: %w", err)
}
for _, m := range msgs {
if _, have := haveCids[m.Cid()]; have {
continue
}
haveCids[m.Cid()] = struct{}{}
mset, ok := result[m.Message.From]
if !ok {
mset = make(map[uint64]*types.SignedMessage)
result[m.Message.From] = mset
}
other, dupNonce := mset[m.Message.Nonce]
if dupNonce {
// duplicate nonce, selfishly keep the message with the highest GasPrice
// if the gas prices are the same, keep the one with the highest GasLimit
switch m.Message.GasPremium.Int.Cmp(other.Message.GasPremium.Int) {
case 0:
if m.Message.GasLimit > other.Message.GasLimit {
mset[m.Message.Nonce] = m
}
case 1:
mset[m.Message.Nonce] = m
}
} else {
mset[m.Message.Nonce] = m
}
}
if curTs.Height() >= ts.Height() {
return result, nil
}
ts, err = mp.api.LoadTipSet(ts.Parents())
if err != nil {
return nil, xerrors.Errorf("error loading parent tipset: %w", err)
}
}
}
func (mp *MessagePool) getGasReward(msg *types.SignedMessage, baseFee types.BigInt, ts *types.TipSet) *big.Int {
maxPremium := types.BigSub(msg.Message.GasFeeCap, baseFee)
if types.BigCmp(maxPremium, msg.Message.GasPremium) < 0 {
maxPremium = msg.Message.GasPremium
}
gasReward := abig.Mul(maxPremium, types.NewInt(uint64(msg.Message.GasLimit)))
return gasReward.Int
}
func (mp *MessagePool) getGasPerf(gasReward *big.Int, gasLimit int64) float64 {
// gasPerf = gasReward * build.BlockGasLimit / gasLimit
a := new(big.Rat).SetInt(new(big.Int).Mul(gasReward, bigBlockGasLimit))
b := big.NewRat(1, gasLimit)
c := new(big.Rat).Mul(a, b)
r, _ := c.Float64()
return r
}
func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint64]*types.SignedMessage, baseFee types.BigInt, ts *types.TipSet) []*msgChain {
// collect all messages
msgs := make([]*types.SignedMessage, 0, len(mset))
for _, m := range mset {
msgs = append(msgs, m)
}
// sort by nonce
sort.Slice(msgs, func(i, j int) bool {
return msgs[i].Message.Nonce < msgs[j].Message.Nonce
})
// sanity checks:
// - there can be no gaps in nonces, starting from the current actor nonce
// if there is a gap, drop messages after the gap, we can't include them
// - all messages must have minimum gas and the total gas for the candidate messages
// cannot exceed the block limit; drop all messages that exceed the limit
// - the total gasReward cannot exceed the actor's balance; drop all messages that exceed
// the balance
a, _ := mp.api.StateGetActor(actor, ts)
curNonce := a.Nonce
balance := a.Balance.Int
gasLimit := int64(0)
skip := 0
i := 0
rewards := make([]*big.Int, 0, len(msgs))
for i = 0; i < len(msgs); i++ {
m := msgs[i]
if m.Message.Nonce < curNonce {
log.Warnf("encountered message from actor %s with nonce (%d) less than the current nonce (%d)",
actor, m.Message.Nonce, curNonce)
skip++
continue
}
if m.Message.Nonce != curNonce {
break
}
curNonce++
minGas := vm.PricelistByEpoch(ts.Height()).OnChainMessage(m.ChainLength()).Total()
if m.Message.GasLimit < minGas {
break
}
gasLimit += m.Message.GasLimit
if gasLimit > build.BlockGasLimit {
break
}
required := m.Message.RequiredFunds().Int
if balance.Cmp(required) < 0 {
break
}
balance = new(big.Int).Sub(balance, required)
value := m.Message.Value.Int
if balance.Cmp(value) >= 0 {
// Note: we only account for the value if the balance doesn't drop below 0
// otherwise the message will fail and the miner can reap the gas rewards
balance = new(big.Int).Sub(balance, value)
}
gasReward := mp.getGasReward(m, baseFee, ts)
rewards = append(rewards, gasReward)
}
// check we have a sane set of messages to construct the chains
if i > skip {
msgs = msgs[skip:i]
} else {
return nil
}
// ok, now we can construct the chains using the messages we have
// invariant: each chain has a bigger gasPerf than the next -- otherwise they can be merged
// and increase the gasPerf of the first chain
// We do this in two passes:
// - in the first pass we create chains that aggreagate messages with non-decreasing gasPerf
// - in the second pass we merge chains to maintain the invariant.
var chains []*msgChain
var curChain *msgChain
newChain := func(m *types.SignedMessage, i int) *msgChain {
chain := new(msgChain)
chain.msgs = []*types.SignedMessage{m}
chain.gasReward = rewards[i]
chain.gasLimit = m.Message.GasLimit
chain.gasPerf = mp.getGasPerf(chain.gasReward, chain.gasLimit)
chain.valid = true
return chain
}
// create the individual chains
for i, m := range msgs {
if curChain == nil {
curChain = newChain(m, i)
continue
}
gasReward := new(big.Int).Add(curChain.gasReward, rewards[i])
gasLimit := curChain.gasLimit + m.Message.GasLimit
gasPerf := mp.getGasPerf(gasReward, gasLimit)
// try to add the message to the current chain -- if it decreases the gasPerf, then make a
// new chain
if gasPerf < curChain.gasPerf {
chains = append(chains, curChain)
curChain = newChain(m, i)
} else {
curChain.msgs = append(curChain.msgs, m)
curChain.gasReward = gasReward
curChain.gasLimit = gasLimit
curChain.gasPerf = gasPerf
}
}
chains = append(chains, curChain)
// merge chains to maintain the invariant
for {
merged := 0
for i := len(chains) - 1; i > 0; i-- {
if chains[i].gasPerf >= chains[i-1].gasPerf {
chains[i-1].msgs = append(chains[i-1].msgs, chains[i].msgs...)
chains[i-1].gasReward = new(big.Int).Add(chains[i-1].gasReward, chains[i].gasReward)
chains[i-1].gasLimit += chains[i].gasLimit
chains[i-1].gasPerf = mp.getGasPerf(chains[i-1].gasReward, chains[i-1].gasLimit)
chains[i].valid = false
merged++
}
}
if merged == 0 {
break
}
// drop invalidated chains
newChains := make([]*msgChain, 0, len(chains)-merged)
for _, c := range chains {
if c.valid {
newChains = append(newChains, c)
}
}
chains = newChains
}
// link dependent chains
for i := 0; i < len(chains)-1; i++ {
chains[i].next = chains[i+1]
}
for i := len(chains) - 1; i > 0; i-- {
chains[i].prev = chains[i-1]
}
return chains
}
func (mc *msgChain) Before(other *msgChain) bool {
return mc.gasPerf > other.gasPerf ||
(mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0)
}
func (mc *msgChain) Trim(gasLimit int64, mp *MessagePool, baseFee types.BigInt, ts *types.TipSet, priority bool) {
i := len(mc.msgs) - 1
for i >= 0 && (mc.gasLimit > gasLimit || (!priority && mc.gasPerf < 0)) {
gasReward := mp.getGasReward(mc.msgs[i], baseFee, ts)
mc.gasReward = new(big.Int).Sub(mc.gasReward, gasReward)
mc.gasLimit -= mc.msgs[i].Message.GasLimit
if mc.gasLimit > 0 {
mc.gasPerf = mp.getGasPerf(mc.gasReward, mc.gasLimit)
if mc.bp != 0 {
mc.setEffPerf()
}
} else {
mc.gasPerf = 0
mc.effPerf = 0
}
i--
}
if i < 0 {
mc.msgs = nil
mc.valid = false
} else {
mc.msgs = mc.msgs[:i+1]
}
if mc.next != nil {
mc.next.Invalidate()
mc.next = nil
}
}
func (mc *msgChain) Invalidate() {
mc.valid = false
mc.msgs = nil
if mc.next != nil {
mc.next.Invalidate()
mc.next = nil
}
}
func (mc *msgChain) SetEffectivePerf(bp float64) {
mc.bp = bp
mc.setEffPerf()
}
func (mc *msgChain) setEffPerf() {
effPerf := mc.gasPerf * mc.bp
if effPerf > 0 && mc.prev != nil {
effPerfWithParent := (effPerf*float64(mc.gasLimit) + mc.prev.effPerf*float64(mc.prev.gasLimit)) / float64(mc.gasLimit+mc.prev.gasLimit)
mc.parentOffset = effPerf - effPerfWithParent
effPerf = effPerfWithParent
}
mc.effPerf = effPerf
}
func (mc *msgChain) SetNullEffectivePerf() {
if mc.gasPerf < 0 {
mc.effPerf = mc.gasPerf
} else {
mc.effPerf = 0
}
}
func (mc *msgChain) BeforeEffective(other *msgChain) bool {
// move merged chains to the front so we can discard them earlier
return (mc.merged && !other.merged) || mc.effPerf > other.effPerf ||
(mc.effPerf == other.effPerf && mc.gasPerf > other.gasPerf) ||
(mc.effPerf == other.effPerf && mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0)
}

1077
chain/messagepool/selection_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

6
chain/metrics/consensus.go
View File

@ -3,7 +3,6 @@ package metrics
import (
"context"
"encoding/json"
"time"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/ipfs/go-cid"
@ -11,6 +10,7 @@ import (
pubsub "github.com/libp2p/go-libp2p-pubsub"
"go.uber.org/fx"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/modules/helpers"
@ -89,7 +89,7 @@ func sendHeadNotifs(ctx context.Context, ps *pubsub.PubSub, topic string, chain
}
// using unix nano time makes very sure we pick a nonce higher than previous restart
nonce := uint64(time.Now().UnixNano())
nonce := uint64(build.Clock.Now().UnixNano())
for {
select {
@ -107,7 +107,7 @@ func sendHeadNotifs(ctx context.Context, ps *pubsub.PubSub, topic string, chain
Height: n.Val.Height(),
Weight: w,
NodeName: nickname,
Time: uint64(time.Now().UnixNano() / 1000_000),
Time: uint64(build.Clock.Now().UnixNano() / 1000_000),
Nonce: nonce,
}

47
chain/state/statetree.go
View File

@ -9,7 +9,6 @@ import (
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/trace"
@ -23,7 +22,7 @@ var log = logging.Logger("statetree")
// StateTree stores actors state by their ID.
type StateTree struct {
root *hamt.Node
root *adt.Map
Store cbor.IpldStore
snaps *stateSnaps
@ -117,15 +116,16 @@ func (ss *stateSnaps) deleteActor(addr address.Address) {
}
func NewStateTree(cst cbor.IpldStore) (*StateTree, error) {
return &StateTree{
root: hamt.NewNode(cst, hamt.UseTreeBitWidth(5)),
root: adt.MakeEmptyMap(adt.WrapStore(context.TODO(), cst)),
Store: cst,
snaps: newStateSnaps(),
}, nil
}
func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) {
nd, err := hamt.LoadNode(context.Background(), cst, c, hamt.UseTreeBitWidth(5))
nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), c)
if err != nil {
log.Errorf("loading hamt node %s failed: %s", c, err)
return nil, err
@ -170,7 +170,10 @@ func (st *StateTree) LookupID(addr address.Address) (address.Address, error) {
return address.Undef, xerrors.Errorf("loading init actor state: %w", err)
}
a, err := ias.ResolveAddress(&AdtStore{st.Store}, addr)
a, found, err := ias.ResolveAddress(&AdtStore{st.Store}, addr)
if err == nil && !found {
err = types.ErrActorNotFound
}
if err != nil {
return address.Undef, xerrors.Errorf("resolve address %s: %w", addr, err)
}
@ -189,7 +192,7 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) {
// Transform `addr` to its ID format.
iaddr, err := st.LookupID(addr)
if err != nil {
if xerrors.Is(err, init_.ErrAddressNotFound) {
if xerrors.Is(err, types.ErrActorNotFound) {
return nil, xerrors.Errorf("resolution lookup failed (%s): %w", addr, err)
}
return nil, xerrors.Errorf("address resolution: %w", err)
@ -206,12 +209,10 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) {
}
var act types.Actor
err = st.root.Find(context.TODO(), string(addr.Bytes()), &act)
if err != nil {
if err == hamt.ErrNotFound {
return nil, types.ErrActorNotFound
}
if found, err := st.root.Get(adt.AddrKey(addr), &act); err != nil {
return nil, xerrors.Errorf("hamt find failed: %w", err)
} else if !found {
return nil, types.ErrActorNotFound
}
st.snaps.setActor(addr, &act)
@ -226,7 +227,7 @@ func (st *StateTree) DeleteActor(addr address.Address) error {
iaddr, err := st.LookupID(addr)
if err != nil {
if xerrors.Is(err, init_.ErrAddressNotFound) {
if xerrors.Is(err, types.ErrActorNotFound) {
return xerrors.Errorf("resolution lookup failed (%s): %w", addr, err)
}
return xerrors.Errorf("address resolution: %w", err)
@ -253,21 +254,17 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
for addr, sto := range st.snaps.layers[0].actors {
if sto.Delete {
if err := st.root.Delete(ctx, string(addr.Bytes())); err != nil {
if err := st.root.Delete(adt.AddrKey(addr)); err != nil {
return cid.Undef, err
}
} else {
if err := st.root.Set(ctx, string(addr.Bytes()), &sto.Act); err != nil {
if err := st.root.Put(adt.AddrKey(addr), &sto.Act); err != nil {
return cid.Undef, err
}
}
}
if err := st.root.Flush(ctx); err != nil {
return cid.Undef, err
}
return st.Store.Put(ctx, st.root)
return st.root.Root()
}
func (st *StateTree) Snapshot(ctx context.Context) error {
@ -339,3 +336,15 @@ func (st *StateTree) MutateActor(addr address.Address, f func(*types.Actor) erro
return st.SetActor(addr, act)
}
func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error {
var act types.Actor
return st.root.ForEach(&act, func(k string) error {
addr, err := address.NewFromBytes([]byte(k))
if err != nil {
return xerrors.Errorf("invalid address (%x) found in state tree key: %w", []byte(k), err)
}
return f(addr, &act)
})
}

2
chain/state/statetree_test.go
View File

@ -272,7 +272,7 @@ func TestStateTreeConsistency(t *testing.T) {
}
fmt.Println("root is: ", root)
if root.String() != "bafy2bzaceadyjnrv3sbjvowfl3jr4pdn5p2bf3exjjie2f3shg4oy5sub7h34" {
if root.String() != "bafy2bzaceb2bhqw75pqp44efoxvlnm73lnctq6djair56bfn5x3gw56epcxbi" {
t.Fatal("MISMATCH!")
}
}

119
chain/stmgr/call.go
View File

@ -4,12 +4,15 @@ import (
"context"
"fmt"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/ipfs/go-cid"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
@ -19,17 +22,31 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw")
defer span.End()
vmi, err := vm.NewVM(bstate, bheight, r, sm.cs.Blockstore(), sm.cs.VMSys())
vmopt := &vm.VMOpts{
StateBase: bstate,
Epoch: bheight,
Rand: r,
Bstore: sm.cs.Blockstore(),
Syscalls: sm.cs.VMSys(),
CircSupplyCalc: sm.GetCirculatingSupply,
BaseFee: types.NewInt(0),
}
vmi, err := vm.NewVM(vmopt)
if err != nil {
return nil, xerrors.Errorf("failed to set up vm: %w", err)
}
if msg.GasLimit == 0 {
msg.GasLimit = 10000000000
msg.GasLimit = build.BlockGasLimit
}
if msg.GasPrice == types.EmptyInt {
msg.GasPrice = types.NewInt(0)
if msg.GasFeeCap == types.EmptyInt {
msg.GasFeeCap = types.NewInt(0)
}
if msg.GasPremium == types.EmptyInt {
msg.GasPremium = types.NewInt(0)
}
if msg.Value == types.EmptyInt {
msg.Value = types.NewInt(0)
}
@ -37,7 +54,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
if span.IsRecordingEvents() {
span.AddAttributes(
trace.Int64Attribute("gas_limit", msg.GasLimit),
trace.Int64Attribute("gas_price", int64(msg.GasPrice.Uint64())),
trace.StringAttribute("gas_feecap", msg.GasFeeCap.String()),
trace.StringAttribute("value", msg.Value.String()),
)
}
@ -83,13 +100,103 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
return sm.CallRaw(ctx, msg, state, r, ts.Height())
}
func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) {
ctx, span := trace.StartSpan(ctx, "statemanager.CallWithGas")
defer span.End()
if ts == nil {
ts = sm.cs.GetHeaviestTipSet()
}
state, _, err := sm.TipSetState(ctx, ts)
if err != nil {
return nil, xerrors.Errorf("computing tipset state: %w", err)
}
r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height())
if span.IsRecordingEvents() {
span.AddAttributes(
trace.Int64Attribute("gas_limit", msg.GasLimit),
trace.StringAttribute("gas_feecap", msg.GasFeeCap.String()),
trace.StringAttribute("value", msg.Value.String()),
)
}
vmopt := &vm.VMOpts{
StateBase: state,
Epoch: ts.Height() + 1,
Rand: r,
Bstore: sm.cs.Blockstore(),
Syscalls: sm.cs.VMSys(),
CircSupplyCalc: sm.GetCirculatingSupply,
BaseFee: ts.Blocks()[0].ParentBaseFee,
}
vmi, err := vm.NewVM(vmopt)
if err != nil {
return nil, xerrors.Errorf("failed to set up vm: %w", err)
}
for i, m := range priorMsgs {
_, err := vmi.ApplyMessage(ctx, m)
if err != nil {
return nil, xerrors.Errorf("applying prior message (%d, %s): %w", i, m.Cid(), err)
}
}
fromActor, err := vmi.StateTree().GetActor(msg.From)
if err != nil {
return nil, xerrors.Errorf("call raw get actor: %s", err)
}
msg.Nonce = fromActor.Nonce
fromKey, err := sm.ResolveToKeyAddress(ctx, msg.From, ts)
if err != nil {
return nil, xerrors.Errorf("could not resolve key: %w", err)
}
var msgApply types.ChainMsg
switch fromKey.Protocol() {
case address.BLS:
msgApply = msg
case address.SECP256K1:
msgApply = &types.SignedMessage{
Message: *msg,
Signature: crypto.Signature{
Type: crypto.SigTypeSecp256k1,
Data: make([]byte, 65),
},
}
}
ret, err := vmi.ApplyMessage(ctx, msgApply)
if err != nil {
return nil, xerrors.Errorf("apply message failed: %w", err)
}
var errs string
if ret.ActorErr != nil {
errs = ret.ActorErr.Error()
}
return &api.InvocResult{
Msg: msg,
MsgRct: &ret.MessageReceipt,
ExecutionTrace: ret.ExecutionTrace,
Error: errs,
Duration: ret.Duration,
}, nil
}
var errHaltExecution = fmt.Errorf("halt")
func (sm *StateManager) Replay(ctx context.Context, ts *types.TipSet, mcid cid.Cid) (*types.Message, *vm.ApplyRet, error) {
var outm *types.Message
var outr *vm.ApplyRet
_, _, err := sm.computeTipSetState(ctx, ts.Blocks(), func(c cid.Cid, m *types.Message, ret *vm.ApplyRet) error {
_, _, err := sm.computeTipSetState(ctx, ts, func(c cid.Cid, m *types.Message, ret *vm.ApplyRet) error {
if c == mcid {
outm = m
outr = ret

21
chain/stmgr/forks.go
View File

@ -3,23 +3,20 @@ package stmgr
import (
"context"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/ipfs/go-cid"
)
var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, cid.Cid) (cid.Cid, error){}
var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, types.StateTree) error{}
func (sm *StateManager) handleStateForks(ctx context.Context, pstate cid.Cid, height, parentH abi.ChainEpoch) (_ cid.Cid, err error) {
for i := parentH; i < height; i++ {
f, ok := ForksAtHeight[i]
if ok {
nstate, err := f(ctx, sm, pstate)
if err != nil {
return cid.Undef, err
}
pstate = nstate
func (sm *StateManager) handleStateForks(ctx context.Context, st types.StateTree, height abi.ChainEpoch) (err error) {
f, ok := ForksAtHeight[height]
if ok {
err := f(ctx, sm, st)
if err != nil {
return err
}
}
return pstate, nil
return nil
}

29
chain/stmgr/forks_test.go
View File

@ -21,7 +21,6 @@ import (
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/stmgr"
. "github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
@ -29,8 +28,6 @@ import (
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log"
cbg "github.com/whyrusleeping/cbor-gen"
@ -122,42 +119,38 @@ func TestForkHeightTriggers(t *testing.T) {
t.Fatal(err)
}
stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, pstate cid.Cid) (cid.Cid, error) {
stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, st types.StateTree) error {
cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
st, err := state.LoadStateTree(cst, pstate)
if err != nil {
return cid.Undef, err
}
act, err := st.GetActor(taddr)
if err != nil {
return cid.Undef, err
return err
}
var tas testActorState
if err := cst.Get(ctx, act.Head, &tas); err != nil {
return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err)
return xerrors.Errorf("in fork handler, failed to run get: %w", err)
}
tas.HasUpgraded = 55
ns, err := cst.Put(ctx, &tas)
if err != nil {
return cid.Undef, err
return err
}
act.Head = ns
if err := st.SetActor(taddr, act); err != nil {
return cid.Undef, err
return err
}
return st.Flush(ctx)
return nil
}
inv.Register(builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{})
sm.SetVMConstructor(func(c cid.Cid, h abi.ChainEpoch, r vm.Rand, b blockstore.Blockstore, s runtime.Syscalls) (*vm.VM, error) {
nvm, err := vm.NewVM(c, h, r, b, s)
sm.SetVMConstructor(func(vmopt *vm.VMOpts) (*vm.VM, error) {
nvm, err := vm.NewVM(vmopt)
if err != nil {
return nil, err
}
@ -179,8 +172,7 @@ func TestForkHeightTriggers(t *testing.T) {
To: builtin.InitActorAddr,
Method: builtin.MethodsInit.Exec,
Params: enc,
GasLimit: 10000,
GasPrice: types.NewInt(0),
GasLimit: types.TestGasLimit,
}
sig, err := cg.Wallet().Sign(ctx, cg.Banker(), m.Cid().Bytes())
if err != nil {
@ -206,8 +198,7 @@ func TestForkHeightTriggers(t *testing.T) {
Method: 2,
Params: nil,
Nonce: nonce,
GasLimit: 10000,
GasPrice: types.NewInt(0),
GasLimit: types.TestGasLimit,
}
nonce++

155
chain/stmgr/read.go Normal file
View File

@ -0,0 +1,155 @@
package stmgr
import (
"context"
"reflect"
"golang.org/x/xerrors"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/util/adt"
)
type StateTreeCB func(state *state.StateTree) error
func (sm *StateManager) WithParentStateTsk(tsk types.TipSetKey, cb StateTreeCB) error {
ts, err := sm.cs.GetTipSetFromKey(tsk)
if err != nil {
return xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
cst := cbor.NewCborStore(sm.cs.Blockstore())
state, err := state.LoadStateTree(cst, sm.parentState(ts))
if err != nil {
return xerrors.Errorf("load state tree: %w", err)
}
return cb(state)
}
func (sm *StateManager) WithParentState(ts *types.TipSet, cb StateTreeCB) error {
cst := cbor.NewCborStore(sm.cs.Blockstore())
state, err := state.LoadStateTree(cst, sm.parentState(ts))
if err != nil {
return xerrors.Errorf("load state tree: %w", err)
}
return cb(state)
}
func (sm *StateManager) WithStateTree(st cid.Cid, cb StateTreeCB) error {
cst := cbor.NewCborStore(sm.cs.Blockstore())
state, err := state.LoadStateTree(cst, st)
if err != nil {
return xerrors.Errorf("load state tree: %w", err)
}
return cb(state)
}
type ActorCB func(act *types.Actor) error
func GetActor(out *types.Actor) ActorCB {
return func(act *types.Actor) error {
*out = *act
return nil
}
}
func (sm *StateManager) WithActor(addr address.Address, cb ActorCB) StateTreeCB {
return func(state *state.StateTree) error {
act, err := state.GetActor(addr)
if err != nil {
return xerrors.Errorf("get actor: %w", err)
}
return cb(act)
}
}
// WithActorState usage:
// Option 1: WithActorState(ctx, idAddr, func(store adt.Store, st *ActorStateType) error {...})
// Option 2: WithActorState(ctx, idAddr, actorStatePtr)
func (sm *StateManager) WithActorState(ctx context.Context, out interface{}) ActorCB {
return func(act *types.Actor) error {
store := sm.cs.Store(ctx)
outCallback := reflect.TypeOf(out).Kind() == reflect.Func
var st reflect.Value
if outCallback {
st = reflect.New(reflect.TypeOf(out).In(1).Elem())
} else {
st = reflect.ValueOf(out)
}
if err := store.Get(ctx, act.Head, st.Interface()); err != nil {
return xerrors.Errorf("read actor head: %w", err)
}
if outCallback {
out := reflect.ValueOf(out).Call([]reflect.Value{reflect.ValueOf(store), st})
if !out[0].IsNil() && out[0].Interface().(error) != nil {
return out[0].Interface().(error)
}
}
return nil
}
}
type DeadlinesCB func(store adt.Store, deadlines *miner.Deadlines) error
func (sm *StateManager) WithDeadlines(cb DeadlinesCB) func(store adt.Store, mas *miner.State) error {
return func(store adt.Store, mas *miner.State) error {
deadlines, err := mas.LoadDeadlines(store)
if err != nil {
return err
}
return cb(store, deadlines)
}
}
type DeadlineCB func(store adt.Store, idx uint64, deadline *miner.Deadline) error
func (sm *StateManager) WithDeadline(idx uint64, cb DeadlineCB) DeadlinesCB {
return func(store adt.Store, deadlines *miner.Deadlines) error {
d, err := deadlines.LoadDeadline(store, idx)
if err != nil {
return err
}
return cb(store, idx, d)
}
}
func (sm *StateManager) WithEachDeadline(cb DeadlineCB) DeadlinesCB {
return func(store adt.Store, deadlines *miner.Deadlines) error {
return deadlines.ForEach(store, func(dlIdx uint64, dl *miner.Deadline) error {
return cb(store, dlIdx, dl)
})
}
}
type PartitionCB func(store adt.Store, idx uint64, partition *miner.Partition) error
func (sm *StateManager) WithEachPartition(cb PartitionCB) DeadlineCB {
return func(store adt.Store, idx uint64, deadline *miner.Deadline) error {
parts, err := deadline.PartitionsArray(store)
if err != nil {
return err
}
var partition miner.Partition
return parts.ForEach(&partition, func(i int64) error {
p := partition
return cb(store, uint64(i), &p)
})
}
}

746
chain/stmgr/stmgr.go
View File

File diff suppressed because it is too large Load Diff

290
chain/stmgr/utils.go
View File

@ -7,15 +7,13 @@ import (
"reflect"
cid "github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
amt "github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/account"
@ -38,19 +36,44 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.NetworkName, error) {
var state init_.State
_, err := sm.LoadActorStateRaw(ctx, builtin.InitActorAddr, &state, st)
err := sm.WithStateTree(st, sm.WithActor(builtin.InitActorAddr, sm.WithActorState(ctx, &state)))
if err != nil {
return "", xerrors.Errorf("(get sset) failed to load init actor state: %w", err)
return "", err
}
return dtypes.NetworkName(state.NetworkName), nil
}
func (sm *StateManager) LoadActorState(ctx context.Context, addr address.Address, out interface{}, ts *types.TipSet) (*types.Actor, error) {
var a *types.Actor
if err := sm.WithParentState(ts, sm.WithActor(addr, func(act *types.Actor) error {
a = act
return sm.WithActorState(ctx, out)(act)
})); err != nil {
return nil, err
}
return a, nil
}
func (sm *StateManager) LoadActorStateRaw(ctx context.Context, addr address.Address, out interface{}, st cid.Cid) (*types.Actor, error) {
var a *types.Actor
if err := sm.WithStateTree(st, sm.WithActor(addr, func(act *types.Actor) error {
a = act
return sm.WithActorState(ctx, out)(act)
})); err != nil {
return nil, err
}
return a, nil
}
func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) {
var mas miner.State
_, err := sm.LoadActorStateRaw(ctx, maddr, &mas, st)
@ -64,7 +87,12 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
return address.Undef, xerrors.Errorf("load state tree: %w", err)
}
return vm.ResolveToKeyAddr(state, cst, mas.Info.Worker)
info, err := mas.GetInfo(sm.cs.Store(ctx))
if err != nil {
return address.Address{}, err
}
return vm.ResolveToKeyAddr(state, cst, info.Worker)
}
func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, error) {
@ -99,35 +127,6 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres
}, nil
}
func SectorSetSizes(ctx context.Context, sm *StateManager, maddr address.Address, ts *types.TipSet) (api.MinerSectors, error) {
var mas miner.State
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return api.MinerSectors{}, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
notProving, err := abi.BitFieldUnion(mas.Faults, mas.Recoveries)
if err != nil {
return api.MinerSectors{}, err
}
npc, err := notProving.Count()
if err != nil {
return api.MinerSectors{}, err
}
blks := cbor.NewCborStore(sm.ChainStore().Blockstore())
ss, err := amt.LoadAMT(ctx, blks, mas.Sectors)
if err != nil {
return api.MinerSectors{}, err
}
return api.MinerSectors{
Sset: ss.Count,
Pset: ss.Count - npc,
}, nil
}
func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (miner.SectorPreCommitOnChainInfo, error) {
var mas miner.State
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
@ -175,31 +174,51 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet,
}
func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]abi.SectorInfo, error) {
var mas miner.State
_, err := sm.LoadActorStateRaw(ctx, maddr, &mas, st)
var partsProving []abi.BitField
var mas *miner.State
var info *miner.MinerInfo
err := sm.WithStateTree(st, sm.WithActor(maddr, sm.WithActorState(ctx, func(store adt.Store, mst *miner.State) error {
var err error
mas = mst
info, err = mas.GetInfo(store)
if err != nil {
return xerrors.Errorf("getting miner info: %w", err)
}
deadlines, err := mas.LoadDeadlines(store)
if err != nil {
return xerrors.Errorf("loading deadlines: %w", err)
}
return deadlines.ForEach(store, func(dlIdx uint64, deadline *miner.Deadline) error {
partitions, err := deadline.PartitionsArray(store)
if err != nil {
return xerrors.Errorf("getting partition array: %w", err)
}
var partition miner.Partition
return partitions.ForEach(&partition, func(partIdx int64) error {
p, err := bitfield.SubtractBitField(partition.Sectors, partition.Faults)
if err != nil {
return xerrors.Errorf("subtract faults from partition sectors: %w", err)
}
partsProving = append(partsProving, p)
return nil
})
})
})))
if err != nil {
return nil, xerrors.Errorf("(get sectors) failed to load miner actor state: %w", err)
return nil, err
}
cst := cbor.NewCborStore(sm.cs.Blockstore())
var deadlines miner.Deadlines
if err := cst.Get(ctx, mas.Deadlines, &deadlines); err != nil {
return nil, xerrors.Errorf("failed to load deadlines: %w", err)
}
notProving, err := abi.BitFieldUnion(mas.Faults, mas.Recoveries)
provingSectors, err := bitfield.MultiMerge(partsProving...)
if err != nil {
return nil, xerrors.Errorf("failed to union faults and recoveries: %w", err)
}
allSectors, err := bitfield.MultiMerge(append(deadlines.Due[:], mas.NewSectors)...)
if err != nil {
return nil, xerrors.Errorf("merging deadline bitfields failed: %w", err)
}
provingSectors, err := bitfield.SubtractBitField(allSectors, notProving)
if err != nil {
return nil, xerrors.Errorf("failed to subtract non-proving sectors from set: %w", err)
return nil, xerrors.Errorf("merge partition proving sets: %w", err)
}
numProvSect, err := provingSectors.Count()
@ -212,7 +231,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S
return nil, nil
}
spt, err := ffiwrapper.SealProofTypeFromSectorSize(mas.Info.SectorSize)
spt, err := ffiwrapper.SealProofTypeFromSectorSize(info.SectorSize)
if err != nil {
return nil, xerrors.Errorf("getting seal proof type: %w", err)
}
@ -237,7 +256,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S
return nil, xerrors.Errorf("failed to enumerate all sector IDs: %w", err)
}
sectorAmt, err := amt.LoadAMT(ctx, cst, mas.Sectors)
sectorAmt, err := adt.AsArray(sm.cs.Store(ctx), mas.Sectors)
if err != nil {
return nil, xerrors.Errorf("failed to load sectors amt: %w", err)
}
@ -247,39 +266,35 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S
sid := sectors[n]
var sinfo miner.SectorOnChainInfo
if err := sectorAmt.Get(ctx, sid, &sinfo); err != nil {
if found, err := sectorAmt.Get(sid, &sinfo); err != nil {
return nil, xerrors.Errorf("failed to get sector %d: %w", sid, err)
} else if !found {
return nil, xerrors.Errorf("failed to find sector %d", sid)
}
out[i] = abi.SectorInfo{
SealProof: spt,
SectorNumber: sinfo.Info.SectorNumber,
SealedCID: sinfo.Info.SealedCID,
SectorNumber: sinfo.SectorNumber,
SealedCID: sinfo.SealedCID,
}
}
return out, nil
}
func StateMinerInfo(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (miner.MinerInfo, error) {
func StateMinerInfo(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*miner.MinerInfo, error) {
var mas miner.State
_, err := sm.LoadActorStateRaw(ctx, maddr, &mas, ts.ParentState())
if err != nil {
return miner.MinerInfo{}, xerrors.Errorf("(get ssize) failed to load miner actor state: %w", err)
return nil, xerrors.Errorf("(get ssize) failed to load miner actor state: %w", err)
}
return mas.Info, nil
return mas.GetInfo(sm.cs.Store(ctx))
}
func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (bool, error) {
var mas miner.State
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return false, xerrors.Errorf("(get miner slashed) failed to load miner actor state")
}
var spas power.State
_, err = sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &spas, ts)
_, err := sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &spas, ts)
if err != nil {
return false, xerrors.Errorf("(get miner slashed) failed to load power actor state")
}
@ -302,53 +317,26 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma
return false, nil
}
func GetMinerDeadlines(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*miner.Deadlines, error) {
var mas miner.State
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return nil, xerrors.Errorf("(get ssize) failed to load miner actor state: %w", err)
}
return mas.LoadDeadlines(sm.cs.Store(ctx))
}
func GetMinerFaults(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*abi.BitField, error) {
var mas miner.State
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return nil, xerrors.Errorf("(get faults) failed to load miner actor state: %w", err)
}
return mas.Faults, nil
}
func GetMinerRecoveries(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*abi.BitField, error) {
var mas miner.State
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return nil, xerrors.Errorf("(get recoveries) failed to load miner actor state: %w", err)
}
return mas.Recoveries, nil
}
func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) {
var state market.State
if _, err := sm.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts); err != nil {
return nil, err
}
store := sm.ChainStore().Store(ctx)
da, err := amt.LoadAMT(ctx, cbor.NewCborStore(sm.ChainStore().Blockstore()), state.Proposals)
da, err := adt.AsArray(store, state.Proposals)
if err != nil {
return nil, err
}
var dp market.DealProposal
if err := da.Get(ctx, uint64(dealID), &dp); err != nil {
if found, err := da.Get(uint64(dealID), &dp); err != nil {
return nil, err
} else if !found {
return nil, xerrors.Errorf("deal %d not found", dealID)
}
sa, err := market.AsDealStateArray(sm.ChainStore().Store(ctx), state.States)
sa, err := market.AsDealStateArray(store, state.States)
if err != nil {
return nil, err
}
@ -400,15 +388,16 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([
}
func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid, filter *abi.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) {
a, err := amt.LoadAMT(ctx, cbor.NewCborStore(bs), ssc)
a, err := adt.AsArray(store.ActorStore(ctx, bs), ssc)
if err != nil {
return nil, err
}
var sset []*api.ChainSectorInfo
if err := a.ForEach(ctx, func(i uint64, v *cbg.Deferred) error {
var v cbg.Deferred
if err := a.ForEach(&v, func(i int64) error {
if filter != nil {
set, err := filter.IsSet(i)
set, err := filter.IsSet(uint64(i))
if err != nil {
return xerrors.Errorf("filter check error: %w", err)
}
@ -443,15 +432,29 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
return cid.Undef, nil, err
}
fstate, err := sm.handleStateForks(ctx, base, height, ts.Height())
r := store.NewChainRand(sm.cs, ts.Cids(), height)
vmopt := &vm.VMOpts{
StateBase: base,
Epoch: height,
Rand: r,
Bstore: sm.cs.Blockstore(),
Syscalls: sm.cs.VMSys(),
CircSupplyCalc: sm.GetCirculatingSupply,
BaseFee: ts.Blocks()[0].ParentBaseFee,
}
vmi, err := vm.NewVM(vmopt)
if err != nil {
return cid.Undef, nil, err
}
r := store.NewChainRand(sm.cs, ts.Cids(), height)
vmi, err := vm.NewVM(fstate, height, r, sm.cs.Blockstore(), sm.cs.VMSys())
if err != nil {
return cid.Undef, nil, err
for i := ts.Height(); i < height; i++ {
// handle state forks
err = sm.handleStateForks(ctx, vmi.StateTree(), i)
if err != nil {
return cid.Undef, nil, xerrors.Errorf("error handling state forks: %w", err)
}
// TODO: should we also run cron here?
}
for i, msg := range msgs {
@ -473,20 +476,6 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
return root, trace, nil
}
func GetProvingSetRaw(ctx context.Context, sm *StateManager, mas miner.State) ([]*api.ChainSectorInfo, error) {
notProving, err := abi.BitFieldUnion(mas.Faults, mas.Recoveries)
if err != nil {
return nil, err
}
provset, err := LoadSectorsFromSet(ctx, sm.cs.Blockstore(), mas.Sectors, notProving, true)
if err != nil {
return nil, xerrors.Errorf("failed to get proving set: %w", err)
}
return provset, nil
}
func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, error) {
var lbr abi.ChainEpoch
if round > build.WinningPoStSectorSetLookback {
@ -570,19 +559,30 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcn beacon.RandomBe
return nil, xerrors.Errorf("failed to get power: %w", err)
}
worker, err := sm.ResolveToKeyAddress(ctx, mas.GetWorker(), ts)
info, err := mas.GetInfo(sm.cs.Store(ctx))
if err != nil {
return nil, err
}
worker, err := sm.ResolveToKeyAddress(ctx, info.Worker, ts)
if err != nil {
return nil, xerrors.Errorf("resolving worker address: %w", err)
}
hmp, err := MinerHasMinPower(ctx, sm, maddr, lbts)
if err != nil {
return nil, xerrors.Errorf("determining if miner has min power failed: %w", err)
}
return &api.MiningBaseInfo{
MinerPower: mpow.QualityAdjPower,
NetworkPower: tpow.QualityAdjPower,
Sectors: sectors,
WorkerKey: worker,
SectorSize: mas.Info.SectorSize,
SectorSize: info.SectorSize,
PrevBeaconEntry: *prev,
BeaconEntries: entries,
HasMinPower: hmp,
}, nil
}
@ -634,11 +634,39 @@ func init() {
}
func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, method abi.MethodNum, ts *types.TipSet) (cbg.CBORUnmarshaler, error) {
act, err := sm.GetActor(to, ts)
if err != nil {
return nil, err
var act types.Actor
if err := sm.WithParentState(ts, sm.WithActor(to, GetActor(&act))); err != nil {
return nil, xerrors.Errorf("getting actor: %w", err)
}
m := MethodsMap[act.Code][method]
return reflect.New(m.Ret.Elem()).Interface().(cbg.CBORUnmarshaler), nil
}
func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) {
var ps power.State
_, err := sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &ps, ts)
if err != nil {
return false, xerrors.Errorf("loading power actor state: %w", err)
}
return ps.MinerNominalPowerMeetsConsensusMinimum(sm.ChainStore().Store(ctx), addr)
}
func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) {
str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState())
if err != nil {
return abi.TokenAmount{}, err
}
sum := types.NewInt(0)
err = str.ForEach(func(a address.Address, act *types.Actor) error {
sum = types.BigAdd(sum, act.Balance)
return nil
})
if err != nil {
return abi.TokenAmount{}, err
}
return sum, nil
}

73
chain/store/basefee.go Normal file
View File

@ -0,0 +1,73 @@
package store
import (
"context"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)
func computeNextBaseFee(baseFee types.BigInt, gasLimitUsed int64, noOfBlocks int) types.BigInt {
// deta := 1/PackingEfficiency * gasLimitUsed/noOfBlocks - build.BlockGasTarget
// change := baseFee * deta / BlockGasTarget / BaseFeeMaxChangeDenom
// nextBaseFee = baseFee + change
// nextBaseFee = max(nextBaseFee, build.MinimumBaseFee)
delta := build.PackingEfficiencyDenom * gasLimitUsed / (int64(noOfBlocks) * build.PackingEfficiencyNum)
delta -= build.BlockGasTarget
// cap change at 12.5% (BaseFeeMaxChangeDenom) by capping delta
if delta > build.BlockGasTarget {
delta = build.BlockGasTarget
}
if delta < -build.BlockGasTarget {
delta = -build.BlockGasTarget
}
change := big.Mul(baseFee, big.NewInt(delta))
change = big.Div(change, big.NewInt(build.BlockGasTarget))
change = big.Div(change, big.NewInt(build.BaseFeeMaxChangeDenom))
nextBaseFee := big.Add(baseFee, change)
if big.Cmp(nextBaseFee, big.NewInt(build.MinimumBaseFee)) < 0 {
nextBaseFee = big.NewInt(build.MinimumBaseFee)
}
return nextBaseFee
}
func (cs *ChainStore) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) {
zero := abi.NewTokenAmount(0)
// totalLimit is sum of GasLimits of unique messages in a tipset
totalLimit := int64(0)
seen := make(map[cid.Cid]struct{})
for _, b := range ts.Blocks() {
msg1, msg2, err := cs.MessagesForBlock(b)
if err != nil {
return zero, xerrors.Errorf("error getting messages for: %s: %w", b.Cid(), err)
}
for _, m := range msg1 {
c := m.Cid()
if _, ok := seen[c]; !ok {
totalLimit += m.GasLimit
seen[c] = struct{}{}
}
}
for _, m := range msg2 {
c := m.Cid()
if _, ok := seen[c]; !ok {
totalLimit += m.Message.GasLimit
seen[c] = struct{}{}
}
}
}
parentBaseFee := ts.Blocks()[0].ParentBaseFee
return computeNextBaseFee(parentBaseFee, totalLimit, len(ts.Blocks())), nil
}

34
chain/store/basefee_test.go Normal file
View File

@ -0,0 +1,34 @@
package store
import (
"fmt"
"testing"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/stretchr/testify/assert"
)
func TestBaseFee(t *testing.T) {
tests := []struct {
basefee uint64
limitUsed int64
noOfBlocks int
output uint64
}{
{100e6, 0, 1, 87.5e6},
{100e6, 0, 5, 87.5e6},
{100e6, build.BlockGasTarget, 1, 103.125e6},
{100e6, build.BlockGasTarget * 2, 2, 103.125e6},
{100e6, build.BlockGasLimit * 2, 2, 112.5e6},
{100e6, build.BlockGasLimit * 1.5, 2, 110937500},
}
for _, test := range tests {
test := test
t.Run(fmt.Sprintf("%v", test), func(t *testing.T) {
output := computeNextBaseFee(types.NewInt(test.basefee), test.limitUsed, test.noOfBlocks)
assert.Equal(t, fmt.Sprintf("%d", test.output), output.String())
})
}
}

4
chain/store/index_test.go
View File

@ -8,10 +8,10 @@ import (
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/specs-actors/actors/abi"
datastore "github.com/ipfs/go-datastore"
syncds "github.com/ipfs/go-datastore/sync"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/stretchr/testify/assert"
)
@ -30,7 +30,7 @@ func TestIndexSeeks(t *testing.T) {
ctx := context.TODO()
nbs := blockstore.NewBlockstore(syncds.MutexWrap(datastore.NewMapDatastore()))
nbs := blockstore.NewTemporarySync()
cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil)
_, err = cs.Import(bytes.NewReader(gencar))

241
chain/store/store.go
View File

@ -15,28 +15,25 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/runtime"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/journal"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/metrics"
"go.opencensus.io/stats"
"go.opencensus.io/trace"
"go.uber.org/multierr"
amt "github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/lotus/chain/types"
lru "github.com/hashicorp/golang-lru"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
blockstore "github.com/ipfs/go-ipfs-blockstore"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log/v2"
car "github.com/ipld/go-car"
@ -96,10 +93,10 @@ type ChainStore struct {
mmCache *lru.ARCCache
tsCache *lru.ARCCache
vmcalls runtime.Syscalls
vmcalls vm.SyscallBuilder
}
func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Syscalls) *ChainStore {
func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder) *ChainStore {
c, _ := lru.NewARC(2048)
tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
cs := &ChainStore{
@ -671,7 +668,7 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) {
return m, nil
}
if err != bstore.ErrNotFound {
log.Warn("GetCMessage: unexpected error getting unsigned message: %s", err)
log.Warnf("GetCMessage: unexpected error getting unsigned message: %s", err)
}
return cs.GetSignedMessage(c)
@ -699,28 +696,39 @@ func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error)
func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) {
ctx := context.TODO()
bs := cbor.NewCborStore(cs.bs)
a, err := amt.LoadAMT(ctx, bs, root)
a, err := adt.AsArray(cs.Store(ctx), root)
if err != nil {
return nil, xerrors.Errorf("amt load: %w", err)
}
var cids []cid.Cid
for i := uint64(0); i < a.Count; i++ {
var c cbg.CborCid
if err := a.Get(ctx, i, &c); err != nil {
return nil, xerrors.Errorf("failed to load cid from amt: %w", err)
}
var (
cids []cid.Cid
cborCid cbg.CborCid
)
if err := a.ForEach(&cborCid, func(i int64) error {
c := cid.Cid(cborCid)
cids = append(cids, c)
return nil
}); err != nil {
return nil, xerrors.Errorf("failed to traverse amt: %w", err)
}
cids = append(cids, cid.Cid(c))
if uint64(len(cids)) != a.Length() {
return nil, xerrors.Errorf("found %d cids, expected %d", len(cids), a.Length())
}
return cids, nil
}
func (cs *ChainStore) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) {
type BlockMessages struct {
Miner address.Address
BlsMessages []types.ChainMsg
SecpkMessages []types.ChainMsg
WinCount int64
}
func (cs *ChainStore) BlockMsgsForTipset(ts *types.TipSet) ([]BlockMessages, error) {
applied := make(map[address.Address]uint64)
balances := make(map[address.Address]types.BigInt)
cst := cbor.NewCborStore(cs.bs)
st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot)
@ -736,43 +744,80 @@ func (cs *ChainStore) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, err
}
applied[a] = act.Nonce
balances[a] = act.Balance
}
return nil
}
var out []types.ChainMsg
selectMsg := func(m *types.Message) (bool, error) {
if err := preloadAddr(m.From); err != nil {
return false, err
}
if applied[m.From] != m.Nonce {
return false, nil
}
applied[m.From]++
return true, nil
}
var out []BlockMessages
for _, b := range ts.Blocks() {
bms, sms, err := cs.MessagesForBlock(b)
if err != nil {
return nil, xerrors.Errorf("failed to get messages for block: %w", err)
}
cmsgs := make([]types.ChainMsg, 0, len(bms)+len(sms))
for _, m := range bms {
cmsgs = append(cmsgs, m)
}
for _, sm := range sms {
cmsgs = append(cmsgs, sm)
bm := BlockMessages{
Miner: b.Miner,
BlsMessages: make([]types.ChainMsg, 0, len(bms)),
SecpkMessages: make([]types.ChainMsg, 0, len(sms)),
WinCount: b.ElectionProof.WinCount,
}
for _, cm := range cmsgs {
m := cm.VMMessage()
if err := preloadAddr(m.From); err != nil {
return nil, err
for _, bmsg := range bms {
b, err := selectMsg(bmsg.VMMessage())
if err != nil {
return nil, xerrors.Errorf("failed to decide whether to select message for block: %w", err)
}
if applied[m.From] != m.Nonce {
continue
if b {
bm.BlsMessages = append(bm.BlsMessages, bmsg)
}
applied[m.From]++
}
if balances[m.From].LessThan(m.RequiredFunds()) {
continue
for _, smsg := range sms {
b, err := selectMsg(smsg.VMMessage())
if err != nil {
return nil, xerrors.Errorf("failed to decide whether to select message for block: %w", err)
}
balances[m.From] = types.BigSub(balances[m.From], m.RequiredFunds())
out = append(out, cm)
if b {
bm.SecpkMessages = append(bm.SecpkMessages, smsg)
}
}
out = append(out, bm)
}
return out, nil
}
func (cs *ChainStore) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) {
bmsgs, err := cs.BlockMsgsForTipset(ts)
if err != nil {
return nil, err
}
var out []types.ChainMsg
for _, bm := range bmsgs {
for _, blsm := range bm.BlsMessages {
out = append(out, blsm)
}
for _, secm := range bm.SecpkMessages {
out = append(out, secm)
}
}
@ -860,15 +905,16 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message,
func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.MessageReceipt, error) {
ctx := context.TODO()
bs := cbor.NewCborStore(cs.bs)
a, err := amt.LoadAMT(ctx, bs, b.ParentMessageReceipts)
a, err := adt.AsArray(cs.Store(ctx), b.ParentMessageReceipts)
if err != nil {
return nil, xerrors.Errorf("amt load: %w", err)
}
var r types.MessageReceipt
if err := a.Get(ctx, uint64(i), &r); err != nil {
if found, err := a.Get(uint64(i), &r); err != nil {
return nil, err
} else if !found {
return nil, xerrors.Errorf("failed to find receipt %d", i)
}
return &r, nil
@ -906,35 +952,15 @@ func (cs *ChainStore) Blockstore() bstore.Blockstore {
return cs.bs
}
func ActorStore(ctx context.Context, bs blockstore.Blockstore) adt.Store {
return &astore{
cst: cbor.NewCborStore(bs),
ctx: ctx,
}
}
type astore struct {
cst cbor.IpldStore
ctx context.Context
}
func (a *astore) Context() context.Context {
return a.ctx
}
func (a *astore) Get(ctx context.Context, c cid.Cid, out interface{}) error {
return a.cst.Get(ctx, c, out)
}
func (a *astore) Put(ctx context.Context, v interface{}) (cid.Cid, error) {
return a.cst.Put(ctx, v)
func ActorStore(ctx context.Context, bs bstore.Blockstore) adt.Store {
return adt.WrapStore(ctx, cbor.NewCborStore(bs))
}
func (cs *ChainStore) Store(ctx context.Context) adt.Store {
return ActorStore(ctx, cs.bs)
}
func (cs *ChainStore) VMSys() runtime.Syscalls {
func (cs *ChainStore) VMSys() vm.SyscallBuilder {
return cs.vmcalls
}
@ -981,8 +1007,42 @@ func DrawRandomness(rbase []byte, pers crypto.DomainSeparationTag, round abi.Cha
return h.Sum(nil), nil
}
func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
_, span := trace.StartSpan(ctx, "store.GetRandomness")
func (cs *ChainStore) GetBeaconRandomness(ctx context.Context, blks []cid.Cid, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
_, span := trace.StartSpan(ctx, "store.GetBeaconRandomness")
defer span.End()
span.AddAttributes(trace.Int64Attribute("round", int64(round)))
ts, err := cs.LoadTipSet(types.NewTipSetKey(blks...))
if err != nil {
return nil, err
}
if round > ts.Height() {
return nil, xerrors.Errorf("cannot draw randomness from the future")
}
searchHeight := round
if searchHeight < 0 {
searchHeight = 0
}
randTs, err := cs.GetTipsetByHeight(ctx, searchHeight, ts, true)
if err != nil {
return nil, err
}
be, err := cs.GetLatestBeaconEntry(randTs)
if err != nil {
return nil, err
}
// if at (or just past -- for null epochs) appropriate epoch
// or at genesis (works for negative epochs)
return DrawRandomness(be.Data, pers, round, entropy)
}
func (cs *ChainStore) GetChainRandomness(ctx context.Context, blks []cid.Cid, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
_, span := trace.StartSpan(ctx, "store.GetChainRandomness")
defer span.End()
span.AddAttributes(trace.Int64Attribute("round", int64(round)))
@ -1049,7 +1109,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t
return cs.LoadTipSet(lbts.Parents())
}
func recurseLinks(bs blockstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
if root.Prefix().Codec != cid.DagCBOR {
return in, nil
}
@ -1059,21 +1119,25 @@ func recurseLinks(bs blockstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.C
return nil, xerrors.Errorf("recurse links get (%s) failed: %w", root, err)
}
top, err := cbg.ScanForLinks(bytes.NewReader(data.RawData()))
var rerr error
err = cbg.ScanForLinks(bytes.NewReader(data.RawData()), func(c cid.Cid) {
if rerr != nil {
// No error return on ScanForLinks :(
return
}
in = append(in, c)
var err error
in, err = recurseLinks(bs, c, in)
if err != nil {
rerr = err
}
})
if err != nil {
return nil, xerrors.Errorf("scanning for links failed: %w", err)
}
in = append(in, top...)
for _, c := range top {
var err error
in, err = recurseLinks(bs, c, in)
if err != nil {
return nil, err
}
}
return in, nil
return in, rerr
}
func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) error {
@ -1113,15 +1177,20 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer)
return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err)
}
for _, p := range b.Parents {
blocksToWalk = append(blocksToWalk, p)
}
cids, err := recurseLinks(cs.bs, b.Messages, []cid.Cid{b.Messages})
if err != nil {
return xerrors.Errorf("recursing messages failed: %w", err)
}
if b.Height > 0 {
for _, p := range b.Parents {
blocksToWalk = append(blocksToWalk, p)
}
} else {
// include the genesis block
cids = append(cids, b.Parents...)
}
out := cids
if b.Height == 0 {
@ -1219,8 +1288,12 @@ func NewChainRand(cs *ChainStore, blks []cid.Cid, bheight abi.ChainEpoch) vm.Ran
}
}
func (cr *chainRand) GetRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
return cr.cs.GetRandomness(ctx, cr.blks, pers, round, entropy)
func (cr *chainRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
return cr.cs.GetChainRandomness(ctx, cr.blks, pers, round, entropy)
}
func (cr *chainRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
return cr.cs.GetBeaconRandomness(ctx, cr.blks, pers, round, entropy)
}
func (cs *ChainStore) GetTipSetFromKey(tsk types.TipSetKey) (*types.TipSet, error) {

6
chain/store/store_test.go
View File

@ -6,7 +6,6 @@ import (
"testing"
datastore "github.com/ipfs/go-datastore"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
@ -18,6 +17,7 @@ import (
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/node/repo"
)
@ -72,7 +72,7 @@ func BenchmarkGetRandomness(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := cs.GetRandomness(context.TODO(), last.Cids(), crypto.DomainSeparationTag_SealRandomness, 500, nil)
_, err := cs.GetChainRandomness(context.TODO(), last.Cids(), crypto.DomainSeparationTag_SealRandomness, 500, nil)
if err != nil {
b.Fatal(err)
}
@ -100,7 +100,7 @@ func TestChainExportImport(t *testing.T) {
t.Fatal(err)
}
nbs := blockstore.NewBlockstore(datastore.NewMapDatastore())
nbs := blockstore.NewTemporary()
cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil)
root, err := cs.Import(buf)

64
chain/store/weight.go
View File

@ -7,7 +7,6 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
big2 "github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
@ -21,11 +20,11 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
if ts == nil {
return types.NewInt(0), nil
}
// >>> w[r] <<< + wFunction(totalPowerAtTipset(ts)) * 2^8 + (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den)
// >>> w[r] <<< + wFunction(totalPowerAtTipset(ts)) * 2^8 + (wFunction(totalPowerAtTipset(ts)) * sum(ts.blocks[].ElectionProof.WinCount) * wRatio_num * 2^8) / (e * wRatio_den)
var out = new(big.Int).Set(ts.Blocks()[0].ParentWeight.Int)
var out = new(big.Int).Set(ts.ParentWeight().Int)
// >>> wFunction(totalPowerAtTipset(ts)) * 2^8 <<< + (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den)
// >>> wFunction(totalPowerAtTipset(ts)) * 2^8 <<< + (wFunction(totalPowerAtTipset(ts)) * sum(ts.blocks[].ElectionProof.WinCount) * wRatio_num * 2^8) / (e * wRatio_den)
tpow := big2.Zero()
{
@ -57,52 +56,19 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
out.Add(out, big.NewInt(log2P<<8))
// (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den)
// (wFunction(totalPowerAtTipset(ts)) * sum(ts.blocks[].ElectionProof.WinCount) * wRatio_num * 2^8) / (e * wRatio_den)
eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * build.WRatioNum) << 8)
eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch*build.WRatioDen)))
out.Add(out, eWeight)
totalJ := int64(0)
for _, b := range ts.Blocks() {
totalJ += b.ElectionProof.WinCount
}
eWeight := big.NewInt((log2P * build.WRatioNum))
eWeight = eWeight.Lsh(eWeight, 8)
eWeight = eWeight.Mul(eWeight, new(big.Int).SetInt64(totalJ))
eWeight = eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch*build.WRatioDen)))
out = out.Add(out, eWeight)
return types.BigInt{Int: out}, nil
}
// todo: dedupe with state manager
func (cs *ChainStore) call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) {
bstate := ts.ParentState()
r := NewChainRand(cs, ts.Cids(), ts.Height())
vmi, err := vm.NewVM(bstate, ts.Height(), r, cs.bs, cs.vmcalls)
if err != nil {
return nil, xerrors.Errorf("failed to set up vm: %w", err)
}
if msg.GasLimit == 0 {
msg.GasLimit = 10000000000
}
if msg.GasPrice == types.EmptyInt {
msg.GasPrice = types.NewInt(0)
}
if msg.Value == types.EmptyInt {
msg.Value = types.NewInt(0)
}
fromActor, err := vmi.StateTree().GetActor(msg.From)
if err != nil {
return nil, xerrors.Errorf("call raw get actor: %s", err)
}
msg.Nonce = fromActor.Nonce
// TODO: maybe just use the invoker directly?
// TODO: use signed message length for secp messages
ret, err := vmi.ApplyMessage(ctx, msg)
if err != nil {
return nil, xerrors.Errorf("apply message failed: %w", err)
}
if ret.ActorErr != nil {
log.Warnf("chain call failed: %s", ret.ActorErr)
}
return &ret.MessageReceipt, nil
}

380
chain/sub/incoming.go
View File

@ -10,12 +10,12 @@ import (
"golang.org/x/xerrors"
address "github.com/filecoin-project/go-address"
amt "github.com/filecoin-project/go-amt-ipld/v2"
miner "github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/util/adt"
lru "github.com/hashicorp/golang-lru"
blocks "github.com/ipfs/go-block-format"
bserv "github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log/v2"
connmgr "github.com/libp2p/go-libp2p-core/connmgr"
@ -32,6 +32,7 @@ import (
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/bufbstore"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/metrics"
@ -39,7 +40,7 @@ import (
var log = logging.Logger("sub")
func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, cmgr connmgr.ConnManager) {
func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bserv bserv.BlockService, cmgr connmgr.ConnManager) {
for {
msg, err := bsub.Next(ctx)
if err != nil {
@ -60,23 +61,23 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
src := msg.GetFrom()
go func() {
start := time.Now()
start := build.Clock.Now()
log.Debug("about to fetch messages for block from pubsub")
bmsgs, err := s.Bsync.FetchMessagesByCids(context.TODO(), blk.BlsMessages)
bmsgs, err := FetchMessagesByCids(context.TODO(), bserv, blk.BlsMessages)
if err != nil {
log.Errorf("failed to fetch all bls messages for block received over pubusb: %s; source: %s", err, src)
return
}
smsgs, err := s.Bsync.FetchSignedMessagesByCids(context.TODO(), blk.SecpkMessages)
smsgs, err := FetchSignedMessagesByCids(context.TODO(), bserv, blk.SecpkMessages)
if err != nil {
log.Errorf("failed to fetch all secpk messages for block received over pubusb: %s; source: %s", err, src)
return
}
took := time.Since(start)
took := build.Clock.Since(start)
log.Infow("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom(), "msgfetch", took)
if delay := time.Now().Unix() - int64(blk.Header.Timestamp); delay > 5 {
if delay := build.Clock.Now().Unix() - int64(blk.Header.Timestamp); delay > 5 {
log.Warnf("Received block with large delay %d from miner %s", delay, blk.Header.Miner)
}
@ -91,7 +92,111 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
}
}
func FetchMessagesByCids(
ctx context.Context,
bserv bserv.BlockService,
cids []cid.Cid,
) ([]*types.Message, error) {
out := make([]*types.Message, len(cids))
err := fetchCids(ctx, bserv, cids, func(i int, b blocks.Block) error {
msg, err := types.DecodeMessage(b.RawData())
if err != nil {
return err
}
// FIXME: We already sort in `fetchCids`, we are duplicating too much work,
// we don't need to pass the index.
if out[i] != nil {
return fmt.Errorf("received duplicate message")
}
out[i] = msg
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
// FIXME: Duplicate of above.
func FetchSignedMessagesByCids(
ctx context.Context,
bserv bserv.BlockService,
cids []cid.Cid,
) ([]*types.SignedMessage, error) {
out := make([]*types.SignedMessage, len(cids))
err := fetchCids(ctx, bserv, cids, func(i int, b blocks.Block) error {
smsg, err := types.DecodeSignedMessage(b.RawData())
if err != nil {
return err
}
if out[i] != nil {
return fmt.Errorf("received duplicate message")
}
out[i] = smsg
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
// Fetch `cids` from the block service, apply `cb` on each of them. Used
// by the fetch message functions above.
// We check that each block is received only once and we do not received
// blocks we did not request.
func fetchCids(
ctx context.Context,
bserv bserv.BlockService,
cids []cid.Cid,
cb func(int, blocks.Block) error,
) error {
// FIXME: Why don't we use the context here?
fetchedBlocks := bserv.GetBlocks(context.TODO(), cids)
cidIndex := make(map[cid.Cid]int)
for i, c := range cids {
cidIndex[c] = i
}
for i := 0; i < len(cids); i++ {
select {
case block, ok := <-fetchedBlocks:
if !ok {
// Closed channel, no more blocks fetched, check if we have all
// of the CIDs requested.
// FIXME: Review this check. We don't call the callback on the
// last index?
if i == len(cids)-1 {
break
}
return fmt.Errorf("failed to fetch all messages")
}
ix, ok := cidIndex[block.Cid()]
if !ok {
return fmt.Errorf("received message we didnt ask for")
}
if err := cb(ix, block); err != nil {
return err
}
}
}
return nil
}
type BlockValidator struct {
self peer.ID
peers *lru.TwoQueueCache
killThresh int
@ -108,9 +213,10 @@ type BlockValidator struct {
keycache map[string]address.Address
}
func NewBlockValidator(chain *store.ChainStore, stmgr *stmgr.StateManager, blacklist func(peer.ID)) *BlockValidator {
func NewBlockValidator(self peer.ID, chain *store.ChainStore, stmgr *stmgr.StateManager, blacklist func(peer.ID)) *BlockValidator {
p, _ := lru.New2Q(4096)
return &BlockValidator{
self: self,
peers: p,
killThresh: 10,
blacklist: blacklist,
@ -140,10 +246,14 @@ func (bv *BlockValidator) flagPeer(p peer.ID) {
}
func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) pubsub.ValidationResult {
if pid == bv.self {
return bv.validateLocalBlock(ctx, msg)
}
// track validation time
begin := time.Now()
begin := build.Clock.Now()
defer func() {
log.Debugf("block validation time: %s", time.Since(begin))
log.Debugf("block validation time: %s", build.Clock.Since(begin))
}()
stats.Record(ctx, metrics.BlockReceived.M(1))
@ -154,25 +264,10 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
bv.flagPeer(pid)
}
// make sure the block can be decoded
blk, err := types.DecodeBlockMsg(msg.GetData())
blk, what, err := bv.decodeAndCheckBlock(msg)
if err != nil {
log.Error("got invalid block over pubsub: ", err)
recordFailure("invalid")
return pubsub.ValidationReject
}
// check the message limit constraints
if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit {
log.Warnf("received block with too many messages over pubsub")
recordFailure("too_many_messages")
return pubsub.ValidationReject
}
// make sure we have a signature
if blk.Header.BlockSig == nil {
log.Warnf("received block without a signature over pubsub")
recordFailure("missing_signature")
recordFailure(what)
return pubsub.ValidationReject
}
@ -191,14 +286,14 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
// if we can't find it, we check whether we are (near) synced in the chain.
// if we are not synced we cannot validate the block and we must ignore it.
// if we are synced and the miner is unknown, then the block is rejcected.
key, err := bv.getMinerWorkerKey(ctx, blk)
key, err := bv.checkPowerAndGetWorkerKey(ctx, blk.Header)
if err != nil {
if bv.isChainNearSynced() {
log.Warnf("received block message from unknown miner over pubsub; rejecting message")
log.Warnf("received block from unknown miner or miner that doesn't meet min power over pubsub; rejecting message")
recordFailure("unknown_miner")
return pubsub.ValidationReject
} else {
log.Warnf("cannot validate block message; unknown miner in unsynced chain")
log.Warnf("cannot validate block message; unknown miner or miner that doesn't meet min power in unsynced chain")
return pubsub.ValidationIgnore
}
}
@ -210,6 +305,12 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
return pubsub.ValidationReject
}
if blk.Header.ElectionProof.WinCount < 1 {
log.Errorf("block is not claiming to be winning")
recordFailure("not_winning")
return pubsub.ValidationReject
}
// it's a good block! make sure we've only seen it once
if bv.recvBlocks.add(blk.Header.Cid()) > 0 {
// TODO: once these changes propagate to the network, we can consider
@ -223,40 +324,84 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
return pubsub.ValidationAccept
}
func (bv *BlockValidator) validateLocalBlock(ctx context.Context, msg *pubsub.Message) pubsub.ValidationResult {
stats.Record(ctx, metrics.BlockPublished.M(1))
blk, what, err := bv.decodeAndCheckBlock(msg)
if err != nil {
log.Errorf("got invalid local block: %s", err)
ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, what))
stats.Record(ctx, metrics.BlockValidationFailure.M(1))
return pubsub.ValidationIgnore
}
if count := bv.recvBlocks.add(blk.Header.Cid()); count > 0 {
log.Warnf("local block has been seen %d times; ignoring", count)
return pubsub.ValidationIgnore
}
msg.ValidatorData = blk
stats.Record(ctx, metrics.BlockValidationSuccess.M(1))
return pubsub.ValidationAccept
}
func (bv *BlockValidator) decodeAndCheckBlock(msg *pubsub.Message) (*types.BlockMsg, string, error) {
blk, err := types.DecodeBlockMsg(msg.GetData())
if err != nil {
return nil, "invalid", xerrors.Errorf("error decoding block: %w", err)
}
if count := len(blk.BlsMessages) + len(blk.SecpkMessages); count > build.BlockMessageLimit {
return nil, "too_many_messages", fmt.Errorf("block contains too many messages (%d)", count)
}
// make sure we have a signature
if blk.Header.BlockSig == nil {
return nil, "missing_signature", fmt.Errorf("block without a signature")
}
return blk, "", nil
}
func (bv *BlockValidator) isChainNearSynced() bool {
ts := bv.chain.GetHeaviestTipSet()
timestamp := ts.MinTimestamp()
now := time.Now().UnixNano()
now := build.Clock.Now().UnixNano()
cutoff := uint64(now) - uint64(6*time.Hour)
return timestamp > cutoff
}
func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error {
var bcids, scids []cbg.CBORMarshaler
for _, m := range msg.BlsMessages {
c := cbg.CborCid(m)
bcids = append(bcids, &c)
}
for _, m := range msg.SecpkMessages {
c := cbg.CborCid(m)
scids = append(scids, &c)
}
// TODO there has to be a simpler way to do this without the blockstore dance
bs := cbor.NewCborStore(bstore.NewBlockstore(dstore.NewMapDatastore()))
store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary()))
bmArr := adt.MakeEmptyArray(store)
smArr := adt.MakeEmptyArray(store)
bmroot, err := amt.FromArray(ctx, bs, bcids)
for i, m := range msg.BlsMessages {
c := cbg.CborCid(m)
if err := bmArr.Set(uint64(i), &c); err != nil {
return err
}
}
for i, m := range msg.SecpkMessages {
c := cbg.CborCid(m)
if err := smArr.Set(uint64(i), &c); err != nil {
return err
}
}
bmroot, err := bmArr.Root()
if err != nil {
return err
}
smroot, err := amt.FromArray(ctx, bs, scids)
smroot, err := smArr.Root()
if err != nil {
return err
}
mrcid, err := bs.Put(ctx, &types.MsgMeta{
mrcid, err := store.Put(store.Context(), &types.MsgMeta{
BlsMessages: bmroot,
SecpkMessages: smroot,
})
@ -272,55 +417,78 @@ func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockM
return nil
}
func (bv *BlockValidator) getMinerWorkerKey(ctx context.Context, msg *types.BlockMsg) (address.Address, error) {
addr := msg.Header.Miner
func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *types.BlockHeader) (address.Address, error) {
addr := bh.Miner
bv.mx.Lock()
key, ok := bv.keycache[addr.String()]
bv.mx.Unlock()
if ok {
return key, nil
if !ok {
// TODO I have a feeling all this can be simplified by cleverer DI to use the API
ts := bv.chain.GetHeaviestTipSet()
st, _, err := bv.stmgr.TipSetState(ctx, ts)
if err != nil {
return address.Undef, err
}
buf := bufbstore.NewBufferedBstore(bv.chain.Blockstore())
cst := cbor.NewCborStore(buf)
state, err := state.LoadStateTree(cst, st)
if err != nil {
return address.Undef, err
}
act, err := state.GetActor(addr)
if err != nil {
return address.Undef, err
}
blk, err := bv.chain.Blockstore().Get(act.Head)
if err != nil {
return address.Undef, err
}
aso := blk.RawData()
var mst miner.State
err = mst.UnmarshalCBOR(bytes.NewReader(aso))
if err != nil {
return address.Undef, err
}
info, err := mst.GetInfo(adt.WrapStore(ctx, cst))
if err != nil {
return address.Undef, err
}
worker := info.Worker
key, err = bv.stmgr.ResolveToKeyAddress(ctx, worker, ts)
if err != nil {
return address.Undef, err
}
bv.mx.Lock()
bv.keycache[addr.String()] = key
bv.mx.Unlock()
}
// TODO I have a feeling all this can be simplified by cleverer DI to use the API
ts := bv.chain.GetHeaviestTipSet()
st, _, err := bv.stmgr.TipSetState(ctx, ts)
if err != nil {
return address.Undef, err
}
buf := bufbstore.NewBufferedBstore(bv.chain.Blockstore())
cst := cbor.NewCborStore(buf)
state, err := state.LoadStateTree(cst, st)
if err != nil {
return address.Undef, err
}
act, err := state.GetActor(addr)
// we check that the miner met the minimum power at the lookback tipset
baseTs := bv.chain.GetHeaviestTipSet()
lbts, err := stmgr.GetLookbackTipSetForRound(ctx, bv.stmgr, baseTs, bh.Height)
if err != nil {
log.Warnf("failed to load lookback tipset for incoming block")
return address.Undef, err
}
blk, err := bv.chain.Blockstore().Get(act.Head)
if err != nil {
return address.Undef, err
}
aso := blk.RawData()
var mst miner.State
err = mst.UnmarshalCBOR(bytes.NewReader(aso))
hmp, err := stmgr.MinerHasMinPower(ctx, bv.stmgr, bh.Miner, lbts)
if err != nil {
log.Warnf("failed to determine if incoming block's miner has minimum power")
return address.Undef, err
}
worker := mst.Info.Worker
key, err = bv.stmgr.ResolveToKeyAddress(ctx, worker, ts)
if err != nil {
return address.Undef, err
if !hmp {
log.Warnf("incoming block's miner does not have minimum power")
return address.Undef, xerrors.New("incoming block's miner does not have minimum power")
}
bv.mx.Lock()
bv.keycache[addr.String()] = key
bv.mx.Unlock()
return key, nil
}
@ -348,14 +516,19 @@ func (brc *blockReceiptCache) add(bcid cid.Cid) int {
}
type MessageValidator struct {
self peer.ID
mpool *messagepool.MessagePool
}
func NewMessageValidator(mp *messagepool.MessagePool) *MessageValidator {
return &MessageValidator{mp}
func NewMessageValidator(self peer.ID, mp *messagepool.MessagePool) *MessageValidator {
return &MessageValidator{self: self, mpool: mp}
}
func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) pubsub.ValidationResult {
if pid == mv.self {
return mv.validateLocalMessage(ctx, msg)
}
stats.Record(ctx, metrics.MessageReceived.M(1))
m, err := types.DecodeSignedMessage(msg.Message.GetData())
if err != nil {
@ -373,7 +546,7 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs
)
stats.Record(ctx, metrics.MessageValidationFailure.M(1))
switch {
case xerrors.Is(err, messagepool.ErrBroadcastAnyway):
case xerrors.Is(err, messagepool.ErrBroadcastAnyway) || xerrors.Is(err, messagepool.ErrRBFTooLowPremium):
return pubsub.ValidationIgnore
default:
return pubsub.ValidationReject
@ -383,6 +556,45 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs
return pubsub.ValidationAccept
}
func (mv *MessageValidator) validateLocalMessage(ctx context.Context, msg *pubsub.Message) pubsub.ValidationResult {
// do some lightweight validation
stats.Record(ctx, metrics.MessagePublished.M(1))
m, err := types.DecodeSignedMessage(msg.Message.GetData())
if err != nil {
log.Warnf("failed to decode local message: %s", err)
stats.Record(ctx, metrics.MessageValidationFailure.M(1))
return pubsub.ValidationIgnore
}
if m.Size() > 32*1024 {
log.Warnf("local message is too large! (%dB)", m.Size())
stats.Record(ctx, metrics.MessageValidationFailure.M(1))
return pubsub.ValidationIgnore
}
if m.Message.To == address.Undef {
log.Warn("local message has invalid destination address")
stats.Record(ctx, metrics.MessageValidationFailure.M(1))
return pubsub.ValidationIgnore
}
if !m.Message.Value.LessThan(types.TotalFilecoinInt) {
log.Warnf("local messages has too high value: %s", m.Message.Value)
stats.Record(ctx, metrics.MessageValidationFailure.M(1))
return pubsub.ValidationIgnore
}
if err := mv.mpool.VerifyMsgSig(m); err != nil {
log.Warnf("signature verification failed for local message: %s", err)
stats.Record(ctx, metrics.MessageValidationFailure.M(1))
return pubsub.ValidationIgnore
}
stats.Record(ctx, metrics.MessageValidationSuccess.M(1))
return pubsub.ValidationAccept
}
func HandleIncomingMessages(ctx context.Context, mpool *messagepool.MessagePool, msub *pubsub.Subscription) {
for {
_, err := msub.Next(ctx)

255
chain/sync.go
View File

@ -13,8 +13,6 @@ import (
"github.com/Gurpartap/async"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/connmgr"
@ -26,8 +24,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
amt "github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
@ -45,6 +42,7 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/lib/sigs/bls"
"github.com/filecoin-project/lotus/metrics"
@ -140,6 +138,12 @@ func NewSyncer(sm *stmgr.StateManager, bsync *blocksync.BlockSync, connmgr connm
incoming: pubsub.New(50),
}
if build.InsecurePoStValidation {
log.Warn("*********************************************************************************************")
log.Warn(" [INSECURE-POST-VALIDATION] Insecure test validation is enabled. If you see this outside of a test, it is a severe bug! ")
log.Warn("*********************************************************************************************")
}
s.syncmgr = NewSyncManager(s.Sync)
return s, nil
}
@ -201,8 +205,8 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool {
syncer.Bsync.AddPeer(from)
bestPweight := syncer.store.GetHeaviestTipSet().Blocks()[0].ParentWeight
targetWeight := fts.TipSet().Blocks()[0].ParentWeight
bestPweight := syncer.store.GetHeaviestTipSet().ParentWeight()
targetWeight := fts.TipSet().ParentWeight()
if targetWeight.LessThan(bestPweight) {
var miners []string
for _, blk := range fts.TipSet().Blocks() {
@ -257,15 +261,13 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
}
// Collect the CIDs of both types of messages separately: BLS and Secpk.
var bcids, scids []cbg.CBORMarshaler
var bcids, scids []cid.Cid
for _, m := range fblk.BlsMessages {
c := cbg.CborCid(m.Cid())
bcids = append(bcids, &c)
bcids = append(bcids, m.Cid())
}
for _, m := range fblk.SecpkMessages {
c := cbg.CborCid(m.Cid())
scids = append(scids, &c)
scids = append(scids, m.Cid())
}
// TODO: IMPORTANT(GARBAGE). These message puts and the msgmeta
@ -355,19 +357,17 @@ func zipTipSetAndMessages(bs cbor.IpldStore, ts *types.TipSet, allbmsgs []*types
}
var smsgs []*types.SignedMessage
var smsgCids []cbg.CBORMarshaler
var smsgCids []cid.Cid
for _, m := range smi[bi] {
smsgs = append(smsgs, allsmsgs[m])
c := cbg.CborCid(allsmsgs[m].Cid())
smsgCids = append(smsgCids, &c)
smsgCids = append(smsgCids, allsmsgs[m].Cid())
}
var bmsgs []*types.Message
var bmsgCids []cbg.CBORMarshaler
var bmsgCids []cid.Cid
for _, m := range bmi[bi] {
bmsgs = append(bmsgs, allbmsgs[m])
c := cbg.CborCid(allbmsgs[m].Cid())
bmsgCids = append(bmsgCids, &c)
bmsgCids = append(bmsgCids, allbmsgs[m].Cid())
}
mrcid, err := computeMsgMeta(bs, bmsgCids, smsgCids)
@ -393,19 +393,36 @@ func zipTipSetAndMessages(bs cbor.IpldStore, ts *types.TipSet, allbmsgs []*types
// computeMsgMeta computes the root CID of the combined arrays of message CIDs
// of both types (BLS and Secpk).
func computeMsgMeta(bs cbor.IpldStore, bmsgCids, smsgCids []cbg.CBORMarshaler) (cid.Cid, error) {
ctx := context.TODO()
bmroot, err := amt.FromArray(ctx, bs, bmsgCids)
func computeMsgMeta(bs cbor.IpldStore, bmsgCids, smsgCids []cid.Cid) (cid.Cid, error) {
store := adt.WrapStore(context.TODO(), bs)
bmArr := adt.MakeEmptyArray(store)
smArr := adt.MakeEmptyArray(store)
for i, m := range bmsgCids {
c := cbg.CborCid(m)
if err := bmArr.Set(uint64(i), &c); err != nil {
return cid.Undef, err
}
}
for i, m := range smsgCids {
c := cbg.CborCid(m)
if err := smArr.Set(uint64(i), &c); err != nil {
return cid.Undef, err
}
}
bmroot, err := bmArr.Root()
if err != nil {
return cid.Undef, err
}
smroot, err := amt.FromArray(ctx, bs, smsgCids)
smroot, err := smArr.Root()
if err != nil {
return cid.Undef, err
}
mrcid, err := bs.Put(ctx, &types.MsgMeta{
mrcid, err := store.Put(store.Context(), &types.MsgMeta{
BlsMessages: bmroot,
SecpkMessages: smroot,
})
@ -621,21 +638,15 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
return nil
}
validationStart := time.Now()
validationStart := build.Clock.Now()
defer func() {
dur := time.Since(validationStart)
durMilli := dur.Seconds() * float64(1000)
stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(durMilli))
log.Infow("block validation", "took", dur, "height", b.Header.Height)
stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(metrics.SinceInMilliseconds(validationStart)))
log.Infow("block validation", "took", time.Since(validationStart), "height", b.Header.Height)
}()
ctx, span := trace.StartSpan(ctx, "validateBlock")
defer span.End()
if build.InsecurePoStValidation {
log.Warn("insecure test validation is enabled, if you see this outside of a test, it is a severe bug!")
}
if err := blockSanityChecks(b.Header); err != nil {
return xerrors.Errorf("incoming header failed basic sanity checks: %w", err)
}
@ -662,23 +673,18 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
return xerrors.Errorf("failed to get latest beacon entry: %w", err)
}
//nulls := h.Height - (baseTs.Height() + 1)
// fast checks first
nulls := h.Height - (baseTs.Height() + 1)
if tgtTs := baseTs.MinTimestamp() + build.BlockDelaySecs*uint64(nulls+1); h.Timestamp != tgtTs {
return xerrors.Errorf("block has wrong timestamp: %d != %d", h.Timestamp, tgtTs)
}
now := uint64(time.Now().Unix())
now := uint64(build.Clock.Now().Unix())
if h.Timestamp > now+build.AllowableClockDriftSecs {
return xerrors.Errorf("block was from the future (now=%d, blk=%d): %w", now, h.Timestamp, ErrTemporal)
}
if h.Timestamp > now {
log.Warn("Got block from the future, but within threshold", h.Timestamp, time.Now().Unix())
}
if h.Timestamp < baseTs.MinTimestamp()+(build.BlockDelaySecs*uint64(h.Height-baseTs.Height())) {
log.Warn("timestamp funtimes: ", h.Timestamp, baseTs.MinTimestamp(), h.Height, baseTs.Height())
diff := (baseTs.MinTimestamp() + (build.BlockDelaySecs * uint64(h.Height-baseTs.Height()))) - h.Timestamp
return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * deltaH:%d; diff %d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelaySecs, h.Height-baseTs.Height(), diff)
log.Warn("Got block from the future, but within threshold", h.Timestamp, build.Clock.Now().Unix())
}
msgsCheck := async.Err(func() error {
@ -695,6 +701,27 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
return nil
})
baseFeeCheck := async.Err(func() error {
baseFee, err := syncer.store.ComputeBaseFee(ctx, baseTs)
if err != nil {
return xerrors.Errorf("computing base fee: %w", err)
}
if types.BigCmp(baseFee, b.Header.ParentBaseFee) != 0 {
return xerrors.Errorf("base fee doesn't match: %s (header) != %s (computed)",
b.Header.ParentBaseFee, baseFee)
}
return nil
})
pweight, err := syncer.store.Weight(ctx, baseTs)
if err != nil {
return xerrors.Errorf("getting parent weight: %w", err)
}
if types.BigCmp(pweight, b.Header.ParentWeight) != 0 {
return xerrors.Errorf("parrent weight different: %s (header) != %s (computed)",
b.Header.ParentWeight, pweight)
}
// Stuff that needs stateroot / worker address
stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs)
if err != nil {
@ -726,6 +753,19 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
}
winnerCheck := async.Err(func() error {
if h.ElectionProof.WinCount < 1 {
return xerrors.Errorf("block is not claiming to be a winner")
}
hp, err := stmgr.MinerHasMinPower(ctx, syncer.sm, h.Miner, lbts)
if err != nil {
return xerrors.Errorf("determining if miner has min power failed: %w", err)
}
if !hp {
return xerrors.New("block's miner does not meet minimum power threshold")
}
rBeacon := *prevBeacon
if len(h.BeaconEntries) != 0 {
rBeacon = h.BeaconEntries[len(h.BeaconEntries)-1]
@ -735,7 +775,6 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
return xerrors.Errorf("failed to marshal miner address to cbor: %w", err)
}
//TODO: DST from spec actors when it is there
vrfBase, err := store.DrawRandomness(rBeacon.Data, crypto.DomainSeparationTag_ElectionProofProduction, h.Height, buf.Bytes())
if err != nil {
return xerrors.Errorf("could not draw randomness: %w", err)
@ -759,8 +798,9 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
return xerrors.Errorf("failed getting power: %w", err)
}
if !types.IsTicketWinner(h.ElectionProof.VRFProof, mpow.QualityAdjPower, tpow.QualityAdjPower) {
return xerrors.Errorf("miner created a block but was not a winner")
j := h.ElectionProof.ComputeWinCount(mpow.QualityAdjPower, tpow.QualityAdjPower)
if h.ElectionProof.WinCount != j {
return xerrors.Errorf("miner claims wrong number of wins: miner: %d, computed: %d", h.ElectionProof.WinCount, j)
}
return nil
@ -824,6 +864,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
wproofCheck,
winnerCheck,
msgsCheck,
baseFeeCheck,
}
var merr error
@ -848,6 +889,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
"%d errors occurred:\n\t%s\n\n",
len(es), strings.Join(points, "\n\t"))
}
return mulErr
}
if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil {
@ -860,13 +902,13 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
func (syncer *Syncer) VerifyWinningPoStProof(ctx context.Context, h *types.BlockHeader, prevBeacon types.BeaconEntry, lbst cid.Cid, waddr address.Address) error {
if build.InsecurePoStValidation {
if len(h.WinPoStProof) == 0 {
return xerrors.Errorf("[TESTING] No winning post proof given")
return xerrors.Errorf("[INSECURE-POST-VALIDATION] No winning post proof given")
}
if string(h.WinPoStProof[0].ProofBytes) == "valid proof" {
return nil
}
return xerrors.Errorf("[TESTING] winning post was invalid")
return xerrors.Errorf("[INSECURE-POST-VALIDATION] winning post was invalid")
}
buf := new(bytes.Buffer)
@ -905,7 +947,7 @@ func (syncer *Syncer) VerifyWinningPoStProof(ctx context.Context, h *types.Block
}
if !ok {
log.Errorf("invalid winning post (%x; %v)", rand, sectors)
log.Errorf("invalid winning post (block: %s, %x; %v)", h.Cid(), rand, sectors)
return xerrors.Errorf("winning post was invalid")
}
@ -947,15 +989,24 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
return xerrors.Errorf("failed to load base state tree: %w", err)
}
pl := vm.PricelistByEpoch(baseTs.Height())
var sumGasLimit int64
checkMsg := func(msg types.ChainMsg) error {
m := msg.VMMessage()
// Phase 1: syntactic validation, as defined in the spec
minGas := vm.PricelistByEpoch(baseTs.Height()).OnChainMessage(msg.ChainLength())
minGas := pl.OnChainMessage(msg.ChainLength())
if err := m.ValidForBlockInclusion(minGas.Total()); err != nil {
return err
}
// ValidForBlockInclusion checks if any single message does not exceed BlockGasLimit
// So below is overflow safe
sumGasLimit += m.GasLimit
if sumGasLimit > build.BlockGasLimit {
return xerrors.Errorf("block gas limit exceeded")
}
// Phase 2: (Partial) semantic validation:
// the sender exists and is an account actor, and the nonces make sense
if _, ok := nonces[m.From]; !ok {
@ -965,7 +1016,6 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
return xerrors.Errorf("failed to get actor: %w", err)
}
// redundant check
if !act.IsAccountActor() {
return xerrors.New("Sender must be an account actor")
}
@ -980,18 +1030,21 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
return nil
}
var blsCids []cbg.CBORMarshaler
store := adt.WrapStore(ctx, cst)
bmArr := adt.MakeEmptyArray(store)
for i, m := range b.BlsMessages {
if err := checkMsg(m); err != nil {
return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err)
}
c := cbg.CborCid(m.Cid())
blsCids = append(blsCids, &c)
if err := bmArr.Set(uint64(i), &c); err != nil {
return xerrors.Errorf("failed to put bls message at index %d: %w", i, err)
}
}
var secpkCids []cbg.CBORMarshaler
smArr := adt.MakeEmptyArray(store)
for i, m := range b.SecpkMessages {
if err := checkMsg(m); err != nil {
return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err)
@ -1009,17 +1062,19 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
}
c := cbg.CborCid(m.Cid())
secpkCids = append(secpkCids, &c)
if err := smArr.Set(uint64(i), &c); err != nil {
return xerrors.Errorf("failed to put secpk message at index %d: %w", i, err)
}
}
bmroot, err := amt.FromArray(ctx, cst, blsCids)
bmroot, err := bmArr.Root()
if err != nil {
return xerrors.Errorf("failed to build amt from bls msg cids: %w", err)
return err
}
smroot, err := amt.FromArray(ctx, cst, secpkCids)
smroot, err := smArr.Root()
if err != nil {
return xerrors.Errorf("failed to build amt from bls msg cids: %w", err)
return err
}
mrcid, err := cst.Put(ctx, &types.MsgMeta{
@ -1049,7 +1104,6 @@ func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signat
msgsS[i] = msgs[i].Bytes()
}
// TODO: empty aggregates are considered valid?
if len(msgs) == 0 {
return nil
}
@ -1208,6 +1262,22 @@ loop:
}
log.Info("Got blocks: ", blks[0].Height(), len(blks))
// Check that the fetched segment of the chain matches what we already
// have. Since we fetch from the head backwards our reassembled chain
// is sorted in reverse here: we have a child -> parent order, our last
// tipset then should be child of the first tipset retrieved.
// FIXME: The reassembly logic should be part of the `BlockSync`
// service, the consumer should not be concerned with the
// `MaxRequestLength` limitation, it should just be able to request
// an segment of arbitrary length. The same burden is put on
// `syncFork()` which needs to be aware this as well.
if blockSet[len(blockSet)-1].IsChildOf(blks[0]) == false {
return nil, xerrors.Errorf("retrieved segments of the chain are not connected at heights %d/%d",
blockSet[len(blockSet)-1].Height(), blks[0].Height())
// A successful `GetBlocks()` call is guaranteed to fetch at least
// one tipset so the acess `blks[0]` is safe.
}
for _, b := range blks {
if b.Height() < untilHeight {
break loop
@ -1231,31 +1301,33 @@ loop:
at = blks[len(blks)-1].Parents()
}
// base is the tipset in the candidate chain at the height equal to our known tipset height.
if base := blockSet[len(blockSet)-1]; !types.CidArrsEqual(base.Parents().Cids(), known.Cids()) {
if base.Parents() == known.Parents() {
// common case: receiving a block thats potentially part of the same tipset as our best block
return blockSet, nil
}
// We have now ascertained that this is *not* a 'fast forward'
log.Warnf("(fork detected) synced header chain (%s - %d) does not link to our best block (%s - %d)", incoming.Cids(), incoming.Height(), known.Cids(), known.Height())
fork, err := syncer.syncFork(ctx, base, known)
if err != nil {
if xerrors.Is(err, ErrForkTooLong) {
// TODO: we're marking this block bad in the same way that we mark invalid blocks bad. Maybe distinguish?
log.Warn("adding forked chain to our bad tipset cache")
for _, b := range incoming.Blocks() {
syncer.bad.Add(b.Cid(), NewBadBlockReason(incoming.Cids(), "fork past finality"))
}
}
return nil, xerrors.Errorf("failed to sync fork: %w", err)
}
blockSet = append(blockSet, fork...)
base := blockSet[len(blockSet)-1]
if base.Parents() == known.Parents() {
// common case: receiving a block thats potentially part of the same tipset as our best block
return blockSet, nil
}
if types.CidArrsEqual(base.Parents().Cids(), known.Cids()) {
// common case: receiving blocks that are building on top of our best tipset
return blockSet, nil
}
// We have now ascertained that this is *not* a 'fast forward'
log.Warnf("(fork detected) synced header chain (%s - %d) does not link to our best block (%s - %d)", incoming.Cids(), incoming.Height(), known.Cids(), known.Height())
fork, err := syncer.syncFork(ctx, base, known)
if err != nil {
if xerrors.Is(err, ErrForkTooLong) {
// TODO: we're marking this block bad in the same way that we mark invalid blocks bad. Maybe distinguish?
log.Warn("adding forked chain to our bad tipset cache")
for _, b := range incoming.Blocks() {
syncer.bad.Add(b.Cid(), NewBadBlockReason(incoming.Cids(), "fork past finality"))
}
}
return nil, xerrors.Errorf("failed to sync fork: %w", err)
}
blockSet = append(blockSet, fork...)
return blockSet, nil
}
@ -1283,7 +1355,7 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know
if !syncer.Genesis.Equals(nts) {
return nil, xerrors.Errorf("somehow synced chain that linked back to a different genesis (bad genesis: %s)", nts.Key())
}
return nil, xerrors.Errorf("synced chain forked at genesis, refusing to sync; incoming: %s")
return nil, xerrors.Errorf("synced chain forked at genesis, refusing to sync; incoming: %s", incoming.Cids())
}
if nts.Equals(tips[cur]) {
@ -1348,7 +1420,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
nextI := (i + 1) - batchSize // want to fetch batchSize values, 'i' points to last one we want to fetch, so its 'inclusive' of our request, thus we need to add one to our request start index
var bstout []*blocksync.BSTipSet
var bstout []*blocksync.CompactedMessages
for len(bstout) < batchSize {
next := headers[nextI]
@ -1364,16 +1436,15 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
for bsi := 0; bsi < len(bstout); bsi++ {
// temp storage so we don't persist data we dont want to
ds := dstore.NewMapDatastore()
bs := bstore.NewBlockstore(ds)
bs := bstore.NewTemporary()
blks := cbor.NewCborStore(bs)
this := headers[i-bsi]
bstip := bstout[len(bstout)-(bsi+1)]
fts, err := zipTipSetAndMessages(blks, this, bstip.BlsMessages, bstip.SecpkMessages, bstip.BlsMsgIncludes, bstip.SecpkMsgIncludes)
fts, err := zipTipSetAndMessages(blks, this, bstip.Bls, bstip.Secpk, bstip.BlsIncludes, bstip.SecpkIncludes)
if err != nil {
log.Warnw("zipping failed", "error", err, "bsi", bsi, "i", i,
"height", this.Height(), "bstip-height", bstip.Blocks[0].Height,
"height", this.Height(),
"next-height", i+batchSize)
return xerrors.Errorf("message processing failed: %w", err)
}
@ -1396,15 +1467,15 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
return nil
}
func persistMessages(bs bstore.Blockstore, bst *blocksync.BSTipSet) error {
for _, m := range bst.BlsMessages {
func persistMessages(bs bstore.Blockstore, bst *blocksync.CompactedMessages) error {
for _, m := range bst.Bls {
//log.Infof("putting BLS message: %s", m.Cid())
if _, err := store.PutMessage(bs, m); err != nil {
log.Errorf("failed to persist messages: %+v", err)
return xerrors.Errorf("BLS message processing failed: %w", err)
}
}
for _, m := range bst.SecpkMessages {
for _, m := range bst.Secpk {
if m.Signature.Type != crypto.SigTypeSecp256k1 {
return xerrors.Errorf("unknown signature type on message %s: %q", m.Cid(), m.Signature.Type)
}
@ -1536,6 +1607,6 @@ func (syncer *Syncer) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool {
return false
}
now := uint64(time.Now().Unix())
now := uint64(build.Clock.Now().Unix())
return epoch > (abi.ChainEpoch((now-g.Timestamp)/build.BlockDelaySecs) + MaxHeightDrift)
}

177
chain/sync_test.go
View File

@ -3,10 +3,12 @@ package chain_test
import (
"context"
"fmt"
"github.com/ipfs/go-cid"
"os"
"testing"
"time"
ds "github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/peer"
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
@ -22,6 +24,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/gen/slashfilter"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
mocktypes "github.com/filecoin-project/lotus/chain/types/mock"
@ -170,7 +173,7 @@ func (tu *syncTestUtil) pushTsExpectErr(to int, fts *store.FullTipSet, experr bo
}
}
func (tu *syncTestUtil) mineOnBlock(blk *store.FullTipSet, src int, miners []int, wait, fail bool) *store.FullTipSet {
func (tu *syncTestUtil) mineOnBlock(blk *store.FullTipSet, to int, miners []int, wait, fail bool, msgs [][]*types.SignedMessage) *store.FullTipSet {
if miners == nil {
for i := range tu.g.Miners {
miners = append(miners, i)
@ -184,20 +187,28 @@ func (tu *syncTestUtil) mineOnBlock(blk *store.FullTipSet, src int, miners []int
fmt.Println("Miner mining block: ", maddrs)
mts, err := tu.g.NextTipSetFromMiners(blk.TipSet(), maddrs)
require.NoError(tu.t, err)
if fail {
tu.pushTsExpectErr(src, mts.TipSet, true)
var nts *store.FullTipSet
var err error
if msgs != nil {
nts, err = tu.g.NextTipSetFromMinersWithMessages(blk.TipSet(), maddrs, msgs)
require.NoError(tu.t, err)
} else {
tu.pushFtsAndWait(src, mts.TipSet, wait)
mt, err := tu.g.NextTipSetFromMiners(blk.TipSet(), maddrs)
require.NoError(tu.t, err)
nts = mt.TipSet
}
return mts.TipSet
if fail {
tu.pushTsExpectErr(to, nts, true)
} else {
tu.pushFtsAndWait(to, nts, wait)
}
return nts
}
func (tu *syncTestUtil) mineNewBlock(src int, miners []int) {
mts := tu.mineOnBlock(tu.g.CurTipset, src, miners, true, false)
mts := tu.mineOnBlock(tu.g.CurTipset, src, miners, true, false, nil)
tu.g.CurTipset = mts
}
@ -414,14 +425,16 @@ func TestSyncBadTimestamp(t *testing.T) {
fmt.Println("BASE: ", base.Cids())
tu.printHeads()
a1 := tu.mineOnBlock(base, 0, nil, false, true)
a1 := tu.mineOnBlock(base, 0, nil, false, true, nil)
tu.g.Timestamper = nil
require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet()))
tu.nds[0].(*impl.FullNodeAPI).SlashFilter = slashfilter.New(ds.NewMapDatastore())
fmt.Println("After mine bad block!")
tu.printHeads()
a2 := tu.mineOnBlock(base, 0, nil, true, false)
a2 := tu.mineOnBlock(base, 0, nil, true, false, nil)
tu.waitUntilSync(0, client)
@ -433,6 +446,41 @@ func TestSyncBadTimestamp(t *testing.T) {
}
}
type badWpp struct{}
func (wpp badWpp) GenerateCandidates(context.Context, abi.PoStRandomness, uint64) ([]uint64, error) {
return []uint64{1}, nil
}
func (wpp badWpp) ComputeProof(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]abi.PoStProof, error) {
return []abi.PoStProof{
abi.PoStProof{
PoStProof: abi.RegisteredPoStProof_StackedDrgWinning2KiBV1,
ProofBytes: []byte("evil"),
},
}, nil
}
func TestSyncBadWinningPoSt(t *testing.T) {
H := 15
tu := prepSyncTest(t, H)
client := tu.addClientNode()
require.NoError(t, tu.mn.LinkAll())
tu.connect(client, 0)
tu.waitUntilSync(0, client)
base := tu.g.CurTipset
// both miners now produce invalid winning posts
tu.g.SetWinningPoStProver(tu.g.Miners[0], &badWpp{})
tu.g.SetWinningPoStProver(tu.g.Miners[1], &badWpp{})
// now ensure that new blocks are not accepted
tu.mineOnBlock(base, client, nil, false, true, nil)
}
func (tu *syncTestUtil) loadChainToNode(to int) {
// utility to simulate incoming blocks without miner process
// TODO: should call syncer directly, this won't work correctly in all cases
@ -475,16 +523,16 @@ func TestSyncFork(t *testing.T) {
fmt.Println("Mining base: ", base.TipSet().Cids(), base.TipSet().Height())
// The two nodes fork at this point into 'a' and 'b'
a1 := tu.mineOnBlock(base, p1, []int{0}, true, false)
a := tu.mineOnBlock(a1, p1, []int{0}, true, false)
a = tu.mineOnBlock(a, p1, []int{0}, true, false)
a1 := tu.mineOnBlock(base, p1, []int{0}, true, false, nil)
a := tu.mineOnBlock(a1, p1, []int{0}, true, false, nil)
a = tu.mineOnBlock(a, p1, []int{0}, true, false, nil)
require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet()))
// chain B will now be heaviest
b := tu.mineOnBlock(base, p2, []int{1}, true, false)
b = tu.mineOnBlock(b, p2, []int{1}, true, false)
b = tu.mineOnBlock(b, p2, []int{1}, true, false)
b = tu.mineOnBlock(b, p2, []int{1}, true, false)
b := tu.mineOnBlock(base, p2, []int{1}, true, false, nil)
b = tu.mineOnBlock(b, p2, []int{1}, true, false, nil)
b = tu.mineOnBlock(b, p2, []int{1}, true, false, nil)
b = tu.mineOnBlock(b, p2, []int{1}, true, false, nil)
fmt.Println("A: ", a.Cids(), a.TipSet().Height())
fmt.Println("B: ", b.Cids(), b.TipSet().Height())
@ -499,6 +547,99 @@ func TestSyncFork(t *testing.T) {
phead()
}
// This test crafts a tipset with 2 blocks, A and B.
// A and B both include _different_ messages from sender X with nonce N (where N is the correct nonce for X).
// We can confirm that the state can be correctly computed, and that `MessagesForTipset` behaves as expected.
func TestDuplicateNonce(t *testing.T) {
H := 10
tu := prepSyncTest(t, H)
base := tu.g.CurTipset
// Produce a message from the banker to the rcvr
makeMsg := func(rcvr address.Address) *types.SignedMessage {
ba, err := tu.nds[0].StateGetActor(context.TODO(), tu.g.Banker(), base.TipSet().Key())
require.NoError(t, err)
msg := types.Message{
To: rcvr,
From: tu.g.Banker(),
Nonce: ba.Nonce,
Value: types.NewInt(1),
Method: 0,
GasLimit: 100_000_000,
GasFeeCap: types.NewInt(0),
GasPremium: types.NewInt(0),
}
sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes())
require.NoError(t, err)
return &types.SignedMessage{
Message: msg,
Signature: *sig,
}
}
msgs := make([][]*types.SignedMessage, 2)
// Each miner includes a message from the banker with the same nonce, but to different addresses
for k, _ := range msgs {
msgs[k] = []*types.SignedMessage{makeMsg(tu.g.Miners[k])}
}
ts1 := tu.mineOnBlock(base, 0, []int{0, 1}, true, false, msgs)
tu.waitUntilSyncTarget(0, ts1.TipSet())
// mine another tipset
ts2 := tu.mineOnBlock(ts1, 0, []int{0, 1}, true, false, make([][]*types.SignedMessage, 2))
tu.waitUntilSyncTarget(0, ts2.TipSet())
var includedMsg cid.Cid
var skippedMsg cid.Cid
r0, err0 := tu.nds[0].StateGetReceipt(context.TODO(), msgs[0][0].Cid(), ts2.TipSet().Key())
r1, err1 := tu.nds[0].StateGetReceipt(context.TODO(), msgs[1][0].Cid(), ts2.TipSet().Key())
if err0 == nil {
require.Error(t, err1, "at least one of the StateGetReceipt calls should fail")
require.True(t, r0.ExitCode.IsSuccess())
includedMsg = msgs[0][0].Message.Cid()
skippedMsg = msgs[1][0].Message.Cid()
} else {
require.NoError(t, err1, "both the StateGetReceipt calls should not fail")
require.True(t, r1.ExitCode.IsSuccess())
includedMsg = msgs[1][0].Message.Cid()
skippedMsg = msgs[0][0].Message.Cid()
}
_, rslts, err := tu.g.StateManager().ExecutionTrace(context.TODO(), ts1.TipSet())
require.NoError(t, err)
found := false
for _, v := range rslts {
if v.Msg.Cid() == skippedMsg {
t.Fatal("skipped message should not be in exec trace")
}
if v.Msg.Cid() == includedMsg {
found = true
}
}
if !found {
t.Fatal("included message should be in exec trace")
}
mft, err := tu.g.ChainStore().MessagesForTipset(ts1.TipSet())
require.NoError(t, err)
require.True(t, len(mft) == 1, "only expecting one message for this tipset")
require.Equal(t, includedMsg, mft[0].VMMessage().Cid(), "messages for tipset didn't contain expected message")
}
func BenchmarkSyncBasic(b *testing.B) {
for i := 0; i < b.N; i++ {
runSyncBenchLength(b, 100)

7
chain/syncstate.go
View File

@ -8,6 +8,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
)
@ -48,7 +49,7 @@ func (ss *SyncerState) SetStage(v api.SyncStateStage) {
defer ss.lk.Unlock()
ss.Stage = v
if v == api.StageSyncComplete {
ss.End = time.Now()
ss.End = build.Clock.Now()
}
}
@ -64,7 +65,7 @@ func (ss *SyncerState) Init(base, target *types.TipSet) {
ss.Stage = api.StageHeaders
ss.Height = 0
ss.Message = ""
ss.Start = time.Now()
ss.Start = build.Clock.Now()
ss.End = time.Time{}
}
@ -87,7 +88,7 @@ func (ss *SyncerState) Error(err error) {
defer ss.lk.Unlock()
ss.Message = err.Error()
ss.Stage = api.StageSyncErrored
ss.End = time.Now()
ss.End = build.Clock.Now()
}
func (ss *SyncerState) Snapshot() SyncerState {

5
chain/types/actor.go
View File

@ -1,13 +1,14 @@
package types
import (
"errors"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/specs-actors/actors/builtin"
init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
)
var ErrActorNotFound = init_.ErrAddressNotFound
var ErrActorNotFound = errors.New("actor not found")
type Actor struct {
// Identifies the type of actor (string coded as a CID), see `chain/actors/actors.go`.

2
chain/types/bigint.go
View File

@ -11,7 +11,7 @@ import (
const BigIntMaxSerializedLen = 128 // is this big enough? or too big?
var TotalFilecoinInt = FromFil(build.TotalFilecoin)
var TotalFilecoinInt = FromFil(build.FilBase)
var EmptyInt = BigInt{}

2
chain/types/bigint_test.go
View File

@ -82,7 +82,7 @@ func TestSizeStrUnitsSymmetry(t *testing.T) {
s := rand.NewSource(time.Now().UnixNano())
r := rand.New(s)
for i := 0; i < 1000000; i++ {
for i := 0; i < 10000; i++ {
n := r.Uint64()
l := strings.ReplaceAll(units.BytesSize(float64(n)), " ", "")
r := strings.ReplaceAll(SizeStr(NewInt(n)), " ", "")

39
chain/types/blockheader.go
View File

@ -11,7 +11,6 @@ import (
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
xerrors "golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
@ -23,8 +22,14 @@ type Ticket struct {
VRFProof []byte
}
type ElectionProof struct {
VRFProof []byte
func (t *Ticket) Quality() float64 {
ticketHash := blake2b.Sum256(t.VRFProof)
ticketNum := BigFromBytes(ticketHash[:]).Int
ticketDenu := big.NewInt(1)
ticketDenu.Lsh(ticketDenu, 256)
tv, _ := new(big.Rat).SetFrac(ticketNum, ticketDenu).Float64()
tq := 1 - tv
return tq
}
type BeaconEntry struct {
@ -70,6 +75,9 @@ type BlockHeader struct {
ForkSignaling uint64 // 14
// ParentBaseFee is the base fee after executing parent tipset
ParentBaseFee abi.TokenAmount // 15
// internal
validated bool // true if the signature has been validated
}
@ -80,8 +88,7 @@ func (blk *BlockHeader) ToStorageBlock() (block.Block, error) {
return nil, err
}
pref := cid.NewPrefixV1(cid.DagCBOR, multihash.BLAKE2B_MIN+31)
c, err := pref.Sum(data)
c, err := abi.CidBuilder.Sum(data)
if err != nil {
return nil, err
}
@ -149,13 +156,12 @@ func (mm *MsgMeta) Cid() cid.Cid {
}
func (mm *MsgMeta) ToStorageBlock() (block.Block, error) {
buf := new(bytes.Buffer)
if err := mm.MarshalCBOR(buf); err != nil {
var buf bytes.Buffer
if err := mm.MarshalCBOR(&buf); err != nil {
return nil, xerrors.Errorf("failed to marshal MsgMeta: %w", err)
}
pref := cid.NewPrefixV1(cid.DagCBOR, multihash.BLAKE2B_MIN+31)
c, err := pref.Sum(buf.Bytes())
c, err := abi.CidBuilder.Sum(buf.Bytes())
if err != nil {
return nil, err
}
@ -182,6 +188,21 @@ func CidArrsEqual(a, b []cid.Cid) bool {
return true
}
func CidArrsSubset(a, b []cid.Cid) bool {
// order ignoring compare...
s := make(map[cid.Cid]bool)
for _, c := range b {
s[c] = true
}
for _, c := range a {
if !s[c] {
return false
}
}
return true
}
func CidArrsContains(a []cid.Cid, b cid.Cid) bool {
for _, elem := range a {
if elem.Equals(b) {

13
chain/types/blockheader_test.go
View File

@ -4,11 +4,11 @@ import (
"bytes"
"encoding/hex"
"fmt"
"github.com/stretchr/testify/require"
"reflect"
"testing"
cid "github.com/ipfs/go-cid"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
@ -44,6 +44,7 @@ func testBlockHeader(t testing.TB) *BlockHeader {
Height: 85919298723,
ParentStateRoot: c,
BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS, Data: []byte("boo! im a signature")},
ParentBaseFee: NewInt(3432432843291),
}
}
@ -80,13 +81,13 @@ func TestInteropBH(t *testing.T) {
}
posts := []abi.PoStProof{
{abi.RegisteredPoStProof_StackedDrgWinning2KiBV1, []byte{0x07}},
{PoStProof: abi.RegisteredPoStProof_StackedDrgWinning2KiBV1, ProofBytes: []byte{0x07}},
}
bh := &BlockHeader{
Miner: newAddr,
Ticket: &Ticket{[]byte{0x01, 0x02, 0x03}},
ElectionProof: &ElectionProof{[]byte{0x0a, 0x0b}},
ElectionProof: &ElectionProof{0, []byte{0x0a, 0x0b}},
BeaconEntries: []BeaconEntry{
{
Round: 5,
@ -107,7 +108,8 @@ func TestInteropBH(t *testing.T) {
Type: crypto.SigTypeBLS,
Data: []byte{0x3},
},
BLSAggregate: &crypto.Signature{},
BLSAggregate: &crypto.Signature{},
ParentBaseFee: NewInt(1000000000),
}
bhsb, err := bh.SigningBytes()
@ -116,8 +118,7 @@ func TestInteropBH(t *testing.T) {
t.Fatal(err)
}
// acquired from go-filecoin
gfc := "8f5501d04cb15021bf6bd003073d79e2238d4e61f1ad22814301020381420a0b818205410c818200410781d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc430003e802d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc410001f603"
gfc := "905501d04cb15021bf6bd003073d79e2238d4e61f1ad2281430102038200420a0b818205410c818200410781d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc430003e802d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc410001f60345003b9aca00"
require.Equal(t, gfc, hex.EncodeToString(bhsb))
}

215
chain/types/cbor_gen.go
View File

@ -16,7 +16,7 @@ import (
var _ = xerrors.Errorf
var lengthBufBlockHeader = []byte{143}
var lengthBufBlockHeader = []byte{144}
func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
if t == nil {
@ -142,10 +142,16 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
return err
}
// t.ParentBaseFee (big.Int) (struct)
if err := t.ParentBaseFee.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
*t = BlockHeader{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -157,7 +163,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 15 {
if extra != 16 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -174,16 +180,14 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
{
pb, err := br.PeekByte()
b, err := br.ReadByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
} else {
t.Ticket = new(Ticket)
if err := t.Ticket.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.Ticket pointer: %w", err)
@ -195,16 +199,14 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
{
pb, err := br.PeekByte()
b, err := br.ReadByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
} else {
t.ElectionProof = new(ElectionProof)
if err := t.ElectionProof.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.ElectionProof pointer: %w", err)
@ -372,16 +374,14 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
{
pb, err := br.PeekByte()
b, err := br.ReadByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
} else {
t.BLSAggregate = new(crypto.Signature)
if err := t.BLSAggregate.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.BLSAggregate pointer: %w", err)
@ -407,16 +407,14 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
{
pb, err := br.PeekByte()
b, err := br.ReadByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
} else {
t.BlockSig = new(crypto.Signature)
if err := t.BlockSig.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.BlockSig pointer: %w", err)
@ -437,6 +435,15 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
}
t.ForkSignaling = uint64(extra)
}
// t.ParentBaseFee (big.Int) (struct)
{
if err := t.ParentBaseFee.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.ParentBaseFee: %w", err)
}
}
return nil
}
@ -463,13 +470,15 @@ func (t *Ticket) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(t.VRFProof); err != nil {
if _, err := w.Write(t.VRFProof[:]); err != nil {
return err
}
return nil
}
func (t *Ticket) UnmarshalCBOR(r io.Reader) error {
*t = Ticket{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -498,14 +507,18 @@ func (t *Ticket) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.VRFProof = make([]byte, extra)
if _, err := io.ReadFull(br, t.VRFProof); err != nil {
if extra > 0 {
t.VRFProof = make([]uint8, extra)
}
if _, err := io.ReadFull(br, t.VRFProof[:]); err != nil {
return err
}
return nil
}
var lengthBufElectionProof = []byte{129}
var lengthBufElectionProof = []byte{130}
func (t *ElectionProof) MarshalCBOR(w io.Writer) error {
if t == nil {
@ -518,6 +531,17 @@ func (t *ElectionProof) MarshalCBOR(w io.Writer) error {
scratch := make([]byte, 9)
// t.WinCount (int64) (int64)
if t.WinCount >= 0 {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.WinCount)); err != nil {
return err
}
} else {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.WinCount-1)); err != nil {
return err
}
}
// t.VRFProof ([]uint8) (slice)
if len(t.VRFProof) > cbg.ByteArrayMaxLen {
return xerrors.Errorf("Byte array in field t.VRFProof was too long")
@ -527,13 +551,15 @@ func (t *ElectionProof) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(t.VRFProof); err != nil {
if _, err := w.Write(t.VRFProof[:]); err != nil {
return err
}
return nil
}
func (t *ElectionProof) UnmarshalCBOR(r io.Reader) error {
*t = ElectionProof{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -545,10 +571,35 @@ func (t *ElectionProof) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.WinCount (int64) (int64)
{
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
var extraI int64
if err != nil {
return err
}
switch maj {
case cbg.MajUnsignedInt:
extraI = int64(extra)
if extraI < 0 {
return fmt.Errorf("int64 positive overflow")
}
case cbg.MajNegativeInt:
extraI = int64(extra)
if extraI < 0 {
return fmt.Errorf("int64 negative oveflow")
}
extraI = -1 - extraI
default:
return fmt.Errorf("wrong type for int64 field: %d", maj)
}
t.WinCount = int64(extraI)
}
// t.VRFProof ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
@ -562,14 +613,18 @@ func (t *ElectionProof) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.VRFProof = make([]byte, extra)
if _, err := io.ReadFull(br, t.VRFProof); err != nil {
if extra > 0 {
t.VRFProof = make([]uint8, extra)
}
if _, err := io.ReadFull(br, t.VRFProof[:]); err != nil {
return err
}
return nil
}
var lengthBufMessage = []byte{137}
var lengthBufMessage = []byte{138}
func (t *Message) MarshalCBOR(w io.Writer) error {
if t == nil {
@ -614,11 +669,6 @@ func (t *Message) MarshalCBOR(w io.Writer) error {
return err
}
// t.GasPrice (big.Int) (struct)
if err := t.GasPrice.MarshalCBOR(w); err != nil {
return err
}
// t.GasLimit (int64) (int64)
if t.GasLimit >= 0 {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.GasLimit)); err != nil {
@ -630,6 +680,16 @@ func (t *Message) MarshalCBOR(w io.Writer) error {
}
}
// t.GasFeeCap (big.Int) (struct)
if err := t.GasFeeCap.MarshalCBOR(w); err != nil {
return err
}
// t.GasPremium (big.Int) (struct)
if err := t.GasPremium.MarshalCBOR(w); err != nil {
return err
}
// t.Method (abi.MethodNum) (uint64)
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Method)); err != nil {
@ -645,13 +705,15 @@ func (t *Message) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(t.Params); err != nil {
if _, err := w.Write(t.Params[:]); err != nil {
return err
}
return nil
}
func (t *Message) UnmarshalCBOR(r io.Reader) error {
*t = Message{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -663,7 +725,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 9 {
if extra != 10 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -732,15 +794,6 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
return xerrors.Errorf("unmarshaling t.Value: %w", err)
}
}
// t.GasPrice (big.Int) (struct)
{
if err := t.GasPrice.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.GasPrice: %w", err)
}
}
// t.GasLimit (int64) (int64)
{
@ -767,6 +820,24 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
t.GasLimit = int64(extraI)
}
// t.GasFeeCap (big.Int) (struct)
{
if err := t.GasFeeCap.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.GasFeeCap: %w", err)
}
}
// t.GasPremium (big.Int) (struct)
{
if err := t.GasPremium.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.GasPremium: %w", err)
}
}
// t.Method (abi.MethodNum) (uint64)
{
@ -794,8 +865,12 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Params = make([]byte, extra)
if _, err := io.ReadFull(br, t.Params); err != nil {
if extra > 0 {
t.Params = make([]uint8, extra)
}
if _, err := io.ReadFull(br, t.Params[:]); err != nil {
return err
}
return nil
@ -825,6 +900,8 @@ func (t *SignedMessage) MarshalCBOR(w io.Writer) error {
}
func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error {
*t = SignedMessage{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -890,6 +967,8 @@ func (t *MsgMeta) MarshalCBOR(w io.Writer) error {
}
func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error {
*t = MsgMeta{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -971,6 +1050,8 @@ func (t *Actor) MarshalCBOR(w io.Writer) error {
}
func (t *Actor) UnmarshalCBOR(r io.Reader) error {
*t = Actor{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -1069,7 +1150,7 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(t.Return); err != nil {
if _, err := w.Write(t.Return[:]); err != nil {
return err
}
@ -1087,6 +1168,8 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error {
}
func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error {
*t = MessageReceipt{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -1140,8 +1223,12 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Return = make([]byte, extra)
if _, err := io.ReadFull(br, t.Return); err != nil {
if extra > 0 {
t.Return = make([]uint8, extra)
}
if _, err := io.ReadFull(br, t.Return[:]); err != nil {
return err
}
// t.GasUsed (int64) (int64)
@ -1221,6 +1308,8 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error {
}
func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error {
*t = BlockMsg{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -1240,16 +1329,14 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error {
{
pb, err := br.PeekByte()
b, err := br.ReadByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
} else {
t.Header = new(BlockHeader)
if err := t.Header.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.Header pointer: %w", err)
@ -1371,6 +1458,8 @@ func (t *ExpTipSet) MarshalCBOR(w io.Writer) error {
}
func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error {
*t = ExpTipSet{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -1499,13 +1588,15 @@ func (t *BeaconEntry) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(t.Data); err != nil {
if _, err := w.Write(t.Data[:]); err != nil {
return err
}
return nil
}
func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) error {
*t = BeaconEntry{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
@ -1548,8 +1639,12 @@ func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Data = make([]byte, extra)
if _, err := io.ReadFull(br, t.Data); err != nil {
if extra > 0 {
t.Data = make([]uint8, extra)
}
if _, err := io.ReadFull(br, t.Data[:]); err != nil {
return err
}
return nil

205
chain/types/electionproof.go Normal file
View File

@ -0,0 +1,205 @@
package types
import (
"math/big"
"github.com/filecoin-project/lotus/build"
"github.com/minio/blake2b-simd"
)
type ElectionProof struct {
WinCount int64
VRFProof []byte
}
const precision = 256
var (
expNumCoef []*big.Int
expDenoCoef []*big.Int
)
func init() {
parse := func(coefs []string) []*big.Int {
out := make([]*big.Int, len(coefs))
for i, coef := range coefs {
c, ok := new(big.Int).SetString(coef, 10)
if !ok {
panic("could not parse exp paramemter")
}
// << 256 (Q.0 to Q.256), >> 128 to transform integer params to coefficients
c = c.Lsh(c, precision-128)
out[i] = c
}
return out
}
// parameters are in integer format,
// coefficients are *2^-128 of that
num := []string{
"-648770010757830093818553637600",
"67469480939593786226847644286976",
"-3197587544499098424029388939001856",
"89244641121992890118377641805348864",
"-1579656163641440567800982336819953664",
"17685496037279256458459817590917169152",
"-115682590513835356866803355398940131328",
"340282366920938463463374607431768211456",
}
expNumCoef = parse(num)
deno := []string{
"1225524182432722209606361",
"114095592300906098243859450",
"5665570424063336070530214243",
"194450132448609991765137938448",
"5068267641632683791026134915072",
"104716890604972796896895427629056",
"1748338658439454459487681798864896",
"23704654329841312470660182937960448",
"259380097567996910282699886670381056",
"2250336698853390384720606936038375424",
"14978272436876548034486263159246028800",
"72144088983913131323343765784380833792",
"224599776407103106596571252037123047424",
"340282366920938463463374607431768211456",
}
expDenoCoef = parse(deno)
}
// expneg accepts x in Q.256 format and computes e^-x.
// It is most precise within [0, 1.725) range, where error is less than 3.4e-30.
// Over the [0, 5) range its error is less than 4.6e-15.
// Output is in Q.256 format.
func expneg(x *big.Int) *big.Int {
// exp is approximated by rational function
// polynomials of the rational function are evaluated using Horner's method
num := polyval(expNumCoef, x) // Q.256
deno := polyval(expDenoCoef, x) // Q.256
num = num.Lsh(num, precision) // Q.512
return num.Div(num, deno) // Q.512 / Q.256 => Q.256
}
// polyval evaluates a polynomial given by coefficients `p` in Q.256 format
// at point `x` in Q.256 format. Output is in Q.256.
// Coefficients should be ordered from the highest order coefficient to the lowest.
func polyval(p []*big.Int, x *big.Int) *big.Int {
// evaluation using Horner's method
res := new(big.Int).Set(p[0]) // Q.256
tmp := new(big.Int) // big.Int.Mul doesn't like when input is reused as output
for _, c := range p[1:] {
tmp = tmp.Mul(res, x) // Q.256 * Q.256 => Q.512
res = res.Rsh(tmp, precision) // Q.512 >> 256 => Q.256
res = res.Add(res, c)
}
return res
}
// computes lambda in Q.256
func lambda(power, totalPower *big.Int) *big.Int {
lam := new(big.Int).Mul(power, blocksPerEpoch.Int) // Q.0
lam = lam.Lsh(lam, precision) // Q.256
lam = lam.Div(lam /* Q.256 */, totalPower /* Q.0 */) // Q.256
return lam
}
var MaxWinCount = 3 * int64(build.BlocksPerEpoch)
type poiss struct {
lam *big.Int
pmf *big.Int
icdf *big.Int
tmp *big.Int // temporary variable for optimization
k uint64
}
// newPoiss starts poisson inverted CDF
// lambda is in Q.256 format
// returns (instance, `1-poisscdf(0, lambda)`)
// CDF value returend is reused when calling `next`
func newPoiss(lambda *big.Int) (*poiss, *big.Int) {
// pmf(k) = (lambda^k)*(e^lambda) / k!
// k = 0 here, so it simplifies to just e^-lambda
elam := expneg(lambda) // Q.256
pmf := new(big.Int).Set(elam)
// icdf(k) = 1 - ∑ᵏᵢ₌₀ pmf(i)
// icdf(0) = 1 - pmf(0)
icdf := big.NewInt(1)
icdf = icdf.Lsh(icdf, precision) // Q.256
icdf = icdf.Sub(icdf, pmf) // Q.256
k := uint64(0)
p := &poiss{
lam: lambda,
pmf: pmf,
tmp: elam,
icdf: icdf,
k: k,
}
return p, icdf
}
// next computes `k++, 1-poisscdf(k, lam)`
// return is in Q.256 format
func (p *poiss) next() *big.Int {
// incrementally compute next pmf and icdf
// pmf(k) = (lambda^k)*(e^lambda) / k!
// so pmf(k) = pmf(k-1) * lambda / k
p.k++
p.tmp.SetUint64(p.k) // Q.0
// calculate pmf for k
p.pmf = p.pmf.Div(p.pmf, p.tmp) // Q.256 / Q.0 => Q.256
// we are using `tmp` as target for multiplication as using an input as output
// for Int.Mul causes allocations
p.tmp = p.tmp.Mul(p.pmf, p.lam) // Q.256 * Q.256 => Q.512
p.pmf = p.pmf.Rsh(p.tmp, precision) // Q.512 >> 256 => Q.256
// calculate output
// icdf(k) = icdf(k-1) - pmf(k)
p.icdf = p.icdf.Sub(p.icdf, p.pmf) // Q.256
return p.icdf
}
// ComputeWinCount uses VRFProof to compute number of wins
// The algorithm is based on Algorand's Sortition with Binomial distribution
// replaced by Poisson distribution.
func (ep *ElectionProof) ComputeWinCount(power BigInt, totalPower BigInt) int64 {
h := blake2b.Sum256(ep.VRFProof)
lhs := BigFromBytes(h[:]).Int // 256bits, assume Q.256 so [0, 1)
// We are calculating upside-down CDF of Poisson distribution with
// rate λ=power*E/totalPower
// Steps:
// 1. calculate λ=power*E/totalPower
// 2. calculate elam = exp(-λ)
// 3. Check how many times we win:
// j = 0
// pmf = elam
// rhs = 1 - pmf
// for h(vrf) < rhs: j++; pmf = pmf * lam / j; rhs = rhs - pmf
lam := lambda(power.Int, totalPower.Int) // Q.256
p, rhs := newPoiss(lam)
var j int64
for lhs.Cmp(rhs) < 0 && j < MaxWinCount {
rhs = p.next()
j++
}
return j
}

Some files were not shown because too many files have changed in this diff Show More