cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/go-data-transfer-v2

Browse Source
This commit is contained in:
Łukasz Magiera 2023-03-15 11:23:10 +01:00
commit 7e414ac390
118 changed files with 2199 additions and 731 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View File

@ -9,15 +9,9 @@ body:
options:
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
required: true
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
required: true
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
required: true
- label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead.
required: true
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
required: true
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
required: true
- label: I did not make any code changes to lotus.
required: false
@ -28,19 +22,11 @@ body:
options:
- label: lotus daemon - chain sync
required: false
- label: lotus miner - mining and block production
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
required: false
- label: lotus miner/worker - sealing
required: false
- label: lotus miner - proving(WindowPoSt)
required: false
- label: lotus miner/market - storage deal
required: false
- label: lotus miner/market - retrieval deal
required: false
- label: lotus miner/market - data transfer
required: false
- label: lotus client
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
required: false
- label: lotus JSON-RPC API
required: false
@ -56,22 +42,33 @@ body:
description: Enter the output of `lotus version` and `lotus-miner version` if applicable.
placeholder: |
e.g.
Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0
Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty
Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0
Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
validations:
required: true
- type: textarea
id: ReproSteps
attributes:
label: Repro Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
validations:
required: false
- type: textarea
id: Description
attributes:
label: Describe the Bug
description: |
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
* What you were doding when you experienced the bug?
* What you were doing when you experienced the bug?
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
* What is the expected behaviour?
* For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
* For proving issues, include the output of `lotus-miner proving` info.
* For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
validations:
required: true
- type: textarea
@ -83,18 +80,6 @@ body:
Please provide debug logs of the problem, remember you can get set log level control for:
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://lotus.filecoin.io/lotus/configure/defaults/#log-level-control).
* lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request we make before furthur diagnosing the problem.
validations:
required: true
- type: textarea
id: RepoSteps
attributes:
label: Repo Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
validations:
required: false
required: true

8
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@ -0,0 +1,8 @@
blank_issues_enabled: true
contact_links:
- name: Ask a question about Lotus or get support
url: https://github.com/filecoin-project/lotus/discussions/new/choose
about: Ask a question or request support for using Lotus
- name: Filecoin protocol feature or enhancement
url: https://github.com/filecoin-project/FIPs/discussions/new/choose
about: Write a discussion in the Filecoin Improvement Proposal repo

34
.github/ISSUE_TEMPLATE/enhancement.yml vendored
View File

@ -7,13 +7,7 @@ body:
label: Checklist
description: Please check off the following boxes before continuing to create an improvement suggestion!
options:
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
required: true
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
required: true
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
required: true
- label: I **have** a specific, actionable, and well motivated improvement to propose.
- label: I **have** a specific, actionable, and well motivated improvement to an existing lotus feature.
required: true
- type: checkboxes
attributes:
@ -22,19 +16,11 @@ body:
options:
- label: lotus daemon - chain sync
required: false
- label: lotus miner - mining and block production
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
required: false
- label: lotus miner/worker - sealing
required: false
- label: lotus miner - proving(WindowPoSt)
required: false
- label: lotus miner/market - storage deal
required: false
- label: lotus miner/market - retrieval deal
required: false
- label: lotus miner/market - data transfer
required: false
- label: lotus client
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
required: false
- label: lotus JSON-RPC API
required: false
@ -45,9 +31,17 @@ body:
- type: textarea
id: request
attributes:
label: Improvement Suggestion
description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement?
placeholder: Ex. Currently lotus... However, as a storage provider, I'd like...
label: Enhancement Suggestion
description: A clear and concise description of the suggested enhancement?
placeholder: Ex. Currently lotus... However it would be great if [enhancement] was implemented... With the ability to...
validations:
required: true
- type: textarea
id: request
attributes:
label: Use-Case
description: How would this enhancement help you?
placeholder: Ex. With the [enhancement] node operators would be able to... For Storage Providers it would enable...
validations:
required: true

17
.github/ISSUE_TEMPLATE/feature_request.yml vendored
View File

@ -7,8 +7,6 @@ body:
label: Checklist
description: Please check off the following boxes before continuing to create a new feature request!
options:
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
required: true
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
required: true
- label: I **have** a specific, actionable, and well motivated feature request to propose.
@ -20,19 +18,11 @@ body:
options:
- label: lotus daemon - chain sync
required: false
- label: lotus miner - mining and block production
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
required: false
- label: lotus miner/worker - sealing
required: false
- label: lotus miner - proving(WindowPoSt)
required: false
- label: lotus miner/market - storage deal
required: false
- label: lotus miner/market - retrieval deal
required: false
- label: lotus miner/market - data transfer
required: false
- label: lotus client
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
required: false
- label: lotus JSON-RPC API
required: false
@ -56,7 +46,7 @@ body:
validations:
required: true
- type: textarea
id: alternates
id: alternatives
attributes:
label: Describe alternatives you've considered
description: A clear and concise description of any alternative solutions or features you've considered.
@ -69,4 +59,3 @@ body:
description: Add any other context, design docs or screenshots about the feature request here.
validations:
required: false

83
.github/ISSUE_TEMPLATE/service_developer_bug_report.yml vendored Normal file
View File

@ -0,0 +1,83 @@
name: "Bug Report - developer/service provider"
description: "Bug report template about FEVM/FVM for developers/service providers"
labels: [need/triage, kind/bug, area/fevm]
body:
- type: checkboxes
attributes:
label: Checklist
description: Please check off the following boxes before continuing to file a bug report!
options:
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
required: true
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
required: true
- label: I did not make any code changes to lotus.
required: false
- type: checkboxes
attributes:
label: Lotus component
description: Please select the lotus component you are filing a bug for
options:
- label: lotus Ethereum RPC
required: false
- label: lotus FVM - Lotus FVM interactions
required: false
- label: FEVM tooling
required: false
- label: Other
required: false
- type: textarea
id: version
attributes:
label: Lotus Version
render: text
description: Enter the output of `lotus version` if applicable.
placeholder: |
e.g.
Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0
Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
validations:
required: true
- type: textarea
id: repro
attributes:
label: Repro Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
validations:
required: false
- type: textarea
id: Description
attributes:
label: Describe the Bug
description: |
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
* What you were doing when you experienced the bug? What are you trying to build?
* Any *error* messages and logs you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
* What is the expected behaviour? Links to the actual code?
validations:
required: true
- type: textarea
id: toolingInfo
attributes:
label: Tooling
render: text
description: |
What kind of tooling are you using:
* Are you using ether.js, Alchemy, Hardhat, etc.
validations:
required: true
- type: textarea
id: extraInfo
attributes:
label: Configuration Options
render: text
description: |
Please provide your updated FEVM related configuration options, or custome enviroment variables related to Lotus FEVM
* lotus: use `lotus config updated` to get your configuration options, and copy the [FEVM] section
validations:
required: true

12
api/api_gateway.go
View File

@ -15,6 +15,7 @@ import (
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
// MODIFYING THE API INTERFACE
@ -25,7 +26,7 @@ import (
// When adding / changing methods in this file:
// * Do the change here
// * Adjust implementation in `node/impl/`
// * Run `make gen` - this will:
// * Run `make clean && make deps && make gen` - this will:
// * Generate proxy structs
// * Generate mocks
// * Generate markdown docs
@ -47,15 +48,18 @@ type Gateway interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainGetGenesis(context.Context) (*types.TipSet, error)
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*InvocResult, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
@ -63,8 +67,10 @@ type Gateway interface {
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error)
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
StateNetworkName(context.Context) (dtypes.NetworkName, error)
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
@ -83,8 +89,6 @@ type Gateway interface {
EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error)
EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkOpt string) (ethtypes.EthUint64, error)
EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error)
EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error)
EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error)
EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error)
EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error)
EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam string) (ethtypes.EthBigInt, error)

4
api/docgen/docgen.go
View File

@ -401,6 +401,10 @@ func init() {
FromBlock: pstring("2301220"),
Address: []ethtypes.EthAddress{ethaddr},
})
percent := types.Percent(123)
addExample(percent)
addExample(&percent)
}
func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {

93
api/proxy_gen.go
View File

@ -692,10 +692,6 @@ type GatewayMethods struct {
EthGetStorageAt func(p0 context.Context, p1 ethtypes.EthAddress, p2 ethtypes.EthBytes, p3 string) (ethtypes.EthBytes, error) ``
EthGetTransactionByBlockHashAndIndex func(p0 context.Context, p1 ethtypes.EthHash, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) ``
EthGetTransactionByBlockNumberAndIndex func(p0 context.Context, p1 ethtypes.EthUint64, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) ``
EthGetTransactionByHash func(p0 context.Context, p1 *ethtypes.EthHash) (*ethtypes.EthTx, error) ``
EthGetTransactionCount func(p0 context.Context, p1 ethtypes.EthAddress, p2 string) (ethtypes.EthUint64, error) ``
@ -724,6 +720,8 @@ type GatewayMethods struct {
GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) ``
MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) ``
MpoolPush func(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) ``
MsigGetAvailableBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (types.BigInt, error) ``
@ -740,8 +738,12 @@ type GatewayMethods struct {
StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``
StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) ``
StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) ``
StateDecodeParams func(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) ``
StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) ``
StateListMiners func(p0 context.Context, p1 types.TipSetKey) ([]address.Address, error) ``
@ -758,9 +760,11 @@ type GatewayMethods struct {
StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) ``
StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) ``
StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) ``
StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) `perm:"read"`
StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) ``
StateSearchMsg func(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) ``
@ -768,6 +772,8 @@ type GatewayMethods struct {
StateVerifiedClientStatus func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) ``
StateVerifierStatus func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) ``
StateWaitMsg func(p0 context.Context, p1 cid.Cid, p2 uint64, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) ``
Version func(p0 context.Context) (APIVersion, error) ``
@ -4413,28 +4419,6 @@ func (s *GatewayStub) EthGetStorageAt(p0 context.Context, p1 ethtypes.EthAddress
return *new(ethtypes.EthBytes), ErrNotSupported
}
func (s *GatewayStruct) EthGetTransactionByBlockHashAndIndex(p0 context.Context, p1 ethtypes.EthHash, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) {
if s.Internal.EthGetTransactionByBlockHashAndIndex == nil {
return *new(ethtypes.EthTx), ErrNotSupported
}
return s.Internal.EthGetTransactionByBlockHashAndIndex(p0, p1, p2)
}
func (s *GatewayStub) EthGetTransactionByBlockHashAndIndex(p0 context.Context, p1 ethtypes.EthHash, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) {
return *new(ethtypes.EthTx), ErrNotSupported
}
func (s *GatewayStruct) EthGetTransactionByBlockNumberAndIndex(p0 context.Context, p1 ethtypes.EthUint64, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) {
if s.Internal.EthGetTransactionByBlockNumberAndIndex == nil {
return *new(ethtypes.EthTx), ErrNotSupported
}
return s.Internal.EthGetTransactionByBlockNumberAndIndex(p0, p1, p2)
}
func (s *GatewayStub) EthGetTransactionByBlockNumberAndIndex(p0 context.Context, p1 ethtypes.EthUint64, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) {
return *new(ethtypes.EthTx), ErrNotSupported
}
func (s *GatewayStruct) EthGetTransactionByHash(p0 context.Context, p1 *ethtypes.EthHash) (*ethtypes.EthTx, error) {
if s.Internal.EthGetTransactionByHash == nil {
return nil, ErrNotSupported
@ -4589,6 +4573,17 @@ func (s *GatewayStub) GasEstimateMessageGas(p0 context.Context, p1 *types.Messag
return nil, ErrNotSupported
}
func (s *GatewayStruct) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) {
if s.Internal.MpoolGetNonce == nil {
return 0, ErrNotSupported
}
return s.Internal.MpoolGetNonce(p0, p1)
}
func (s *GatewayStub) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) {
return 0, ErrNotSupported
}
func (s *GatewayStruct) MpoolPush(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) {
if s.Internal.MpoolPush == nil {
return *new(cid.Cid), ErrNotSupported
@ -4677,6 +4672,17 @@ func (s *GatewayStub) StateAccountKey(p0 context.Context, p1 address.Address, p2
return *new(address.Address), ErrNotSupported
}
func (s *GatewayStruct) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) {
if s.Internal.StateCall == nil {
return nil, ErrNotSupported
}
return s.Internal.StateCall(p0, p1, p2)
}
func (s *GatewayStub) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) {
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateDealProviderCollateralBounds(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) {
if s.Internal.StateDealProviderCollateralBounds == nil {
return *new(DealCollateralBounds), ErrNotSupported
@ -4688,6 +4694,17 @@ func (s *GatewayStub) StateDealProviderCollateralBounds(p0 context.Context, p1 a
return *new(DealCollateralBounds), ErrNotSupported
}
func (s *GatewayStruct) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) {
if s.Internal.StateDecodeParams == nil {
return nil, ErrNotSupported
}
return s.Internal.StateDecodeParams(p0, p1, p2, p3, p4)
}
func (s *GatewayStub) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) {
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateGetActor(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) {
if s.Internal.StateGetActor == nil {
return nil, ErrNotSupported
@ -4776,6 +4793,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) {
if s.Internal.StateNetworkName == nil {
return *new(dtypes.NetworkName), ErrNotSupported
}
return s.Internal.StateNetworkName(p0)
}
func (s *GatewayStub) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) {
return *new(dtypes.NetworkName), ErrNotSupported
}
func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) {
if s.Internal.StateNetworkVersion == nil {
return *new(apitypes.NetworkVersion), ErrNotSupported
@ -4831,6 +4859,17 @@ func (s *GatewayStub) StateVerifiedClientStatus(p0 context.Context, p1 address.A
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateVerifierStatus(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) {
if s.Internal.StateVerifierStatus == nil {
return nil, ErrNotSupported
}
return s.Internal.StateVerifierStatus(p0, p1, p2)
}
func (s *GatewayStub) StateVerifierStatus(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) {
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateWaitMsg(p0 context.Context, p1 cid.Cid, p2 uint64, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) {
if s.Internal.StateWaitMsg == nil {
return nil, ErrNotSupported

5
api/v0api/gateway.go
View File

@ -14,6 +14,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
// MODIFYING THE API INTERFACE
@ -44,12 +45,15 @@ type Gateway interface {
ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error)
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
@ -59,6 +63,7 @@ type Gateway interface {
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error)
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
StateNetworkName(context.Context) (dtypes.NetworkName, error)
StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error)
StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)

52
api/v0api/proxy_gen.go
View File

@ -451,6 +451,8 @@ type GatewayMethods struct {
GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *api.MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) ``
MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) ``
MpoolPush func(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) ``
MsigGetAvailableBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (types.BigInt, error) ``
@ -461,8 +463,12 @@ type GatewayMethods struct {
StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``
StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) ``
StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (api.DealCollateralBounds, error) ``
StateDecodeParams func(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) ``
StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) ``
StateGetReceipt func(p0 context.Context, p1 cid.Cid, p2 types.TipSetKey) (*types.MessageReceipt, error) ``
@ -481,6 +487,8 @@ type GatewayMethods struct {
StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) ``
StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) ``
StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) ``
StateSearchMsg func(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) ``
@ -2677,6 +2685,17 @@ func (s *GatewayStub) GasEstimateMessageGas(p0 context.Context, p1 *types.Messag
return nil, ErrNotSupported
}
func (s *GatewayStruct) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) {
if s.Internal.MpoolGetNonce == nil {
return 0, ErrNotSupported
}
return s.Internal.MpoolGetNonce(p0, p1)
}
func (s *GatewayStub) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) {
return 0, ErrNotSupported
}
func (s *GatewayStruct) MpoolPush(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) {
if s.Internal.MpoolPush == nil {
return *new(cid.Cid), ErrNotSupported
@ -2732,6 +2751,17 @@ func (s *GatewayStub) StateAccountKey(p0 context.Context, p1 address.Address, p2
return *new(address.Address), ErrNotSupported
}
func (s *GatewayStruct) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) {
if s.Internal.StateCall == nil {
return nil, ErrNotSupported
}
return s.Internal.StateCall(p0, p1, p2)
}
func (s *GatewayStub) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) {
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateDealProviderCollateralBounds(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (api.DealCollateralBounds, error) {
if s.Internal.StateDealProviderCollateralBounds == nil {
return *new(api.DealCollateralBounds), ErrNotSupported
@ -2743,6 +2773,17 @@ func (s *GatewayStub) StateDealProviderCollateralBounds(p0 context.Context, p1 a
return *new(api.DealCollateralBounds), ErrNotSupported
}
func (s *GatewayStruct) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) {
if s.Internal.StateDecodeParams == nil {
return nil, ErrNotSupported
}
return s.Internal.StateDecodeParams(p0, p1, p2, p3, p4)
}
func (s *GatewayStub) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) {
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateGetActor(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) {
if s.Internal.StateGetActor == nil {
return nil, ErrNotSupported
@ -2842,6 +2883,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) {
if s.Internal.StateNetworkName == nil {
return *new(dtypes.NetworkName), ErrNotSupported
}
return s.Internal.StateNetworkName(p0)
}
func (s *GatewayStub) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) {
return *new(dtypes.NetworkName), ErrNotSupported
}
func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) {
if s.Internal.StateNetworkVersion == nil {
return *new(abinetwork.Version), ErrNotSupported

12
blockstore/badger/blockstore.go
View File

@ -601,6 +601,18 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
})
}
func (b *Blockstore) Flush(context.Context) error {
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
return b.db.Sync()
}
// Has implements Blockstore.Has.
func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
if err := b.access(); err != nil {

12
blockstore/blockstore.go
View File

@ -18,6 +18,7 @@ type Blockstore interface {
blockstore.Blockstore
blockstore.Viewer
BatchDeleter
Flusher
}
// BasicBlockstore is an alias to the original IPFS Blockstore.
@ -25,6 +26,10 @@ type BasicBlockstore = blockstore.Blockstore
type Viewer = blockstore.Viewer
type Flusher interface {
Flush(context.Context) error
}
type BatchDeleter interface {
DeleteMany(ctx context.Context, cids []cid.Cid) error
}
@ -106,6 +111,13 @@ type adaptedBlockstore struct {
var _ Blockstore = (*adaptedBlockstore)(nil)
func (a *adaptedBlockstore) Flush(ctx context.Context) error {
if flusher, canFlush := a.Blockstore.(Flusher); canFlush {
return flusher.Flush(ctx)
}
return nil
}
func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
blk, err := a.Get(ctx, cid)
if err != nil {

2
blockstore/buffered.go
View File

@ -46,6 +46,8 @@ var (
_ Viewer = (*BufferedBlockstore)(nil)
)
func (bs *BufferedBlockstore) Flush(ctx context.Context) error { return bs.write.Flush(ctx) }
func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
a, err := bs.read.AllKeysChan(ctx)
if err != nil {

4
blockstore/discard.go
View File

@ -38,6 +38,10 @@ func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) err
return b.bs.View(ctx, cid, f)
}
func (b *discardstore) Flush(ctx context.Context) error {
return nil
}
func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error {
return nil
}

4
blockstore/idstore.go
View File

@ -179,3 +179,7 @@ func (b *idstore) Close() error {
}
return nil
}
func (b *idstore) Flush(ctx context.Context) error {
return b.bs.Flush(ctx)
}

2
blockstore/mem.go
View File

@ -17,6 +17,8 @@ func NewMemory() MemBlockstore {
// To match behavior of badger blockstore we index by multihash only.
type MemBlockstore map[string]blocks.Block
func (MemBlockstore) Flush(context.Context) error { return nil }
func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
delete(m, string(k.Hash()))
return nil

2
blockstore/net.go
View File

@ -410,6 +410,8 @@ func (n *NetworkStore) HashOnRead(enabled bool) {
return
}
func (*NetworkStore) Flush(context.Context) error { return nil }
func (n *NetworkStore) Stop(ctx context.Context) error {
close(n.closing)

46
blockstore/splitstore/splitstore.go
View File

@ -115,6 +115,23 @@ type Config struct {
// A positive value is the number of compactions before a full GC is performed;
// a value of 1 will perform full GC in every compaction.
HotStoreFullGCFrequency uint64
// HotstoreMaxSpaceTarget suggests the max allowed space the hotstore can take.
// This is not a hard limit, it is possible for the hotstore to exceed the target
// for example if state grows massively between compactions. The splitstore
// will make a best effort to avoid overflowing the target and in practice should
// never overflow. This field is used when doing GC at the end of a compaction to
// adaptively choose moving GC
HotstoreMaxSpaceTarget uint64
// Moving GC will be triggered when total moving size exceeds
// HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold
HotstoreMaxSpaceThreshold uint64
// Safety buffer to prevent moving GC from overflowing disk.
// Moving GC will not occur when total moving size exceeds
// HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer
HotstoreMaxSpaceSafetyBuffer uint64
}
// ChainAccessor allows the Splitstore to access the chain. It will most likely
@ -165,6 +182,7 @@ type SplitStore struct {
compactionIndex int64
pruneIndex int64
onlineGCCnt int64
ctx context.Context
cancel func()
@ -195,6 +213,17 @@ type SplitStore struct {
// registered protectors
protectors []func(func(cid.Cid) error) error
// dag sizes measured during latest compaction
// logged and used for GC strategy
// protected by compaction lock
szWalk int64
szProtectedTxns int64
szKeys int64 // approximate, not counting keys protected when entering critical section
// protected by txnLk
szMarkedLiveRefs int64
}
var _ bstore.Blockstore = (*SplitStore)(nil)
@ -447,6 +476,23 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
}
}
func (s *SplitStore) Flush(ctx context.Context) error {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
if err := s.cold.Flush(ctx); err != nil {
return err
}
if err := s.hot.Flush(ctx); err != nil {
return err
}
if err := s.ds.Sync(ctx, dstore.Key{}); err != nil {
return err
}
return nil
}
func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
if isIdentiyCid(blk.Cid()) {
return nil

4
blockstore/splitstore/splitstore_check.go
View File

@ -95,7 +95,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
}
defer visitor.Close() //nolint
err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
size := s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
@ -133,7 +133,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
return err
}
log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt, "walk size", size)
write("--")
write("cold: %d missing: %d", *coldCnt, *missingCnt)
write("DONE")

152
blockstore/splitstore/splitstore_compact.go
View File

@ -66,7 +66,8 @@ var (
)
const (
batchSize = 16384
batchSize = 16384
cidKeySize = 128
)
func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@ -199,9 +200,11 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
log.Debugf("marking %d live refs", len(cids))
startMark := time.Now()
szMarked := new(int64)
count := new(int32)
visitor := newConcurrentVisitor()
walkObject := func(c cid.Cid) error {
walkObject := func(c cid.Cid) (int64, error) {
return s.walkObjectIncomplete(c, visitor,
func(c cid.Cid) error {
if isUnitaryObject(c) {
@ -228,10 +231,12 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
// optimize the common case of single put
if len(cids) == 1 {
if err := walkObject(cids[0]); err != nil {
sz, err := walkObject(cids[0])
if err != nil {
log.Errorf("error marking tipset refs: %s", err)
}
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
atomic.AddInt64(szMarked, sz)
return
}
@ -243,9 +248,11 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
worker := func() error {
for c := range workch {
if err := walkObject(c); err != nil {
sz, err := walkObject(c)
if err != nil {
return err
}
atomic.AddInt64(szMarked, sz)
}
return nil
@ -268,7 +275,8 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
log.Errorf("error marking tipset refs: %s", err)
}
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count, "size marked", *szMarked)
s.szMarkedLiveRefs += atomic.LoadInt64(szMarked)
}
// transactionally protect a view
@ -361,6 +369,7 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
log.Infow("protecting transactional references", "refs", len(txnRefs))
count := 0
sz := new(int64)
workch := make(chan cid.Cid, len(txnRefs))
startProtect := time.Now()
@ -393,10 +402,11 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
worker := func() error {
for c := range workch {
err := s.doTxnProtect(c, markSet)
szTxn, err := s.doTxnProtect(c, markSet)
if err != nil {
return xerrors.Errorf("error protecting transactional references to %s: %w", c, err)
}
atomic.AddInt64(sz, szTxn)
}
return nil
}
@ -409,16 +419,16 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
if err := g.Wait(); err != nil {
return err
}
log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count)
s.szProtectedTxns += atomic.LoadInt64(sz)
log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count, "protected size", sz)
}
}
// transactionally protect a reference by walking the object and marking.
// concurrent markings are short circuited by checking the markset.
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error {
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) (int64, error) {
if err := s.checkClosing(); err != nil {
return err
return 0, err
}
// Note: cold objects are deleted heaviest first, so the consituents of an object
@ -509,6 +519,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
// might be potentially inconsistent; abort compaction and notify the user to intervene.
return xerrors.Errorf("checkpoint exists; aborting compaction")
}
s.clearSizeMeasurements()
currentEpoch := curTs.Height()
boundaryEpoch := currentEpoch - CompactionBoundary
@ -598,7 +609,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
}
err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{}, fHot, fCold)
if err != nil {
return xerrors.Errorf("error marking: %w", err)
}
@ -638,7 +648,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
defer purgew.Close() //nolint:errcheck
// some stats for logging
var hotCnt, coldCnt, purgeCnt int
var hotCnt, coldCnt, purgeCnt int64
err = s.hot.ForEachKey(func(c cid.Cid) error {
// was it marked?
mark, err := markSet.Has(c)
@ -690,8 +700,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
log.Infow("cold collection done", "took", time.Since(startCollect))
log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "purge", purgeCnt)
stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt)))
stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt)))
s.szKeys = hotCnt * cidKeySize
stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(hotCnt))
stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(coldCnt))
if err := s.checkClosing(); err != nil {
return err
@ -773,8 +784,8 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return xerrors.Errorf("error purging cold objects: %w", err)
}
log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
s.endCriticalSection()
log.Infow("critical section done", "total protected size", s.szProtectedTxns, "total marked live size", s.szMarkedLiveRefs)
if err := checkpoint.Close(); err != nil {
log.Warnf("error closing checkpoint: %s", err)
@ -907,6 +918,7 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
copy(toWalk, ts.Cids())
walkCnt := new(int64)
scanCnt := new(int64)
szWalk := new(int64)
tsRef := func(blkCids []cid.Cid) (cid.Cid, error) {
return types.NewTipSetKey(blkCids...).Cid()
@ -942,48 +954,64 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
if err != nil {
return xerrors.Errorf("error computing cid reference to parent tipset")
}
if err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk); err != nil {
sz, err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk)
if err != nil {
return xerrors.Errorf("error walking parent tipset cid reference")
}
atomic.AddInt64(szWalk, sz)
// message are retained if within the inclMsgs boundary
if hdr.Height >= inclMsgs && hdr.Height > 0 {
if inclMsgs < inclState {
// we need to use walkObjectIncomplete here, as messages/receipts may be missing early on if we
// synced from snapshot and have a long HotStoreMessageRetentionPolicy.
if err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk); err != nil {
sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk)
if err != nil {
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
}
atomic.AddInt64(szWalk, sz)
if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk); err != nil {
sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk)
if err != nil {
return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
}
atomic.AddInt64(szWalk, sz)
} else {
if err := s.walkObject(hdr.Messages, visitor, fHot); err != nil {
sz, err = s.walkObject(hdr.Messages, visitor, fHot)
if err != nil {
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
}
atomic.AddInt64(szWalk, sz)
if err := s.walkObject(hdr.ParentMessageReceipts, visitor, fHot); err != nil {
sz, err := s.walkObject(hdr.ParentMessageReceipts, visitor, fHot)
if err != nil {
return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
}
atomic.AddInt64(szWalk, sz)
}
}
// messages and receipts outside of inclMsgs are included in the cold store
if hdr.Height < inclMsgs && hdr.Height > 0 {
if err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk); err != nil {
sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk)
if err != nil {
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
}
if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk); err != nil {
atomic.AddInt64(szWalk, sz)
sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk)
if err != nil {
return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
}
atomic.AddInt64(szWalk, sz)
}
// state is only retained if within the inclState boundary, with the exception of genesis
if hdr.Height >= inclState || hdr.Height == 0 {
if err := s.walkObject(hdr.ParentStateRoot, visitor, fHot); err != nil {
sz, err := s.walkObject(hdr.ParentStateRoot, visitor, fHot)
if err != nil {
return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err)
}
atomic.AddInt64(szWalk, sz)
atomic.AddInt64(scanCnt, 1)
}
@ -1001,9 +1029,11 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
if err != nil {
return xerrors.Errorf("error computing cid reference to parent tipset")
}
if err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk); err != nil {
sz, err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk)
if err != nil {
return xerrors.Errorf("error walking parent tipset cid reference")
}
atomic.AddInt64(szWalk, sz)
for len(toWalk) > 0 {
// walking can take a while, so check this with every opportunity
@ -1047,123 +1077,129 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
}
}
log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt)
log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt, "walk size", szWalk)
s.szWalk = atomic.LoadInt64(szWalk)
return nil
}
func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) error {
func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) (int64, error) {
var sz int64
visit, err := visitor.Visit(c)
if err != nil {
return xerrors.Errorf("error visiting object: %w", err)
return 0, xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return nil
return sz, nil
}
if err := f(c); err != nil {
if err == errStopWalk {
return nil
return sz, nil
}
return err
return 0, err
}
if c.Prefix().Codec != cid.DagCBOR {
return nil
return sz, nil
}
// check this before recursing
if err := s.checkClosing(); err != nil {
return err
return 0, err
}
var links []cid.Cid
err = s.view(c, func(data []byte) error {
sz += int64(len(data))
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
links = append(links, c)
})
})
if err != nil {
return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
}
for _, c := range links {
err := s.walkObject(c, visitor, f)
szLink, err := s.walkObject(c, visitor, f)
if err != nil {
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err)
}
sz += szLink
}
return nil
return sz, nil
}
// like walkObject, but the object may be potentially incomplete (references missing)
func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) error {
func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) (int64, error) {
var sz int64
visit, err := visitor.Visit(c)
if err != nil {
return xerrors.Errorf("error visiting object: %w", err)
return 0, xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return nil
return sz, nil
}
// occurs check -- only for DAGs
if c.Prefix().Codec == cid.DagCBOR {
has, err := s.has(c)
if err != nil {
return xerrors.Errorf("error occur checking %s: %w", c, err)
return 0, xerrors.Errorf("error occur checking %s: %w", c, err)
}
if !has {
err = missing(c)
if err == errStopWalk {
return nil
return sz, nil
}
return err
return 0, err
}
}
if err := f(c); err != nil {
if err == errStopWalk {
return nil
return sz, nil
}
return err
return 0, err
}
if c.Prefix().Codec != cid.DagCBOR {
return nil
return sz, nil
}
// check this before recursing
if err := s.checkClosing(); err != nil {
return err
return sz, err
}
var links []cid.Cid
err = s.view(c, func(data []byte) error {
sz += int64(len(data))
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
links = append(links, c)
})
})
if err != nil {
return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
}
for _, c := range links {
err := s.walkObjectIncomplete(c, visitor, f, missing)
szLink, err := s.walkObjectIncomplete(c, visitor, f, missing)
if err != nil {
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err)
}
sz += szLink
}
return nil
return sz, nil
}
// internal version used during compaction and related operations
@ -1429,8 +1465,9 @@ func (s *SplitStore) completeCompaction() error {
}
s.compactType = none
// Note: at this point we can start the splitstore; a compaction should run on
// the first head change, which will trigger gc on the hotstore.
// Note: at this point we can start the splitstore; base epoch is not
// incremented here so a compaction should run on the first head
// change, which will trigger gc on the hotstore.
// We don't mind the second (back-to-back) compaction as the head will
// have advanced during marking and coldset accumulation.
return nil
@ -1488,6 +1525,13 @@ func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint,
return nil
}
func (s *SplitStore) clearSizeMeasurements() {
s.szKeys = 0
s.szMarkedLiveRefs = 0
s.szProtectedTxns = 0
s.szWalk = 0
}
// I really don't like having this code, but we seem to have some occasional DAG references with
// missing constituents. During testing in mainnet *some* of these references *sometimes* appeared
// after a little bit.
@ -1528,7 +1572,7 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSet) {
missing = make(map[cid.Cid]struct{})
for c := range towalk {
err := s.walkObjectIncomplete(c, visitor,
_, err := s.walkObjectIncomplete(c, visitor,
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk

4
blockstore/splitstore/splitstore_expose.go
View File

@ -77,6 +77,10 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error
return size, err
}
func (es *exposedSplitStore) Flush(ctx context.Context) error {
return es.s.Flush(ctx)
}
func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
return es.s.Put(ctx, blk)
}

48
blockstore/splitstore/splitstore_gc.go
View File

@ -7,15 +7,61 @@ import (
bstore "github.com/filecoin-project/lotus/blockstore"
)
const (
// Fraction of garbage in badger vlog for online GC traversal to collect garbage
AggressiveOnlineGCThreshold = 0.0001
)
func (s *SplitStore) gcHotAfterCompaction() {
// Measure hotstore size, determine if we should do full GC, determine if we can do full GC.
// We should do full GC if
// FullGCFrequency is specified and compaction index matches frequency
// OR HotstoreMaxSpaceTarget is specified and total moving space is within 150 GB of target
// We can do full if
// HotstoreMaxSpaceTarget is not specified
// OR total moving space would not exceed 50 GB below target
//
// a) If we should not do full GC => online GC
// b) If we should do full GC and can => moving GC
// c) If we should do full GC and can't => aggressive online GC
getSize := func() int64 {
sizer, ok := s.hot.(bstore.BlockstoreSize)
if ok {
size, err := sizer.Size()
if err != nil {
log.Warnf("error getting hotstore size: %s, estimating empty hot store for targeting", err)
return 0
}
return size
}
log.Errorf("Could not measure hotstore size, assuming it is 0 bytes, which it is not")
return 0
}
hotSize := getSize()
copySizeApprox := s.szKeys + s.szMarkedLiveRefs + s.szProtectedTxns + s.szWalk
shouldTarget := s.cfg.HotstoreMaxSpaceTarget > 0 && hotSize+copySizeApprox > int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceThreshold)
shouldFreq := s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0
shouldDoFull := shouldTarget || shouldFreq
canDoFull := s.cfg.HotstoreMaxSpaceTarget == 0 || hotSize+copySizeApprox < int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceSafetyBuffer)
log.Debugw("approximating new hot store size", "key size", s.szKeys, "marked live refs", s.szMarkedLiveRefs, "protected txns", s.szProtectedTxns, "walked DAG", s.szWalk)
log.Infof("measured hot store size: %d, approximate new size: %d, should do full %t, can do full %t", hotSize, copySizeApprox, shouldDoFull, canDoFull)
var opts []bstore.BlockstoreGCOption
if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 {
if shouldDoFull && canDoFull {
opts = append(opts, bstore.WithFullGC(true))
} else if shouldDoFull && !canDoFull {
log.Warnf("Attention! Estimated moving GC size %d is not within safety buffer %d of target max %d, performing aggressive online GC to attempt to bring hotstore size down safely", copySizeApprox, s.cfg.HotstoreMaxSpaceSafetyBuffer, s.cfg.HotstoreMaxSpaceTarget)
log.Warn("If problem continues you can 1) temporarily allocate more disk space to hotstore and 2) reflect in HotstoreMaxSpaceTarget OR trigger manual move with `lotus chain prune hot-moving`")
log.Warn("If problem continues and you do not have any more disk space you can run continue to manually trigger online GC at aggressive thresholds (< 0.01) with `lotus chain prune hot`")
opts = append(opts, bstore.WithThreshold(AggressiveOnlineGCThreshold))
}
if err := s.gcBlockstore(s.hot, opts); err != nil {
log.Warnf("error garbage collecting hostore: %s", err)
}
log.Infof("measured hot store size after GC: %d", getSize())
}
func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {

2
blockstore/splitstore/splitstore_reify.go
View File

@ -101,7 +101,7 @@ func (s *SplitStore) doReify(c cid.Cid) {
defer s.txnLk.RUnlock()
count := 0
err := s.walkObjectIncomplete(c, newTmpVisitor(),
_, err := s.walkObjectIncomplete(c, newTmpVisitor(),
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk

2
blockstore/splitstore/splitstore_test.go
View File

@ -757,6 +757,8 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
return nil
}
func (b *mockStore) Flush(context.Context) error { return nil }
func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, errors.New("not implemented")
}

2
blockstore/sync.go
View File

@ -20,6 +20,8 @@ type SyncBlockstore struct {
bs MemBlockstore // specifically use a memStore to save indirection overhead.
}
func (*SyncBlockstore) Flush(context.Context) error { return nil }
func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
m.mu.Lock()
defer m.mu.Unlock()

10
blockstore/timed.go
View File

@ -93,6 +93,16 @@ func (t *TimedCacheBlockstore) rotate() {
t.mu.Unlock()
}
func (t *TimedCacheBlockstore) Flush(ctx context.Context) error {
t.mu.Lock()
defer t.mu.Unlock()
if err := t.active.Flush(ctx); err != nil {
return err
}
return t.inactive.Flush(ctx)
}
func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error {
// Don't check the inactive set here. We want to keep this block for at
// least one interval.

9
blockstore/union.go
View File

@ -55,6 +55,15 @@ func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, er
return size, err
}
func (m unionBlockstore) Flush(ctx context.Context) (err error) {
for _, bs := range m {
if err = bs.Flush(ctx); err != nil {
break
}
}
return err
}
func (m unionBlockstore) Put(ctx context.Context, block blocks.Block) (err error) {
for _, bs := range m {
if err = bs.Put(ctx, block); err != nil {

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/gateway.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

13
chain/badtscache.go
View File

@ -3,14 +3,14 @@ package chain
import (
"fmt"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/build"
)
type BadBlockCache struct {
badBlocks *lru.ARCCache
badBlocks *lru.ARCCache[cid.Cid, BadBlockReason]
}
type BadBlockReason struct {
@ -43,7 +43,7 @@ func (bbr BadBlockReason) String() string {
}
func NewBadBlockCache() *BadBlockCache {
cache, err := lru.NewARC(build.BadBlockCacheSize)
cache, err := lru.NewARC[cid.Cid, BadBlockReason](build.BadBlockCacheSize)
if err != nil {
panic(err) // ok
}
@ -66,10 +66,5 @@ func (bts *BadBlockCache) Purge() {
}
func (bts *BadBlockCache) Has(c cid.Cid) (BadBlockReason, bool) {
rval, ok := bts.badBlocks.Get(c)
if !ok {
return BadBlockReason{}, false
}
return rval.(BadBlockReason), true
return bts.badBlocks.Get(c)
}

16
chain/beacon/drand/drand.go
View File

@ -12,7 +12,7 @@ import (
dlog "github.com/drand/drand/log"
gclient "github.com/drand/drand/lp2p/client"
"github.com/drand/kyber"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
logging "github.com/ipfs/go-log/v2"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"go.uber.org/zap"
@ -61,7 +61,7 @@ type DrandBeacon struct {
filGenTime uint64
filRoundTime uint64
localCache *lru.Cache
localCache *lru.Cache[uint64, *types.BeaconEntry]
}
// DrandHTTPClient interface overrides the user agent used by drand
@ -119,7 +119,7 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes
return nil, xerrors.Errorf("creating drand client: %w", err)
}
lc, err := lru.New(1024)
lc, err := lru.New[uint64, *types.BeaconEntry](1024)
if err != nil {
return nil, err
}
@ -169,16 +169,12 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
return out
}
func (db *DrandBeacon) cacheValue(e types.BeaconEntry) {
db.localCache.Add(e.Round, e)
db.localCache.Add(e.Round, &e)
}
func (db *DrandBeacon) getCachedValue(round uint64) *types.BeaconEntry {
v, ok := db.localCache.Get(round)
if !ok {
return nil
}
e, _ := v.(types.BeaconEntry)
return &e
v, _ := db.localCache.Get(round)
return v
}
func (db *DrandBeacon) VerifyEntry(curr types.BeaconEntry, prev types.BeaconEntry) error {

12
chain/block_receipt_tracker.go
View File

@ -5,7 +5,7 @@ import (
"sync"
"time"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/filecoin-project/lotus/build"
@ -17,7 +17,7 @@ type blockReceiptTracker struct {
// using an LRU cache because i don't want to handle all the edge cases for
// manual cleanup and maintenance of a fixed size set
cache *lru.Cache
cache *lru.Cache[types.TipSetKey, *peerSet]
}
type peerSet struct {
@ -25,7 +25,7 @@ type peerSet struct {
}
func newBlockReceiptTracker() *blockReceiptTracker {
c, _ := lru.New(512)
c, _ := lru.New[types.TipSetKey, *peerSet](512)
return &blockReceiptTracker{
cache: c,
}
@ -46,20 +46,18 @@ func (brt *blockReceiptTracker) Add(p peer.ID, ts *types.TipSet) {
return
}
val.(*peerSet).peers[p] = build.Clock.Now()
val.peers[p] = build.Clock.Now()
}
func (brt *blockReceiptTracker) GetPeers(ts *types.TipSet) []peer.ID {
brt.lk.Lock()
defer brt.lk.Unlock()
val, ok := brt.cache.Get(ts.Key())
ps, ok := brt.cache.Get(ts.Key())
if !ok {
return nil
}
ps := val.(*peerSet)
out := make([]peer.ID, 0, len(ps.peers))
for p := range ps.peers {
out = append(out, p)

22
chain/consensus/compute_state.go
View File

@ -3,6 +3,7 @@ package consensus
import (
"context"
"sync/atomic"
"time"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
@ -113,6 +114,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
return sm.VMConstructor()(ctx, vmopt)
}
var cronGas int64
runCron := func(vmCron vm.Interface, epoch abi.ChainEpoch) error {
cronMsg := &types.Message{
To: cron.Address,
@ -130,6 +133,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
return xerrors.Errorf("running cron: %w", err)
}
cronGas += ret.GasUsed
if em != nil {
if err := em.MessageApplied(ctx, ts, cronMsg.Cid(), cronMsg, ret, true); err != nil {
return xerrors.Errorf("callback failed on cron message: %w", err)
@ -181,7 +186,9 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
}
}
partDone()
vmEarly := partDone()
earlyCronGas := cronGas
cronGas = 0
partDone = metrics.Timer(ctx, metrics.VMApplyMessages)
vmi, err := makeVm(pstate, epoch, ts.MinTimestamp())
@ -196,6 +203,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
processedMsgs = make(map[cid.Cid]struct{})
)
var msgGas int64
for _, b := range bms {
penalty := types.NewInt(0)
gasReward := big.Zero()
@ -210,6 +219,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
return cid.Undef, cid.Undef, err
}
msgGas += r.GasUsed
receipts = append(receipts, &r.MessageReceipt)
gasReward = big.Add(gasReward, r.GasCosts.MinerTip)
penalty = big.Add(penalty, r.GasCosts.MinerPenalty)
@ -239,14 +250,14 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
}
}
partDone()
vmMsg := partDone()
partDone = metrics.Timer(ctx, metrics.VMApplyCron)
if err := runCron(vmi, epoch); err != nil {
return cid.Cid{}, cid.Cid{}, err
}
partDone()
vmCron := partDone()
partDone = metrics.Timer(ctx, metrics.VMApplyFlush)
rectarr := blockadt.MakeEmptyArray(sm.ChainStore().ActorStore(ctx))
@ -282,6 +293,11 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
return cid.Undef, cid.Undef, xerrors.Errorf("vm flush failed: %w", err)
}
vmFlush := partDone()
partDone = func() time.Duration { return time.Duration(0) }
log.Infow("ApplyBlocks stats", "early", vmEarly, "earlyCronGas", earlyCronGas, "vmMsg", vmMsg, "msgGas", msgGas, "vmCron", vmCron, "cronGas", cronGas, "vmFlush", vmFlush, "epoch", epoch, "tsk", ts.Key())
stats.Record(ctx, metrics.VMSends.M(int64(atomic.LoadUint64(&vm.StatSends))),
metrics.VMApplied.M(int64(atomic.LoadUint64(&vm.StatApplied))))

18
chain/events/message_cache.go
View File

@ -4,7 +4,7 @@ import (
"context"
"sync"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/api"
@ -14,14 +14,14 @@ type messageCache struct {
api EventAPI
blockMsgLk sync.Mutex
blockMsgCache *lru.ARCCache
blockMsgCache *lru.ARCCache[cid.Cid, *api.BlockMessages]
}
func newMessageCache(api EventAPI) *messageCache {
blsMsgCache, _ := lru.NewARC(500)
func newMessageCache(a EventAPI) *messageCache {
blsMsgCache, _ := lru.NewARC[cid.Cid, *api.BlockMessages](500)
return &messageCache{
api: api,
api: a,
blockMsgCache: blsMsgCache,
}
}
@ -30,14 +30,14 @@ func (c *messageCache) ChainGetBlockMessages(ctx context.Context, blkCid cid.Cid
c.blockMsgLk.Lock()
defer c.blockMsgLk.Unlock()
msgsI, ok := c.blockMsgCache.Get(blkCid)
msgs, ok := c.blockMsgCache.Get(blkCid)
var err error
if !ok {
msgsI, err = c.api.ChainGetBlockMessages(ctx, blkCid)
msgs, err = c.api.ChainGetBlockMessages(ctx, blkCid)
if err != nil {
return nil, err
}
c.blockMsgCache.Add(blkCid, msgsI)
c.blockMsgCache.Add(blkCid, msgs)
}
return msgsI.(*api.BlockMessages), nil
return msgs, nil
}

2
chain/gen/gen.go
View File

@ -256,7 +256,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS
//return nil, xerrors.Errorf("creating drand beacon: %w", err)
//}
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac, ds)
if err != nil {
return nil, xerrors.Errorf("initing stmgr: %w", err)
}

14
chain/messagepool/config.go
View File

@ -13,7 +13,11 @@ import (
)
var (
ReplaceByFeeRatioDefault = 1.25
ReplaceByFeePercentageMinimum types.Percent = 110
ReplaceByFeePercentageDefault types.Percent = 125
)
var (
MemPoolSizeLimitHiDefault = 30000
MemPoolSizeLimitLoDefault = 20000
PruneCooldownDefault = time.Minute
@ -60,9 +64,9 @@ func (mp *MessagePool) getConfig() *types.MpoolConfig {
}
func validateConfg(cfg *types.MpoolConfig) error {
if cfg.ReplaceByFeeRatio < ReplaceByFeeRatioDefault {
return fmt.Errorf("'ReplaceByFeeRatio' is less than required %f < %f",
cfg.ReplaceByFeeRatio, ReplaceByFeeRatioDefault)
if cfg.ReplaceByFeeRatio < ReplaceByFeePercentageMinimum {
return fmt.Errorf("'ReplaceByFeeRatio' is less than required %s < %s",
cfg.ReplaceByFeeRatio, ReplaceByFeePercentageMinimum)
}
if cfg.GasLimitOverestimation < 1 {
return fmt.Errorf("'GasLimitOverestimation' cannot be less than 1")
@ -91,7 +95,7 @@ func DefaultConfig() *types.MpoolConfig {
return &types.MpoolConfig{
SizeLimitHigh: MemPoolSizeLimitHiDefault,
SizeLimitLow: MemPoolSizeLimitLoDefault,
ReplaceByFeeRatio: ReplaceByFeeRatioDefault,
ReplaceByFeeRatio: ReplaceByFeePercentageDefault,
PruneCooldown: PruneCooldownDefault,
GasLimitOverestimation: GasLimitOverestimation,
}

37
chain/messagepool/messagepool.go
View File

@ -12,7 +12,7 @@ import (
"time"
"github.com/hashicorp/go-multierror"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
@ -47,10 +47,8 @@ var log = logging.Logger("messagepool")
var futureDebug = false
var rbfNumBig = types.NewInt(uint64((ReplaceByFeeRatioDefault - 1) * RbfDenom))
var rbfDenomBig = types.NewInt(RbfDenom)
const RbfDenom = 256
var rbfNumBig = types.NewInt(uint64(ReplaceByFeePercentageMinimum))
var rbfDenomBig = types.NewInt(100)
var RepublishInterval = time.Duration(10*build.BlockDelaySecs+build.PropagationDelaySecs) * time.Second
@ -161,7 +159,7 @@ type MessagePool struct {
// pruneCooldown is a channel used to allow a cooldown time between prunes
pruneCooldown chan struct{}
blsSigCache *lru.TwoQueueCache
blsSigCache *lru.TwoQueueCache[cid.Cid, crypto.Signature]
changes *lps.PubSub
@ -169,9 +167,9 @@ type MessagePool struct {
netName dtypes.NetworkName
sigValCache *lru.TwoQueueCache
sigValCache *lru.TwoQueueCache[string, struct{}]
nonceCache *lru.Cache
nonceCache *lru.Cache[nonceCacheKey, uint64]
evtTypes [3]journal.EventType
journal journal.Journal
@ -197,7 +195,13 @@ func newMsgSet(nonce uint64) *msgSet {
}
func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount {
minPrice := types.BigAdd(curPrem, types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig))
minPrice := types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig)
return types.BigAdd(minPrice, types.NewInt(1))
}
func ComputeRBF(curPrem abi.TokenAmount, replaceByFeeRatio types.Percent) abi.TokenAmount {
rbfNumBig := types.NewInt(uint64(replaceByFeeRatio))
minPrice := types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig)
return types.BigAdd(minPrice, types.NewInt(1))
}
@ -365,9 +369,9 @@ func (ms *msgSet) toSlice() []*types.SignedMessage {
}
func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) {
cache, _ := lru.New2Q(build.BlsSignatureCacheSize)
verifcache, _ := lru.New2Q(build.VerifSigCacheSize)
noncecache, _ := lru.New(256)
cache, _ := lru.New2Q[cid.Cid, crypto.Signature](build.BlsSignatureCacheSize)
verifcache, _ := lru.New2Q[string, struct{}](build.VerifSigCacheSize)
noncecache, _ := lru.New[nonceCacheKey, uint64](256)
cfg, err := loadConfig(ctx, ds)
if err != nil {
@ -1049,7 +1053,7 @@ func (mp *MessagePool) getStateNonce(ctx context.Context, addr address.Address,
n, ok := mp.nonceCache.Get(nk)
if ok {
return n.(uint64), nil
return n, nil
}
act, err := mp.api.GetActorAfter(addr, ts)
@ -1469,15 +1473,10 @@ func (mp *MessagePool) MessagesForBlocks(ctx context.Context, blks []*types.Bloc
}
func (mp *MessagePool) RecoverSig(msg *types.Message) *types.SignedMessage {
val, ok := mp.blsSigCache.Get(msg.Cid())
sig, ok := mp.blsSigCache.Get(msg.Cid())
if !ok {
return nil
}
sig, ok := val.(crypto.Signature)
if !ok {
log.Errorf("value in signature cache was not a signature (got %T)", val)
return nil
}
return &types.SignedMessage{
Message: *msg,

62
chain/stmgr/execute.go
View File

@ -8,6 +8,7 @@ import (
"go.opencensus.io/trace"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
)
@ -52,6 +53,22 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c
sm.stlk.Unlock()
if ts.Height() == 0 {
// NB: This is here because the process that executes blocks requires that the
// block miner reference a valid miner in the state tree. Unless we create some
// magical genesis miner, this won't work properly, so we short circuit here
// This avoids the question of 'who gets paid the genesis block reward'.
// This also makes us not attempt to lookup the tipset state with
// tryLookupTipsetState, which would cause a very long, very slow walk.
return ts.Blocks()[0].ParentStateRoot, ts.Blocks()[0].ParentMessageReceipts, nil
}
// First, try to find the tipset in the current chain. If found, we can avoid re-executing
// it.
if st, rec, found := tryLookupTipsetState(ctx, sm.cs, ts); found {
return st, rec, nil
}
st, rec, err = sm.tsExec.ExecuteTipSet(ctx, sm, ts, sm.tsExecMonitor, false)
if err != nil {
return cid.Undef, cid.Undef, err
@ -60,6 +77,51 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c
return st, rec, nil
}
// Try to lookup a state & receipt CID for a given tipset by walking the chain instead of executing
// it. This will only successfully return the state/receipt CIDs if they're found in the state
// store.
//
// NOTE: This _won't_ recursively walk the receipt/state trees. It assumes that having the root
// implies having the rest of the tree. However, lotus generally makes that assumption anyways.
func tryLookupTipsetState(ctx context.Context, cs *store.ChainStore, ts *types.TipSet) (cid.Cid, cid.Cid, bool) {
nextTs, err := cs.GetTipsetByHeight(ctx, ts.Height()+1, nil, false)
if err != nil {
// Nothing to see here. The requested height may be beyond the current head.
return cid.Undef, cid.Undef, false
}
// Make sure we're on the correct fork.
if nextTs.Parents() != ts.Key() {
// Also nothing to see here. This just means that the requested tipset is on a
// different fork.
return cid.Undef, cid.Undef, false
}
stateCid := nextTs.ParentState()
receiptCid := nextTs.ParentMessageReceipts()
// Make sure we have the parent state.
if hasState, err := cs.StateBlockstore().Has(ctx, stateCid); err != nil {
log.Errorw("failed to lookup state-root in blockstore", "cid", stateCid, "error", err)
return cid.Undef, cid.Undef, false
} else if !hasState {
// We have the chain but don't have the state. It looks like we need to try
// executing?
return cid.Undef, cid.Undef, false
}
// Make sure we have the receipts.
if hasReceipts, err := cs.ChainBlockstore().Has(ctx, receiptCid); err != nil {
log.Errorw("failed to lookup receipts in blockstore", "cid", receiptCid, "error", err)
return cid.Undef, cid.Undef, false
} else if !hasReceipts {
// If we don't have the receipts, re-execute and try again.
return cid.Undef, cid.Undef, false
}
return stateCid, receiptCid, true
}
func (sm *StateManager) ExecutionTraceWithMonitor(ctx context.Context, ts *types.TipSet, em ExecMonitor) (cid.Cid, error) {
st, _, err := sm.tsExec.ExecuteTipSet(ctx, sm, ts, em, true)
return st, err

14
chain/stmgr/forks.go
View File

@ -174,9 +174,16 @@ func (us UpgradeSchedule) GetNtwkVersion(e abi.ChainEpoch) (network.Version, err
func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecMonitor, ts *types.TipSet) (cid.Cid, error) {
retCid := root
var err error
u := sm.stateMigrations[height]
if u != nil && u.upgrade != nil {
migCid, ok, err := u.migrationResultCache.Get(ctx, root)
if err == nil && ok {
log.Infow("CACHED migration", "height", height, "from", root, "to", migCid)
return migCid, nil
} else if err != nil {
log.Errorw("failed to lookup previous migration result", "err", err)
}
startTime := time.Now()
log.Warnw("STARTING migration", "height", height, "from", root)
// Yes, we clone the cache, even for the final upgrade epoch. Why? Reverts. We may
@ -197,6 +204,11 @@ func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, heig
"to", retCid,
"duration", time.Since(startTime),
)
// Only set if migration ran, we do not want a root => root mapping
if err := u.migrationResultCache.Store(ctx, root, retCid); err != nil {
log.Errorw("failed to store migration result", "err", err)
}
}
return retCid, nil

108
chain/stmgr/forks_test.go
View File

@ -10,6 +10,7 @@ import (
"testing"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
ipldcbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log/v2"
"github.com/stretchr/testify/require"
@ -35,6 +36,7 @@ import (
"github.com/filecoin-project/lotus/chain/consensus"
"github.com/filecoin-project/lotus/chain/consensus/filcns"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/stmgr"
. "github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
@ -166,7 +168,7 @@ func TestForkHeightTriggers(t *testing.T) {
}
return st.Flush(ctx)
}}}, cg.BeaconSchedule())
}}}, cg.BeaconSchedule(), datastore.NewMapDatastore())
if err != nil {
t.Fatal(err)
}
@ -284,7 +286,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
migrationCount++
return root, nil
}}}, cg.BeaconSchedule())
}}}, cg.BeaconSchedule(), datastore.NewMapDatastore())
if err != nil {
t.Fatal(err)
}
@ -502,7 +504,7 @@ func TestForkPreMigration(t *testing.T) {
return nil
},
}}},
}, cg.BeaconSchedule())
}, cg.BeaconSchedule(), datastore.NewMapDatastore())
if err != nil {
t.Fatal(err)
}
@ -576,6 +578,7 @@ func TestDisablePreMigration(t *testing.T) {
}}},
},
cg.BeaconSchedule(),
datastore.NewMapDatastore(),
)
require.NoError(t, err)
require.NoError(t, sm.Start(context.Background()))
@ -603,3 +606,102 @@ func TestDisablePreMigration(t *testing.T) {
require.Equal(t, 1, len(counter))
}
func TestMigrtionCache(t *testing.T) {
logging.SetAllLoggers(logging.LevelInfo)
cg, err := gen.NewGenerator()
require.NoError(t, err)
counter := make(chan struct{}, 10)
metadataDs := datastore.NewMapDatastore()
sm, err := NewStateManager(
cg.ChainStore(),
consensus.NewTipSetExecutor(filcns.RewardFunc),
cg.StateManager().VMSys(),
UpgradeSchedule{{
Network: network.Version1,
Height: testForkHeight,
Migration: func(_ context.Context, _ *StateManager, _ MigrationCache, _ ExecMonitor,
root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) {
counter <- struct{}{}
return root, nil
}},
},
cg.BeaconSchedule(),
metadataDs,
)
require.NoError(t, err)
require.NoError(t, sm.Start(context.Background()))
defer func() {
require.NoError(t, sm.Stop(context.Background()))
}()
inv := consensus.NewActorRegistry()
registry := builtin.MakeRegistryLegacy([]rtt.VMActor{testActor{}})
inv.Register(actorstypes.Version0, nil, registry)
sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) {
nvm, err := vm.NewLegacyVM(ctx, vmopt)
require.NoError(t, err)
nvm.SetInvoker(inv)
return nvm, nil
})
cg.SetStateManager(sm)
for i := 0; i < 50; i++ {
_, err := cg.NextTipSet()
require.NoError(t, err)
}
ts, err := cg.ChainStore().GetTipsetByHeight(context.Background(), testForkHeight, nil, false)
require.NoError(t, err)
root, _, err := stmgr.ComputeState(context.Background(), sm, testForkHeight+1, []*types.Message{}, ts)
require.NoError(t, err)
t.Log(root)
require.Equal(t, 1, len(counter))
{
sm, err := NewStateManager(
cg.ChainStore(),
consensus.NewTipSetExecutor(filcns.RewardFunc),
cg.StateManager().VMSys(),
UpgradeSchedule{{
Network: network.Version1,
Height: testForkHeight,
Migration: func(_ context.Context, _ *StateManager, _ MigrationCache, _ ExecMonitor,
root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) {
counter <- struct{}{}
return root, nil
}},
},
cg.BeaconSchedule(),
metadataDs,
)
require.NoError(t, err)
sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) {
nvm, err := vm.NewLegacyVM(ctx, vmopt)
require.NoError(t, err)
nvm.SetInvoker(inv)
return nvm, nil
})
ctx := context.Background()
base, _, err := sm.ExecutionTrace(ctx, ts)
require.NoError(t, err)
_, err = sm.HandleStateForks(context.Background(), base, ts.Height(), nil, ts)
require.NoError(t, err)
// Should not have increased as we should be using the cached results in the metadataDs
require.Equal(t, 1, len(counter))
}
}

59
chain/stmgr/stmgr.go
View File

@ -2,10 +2,13 @@ package stmgr
import (
"context"
"fmt"
"sync"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
cbor "github.com/ipfs/go-ipld-cbor"
ipld "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
@ -51,9 +54,47 @@ type versionSpec struct {
}
type migration struct {
upgrade MigrationFunc
preMigrations []PreMigration
cache *nv16.MemMigrationCache
upgrade MigrationFunc
preMigrations []PreMigration
cache *nv16.MemMigrationCache
migrationResultCache *migrationResultCache
}
type migrationResultCache struct {
ds dstore.Batching
keyPrefix string
}
func (m *migrationResultCache) keyForMigration(root cid.Cid) dstore.Key {
kStr := fmt.Sprintf("%s/%s", m.keyPrefix, root)
return dstore.NewKey(kStr)
}
func (m *migrationResultCache) Get(ctx context.Context, root cid.Cid) (cid.Cid, bool, error) {
k := m.keyForMigration(root)
bs, err := m.ds.Get(ctx, k)
if ipld.IsNotFound(err) {
return cid.Undef, false, nil
} else if err != nil {
return cid.Undef, false, xerrors.Errorf("error loading migration result: %w", err)
}
c, err := cid.Parse(bs)
if err != nil {
return cid.Undef, false, xerrors.Errorf("error parsing migration result: %w", err)
}
return c, true, nil
}
func (m *migrationResultCache) Store(ctx context.Context, root cid.Cid, resultCid cid.Cid) error {
k := m.keyForMigration(root)
if err := m.ds.Put(ctx, k, resultCid.Bytes()); err != nil {
return err
}
return nil
}
type Executor interface {
@ -103,7 +144,7 @@ type treeCache struct {
tree *state.StateTree
}
func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule) (*StateManager, error) {
func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule, metadataDs dstore.Batching) (*StateManager, error) {
// If we have upgrades, make sure they're in-order and make sense.
if err := us.Validate(); err != nil {
return nil, err
@ -122,12 +163,18 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder,
upgrade: upgrade.Migration,
preMigrations: upgrade.PreMigrations,
cache: nv16.NewMemMigrationCache(),
migrationResultCache: &migrationResultCache{
keyPrefix: fmt.Sprintf("/migration-cache/nv%d", upgrade.Network),
ds: metadataDs,
},
}
stateMigrations[upgrade.Height] = migration
}
if upgrade.Expensive {
expensiveUpgrades[upgrade.Height] = struct{}{}
}
networkVersions = append(networkVersions, versionSpec{
networkVersion: lastVersion,
atOrBelow: upgrade.Height,
@ -155,8 +202,8 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder,
}, nil
}
func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor) (*StateManager, error) {
sm, err := NewStateManager(cs, exec, sys, us, b)
func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor, metadataDs dstore.Batching) (*StateManager, error) {
sm, err := NewStateManager(cs, exec, sys, us, b, metadataDs)
if err != nil {
return nil, err
}

56
chain/store/index.go
View File

@ -4,8 +4,8 @@ import (
"context"
"os"
"strconv"
"sync"
lru "github.com/hashicorp/golang-lru"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
@ -13,7 +13,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
)
var DefaultChainIndexCacheSize = 32 << 10
var DefaultChainIndexCacheSize = 32 << 15
func init() {
if s := os.Getenv("LOTUS_CHAIN_INDEX_CACHE"); s != "" {
@ -27,7 +27,8 @@ func init() {
}
type ChainIndex struct {
skipCache *lru.ARCCache
indexCacheLk sync.Mutex
indexCache map[types.TipSetKey]*lbEntry
loadTipSet loadTipSetFunc
@ -36,17 +37,14 @@ type ChainIndex struct {
type loadTipSetFunc func(context.Context, types.TipSetKey) (*types.TipSet, error)
func NewChainIndex(lts loadTipSetFunc) *ChainIndex {
sc, _ := lru.NewARC(DefaultChainIndexCacheSize)
return &ChainIndex{
skipCache: sc,
indexCache: make(map[types.TipSetKey]*lbEntry, DefaultChainIndexCacheSize),
loadTipSet: lts,
skipLength: 20,
}
}
type lbEntry struct {
ts *types.TipSet
parentHeight abi.ChainEpoch
targetHeight abi.ChainEpoch
target types.TipSetKey
}
@ -58,25 +56,36 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet,
rounded, err := ci.roundDown(ctx, from)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed to round down: %w", err)
}
ci.indexCacheLk.Lock()
defer ci.indexCacheLk.Unlock()
cur := rounded.Key()
for {
cval, ok := ci.skipCache.Get(cur)
lbe, ok := ci.indexCache[cur]
if !ok {
fc, err := ci.fillCache(ctx, cur)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed to fill cache: %w", err)
}
cval = fc
lbe = fc
}
lbe := cval.(*lbEntry)
if lbe.ts.Height() == to || lbe.parentHeight < to {
return lbe.ts, nil
} else if to > lbe.targetHeight {
return ci.walkBack(ctx, lbe.ts, to)
if to == lbe.targetHeight {
ts, err := ci.loadTipSet(ctx, lbe.target)
if err != nil {
return nil, xerrors.Errorf("failed to load tipset: %w", err)
}
return ts, nil
}
if to > lbe.targetHeight {
ts, err := ci.loadTipSet(ctx, cur)
if err != nil {
return nil, xerrors.Errorf("failed to load tipset: %w", err)
}
return ci.walkBack(ctx, ts, to)
}
cur = lbe.target
@ -87,16 +96,17 @@ func (ci *ChainIndex) GetTipsetByHeightWithoutCache(ctx context.Context, from *t
return ci.walkBack(ctx, from, to)
}
// Caller must hold indexCacheLk
func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEntry, error) {
ts, err := ci.loadTipSet(ctx, tsk)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed to load tipset: %w", err)
}
if ts.Height() == 0 {
return &lbEntry{
ts: ts,
parentHeight: 0,
targetHeight: 0,
target: tsk,
}, nil
}
@ -124,12 +134,10 @@ func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEn
}
lbe := &lbEntry{
ts: ts,
parentHeight: parent.Height(),
targetHeight: skipTarget.Height(),
target: skipTarget.Key(),
}
ci.skipCache.Add(tsk, lbe)
ci.indexCache[tsk] = lbe
return lbe, nil
}
@ -144,7 +152,7 @@ func (ci *ChainIndex) roundDown(ctx context.Context, ts *types.TipSet) (*types.T
rounded, err := ci.walkBack(ctx, ts, target)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed to walk back: %w", err)
}
return rounded, nil
@ -164,7 +172,7 @@ func (ci *ChainIndex) walkBack(ctx context.Context, from *types.TipSet, to abi.C
for {
pts, err := ci.loadTipSet(ctx, ts.Parents())
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed to load tipset: %w", err)
}
if to > pts.Height() {

26
chain/store/messages.go
View File

@ -207,9 +207,7 @@ type mmCids struct {
}
func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) {
o, ok := cs.mmCache.Get(mmc)
if ok {
mmcids := o.(*mmCids)
if mmcids, ok := cs.mmCache.Get(mmc); ok {
return mmcids.bls, mmcids.secpk, nil
}
@ -229,7 +227,7 @@ func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.C
return nil, nil, xerrors.Errorf("loading secpk message cids for block: %w", err)
}
cs.mmCache.Add(mmc, &mmCids{
cs.mmCache.Add(mmc, mmCids{
bls: blscids,
secpk: secpkcids,
})
@ -237,6 +235,26 @@ func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.C
return blscids, secpkcids, nil
}
func (cs *ChainStore) ReadReceipts(ctx context.Context, root cid.Cid) ([]types.MessageReceipt, error) {
a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
if err != nil {
return nil, err
}
receipts := make([]types.MessageReceipt, 0, a.Length())
var rcpt types.MessageReceipt
if err := a.ForEach(&rcpt, func(i int64) error {
if int64(len(receipts)) != i {
return xerrors.Errorf("missing receipt %d", i)
}
receipts = append(receipts, rcpt)
return nil
}); err != nil {
return nil, err
}
return receipts, nil
}
func (cs *ChainStore) MessagesForBlock(ctx context.Context, b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
blscids, secpkcids, err := cs.ReadMsgMetaCids(ctx, b.Messages)
if err != nil {

15
chain/store/store.go
View File

@ -11,7 +11,7 @@ import (
"sync"
"time"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
@ -120,8 +120,8 @@ type ChainStore struct {
reorgCh chan<- reorg
reorgNotifeeCh chan ReorgNotifee
mmCache *lru.ARCCache // msg meta cache (mh.Messages -> secp, bls []cid)
tsCache *lru.ARCCache
mmCache *lru.ARCCache[cid.Cid, mmCids]
tsCache *lru.ARCCache[types.TipSetKey, *types.TipSet]
evtTypes [1]journal.EventType
journal journal.Journal
@ -133,8 +133,8 @@ type ChainStore struct {
}
func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dstore.Batching, weight WeightFunc, j journal.Journal) *ChainStore {
c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
c, _ := lru.NewARC[cid.Cid, mmCids](DefaultMsgMetaCacheSize)
tsc, _ := lru.NewARC[types.TipSetKey, *types.TipSet](DefaultTipSetCacheSize)
if j == nil {
j = journal.NilJournal()
}
@ -818,9 +818,8 @@ func (cs *ChainStore) GetBlock(ctx context.Context, c cid.Cid) (*types.BlockHead
}
func (cs *ChainStore) LoadTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
v, ok := cs.tsCache.Get(tsk)
if ok {
return v.(*types.TipSet), nil
if ts, ok := cs.tsCache.Get(tsk); ok {
return ts, nil
}
// Fetch tipset block headers from blockstore in parallel

5
chain/store/store_test.go
View File

@ -196,7 +196,8 @@ func TestChainExportImportFull(t *testing.T) {
}
nbs := blockstore.NewMemorySync()
cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), filcns.Weight, nil)
ds := datastore.NewMapDatastore()
cs := store.NewChainStore(nbs, nbs, ds, filcns.Weight, nil)
defer cs.Close() //nolint:errcheck
root, err := cs.Import(context.TODO(), buf)
@ -213,7 +214,7 @@ func TestChainExportImportFull(t *testing.T) {
t.Fatal("imported chain differed from exported chain")
}
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule())
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule(), ds)
if err != nil {
t.Fatal(err)
}

37
chain/sub/incoming.go
View File

@ -7,7 +7,7 @@ import (
"sync"
"time"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
bserv "github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
@ -217,7 +217,7 @@ func fetchCids(
type BlockValidator struct {
self peer.ID
peers *lru.TwoQueueCache
peers *lru.TwoQueueCache[peer.ID, int]
killThresh int
@ -231,7 +231,7 @@ type BlockValidator struct {
}
func NewBlockValidator(self peer.ID, chain *store.ChainStore, cns consensus.Consensus, blacklist func(peer.ID)) *BlockValidator {
p, _ := lru.New2Q(4096)
p, _ := lru.New2Q[peer.ID, int](4096)
return &BlockValidator{
self: self,
peers: p,
@ -244,21 +244,19 @@ func NewBlockValidator(self peer.ID, chain *store.ChainStore, cns consensus.Cons
}
func (bv *BlockValidator) flagPeer(p peer.ID) {
v, ok := bv.peers.Get(p)
val, ok := bv.peers.Get(p)
if !ok {
bv.peers.Add(p, int(1))
bv.peers.Add(p, 1)
return
}
val := v.(int)
if val >= bv.killThresh {
log.Warnf("blacklisting peer %s", p)
bv.blacklist(p)
return
}
bv.peers.Add(p, v.(int)+1)
bv.peers.Add(p, val+1)
}
func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) (res pubsub.ValidationResult) {
@ -293,11 +291,11 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
}
type blockReceiptCache struct {
blocks *lru.TwoQueueCache
blocks *lru.TwoQueueCache[cid.Cid, int]
}
func newBlockReceiptCache() *blockReceiptCache {
c, _ := lru.New2Q(8192)
c, _ := lru.New2Q[cid.Cid, int](8192)
return &blockReceiptCache{
blocks: c,
@ -307,12 +305,12 @@ func newBlockReceiptCache() *blockReceiptCache {
func (brc *blockReceiptCache) add(bcid cid.Cid) int {
val, ok := brc.blocks.Get(bcid)
if !ok {
brc.blocks.Add(bcid, int(1))
brc.blocks.Add(bcid, 1)
return 0
}
brc.blocks.Add(bcid, val.(int)+1)
return val.(int)
brc.blocks.Add(bcid, val+1)
return val
}
type MessageValidator struct {
@ -466,13 +464,13 @@ type peerMsgInfo struct {
type IndexerMessageValidator struct {
self peer.ID
peerCache *lru.TwoQueueCache
peerCache *lru.TwoQueueCache[address.Address, *peerMsgInfo]
chainApi full.ChainModuleAPI
stateApi full.StateModuleAPI
}
func NewIndexerMessageValidator(self peer.ID, chainApi full.ChainModuleAPI, stateApi full.StateModuleAPI) *IndexerMessageValidator {
peerCache, _ := lru.New2Q(8192)
peerCache, _ := lru.New2Q[address.Address, *peerMsgInfo](8192)
return &IndexerMessageValidator{
self: self,
@ -515,15 +513,12 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
return pubsub.ValidationReject
}
minerID := minerAddr.String()
msgCid := idxrMsg.Cid
var msgInfo *peerMsgInfo
val, ok := v.peerCache.Get(minerID)
msgInfo, ok := v.peerCache.Get(minerAddr)
if !ok {
msgInfo = &peerMsgInfo{}
} else {
msgInfo = val.(*peerMsgInfo)
}
// Lock this peer's message info.
@ -544,7 +539,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
// Check that the miner ID maps to the peer that sent the message.
err = v.authenticateMessage(ctx, minerAddr, originPeer)
if err != nil {
log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerID)
log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerAddr)
stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
return pubsub.ValidationReject
}
@ -554,7 +549,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
// messages from the same peer are handled concurrently, there is a
// small chance that one msgInfo could replace the other here when
// the info is first cached. This is OK, so no need to prevent it.
v.peerCache.Add(minerID, msgInfo)
v.peerCache.Add(minerAddr, msgInfo)
}
}

8
chain/types/ethtypes/eth_transactions.go
View File

@ -44,14 +44,6 @@ type EthTx struct {
S EthBigInt `json:"s"`
}
func (tx *EthTx) Reward(blkBaseFee big.Int) EthBigInt {
availablePriorityFee := big.Sub(big.Int(tx.MaxFeePerGas), blkBaseFee)
if big.Cmp(big.Int(tx.MaxPriorityFeePerGas), availablePriorityFee) <= 0 {
return tx.MaxPriorityFeePerGas
}
return EthBigInt(availablePriorityFee)
}
type EthTxArgs struct {
ChainID int `json:"chainId"`
Nonce int `json:"nonce"`

23
chain/types/ethtypes/eth_types.go
View File

@ -295,17 +295,21 @@ func EthAddressFromPubKey(pubk []byte) ([]byte, error) {
return ethAddr, nil
}
var maskedIDPrefix = [20 - 8]byte{0xff}
func IsEthAddress(addr address.Address) bool {
if addr.Protocol() != address.Delegated {
return false
}
payload := addr.Payload()
namespace, _, err := varint.FromUvarint(payload)
namespace, offset, err := varint.FromUvarint(payload)
if err != nil {
return false
}
return namespace == builtintypes.EthereumAddressManagerActorID
payload = payload[offset:]
return namespace == builtintypes.EthereumAddressManagerActorID && len(payload) == 20 && !bytes.HasPrefix(payload, maskedIDPrefix[:])
}
func EthAddressFromFilecoinAddress(addr address.Address) (EthAddress, error) {
@ -326,9 +330,17 @@ func EthAddressFromFilecoinAddress(addr address.Address) (EthAddress, error) {
return EthAddress{}, xerrors.Errorf("invalid delegated address namespace in: %s", addr)
}
payload = payload[n:]
if namespace == builtintypes.EthereumAddressManagerActorID {
return CastEthAddress(payload)
if namespace != builtintypes.EthereumAddressManagerActorID {
return EthAddress{}, ErrInvalidAddress
}
ethAddr, err := CastEthAddress(payload)
if err != nil {
return EthAddress{}, err
}
if ethAddr.IsMaskedID() {
return EthAddress{}, xerrors.Errorf("f410f addresses cannot embed masked-ID payloads: %s", ethAddr)
}
return ethAddr, nil
}
return EthAddress{}, ErrInvalidAddress
}
@ -376,8 +388,7 @@ func (ea *EthAddress) UnmarshalJSON(b []byte) error {
}
func (ea EthAddress) IsMaskedID() bool {
idmask := [12]byte{0xff}
return bytes.Equal(ea[:12], idmask[:])
return bytes.HasPrefix(ea[:], maskedIDPrefix[:])
}
func (ea EthAddress) ToFilecoinAddress() (address.Address, error) {

15
chain/types/ethtypes/eth_types_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin"
)
type TestCase struct {
@ -178,6 +179,20 @@ func TestParseEthAddr(t *testing.T) {
}
}
func TestMaskedIDInF4(t *testing.T) {
addr, err := address.NewIDAddress(100)
require.NoError(t, err)
eaddr, err := EthAddressFromFilecoinAddress(addr)
require.NoError(t, err)
badaddr, err := address.NewDelegatedAddress(builtin.EthereumAddressManagerActorID, eaddr[:])
require.NoError(t, err)
_, err = EthAddressFromFilecoinAddress(badaddr)
require.Error(t, err)
}
func TestUnmarshalEthCall(t *testing.T) {
data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":""}`

13
chain/types/message.go
View File

@ -219,4 +219,17 @@ func (m *Message) ValidForBlockInclusion(minGas int64, version network.Version)
return nil
}
// EffectiveGasPremium returns the effective gas premium claimable by the miner
// given the supplied base fee.
//
// Filecoin clamps the gas premium at GasFeeCap - BaseFee, if lower than the
// specified premium.
func (m *Message) EffectiveGasPremium(baseFee abi.TokenAmount) abi.TokenAmount {
available := big.Sub(m.GasFeeCap, baseFee)
if big.Cmp(m.GasPremium, available) <= 0 {
return m.GasPremium
}
return available
}
const TestGasLimit = 100e6

2
chain/types/mpool.go
View File

@ -10,7 +10,7 @@ type MpoolConfig struct {
PriorityAddrs []address.Address
SizeLimitHigh int
SizeLimitLow int
ReplaceByFeeRatio float64
ReplaceByFeeRatio Percent
PruneCooldown time.Duration
GasLimitOverestimation float64
}

39
chain/types/percent.go Normal file
View File

@ -0,0 +1,39 @@
package types
import (
"fmt"
"math"
"strconv"
"golang.org/x/xerrors"
)
// Percent stores a signed percentage as an int64. When converted to a string (or json), it's stored
// as a decimal with two places (e.g., 100% -> 1.00).
type Percent int64
func (p Percent) String() string {
abs := p
sign := ""
if abs < 0 {
abs = -abs
sign = "-"
}
return fmt.Sprintf(`%s%d.%d`, sign, abs/100, abs%100)
}
func (p Percent) MarshalJSON() ([]byte, error) {
return []byte(p.String()), nil
}
func (p *Percent) UnmarshalJSON(b []byte) error {
flt, err := strconv.ParseFloat(string(b)+"e2", 64)
if err != nil {
return xerrors.Errorf("unable to parse ratio %s: %w", string(b), err)
}
if math.Trunc(flt) != flt {
return xerrors.Errorf("ratio may only have two decimals: %s", string(b))
}
*p = Percent(flt)
return nil
}

34
chain/types/percent_test.go Normal file
View File

@ -0,0 +1,34 @@
package types
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestPercent(t *testing.T) {
for _, tc := range []struct {
p Percent
s string
}{
{100, "1.0"},
{111, "1.11"},
{12, "0.12"},
{-12, "-0.12"},
{1012, "10.12"},
{-1012, "-10.12"},
{0, "0.0"},
} {
tc := tc
t.Run(fmt.Sprintf("%d <> %s", tc.p, tc.s), func(t *testing.T) {
m, err := tc.p.MarshalJSON()
require.NoError(t, err)
require.Equal(t, tc.s, string(m))
var p Percent
require.NoError(t, p.UnmarshalJSON([]byte(tc.s)))
require.Equal(t, tc.p, p)
})
}
}

4
chain/types/tipset.go
View File

@ -234,6 +234,10 @@ func (ts *TipSet) MinTicketBlock() *BlockHeader {
return min
}
func (ts *TipSet) ParentMessageReceipts() cid.Cid {
return ts.blks[0].ParentMessageReceipts
}
func (ts *TipSet) ParentState() cid.Cid {
return ts.blks[0].ParentStateRoot
}

49
cli/evm.go
View File

@ -35,6 +35,7 @@ var EvmCmd = &cli.Command{
EvmGetInfoCmd,
EvmCallSimulateCmd,
EvmGetContractAddress,
EvmGetBytecode,
},
}
@ -486,3 +487,51 @@ func ethAddrFromFilecoinAddress(ctx context.Context, addr address.Address, fnapi
return ethAddr, faddr, nil
}
var EvmGetBytecode = &cli.Command{
Name: "bytecode",
Usage: "Write the bytecode of a smart contract to a file",
ArgsUsage: "[contract-address] [file-name]",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "bin",
Usage: "write the bytecode as raw binary and don't hex-encode",
},
},
Action: func(cctx *cli.Context) error {
if cctx.NArg() != 2 {
return IncorrectNumArgs(cctx)
}
contractAddr, err := ethtypes.ParseEthAddress(cctx.Args().Get(0))
if err != nil {
return err
}
fileName := cctx.Args().Get(1)
api, closer, err := GetFullNodeAPIV1(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
code, err := api.EthGetCode(ctx, contractAddr, "latest")
if err != nil {
return err
}
if !cctx.Bool("bin") {
newCode := make([]byte, hex.EncodedLen(len(code)))
hex.Encode(newCode, code)
code = newCode
}
if err := os.WriteFile(fileName, code, 0o666); err != nil {
return xerrors.Errorf("failed to write bytecode to file %s: %w", fileName, err)
}
fmt.Printf("Code for %s written to %s\n", contractAddr, fileName)
return nil
},
}

9
cli/mpool.go
View File

@ -461,7 +461,12 @@ var MpoolReplaceCmd = &cli.Command{
msg := found.Message
if cctx.Bool("auto") {
minRBF := messagepool.ComputeMinRBF(msg.GasPremium)
cfg, err := api.MpoolGetConfig(ctx)
if err != nil {
return xerrors.Errorf("failed to lookup the message pool config: %w", err)
}
defaultRBF := messagepool.ComputeRBF(msg.GasPremium, cfg.ReplaceByFeeRatio)
var mss *lapi.MessageSendSpec
if cctx.IsSet("fee-limit") {
@ -482,7 +487,7 @@ var MpoolReplaceCmd = &cli.Command{
return xerrors.Errorf("failed to estimate gas values: %w", err)
}
msg.GasPremium = big.Max(retm.GasPremium, minRBF)
msg.GasPremium = big.Max(retm.GasPremium, defaultRBF)
msg.GasFeeCap = big.Max(retm.GasFeeCap, msg.GasPremium)
mff := func() (abi.TokenAmount, error) {

7
cli/mpool_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/filecoin-project/lotus/chain/wallet"
@ -298,6 +299,7 @@ func TestReplace(t *testing.T) {
mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil),
mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
mockApi.EXPECT().MpoolGetConfig(ctx).Return(messagepool.DefaultConfig(), nil),
// use gomock.any to match the message in expected api calls
// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
@ -342,6 +344,7 @@ func TestReplace(t *testing.T) {
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
mockApi.EXPECT().MpoolGetConfig(ctx).Return(messagepool.DefaultConfig(), nil),
// use gomock.any to match the message in expected api calls
// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
@ -538,7 +541,7 @@ func TestConfig(t *testing.T) {
t.Fatal(err)
}
mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: 0.25}
mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: types.Percent(25)}
gomock.InOrder(
mockApi.EXPECT().MpoolGetConfig(ctx).Return(mpoolCfg, nil),
)
@ -566,7 +569,7 @@ func TestConfig(t *testing.T) {
t.Fatal(err)
}
mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: 0.33}
mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: types.Percent(33)}
gomock.InOrder(
mockApi.EXPECT().MpoolSetConfig(ctx, mpoolCfg).Return(nil),
)

2
cmd/lotus-bench/import.go
View File

@ -229,7 +229,7 @@ var importBenchCmd = &cli.Command{
defer cs.Close() //nolint:errcheck
// TODO: We need to supply the actual beacon after v14
stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil)
stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil, metadataDs)
if err != nil {
return err
}

38
cmd/lotus-miner/proving.go
View File

@ -19,6 +19,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/proof"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
@ -640,7 +641,42 @@ It will not send any messages to the chain.`,
if err != nil {
return err
}
jr, err := json.Marshal(res)
//convert sector information into easily readable information
type PoStPartition struct {
Index uint64
Skipped []uint64
}
type SubmitWindowedPoStParams struct {
Deadline uint64
Partitions []PoStPartition
Proofs []proof.PoStProof
ChainCommitEpoch abi.ChainEpoch
ChainCommitRand abi.Randomness
}
var postParams []SubmitWindowedPoStParams
for _, i := range res {
var postParam SubmitWindowedPoStParams
postParam.Deadline = i.Deadline
for id, part := range i.Partitions {
postParam.Partitions[id].Index = part.Index
count, err := part.Skipped.Count()
if err != nil {
return err
}
sectors, err := part.Skipped.All(count)
if err != nil {
return err
}
postParam.Partitions[id].Skipped = sectors
}
postParam.Proofs = i.Proofs
postParam.ChainCommitEpoch = i.ChainCommitEpoch
postParam.ChainCommitRand = i.ChainCommitRand
postParams = append(postParams, postParam)
}
jr, err := json.MarshalIndent(postParams, "", " ")
if err != nil {
return err
}

4
cmd/lotus-shed/balances.go
View File

@ -513,7 +513,7 @@ var chainBalanceStateCmd = &cli.Command{
cst := cbor.NewCborStore(bs)
store := adt.WrapStore(ctx, cst)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
if err != nil {
return err
}
@ -737,7 +737,7 @@ var chainPledgeCmd = &cli.Command{
cst := cbor.NewCborStore(bs)
store := adt.WrapStore(ctx, cst)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
if err != nil {
return err
}

4
cmd/lotus-shed/gas-estimation.go
View File

@ -111,7 +111,7 @@ var gasTraceCmd = &cli.Command{
cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
defer cs.Close() //nolint:errcheck
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds)
if err != nil {
return err
}
@ -212,7 +212,7 @@ var replayOfflineCmd = &cli.Command{
cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
defer cs.Close() //nolint:errcheck
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds)
if err != nil {
return err
}

2
cmd/lotus-shed/invariants.go
View File

@ -90,7 +90,7 @@ var invariantsCmd = &cli.Command{
cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
defer cs.Close() //nolint:errcheck
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil)
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
if err != nil {
return err
}

4
cmd/lotus-shed/migrations.go
View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/urfave/cli/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
@ -121,7 +122,8 @@ var migrationsCmd = &cli.Command{
cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
defer cs.Close() //nolint:errcheck
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil)
// Note: we use a map datastore for the metadata to avoid writing / using cached migration results in the metadata store
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, datastore.NewMapDatastore())
if err != nil {
return err
}

10
cmd/lotus-shed/state-stats.go
View File

@ -9,7 +9,7 @@ import (
"sync"
"github.com/docker/go-units"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
offline "github.com/ipfs/go-ipfs-exchange-offline"
@ -50,13 +50,13 @@ type fieldItem struct {
type cacheNodeGetter struct {
ds format.NodeGetter
cache *lru.TwoQueueCache
cache *lru.TwoQueueCache[cid.Cid, format.Node]
}
func newCacheNodeGetter(d format.NodeGetter, size int) (*cacheNodeGetter, error) {
cng := &cacheNodeGetter{ds: d}
cache, err := lru.New2Q(size)
cache, err := lru.New2Q[cid.Cid, format.Node](size)
if err != nil {
return nil, err
}
@ -68,7 +68,7 @@ func newCacheNodeGetter(d format.NodeGetter, size int) (*cacheNodeGetter, error)
func (cng *cacheNodeGetter) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
if n, ok := cng.cache.Get(c); ok {
return n.(format.Node), nil
return n, nil
}
n, err := cng.ds.Get(ctx, c)
@ -308,7 +308,7 @@ to reduce the number of decode operations performed by caching the decoded objec
}
tsExec := consensus.NewTipSetExecutor(filcns.RewardFunc)
sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil)
sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
if err != nil {
return err
}

4
cmd/lotus-sim/simulation/node.go
View File

@ -106,7 +106,7 @@ func (nd *Node) LoadSim(ctx context.Context, name string) (*Simulation, error) {
if err != nil {
return nil, xerrors.Errorf("failed to create upgrade schedule for simulation %s: %w", name, err)
}
sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil)
sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil, nd.MetadataDS)
if err != nil {
return nil, xerrors.Errorf("failed to create state manager for simulation %s: %w", name, err)
}
@ -125,7 +125,7 @@ func (nd *Node) CreateSim(ctx context.Context, name string, head *types.TipSet)
if err != nil {
return nil, err
}
sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil)
sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil, nd.MetadataDS)
if err != nil {
return nil, xerrors.Errorf("creating state manager: %w", err)
}

2
cmd/lotus-sim/simulation/simulation.go
View File

@ -201,7 +201,7 @@ func (sim *Simulation) SetUpgradeHeight(nv network.Version, epoch abi.ChainEpoch
if err != nil {
return err
}
sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil)
sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil, sim.Node.MetadataDS)
if err != nil {
return err
}

2
cmd/lotus/daemon.go
View File

@ -540,7 +540,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
}
// TODO: We need to supply the actual beacon after v14
stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil)
stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
if err != nil {
return err
}

2
conformance/driver.go
View File

@ -108,7 +108,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, params
cs = store.NewChainStore(bs, bs, ds, filcns.Weight, nil)
tse = consensus.NewTipSetExecutor(filcns.RewardFunc)
sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil)
sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil, ds)
)
if err != nil {
return nil, err

4
documentation/en/api-v0-methods.md
View File

@ -2879,7 +2879,7 @@ Response:
],
"SizeLimitHigh": 123,
"SizeLimitLow": 123,
"ReplaceByFeeRatio": 12.3,
"ReplaceByFeeRatio": 1.23,
"PruneCooldown": 60000000000,
"GasLimitOverestimation": 12.3
}
@ -3167,7 +3167,7 @@ Inputs:
],
"SizeLimitHigh": 123,
"SizeLimitLow": 123,
"ReplaceByFeeRatio": 12.3,
"ReplaceByFeeRatio": 1.23,
"PruneCooldown": 60000000000,
"GasLimitOverestimation": 12.3
}

4
documentation/en/api-v1-unstable-methods.md
View File

@ -3882,7 +3882,7 @@ Response:
],
"SizeLimitHigh": 123,
"SizeLimitLow": 123,
"ReplaceByFeeRatio": 12.3,
"ReplaceByFeeRatio": 1.23,
"PruneCooldown": 60000000000,
"GasLimitOverestimation": 12.3
}
@ -4170,7 +4170,7 @@ Inputs:
],
"SizeLimitHigh": 123,
"SizeLimitLow": 123,
"ReplaceByFeeRatio": 12.3,
"ReplaceByFeeRatio": 1.23,
"PruneCooldown": 60000000000,
"GasLimitOverestimation": 12.3
}

14
documentation/en/cli-lotus.md
View File

@ -2647,6 +2647,7 @@ COMMANDS:
stat Print eth/filecoin addrs and code cid
call Simulate an eth contract call
contract-address Generate contract address from smart contract code
bytecode Write the bytecode of a smart contract to a file
help, h Shows a list of commands or help for one command
OPTIONS:
@ -2721,6 +2722,19 @@ OPTIONS:
```
### lotus evm bytecode
```
NAME:
lotus evm bytecode - Write the bytecode of a smart contract to a file
USAGE:
lotus evm bytecode [command options] [contract-address] [file-name]
OPTIONS:
--bin write the bytecode as raw binary and don't hex-encode (default: false)
```
## lotus net
```
NAME:

33
documentation/en/default-lotus-config.toml
View File

@ -191,7 +191,7 @@
[Chainstore]
# type: bool
# env var: LOTUS_CHAINSTORE_ENABLESPLITSTORE
#EnableSplitstore = false
#EnableSplitstore = true
[Chainstore.Splitstore]
# ColdStoreType specifies the type of the coldstore.
@ -199,7 +199,7 @@
#
# type: string
# env var: LOTUS_CHAINSTORE_SPLITSTORE_COLDSTORETYPE
#ColdStoreType = "messages"
#ColdStoreType = "discard"
# HotStoreType specifies the type of the hotstore.
# Only currently supported value is "badger".
@ -230,6 +230,35 @@
# env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREFULLGCFREQUENCY
#HotStoreFullGCFrequency = 20
# HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC
# will run moving GC if disk utilization gets within a threshold (150 GB) of the target.
# Splitstore GC will NOT run moving GC if the total size of the move would get
# within 50 GB of the target, and instead will run a more aggressive online GC.
# If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore
# GC will trigger moving GC if either configuration condition is met.
# A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer.
# At this minimum size moving GC happens every time, any smaller and moving GC won't
# be able to run. In spring 2023 this minimum is ~550 GB.
#
# type: uint64
# env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACETARGET
#HotStoreMaxSpaceTarget = 0
# When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size
# exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold
#
# type: uint64
# env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACETHRESHOLD
#HotStoreMaxSpaceThreshold = 150000000000
# Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget
# is set. Moving GC will not occur when total moving size exceeds
# HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer
#
# type: uint64
# env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACESAFETYBUFFER
#HotstoreMaxSpaceSafetyBuffer = 50000000000
[Cluster]
# EXPERIMENTAL. config to enabled node cluster with raft consensus

2
extern/filecoin-ffi vendored

@ -1 +1 @@
Subproject commit 88dc4c4ceb322337818b6508c8e9c23948f36cb1
Subproject commit 28e3cd44d91681c074aba362d1e5c954db110ad9

9
gateway/node.go
View File

@ -27,6 +27,7 @@ import (
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
const (
@ -59,18 +60,22 @@ type TargetAPI interface {
ChainPutObj(context.Context, blocks.Block) error
ChainGetGenesis(context.Context) (*types.TipSet, error)
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error)
MsigGetPending(ctx context.Context, addr address.Address, ts types.TipSetKey) ([]*api.MsigTransaction, error)
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
StateNetworkName(context.Context) (dtypes.NetworkName, error)
StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error)
StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error)
@ -85,6 +90,7 @@ type TargetAPI interface {
StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error)
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error)
WalletBalance(context.Context, address.Address) (types.BigInt, error)
@ -188,6 +194,9 @@ func (gw *Node) checkTipset(ts *types.TipSet) error {
}
func (gw *Node) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error {
if h > ts.Height() {
return fmt.Errorf("tipset height in future")
}
tsBlock := ts.Blocks()[0]
heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second
timeAtHeight := time.Unix(int64(tsBlock.Timestamp), 0).Add(-heightDelta)

108
gateway/proxy_eth.go
View File

@ -21,7 +21,7 @@ import (
)
func (gw *Node) Web3ClientVersion(ctx context.Context) (string, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
if err := gw.limit(ctx, basicRateLimitTokens); err != nil {
return "", err
}
@ -34,7 +34,7 @@ func (gw *Node) EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error)
}
func (gw *Node) EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
return 0, err
}
@ -80,40 +80,40 @@ func (gw *Node) checkBlkHash(ctx context.Context, blkHash ethtypes.EthHash) erro
return gw.checkTipsetKey(ctx, tsk)
}
func (gw *Node) checkBlkParam(ctx context.Context, blkParam string) error {
func (gw *Node) checkBlkParam(ctx context.Context, blkParam string, lookback ethtypes.EthUint64) error {
if blkParam == "earliest" {
// also not supported in node impl
return fmt.Errorf("block param \"earliest\" is not supported")
}
switch blkParam {
case "pending", "latest":
// those will be recent enough, so we don't need to check
return nil
default:
var num ethtypes.EthUint64
err := num.UnmarshalJSON([]byte(`"` + blkParam + `"`))
if err != nil {
return fmt.Errorf("cannot parse block number: %v", err)
}
head, err := gw.target.ChainHead(ctx)
if err != nil {
return err
}
if err := gw.checkTipsetHeight(head, abi.ChainEpoch(num)); err != nil {
return err
}
head, err := gw.target.ChainHead(ctx)
if err != nil {
return err
}
return nil
var num ethtypes.EthUint64
switch blkParam {
case "pending", "latest":
// Head is always ok.
if lookback == 0 {
return nil
}
// Can't look beyond 0 anyways.
if lookback > ethtypes.EthUint64(head.Height()) {
break
}
num = ethtypes.EthUint64(head.Height()) - lookback
default:
if err := num.UnmarshalJSON([]byte(`"` + blkParam + `"`)); err != nil {
return fmt.Errorf("cannot parse block number: %v", err)
}
}
return gw.checkTipsetHeight(head, abi.ChainEpoch(num))
}
func (gw *Node) EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return 0, err
}
if err := gw.checkBlkHash(ctx, blkHash); err != nil {
if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
return 0, err
}
@ -137,7 +137,7 @@ func (gw *Node) EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxIn
return ethtypes.EthBlock{}, err
}
if err := gw.checkBlkParam(ctx, blkNum); err != nil {
if err := gw.checkBlkParam(ctx, blkNum, 0); err != nil {
return ethtypes.EthBlock{}, err
}
@ -161,7 +161,7 @@ func (gw *Node) EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*e
}
func (gw *Node) EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
return nil, err
}
@ -173,7 +173,7 @@ func (gw *Node) EthGetTransactionCount(ctx context.Context, sender ethtypes.EthA
return 0, err
}
if err := gw.checkBlkParam(ctx, blkOpt); err != nil {
if err := gw.checkBlkParam(ctx, blkOpt, 0); err != nil {
return 0, err
}
@ -188,40 +188,12 @@ func (gw *Node) EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.Et
return gw.target.EthGetTransactionReceipt(ctx, txHash)
}
func (gw *Node) EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return ethtypes.EthTx{}, err
}
if err := gw.checkBlkHash(ctx, blkHash); err != nil {
return ethtypes.EthTx{}, err
}
return gw.target.EthGetTransactionByBlockHashAndIndex(ctx, blkHash, txIndex)
}
func (gw *Node) EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return ethtypes.EthTx{}, err
}
head, err := gw.target.ChainHead(ctx)
if err != nil {
return ethtypes.EthTx{}, err
}
if err := gw.checkTipsetHeight(head, abi.ChainEpoch(blkNum)); err != nil {
return ethtypes.EthTx{}, err
}
return gw.target.EthGetTransactionByBlockNumberAndIndex(ctx, blkNum, txIndex)
}
func (gw *Node) EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return nil, err
}
if err := gw.checkBlkParam(ctx, blkOpt); err != nil {
if err := gw.checkBlkParam(ctx, blkOpt, 0); err != nil {
return nil, err
}
@ -233,7 +205,7 @@ func (gw *Node) EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress
return nil, err
}
if err := gw.checkBlkParam(ctx, blkParam); err != nil {
if err := gw.checkBlkParam(ctx, blkParam, 0); err != nil {
return nil, err
}
@ -245,7 +217,7 @@ func (gw *Node) EthGetBalance(ctx context.Context, address ethtypes.EthAddress,
return ethtypes.EthBigInt(big.Zero()), err
}
if err := gw.checkBlkParam(ctx, blkParam); err != nil {
if err := gw.checkBlkParam(ctx, blkParam, 0); err != nil {
return ethtypes.EthBigInt(big.Zero()), err
}
@ -253,7 +225,7 @@ func (gw *Node) EthGetBalance(ctx context.Context, address ethtypes.EthAddress,
}
func (gw *Node) EthChainId(ctx context.Context) (ethtypes.EthUint64, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
if err := gw.limit(ctx, basicRateLimitTokens); err != nil {
return 0, err
}
@ -269,7 +241,7 @@ func (gw *Node) NetVersion(ctx context.Context) (string, error) {
}
func (gw *Node) NetListening(ctx context.Context) (bool, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
if err := gw.limit(ctx, basicRateLimitTokens); err != nil {
return false, err
}
@ -285,7 +257,7 @@ func (gw *Node) EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, err
}
func (gw *Node) EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
return ethtypes.EthBigInt(big.Zero()), err
}
@ -304,7 +276,7 @@ func (gw *Node) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtype
return ethtypes.EthFeeHistory{}, err
}
if err := gw.checkBlkParam(ctx, params.NewestBlkNum); err != nil {
if err := gw.checkBlkParam(ctx, params.NewestBlkNum, params.BlkCount); err != nil {
return ethtypes.EthFeeHistory{}, err
}
@ -316,7 +288,7 @@ func (gw *Node) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtype
}
func (gw *Node) EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
return ethtypes.EthBigInt(big.Zero()), err
}
@ -337,7 +309,7 @@ func (gw *Node) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam strin
return nil, err
}
if err := gw.checkBlkParam(ctx, blkParam); err != nil {
if err := gw.checkBlkParam(ctx, blkParam, 0); err != nil {
return nil, err
}
@ -359,12 +331,12 @@ func (gw *Node) EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec)
}
if filter.FromBlock != nil {
if err := gw.checkBlkParam(ctx, *filter.FromBlock); err != nil {
if err := gw.checkBlkParam(ctx, *filter.FromBlock, 0); err != nil {
return nil, err
}
}
if filter.ToBlock != nil {
if err := gw.checkBlkParam(ctx, *filter.ToBlock); err != nil {
if err := gw.checkBlkParam(ctx, *filter.ToBlock, 0); err != nil {
return nil, err
}
}

48
gateway/proxy_fil.go
View File

@ -20,6 +20,7 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func (gw *Node) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
@ -187,6 +188,13 @@ func (gw *Node) GasEstimateMessageGas(ctx context.Context, msg *types.Message, s
return gw.target.GasEstimateMessageGas(ctx, msg, spec, tsk)
}
func (gw *Node) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return 0, err
}
return gw.target.MpoolGetNonce(ctx, addr)
}
func (gw *Node) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return cid.Cid{}, err
@ -209,9 +217,6 @@ func (gw *Node) MsigGetVested(ctx context.Context, addr address.Address, start t
if err := gw.limit(ctx, walletRateLimitTokens); err != nil {
return types.BigInt{}, err
}
if err := gw.checkTipsetKey(ctx, start); err != nil {
return types.NewInt(0), err
}
if err := gw.checkTipsetKey(ctx, end); err != nil {
return types.NewInt(0), err
}
@ -248,6 +253,16 @@ func (gw *Node) StateAccountKey(ctx context.Context, addr address.Address, tsk t
return gw.target.StateAccountKey(ctx, addr, tsk)
}
func (gw *Node) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return nil, err
}
if err := gw.checkTipsetKey(ctx, tsk); err != nil {
return nil, err
}
return gw.target.StateCall(ctx, msg, tsk)
}
func (gw *Node) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return api.DealCollateralBounds{}, err
@ -258,6 +273,16 @@ func (gw *Node) StateDealProviderCollateralBounds(ctx context.Context, size abi.
return gw.target.StateDealProviderCollateralBounds(ctx, size, verified, tsk)
}
func (gw *Node) StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return nil, err
}
if err := gw.checkTipsetKey(ctx, tsk); err != nil {
return nil, err
}
return gw.target.StateDecodeParams(ctx, toAddr, method, params, tsk)
}
func (gw *Node) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return nil, err
@ -308,6 +333,13 @@ func (gw *Node) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, t
return gw.target.StateMarketStorageDeal(ctx, dealId, tsk)
}
func (gw *Node) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return *new(dtypes.NetworkName), err
}
return gw.target.StateNetworkName(ctx)
}
func (gw *Node) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return network.VersionMax, err
@ -457,6 +489,16 @@ func (gw *Node) StateVerifiedClientStatus(ctx context.Context, addr address.Addr
return gw.target.StateVerifiedClientStatus(ctx, addr, tsk)
}
func (gw *Node) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return nil, err
}
if err := gw.checkTipsetKey(ctx, tsk); err != nil {
return nil, err
}
return gw.target.StateVerifierStatus(ctx, addr, tsk)
}
func (gw *Node) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) {
if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
return api.CirculatingSupply{}, err

6
go.mod
View File

@ -69,12 +69,13 @@ require (
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
github.com/hashicorp/go-hclog v1.3.0
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/golang-lru v0.5.4
github.com/hashicorp/golang-lru/v2 v2.0.2
github.com/hashicorp/raft v1.3.10
github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94
github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab
github.com/ipfs/bbloom v0.0.4
github.com/ipfs/go-block-format v0.1.1
github.com/ipfs/go-blockservice v0.5.0
github.com/ipfs/go-cid v0.3.2
github.com/ipfs/go-cidutil v0.1.0
@ -227,11 +228,10 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/iancoleman/orderedmap v0.1.0 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect
github.com/ipfs/go-block-format v0.1.1 // indirect
github.com/ipfs/go-filestore v1.2.0 // indirect
github.com/ipfs/go-ipfs-cmds v0.8.2 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect

7
go.sum
View File

@ -633,10 +633,11 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=
github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU=
github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=

15
itests/eth_api_test.go
View File

@ -2,6 +2,7 @@ package itests
import (
"context"
"strconv"
"testing"
"time"
@ -9,6 +10,7 @@ import (
"github.com/filecoin-project/go-state-types/builtin"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/chain/wallet/key"
@ -109,3 +111,16 @@ func TestEthGetGenesis(t *testing.T) {
require.NoError(t, err)
require.Equal(t, ethBlk.Hash, genesisHash)
}
func TestNetVersion(t *testing.T) {
blockTime := 100 * time.Millisecond
client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC())
ens.InterconnectAll().BeginMining(blockTime)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
version, err := client.NetVersion(ctx)
require.NoError(t, err)
require.Equal(t, strconv.Itoa(build.Eip155ChainId), version)
}

8
itests/eth_block_hash_test.go
View File

@ -3,6 +3,7 @@ package itests
import (
"context"
"fmt"
"strings"
"testing"
"time"
@ -48,13 +49,14 @@ func TestEthBlockHashesCorrect_MultiBlockTipset(t *testing.T) {
// let the chain run a little bit longer to minimise the chance of reorgs
n2.WaitTillChain(ctx, kit.HeightAtLeast(head.Height()+50))
head, err = n2.ChainHead(context.Background())
require.NoError(t, err)
for i := 1; i <= int(head.Height()); i++ {
hex := fmt.Sprintf("0x%x", i)
ethBlockA, err := n2.EthGetBlockByNumber(ctx, hex, true)
// Cannot use static ErrFullRound error for comparison since it gets reserialized as a JSON RPC error.
if err != nil && strings.Contains(err.Error(), "null round") {
continue
}
require.NoError(t, err)
ethBlockB, err := n2.EthGetBlockByHash(ctx, ethBlockA.Hash, true)

8
itests/eth_conformance_test.go
View File

@ -236,14 +236,6 @@ func TestEthOpenRPCConformance(t *testing.T) {
skipReason: "earliest block is not supported",
},
{
method: "eth_getBlockByNumber",
variant: "pending",
call: func(a *ethAPIRaw) (json.RawMessage, error) {
return ethapi.EthGetBlockByNumber(context.Background(), "pending", true)
},
},
{
method: "eth_getBlockByNumber",
call: func(a *ethAPIRaw) (json.RawMessage, error) {

135
itests/eth_fee_history_test.go
View File

@ -3,18 +3,42 @@ package itests
import (
"context"
"encoding/json"
"sort"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/lib/result"
"github.com/filecoin-project/lotus/node/impl/full"
)
// calculateExpectations calculates the expected number of items to be included in the response
// of eth_feeHistory. It takes care of null rounds by finding the closet tipset with height
// smaller than startHeight, and then looks back at requestAmount of items. It also considers
// scenarios where there are not enough items to look back.
func calculateExpectations(tsHeights []int, requestAmount, startHeight int) (count, oldestHeight int) {
latestIdx := sort.SearchInts(tsHeights, startHeight)
// SearchInts returns the index of the number that's larger than the target if the target
// doesn't exist. However, we're looking for the closet number that's smaller that the target
for tsHeights[latestIdx] > startHeight {
latestIdx--
}
cnt := requestAmount
oldestIdx := latestIdx - requestAmount + 1
if oldestIdx < 0 {
cnt = latestIdx + 1
oldestIdx = 0
}
return cnt, tsHeights[oldestIdx]
}
func TestEthFeeHistory(t *testing.T) {
require := require.New(t)
@ -22,70 +46,136 @@ func TestEthFeeHistory(t *testing.T) {
blockTime := 100 * time.Millisecond
client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC())
ens.InterconnectAll().BeginMining(blockTime)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
// Wait for the network to create 20 blocks
heads, err := client.ChainNotify(ctx)
require.NoError(err)
// Save the full view of the tipsets to calculate the answer when there are null rounds
tsHeights := []int{1}
go func() {
for chg := range heads {
for _, c := range chg {
tsHeights = append(tsHeights, int(c.Val.Height()))
}
}
}()
miner := ens.InterconnectAll().BeginMining(blockTime)
client.WaitTillChain(ctx, kit.HeightAtLeast(7))
miner[0].InjectNulls(abi.ChainEpoch(5))
// Wait for the network to create at least 20 tipsets
client.WaitTillChain(ctx, kit.HeightAtLeast(20))
for _, m := range miner {
m.Pause()
}
ch, err := client.ChainNotify(ctx)
require.NoError(err)
// Wait for 5 seconds of inactivity
func() {
for {
select {
case <-ch:
continue
case <-time.After(5 * time.Second):
return
}
}
}()
sort.Ints(tsHeights)
// because of the deferred execution, the last tipset is not executed yet,
// and the one before the last one is the last executed tipset,
// which corresponds to the "latest" tag in EthGetBlockByNumber
latestBlk := ethtypes.EthUint64(tsHeights[len(tsHeights)-2])
blk, err := client.EthGetBlockByNumber(ctx, "latest", false)
require.NoError(err)
require.Equal(blk.Number, latestBlk)
assertHistory := func(history *ethtypes.EthFeeHistory, requestAmount, startHeight int) {
amount, oldest := calculateExpectations(tsHeights, requestAmount, startHeight)
require.Equal(amount+1, len(history.BaseFeePerGas))
require.Equal(amount, len(history.GasUsedRatio))
require.Equal(ethtypes.EthUint64(oldest), history.OldestBlock)
}
history, err := client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{5, "0x10"}),
).Assert(require.NoError))
require.NoError(err)
require.Equal(6, len(history.BaseFeePerGas))
require.Equal(5, len(history.GasUsedRatio))
require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock)
assertHistory(&history, 5, 16)
require.Nil(history.Reward)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{"5", "0x10"}),
).Assert(require.NoError))
require.NoError(err)
require.Equal(6, len(history.BaseFeePerGas))
require.Equal(5, len(history.GasUsedRatio))
require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock)
assertHistory(&history, 5, 16)
require.Nil(history.Reward)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{5, "latest"}),
).Assert(require.NoError))
require.NoError(err)
assertHistory(&history, 5, int(latestBlk))
require.Nil(history.Reward)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{"0x10", "0x12"}),
).Assert(require.NoError))
require.NoError(err)
require.Equal(17, len(history.BaseFeePerGas))
require.Equal(16, len(history.GasUsedRatio))
require.Equal(ethtypes.EthUint64(18-16+1), history.OldestBlock)
assertHistory(&history, 16, 18)
require.Nil(history.Reward)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{5, "0x10"}),
).Assert(require.NoError))
require.NoError(err)
require.Equal(6, len(history.BaseFeePerGas))
require.Equal(5, len(history.GasUsedRatio))
require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock)
assertHistory(&history, 5, 16)
require.Nil(history.Reward)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{5, "10"}),
).Assert(require.NoError))
require.NoError(err)
require.Equal(6, len(history.BaseFeePerGas))
require.Equal(5, len(history.GasUsedRatio))
require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock)
assertHistory(&history, 5, 10)
require.Nil(history.Reward)
// test when the requested number of blocks is longer than chain length
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{"0x30", "latest"}),
).Assert(require.NoError))
require.NoError(err)
assertHistory(&history, 48, int(latestBlk))
require.Nil(history.Reward)
// test when the requested number of blocks is longer than chain length
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{"0x30", "10"}),
).Assert(require.NoError))
require.NoError(err)
assertHistory(&history, 48, 10)
require.Nil(history.Reward)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{5, "10", &[]float64{25, 50, 75}}),
).Assert(require.NoError))
require.NoError(err)
require.Equal(6, len(history.BaseFeePerGas))
require.Equal(5, len(history.GasUsedRatio))
require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock)
assertHistory(&history, 5, 10)
require.NotNil(history.Reward)
require.Equal(5, len(*history.Reward))
for _, arr := range *history.Reward {
require.Equal(3, len(arr))
for _, item := range arr {
require.Equal(ethtypes.EthBigInt(types.NewInt(full.MinGasPremium)), item)
}
}
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
@ -93,6 +183,11 @@ func TestEthFeeHistory(t *testing.T) {
).Assert(require.NoError))
require.Error(err)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{5, "10", &[]float64{75, 50}}),
).Assert(require.NoError))
require.Error(err)
history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
json.Marshal([]interface{}{5, "10", &[]float64{}}),
).Assert(require.NoError))

52
itests/eth_transactions_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/itests/kit"
@ -270,6 +271,57 @@ func TestContractInvocation(t *testing.T) {
require.EqualValues(t, ethtypes.EthUint64(0x1), receipt.Status)
}
func TestGetBlockByNumber(t *testing.T) {
blockTime := 100 * time.Millisecond
client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC())
bms := ens.InterconnectAll().BeginMining(blockTime)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
// create a new Ethereum account
_, ethAddr, filAddr := client.EVM().NewAccount()
// send some funds to the f410 address
kit.SendFunds(ctx, t, client, filAddr, types.FromFil(10))
latest, err := client.EthBlockNumber(ctx)
require.NoError(t, err)
// can get the latest block
_, err = client.EthGetBlockByNumber(ctx, latest.Hex(), true)
require.NoError(t, err)
// fail to get a future block
_, err = client.EthGetBlockByNumber(ctx, (latest + 10000).Hex(), true)
require.Error(t, err)
// inject 10 null rounds
bms[0].InjectNulls(10)
// wait until we produce blocks again
tctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
ch, err := client.ChainNotify(tctx)
require.NoError(t, err)
<-ch // current
hc := <-ch // wait for next block
require.Equal(t, store.HCApply, hc[0].Type)
afterNullHeight := hc[0].Val.Height()
// Fail when trying to fetch a null round.
_, err = client.EthGetBlockByNumber(ctx, (ethtypes.EthUint64(afterNullHeight - 1)).Hex(), true)
require.Error(t, err)
// Fetch balance on a null round; should not fail and should return previous balance.
// Should be lower than original balance.
bal, err := client.EthGetBalance(ctx, ethAddr, (ethtypes.EthUint64(afterNullHeight - 1)).Hex())
require.NoError(t, err)
require.NotEqual(t, big.Zero(), bal)
require.Equal(t, types.FromFil(10).Int, bal.Int)
}
func deployContractTx(ctx context.Context, client *kit.TestFullNode, ethAddr ethtypes.EthAddress, contract []byte) (*ethtypes.EthTxArgs, error) {
gaslimit, err := client.EthEstimateGas(ctx, ethtypes.EthCall{
From: &ethAddr,

7
itests/kit/blockminer.go
View File

@ -192,12 +192,15 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
reportSuccessFn := func(success bool, epoch abi.ChainEpoch, err error) {
// if api shuts down before mining, we may get an error which we should probably just ignore
// (fixing it will require rewriting most of the mining loop)
if err != nil && !strings.Contains(err.Error(), "websocket connection closed") && !api.ErrorIsIn(err, []error{new(jsonrpc.RPCConnectionError)}) {
if err != nil && ctx.Err() == nil && !strings.Contains(err.Error(), "websocket connection closed") && !api.ErrorIsIn(err, []error{new(jsonrpc.RPCConnectionError)}) {
require.NoError(bm.t, err)
}
target = epoch
wait <- success
select {
case wait <- success:
case <-ctx.Done():
}
}
var success bool

4
lib/rpcenc/reader.go
View File

@ -221,7 +221,7 @@ type RpcReader struct {
res chan readRes
beginOnce *sync.Once
closeOnce sync.Once
closeOnce *sync.Once
}
var ErrHasBody = errors.New("RPCReader has body, either already read from or from a client with no redirect support")
@ -265,6 +265,7 @@ func (w *RpcReader) beginPost() {
w.postBody = nr.postBody
w.res = nr.res
w.beginOnce = nr.beginOnce
w.closeOnce = nr.closeOnce
}
}
@ -355,6 +356,7 @@ func ReaderParamDecoder() (http.HandlerFunc, jsonrpc.ServerOption) {
res: make(chan readRes),
next: ch,
beginOnce: &sync.Once{},
closeOnce: &sync.Once{},
}
switch req.Method {

25
lib/rpcenc/reader_test.go
View File

@ -3,8 +3,8 @@ package rpcenc
import (
"context"
"fmt"
"io"
"io/ioutil"
"net/http/httptest"
"strings"
"sync"
@ -77,7 +77,12 @@ func (h *ReaderHandler) CloseReader(ctx context.Context, r io.Reader) error {
}
func (h *ReaderHandler) ReadAll(ctx context.Context, r io.Reader) ([]byte, error) {
return ioutil.ReadAll(r)
b, err := io.ReadAll(r)
if err != nil {
return nil, xerrors.Errorf("readall: %w", err)
}
return b, nil
}
func (h *ReaderHandler) ReadNullLen(ctx context.Context, r io.Reader) (int64, error) {
@ -219,9 +224,15 @@ func TestReaderRedirect(t *testing.T) {
}
func TestReaderRedirectDrop(t *testing.T) {
for i := 0; i < 10; i++ {
t.Run(fmt.Sprintf("test %d", i), testReaderRedirectDrop)
}
}
func testReaderRedirectDrop(t *testing.T) {
// lower timeout so that the dangling connection between client and reader is dropped quickly
// after the test. Otherwise httptest.Close is blocked.
Timeout = 200 * time.Millisecond
Timeout = 90 * time.Millisecond
var allClient struct {
ReadAll func(ctx context.Context, r io.Reader) ([]byte, error)
@ -294,6 +305,8 @@ func TestReaderRedirectDrop(t *testing.T) {
done.Wait()
fmt.Println("---------------------")
// Redir client drops before subcall
done.Add(1)
@ -322,5 +335,9 @@ func TestReaderRedirectDrop(t *testing.T) {
// wait for subcall to finish
<-contCh
require.ErrorContains(t, allServerHandler.subErr, "decoding params for 'ReaderHandler.ReadAll' (param: 0; custom decoder): context canceled")
estr := allServerHandler.subErr.Error()
require.True(t,
strings.Contains(estr, "decoding params for 'ReaderHandler.ReadAll' (param: 0; custom decoder): context canceled") ||
strings.Contains(estr, "readall: unexpected EOF"), "unexpected error: %s", estr)
}

5
metrics/metrics.go
View File

@ -775,9 +775,10 @@ func SinceInMilliseconds(startTime time.Time) float64 {
// Timer is a function stopwatch, calling it starts the timer,
// calling the returned function will record the duration.
func Timer(ctx context.Context, m *stats.Float64Measure) func() {
func Timer(ctx context.Context, m *stats.Float64Measure) func() time.Duration {
start := time.Now()
return func() {
return func() time.Duration {
stats.Record(ctx, m.M(SinceInMilliseconds(start)))
return time.Since(start)
}
}

11
miner/miner.go
View File

@ -10,7 +10,7 @@ import (
"sync"
"time"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
@ -61,7 +61,7 @@ func randTimeOffset(width time.Duration) time.Duration {
// NewMiner instantiates a miner with a concrete WinningPoStProver and a miner
// address (which can be different from the worker's address).
func NewMiner(api v1api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, j journal.Journal) *Miner {
arc, err := lru.NewARC(10000)
arc, err := lru.NewARC[abi.ChainEpoch, bool](10000)
if err != nil {
panic(err)
}
@ -122,7 +122,7 @@ type Miner struct {
// minedBlockHeights is a safeguard that caches the last heights we mined.
// It is consulted before publishing a newly mined block, for a sanity check
// intended to avoid slashings in case of a bug.
minedBlockHeights *lru.ARCCache
minedBlockHeights *lru.ARCCache[abi.ChainEpoch, bool]
evtTypes [1]journal.EventType
journal journal.Journal
@ -331,13 +331,12 @@ minerLoop:
}
}
blkKey := fmt.Sprintf("%d", b.Header.Height)
if _, ok := m.minedBlockHeights.Get(blkKey); ok {
if _, ok := m.minedBlockHeights.Get(b.Header.Height); ok {
log.Warnw("Created a block at the same height as another block we've created", "height", b.Header.Height, "miner", b.Header.Miner, "parents", b.Header.Parents)
continue
}
m.minedBlockHeights.Add(blkKey, true)
m.minedBlockHeights.Add(b.Header.Height, true)
if err := m.api.SyncSubmitBlock(ctx, b); err != nil {
log.Errorf("failed to submit newly mined block: %+v", err)

4
miner/testminer.go
View File

@ -3,7 +3,7 @@ package miner
import (
"context"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
ds "github.com/ipfs/go-datastore"
"github.com/filecoin-project/go-address"
@ -22,7 +22,7 @@ type MineReq struct {
func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(v1api.FullNode, gen.WinningPoStProver) *Miner {
return func(api v1api.FullNode, epp gen.WinningPoStProver) *Miner {
arc, err := lru.NewARC(10000)
arc, err := lru.NewARC[abi.ChainEpoch, bool](10000)
if err != nil {
panic(err)
}

8
node/config/def.go
View File

@ -89,13 +89,15 @@ func DefaultFullNode() *FullNode {
SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers,
},
Chainstore: Chainstore{
EnableSplitstore: false,
EnableSplitstore: true,
Splitstore: Splitstore{
ColdStoreType: "messages",
ColdStoreType: "discard",
HotStoreType: "badger",
MarkSetType: "badger",
HotStoreFullGCFrequency: 20,
HotStoreFullGCFrequency: 20,
HotStoreMaxSpaceThreshold: 150_000_000_000,
HotstoreMaxSpaceSafetyBuffer: 50_000_000_000,
},
},
Cluster: *DefaultUserRaftConfig(),

29
node/config/doc_gen.go
View File

@ -1286,6 +1286,35 @@ the compaction boundary; default is 0.`,
A value of 0 disables, while a value 1 will do full GC in every compaction.
Default is 20 (about once a week).`,
},
{
Name: "HotStoreMaxSpaceTarget",
Type: "uint64",
Comment: `HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC
will run moving GC if disk utilization gets within a threshold (150 GB) of the target.
Splitstore GC will NOT run moving GC if the total size of the move would get
within 50 GB of the target, and instead will run a more aggressive online GC.
If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore
GC will trigger moving GC if either configuration condition is met.
A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer.
At this minimum size moving GC happens every time, any smaller and moving GC won't
be able to run. In spring 2023 this minimum is ~550 GB.`,
},
{
Name: "HotStoreMaxSpaceThreshold",
Type: "uint64",
Comment: `When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size
exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold`,
},
{
Name: "HotstoreMaxSpaceSafetyBuffer",
Type: "uint64",
Comment: `Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget
is set. Moving GC will not occur when total moving size exceeds
HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`,
},
},
"StorageMiner": []DocField{
{

19
node/config/types.go
View File

@ -601,6 +601,25 @@ type Splitstore struct {
// A value of 0 disables, while a value 1 will do full GC in every compaction.
// Default is 20 (about once a week).
HotStoreFullGCFrequency uint64
// HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC
// will run moving GC if disk utilization gets within a threshold (150 GB) of the target.
// Splitstore GC will NOT run moving GC if the total size of the move would get
// within 50 GB of the target, and instead will run a more aggressive online GC.
// If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore
// GC will trigger moving GC if either configuration condition is met.
// A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer.
// At this minimum size moving GC happens every time, any smaller and moving GC won't
// be able to run. In spring 2023 this minimum is ~550 GB.
HotStoreMaxSpaceTarget uint64
// When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size
// exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold
HotStoreMaxSpaceThreshold uint64
// Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget
// is set. Moving GC will not occur when total moving size exceeds
// HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer
HotstoreMaxSpaceSafetyBuffer uint64
}
// // Full Node

19
node/impl/full/chain.go
View File

@ -193,25 +193,14 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([]
return nil, nil
}
// TODO: need to get the number of messages better than this
pts, err := a.Chain.LoadTipSet(ctx, types.NewTipSetKey(b.Parents...))
receipts, err := a.Chain.ReadReceipts(ctx, b.ParentMessageReceipts)
if err != nil {
return nil, err
}
cm, err := a.Chain.MessagesForTipset(ctx, pts)
if err != nil {
return nil, err
}
var out []*types.MessageReceipt
for i := 0; i < len(cm); i++ {
r, err := a.Chain.GetParentReceipt(ctx, b, i)
if err != nil {
return nil, err
}
out = append(out, r)
out := make([]*types.MessageReceipt, len(receipts))
for i := range receipts {
out[i] = &receipts[i]
}
return out, nil

227
node/impl/full/eth.go
View File

@ -39,9 +39,12 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
var ErrUnsupported = errors.New("unsupported method")
type EthModuleAPI interface {
EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error)
EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error)
@ -54,8 +57,6 @@ type EthModuleAPI interface {
EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error)
EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkOpt string) (ethtypes.EthUint64, error)
EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*api.EthTxReceipt, error)
EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error)
EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error)
EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error)
EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error)
EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam string) (ethtypes.EthBigInt, error)
@ -153,6 +154,8 @@ type EthAPI struct {
EthEventAPI
}
var ErrNullRound = errors.New("requested epoch was a null round")
func (a *EthModule) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
return stmgr.GetNetworkName(ctx, a.StateManager, a.Chain.GetHeaviestTipSet().ParentState())
}
@ -231,15 +234,14 @@ func (a *EthModule) EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthH
return newEthBlockFromFilecoinTipSet(ctx, ts, fullTxInfo, a.Chain, a.StateAPI)
}
func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string) (tipset *types.TipSet, err error) {
if blkParam == "earliest" {
return nil, fmt.Errorf("block param \"earliest\" is not supported")
func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string, strict bool) (tipset *types.TipSet, err error) {
switch blkParam {
case "earliest", "pending":
return nil, fmt.Errorf("block param %q is not supported", blkParam)
}
head := a.Chain.GetHeaviestTipSet()
switch blkParam {
case "pending":
return head, nil
case "latest":
parent, err := a.Chain.GetTipSetFromKey(ctx, head.Parents())
if err != nil {
@ -252,16 +254,22 @@ func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string) (tipset
if err != nil {
return nil, fmt.Errorf("cannot parse block number: %v", err)
}
ts, err := a.Chain.GetTipsetByHeight(ctx, abi.ChainEpoch(num), nil, false)
if abi.ChainEpoch(num) > head.Height()-1 {
return nil, fmt.Errorf("requested a future epoch (beyond 'latest')")
}
ts, err := a.ChainAPI.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(num), head.Key())
if err != nil {
return nil, fmt.Errorf("cannot get tipset at height: %v", num)
}
if strict && ts.Height() != abi.ChainEpoch(num) {
return nil, ErrNullRound
}
return ts, nil
}
}
func (a *EthModule) EthGetBlockByNumber(ctx context.Context, blkParam string, fullTxInfo bool) (ethtypes.EthBlock, error) {
ts, err := a.parseBlkParam(ctx, blkParam)
ts, err := a.parseBlkParam(ctx, blkParam, true)
if err != nil {
return ethtypes.EthBlock{}, err
}
@ -335,13 +343,13 @@ func (a *EthModule) EthGetMessageCidByTransactionHash(ctx context.Context, txHas
c = txHash.ToCid()
}
_, err = a.StateAPI.Chain.GetSignedMessage(ctx, c)
_, err = a.Chain.GetSignedMessage(ctx, c)
if err == nil {
// This is an Eth Tx, Secp message, Or BLS message in the mpool
return &c, nil
}
_, err = a.StateAPI.Chain.GetMessage(ctx, c)
_, err = a.Chain.GetMessage(ctx, c)
if err == nil {
// This is a BLS message
return &c, nil
@ -367,7 +375,7 @@ func (a *EthModule) EthGetTransactionCount(ctx context.Context, sender ethtypes.
return ethtypes.EthUint64(0), nil
}
ts, err := a.parseBlkParam(ctx, blkParam)
ts, err := a.parseBlkParam(ctx, blkParam, false)
if err != nil {
return ethtypes.EthUint64(0), xerrors.Errorf("cannot parse block param: %s", blkParam)
}
@ -420,20 +428,15 @@ func (a *EthModule) EthGetTransactionReceipt(ctx context.Context, txHash ethtype
return nil, nil
}
replay, err := a.StateAPI.StateReplay(ctx, types.EmptyTSK, c)
if err != nil {
return nil, nil
}
var events []types.Event
if rct := replay.MsgRct; rct != nil && rct.EventsRoot != nil {
if rct := msgLookup.Receipt; rct.EventsRoot != nil {
events, err = a.ChainAPI.ChainGetEvents(ctx, *rct.EventsRoot)
if err != nil {
return nil, nil
}
}
receipt, err := newEthTxReceipt(ctx, tx, msgLookup, replay, events, a.StateAPI)
receipt, err := newEthTxReceipt(ctx, tx, msgLookup, events, a.Chain, a.StateAPI)
if err != nil {
return nil, nil
}
@ -441,12 +444,12 @@ func (a *EthModule) EthGetTransactionReceipt(ctx context.Context, txHash ethtype
return &receipt, nil
}
func (a *EthModule) EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) {
return ethtypes.EthTx{}, nil
func (a *EthAPI) EthGetTransactionByBlockHashAndIndex(context.Context, ethtypes.EthHash, ethtypes.EthUint64) (ethtypes.EthTx, error) {
return ethtypes.EthTx{}, ErrUnsupported
}
func (a *EthModule) EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) {
return ethtypes.EthTx{}, nil
func (a *EthAPI) EthGetTransactionByBlockNumberAndIndex(context.Context, ethtypes.EthUint64, ethtypes.EthUint64) (ethtypes.EthTx, error) {
return ethtypes.EthTx{}, ErrUnsupported
}
// EthGetCode returns string value of the compiled bytecode
@ -456,7 +459,7 @@ func (a *EthModule) EthGetCode(ctx context.Context, ethAddr ethtypes.EthAddress,
return nil, xerrors.Errorf("cannot get Filecoin address: %w", err)
}
ts, err := a.parseBlkParam(ctx, blkParam)
ts, err := a.parseBlkParam(ctx, blkParam, false)
if err != nil {
return nil, xerrors.Errorf("cannot parse block param: %s", blkParam)
}
@ -535,7 +538,7 @@ func (a *EthModule) EthGetCode(ctx context.Context, ethAddr ethtypes.EthAddress,
}
func (a *EthModule) EthGetStorageAt(ctx context.Context, ethAddr ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error) {
ts, err := a.parseBlkParam(ctx, blkParam)
ts, err := a.parseBlkParam(ctx, blkParam, false)
if err != nil {
return nil, xerrors.Errorf("cannot parse block param: %s", blkParam)
}
@ -631,7 +634,7 @@ func (a *EthModule) EthGetBalance(ctx context.Context, address ethtypes.EthAddre
return ethtypes.EthBigInt{}, err
}
ts, err := a.parseBlkParam(ctx, blkParam)
ts, err := a.parseBlkParam(ctx, blkParam, false)
if err != nil {
return ethtypes.EthBigInt{}, xerrors.Errorf("cannot parse block param: %s", blkParam)
}
@ -676,57 +679,48 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
}
}
ts, err := a.parseBlkParam(ctx, params.NewestBlkNum)
ts, err := a.parseBlkParam(ctx, params.NewestBlkNum, false)
if err != nil {
return ethtypes.EthFeeHistory{}, fmt.Errorf("bad block parameter %s: %s", params.NewestBlkNum, err)
}
// Deal with the case that the chain is shorter than the number of requested blocks.
oldestBlkHeight := uint64(1)
if abi.ChainEpoch(params.BlkCount) <= ts.Height() {
oldestBlkHeight = uint64(ts.Height()) - uint64(params.BlkCount) + 1
}
var (
basefee = ts.Blocks()[0].ParentBaseFee
oldestBlkHeight = uint64(1)
// NOTE: baseFeePerGas should include the next block after the newest of the returned range,
// because the next base fee can be inferred from the messages in the newest block.
// However, this is NOT the case in Filecoin due to deferred execution, so the best
// we can do is duplicate the last value.
baseFeeArray := []ethtypes.EthBigInt{ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)}
gasUsedRatioArray := []float64{}
rewardsArray := make([][]ethtypes.EthBigInt, 0)
// NOTE: baseFeePerGas should include the next block after the newest of the returned range,
// because the next base fee can be inferred from the messages in the newest block.
// However, this is NOT the case in Filecoin due to deferred execution, so the best
// we can do is duplicate the last value.
baseFeeArray = []ethtypes.EthBigInt{ethtypes.EthBigInt(basefee)}
rewardsArray = make([][]ethtypes.EthBigInt, 0)
gasUsedRatioArray = []float64{}
blocksIncluded int
)
for ts.Height() >= abi.ChainEpoch(oldestBlkHeight) {
// Unfortunately we need to rebuild the full message view so we can
// totalize gas used in the tipset.
msgs, err := a.Chain.MessagesForTipset(ctx, ts)
for blocksIncluded < int(params.BlkCount) && ts.Height() > 0 {
msgs, rcpts, err := messagesAndReceipts(ctx, ts, a.Chain, a.StateAPI)
if err != nil {
return ethtypes.EthFeeHistory{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
return ethtypes.EthFeeHistory{}, xerrors.Errorf("failed to retrieve messages and receipts for height %d: %w", ts.Height(), err)
}
txGasRewards := gasRewardSorter{}
for txIdx, msg := range msgs {
msgLookup, err := a.StateAPI.StateSearchMsg(ctx, types.EmptyTSK, msg.Cid(), api.LookbackNoLimit, false)
if err != nil || msgLookup == nil {
return ethtypes.EthFeeHistory{}, nil
}
tx, err := newEthTxFromMessageLookup(ctx, msgLookup, txIdx, a.Chain, a.StateAPI)
if err != nil {
return ethtypes.EthFeeHistory{}, nil
}
for i, msg := range msgs {
effectivePremium := msg.VMMessage().EffectiveGasPremium(basefee)
txGasRewards = append(txGasRewards, gasRewardTuple{
reward: tx.Reward(ts.Blocks()[0].ParentBaseFee),
gas: uint64(msgLookup.Receipt.GasUsed),
premium: effectivePremium,
gasUsed: rcpts[i].GasUsed,
})
}
rewards, totalGasUsed := calculateRewardsAndGasUsed(rewardPercentiles, txGasRewards)
// arrays should be reversed at the end
baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee))
baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(basefee))
gasUsedRatioArray = append(gasUsedRatioArray, float64(totalGasUsed)/float64(build.BlockGasLimit))
rewardsArray = append(rewardsArray, rewards)
oldestBlkHeight = uint64(ts.Height())
blocksIncluded++
parentTsKey := ts.Parents()
ts, err = a.Chain.LoadTipSet(ctx, parentTsKey)
@ -757,13 +751,8 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
return ret, nil
}
func (a *EthModule) NetVersion(ctx context.Context) (string, error) {
// Note that networkId is not encoded in hex
nv, err := a.StateNetworkVersion(ctx, types.EmptyTSK)
if err != nil {
return "", err
}
return strconv.FormatUint(uint64(nv), 10), nil
func (a *EthModule) NetVersion(_ context.Context) (string, error) {
return strconv.FormatInt(build.Eip155ChainId, 10), nil
}
func (a *EthModule) NetListening(ctx context.Context) (bool, error) {
@ -1066,7 +1055,7 @@ func (a *EthModule) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam s
return nil, xerrors.Errorf("failed to convert ethcall to filecoin message: %w", err)
}
ts, err := a.parseBlkParam(ctx, blkParam)
ts, err := a.parseBlkParam(ctx, blkParam, false)
if err != nil {
return nil, xerrors.Errorf("cannot parse block param: %s", blkParam)
}
@ -1783,37 +1772,37 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx
return ethtypes.EthBlock{}, err
}
msgs, err := cs.MessagesForTipset(ctx, ts)
msgs, rcpts, err := messagesAndReceipts(ctx, ts, cs, sa)
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
return ethtypes.EthBlock{}, xerrors.Errorf("failed to retrieve messages and receipts: %w", err)
}
block := ethtypes.NewEthBlock(len(msgs) > 0)
gasUsed := int64(0)
compOutput, err := sa.StateCompute(ctx, ts.Height(), nil, ts.Key())
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("failed to compute state: %w", err)
}
for txIdx, msg := range compOutput.Trace {
// skip system messages like reward application and cron
if msg.Msg.From == builtintypes.SystemActorAddr {
continue
for i, msg := range msgs {
rcpt := rcpts[i]
ti := ethtypes.EthUint64(i)
gasUsed += rcpt.GasUsed
var smsg *types.SignedMessage
switch msg := msg.(type) {
case *types.SignedMessage:
smsg = msg
case *types.Message:
smsg = &types.SignedMessage{
Message: *msg,
Signature: crypto.Signature{
Type: crypto.SigTypeBLS,
},
}
default:
return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.Cid(), err)
}
gasUsed += msg.MsgRct.GasUsed
smsgCid, err := getSignedMessage(ctx, cs, msg.MsgCid)
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.MsgCid, err)
}
tx, err := newEthTxFromSignedMessage(ctx, smsgCid, sa)
tx, err := newEthTxFromSignedMessage(ctx, smsg, sa)
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("failed to convert msg to ethTx: %w", err)
}
ti := ethtypes.EthUint64(txIdx)
tx.ChainID = ethtypes.EthUint64(build.Eip155ChainId)
tx.BlockHash = &blkHash
tx.BlockNumber = &bn
@ -1835,6 +1824,29 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx
return block, nil
}
func messagesAndReceipts(ctx context.Context, ts *types.TipSet, cs *store.ChainStore, sa StateAPI) ([]types.ChainMsg, []types.MessageReceipt, error) {
msgs, err := cs.MessagesForTipset(ctx, ts)
if err != nil {
return nil, nil, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
}
_, rcptRoot, err := sa.StateManager.TipSetState(ctx, ts)
if err != nil {
return nil, nil, xerrors.Errorf("failed to compute state: %w", err)
}
rcpts, err := cs.ReadReceipts(ctx, rcptRoot)
if err != nil {
return nil, nil, xerrors.Errorf("error loading receipts for tipset: %v: %w", ts, err)
}
if len(msgs) != len(rcpts) {
return nil, nil, xerrors.Errorf("receipts and message array lengths didn't match for tipset: %v: %w", ts, err)
}
return msgs, rcpts, nil
}
// lookupEthAddress makes its best effort at finding the Ethereum address for a
// Filecoin address. It does the following:
//
@ -2030,7 +2042,7 @@ func newEthTxFromMessageLookup(ctx context.Context, msgLookup *api.MsgLookup, tx
return tx, nil
}
func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLookup, replay *api.InvocResult, events []types.Event, sa StateAPI) (api.EthTxReceipt, error) {
func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLookup, events []types.Event, cs *store.ChainStore, sa StateAPI) (api.EthTxReceipt, error) {
var (
transactionIndex ethtypes.EthUint64
blockHash ethtypes.EthHash
@ -2061,8 +2073,7 @@ func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLook
if lookup.Receipt.ExitCode.IsSuccess() {
receipt.Status = 1
}
if lookup.Receipt.ExitCode.IsError() {
} else {
receipt.Status = 0
}
@ -2071,7 +2082,17 @@ func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLook
// TODO: handle CumulativeGasUsed
receipt.CumulativeGasUsed = ethtypes.EmptyEthInt
effectiveGasPrice := big.Div(replay.GasCost.TotalCost, big.NewInt(lookup.Receipt.GasUsed))
// TODO: avoid loading the tipset twice (once here, once when we convert the message to a txn)
ts, err := cs.GetTipSetFromKey(ctx, lookup.TipSet)
if err != nil {
return api.EthTxReceipt{}, xerrors.Errorf("failed to lookup tipset %s when constructing the eth txn receipt: %w", lookup.TipSet, err)
}
baseFee := ts.Blocks()[0].ParentBaseFee
gasOutputs := vm.ComputeGasOutputs(lookup.Receipt.GasUsed, int64(tx.Gas), baseFee, big.Int(tx.MaxFeePerGas), big.Int(tx.MaxPriorityFeePerGas), true)
totalSpent := big.Sum(gasOutputs.BaseFeeBurn, gasOutputs.MinerTip, gasOutputs.OverEstimationBurn)
effectiveGasPrice := big.Div(totalSpent, big.NewInt(lookup.Receipt.GasUsed))
receipt.EffectiveGasPrice = ethtypes.EthBigInt(effectiveGasPrice)
if receipt.To == nil && lookup.Receipt.ExitCode.IsSuccess() {
@ -2335,35 +2356,35 @@ func parseEthRevert(ret []byte) string {
return ethtypes.EthBytes(cbytes).String()
}
func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, uint64) {
var totalGasUsed uint64
func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, int64) {
var gasUsedTotal int64
for _, tx := range txGasRewards {
totalGasUsed += tx.gas
gasUsedTotal += tx.gasUsed
}
rewards := make([]ethtypes.EthBigInt, len(rewardPercentiles))
for i := range rewards {
rewards[i] = ethtypes.EthBigIntZero
rewards[i] = ethtypes.EthBigInt(types.NewInt(MinGasPremium))
}
if len(txGasRewards) == 0 {
return rewards, totalGasUsed
return rewards, gasUsedTotal
}
sort.Stable(txGasRewards)
var idx int
var sum uint64
var sum int64
for i, percentile := range rewardPercentiles {
threshold := uint64(float64(totalGasUsed) * percentile / 100)
threshold := int64(float64(gasUsedTotal) * percentile / 100)
for sum < threshold && idx < len(txGasRewards)-1 {
sum += txGasRewards[idx].gas
sum += txGasRewards[idx].gasUsed
idx++
}
rewards[i] = txGasRewards[idx].reward
rewards[i] = ethtypes.EthBigInt(txGasRewards[idx].premium)
}
return rewards, totalGasUsed
return rewards, gasUsedTotal
}
func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) (*types.SignedMessage, error) {
@ -2386,8 +2407,8 @@ func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid)
}
type gasRewardTuple struct {
gas uint64
reward ethtypes.EthBigInt
gasUsed int64
premium abi.TokenAmount
}
// sorted in ascending order
@ -2398,5 +2419,5 @@ func (g gasRewardSorter) Swap(i, j int) {
g[i], g[j] = g[j], g[i]
}
func (g gasRewardSorter) Less(i, j int) bool {
return g[i].reward.Int.Cmp(g[j].reward.Int) == -1
return g[i].premium.Int.Cmp(g[j].premium.Int) == -1
}

23
node/impl/full/eth_test.go
View File

@ -117,11 +117,8 @@ func TestReward(t *testing.T) {
{maxFeePerGas: big.NewInt(50), maxPriorityFeePerGas: big.NewInt(200), answer: big.NewInt(-50)},
}
for _, tc := range testcases {
tx := ethtypes.EthTx{
MaxFeePerGas: ethtypes.EthBigInt(tc.maxFeePerGas),
MaxPriorityFeePerGas: ethtypes.EthBigInt(tc.maxPriorityFeePerGas),
}
reward := tx.Reward(baseFee)
msg := &types.Message{GasFeeCap: tc.maxFeePerGas, GasPremium: tc.maxPriorityFeePerGas}
reward := msg.EffectiveGasPremium(baseFee)
require.Equal(t, 0, reward.Int.Cmp(tc.answer.Int), reward, tc.answer)
}
}
@ -135,25 +132,25 @@ func TestRewardPercentiles(t *testing.T) {
{
percentiles: []float64{25, 50, 75},
txGasRewards: []gasRewardTuple{},
answer: []int64{0, 0, 0},
answer: []int64{MinGasPremium, MinGasPremium, MinGasPremium},
},
{
percentiles: []float64{25, 50, 75, 100},
txGasRewards: []gasRewardTuple{
{gas: uint64(0), reward: ethtypes.EthBigInt(big.NewInt(300))},
{gas: uint64(100), reward: ethtypes.EthBigInt(big.NewInt(200))},
{gas: uint64(350), reward: ethtypes.EthBigInt(big.NewInt(100))},
{gas: uint64(500), reward: ethtypes.EthBigInt(big.NewInt(600))},
{gas: uint64(300), reward: ethtypes.EthBigInt(big.NewInt(700))},
{gasUsed: int64(0), premium: big.NewInt(300)},
{gasUsed: int64(100), premium: big.NewInt(200)},
{gasUsed: int64(350), premium: big.NewInt(100)},
{gasUsed: int64(500), premium: big.NewInt(600)},
{gasUsed: int64(300), premium: big.NewInt(700)},
},
answer: []int64{200, 700, 700, 700},
},
}
for _, tc := range testcases {
rewards, totalGasUsed := calculateRewardsAndGasUsed(tc.percentiles, tc.txGasRewards)
gasUsed := uint64(0)
var gasUsed int64
for _, tx := range tc.txGasRewards {
gasUsed += tx.gas
gasUsed += tx.gasUsed
}
ans := []ethtypes.EthBigInt{}
for _, bi := range tc.answer {

8
node/impl/full/gas.go
View File

@ -6,7 +6,7 @@ import (
"math/rand"
"sort"
lru "github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru/v2"
"go.uber.org/fx"
"golang.org/x/xerrors"
@ -61,7 +61,7 @@ type GasAPI struct {
func NewGasPriceCache() *GasPriceCache {
// 50 because we usually won't access more than 40
c, err := lru.New2Q(50)
c, err := lru.New2Q[types.TipSetKey, []GasMeta](50)
if err != nil {
// err only if parameter is bad
panic(err)
@ -73,7 +73,7 @@ func NewGasPriceCache() *GasPriceCache {
}
type GasPriceCache struct {
c *lru.TwoQueueCache
c *lru.TwoQueueCache[types.TipSetKey, []GasMeta]
}
type GasMeta struct {
@ -84,7 +84,7 @@ type GasMeta struct {
func (g *GasPriceCache) GetTSGasStats(ctx context.Context, cstore *store.ChainStore, ts *types.TipSet) ([]GasMeta, error) {
i, has := g.c.Get(ts.Key())
if has {
return i.([]GasMeta), nil
return i, nil
}
var prices []GasMeta

13
node/modules/blockstore.go
View File

@ -82,11 +82,14 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked
}
cfg := &splitstore.Config{
MarkSetType: cfg.Splitstore.MarkSetType,
DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard",
UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal",
HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention,
HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency,
MarkSetType: cfg.Splitstore.MarkSetType,
DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard",
UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal",
HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention,
HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency,
HotstoreMaxSpaceTarget: cfg.Splitstore.HotStoreMaxSpaceTarget,
HotstoreMaxSpaceThreshold: cfg.Splitstore.HotStoreMaxSpaceThreshold,
HotstoreMaxSpaceSafetyBuffer: cfg.Splitstore.HotstoreMaxSpaceSafetyBuffer,
}
ss, err := splitstore.Open(path, ds, hot, cold, cfg)
if err != nil {

2
node/modules/chain.go
View File

@ -123,7 +123,7 @@ func NetworkName(mctx helpers.MetricsCtx,
ctx := helpers.LifecycleCtx(mctx, lc)
sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil)
sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil, nil)
if err != nil {
return "", err
}

Some files were not shown because too many files have changed in this diff Show More